diff --git a/scripts/lib/paid-report.ts b/scripts/lib/paid-report.ts index 3fdc4ec4b..998635ebb 100644 --- a/scripts/lib/paid-report.ts +++ b/scripts/lib/paid-report.ts @@ -33,7 +33,9 @@ import { getProjectEvalDir, getClaudeCliVersion, isFinalizedEvalResultFile, evalEntryOutcome, failureClassOf, panelVerdict, sanitizeTrialError, formatTrialOutcomes, CONTRACT_VIOLATIONS_FILE, TRIAL_ENV, TRIAL_OUTCOME_SCHEMA, TRIAL_OUTCOMES_FILE, type EvalCaseKind, type PanelShape, type PanelVerdict, type TrialFailureClass, type TrialOutcome, type TrialOutcomeRecord, + trialFailureFields, trialSessions, } from '../../test/helpers/eval-store'; +import { readSessionLedger, type SessionLedgerRow } from '../../test/helpers/session-ledger'; import { E2E_KINDS } from '../../test/helpers/touchfiles-data'; import { manualReviewProblem } from '../../test/helpers/cookie-workflow-manual-review'; import { preflightAnthropicApi } from '../../test/helpers/anthropic-preflight'; @@ -225,6 +227,11 @@ export interface JUnitCensus { const bump = (counts: Map, key: string) => counts.set(key, (counts.get(key) ?? 0) + 1); +/** Ledger rows of one JUnit case: its own id or test name, else the shard's only case. */ +function caseSessions(rows: SessionLedgerRow[], id: string, onlyCase: boolean): SessionLedgerRow[] { + return rows.filter(row => row.case === id || (!row.case && (onlyCase || row.test_name === id || row.test_name === CASE_TEST_NAMES[id]))); +} + export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact[], recordsByShard: Map, base: Record, cliVersion?: string): JUnitCensus { const census: JUnitCensus = { ruleCases: [], history: [], failedShards: new Set(), skipped: [], deselected: new Map(), shardSkipReasons: new Map(), unattributed: [] }; @@ -245,6 +252,7 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact else census.unattributed.push(`${key} :: ${tc.name || '(no name)'} (${tc.outcome})`); } const records = recordsByShard.get(key) ?? []; + const ledger = readSessionLedger(path.join(root, 'shards', shardSlug([key]))); for (const [id, cases] of byCase) { const kind = (E2E_KINDS[id] ?? 'rule') as EvalCaseKind; const failed = cases.find(tc => tc.outcome === 'failed'); @@ -255,6 +263,9 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact : failed.failureType === 'TimeoutError' ? 'timeout' : failedRecord ? failureClassOf(failedRecord) : 'assertion'; const error = sanitizeTrialError(failedRecord?.error ?? failed?.message); const rerun = Object.hasOwn(E2E_TIERS, id) ? rerunCommand(manifest.tier, id, 1) : `EVALS=1 EVALS_TIER=${manifest.tier} bun test ${shardFile(key)}`; + const sessions = caseSessions(ledger, id, byCase.size === 1); + const diagnosis = failed && failureClass ? trialFailureFields({ failure_class: failureClass, exit_reason: failedRecord?.exit_reason, + error: failedRecord?.error ?? failed.message, sessions, record: failedRecord }) : undefined; if (failed) census.failedShards.add(key); if (caseOutcome === 'skipped') { bump(reasons, RUNTIME_SKIP_REASON); @@ -264,7 +275,7 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact ...(failed ? { line: `✗ ${id} ${kind} FAIL ${failureClass}${failedRecord?.exit_reason === 'timeout' && failedRecord?.timeout_at_turn !== undefined ? ` at turn ${failedRecord.timeout_at_turn}` : ''}${error ? ` — ${error}` : ''} [slice ${result.sliceIndex}, attempt ${result.attempt ?? 1}] rerun: ${rerun}` } : {}) }); census.history.push({ ...base, case: id, file: shardFile(key), kind, trial: 1, panel: { n: 1, k: 1 }, outcome: caseOutcome, ...(failureClass ? { failure_class: failureClass } : {}), ...(failedRecord?.exit_reason ? { exit_reason: String(failedRecord.exit_reason) } : {}), - ...(error && failed ? { error } : {}), duration_ms: cases.reduce((sum, tc) => sum + tc.timeMs, 0), + ...(error && failed ? { error } : {}), ...diagnosis, ...trialSessions(sessions), duration_ms: cases.reduce((sum, tc) => sum + tc.timeMs, 0), cost_usd: Math.round(mine.reduce((sum: number, r: any) => sum + (Number(r.cost_usd) || 0), 0) * 100) / 100, ...(typeof mine[0]?.model === 'string' ? { model: mine[0].model } : {}), ...(cliVersion ? { cli_version: cliVersion } : {}), quarantined: false, execution: outcome.reused ? 'reused' : 'executed', source: 'junit' } as TrialOutcomeRecord); @@ -576,6 +587,9 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo history.push({ ...common(attempt), case: t.case, file: shardFile(outcome.files[0]!), kind: t.kind, trial: t.trial, panel: t.panel, outcome: t.outcome, ...(t.outcome === 'failed' ? { failure_class: t.failure_class ?? 'assertion' } : {}), ...(t.exit_reason ? { exit_reason: t.exit_reason } : {}), ...(t.error ? { error: t.error } : {}), + ...(t.outcome === 'failed' && t.failure_cause ? { failure_cause: t.failure_cause } : {}), + ...(t.outcome === 'failed' && t.failure_cause_evidence ? { failure_cause_evidence: t.failure_cause_evidence } : {}), + ...(t.outcome === 'failed' && t.failure_detail ? { failure_detail: t.failure_detail } : {}), ...(t.sessions ? { sessions: t.sessions } : {}), duration_ms: t.duration_ms, cost_usd: t.cost_usd, ...(t.model ? { model: t.model } : {}), ...(outcome.reused ? { input_identity: outcome.reused.inputKey } : {}), quarantined: t.quarantined, execution: outcome.reused ? 'reused' : 'executed', source: 'shard' } as TrialOutcomeRecord); diff --git a/scripts/test-paid-shards.ts b/scripts/test-paid-shards.ts index 5fe708eb6..63ce29518 100644 --- a/scripts/test-paid-shards.ts +++ b/scripts/test-paid-shards.ts @@ -83,7 +83,9 @@ import { getProjectEvalDir, getClaudeCliVersion, isFinalizedEvalResultFile, evalEntryOutcome, failureClassOf, panelVerdict, sanitizeTrialError, formatTrialOutcomes, CONTRACT_VIOLATIONS_FILE, TRIAL_ENV, TRIAL_OUTCOME_SCHEMA, TRIAL_OUTCOMES_FILE, type EvalCaseKind, type PanelShape, type PanelVerdict, type TrialFailureClass, type TrialOutcome, type TrialOutcomeRecord, + trialFailureFields, trialSessions, type TrialFailureCause, type TrialFailureDetail, type TrialSessionSummary, } from '../test/helpers/eval-store'; +import { readSessionLedger, type SessionLedgerRow } from '../test/helpers/session-ledger'; import { E2E_KINDS } from '../test/helpers/touchfiles-data'; import { manualReviewProblem } from '../test/helpers/cookie-workflow-manual-review'; import { preflightAnthropicApi } from '../test/helpers/anthropic-preflight'; @@ -720,10 +722,16 @@ export interface ShardTrialRecord { cost_usd: number; duration_ms: number; model?: string; + failure_cause?: TrialFailureCause; + failure_cause_evidence?: string; + failure_detail?: TrialFailureDetail; + sessions?: TrialSessionSummary[]; } -/** Records and contract evidence an isolated shard left in its eval dir. */ -export function readTrialEvidence(evalDir: string | undefined): { records: any[]; contract: string | null } { +type TrialEvidence = { records: any[]; contract: string | null; sessions?: SessionLedgerRow[] }; + +/** Records, contract evidence and session-ledger rows an isolated shard left in its eval dir. */ +export function readTrialEvidence(evalDir: string | undefined): TrialEvidence { if (!evalDir || !fs.existsSync(evalDir)) return { records: [], contract: null }; const names = fs.readdirSync(evalDir); const parse = (name: string) => { try { return JSON.parse(fs.readFileSync(path.join(evalDir, name), 'utf8')); } catch { return null; } }; @@ -736,28 +744,31 @@ export function readTrialEvidence(evalDir: string | undefined): { records: any[] const line = fs.readFileSync(path.join(evalDir, CONTRACT_VIOLATIONS_FILE), 'utf8').split('\n').find(l => l.trim()); if (line) contract = String(JSON.parse(line).message ?? 'contract violation'); } catch { /* no sidecar */ } - return { records, contract }; + return { records, contract, sessions: readSessionLedger(evalDir) }; } /** Classify one isolated trial shard. Contract evidence always fails the trial. */ export function classifyTrialShard( outcome: Pick, caseId: string, trial: number, plan: CaseTrialPlan, - evidence: { records: any[]; contract: string | null }, + evidence: TrialEvidence, ): ShardTrialRecord { const failedRecord = evidence.records.find(record => record.passed === false) ?? evidence.records[0]; const base: ShardTrialRecord = { case: caseId, trial, kind: plan.kind, panel: plan.panel, quarantined: plan.quarantined, outcome: null, cost_usd: Math.round(evidence.records.reduce((sum, record) => sum + (Number(record.cost_usd) || 0), 0) * 100) / 100, duration_ms: outcome.elapsedMs, - ...(typeof failedRecord?.model === 'string' ? { model: failedRecord.model } : {}), + ...(typeof failedRecord?.model === 'string' ? { model: failedRecord.model } : {}), ...trialSessions(evidence.sessions ?? []), + }; + const failed = (failureClass: TrialFailureClass, error?: string): ShardTrialRecord => { + const cls = evidence.contract !== null ? 'contract' : failureClass; + const raw = evidence.contract ?? failedRecord?.error ?? error; + return { ...base, outcome: 'failed', failure_class: cls, + ...(failedRecord?.exit_reason ? { exit_reason: String(failedRecord.exit_reason) } : {}), + ...(Number.isInteger(failedRecord?.timeout_at_turn) ? { timeout_at_turn: failedRecord.timeout_at_turn } : {}), + ...(sanitizeTrialError(raw) ? { error: sanitizeTrialError(raw) } : {}), + ...trialFailureFields({ failure_class: cls, exit_reason: failedRecord?.exit_reason, error: raw, sessions: evidence.sessions, record: failedRecord }) }; }; - const failed = (failureClass: TrialFailureClass, error?: string): ShardTrialRecord => ({ - ...base, outcome: 'failed', failure_class: evidence.contract !== null ? 'contract' : failureClass, - ...(failedRecord?.exit_reason ? { exit_reason: String(failedRecord.exit_reason) } : {}), - ...(Number.isInteger(failedRecord?.timeout_at_turn) ? { timeout_at_turn: failedRecord.timeout_at_turn } : {}), - ...(sanitizeTrialError(evidence.contract ?? failedRecord?.error ?? error) ? { error: sanitizeTrialError(evidence.contract ?? failedRecord?.error ?? error) } : {}), - }); if (outcome.runnerError !== undefined) return { ...base, harness: `runner error: ${sanitizeTrialError(outcome.runnerError) ?? 'unknown'}` }; if (outcome.status === 'never-started') return { ...base, harness: 'never started' }; if (outcome.status === 'passed-empty') return { ...base, harness: 'hollow: executed no case' }; diff --git a/test/fixtures/trial-cause/replays.json b/test/fixtures/trial-cause/replays.json new file mode 100644 index 000000000..711d53f39 --- /dev/null +++ b/test/fixtures/trial-cause/replays.json @@ -0,0 +1 @@ +{"source":"stored census records; events reduced to type, subtype, ids, stop reasons and the first line of result text","cases":[{"run":"37195203538","case":"plan-ceo-review-format-approach","trial":1,"runner":"claude-p","exit_reason":"exit_code_1","error":"expect(received).toContain(expected)\n\nExpected to contain: \"exit_code_1\"\nReceived: [ \"success\", \"error_max_turns\" ]\n","failure_class":"assertion","expected":"refusal","events":[[0,{"type":"system","subtype":"init"}],[1,{"type":"assistant","message":{"id":"mt0","content":[]}}],[2,{"type":"assistant","message":{"id":"mt0","content":[{"type":"tool_use","id":"t1","name":"Bash"}]}}],[3,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t1"}]}}],[4,{"type":"system","subtype":"informational"}],[5,{"type":"user","message":{"content":[]}}],[6,{"type":"system","subtype":"thinking_tokens"}],[7,{"type":"system","subtype":"thinking_tokens"}],[8,{"type":"assistant","message":{"id":"mt2","content":[]}}],[9,{"type":"assistant","message":{"id":"mt2","content":[]}}],[10,{"type":"assistant","message":{"id":"mt2","content":[{"type":"tool_use","id":"t3","name":"Bash"}]}}],[11,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t3"}]}}],[12,{"type":"system","subtype":"model_refusal_no_fallback","api_refusal_category":"reasoning_extraction"}],[13,{"type":"assistant","message":{"id":"mt4","model":"","stop_reason":"refusal","stop_details":{"category":"reasoning_extraction"},"content":[{"type":"text","text":"API Error: Fable 5.1's safeguards flagged this message (https://www.anthropic.com/legal/aup). This sometimes happens with safe, normal conversations. Claude Code can't respond to this message with Fable 5.1."}]}}],[14,{"type":"result","subtype":"success","is_error":true,"stop_reason":"refusal","result":"API Error: Fable 5.1's safeguards flagged this message (https://www.anthropic.com/legal/aup). This sometimes happens with safe, normal conversations. Claude Code can't respond to this message with Fable 5.1."}]]},{"run":"37195203538","case":"plan-ceo-review-format-approach","trial":2,"runner":"claude-p","exit_reason":"exit_code_1","error":"expect(received).toContain(expected)\n\nExpected to contain: \"exit_code_1\"\nReceived: [ \"success\", \"error_max_turns\" ]\n","failure_class":"assertion","expected":"refusal","events":[[0,{"type":"system","subtype":"init"}],[1,{"type":"assistant","message":{"id":"mt0","content":[]}}],[2,{"type":"assistant","message":{"id":"mt0","content":[{"type":"tool_use","id":"t1","name":"Bash"}]}}],[3,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t1"}]}}],[4,{"type":"assistant","message":{"id":"mt2","content":[{"type":"tool_use","id":"t3","name":"Bash"}]}}],[5,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t3"}]}}],[6,{"type":"assistant","message":{"id":"mt4","content":[{"type":"tool_use","id":"t5","name":"Bash"}]}}],[7,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t5"}]}}],[8,{"type":"system","subtype":"thinking_tokens"}],[9,{"type":"system","subtype":"thinking_tokens"}],[10,{"type":"assistant","message":{"id":"mt6","content":[]}}],[11,{"type":"assistant","message":{"id":"mt6","content":[{"type":"tool_use","id":"t7","name":"Bash"}]}}],[12,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t7"}]}}],[13,{"type":"system","subtype":"thinking_tokens"}],[14,{"type":"system","subtype":"thinking_tokens"}],[15,{"type":"system","subtype":"thinking_tokens"}],[16,{"type":"system","subtype":"thinking_tokens"}],[17,{"type":"system","subtype":"thinking_tokens"}],[18,{"type":"system","subtype":"thinking_tokens"}],[19,{"type":"assistant","message":{"id":"mt8","content":[]}}],[20,{"type":"system","subtype":"informational"}],[21,{"type":"user","message":{"content":[]}}],[22,{"type":"system","subtype":"thinking_tokens"}],[23,{"type":"system","subtype":"thinking_tokens"}],[24,{"type":"assistant","message":{"id":"mt9","content":[]}}],[25,{"type":"assistant","message":{"id":"mt9","content":[{"type":"tool_use","id":"t10","name":"Write"}]}}],[26,{"type":"system","subtype":"model_refusal_no_fallback","api_refusal_category":"reasoning_extraction"}],[27,{"type":"assistant","message":{"id":"mt11","model":"","stop_reason":"refusal","stop_details":{"category":"reasoning_extraction"},"content":[{"type":"text","text":"API Error: Fable 5.1's safeguards flagged this message (https://www.anthropic.com/legal/aup). This sometimes happens with safe, normal conversations. Claude Code can't respond to this message with Fable 5.1."}]}}],[28,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t10"}]}}],[29,{"type":"assistant","message":{"id":"mt12","content":[]}}],[30,{"type":"assistant","message":{"id":"mt12","content":[{"type":"tool_use","id":"t13","name":"Write"}]}}],[31,{"type":"system","subtype":"model_refusal_no_fallback","api_refusal_category":"reasoning_extraction"}],[32,{"type":"assistant","message":{"id":"mt14","model":"","stop_reason":"refusal","stop_details":{"category":"reasoning_extraction"},"content":[{"type":"text","text":"API Error: Fable 5.1's safeguards flagged this message (https://www.anthropic.com/legal/aup). This sometimes happens with safe, normal conversations. Claude Code can't respond to this message with Fable 5.1."}]}}],[33,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t13"}]}}],[34,{"type":"system","subtype":"thinking_tokens"}],[35,{"type":"system","subtype":"thinking_tokens"}],[36,{"type":"assistant","message":{"id":"mt15","content":[]}}],[37,{"type":"assistant","message":{"id":"mt15","content":[{"type":"tool_use","id":"t16","name":"Bash"}]}}],[38,{"type":"system","subtype":"model_refusal_no_fallback","api_refusal_category":"reasoning_extraction"}],[39,{"type":"assistant","message":{"id":"mt17","model":"","stop_reason":"refusal","stop_details":{"category":"reasoning_extraction"},"content":[{"type":"text","text":"API Error: Fable 5.1's safeguards flagged this message (https://www.anthropic.com/legal/aup). This sometimes happens with safe, normal conversations. Claude Code can't respond to this message with Fable 5.1."}]}}],[40,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t16"}]}}],[41,{"type":"assistant","message":{"id":"mt18","content":[]}}],[42,{"type":"assistant","message":{"id":"mt18","content":[{"type":"tool_use","id":"t19","name":"Bash"}]}}],[43,{"type":"system","subtype":"model_refusal_no_fallback","api_refusal_category":"reasoning_extraction"}],[44,{"type":"assistant","message":{"id":"mt20","model":"","stop_reason":"refusal","stop_details":{"category":"reasoning_extraction"},"content":[{"type":"text","text":"API Error: Fable 5.1's safeguards flagged this message (https://www.anthropic.com/legal/aup). This sometimes happens with safe, normal conversations. Claude Code can't respond to this message with Fable 5.1."}]}}],[45,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t19"}]}}],[46,{"type":"system","subtype":"model_refusal_no_fallback","api_refusal_category":"reasoning_extraction"}],[47,{"type":"assistant","message":{"id":"mt21","model":"","stop_reason":"refusal","stop_details":{"category":"reasoning_extraction"},"content":[{"type":"text","text":"API Error: Fable 5.1's safeguards flagged this message (https://www.anthropic.com/legal/aup). This sometimes happens with safe, normal conversations. Claude Code can't respond to this message with Fable 5.1."}]}}],[48,{"type":"result","subtype":"success","is_error":true,"stop_reason":"refusal","result":"API Error: Fable 5.1's safeguards flagged this message (https://www.anthropic.com/legal/aup). This sometimes happens with safe, normal conversations. Claude Code can't respond to this message with Fable 5.1."}]]},{"run":"37186854666","case":"shared-libs-review-revalidation","trial":1,"runner":"agent-sdk","exit_reason":"timeout","error":"Error: Claude Code process aborted by user\nNo terminal billing event; actual cost is unknown. Raw usage is retained in the transcript.\nError: Claude Code process aborted by user","failure_class":"timeout","duration_ms":657194,"expected":"session_timeout","events":[[0,{"type":"system","subtype":"init"}],[0,{"type":"system","subtype":"thinking_tokens"}],[0,{"type":"system","subtype":"thinking_tokens"}],[0,{"type":"assistant","message":{"id":"mt0","content":[]}}],[679,{"type":"assistant","message":{"id":"mt0","content":[]}}],[1414,{"type":"assistant","message":{"id":"mt0","content":[{"type":"tool_use","id":"t1","name":"Read"}]}}],[1421,{"type":"assistant","message":{"id":"mt0","content":[{"type":"tool_use","id":"t2","name":"Read"}]}}],[1426,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t2"}]}}],[1589,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t1"}]}}],[2190,{"type":"assistant","message":{"id":"mt0","content":[{"type":"tool_use","id":"t3","name":"Read"}]}}],[2195,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t3"}]}}],[3527,{"type":"assistant","message":{"id":"mt0","content":[{"type":"tool_use","id":"t4","name":"Read"}]}}],[3533,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t4"}]}}],[3629,{"type":"assistant","message":{"id":"mt0","content":[{"type":"tool_use","id":"t5","name":"Read"}]}}],[3634,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t5"}]}}],[7348,{"type":"assistant","message":{"id":"mt0","content":[{"type":"tool_use","id":"t6","name":"Bash"}]}}],[7571,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t6"}]}}],[11333,{"type":"assistant","message":{"id":"mt7","content":[]}}],[11334,{"type":"assistant","message":{"id":"mt7","content":[]}}],[12410,{"type":"assistant","message":{"id":"mt7","content":[{"type":"tool_use","id":"t8","name":"Bash"}]}}],[12631,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t8"}]}}],[16387,{"type":"assistant","message":{"id":"mt9","content":[{"type":"tool_use","id":"t10","name":"Read"}]}}],[16394,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t10"}]}}],[16394,{"type":"system","subtype":"thinking_tokens"}],[16394,{"type":"system","subtype":"thinking_tokens"}],[16394,{"type":"system","subtype":"thinking_tokens"}],[23696,{"type":"assistant","message":{"id":"mt11","content":[]}}],[23696,{"type":"system","subtype":"thinking_tokens"}],[23696,{"type":"system","subtype":"thinking_tokens"}],[23696,{"type":"system","subtype":"thinking_tokens"}],[23696,{"type":"system","subtype":"thinking_tokens"}],[23696,{"type":"system","subtype":"thinking_tokens"}],[23696,{"type":"system","subtype":"thinking_tokens"}],[23696,{"type":"system","subtype":"thinking_tokens"}],[23696,{"type":"system","subtype":"thinking_tokens"}],[23696,{"type":"system","subtype":"thinking_tokens"}],[23696,{"type":"system","subtype":"thinking_tokens"}],[23696,{"type":"system","subtype":"thinking_tokens"}],[23696,{"type":"system","subtype":"thinking_tokens"}],[23696,{"type":"system","subtype":"thinking_tokens"}],[23698,{"type":"assistant","message":{"id":"mt11","content":[]}}],[25532,{"type":"assistant","message":{"id":"mt11","content":[{"type":"tool_use","id":"t12","name":"Bash"}]}}],[25602,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t12"}]}}],[26906,{"type":"assistant","message":{"id":"mt11","content":[{"type":"tool_use","id":"t13","name":"Bash"}]}}],[27100,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t13"}]}}],[27783,{"type":"assistant","message":{"id":"mt11","content":[{"type":"tool_use","id":"t14","name":"Read"}]}}],[27787,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t14"}]}}],[28658,{"type":"assistant","message":{"id":"mt11","content":[{"type":"tool_use","id":"t15","name":"Read"}]}}],[28662,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t15"}]}}],[29537,{"type":"assistant","message":{"id":"mt11","content":[{"type":"tool_use","id":"t16","name":"Read"}]}}],[29541,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t16"}]}}],[30350,{"type":"assistant","message":{"id":"mt11","content":[{"type":"tool_use","id":"t17","name":"Read"}]}}],[30354,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t17"}]}}],[31284,{"type":"assistant","message":{"id":"mt11","content":[{"type":"tool_use","id":"t18","name":"Read"}]}}],[31289,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t18"}]}}],[31975,{"type":"assistant","message":{"id":"mt11","content":[{"type":"tool_use","id":"t19","name":"Read"}]}}],[31982,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t19"}]}}],[31982,{"type":"system","subtype":"thinking_tokens"}],[31982,{"type":"system","subtype":"thinking_tokens"}],[31982,{"type":"system","subtype":"thinking_tokens"}],[41380,{"type":"assistant","message":{"id":"mt20","content":[]}}],[41380,{"type":"system","subtype":"thinking_tokens"}],[41380,{"type":"system","subtype":"thinking_tokens"}],[41380,{"type":"system","subtype":"thinking_tokens"}],[41380,{"type":"system","subtype":"thinking_tokens"}],[41380,{"type":"system","subtype":"thinking_tokens"}],[41380,{"type":"system","subtype":"thinking_tokens"}],[41380,{"type":"system","subtype":"thinking_tokens"}],[41380,{"type":"system","subtype":"thinking_tokens"}],[41380,{"type":"system","subtype":"thinking_tokens"}],[41380,{"type":"system","subtype":"thinking_tokens"}],[41380,{"type":"system","subtype":"thinking_tokens"}],[41380,{"type":"system","subtype":"thinking_tokens"}],[41380,{"type":"system","subtype":"thinking_tokens"}],[41380,{"type":"system","subtype":"thinking_tokens"}],[41380,{"type":"system","subtype":"thinking_tokens"}],[41380,{"type":"system","subtype":"thinking_tokens"}],[41380,{"type":"system","subtype":"thinking_tokens"}],[41380,{"type":"system","subtype":"thinking_tokens"}],[41380,{"type":"system","subtype":"thinking_tokens"}],[41380,{"type":"system","subtype":"thinking_tokens"}],[41382,{"type":"assistant","message":{"id":"mt20","content":[]}}],[47062,{"type":"assistant","message":{"id":"mt20","content":[{"type":"tool_use","id":"t21","name":"Bash"}]}}],[47604,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t21"}]}}],[47604,{"type":"system","subtype":"thinking_tokens"}],[47604,{"type":"system","subtype":"thinking_tokens"}],[47604,{"type":"system","subtype":"thinking_tokens"}],[47604,{"type":"system","subtype":"thinking_tokens"}],[47604,{"type":"system","subtype":"thinking_tokens"}],[47604,{"type":"system","subtype":"thinking_tokens"}],[47604,{"type":"system","subtype":"thinking_tokens"}],[47604,{"type":"system","subtype":"thinking_tokens"}],[47604,{"type":"system","subtype":"thinking_tokens"}],[63786,{"type":"assistant","message":{"id":"mt22","content":[]}}],[63786,{"type":"system","subtype":"thinking_tokens"}],[63786,{"type":"system","subtype":"thinking_tokens"}],[63786,{"type":"system","subtype":"thinking_tokens"}],[63786,{"type":"system","subtype":"thinking_tokens"}],[63786,{"type":"system","subtype":"thinking_tokens"}],[63786,{"type":"system","subtype":"thinking_tokens"}],[63786,{"type":"system","subtype":"thinking_tokens"}],[63786,{"type":"system","subtype":"thinking_tokens"}],[63786,{"type":"system","subtype":"thinking_tokens"}],[63786,{"type":"system","subtype":"thinking_tokens"}],[63786,{"type":"system","subtype":"thinking_tokens"}],[63786,{"type":"system","subtype":"thinking_tokens"}],[63786,{"type":"system","subtype":"thinking_tokens"}],[63786,{"type":"system","subtype":"thinking_tokens"}],[63786,{"type":"system","subtype":"thinking_tokens"}],[63786,{"type":"system","subtype":"thinking_tokens"}],[63786,{"type":"system","subtype":"thinking_tokens"}],[63786,{"type":"system","subtype":"thinking_tokens"}],[63786,{"type":"system","subtype":"thinking_tokens"}],[63786,{"type":"system","subtype":"thinking_tokens"}],[63786,{"type":"system","subtype":"thinking_tokens"}],[63790,{"type":"assistant","message":{"id":"mt22","content":[]}}],[66543,{"type":"assistant","message":{"id":"mt22","content":[{"type":"tool_use","id":"t23","name":"Bash"}]}}],[66672,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t23"}]}}],[69997,{"type":"assistant","message":{"id":"mt24","content":[]}}],[69998,{"type":"assistant","message":{"id":"mt24","content":[]}}],[72860,{"type":"assistant","message":{"id":"mt24","content":[{"type":"tool_use","id":"t25","name":"Bash"}]}}],[73024,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t25"}]}}],[73024,{"type":"system","subtype":"thinking_tokens"}],[73024,{"type":"system","subtype":"thinking_tokens"}],[73024,{"type":"system","subtype":"thinking_tokens"}],[80192,{"type":"assistant","message":{"id":"mt26","content":[]}}],[80192,{"type":"system","subtype":"thinking_tokens"}],[80192,{"type":"system","subtype":"thinking_tokens"}],[80192,{"type":"system","subtype":"thinking_tokens"}],[80192,{"type":"system","subtype":"thinking_tokens"}],[80192,{"type":"system","subtype":"thinking_tokens"}],[80192,{"type":"system","subtype":"thinking_tokens"}],[80192,{"type":"system","subtype":"thinking_tokens"}],[80192,{"type":"system","subtype":"thinking_tokens"}],[80192,{"type":"system","subtype":"thinking_tokens"}],[80192,{"type":"system","subtype":"thinking_tokens"}],[80192,{"type":"system","subtype":"thinking_tokens"}],[80192,{"type":"system","subtype":"thinking_tokens"}],[80192,{"type":"system","subtype":"thinking_tokens"}],[80194,{"type":"assistant","message":{"id":"mt26","content":[]}}],[82929,{"type":"assistant","message":{"id":"mt26","content":[{"type":"tool_use","id":"t27","name":"Bash"}]}}],[83331,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t27"}]}}],[92916,{"type":"assistant","message":{"id":"mt28","content":[]}}],[92916,{"type":"result","subtype":"success","is_error":false,"stop_reason":"end_turn","result":"Persistence succeeded and the read-back shows the new record bound to my start token with a verified binding and matching start and end working-tree digests."}],[92916,{"type":"system","subtype":"init"}],[92916,{"type":"system","subtype":"thinking_tokens"}],[92916,{"type":"system","subtype":"thinking_tokens"}],[-914,{"type":"assistant","message":{"id":"mt29","content":[]}}],[-161,{"type":"assistant","message":{"id":"mt29","content":[]}}],[1041,{"type":"assistant","message":{"id":"mt29","content":[{"type":"tool_use","id":"t30","name":"Read"}]}}],[1156,{"type":"assistant","message":{"id":"mt29","content":[{"type":"tool_use","id":"t31","name":"Read"}]}}],[1162,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t31"}]}}],[1203,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t30"}]}}],[1970,{"type":"assistant","message":{"id":"mt29","content":[{"type":"tool_use","id":"t32","name":"Read"}]}}],[1975,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t32"}]}}],[2845,{"type":"assistant","message":{"id":"mt29","content":[{"type":"tool_use","id":"t33","name":"Read"}]}}],[2851,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t33"}]}}],[3409,{"type":"assistant","message":{"id":"mt29","content":[{"type":"tool_use","id":"t34","name":"Read"}]}}],[3414,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t34"}]}}],[8170,{"type":"assistant","message":{"id":"mt29","content":[{"type":"tool_use","id":"t35","name":"Bash"}]}}],[8418,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t35"}]}}],[8418,{"type":"system","subtype":"thinking_tokens"}],[8418,{"type":"system","subtype":"thinking_tokens"}],[12906,{"type":"assistant","message":{"id":"mt36","content":[]}}],[12907,{"type":"assistant","message":{"id":"mt36","content":[]}}],[13920,{"type":"assistant","message":{"id":"mt36","content":[{"type":"tool_use","id":"t37","name":"Bash"}]}}],[14144,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t37"}]}}],[18288,{"type":"assistant","message":{"id":"mt38","content":[{"type":"tool_use","id":"t39","name":"Read"}]}}],[18296,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t39"}]}}],[18296,{"type":"system","subtype":"thinking_tokens"}],[18296,{"type":"system","subtype":"thinking_tokens"}],[18296,{"type":"system","subtype":"thinking_tokens"}],[18296,{"type":"system","subtype":"thinking_tokens"}],[18296,{"type":"system","subtype":"thinking_tokens"}],[26436,{"type":"assistant","message":{"id":"mt40","content":[]}}],[26438,{"type":"assistant","message":{"id":"mt40","content":[]}}],[31104,{"type":"assistant","message":{"id":"mt40","content":[{"type":"tool_use","id":"t41","name":"Bash"}]}}],[31387,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t41"}]}}],[31918,{"type":"assistant","message":{"id":"mt40","content":[{"type":"tool_use","id":"t42","name":"Read"}]}}],[31923,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t42"}]}}],[32795,{"type":"assistant","message":{"id":"mt40","content":[{"type":"tool_use","id":"t43","name":"Read"}]}}],[32799,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t43"}]}}],[33608,{"type":"assistant","message":{"id":"mt40","content":[{"type":"tool_use","id":"t44","name":"Read"}]}}],[33612,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t44"}]}}],[34484,{"type":"assistant","message":{"id":"mt40","content":[{"type":"tool_use","id":"t45","name":"Read"}]}}],[34492,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t45"}]}}],[35360,{"type":"assistant","message":{"id":"mt40","content":[{"type":"tool_use","id":"t46","name":"Read"}]}}],[35364,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t46"}]}}],[36005,{"type":"assistant","message":{"id":"mt40","content":[{"type":"tool_use","id":"t47","name":"Read"}]}}],[36012,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t47"}]}}],[36012,{"type":"system","subtype":"thinking_tokens"}],[36012,{"type":"system","subtype":"thinking_tokens"}],[36012,{"type":"system","subtype":"thinking_tokens"}],[36012,{"type":"system","subtype":"thinking_tokens"}],[36012,{"type":"system","subtype":"thinking_tokens"}],[36012,{"type":"system","subtype":"thinking_tokens"}],[36012,{"type":"system","subtype":"thinking_tokens"}],[36012,{"type":"system","subtype":"thinking_tokens"}],[50298,{"type":"assistant","message":{"id":"mt48","content":[]}}],[50298,{"type":"system","subtype":"thinking_tokens"}],[50298,{"type":"system","subtype":"thinking_tokens"}],[50298,{"type":"system","subtype":"thinking_tokens"}],[50298,{"type":"system","subtype":"thinking_tokens"}],[50298,{"type":"system","subtype":"thinking_tokens"}],[50298,{"type":"system","subtype":"thinking_tokens"}],[50298,{"type":"system","subtype":"thinking_tokens"}],[50298,{"type":"system","subtype":"thinking_tokens"}],[50298,{"type":"system","subtype":"thinking_tokens"}],[50298,{"type":"system","subtype":"thinking_tokens"}],[50298,{"type":"system","subtype":"thinking_tokens"}],[50298,{"type":"system","subtype":"thinking_tokens"}],[50298,{"type":"system","subtype":"thinking_tokens"}],[50304,{"type":"assistant","message":{"id":"mt48","content":[]}}],[53738,{"type":"assistant","message":{"id":"mt48","content":[{"type":"tool_use","id":"t49","name":"Bash"}]}}],[54268,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t49"}]}}],[54268,{"type":"system","subtype":"thinking_tokens"}],[54268,{"type":"system","subtype":"thinking_tokens"}],[54268,{"type":"system","subtype":"thinking_tokens"}],[54268,{"type":"system","subtype":"thinking_tokens"}],[54268,{"type":"system","subtype":"thinking_tokens"}],[54268,{"type":"system","subtype":"thinking_tokens"}],[54268,{"type":"system","subtype":"thinking_tokens"}],[54268,{"type":"system","subtype":"thinking_tokens"}],[54268,{"type":"system","subtype":"thinking_tokens"}],[54268,{"type":"system","subtype":"thinking_tokens"}],[54268,{"type":"system","subtype":"thinking_tokens"}],[54268,{"type":"system","subtype":"thinking_tokens"}],[54268,{"type":"system","subtype":"thinking_tokens"}],[54268,{"type":"system","subtype":"thinking_tokens"}],[76924,{"type":"assistant","message":{"id":"mt50","content":[]}}],[76924,{"type":"system","subtype":"thinking_tokens"}],[76924,{"type":"system","subtype":"thinking_tokens"}],[76924,{"type":"system","subtype":"thinking_tokens"}],[76924,{"type":"system","subtype":"thinking_tokens"}],[76924,{"type":"system","subtype":"thinking_tokens"}],[76924,{"type":"system","subtype":"thinking_tokens"}],[76924,{"type":"system","subtype":"thinking_tokens"}],[76924,{"type":"system","subtype":"thinking_tokens"}],[76924,{"type":"system","subtype":"thinking_tokens"}],[76924,{"type":"system","subtype":"thinking_tokens"}],[76924,{"type":"system","subtype":"thinking_tokens"}],[76924,{"type":"system","subtype":"thinking_tokens"}],[76924,{"type":"system","subtype":"thinking_tokens"}],[76930,{"type":"assistant","message":{"id":"mt50","content":[]}}],[80683,{"type":"assistant","message":{"id":"mt50","content":[{"type":"tool_use","id":"t51","name":"AskUserQuestion"}]}}],[80690,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t51"}]}}],[80690,{"type":"system","subtype":"thinking_tokens"}],[80690,{"type":"system","subtype":"thinking_tokens"}],[84902,{"type":"assistant","message":{"id":"mt52","content":[]}}],[84903,{"type":"assistant","message":{"id":"mt52","content":[]}}],[90405,{"type":"assistant","message":{"id":"mt52","content":[{"type":"tool_use","id":"t53","name":"Bash"}]}}],[90571,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t53"}]}}],[93365,{"type":"assistant","message":{"id":"mt54","content":[]}}],[96013,{"type":"assistant","message":{"id":"mt54","content":[{"type":"tool_use","id":"t55","name":"Bash"}]}}],[96175,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t55"}]}}],[96175,{"type":"system","subtype":"thinking_tokens"}],[96175,{"type":"system","subtype":"thinking_tokens"}],[96175,{"type":"system","subtype":"thinking_tokens"}],[96175,{"type":"system","subtype":"thinking_tokens"}],[102635,{"type":"assistant","message":{"id":"mt56","content":[]}}],[102636,{"type":"assistant","message":{"id":"mt56","content":[]}}],[111228,{"type":"assistant","message":{"id":"mt56","content":[{"type":"tool_use","id":"t57","name":"Bash"}]}}],[111963,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t57"}]}}],[122245,{"type":"assistant","message":{"id":"mt58","content":[]}}],[122245,{"type":"result","subtype":"success","is_error":false,"stop_reason":"end_turn","result":"Persistence succeeded and the read-back shows the new record bound to this pass's start token with a verified binding and logger-computed coverage over all three paths."}],[122245,{"type":"system","subtype":"init"}],[122245,{"type":"system","subtype":"thinking_tokens"}],[122245,{"type":"system","subtype":"thinking_tokens"}],[-809,{"type":"assistant","message":{"id":"mt59","content":[]}}],[-121,{"type":"assistant","message":{"id":"mt59","content":[]}}],[695,{"type":"assistant","message":{"id":"mt59","content":[{"type":"tool_use","id":"t60","name":"Read"}]}}],[849,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t60"}]}}],[1204,{"type":"assistant","message":{"id":"mt59","content":[{"type":"tool_use","id":"t61","name":"Read"}]}}],[1209,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t61"}]}}],[2072,{"type":"assistant","message":{"id":"mt59","content":[{"type":"tool_use","id":"t62","name":"Read"}]}}],[2078,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t62"}]}}],[2886,{"type":"assistant","message":{"id":"mt59","content":[{"type":"tool_use","id":"t63","name":"Read"}]}}],[2891,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t63"}]}}],[3448,{"type":"assistant","message":{"id":"mt59","content":[{"type":"tool_use","id":"t64","name":"Read"}]}}],[3453,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t64"}]}}],[8240,{"type":"assistant","message":{"id":"mt59","content":[{"type":"tool_use","id":"t65","name":"Bash"}]}}],[8472,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t65"}]}}],[11730,{"type":"assistant","message":{"id":"mt66","content":[]}}],[12788,{"type":"assistant","message":{"id":"mt66","content":[{"type":"tool_use","id":"t67","name":"Bash"}]}}],[13012,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t67"}]}}],[17042,{"type":"assistant","message":{"id":"mt68","content":[{"type":"tool_use","id":"t69","name":"Read"}]}}],[17049,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t69"}]}}],[17049,{"type":"system","subtype":"thinking_tokens"}],[17049,{"type":"system","subtype":"thinking_tokens"}],[17049,{"type":"system","subtype":"thinking_tokens"}],[17049,{"type":"system","subtype":"thinking_tokens"}],[25401,{"type":"assistant","message":{"id":"mt70","content":[]}}],[25401,{"type":"system","subtype":"thinking_tokens"}],[25401,{"type":"system","subtype":"thinking_tokens"}],[25401,{"type":"system","subtype":"thinking_tokens"}],[25401,{"type":"system","subtype":"thinking_tokens"}],[25401,{"type":"system","subtype":"thinking_tokens"}],[25401,{"type":"system","subtype":"thinking_tokens"}],[25401,{"type":"system","subtype":"thinking_tokens"}],[25401,{"type":"system","subtype":"thinking_tokens"}],[25401,{"type":"system","subtype":"thinking_tokens"}],[25401,{"type":"system","subtype":"thinking_tokens"}],[25401,{"type":"system","subtype":"thinking_tokens"}],[25401,{"type":"system","subtype":"thinking_tokens"}],[25401,{"type":"system","subtype":"thinking_tokens"}],[25401,{"type":"system","subtype":"thinking_tokens"}],[25403,{"type":"assistant","message":{"id":"mt70","content":[]}}],[27167,{"type":"assistant","message":{"id":"mt70","content":[{"type":"tool_use","id":"t71","name":"Bash"}]}}],[27234,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t71"}]}}],[28045,{"type":"assistant","message":{"id":"mt70","content":[{"type":"tool_use","id":"t72","name":"Read"}]}}],[28050,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t72"}]}}],[28920,{"type":"assistant","message":{"id":"mt70","content":[{"type":"tool_use","id":"t73","name":"Read"}]}}],[28925,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t73"}]}}],[29734,{"type":"assistant","message":{"id":"mt70","content":[{"type":"tool_use","id":"t74","name":"Read"}]}}],[29738,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t74"}]}}],[30609,{"type":"assistant","message":{"id":"mt70","content":[{"type":"tool_use","id":"t75","name":"Read"}]}}],[30614,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t75"}]}}],[31489,{"type":"assistant","message":{"id":"mt70","content":[{"type":"tool_use","id":"t76","name":"Read"}]}}],[31493,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t76"}]}}],[32298,{"type":"assistant","message":{"id":"mt70","content":[{"type":"tool_use","id":"t77","name":"Read"}]}}],[32303,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t77"}]}}],[33168,{"type":"assistant","message":{"id":"mt70","content":[{"type":"tool_use","id":"t78","name":"Bash"}]}}],[33363,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t78"}]}}],[33363,{"type":"system","subtype":"thinking_tokens"}],[33363,{"type":"system","subtype":"thinking_tokens"}],[33363,{"type":"system","subtype":"thinking_tokens"}],[33363,{"type":"system","subtype":"thinking_tokens"}],[33363,{"type":"system","subtype":"thinking_tokens"}],[44070,{"type":"assistant","message":{"id":"mt79","content":[]}}],[44070,{"type":"system","subtype":"thinking_tokens"}],[44070,{"type":"system","subtype":"thinking_tokens"}],[44070,{"type":"system","subtype":"thinking_tokens"}],[44070,{"type":"system","subtype":"thinking_tokens"}],[44070,{"type":"system","subtype":"thinking_tokens"}],[44070,{"type":"system","subtype":"thinking_tokens"}],[44070,{"type":"system","subtype":"thinking_tokens"}],[44070,{"type":"system","subtype":"thinking_tokens"}],[44070,{"type":"system","subtype":"thinking_tokens"}],[44070,{"type":"system","subtype":"thinking_tokens"}],[44070,{"type":"system","subtype":"thinking_tokens"}],[44070,{"type":"system","subtype":"thinking_tokens"}],[44070,{"type":"system","subtype":"thinking_tokens"}],[44070,{"type":"system","subtype":"thinking_tokens"}],[44070,{"type":"system","subtype":"thinking_tokens"}],[44072,{"type":"assistant","message":{"id":"mt79","content":[]}}],[49989,{"type":"assistant","message":{"id":"mt79","content":[{"type":"tool_use","id":"t80","name":"Bash"}]}}],[50540,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t80"}]}}],[50540,{"type":"system","subtype":"thinking_tokens"}],[50540,{"type":"system","subtype":"thinking_tokens"}],[50540,{"type":"system","subtype":"thinking_tokens"}],[50540,{"type":"system","subtype":"thinking_tokens"}],[50540,{"type":"system","subtype":"thinking_tokens"}],[50540,{"type":"system","subtype":"thinking_tokens"}],[50540,{"type":"system","subtype":"thinking_tokens"}],[50540,{"type":"system","subtype":"thinking_tokens"}],[50540,{"type":"system","subtype":"thinking_tokens"}],[50540,{"type":"system","subtype":"thinking_tokens"}],[50540,{"type":"system","subtype":"thinking_tokens"}],[50540,{"type":"system","subtype":"thinking_tokens"}],[50540,{"type":"system","subtype":"thinking_tokens"}],[50540,{"type":"system","subtype":"thinking_tokens"}],[50540,{"type":"system","subtype":"thinking_tokens"}],[77752,{"type":"assistant","message":{"id":"mt81","content":[]}}],[77752,{"type":"system","subtype":"thinking_tokens"}],[77752,{"type":"system","subtype":"thinking_tokens"}],[77752,{"type":"system","subtype":"thinking_tokens"}],[77752,{"type":"system","subtype":"thinking_tokens"}],[77752,{"type":"system","subtype":"thinking_tokens"}],[77752,{"type":"system","subtype":"thinking_tokens"}],[77752,{"type":"system","subtype":"thinking_tokens"}],[77752,{"type":"system","subtype":"thinking_tokens"}],[77752,{"type":"system","subtype":"thinking_tokens"}],[77752,{"type":"system","subtype":"thinking_tokens"}],[77752,{"type":"system","subtype":"thinking_tokens"}],[77752,{"type":"system","subtype":"thinking_tokens"}],[77752,{"type":"system","subtype":"thinking_tokens"}],[77752,{"type":"system","subtype":"thinking_tokens"}],[77752,{"type":"system","subtype":"thinking_tokens"}],[77752,{"type":"system","subtype":"thinking_tokens"}],[77752,{"type":"system","subtype":"thinking_tokens"}],[77752,{"type":"system","subtype":"thinking_tokens"}],[77752,{"type":"system","subtype":"thinking_tokens"}],[77752,{"type":"system","subtype":"thinking_tokens"}],[77752,{"type":"system","subtype":"thinking_tokens"}],[77752,{"type":"system","subtype":"thinking_tokens"}],[77752,{"type":"system","subtype":"thinking_tokens"}],[77752,{"type":"system","subtype":"thinking_tokens"}],[77752,{"type":"system","subtype":"thinking_tokens"}],[77757,{"type":"assistant","message":{"id":"mt81","content":[]}}],[80707,{"type":"assistant","message":{"id":"mt81","content":[{"type":"tool_use","id":"t82","name":"AskUserQuestion"}]}}],[80713,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t82"}]}}],[80713,{"type":"system","subtype":"thinking_tokens"}],[80713,{"type":"system","subtype":"thinking_tokens"}],[80713,{"type":"system","subtype":"thinking_tokens"}],[86338,{"type":"assistant","message":{"id":"mt83","content":[]}}],[86339,{"type":"assistant","message":{"id":"mt83","content":[]}}],[93438,{"type":"assistant","message":{"id":"mt83","content":[{"type":"tool_use","id":"t84","name":"Bash"}]}}],[93666,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t84"}]}}],[97111,{"type":"assistant","message":{"id":"mt85","content":[]}}],[99823,{"type":"assistant","message":{"id":"mt85","content":[{"type":"tool_use","id":"t86","name":"Bash"}]}}],[99985,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t86"}]}}],[99985,{"type":"system","subtype":"thinking_tokens"}],[99985,{"type":"system","subtype":"thinking_tokens"}],[99985,{"type":"system","subtype":"thinking_tokens"}],[99985,{"type":"system","subtype":"thinking_tokens"}],[109560,{"type":"assistant","message":{"id":"mt87","content":[]}}],[109560,{"type":"system","subtype":"thinking_tokens"}],[109560,{"type":"system","subtype":"thinking_tokens"}],[109560,{"type":"system","subtype":"thinking_tokens"}],[109560,{"type":"system","subtype":"thinking_tokens"}],[109560,{"type":"system","subtype":"thinking_tokens"}],[109560,{"type":"system","subtype":"thinking_tokens"}],[109560,{"type":"system","subtype":"thinking_tokens"}],[109560,{"type":"system","subtype":"thinking_tokens"}],[109560,{"type":"system","subtype":"thinking_tokens"}],[109563,{"type":"assistant","message":{"id":"mt87","content":[]}}],[116794,{"type":"assistant","message":{"id":"mt87","content":[{"type":"tool_use","id":"t88","name":"Bash"}]}}],[117534,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t88"}]}}],[126855,{"type":"assistant","message":{"id":"mt89","content":[]}}],[126855,{"type":"result","subtype":"success","is_error":false,"stop_reason":"end_turn","result":"Persistence succeeded. The read-back shows the new record bound to start and end working-tree digest 858ac83c, with a verified binding, freshness CURRENT, and logger-computed snapshot coverage over all three evidence paths."}],[126855,{"type":"system","subtype":"init"}],[126855,{"type":"system","subtype":"thinking_tokens"}],[126855,{"type":"system","subtype":"thinking_tokens"}],[96739,{"type":"assistant","message":{"id":"mt90","content":[]}}],[97496,{"type":"assistant","message":{"id":"mt90","content":[]}}],[98710,{"type":"assistant","message":{"id":"mt90","content":[{"type":"tool_use","id":"t91","name":"Read"}]}}],[98816,{"type":"assistant","message":{"id":"mt90","content":[{"type":"tool_use","id":"t92","name":"Read"}]}}],[98822,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t92"}]}}],[98880,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t91"}]}}],[99693,{"type":"assistant","message":{"id":"mt90","content":[{"type":"tool_use","id":"t93","name":"Read"}]}}],[99698,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t93"}]}}],[100506,{"type":"assistant","message":{"id":"mt90","content":[{"type":"tool_use","id":"t94","name":"Read"}]}}],[100511,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t94"}]}}],[101069,{"type":"assistant","message":{"id":"mt90","content":[{"type":"tool_use","id":"t95","name":"Read"}]}}],[101075,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t95"}]}}],[106044,{"type":"assistant","message":{"id":"mt90","content":[{"type":"tool_use","id":"t96","name":"Bash"}]}}],[106284,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t96"}]}}],[106284,{"type":"system","subtype":"thinking_tokens"}],[106284,{"type":"system","subtype":"thinking_tokens"}],[106284,{"type":"system","subtype":"thinking_tokens"}],[106284,{"type":"system","subtype":"thinking_tokens"}],[116104,{"type":"assistant","message":{"id":"mt97","content":[]}}],[116104,{"type":"system","subtype":"thinking_tokens"}],[116104,{"type":"system","subtype":"thinking_tokens"}],[116104,{"type":"system","subtype":"thinking_tokens"}],[116104,{"type":"system","subtype":"thinking_tokens"}],[116104,{"type":"system","subtype":"thinking_tokens"}],[116104,{"type":"system","subtype":"thinking_tokens"}],[116104,{"type":"system","subtype":"thinking_tokens"}],[116104,{"type":"system","subtype":"thinking_tokens"}],[116104,{"type":"system","subtype":"thinking_tokens"}],[116104,{"type":"system","subtype":"thinking_tokens"}],[116104,{"type":"system","subtype":"thinking_tokens"}],[116104,{"type":"system","subtype":"thinking_tokens"}],[116104,{"type":"system","subtype":"thinking_tokens"}],[116104,{"type":"system","subtype":"thinking_tokens"}],[116104,{"type":"system","subtype":"thinking_tokens"}],[116106,{"type":"assistant","message":{"id":"mt97","content":[]}}],[116110,{"type":"assistant","message":{"id":"mt97","content":[{"type":"tool_use","id":"t98","name":"Bash"}]}}],[116337,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t98"}]}}],[119685,{"type":"assistant","message":{"id":"mt99","content":[{"type":"tool_use","id":"t100","name":"Read"}]}}],[119693,{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t100"}]}}],[119693,{"type":"system","subtype":"thinking_tokens"}],[119693,{"type":"system","subtype":"thinking_tokens"}],[119693,{"type":"system","subtype":"thinking_tokens"}],[119693,{"type":"system","subtype":"thinking_tokens"}],[119693,{"type":"system","subtype":"thinking_tokens"}],[129359,{"type":"assistant","message":{"id":"mt101","content":[]}}],[129359,{"type":"system","subtype":"thinking_tokens"}],[129359,{"type":"system","subtype":"thinking_tokens"}],[129359,{"type":"system","subtype":"thinking_tokens"}],[129359,{"type":"system","subtype":"thinking_tokens"}],[129359,{"type":"system","subtype":"thinking_tokens"}],[129359,{"type":"system","subtype":"thinking_tokens"}],[129359,{"type":"system","subtype":"thinking_tokens"}],[129359,{"type":"system","subtype":"thinking_tokens"}],[129359,{"type":"system","subtype":"thinking_tokens"}],[129359,{"type":"system","subtype":"thinking_tokens"}],[129359,{"type":"system","subtype":"thinking_tokens"}],[129359,{"type":"system","subtype":"thinking_tokens"}],[129359,{"type":"system","subtype":"thinking_tokens"}],[129362,{"type":"assistant","message":{"id":"mt101","content":[]}}]]},{"run":"37174266054","case":"plan-eng-review-artifact","trial":1,"runner":"pty","error":"plan-eng-review-artifact: outcome=timeout (no terminal outcome within 600000ms total budget (including startup and 5000ms cleanup reserve; step0=6, review=0); idleFor=234ms)","failure_class":"assertion","expected":"observer_timeout","events":[]},{"run":"37176837432","case":"auto-decide-preserved","trial":1,"runner":"pty","error":"AUTO_DECIDE preserve test inconclusive: outcome=timeout","failure_class":"assertion","expected":"observer_timeout","events":[]}]} \ No newline at end of file diff --git a/test/fixtures/trial-cause/thinking-captures-37198445662.json b/test/fixtures/trial-cause/thinking-captures-37198445662.json new file mode 100644 index 000000000..83520ac79 --- /dev/null +++ b/test/fixtures/trial-cause/thinking-captures-37198445662.json @@ -0,0 +1 @@ +{"source":"census 37198445662 (Claude Code 2.1.284), publicStreamDiagnostics sessions; arrival times from elapsedMs, non-stream rows stamped with the preceding stream row; same-kind deltas thinned to one per second (every gap of 1 s or more kept); ids renamed; no message text","sessions":[{"case":"/review army consensus","artifact":"paid-slice-11-a1","events":["0 y init","0 y status","1704 s message_start","1704 s content_block_start","2361 s content_block_delta:thinking_delta","2361 s content_block_delta:signature_delta","2361 a","2362 s content_block_stop","2362 s content_block_start","2362 s content_block_delta:text_delta","2455 a","2602 s content_block_stop","2602 s content_block_start","2602 s content_block_delta:input_json_delta","3622 s content_block_delta:input_json_delta","4647 s content_block_delta:input_json_delta","4791 a t0:Bash","4802 s content_block_stop","4809 s message_delta","4809 s message_stop","4809 u t0","4809 y status","7496 s message_start","7497 s content_block_start","7497 s content_block_delta:text_delta","7504 a","7505 s content_block_stop","7505 s content_block_start","7505 s content_block_delta:input_json_delta","8566 s content_block_delta:input_json_delta","9625 s content_block_delta:input_json_delta","10627 s content_block_delta:input_json_delta","11629 s content_block_delta:input_json_delta","12629 s content_block_delta:input_json_delta","13630 s content_block_delta:input_json_delta","14631 s content_block_delta:input_json_delta","15693 s content_block_delta:input_json_delta","16070 a t1:Agent","16073 s content_block_stop","16073 y task_started","16073 u","16081 s content_block_start","16085 s content_block_delta:input_json_delta","17132 s content_block_delta:input_json_delta","18133 s content_block_delta:input_json_delta","19135 s content_block_delta:input_json_delta","20196 s content_block_delta:input_json_delta","21196 s content_block_delta:input_json_delta","21384 y task_progress","21384 a* t2:Bash","21447 s content_block_delta:input_json_delta","21509 u t2","21572 s content_block_delta:input_json_delta","22438 a t3:Agent","22440 s content_block_stop","22440 y task_started","22440 u","22448 s message_delta","22448 s message_stop","22448 y task_progress","22448 a* t4:Read","22448 u t4","22448 y task_progress","22448 a* t5:Bash","22448 u t5","22448 y task_progress","22448 a* t6:Bash","22448 u t6","22448 y task_progress","22448 a* t7:Bash","22448 u t7","22448 y task_updated","22448 y task_notification","22448 u t3","22448 y task_updated","22448 y task_notification","22448 u t1","22448 y status","56707 s message_start","57220 s content_block_start","57220 y thinking_tokens","57512 s content_block_delta:thinking_delta","57512 y thinking_tokens","58833 s content_block_delta:thinking_delta","58833 y thinking_tokens","59900 s content_block_delta:thinking_delta","59900 y thinking_tokens","61276 s content_block_delta:thinking_delta","61276 y thinking_tokens","62658 s content_block_delta:thinking_delta","62658 y thinking_tokens","63843 s content_block_delta:thinking_delta","63843 y thinking_tokens","65031 s content_block_delta:thinking_delta","65031 y thinking_tokens","66220 s content_block_delta:thinking_delta","66220 y thinking_tokens","67721 s content_block_delta:thinking_delta","67721 y thinking_tokens","68783 s content_block_delta:thinking_delta","68783 y thinking_tokens","69849 s content_block_delta:thinking_delta","69849 y thinking_tokens","70909 s content_block_delta:thinking_delta","73753 s content_block_delta:thinking_delta","73753 s content_block_delta:signature_delta","73753 a","73755 s content_block_stop","73755 s content_block_start","73755 y thinking_tokens","73755 s content_block_delta:thinking_delta","73755 y thinking_tokens","73755 s content_block_delta:thinking_delta","73755 y thinking_tokens","73755 s content_block_delta:thinking_delta","73755 y thinking_tokens","73755 s content_block_delta:thinking_delta","73755 y thinking_tokens","73755 s content_block_delta:thinking_delta","73755 y thinking_tokens","73755 s content_block_delta:thinking_delta","73755 y thinking_tokens","73755 s content_block_delta:thinking_delta","73755 y thinking_tokens","73755 s content_block_delta:thinking_delta","73755 y thinking_tokens","73755 s content_block_delta:thinking_delta","73755 y thinking_tokens","73756 s content_block_delta:thinking_delta","73756 y thinking_tokens","73756 s content_block_delta:thinking_delta","73756 y thinking_tokens","73756 s content_block_delta:thinking_delta","73756 y thinking_tokens","73756 s content_block_delta:thinking_delta","73756 y thinking_tokens","73756 s content_block_delta:signature_delta","73756 a","73756 s content_block_stop","73756 s content_block_start","73756 s content_block_delta:input_json_delta","74781 s content_block_delta:input_json_delta","75781 s content_block_delta:input_json_delta","76781 s content_block_delta:input_json_delta","77782 s content_block_delta:input_json_delta","78843 s content_block_delta:input_json_delta","79845 s content_block_delta:input_json_delta","80845 s content_block_delta:input_json_delta","81845 s content_block_delta:input_json_delta","82846 s content_block_delta:input_json_delta","83846 s content_block_delta:input_json_delta","84846 s content_block_delta:input_json_delta","85847 s content_block_delta:input_json_delta","86909 s content_block_delta:input_json_delta","87143 a t8:Bash","87152 s content_block_stop","87156 s message_delta","87156 s message_stop","87156 u t8","87156 y status","89770 s message_start","89770 s content_block_start","89770 s content_block_delta:text_delta","89780 a","89781 s content_block_stop","89781 s message_delta","89781 s message_stop","89781 r"]},{"case":"cso-infra-scope","artifact":"paid-slice-10-a1","events":["0 y init","0 y status","4452 s message_start","4452 s content_block_start","4647 s content_block_delta:thinking_delta","4647 s content_block_delta:signature_delta","4647 a","4648 s content_block_stop","4648 s content_block_start","4648 s content_block_delta:text_delta","5345 a","5358 s content_block_stop","5358 s content_block_start","5358 s content_block_delta:input_json_delta","6766 s content_block_delta:input_json_delta","6768 a t0:Bash","6775 s content_block_stop","6780 s message_delta","6780 s message_stop","6780 u t0","6780 y status","8998 s message_start","9746 s content_block_start","9746 y thinking_tokens","10570 s content_block_delta:thinking_delta","11933 s content_block_delta:thinking_delta","11933 y thinking_tokens","11934 s content_block_delta:signature_delta","11934 a","11934 s content_block_stop","11934 s content_block_start","11934 s content_block_delta:text_delta","11935 a","11935 s content_block_stop","11935 s content_block_start","11935 s content_block_delta:input_json_delta","12785 a t1:Bash","12788 s content_block_stop","12790 s content_block_start","12790 s content_block_delta:input_json_delta","12790 u t1","13341 s content_block_delta:input_json_delta","14111 a t2:Bash","14115 s content_block_stop","14119 s content_block_start","14119 s content_block_delta:input_json_delta","14350 u t2","14413 s content_block_delta:input_json_delta","14531 a t3:Bash","14534 s content_block_stop","14538 s message_delta","14538 s message_stop","14538 u t3","14538 y status","17531 s message_start","17531 s content_block_start","17531 s content_block_delta:text_delta","17542 a","17543 s content_block_stop","17543 s content_block_start","17543 s content_block_delta:input_json_delta","18587 s content_block_delta:input_json_delta","19812 s content_block_delta:input_json_delta","20157 a t4:Bash","20162 s content_block_stop","20165 s message_delta","20165 s message_stop","20165 u t4","20165 y status","22843 s message_start","23960 s content_block_start","24713 s content_block_delta:thinking_delta","24713 s content_block_delta:signature_delta","24713 a","24714 s content_block_stop","24714 s content_block_start","24714 s content_block_delta:text_delta","24714 a","24714 s content_block_stop","24714 s content_block_start","24714 s content_block_delta:input_json_delta","26246 s content_block_delta:input_json_delta","26247 a t5:Bash","26251 s content_block_stop","26253 s content_block_start","26253 s content_block_delta:input_json_delta","26253 u t5","26965 s content_block_delta:input_json_delta","27173 a t6:Bash","27176 s content_block_stop","27180 s message_delta","27180 s message_stop","27180 u t6","27180 y status","30295 s message_start","30295 s content_block_start","30295 s content_block_delta:text_delta","30304 a","30305 s content_block_stop","30305 s content_block_start","30305 s content_block_delta:input_json_delta","31724 s content_block_delta:input_json_delta","32529 a t7:Bash","32534 s content_block_stop","32535 s content_block_start","32535 s content_block_delta:input_json_delta","32792 u t7","33169 s content_block_delta:input_json_delta","33486 a t8:Bash","33489 s content_block_stop","33491 s content_block_start","33491 s content_block_delta:input_json_delta","33491 u t8","33848 s content_block_delta:input_json_delta","34523 a t9:Bash","34526 s content_block_stop","34528 s content_block_start","34528 s content_block_delta:input_json_delta","34740 u t9","34803 s content_block_delta:input_json_delta","35804 s content_block_delta:input_json_delta","36102 a t10:Bash","36108 s content_block_stop","36111 s message_delta","36111 s message_stop","36111 u t10","36111 y status","38680 s message_start","39596 s content_block_start","39596 y thinking_tokens","39597 s content_block_delta:thinking_delta","39597 y thinking_tokens","41102 s content_block_delta:thinking_delta","41102 y thinking_tokens","42620 s content_block_delta:thinking_delta","42620 y thinking_tokens","43819 s content_block_delta:thinking_delta","43819 y thinking_tokens","45072 s content_block_delta:thinking_delta","45072 y thinking_tokens","46555 s content_block_delta:thinking_delta","46555 y thinking_tokens","47927 s content_block_delta:thinking_delta","47927 y thinking_tokens","49051 s content_block_delta:thinking_delta","49051 y thinking_tokens","50305 s content_block_delta:thinking_delta","50305 y thinking_tokens","51678 s content_block_delta:thinking_delta","51678 y thinking_tokens","53053 s content_block_delta:thinking_delta","53053 y thinking_tokens","54301 s content_block_delta:thinking_delta","54301 y thinking_tokens","55504 s content_block_delta:thinking_delta","55504 y thinking_tokens","56989 s content_block_delta:thinking_delta","56989 y thinking_tokens","58116 s content_block_delta:thinking_delta","58116 y thinking_tokens","59306 s content_block_delta:thinking_delta","59306 y thinking_tokens","60613 s content_block_delta:thinking_delta","60613 y thinking_tokens","61864 s content_block_delta:thinking_delta","61864 y thinking_tokens","62990 s content_block_delta:thinking_delta","62990 y thinking_tokens","64129 s content_block_delta:thinking_delta","64129 y thinking_tokens","65426 s content_block_delta:thinking_delta","65426 y thinking_tokens","66743 s content_block_delta:thinking_delta","66743 y thinking_tokens","68309 s content_block_delta:thinking_delta","68309 y thinking_tokens","69368 s content_block_delta:thinking_delta","69368 y thinking_tokens","70992 s content_block_delta:thinking_delta","70992 y thinking_tokens","72368 s content_block_delta:thinking_delta","72368 y thinking_tokens","73554 s content_block_delta:thinking_delta","73554 y thinking_tokens","74804 s content_block_delta:thinking_delta","74804 y thinking_tokens","76179 s content_block_delta:thinking_delta","76179 y thinking_tokens","77304 s content_block_delta:thinking_delta","77304 y thinking_tokens","78492 s content_block_delta:thinking_delta","78492 y thinking_tokens","80055 s content_block_delta:thinking_delta","80055 y thinking_tokens","81119 s content_block_delta:thinking_delta","81119 y thinking_tokens","82429 s content_block_delta:thinking_delta","82429 y thinking_tokens","83805 s content_block_delta:thinking_delta","83805 y thinking_tokens","84993 s content_block_delta:thinking_delta","84993 y thinking_tokens","86254 s content_block_delta:thinking_delta","86254 y thinking_tokens","87751 s content_block_delta:thinking_delta","87751 y thinking_tokens","88868 s content_block_delta:thinking_delta","88868 y thinking_tokens","90117 s content_block_delta:thinking_delta","90117 y thinking_tokens","91247 s content_block_delta:thinking_delta","91247 y thinking_tokens","92803 s content_block_delta:thinking_delta","92803 y thinking_tokens","93992 s content_block_delta:thinking_delta","93992 y thinking_tokens","95178 s content_block_delta:thinking_delta","95178 y thinking_tokens","96303 s content_block_delta:thinking_delta","96303 y thinking_tokens","97552 s content_block_delta:thinking_delta","97552 y thinking_tokens","98740 s content_block_delta:thinking_delta","98740 y thinking_tokens","99990 s content_block_delta:thinking_delta","99990 y thinking_tokens","101115 s content_block_delta:thinking_delta","101115 y thinking_tokens","102303 s content_block_delta:thinking_delta","102303 y thinking_tokens","103616 s content_block_delta:thinking_delta","103616 y thinking_tokens","104865 s content_block_delta:thinking_delta","104865 y thinking_tokens","105992 s content_block_delta:thinking_delta","105992 y thinking_tokens","107053 s content_block_delta:thinking_delta","107053 y thinking_tokens","108240 s content_block_delta:thinking_delta","111497 s content_block_delta:thinking_delta","111498 s content_block_delta:signature_delta","111498 a","111500 s content_block_stop","111500 s content_block_start","111500 y thinking_tokens","111500 s content_block_delta:thinking_delta","111500 y thinking_tokens","111500 s content_block_delta:thinking_delta","111500 y thinking_tokens","111500 s content_block_delta:thinking_delta","111500 y thinking_tokens","111500 s content_block_delta:thinking_delta","111500 y thinking_tokens","111500 s content_block_delta:thinking_delta","111500 y thinking_tokens","111500 s content_block_delta:thinking_delta","111500 y thinking_tokens","111500 s content_block_delta:thinking_delta","111500 y thinking_tokens","111500 s content_block_delta:thinking_delta","111500 y thinking_tokens","111501 s content_block_delta:thinking_delta","111501 y thinking_tokens","111501 s content_block_delta:thinking_delta","111501 y thinking_tokens","111501 s content_block_delta:thinking_delta","111501 y thinking_tokens","111501 s content_block_delta:thinking_delta","111501 y thinking_tokens","111501 s content_block_delta:thinking_delta","111501 y thinking_tokens","111501 s content_block_delta:thinking_delta","111501 y thinking_tokens","111501 s content_block_delta:thinking_delta","111501 y thinking_tokens","111501 s content_block_delta:thinking_delta","111501 y thinking_tokens","111501 s content_block_delta:thinking_delta","111501 y thinking_tokens","111501 s content_block_delta:signature_delta","111501 a","111502 s content_block_stop","111502 s content_block_start","111502 s content_block_delta:input_json_delta","112429 a t11:Bash","112432 s content_block_stop","112435 s content_block_start","112436 s content_block_delta:input_json_delta","112678 u t11","112754 s content_block_delta:input_json_delta","113742 a t12:Bash","113745 s content_block_stop","113748 s content_block_start","113748 s content_block_delta:input_json_delta","113992 u t12","114055 s content_block_delta:input_json_delta","114971 a t13:Bash","114974 s content_block_stop","114978 s message_delta","114978 s message_stop","114978 u t13","114978 y status","117790 s message_start","118599 s content_block_start","118599 y thinking_tokens","119364 s content_block_delta:thinking_delta","124672 s content_block_delta:thinking_delta","124672 y thinking_tokens","124672 s content_block_delta:signature_delta","124672 a","124673 s content_block_stop","124673 s content_block_start","124673 y thinking_tokens","124673 s content_block_delta:thinking_delta","124673 y thinking_tokens","124673 s content_block_delta:thinking_delta","124673 y thinking_tokens","124673 s content_block_delta:thinking_delta","124673 y thinking_tokens","124673 s content_block_delta:thinking_delta","124673 y thinking_tokens","124673 s content_block_delta:thinking_delta","124673 y thinking_tokens","124673 s content_block_delta:thinking_delta","124673 y thinking_tokens","124673 s content_block_delta:thinking_delta","124673 y thinking_tokens","124673 s content_block_delta:thinking_delta","124673 y thinking_tokens","124674 s content_block_delta:thinking_delta","124674 y thinking_tokens","124674 s content_block_delta:thinking_delta","124674 y thinking_tokens","124674 s content_block_delta:thinking_delta","124674 y thinking_tokens","124674 s content_block_delta:thinking_delta","124674 y thinking_tokens","124674 s content_block_delta:thinking_delta","124674 y thinking_tokens","124674 s content_block_delta:thinking_delta","124674 y thinking_tokens","124674 s content_block_delta:thinking_delta","124674 y thinking_tokens","124674 s content_block_delta:thinking_delta","124674 y thinking_tokens","124674 s content_block_delta:thinking_delta","124674 y thinking_tokens","124674 s content_block_delta:thinking_delta","124674 y thinking_tokens","124674 s content_block_delta:thinking_delta","124674 y thinking_tokens","124674 s content_block_delta:thinking_delta","124674 y thinking_tokens","124675 s content_block_delta:thinking_delta","124675 y thinking_tokens","124675 s content_block_delta:thinking_delta","124675 y thinking_tokens","124675 s content_block_delta:signature_delta","124675 a","124675 s content_block_stop","124675 s content_block_start","124675 s content_block_delta:input_json_delta","125696 s content_block_delta:input_json_delta","126703 s content_block_delta:input_json_delta","127760 s content_block_delta:input_json_delta","128822 s content_block_delta:input_json_delta","129823 s content_block_delta:input_json_delta","130823 s content_block_delta:input_json_delta","131823 s content_block_delta:input_json_delta","132884 s content_block_delta:input_json_delta","133885 s content_block_delta:input_json_delta","134885 s content_block_delta:input_json_delta","135886 s content_block_delta:input_json_delta","136886 s content_block_delta:input_json_delta","137888 s content_block_delta:input_json_delta","138948 s content_block_delta:input_json_delta","139948 s content_block_delta:input_json_delta","140948 s content_block_delta:input_json_delta","141948 s content_block_delta:input_json_delta","142948 s content_block_delta:input_json_delta","143948 s content_block_delta:input_json_delta","145010 s content_block_delta:input_json_delta","146011 s content_block_delta:input_json_delta","147073 s content_block_delta:input_json_delta","148075 s content_block_delta:input_json_delta","149076 s content_block_delta:input_json_delta","150076 s content_block_delta:input_json_delta","151136 s content_block_delta:input_json_delta","152149 s content_block_delta:input_json_delta","153199 s content_block_delta:input_json_delta","154261 s content_block_delta:input_json_delta","155261 s content_block_delta:input_json_delta","156261 s content_block_delta:input_json_delta","157261 s content_block_delta:input_json_delta","158261 s content_block_delta:input_json_delta","159323 s content_block_delta:input_json_delta","160324 s content_block_delta:input_json_delta","161386 s content_block_delta:input_json_delta","162386 s content_block_delta:input_json_delta","163386 s content_block_delta:input_json_delta","164448 s content_block_delta:input_json_delta","165448 s content_block_delta:input_json_delta","166448 s content_block_delta:input_json_delta","167449 s content_block_delta:input_json_delta","168449 s content_block_delta:input_json_delta","169450 s content_block_delta:input_json_delta","170511 s content_block_delta:input_json_delta","171512 s content_block_delta:input_json_delta","172576 s content_block_delta:input_json_delta","173636 s content_block_delta:input_json_delta","174636 s content_block_delta:input_json_delta","175636 s content_block_delta:input_json_delta","176636 s content_block_delta:input_json_delta","177636 s content_block_delta:input_json_delta","178636 s content_block_delta:input_json_delta","179699 s content_block_delta:input_json_delta","180722 s content_block_delta:input_json_delta","181761 s content_block_delta:input_json_delta","182761 s content_block_delta:input_json_delta","183761 s content_block_delta:input_json_delta","184761 s content_block_delta:input_json_delta","185761 s content_block_delta:input_json_delta","186761 s content_block_delta:input_json_delta","187761 s content_block_delta:input_json_delta","188762 s content_block_delta:input_json_delta","189765 s content_block_delta:input_json_delta","190829 s content_block_delta:input_json_delta","191887 s content_block_delta:input_json_delta","193074 s content_block_delta:input_json_delta","193643 a t14:Bash","193656 s content_block_stop","193661 s message_delta","193661 s message_stop","193661 u t14","193661 y status","197469 s message_start","198263 s content_block_start","198263 y thinking_tokens","198263 s content_block_delta:thinking_delta","199498 s content_block_delta:thinking_delta","199498 y thinking_tokens","199498 s content_block_delta:signature_delta","199498 a","199499 s content_block_stop","199499 s content_block_start","199499 s content_block_delta:text_delta","199499 a","199499 s content_block_stop","199499 s content_block_start","199499 s content_block_delta:input_json_delta","200921 s content_block_delta:input_json_delta","202394 s content_block_delta:input_json_delta","203789 s content_block_delta:input_json_delta","204814 s content_block_delta:input_json_delta","205522 a t15:Bash","205527 s content_block_stop","205537 s message_delta","205537 s message_stop","205537 u t15","205537 y status","208121 s message_start","208929 s content_block_start","208929 y thinking_tokens","208929 s content_block_delta:thinking_delta","208929 y thinking_tokens","210410 s content_block_delta:thinking_delta","210410 y thinking_tokens","211861 s content_block_delta:thinking_delta","213774 s content_block_delta:thinking_delta","213774 y thinking_tokens","213774 s content_block_delta:signature_delta","213774 a","213774 s content_block_stop","213774 s content_block_start","213774 s content_block_delta:text_delta","213775 a","213775 s content_block_stop","213775 s content_block_start","213775 s content_block_delta:input_json_delta","215282 s content_block_delta:input_json_delta","215628 a t16:Bash","215631 s content_block_stop","215632 s message_delta","215633 s message_stop","215633 u t16","215633 y status","227726 s message_start","227726 s content_block_start","227726 s content_block_delta:text_delta","228782 s content_block_delta:text_delta","229890 s content_block_delta:text_delta","230908 s content_block_delta:text_delta","232138 s content_block_delta:text_delta","233158 s content_block_delta:text_delta","234512 s content_block_delta:text_delta","235282 a","235333 s content_block_stop","235334 s message_delta","235334 s message_stop","235334 r"]},{"case":"cso-full-audit","artifact":"paid-slice-10-a1","events":["0 y init","0 y status","2970 s message_start","2970 s content_block_start","3086 s content_block_delta:thinking_delta","3086 s content_block_delta:signature_delta","3086 a","3087 s content_block_stop","3087 s content_block_start","3087 s content_block_delta:text_delta","3751 a","3769 s content_block_stop","3769 s content_block_start","3770 s content_block_delta:input_json_delta","5256 s content_block_delta:input_json_delta","5258 a t0:Bash","5265 s content_block_stop","5269 s message_delta","5269 s message_stop","5269 u t0","5269 y status","7231 s message_start","8069 s content_block_start","8069 y thinking_tokens","8725 s content_block_delta:thinking_delta","9689 y thinking_tokens","9689 s content_block_delta:signature_delta","9689 a","9690 s content_block_stop","9690 s content_block_start","9690 s content_block_delta:text_delta","9690 a","9691 s content_block_stop","9691 s content_block_start","9691 s content_block_delta:input_json_delta","10515 a t1:Bash","10519 s content_block_stop","10523 s content_block_start","10523 s content_block_delta:input_json_delta","10523 u t1","10988 s content_block_delta:input_json_delta","11161 a t2:Bash","11164 s content_block_stop","11166 s message_delta","11166 s message_stop","11166 u t2","11166 y status","14177 s message_start","14177 s content_block_start","14177 s content_block_delta:text_delta","14188 a","14189 s content_block_stop","14189 s content_block_start","14189 s content_block_delta:input_json_delta","15706 s content_block_delta:input_json_delta","17248 s content_block_delta:input_json_delta","17248 a t3:Bash","17253 s content_block_stop","17255 s content_block_start","17255 s content_block_delta:input_json_delta","17425 u t3","17892 s content_block_delta:input_json_delta","18099 a t4:Bash","18102 s content_block_stop","18106 s message_delta","18106 s message_stop","18106 u t4","18106 y status","20639 s message_start","22148 s content_block_start","22148 y thinking_tokens","22148 s content_block_delta:thinking_delta","23679 s content_block_delta:thinking_delta","23679 y thinking_tokens","23680 s content_block_delta:signature_delta","23680 a","23680 s content_block_stop","23680 s content_block_start","23681 s content_block_delta:text_delta","23681 a","23681 s content_block_stop","23681 s content_block_start","23682 s content_block_delta:input_json_delta","25226 s content_block_delta:input_json_delta","25226 a t5:Bash","25229 s content_block_stop","25233 s content_block_start","25233 s content_block_delta:input_json_delta","25408 u t5","25732 s content_block_delta:input_json_delta","25949 a t6:Bash","25951 s content_block_stop","25956 s message_delta","25956 s message_stop","25956 u t6","25956 y status","29196 s message_start","29196 s content_block_start","29196 s content_block_delta:input_json_delta","30731 s content_block_delta:input_json_delta","31838 s content_block_delta:input_json_delta","31859 a t7:Bash","31864 s content_block_stop","31869 s message_delta","31869 s message_stop","31869 u t7","31869 y status","34604 s message_start","35395 s content_block_start","35395 y thinking_tokens","35396 s content_block_delta:thinking_delta","37937 s content_block_delta:thinking_delta","37937 y thinking_tokens","37937 s content_block_delta:signature_delta","37937 a","37937 s content_block_stop","37937 s content_block_start","37937 y thinking_tokens","37937 s content_block_delta:thinking_delta","37937 y thinking_tokens","37938 s content_block_delta:thinking_delta","37938 y thinking_tokens","37938 s content_block_delta:thinking_delta","37938 y thinking_tokens","37938 s content_block_delta:thinking_delta","37938 y thinking_tokens","37938 s content_block_delta:thinking_delta","37938 y thinking_tokens","37938 s content_block_delta:thinking_delta","37938 y thinking_tokens","37938 s content_block_delta:thinking_delta","37938 y thinking_tokens","37938 s content_block_delta:thinking_delta","37938 y thinking_tokens","37938 s content_block_delta:thinking_delta","37938 y thinking_tokens","37938 s content_block_delta:thinking_delta","37938 y thinking_tokens","37938 s content_block_delta:thinking_delta","37938 y thinking_tokens","37938 s content_block_delta:signature_delta","37938 a","37939 s content_block_stop","37939 s content_block_start","37939 s content_block_delta:input_json_delta","39272 s content_block_delta:input_json_delta","39273 a t8:Bash","39276 s content_block_stop","39278 s content_block_start","39278 s content_block_delta:input_json_delta","39278 u t8","39963 s content_block_delta:input_json_delta","39963 a t9:Bash","39966 s content_block_stop","39968 s content_block_start","39968 s content_block_delta:input_json_delta","40003 u t9","40614 s content_block_delta:input_json_delta","41388 a t10:Bash","41396 s content_block_stop","41401 s content_block_start","41401 s content_block_delta:input_json_delta","41512 u t10","42063 s content_block_delta:input_json_delta","42499 a t11:Bash","42505 s content_block_stop","42508 s message_delta","42508 s message_stop","42508 u t11","42508 y status","45311 s message_start","46234 s content_block_start","46234 y thinking_tokens","46234 s content_block_delta:thinking_delta","46234 y thinking_tokens","47700 s content_block_delta:thinking_delta","47700 y thinking_tokens","49186 s content_block_delta:thinking_delta","49186 y thinking_tokens","50458 s content_block_delta:thinking_delta","50458 y thinking_tokens","51949 s content_block_delta:thinking_delta","51949 y thinking_tokens","53541 s content_block_delta:thinking_delta","53541 y thinking_tokens","54970 s content_block_delta:thinking_delta","54970 y thinking_tokens","56278 s content_block_delta:thinking_delta","56278 y thinking_tokens","57705 s content_block_delta:thinking_delta","57705 y thinking_tokens","58878 s content_block_delta:thinking_delta","58878 y thinking_tokens","60129 s content_block_delta:thinking_delta","60129 y thinking_tokens","61443 s content_block_delta:thinking_delta","61443 y thinking_tokens","62505 s content_block_delta:thinking_delta","62505 y thinking_tokens","63754 s content_block_delta:thinking_delta","63754 y thinking_tokens","64817 s content_block_delta:thinking_delta","64817 y thinking_tokens","65878 s content_block_delta:thinking_delta","65878 y thinking_tokens","67066 s content_block_delta:thinking_delta","67066 y thinking_tokens","68576 s content_block_delta:thinking_delta","68576 y thinking_tokens","70066 s content_block_delta:thinking_delta","70066 y thinking_tokens","71193 s content_block_delta:thinking_delta","71193 y thinking_tokens","72629 s content_block_delta:thinking_delta","72629 y thinking_tokens","73942 s content_block_delta:thinking_delta","73942 y thinking_tokens","75319 s content_block_delta:thinking_delta","75319 y thinking_tokens","76379 s content_block_delta:thinking_delta","76379 y thinking_tokens","77692 s content_block_delta:thinking_delta","77692 y thinking_tokens","79192 s content_block_delta:thinking_delta","79192 y thinking_tokens","80318 s content_block_delta:thinking_delta","80318 y thinking_tokens","81690 s content_block_delta:thinking_delta","81690 y thinking_tokens","82753 s content_block_delta:thinking_delta","82753 y thinking_tokens","84377 s content_block_delta:thinking_delta","84377 y thinking_tokens","85441 s content_block_delta:thinking_delta","85441 y thinking_tokens","86628 s content_block_delta:thinking_delta","86628 y thinking_tokens","88003 s content_block_delta:thinking_delta","88003 y thinking_tokens","89067 s content_block_delta:thinking_delta","89067 y thinking_tokens","90315 s content_block_delta:thinking_delta","90315 y thinking_tokens","91628 s content_block_delta:thinking_delta","91628 y thinking_tokens","92690 s content_block_delta:thinking_delta","92690 y thinking_tokens","94065 s content_block_delta:thinking_delta","94065 y thinking_tokens","95128 s content_block_delta:thinking_delta","95128 y thinking_tokens","96191 s content_block_delta:thinking_delta","96191 y thinking_tokens","97691 s content_block_delta:thinking_delta","97691 y thinking_tokens","98941 s content_block_delta:thinking_delta","98941 y thinking_tokens","100065 s content_block_delta:thinking_delta","109630 s content_block_delta:thinking_delta","109631 s content_block_delta:signature_delta","109631 a","109633 s content_block_stop","109633 s content_block_start","109633 y thinking_tokens","109633 s content_block_delta:thinking_delta","109633 y thinking_tokens","109633 s content_block_delta:thinking_delta","109633 y thinking_tokens","109633 s content_block_delta:thinking_delta","109633 y thinking_tokens","109633 s content_block_delta:thinking_delta","109633 y thinking_tokens","109633 s content_block_delta:thinking_delta","109633 y thinking_tokens","109634 s content_block_delta:thinking_delta","109634 y thinking_tokens","109634 s content_block_delta:thinking_delta","109634 y thinking_tokens","109634 s content_block_delta:thinking_delta","109634 y thinking_tokens","109634 s content_block_delta:thinking_delta","109634 y thinking_tokens","109634 s content_block_delta:thinking_delta","109634 y thinking_tokens","109634 s content_block_delta:thinking_delta","109634 y thinking_tokens","109634 s content_block_delta:thinking_delta","109634 y thinking_tokens","109634 s content_block_delta:thinking_delta","109634 y thinking_tokens","109634 s content_block_delta:thinking_delta","109634 y thinking_tokens","109634 s content_block_delta:thinking_delta","109634 y thinking_tokens","109635 s content_block_delta:thinking_delta","109635 y thinking_tokens","109635 s content_block_delta:thinking_delta","109635 y thinking_tokens","109635 s content_block_delta:thinking_delta","109635 y thinking_tokens","109635 s content_block_delta:thinking_delta","109635 y thinking_tokens","109635 s content_block_delta:thinking_delta","109635 y thinking_tokens","109635 s content_block_delta:signature_delta","109635 a","109635 s content_block_stop","109635 s content_block_start","109636 s content_block_delta:input_json_delta","110692 s content_block_delta:input_json_delta","111692 s content_block_delta:input_json_delta","112694 s content_block_delta:input_json_delta","113755 s content_block_delta:input_json_delta","114760 s content_block_delta:input_json_delta","115822 s content_block_delta:input_json_delta","116883 s content_block_delta:input_json_delta","117943 s content_block_delta:input_json_delta","118943 s content_block_delta:input_json_delta","119944 s content_block_delta:input_json_delta","121005 s content_block_delta:input_json_delta","122006 s content_block_delta:input_json_delta","123006 s content_block_delta:input_json_delta","124006 s content_block_delta:input_json_delta","125006 s content_block_delta:input_json_delta","126006 s content_block_delta:input_json_delta","127006 s content_block_delta:input_json_delta","128006 s content_block_delta:input_json_delta","129007 s content_block_delta:input_json_delta","130007 s content_block_delta:input_json_delta","131069 s content_block_delta:input_json_delta","132069 s content_block_delta:input_json_delta","133069 s content_block_delta:input_json_delta","134069 s content_block_delta:input_json_delta","135072 s content_block_delta:input_json_delta","136131 s content_block_delta:input_json_delta","137131 s content_block_delta:input_json_delta","138132 s content_block_delta:input_json_delta","139140 s content_block_delta:input_json_delta","140144 s content_block_delta:input_json_delta","141194 s content_block_delta:input_json_delta","142194 s content_block_delta:input_json_delta","143194 s content_block_delta:input_json_delta","144194 s content_block_delta:input_json_delta","145194 s content_block_delta:input_json_delta","146194 s content_block_delta:input_json_delta","147194 s content_block_delta:input_json_delta","148195 s content_block_delta:input_json_delta","149259 s content_block_delta:input_json_delta","150320 s content_block_delta:input_json_delta","151383 s content_block_delta:input_json_delta","152450 s content_block_delta:input_json_delta","153507 s content_block_delta:input_json_delta","154507 s content_block_delta:input_json_delta","155569 s content_block_delta:input_json_delta","156569 s content_block_delta:input_json_delta","157569 s content_block_delta:input_json_delta","158570 s content_block_delta:input_json_delta","159570 s content_block_delta:input_json_delta","160570 s content_block_delta:input_json_delta","161573 s content_block_delta:input_json_delta","162635 s content_block_delta:input_json_delta","163696 s content_block_delta:input_json_delta","164696 s content_block_delta:input_json_delta","165758 s content_block_delta:input_json_delta","166760 s content_block_delta:input_json_delta","167760 s content_block_delta:input_json_delta","168823 s content_block_delta:input_json_delta","169884 s content_block_delta:input_json_delta","170952 s content_block_delta:input_json_delta","172008 s content_block_delta:input_json_delta","173008 s content_block_delta:input_json_delta","174012 s content_block_delta:input_json_delta","175071 s content_block_delta:input_json_delta","176071 s content_block_delta:input_json_delta","177071 s content_block_delta:input_json_delta","178071 s content_block_delta:input_json_delta","179071 s content_block_delta:input_json_delta","180071 s content_block_delta:input_json_delta","181071 s content_block_delta:input_json_delta","182071 s content_block_delta:input_json_delta","183071 s content_block_delta:input_json_delta","184137 s content_block_delta:input_json_delta","184851 a t12:Bash","184864 s content_block_stop","184866 s message_delta","184866 s message_stop","184866 u t12","184866 y status","189854 s message_start","190649 s content_block_start","190649 y thinking_tokens","191448 s content_block_delta:thinking_delta","191448 y thinking_tokens","192867 s content_block_delta:thinking_delta","192867 y thinking_tokens","194263 s content_block_delta:thinking_delta","197534 s content_block_delta:thinking_delta","197534 y thinking_tokens","197534 s content_block_delta:signature_delta","197534 a","197535 s content_block_stop","197535 s content_block_start","197535 y thinking_tokens","197535 s content_block_delta:thinking_delta","197535 y thinking_tokens","197535 s content_block_delta:thinking_delta","197535 y thinking_tokens","197535 s content_block_delta:thinking_delta","197535 y thinking_tokens","197535 s content_block_delta:thinking_delta","197535 y thinking_tokens","197535 s content_block_delta:thinking_delta","197535 y thinking_tokens","197535 s content_block_delta:thinking_delta","197535 y thinking_tokens","197535 s content_block_delta:thinking_delta","197535 y thinking_tokens","197535 s content_block_delta:thinking_delta","197535 y thinking_tokens","197535 s content_block_delta:thinking_delta","197535 y thinking_tokens","197535 s content_block_delta:thinking_delta","197535 y thinking_tokens","197535 s content_block_delta:signature_delta","197535 a","197535 s content_block_stop","197536 s content_block_start","197536 s content_block_delta:input_json_delta","198605 s content_block_delta:input_json_delta","199628 s content_block_delta:input_json_delta","200777 s content_block_delta:input_json_delta","201818 s content_block_delta:input_json_delta","202819 s content_block_delta:input_json_delta","203819 s content_block_delta:input_json_delta","204820 s content_block_delta:input_json_delta","205821 s content_block_delta:input_json_delta","206883 s content_block_delta:input_json_delta","207883 s content_block_delta:input_json_delta","208884 s content_block_delta:input_json_delta","209950 s content_block_delta:input_json_delta","211009 s content_block_delta:input_json_delta","212009 s content_block_delta:input_json_delta","213009 s content_block_delta:input_json_delta","214009 s content_block_delta:input_json_delta","215009 s content_block_delta:input_json_delta","216011 s content_block_delta:input_json_delta","217072 s content_block_delta:input_json_delta","218074 s content_block_delta:input_json_delta","219076 s content_block_delta:input_json_delta","220136 s content_block_delta:input_json_delta","221197 s content_block_delta:input_json_delta","222198 s content_block_delta:input_json_delta","223198 s content_block_delta:input_json_delta","224198 s content_block_delta:input_json_delta","225205 s content_block_delta:input_json_delta","226262 s content_block_delta:input_json_delta","227323 s content_block_delta:input_json_delta","228323 s content_block_delta:input_json_delta","229323 s content_block_delta:input_json_delta","230323 s content_block_delta:input_json_delta","231326 s content_block_delta:input_json_delta","232328 s content_block_delta:input_json_delta","233387 s content_block_delta:input_json_delta","234451 s content_block_delta:input_json_delta","235451 s content_block_delta:input_json_delta","236511 s content_block_delta:input_json_delta","237511 s content_block_delta:input_json_delta","238511 s content_block_delta:input_json_delta","239511 s content_block_delta:input_json_delta","240511 s content_block_delta:input_json_delta","241511 s content_block_delta:input_json_delta","242511 s content_block_delta:input_json_delta","243512 s content_block_delta:input_json_delta","244574 s content_block_delta:input_json_delta","245574 s content_block_delta:input_json_delta","246574 s content_block_delta:input_json_delta","247576 s content_block_delta:input_json_delta","248638 s content_block_delta:input_json_delta","249638 s content_block_delta:input_json_delta","250639 s content_block_delta:input_json_delta","251699 s content_block_delta:input_json_delta","252699 s content_block_delta:input_json_delta","253700 s content_block_delta:input_json_delta","254762 s content_block_delta:input_json_delta","255824 s content_block_delta:input_json_delta","256623 a t13:Bash","256632 s content_block_stop","256640 s message_delta","256640 s message_stop","256640 u t13","256640 y status","259417 s message_start","260249 s content_block_start","260249 y thinking_tokens","260249 s content_block_delta:thinking_delta","261237 y thinking_tokens","261237 s content_block_delta:signature_delta","261237 a","261237 s content_block_stop","261237 s content_block_start","261237 s content_block_delta:text_delta","261237 a","261238 s content_block_stop","261238 s content_block_start","261238 s content_block_delta:input_json_delta","262799 s content_block_delta:input_json_delta","262809 a t14:Bash","262813 s content_block_stop","262817 s message_delta","262817 s message_stop","262817 u t14","262817 y status","267448 s message_start","267448 s content_block_start","267448 s content_block_delta:input_json_delta","269112 s content_block_delta:input_json_delta","270583 s content_block_delta:input_json_delta","271301 a t15:Bash","271304 s content_block_stop","271311 s content_block_start","271312 s content_block_delta:input_json_delta","271362 u t15","272026 s content_block_delta:input_json_delta","272711 a t16:Bash","272713 s content_block_stop","272716 s content_block_start","272717 s content_block_delta:input_json_delta","272920 u t16","273438 s content_block_delta:input_json_delta","274127 a t17:Bash","274130 s content_block_stop","274132 s message_delta","274132 s message_stop","274132 u t17","274132 y status","276840 s message_start","277678 s content_block_start","277678 y thinking_tokens","277678 s content_block_delta:thinking_delta","278556 y thinking_tokens","278556 s content_block_delta:signature_delta","278556 a","278557 s content_block_stop","278557 s content_block_start","278557 s content_block_delta:input_json_delta","280034 s content_block_delta:input_json_delta","280950 a t18:Bash","280952 s content_block_stop","280954 s message_delta","280954 s message_stop","280954 u t18","280954 y status","295410 s message_start","295411 s content_block_start","295411 s content_block_delta:text_delta","296412 s content_block_delta:text_delta","297419 s content_block_delta:text_delta","298475 s content_block_delta:text_delta","299475 s content_block_delta:text_delta","300537 s content_block_delta:text_delta","301599 s content_block_delta:text_delta","302600 s content_block_delta:text_delta","303412 a","303421 s content_block_stop","303422 s message_delta","303422 s message_stop","303422 r"]},{"case":"/review army consensus","artifact":"paid-slice-10-a1","events":["0 y init","0 y status","2787 s message_start","2787 s content_block_start","2787 y thinking_tokens","3628 s content_block_delta:thinking_delta","3628 y thinking_tokens","3629 s content_block_delta:signature_delta","3629 a","3630 s content_block_stop","3630 s content_block_start","3630 s content_block_delta:text_delta","4202 a","4216 s content_block_stop","4216 s content_block_start","4217 s content_block_delta:input_json_delta","5552 s content_block_delta:input_json_delta","7077 s content_block_delta:input_json_delta","7080 a t0:Bash","7091 s content_block_stop","7097 s message_delta","7097 s message_stop","7097 u t0","7097 y status","11350 s message_start","11350 s content_block_start","11350 y thinking_tokens","11364 s content_block_delta:thinking_delta","11364 y thinking_tokens","11364 s content_block_delta:thinking_delta","11364 y thinking_tokens","11365 s content_block_delta:thinking_delta","11365 y thinking_tokens","11365 s content_block_delta:thinking_delta","11365 y thinking_tokens","11365 s content_block_delta:thinking_delta","11365 y thinking_tokens","11365 s content_block_delta:thinking_delta","11365 y thinking_tokens","11365 s content_block_delta:thinking_delta","11365 y thinking_tokens","11365 s content_block_delta:thinking_delta","11365 y thinking_tokens","11365 s content_block_delta:signature_delta","11365 a","11366 s content_block_stop","11366 s content_block_start","11366 s content_block_delta:input_json_delta","12457 s content_block_delta:input_json_delta","13490 s content_block_delta:input_json_delta","14548 s content_block_delta:input_json_delta","15548 s content_block_delta:input_json_delta","16548 s content_block_delta:input_json_delta","17549 s content_block_delta:input_json_delta","18549 s content_block_delta:input_json_delta","18924 a t1:Agent","18927 s content_block_stop","18927 y task_started","18927 u","18934 s content_block_start","18936 s content_block_delta:input_json_delta","19987 s content_block_delta:input_json_delta","20988 s content_block_delta:input_json_delta","22054 s content_block_delta:input_json_delta","22863 y task_progress","22863 a* t2:Read","22930 s content_block_delta:input_json_delta","22930 u t2","22988 s content_block_delta:input_json_delta","23988 s content_block_delta:input_json_delta","24676 y task_progress","24676 a* t3:Bash","24742 s content_block_delta:input_json_delta","24742 u t3","24801 s content_block_delta:input_json_delta","25563 a t4:Agent","25565 s content_block_stop","25565 y task_started","25565 u","25572 s message_delta","25572 s message_stop","25572 y task_progress","25572 a* t5:Read","25572 u t5","25572 y task_progress","25572 a* t6:Bash","25572 u t6","25572 y task_progress","25572 a* t7:Bash","25572 u t7","25572 y task_updated","25572 y task_notification","25572 u t1","25572 y task_updated","25572 y task_notification","25572 u t4","25572 y status","58204 s message_start","58951 s content_block_start","58951 y thinking_tokens","58952 s content_block_delta:thinking_delta","58952 y thinking_tokens","60438 s content_block_delta:thinking_delta","60438 y thinking_tokens","61935 s content_block_delta:thinking_delta","61935 y thinking_tokens","63417 s content_block_delta:thinking_delta","63417 y thinking_tokens","64913 s content_block_delta:thinking_delta","64913 y thinking_tokens","66406 s content_block_delta:thinking_delta","66406 y thinking_tokens","67898 s content_block_delta:thinking_delta","67898 y thinking_tokens","69450 s content_block_delta:thinking_delta","69450 y thinking_tokens","70991 s content_block_delta:thinking_delta","70991 y thinking_tokens","72435 s content_block_delta:thinking_delta","72435 y thinking_tokens","73875 s content_block_delta:thinking_delta","73875 y thinking_tokens","75302 s content_block_delta:thinking_delta","75302 y thinking_tokens","76806 s content_block_delta:thinking_delta","76806 y thinking_tokens","78274 s content_block_delta:thinking_delta","78274 y thinking_tokens","79694 s content_block_delta:thinking_delta","79694 y thinking_tokens","81129 s content_block_delta:thinking_delta","84116 s content_block_delta:thinking_delta","84117 s content_block_delta:signature_delta","84117 a","84118 s content_block_stop","84118 s content_block_start","84118 y thinking_tokens","84118 s content_block_delta:thinking_delta","84118 y thinking_tokens","84118 s content_block_delta:thinking_delta","84118 y thinking_tokens","84119 s content_block_delta:thinking_delta","84119 y thinking_tokens","84119 s content_block_delta:thinking_delta","84119 y thinking_tokens","84119 s content_block_delta:thinking_delta","84119 y thinking_tokens","84119 s content_block_delta:thinking_delta","84119 y thinking_tokens","84119 s content_block_delta:thinking_delta","84119 y thinking_tokens","84119 s content_block_delta:thinking_delta","84119 y thinking_tokens","84119 s content_block_delta:thinking_delta","84119 y thinking_tokens","84119 s content_block_delta:thinking_delta","84119 y thinking_tokens","84119 s content_block_delta:thinking_delta","84119 y thinking_tokens","84119 s content_block_delta:thinking_delta","84119 y thinking_tokens","84119 s content_block_delta:thinking_delta","84119 y thinking_tokens","84119 s content_block_delta:thinking_delta","84119 y thinking_tokens","84119 s content_block_delta:thinking_delta","84119 y thinking_tokens","84119 s content_block_delta:thinking_delta","84119 y thinking_tokens","84120 s content_block_delta:signature_delta","84120 a","84120 s content_block_stop","84120 s content_block_start","84120 s content_block_delta:input_json_delta","85158 s content_block_delta:input_json_delta","86416 s content_block_delta:input_json_delta","87472 s content_block_delta:input_json_delta","88626 s content_block_delta:input_json_delta","89660 s content_block_delta:input_json_delta","90814 s content_block_delta:input_json_delta","91848 s content_block_delta:input_json_delta","92875 s content_block_delta:input_json_delta","93911 s content_block_delta:input_json_delta","94859 a t8:Bash","94868 s content_block_stop","94871 s message_delta","94871 s message_stop","94871 u t8","94871 y status","98921 s message_start","98921 s content_block_start","98922 s content_block_delta:text_delta","98939 a","98939 s content_block_stop","98940 s message_delta","98940 s message_stop","98940 r"]},{"case":"plan-ceo-section-loading","artifact":"paid-slice-6-a1","events":["0 y init","0 y status","2573 s message_start","2573 s content_block_start","2573 y thinking_tokens","3560 s content_block_delta:thinking_delta","3560 y thinking_tokens","4671 s content_block_delta:thinking_delta","5168 y thinking_tokens","5168 s content_block_delta:signature_delta","5168 a","5169 s content_block_stop","5169 s content_block_start","5170 s content_block_delta:text_delta","5554 a","5681 s content_block_stop","5681 s content_block_start","5681 s content_block_delta:input_json_delta","6492 a t0:Bash","6499 s content_block_stop","6503 s message_delta","6503 s message_stop","6503 u t0","6503 y status","8980 s message_start","8980 s content_block_start","8980 s content_block_delta:input_json_delta","10594 s content_block_delta:input_json_delta","11240 a t1:Bash","11244 s content_block_stop","11245 s content_block_start","11245 s content_block_delta:input_json_delta","11245 u t1","11298 s content_block_delta:input_json_delta","12001 a t2:Bash","12004 s content_block_stop","12006 s content_block_start","12006 s content_block_delta:input_json_delta","12006 u t2","12127 s content_block_delta:input_json_delta","13193 s content_block_delta:input_json_delta","13793 a t3:Bash","13798 s content_block_stop","13801 s message_delta","13801 s message_stop","13801 u t3","13801 y status","16164 s message_start","16956 s content_block_start","16956 y thinking_tokens","16956 s content_block_delta:thinking_delta","18011 s content_block_delta:thinking_delta","18011 y thinking_tokens","18011 s content_block_delta:signature_delta","18011 a","18012 s content_block_stop","18012 s content_block_start","18012 s content_block_delta:text_delta","18012 a","18012 s content_block_stop","18012 s content_block_start","18013 s content_block_delta:input_json_delta","19326 s content_block_delta:input_json_delta","19987 a t4:Bash","19993 s content_block_stop","19995 s content_block_start","19995 s content_block_delta:input_json_delta","19995 u t4","20017 s content_block_delta:input_json_delta","21082 s content_block_delta:input_json_delta","21363 a t5:Bash","21367 s content_block_stop","21370 s message_delta","21370 s message_stop","21370 u t5","21370 y status","23615 s message_start","24057 s content_block_start","24844 s content_block_delta:thinking_delta","24845 s content_block_delta:signature_delta","24845 a","24845 s content_block_stop","24846 s content_block_start","24846 s content_block_delta:text_delta","24846 a","24846 s content_block_stop","24846 s content_block_start","24846 s content_block_delta:input_json_delta","26175 s content_block_delta:input_json_delta","26175 a t6:Bash","26181 s content_block_stop","26183 s content_block_start","26183 s content_block_delta:input_json_delta","26183 u t6","26236 s content_block_delta:input_json_delta","26846 a t7:Bash","26851 s content_block_stop","26854 s message_delta","26854 s message_stop","26854 u t7","26854 y status","29199 s message_start","30169 s content_block_start","30169 y thinking_tokens","30169 s content_block_delta:thinking_delta","30169 y thinking_tokens","31413 s content_block_delta:thinking_delta","31413 y thinking_tokens","32730 s content_block_delta:thinking_delta","32730 y thinking_tokens","33842 s content_block_delta:thinking_delta","33842 y thinking_tokens","35033 s content_block_delta:thinking_delta","35033 y thinking_tokens","36093 s content_block_delta:thinking_delta","36093 y thinking_tokens","37407 s content_block_delta:thinking_delta","37407 y thinking_tokens","38596 s content_block_delta:thinking_delta","38596 y thinking_tokens","39722 s content_block_delta:thinking_delta","39722 y thinking_tokens","41033 s content_block_delta:thinking_delta","41033 y thinking_tokens","42283 s content_block_delta:thinking_delta","42283 y thinking_tokens","43719 s content_block_delta:thinking_delta","43719 y thinking_tokens","44844 s content_block_delta:thinking_delta","44844 y thinking_tokens","46033 s content_block_delta:thinking_delta","46033 y thinking_tokens","47406 s content_block_delta:thinking_delta","47406 y thinking_tokens","48531 s content_block_delta:thinking_delta","48531 y thinking_tokens","49594 s content_block_delta:thinking_delta","49594 y thinking_tokens","51219 s content_block_delta:thinking_delta","51219 y thinking_tokens","52530 s content_block_delta:thinking_delta","52530 y thinking_tokens","53656 s content_block_delta:thinking_delta","53656 y thinking_tokens","54844 s content_block_delta:thinking_delta","54844 y thinking_tokens","55968 s content_block_delta:thinking_delta","55968 y thinking_tokens","57218 s content_block_delta:thinking_delta","57218 y thinking_tokens","58343 s content_block_delta:thinking_delta","58343 y thinking_tokens","59656 s content_block_delta:thinking_delta","59656 y thinking_tokens","60719 s content_block_delta:thinking_delta","60719 y thinking_tokens","61844 s content_block_delta:thinking_delta","61844 y thinking_tokens","62969 s content_block_delta:thinking_delta","62969 y thinking_tokens","64156 s content_block_delta:thinking_delta","64156 y thinking_tokens","65593 s content_block_delta:thinking_delta","65593 y thinking_tokens","66843 s content_block_delta:thinking_delta","66843 y thinking_tokens","68093 s content_block_delta:thinking_delta","68093 y thinking_tokens","69281 s content_block_delta:thinking_delta","69281 y thinking_tokens","70468 s content_block_delta:thinking_delta","70468 y thinking_tokens","71655 s content_block_delta:thinking_delta","71655 y thinking_tokens","73031 s content_block_delta:thinking_delta","73031 y thinking_tokens","74280 s content_block_delta:thinking_delta","78310 s content_block_delta:thinking_delta","78310 s content_block_delta:signature_delta","78310 a","78313 s content_block_stop","78313 s content_block_start","78313 y thinking_tokens","78313 s content_block_delta:thinking_delta","78313 y thinking_tokens","78313 s content_block_delta:thinking_delta","78313 y thinking_tokens","78313 s content_block_delta:thinking_delta","78313 y thinking_tokens","78313 s content_block_delta:thinking_delta","78313 y thinking_tokens","78313 s content_block_delta:thinking_delta","78313 y thinking_tokens","78313 s content_block_delta:thinking_delta","78313 y thinking_tokens","78313 s content_block_delta:thinking_delta","78313 y thinking_tokens","78313 s content_block_delta:thinking_delta","78313 y thinking_tokens","78313 s content_block_delta:thinking_delta","78313 y thinking_tokens","78313 s content_block_delta:thinking_delta","78313 y thinking_tokens","78314 s content_block_delta:thinking_delta","78314 y thinking_tokens","78314 s content_block_delta:thinking_delta","78314 y thinking_tokens","78314 s content_block_delta:thinking_delta","78314 y thinking_tokens","78314 s content_block_delta:thinking_delta","78314 y thinking_tokens","78314 s content_block_delta:thinking_delta","78314 y thinking_tokens","78314 s content_block_delta:thinking_delta","78314 y thinking_tokens","78314 s content_block_delta:signature_delta","78314 a","78314 s content_block_stop","78314 s content_block_start","78314 s content_block_delta:input_json_delta","79343 s content_block_delta:input_json_delta","80343 s content_block_delta:input_json_delta","81405 s content_block_delta:input_json_delta","82405 s content_block_delta:input_json_delta","83405 s content_block_delta:input_json_delta","84405 s content_block_delta:input_json_delta","85406 s content_block_delta:input_json_delta","86470 s content_block_delta:input_json_delta","87531 s content_block_delta:input_json_delta","88531 s content_block_delta:input_json_delta","89531 s content_block_delta:input_json_delta","90531 s content_block_delta:input_json_delta","91531 s content_block_delta:input_json_delta","92532 s content_block_delta:input_json_delta","93595 s content_block_delta:input_json_delta","94657 s content_block_delta:input_json_delta","95719 s content_block_delta:input_json_delta","96781 s content_block_delta:input_json_delta","97781 s content_block_delta:input_json_delta","98781 s content_block_delta:input_json_delta","99782 s content_block_delta:input_json_delta","100784 s content_block_delta:input_json_delta","101845 s content_block_delta:input_json_delta","102907 s content_block_delta:input_json_delta","103907 s content_block_delta:input_json_delta","104910 s content_block_delta:input_json_delta","105970 s content_block_delta:input_json_delta","106970 s content_block_delta:input_json_delta","107973 s content_block_delta:input_json_delta","109033 s content_block_delta:input_json_delta","110033 s content_block_delta:input_json_delta","111095 s content_block_delta:input_json_delta","112098 s content_block_delta:input_json_delta","113157 s content_block_delta:input_json_delta","114158 s content_block_delta:input_json_delta","115160 s content_block_delta:input_json_delta","116220 s content_block_delta:input_json_delta","117220 s content_block_delta:input_json_delta","118221 s content_block_delta:input_json_delta","119221 s content_block_delta:input_json_delta","120354 s content_block_delta:input_json_delta","120727 a t8:Bash","120739 s content_block_stop","120743 s message_delta","120743 s message_stop","120743 u t8","120743 y status","123085 s message_start","123868 s content_block_start","123868 y thinking_tokens","123965 s content_block_delta:thinking_delta","125284 s content_block_delta:thinking_delta","125284 y thinking_tokens","125284 s content_block_delta:signature_delta","125284 a","125285 s content_block_stop","125285 s content_block_start","125285 s content_block_delta:text_delta","125285 a","125285 s content_block_stop","125285 s content_block_start","125285 s content_block_delta:input_json_delta","126663 s content_block_delta:input_json_delta","127394 a t9:Bash","127397 s content_block_stop","127398 s content_block_start","127398 s content_block_delta:input_json_delta","127399 u t9","127423 s content_block_delta:input_json_delta","128095 a t10:Read","128099 s content_block_stop","128100 s message_delta","128100 s message_stop","128100 u t10","128100 y status","130571 s message_start","131454 s content_block_start","131454 y thinking_tokens","131454 s content_block_delta:thinking_delta","131454 y thinking_tokens","133046 s content_block_delta:thinking_delta","135697 s content_block_delta:thinking_delta","135697 y thinking_tokens","135697 s content_block_delta:signature_delta","135697 a","135698 s content_block_stop","135698 s content_block_start","135698 y thinking_tokens","135698 s content_block_delta:thinking_delta","135698 y thinking_tokens","135698 s content_block_delta:thinking_delta","135698 y thinking_tokens","135698 s content_block_delta:thinking_delta","135698 y thinking_tokens","135698 s content_block_delta:thinking_delta","135698 y thinking_tokens","135698 s content_block_delta:thinking_delta","135698 y thinking_tokens","135698 s content_block_delta:thinking_delta","135698 y thinking_tokens","135698 s content_block_delta:thinking_delta","135698 y thinking_tokens","135698 s content_block_delta:thinking_delta","135698 y thinking_tokens","135698 s content_block_delta:thinking_delta","135698 y thinking_tokens","135699 s content_block_delta:thinking_delta","135699 y thinking_tokens","135699 s content_block_delta:thinking_delta","135699 y thinking_tokens","135699 s content_block_delta:thinking_delta","135699 y thinking_tokens","135699 s content_block_delta:thinking_delta","135699 y thinking_tokens","135699 s content_block_delta:signature_delta","135699 a","135699 s content_block_stop","135699 s content_block_start","135699 s content_block_delta:input_json_delta","135755 a t11:Read","135758 s content_block_stop","135758 s content_block_start","135759 s content_block_delta:input_json_delta","135759 u t11","135824 s content_block_delta:input_json_delta","136828 s content_block_delta:input_json_delta","137885 s content_block_delta:input_json_delta","138944 s content_block_delta:input_json_delta","140005 s content_block_delta:input_json_delta","141005 s content_block_delta:input_json_delta","142006 s content_block_delta:input_json_delta","143006 s content_block_delta:input_json_delta","144007 s content_block_delta:input_json_delta","145011 s content_block_delta:input_json_delta","146074 s content_block_delta:input_json_delta","147135 s content_block_delta:input_json_delta","148137 s content_block_delta:input_json_delta","149267 s content_block_delta:input_json_delta","149609 a t12:Bash","149617 s content_block_stop","149620 s message_delta","149620 s message_stop","149620 u t12","149620 y status","151864 s message_start","152797 s content_block_start","152797 y thinking_tokens","152797 s content_block_delta:thinking_delta","152797 y thinking_tokens","154164 s content_block_delta:thinking_delta","154164 y thinking_tokens","155373 s content_block_delta:thinking_delta","155373 y thinking_tokens","156749 s content_block_delta:thinking_delta","156749 y thinking_tokens","157882 s content_block_delta:thinking_delta","157882 y thinking_tokens","159255 s content_block_delta:thinking_delta","159255 y thinking_tokens","160567 s content_block_delta:thinking_delta","160567 y thinking_tokens","161692 s content_block_delta:thinking_delta","161692 y thinking_tokens","162755 s content_block_delta:thinking_delta","162755 y thinking_tokens","164192 s content_block_delta:thinking_delta","164192 y thinking_tokens","165442 s content_block_delta:thinking_delta","165442 y thinking_tokens","166817 s content_block_delta:thinking_delta","166817 y thinking_tokens","168067 s content_block_delta:thinking_delta","168067 y thinking_tokens","169504 s content_block_delta:thinking_delta","169504 y thinking_tokens","170816 s content_block_delta:thinking_delta","170816 y thinking_tokens","172378 s content_block_delta:thinking_delta","172378 y thinking_tokens","173442 s content_block_delta:thinking_delta","173442 y thinking_tokens","174629 s content_block_delta:thinking_delta","174629 y thinking_tokens","175816 s content_block_delta:thinking_delta","175816 y thinking_tokens","177066 s content_block_delta:thinking_delta","179950 s content_block_delta:thinking_delta","179950 s content_block_delta:signature_delta","179950 a","179952 s content_block_stop","179952 s content_block_start","179952 y thinking_tokens","179952 s content_block_delta:thinking_delta","179952 y thinking_tokens","179952 s content_block_delta:thinking_delta","179952 y thinking_tokens","179952 s content_block_delta:thinking_delta","179952 y thinking_tokens","179952 s content_block_delta:thinking_delta","179952 y thinking_tokens","179952 s content_block_delta:thinking_delta","179952 y thinking_tokens","179952 s content_block_delta:thinking_delta","179952 y thinking_tokens","179952 s content_block_delta:thinking_delta","179952 y thinking_tokens","179952 s content_block_delta:thinking_delta","179952 y thinking_tokens","179952 s content_block_delta:thinking_delta","179952 y thinking_tokens","179952 s content_block_delta:thinking_delta","179952 y thinking_tokens","179952 s content_block_delta:thinking_delta","179952 y thinking_tokens","179952 s content_block_delta:signature_delta","179952 a","179953 s content_block_stop","179953 s content_block_start","179953 s content_block_delta:input_json_delta","181003 s content_block_delta:input_json_delta","182003 s content_block_delta:input_json_delta","183004 s content_block_delta:input_json_delta","184005 s content_block_delta:input_json_delta","185005 s content_block_delta:input_json_delta","186067 s content_block_delta:input_json_delta","187067 s content_block_delta:input_json_delta","188067 s content_block_delta:input_json_delta","189068 s content_block_delta:input_json_delta","190130 s content_block_delta:input_json_delta","191133 s content_block_delta:input_json_delta","192193 s content_block_delta:input_json_delta","193193 s content_block_delta:input_json_delta","194194 s content_block_delta:input_json_delta","195194 s content_block_delta:input_json_delta","196194 s content_block_delta:input_json_delta","197195 s content_block_delta:input_json_delta","198195 s content_block_delta:input_json_delta","199195 s content_block_delta:input_json_delta","200197 s content_block_delta:input_json_delta","201257 s content_block_delta:input_json_delta","202257 s content_block_delta:input_json_delta","203257 s content_block_delta:input_json_delta","204257 s content_block_delta:input_json_delta","205258 s content_block_delta:input_json_delta","206258 s content_block_delta:input_json_delta","207258 s content_block_delta:input_json_delta","208258 s content_block_delta:input_json_delta","209259 s content_block_delta:input_json_delta","210321 s content_block_delta:input_json_delta","211383 s content_block_delta:input_json_delta","212383 s content_block_delta:input_json_delta","213383 s content_block_delta:input_json_delta","214383 s content_block_delta:input_json_delta","215384 s content_block_delta:input_json_delta","216446 s content_block_delta:input_json_delta","217446 s content_block_delta:input_json_delta","218448 s content_block_delta:input_json_delta","219508 s content_block_delta:input_json_delta","220508 s content_block_delta:input_json_delta","221508 s content_block_delta:input_json_delta","222509 s content_block_delta:input_json_delta","223570 s content_block_delta:input_json_delta","224570 s content_block_delta:input_json_delta","225570 s content_block_delta:input_json_delta","226571 s content_block_delta:input_json_delta","227571 s content_block_delta:input_json_delta","228571 s content_block_delta:input_json_delta","229571 s content_block_delta:input_json_delta","230571 s content_block_delta:input_json_delta","231574 s content_block_delta:input_json_delta","232633 s content_block_delta:input_json_delta","233633 s content_block_delta:input_json_delta","234633 s content_block_delta:input_json_delta","235633 s content_block_delta:input_json_delta","236633 s content_block_delta:input_json_delta","237634 s content_block_delta:input_json_delta","238634 s content_block_delta:input_json_delta","239634 s content_block_delta:input_json_delta","240635 s content_block_delta:input_json_delta","241696 s content_block_delta:input_json_delta","242704 s content_block_delta:input_json_delta","243761 s content_block_delta:input_json_delta","244762 s content_block_delta:input_json_delta","245822 s content_block_delta:input_json_delta","246826 s content_block_delta:input_json_delta","247885 s content_block_delta:input_json_delta","248887 s content_block_delta:input_json_delta","249948 s content_block_delta:input_json_delta","251010 s content_block_delta:input_json_delta","252010 s content_block_delta:input_json_delta","253010 s content_block_delta:input_json_delta","254010 s content_block_delta:input_json_delta","255010 s content_block_delta:input_json_delta","256011 s content_block_delta:input_json_delta","257011 s content_block_delta:input_json_delta","258011 s content_block_delta:input_json_delta","259011 s content_block_delta:input_json_delta","260011 s content_block_delta:input_json_delta","261015 s content_block_delta:input_json_delta","262074 s content_block_delta:input_json_delta","263074 s content_block_delta:input_json_delta","264136 s content_block_delta:input_json_delta","265136 s content_block_delta:input_json_delta","266136 s content_block_delta:input_json_delta","267136 s content_block_delta:input_json_delta","268136 s content_block_delta:input_json_delta","269136 s content_block_delta:input_json_delta","270136 s content_block_delta:input_json_delta","271136 s content_block_delta:input_json_delta","272136 s content_block_delta:input_json_delta","273137 s content_block_delta:input_json_delta","274137 s content_block_delta:input_json_delta","275137 s content_block_delta:input_json_delta","276138 s content_block_delta:input_json_delta","277138 s content_block_delta:input_json_delta","278138 s content_block_delta:input_json_delta","279141 s content_block_delta:input_json_delta","280201 s content_block_delta:input_json_delta","281201 s content_block_delta:input_json_delta","281365 a t13:Bash","281370 s content_block_stop","281371 s message_delta","281371 s message_stop","281371 u t13","281371 y status","283823 s message_start","284622 s content_block_start","284622 y thinking_tokens","284644 s content_block_delta:thinking_delta","286279 s content_block_delta:thinking_delta","286279 y thinking_tokens","286279 s content_block_delta:signature_delta","286279 a","286280 s content_block_stop","286280 s content_block_start","286280 s content_block_delta:text_delta","286280 a","286280 s content_block_stop","286280 s content_block_start","286280 s content_block_delta:input_json_delta","287371 s content_block_delta:input_json_delta","288395 s content_block_delta:input_json_delta","289402 s content_block_delta:input_json_delta","290402 s content_block_delta:input_json_delta","291402 s content_block_delta:input_json_delta","292346 a t14:Bash","292352 s content_block_stop","292365 s message_delta","292366 s message_stop","292366 u t14","292366 y status","302038 s message_start","302039 s content_block_start","302039 s content_block_delta:text_delta","302048 a","302049 s content_block_stop","302050 s message_delta","302050 s message_stop","302050 r"]},{"case":"/review army consensus","artifact":"paid-slice-12-a1","events":["0 y init","0 y status","3084 s message_start","3085 s content_block_start","3328 s content_block_delta:thinking_delta","3329 s content_block_delta:signature_delta","3329 a","3330 s content_block_stop","3330 s content_block_start","3330 s content_block_delta:text_delta","4084 a","4096 s content_block_stop","4097 s content_block_start","4097 s content_block_delta:input_json_delta","5417 s content_block_delta:input_json_delta","7114 s content_block_delta:input_json_delta","7116 a t0:Bash","7126 s content_block_stop","7132 s message_delta","7132 s message_stop","7132 u t0","7132 y status","10602 s message_start","10602 s content_block_start","10602 s content_block_delta:text_delta","10612 a","10612 s content_block_stop","10613 s content_block_start","10613 s content_block_delta:input_json_delta","11675 s content_block_delta:input_json_delta","12770 s content_block_delta:input_json_delta","13798 s content_block_delta:input_json_delta","15056 s content_block_delta:input_json_delta","16115 s content_block_delta:input_json_delta","17315 s content_block_delta:input_json_delta","18363 s content_block_delta:input_json_delta","19180 a t1:Agent","19184 s content_block_stop","19184 y task_started","19184 u","19196 s content_block_start","19199 s content_block_delta:input_json_delta","20240 s content_block_delta:input_json_delta","21240 s content_block_delta:input_json_delta","22241 s content_block_delta:input_json_delta","23241 s content_block_delta:input_json_delta","23241 y task_progress","23241 a* t2:Read","23305 s content_block_delta:input_json_delta","23305 u t2","23367 s content_block_delta:input_json_delta","24367 s content_block_delta:input_json_delta","25242 y task_progress","25242 a* t3:Bash","25304 s content_block_delta:input_json_delta","25429 u t3","25493 s content_block_delta:input_json_delta","25680 a t4:Agent","25682 s content_block_stop","25682 y task_started","25682 u","25689 s message_delta","25690 s message_stop","25690 y task_progress","25690 a* t5:Bash","25690 u t5","25690 y task_updated","25690 y task_notification","25690 u t1","25690 y task_updated","25690 y task_notification","25690 u t4","25690 y status","55795 s message_start","56553 s content_block_start","56553 y thinking_tokens","56553 s content_block_delta:thinking_delta","56553 y thinking_tokens","58052 s content_block_delta:thinking_delta","58052 y thinking_tokens","59493 s content_block_delta:thinking_delta","59493 y thinking_tokens","60765 s content_block_delta:thinking_delta","60765 y thinking_tokens","61867 s content_block_delta:thinking_delta","61867 y thinking_tokens","62994 s content_block_delta:thinking_delta","62994 y thinking_tokens","64185 s content_block_delta:thinking_delta","64185 y thinking_tokens","65686 s content_block_delta:thinking_delta","65686 y thinking_tokens","66938 s content_block_delta:thinking_delta","66938 y thinking_tokens","68186 s content_block_delta:thinking_delta","68186 y thinking_tokens","69375 s content_block_delta:thinking_delta","69375 y thinking_tokens","70562 s content_block_delta:thinking_delta","70562 y thinking_tokens","71936 s content_block_delta:thinking_delta","71936 y thinking_tokens","73248 s content_block_delta:thinking_delta","73248 y thinking_tokens","74316 s content_block_delta:thinking_delta","74316 y thinking_tokens","75686 s content_block_delta:thinking_delta","75686 y thinking_tokens","76874 s content_block_delta:thinking_delta","76874 y thinking_tokens","78186 s content_block_delta:thinking_delta","78186 y thinking_tokens","79436 s content_block_delta:thinking_delta","79436 y thinking_tokens","80812 s content_block_delta:thinking_delta","80812 y thinking_tokens","81874 s content_block_delta:thinking_delta","81874 y thinking_tokens","83188 s content_block_delta:thinking_delta","83188 y thinking_tokens","84314 s content_block_delta:thinking_delta","84314 y thinking_tokens","85750 s content_block_delta:thinking_delta","89831 s content_block_delta:thinking_delta","89832 s content_block_delta:signature_delta","89832 a","89834 s content_block_stop","89834 s content_block_start","89834 y thinking_tokens","89834 s content_block_delta:thinking_delta","89834 y thinking_tokens","89834 s content_block_delta:thinking_delta","89834 y thinking_tokens","89834 s content_block_delta:thinking_delta","89834 y thinking_tokens","89834 s content_block_delta:thinking_delta","89834 y thinking_tokens","89834 s content_block_delta:thinking_delta","89834 y thinking_tokens","89834 s content_block_delta:thinking_delta","89834 y thinking_tokens","89834 s content_block_delta:thinking_delta","89834 y thinking_tokens","89834 s content_block_delta:thinking_delta","89834 y thinking_tokens","89834 s content_block_delta:thinking_delta","89834 y thinking_tokens","89835 s content_block_delta:thinking_delta","89835 y thinking_tokens","89835 s content_block_delta:thinking_delta","89835 y thinking_tokens","89835 s content_block_delta:thinking_delta","89835 y thinking_tokens","89835 s content_block_delta:thinking_delta","89835 y thinking_tokens","89835 s content_block_delta:thinking_delta","89835 y thinking_tokens","89835 s content_block_delta:thinking_delta","89835 y thinking_tokens","89835 s content_block_delta:thinking_delta","89835 y thinking_tokens","89835 s content_block_delta:signature_delta","89835 a","89835 s content_block_stop","89835 s content_block_start","89835 s content_block_delta:input_json_delta","90876 s content_block_delta:input_json_delta","91936 s content_block_delta:input_json_delta","92937 s content_block_delta:input_json_delta","93999 s content_block_delta:input_json_delta","95005 s content_block_delta:input_json_delta","96062 s content_block_delta:input_json_delta","97066 s content_block_delta:input_json_delta","98126 s content_block_delta:input_json_delta","99126 s content_block_delta:input_json_delta","100126 s content_block_delta:input_json_delta","101126 s content_block_delta:input_json_delta","101857 a t6:Bash","101866 s content_block_stop","101870 s message_delta","101870 s message_stop","101870 u t6","101870 y status","106663 s message_start","106664 s content_block_start","106664 s content_block_delta:text_delta","106674 a","106675 s content_block_stop","106675 s message_delta","106675 s message_stop","106675 r"]},{"case":"/plan-design-review plan-mode","artifact":"paid-slice-9-a1","events":["0 y init","0 y status","1974 s message_start","1974 s content_block_start","1974 y thinking_tokens","2693 s content_block_delta:thinking_delta","3129 y thinking_tokens","3130 s content_block_delta:signature_delta","3130 a","3131 s content_block_stop","3131 s content_block_start","3131 s content_block_delta:text_delta","4191 s content_block_delta:text_delta","4191 a","4264 s content_block_stop","4264 s content_block_start","4264 s content_block_delta:input_json_delta","5443 s content_block_delta:input_json_delta","5443 a t0:Read","5449 s content_block_stop","5450 s content_block_start","5450 s content_block_delta:input_json_delta","5569 u t0","5631 s content_block_delta:input_json_delta","6324 a t1:Read","6327 s content_block_stop","6328 s content_block_start","6328 s content_block_delta:input_json_delta","6448 u t1","6517 s content_block_delta:input_json_delta","6979 a t2:Read","6982 s content_block_stop","6984 s message_delta","6984 s message_stop","6984 u t2","6984 y status","10028 s message_start","10028 s content_block_start","10028 s content_block_delta:text_delta","10038 a","10038 s content_block_stop","10038 s content_block_start","10038 s content_block_delta:input_json_delta","11154 s content_block_delta:input_json_delta","11154 a t3:Read","11157 s content_block_stop","11158 s message_delta","11158 s message_stop","11158 u t3","11158 y status","12170 s message_start","12666 s content_block_start","12666 y thinking_tokens","12920 s content_block_delta:thinking_delta","12920 y thinking_tokens","14292 s content_block_delta:thinking_delta","14292 y thinking_tokens","15422 s content_block_delta:thinking_delta","15422 y thinking_tokens","16864 s content_block_delta:thinking_delta","16864 y thinking_tokens","18240 s content_block_delta:thinking_delta","18240 y thinking_tokens","19737 s content_block_delta:thinking_delta","19737 y thinking_tokens","20800 s content_block_delta:thinking_delta","20800 y thinking_tokens","22049 s content_block_delta:thinking_delta","22049 y thinking_tokens","23549 s content_block_delta:thinking_delta","23549 y thinking_tokens","24738 s content_block_delta:thinking_delta","24738 y thinking_tokens","25800 s content_block_delta:thinking_delta","25800 y thinking_tokens","27175 s content_block_delta:thinking_delta","27175 y thinking_tokens","28490 s content_block_delta:thinking_delta","28490 y thinking_tokens","29550 s content_block_delta:thinking_delta","29550 y thinking_tokens","30923 s content_block_delta:thinking_delta","30923 y thinking_tokens","32236 s content_block_delta:thinking_delta","32236 y thinking_tokens","33297 s content_block_delta:thinking_delta","33297 y thinking_tokens","34609 s content_block_delta:thinking_delta","34609 y thinking_tokens","36109 s content_block_delta:thinking_delta","36109 y thinking_tokens","37296 s content_block_delta:thinking_delta","37296 y thinking_tokens","38925 s content_block_delta:thinking_delta","38925 y thinking_tokens","40232 s content_block_delta:thinking_delta","40232 y thinking_tokens","41420 s content_block_delta:thinking_delta","41420 y thinking_tokens","42607 s content_block_delta:thinking_delta","42607 y thinking_tokens","43732 s content_block_delta:thinking_delta","43732 y thinking_tokens","44919 s content_block_delta:thinking_delta","44919 y thinking_tokens","46044 s content_block_delta:thinking_delta","46044 y thinking_tokens","47106 s content_block_delta:thinking_delta","47106 y thinking_tokens","48230 s content_block_delta:thinking_delta","48230 y thinking_tokens","49357 s content_block_delta:thinking_delta","49357 y thinking_tokens","50734 s content_block_delta:thinking_delta","50734 y thinking_tokens","52043 s content_block_delta:thinking_delta","52043 y thinking_tokens","53293 s content_block_delta:thinking_delta","53293 y thinking_tokens","54544 s content_block_delta:thinking_delta","54544 y thinking_tokens","55982 s content_block_delta:thinking_delta","55982 y thinking_tokens","57482 s content_block_delta:thinking_delta","57482 y thinking_tokens","58608 s content_block_delta:thinking_delta","59632 s content_block_delta:thinking_delta","59633 s content_block_delta:signature_delta","59633 a","59634 s content_block_stop","59634 s content_block_start","59634 s content_block_delta:text_delta","59635 a","59635 s content_block_stop","59636 s content_block_start","59636 s content_block_delta:input_json_delta","60670 s content_block_delta:input_json_delta","61671 s content_block_delta:input_json_delta","62672 s content_block_delta:input_json_delta","63672 s content_block_delta:input_json_delta","64734 s content_block_delta:input_json_delta","65735 s content_block_delta:input_json_delta","66797 s content_block_delta:input_json_delta","67797 s content_block_delta:input_json_delta","68797 s content_block_delta:input_json_delta","69798 s content_block_delta:input_json_delta","70924 s content_block_delta:input_json_delta","71984 s content_block_delta:input_json_delta","72984 s content_block_delta:input_json_delta","73984 s content_block_delta:input_json_delta","74985 s content_block_delta:input_json_delta","75986 s content_block_delta:input_json_delta","77004 s content_block_delta:input_json_delta","78048 s content_block_delta:input_json_delta","79048 s content_block_delta:input_json_delta","80048 s content_block_delta:input_json_delta","81048 s content_block_delta:input_json_delta","82049 s content_block_delta:input_json_delta","83111 s content_block_delta:input_json_delta","84111 s content_block_delta:input_json_delta","85111 s content_block_delta:input_json_delta","86111 s content_block_delta:input_json_delta","87111 s content_block_delta:input_json_delta","88112 s content_block_delta:input_json_delta","89174 s content_block_delta:input_json_delta","90174 s content_block_delta:input_json_delta","91174 s content_block_delta:input_json_delta","92174 s content_block_delta:input_json_delta","93176 s content_block_delta:input_json_delta","94237 s content_block_delta:input_json_delta","95242 s content_block_delta:input_json_delta","96300 s content_block_delta:input_json_delta","97301 s content_block_delta:input_json_delta","98363 s content_block_delta:input_json_delta","99363 s content_block_delta:input_json_delta","100425 s content_block_delta:input_json_delta","101425 s content_block_delta:input_json_delta","102425 s content_block_delta:input_json_delta","103425 s content_block_delta:input_json_delta","104425 s content_block_delta:input_json_delta","105425 s content_block_delta:input_json_delta","106425 s content_block_delta:input_json_delta","107425 s content_block_delta:input_json_delta","108425 s content_block_delta:input_json_delta","109429 s content_block_delta:input_json_delta","110503 s content_block_delta:input_json_delta","111552 s content_block_delta:input_json_delta","112552 s content_block_delta:input_json_delta","113614 s content_block_delta:input_json_delta","114614 s content_block_delta:input_json_delta","115677 s content_block_delta:input_json_delta","116677 s content_block_delta:input_json_delta","117677 s content_block_delta:input_json_delta","118739 s content_block_delta:input_json_delta","119741 s content_block_delta:input_json_delta","120746 s content_block_delta:input_json_delta","121864 s content_block_delta:input_json_delta","122864 s content_block_delta:input_json_delta","123865 s content_block_delta:input_json_delta","124865 s content_block_delta:input_json_delta","125927 s content_block_delta:input_json_delta","126927 s content_block_delta:input_json_delta","127927 s content_block_delta:input_json_delta","128927 s content_block_delta:input_json_delta","129927 s content_block_delta:input_json_delta","130927 s content_block_delta:input_json_delta","131927 s content_block_delta:input_json_delta","132927 s content_block_delta:input_json_delta","133927 s content_block_delta:input_json_delta","134927 s content_block_delta:input_json_delta","135927 s content_block_delta:input_json_delta","136928 s content_block_delta:input_json_delta","137990 s content_block_delta:input_json_delta","138991 s content_block_delta:input_json_delta","140053 s content_block_delta:input_json_delta","141054 s content_block_delta:input_json_delta","142115 s content_block_delta:input_json_delta","143117 s content_block_delta:input_json_delta","144121 s content_block_delta:input_json_delta","145178 s content_block_delta:input_json_delta","146180 s content_block_delta:input_json_delta","147304 s content_block_delta:input_json_delta","148304 s content_block_delta:input_json_delta","148745 a t4:Write","148749 s content_block_stop","148750 s message_delta","148751 s message_stop","148751 u t4","148751 y status","150776 s message_start","150777 s content_block_start","150777 s content_block_delta:text_delta","150788 a","150789 s content_block_stop","150789 s content_block_start","150789 s content_block_delta:input_json_delta","151425 a t5:Read","151429 s content_block_stop","151430 s message_delta","151430 s message_stop","151430 u t5","151430 y status","152573 s message_start","153081 s content_block_start","153081 y thinking_tokens","153236 s content_block_delta:thinking_delta","155122 s content_block_delta:thinking_delta","155122 y thinking_tokens","155122 s content_block_delta:signature_delta","155122 a","155123 s content_block_stop","155123 s content_block_start","155123 s content_block_delta:text_delta","155123 a","155124 s content_block_stop","155124 s content_block_start","155124 s content_block_delta:input_json_delta","156172 s content_block_delta:input_json_delta","157304 s content_block_delta:input_json_delta","157806 a t6:Edit","157808 s content_block_stop","157808 s content_block_start","157808 s content_block_delta:input_json_delta","157808 u t6","157932 s content_block_delta:input_json_delta","159127 s content_block_delta:input_json_delta","159127 a t7:Edit","159129 s content_block_stop","159129 s content_block_start","159129 s content_block_delta:input_json_delta","159129 u t7","159253 s content_block_delta:input_json_delta","160257 s content_block_delta:input_json_delta","160477 a t8:Edit","160478 s content_block_stop","160478 s message_delta","160479 s message_stop","160479 u t8","160479 y status","168488 s message_start","168488 s content_block_start","168488 s content_block_delta:text_delta","168502 a","168503 s content_block_stop","168503 s message_delta","168503 s message_stop","168503 r"]}]} \ No newline at end of file diff --git a/test/fixtures/trial-record-v1-reader.ts b/test/fixtures/trial-record-v1-reader.ts new file mode 100644 index 000000000..2827c1a9d --- /dev/null +++ b/test/fixtures/trial-record-v1-reader.ts @@ -0,0 +1,34 @@ +/** + * FROZEN: trialRecordProblems exactly as EVAL_POLICY v1 readers ship it + * (origin/main 2db0b3adc, test/helpers/eval-store.ts). Never edit: it stands in + * for older checkouts reading trial records written by this one, so new + * optional fields must stay invisible to it (test/helpers/eval-store.test.ts). + */ +const TRIAL_OUTCOME_SCHEMA = 'gstack-trial-outcome/v1'; +const TRIAL_ERROR_MAX = 300; +const EVAL_KINDS: readonly string[] = ['rule', 'behavior', 'judge']; +const FAILURE_CLASSES: readonly string[] = ['assertion', 'contract', 'timeout', 'infra']; + +export function trialRecordProblems(r: any): string[] { + const problems: string[] = []; + if (!r || typeof r !== 'object' || Array.isArray(r)) return ['not an object']; + if (r.schema !== TRIAL_OUTCOME_SCHEMA) problems.push(`schema ${String(r.schema)}`); + for (const key of ['case', 'file', 'tier'] as const) if (typeof r[key] !== 'string' || r[key].length === 0) problems.push(`${key} missing`); + if (!EVAL_KINDS.includes(r.kind)) problems.push(`kind ${String(r.kind)}`); + const n = r.panel?.n, k = r.panel?.k; + if (!Number.isInteger(n) || !Number.isInteger(k) || n < 1 || k < 1 || k > n) problems.push('panel invalid'); + if (!Number.isInteger(r.trial) || r.trial < 1 || (Number.isInteger(n) && r.trial > n)) problems.push('trial invalid'); + if (!Number.isInteger(r.attempt) || r.attempt < 1) problems.push('attempt invalid'); + if (!['passed', 'failed', 'skipped'].includes(r.outcome)) problems.push(`outcome ${String(r.outcome)}`); + if (r.outcome === 'failed' && !FAILURE_CLASSES.includes(r.failure_class)) problems.push('failed without failure_class'); + if (r.outcome !== 'failed' && r.failure_class !== undefined) problems.push('failure_class on a non-failed trial'); + if (typeof r.duration_ms !== 'number' || !Number.isFinite(r.duration_ms) || r.duration_ms < 0) problems.push('duration_ms invalid'); + if (typeof r.cost_usd !== 'number' || !Number.isFinite(r.cost_usd) || r.cost_usd < 0) problems.push('cost_usd invalid'); + if (!Number.isInteger(r.policy_version) || r.policy_version < 0) problems.push('policy_version invalid'); + if (typeof r.quarantined !== 'boolean') problems.push('quarantined invalid'); + if (r.execution !== 'executed' && r.execution !== 'reused') problems.push('execution invalid'); + if (!['shard', 'junit', 'backfill'].includes(r.source)) problems.push('source invalid'); + if (r.error !== undefined && (typeof r.error !== 'string' || r.error.length > TRIAL_ERROR_MAX)) problems.push('error invalid'); + if (r.series_identity !== undefined && (typeof r.series_identity !== 'string' || !/^[\w.-]{1,64}$/.test(r.series_identity))) problems.push('series_identity invalid'); + return problems; +} diff --git a/test/helpers/eval-budgets.ts b/test/helpers/eval-budgets.ts index 824be22c1..667d0e09b 100644 --- a/test/helpers/eval-budgets.ts +++ b/test/helpers/eval-budgets.ts @@ -130,3 +130,20 @@ export function assertPaidTestBudget(file: string, ms: number): void { throw new Error(`Unregistered paid test budget: ${file}: ${ms}`); } } + +/** + * Diagnostic thresholds, outside EVAL_POLICY: v1 verdicts never read them. + * A session above HEADROOM_WARN of the timeout it armed is flagged by + * `eval:pass-rates --headroom`; above HEADROOM_FAIL the weekly `--gate` alarms. + * Cut work to get under them; budgets are never raised. + */ +export const HEADROOM_WARN = 0.75; +export const HEADROOM_FAIL = 0.85; + +/** + * In-request stream silence that labels a provider stall (failure_cause + * provider_stall, still a failure under EVAL_POLICY v1). Pre-registered from + * census 37198445662 on Claude Code 2.1.284: the longest gap inside one model + * message across 50 messages was 9.6 s, thinking deltas included. + */ +export const STALL_WINDOW_MS = 120_000; diff --git a/test/helpers/eval-store.test.ts b/test/helpers/eval-store.test.ts index 8c55906c0..a2d6d6422 100644 --- a/test/helpers/eval-store.test.ts +++ b/test/helpers/eval-store.test.ts @@ -25,10 +25,12 @@ import { parseTrialOutcomes, sanitizeTrialError, trialContextFromEnv, + TRIAL_ERROR_MAX, } from './eval-store'; import type { EvalResult, EvalTestEntry, ComparisonResult, PanelTrial, TrialOutcomeRecord } from './eval-store'; import { EVAL_POLICY } from './periodic-exclude-data'; import { manualReviewFixture } from './manual-judge-review-fixture'; +import { trialRecordProblems as trialRecordProblemsV1 } from '../fixtures/trial-record-v1-reader'; let tmpDir: string; @@ -1151,6 +1153,35 @@ describe('trial-outcomes JSONL', () => { expect(parseTrialOutcomes(text, { maxBytes: 10 }).errors[0]).toContain('exceed'); }); + test('new optional fields: valid records pass the current reader and the frozen v1 reader', () => { + const failed = record({ trial: 2, outcome: 'failed', failure_class: 'assertion', exit_reason: 'success', + error: sanitizeTrialError('expect(received).toBe(expected)\n\nExpected: "b"\nReceived: "a"'), + failure_cause: 'assertion', failure_cause_evidence: 'session completed; check failed', + failure_detail: { expected: '"b"', received: '"a"' }, cost_known: false, + sessions: [{ key: 'claude-p:case-x#1', runner: 'claude-p', elapsed_ms: 1200, budget_ms: 300_000, end: 'completed' }] }); + const judge = record({ trial: 3, outcome: 'failed', failure_class: 'assertion', failure_cause: 'assertion', + failure_detail: { judge: [{ dimension: 'clarity', mean: 3.67, threshold: 4, samples: 3, rationale: 'Clarity suffers from density.' }] } }); + const passed = record({ sessions: [{ key: 'pty:case-x#1', runner: 'pty', elapsed_ms: 10, end: 'completed' }] }); + for (const r of [failed, judge, passed]) { + expect(trialRecordProblemsV1(JSON.parse(JSON.stringify(r)))).toEqual([]); + expect(r.error === undefined || r.error.length <= TRIAL_ERROR_MAX).toBe(true); + } + expect(parseTrialOutcomes(formatTrialOutcomes([failed, judge, passed]))).toEqual({ records: [failed, judge, passed], errors: [] }); + }); + + test('new optional fields are bounded and checked only when present', () => { + const failed = (extra: Partial | Record) => record({ outcome: 'failed', failure_class: 'assertion', ...extra } as Partial); + const problems = (r: unknown) => parseTrialOutcomes(JSON.stringify(r)).errors; + expect(problems(record({ failure_cause: 'assertion' }))[0]).toContain('failure_cause on a non-failed trial'); + expect(problems(failed({ failure_cause: 'detector' }))[0]).toContain("unknown failure_cause \"detector\" (written by a newer gstack; update this checkout to read it)"); + expect(problems(failed({ failure_cause_evidence: 'x'.repeat(301) }))[0]).toContain('failure_cause_evidence invalid'); + expect(problems(failed({ failure_detail: { expected: 'x'.repeat(201), received: '' } }))[0]).toContain('failure_detail invalid'); + expect(problems(failed({ failure_detail: { judge: [] } }))[0]).toContain('failure_detail invalid'); + expect(problems(record({ sessions: [{ key: 'k', runner: 'pty', elapsed_ms: -1, end: 'completed' }] }))[0]).toContain('sessions invalid'); + expect(problems(record({ cost_known: 'no' } as any))[0]).toContain('cost_known invalid'); + expect(problems(failed({}))).toEqual([]); + }); + test('sanitizeTrialError keeps one capped line without mentions', () => { expect(sanitizeTrialError('\n expected @garrytan to `see`\nsecond')).toBe("expected @\u200bgarrytan to 'see'"); expect(sanitizeTrialError('x'.repeat(1000))!.length).toBe(300); diff --git a/test/helpers/eval-store.ts b/test/helpers/eval-store.ts index 23ec90ffd..8e1b8ea9d 100644 --- a/test/helpers/eval-store.ts +++ b/test/helpers/eval-store.ts @@ -14,6 +14,7 @@ import * as os from 'os'; import { spawnSync } from 'child_process'; import { isManualReviewEntry } from './cookie-workflow-manual-review'; import type { ManualJudgeReview } from './cookie-workflow-manual-review'; +import { STALL_WINDOW_MS } from './eval-budgets'; // v2: EvalTestEntry.harvest gains optional {insertions, deletions, net} and // may be explicitly null (arm-benchmark harvest-failure taxonomy). Readers @@ -210,6 +211,151 @@ export function failureClassOf(entry: Pick { + const first = text?.split('\n').map((l) => l.trim()).find((l) => l.length > 0); + if (!first) return undefined; + // eslint-disable-next-line no-control-regex + const clean = first.replace(/[\u0000-\u001f\u007f]/g, ' ').replace(/@(?=[A-Za-z0-9_-])/g, '@\u200b'); + return clean.length > max ? `${clean.slice(0, max - 1)}…` : clean; +}; + +/** + * Pure and dependency-free: the cause of one failed trial from its recorded + * facts. Precedence contract > pre_turn_infra > api_error > refusal > + * provider_stall > session_timeout > observer_timeout > assertion > unknown; + * the highest cause with evidence wins and its evidence line is returned. + * A provider stall needs a session that streamed partial messages, did not + * complete, and shows STALL_WINDOW_MS of silence with a request in flight and + * nothing outstanding; sessions without partial messages are never stalls. + */ +export function failureCauseOf(facts: FailureCauseFacts, stallWindowMs = STALL_WINDOW_MS): { cause: TrialFailureCause; evidence?: string } { + const sessions = facts.sessions ?? []; + const ended = (end: string) => sessions.find((s) => s.end === end); + const out = (cause: TrialFailureCause, evidence?: string) => { + const line = oneLine(evidence, FAILURE_CAUSE_EVIDENCE_MAX); + return line ? { cause, evidence: line } : { cause }; + }; + if (facts.failure_class === 'contract') return out('contract', facts.error); + if (facts.failure_class === 'infra') return out('pre_turn_infra', facts.error ?? facts.exit_reason); + const api = ended('api_error'); + if (api) return out('api_error', api.evidence); + if (facts.exit_reason === 'error_api') return out('api_error', facts.error); + const refusal = ended('refusal'); + if (refusal) return out('refusal', refusal.evidence); + const stall = sessions.find((s) => s.end !== 'completed' && s.liveness?.partial === true && s.liveness.max_request_silence_ms >= stallWindowMs); + if (stall?.liveness) { + const l = stall.liveness; + return out('provider_stall', `no stream event for ${Math.round(l.max_request_silence_ms / 1000)}s` + + `${l.silence_started_ms !== undefined ? ` from ${Math.round(l.silence_started_ms / 1000)}s` : ''}` + + `${l.silence_after ? ` (last: ${l.silence_after})` : ''}; model request in flight, no tool outstanding`); + } + const clock = (s: SessionCauseFacts) => `${s.key ?? 'session'} ran ${Math.round((s.elapsed_ms ?? 0) / 1000)}s` + + `${s.budget_ms ? ` of its ${Math.round(s.budget_ms / 1000)}s budget` : ''}`; + const timedOut = ended('session_timeout'); + if (timedOut) return out('session_timeout', timedOut.evidence ?? clock(timedOut)); + if (facts.exit_reason === 'timeout') return out('session_timeout', 'the runner\'s armed session timeout fired'); + const observerText = /\boutcome=timeout\b/.test(facts.error ?? '') ? facts.error : undefined; + const observer = ended('observer_timeout'); + if (observer) return out('observer_timeout', observer.evidence ?? observerText ?? clock(observer)); + if (observerText) return out('observer_timeout', observerText); + if (facts.failure_class === 'assertion') return out('assertion', facts.exit_reason === 'success' ? 'session completed; check failed' : undefined); + return out('unknown', facts.failure_class === 'timeout' ? 'case budget expired with no session evidence' : undefined); +} + +/** Expected/Received values of a failed Bun matcher, or a judge's failing dimensions. */ +export type TrialFailureDetail = + | { expected: string; received: string } + | { judge: Array<{ dimension: string; mean: number; threshold: number; samples: number; rationale?: string }> }; + +const detailPart = (text: string): string => { + // eslint-disable-next-line no-control-regex + const flat = text.replace(/[\u0000-\u001f\u007f]+/g, ' ').replace(/\s+/g, ' ').trim().replace(/@(?=[A-Za-z0-9_-])/g, '@\u200b'); + return flat.length > FAILURE_DETAIL_PART_MAX ? `${flat.slice(0, FAILURE_DETAIL_PART_MAX - 1)}…` : flat; +}; + +/** + * Detail the stored error line cannot carry (it keeps only the matcher header). + * A judge record with a `>= N` threshold names each dimension below it, its + * mean, its sample count and the first sample sentence naming that dimension; + * otherwise a Bun matcher message yields its Expected/Received values. + */ +export function failureDetailOf(text: string | undefined, + record?: { judge_scores?: unknown; judge_reasoning?: unknown }): TrialFailureDetail | undefined { + if (!text) return undefined; + const threshold = /^Expected:\s*>=?\s*(-?[\d.]+)\s*$/m.exec(text); + const scores = record?.judge_scores; + if (threshold && scores && typeof scores === 'object' && !Array.isArray(scores)) { + const limit = Number(threshold[1]); + const reasoning = typeof record?.judge_reasoning === 'string' ? record.judge_reasoning : ''; + const samples = reasoning.split(/\[sample \d+\]\s*/).map((s) => s.trim()).filter(Boolean); + const judge = Object.entries(scores as Record) + .filter((entry): entry is [string, number] => typeof entry[1] === 'number' && Number.isFinite(entry[1]) && entry[1] < limit) + .slice(0, 8).map(([dimension, mean]) => { + const word = dimension.replace(/_/g, ' ').toLowerCase(); + const sentence = samples.flatMap((s) => s.split(/(?<=[.!?])\s+/)).find((s) => s.toLowerCase().includes(word)); + return { dimension: detailPart(dimension), mean: Math.round(mean * 100) / 100, threshold: limit, + samples: Math.max(1, samples.length), ...(sentence ? { rationale: detailPart(sentence.slice(0, 150)) } : {}) }; + }); + if (judge.length) return { judge }; + } + const lines = text.split('\n'); + const expected = lines.map((l) => /^Expected(?:[^:]*)?:\s?(.*)$/.exec(l)).find(Boolean); + const received = lines.map((l) => /^Received(?:[^:]*)?:\s?(.*)$/.exec(l)).find(Boolean); + if (expected && received && !/^\s*[-+]\s*\d+\s*$/.test(expected[1]!)) { + return { expected: detailPart(expected[1]!), received: detailPart(received[1]!) }; + } + const minus = lines.filter((l) => /^-\s/.test(l) && !/^- Expected\s/.test(l)).map((l) => l.slice(1).trim()); + const plus = lines.filter((l) => /^\+\s/.test(l) && !/^\+ Received\s/.test(l)).map((l) => l.slice(1).trim()); + if (minus.length || plus.length) return { expected: detailPart(minus.join(' ')), received: detailPart(plus.join(' ')) }; + return undefined; +} + +export interface TrialSessionSummary { key: string; runner: string; elapsed_ms: number; budget_ms?: number; end: string } + +/** The compact per-session view a trial record carries (ledger evidence and liveness stay in the artifact). */ +export function trialSessions(rows: ReadonlyArray): { sessions?: TrialSessionSummary[] } { + const sessions = rows.filter((r) => typeof r.key === 'string' && Number.isFinite(r.elapsed_ms)).slice(0, 32).map((r) => ({ + key: r.key!.slice(0, 160), runner: String(r.runner ?? 'unknown').slice(0, 20), elapsed_ms: r.elapsed_ms!, + ...(Number.isFinite(r.budget_ms) ? { budget_ms: r.budget_ms } : {}), end: r.end.slice(0, 20) })); + return sessions.length ? { sessions } : {}; +} + +/** Cause, cause evidence and detail of one failed trial, for both record builders. */ +export function trialFailureFields(input: FailureCauseFacts & { record?: { judge_scores?: unknown; judge_reasoning?: unknown } }): { + failure_cause: TrialFailureCause; failure_cause_evidence?: string; failure_detail?: TrialFailureDetail; +} { + const { cause, evidence } = failureCauseOf(input); + const detail = failureDetailOf(input.error, input.record); + return { failure_cause: cause, ...(evidence ? { failure_cause_evidence: evidence } : {}), ...(detail ? { failure_detail: detail } : {}) }; +} + export class ContractViolation extends Error { constructor(message: string) { super(`CONTRACT: ${message}`); @@ -404,6 +550,14 @@ export interface TrialOutcomeRecord { recorded_at?: string; /** History series key: a hash of the case's own touchfiles (GLOBAL_TOUCHFILES excluded), stamped by the report job. */ series_identity?: string; + /** Failed trials only: what the runners observed (failureCauseOf); verdicts never read it. */ + failure_cause?: TrialFailureCause; + failure_cause_evidence?: string; + failure_detail?: TrialFailureDetail; + /** Ledger sessions of this trial; absent means unknown, never scored. */ + sessions?: TrialSessionSummary[]; + /** False when the harness captured no billing, so cost_usd 0 means unknown. Absent means known. */ + cost_known?: boolean; } /** First line of free text, stripped of @-mentions and control characters, capped. */ @@ -437,6 +591,34 @@ function trialRecordProblems(r: any): string[] { if (!['shard', 'junit', 'backfill'].includes(r.source)) problems.push('source invalid'); if (r.error !== undefined && (typeof r.error !== 'string' || r.error.length > TRIAL_ERROR_MAX)) problems.push('error invalid'); if (r.series_identity !== undefined && (typeof r.series_identity !== 'string' || !/^[\w.-]{1,64}$/.test(r.series_identity))) problems.push('series_identity invalid'); + problems.push(...optionalFieldProblems(r)); + return problems; +} + +const boundedString = (v: unknown, max: number) => typeof v === 'string' && v.length <= max; +const finiteNonNegative = (v: unknown) => typeof v === 'number' && Number.isFinite(v) && v >= 0; + +/** The optional diagnostic fields, checked only when present. */ +function optionalFieldProblems(r: any): string[] { + const problems: string[] = []; + if (r.failure_cause !== undefined) { + if (r.outcome !== 'failed') problems.push('failure_cause on a non-failed trial'); + else if (!FAILURE_CAUSES.includes(r.failure_cause)) problems.push(`unknown failure_cause ${JSON.stringify(String(r.failure_cause).slice(0, 40))} (written by a newer gstack; update this checkout to read it)`); + } + if (r.failure_cause_evidence !== undefined && !boundedString(r.failure_cause_evidence, FAILURE_CAUSE_EVIDENCE_MAX)) problems.push('failure_cause_evidence invalid'); + const d = r.failure_detail; + if (d !== undefined) { + const assertion = d && typeof d === 'object' && !Array.isArray(d) && Object.keys(d).length === 2 + && boundedString(d.expected, FAILURE_DETAIL_PART_MAX) && boundedString(d.received, FAILURE_DETAIL_PART_MAX); + const judge = d && typeof d === 'object' && Object.keys(d).length === 1 && Array.isArray(d.judge) && d.judge.length >= 1 && d.judge.length <= 8 + && d.judge.every((j: any) => j && boundedString(j.dimension, FAILURE_DETAIL_PART_MAX) && Number.isFinite(j.mean) && Number.isFinite(j.threshold) + && Number.isInteger(j.samples) && j.samples >= 1 && j.samples <= 20 && (j.rationale === undefined || boundedString(j.rationale, FAILURE_DETAIL_PART_MAX))); + if (!assertion && !judge) problems.push('failure_detail invalid'); + } + if (r.sessions !== undefined && !(Array.isArray(r.sessions) && r.sessions.length <= 32 && r.sessions.every((x: any) => x + && boundedString(x.key, 160) && boundedString(x.runner, 20) && finiteNonNegative(x.elapsed_ms) && boundedString(x.end, 20) + && (x.budget_ms === undefined || finiteNonNegative(x.budget_ms))))) problems.push('sessions invalid'); + if (r.cost_known !== undefined && typeof r.cost_known !== 'boolean') problems.push('cost_known invalid'); return problems; } diff --git a/test/paid-report-census.test.ts b/test/paid-report-census.test.ts index e9ff0bb93..9d0b3dfcb 100644 --- a/test/paid-report-census.test.ts +++ b/test/paid-report-census.test.ts @@ -13,6 +13,7 @@ import * as os from 'node:os'; import * as path from 'node:path'; import { DESELECTION_REASONS, RUNTIME_SKIP_REASON, attributeJUnitCase, formatPanelLine, runPaidReport, type JUnitCase } from '../scripts/test-paid-shards'; import { panelVerdict } from './helpers/eval-store'; +import { trialRecordProblems as trialRecordProblemsV1 } from './fixtures/trial-record-v1-reader'; const FIXTURES = path.join(import.meta.dir, 'fixtures', 'paid-report-census'); @@ -92,6 +93,37 @@ describe('census attribution from the 2026-10-03 artifacts', () => { }); }); +describe('record builders attach ledger sessions and failure causes (plan 0.1/0.2)', () => { + const noOp = (dir: string) => path.join(dir, 'paid-slice-1-a1', 'shards', 'skill-e2e-plan-mode-no-op'); + const failNoOp = (dir: string) => { + const file = path.join(noOp(dir), 'junit.xml'); + fs.writeFileSync(file, fs.readFileSync(file, 'utf8') + .replace('failures="0"', 'failures="1"') + .replace(/(]*?) \/>/, + '$1>')); + fs.writeFileSync(path.join(noOp(dir), 'session-ledger.jsonl'), [ + { key: 'pty:session#1', runner: 'pty', started_at: 't', budget_ms: 180_000, elapsed_ms: 16_800, end: 'completed' }, + { key: 'pty:session#2', runner: 'pty', started_at: 't', budget_ms: 180_000, elapsed_ms: 180_050, end: 'observer_timeout' }, + ].map(row => JSON.stringify(row)).join('\n') + '\nnot json\n'); + }; + + test('a JUnit-only PTY case gets failure_cause and sessions from its ledger; failure_class is unchanged', () => { + const r = report('gate', failNoOp); + const noOpRecord = r.history.find(record => record.case === 'plan-mode-no-op'); + expect(noOpRecord).toMatchObject({ outcome: 'failed', failure_class: 'assertion', failure_cause: 'observer_timeout', + failure_cause_evidence: 'plan-ceo-review: outcome=timeout (no terminal outcome within 180000ms)', + sessions: [{ key: 'pty:session#1', runner: 'pty', elapsed_ms: 16_800, budget_ms: 180_000, end: 'completed' }, + { key: 'pty:session#2', runner: 'pty', elapsed_ms: 180_050, budget_ms: 180_000, end: 'observer_timeout' }] }); + for (const record of r.history) expect(trialRecordProblemsV1(record), record.case).toEqual([]); + const plain = report('gate', dir => { failNoOp(dir); fs.rmSync(path.join(noOp(dir), 'session-ledger.jsonl')); }); + const without = plain.history.find(record => record.case === 'plan-mode-no-op'); + expect(without.failure_class).toBe(noOpRecord.failure_class); + expect(without.sessions).toBeUndefined(); + const collector = (dir: string) => JSON.parse(fs.readFileSync(path.join(dir, 'collector-outcomes.json'), 'utf8')); + expect(collector(plain.dir).verdict).toEqual(collector(r.dir).verdict); + }); +}); + describe('attributeJUnitCase', () => { const tc = (name: string, outcome: JUnitCase['outcome'], line: number | null = 1): JUnitCase => ({ name, classname: 's', outcome, timeMs: 0, ...(line === null ? {} : { line }) }); diff --git a/test/paid-shards.test.ts b/test/paid-shards.test.ts index 2fedf487c..6d94b3554 100644 --- a/test/paid-shards.test.ts +++ b/test/paid-shards.test.ts @@ -686,6 +686,33 @@ describe('isolated trial shards: record, classification and slice exit', () => { } }); + test('classification adds cause, detail and ledger sessions; failure_class and outcome never change', () => { + const sessions = [{ key: 'claude-p:review-sql-injection#1', runner: 'claude-p' as const, started_at: 't', budget_ms: 300_000, elapsed_ms: 300_100, + end: 'session_timeout' as const, evidence: 'armed 300000ms session timeout fired', liveness: { partial: true, events: 9, turns: 2, max_request_silence_ms: 150_000 } }]; + const inputs: Array<[Partial, { records: any[]; contract: string | null }]> = [ + [{}, none], [{}, { records: [], contract: 'handoff missing' }], + [{ status: 'failed' }, { records: [{ passed: false, exit_reason: 'timeout', timeout_at_turn: 9, error: 'x' }], contract: null }], + [{ status: 'failed' }, { records: [{ passed: false, exit_reason: 'success', error: 'expect(received).toBe(expected)\n\nExpected: 3\nReceived: 2\n' }], contract: null }], + [{ status: 'timed-out', executedTests: null, skippedTests: null }, none], + [{ status: 'failed', executedTests: null, skippedTests: null }, none], + ]; + for (const [over, evidence] of inputs) { + const before = classifyTrialShard({ ...base, ...over }, 'review-sql-injection', 1, plan, evidence); + const after = classifyTrialShard({ ...base, ...over }, 'review-sql-injection', 1, plan, { ...evidence, sessions }); + expect([after.outcome, after.failure_class, after.error], JSON.stringify(over)).toEqual([before.outcome, before.failure_class, before.error]); + expect(after.sessions).toEqual([{ key: 'claude-p:review-sql-injection#1', runner: 'claude-p', elapsed_ms: 300_100, budget_ms: 300_000, end: 'session_timeout' }]); + expect(after.failure_cause === undefined).toBe(after.outcome !== 'failed'); + } + const c = (over: Partial, evidence: { records: any[]; contract: string | null; sessions?: any[] }) => + classifyTrialShard({ ...base, ...over }, 'review-sql-injection', 1, plan, evidence); + expect(c({ status: 'failed' }, { records: [{ passed: false, exit_reason: 'success', error: 'expect(received).toBe(expected)\n\nExpected: 3\nReceived: 2\n' }], contract: null })) + .toMatchObject({ failure_class: 'assertion', error: 'expect(received).toBe(expected)', failure_cause: 'assertion', + failure_cause_evidence: 'session completed; check failed', failure_detail: { expected: '3', received: '2' } }); + expect(c({ status: 'failed' }, { records: [{ passed: false, exit_reason: 'timeout', error: 'Error: Claude Code process aborted by user' }], contract: null, sessions })) + .toMatchObject({ failure_class: 'timeout', failure_cause: 'provider_stall' }); + expect(c({}, { records: [], contract: 'handoff missing', sessions })).toMatchObject({ failure_class: 'contract', failure_cause: 'contract', failure_cause_evidence: 'handoff missing' }); + }); + test('slice exit: rule shards stay strict; failed trials never red the runner, missing records do', () => { const trial = (outcome: 'passed' | 'failed' | null) => ({ status: outcome === 'failed' ? 'failed' as const : 'passed' as const, trial: { case: 'c', trial: 1, ...plan, outcome, cost_usd: 0, duration_ms: 1, ...(outcome === null ? { harness: 'never started' } : {}) } }); diff --git a/test/trial-cause.test.ts b/test/trial-cause.test.ts new file mode 100644 index 000000000..8b19ddab2 --- /dev/null +++ b/test/trial-cause.test.ts @@ -0,0 +1,273 @@ +/** + * failure_cause (plan 0.2): the pure failureCauseOf() precedence table, the + * session observer's liveness and structured refusal/API-error facts, and + * replays of stored census evidence. failure_class is never an output here: + * verdicts keep reading it unchanged. + */ +import { describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { FAILURE_CAUSES, failureCauseOf, failureDetailOf, type FailureCauseFacts, type SessionCauseFacts } from './helpers/eval-store'; +import { SessionObserver, readSessionLedger, type SessionLedgerRow } from './helpers/session-ledger'; +import { STALL_WINDOW_MS } from './helpers/eval-budgets'; +import { runPlanSkillObservation } from './helpers/claude-pty-runner'; +import { createFakePtyDriver } from './helpers/pty/fake-session'; +import { runAgentSdkTest, __resetSemaphoreForTests, type QueryProvider } from './helpers/agent-sdk-runner'; +import { runRecordedCodexEval } from './helpers/codex-eval'; + +const FIXTURES = path.join(import.meta.dir, 'fixtures', 'trial-cause'); +const thinking = JSON.parse(fs.readFileSync(path.join(FIXTURES, 'thinking-captures-37198445662.json'), 'utf8')); +const replays = JSON.parse(fs.readFileSync(path.join(FIXTURES, 'replays.json'), 'utf8')); + +type Timed = [number, any]; +/** Decode one compact capture line: ` s [:]`, `a[*] :,...`, `u ,...`, `y `, `r`. */ +function decode(line: string): Timed { + const [ms, code, arg = ''] = line.split(' '); + const at = Number(ms); + if (code === 's') { + const [type, delta] = arg.split(':'); + return [at, { type: 'stream_event', event: { type, ...(delta ? { delta: { type: delta } } : {}) } }]; + } + if (code === 'a' || code === 'a*') { + const content = arg ? arg.split(',').map(t => { const [id, name] = t.split(':'); return { type: 'tool_use', id, name }; }) : []; + return [at, { type: 'assistant', ...(code === 'a*' ? { parent_tool_use_id: 'parent' } : {}), message: { content } }]; + } + if (code === 'u') return [at, { type: 'user', message: { content: arg ? arg.split(',').map(id => ({ type: 'tool_result', tool_use_id: id })) : [] } }]; + if (code === 'y') return [at, { type: 'system', subtype: arg }]; + return [at, { type: 'result', subtype: 'success' }]; +} + +function observe(events: Timed[], endAt: number, hooks: { permissionAt?: number } = {}) { + const observer = new SessionObserver(0); + for (const [at, event] of events) { + if (hooks.permissionAt !== undefined && at > hooks.permissionAt) { observer.permission(1); hooks.permissionAt = undefined; } + observer.observe(event, at); + } + if (hooks.permissionAt !== undefined) observer.permission(1); + return { observer, liveness: observer.summary(endAt) }; +} + +const timedOutSession = (liveness: SessionCauseFacts['liveness']): SessionCauseFacts => ({ key: 'claude-p:x#1', end: 'session_timeout', elapsed_ms: 600_000, budget_ms: 600_000, liveness }); +const causeOf = (facts: Partial) => failureCauseOf({ failure_class: 'assertion', ...facts }); + +describe('failureCauseOf precedence', () => { + const stalled = { partial: true, max_request_silence_ms: STALL_WINDOW_MS + 10_000, silence_started_ms: 40_000, silence_after: 'thinking_delta' }; + const rows: Array<[string, Partial]> = [ + ['contract', { failure_class: 'contract', error: 'CONTRACT: no write', sessions: [{ end: 'api_error', evidence: 'API Error: 500' }] }], + ['pre_turn_infra', { failure_class: 'infra', exit_reason: 'timeout_startup', sessions: [{ end: 'refusal' }] }], + ['api_error', { sessions: [{ end: 'refusal', evidence: 'refusal at turn 2' }, { end: 'api_error', evidence: 'API Error: Connection lost mid-response.' }] }], + ['refusal', { sessions: [timedOutSession(stalled), { end: 'refusal', evidence: 'refusal at turn 3 (model_refusal_no_fallback) — reasoning_extraction' }] }], + ['provider_stall', { failure_class: 'timeout', exit_reason: 'timeout', sessions: [timedOutSession(stalled)] }], + ['session_timeout', { failure_class: 'timeout', exit_reason: 'timeout', sessions: [timedOutSession({ partial: false, max_request_silence_ms: 0 }), { end: 'observer_timeout' }] }], + ['observer_timeout', { sessions: [{ end: 'observer_timeout' }] }], + ['assertion', { exit_reason: 'success', error: 'expect(received).toBe(expected)' }], + ['unknown', { failure_class: 'timeout', error: 'shard wall reached' }], + ]; + + test('one row per cause, in precedence order, and nothing else', () => { + expect(rows.map(([cause]) => cause)).toEqual([...FAILURE_CAUSES]); + expect(FAILURE_CAUSES).not.toContain('detector'); + }); + + test.each(rows)('%s', (cause, facts) => { + expect(causeOf(facts).cause).toBe(cause as any); + }); + + test('evidence is one bounded line naming what was observed', () => { + expect(causeOf(rows[4]![1]).evidence).toBe('no stream event for 130s from 40s (last: thinking_delta); model request in flight, no tool outstanding'); + expect(causeOf({ exit_reason: 'success' }).evidence).toBe('session completed; check failed'); + const long = causeOf({ failure_class: 'contract', error: `CONTRACT: ${'x'.repeat(1000)}\nsecond line @team` }); + expect(long.evidence!.length).toBeLessThanOrEqual(300); + expect(long.evidence).not.toContain('second line'); + }); + + test('a stall needs partial messages and a session that did not complete', () => { + const quiet = { partial: false, max_request_silence_ms: 0 }; + expect(causeOf({ failure_class: 'timeout', sessions: [timedOutSession(quiet)] }).cause).toBe('session_timeout'); + const recovered = { key: 'k', end: 'completed', liveness: { partial: true, max_request_silence_ms: STALL_WINDOW_MS * 2 } }; + expect(causeOf({ sessions: [recovered] }).cause).toBe('assertion'); + }); +}); + +describe('liveness on real thinking captures (Claude Code 2.1.284, census 37198445662)', () => { + test('the seven partial-message sessions never reach the stall window', () => { + expect(thinking.sessions).toHaveLength(7); + for (const session of thinking.sessions) { + const events: Timed[] = session.events.map(decode); + const { liveness } = observe(events, events.at(-1)![0]); + expect(liveness.partial).toBe(true); + expect(liveness.max_request_silence_ms, session.case).toBeLessThan(STALL_WINDOW_MS); + expect(causeOf({ failure_class: 'timeout', sessions: [timedOutSession(liveness)] }).cause).toBe('session_timeout'); + } + }); + + const base: Timed[] = thinking.sessions.find((s: any) => s.case === '/review army consensus').events.map(decode); + const SILENCE = STALL_WINDOW_MS + 10_000; + const cutAfter = (pick: (event: any, i: number) => boolean) => { + const at = base.findIndex(([, e], i) => i > 20 && pick(e, i)); + expect(at).toBeGreaterThan(0); + return base.slice(0, at + 1); + }; + + test('synthetic positive: a 130 s silence inside a model message is a provider stall', () => { + const events = cutAfter(e => e.type === 'stream_event' && e.event.delta?.type === 'thinking_delta'); + const { liveness } = observe(events, events.at(-1)![0] + SILENCE); + expect(liveness.max_request_silence_ms).toBe(SILENCE); + expect(liveness.silence_after).toBe('thinking_delta'); + expect(causeOf({ failure_class: 'timeout', exit_reason: 'timeout', sessions: [timedOutSession(liveness)] }).cause).toBe('provider_stall'); + }); + + test('controls: hung tool, hook, permission prompt, subagent and long thinking are not stalls', () => { + const toolOpen = cutAfter(e => e.type === 'assistant' && e.message.content.length > 0); + const hook: Timed[] = [...cutAfter(e => e.type === 'user'), [0, { type: 'system', subtype: 'hook_started' }]]; + hook[hook.length - 1]![0] = hook[hook.length - 2]![0]; + const subagent: Timed[] = [[0, { type: 'stream_event', event: { type: 'message_start' } }], + [1000, { type: 'assistant', message: { content: [{ type: 'tool_use', id: 'agent', name: 'Agent' }] } }], + [1500, { type: 'stream_event', event: { type: 'message_stop' } }], + [2000, { type: 'assistant', parent_tool_use_id: 'agent', message: { content: [{ type: 'tool_use', id: 'child', name: 'Bash' }] } }]]; + const thinkingLong: Timed[] = [[0, { type: 'stream_event', event: { type: 'message_start' } }], + ...Array.from({ length: 40 }, (_, i): Timed => [(i + 1) * 9_000, { type: 'stream_event', event: { type: 'content_block_delta', delta: { type: 'thinking_delta' } } }])]; + const permission = cutAfter(e => e.type === 'user'); + for (const [name, events, hooks] of [['hung tool', toolOpen, {}], ['hook', hook, {}], ['subagent', subagent, {}], + ['long thinking', thinkingLong, {}], ['permission', permission, { permissionAt: permission.at(-1)![0] }]] as const) { + // Long thinking runs 360 s of deltas 9 s apart and ends at the next one; the others go silent for 130 s. + const { liveness } = observe(events as Timed[], (events as Timed[]).at(-1)![0] + (name === 'long thinking' ? 9_000 : SILENCE), { ...hooks }); + expect(liveness.max_request_silence_ms, name).toBeLessThan(STALL_WINDOW_MS); + expect(causeOf({ failure_class: 'timeout', sessions: [timedOutSession(liveness)] }).cause, name).toBe('session_timeout'); + } + }); +}); + +describe('replays of stored census evidence', () => { + const byCase = (run: string, c: string, trial = 1) => replays.cases.find((r: any) => r.run === run && r.case === c && r.trial === trial); + + test.each([1, 2])('37195203538 plan-ceo-review-format-approach t%d is a refusal from the structured stream', trial => { + const r = byCase('37195203538', 'plan-ceo-review-format-approach', trial); + const { observer } = observe(r.events, r.events.at(-1)[0]); + const verdict = observer.verdict()!; + expect(verdict.end).toBe('refusal'); + expect(verdict.evidence).toMatch(/^refusal at turn \d+ \(model_refusal_no_fallback\) — reasoning_extraction$/); + const cause = causeOf({ exit_reason: r.exit_reason, error: r.error, sessions: [{ key: 'k', end: verdict.end, evidence: verdict.evidence }] }); + expect(cause.cause).toBe('refusal'); + }); + + test('a tool result quoting refusal text is not a refusal', () => { + const observer = new SessionObserver(0); + observer.observe({ type: 'assistant', message: { content: [{ type: 'tool_use', id: 't', name: 'Read' }] } }, 1); + observer.observe({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 't', + content: "API Error: Fable 5.1's safeguards flagged this message. Claude Code can't respond to this message." }] } }, 2); + observer.observe({ type: 'result', subtype: 'success', is_error: false, result: 'Quoted the refusal text from docs.' }, 3); + expect(observer.verdict()).toBeNull(); + }); + + test('37186854666 shared-libs-review-revalidation ran without partial messages, so it is a session timeout', () => { + const r = byCase('37186854666', 'shared-libs-review-revalidation'); + const { liveness } = observe(r.events, r.duration_ms); + expect(liveness.partial).toBe(false); + expect(liveness.max_request_silence_ms).toBe(0); + expect(causeOf({ failure_class: r.failure_class, exit_reason: r.exit_reason, error: r.error }).cause).toBe('session_timeout'); + expect(causeOf({ failure_class: r.failure_class, exit_reason: r.exit_reason, error: r.error, + sessions: [{ key: 'agent-sdk:x#1', end: 'session_timeout', elapsed_ms: r.duration_ms, budget_ms: 300_000, liveness }] }).cause).toBe('session_timeout'); + }); + + test.each([['37174266054', 'plan-eng-review-artifact'], ['37176837432', 'auto-decide-preserved']])('%s %s is an observer timeout', (run, c) => { + const r = byCase(run, c); + expect(r.failure_class).toBe('assertion'); + expect(causeOf({ failure_class: r.failure_class, exit_reason: r.exit_reason, error: r.error }).cause).toBe('observer_timeout'); + }); +}); + +async function withEvalDir(run: (dir: string) => Promise): Promise<{ value: T; rows: SessionLedgerRow[] }> { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'trial-cause-ledger-')); + const saved = { dir: process.env.GSTACK_EVAL_DIR, run: process.env.EVALS_RUN_ID, id: process.env.GSTACK_EVAL_CASE_ID }; + process.env.GSTACK_EVAL_DIR = dir; + delete process.env.EVALS_RUN_ID; delete process.env.GSTACK_EVAL_CASE_ID; + try { + const value = await run(dir); + return { value, rows: readSessionLedger(dir) }; + } finally { + for (const [key, name] of [['dir', 'GSTACK_EVAL_DIR'], ['run', 'EVALS_RUN_ID'], ['id', 'GSTACK_EVAL_CASE_ID']] as const) { + if (saved[key] === undefined) delete process.env[name]; else process.env[name] = saved[key]; + } + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +describe('one fixture per runner writes the ledger the classifier reads', () => { + test('PTY: the captured API Error idle turn records api_error with its panel line', async () => { + const captures = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures', 'pty-idle-turn-end.json'), 'utf8')); + const configDir = fs.mkdtempSync(path.join(os.tmpdir(), 'trial-cause-pty-')); + try { + const { value: obs, rows } = await withEvalDir(async () => { + const fake = createFakePtyDriver({ frames: [{ screen: 'Claude Code\n> ' }, { onInput: '/plan-eng-review\r', screen: captures.apiErrorIdle.visible }], configDir }); + return runPlanSkillObservation({ skillName: 'plan-eng-review', inPlanMode: false, model: 'fake-model', driver: fake.driver, + extraArgs: ['--disallowedTools', 'AskUserQuestion'], requireProseEvidence: true, timeoutMs: 300_000 }); + }); + expect(obs.outcome).toBe('timeout'); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ runner: 'pty', budget_ms: 300_000, end: 'api_error', evidence: '●API Error: Connection lost mid-response. The response above may be incomplete.' }); + expect(causeOf({ error: obs.summary, sessions: rows }).cause).toBe('api_error'); + } finally { fs.rmSync(configDir, { recursive: true, force: true }); } + }); + + test('Agent SDK: an abort at the armed budget is a session timeout; partial-message silence is a stall', async () => { + __resetSemaphoreForTests(4); + const hangingQuery = (events: any[]): QueryProvider => ((args: any) => { + const signal: AbortSignal = args.options.abortController.signal; + const iterator = (async function* () { + for (const event of events) yield event; + await new Promise((_, reject) => signal.addEventListener('abort', () => reject(new Error('Claude Code process aborted by user')), { once: true })); + })(); + setTimeout(() => args.options.abortController.abort(), 400); + return Object.assign(iterator, { close: () => {} }); + }) as unknown as QueryProvider; + const run = (events: any[]) => withEvalDir(() => runAgentSdkTest({ systemPrompt: '', userPrompt: 'x', workingDirectory: os.tmpdir(), maxRetries: 0, + testName: 'sdk-fixture', sessionBudgetMs: 400, streamLiveness: true, queryProvider: hangingQuery(events) }).catch(error => error)); + const quiet = await run([{ type: 'system', subtype: 'init' }]); + expect(quiet.rows).toEqual([expect.objectContaining({ key: 'agent-sdk:sdk-fixture#1', runner: 'agent-sdk', budget_ms: 400, end: 'session_timeout', + liveness: expect.objectContaining({ partial: false }) })]); + expect(causeOf({ failure_class: 'timeout', exit_reason: 'timeout', sessions: quiet.rows }).cause).toBe('session_timeout'); + const streaming = await run([{ type: 'system', subtype: 'init' }, { type: 'stream_event', event: { type: 'message_start' } }, + { type: 'stream_event', event: { type: 'content_block_delta', delta: { type: 'thinking_delta' } } }]); + expect(streaming.rows[0]!.liveness).toMatchObject({ partial: true, silence_after: 'thinking_delta' }); + expect(failureCauseOf({ failure_class: 'timeout', sessions: streaming.rows }, 300).cause).toBe('provider_stall'); + expect(failureCauseOf({ failure_class: 'timeout', sessions: streaming.rows }).cause).toBe('session_timeout'); + }); + + test('Codex: an expired budget is a session timeout', async () => { + const { rows } = await withEvalDir(() => runRecordedCodexEval({ name: 'codex-fixture', suite: 's', budgetMs: 50, drainGraceMs: 10, + run: signal => new Promise((_, reject) => signal.addEventListener('abort', () => reject(signal.reason), { once: true })), + validate: () => {}, record: () => {} }).catch(() => undefined)); + expect(rows).toEqual([expect.objectContaining({ key: 'codex:codex-fixture#1', runner: 'codex', budget_ms: 50, end: 'session_timeout' })]); + expect(causeOf({ failure_class: 'assertion', exit_reason: 'timeout', sessions: rows }).cause).toBe('session_timeout'); + }); +}); + +describe('failureDetailOf (Bun 1.4.0 matcher shapes)', () => { + test.each([ + ['expect(received).toBe(expected)\n\nExpected: "b"\nReceived: "a"\n', { expected: '"b"', received: '"a"' }], + ['expect(received).toContain(expected)\n\nExpected to contain: "xyz"\nReceived: "hello world"\n', { expected: '"xyz"', received: '"hello world"' }], + ['expect(received).not.toMatch(expected)\n\nExpected substring or pattern: not /taste/\nReceived: "taste call"\n', { expected: 'not /taste/', received: '"taste call"' }], + ['expect(received).toEqual(expected)\n\n {\n- "a": 2,\n+ "a": 1,\n }\n\n- Expected - 1\n+ Received + 1\n', { expected: '"a": 2,', received: '"a": 1,' }], + ])('%#', (message, detail) => { + expect(failureDetailOf(message)).toEqual(detail); + }); + + test('the live message shape on this Bun matches the fixtures', () => { + let message = ''; + try { expect('a').toBe('b'); } catch (error) { message = (error as Error).message; } + expect(failureDetailOf(message)).toEqual({ expected: '"b"', received: '"a"' }); + }); + + test('a judge red names each failing dimension, its mean, sample count and a rationale', () => { + const detail = failureDetailOf('expect(received).toBeGreaterThanOrEqual(expected)\n\nExpected: >= 4\nReceived: 3.6666666666666665\n', { + judge_scores: { clarity: 3.6666666666666665, completeness: 4.333333333333333, actionability: 3.6666666666666665 }, + judge_reasoning: '[sample 1] Thorough. Clarity suffers from extreme density. Two actionability concerns: the budget is infeasible.\n[sample 2] Fine.\n[sample 3] Ok.', + }); + expect(detail).toEqual({ judge: [ + { dimension: 'clarity', mean: 3.67, threshold: 4, samples: 3, rationale: 'Clarity suffers from extreme density.' }, + { dimension: 'actionability', mean: 3.67, threshold: 4, samples: 3, rationale: 'Two actionability concerns: the budget is infeasible.' }, + ] }); + }); +});