fix(evals): a rejected API key is INFRA, never 131 assertion reds and reruns

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

test/credential-rejection.test.ts

@@ -0,0 +1,191 @@
+/**
+ * Credential rejections are INFRA (Captain, 2026-10-23). Train gate run
+ * 39696775546 ran with a rejected ANTHROPIC_API_KEY: every case failed on a
+ * 401 before the model ran, and the report called them 131 assertion reds and
+ * launched 131 rule-16 reruns with the same key. A rejected key fails every
+ * case the same way, so these failures are INFRA: never rerun, and counted as
+ * INFRA in the verdict. The captures below are trimmed from that run
+ * (test/fixtures/credential-rejection-39696775546.json). Each runner's own
+ * evidence path is replayed: the claude -p stream observer, the PTY screen,
+ * the judge's API error and the Agent SDK's error result. The negative controls
+ * are real assertion failures that mention a key; they keep their class and
+ * are still rerun.
+ */
+import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import { spawnSync } from 'node:child_process';
+import { credentialRejection } from './helpers/eval-store';
+import { SessionObserver } from './helpers/session-ledger';
+import { safePanelEnd } from './helpers/pty/session';
+import { classifyTrialShard } from '../scripts/test-paid-shards';
+
+const ROOT = path.resolve(import.meta.dir, '..');
+const CAPTURE = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/credential-rejection-39696775546.json'), 'utf8'));
+const REJECTED = 'Invalid API key · Fix external API key';
+/** Real assertion failures that mention a key, an API key or an auth word, with no provider rejection. */
+const KEY_ASSERTIONS = [
+  'expect(received).toContain(expected)\n\nExpected to contain: "Invalid API key"\nReceived: "ok"',
+  "Expected config to contain key \"api_key\"; the Invalid API key handling section is missing",
+  'README should explain how to rotate the ANTHROPIC_API_KEY',
+  'authentication rejected by the fixture server was not reported to the user',
+  'expect(received).toBe(expected)\n\nExpected: "success"\nReceived: "exit_code_1"',
+];
+
+/** The ledger row the claude -p runner writes from its observer (session-runner.ts: structured end, else error). */
+function claudePLedgerRow(events: unknown[]): Record<string, unknown> {
+  const observer = new SessionObserver(0);
+  events.forEach((event, i) => observer.observe(event, i * 100));
+  const verdict = observer.verdict();
+  return { key: 'claude-p:ship-docsync-store#1', test_name: 'ship-docsync-store', runner: 'claude-p', started_at: '2026-10-22T00:00:33.527Z',
+    budget_ms: 584957, elapsed_ms: 1540, end: verdict?.end ?? 'error', ...(verdict ? { evidence: verdict.evidence } : {}), billed: true };
+}
+/** The ledger row the PTY runner writes: its observer timed out, unless the screen shows an end panel. */
+function ptyLedgerRow(screen: string): Record<string, unknown> {
+  return { key: 'pty:session#1', runner: 'pty', started_at: '2026-10-22T00:00:35.294Z', budget_ms: 420000, elapsed_ms: 420002,
+    end: 'observer_timeout', ...safePanelEnd({ visibleText: () => screen }), billed: false };
+}
+
+describe('each runner records the rejection it saw (run 39696775546)', () => {
+  test('claude -p: the CLI\'s synthetic 401 message ends the session as an API error', () => {
+    expect(claudePLedgerRow(CAPTURE.claude_p_events)).toMatchObject({ end: 'api_error', evidence: REJECTED });
+  });
+
+  test('PTY: a session parked at the plan-mode footer after the rejection still records it', () => {
+    expect(ptyLedgerRow(CAPTURE.pty_screen)).toMatchObject({ end: 'api_error', evidence: expect.stringContaining(REJECTED) });
+    expect(ptyLedgerRow('❯ /plan-ceo-review\n✻Baked for 3s · done 12:00 AM\n❯ \n')).toMatchObject({ end: 'observer_timeout' });
+  });
+
+  test('every runner\'s evidence reads as a credential rejection', () => {
+    expect(credentialRejection({ error: CAPTURE.judge_error })).toContain('"type":"authentication_error"');
+    expect(credentialRejection({ error: CAPTURE.agent_sdk_error })).toContain(REJECTED);
+    expect(credentialRejection({ error: CAPTURE.pty_junit_message + CAPTURE.pty_screen })).toContain(REJECTED);
+    expect(credentialRejection({ error: 'x', sessions: [{ end: 'api_error', evidence: REJECTED }] })).toBe(REJECTED);
+    expect(credentialRejection({ error: '401 {"error":{"code":"invalid_api_key","message":"Incorrect API key provided"}}' })).toContain('invalid_api_key');
+  });
+});
+
+describe('negative controls: a failure that only mentions a key keeps its class', () => {
+  test('assertion text alone is never a rejection', () => {
+    for (const error of KEY_ASSERTIONS) expect(credentialRejection({ error }), error).toBeUndefined();
+  });
+
+  test('a session must have ended as an API error, and a non-credential API error is not one', () => {
+    expect(credentialRejection({ sessions: [{ end: 'completed', evidence: REJECTED }] })).toBeUndefined();
+    expect(credentialRejection({ sessions: [{ end: 'api_error', evidence: 'API Error: 529 {"type":"error","error":{"type":"overloaded_error"}}' }] })).toBeUndefined();
+  });
+});
+
+describe('trial panels: classifyTrialShard', () => {
+  const plan = { kind: 'behavior' as const, panel: { n: 3, k: 2 }, quarantined: false };
+  const shard = { status: 'failed' as const, executedTests: 1, skippedTests: 0, elapsedMs: 1540 };
+  const rejected = { sessions: [claudePLedgerRow(CAPTURE.claude_p_events) as any] };
+
+  test('a rejected session makes the trial INFRA, and its error names the rejection', () => {
+    expect(classifyTrialShard(shard, 'ship-docsync-store', 1, plan, { records: [CAPTURE.claude_p_record], contract: null, ...rejected }))
+      .toMatchObject({ outcome: 'failed', failure_class: 'infra', error: `credentials rejected: ${REJECTED}`, failure_cause: 'pre_turn_infra' });
+    expect(classifyTrialShard(shard, 'c', 1, plan, { records: [{ passed: false, error: CAPTURE.agent_sdk_error }], contract: null }))
+      .toMatchObject({ failure_class: 'infra' });
+  });
+
+  test('contract evidence still wins, and a key-mentioning assertion stays an assertion', () => {
+    expect(classifyTrialShard(shard, 'c', 1, plan, { records: [CAPTURE.claude_p_record], contract: 'handoff missing', ...rejected }))
+      .toMatchObject({ failure_class: 'contract' });
+    for (const error of KEY_ASSERTIONS) {
+      expect(classifyTrialShard(shard, 'c', 1, plan, { records: [{ passed: false, error }], contract: null, sessions: [] }), error)
+        .toMatchObject({ failure_class: 'assertion' });
+    }
+  });
+});
+
+// ── The real --report and --rerun-plan CLI paths on a trimmed two-shard capture ──
+const STORE = 'test/skill-e2e-ship-docsync.test.ts#ship-docsync-store';
+const PTY = 'test/skill-e2e-plan-ceo-plan-mode.test.ts';
+const CASES = ['ship-docsync-store', 'plan-ceo-review-plan-mode'];
+const xml = (s: string) => s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;').replace(/\n/g, '&#10;').replace(/\r/g, '&#13;');
+const junit = (file: string, suite: string, name: string, message: string) => `<?xml version="1.0" encoding="UTF-8"?>
+<testsuites name="bun test" tests="1" assertions="0" failures="1" skipped="0" time="1.6">
+  <testsuite name="${file}" file="${file}" tests="1" assertions="0" failures="1" skipped="0" time="1.6">
+    <testsuite name="${suite}" file="${file}" line="1" tests="1" assertions="0" failures="1" skipped="0" time="1.6">
+      <testcase name="${name}" classname="${suite}" time="1.6" file="${file}" line="1" assertions="0">
+        <failure type="Error" message="${xml(message)}" />
+      </testcase>
+    </testsuite>
+  </testsuite>
+</testsuites>
+`;
+interface Shard { key: string; slug: string; junit: string; ledger: Record<string, unknown>; record?: Record<string, unknown>; budget?: Record<string, unknown> }
+const storeShard = (message: string, ledger: Record<string, unknown>): Shard => ({ key: STORE, slug: 'skill-e2e-ship-docsync--ship-docsync-store', ledger,
+  budget: { timeoutMs: 720000, source: 'registered', policyId: 'skill-e2e-ship-docsync-existing-retry-v1' },
+  junit: junit('test/skill-e2e-ship-docsync.test.ts', 'Ship doc-sync lifecycle > Ship doc-sync lifecycle E2E (gate)', 'ship-docsync-store', message),
+  record: { ...CAPTURE.claude_p_record, transcript: CAPTURE.claude_p_events } });
+const ptyShard: Shard = { key: PTY, slug: 'skill-e2e-plan-ceo-plan-mode', ledger: ptyLedgerRow(CAPTURE.pty_screen),
+  junit: junit(PTY, 'plan-ceo-review plan-mode smoke (gate)', 'first terminal outcome is asked (Step 0 fires before any plan write)', CAPTURE.pty_junit_message + CAPTURE.pty_screen) };
+
+let base: string;
+let counter = 0;
+beforeAll(() => { base = fs.mkdtempSync(path.join(os.tmpdir(), 'credential-rejection-')); });
+afterAll(() => { fs.rmSync(base, { recursive: true, force: true }); });
+
+function reportDir(shards: Shard[]): string {
+  const dir = path.join(base, `case-${++counter}`);
+  const e2e = CASES.filter((_, i) => shards.some(s => s.key === [STORE, PTY][i]));
+  const selection = { e2e, judges: [] };
+  fs.mkdirSync(dir, { recursive: true });
+  fs.writeFileSync(path.join(dir, 'manifest.json'), JSON.stringify({ version: 1, tier: 'gate', evalsAll: false, sliceCount: 1, profile: 'pr', selection,
+    selectionReason: 'trimmed replay of run 39696775546',
+    prCoverage: { mode: 'dependents', e2e, judges: [], deferred: [], unknownFiles: [], unknownFileLabels: [], directCases: [], derivedFiles: [], noConsumerFiles: [],
+      deferredPromptFiles: [], missingCoverage: [], needsFullValidation: false, reasons: [] },
+    plan: { sliceBudgetMs: 420000, jobs: 2, estimatedSliceMs: [420000], ciTimeoutMinutes: 50, sliceCiTimeoutMinutes: [50], codexSlices: [] },
+    entries: shards.map(s => ({ file: s.key, slice: 1, status: 'planned', estimatedMs: 120000, ...(s.budget ? { budget: s.budget } : {}) })) }));
+  for (const s of shards) {
+    const shardDir = path.join(dir, 'paid-slice-1-a1', 'shards', s.slug);
+    fs.mkdirSync(shardDir, { recursive: true });
+    fs.writeFileSync(path.join(shardDir, 'junit.xml'), s.junit);
+    fs.writeFileSync(path.join(shardDir, 'session-ledger.jsonl'), JSON.stringify(s.ledger) + '\n');
+    if (s.record) {
+      fs.writeFileSync(path.join(shardDir, '1.91.176-train-gate.train-437b2f2a3e80-514fbfdc9799-5bc2e6-e2e-ship-docsync-2026-10-22-0000.json'),
+        JSON.stringify({ schema_version: 2, version: '1.91.176', tier: 'e2e-ship-docsync', total_tests: 1, executed_tests: 1, passed: 0, failed: 1, total_cost_usd: 0, tests: [s.record] }));
+    }
+  }
+  fs.writeFileSync(path.join(dir, 'paid-slice-1-a1', 'slice-1.json'), JSON.stringify({ version: 1, tier: 'gate', profile: 'pr', selection, sliceIndex: 1, sliceCount: 1, attempt: 1,
+    startedAt: 1792627232000, finishedAt: 1792627653000,
+    outcomes: shards.map(s => ({ files: [s.key], status: 'failed', exitCode: 1, elapsedMs: 1655, executedTests: 1, skippedTests: 0,
+      budget: s.budget ?? { timeoutMs: 1800000, source: 'default', policyId: null } })) }));
+  return dir;
+}
+const cli = (args: string[]) => spawnSync(process.execPath, ['--no-env-file', '--config=/dev/null', path.join(ROOT, 'scripts/test-paid-shards.ts'), '--tier', 'gate', ...args],
+  { cwd: ROOT, encoding: 'utf8', timeout: 60_000,
+    env: { ...process.env, EVALS_TIER: 'gate', GITHUB_RUN_ID: '39696775546', GITHUB_RUN_ATTEMPT: '1', GITHUB_SHA: '', GSTACK_GATE_PR: '', GSTACK_GATE_HEAD_SHA: '' } });
+function planRerun(dir: string) {
+  const out = path.join(dir, 'rerun-plan-a1');
+  const r = cli(['--report', dir, '--rerun-plan', out]);
+  expect(r.status, r.stdout + r.stderr).toBe(0);
+  const file = path.join(out, 'manifest.json');
+  return { out: r.stdout, manifest: fs.existsSync(file) ? JSON.parse(fs.readFileSync(file, 'utf8')) : null };
+}
+
+describe('replay of run 39696775546 through --report and --rerun-plan', () => {
+  test('every case died on the rejected key: no rule-16 rerun, and the verdict counts INFRA, not assertion reds', () => {
+    const dir = reportDir([storeShard(CAPTURE.claude_p_junit_message, claudePLedgerRow(CAPTURE.claude_p_events)), ptyShard]);
+    const plan = planRerun(dir);
+    expect(plan.manifest).toBeNull();
+    for (const id of CASES) expect(plan.out).toContain(`not rerun: ${id} (never rerun: INFRA failure class)`);
+    fs.rmSync(path.join(dir, 'collector-outcomes.json'), { force: true });
+    const r = cli(['--report', dir]);
+    const out = r.stdout + r.stderr;
+    expect(r.status, out).toBe(1);
+    expect(out).toContain('VERDICT RED');
+    expect(out).toContain('INFRA 2');
+    for (const id of CASES) expect(out).toMatch(new RegExp(`✗ ${id}  rule  FAIL  infra — credentials rejected: `));
+    expect(out).not.toMatch(/FAIL {2}assertion/);
+  });
+
+  test('negative control: a real assertion red that mentions a key is still rerun once', () => {
+    const completed = { ...claudePLedgerRow([]), end: 'completed' };
+    const dir = reportDir([{ ...storeShard(KEY_ASSERTIONS[1], completed), record: { ...CAPTURE.claude_p_record, output: 'done', exit_reason: 'success', error: KEY_ASSERTIONS[1] } }]);
+    const plan = planRerun(dir);
+    expect(plan.manifest?.rerun).toEqual({ attempt: 1, cases: ['ship-docsync-store'] });
+  });
+});

test/fixtures/credential-rejection-39696775546.json

@@ -0,0 +1,61 @@
+{
+  "source": "GitHub Actions run 39696775546 (train gate, 2026-10-22): every gated case failed on a rejected ANTHROPIC_API_KEY; trimmed captures",
+  "claude_p_events": [
+    {
+      "type": "system",
+      "subtype": "status",
+      "status": "requesting",
+      "session_id": "fc99f0a9-26d1-4c20-b835-7e2c7bd5a629"
+    },
+    {
+      "type": "assistant",
+      "message": {
+        "id": "c885e094-9dfe-45d9-ba48-1d7955f9ba48",
+        "model": "<synthetic>",
+        "role": "assistant",
+        "stop_reason": "stop_sequence",
+        "type": "message",
+        "content": [
+          {
+            "type": "text",
+            "text": "Invalid API key · Fix external API key"
+          }
+        ]
+      },
+      "parent_tool_use_id": null,
+      "session_id": "fc99f0a9-26d1-4c20-b835-7e2c7bd5a629",
+      "error": "authentication_failed",
+      "is_api_error_message": true
+    },
+    {
+      "type": "result",
+      "subtype": "success",
+      "is_error": true,
+      "api_error_status": 401,
+      "terminal_reason": "api_error",
+      "num_turns": 1,
+      "total_cost_usd": 0,
+      "result": "Invalid API key · Fix external API key",
+      "session_id": "fc99f0a9-26d1-4c20-b835-7e2c7bd5a629"
+    }
+  ],
+  "claude_p_record": {
+    "name": "ship-docsync-store",
+    "suite": "Ship doc-sync lifecycle",
+    "tier": "e2e",
+    "passed": false,
+    "duration_ms": 1540,
+    "cost_usd": 0,
+    "output": "Invalid API key · Fix external API key",
+    "turns_used": 1,
+    "tokens_used": 0,
+    "browse_errors": [],
+    "exit_reason": "exit_code_1",
+    "model": "claude-opus-5-5"
+  },
+  "claude_p_junit_message": "expect(received).toBeGreaterThanOrEqual(expected)\n\nExpected: >= 0\nReceived: -1\n",
+  "pty_junit_message": "plan-ceo-review smoke FAILED: outcome=timeout\nTimeout means the agent neither asked nor completed within the budget — likely hung mid-question or stuck on a permission dialog.\nsummary: no terminal outcome within 420000ms (last LLM judge: state=unknown — claude exited 1 ())\nelapsed: 420002ms\n--- evidence (last 2KB visible) ---\n",
+  "pty_screen": "\r❯ /plan-ceo-review\r\r✢ Ebbing… \r\r\r\n────────────────────────────────────────\r\r\n❯  \r\r\n────────────────────────────────────────\r\r\n⏸planmodeon (shift+tabtocycle)·esctointerrupt·←foragents◐medium·/effort\r\r\n\r·n\r\r\n\r  ⎿  Invalid API key · Fix external API key\r\r✻Baked for 0s · done 12:00 AM\r\r────────────────────────────────────────\r❯  \r────────────────────────────────────────\r\r\n⏸planmodeon (shift+tabtocycle)·←foragents◐medium·/effort\r\r\n\r\r\r\n",
+  "judge_error": "401 {\"type\":\"error\",\"error\":{\"type\":\"authentication_error\",\"message\":\"invalid x-api-key\"},\"request_id\":\"req_011CgGHuinAhnU3wapQEbUe1\"}",
+  "agent_sdk_error": "Error: Claude Code returned an error result: Invalid API key · Fix external API key"
+}

test/helpers/eval-store.ts

@@ -15,6 +15,7 @@ import { spawnSync } from 'child_process';
 import { isManualReviewEntry } from './cookie-workflow-manual-review';
 import type { ManualJudgeReview } from './cookie-workflow-manual-review';
 import type { CodexTokenUsage } from './codex-session-runner';
+import { CREDENTIAL_REJECTION } from './session-ledger';
 
 // v2: EvalTestEntry.harvest gains optional {insertions, deletions, net} and
 // may be explicitly null (arm-benchmark harvest-failure taxonomy). Readers
@@ -215,6 +216,15 @@ export function failureClassOf(entry: Pick<EvalTestEntry, 'failure_class' | 'exi
   return entry.exit_reason === 'timeout' ? 'timeout' : 'assertion';
 }
 
+/** Evidence that a failed trial died on rejected credentials, from a session its runner saw end that way or
+ *  from its failure text. The run's key is shared, so a rerun cannot help: the trial is INFRA, never a verdict.
+ *  A failure that merely mentions a key matches no provider rejection and keeps its own class. */
+export function credentialRejection(facts: { error?: string; sessions?: readonly Pick<SessionCauseFacts, 'end' | 'evidence'>[] }): string | undefined {
+  const texts = [...(facts.sessions ?? []).filter((s) => s.end === 'api_error').map((s) => s.evidence), facts.error];
+  const line = texts.flatMap((text) => text?.split(/\r\n?|\n/) ?? []).find((l) => CREDENTIAL_REJECTION.test(l));
+  return oneLine(line, FAILURE_CAUSE_EVIDENCE_MAX);
+}
+
 /** Machine-recorded cause of a failed trial, beside its policy class. failure_class
  *  (what verdicts read) is never changed by it; whether an assertion was a
  *  detector, harness or product fault is a human diagnosis, never a cause. */

test/helpers/pty/launch.ts

@@ -13,7 +13,7 @@ import { createFilePermissionRecorder } from '../plan-count-file-permission';
 import { createAutoplanArtifactRecorder } from '../autoplan-artifact-recorder';
 import { trustDialogInput } from '../pty-trust-dialog';
 import { resolveClaudeBinary } from './binary';
-import { createPtyScreen, idlePanelEnd, stripAnsi } from './screen';
+import { createPtyScreen, screenPanelEnd, stripAnsi } from './screen';
 import { appendSessionLedger, sessionKey } from '../session-ledger';
 
 export interface ClaudePtyOptions {
@@ -249,7 +249,7 @@ export async function launchClaudePty(
       recorded = true;
       const elapsed = performance.now() - startedMono;
       appendSessionLedger({ key: sessionKey('pty', undefined), runner: 'pty', started_at: startedAt, budget_ms: timeoutMs,
-        elapsed_ms: elapsed, end: elapsed >= timeoutMs ? 'session_timeout' : 'completed', ...idlePanelEnd(handle.visibleText()), billed: false });
+        elapsed_ms: elapsed, end: elapsed >= timeoutMs ? 'session_timeout' : 'completed', ...screenPanelEnd(handle.visibleText()), billed: false });
     }
     return close();
   } });

test/helpers/pty/screen.ts

@@ -7,6 +7,7 @@ import * as fs from 'node:fs';
 import * as path from 'node:path';
 import { currentFilePermissionTarget, currentBashPermissionCard, currentReadPermissionCard, currentWebFetchPermissionCard, hasCurrentBashPermissionHeading, hasCurrentReadPermissionHeading, hasCurrentWebFetchPermissionHeading } from '../plan-skill-questions';
 import { createRequire } from 'node:module';
+import { CREDENTIAL_REJECTION } from '../session-ledger';
 
 /** The existing xterm package ships this headless source beside its browser bundle. */
 let terminalConstructor: Promise<any> | undefined;
@@ -413,7 +414,7 @@ export function isNumberedOptionListVisible(visible: string): boolean {
 }
 
 /** Start of the CLI's public "API Error:" panel. The native AUQ capture reads the same panel. */
-export const API_ERROR_PANEL = /(?:^|\n)[\t │┃]*(?:[⎿●⏺]\s*)?API Error:/i;
+export const API_ERROR_PANEL = /(?:^|\n)[\t │┃]*(?:[⎿●⏺]\s*)?(?:API Error:|Invalid API key\s*·)/i;
 
 /** Session-ledger end from the last idle turn's public error panel: a safeguards panel is a refusal, any other an API error. */
 export function idlePanelEnd(visible: string): { end: 'refusal' | 'api_error'; evidence: string } | null {
@@ -422,6 +423,13 @@ export function idlePanelEnd(visible: string): { end: 'refusal' | 'api_error'; e
   return { end: /safeguards flagged|can't respond to this message/i.test(panel) ? 'refusal' : 'api_error', evidence: panel };
 }
 
+/** The last idle turn's error panel, else the CLI's rejected-credentials line anywhere on screen: after one the
+ *  CLI can sit at a footer the idle check does not know (plan mode, manual mode), and no turn can run. */
+export function screenPanelEnd(visible: string): { end: 'refusal' | 'api_error'; evidence: string } | null {
+  const credential = visible.split(/\r\n?|\n/).reverse().find(line => CREDENTIAL_REJECTION.test(line))?.trim();
+  return idlePanelEnd(visible) ?? (credential ? { end: 'api_error', evidence: credential } : null);
+}
+
 /** The CLI's end-of-turn line, e.g. "✻ Cooked for 24s · done 12:46 AM" (spaces may collapse in the PTY text). */
 const TURN_DONE_RE = /✻[ \t]*\p{L}[\p{L}'’-]*[ \t]*for[ \t]*(?=\d)(?:\d+h[ \t]*)?(?:\d+m[ \t]*)?(?:\d+s)?[ \t]*·[ \t]*done[^\n]*/gu;
 /** An empty input prompt and its known idle footers, nothing else. */

test/helpers/pty/session.ts

@@ -8,7 +8,7 @@
  * former claude-pty-runner.ts. Tests import through claude-pty-runner.ts.
  */
 import { launchClaudePty, type ClaudePtyOptions, type ClaudePtySession } from './launch';
-import { idlePanelEnd } from './screen';
+import { screenPanelEnd } from './screen';
 import { appendSessionLedger, sessionKey, type SessionEnd } from '../session-ledger';
 
 /**
@@ -33,9 +33,9 @@ export const realPtyDriver: PtyDriver = {
   sleep: ms => Bun.sleep(ms),
 };
 
-/** The last idle turn's error panel, if the session can show its text; diagnostics never throw. */
-export function safePanelEnd(session: Pick<ClaudePtySession, 'visibleText'>): ReturnType<typeof idlePanelEnd> {
-  try { return idlePanelEnd(session.visibleText()); } catch { return null; }
+/** The session's end panel (screenPanelEnd), if the session can show its text; diagnostics never throw. */
+export function safePanelEnd(session: Pick<ClaudePtySession, 'visibleText'>): ReturnType<typeof screenPanelEnd> {
+  try { return screenPanelEnd(session.visibleText()); } catch { return null; }
 }
 
 /** One poll step: finish with a result, keep polling, or end the loop as a timeout. */

test/helpers/session-ledger.ts

@@ -19,6 +19,9 @@ export const SESSION_EVIDENCE_MAX = 200;
 export type SessionRunner = 'claude-p' | 'agent-sdk' | 'pty' | 'codex';
 export type SessionEnd = 'completed' | 'session_timeout' | 'observer_timeout' | 'api_error' | 'refusal' | 'aborted' | 'error';
 export const SESSION_ENDS: readonly SessionEnd[] = ['completed', 'session_timeout', 'observer_timeout', 'api_error', 'refusal', 'aborted', 'error'];
+/** A provider rejected the run's credentials: Anthropic's 401 authentication_error body, the CLI's
+ *  authentication_failed code or its "Invalid API key · …" message, or OpenAI's invalid_api_key code. */
+export const CREDENTIAL_REJECTION = /"type"\s*:\s*"authentication_error"|\bauthentication_failed\b|\bInvalid API key\s*·|"code"\s*:\s*"invalid_api_key"/i;
 
 export interface LivenessSummary {
   /** The session streamed partial messages. Without them no stall is ever inferred. */
@@ -189,7 +192,7 @@ export class SessionObserver {
   }
 
   private noteResultText(text: string): void {
-    const line = text.split('\n').map(l => l.trim()).find(l => /^API Error:/i.test(l));
+    const line = text.split('\n').map(l => l.trim()).find(l => /^API Error:/i.test(l) || CREDENTIAL_REJECTION.test(l));
     if (!line) return;
     if (REFUSAL_PHRASE.test(text)) this.phraseRefusal ??= `refusal at turn ${Math.max(1, this.turnIds.size)} (phrase) — ${line}`;
     else this.apiError ??= line;