diff --git a/CHANGELOG.md b/CHANGELOG.md index e74a6d62b..3ede054fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## [1.91.183.0] - 2026-10-23 + +**A rejected API key no longer turns a gate into 131 assertion reds and 131 reruns.** Train gate run 39696775546 ran with a rejected `ANTHROPIC_API_KEY`. Every case failed on a 401 before the model ran, and the report counted them as assertion failures. Rule 16 then reran all 131 cases with the same key. A trial whose provider rejected the run's credentials is now INFRA: the gate never reruns it, the verdict counts it under INFRA, and its failure line reads `credentials rejected: Invalid API key · …`. A real assertion failure that only mentions a key keeps its class and is still rerun. + +### Fixed +- **`test/helpers/eval-store.ts`**: `credentialRejection()` finds the evidence in a trial's API-error sessions or its failure text: Anthropic's 401 `authentication_error` body, the CLI's `authentication_failed` code or `Invalid API key ·` message, or OpenAI's `invalid_api_key` code. Both record builders use it. The JUnit census (`scripts/lib/paid-report.ts`, which feeds the rerun plan) and `classifyTrialShard` (trial panels) mark such a trial `infra`. Contract evidence still wins. +- **Session ledger**: the claude -p and Agent SDK observer (`SessionObserver`) ends a session as `api_error` on the CLI's synthetic `Invalid API key` message, not just on `API Error:` lines. Both PTY ledger writers (`screenPanelEnd` in `test/helpers/pty/screen.ts`) record the rejection line wherever it is on screen, because after one the CLI sits at a plan-mode or manual-mode footer that the idle check doesn't recognize. Before this, `autoplan-guard-pty` recorded `completed` with no evidence. +- **Report headline**: the INFRA count includes rule cases with an INFRA-class failure, not only trial panels. +- **Gate integrity (a gate edit):** a credential rejection was already a red, and it is still a red. Only its class changes, from assertion to INFRA, so rule 16 no longer reruns it. No case's pass or fail changes. No threshold, trial count, case kind, judge model, `EVAL_POLICY` constant or `HARNESS_VERSION` changed; the edit is recorded as non-behavioral in `scripts/harness-version.json`. Known gap: local `--case` diagnosis and the train bisect's file-mode trials still classify by shard status. + +### For contributors +- **`test/credential-rejection.test.ts`** replays trimmed captures from run 39696775546 (`test/fixtures/credential-rejection-39696775546.json`) through each runner's evidence path and the real `--report` and `--rerun-plan` CLI paths. Negative controls cover assertion failures that mention a key, an API key or an auth word, and a non-credential API error. + ## [1.91.182.0] - 2026-10-23 **gstack installs on a project that pins Bun 1.4.4 again.** gbrain master moved its CI to Bun 1.4.4 (gbrain #7374, the Brotli CVE-2026-111497 fix), and gstack's Capy installer then stopped every gbrain session with `RUNTIME_PIN_CONFLICT` ("verified archives: 1.4.2 1.4.3 … pins exactly 1.4.4"), because it installs only Bun archives whose SHA-256 it carries. It now carries 1.4.4, so the same run installs the pinned Bun and ends `gstack: ok`. diff --git a/VERSION b/VERSION index 1f6481848..ac9268334 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.182.0 +1.91.183.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 275434383..c433d5206 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.182.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.183.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/docs/TESTING_INTERNALS.md b/docs/TESTING_INTERNALS.md index aec69f4e7..8528a1609 100644 --- a/docs/TESTING_INTERNALS.md +++ b/docs/TESTING_INTERNALS.md @@ -476,6 +476,14 @@ plans nothing when the red is a harness failure (runner error, slice deadline, unfinalized checkpoint, a different executed case count, a reused result), an INFRA or INCOMPLETE panel, an INFRA-class or unattributed testcase, a case already red twice, or when any other problem would keep the gate red anyway. +A trial whose provider rejected the run's credentials is INFRA-class +(`credentialRejection()` in `test/helpers/eval-store.ts`): a session its runner +saw end on Anthropic's 401 `authentication_error`, the CLI's +`authentication_failed` or `Invalid API key ·` message (claude -p, Agent SDK, +or anywhere on a PTY screen), or OpenAI's `invalid_api_key`, or the same +evidence in its failure text. The run's key is shared, so a rerun cannot help; +the report counts these under INFRA and names them `credentials rejected:`. +Assertion text that only mentions a key keeps its class. `slices-report` reads every attempt's slices (`paid-slice-*`), each attempt's rerun (`paid-rerun-*`, bound to its own `rerun-plan-a` manifest) and @@ -612,7 +620,7 @@ The highest-precedence cause with evidence wins: | `failure_cause` | Recorded when | First step | |---|---|---| | `contract` | `expectContract()` stamped a contract violation | fix the product; a violation fails a behavior panel outright | -| `pre_turn_infra` | the trial failed before the first model turn (`failure_class: 'infra'`) | check the runner and API status; an all-INFRA census is re-dispatched once | +| `pre_turn_infra` | the trial failed before the first model turn (`failure_class: 'infra'`), a rejected credential included | check the runner and API status (for `credentials rejected:`, the key secret); an all-INFRA census is re-dispatched once | | `api_error` | a session ended on an API error (for PTY, the `API Error:` panel on an idle turn) or the runner's exit reason was `error_api` | read the evidence line; a provider outage is a named red | | `refusal` | the structured stop reason was a refusal (phrases only as a last resort; quoted refusal text in a tool result does not count) | check whether the fixture prompt invites the refusal | | `provider_stall` | a session that streamed partial messages saw no event of any kind, thinking deltas included, for `STALL_WINDOW_MS` (120 s) while a model request was in flight and no tool, permission prompt, hook or subagent was outstanding; sessions without partial messages are never stalls | read the session ledger row; still a failed trial under v1 | diff --git a/package.json b/package.json index 86dbd66d0..ff3635537 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.182", + "version": "1.91.183", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/scripts/harness-version.json b/scripts/harness-version.json index 779878132..3ee92c6d1 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -13,10 +13,10 @@ "files": { "scripts/lib/paid-cases.ts": {"blob":"f862cf5241580c7593b9ee62eacf5ef046248d26","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "scripts/lib/paid-plan.ts": {"blob":"e8ce3cb98474dded38c5128a631d283b0ca35b63","since":6,"nonBehavioral":"F1 rule-16 flaky rerun: report aggregation, rerun planning and the gate slice exit code; no case's execution, trial outcome, trial record or panel verdict changes, so no pass-rate series resets"}, - "scripts/lib/paid-report.ts": {"blob":"df19605183b8c6bfabd01bbe23fbe821d97a405e","since":6,"nonBehavioral":"F1 rule-16 flaky rerun: report aggregation, rerun planning and the gate slice exit code; no case's execution, trial outcome, trial record or panel verdict changes, so no pass-rate series resets"}, + "scripts/lib/paid-report.ts": {"blob":"19a87cb02bb3cd0f57da7719a5c30b7a4df81b2b","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6}, "scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":6}, - "scripts/test-paid-shards.ts": {"blob":"b3b4786e406ca6793df2eb24fafac27cfb99a48e","since":6,"nonBehavioral":"F1 rule-16 flaky rerun: report aggregation, rerun planning and the gate slice exit code; no case's execution, trial outcome, trial record or panel verdict changes, so no pass-rate series resets"}, + "scripts/test-paid-shards.ts": {"blob":"5439d0a8f6e1569fa7c924af20d67c30c81d5d30","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, "scripts/test-pr-profile.ts": {"blob":"d334195b0d0393622e2c380dc3e2feebe099b6e6","since":6,"nonBehavioral":"classify the new evals-main-backstop.yml workflow as free-lane only: it dispatches evals.yml on main and runs inside no case, so no case's selection on a PR, input or verdict changes"}, "scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":6}, "test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":6}, @@ -27,7 +27,7 @@ "test/helpers/e2e-gate.ts": {"blob":"efb7e197dd5f66c3ad0e34074b14e640754f7a45","since":6}, "test/helpers/e2e-helpers.ts": {"blob":"6ee6937e6c3ca081c77131b5a755278c7b714834","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "test/helpers/eval-budgets.ts": {"blob":"667d0e09b20fe5d7366f57b77b5fb9b04baa37ef","since":6}, - "test/helpers/eval-store.ts": {"blob":"ed8721f13d39f4ef62e0a1d814efbe9352da781a","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, + "test/helpers/eval-store.ts": {"blob":"a50ea2c28f508ad9f35ea9353dd2f502ee0dcfc2","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, "test/helpers/hermetic-env.ts": {"blob":"cc072895461dc52ea317d1beff72121342420d30","since":6}, "test/helpers/llm-judge.ts": {"blob":"d0bc6bbbe9f478603d11a1ece95e66c812ad34e6","since":6}, "test/helpers/paid-test-set.ts": {"blob":"ff9bfe43c43ec2bdd11114f7b8ec7c5736116f75","since":6}, @@ -38,15 +38,15 @@ "test/helpers/pty/classify.ts": {"blob":"3f2daa12ff6484f96b8d28d910a50231c4bb3ae8","since":6}, "test/helpers/pty/fake-session.ts": {"blob":"554e064ef259b8444280eddfe9b0db93c66a8202","since":6}, "test/helpers/pty/judge.ts": {"blob":"81c5bb4c48ab928aca2f186a1ade9df7169d7fbe","since":6}, - "test/helpers/pty/launch.ts": {"blob":"884d1008b3349e8b16d6c38f9f0aa7ae514f0f37","since":6}, + "test/helpers/pty/launch.ts": {"blob":"e4413de380b799659170e7da248d51afda3b0e3f","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, "test/helpers/pty/plan-native.ts": {"blob":"ddba9cb68c7c0da935cff7c1cea258dee2dff740","since":6}, "test/helpers/pty/runners/counting.ts": {"blob":"8c5c05e54abf961259d34d9c3d77db67138d7794","since":6}, "test/helpers/pty/runners/floor.ts": {"blob":"57fa44c31300754b00473276cf7b972d4e9e47d8","since":6}, "test/helpers/pty/runners/observation.ts": {"blob":"1c2b75fbb984708725abaa8939125f7cab969f9f","since":6}, - "test/helpers/pty/screen.ts": {"blob":"ba7a80bb68b1959296ab2a827cfacdaa46f4ba2d","since":6}, - "test/helpers/pty/session.ts": {"blob":"58533cea9d81cc87bb3b499bdfe0db8e3a8709c5","since":6}, + "test/helpers/pty/screen.ts": {"blob":"f3743152fc9bb7a8e8c70002f75bf45914f260d5","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, + "test/helpers/pty/session.ts": {"blob":"54fe04b61d953e4e55e2fc251532c04a49fab66f","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, "test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":6}, - "test/helpers/session-ledger.ts": {"blob":"425323284a368f5e77937e03884db4dea4e6134e","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, + "test/helpers/session-ledger.ts": {"blob":"a10bf28bf100f135b93c73818670dd147253cdd3","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, "test/helpers/session-runner.ts": {"blob":"cc65a66adafa9c053e3da3c6f17b5ac86c0eade7","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "test/helpers/skill-fixture.ts": {"blob":"ffece830f35b0f1562005105e9296dc4d8ca50c2","since":6,"nonBehavioral":"skill-fixture.ts drops 'When to invoke this skill' from the review, review-army and retro section lists because #3129 stops generating that heading; extraction would throw otherwise. The fixture bytes follow the regenerated SKILL.md, whose owned prompt bytes already start those cases' new series."}, "test/helpers/test-selection.ts": {"blob":"51655f89f66be7a5ec487e2abfe4744300ebd5f6","since":6}, diff --git a/scripts/lib/paid-report.ts b/scripts/lib/paid-report.ts index df1960518..19a87cb02 100644 --- a/scripts/lib/paid-report.ts +++ b/scripts/lib/paid-report.ts @@ -7,7 +7,7 @@ import { normalizeRelativePath } from './shard-engine'; import { EVAL_POLICY } from '../../test/helpers/periodic-exclude-data'; import { STALL_WINDOW_MS } from '../../test/helpers/eval-budgets'; import { - isFinalizedEvalResultFile, failureClassOf, panelVerdict, sanitizeTrialError, formatTrialOutcomes, TRIAL_OUTCOME_SCHEMA, TRIAL_OUTCOMES_FILE, + isFinalizedEvalResultFile, credentialRejection, failureClassOf, panelVerdict, sanitizeTrialError, formatTrialOutcomes, TRIAL_OUTCOME_SCHEMA, TRIAL_OUTCOMES_FILE, type EvalCaseKind, type PanelVerdict, type TrialFailureClass, type TrialOutcome, type TrialOutcomeRecord, trialCostKnown, trialFailureFields, trialSessions, type TrialFailureDetail, } from '../../test/helpers/eval-store'; @@ -277,13 +277,16 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact const caseOutcome: TrialOutcome = failed ? 'failed' : cases.some(tc => tc.outcome === 'passed') ? 'passed' : 'skipped'; const mine = records.filter((r: any) => r?.name === id || r?.case_id === id); const failedRecord = mine.find((r: any) => r.passed === false); - const failureClass: TrialFailureClass | undefined = !failed ? undefined + const recorded: TrialFailureClass | undefined = !failed ? undefined : failed.failureType === 'TimeoutError' ? 'timeout' : failedRecord ? failureClassOf(failedRecord) : 'assertion'; - const error = sanitizeTrialError(failedRecord?.error ?? failed?.message); const sessions = caseSessions(ledger, id, byCase.size === 1); + const credentials = failed && recorded !== 'contract' ? credentialRejection({ error: `${failedRecord?.error ?? ''}\n${failed.message ?? ''}`, sessions }) : undefined; + const failureClass = credentials ? 'infra' : recorded; + const rawError = credentials ? `credentials rejected: ${credentials}` : failedRecord?.error ?? failed?.message; + const error = sanitizeTrialError(rawError); // A case's JUnit times sum its concurrent tests (plan-mode-no-op: 346 s in a 176 s shard), so its wall is capped by the shard's. const diagnosis = failed && failureClass ? trialFailureFields({ failure_class: failureClass, exit_reason: failedRecord?.exit_reason, - error: failedRecord?.error ?? failed.message, sessions, record: failedRecord }, STALL_WINDOW_MS) : undefined; + error: rawError, sessions, record: failedRecord }, STALL_WINDOW_MS) : undefined; if (failed) census.failedShards.add(key); if (caseOutcome === 'skipped') { bump(reasons, RUNTIME_SKIP_REASON); @@ -762,7 +765,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo judge: count('judge'), quarantined: { total: panels.filter(p => p.quarantined).length, failingLane: panels.filter(p => p.quarantined && p.failsLane).length }, skipped: hollowWithoutJUnit.length + panels.filter(p => p.status === 'SKIPPED').length + ruleCases.filter(c => c.outcome === 'skipped').length, - infra: primaryTrials.filter(t => t.failure_class === 'infra').length + infra: primaryTrials.filter(t => t.failure_class === 'infra').length + ruleCases.filter(c => c.outcome === 'failed' && c.failureClass === 'infra').length + results.flatMap(r => r.outcomes).filter(o => !o.trial && o.runnerError !== undefined).length, incomplete: panels.filter(p => p.status === 'INCOMPLETE').length, unattributed: census.unattributed.length, diff --git a/scripts/lib/windows-curation.ts b/scripts/lib/windows-curation.ts index a05efcf83..8d153facf 100644 --- a/scripts/lib/windows-curation.ts +++ b/scripts/lib/windows-curation.ts @@ -51,6 +51,8 @@ const WINDOWS_FRAGILE_PATTERNS: Array<{ pattern: RegExp; reason: string }> = [ // when possible; this list is for environment-/runtime-specific tests where // the failure mode is structural rather than detectable via source-file scan. export const KNOWN_WINDOWS_INCOMPATIBLE: Array<{ file: string; reason: string }> = [ + // Like test/paid-rerun.test.ts (excluded by pattern), it replays a gate capture through the paid lane's --report and --rerun-plan CLI. + { file: 'test/credential-rejection.test.ts', reason: 'replays run 39696775546 through the --report and --rerun-plan CLI of the paid lane, which runs only on Linux CI; on windows-latest that CLI exits 1 with EINVAL in open() (windows-free-shard run 39982914852)' }, // Field list 2, W11: the review-voice install is a POSIX launcher contract, exercised end to end on Linux and macOS. { file: 'test/capy-review-voice.test.ts', reason: 'copies the checkout with tar -C into the temp root (GNU tar reads a D: path as a remote host) and launches the bash --for review-voice installer with a colon-joined GSTACK_LAUNCH PATH' }, // C-pregate. Since fix wave PR 5 (item 10b) test/helpers/pregate-fixture.ts and restamp-fixture.ts diff --git a/scripts/test-paid-shards.ts b/scripts/test-paid-shards.ts index b3b4786e4..5439d0a8f 100644 --- a/scripts/test-paid-shards.ts +++ b/scripts/test-paid-shards.ts @@ -64,7 +64,7 @@ import { import { PAID_TEST_GLOBS, isPaidTestFile } from '../test/helpers/paid-test-set'; import { EVAL_POLICY, PERIODIC_CI_EXCLUDE } from '../test/helpers/periodic-exclude-data'; import { - getProjectEvalDir, getClaudeCliVersion, isFinalizedEvalResultFile, failureClassOf, panelVerdict, + getProjectEvalDir, getClaudeCliVersion, isFinalizedEvalResultFile, credentialRejection, failureClassOf, panelVerdict, sanitizeTrialError, CONTRACT_VIOLATIONS_FILE, TRIAL_ENV, type TrialFailureClass, trialCostKnown, trialFailureFields, trialSessions, } from '../test/helpers/eval-store'; @@ -134,8 +134,9 @@ export function classifyTrialShard( ...trialCostKnown(evidence.records, evidence.sessions), }; const failed = (failureClass: TrialFailureClass, error?: string): ShardTrialRecord => { - const cls = evidence.contract !== null ? 'contract' : failureClass; - const raw = evidence.contract ?? failedRecord?.error ?? error; + const credentials = evidence.contract === null ? credentialRejection({ error: failedRecord?.error ?? error, sessions: evidence.sessions }) : undefined; + const cls = evidence.contract !== null ? 'contract' : credentials ? 'infra' : failureClass; + const raw = evidence.contract ?? (credentials ? `credentials rejected: ${credentials}` : failedRecord?.error ?? error); return { ...base, outcome: 'failed', failure_class: cls, ...(failedRecord?.exit_reason ? { exit_reason: String(failedRecord.exit_reason) } : {}), ...(Number.isInteger(failedRecord?.timeout_at_turn) ? { timeout_at_turn: failedRecord.timeout_at_turn } : {}), diff --git a/test/credential-rejection.test.ts b/test/credential-rejection.test.ts new file mode 100644 index 000000000..694feee3d --- /dev/null +++ b/test/credential-rejection.test.ts @@ -0,0 +1,191 @@ +/** + * Credential rejections are INFRA (Captain, 2026-10-23). Train gate run + * 39696775546 ran with a rejected ANTHROPIC_API_KEY: every case failed on a + * 401 before the model ran, and the report called them 131 assertion reds and + * launched 131 rule-16 reruns with the same key. A rejected key fails every + * case the same way, so these failures are INFRA: never rerun, and counted as + * INFRA in the verdict. The captures below are trimmed from that run + * (test/fixtures/credential-rejection-39696775546.json). Each runner's own + * evidence path is replayed: the claude -p stream observer, the PTY screen, + * the judge's API error and the Agent SDK's error result. The negative controls + * are real assertion failures that mention a key; they keep their class and + * are still rerun. + */ +import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { credentialRejection } from './helpers/eval-store'; +import { SessionObserver } from './helpers/session-ledger'; +import { safePanelEnd } from './helpers/pty/session'; +import { classifyTrialShard } from '../scripts/test-paid-shards'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const CAPTURE = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/credential-rejection-39696775546.json'), 'utf8')); +const REJECTED = 'Invalid API key · Fix external API key'; +/** Real assertion failures that mention a key, an API key or an auth word, with no provider rejection. */ +const KEY_ASSERTIONS = [ + 'expect(received).toContain(expected)\n\nExpected to contain: "Invalid API key"\nReceived: "ok"', + "Expected config to contain key \"api_key\"; the Invalid API key handling section is missing", + 'README should explain how to rotate the ANTHROPIC_API_KEY', + 'authentication rejected by the fixture server was not reported to the user', + 'expect(received).toBe(expected)\n\nExpected: "success"\nReceived: "exit_code_1"', +]; + +/** The ledger row the claude -p runner writes from its observer (session-runner.ts: structured end, else error). */ +function claudePLedgerRow(events: unknown[]): Record { + const observer = new SessionObserver(0); + events.forEach((event, i) => observer.observe(event, i * 100)); + const verdict = observer.verdict(); + return { key: 'claude-p:ship-docsync-store#1', test_name: 'ship-docsync-store', runner: 'claude-p', started_at: '2026-10-22T00:00:33.527Z', + budget_ms: 584957, elapsed_ms: 1540, end: verdict?.end ?? 'error', ...(verdict ? { evidence: verdict.evidence } : {}), billed: true }; +} +/** The ledger row the PTY runner writes: its observer timed out, unless the screen shows an end panel. */ +function ptyLedgerRow(screen: string): Record { + return { key: 'pty:session#1', runner: 'pty', started_at: '2026-10-22T00:00:35.294Z', budget_ms: 420000, elapsed_ms: 420002, + end: 'observer_timeout', ...safePanelEnd({ visibleText: () => screen }), billed: false }; +} + +describe('each runner records the rejection it saw (run 39696775546)', () => { + test('claude -p: the CLI\'s synthetic 401 message ends the session as an API error', () => { + expect(claudePLedgerRow(CAPTURE.claude_p_events)).toMatchObject({ end: 'api_error', evidence: REJECTED }); + }); + + test('PTY: a session parked at the plan-mode footer after the rejection still records it', () => { + expect(ptyLedgerRow(CAPTURE.pty_screen)).toMatchObject({ end: 'api_error', evidence: expect.stringContaining(REJECTED) }); + expect(ptyLedgerRow('❯ /plan-ceo-review\n✻Baked for 3s · done 12:00 AM\n❯ \n')).toMatchObject({ end: 'observer_timeout' }); + }); + + test('every runner\'s evidence reads as a credential rejection', () => { + expect(credentialRejection({ error: CAPTURE.judge_error })).toContain('"type":"authentication_error"'); + expect(credentialRejection({ error: CAPTURE.agent_sdk_error })).toContain(REJECTED); + expect(credentialRejection({ error: CAPTURE.pty_junit_message + CAPTURE.pty_screen })).toContain(REJECTED); + expect(credentialRejection({ error: 'x', sessions: [{ end: 'api_error', evidence: REJECTED }] })).toBe(REJECTED); + expect(credentialRejection({ error: '401 {"error":{"code":"invalid_api_key","message":"Incorrect API key provided"}}' })).toContain('invalid_api_key'); + }); +}); + +describe('negative controls: a failure that only mentions a key keeps its class', () => { + test('assertion text alone is never a rejection', () => { + for (const error of KEY_ASSERTIONS) expect(credentialRejection({ error }), error).toBeUndefined(); + }); + + test('a session must have ended as an API error, and a non-credential API error is not one', () => { + expect(credentialRejection({ sessions: [{ end: 'completed', evidence: REJECTED }] })).toBeUndefined(); + expect(credentialRejection({ sessions: [{ end: 'api_error', evidence: 'API Error: 529 {"type":"error","error":{"type":"overloaded_error"}}' }] })).toBeUndefined(); + }); +}); + +describe('trial panels: classifyTrialShard', () => { + const plan = { kind: 'behavior' as const, panel: { n: 3, k: 2 }, quarantined: false }; + const shard = { status: 'failed' as const, executedTests: 1, skippedTests: 0, elapsedMs: 1540 }; + const rejected = { sessions: [claudePLedgerRow(CAPTURE.claude_p_events) as any] }; + + test('a rejected session makes the trial INFRA, and its error names the rejection', () => { + expect(classifyTrialShard(shard, 'ship-docsync-store', 1, plan, { records: [CAPTURE.claude_p_record], contract: null, ...rejected })) + .toMatchObject({ outcome: 'failed', failure_class: 'infra', error: `credentials rejected: ${REJECTED}`, failure_cause: 'pre_turn_infra' }); + expect(classifyTrialShard(shard, 'c', 1, plan, { records: [{ passed: false, error: CAPTURE.agent_sdk_error }], contract: null })) + .toMatchObject({ failure_class: 'infra' }); + }); + + test('contract evidence still wins, and a key-mentioning assertion stays an assertion', () => { + expect(classifyTrialShard(shard, 'c', 1, plan, { records: [CAPTURE.claude_p_record], contract: 'handoff missing', ...rejected })) + .toMatchObject({ failure_class: 'contract' }); + for (const error of KEY_ASSERTIONS) { + expect(classifyTrialShard(shard, 'c', 1, plan, { records: [{ passed: false, error }], contract: null, sessions: [] }), error) + .toMatchObject({ failure_class: 'assertion' }); + } + }); +}); + +// ── The real --report and --rerun-plan CLI paths on a trimmed two-shard capture ── +const STORE = 'test/skill-e2e-ship-docsync.test.ts#ship-docsync-store'; +const PTY = 'test/skill-e2e-plan-ceo-plan-mode.test.ts'; +const CASES = ['ship-docsync-store', 'plan-ceo-review-plan-mode']; +const xml = (s: string) => s.replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"').replace(/\n/g, ' ').replace(/\r/g, ' '); +const junit = (file: string, suite: string, name: string, message: string) => ` + + + + + + + + + +`; +interface Shard { key: string; slug: string; junit: string; ledger: Record; record?: Record; budget?: Record } +const storeShard = (message: string, ledger: Record): Shard => ({ key: STORE, slug: 'skill-e2e-ship-docsync--ship-docsync-store', ledger, + budget: { timeoutMs: 720000, source: 'registered', policyId: 'skill-e2e-ship-docsync-existing-retry-v1' }, + junit: junit('test/skill-e2e-ship-docsync.test.ts', 'Ship doc-sync lifecycle > Ship doc-sync lifecycle E2E (gate)', 'ship-docsync-store', message), + record: { ...CAPTURE.claude_p_record, transcript: CAPTURE.claude_p_events } }); +const ptyShard: Shard = { key: PTY, slug: 'skill-e2e-plan-ceo-plan-mode', ledger: ptyLedgerRow(CAPTURE.pty_screen), + junit: junit(PTY, 'plan-ceo-review plan-mode smoke (gate)', 'first terminal outcome is asked (Step 0 fires before any plan write)', CAPTURE.pty_junit_message + CAPTURE.pty_screen) }; + +let base: string; +let counter = 0; +beforeAll(() => { base = fs.mkdtempSync(path.join(os.tmpdir(), 'credential-rejection-')); }); +afterAll(() => { fs.rmSync(base, { recursive: true, force: true }); }); + +function reportDir(shards: Shard[]): string { + const dir = path.join(base, `case-${++counter}`); + const e2e = CASES.filter((_, i) => shards.some(s => s.key === [STORE, PTY][i])); + const selection = { e2e, judges: [] }; + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, 'manifest.json'), JSON.stringify({ version: 1, tier: 'gate', evalsAll: false, sliceCount: 1, profile: 'pr', selection, + selectionReason: 'trimmed replay of run 39696775546', + prCoverage: { mode: 'dependents', e2e, judges: [], deferred: [], unknownFiles: [], unknownFileLabels: [], directCases: [], derivedFiles: [], noConsumerFiles: [], + deferredPromptFiles: [], missingCoverage: [], needsFullValidation: false, reasons: [] }, + plan: { sliceBudgetMs: 420000, jobs: 2, estimatedSliceMs: [420000], ciTimeoutMinutes: 50, sliceCiTimeoutMinutes: [50], codexSlices: [] }, + entries: shards.map(s => ({ file: s.key, slice: 1, status: 'planned', estimatedMs: 120000, ...(s.budget ? { budget: s.budget } : {}) })) })); + for (const s of shards) { + const shardDir = path.join(dir, 'paid-slice-1-a1', 'shards', s.slug); + fs.mkdirSync(shardDir, { recursive: true }); + fs.writeFileSync(path.join(shardDir, 'junit.xml'), s.junit); + fs.writeFileSync(path.join(shardDir, 'session-ledger.jsonl'), JSON.stringify(s.ledger) + '\n'); + if (s.record) { + fs.writeFileSync(path.join(shardDir, '1.91.176-train-gate.train-437b2f2a3e80-514fbfdc9799-5bc2e6-e2e-ship-docsync-2026-10-22-0000.json'), + JSON.stringify({ schema_version: 2, version: '1.91.176', tier: 'e2e-ship-docsync', total_tests: 1, executed_tests: 1, passed: 0, failed: 1, total_cost_usd: 0, tests: [s.record] })); + } + } + fs.writeFileSync(path.join(dir, 'paid-slice-1-a1', 'slice-1.json'), JSON.stringify({ version: 1, tier: 'gate', profile: 'pr', selection, sliceIndex: 1, sliceCount: 1, attempt: 1, + startedAt: 1792627232000, finishedAt: 1792627653000, + outcomes: shards.map(s => ({ files: [s.key], status: 'failed', exitCode: 1, elapsedMs: 1655, executedTests: 1, skippedTests: 0, + budget: s.budget ?? { timeoutMs: 1800000, source: 'default', policyId: null } })) })); + return dir; +} +const cli = (args: string[]) => spawnSync(process.execPath, ['--no-env-file', '--config=/dev/null', path.join(ROOT, 'scripts/test-paid-shards.ts'), '--tier', 'gate', ...args], + { cwd: ROOT, encoding: 'utf8', timeout: 60_000, + env: { ...process.env, EVALS_TIER: 'gate', GITHUB_RUN_ID: '39696775546', GITHUB_RUN_ATTEMPT: '1', GITHUB_SHA: '', GSTACK_GATE_PR: '', GSTACK_GATE_HEAD_SHA: '' } }); +function planRerun(dir: string) { + const out = path.join(dir, 'rerun-plan-a1'); + const r = cli(['--report', dir, '--rerun-plan', out]); + expect(r.status, r.stdout + r.stderr).toBe(0); + const file = path.join(out, 'manifest.json'); + return { out: r.stdout, manifest: fs.existsSync(file) ? JSON.parse(fs.readFileSync(file, 'utf8')) : null }; +} + +describe('replay of run 39696775546 through --report and --rerun-plan', () => { + test('every case died on the rejected key: no rule-16 rerun, and the verdict counts INFRA, not assertion reds', () => { + const dir = reportDir([storeShard(CAPTURE.claude_p_junit_message, claudePLedgerRow(CAPTURE.claude_p_events)), ptyShard]); + const plan = planRerun(dir); + expect(plan.manifest).toBeNull(); + for (const id of CASES) expect(plan.out).toContain(`not rerun: ${id} (never rerun: INFRA failure class)`); + fs.rmSync(path.join(dir, 'collector-outcomes.json'), { force: true }); + const r = cli(['--report', dir]); + const out = r.stdout + r.stderr; + expect(r.status, out).toBe(1); + expect(out).toContain('VERDICT RED'); + expect(out).toContain('INFRA 2'); + for (const id of CASES) expect(out).toMatch(new RegExp(`✗ ${id} rule FAIL infra — credentials rejected: `)); + expect(out).not.toMatch(/FAIL {2}assertion/); + }); + + test('negative control: a real assertion red that mentions a key is still rerun once', () => { + const completed = { ...claudePLedgerRow([]), end: 'completed' }; + const dir = reportDir([{ ...storeShard(KEY_ASSERTIONS[1], completed), record: { ...CAPTURE.claude_p_record, output: 'done', exit_reason: 'success', error: KEY_ASSERTIONS[1] } }]); + const plan = planRerun(dir); + expect(plan.manifest?.rerun).toEqual({ attempt: 1, cases: ['ship-docsync-store'] }); + }); +}); diff --git a/test/fixtures/credential-rejection-39696775546.json b/test/fixtures/credential-rejection-39696775546.json new file mode 100644 index 000000000..5f6e5f1d7 --- /dev/null +++ b/test/fixtures/credential-rejection-39696775546.json @@ -0,0 +1,61 @@ +{ + "source": "GitHub Actions run 39696775546 (train gate, 2026-10-22): every gated case failed on a rejected ANTHROPIC_API_KEY; trimmed captures", + "claude_p_events": [ + { + "type": "system", + "subtype": "status", + "status": "requesting", + "session_id": "fc99f0a9-26d1-4c20-b835-7e2c7bd5a629" + }, + { + "type": "assistant", + "message": { + "id": "c885e094-9dfe-45d9-ba48-1d7955f9ba48", + "model": "", + "role": "assistant", + "stop_reason": "stop_sequence", + "type": "message", + "content": [ + { + "type": "text", + "text": "Invalid API key · Fix external API key" + } + ] + }, + "parent_tool_use_id": null, + "session_id": "fc99f0a9-26d1-4c20-b835-7e2c7bd5a629", + "error": "authentication_failed", + "is_api_error_message": true + }, + { + "type": "result", + "subtype": "success", + "is_error": true, + "api_error_status": 401, + "terminal_reason": "api_error", + "num_turns": 1, + "total_cost_usd": 0, + "result": "Invalid API key · Fix external API key", + "session_id": "fc99f0a9-26d1-4c20-b835-7e2c7bd5a629" + } + ], + "claude_p_record": { + "name": "ship-docsync-store", + "suite": "Ship doc-sync lifecycle", + "tier": "e2e", + "passed": false, + "duration_ms": 1540, + "cost_usd": 0, + "output": "Invalid API key · Fix external API key", + "turns_used": 1, + "tokens_used": 0, + "browse_errors": [], + "exit_reason": "exit_code_1", + "model": "claude-opus-5-5" + }, + "claude_p_junit_message": "expect(received).toBeGreaterThanOrEqual(expected)\n\nExpected: >= 0\nReceived: -1\n", + "pty_junit_message": "plan-ceo-review smoke FAILED: outcome=timeout\nTimeout means the agent neither asked nor completed within the budget — likely hung mid-question or stuck on a permission dialog.\nsummary: no terminal outcome within 420000ms (last LLM judge: state=unknown — claude exited 1 ())\nelapsed: 420002ms\n--- evidence (last 2KB visible) ---\n", + "pty_screen": "\r❯ /plan-ceo-review\r\r✢ Ebbing… \r\r\r\n────────────────────────────────────────\r\r\n❯  \r\r\n────────────────────────────────────────\r\r\n⏸planmodeon (shift+tabtocycle)·esctointerrupt·←foragents◐medium·/effort\r\r\n\r·n\r\r\n\r ⎿  Invalid API key · Fix external API key\r\r✻Baked for 0s · done 12:00 AM\r\r────────────────────────────────────────\r❯  \r────────────────────────────────────────\r\r\n⏸planmodeon (shift+tabtocycle)·←foragents◐medium·/effort\r\r\n\r\r\r\n", + "judge_error": "401 {\"type\":\"error\",\"error\":{\"type\":\"authentication_error\",\"message\":\"invalid x-api-key\"},\"request_id\":\"req_011CgGHuinAhnU3wapQEbUe1\"}", + "agent_sdk_error": "Error: Claude Code returned an error result: Invalid API key · Fix external API key" +} diff --git a/test/helpers/eval-store.ts b/test/helpers/eval-store.ts index ed8721f13..a50ea2c28 100644 --- a/test/helpers/eval-store.ts +++ b/test/helpers/eval-store.ts @@ -15,6 +15,7 @@ import { spawnSync } from 'child_process'; import { isManualReviewEntry } from './cookie-workflow-manual-review'; import type { ManualJudgeReview } from './cookie-workflow-manual-review'; import type { CodexTokenUsage } from './codex-session-runner'; +import { CREDENTIAL_REJECTION } from './session-ledger'; // v2: EvalTestEntry.harvest gains optional {insertions, deletions, net} and // may be explicitly null (arm-benchmark harvest-failure taxonomy). Readers @@ -215,6 +216,15 @@ export function failureClassOf(entry: Pick[] }): string | undefined { + const texts = [...(facts.sessions ?? []).filter((s) => s.end === 'api_error').map((s) => s.evidence), facts.error]; + const line = texts.flatMap((text) => text?.split(/\r\n?|\n/) ?? []).find((l) => CREDENTIAL_REJECTION.test(l)); + return oneLine(line, FAILURE_CAUSE_EVIDENCE_MAX); +} + /** Machine-recorded cause of a failed trial, beside its policy class. failure_class * (what verdicts read) is never changed by it; whether an assertion was a * detector, harness or product fault is a human diagnosis, never a cause. */ diff --git a/test/helpers/free-fixtures-data.ts b/test/helpers/free-fixtures-data.ts index 40ed88351..aaca3c551 100644 --- a/test/helpers/free-fixtures-data.ts +++ b/test/helpers/free-fixtures-data.ts @@ -65,6 +65,7 @@ export const FREE_FIXTURES: Readonly> = { 'test/fixtures/coverage-audit-separator-collision.json': ['test/coverage-audit-evidence.test.ts'], 'test/fixtures/coverage-audit-summary.json': ['test/coverage-audit-evidence.test.ts'], 'test/fixtures/coverage-audit-wc-anchor.json': ['test/coverage-audit-evidence.test.ts'], + 'test/fixtures/credential-rejection-39696775546.json': ['test/credential-rejection.test.ts'], 'test/fixtures/cso-eval/**': ['scripts/cso-eval.ts', 'test/cso-eval.test.ts', 'test/cso-stack-cold-integration.test.ts', 'test/helpers/cso-eval-oracles.ts'], 'test/fixtures/cso-http-smoke.c': ['test/cso-docker-integration.test.ts'], 'test/fixtures/cso-sarif/**': ['test/cso-scanner-cli.test.ts', 'test/cso-scanners.test.ts'], diff --git a/test/helpers/pty/launch.ts b/test/helpers/pty/launch.ts index 884d1008b..e4413de38 100644 --- a/test/helpers/pty/launch.ts +++ b/test/helpers/pty/launch.ts @@ -13,7 +13,7 @@ import { createFilePermissionRecorder } from '../plan-count-file-permission'; import { createAutoplanArtifactRecorder } from '../autoplan-artifact-recorder'; import { trustDialogInput } from '../pty-trust-dialog'; import { resolveClaudeBinary } from './binary'; -import { createPtyScreen, idlePanelEnd, stripAnsi } from './screen'; +import { createPtyScreen, screenPanelEnd, stripAnsi } from './screen'; import { appendSessionLedger, sessionKey } from '../session-ledger'; export interface ClaudePtyOptions { @@ -249,7 +249,7 @@ export async function launchClaudePty( recorded = true; const elapsed = performance.now() - startedMono; appendSessionLedger({ key: sessionKey('pty', undefined), runner: 'pty', started_at: startedAt, budget_ms: timeoutMs, - elapsed_ms: elapsed, end: elapsed >= timeoutMs ? 'session_timeout' : 'completed', ...idlePanelEnd(handle.visibleText()), billed: false }); + elapsed_ms: elapsed, end: elapsed >= timeoutMs ? 'session_timeout' : 'completed', ...screenPanelEnd(handle.visibleText()), billed: false }); } return close(); } }); diff --git a/test/helpers/pty/screen.ts b/test/helpers/pty/screen.ts index ba7a80bb6..f3743152f 100644 --- a/test/helpers/pty/screen.ts +++ b/test/helpers/pty/screen.ts @@ -7,6 +7,7 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import { currentFilePermissionTarget, currentBashPermissionCard, currentReadPermissionCard, currentWebFetchPermissionCard, hasCurrentBashPermissionHeading, hasCurrentReadPermissionHeading, hasCurrentWebFetchPermissionHeading } from '../plan-skill-questions'; import { createRequire } from 'node:module'; +import { CREDENTIAL_REJECTION } from '../session-ledger'; /** The existing xterm package ships this headless source beside its browser bundle. */ let terminalConstructor: Promise | undefined; @@ -413,7 +414,7 @@ export function isNumberedOptionListVisible(visible: string): boolean { } /** Start of the CLI's public "API Error:" panel. The native AUQ capture reads the same panel. */ -export const API_ERROR_PANEL = /(?:^|\n)[\t │┃]*(?:[⎿●⏺]\s*)?API Error:/i; +export const API_ERROR_PANEL = /(?:^|\n)[\t │┃]*(?:[⎿●⏺]\s*)?(?:API Error:|Invalid API key\s*·)/i; /** Session-ledger end from the last idle turn's public error panel: a safeguards panel is a refusal, any other an API error. */ export function idlePanelEnd(visible: string): { end: 'refusal' | 'api_error'; evidence: string } | null { @@ -422,6 +423,13 @@ export function idlePanelEnd(visible: string): { end: 'refusal' | 'api_error'; e return { end: /safeguards flagged|can't respond to this message/i.test(panel) ? 'refusal' : 'api_error', evidence: panel }; } +/** The last idle turn's error panel, else the CLI's rejected-credentials line anywhere on screen: after one the + * CLI can sit at a footer the idle check does not know (plan mode, manual mode), and no turn can run. */ +export function screenPanelEnd(visible: string): { end: 'refusal' | 'api_error'; evidence: string } | null { + const credential = visible.split(/\r\n?|\n/).reverse().find(line => CREDENTIAL_REJECTION.test(line))?.trim(); + return idlePanelEnd(visible) ?? (credential ? { end: 'api_error', evidence: credential } : null); +} + /** The CLI's end-of-turn line, e.g. "✻ Cooked for 24s · done 12:46 AM" (spaces may collapse in the PTY text). */ const TURN_DONE_RE = /✻[ \t]*\p{L}[\p{L}'’-]*[ \t]*for[ \t]*(?=\d)(?:\d+h[ \t]*)?(?:\d+m[ \t]*)?(?:\d+s)?[ \t]*·[ \t]*done[^\n]*/gu; /** An empty input prompt and its known idle footers, nothing else. */ diff --git a/test/helpers/pty/session.ts b/test/helpers/pty/session.ts index 58533cea9..54fe04b61 100644 --- a/test/helpers/pty/session.ts +++ b/test/helpers/pty/session.ts @@ -8,7 +8,7 @@ * former claude-pty-runner.ts. Tests import through claude-pty-runner.ts. */ import { launchClaudePty, type ClaudePtyOptions, type ClaudePtySession } from './launch'; -import { idlePanelEnd } from './screen'; +import { screenPanelEnd } from './screen'; import { appendSessionLedger, sessionKey, type SessionEnd } from '../session-ledger'; /** @@ -33,9 +33,9 @@ export const realPtyDriver: PtyDriver = { sleep: ms => Bun.sleep(ms), }; -/** The last idle turn's error panel, if the session can show its text; diagnostics never throw. */ -export function safePanelEnd(session: Pick): ReturnType { - try { return idlePanelEnd(session.visibleText()); } catch { return null; } +/** The session's end panel (screenPanelEnd), if the session can show its text; diagnostics never throw. */ +export function safePanelEnd(session: Pick): ReturnType { + try { return screenPanelEnd(session.visibleText()); } catch { return null; } } /** One poll step: finish with a result, keep polling, or end the loop as a timeout. */ diff --git a/test/helpers/session-ledger.ts b/test/helpers/session-ledger.ts index 425323284..a10bf28bf 100644 --- a/test/helpers/session-ledger.ts +++ b/test/helpers/session-ledger.ts @@ -19,6 +19,9 @@ export const SESSION_EVIDENCE_MAX = 200; export type SessionRunner = 'claude-p' | 'agent-sdk' | 'pty' | 'codex'; export type SessionEnd = 'completed' | 'session_timeout' | 'observer_timeout' | 'api_error' | 'refusal' | 'aborted' | 'error'; export const SESSION_ENDS: readonly SessionEnd[] = ['completed', 'session_timeout', 'observer_timeout', 'api_error', 'refusal', 'aborted', 'error']; +/** A provider rejected the run's credentials: Anthropic's 401 authentication_error body, the CLI's + * authentication_failed code or its "Invalid API key · …" message, or OpenAI's invalid_api_key code. */ +export const CREDENTIAL_REJECTION = /"type"\s*:\s*"authentication_error"|\bauthentication_failed\b|\bInvalid API key\s*·|"code"\s*:\s*"invalid_api_key"/i; export interface LivenessSummary { /** The session streamed partial messages. Without them no stall is ever inferred. */ @@ -189,7 +192,7 @@ export class SessionObserver { } private noteResultText(text: string): void { - const line = text.split('\n').map(l => l.trim()).find(l => /^API Error:/i.test(l)); + const line = text.split('\n').map(l => l.trim()).find(l => /^API Error:/i.test(l) || CREDENTIAL_REJECTION.test(l)); if (!line) return; if (REFUSAL_PHRASE.test(text)) this.phraseRefusal ??= `refusal at turn ${Math.max(1, this.turnIds.size)} (phrase) — ${line}`; else this.apiError ??= line;