v1.91.183.0 fix(evals): a rejected API key is INFRA, never 131 assertion reds and reruns (#3361)

* fix(evals): a rejected API key is INFRA, never 131 assertion reds and reruns

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* test: register the credential-rejection fixture as free

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* test: keep the credential-rejection CLI replay off the Windows subset, like paid-rerun

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* test: name the Windows run in the curation reason

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* v1.91.183.0 fix(evals): a rejected API key is INFRA, never 131 assertion reds and reruns (#3361)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

---------

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

scripts/harness-version.json

@@ -13,10 +13,10 @@
   "files": {
     "scripts/lib/paid-cases.ts": {"blob":"f862cf5241580c7593b9ee62eacf5ef046248d26","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
     "scripts/lib/paid-plan.ts": {"blob":"e8ce3cb98474dded38c5128a631d283b0ca35b63","since":6,"nonBehavioral":"F1 rule-16 flaky rerun: report aggregation, rerun planning and the gate slice exit code; no case's execution, trial outcome, trial record or panel verdict changes, so no pass-rate series resets"},
-    "scripts/lib/paid-report.ts": {"blob":"df19605183b8c6bfabd01bbe23fbe821d97a405e","since":6,"nonBehavioral":"F1 rule-16 flaky rerun: report aggregation, rerun planning and the gate slice exit code; no case's execution, trial outcome, trial record or panel verdict changes, so no pass-rate series resets"},
+    "scripts/lib/paid-report.ts": {"blob":"19a87cb02bb3cd0f57da7719a5c30b7a4df81b2b","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
     "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6},
     "scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":6},
-    "scripts/test-paid-shards.ts": {"blob":"b3b4786e406ca6793df2eb24fafac27cfb99a48e","since":6,"nonBehavioral":"F1 rule-16 flaky rerun: report aggregation, rerun planning and the gate slice exit code; no case's execution, trial outcome, trial record or panel verdict changes, so no pass-rate series resets"},
+    "scripts/test-paid-shards.ts": {"blob":"5439d0a8f6e1569fa7c924af20d67c30c81d5d30","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
     "scripts/test-pr-profile.ts": {"blob":"d334195b0d0393622e2c380dc3e2feebe099b6e6","since":6,"nonBehavioral":"classify the new evals-main-backstop.yml workflow as free-lane only: it dispatches evals.yml on main and runs inside no case, so no case's selection on a PR, input or verdict changes"},
     "scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":6},
     "test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":6},
@@ -27,7 +27,7 @@
     "test/helpers/e2e-gate.ts": {"blob":"efb7e197dd5f66c3ad0e34074b14e640754f7a45","since":6},
     "test/helpers/e2e-helpers.ts": {"blob":"6ee6937e6c3ca081c77131b5a755278c7b714834","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
     "test/helpers/eval-budgets.ts": {"blob":"667d0e09b20fe5d7366f57b77b5fb9b04baa37ef","since":6},
-    "test/helpers/eval-store.ts": {"blob":"ed8721f13d39f4ef62e0a1d814efbe9352da781a","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
+    "test/helpers/eval-store.ts": {"blob":"a50ea2c28f508ad9f35ea9353dd2f502ee0dcfc2","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
     "test/helpers/hermetic-env.ts": {"blob":"cc072895461dc52ea317d1beff72121342420d30","since":6},
     "test/helpers/llm-judge.ts": {"blob":"d0bc6bbbe9f478603d11a1ece95e66c812ad34e6","since":6},
     "test/helpers/paid-test-set.ts": {"blob":"ff9bfe43c43ec2bdd11114f7b8ec7c5736116f75","since":6},
@@ -38,15 +38,15 @@
     "test/helpers/pty/classify.ts": {"blob":"3f2daa12ff6484f96b8d28d910a50231c4bb3ae8","since":6},
     "test/helpers/pty/fake-session.ts": {"blob":"554e064ef259b8444280eddfe9b0db93c66a8202","since":6},
     "test/helpers/pty/judge.ts": {"blob":"81c5bb4c48ab928aca2f186a1ade9df7169d7fbe","since":6},
-    "test/helpers/pty/launch.ts": {"blob":"884d1008b3349e8b16d6c38f9f0aa7ae514f0f37","since":6},
+    "test/helpers/pty/launch.ts": {"blob":"e4413de380b799659170e7da248d51afda3b0e3f","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
     "test/helpers/pty/plan-native.ts": {"blob":"ddba9cb68c7c0da935cff7c1cea258dee2dff740","since":6},
     "test/helpers/pty/runners/counting.ts": {"blob":"8c5c05e54abf961259d34d9c3d77db67138d7794","since":6},
     "test/helpers/pty/runners/floor.ts": {"blob":"57fa44c31300754b00473276cf7b972d4e9e47d8","since":6},
     "test/helpers/pty/runners/observation.ts": {"blob":"1c2b75fbb984708725abaa8939125f7cab969f9f","since":6},
-    "test/helpers/pty/screen.ts": {"blob":"ba7a80bb68b1959296ab2a827cfacdaa46f4ba2d","since":6},
-    "test/helpers/pty/session.ts": {"blob":"58533cea9d81cc87bb3b499bdfe0db8e3a8709c5","since":6},
+    "test/helpers/pty/screen.ts": {"blob":"f3743152fc9bb7a8e8c70002f75bf45914f260d5","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
+    "test/helpers/pty/session.ts": {"blob":"54fe04b61d953e4e55e2fc251532c04a49fab66f","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
     "test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":6},
-    "test/helpers/session-ledger.ts": {"blob":"425323284a368f5e77937e03884db4dea4e6134e","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
+    "test/helpers/session-ledger.ts": {"blob":"a10bf28bf100f135b93c73818670dd147253cdd3","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
     "test/helpers/session-runner.ts": {"blob":"cc65a66adafa9c053e3da3c6f17b5ac86c0eade7","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
     "test/helpers/skill-fixture.ts": {"blob":"ffece830f35b0f1562005105e9296dc4d8ca50c2","since":6,"nonBehavioral":"skill-fixture.ts drops 'When to invoke this skill' from the review, review-army and retro section lists because #3129 stops generating that heading; extraction would throw otherwise. The fixture bytes follow the regenerated SKILL.md, whose owned prompt bytes already start those cases' new series."},
     "test/helpers/test-selection.ts": {"blob":"51655f89f66be7a5ec487e2abfe4744300ebd5f6","since":6},

scripts/lib/paid-report.ts

@@ -7,7 +7,7 @@ import { normalizeRelativePath } from './shard-engine';
 import { EVAL_POLICY } from '../../test/helpers/periodic-exclude-data';
 import { STALL_WINDOW_MS } from '../../test/helpers/eval-budgets';
 import {
-  isFinalizedEvalResultFile, failureClassOf, panelVerdict, sanitizeTrialError, formatTrialOutcomes, TRIAL_OUTCOME_SCHEMA, TRIAL_OUTCOMES_FILE,
+  isFinalizedEvalResultFile, credentialRejection, failureClassOf, panelVerdict, sanitizeTrialError, formatTrialOutcomes, TRIAL_OUTCOME_SCHEMA, TRIAL_OUTCOMES_FILE,
   type EvalCaseKind, type PanelVerdict, type TrialFailureClass, type TrialOutcome, type TrialOutcomeRecord,
   trialCostKnown, trialFailureFields, trialSessions, type TrialFailureDetail,
 } from '../../test/helpers/eval-store';
@@ -277,13 +277,16 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact
         const caseOutcome: TrialOutcome = failed ? 'failed' : cases.some(tc => tc.outcome === 'passed') ? 'passed' : 'skipped';
         const mine = records.filter((r: any) => r?.name === id || r?.case_id === id);
         const failedRecord = mine.find((r: any) => r.passed === false);
-        const failureClass: TrialFailureClass | undefined = !failed ? undefined
+        const recorded: TrialFailureClass | undefined = !failed ? undefined
           : failed.failureType === 'TimeoutError' ? 'timeout' : failedRecord ? failureClassOf(failedRecord) : 'assertion';
-        const error = sanitizeTrialError(failedRecord?.error ?? failed?.message);
         const sessions = caseSessions(ledger, id, byCase.size === 1);
+        const credentials = failed && recorded !== 'contract' ? credentialRejection({ error: `${failedRecord?.error ?? ''}\n${failed.message ?? ''}`, sessions }) : undefined;
+        const failureClass = credentials ? 'infra' : recorded;
+        const rawError = credentials ? `credentials rejected: ${credentials}` : failedRecord?.error ?? failed?.message;
+        const error = sanitizeTrialError(rawError);
         // A case's JUnit times sum its concurrent tests (plan-mode-no-op: 346 s in a 176 s shard), so its wall is capped by the shard's.
         const diagnosis = failed && failureClass ? trialFailureFields({ failure_class: failureClass, exit_reason: failedRecord?.exit_reason,
-          error: failedRecord?.error ?? failed.message, sessions, record: failedRecord }, STALL_WINDOW_MS) : undefined;
+          error: rawError, sessions, record: failedRecord }, STALL_WINDOW_MS) : undefined;
         if (failed) census.failedShards.add(key);
         if (caseOutcome === 'skipped') {
           bump(reasons, RUNTIME_SKIP_REASON);
@@ -762,7 +765,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
       judge: count('judge'),
       quarantined: { total: panels.filter(p => p.quarantined).length, failingLane: panels.filter(p => p.quarantined && p.failsLane).length },
       skipped: hollowWithoutJUnit.length + panels.filter(p => p.status === 'SKIPPED').length + ruleCases.filter(c => c.outcome === 'skipped').length,
-      infra: primaryTrials.filter(t => t.failure_class === 'infra').length
+      infra: primaryTrials.filter(t => t.failure_class === 'infra').length + ruleCases.filter(c => c.outcome === 'failed' && c.failureClass === 'infra').length
         + results.flatMap(r => r.outcomes).filter(o => !o.trial && o.runnerError !== undefined).length,
       incomplete: panels.filter(p => p.status === 'INCOMPLETE').length,
       unattributed: census.unattributed.length,

scripts/lib/windows-curation.ts

@@ -51,6 +51,8 @@ const WINDOWS_FRAGILE_PATTERNS: Array<{ pattern: RegExp; reason: string }> = [
 // when possible; this list is for environment-/runtime-specific tests where
 // the failure mode is structural rather than detectable via source-file scan.
 export const KNOWN_WINDOWS_INCOMPATIBLE: Array<{ file: string; reason: string }> = [
+  // Like test/paid-rerun.test.ts (excluded by pattern), it replays a gate capture through the paid lane's --report and --rerun-plan CLI.
+  { file: 'test/credential-rejection.test.ts', reason: 'replays run 39696775546 through the --report and --rerun-plan CLI of the paid lane, which runs only on Linux CI; on windows-latest that CLI exits 1 with EINVAL in open() (windows-free-shard run 39982914852)' },
   // Field list 2, W11: the review-voice install is a POSIX launcher contract, exercised end to end on Linux and macOS.
   { file: 'test/capy-review-voice.test.ts', reason: 'copies the checkout with tar -C into the temp root (GNU tar reads a D: path as a remote host) and launches the bash --for review-voice installer with a colon-joined GSTACK_LAUNCH PATH' },
   // C-pregate. Since fix wave PR 5 (item 10b) test/helpers/pregate-fixture.ts and restamp-fixture.ts

scripts/test-paid-shards.ts

@@ -64,7 +64,7 @@ import {
 import { PAID_TEST_GLOBS, isPaidTestFile } from '../test/helpers/paid-test-set';
 import { EVAL_POLICY, PERIODIC_CI_EXCLUDE } from '../test/helpers/periodic-exclude-data';
 import {
-  getProjectEvalDir, getClaudeCliVersion, isFinalizedEvalResultFile, failureClassOf, panelVerdict,
+  getProjectEvalDir, getClaudeCliVersion, isFinalizedEvalResultFile, credentialRejection, failureClassOf, panelVerdict,
   sanitizeTrialError, CONTRACT_VIOLATIONS_FILE, TRIAL_ENV, type TrialFailureClass,
   trialCostKnown, trialFailureFields, trialSessions,
 } from '../test/helpers/eval-store';
@@ -134,8 +134,9 @@ export function classifyTrialShard(
     ...trialCostKnown(evidence.records, evidence.sessions),
   };
   const failed = (failureClass: TrialFailureClass, error?: string): ShardTrialRecord => {
-    const cls = evidence.contract !== null ? 'contract' : failureClass;
-    const raw = evidence.contract ?? failedRecord?.error ?? error;
+    const credentials = evidence.contract === null ? credentialRejection({ error: failedRecord?.error ?? error, sessions: evidence.sessions }) : undefined;
+    const cls = evidence.contract !== null ? 'contract' : credentials ? 'infra' : failureClass;
+    const raw = evidence.contract ?? (credentials ? `credentials rejected: ${credentials}` : failedRecord?.error ?? error);
     return { ...base, outcome: 'failed', failure_class: cls,
       ...(failedRecord?.exit_reason ? { exit_reason: String(failedRecord.exit_reason) } : {}),
       ...(Number.isInteger(failedRecord?.timeout_at_turn) ? { timeout_at_turn: failedRecord.timeout_at_turn } : {}),