From 15899f57d39afc55aa2f8f14f4174dd0276dbe61 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 24 Oct 2026 23:18:38 +0000 Subject: [PATCH] test: classify the failure-routing gate cases as skill-tool Claude-only (#3157/#3158) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 2 +- test/codex-host.test.ts | 4 ++-- test/helpers/e2e-hosts.ts | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c19d516a2..07d295dce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ ### Gate integrity - No threshold, budget, timeout, trial count or existing case kind changed. The ship skill's parity size stays under its cap (313,277 of 313,281 bytes). The new eval's negative controls are the two /investigate cases, and on the base router the triage cases go red. -- One pin rises with a stated reason: `test/eng-finding-retry-budget.test.ts` counts 110 paid test files (was 109) for the new eval file. +- Registries and pins the new gate-tier file needs, each with a stated reason: `test/eng-finding-retry-budget.test.ts` counts 110 paid test files (was 109). `test/helpers/e2e-hosts.ts` lists the four cases as `skill-tool` Claude-only, because they assert on the Claude Skill tool call. `test/codex-host.test.ts` counts 33 Claude-specific gate cases (was 29). - Each train gate now runs the four new cases, one trial each, for about $0.37 (measured at this head: $0.06, $0.06, $0.11 and $0.14; about 67 s wall when they run in parallel). ## [1.91.186.0] - 2026-10-24 diff --git a/test/codex-host.test.ts b/test/codex-host.test.ts index 0976a72e6..d6dda8b79 100644 --- a/test/codex-host.test.ts +++ b/test/codex-host.test.ts @@ -128,13 +128,13 @@ describe('per-case host metadata', () => { for (const id of Object.keys(E2E_HOSTS)) expect(Object.hasOwn(E2E_TOUCHFILES, id), id).toBe(true); }); - test('classifies 52 adapter-only, 29 Claude-specific, 26 rework and 1 host-neutral case', () => { + test('classifies 52 adapter-only, 33 Claude-specific, 26 rework and 1 host-neutral case', () => { const counts: Record = {}; for (const support of Object.values(E2E_HOSTS)) counts[support] = (counts[support] ?? 0) + 1; expect(counts.both).toBe(52); expect(counts.rework).toBe(26); expect(counts['host-neutral']).toBe(1); - expect(['pty', 'sdk', 'subagents', 'skill-tool', 'hermetic', 'self-host'].reduce((sum, reason) => sum + (counts[reason] ?? 0), 0)).toBe(29); + expect(['pty', 'sdk', 'subagents', 'skill-tool', 'hermetic', 'self-host'].reduce((sum, reason) => sum + (counts[reason] ?? 0), 0)).toBe(33); }); }); diff --git a/test/helpers/e2e-hosts.ts b/test/helpers/e2e-hosts.ts index 622c3756d..90ca0344f 100644 --- a/test/helpers/e2e-hosts.ts +++ b/test/helpers/e2e-hosts.ts @@ -98,7 +98,8 @@ const CLAUDE_ONLY: Readonly> = { 'shared-libs-review-prior-coverage', 'shared-libs-review-revalidation', 'ship-local-hook-preservation', 'ship-managed-hook-refresh', 'ship-skipped-queued-finding', 'ship-unmanaged-hook-consent'], subagents: ['docsync-spawned', 'review-army-perf-n-plus-one'], - 'skill-tool': ['skillify-approval-reject', 'skillify-happy-path', 'skillify-provenance-refusal'], + 'skill-tool': ['router-failure-contradicted-cause', 'router-failure-triage-resolves', 'router-failure-triage-unclear', + 'router-failure-verified-cause', 'skillify-approval-reject', 'skillify-happy-path', 'skillify-provenance-refusal'], hermetic: ['hermetic-canary', 'hermetic-sentinel'], 'self-host': ['safety-codex-consult-embed'], rework: ['qa-functional-cli-fix', 'qa-functional-cli-report', 'qa-functional-webhook-fix', 'qa-functional-webhook-report',