test: classify the failure-routing gate cases as skill-tool Claude-only (#3157/#3158)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed 2026-10-24 23:18:38 +00:00
1 parent b277e335a2
commit 15899f57d3
3 files changed
+5 -4

No files matched your search

+1 -1
View File
@@ -15,7 +15,7 @@
### Gate integrity
- No threshold, budget, timeout, trial count or existing case kind changed. The ship skill's parity size stays under its cap (313,277 of 313,281 bytes). The new eval's negative controls are the two /investigate cases, and on the base router the triage cases go red.
- One pin rises with a stated reason: `test/eng-finding-retry-budget.test.ts` counts 110 paid test files (was 109) for the new eval file.
- Registries and pins the new gate-tier file needs, each with a stated reason: `test/eng-finding-retry-budget.test.ts` counts 110 paid test files (was 109). `test/helpers/e2e-hosts.ts` lists the four cases as `skill-tool` Claude-only, because they assert on the Claude Skill tool call. `test/codex-host.test.ts` counts 33 Claude-specific gate cases (was 29).
- Each train gate now runs the four new cases, one trial each, for about $0.37 (measured at this head: $0.06, $0.06, $0.11 and $0.14; about 67 s wall when they run in parallel).
## [1.91.186.0] - 2026-10-24
+2 -2
View File
@@ -128,13 +128,13 @@ describe('per-case host metadata', () => {
for (const id of Object.keys(E2E_HOSTS)) expect(Object.hasOwn(E2E_TOUCHFILES, id), id).toBe(true);
});
test('classifies 52 adapter-only, 29 Claude-specific, 26 rework and 1 host-neutral case', () => {
test('classifies 52 adapter-only, 33 Claude-specific, 26 rework and 1 host-neutral case', () => {
const counts: Record<string, number> = {};
for (const support of Object.values(E2E_HOSTS)) counts[support] = (counts[support] ?? 0) + 1;
expect(counts.both).toBe(52);
expect(counts.rework).toBe(26);
expect(counts['host-neutral']).toBe(1);
expect(['pty', 'sdk', 'subagents', 'skill-tool', 'hermetic', 'self-host'].reduce((sum, reason) => sum + (counts[reason] ?? 0), 0)).toBe(29);
expect(['pty', 'sdk', 'subagents', 'skill-tool', 'hermetic', 'self-host'].reduce((sum, reason) => sum + (counts[reason] ?? 0), 0)).toBe(33);
});
});
+2 -1
View File
@@ -98,7 +98,8 @@ const CLAUDE_ONLY: Readonly<Record<ClaudeOnlyReason, readonly string[]>> = {
'shared-libs-review-prior-coverage', 'shared-libs-review-revalidation', 'ship-local-hook-preservation',
'ship-managed-hook-refresh', 'ship-skipped-queued-finding', 'ship-unmanaged-hook-consent'],
subagents: ['docsync-spawned', 'review-army-perf-n-plus-one'],
'skill-tool': ['skillify-approval-reject', 'skillify-happy-path', 'skillify-provenance-refusal'],
'skill-tool': ['router-failure-contradicted-cause', 'router-failure-triage-resolves', 'router-failure-triage-unclear',
'router-failure-verified-cause', 'skillify-approval-reject', 'skillify-happy-path', 'skillify-provenance-refusal'],
hermetic: ['hermetic-canary', 'hermetic-sentinel'],
'self-host': ['safety-codex-consult-embed'],
rework: ['qa-functional-cli-fix', 'qa-functional-cli-report', 'qa-functional-webhook-fix', 'qa-functional-webhook-report',