mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-05 00:03:15 +01:00
test: classify the failure-routing gate cases as skill-tool Claude-only (#3157/#3158)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
b277e335a2
commit
15899f57d3
3 files changed
+5
-4
No files matched your search
+1
-1
@@ -15,7 +15,7 @@
|
||||
|
||||
### Gate integrity
|
||||
- No threshold, budget, timeout, trial count or existing case kind changed. The ship skill's parity size stays under its cap (313,277 of 313,281 bytes). The new eval's negative controls are the two /investigate cases, and on the base router the triage cases go red.
|
||||
- One pin rises with a stated reason: `test/eng-finding-retry-budget.test.ts` counts 110 paid test files (was 109) for the new eval file.
|
||||
- Registries and pins the new gate-tier file needs, each with a stated reason: `test/eng-finding-retry-budget.test.ts` counts 110 paid test files (was 109). `test/helpers/e2e-hosts.ts` lists the four cases as `skill-tool` Claude-only, because they assert on the Claude Skill tool call. `test/codex-host.test.ts` counts 33 Claude-specific gate cases (was 29).
|
||||
- Each train gate now runs the four new cases, one trial each, for about $0.37 (measured at this head: $0.06, $0.06, $0.11 and $0.14; about 67 s wall when they run in parallel).
|
||||
|
||||
## [1.91.186.0] - 2026-10-24
|
||||
|
||||
@@ -128,13 +128,13 @@ describe('per-case host metadata', () => {
|
||||
for (const id of Object.keys(E2E_HOSTS)) expect(Object.hasOwn(E2E_TOUCHFILES, id), id).toBe(true);
|
||||
});
|
||||
|
||||
test('classifies 52 adapter-only, 29 Claude-specific, 26 rework and 1 host-neutral case', () => {
|
||||
test('classifies 52 adapter-only, 33 Claude-specific, 26 rework and 1 host-neutral case', () => {
|
||||
const counts: Record<string, number> = {};
|
||||
for (const support of Object.values(E2E_HOSTS)) counts[support] = (counts[support] ?? 0) + 1;
|
||||
expect(counts.both).toBe(52);
|
||||
expect(counts.rework).toBe(26);
|
||||
expect(counts['host-neutral']).toBe(1);
|
||||
expect(['pty', 'sdk', 'subagents', 'skill-tool', 'hermetic', 'self-host'].reduce((sum, reason) => sum + (counts[reason] ?? 0), 0)).toBe(29);
|
||||
expect(['pty', 'sdk', 'subagents', 'skill-tool', 'hermetic', 'self-host'].reduce((sum, reason) => sum + (counts[reason] ?? 0), 0)).toBe(33);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -98,7 +98,8 @@ const CLAUDE_ONLY: Readonly<Record<ClaudeOnlyReason, readonly string[]>> = {
|
||||
'shared-libs-review-prior-coverage', 'shared-libs-review-revalidation', 'ship-local-hook-preservation',
|
||||
'ship-managed-hook-refresh', 'ship-skipped-queued-finding', 'ship-unmanaged-hook-consent'],
|
||||
subagents: ['docsync-spawned', 'review-army-perf-n-plus-one'],
|
||||
'skill-tool': ['skillify-approval-reject', 'skillify-happy-path', 'skillify-provenance-refusal'],
|
||||
'skill-tool': ['router-failure-contradicted-cause', 'router-failure-triage-resolves', 'router-failure-triage-unclear',
|
||||
'router-failure-verified-cause', 'skillify-approval-reject', 'skillify-happy-path', 'skillify-provenance-refusal'],
|
||||
hermetic: ['hermetic-canary', 'hermetic-sentinel'],
|
||||
'self-host': ['safety-codex-consult-embed'],
|
||||
rework: ['qa-functional-cli-fix', 'qa-functional-cli-report', 'qa-functional-webhook-fix', 'qa-functional-webhook-report',
|
||||
|
||||
Reference in new issue
Block a user