diff --git a/scripts/paid-test-durations.json b/scripts/paid-test-durations.json index 190cdee33..58e175ff5 100644 --- a/scripts/paid-test-durations.json +++ b/scripts/paid-test-durations.json @@ -157,6 +157,7 @@ "test/skill-e2e-outside-plan-disabled.test.ts": 53292, "test/skill-e2e-outside-voice.test.ts": 1000, "test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-sonnet.test.ts": 85282, + "test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5.test.ts": 85282, "test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash.test.ts": 80465, "test/skill-e2e-overlay-harness-opus-4-7-effort-match-trivial.test.ts": 49282, "test/skill-e2e-overlay-harness-opus-4-7-literal-interpretation.test.ts": 318532, diff --git a/test/fixtures/overlay-nudges.ts b/test/fixtures/overlay-nudges.ts index 0621e3fb4..92b7f80b1 100644 --- a/test/fixtures/overlay-nudges.ts +++ b/test/fixtures/overlay-nudges.ts @@ -324,6 +324,34 @@ export const OVERLAY_FIXTURES: OverlayFixture[] = [ comparison: { direction: 'lower_is_better', minimum: 0 }, pass: lowerIsBetter20Pct, }, + { + id: 'claude-dedicated-tools-vs-bash-opus-5-5', + overlayPath: 'model-overlays/claude.md', + model: 'claude-opus-5-5', + trials: 10, + concurrency: 3, + direction: 'lower_is_better', + maxTurns: 15, + setupWorkspace: (dir) => { + fs.mkdirSync(path.join(dir, 'src'), { recursive: true }); + fs.writeFileSync(path.join(dir, 'src', 'index.ts'), "export const x = 1;\n"); + fs.writeFileSync(path.join(dir, 'src', 'util.ts'), "export function util() { return 42; }\n"); + fs.writeFileSync(path.join(dir, 'src', 'types.ts'), "export type Foo = { a: number };\n"); + fs.writeFileSync(path.join(dir, 'src', 'config.ts'), "export const c = { n: 'demo' };\n"); + fs.writeFileSync(path.join(dir, 'src', 'api.ts'), "export async function fetchFoo() { return null; }\n"); + }, + userPrompt: + "List every TypeScript file under src/ and tell me what each exports. " + + "Return only a JSON object mapping each src/ path to an array of its exported symbol names. " + + "The final message is consumed directly by JSON.parse: return the exact JSON object, with no Markdown fences and no other prose. " + + "You may use any tools available.", + metric: bashToolCallCount, + metricName: 'bash_tool_calls', + verify: (r) => assertFinalJson(r, { 'src/index.ts': ['x'], 'src/util.ts': ['util'], 'src/types.ts': ['Foo'], 'src/config.ts': ['c'], 'src/api.ts': ['fetchFoo'] }), + gate: lowerIsBetter20PctOrZeroBaseline, + comparison: { direction: 'lower_is_better', minimum: 0 }, + pass: lowerIsBetter20Pct, + }, ]; diff --git a/test/helpers/overlay-case-policy.ts b/test/helpers/overlay-case-policy.ts index b29caff8b..6316c3c5d 100644 --- a/test/helpers/overlay-case-policy.ts +++ b/test/helpers/overlay-case-policy.ts @@ -23,4 +23,5 @@ export const OVERLAY_CASE_FILES: Record = { 'test/skill-e2e-overlay-harness-opus-4-7-effort-match-trivial.test.ts': 'opus-4-7-effort-match-trivial', 'test/skill-e2e-overlay-harness-opus-4-7-literal-interpretation.test.ts': 'opus-4-7-literal-interpretation', 'test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-sonnet.test.ts': 'claude-dedicated-tools-vs-bash-sonnet', + 'test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5.test.ts': 'claude-dedicated-tools-vs-bash-opus-5-5', }; diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index 8b178f20b..a7e5ccebe 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -1096,6 +1096,7 @@ export const E2E_TOUCHFILES: Record = { 'overlay-harness-opus-4-7-effort-match-trivial': ['model-overlays/**', 'test/fixtures/overlay-nudges.ts', 'test/helpers/agent-sdk-runner.ts', 'scripts/resolvers/model-overlay.ts', 'test/skill-e2e-overlay-harness-opus-4-7-effort-match-trivial.test.ts', 'test/helpers/overlay-measurement.ts', 'test/helpers/overlay-workspace.ts', 'test/helpers/overlay-attempt.ts', 'test/helpers/overlay-case.ts', 'test/helpers/overlay-case-policy.ts', 'test/helpers/overlay-lifecycle.ts', 'test/fixtures/overlay-admission-child.ts', 'test/helpers/e2e-gate.ts'], 'overlay-harness-opus-4-7-literal-interpretation': ['model-overlays/**', 'test/fixtures/overlay-nudges.ts', 'test/helpers/agent-sdk-runner.ts', 'scripts/resolvers/model-overlay.ts', 'test/skill-e2e-overlay-harness-opus-4-7-literal-interpretation.test.ts', 'test/helpers/overlay-measurement.ts', 'test/helpers/overlay-workspace.ts', 'test/helpers/overlay-attempt.ts', 'test/helpers/overlay-case.ts', 'test/helpers/overlay-case-policy.ts', 'test/helpers/overlay-lifecycle.ts', 'test/fixtures/overlay-admission-child.ts', 'test/helpers/e2e-gate.ts'], 'overlay-harness-claude-dedicated-tools-vs-bash-sonnet': ['model-overlays/**', 'test/fixtures/overlay-nudges.ts', 'test/helpers/agent-sdk-runner.ts', 'scripts/resolvers/model-overlay.ts', 'test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-sonnet.test.ts', 'test/helpers/overlay-measurement.ts', 'test/helpers/overlay-workspace.ts', 'test/helpers/overlay-attempt.ts', 'test/helpers/overlay-case.ts', 'test/helpers/overlay-case-policy.ts', 'test/helpers/overlay-lifecycle.ts', 'test/fixtures/overlay-admission-child.ts', 'test/helpers/e2e-gate.ts'], + 'overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5': ['model-overlays/**', 'test/fixtures/overlay-nudges.ts', 'test/helpers/agent-sdk-runner.ts', 'scripts/resolvers/model-overlay.ts', 'test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5.test.ts', 'test/helpers/overlay-measurement.ts', 'test/helpers/overlay-workspace.ts', 'test/helpers/overlay-attempt.ts', 'test/helpers/overlay-case.ts', 'test/helpers/overlay-case-policy.ts', 'test/helpers/overlay-lifecycle.ts', 'test/fixtures/overlay-admission-child.ts', 'test/helpers/e2e-gate.ts'], 'journey-negatives': [ @@ -1477,6 +1478,7 @@ export const E2E_TIERS: Record = { 'overlay-harness-opus-4-7-effort-match-trivial': 'periodic', 'overlay-harness-opus-4-7-literal-interpretation': 'periodic', 'overlay-harness-claude-dedicated-tools-vs-bash-sonnet': 'periodic', + 'overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5': 'periodic', 'journey-negatives': 'periodic', }; @@ -1803,6 +1805,7 @@ export const E2E_KINDS: Record = { 'overlay-harness-opus-4-7-effort-match-trivial': 'rule', 'overlay-harness-opus-4-7-literal-interpretation': 'rule', 'overlay-harness-claude-dedicated-tools-vs-bash-sonnet': 'rule', + 'overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5': 'rule', 'journey-negatives': 'rule', 'review/SKILL.md workflow': 'judge', 'setup-browser-cookies/SKILL.md workflow': 'judge', diff --git a/test/overlay-measurement.test.ts b/test/overlay-measurement.test.ts index 2f18da844..c7e875e83 100644 --- a/test/overlay-measurement.test.ts +++ b/test/overlay-measurement.test.ts @@ -255,11 +255,12 @@ describe('literal fixture task correctness', () => { expect(() => assertReadOnlyWorkspace(before, snapshotWorkspace(dir))).toThrow('b'); })); test('registry retires absent-nudge fanout cases and preserves remaining model/trial budgets', () => { - expect(OVERLAY_FIXTURES).toHaveLength(4); + expect(OVERLAY_FIXTURES).toHaveLength(5); expect(OVERLAY_FIXTURES.some((f) => f.id.includes('fanout') || f.comparison?.unsupportedHypothesis)).toBe(false); expect(OVERLAY_FIXTURES.every((f) => f.trials === 10 && f.comparison)).toBe(true); expect(OVERLAY_FIXTURES.filter((f) => f.model === 'claude-opus-4-7')).toHaveLength(3); expect(OVERLAY_FIXTURES.filter((f) => f.model === 'claude-sonnet-4-6')).toHaveLength(1); + expect(OVERLAY_FIXTURES.filter((f) => f.model === 'claude-opus-5-5')).toHaveLength(1); }); }); diff --git a/test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5.test.ts b/test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5.test.ts new file mode 100644 index 000000000..4c4240560 --- /dev/null +++ b/test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5.test.ts @@ -0,0 +1,6 @@ +import { describeE2ETier } from './helpers/e2e-gate'; +import { registerOverlayCase } from './helpers/overlay-case'; + +describeE2ETier('periodic')('overlay behavior contract v2 (SDK)', () => { + registerOverlayCase('claude-dedicated-tools-vs-bash-opus-5-5'); +});