diff --git a/scripts/paid-test-durations.json b/scripts/paid-test-durations.json index 845d718cc..834bab79a 100644 --- a/scripts/paid-test-durations.json +++ b/scripts/paid-test-durations.json @@ -148,6 +148,7 @@ "test/skill-e2e-office-hours.test.ts": 136970, "test/skill-e2e-outside-plan-disabled.test.ts": 53292, "test/skill-e2e-outside-voice.test.ts": 1000, + "test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5.test.ts": 85282, "test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-sonnet.test.ts": 85282, "test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash.test.ts": 80465, "test/skill-e2e-overlay-harness-opus-4-7-effort-match-trivial.test.ts": 49282, diff --git a/test/eng-finding-retry-budget.test.ts b/test/eng-finding-retry-budget.test.ts index afca16aaa..1d5ab48f1 100644 --- a/test/eng-finding-retry-budget.test.ts +++ b/test/eng-finding-retry-budget.test.ts @@ -133,13 +133,13 @@ test('live periodic census fits the declared CI wall including setup', () => { const plannedFiles = new Set(m.entries.filter(e => e.status === 'planned').map(e => shardFile(e.file))); expect(plannedFiles).toEqual(new Set(selectPaidTestFiles(collectPaidTestFiles(), 'periodic').selected)); const overlays = m.entries.filter(e => e.status === 'planned' && e.slice === m.sliceCount); - expect(overlays).toHaveLength(4); + expect(overlays).toHaveLength(5); expect(overlays.every(e => isOverlayTestFile(e.file))).toBe(true); }); test('registered allocation is deterministic and preserves every discovered file', () => { const files = collectPaidTestFiles(); - expect(files).toHaveLength(106); + expect(files).toHaveLength(107); expect(files).toContain('test/skill-e2e-ship-skip.test.ts'); const m = livePlan(files); expect(livePlan([...files].reverse())).toEqual(m); @@ -192,7 +192,7 @@ test('current detach supervision covers the live-census floor', () => { expect(floorFor('gate')).toBe(21_725); expect(gateTimeout).toBe(49_320); expect(gateTimeout).toBeGreaterThanOrEqual(floorFor('gate')); - expect(floorFor('periodic')).toBe(35_711); + expect(floorFor('periodic')).toBe(35_742); expect(periodicTimeout).toBeGreaterThanOrEqual(floorFor('periodic')); }); diff --git a/test/fixtures/overlay-nudges.ts b/test/fixtures/overlay-nudges.ts index 0621e3fb4..0b499d785 100644 --- a/test/fixtures/overlay-nudges.ts +++ b/test/fixtures/overlay-nudges.ts @@ -325,6 +325,37 @@ export const OVERLAY_FIXTURES: OverlayFixture[] = [ pass: lowerIsBetter20Pct, }, + // Opus 5.5 measures the generic claude overlay (resolveModel maps + // claude-opus-5-5 to `claude`); it adds no Opus 5.5 nudges. + { + id: 'claude-dedicated-tools-vs-bash-opus-5-5', + overlayPath: 'model-overlays/claude.md', + model: 'claude-opus-5-5', + trials: 10, + concurrency: 3, + direction: 'lower_is_better', + maxTurns: 15, + setupWorkspace: (dir) => { + fs.mkdirSync(path.join(dir, 'src'), { recursive: true }); + fs.writeFileSync(path.join(dir, 'src', 'index.ts'), "export const x = 1;\n"); + fs.writeFileSync(path.join(dir, 'src', 'util.ts'), "export function util() { return 42; }\n"); + fs.writeFileSync(path.join(dir, 'src', 'types.ts'), "export type Foo = { a: number };\n"); + fs.writeFileSync(path.join(dir, 'src', 'config.ts'), "export const c = { n: 'demo' };\n"); + fs.writeFileSync(path.join(dir, 'src', 'api.ts'), "export async function fetchFoo() { return null; }\n"); + }, + userPrompt: + "List every TypeScript file under src/ and tell me what each exports. " + + "Return only a JSON object mapping each src/ path to an array of its exported symbol names. " + + "The final message is consumed directly by JSON.parse: return the exact JSON object, with no Markdown fences and no other prose. " + + "You may use any tools available.", + metric: bashToolCallCount, + metricName: 'bash_tool_calls', + verify: (r) => assertFinalJson(r, { 'src/index.ts': ['x'], 'src/util.ts': ['util'], 'src/types.ts': ['Foo'], 'src/config.ts': ['c'], 'src/api.ts': ['fetchFoo'] }), + gate: lowerIsBetter20PctOrZeroBaseline, + comparison: { direction: 'lower_is_better', minimum: 0 }, + pass: lowerIsBetter20Pct, + }, + ]; // Validate at module load so a broken fixture fails fast at test startup, diff --git a/test/helpers/overlay-case-policy.ts b/test/helpers/overlay-case-policy.ts index b29caff8b..6316c3c5d 100644 --- a/test/helpers/overlay-case-policy.ts +++ b/test/helpers/overlay-case-policy.ts @@ -23,4 +23,5 @@ export const OVERLAY_CASE_FILES: Record = { 'test/skill-e2e-overlay-harness-opus-4-7-effort-match-trivial.test.ts': 'opus-4-7-effort-match-trivial', 'test/skill-e2e-overlay-harness-opus-4-7-literal-interpretation.test.ts': 'opus-4-7-literal-interpretation', 'test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-sonnet.test.ts': 'claude-dedicated-tools-vs-bash-sonnet', + 'test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5.test.ts': 'claude-dedicated-tools-vs-bash-opus-5-5', }; diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index d1c3d3ac7..258cc3669 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -1063,6 +1063,7 @@ export const E2E_TOUCHFILES: Record = { 'overlay-harness-opus-4-7-effort-match-trivial': ['model-overlays/**', 'test/fixtures/overlay-nudges.ts', 'test/helpers/agent-sdk-runner.ts', 'scripts/resolvers/model-overlay.ts', 'test/skill-e2e-overlay-harness-opus-4-7-effort-match-trivial.test.ts', 'test/helpers/overlay-measurement.ts', 'test/helpers/overlay-workspace.ts', 'test/helpers/overlay-attempt.ts', 'test/helpers/overlay-case.ts', 'test/helpers/overlay-case-policy.ts', 'test/helpers/overlay-lifecycle.ts', 'test/fixtures/overlay-admission-child.ts', 'test/helpers/e2e-gate.ts'], 'overlay-harness-opus-4-7-literal-interpretation': ['model-overlays/**', 'test/fixtures/overlay-nudges.ts', 'test/helpers/agent-sdk-runner.ts', 'scripts/resolvers/model-overlay.ts', 'test/skill-e2e-overlay-harness-opus-4-7-literal-interpretation.test.ts', 'test/helpers/overlay-measurement.ts', 'test/helpers/overlay-workspace.ts', 'test/helpers/overlay-attempt.ts', 'test/helpers/overlay-case.ts', 'test/helpers/overlay-case-policy.ts', 'test/helpers/overlay-lifecycle.ts', 'test/fixtures/overlay-admission-child.ts', 'test/helpers/e2e-gate.ts'], 'overlay-harness-claude-dedicated-tools-vs-bash-sonnet': ['model-overlays/**', 'test/fixtures/overlay-nudges.ts', 'test/helpers/agent-sdk-runner.ts', 'scripts/resolvers/model-overlay.ts', 'test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-sonnet.test.ts', 'test/helpers/overlay-measurement.ts', 'test/helpers/overlay-workspace.ts', 'test/helpers/overlay-attempt.ts', 'test/helpers/overlay-case.ts', 'test/helpers/overlay-case-policy.ts', 'test/helpers/overlay-lifecycle.ts', 'test/fixtures/overlay-admission-child.ts', 'test/helpers/e2e-gate.ts'], + 'overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5': ['model-overlays/**', 'test/fixtures/overlay-nudges.ts', 'test/helpers/agent-sdk-runner.ts', 'scripts/resolvers/model-overlay.ts', 'test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5.test.ts', 'test/helpers/overlay-measurement.ts', 'test/helpers/overlay-workspace.ts', 'test/helpers/overlay-attempt.ts', 'test/helpers/overlay-case.ts', 'test/helpers/overlay-case-policy.ts', 'test/helpers/overlay-lifecycle.ts', 'test/fixtures/overlay-admission-child.ts', 'test/helpers/e2e-gate.ts'], 'journey-negatives': [ @@ -1434,6 +1435,7 @@ export const E2E_TIERS: Record = { 'overlay-harness-opus-4-7-effort-match-trivial': 'periodic', 'overlay-harness-opus-4-7-literal-interpretation': 'periodic', 'overlay-harness-claude-dedicated-tools-vs-bash-sonnet': 'periodic', + 'overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5': 'periodic', 'journey-negatives': 'periodic', }; @@ -1752,6 +1754,7 @@ export const E2E_KINDS: Record = { 'overlay-harness-opus-4-7-effort-match-trivial': 'rule', 'overlay-harness-opus-4-7-literal-interpretation': 'rule', 'overlay-harness-claude-dedicated-tools-vs-bash-sonnet': 'rule', + 'overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5': 'rule', 'journey-negatives': 'rule', 'review/SKILL.md workflow': 'judge', 'setup-browser-cookies/SKILL.md workflow': 'judge', diff --git a/test/overlay-lifecycle.test.ts b/test/overlay-lifecycle.test.ts index a14d76dac..b27124611 100644 --- a/test/overlay-lifecycle.test.ts +++ b/test/overlay-lifecycle.test.ts @@ -27,8 +27,8 @@ test('every fixture owns one paid wrapper; public models/trials/concurrency/turn } expect(isPaidTestFile('test/overlay-lifecycle.test.ts')).toBe(false); expect(OVERLAY_FIXTURES.every(f => f.trials === 10 && f.concurrency === 3)).toBe(true); - expect(OVERLAY_FIXTURES.map(f => f.maxTurns ?? 5)).toEqual([15,8,15,15]); - expect(OVERLAY_FIXTURES.map(f => f.model)).toEqual([...Array(3).fill('claude-opus-4-7'), 'claude-sonnet-4-6']); + expect(OVERLAY_FIXTURES.map(f => f.maxTurns ?? 5)).toEqual([15,8,15,15,15]); + expect(OVERLAY_FIXTURES.map(f => f.model)).toEqual([...Array(3).fill('claude-opus-4-7'), 'claude-sonnet-4-6', 'claude-opus-5-5']); expect([OVERLAY_CASE_WORK_MS, OVERLAY_RECORD_GRACE_MS, OVERLAY_CASE_OUTER_MS, OVERLAY_MIN_FILE_WALL_MS]).toEqual([1_800_000, 5_000, 1_810_000, 1_830_000]); }); diff --git a/test/overlay-measurement.test.ts b/test/overlay-measurement.test.ts index 2f18da844..c7e875e83 100644 --- a/test/overlay-measurement.test.ts +++ b/test/overlay-measurement.test.ts @@ -255,11 +255,12 @@ describe('literal fixture task correctness', () => { expect(() => assertReadOnlyWorkspace(before, snapshotWorkspace(dir))).toThrow('b'); })); test('registry retires absent-nudge fanout cases and preserves remaining model/trial budgets', () => { - expect(OVERLAY_FIXTURES).toHaveLength(4); + expect(OVERLAY_FIXTURES).toHaveLength(5); expect(OVERLAY_FIXTURES.some((f) => f.id.includes('fanout') || f.comparison?.unsupportedHypothesis)).toBe(false); expect(OVERLAY_FIXTURES.every((f) => f.trials === 10 && f.comparison)).toBe(true); expect(OVERLAY_FIXTURES.filter((f) => f.model === 'claude-opus-4-7')).toHaveLength(3); expect(OVERLAY_FIXTURES.filter((f) => f.model === 'claude-sonnet-4-6')).toHaveLength(1); + expect(OVERLAY_FIXTURES.filter((f) => f.model === 'claude-opus-5-5')).toHaveLength(1); }); }); diff --git a/test/paid-overlay-scheduling.test.ts b/test/paid-overlay-scheduling.test.ts index 7b08b466c..8fef216d0 100644 --- a/test/paid-overlay-scheduling.test.ts +++ b/test/paid-overlay-scheduling.test.ts @@ -52,7 +52,7 @@ describe('overlay file policy', () => { }); test('only the exact wrapper family gets one attempt and the extra process grace', () => { - expect(overlayFiles).toHaveLength(4); + expect(overlayFiles).toHaveLength(5); expect(OVERLAY_MAX_ACTIVE_SHARDS).toBe(1); expect(OVERLAY_MIN_FILE_WALL_MS).toBe(1_830_000); for (const file of overlayFiles) { @@ -125,16 +125,16 @@ describe('overlay file policy', () => { }); describe('overlay manifest affinity and CI capacity', () => { - test('actual lifecycle wrapper guards include all four in periodic and exclude all four from gate', () => { + test('actual lifecycle wrapper guards include all five in periodic and exclude all five from gate', () => { for (const tier of ['periodic', 'gate'] as const) { const manifest = buildRunManifest({ tier, sliceCount: 6, evalsAll: true, env: { EVALS_ALL: '1' } }); const entries = manifest.entries.filter(entry => isOverlayTestFile(entry.file)); - expect(entries).toHaveLength(4); + expect(entries).toHaveLength(5); expect(entries.every(entry => entry.status === (tier === 'periodic' ? 'planned' : 'excluded'))).toBe(true); } }); - test('93 files retain every case, reserve slice six, and fit 330 minutes with actual family walls', () => { + test('94 files retain every case, reserve slice six, and fit 330 minutes with actual family walls', () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'overlay-affinity-')); const normalFiles = Array.from({ length: 89 }, (_, i) => `test/skill-e2e-normal-${i.toString().padStart(2, '0')}.test.ts`); const discovered = [...normalFiles, ...overlayFiles]; @@ -146,11 +146,11 @@ describe('overlay manifest affinity and CI capacity', () => { } const opts = { tier: 'periodic' as const, sliceCount: 6, evalsAll: true, discovered, rootDir: dir, env: { EVALS_ALL: '1' } }; const manifest = buildRunManifest(opts); - expect(manifest.entries).toHaveLength(93); - expect(new Set(manifest.entries.map(e => e.file)).size).toBe(93); + expect(manifest.entries).toHaveLength(94); + expect(new Set(manifest.entries.map(e => e.file)).size).toBe(94); expect(manifest.entries.every(e => e.status === 'planned')).toBe(true); const counts = [1, 2, 3, 4, 5, 6].map(slice => manifest.entries.filter(e => e.slice === slice).length); - expect(counts).toEqual([18, 18, 18, 18, 17, 4]); + expect(counts).toEqual([18, 18, 18, 18, 17, 5]); expect(manifest.entries.filter(e => e.slice === 6).map(e => e.file).sort()).toEqual([...overlayFiles].sort()); expect(buildRunManifest({ ...opts, discovered: [...discovered].reverse() })).toEqual(manifest); expect(parseRunManifest(JSON.stringify(manifest))).toEqual(manifest); @@ -175,8 +175,8 @@ describe('overlay manifest affinity and CI capacity', () => { const overlayMinutes = Math.ceil(overlayFiles.length / OVERLAY_MAX_ACTIVE_SHARDS) * Math.max(...overlayFiles.map(file => resolvePaidShardTimeoutMs([file]))) / 60_000; expect(normalMinutes).toBe(270); - expect(overlayMinutes).toBe(122); - // The CI budget plan (what the workflow runs) keeps the four overlays + expect(overlayMinutes).toBe(152.5); + // The CI budget plan (what the workflow runs) keeps the five overlays // in one final one-at-a-time slice and its job cap covers them. const budget = buildRunManifest({ ...opts, sliceCount: undefined, sliceBudgetMs: 540_000, jobs }); const lastSlice = budget.entries.filter(e => e.slice === budget.sliceCount).map(e => e.file).sort(); diff --git a/test/paid-retry-supervision.test.ts b/test/paid-retry-supervision.test.ts index e5cec82d4..f8866e532 100644 --- a/test/paid-retry-supervision.test.ts +++ b/test/paid-retry-supervision.test.ts @@ -229,8 +229,8 @@ test('detached PR fallback and release commands cover their actual default worke )) / 1000 * 1.05)); } const detachedReleaseWall = Number(scripts['eval:bg:release'].match(/--timeout (\d+)/)?.[1]) * 1000; - expect(releaseFloors).toEqual([21_725, 35_711]); - expect(releaseFloors.reduce((total, floor) => total + floor, 0)).toBe(57_436); + expect(releaseFloors).toEqual([21_725, 35_742]); + expect(releaseFloors.reduce((total, floor) => total + floor, 0)).toBe(57_467); expect(detachedReleaseWall).toBe(116_700_000); expect(detachedReleaseWall).toBeGreaterThanOrEqual(releaseWall + 120_000); }); diff --git a/test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5.test.ts b/test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5.test.ts new file mode 100644 index 000000000..4c4240560 --- /dev/null +++ b/test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5.test.ts @@ -0,0 +1,6 @@ +import { describeE2ETier } from './helpers/e2e-gate'; +import { registerOverlayCase } from './helpers/overlay-case'; + +describeE2ETier('periodic')('overlay behavior contract v2 (SDK)', () => { + registerOverlayCase('claude-dedicated-tools-vs-bash-opus-5-5'); +});