W4 T6: register Opus 5.5 dedicated-tools overlay case (periodic) with census pins

scripts/paid-test-durations.json

@@ -148,6 +148,7 @@
       "test/skill-e2e-office-hours.test.ts": 136970,
       "test/skill-e2e-outside-plan-disabled.test.ts": 53292,
       "test/skill-e2e-outside-voice.test.ts": 1000,
+      "test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5.test.ts": 85282,
       "test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-sonnet.test.ts": 85282,
       "test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash.test.ts": 80465,
       "test/skill-e2e-overlay-harness-opus-4-7-effort-match-trivial.test.ts": 49282,

test/eng-finding-retry-budget.test.ts

@@ -133,13 +133,13 @@ test('live periodic census fits the declared CI wall including setup', () => {
   const plannedFiles = new Set(m.entries.filter(e => e.status === 'planned').map(e => shardFile(e.file)));
   expect(plannedFiles).toEqual(new Set(selectPaidTestFiles(collectPaidTestFiles(), 'periodic').selected));
   const overlays = m.entries.filter(e => e.status === 'planned' && e.slice === m.sliceCount);
-  expect(overlays).toHaveLength(4);
+  expect(overlays).toHaveLength(5);
   expect(overlays.every(e => isOverlayTestFile(e.file))).toBe(true);
 });
 
 test('registered allocation is deterministic and preserves every discovered file', () => {
   const files = collectPaidTestFiles();
-  expect(files).toHaveLength(106);
+  expect(files).toHaveLength(107);
   expect(files).toContain('test/skill-e2e-ship-skip.test.ts');
   const m = livePlan(files);
   expect(livePlan([...files].reverse())).toEqual(m);
@@ -192,7 +192,7 @@ test('current detach supervision covers the live-census floor', () => {
   expect(floorFor('gate')).toBe(21_725);
   expect(gateTimeout).toBe(49_320);
   expect(gateTimeout).toBeGreaterThanOrEqual(floorFor('gate'));
-  expect(floorFor('periodic')).toBe(35_711);
+  expect(floorFor('periodic')).toBe(35_742);
   expect(periodicTimeout).toBeGreaterThanOrEqual(floorFor('periodic'));
 });
 

test/fixtures/overlay-nudges.ts

@@ -325,6 +325,37 @@ export const OVERLAY_FIXTURES: OverlayFixture[] = [
     pass: lowerIsBetter20Pct,
   },
 
+  // Opus 5.5 measures the generic claude overlay (resolveModel maps
+  // claude-opus-5-5 to `claude`); it adds no Opus 5.5 nudges.
+  {
+    id: 'claude-dedicated-tools-vs-bash-opus-5-5',
+    overlayPath: 'model-overlays/claude.md',
+    model: 'claude-opus-5-5',
+    trials: 10,
+    concurrency: 3,
+    direction: 'lower_is_better',
+    maxTurns: 15,
+    setupWorkspace: (dir) => {
+      fs.mkdirSync(path.join(dir, 'src'), { recursive: true });
+      fs.writeFileSync(path.join(dir, 'src', 'index.ts'), "export const x = 1;\n");
+      fs.writeFileSync(path.join(dir, 'src', 'util.ts'), "export function util() { return 42; }\n");
+      fs.writeFileSync(path.join(dir, 'src', 'types.ts'), "export type Foo = { a: number };\n");
+      fs.writeFileSync(path.join(dir, 'src', 'config.ts'), "export const c = { n: 'demo' };\n");
+      fs.writeFileSync(path.join(dir, 'src', 'api.ts'), "export async function fetchFoo() { return null; }\n");
+    },
+    userPrompt:
+      "List every TypeScript file under src/ and tell me what each exports. " +
+      "Return only a JSON object mapping each src/ path to an array of its exported symbol names. " +
+      "The final message is consumed directly by JSON.parse: return the exact JSON object, with no Markdown fences and no other prose. " +
+      "You may use any tools available.",
+    metric: bashToolCallCount,
+    metricName: 'bash_tool_calls',
+    verify: (r) => assertFinalJson(r, { 'src/index.ts': ['x'], 'src/util.ts': ['util'], 'src/types.ts': ['Foo'], 'src/config.ts': ['c'], 'src/api.ts': ['fetchFoo'] }),
+    gate: lowerIsBetter20PctOrZeroBaseline,
+    comparison: { direction: 'lower_is_better', minimum: 0 },
+    pass: lowerIsBetter20Pct,
+  },
+
 ];
 
 // Validate at module load so a broken fixture fails fast at test startup,

test/helpers/overlay-case-policy.ts

@@ -23,4 +23,5 @@ export const OVERLAY_CASE_FILES: Record<string, string> = {
   'test/skill-e2e-overlay-harness-opus-4-7-effort-match-trivial.test.ts': 'opus-4-7-effort-match-trivial',
   'test/skill-e2e-overlay-harness-opus-4-7-literal-interpretation.test.ts': 'opus-4-7-literal-interpretation',
   'test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-sonnet.test.ts': 'claude-dedicated-tools-vs-bash-sonnet',
+  'test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5.test.ts': 'claude-dedicated-tools-vs-bash-opus-5-5',
 };

test/helpers/touchfiles-data.ts

@@ -1063,6 +1063,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
   'overlay-harness-opus-4-7-effort-match-trivial': ['model-overlays/**', 'test/fixtures/overlay-nudges.ts', 'test/helpers/agent-sdk-runner.ts',  'scripts/resolvers/model-overlay.ts', 'test/skill-e2e-overlay-harness-opus-4-7-effort-match-trivial.test.ts', 'test/helpers/overlay-measurement.ts', 'test/helpers/overlay-workspace.ts', 'test/helpers/overlay-attempt.ts',  'test/helpers/overlay-case.ts', 'test/helpers/overlay-case-policy.ts', 'test/helpers/overlay-lifecycle.ts',     'test/fixtures/overlay-admission-child.ts', 'test/helpers/e2e-gate.ts'],
   'overlay-harness-opus-4-7-literal-interpretation': ['model-overlays/**', 'test/fixtures/overlay-nudges.ts', 'test/helpers/agent-sdk-runner.ts',  'scripts/resolvers/model-overlay.ts', 'test/skill-e2e-overlay-harness-opus-4-7-literal-interpretation.test.ts', 'test/helpers/overlay-measurement.ts', 'test/helpers/overlay-workspace.ts', 'test/helpers/overlay-attempt.ts',  'test/helpers/overlay-case.ts', 'test/helpers/overlay-case-policy.ts', 'test/helpers/overlay-lifecycle.ts',     'test/fixtures/overlay-admission-child.ts', 'test/helpers/e2e-gate.ts'],
   'overlay-harness-claude-dedicated-tools-vs-bash-sonnet': ['model-overlays/**', 'test/fixtures/overlay-nudges.ts', 'test/helpers/agent-sdk-runner.ts',  'scripts/resolvers/model-overlay.ts', 'test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-sonnet.test.ts', 'test/helpers/overlay-measurement.ts', 'test/helpers/overlay-workspace.ts', 'test/helpers/overlay-attempt.ts',  'test/helpers/overlay-case.ts', 'test/helpers/overlay-case-policy.ts', 'test/helpers/overlay-lifecycle.ts',     'test/fixtures/overlay-admission-child.ts', 'test/helpers/e2e-gate.ts'],
+  'overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5': ['model-overlays/**', 'test/fixtures/overlay-nudges.ts', 'test/helpers/agent-sdk-runner.ts',  'scripts/resolvers/model-overlay.ts', 'test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5.test.ts', 'test/helpers/overlay-measurement.ts', 'test/helpers/overlay-workspace.ts', 'test/helpers/overlay-attempt.ts',  'test/helpers/overlay-case.ts', 'test/helpers/overlay-case-policy.ts', 'test/helpers/overlay-lifecycle.ts',     'test/fixtures/overlay-admission-child.ts', 'test/helpers/e2e-gate.ts'],
   'journey-negatives':      [
     
 
@@ -1434,6 +1435,7 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic' | 'marathon'> = {
   'overlay-harness-opus-4-7-effort-match-trivial': 'periodic',
   'overlay-harness-opus-4-7-literal-interpretation': 'periodic',
   'overlay-harness-claude-dedicated-tools-vs-bash-sonnet': 'periodic',
+  'overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5': 'periodic',
   'journey-negatives': 'periodic',
 };
 
@@ -1752,6 +1754,7 @@ export const E2E_KINDS: Record<string, 'rule' | 'behavior' | 'judge'> = {
   'overlay-harness-opus-4-7-effort-match-trivial': 'rule',
   'overlay-harness-opus-4-7-literal-interpretation': 'rule',
   'overlay-harness-claude-dedicated-tools-vs-bash-sonnet': 'rule',
+  'overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5': 'rule',
   'journey-negatives': 'rule',
   'review/SKILL.md workflow': 'judge',
   'setup-browser-cookies/SKILL.md workflow': 'judge',

test/overlay-lifecycle.test.ts

@@ -27,8 +27,8 @@ test('every fixture owns one paid wrapper; public models/trials/concurrency/turn
   }
   expect(isPaidTestFile('test/overlay-lifecycle.test.ts')).toBe(false);
   expect(OVERLAY_FIXTURES.every(f => f.trials === 10 && f.concurrency === 3)).toBe(true);
-  expect(OVERLAY_FIXTURES.map(f => f.maxTurns ?? 5)).toEqual([15,8,15,15]);
-  expect(OVERLAY_FIXTURES.map(f => f.model)).toEqual([...Array(3).fill('claude-opus-4-7'), 'claude-sonnet-4-6']);
+  expect(OVERLAY_FIXTURES.map(f => f.maxTurns ?? 5)).toEqual([15,8,15,15,15]);
+  expect(OVERLAY_FIXTURES.map(f => f.model)).toEqual([...Array(3).fill('claude-opus-4-7'), 'claude-sonnet-4-6', 'claude-opus-5-5']);
   expect([OVERLAY_CASE_WORK_MS, OVERLAY_RECORD_GRACE_MS, OVERLAY_CASE_OUTER_MS, OVERLAY_MIN_FILE_WALL_MS]).toEqual([1_800_000, 5_000, 1_810_000, 1_830_000]);
 });
 

test/overlay-measurement.test.ts

@@ -255,11 +255,12 @@ describe('literal fixture task correctness', () => {
     expect(() => assertReadOnlyWorkspace(before, snapshotWorkspace(dir))).toThrow('b');
   }));
   test('registry retires absent-nudge fanout cases and preserves remaining model/trial budgets', () => {
-    expect(OVERLAY_FIXTURES).toHaveLength(4);
+    expect(OVERLAY_FIXTURES).toHaveLength(5);
     expect(OVERLAY_FIXTURES.some((f) => f.id.includes('fanout') || f.comparison?.unsupportedHypothesis)).toBe(false);
     expect(OVERLAY_FIXTURES.every((f) => f.trials === 10 && f.comparison)).toBe(true);
     expect(OVERLAY_FIXTURES.filter((f) => f.model === 'claude-opus-4-7')).toHaveLength(3);
     expect(OVERLAY_FIXTURES.filter((f) => f.model === 'claude-sonnet-4-6')).toHaveLength(1);
+    expect(OVERLAY_FIXTURES.filter((f) => f.model === 'claude-opus-5-5')).toHaveLength(1);
   });
 });
 

test/paid-overlay-scheduling.test.ts

@@ -52,7 +52,7 @@ describe('overlay file policy', () => {
   });
 
   test('only the exact wrapper family gets one attempt and the extra process grace', () => {
-    expect(overlayFiles).toHaveLength(4);
+    expect(overlayFiles).toHaveLength(5);
     expect(OVERLAY_MAX_ACTIVE_SHARDS).toBe(1);
     expect(OVERLAY_MIN_FILE_WALL_MS).toBe(1_830_000);
     for (const file of overlayFiles) {
@@ -125,16 +125,16 @@ describe('overlay file policy', () => {
 });
 
 describe('overlay manifest affinity and CI capacity', () => {
-  test('actual lifecycle wrapper guards include all four in periodic and exclude all four from gate', () => {
+  test('actual lifecycle wrapper guards include all five in periodic and exclude all five from gate', () => {
     for (const tier of ['periodic', 'gate'] as const) {
       const manifest = buildRunManifest({ tier, sliceCount: 6, evalsAll: true, env: { EVALS_ALL: '1' } });
       const entries = manifest.entries.filter(entry => isOverlayTestFile(entry.file));
-      expect(entries).toHaveLength(4);
+      expect(entries).toHaveLength(5);
       expect(entries.every(entry => entry.status === (tier === 'periodic' ? 'planned' : 'excluded'))).toBe(true);
     }
   });
 
-  test('93 files retain every case, reserve slice six, and fit 330 minutes with actual family walls', () => {
+  test('94 files retain every case, reserve slice six, and fit 330 minutes with actual family walls', () => {
     const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'overlay-affinity-'));
     const normalFiles = Array.from({ length: 89 }, (_, i) => `test/skill-e2e-normal-${i.toString().padStart(2, '0')}.test.ts`);
     const discovered = [...normalFiles, ...overlayFiles];
@@ -146,11 +146,11 @@ describe('overlay manifest affinity and CI capacity', () => {
       }
       const opts = { tier: 'periodic' as const, sliceCount: 6, evalsAll: true, discovered, rootDir: dir, env: { EVALS_ALL: '1' } };
       const manifest = buildRunManifest(opts);
-      expect(manifest.entries).toHaveLength(93);
-      expect(new Set(manifest.entries.map(e => e.file)).size).toBe(93);
+      expect(manifest.entries).toHaveLength(94);
+      expect(new Set(manifest.entries.map(e => e.file)).size).toBe(94);
       expect(manifest.entries.every(e => e.status === 'planned')).toBe(true);
       const counts = [1, 2, 3, 4, 5, 6].map(slice => manifest.entries.filter(e => e.slice === slice).length);
-      expect(counts).toEqual([18, 18, 18, 18, 17, 4]);
+      expect(counts).toEqual([18, 18, 18, 18, 17, 5]);
       expect(manifest.entries.filter(e => e.slice === 6).map(e => e.file).sort()).toEqual([...overlayFiles].sort());
       expect(buildRunManifest({ ...opts, discovered: [...discovered].reverse() })).toEqual(manifest);
       expect(parseRunManifest(JSON.stringify(manifest))).toEqual(manifest);
@@ -175,8 +175,8 @@ describe('overlay manifest affinity and CI capacity', () => {
       const overlayMinutes = Math.ceil(overlayFiles.length / OVERLAY_MAX_ACTIVE_SHARDS)
         * Math.max(...overlayFiles.map(file => resolvePaidShardTimeoutMs([file]))) / 60_000;
       expect(normalMinutes).toBe(270);
-      expect(overlayMinutes).toBe(122);
-      // The CI budget plan (what the workflow runs) keeps the four overlays
+      expect(overlayMinutes).toBe(152.5);
+      // The CI budget plan (what the workflow runs) keeps the five overlays
       // in one final one-at-a-time slice and its job cap covers them.
       const budget = buildRunManifest({ ...opts, sliceCount: undefined, sliceBudgetMs: 540_000, jobs });
       const lastSlice = budget.entries.filter(e => e.slice === budget.sliceCount).map(e => e.file).sort();

test/paid-retry-supervision.test.ts

@@ -229,8 +229,8 @@ test('detached PR fallback and release commands cover their actual default worke
     )) / 1000 * 1.05));
   }
   const detachedReleaseWall = Number(scripts['eval:bg:release'].match(/--timeout (\d+)/)?.[1]) * 1000;
-  expect(releaseFloors).toEqual([21_725, 35_711]);
-  expect(releaseFloors.reduce((total, floor) => total + floor, 0)).toBe(57_436);
+  expect(releaseFloors).toEqual([21_725, 35_742]);
+  expect(releaseFloors.reduce((total, floor) => total + floor, 0)).toBe(57_467);
   expect(detachedReleaseWall).toBe(116_700_000);
   expect(detachedReleaseWall).toBeGreaterThanOrEqual(releaseWall + 120_000);
 });

test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5.test.ts

@@ -0,0 +1,6 @@
+import { describeE2ETier } from './helpers/e2e-gate';
+import { registerOverlayCase } from './helpers/overlay-case';
+
+describeE2ETier('periodic')('overlay behavior contract v2 (SDK)', () => {
+  registerOverlayCase('claude-dedicated-tools-vs-bash-opus-5-5');
+});