From d9749f7375e042d6ff0fb531294671f210fc7201 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sat, 30 May 2026 09:20:25 -0700 Subject: [PATCH] =?UTF-8?q?test(parity):=20sectioned-skill=20parity=20capa?= =?UTF-8?q?bility=20=E2=80=94=20guards=20the=20carve=20(T9)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Carved skills (skeleton + sections/*.md) need parity checks that see relocated content, or moving a phrase into a section reads as 'lost': - readSkillForParity(): union skeleton + all sections/*.md - checkSkillParity sectioned mode: content checks against the union; minBytes/ maxSizeRatio against union bytes (total behavior preserved); maxSkeletonBytes asserts the always-loaded skeleton actually shrank. Lowering minBytes to fit a small skeleton would otherwise make the size floor toothless [Codex #12]. Built + tested BEFORE the carve so ship's invariant can flip to sectioned in the same commit it lands. Monolith path byte-identical (verified: pre-existing investigate 1.053 ratio drift fails the same with this change stashed). 7 sectioned-parity tests + existing parity tests green. Co-Authored-By: Claude Opus 4.8 (1M context) --- test/helpers/parity-harness.ts | 107 ++++++++++++++++++++++++++------- test/parity-sectioned.test.ts | 88 +++++++++++++++++++++++++++ 2 files changed, 172 insertions(+), 23 deletions(-) create mode 100644 test/parity-sectioned.test.ts diff --git a/test/helpers/parity-harness.ts b/test/helpers/parity-harness.ts index 4071a6cae..fcde00859 100644 --- a/test/helpers/parity-harness.ts +++ b/test/helpers/parity-harness.ts @@ -33,6 +33,22 @@ export interface ParityInvariant { maxSizeRatio?: number; /** Minimum byte size (catches over-stripping cliffs). */ minBytes?: number; + /** + * Carved skill (v2 plan T9): the skill is a skeleton SKILL.md plus on-demand + * sections/*.md. When true: + * - mustContain / mustHaveHeadings run against skeleton + ALL sections unioned, + * so a phrase that moved into a section still counts (content preserved, just + * relocated — that's the whole point of the carve). + * - minBytes / maxSizeRatio run against the UNION bytes, not the skeleton alone + * (total behavior must not shrink; the win is what's no longer always-loaded, + * which the union size deliberately does NOT measure — maxSkeletonBytes does). + * - maxSkeletonBytes asserts the always-loaded skeleton actually shrank. + * Without this, lowering minBytes to fit a 65KB skeleton would make the size + * floor toothless (Codex outside-voice #12). + */ + sectioned?: boolean; + /** Max bytes for the always-loaded skeleton SKILL.md (carved skills only). */ + maxSkeletonBytes?: number; } export interface ParityCheckResult { @@ -41,6 +57,35 @@ export interface ParityCheckResult { failures: string[]; } +/** + * Read a skill's check text + sizes. For a carved skill, union the skeleton with + * every sections/*.md so relocated content still counts and the union size + * measures total preserved behavior; skeletonBytes is reported separately so the + * always-loaded shrink can be asserted. For a monolith, text == skeleton. + */ +export function readSkillForParity( + repoRoot: string, + skill: string, + sectioned: boolean, +): { text: string; unionBytes: number; skeletonBytes: number } { + const skeleton = fs.readFileSync(path.join(repoRoot, skill, 'SKILL.md'), 'utf-8'); + const skeletonBytes = Buffer.byteLength(skeleton, 'utf-8'); + if (!sectioned) return { text: skeleton, unionBytes: skeletonBytes, skeletonBytes }; + + let text = skeleton; + let unionBytes = skeletonBytes; + const sectionsDir = path.join(repoRoot, skill, 'sections'); + if (fs.existsSync(sectionsDir)) { + for (const f of fs.readdirSync(sectionsDir).sort()) { + if (!f.endsWith('.md')) continue; + const sec = fs.readFileSync(path.join(sectionsDir, f), 'utf-8'); + text += '\n' + sec; + unionBytes += Buffer.byteLength(sec, 'utf-8'); + } + } + return { text, unionBytes, skeletonBytes }; +} + export function checkSkillParity( invariant: ParityInvariant, current: SkillBaselineEntry, @@ -48,38 +93,54 @@ export function checkSkillParity( repoRoot: string, ): ParityCheckResult { const failures: string[] = []; + const needText = !!(invariant.mustContain?.length || invariant.mustHaveHeadings?.length); - // SIZE checks + // Resolve the text + size to check against. Carved skills union skeleton + + // sections; monoliths use the skeleton alone. Read on demand so size-only + // invariants don't pay for a file read they don't need (monolith path). + let checkText: string | null = null; + let checkBytes = current.skillMdBytes; + if (invariant.sectioned) { + try { + const r = readSkillForParity(repoRoot, invariant.skill, true); + checkText = r.text; + checkBytes = r.unionBytes; + if (invariant.maxSkeletonBytes !== undefined && r.skeletonBytes > invariant.maxSkeletonBytes) { + failures.push(`skeleton ${r.skeletonBytes} > maxSkeletonBytes ${invariant.maxSkeletonBytes}`); + } + } catch (err) { + failures.push(`cannot read carved skill ${invariant.skill}: ${(err as Error).message}`); + } + } else if (needText) { + try { + checkText = fs.readFileSync(path.join(repoRoot, invariant.skill, 'SKILL.md'), 'utf-8'); + } catch (err) { + failures.push(`cannot read ${path.join(repoRoot, invariant.skill, 'SKILL.md')}: ${(err as Error).message}`); + } + } + + // SIZE checks (union bytes for carved skills, skeleton bytes for monoliths) if (invariant.maxSizeRatio !== undefined && baseline) { - const ratio = current.skillMdBytes / baseline.skillMdBytes; + const ratio = checkBytes / baseline.skillMdBytes; if (ratio > invariant.maxSizeRatio) { failures.push(`size ratio ${ratio.toFixed(3)} > maxSizeRatio ${invariant.maxSizeRatio}`); } } - if (invariant.minBytes !== undefined && current.skillMdBytes < invariant.minBytes) { - failures.push(`size ${current.skillMdBytes} < minBytes ${invariant.minBytes}`); + if (invariant.minBytes !== undefined && checkBytes < invariant.minBytes) { + failures.push(`size ${checkBytes} < minBytes ${invariant.minBytes}`); } - // CONTENT checks (read live file for fresh content) - if (invariant.mustContain?.length || invariant.mustHaveHeadings?.length) { - const skillMdPath = path.join(repoRoot, invariant.skill, 'SKILL.md'); - let content: string | null = null; - try { - content = fs.readFileSync(skillMdPath, 'utf-8'); - } catch (err) { - failures.push(`cannot read ${skillMdPath}: ${(err as Error).message}`); - } - if (content) { - const lower = content.toLowerCase(); - for (const phrase of invariant.mustContain ?? []) { - if (!lower.includes(phrase.toLowerCase())) { - failures.push(`missing required phrase: "${phrase}"`); - } + // CONTENT checks + if (needText && checkText !== null) { + const lower = checkText.toLowerCase(); + for (const phrase of invariant.mustContain ?? []) { + if (!lower.includes(phrase.toLowerCase())) { + failures.push(`missing required phrase: "${phrase}"`); } - for (const heading of invariant.mustHaveHeadings ?? []) { - if (!content.includes(heading)) { - failures.push(`missing required heading: "${heading}"`); - } + } + for (const heading of invariant.mustHaveHeadings ?? []) { + if (!checkText.includes(heading)) { + failures.push(`missing required heading: "${heading}"`); } } } diff --git a/test/parity-sectioned.test.ts b/test/parity-sectioned.test.ts new file mode 100644 index 000000000..3b3cfab2e --- /dev/null +++ b/test/parity-sectioned.test.ts @@ -0,0 +1,88 @@ +/** + * Unit coverage for the sectioned-parity capability (v2 plan T9, guards the + * carve). Proves that a carved skill's relocated content still counts (union of + * skeleton + sections), the always-loaded skeleton shrink is asserted + * separately (maxSkeletonBytes), and size floors run against the union so they + * stay meaningful (Codex outside-voice #12). Synthetic fixture — no ship carve + * needed to validate the logic. + */ + +import { describe, test, expect, afterAll } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { checkSkillParity, readSkillForParity, type ParityInvariant } from './helpers/parity-harness'; +import type { SkillBaselineEntry } from './helpers/capture-parity-baseline'; + +const root = fs.mkdtempSync(path.join(os.tmpdir(), 'parity-sectioned-')); +afterAll(() => { try { fs.rmSync(root, { recursive: true, force: true }); } catch { /* noop */ } }); + +// Carved "ship": a small skeleton + two sections holding the relocated prose. +fs.mkdirSync(path.join(root, 'ship', 'sections'), { recursive: true }); +fs.writeFileSync(path.join(root, 'ship', 'SKILL.md'), + '## Preamble\nskeleton body, decision tree, VERSION bump step calls the CLI.\n## When to invoke\n'); +fs.writeFileSync(path.join(root, 'ship', 'sections', 'changelog.md'), '# Changelog\nWrite the CHANGELOG entry here.\n'); +fs.writeFileSync(path.join(root, 'ship', 'sections', 'review-army.md'), '# Review\nDispatch the pre-landing review army.\n'); + +// A monolith control skill. +fs.mkdirSync(path.join(root, 'mono'), { recursive: true }); +fs.writeFileSync(path.join(root, 'mono', 'SKILL.md'), '## Preamble\nVERSION CHANGELOG review all inline here.\n'); + +const skeletonBytes = Buffer.byteLength(fs.readFileSync(path.join(root, 'ship', 'SKILL.md'), 'utf-8'), 'utf-8'); +const unionBytes = readSkillForParity(root, 'ship', true).unionBytes; +const baseline: SkillBaselineEntry = { skillMdBytes: unionBytes } as SkillBaselineEntry; + +describe('readSkillForParity', () => { + test('unions skeleton + sections for carved skills', () => { + const r = readSkillForParity(root, 'ship', true); + expect(r.text).toContain('CHANGELOG'); // from changelog.md + expect(r.text).toContain('review army'); // from review-army.md + expect(r.skeletonBytes).toBe(skeletonBytes); + expect(r.unionBytes).toBeGreaterThan(r.skeletonBytes); + }); + test('monolith text == skeleton, union == skeleton', () => { + const r = readSkillForParity(root, 'mono', false); + expect(r.unionBytes).toBe(r.skeletonBytes); + }); +}); + +describe('checkSkillParity (sectioned)', () => { + test('finds phrases that moved into sections (union content check)', () => { + const inv: ParityInvariant = { + skill: 'ship', sectioned: true, + mustContain: ['VERSION', 'CHANGELOG', 'review army'], + mustHaveHeadings: ['## Preamble', '## When to invoke'], + }; + const res = checkSkillParity(inv, { skillMdBytes: skeletonBytes } as SkillBaselineEntry, baseline, root); + expect(res.passed).toBe(true); + }); + + test('maxSkeletonBytes catches a skeleton that did not shrink', () => { + const inv: ParityInvariant = { skill: 'ship', sectioned: true, maxSkeletonBytes: 10 }; + const res = checkSkillParity(inv, { skillMdBytes: skeletonBytes } as SkillBaselineEntry, baseline, root); + expect(res.passed).toBe(false); + expect(res.failures.join()).toContain('maxSkeletonBytes'); + }); + + test('minBytes runs against the union, not the skeleton (content preserved)', () => { + // A floor between skeletonBytes and unionBytes must PASS for sectioned skills, + // because the union (total behavior) is what must not shrink. + const floor = Math.floor((skeletonBytes + unionBytes) / 2); + const inv: ParityInvariant = { skill: 'ship', sectioned: true, minBytes: floor }; + const res = checkSkillParity(inv, { skillMdBytes: skeletonBytes } as SkillBaselineEntry, baseline, root); + expect(res.passed).toBe(true); + }); + + test('flags a phrase that truly went missing', () => { + const inv: ParityInvariant = { skill: 'ship', sectioned: true, mustContain: ['this-phrase-is-not-anywhere'] }; + const res = checkSkillParity(inv, { skillMdBytes: skeletonBytes } as SkillBaselineEntry, baseline, root); + expect(res.passed).toBe(false); + expect(res.failures.join()).toContain('missing required phrase'); + }); + + test('maxSizeRatio uses union bytes vs baseline (carve preserves ~total size)', () => { + const inv: ParityInvariant = { skill: 'ship', sectioned: true, maxSizeRatio: 1.05 }; + const res = checkSkillParity(inv, { skillMdBytes: skeletonBytes } as SkillBaselineEntry, baseline, root); + expect(res.passed).toBe(true); // union == baseline here → ratio 1.0 + }); +});