test(parity): sectioned-skill parity capability — guards the carve (T9)

Carved skills (skeleton + sections/*.md) need parity checks that see relocated
content, or moving a phrase into a section reads as 'lost':
- readSkillForParity(): union skeleton + all sections/*.md
- checkSkillParity sectioned mode: content checks against the union; minBytes/
  maxSizeRatio against union bytes (total behavior preserved); maxSkeletonBytes
  asserts the always-loaded skeleton actually shrank. Lowering minBytes to fit a
  small skeleton would otherwise make the size floor toothless [Codex #12].

Built + tested BEFORE the carve so ship's invariant can flip to sectioned in the
same commit it lands. Monolith path byte-identical (verified: pre-existing
investigate 1.053 ratio drift fails the same with this change stashed).

7 sectioned-parity tests + existing parity tests green.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-05-30 09:20:25 -07:00
parent 4c4d136001
commit d9749f7375
2 changed files with 172 additions and 23 deletions
+84 -23
View File
@@ -33,6 +33,22 @@ export interface ParityInvariant {
maxSizeRatio?: number;
/** Minimum byte size (catches over-stripping cliffs). */
minBytes?: number;
/**
* Carved skill (v2 plan T9): the skill is a skeleton SKILL.md plus on-demand
* sections/*.md. When true:
* - mustContain / mustHaveHeadings run against skeleton + ALL sections unioned,
* so a phrase that moved into a section still counts (content preserved, just
* relocated — that's the whole point of the carve).
* - minBytes / maxSizeRatio run against the UNION bytes, not the skeleton alone
* (total behavior must not shrink; the win is what's no longer always-loaded,
* which the union size deliberately does NOT measure — maxSkeletonBytes does).
* - maxSkeletonBytes asserts the always-loaded skeleton actually shrank.
* Without this, lowering minBytes to fit a 65KB skeleton would make the size
* floor toothless (Codex outside-voice #12).
*/
sectioned?: boolean;
/** Max bytes for the always-loaded skeleton SKILL.md (carved skills only). */
maxSkeletonBytes?: number;
}
export interface ParityCheckResult {
@@ -41,6 +57,35 @@ export interface ParityCheckResult {
failures: string[];
}
/**
* Read a skill's check text + sizes. For a carved skill, union the skeleton with
* every sections/*.md so relocated content still counts and the union size
* measures total preserved behavior; skeletonBytes is reported separately so the
* always-loaded shrink can be asserted. For a monolith, text == skeleton.
*/
export function readSkillForParity(
repoRoot: string,
skill: string,
sectioned: boolean,
): { text: string; unionBytes: number; skeletonBytes: number } {
const skeleton = fs.readFileSync(path.join(repoRoot, skill, 'SKILL.md'), 'utf-8');
const skeletonBytes = Buffer.byteLength(skeleton, 'utf-8');
if (!sectioned) return { text: skeleton, unionBytes: skeletonBytes, skeletonBytes };
let text = skeleton;
let unionBytes = skeletonBytes;
const sectionsDir = path.join(repoRoot, skill, 'sections');
if (fs.existsSync(sectionsDir)) {
for (const f of fs.readdirSync(sectionsDir).sort()) {
if (!f.endsWith('.md')) continue;
const sec = fs.readFileSync(path.join(sectionsDir, f), 'utf-8');
text += '\n' + sec;
unionBytes += Buffer.byteLength(sec, 'utf-8');
}
}
return { text, unionBytes, skeletonBytes };
}
export function checkSkillParity(
invariant: ParityInvariant,
current: SkillBaselineEntry,
@@ -48,38 +93,54 @@ export function checkSkillParity(
repoRoot: string,
): ParityCheckResult {
const failures: string[] = [];
const needText = !!(invariant.mustContain?.length || invariant.mustHaveHeadings?.length);
// SIZE checks
// Resolve the text + size to check against. Carved skills union skeleton +
// sections; monoliths use the skeleton alone. Read on demand so size-only
// invariants don't pay for a file read they don't need (monolith path).
let checkText: string | null = null;
let checkBytes = current.skillMdBytes;
if (invariant.sectioned) {
try {
const r = readSkillForParity(repoRoot, invariant.skill, true);
checkText = r.text;
checkBytes = r.unionBytes;
if (invariant.maxSkeletonBytes !== undefined && r.skeletonBytes > invariant.maxSkeletonBytes) {
failures.push(`skeleton ${r.skeletonBytes} > maxSkeletonBytes ${invariant.maxSkeletonBytes}`);
}
} catch (err) {
failures.push(`cannot read carved skill ${invariant.skill}: ${(err as Error).message}`);
}
} else if (needText) {
try {
checkText = fs.readFileSync(path.join(repoRoot, invariant.skill, 'SKILL.md'), 'utf-8');
} catch (err) {
failures.push(`cannot read ${path.join(repoRoot, invariant.skill, 'SKILL.md')}: ${(err as Error).message}`);
}
}
// SIZE checks (union bytes for carved skills, skeleton bytes for monoliths)
if (invariant.maxSizeRatio !== undefined && baseline) {
const ratio = current.skillMdBytes / baseline.skillMdBytes;
const ratio = checkBytes / baseline.skillMdBytes;
if (ratio > invariant.maxSizeRatio) {
failures.push(`size ratio ${ratio.toFixed(3)} > maxSizeRatio ${invariant.maxSizeRatio}`);
}
}
if (invariant.minBytes !== undefined && current.skillMdBytes < invariant.minBytes) {
failures.push(`size ${current.skillMdBytes} < minBytes ${invariant.minBytes}`);
if (invariant.minBytes !== undefined && checkBytes < invariant.minBytes) {
failures.push(`size ${checkBytes} < minBytes ${invariant.minBytes}`);
}
// CONTENT checks (read live file for fresh content)
if (invariant.mustContain?.length || invariant.mustHaveHeadings?.length) {
const skillMdPath = path.join(repoRoot, invariant.skill, 'SKILL.md');
let content: string | null = null;
try {
content = fs.readFileSync(skillMdPath, 'utf-8');
} catch (err) {
failures.push(`cannot read ${skillMdPath}: ${(err as Error).message}`);
}
if (content) {
const lower = content.toLowerCase();
for (const phrase of invariant.mustContain ?? []) {
if (!lower.includes(phrase.toLowerCase())) {
failures.push(`missing required phrase: "${phrase}"`);
}
// CONTENT checks
if (needText && checkText !== null) {
const lower = checkText.toLowerCase();
for (const phrase of invariant.mustContain ?? []) {
if (!lower.includes(phrase.toLowerCase())) {
failures.push(`missing required phrase: "${phrase}"`);
}
for (const heading of invariant.mustHaveHeadings ?? []) {
if (!content.includes(heading)) {
failures.push(`missing required heading: "${heading}"`);
}
}
for (const heading of invariant.mustHaveHeadings ?? []) {
if (!checkText.includes(heading)) {
failures.push(`missing required heading: "${heading}"`);
}
}
}