/** * Per-skill SKILL.md size budget regression (v1.46.0.0 T5). * * Asserts that no skill's generated SKILL.md grew beyond the v1.47.0.0 * baseline. Catches preamble/resolver changes that bloat skills back to * the pre-compression size. Free — pure file IO + JSON diff. * * Baseline rebased v1.44.1 → v1.47.0.0 in the AskUserQuestion split-rule * PR after main merged GSTACK_PLAN_MODE + /spec, pushing the v1.44.1 * anchor past the 5% ratchet. Historical v1.44.1.json and v1.46.0.0.json * are retained in test/fixtures/ for reference. * * Why a separate test from skill-budget-regression.test.ts: that one * compares LIVE eval runs (tool calls, turns, cost); this one compares * static SKILL.md sizes. Both gate-tier. * * The baseline lives at test/fixtures/parity-baseline-v1.47.0.0.json, * captured by scripts/capture-baseline.ts before any Phase A work landed. * * Override: * - GSTACK_SIZE_BUDGET_RATIO= changes the per-skill regression ratio. * Default 1.0 (no growth allowed). Set to 1.10 to permit 10% growth * (e.g., during deliberate feature additions that the catalog trim * doesn't offset). * - GSTACK_SIZE_BUDGET_OVERRIDE_REASON="text" allows a regression to * pass and logs the reason to ~/.gstack/analytics/spend-overrides.jsonl * for audit. Use sparingly; the next baseline should bake in the new * size. */ import { describe, test, expect } from 'bun:test'; import * as fs from 'fs'; import * as path from 'path'; import { captureBaseline, type ParityBaseline } from './helpers/capture-parity-baseline'; import { logBudgetOverride } from './helpers/budget-override'; const REPO_ROOT = path.resolve(import.meta.dir, '..'); const BASELINE_PATH = path.join(REPO_ROOT, 'test', 'fixtures', 'parity-baseline-v1.47.0.0.json'); // Default per-skill ratio is 1.05 (5% growth tolerance). T4 catalog trim // MOVES text from frontmatter (always-loaded catalog) to a body section // ("## When to invoke"), so small skills with already-short descriptions // see a tiny body growth from the section header itself (~20 bytes). The // 5% per-skill tolerance accommodates that while still catching real bloat; // the always-loaded catalog cost is enforced separately with a hard ceiling. const DEFAULT_RATIO = 1.05; const RATIO = Number(process.env.GSTACK_SIZE_BUDGET_RATIO) || DEFAULT_RATIO; interface Regression { skill: string; beforeBytes: number; afterBytes: number; growth: number; } describe('SKILL.md size budget regression (gate, free)', () => { test('parity-baseline-v1.47.0.0.json exists', () => { expect(fs.existsSync(BASELINE_PATH)).toBe(true); }); test('no skill exceeds v1.47.0.0 baseline size × ratio', () => { const baseline: ParityBaseline = JSON.parse(fs.readFileSync(BASELINE_PATH, 'utf-8')); const current = captureBaseline({ repoRoot: REPO_ROOT }); const regressions: Regression[] = []; for (const [skill, before] of Object.entries(baseline.skills)) { const after = current.skills[skill]; if (!after) continue; // skill removed since v1.44 — not a regression if (after.skillMdBytes <= before.skillMdBytes * RATIO) continue; regressions.push({ skill, beforeBytes: before.skillMdBytes, afterBytes: after.skillMdBytes, growth: after.skillMdBytes / before.skillMdBytes, }); } if (regressions.length === 0) return; const overrideReason = process.env.GSTACK_SIZE_BUDGET_OVERRIDE_REASON?.trim(); if (overrideReason) { logBudgetOverride({ scope: 'skill-size-budget', reason: overrideReason, details: { ratio: RATIO, regressions }, }); // eslint-disable-next-line no-console console.warn( `[skill-size-budget] OVERRIDE APPLIED (${overrideReason}) — ${regressions.length} regression(s) allowed:`, ); for (const r of regressions) { // eslint-disable-next-line no-console console.warn(` ${r.skill}: ${r.beforeBytes} → ${r.afterBytes} bytes (×${r.growth.toFixed(2)})`); } return; } const msg = regressions.map(r => ` ${r.skill}: ${r.beforeBytes} → ${r.afterBytes} bytes (×${r.growth.toFixed(2)})`, ).join('\n'); throw new Error( `${regressions.length} skill(s) regressed past v1.47.0.0 baseline × ${RATIO}:\n${msg}\n` + `Override: set GSTACK_SIZE_BUDGET_OVERRIDE_REASON="why this is OK" to allow and audit-log.`, ); }); test('total corpus byte count does not regress past baseline × ratio', () => { const baseline: ParityBaseline = JSON.parse(fs.readFileSync(BASELINE_PATH, 'utf-8')); const current = captureBaseline({ repoRoot: REPO_ROOT }); const ratio = current.totalCorpusBytes / baseline.totalCorpusBytes; if (current.totalCorpusBytes <= baseline.totalCorpusBytes * RATIO) { // eslint-disable-next-line no-console console.log( `[skill-size-budget] corpus OK: ${baseline.totalCorpusBytes} → ${current.totalCorpusBytes} bytes (×${ratio.toFixed(3)})`, ); return; } const overrideReason = process.env.GSTACK_SIZE_BUDGET_OVERRIDE_REASON?.trim(); if (overrideReason) { logBudgetOverride({ scope: 'skill-size-budget-corpus', reason: overrideReason, details: { ratio: RATIO, observed: ratio, before: baseline.totalCorpusBytes, after: current.totalCorpusBytes }, }); return; } throw new Error( `Total corpus regressed past v1.47.0.0 baseline × ${RATIO}: ` + `${baseline.totalCorpusBytes} → ${current.totalCorpusBytes} bytes (×${ratio.toFixed(3)}). ` + `Override: set GSTACK_SIZE_BUDGET_OVERRIDE_REASON to allow.`, ); }); /** * Gap E (v1.46.0.0): per-skill min-size floor. * * The existing skill-coverage-floor enforces body ≥ 200 bytes, which is * a tiny noise floor. A skill that was 100 KB at v1.47.0.0 and shrinks to * 250 bytes passes that check despite losing 99.75% of content. The * parity-suite content invariants cover this for 10 hand-picked skills * (cso, ship, plan-ceo, etc.); the remaining 41 skills had no per-skill * shrinkage floor. * * Floor: 80% of the v1.47.0.0 baseline. v1.46 actual shrinkage is <1% per * skill, so this is a comfortable ceiling that still catches accidental * mass deletion (e.g., a refactor that strips the body of a skill). * * v2.0.0.0 will introduce the sections/ pattern for 5 heavyweights * (ship, plan-ceo-review, office-hours, plan-eng-review, * plan-design-review). Those skills will legitimately shrink to ~15 KB * skeletons. When that lands, add them to SECTIONS_EXTRACTED so the floor * relaxes for them. */ test('no skill shrinks past 80% of v1.47.0.0 baseline (catches accidental body strip)', () => { const baseline: ParityBaseline = JSON.parse(fs.readFileSync(BASELINE_PATH, 'utf-8')); const current = captureBaseline({ repoRoot: REPO_ROOT }); const MIN_RATIO = 0.80; // a skill at <80% of its v1.44 size signals mass-deletion const SECTIONS_EXTRACTED = new Set(); // populate in v2.0.0.0 when sections/ lands const undershoots: Array<{ skill: string; beforeBytes: number; afterBytes: number; ratio: number; }> = []; for (const [skill, before] of Object.entries(baseline.skills)) { if (SECTIONS_EXTRACTED.has(skill)) continue; const after = current.skills[skill]; if (!after) continue; // skill removed since baseline — separate concern const ratio = after.skillMdBytes / before.skillMdBytes; if (ratio < MIN_RATIO) { undershoots.push({ skill, beforeBytes: before.skillMdBytes, afterBytes: after.skillMdBytes, ratio, }); } } if (undershoots.length === 0) return; const overrideReason = process.env.GSTACK_SIZE_BUDGET_OVERRIDE_REASON?.trim(); if (overrideReason) { logBudgetOverride({ scope: 'skill-size-budget-floor', reason: overrideReason, details: { min_ratio: MIN_RATIO, undershoots }, }); // eslint-disable-next-line no-console console.warn( `[skill-size-budget-floor] OVERRIDE APPLIED (${overrideReason}) — ${undershoots.length} undershoot(s) allowed`, ); return; } const msg = undershoots.map(u => ` ${u.skill}: ${u.beforeBytes} → ${u.afterBytes} bytes (×${u.ratio.toFixed(2)} — below ${MIN_RATIO} floor)`, ).join('\n'); throw new Error( `${undershoots.length} skill(s) shrunk past v1.47.0.0 × ${MIN_RATIO} floor:\n${msg}\n` + `This usually signals accidental body strip (e.g., a resolver returning empty, a ` + `template losing a section). If the shrinkage is intentional (e.g., the skill moved ` + `to the sections/ pattern), add it to SECTIONS_EXTRACTED in this test. Override: ` + `GSTACK_SIZE_BUDGET_OVERRIDE_REASON="why" allows + audit-logs.`, ); }); test('catalog token estimate stays compressed (v1.45 target ≤ 7000)', () => { const current = captureBaseline({ repoRoot: REPO_ROOT }); const v145Target = 7000; if (current.estTotalCatalogTokens <= v145Target) { // eslint-disable-next-line no-console console.log(`[skill-size-budget] catalog OK: ~${current.estTotalCatalogTokens} tokens (target ≤${v145Target})`); return; } const overrideReason = process.env.GSTACK_SIZE_BUDGET_OVERRIDE_REASON?.trim(); if (overrideReason) { logBudgetOverride({ scope: 'skill-size-budget-catalog', reason: overrideReason, details: { target: v145Target, observed: current.estTotalCatalogTokens }, }); return; } throw new Error( `Catalog token estimate regressed past v1.45 target: ${current.estTotalCatalogTokens} tokens > ${v145Target}. ` + `T4 catalog trim should keep this under control. Override: set GSTACK_SIZE_BUDGET_OVERRIDE_REASON to allow.`, ); }); });