diff --git a/test/context-budget-ratchet.test.ts b/test/context-budget-ratchet.test.ts new file mode 100644 index 000000000..531e9f243 --- /dev/null +++ b/test/context-budget-ratchet.test.ts @@ -0,0 +1,78 @@ +/** + * Context-budget ratchet — CI-enforced ceilings on the two token ledgers + * nothing else guards (plan OV8): + * + * ALWAYS-ON — full frontmatter bytes every session's skill scanner loads + * (catalog-budget.test.ts caps name+description only; this + * catches growth in the OTHER frontmatter keys). + * EAGER — per-invocation SKILL.md + forced-read references, per skill + * (skill-size-budget floors catch shrink; parity-suite catches + * growth RATIOS vs an old baseline; this pins absolute token + * ceilings that ratchet DOWN as reduction phases land). + * + * Fails when a skill's eager tokens exceed its fixture ceiling, when the + * always-on aggregate exceeds its ceiling, or when a skill exists with no + * ceiling at all (new skills must be consciously budgeted). + * + * RATCHET PROTOCOL (on failure): + * 1. If the growth is a real feature: re-run + * bun test/helpers/capture-context-budget.ts + * and commit the refreshed fixture in the SAME commit as the feature, + * so the growth is a visible, conscious decision in the diff. + * 2. If the growth is accidental (resolver bloat, duplicated block, + * copy-paste): fix the bloat instead. + * 3. After a token-reduction phase lands: re-run the capture so ceilings + * ratchet down and the win is locked against regression. + */ +import { describe, test, expect } from 'bun:test'; +import * as fs from 'fs'; +import { checkBudget } from '../lib/context-bill'; +import { + buildRatchetBill, + BUDGET_FIXTURE_PATH, + type ContextBudget, +} from './helpers/capture-context-budget'; + +const RATCHET_PROTOCOL = + 'Ratchet protocol: legitimate feature growth -> re-run `bun test/helpers/capture-context-budget.ts` ' + + 'and commit the refreshed fixture in the same commit; accidental bloat -> fix the bloat; ' + + 'after a reduction lands -> re-run the capture so the ceilings ratchet down.'; + +const budget: ContextBudget = JSON.parse(fs.readFileSync(BUDGET_FIXTURE_PATH, 'utf-8')); +const bill = buildRatchetBill(); + +describe('context-budget ratchet', () => { + test('always-on + eager ledgers stay under the fixture ceilings', () => { + const violations = checkBudget(bill, { + alwaysOnTotal: budget.alwaysOnTotal, + eagerPerInvocation: budget.eagerPerInvocation, + }); + const detail = violations + .map((v) => ` ${v.ceiling}: ${v.actual} tok > limit ${v.limit}\n ${v.files.join('\n ')}`) + .join('\n'); + expect( + violations.length, + `Context-budget ceilings exceeded:\n${detail}\n${RATCHET_PROTOCOL}`, + ).toBe(0); + }); + + test('every skill in the tree has an eager ceiling (new skills are consciously budgeted)', () => { + const missing = bill.skills + .map((s) => s.name) + .filter((name) => !(name in budget.eagerPerInvocation)); + expect( + missing, + `Skills without a context-budget ceiling: ${missing.join(', ')}.\n` + + `Add them by re-running the capture. ${RATCHET_PROTOCOL}`, + ).toEqual([]); + }); + + test('fixture has no ceilings for skills that no longer exist', () => { + const live = new Set(bill.skills.map((s) => s.name)); + const stale = Object.keys(budget.eagerPerInvocation).filter((name) => !live.has(name)); + expect( + stale, + `Fixture carries ceilings for removed skills: ${stale.join(', ')}. Re-run the capture.`, + ).toEqual([]); + }); +}); diff --git a/test/fixtures/context-budget.json b/test/fixtures/context-budget.json new file mode 100644 index 000000000..5c41212e3 --- /dev/null +++ b/test/fixtures/context-budget.json @@ -0,0 +1,66 @@ +{ + "_comment": "Context-budget ratchet ceilings (~tokens). Regenerate: bun test/helpers/capture-context-budget.ts. Headroom: alwaysOnTotal x1.05, eagerPerInvocation x1.1. Graded by test/context-budget-ratchet.test.ts via lib/context-bill.ts checkBudget.", + "alwaysOnTotal": 6423, + "eagerPerInvocation": { + "autoplan": 27445, + "benchmark": 10114, + "benchmark-models": 9086, + "browse": 15469, + "browser-skills/hackernews-frontpage": 371, + "canary": 15492, + "careful": 919, + "codex": 26760, + "connect-chrome": 9687, + "context-restore": 14506, + "context-save": 15123, + "cso": 20096, + "design-consultation": 18557, + "design-html": 20573, + "design-review": 29138, + "design-shotgun": 19603, + "devex-review": 20496, + "diagram": 9104, + "document-generate": 17251, + "document-release": 15048, + "freeze": 990, + "gstack": 8806, + "gstack-upgrade": 3981, + "guard": 889, + "health": 15704, + "investigate": 16357, + "ios-clean": 13881, + "ios-design-review": 14062, + "ios-fix": 13835, + "ios-qa": 16132, + "ios-sync": 14005, + "land-and-deploy": 28990, + "landing-report": 14416, + "learn": 14086, + "make-pdf": 10214, + "office-hours": 26763, + "open-gstack-browser": 9687, + "openclaw/skills/gstack-openclaw-ceo-review": 2764, + "openclaw/skills/gstack-openclaw-investigate": 1429, + "openclaw/skills/gstack-openclaw-office-hours": 4433, + "openclaw/skills/gstack-openclaw-retro": 2542, + "pair-agent": 16436, + "plan-ceo-review": 24431, + "plan-design-review": 23872, + "plan-devex-review": 21713, + "plan-eng-review": 18652, + "plan-tune": 19660, + "qa": 23445, + "qa-only": 17927, + "retro": 24885, + "review": 29309, + "scrape": 9196, + "setup-browser-cookies": 8377, + "setup-deploy": 15246, + "setup-gbrain": 25164, + "ship": 23847, + "skillify": 17092, + "spec": 22662, + "sync-gbrain": 18874, + "unfreeze": 393 + } +} diff --git a/test/helpers/capture-context-budget.ts b/test/helpers/capture-context-budget.ts new file mode 100644 index 000000000..19aaef866 --- /dev/null +++ b/test/helpers/capture-context-budget.ts @@ -0,0 +1,91 @@ +/** + * Context-budget capture — the ratchet's write side. + * + * Captures the current ALWAYS-ON + EAGER token ledgers from + * `lib/context-bill.ts` into `test/fixtures/context-budget.json`, with + * deliberate headroom baked into every ceiling: + * + * - alwaysOnTotal: actual × 1.05 (full-frontmatter catalog, aggregate) + * - eagerPerInvocation: actual × 1.10 (per-skill SKILL.md + forced refs) + * + * Why these two ledgers and no others: they are the ledgers nothing else + * measures (plan OV8). The shrink floor lives in skill-size-budget.test.ts, + * growth ratios + minBytes floors in parity-suite.test.ts, the name+description + * discovery cap in catalog-budget.test.ts. TOTAL overlaps those guards, so it + * is deliberately not budgeted here. + * + * Ratchet protocol (mirrors catalog-budget.test.ts): + * - Legitimate growth (a real feature grew a skill past its ceiling): + * re-run `bun test/helpers/capture-context-budget.ts` and commit the + * refreshed fixture IN THE SAME COMMIT as the growth, so the diff shows + * the conscious decision. + * - After a reduction phase lands: re-run the capture so the ceilings + * ratchet DOWN and the win is locked. + * + * Test-fixture skill trees under test/fixtures/ are excluded — they exist to + * test context-bill itself and must not couple the ratchet to test data. + */ +import * as fs from 'fs'; +import * as path from 'path'; +import { buildBill, type Bill } from '../../lib/context-bill'; + +export const REPO_ROOT = path.resolve(import.meta.dir, '..', '..'); +export const BUDGET_FIXTURE_PATH = path.join(REPO_ROOT, 'test', 'fixtures', 'context-budget.json'); + +export const ALWAYS_ON_HEADROOM = 1.05; +export const EAGER_HEADROOM = 1.10; + +/** Skills that exist only as context-bill test data — never budgeted. */ +export function isFixtureSkill(name: string): boolean { + return name.startsWith('test/'); +} + +export interface ContextBudget { + _comment: string; + alwaysOnTotal: number; + eagerPerInvocation: Record; +} + +/** The bill the ratchet grades: repo tree minus test-fixture skill dirs. */ +export function buildRatchetBill(root: string = REPO_ROOT): Bill { + const bill = buildBill(root); + const skills = bill.skills.filter((s) => !isFixtureSkill(s.name)); + return { + ...bill, + skills, + totals: { + ...bill.totals, + skillCount: skills.length, + alwaysOnBytes: skills.reduce((n, s) => n + s.frontmatterBytes, 0), + alwaysOnTokens: skills.reduce((n, s) => n + s.frontmatterTokens, 0), + eagerBytesBySkill: Object.fromEntries(skills.map((s) => [s.name, s.eagerBytes])), + eagerTokensBySkill: Object.fromEntries(skills.map((s) => [s.name, Math.round(s.eagerTokens)])), + }, + }; +} + +export function captureContextBudget(root: string = REPO_ROOT): ContextBudget { + const bill = buildRatchetBill(root); + const eagerPerInvocation: Record = {}; + for (const s of [...bill.skills].sort((a, b) => a.name.localeCompare(b.name))) { + eagerPerInvocation[s.name] = Math.ceil(s.eagerTokens * EAGER_HEADROOM); + } + return { + _comment: + 'Context-budget ratchet ceilings (~tokens). Regenerate: bun test/helpers/capture-context-budget.ts. ' + + `Headroom: alwaysOnTotal x${ALWAYS_ON_HEADROOM}, eagerPerInvocation x${EAGER_HEADROOM}. ` + + 'Graded by test/context-budget-ratchet.test.ts via lib/context-bill.ts checkBudget.', + alwaysOnTotal: Math.ceil(bill.totals.alwaysOnTokens * ALWAYS_ON_HEADROOM), + eagerPerInvocation, + }; +} + +// CLI: write the fixture. +if (import.meta.main) { + const budget = captureContextBudget(); + fs.writeFileSync(BUDGET_FIXTURE_PATH, JSON.stringify(budget, null, 2) + '\n'); + const n = Object.keys(budget.eagerPerInvocation).length; + console.log( + `Wrote ${path.relative(REPO_ROOT, BUDGET_FIXTURE_PATH)}: alwaysOnTotal=${budget.alwaysOnTotal} tok, ${n} eager ceilings`, + ); +}