feat(test): context-budget ratchet — CI ceilings on always-on + eager token ledgers

New free test grades the two ledgers nothing else guards: the full-frontmatter
always-on catalog (aggregate) and per-skill eager tokens (SKILL.md +
forced-read refs), via checkBudget from lib/context-bill.ts. Ceilings live in
test/fixtures/context-budget.json with x1.05/x1.10 headroom; regenerate with
bun test/helpers/capture-context-budget.ts. New skills fail until consciously
budgeted; removed skills fail until the fixture is refreshed; reductions
ratchet the ceilings down so wins lock in.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-08-24 16:00:58 +00:00
co-authored by Claude Fable 5
parent 51ef97974a
commit e4dce128de
3 changed files with 235 additions and 0 deletions
+91
View File
@@ -0,0 +1,91 @@
/**
* Context-budget capture — the ratchet's write side.
*
* Captures the current ALWAYS-ON + EAGER token ledgers from
* `lib/context-bill.ts` into `test/fixtures/context-budget.json`, with
* deliberate headroom baked into every ceiling:
*
* - alwaysOnTotal: actual × 1.05 (full-frontmatter catalog, aggregate)
* - eagerPerInvocation: actual × 1.10 (per-skill SKILL.md + forced refs)
*
* Why these two ledgers and no others: they are the ledgers nothing else
* measures (plan OV8). The shrink floor lives in skill-size-budget.test.ts,
* growth ratios + minBytes floors in parity-suite.test.ts, the name+description
* discovery cap in catalog-budget.test.ts. TOTAL overlaps those guards, so it
* is deliberately not budgeted here.
*
* Ratchet protocol (mirrors catalog-budget.test.ts):
* - Legitimate growth (a real feature grew a skill past its ceiling):
* re-run `bun test/helpers/capture-context-budget.ts` and commit the
* refreshed fixture IN THE SAME COMMIT as the growth, so the diff shows
* the conscious decision.
* - After a reduction phase lands: re-run the capture so the ceilings
* ratchet DOWN and the win is locked.
*
* Test-fixture skill trees under test/fixtures/ are excluded — they exist to
* test context-bill itself and must not couple the ratchet to test data.
*/
import * as fs from 'fs';
import * as path from 'path';
import { buildBill, type Bill } from '../../lib/context-bill';
export const REPO_ROOT = path.resolve(import.meta.dir, '..', '..');
export const BUDGET_FIXTURE_PATH = path.join(REPO_ROOT, 'test', 'fixtures', 'context-budget.json');
export const ALWAYS_ON_HEADROOM = 1.05;
export const EAGER_HEADROOM = 1.10;
/** Skills that exist only as context-bill test data — never budgeted. */
export function isFixtureSkill(name: string): boolean {
return name.startsWith('test/');
}
export interface ContextBudget {
_comment: string;
alwaysOnTotal: number;
eagerPerInvocation: Record<string, number>;
}
/** The bill the ratchet grades: repo tree minus test-fixture skill dirs. */
export function buildRatchetBill(root: string = REPO_ROOT): Bill {
const bill = buildBill(root);
const skills = bill.skills.filter((s) => !isFixtureSkill(s.name));
return {
...bill,
skills,
totals: {
...bill.totals,
skillCount: skills.length,
alwaysOnBytes: skills.reduce((n, s) => n + s.frontmatterBytes, 0),
alwaysOnTokens: skills.reduce((n, s) => n + s.frontmatterTokens, 0),
eagerBytesBySkill: Object.fromEntries(skills.map((s) => [s.name, s.eagerBytes])),
eagerTokensBySkill: Object.fromEntries(skills.map((s) => [s.name, Math.round(s.eagerTokens)])),
},
};
}
export function captureContextBudget(root: string = REPO_ROOT): ContextBudget {
const bill = buildRatchetBill(root);
const eagerPerInvocation: Record<string, number> = {};
for (const s of [...bill.skills].sort((a, b) => a.name.localeCompare(b.name))) {
eagerPerInvocation[s.name] = Math.ceil(s.eagerTokens * EAGER_HEADROOM);
}
return {
_comment:
'Context-budget ratchet ceilings (~tokens). Regenerate: bun test/helpers/capture-context-budget.ts. ' +
`Headroom: alwaysOnTotal x${ALWAYS_ON_HEADROOM}, eagerPerInvocation x${EAGER_HEADROOM}. ` +
'Graded by test/context-budget-ratchet.test.ts via lib/context-bill.ts checkBudget.',
alwaysOnTotal: Math.ceil(bill.totals.alwaysOnTokens * ALWAYS_ON_HEADROOM),
eagerPerInvocation,
};
}
// CLI: write the fixture.
if (import.meta.main) {
const budget = captureContextBudget();
fs.writeFileSync(BUDGET_FIXTURE_PATH, JSON.stringify(budget, null, 2) + '\n');
const n = Object.keys(budget.eagerPerInvocation).length;
console.log(
`Wrote ${path.relative(REPO_ROOT, BUDGET_FIXTURE_PATH)}: alwaysOnTotal=${budget.alwaysOnTotal} tok, ${n} eager ceilings`,
);
}