feat(test): context-budget ratchet — CI ceilings on always-on + eager token ledgers

New free test grades the two ledgers nothing else guards: the full-frontmatter always-on catalog (aggregate) and per-skill eager tokens (SKILL.md + forced-read refs), via checkBudget from lib/context-bill.ts. Ceilings live in test/fixtures/context-budget.json with x1.05/x1.10 headroom; regenerate with bun test/helpers/capture-context-budget.ts. New skills fail until consciously budgeted; removed skills fail until the fixture is refreshed; reductions ratchet the ceilings down so wins lock in. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-29 09:20:39 +02:00 · 2026-08-24 16:00:58 +00:00
parent 51ef97974a
commit e4dce128de
3 changed files with 235 additions and 0 deletions
@@ -0,0 +1,78 @@
+/**
+ * Context-budget ratchet — CI-enforced ceilings on the two token ledgers
+ * nothing else guards (plan OV8):
+ *
+ *   ALWAYS-ON  — full frontmatter bytes every session's skill scanner loads
+ *                (catalog-budget.test.ts caps name+description only; this
+ *                catches growth in the OTHER frontmatter keys).
+ *   EAGER      — per-invocation SKILL.md + forced-read references, per skill
+ *                (skill-size-budget floors catch shrink; parity-suite catches
+ *                growth RATIOS vs an old baseline; this pins absolute token
+ *                ceilings that ratchet DOWN as reduction phases land).
+ *
+ * Fails when a skill's eager tokens exceed its fixture ceiling, when the
+ * always-on aggregate exceeds its ceiling, or when a skill exists with no
+ * ceiling at all (new skills must be consciously budgeted).
+ *
+ * RATCHET PROTOCOL (on failure):
+ *   1. If the growth is a real feature: re-run
+ *        bun test/helpers/capture-context-budget.ts
+ *      and commit the refreshed fixture in the SAME commit as the feature,
+ *      so the growth is a visible, conscious decision in the diff.
+ *   2. If the growth is accidental (resolver bloat, duplicated block,
+ *      copy-paste): fix the bloat instead.
+ *   3. After a token-reduction phase lands: re-run the capture so ceilings
+ *      ratchet down and the win is locked against regression.
+ */
+import { describe, test, expect } from 'bun:test';
+import * as fs from 'fs';
+import { checkBudget } from '../lib/context-bill';
+import {
+  buildRatchetBill,
+  BUDGET_FIXTURE_PATH,
+  type ContextBudget,
+} from './helpers/capture-context-budget';
+
+const RATCHET_PROTOCOL =
+  'Ratchet protocol: legitimate feature growth -> re-run `bun test/helpers/capture-context-budget.ts` ' +
+  'and commit the refreshed fixture in the same commit; accidental bloat -> fix the bloat; ' +
+  'after a reduction lands -> re-run the capture so the ceilings ratchet down.';
+
+const budget: ContextBudget = JSON.parse(fs.readFileSync(BUDGET_FIXTURE_PATH, 'utf-8'));
+const bill = buildRatchetBill();
+
+describe('context-budget ratchet', () => {
+  test('always-on + eager ledgers stay under the fixture ceilings', () => {
+    const violations = checkBudget(bill, {
+      alwaysOnTotal: budget.alwaysOnTotal,
+      eagerPerInvocation: budget.eagerPerInvocation,
+    });
+    const detail = violations
+      .map((v) => `  ${v.ceiling}: ${v.actual} tok > limit ${v.limit}\n    ${v.files.join('\n    ')}`)
+      .join('\n');
+    expect(
+      violations.length,
+      `Context-budget ceilings exceeded:\n${detail}\n${RATCHET_PROTOCOL}`,
+    ).toBe(0);
+  });
+
+  test('every skill in the tree has an eager ceiling (new skills are consciously budgeted)', () => {
+    const missing = bill.skills
+      .map((s) => s.name)
+      .filter((name) => !(name in budget.eagerPerInvocation));
+    expect(
+      missing,
+      `Skills without a context-budget ceiling: ${missing.join(', ')}.\n` +
+        `Add them by re-running the capture. ${RATCHET_PROTOCOL}`,
+    ).toEqual([]);
+  });
+
+  test('fixture has no ceilings for skills that no longer exist', () => {
+    const live = new Set(bill.skills.map((s) => s.name));
+    const stale = Object.keys(budget.eagerPerInvocation).filter((name) => !live.has(name));
+    expect(
+      stale,
+      `Fixture carries ceilings for removed skills: ${stale.join(', ')}. Re-run the capture.`,
+    ).toEqual([]);
+  });
+});
@@ -0,0 +1,66 @@
+{
+  "_comment": "Context-budget ratchet ceilings (~tokens). Regenerate: bun test/helpers/capture-context-budget.ts. Headroom: alwaysOnTotal x1.05, eagerPerInvocation x1.1. Graded by test/context-budget-ratchet.test.ts via lib/context-bill.ts checkBudget.",
+  "alwaysOnTotal": 6423,
+  "eagerPerInvocation": {
+    "autoplan": 27445,
+    "benchmark": 10114,
+    "benchmark-models": 9086,
+    "browse": 15469,
+    "browser-skills/hackernews-frontpage": 371,
+    "canary": 15492,
+    "careful": 919,
+    "codex": 26760,
+    "connect-chrome": 9687,
+    "context-restore": 14506,
+    "context-save": 15123,
+    "cso": 20096,
+    "design-consultation": 18557,
+    "design-html": 20573,
+    "design-review": 29138,
+    "design-shotgun": 19603,
+    "devex-review": 20496,
+    "diagram": 9104,
+    "document-generate": 17251,
+    "document-release": 15048,
+    "freeze": 990,
+    "gstack": 8806,
+    "gstack-upgrade": 3981,
+    "guard": 889,
+    "health": 15704,
+    "investigate": 16357,
+    "ios-clean": 13881,
+    "ios-design-review": 14062,
+    "ios-fix": 13835,
+    "ios-qa": 16132,
+    "ios-sync": 14005,
+    "land-and-deploy": 28990,
+    "landing-report": 14416,
+    "learn": 14086,
+    "make-pdf": 10214,
+    "office-hours": 26763,
+    "open-gstack-browser": 9687,
+    "openclaw/skills/gstack-openclaw-ceo-review": 2764,
+    "openclaw/skills/gstack-openclaw-investigate": 1429,
+    "openclaw/skills/gstack-openclaw-office-hours": 4433,
+    "openclaw/skills/gstack-openclaw-retro": 2542,
+    "pair-agent": 16436,
+    "plan-ceo-review": 24431,
+    "plan-design-review": 23872,
+    "plan-devex-review": 21713,
+    "plan-eng-review": 18652,
+    "plan-tune": 19660,
+    "qa": 23445,
+    "qa-only": 17927,
+    "retro": 24885,
+    "review": 29309,
+    "scrape": 9196,
+    "setup-browser-cookies": 8377,
+    "setup-deploy": 15246,
+    "setup-gbrain": 25164,
+    "ship": 23847,
+    "skillify": 17092,
+    "spec": 22662,
+    "sync-gbrain": 18874,
+    "unfreeze": 393
+  }
+}
@@ -0,0 +1,91 @@
+/**
+ * Context-budget capture — the ratchet's write side.
+ *
+ * Captures the current ALWAYS-ON + EAGER token ledgers from
+ * `lib/context-bill.ts` into `test/fixtures/context-budget.json`, with
+ * deliberate headroom baked into every ceiling:
+ *
+ *   - alwaysOnTotal:       actual × 1.05  (full-frontmatter catalog, aggregate)
+ *   - eagerPerInvocation:  actual × 1.10  (per-skill SKILL.md + forced refs)
+ *
+ * Why these two ledgers and no others: they are the ledgers nothing else
+ * measures (plan OV8). The shrink floor lives in skill-size-budget.test.ts,
+ * growth ratios + minBytes floors in parity-suite.test.ts, the name+description
+ * discovery cap in catalog-budget.test.ts. TOTAL overlaps those guards, so it
+ * is deliberately not budgeted here.
+ *
+ * Ratchet protocol (mirrors catalog-budget.test.ts):
+ *   - Legitimate growth (a real feature grew a skill past its ceiling):
+ *     re-run `bun test/helpers/capture-context-budget.ts` and commit the
+ *     refreshed fixture IN THE SAME COMMIT as the growth, so the diff shows
+ *     the conscious decision.
+ *   - After a reduction phase lands: re-run the capture so the ceilings
+ *     ratchet DOWN and the win is locked.
+ *
+ * Test-fixture skill trees under test/fixtures/ are excluded — they exist to
+ * test context-bill itself and must not couple the ratchet to test data.
+ */
+import * as fs from 'fs';
+import * as path from 'path';
+import { buildBill, type Bill } from '../../lib/context-bill';
+
+export const REPO_ROOT = path.resolve(import.meta.dir, '..', '..');
+export const BUDGET_FIXTURE_PATH = path.join(REPO_ROOT, 'test', 'fixtures', 'context-budget.json');
+
+export const ALWAYS_ON_HEADROOM = 1.05;
+export const EAGER_HEADROOM = 1.10;
+
+/** Skills that exist only as context-bill test data — never budgeted. */
+export function isFixtureSkill(name: string): boolean {
+  return name.startsWith('test/');
+}
+
+export interface ContextBudget {
+  _comment: string;
+  alwaysOnTotal: number;
+  eagerPerInvocation: Record<string, number>;
+}
+
+/** The bill the ratchet grades: repo tree minus test-fixture skill dirs. */
+export function buildRatchetBill(root: string = REPO_ROOT): Bill {
+  const bill = buildBill(root);
+  const skills = bill.skills.filter((s) => !isFixtureSkill(s.name));
+  return {
+    ...bill,
+    skills,
+    totals: {
+      ...bill.totals,
+      skillCount: skills.length,
+      alwaysOnBytes: skills.reduce((n, s) => n + s.frontmatterBytes, 0),
+      alwaysOnTokens: skills.reduce((n, s) => n + s.frontmatterTokens, 0),
+      eagerBytesBySkill: Object.fromEntries(skills.map((s) => [s.name, s.eagerBytes])),
+      eagerTokensBySkill: Object.fromEntries(skills.map((s) => [s.name, Math.round(s.eagerTokens)])),
+    },
+  };
+}
+
+export function captureContextBudget(root: string = REPO_ROOT): ContextBudget {
+  const bill = buildRatchetBill(root);
+  const eagerPerInvocation: Record<string, number> = {};
+  for (const s of [...bill.skills].sort((a, b) => a.name.localeCompare(b.name))) {
+    eagerPerInvocation[s.name] = Math.ceil(s.eagerTokens * EAGER_HEADROOM);
+  }
+  return {
+    _comment:
+      'Context-budget ratchet ceilings (~tokens). Regenerate: bun test/helpers/capture-context-budget.ts. ' +
+      `Headroom: alwaysOnTotal x${ALWAYS_ON_HEADROOM}, eagerPerInvocation x${EAGER_HEADROOM}. ` +
+      'Graded by test/context-budget-ratchet.test.ts via lib/context-bill.ts checkBudget.',
+    alwaysOnTotal: Math.ceil(bill.totals.alwaysOnTokens * ALWAYS_ON_HEADROOM),
+    eagerPerInvocation,
+  };
+}
+
+// CLI: write the fixture.
+if (import.meta.main) {
+  const budget = captureContextBudget();
+  fs.writeFileSync(BUDGET_FIXTURE_PATH, JSON.stringify(budget, null, 2) + '\n');
+  const n = Object.keys(budget.eagerPerInvocation).length;
+  console.log(
+    `Wrote ${path.relative(REPO_ROOT, BUDGET_FIXTURE_PATH)}: alwaysOnTotal=${budget.alwaysOnTotal} tok, ${n} eager ceilings`,
+  );
+}