gstack/test/skill-size-budget.test.ts

/**
 * Per-skill SKILL.md size budget regression (v1.46.0.0 T5).
 *
 * Asserts that no skill's startup context grew beyond the baseline: its
 * generated SKILL.md, plus sections/preamble.md for a tier 2+ skill that
 * points to the shared preamble (readSkillStartup), the same measure as the
 * shrink floor below. Catches preamble/resolver changes that bloat skills
 * back to the pre-compression size. Free — pure file IO + JSON diff.
 *
 * Baseline rebased v1.44.1 → v1.47.0.0 in the AskUserQuestion split-rule
 * PR after main merged GSTACK_PLAN_MODE + /spec, pushing the v1.44.1
 * anchor past the 5% ratchet. Historical v1.44.1.json and v1.46.0.0.json
 * are retained in test/fixtures/ for reference.
 *
 * Why a separate test from skill-budget-regression.test.ts: that one
 * compares LIVE eval runs (tool calls, turns, cost); this one compares
 * static SKILL.md sizes. Both gate-tier.
 *
 * Baseline rebased v1.69.1.0 → v1.81.0.0: the Aside-first browser contract
 * ({{ASIDE_SETUP}}) plus the gstack-browser fallback block now ride in every
 * browsing skill (~9KB), which pushed benchmark and scrape past 1.5× of the
 * v1.69.1.0 anchor. Deliberate, corpus-wide, receipted in the v1.81.0.0
 * CHANGELOG; the v1.69.1.0 fixture is in git history.
 *
 * The previous baseline lived at test/fixtures/parity-baseline-v1.69.1.0.json,
 * re-captured 2026-08-25 during token-reduction Phase 1 (bash consolidation
 * moved ~11-13KB of inline preamble bash per skill into bin/gstack-skill-start
 * and bin/gstack-skill-end — a deliberate corpus-wide shrink; receipt:
 * gstack-context-bill --diff in PR #2691). The prior v1.47.0.0 fixture is in
 * git history. Live pins at capture time: this test (shrink floor)
 * and test/parity-suite.test.ts vs parity-baseline-v1.64.1.0.json (growth).
 *
 * Override:
 * - GSTACK_SIZE_BUDGET_RATIO=<n> changes the per-skill regression ratio.
 *   Default 1.0 (no growth allowed). Set to 1.10 to permit 10% growth
 *   (e.g., during deliberate feature additions that the catalog trim
 *   doesn't offset).
 * - GSTACK_SIZE_BUDGET_OVERRIDE_REASON="text" allows a regression to
 *   pass and logs the reason to ~/.gstack/analytics/spend-overrides.jsonl
 *   for audit. Use sparingly; the next baseline should bake in the new
 *   size.
 */

import { describe, test, expect } from 'bun:test';
import * as fs from 'fs';
import * as path from 'path';
import { execSync } from 'child_process';
import { captureBaseline, extractDescription, type ParityBaseline } from './helpers/capture-parity-baseline';
import { logBudgetOverride } from './helpers/budget-override';
import { CARVED_SKILLS } from './helpers/carve-guards';
import { readSkillStartup } from './helpers/skill-startup';

const REPO_ROOT = path.resolve(import.meta.dir, '..');
const BASELINE_PATH = path.join(REPO_ROOT, 'test', 'fixtures', 'parity-baseline-v1.81.0.0.json');

// Default per-skill ratio is 1.50 (50% growth tolerance). Adjusted v1.52.0.0
// (cathedral cap audit) from 1.05 → 1.50: a 5% ratio tripped on legitimate
// feature additions (e.g., plan-tune cathedral T13 grew SKILL.md ×1.24
// adding load-bearing Dream cycle + Audit unmarked + Recent auto-decisions
// surfaces). Real bloat is 2-3×; this catches that while not tripping on
// normal feature scope. The always-loaded catalog cost is enforced
// separately with a hard ceiling.
const DEFAULT_RATIO = 1.50;
const RATIO = Number(process.env.GSTACK_SIZE_BUDGET_RATIO) || DEFAULT_RATIO;
/** What a skill loads at startup: SKILL.md plus a tier 2+ skill's shared sections/preamble.md. */
const startupBytes = (skill: string): number =>
  Buffer.byteLength(readSkillStartup(path.join(REPO_ROOT, skill, 'SKILL.md')), 'utf-8');

interface Regression {
  skill: string;
  beforeBytes: number;
  afterBytes: number;
  growth: number;
}

describe('SKILL.md size budget regression (gate, free)', () => {
  test('parity-baseline-v1.81.0.0.json exists', () => {
    expect(fs.existsSync(BASELINE_PATH)).toBe(true);
  });

  test('no skill exceeds v1.81.0.0 baseline size × ratio', () => {
    const baseline: ParityBaseline = JSON.parse(fs.readFileSync(BASELINE_PATH, 'utf-8'));
    const current = captureBaseline({ repoRoot: REPO_ROOT });

    const regressions: Regression[] = [];
    for (const [skill, before] of Object.entries(baseline.skills)) {
      const after = current.skills[skill];
      if (!after) continue; // skill removed since v1.44 — not a regression
      const afterBytes = startupBytes(skill);
      if (afterBytes <= before.skillMdBytes * RATIO) continue;
      regressions.push({
        skill,
        beforeBytes: before.skillMdBytes,
        afterBytes,
        growth: afterBytes / before.skillMdBytes,
      });
    }

    if (regressions.length === 0) return;

    const overrideReason = process.env.GSTACK_SIZE_BUDGET_OVERRIDE_REASON?.trim();
    if (overrideReason) {
      logBudgetOverride({
        scope: 'skill-size-budget',
        reason: overrideReason,
        details: { ratio: RATIO, regressions },
      });
      // eslint-disable-next-line no-console
      console.warn(
        `[skill-size-budget] OVERRIDE APPLIED (${overrideReason}) — ${regressions.length} regression(s) allowed:`,
      );
      for (const r of regressions) {
        // eslint-disable-next-line no-console
        console.warn(`  ${r.skill}: ${r.beforeBytes} → ${r.afterBytes} bytes (×${r.growth.toFixed(2)})`);
      }
      return;
    }

    const msg = regressions.map(r =>
      `  ${r.skill}: ${r.beforeBytes} → ${r.afterBytes} bytes (×${r.growth.toFixed(2)})`,
    ).join('\n');
    throw new Error(
      `${regressions.length} skill(s) regressed past v1.47.0.0 baseline × ${RATIO}:\n${msg}\n` +
      `Override: set GSTACK_SIZE_BUDGET_OVERRIDE_REASON="why this is OK" to allow and audit-log.`,
    );
  });

  test('total corpus byte count does not regress past baseline × ratio', () => {
    const baseline: ParityBaseline = JSON.parse(fs.readFileSync(BASELINE_PATH, 'utf-8'));
    const currentBytes = Object.keys(captureBaseline({ repoRoot: REPO_ROOT }).skills)
      .reduce((sum, skill) => sum + startupBytes(skill), 0);
    const ratio = currentBytes / baseline.totalCorpusBytes;
    if (currentBytes <= baseline.totalCorpusBytes * RATIO) {
      // eslint-disable-next-line no-console
      console.log(
        `[skill-size-budget] corpus OK: ${baseline.totalCorpusBytes} → ${currentBytes} bytes (×${ratio.toFixed(3)})`,
      );
      return;
    }
    const overrideReason = process.env.GSTACK_SIZE_BUDGET_OVERRIDE_REASON?.trim();
    if (overrideReason) {
      logBudgetOverride({
        scope: 'skill-size-budget-corpus',
        reason: overrideReason,
        details: { ratio: RATIO, observed: ratio, before: baseline.totalCorpusBytes, after: currentBytes },
      });
      return;
    }
    throw new Error(
      `Total corpus regressed past v1.47.0.0 baseline × ${RATIO}: ` +
      `${baseline.totalCorpusBytes} → ${currentBytes} bytes (×${ratio.toFixed(3)}). ` +
      `Override: set GSTACK_SIZE_BUDGET_OVERRIDE_REASON to allow.`,
    );
  });

  /**
   * Gap E (v1.46.0.0): per-skill min-size floor.
   *
   * The existing skill-coverage-floor enforces body ≥ 200 bytes, which is
   * a tiny noise floor. A skill that was 100 KB at v1.47.0.0 and shrinks to
   * 250 bytes passes that check despite losing 99.75% of content. The
   * parity-suite content invariants cover this for 10 hand-picked skills
   * (cso, ship, plan-ceo, etc.); the remaining 41 skills had no per-skill
   * shrinkage floor.
   *
   * Floor: 80% of the v1.47.0.0 baseline. v1.46 actual shrinkage is <1% per
   * skill, so this is a comfortable ceiling that still catches accidental
   * mass deletion (e.g., a refactor that strips the body of a skill).
   *
   * v2.0.0.0 introduces the sections/ pattern for 5 heavyweights
   * (ship, plan-ceo-review, office-hours, plan-eng-review,
   * plan-design-review). Carved so far: ship (skeleton ~83 KB) and
   * plan-ceo-review (skeleton ~81 KB, down from the 138 KB monolith). Those
   * skeletons legitimately fall below the 80% body-strip floor, so each carved
   * skill is added to SECTIONS_EXTRACTED; its union is guarded instead by the
   * sectioned invariant in parity-harness.ts (minBytes on skeleton+sections).
   * Add the remaining three here as they carve.
   */
  test('no skill shrinks past 80% of v1.69.1.0 baseline (catches accidental body strip)', () => {
    const baseline: ParityBaseline = JSON.parse(fs.readFileSync(BASELINE_PATH, 'utf-8'));
    const current = captureBaseline({ repoRoot: REPO_ROOT });
    const MIN_RATIO = 0.80; // a skill at <80% of its v1.44 size signals mass-deletion
    // Carved skills (v2 plan T9): the skeleton SKILL.md intentionally shrinks
    // because prose moved into sections/*.md. The union size is guarded instead
    // by the sectioned invariant in parity-harness.ts (minBytes on the
    // skeleton+sections union), so exempt the skeleton from the body-strip floor.
    // EQ1: derived from the canonical CARVE_GUARDS registry — no parallel list.
    const SECTIONS_EXTRACTED = new Set<string>(CARVED_SKILLS);
    // Intentional one-off shrinks vs the frozen baseline (each needs a reason):
    // - spec: the baseline measured a template bug — prose at Phase 5 mentioned
    //   {{PREAMBLE}} literally, so the generator expanded the ENTIRE preamble a
    //   second time mid-sentence (~47 KB of duplication). Fixed by rewording the
    //   prose; spec/SKILL.md now carries exactly one preamble (~80.9 KB, ×0.79).
    // - scrape/diagram/open-gstack-browser/landing-report/pair-agent/skillify:
    //   the baseline measured these at the silent tier-4 default (a missing
    //   preamble-tier frontmatter fell through `?? 4`). Their tiers are now
    //   declared correctly (1-2), shedding the tier-2..4 onboarding prose they
    //   never should have carried (-271 lines each for tier 1).
    // - browse: the baseline measured the headless-browse skill with its ~17 KB
    //   $B command reference + snapshot-flag tables inline. /browse drives the
    //   Aside browser first and carries the Aside contract in the skeleton; the
    //   command tables live in the carved browse/sections/command-list.md.
    const INTENTIONAL_SHRINKS = new Set<string>([
      'spec',
      'scrape', 'diagram', 'open-gstack-browser',
      'landing-report', 'pair-agent', 'skillify',
      'browse',
    ]);

    const undershoots: Array<{
      skill: string; beforeBytes: number; afterBytes: number; ratio: number;
    }> = [];
    for (const [skill, before] of Object.entries(baseline.skills)) {
      if (SECTIONS_EXTRACTED.has(skill)) continue;
      if (INTENTIONAL_SHRINKS.has(skill)) continue;
      const after = current.skills[skill];
      if (!after) continue; // skill removed since baseline — separate concern
      // A tier 2+ skill's shared preamble moved to sections/preamble.md, which
      // its SKILL.md requires at startup: measure what it loads, not the move.
      const afterBytes = startupBytes(skill);
      const ratio = afterBytes / before.skillMdBytes;
      if (ratio < MIN_RATIO) {
        undershoots.push({
          skill, beforeBytes: before.skillMdBytes, afterBytes, ratio,
        });
      }
    }

    if (undershoots.length === 0) return;

    const overrideReason = process.env.GSTACK_SIZE_BUDGET_OVERRIDE_REASON?.trim();
    if (overrideReason) {
      logBudgetOverride({
        scope: 'skill-size-budget-floor',
        reason: overrideReason,
        details: { min_ratio: MIN_RATIO, undershoots },
      });
      // eslint-disable-next-line no-console
      console.warn(
        `[skill-size-budget-floor] OVERRIDE APPLIED (${overrideReason}) — ${undershoots.length} undershoot(s) allowed`,
      );
      return;
    }

    const msg = undershoots.map(u =>
      `  ${u.skill}: ${u.beforeBytes} → ${u.afterBytes} bytes (×${u.ratio.toFixed(2)} — below ${MIN_RATIO} floor)`,
    ).join('\n');
    throw new Error(
      `${undershoots.length} skill(s) shrunk past v1.47.0.0 × ${MIN_RATIO} floor:\n${msg}\n` +
      `This usually signals accidental body strip (e.g., a resolver returning empty, a ` +
      `template losing a section). If the shrinkage is intentional (e.g., the skill moved ` +
      `to the sections/ pattern), add it to SECTIONS_EXTRACTED in this test. Override: ` +
      `GSTACK_SIZE_BUDGET_OVERRIDE_REASON="why" allows + audit-logs.`,
    );
  });

  test('catalog token estimate stays compressed (v1.45 target ≤ 7000)', () => {
    // Measure COMMITTED content (git show HEAD:), not the live tree. Under
    // the parallel free-suite runner, sibling workers regenerate real
    // SKILL.md files mid-run (gen-skill-docs regen tests), so the live-tree
    // estimate was a moving target: 4177 solo, 8356 and 8041 in two parallel
    // runs. A repo-budget ratchet measures the catalog that ships; CI always
    // checks the PR's committed tree anyway.
    // List paths from HEAD too: a staged-but-uncommitted skill is in the index
    // but not yet in HEAD, so `git ls-files` + `git show HEAD:` disagree.
    const trackedPaths = execSync('git ls-tree -r --name-only HEAD', { cwd: REPO_ROOT, encoding: 'utf-8', timeout: 30_000, maxBuffer: 16 * 1024 * 1024 })
      .split('\n')
      .filter((p) => p.endsWith('/SKILL.md') && p.split('/').length === 2);
    let descriptionBytes = 0;
    for (const rel of trackedPaths) {
      const committed = execSync(`git show HEAD:${JSON.stringify(rel)}`, {
        cwd: REPO_ROOT,
        encoding: 'utf-8',
        maxBuffer: 8 * 1024 * 1024,
        timeout: 30_000,
      });
      descriptionBytes += Buffer.byteLength(extractDescription(committed), 'utf-8');
    }
    const catalogTokens = Math.round(descriptionBytes / 4);
    const trackedCount = trackedPaths.length;
    const v145Target = 7000;
    if (catalogTokens <= v145Target) {
      // eslint-disable-next-line no-console
      console.log(`[skill-size-budget] catalog OK: ~${catalogTokens} tokens (target ≤${v145Target}, ${trackedCount} tracked skills)`);
      return;
    }
    const overrideReason = process.env.GSTACK_SIZE_BUDGET_OVERRIDE_REASON?.trim();
    if (overrideReason) {
      logBudgetOverride({
        scope: 'skill-size-budget-catalog',
        reason: overrideReason,
        details: { target: v145Target, observed: catalogTokens },
      });
      return;
    }
    throw new Error(
      `Catalog token estimate regressed past v1.45 target: ${catalogTokens} tokens > ${v145Target}. ` +
      `T4 catalog trim should keep this under control. Override: set GSTACK_SIZE_BUDGET_OVERRIDE_REASON to allow.`,
    );
  });
});