From 4510ad3e63472a633059aafb64877f9bb992fb56 Mon Sep 17 00:00:00 2001 From: garrytan Date: Mon, 26 Oct 2026 13:02:40 +0000 Subject: [PATCH] test: corpus budget measures startup bytes; carve-safety counts shared preamble as uncarved (ship review pass 2) skill-size-budget's growth ceiling and shrink floor already measured SKILL.md plus sections/preamble.md, but the total-corpus check still summed hoisted SKILL.md bytes against the pre-hoist baseline, hiding about 0.9 MB of headroom. It now sums the same startup bytes (3,114,885 -> 2,590,484, x0.832); the unreachable SKILL.md-missing fallback is gone. auq-format-always-loaded's carve-safety guard treated any sections/ dir as carved, so every skill with only the shared preamble ran it and its sanity check passed on preamble.md. It now uses setup --status's rule: a section other than the shared preamble. Those skills keep the per-skill checks. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 2 +- test/auq-format-always-loaded.test.ts | 9 ++++++--- test/skill-size-budget.test.ts | 22 ++++++++++++---------- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 913c253fc..a850d87d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,7 @@ - `test/helpers/touchfiles-data.ts`: `ETHOS.md` is registered on the 41 paid cases whose touchfiles cover `plan-ceo-review` or `office-hours`, the two skills that tell the agent to read it, so an ETHOS.md change selects those cases instead of restoring the full gate. The unknown-dependency examples in `test/paid-pr-profile.test.ts`, `test/paid-free-boundary.test.ts` and `test/pr-dependencies.test.ts` now use `SECURITY.md` and `LICENSE`, and the first checks that ETHOS.md maps to its reader cases. ### Gate integrity -- Raised: the ten parity caps above, by owner decision. No other threshold, budget, timeout, trial count, case kind or assertion weakened. The size floor and the growth ceiling in `test/skill-size-budget.test.ts` now measure SKILL.md plus its shared file, which is what a skill loads at startup. +- Raised: the ten parity caps above, by owner decision. No other threshold, budget, timeout, trial count, case kind or assertion weakened. The size floor, the growth ceiling and the total-corpus budget in `test/skill-size-budget.test.ts` now measure SKILL.md plus its shared file, which is what a skill loads at startup (corpus 3,114,885 → 2,590,484 bytes, ×0.832). `test/auq-format-always-loaded.test.ts`'s carve-safety guard counts a skill as carved only when `sections/` holds a file other than the shared `preamble.md`, the rule `setup --status` uses; the skills that leaves out keep the same AskUserQuestion checks in its per-skill loop. **The Completeness Principle now asks for complete coverage with minimal code.** Every tier-2+ skill's preamble said "the complete thing is the goal", and ETHOS.md told models to "always prefer A" because "the 70-line delta costs seconds". Models read that as permission to write more code, which is the "50-line job arriving as 400" complaint (GSTA-93). Coverage of the requirement is still non-negotiable; code beyond it is no longer a virtue. diff --git a/test/auq-format-always-loaded.test.ts b/test/auq-format-always-loaded.test.ts index fac803770..4ee372853 100644 --- a/test/auq-format-always-loaded.test.ts +++ b/test/auq-format-always-loaded.test.ts @@ -33,7 +33,7 @@ import { describe, test, expect } from 'bun:test'; import * as fs from 'node:fs'; import * as path from 'node:path'; -import { readSkillStartup } from './helpers/skill-startup'; +import { readSkillStartup, SHARED_PREAMBLE_FILE } from './helpers/skill-startup'; const ROOT = path.resolve(__dirname, '..'); @@ -85,10 +85,13 @@ function discoverSkills(): Array<{ skill: string; skillMd: string; sectionsDir: .filter(skill => fs.existsSync(path.join(ROOT, skill, 'SKILL.md'))) .map(skill => { const sectionsDir = path.join(ROOT, skill, 'sections'); + // Carved means a section beyond the shared preamble every tier 2+ skill ships. + const carved = fs.existsSync(sectionsDir) + && fs.readdirSync(sectionsDir).some(f => f.endsWith('.md') && f !== SHARED_PREAMBLE_FILE); return { skill, skillMd: path.join(ROOT, skill, 'SKILL.md'), - sectionsDir: fs.existsSync(sectionsDir) ? sectionsDir : null, + sectionsDir: carved ? sectionsDir : null, }; }); } @@ -181,7 +184,7 @@ describe('AUQ format is always-loaded (token-reduction safety net)', () => { expect(body).toMatch(/ELI10\s*:/i); expect(body).toMatch(/Recommendation\s*:/i); // sanity: confirm there really is a section dir we're guarding against - expect(fs.readdirSync(sectionsDir!).some(f => f.endsWith('.md'))).toBe(true); + expect(fs.readdirSync(sectionsDir!).some(f => f.endsWith('.md') && f !== SHARED_PREAMBLE_FILE)).toBe(true); }); } diff --git a/test/skill-size-budget.test.ts b/test/skill-size-budget.test.ts index 44b09f082..5fb839e87 100644 --- a/test/skill-size-budget.test.ts +++ b/test/skill-size-budget.test.ts @@ -62,6 +62,9 @@ const BASELINE_PATH = path.join(REPO_ROOT, 'test', 'fixtures', 'parity-baseline- // separately with a hard ceiling. const DEFAULT_RATIO = 1.50; const RATIO = Number(process.env.GSTACK_SIZE_BUDGET_RATIO) || DEFAULT_RATIO; +/** What a skill loads at startup: SKILL.md plus a tier 2+ skill's shared sections/preamble.md. */ +const startupBytes = (skill: string): number => + Buffer.byteLength(readSkillStartup(path.join(REPO_ROOT, skill, 'SKILL.md')), 'utf-8'); interface Regression { skill: string; @@ -83,8 +86,7 @@ describe('SKILL.md size budget regression (gate, free)', () => { for (const [skill, before] of Object.entries(baseline.skills)) { const after = current.skills[skill]; if (!after) continue; // skill removed since v1.44 — not a regression - const skillMd = path.join(REPO_ROOT, skill, 'SKILL.md'); - const afterBytes = fs.existsSync(skillMd) ? Buffer.byteLength(readSkillStartup(skillMd), 'utf-8') : after.skillMdBytes; + const afterBytes = startupBytes(skill); if (afterBytes <= before.skillMdBytes * RATIO) continue; regressions.push({ skill, @@ -125,12 +127,13 @@ describe('SKILL.md size budget regression (gate, free)', () => { test('total corpus byte count does not regress past baseline × ratio', () => { const baseline: ParityBaseline = JSON.parse(fs.readFileSync(BASELINE_PATH, 'utf-8')); - const current = captureBaseline({ repoRoot: REPO_ROOT }); - const ratio = current.totalCorpusBytes / baseline.totalCorpusBytes; - if (current.totalCorpusBytes <= baseline.totalCorpusBytes * RATIO) { + const currentBytes = Object.keys(captureBaseline({ repoRoot: REPO_ROOT }).skills) + .reduce((sum, skill) => sum + startupBytes(skill), 0); + const ratio = currentBytes / baseline.totalCorpusBytes; + if (currentBytes <= baseline.totalCorpusBytes * RATIO) { // eslint-disable-next-line no-console console.log( - `[skill-size-budget] corpus OK: ${baseline.totalCorpusBytes} → ${current.totalCorpusBytes} bytes (×${ratio.toFixed(3)})`, + `[skill-size-budget] corpus OK: ${baseline.totalCorpusBytes} → ${currentBytes} bytes (×${ratio.toFixed(3)})`, ); return; } @@ -139,13 +142,13 @@ describe('SKILL.md size budget regression (gate, free)', () => { logBudgetOverride({ scope: 'skill-size-budget-corpus', reason: overrideReason, - details: { ratio: RATIO, observed: ratio, before: baseline.totalCorpusBytes, after: current.totalCorpusBytes }, + details: { ratio: RATIO, observed: ratio, before: baseline.totalCorpusBytes, after: currentBytes }, }); return; } throw new Error( `Total corpus regressed past v1.47.0.0 baseline × ${RATIO}: ` + - `${baseline.totalCorpusBytes} → ${current.totalCorpusBytes} bytes (×${ratio.toFixed(3)}). ` + + `${baseline.totalCorpusBytes} → ${currentBytes} bytes (×${ratio.toFixed(3)}). ` + `Override: set GSTACK_SIZE_BUDGET_OVERRIDE_REASON to allow.`, ); }); @@ -214,8 +217,7 @@ describe('SKILL.md size budget regression (gate, free)', () => { if (!after) continue; // skill removed since baseline — separate concern // A tier 2+ skill's shared preamble moved to sections/preamble.md, which // its SKILL.md requires at startup: measure what it loads, not the move. - const skillMd = path.join(REPO_ROOT, skill, 'SKILL.md'); - const afterBytes = fs.existsSync(skillMd) ? Buffer.byteLength(readSkillStartup(skillMd), 'utf-8') : after.skillMdBytes; + const afterBytes = startupBytes(skill); const ratio = afterBytes / before.skillMdBytes; if (ratio < MIN_RATIO) { undershoots.push({