test: corpus budget measures startup bytes; carve-safety counts shared preamble as uncarved (ship review pass 2)

skill-size-budget's growth ceiling and shrink floor already measured
SKILL.md plus sections/preamble.md, but the total-corpus check still summed
hoisted SKILL.md bytes against the pre-hoist baseline, hiding about 0.9 MB
of headroom. It now sums the same startup bytes (3,114,885 -> 2,590,484,
x0.832); the unreachable SKILL.md-missing fallback is gone.

auq-format-always-loaded's carve-safety guard treated any sections/ dir as
carved, so every skill with only the shared preamble ran it and its sanity
check passed on preamble.md. It now uses setup --status's rule: a section
other than the shared preamble. Those skills keep the per-skill checks.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed 2026-10-26 13:02:40 +00:00
1 parent a1f41ee9f3
commit 4510ad3e63
3 files changed
+19 -14

No files matched your search

+1 -1
View File
@@ -21,7 +21,7 @@
- `test/helpers/touchfiles-data.ts`: `ETHOS.md` is registered on the 41 paid cases whose touchfiles cover `plan-ceo-review` or `office-hours`, the two skills that tell the agent to read it, so an ETHOS.md change selects those cases instead of restoring the full gate. The unknown-dependency examples in `test/paid-pr-profile.test.ts`, `test/paid-free-boundary.test.ts` and `test/pr-dependencies.test.ts` now use `SECURITY.md` and `LICENSE`, and the first checks that ETHOS.md maps to its reader cases.
### Gate integrity
- Raised: the ten parity caps above, by owner decision. No other threshold, budget, timeout, trial count, case kind or assertion weakened. The size floor and the growth ceiling in `test/skill-size-budget.test.ts` now measure SKILL.md plus its shared file, which is what a skill loads at startup.
- Raised: the ten parity caps above, by owner decision. No other threshold, budget, timeout, trial count, case kind or assertion weakened. The size floor, the growth ceiling and the total-corpus budget in `test/skill-size-budget.test.ts` now measure SKILL.md plus its shared file, which is what a skill loads at startup (corpus 3,114,885 → 2,590,484 bytes, ×0.832). `test/auq-format-always-loaded.test.ts`'s carve-safety guard counts a skill as carved only when `sections/` holds a file other than the shared `preamble.md`, the rule `setup --status` uses; the skills that leaves out keep the same AskUserQuestion checks in its per-skill loop.
**The Completeness Principle now asks for complete coverage with minimal code.** Every tier-2+ skill's preamble said "the complete thing is the goal", and ETHOS.md told models to "always prefer A" because "the 70-line delta costs seconds". Models read that as permission to write more code, which is the "50-line job arriving as 400" complaint (GSTA-93). Coverage of the requirement is still non-negotiable; code beyond it is no longer a virtue.
+6 -3
View File
@@ -33,7 +33,7 @@
import { describe, test, expect } from 'bun:test';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { readSkillStartup } from './helpers/skill-startup';
import { readSkillStartup, SHARED_PREAMBLE_FILE } from './helpers/skill-startup';
const ROOT = path.resolve(__dirname, '..');
@@ -85,10 +85,13 @@ function discoverSkills(): Array<{ skill: string; skillMd: string; sectionsDir:
.filter(skill => fs.existsSync(path.join(ROOT, skill, 'SKILL.md')))
.map(skill => {
const sectionsDir = path.join(ROOT, skill, 'sections');
// Carved means a section beyond the shared preamble every tier 2+ skill ships.
const carved = fs.existsSync(sectionsDir)
&& fs.readdirSync(sectionsDir).some(f => f.endsWith('.md') && f !== SHARED_PREAMBLE_FILE);
return {
skill,
skillMd: path.join(ROOT, skill, 'SKILL.md'),
sectionsDir: fs.existsSync(sectionsDir) ? sectionsDir : null,
sectionsDir: carved ? sectionsDir : null,
};
});
}
@@ -181,7 +184,7 @@ describe('AUQ format is always-loaded (token-reduction safety net)', () => {
expect(body).toMatch(/ELI10\s*:/i);
expect(body).toMatch(/Recommendation\s*:/i);
// sanity: confirm there really is a section dir we're guarding against
expect(fs.readdirSync(sectionsDir!).some(f => f.endsWith('.md'))).toBe(true);
expect(fs.readdirSync(sectionsDir!).some(f => f.endsWith('.md') && f !== SHARED_PREAMBLE_FILE)).toBe(true);
});
}
+12 -10
View File
@@ -62,6 +62,9 @@ const BASELINE_PATH = path.join(REPO_ROOT, 'test', 'fixtures', 'parity-baseline-
// separately with a hard ceiling.
const DEFAULT_RATIO = 1.50;
const RATIO = Number(process.env.GSTACK_SIZE_BUDGET_RATIO) || DEFAULT_RATIO;
/** What a skill loads at startup: SKILL.md plus a tier 2+ skill's shared sections/preamble.md. */
const startupBytes = (skill: string): number =>
Buffer.byteLength(readSkillStartup(path.join(REPO_ROOT, skill, 'SKILL.md')), 'utf-8');
interface Regression {
skill: string;
@@ -83,8 +86,7 @@ describe('SKILL.md size budget regression (gate, free)', () => {
for (const [skill, before] of Object.entries(baseline.skills)) {
const after = current.skills[skill];
if (!after) continue; // skill removed since v1.44 — not a regression
const skillMd = path.join(REPO_ROOT, skill, 'SKILL.md');
const afterBytes = fs.existsSync(skillMd) ? Buffer.byteLength(readSkillStartup(skillMd), 'utf-8') : after.skillMdBytes;
const afterBytes = startupBytes(skill);
if (afterBytes <= before.skillMdBytes * RATIO) continue;
regressions.push({
skill,
@@ -125,12 +127,13 @@ describe('SKILL.md size budget regression (gate, free)', () => {
test('total corpus byte count does not regress past baseline × ratio', () => {
const baseline: ParityBaseline = JSON.parse(fs.readFileSync(BASELINE_PATH, 'utf-8'));
const current = captureBaseline({ repoRoot: REPO_ROOT });
const ratio = current.totalCorpusBytes / baseline.totalCorpusBytes;
if (current.totalCorpusBytes <= baseline.totalCorpusBytes * RATIO) {
const currentBytes = Object.keys(captureBaseline({ repoRoot: REPO_ROOT }).skills)
.reduce((sum, skill) => sum + startupBytes(skill), 0);
const ratio = currentBytes / baseline.totalCorpusBytes;
if (currentBytes <= baseline.totalCorpusBytes * RATIO) {
// eslint-disable-next-line no-console
console.log(
`[skill-size-budget] corpus OK: ${baseline.totalCorpusBytes} → ${current.totalCorpusBytes} bytes (×${ratio.toFixed(3)})`,
`[skill-size-budget] corpus OK: ${baseline.totalCorpusBytes} → ${currentBytes} bytes (×${ratio.toFixed(3)})`,
);
return;
}
@@ -139,13 +142,13 @@ describe('SKILL.md size budget regression (gate, free)', () => {
logBudgetOverride({
scope: 'skill-size-budget-corpus',
reason: overrideReason,
details: { ratio: RATIO, observed: ratio, before: baseline.totalCorpusBytes, after: current.totalCorpusBytes },
details: { ratio: RATIO, observed: ratio, before: baseline.totalCorpusBytes, after: currentBytes },
});
return;
}
throw new Error(
`Total corpus regressed past v1.47.0.0 baseline × ${RATIO}: ` +
`${baseline.totalCorpusBytes} → ${current.totalCorpusBytes} bytes (×${ratio.toFixed(3)}). ` +
`${baseline.totalCorpusBytes} → ${currentBytes} bytes (×${ratio.toFixed(3)}). ` +
`Override: set GSTACK_SIZE_BUDGET_OVERRIDE_REASON to allow.`,
);
});
@@ -214,8 +217,7 @@ describe('SKILL.md size budget regression (gate, free)', () => {
if (!after) continue; // skill removed since baseline — separate concern
// A tier 2+ skill's shared preamble moved to sections/preamble.md, which
// its SKILL.md requires at startup: measure what it loads, not the move.
const skillMd = path.join(REPO_ROOT, skill, 'SKILL.md');
const afterBytes = fs.existsSync(skillMd) ? Buffer.byteLength(readSkillStartup(skillMd), 'utf-8') : after.skillMdBytes;
const afterBytes = startupBytes(skill);
const ratio = afterBytes / before.skillMdBytes;
if (ratio < MIN_RATIO) {
undershoots.push({