mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-01 22:45:02 +01:00
test: corpus budget measures startup bytes; carve-safety counts shared preamble as uncarved (ship review pass 2)
skill-size-budget's growth ceiling and shrink floor already measured SKILL.md plus sections/preamble.md, but the total-corpus check still summed hoisted SKILL.md bytes against the pre-hoist baseline, hiding about 0.9 MB of headroom. It now sums the same startup bytes (3,114,885 -> 2,590,484, x0.832); the unreachable SKILL.md-missing fallback is gone. auq-format-always-loaded's carve-safety guard treated any sections/ dir as carved, so every skill with only the shared preamble ran it and its sanity check passed on preamble.md. It now uses setup --status's rule: a section other than the shared preamble. Those skills keep the per-skill checks. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
a1f41ee9f3
commit
4510ad3e63
3 files changed
+19
-14
No files matched your search
+1
-1
@@ -21,7 +21,7 @@
|
||||
- `test/helpers/touchfiles-data.ts`: `ETHOS.md` is registered on the 41 paid cases whose touchfiles cover `plan-ceo-review` or `office-hours`, the two skills that tell the agent to read it, so an ETHOS.md change selects those cases instead of restoring the full gate. The unknown-dependency examples in `test/paid-pr-profile.test.ts`, `test/paid-free-boundary.test.ts` and `test/pr-dependencies.test.ts` now use `SECURITY.md` and `LICENSE`, and the first checks that ETHOS.md maps to its reader cases.
|
||||
|
||||
### Gate integrity
|
||||
- Raised: the ten parity caps above, by owner decision. No other threshold, budget, timeout, trial count, case kind or assertion weakened. The size floor and the growth ceiling in `test/skill-size-budget.test.ts` now measure SKILL.md plus its shared file, which is what a skill loads at startup.
|
||||
- Raised: the ten parity caps above, by owner decision. No other threshold, budget, timeout, trial count, case kind or assertion weakened. The size floor, the growth ceiling and the total-corpus budget in `test/skill-size-budget.test.ts` now measure SKILL.md plus its shared file, which is what a skill loads at startup (corpus 3,114,885 → 2,590,484 bytes, ×0.832). `test/auq-format-always-loaded.test.ts`'s carve-safety guard counts a skill as carved only when `sections/` holds a file other than the shared `preamble.md`, the rule `setup --status` uses; the skills that leaves out keep the same AskUserQuestion checks in its per-skill loop.
|
||||
|
||||
**The Completeness Principle now asks for complete coverage with minimal code.** Every tier-2+ skill's preamble said "the complete thing is the goal", and ETHOS.md told models to "always prefer A" because "the 70-line delta costs seconds". Models read that as permission to write more code, which is the "50-line job arriving as 400" complaint (GSTA-93). Coverage of the requirement is still non-negotiable; code beyond it is no longer a virtue.
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { readSkillStartup } from './helpers/skill-startup';
|
||||
import { readSkillStartup, SHARED_PREAMBLE_FILE } from './helpers/skill-startup';
|
||||
|
||||
const ROOT = path.resolve(__dirname, '..');
|
||||
|
||||
@@ -85,10 +85,13 @@ function discoverSkills(): Array<{ skill: string; skillMd: string; sectionsDir:
|
||||
.filter(skill => fs.existsSync(path.join(ROOT, skill, 'SKILL.md')))
|
||||
.map(skill => {
|
||||
const sectionsDir = path.join(ROOT, skill, 'sections');
|
||||
// Carved means a section beyond the shared preamble every tier 2+ skill ships.
|
||||
const carved = fs.existsSync(sectionsDir)
|
||||
&& fs.readdirSync(sectionsDir).some(f => f.endsWith('.md') && f !== SHARED_PREAMBLE_FILE);
|
||||
return {
|
||||
skill,
|
||||
skillMd: path.join(ROOT, skill, 'SKILL.md'),
|
||||
sectionsDir: fs.existsSync(sectionsDir) ? sectionsDir : null,
|
||||
sectionsDir: carved ? sectionsDir : null,
|
||||
};
|
||||
});
|
||||
}
|
||||
@@ -181,7 +184,7 @@ describe('AUQ format is always-loaded (token-reduction safety net)', () => {
|
||||
expect(body).toMatch(/ELI10\s*:/i);
|
||||
expect(body).toMatch(/Recommendation\s*:/i);
|
||||
// sanity: confirm there really is a section dir we're guarding against
|
||||
expect(fs.readdirSync(sectionsDir!).some(f => f.endsWith('.md'))).toBe(true);
|
||||
expect(fs.readdirSync(sectionsDir!).some(f => f.endsWith('.md') && f !== SHARED_PREAMBLE_FILE)).toBe(true);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -62,6 +62,9 @@ const BASELINE_PATH = path.join(REPO_ROOT, 'test', 'fixtures', 'parity-baseline-
|
||||
// separately with a hard ceiling.
|
||||
const DEFAULT_RATIO = 1.50;
|
||||
const RATIO = Number(process.env.GSTACK_SIZE_BUDGET_RATIO) || DEFAULT_RATIO;
|
||||
/** What a skill loads at startup: SKILL.md plus a tier 2+ skill's shared sections/preamble.md. */
|
||||
const startupBytes = (skill: string): number =>
|
||||
Buffer.byteLength(readSkillStartup(path.join(REPO_ROOT, skill, 'SKILL.md')), 'utf-8');
|
||||
|
||||
interface Regression {
|
||||
skill: string;
|
||||
@@ -83,8 +86,7 @@ describe('SKILL.md size budget regression (gate, free)', () => {
|
||||
for (const [skill, before] of Object.entries(baseline.skills)) {
|
||||
const after = current.skills[skill];
|
||||
if (!after) continue; // skill removed since v1.44 — not a regression
|
||||
const skillMd = path.join(REPO_ROOT, skill, 'SKILL.md');
|
||||
const afterBytes = fs.existsSync(skillMd) ? Buffer.byteLength(readSkillStartup(skillMd), 'utf-8') : after.skillMdBytes;
|
||||
const afterBytes = startupBytes(skill);
|
||||
if (afterBytes <= before.skillMdBytes * RATIO) continue;
|
||||
regressions.push({
|
||||
skill,
|
||||
@@ -125,12 +127,13 @@ describe('SKILL.md size budget regression (gate, free)', () => {
|
||||
|
||||
test('total corpus byte count does not regress past baseline × ratio', () => {
|
||||
const baseline: ParityBaseline = JSON.parse(fs.readFileSync(BASELINE_PATH, 'utf-8'));
|
||||
const current = captureBaseline({ repoRoot: REPO_ROOT });
|
||||
const ratio = current.totalCorpusBytes / baseline.totalCorpusBytes;
|
||||
if (current.totalCorpusBytes <= baseline.totalCorpusBytes * RATIO) {
|
||||
const currentBytes = Object.keys(captureBaseline({ repoRoot: REPO_ROOT }).skills)
|
||||
.reduce((sum, skill) => sum + startupBytes(skill), 0);
|
||||
const ratio = currentBytes / baseline.totalCorpusBytes;
|
||||
if (currentBytes <= baseline.totalCorpusBytes * RATIO) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(
|
||||
`[skill-size-budget] corpus OK: ${baseline.totalCorpusBytes} → ${current.totalCorpusBytes} bytes (×${ratio.toFixed(3)})`,
|
||||
`[skill-size-budget] corpus OK: ${baseline.totalCorpusBytes} → ${currentBytes} bytes (×${ratio.toFixed(3)})`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
@@ -139,13 +142,13 @@ describe('SKILL.md size budget regression (gate, free)', () => {
|
||||
logBudgetOverride({
|
||||
scope: 'skill-size-budget-corpus',
|
||||
reason: overrideReason,
|
||||
details: { ratio: RATIO, observed: ratio, before: baseline.totalCorpusBytes, after: current.totalCorpusBytes },
|
||||
details: { ratio: RATIO, observed: ratio, before: baseline.totalCorpusBytes, after: currentBytes },
|
||||
});
|
||||
return;
|
||||
}
|
||||
throw new Error(
|
||||
`Total corpus regressed past v1.47.0.0 baseline × ${RATIO}: ` +
|
||||
`${baseline.totalCorpusBytes} → ${current.totalCorpusBytes} bytes (×${ratio.toFixed(3)}). ` +
|
||||
`${baseline.totalCorpusBytes} → ${currentBytes} bytes (×${ratio.toFixed(3)}). ` +
|
||||
`Override: set GSTACK_SIZE_BUDGET_OVERRIDE_REASON to allow.`,
|
||||
);
|
||||
});
|
||||
@@ -214,8 +217,7 @@ describe('SKILL.md size budget regression (gate, free)', () => {
|
||||
if (!after) continue; // skill removed since baseline — separate concern
|
||||
// A tier 2+ skill's shared preamble moved to sections/preamble.md, which
|
||||
// its SKILL.md requires at startup: measure what it loads, not the move.
|
||||
const skillMd = path.join(REPO_ROOT, skill, 'SKILL.md');
|
||||
const afterBytes = fs.existsSync(skillMd) ? Buffer.byteLength(readSkillStartup(skillMd), 'utf-8') : after.skillMdBytes;
|
||||
const afterBytes = startupBytes(skill);
|
||||
const ratio = afterBytes / before.skillMdBytes;
|
||||
if (ratio < MIN_RATIO) {
|
||||
undershoots.push({
|
||||
|
||||
Reference in new issue
Block a user