mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-01 06:24:41 +01:00
test(skills): guard generated SKILL.md size at 100,000 chars on every host (#3199)
A free test renders every host and fails when a generated SKILL.md exceeds 100,000 characters, naming host, skill, actual size, limit and the fix. The 89 current offenders are pinned at their exact size in test/fixtures/skill-md-size-allowlist.json: growth past a pin fails, and a shrink fails until the pin is lowered. The shared-preamble hoist is deferred (owner Garry, revisit 2026-11-15). Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed
1 parent
a5bc38fd63
commit
93748dc31e
3 files changed
+214
No files matched your search
+95
@@ -0,0 +1,95 @@
|
||||
{
|
||||
"_comment": "Generated SKILL.md files over the limit, pinned at their exact character count (#3199). test/skill-md-size-guard.test.ts fails when a pinned skill grows past its pin or shrinks below it: lower the pin when a skill shrinks and delete it once the skill is within the limit. Raising a pin is an owner decision. The shared-preamble hoist that would retire most pins is deferred: owner Garry, revisit 2026-11-15.",
|
||||
"limit": 100000,
|
||||
"pins": {
|
||||
"agy/gstack-autoplan": 130280,
|
||||
"agy/gstack-design-consultation": 108034,
|
||||
"agy/gstack-design-review": 143383,
|
||||
"agy/gstack-land-and-deploy": 116595,
|
||||
"agy/gstack-plan-design-review": 139639,
|
||||
"agy/gstack-plan-devex-review": 132068,
|
||||
"agy/gstack-plan-eng-review": 152679,
|
||||
"agy/gstack-review": 124266,
|
||||
"claude/design-review": 136823,
|
||||
"codex/gstack-autoplan": 130239,
|
||||
"codex/gstack-design-consultation": 109236,
|
||||
"codex/gstack-design-review": 144817,
|
||||
"codex/gstack-land-and-deploy": 121172,
|
||||
"codex/gstack-office-hours": 102128,
|
||||
"codex/gstack-plan-design-review": 142026,
|
||||
"codex/gstack-plan-devex-review": 134891,
|
||||
"codex/gstack-plan-eng-review": 155159,
|
||||
"codex/gstack-review": 125071,
|
||||
"copilot/gstack-autoplan": 131413,
|
||||
"copilot/gstack-codex": 100103,
|
||||
"copilot/gstack-design-consultation": 109386,
|
||||
"copilot/gstack-design-review": 144688,
|
||||
"copilot/gstack-land-and-deploy": 117865,
|
||||
"copilot/gstack-office-hours": 100416,
|
||||
"copilot/gstack-plan-design-review": 140510,
|
||||
"copilot/gstack-plan-devex-review": 133220,
|
||||
"copilot/gstack-plan-eng-review": 153404,
|
||||
"copilot/gstack-review": 125537,
|
||||
"cursor/gstack-autoplan": 129054,
|
||||
"cursor/gstack-design-consultation": 107072,
|
||||
"cursor/gstack-design-review": 142362,
|
||||
"cursor/gstack-land-and-deploy": 115534,
|
||||
"cursor/gstack-plan-design-review": 138594,
|
||||
"cursor/gstack-plan-devex-review": 130899,
|
||||
"cursor/gstack-plan-eng-review": 151533,
|
||||
"cursor/gstack-review": 140664,
|
||||
"factory/gstack-autoplan": 131338,
|
||||
"factory/gstack-codex": 100523,
|
||||
"factory/gstack-design-consultation": 108520,
|
||||
"factory/gstack-design-review": 144152,
|
||||
"factory/gstack-land-and-deploy": 119717,
|
||||
"factory/gstack-office-hours": 101295,
|
||||
"factory/gstack-plan-design-review": 141375,
|
||||
"factory/gstack-plan-devex-review": 134268,
|
||||
"factory/gstack-plan-eng-review": 154647,
|
||||
"factory/gstack-review": 144443,
|
||||
"gbrain/gstack-autoplan": 129591,
|
||||
"gbrain/gstack-design-review": 132730,
|
||||
"gbrain/gstack-land-and-deploy": 116056,
|
||||
"gbrain/gstack-plan-design-review": 127514,
|
||||
"gbrain/gstack-plan-devex-review": 111470,
|
||||
"gbrain/gstack-plan-eng-review": 134298,
|
||||
"hermes/gstack-autoplan": 129681,
|
||||
"hermes/gstack-design-review": 132837,
|
||||
"hermes/gstack-land-and-deploy": 116158,
|
||||
"hermes/gstack-plan-design-review": 127623,
|
||||
"hermes/gstack-plan-devex-review": 111556,
|
||||
"hermes/gstack-plan-eng-review": 134383,
|
||||
"kiro/gstack-autoplan": 128816,
|
||||
"kiro/gstack-design-consultation": 106914,
|
||||
"kiro/gstack-design-review": 142176,
|
||||
"kiro/gstack-land-and-deploy": 115424,
|
||||
"kiro/gstack-plan-design-review": 138434,
|
||||
"kiro/gstack-plan-devex-review": 130739,
|
||||
"kiro/gstack-plan-eng-review": 151383,
|
||||
"kiro/gstack-review": 140476,
|
||||
"openclaw/gstack-autoplan": 129296,
|
||||
"openclaw/gstack-design-review": 131535,
|
||||
"openclaw/gstack-land-and-deploy": 115644,
|
||||
"openclaw/gstack-plan-design-review": 126365,
|
||||
"openclaw/gstack-plan-devex-review": 110309,
|
||||
"openclaw/gstack-plan-eng-review": 132695,
|
||||
"opencode/gstack-autoplan": 131416,
|
||||
"opencode/gstack-design-consultation": 109207,
|
||||
"opencode/gstack-design-review": 144560,
|
||||
"opencode/gstack-land-and-deploy": 117642,
|
||||
"opencode/gstack-office-hours": 100250,
|
||||
"opencode/gstack-plan-design-review": 140364,
|
||||
"opencode/gstack-plan-devex-review": 133120,
|
||||
"opencode/gstack-plan-eng-review": 153285,
|
||||
"opencode/gstack-review": 142899,
|
||||
"slate/gstack-autoplan": 128935,
|
||||
"slate/gstack-design-consultation": 106993,
|
||||
"slate/gstack-design-review": 142269,
|
||||
"slate/gstack-land-and-deploy": 115479,
|
||||
"slate/gstack-plan-design-review": 138514,
|
||||
"slate/gstack-plan-devex-review": 130819,
|
||||
"slate/gstack-plan-eng-review": 151458,
|
||||
"slate/gstack-review": 140570
|
||||
}
|
||||
}
|
||||
@@ -169,6 +169,7 @@ export const FREE_FIXTURES: Readonly<Record<string, readonly string[]>> = {
|
||||
'test/fixtures/shared-libs-plan-callers-verbless-scope-37186854666.json': ['test/shared-libs-plan-actor.test.ts'],
|
||||
'test/fixtures/shared-libs/**': ['test/shared-libs-checker-interface-evidence.test.ts', 'test/shared-libs-fixture.test.ts', 'test/shared-libs-plan-actor.test.ts', 'test/shared-libs-revalidation-prompt.test.ts', 'test/shared-libs-review-start-evidence.test.ts', 'test/shared-libs-source-reads.test.ts', 'test/shared-libs-stage-actor.test.ts'],
|
||||
'test/fixtures/ship-triage/**': ['test/ship-triage-labels.test.ts'],
|
||||
'test/fixtures/skill-md-size-allowlist.json': ['test/skill-md-size-guard.test.ts'],
|
||||
'test/fixtures/skillify-narration/**': ['test/skill-fixture.test.ts'],
|
||||
'test/fixtures/test-value-seam-evidence.json': ['test/test-value-bar.test.ts'],
|
||||
'test/fixtures/third-party-actions-recovery-public.json': ['test/third-party-actions-recording.test.ts', 'test/third-party-actions.test.ts'],
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
/**
|
||||
* Generated SKILL.md size guard (#3199). Several agent runtimes and skill
|
||||
* loaders cap one SKILL.md at 100,000 characters; Hermes' skill editor refuses
|
||||
* every edit to a larger skill without naming the size. This free test renders
|
||||
* every host and fails when any generated SKILL.md is over the limit.
|
||||
*
|
||||
* Skills already over the limit are pinned at their exact size in
|
||||
* test/fixtures/skill-md-size-allowlist.json (a ratchet): growth past a pin
|
||||
* fails, and a shrink below a pin fails until the pin is lowered. The shared
|
||||
* preamble hoist that would retire most pins is deferred (owner Garry,
|
||||
* revisit 2026-11-15).
|
||||
*/
|
||||
import { afterAll, describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { ALL_HOST_CONFIGS } from '../hosts/index';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const ALLOWLIST_FILE = 'test/fixtures/skill-md-size-allowlist.json';
|
||||
|
||||
interface Allowlist {
|
||||
limit: number;
|
||||
pins: Record<string, number>;
|
||||
}
|
||||
|
||||
function countChars(text: string): number {
|
||||
let count = 0;
|
||||
for (const _ of text) count++;
|
||||
return count;
|
||||
}
|
||||
|
||||
function sizeViolations(sizes: Record<string, number>, allowlist: Allowlist): string[] {
|
||||
const { limit, pins } = allowlist;
|
||||
const violations: string[] = [];
|
||||
for (const [key, actual] of Object.entries(sizes).sort(([a], [b]) => a.localeCompare(b))) {
|
||||
const [host, skill] = key.split('/');
|
||||
const pin = pins[key];
|
||||
const where = `host ${host}, skill ${skill}: ${actual} chars, limit ${limit}`;
|
||||
if (pin === undefined) {
|
||||
if (actual > limit) violations.push(`${where}, not pinned. Shrink it to ${limit} chars or fewer.`);
|
||||
} else if (actual > pin) {
|
||||
violations.push(`${where}, pinned at ${pin}: grew by ${actual - pin}. Shrink it back to ${pin} chars or fewer.`);
|
||||
} else if (actual < pin) {
|
||||
violations.push(actual > limit
|
||||
? `${where}, pinned at ${pin}: shrank by ${pin - actual}. Lower the pin "${key}" to ${actual} in ${ALLOWLIST_FILE}.`
|
||||
: `${where}, pinned at ${pin}: now within the limit. Delete the pin "${key}" from ${ALLOWLIST_FILE}.`);
|
||||
}
|
||||
}
|
||||
for (const key of Object.keys(pins).sort()) {
|
||||
if (!(key in sizes)) violations.push(`pin "${key}" names no generated SKILL.md. Delete it from ${ALLOWLIST_FILE}.`);
|
||||
}
|
||||
return violations;
|
||||
}
|
||||
|
||||
const SIZE_FIX = 'Fix: move skill-specific prose into sections/*.md that the skill reads on demand (on non-Claude hosts, also add the skill '
|
||||
+ 'to CARVED_ON_EVERY_HOST in scripts/resolvers/sections.ts so its sections stop being inlined), or cut duplicated text; '
|
||||
+ 'then run `bun run gen:skill-docs` and rerun this test. A pin may only go down: when a pinned skill shrinks, '
|
||||
+ `lower its pin in ${ALLOWLIST_FILE} to the new size, and delete it once the skill is within the limit. `
|
||||
+ 'Raising a pin is an owner decision. The shared-preamble hoist (#3199) is deferred: owner Garry, revisit 2026-11-15.';
|
||||
|
||||
function renderAllHosts(outDir: string): Record<string, number> {
|
||||
const render = Bun.spawnSync(
|
||||
['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'all', '--out-dir', outDir],
|
||||
{ cwd: ROOT, stdout: 'pipe', stderr: 'pipe', timeout: 180_000 },
|
||||
);
|
||||
if (render.exitCode !== 0) {
|
||||
throw new Error(`gen-skill-docs --host all --out-dir failed (exit ${render.exitCode}):\n${render.stderr.toString()}`);
|
||||
}
|
||||
const sizes: Record<string, number> = {};
|
||||
for (const host of ALL_HOST_CONFIGS) {
|
||||
// Claude's tracked tree is what ships: an out-dir Claude render repoints
|
||||
// section paths into the render, so it is not byte-identical.
|
||||
const skillsRoot = host.name === 'claude' ? ROOT : path.join(outDir, host.hostSubdir, 'skills');
|
||||
const skillDirs = fs.readdirSync(skillsRoot, { withFileTypes: true })
|
||||
.filter(entry => entry.isDirectory() && !entry.name.startsWith('.'))
|
||||
.map(entry => entry.name);
|
||||
if (host.name === 'claude') skillDirs.push('.');
|
||||
for (const dir of skillDirs) {
|
||||
const file = path.join(skillsRoot, dir, 'SKILL.md');
|
||||
if (!fs.existsSync(file)) continue;
|
||||
if (host.name === 'claude' && !fs.existsSync(`${file}.tmpl`)) continue;
|
||||
sizes[`${host.name}/${dir === '.' ? 'gstack' : dir}`] = countChars(fs.readFileSync(file, 'utf-8'));
|
||||
}
|
||||
}
|
||||
return sizes;
|
||||
}
|
||||
|
||||
describe('generated SKILL.md size guard (#3199)', () => {
|
||||
const outDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-size-guard-'));
|
||||
afterAll(() => fs.rmSync(outDir, { recursive: true, force: true }));
|
||||
|
||||
test('the ratchet flags over-limit, grown, shrunk, within-limit and stale entries', () => {
|
||||
const allowlist = { limit: 100, pins: { 'h/grew': 150, 'h/shrank': 150, 'h/under': 150, 'h/same': 150, 'h/gone': 150 } };
|
||||
const violations = sizeViolations({ 'h/new': 101, 'h/ok': 100, 'h/grew': 151, 'h/shrank': 149, 'h/under': 99, 'h/same': 150 }, allowlist);
|
||||
expect(violations).toEqual([
|
||||
'host h, skill grew: 151 chars, limit 100, pinned at 150: grew by 1. Shrink it back to 150 chars or fewer.',
|
||||
'host h, skill new: 101 chars, limit 100, not pinned. Shrink it to 100 chars or fewer.',
|
||||
`host h, skill shrank: 149 chars, limit 100, pinned at 150: shrank by 1. Lower the pin "h/shrank" to 149 in ${ALLOWLIST_FILE}.`,
|
||||
`host h, skill under: 99 chars, limit 100, pinned at 150: now within the limit. Delete the pin "h/under" from ${ALLOWLIST_FILE}.`,
|
||||
`pin "h/gone" names no generated SKILL.md. Delete it from ${ALLOWLIST_FILE}.`,
|
||||
]);
|
||||
expect(countChars('é—x')).toBe(3);
|
||||
});
|
||||
|
||||
test('every host\'s generated SKILL.md is within 100,000 chars or at its exact pin', () => {
|
||||
const allowlist: Allowlist = JSON.parse(fs.readFileSync(path.join(ROOT, ALLOWLIST_FILE), 'utf-8'));
|
||||
expect(allowlist.limit).toBe(100_000);
|
||||
const sizes = renderAllHosts(outDir);
|
||||
for (const host of ALL_HOST_CONFIGS) {
|
||||
expect(Object.keys(sizes).some(key => key.startsWith(`${host.name}/`)), `no rendered skills for ${host.name}`).toBe(true);
|
||||
}
|
||||
const violations = sizeViolations(sizes, allowlist);
|
||||
if (violations.length > 0) {
|
||||
throw new Error(`${violations.length} generated SKILL.md size violation(s):\n ${violations.join('\n ')}\n${SIZE_FIX}`);
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user