diff --git a/test/fixtures/skill-md-size-allowlist.json b/test/fixtures/skill-md-size-allowlist.json new file mode 100644 index 000000000..88b8e9d99 --- /dev/null +++ b/test/fixtures/skill-md-size-allowlist.json @@ -0,0 +1,95 @@ +{ + "_comment": "Generated SKILL.md files over the limit, pinned at their exact character count (#3199). test/skill-md-size-guard.test.ts fails when a pinned skill grows past its pin or shrinks below it: lower the pin when a skill shrinks and delete it once the skill is within the limit. Raising a pin is an owner decision. The shared-preamble hoist that would retire most pins is deferred: owner Garry, revisit 2026-11-15.", + "limit": 100000, + "pins": { + "agy/gstack-autoplan": 130280, + "agy/gstack-design-consultation": 108034, + "agy/gstack-design-review": 143383, + "agy/gstack-land-and-deploy": 116595, + "agy/gstack-plan-design-review": 139639, + "agy/gstack-plan-devex-review": 132068, + "agy/gstack-plan-eng-review": 152679, + "agy/gstack-review": 124266, + "claude/design-review": 136823, + "codex/gstack-autoplan": 130239, + "codex/gstack-design-consultation": 109236, + "codex/gstack-design-review": 144817, + "codex/gstack-land-and-deploy": 121172, + "codex/gstack-office-hours": 102128, + "codex/gstack-plan-design-review": 142026, + "codex/gstack-plan-devex-review": 134891, + "codex/gstack-plan-eng-review": 155159, + "codex/gstack-review": 125071, + "copilot/gstack-autoplan": 131413, + "copilot/gstack-codex": 100103, + "copilot/gstack-design-consultation": 109386, + "copilot/gstack-design-review": 144688, + "copilot/gstack-land-and-deploy": 117865, + "copilot/gstack-office-hours": 100416, + "copilot/gstack-plan-design-review": 140510, + "copilot/gstack-plan-devex-review": 133220, + "copilot/gstack-plan-eng-review": 153404, + "copilot/gstack-review": 125537, + "cursor/gstack-autoplan": 129054, + "cursor/gstack-design-consultation": 107072, + "cursor/gstack-design-review": 142362, + "cursor/gstack-land-and-deploy": 115534, + "cursor/gstack-plan-design-review": 138594, + "cursor/gstack-plan-devex-review": 130899, + "cursor/gstack-plan-eng-review": 151533, + "cursor/gstack-review": 140664, + "factory/gstack-autoplan": 131338, + "factory/gstack-codex": 100523, + "factory/gstack-design-consultation": 108520, + "factory/gstack-design-review": 144152, + "factory/gstack-land-and-deploy": 119717, + "factory/gstack-office-hours": 101295, + "factory/gstack-plan-design-review": 141375, + "factory/gstack-plan-devex-review": 134268, + "factory/gstack-plan-eng-review": 154647, + "factory/gstack-review": 144443, + "gbrain/gstack-autoplan": 129591, + "gbrain/gstack-design-review": 132730, + "gbrain/gstack-land-and-deploy": 116056, + "gbrain/gstack-plan-design-review": 127514, + "gbrain/gstack-plan-devex-review": 111470, + "gbrain/gstack-plan-eng-review": 134298, + "hermes/gstack-autoplan": 129681, + "hermes/gstack-design-review": 132837, + "hermes/gstack-land-and-deploy": 116158, + "hermes/gstack-plan-design-review": 127623, + "hermes/gstack-plan-devex-review": 111556, + "hermes/gstack-plan-eng-review": 134383, + "kiro/gstack-autoplan": 128816, + "kiro/gstack-design-consultation": 106914, + "kiro/gstack-design-review": 142176, + "kiro/gstack-land-and-deploy": 115424, + "kiro/gstack-plan-design-review": 138434, + "kiro/gstack-plan-devex-review": 130739, + "kiro/gstack-plan-eng-review": 151383, + "kiro/gstack-review": 140476, + "openclaw/gstack-autoplan": 129296, + "openclaw/gstack-design-review": 131535, + "openclaw/gstack-land-and-deploy": 115644, + "openclaw/gstack-plan-design-review": 126365, + "openclaw/gstack-plan-devex-review": 110309, + "openclaw/gstack-plan-eng-review": 132695, + "opencode/gstack-autoplan": 131416, + "opencode/gstack-design-consultation": 109207, + "opencode/gstack-design-review": 144560, + "opencode/gstack-land-and-deploy": 117642, + "opencode/gstack-office-hours": 100250, + "opencode/gstack-plan-design-review": 140364, + "opencode/gstack-plan-devex-review": 133120, + "opencode/gstack-plan-eng-review": 153285, + "opencode/gstack-review": 142899, + "slate/gstack-autoplan": 128935, + "slate/gstack-design-consultation": 106993, + "slate/gstack-design-review": 142269, + "slate/gstack-land-and-deploy": 115479, + "slate/gstack-plan-design-review": 138514, + "slate/gstack-plan-devex-review": 130819, + "slate/gstack-plan-eng-review": 151458, + "slate/gstack-review": 140570 + } +} diff --git a/test/helpers/free-fixtures-data.ts b/test/helpers/free-fixtures-data.ts index 06d53089a..a3be17eba 100644 --- a/test/helpers/free-fixtures-data.ts +++ b/test/helpers/free-fixtures-data.ts @@ -169,6 +169,7 @@ export const FREE_FIXTURES: Readonly> = { 'test/fixtures/shared-libs-plan-callers-verbless-scope-37186854666.json': ['test/shared-libs-plan-actor.test.ts'], 'test/fixtures/shared-libs/**': ['test/shared-libs-checker-interface-evidence.test.ts', 'test/shared-libs-fixture.test.ts', 'test/shared-libs-plan-actor.test.ts', 'test/shared-libs-revalidation-prompt.test.ts', 'test/shared-libs-review-start-evidence.test.ts', 'test/shared-libs-source-reads.test.ts', 'test/shared-libs-stage-actor.test.ts'], 'test/fixtures/ship-triage/**': ['test/ship-triage-labels.test.ts'], + 'test/fixtures/skill-md-size-allowlist.json': ['test/skill-md-size-guard.test.ts'], 'test/fixtures/skillify-narration/**': ['test/skill-fixture.test.ts'], 'test/fixtures/test-value-seam-evidence.json': ['test/test-value-bar.test.ts'], 'test/fixtures/third-party-actions-recovery-public.json': ['test/third-party-actions-recording.test.ts', 'test/third-party-actions.test.ts'], diff --git a/test/skill-md-size-guard.test.ts b/test/skill-md-size-guard.test.ts new file mode 100644 index 000000000..75e2f1afb --- /dev/null +++ b/test/skill-md-size-guard.test.ts @@ -0,0 +1,118 @@ +/** + * Generated SKILL.md size guard (#3199). Several agent runtimes and skill + * loaders cap one SKILL.md at 100,000 characters; Hermes' skill editor refuses + * every edit to a larger skill without naming the size. This free test renders + * every host and fails when any generated SKILL.md is over the limit. + * + * Skills already over the limit are pinned at their exact size in + * test/fixtures/skill-md-size-allowlist.json (a ratchet): growth past a pin + * fails, and a shrink below a pin fails until the pin is lowered. The shared + * preamble hoist that would retire most pins is deferred (owner Garry, + * revisit 2026-11-15). + */ +import { afterAll, describe, expect, test } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { ALL_HOST_CONFIGS } from '../hosts/index'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const ALLOWLIST_FILE = 'test/fixtures/skill-md-size-allowlist.json'; + +interface Allowlist { + limit: number; + pins: Record; +} + +function countChars(text: string): number { + let count = 0; + for (const _ of text) count++; + return count; +} + +function sizeViolations(sizes: Record, allowlist: Allowlist): string[] { + const { limit, pins } = allowlist; + const violations: string[] = []; + for (const [key, actual] of Object.entries(sizes).sort(([a], [b]) => a.localeCompare(b))) { + const [host, skill] = key.split('/'); + const pin = pins[key]; + const where = `host ${host}, skill ${skill}: ${actual} chars, limit ${limit}`; + if (pin === undefined) { + if (actual > limit) violations.push(`${where}, not pinned. Shrink it to ${limit} chars or fewer.`); + } else if (actual > pin) { + violations.push(`${where}, pinned at ${pin}: grew by ${actual - pin}. Shrink it back to ${pin} chars or fewer.`); + } else if (actual < pin) { + violations.push(actual > limit + ? `${where}, pinned at ${pin}: shrank by ${pin - actual}. Lower the pin "${key}" to ${actual} in ${ALLOWLIST_FILE}.` + : `${where}, pinned at ${pin}: now within the limit. Delete the pin "${key}" from ${ALLOWLIST_FILE}.`); + } + } + for (const key of Object.keys(pins).sort()) { + if (!(key in sizes)) violations.push(`pin "${key}" names no generated SKILL.md. Delete it from ${ALLOWLIST_FILE}.`); + } + return violations; +} + +const SIZE_FIX = 'Fix: move skill-specific prose into sections/*.md that the skill reads on demand (on non-Claude hosts, also add the skill ' + + 'to CARVED_ON_EVERY_HOST in scripts/resolvers/sections.ts so its sections stop being inlined), or cut duplicated text; ' + + 'then run `bun run gen:skill-docs` and rerun this test. A pin may only go down: when a pinned skill shrinks, ' + + `lower its pin in ${ALLOWLIST_FILE} to the new size, and delete it once the skill is within the limit. ` + + 'Raising a pin is an owner decision. The shared-preamble hoist (#3199) is deferred: owner Garry, revisit 2026-11-15.'; + +function renderAllHosts(outDir: string): Record { + const render = Bun.spawnSync( + ['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'all', '--out-dir', outDir], + { cwd: ROOT, stdout: 'pipe', stderr: 'pipe', timeout: 180_000 }, + ); + if (render.exitCode !== 0) { + throw new Error(`gen-skill-docs --host all --out-dir failed (exit ${render.exitCode}):\n${render.stderr.toString()}`); + } + const sizes: Record = {}; + for (const host of ALL_HOST_CONFIGS) { + // Claude's tracked tree is what ships: an out-dir Claude render repoints + // section paths into the render, so it is not byte-identical. + const skillsRoot = host.name === 'claude' ? ROOT : path.join(outDir, host.hostSubdir, 'skills'); + const skillDirs = fs.readdirSync(skillsRoot, { withFileTypes: true }) + .filter(entry => entry.isDirectory() && !entry.name.startsWith('.')) + .map(entry => entry.name); + if (host.name === 'claude') skillDirs.push('.'); + for (const dir of skillDirs) { + const file = path.join(skillsRoot, dir, 'SKILL.md'); + if (!fs.existsSync(file)) continue; + if (host.name === 'claude' && !fs.existsSync(`${file}.tmpl`)) continue; + sizes[`${host.name}/${dir === '.' ? 'gstack' : dir}`] = countChars(fs.readFileSync(file, 'utf-8')); + } + } + return sizes; +} + +describe('generated SKILL.md size guard (#3199)', () => { + const outDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-size-guard-')); + afterAll(() => fs.rmSync(outDir, { recursive: true, force: true })); + + test('the ratchet flags over-limit, grown, shrunk, within-limit and stale entries', () => { + const allowlist = { limit: 100, pins: { 'h/grew': 150, 'h/shrank': 150, 'h/under': 150, 'h/same': 150, 'h/gone': 150 } }; + const violations = sizeViolations({ 'h/new': 101, 'h/ok': 100, 'h/grew': 151, 'h/shrank': 149, 'h/under': 99, 'h/same': 150 }, allowlist); + expect(violations).toEqual([ + 'host h, skill grew: 151 chars, limit 100, pinned at 150: grew by 1. Shrink it back to 150 chars or fewer.', + 'host h, skill new: 101 chars, limit 100, not pinned. Shrink it to 100 chars or fewer.', + `host h, skill shrank: 149 chars, limit 100, pinned at 150: shrank by 1. Lower the pin "h/shrank" to 149 in ${ALLOWLIST_FILE}.`, + `host h, skill under: 99 chars, limit 100, pinned at 150: now within the limit. Delete the pin "h/under" from ${ALLOWLIST_FILE}.`, + `pin "h/gone" names no generated SKILL.md. Delete it from ${ALLOWLIST_FILE}.`, + ]); + expect(countChars('é—x')).toBe(3); + }); + + test('every host\'s generated SKILL.md is within 100,000 chars or at its exact pin', () => { + const allowlist: Allowlist = JSON.parse(fs.readFileSync(path.join(ROOT, ALLOWLIST_FILE), 'utf-8')); + expect(allowlist.limit).toBe(100_000); + const sizes = renderAllHosts(outDir); + for (const host of ALL_HOST_CONFIGS) { + expect(Object.keys(sizes).some(key => key.startsWith(`${host.name}/`)), `no rendered skills for ${host.name}`).toBe(true); + } + const violations = sizeViolations(sizes, allowlist); + if (violations.length > 0) { + throw new Error(`${violations.length} generated SKILL.md size violation(s):\n ${violations.join('\n ')}\n${SIZE_FIX}`); + } + }); +});