feat(write-skill): /write-skill skill, gstack-skill-bill, static skill-to-bin drift ratchet, shared skill eval contract

Skill-Size: 0→3245 bytes (write-skill/SKILL.md.tmpl)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed 2026-10-27 23:49:41 +00:00
1 parent b286989895
commit 92c90ba66e
32 files changed
+1948 -11

No files matched your search

+9 -1
View File
@@ -80,8 +80,16 @@ import { skillCensus } from './helpers/skill-census';
* New-skill ratchet: previous ceiling 1,314 + ceil(178 / 4) = 1,359
* token-equivalents (5,436 bytes), leaving 9 bytes. Existing descriptions
* are unchanged.
* ref write-skill addition on base 10b77de (2026-10-27), Garry
* 2026-10-27 'Rec' (GSTA-133 plan, autoplan-20261027230116)
* result pre-addition aggregate 5,427 bytes; write-skill adds 135 bytes
* (name plus the one-sentence catalog lead), yielding 5,562 bytes
* = 1,391 token-equivalents including the root router alias.
* New-skill ratchet: previous ceiling 1,359 + ceil(135 / 4) = 1,393
* token-equivalents (5,572 bytes), leaving 10 bytes. Existing descriptions
* are unchanged.
*/
const CATALOG_BUDGET_TOKEN_EQUIVALENTS = 1_359;
const CATALOG_BUDGET_TOKEN_EQUIVALENTS = 1_393;
// Largest today: design-consultation at 229 bytes. A description that needs
// more than 260 bytes is a body paragraph, not a catalog entry.
+4 -3
View File
@@ -1,6 +1,6 @@
{
"_comment": "Context-budget ratchet ceilings (~tokens). Regenerate: bun test/helpers/capture-context-budget.ts. Headroom: alwaysOnTotal x1.05, eagerPerInvocation x1.1. Graded by test/context-budget-ratchet.test.ts via lib/context-bill.ts checkBudget.",
"alwaysOnTotal": 7880,
"alwaysOnTotal": 7985,
"eagerPerInvocation": {
"autoplan": 11289,
"benchmark": 8043,
@@ -39,7 +39,7 @@
"ios-sync": 2465,
"land-and-deploy": 13585,
"landing-report": 3269,
"learn": 2931,
"learn": 2092,
"make-pdf": 5204,
"modernize": 3772,
"office-hours": 19139,
@@ -68,6 +68,7 @@
"sync-gbrain": 9749,
"test-audit": 4622,
"ubicloud": 3594,
"unfreeze": 409
"unfreeze": 409,
"write-skill": 2264
}
}
+6
View File
@@ -0,0 +1,6 @@
{
"_comment": "Shrink-only allowlist for test/skill-bin-drift.test.ts (scripts/skill-bin-drift.ts). Keys are <file>:<bin>:<flag> for DRIFT_UNKNOWN_FLAG and <file>:<bin> for DRIFT_UNKNOWN_BIN, recorded 2026-10-27 (GSTA-133). Never add an entry: fix the template or the bin's parser instead. Delete an entry as soon as it stops reproducing; the test fails on stale entries so this list can only shrink. The one entry today is a scanner limit: gstack-safe-git forwards its subcommand's arguments to git, so --is-inside-work-tree belongs to `git rev-parse`.",
"entries": [
"deslop-shared-libs/SKILL.md:gstack-safe-git:--is-inside-work-tree"
]
}
+28
View File
@@ -0,0 +1,28 @@
#!/bin/sh
# Seeds a /write-skill eval work dir. Usage: sh "$SKILL_EVAL_FIXTURES/seed.sh" lesson|new
set -eu
mkdir -p deploy
cat > deploy/SKILL.md <<'EOF'
---
name: deploy
description: Deploy the web app to production.
---
# Deploy
1. Build the release image with `make release`.
2. Run database migrations before restarting the web servers.
3. Restart the web servers one at a time.
4. Check the health endpoint returns 200.
EOF
cat > CHANGELOG.md <<'EOF'
# Changelog
## Unreleased
- Fix login redirect loop.
## 1.2.0
- Add CSV export.
EOF
+17
View File
@@ -0,0 +1,17 @@
[
{
"id": "add-lesson",
"prompt": "Add this lesson to the deploy skill in ./deploy: \"Last Tuesday we restarted the web servers while migrations were still running and the site returned 500s for ten minutes. Migrations must finish before any restart.\"",
"setup": "sh \"$SKILL_EVAL_FIXTURES/seed.sh\" lesson",
"must": [
"\\d+\\s*(B|bytes)?\\s*(→|->|to)\\s*\\d+"
],
"check": "test $(wc -l < deploy/SKILL.md) -le 12 && ! grep -qiE 'tuesday|500s|ten minutes' deploy/SKILL.md && grep -i migration deploy/SKILL.md | grep -qiE 'finish|complete|wait'"
},
{
"id": "new-skill",
"prompt": "Write a new agent skill in ./skills/changelog that tells an agent how to add an entry to this repo's CHANGELOG.md: newest first, one line per change, under ## Unreleased.",
"setup": "sh \"$SKILL_EVAL_FIXTURES/seed.sh\" new",
"check": "f=skills/changelog/SKILL.md && test -f $f && head -1 $f | grep -q '^---' && grep -q '^name: *changelog' $f && grep -q '^description:' $f && test $(wc -c < $f) -le 2500 && grep -q Unreleased $f"
}
]
+3
View File
@@ -683,6 +683,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
'cso-infra-scope': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'cso/**', 'lib/cso/**', 'lib/redact-engine.ts', 'lib/redact-patterns.ts', 'test/skill-e2e-cso.test.ts'],
// Learnings
'write-skill-add-lesson': ['write-skill/**', 'bin/gstack-skill-bill', 'lib/skill-bill.ts', 'lib/context-bill.ts', 'test/skill-e2e-write-skill.test.ts'],
'learnings-show': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'learn/**', 'bin/gstack-learnings-search', 'bin/gstack-learnings-log', 'scripts/resolvers/learnings.ts', 'test/skill-e2e-learnings.test.ts'],
// W1 safety-rule evals (test/helpers/safety-rules.ts registry; E3).
'safety-design-risk-stop': ['design-review/**', 'scripts/resolvers/browse.ts', 'scripts/resolvers/testing.ts', 'lib/aside-render.ts', 'test/skill-e2e-safety-design-risk.test.ts', 'test/helpers/browser-available.ts', 'test/helpers/aside-available.ts', 'test/helpers/safety-rules.ts', 'test/helpers/skill-fixture.ts', 'test/helpers/llm-judge.ts'],
@@ -1434,6 +1435,7 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic' | 'marathon'> = {
// Learnings — gate (functional guardrail: seeded learnings must appear)
'learnings-show': 'gate',
'write-skill-add-lesson': 'periodic',
// W1 safety-rule evals.
'safety-design-risk-stop': 'periodic',
'safety-codex-boundary': 'periodic',
@@ -1855,6 +1857,7 @@ export const E2E_KINDS: Record<string, 'rule' | 'behavior' | 'judge'> = {
'cso-diff-mode': 'rule',
'cso-infra-scope': 'rule',
'learnings-show': 'rule',
'write-skill-add-lesson': 'rule',
// W1 safety-rule evals.
'safety-design-risk-stop': 'rule',
'safety-codex-boundary': 'rule',
+115
View File
@@ -0,0 +1,115 @@
import { describe, test, expect, afterAll } from 'bun:test';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import { spawnSync } from 'child_process';
const BIN = path.resolve(import.meta.dir, '..', 'bin', 'gstack-skill-bill');
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-bill-'));
const home = path.join(tmp, 'home');
fs.mkdirSync(home);
function sh(cwd: string, cmd: string, args: string[]) {
const r = spawnSync(cmd, args, { cwd, encoding: 'utf-8', timeout: 30_000 });
if (r.status !== 0) throw new Error(`${cmd} ${args.join(' ')} failed: ${r.stderr}`);
return r.stdout.trim();
}
function bill(cwd: string, args: string[]) {
return spawnSync(BIN, args, { cwd, encoding: 'utf-8', timeout: 30_000, env: { ...process.env, HOME: home, CODEX_HOME: path.join(home, '.codex') } });
}
function skill(dir: string, name: string, body: string, file = 'SKILL.md') {
fs.mkdirSync(dir, { recursive: true });
fs.writeFileSync(path.join(dir, file), `---\nname: ${name}\ndescription: |\n Does ${name} things.\n---\n${body}`);
}
function gstackRepo(): string {
const root = path.join(tmp, 'gstack');
fs.mkdirSync(path.join(root, 'bin'), { recursive: true });
fs.writeFileSync(path.join(root, 'SKILL.md.tmpl'), 'router');
fs.writeFileSync(path.join(root, 'VERSION'), '1.0.0');
skill(path.join(root, 'demo'), 'demo', 'x'.repeat(1000) + '\n', 'SKILL.md.tmpl');
fs.writeFileSync(path.join(root, 'demo', 'SKILL.md'), '---\nname: demo\ndescription: Does demo things.\n---\n<!-- AUTO-GENERATED from SKILL.md.tmpl -->\nbody\n');
fs.mkdirSync(path.join(root, 'demo', 'references'));
fs.writeFileSync(path.join(root, 'demo', 'references', 'deep.md'), 'deep\n');
sh(root, 'git', ['init', '-q', '-b', 'main']);
sh(root, 'git', ['config', 'user.email', 'test@example.com']);
sh(root, 'git', ['config', 'user.name', 'Test']);
sh(root, 'git', ['add', '-A']);
sh(root, 'git', ['commit', '-qm', 'diet demo\n\nSkill-Diet: demo']);
return root;
}
afterAll(() => fs.rmSync(tmp, { recursive: true, force: true }));
describe('gstack-skill-bill', () => {
const root = gstackRepo();
test('a gstack checkout skill reports every level and its last diet', () => {
const r = bill(root, ['demo']);
expect(r.status).toBe(0);
expect(r.stdout).toContain('SKILL: demo (gstack-checkout)');
expect(r.stdout).toMatch(/template\s+SKILL\.md\.tmpl/);
expect(r.stdout).toMatch(/on-demand\s+references\/deep\.md/);
expect(r.stdout).toContain('BASELINE: ');
expect(r.stdout).not.toContain('DUE');
});
test('growth over 10% since the Skill-Diet commit prints DUE, and an unrendered template is RENDER_STALE', () => {
fs.appendFileSync(path.join(root, 'demo', 'SKILL.md.tmpl'), 'y'.repeat(200));
const r = bill(root, ['demo', '--json']);
const report = JSON.parse(r.stdout);
expect(report.baseline.status).toBe('found');
expect(report.baseline.due).toBe(true);
expect(report.warnings.join('\n')).toContain('RENDER_STALE');
sh(root, 'git', ['checkout', '--', 'demo/SKILL.md.tmpl']);
});
test('a shallow clone says the baseline is unknown instead of none', () => {
const shallow = path.join(tmp, 'shallow');
sh(root, 'git', ['commit', '-q', '--allow-empty', '-m', 'later']);
sh(tmp, 'git', ['clone', '-q', '--depth', '1', 'file://' + root, shallow]);
const report = JSON.parse(bill(shallow, ['demo', '--json']).stdout);
expect(report.baseline.status).toBe('unknown');
expect(report.baseline.reason).toContain('shallow');
});
test('a standalone folder outside git is a user skill with an unknown baseline', () => {
const own = path.join(tmp, 'mine', 'deploy');
skill(own, 'deploy', 'steps\n');
const r = bill(tmp, ['./mine/deploy']);
expect(r.status).toBe(0);
expect(r.stdout).toContain('SKILL: deploy (user)');
expect(r.stdout).toContain('not a git folder');
});
test('names resolve exactly: an installed gstack copy yields to the user skill, a near miss lists the closest names', () => {
skill(path.join(home, '.claude', 'skills', 'deploy'), 'deploy', 'mine\n');
skill(path.join(home, '.agents', 'skills', 'deploy'), 'deploy', 'also mine\n');
skill(path.join(home, '.claude', 'skills', 'gstack', 'ship'), 'ship', '<!-- AUTO-GENERATED from SKILL.md.tmpl -->\n');
const amb = bill(tmp, ['deploy']);
expect(amb.status).toBe(2);
expect(amb.stderr).toContain('(SKILL_AMBIGUOUS)');
const installed = bill(tmp, ['ship']);
expect(installed.status).toBe(0);
expect(installed.stdout).toContain('(gstack-install)');
expect(installed.stdout).toContain('INSTALLED_GSTACK_SKILL');
const miss = bill(tmp, ['shp']);
expect(miss.status).toBe(2);
expect(miss.stderr).toContain('(SKILL_NOT_FOUND)');
expect(miss.stderr).toContain('closest: ship');
});
test('a folder without SKILL.md and an unknown flag are errors with a fix line', () => {
fs.mkdirSync(path.join(tmp, 'empty-dir'));
const missing = bill(tmp, ['./empty-dir']);
expect(missing.status).toBe(2);
expect(missing.stderr).toContain('(SKILL_MD_MISSING)');
expect(missing.stderr).toContain('fix:');
const flag = bill(tmp, ['demo', '--jsno']);
expect(flag.status).toBe(2);
expect(flag.stderr).toContain("unknown argument '--jsno'");
expect(bill(tmp, ['--help']).stdout).toContain('Usage: gstack-skill-bill');
});
});
+223
View File
@@ -0,0 +1,223 @@
/**
* GSTA-133 ratchet: every gstack bin a skill invokes exists, and every --flag a
* skill passes to it appears in that bin's parser. scripts/skill-bin-drift.ts
* reads templates, carved sections and Claude-host renders statically (it never
* executes a bin). Today's misses live in test/fixtures/skill-bin-drift-allowlist.json,
* which may only shrink: a new finding fails, and so does an entry that no
* longer reproduces. PERMISSIVE and NO_HELP bins are reported, never failed.
*/
import { afterAll, describe, expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { bunIsolationArgs } from '../lib/bun-isolation';
import {
ALLOWLIST_FILE,
FIX_LINE,
compareAllowlist,
formatFinding,
loadAllowlist,
scanRepo,
scanText,
} from '../scripts/skill-bin-drift';
const ROOT = path.resolve(import.meta.dir, '..');
const SCRIPT = path.join(ROOT, 'scripts', 'skill-bin-drift.ts');
describe('skill-bin-drift: repository ratchet', () => {
const report = scanRepo(ROOT);
const allowlist = loadAllowlist(path.join(ROOT, ALLOWLIST_FILE));
test('reads templates, carved sections and Claude-host renders', () => {
expect(report.inputs).toEqual(expect.arrayContaining(['learn/SKILL.md.tmpl', 'learn/SKILL.md', 'ship/sections/pr-body.md.tmpl', 'ship/sections/pr-body.md']));
expect(report.calls.length).toBeGreaterThan(500);
expect(report.binsInvoked.length).toBeGreaterThan(50);
});
test('no drift outside the shrink-only allowlist', () => {
const { unlisted } = compareAllowlist(report.findings, allowlist);
if (unlisted.length > 0) throw new Error(`${unlisted.length} skill-to-bin drift finding(s):\n${unlisted.map(formatFinding).join('\n')}`);
});
test('every allowlist entry still reproduces, so the list only shrinks', () => {
const { stale } = compareAllowlist(report.findings, allowlist);
if (stale.length > 0) throw new Error(`delete these entries from ${ALLOWLIST_FILE}; they no longer reproduce:\n${stale.join('\n')}`);
});
test('allowlist is sorted, unique and explains itself', () => {
expect(allowlist._comment).toContain('Shrink-only');
expect(allowlist.entries).toEqual([...new Set(allowlist.entries)].sort());
});
test('help backlog and permissive parsers are listed, not failed', () => {
for (const bin of [...report.noHelp, ...report.permissive]) expect(report.binsInvoked).toContain(bin);
for (const bin of ['gstack-learnings-search', 'gstack-learnings-log']) {
expect(report.permissive).not.toContain(bin);
expect(report.noHelp).not.toContain(bin);
}
});
});
describe('skill-bin-drift: fixtures', () => {
const roots: string[] = [];
afterAll(() => { for (const root of roots) fs.rmSync(root, { recursive: true, force: true }); });
function fixture(files: Record<string, string>): string {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-skill-bin-drift-'));
roots.push(root);
for (const [rel, text] of Object.entries(files)) {
fs.mkdirSync(path.dirname(path.join(root, rel)), { recursive: true });
fs.writeFileSync(path.join(root, rel), text);
}
return root;
}
const BASH_BIN = ['#!/usr/bin/env bash',
'# gstack-notes-search — read notes',
'# Usage: gstack-notes-search [--type TYPE] [--limit N]',
'set -euo pipefail',
'TYPE=""; LIMIT=10',
'while [[ $# -gt 0 ]]; do',
' case "$1" in',
' --type) TYPE="$2"; shift 2 ;;',
' --limit|--max) LIMIT="$2"; shift 2 ;;',
' -h|--help) sed -n 2,3p "$0"; exit 0 ;;',
' -*) echo "gstack-notes-search: unknown flag $1" >&2; exit 2 ;;',
' *) shift ;;',
' esac',
'done',
'echo "$TYPE $LIMIT"',
''].join('\n');
const PERMISSIVE_BIN = ['#!/usr/bin/env bash',
'while [[ $# -gt 0 ]]; do',
' case "$1" in',
' --type) TYPE="$2"; shift 2 ;;',
' *) shift ;;',
' esac',
'done',
''].join('\n');
const skill = (body: string) => `---\nname: demo\n---\n# Demo\n\n${body}\n`;
const fence = (cmd: string) => ['```bash', cmd, '```'].join('\n');
test('a real-shaped bash bin with case arms and a valid flag passes', () => {
const root = fixture({
'bin/gstack-notes-search': BASH_BIN,
'demo/SKILL.md.tmpl': skill(fence('~/.claude/skills/gstack/bin/gstack-notes-search --type pitfall --max 5 2>/dev/null || true')),
});
const report = scanRepo(root);
expect(report.calls).toEqual([{ file: 'demo/SKILL.md.tmpl', line: 7, bin: 'gstack-notes-search', flags: ['--type', '--max'] }]);
expect(report.findings).toEqual([]);
expect(report.permissive).toEqual([]);
expect(report.noHelp).toEqual([]);
});
test('negative control: an invented flag on a real bin fails with file:line and the fix line', () => {
const root = fixture({
'bin/gstack-notes-search': BASH_BIN,
'demo/SKILL.md.tmpl': skill(fence('~/.claude/skills/gstack/bin/gstack-notes-search --type pitfall --invented-flag')),
});
const [finding, ...rest] = scanRepo(root).findings;
expect(rest).toEqual([]);
expect(finding).toMatchObject({ code: 'DRIFT_UNKNOWN_FLAG', file: 'demo/SKILL.md.tmpl', line: 7, bin: 'gstack-notes-search', flag: '--invented-flag', key: 'demo/SKILL.md.tmpl:gstack-notes-search:--invented-flag', fix: FIX_LINE });
expect(formatFinding(finding!)).toContain('DRIFT_UNKNOWN_FLAG demo/SKILL.md.tmpl:7');
});
test('flags after a pipe, ;, &&, a redirect or $( ) belong to the next command', () => {
const cmds = [
'bin/gstack-notes-search --type a | grep --count --fixed-strings x',
'bin/gstack-notes-search --limit 3; git log --oneline',
'bin/gstack-notes-search && jq --raw-output .x',
'bin/gstack-notes-search --type a 2>/dev/null --bogus',
'X=$(bin/gstack-notes-search --type a) || sort --unique',
].join('\n');
const root = fixture({ 'bin/gstack-notes-search': BASH_BIN, 'demo/SKILL.md.tmpl': skill(fence(cmds)) });
const report = scanRepo(root);
expect(report.calls.map(c => c.flags)).toEqual([['--type'], ['--limit'], [], ['--type'], ['--type']]);
expect(report.findings).toEqual([]);
});
test('a TS bin with string-literal flags one relative import deep passes; two deep does not count', () => {
const root = fixture({
'bin/gstack-ts-tool': `#!/usr/bin/env -S bun --no-env-file --config=/dev/null\nimport { parse } from '../lib/tool-args';\nparse(process.argv.slice(2));\n`,
'lib/tool-args.ts': `import { deeper } from './tool-deeper';\nexport function parse(argv: string[]) { return argv.includes('--dry-run') || argv.includes("--json") || deeper(argv); }\n`,
'lib/tool-deeper.ts': `export function deeper(argv: string[]) { return argv.includes('--too-deep'); }\n`,
'demo/SKILL.md.tmpl': skill('Run `bin/gstack-ts-tool --dry-run --json`, then `bin/gstack-ts-tool --too-deep`.'),
});
const report = scanRepo(root);
expect(report.findings.map(f => f.key)).toEqual(['demo/SKILL.md.tmpl:gstack-ts-tool:--too-deep']);
});
test('a bash bin that hands off to a .ts file gains its flags', () => {
const root = fixture({
'bin/gstack-wrapper': '#!/usr/bin/env bash\n# Usage: gstack-wrapper [--fast]\nexec bun run "$(dirname "$0")/../lib/wrapper-impl.ts" "$@"\n',
'lib/wrapper-impl.ts': "const fast = process.argv.includes('--fast');\n",
'demo/SKILL.md.tmpl': skill(fence('$GSTACK_BIN/gstack-wrapper --fast')),
});
expect(scanRepo(root).findings).toEqual([]);
});
test('a catch-all arg loop is PERMISSIVE, and its unknown flags still fail', () => {
const root = fixture({
'bin/gstack-loose': PERMISSIVE_BIN,
'demo/SKILL.md.tmpl': skill(fence('{{BIN_DIR}}/gstack-loose --type x --ignored')),
});
const report = scanRepo(root);
expect(report.permissive).toEqual(['gstack-loose']);
expect(report.noHelp).toEqual(['gstack-loose']);
expect(report.findings.map(f => f.key)).toEqual(['demo/SKILL.md.tmpl:gstack-loose:--ignored']);
});
test('a missing bin is DRIFT_UNKNOWN_BIN keyed without a flag', () => {
const root = fixture({ 'bin/.keep': '', 'demo/SKILL.md.tmpl': skill(fence('~/.claude/skills/gstack/bin/gstack-gone --x')) });
expect(scanRepo(root).findings.map(f => [f.code, f.key])).toEqual([['DRIFT_UNKNOWN_BIN', 'demo/SKILL.md.tmpl:gstack-gone']]);
});
test('prose and placeholders do not false-positive', () => {
const text = [
'Pass bin/gstack-notes-search --nope <VALUE> in prose: not code, not scanned.',
'`bin/gstack-notes-search --type <TYPE> --limit {{LIMIT}}`',
fence('bin/gstack-notes-search --type <kind of note> --limit "$N" # --not-a-flag in a comment'),
].join('\n');
const calls = scanText(text, 'demo/SKILL.md.tmpl');
expect(calls.map(c => [c.line, c.flags])).toEqual([[2, ['--type', '--limit']], [4, ['--type', '--limit']]]);
});
test('a render finding its own template already reports is dropped; render-only findings stay', () => {
const root = fixture({
'bin/gstack-notes-search': BASH_BIN,
'demo/SKILL.md.tmpl': skill(fence('bin/gstack-notes-search --bad')),
'demo/SKILL.md': `<!-- AUTO-GENERATED from SKILL.md.tmpl — do not edit directly -->\n${fence('bin/gstack-notes-search --bad')}\n${fence('bin/gstack-notes-search --resolver-made')}\n`,
});
expect(scanRepo(root).findings.map(f => f.key)).toEqual(['demo/SKILL.md:gstack-notes-search:--resolver-made', 'demo/SKILL.md.tmpl:gstack-notes-search:--bad']);
});
test('allowlist comparison reports unlisted findings and stale entries', () => {
const root = fixture({ 'bin/gstack-notes-search': BASH_BIN, 'demo/SKILL.md.tmpl': skill(fence('bin/gstack-notes-search --bad')) });
const { findings } = scanRepo(root);
expect(compareAllowlist(findings, { entries: [] }).unlisted.map(f => f.key)).toEqual(['demo/SKILL.md.tmpl:gstack-notes-search:--bad']);
const result = compareAllowlist(findings, { entries: ['demo/SKILL.md.tmpl:gstack-notes-search:--bad', 'other/SKILL.md.tmpl:gstack-x:--y'] });
expect(result).toEqual({ unlisted: [], stale: ['other/SKILL.md.tmpl:gstack-x:--y'] });
expect(compareAllowlist(findings, { entries: ['other/SKILL.md.tmpl:gstack-x:--y'] }, ['demo/SKILL.md.tmpl']).stale).toEqual([]);
});
test('CLI: --skill scans one skill, --json reports, exit codes follow the allowlist', () => {
const root = fixture({
'bin/gstack-notes-search': BASH_BIN,
'demo/SKILL.md.tmpl': skill(fence('bin/gstack-notes-search --bad')),
'other/SKILL.md.tmpl': skill(fence('bin/gstack-notes-search --type ok')),
});
const run = (...args: string[]) => {
const child = Bun.spawnSync([process.execPath, ...bunIsolationArgs(), SCRIPT, '--root', root, ...args], { stdout: 'pipe', stderr: 'pipe', timeout: 20_000 });
return { code: child.exitCode, out: child.stdout.toString() + child.stderr.toString() };
};
const other = run('--skill', 'other', '--json');
expect(other.code).toBe(0);
expect(JSON.parse(other.out)).toMatchObject({ inputs: 1, calls: 1, unknownFlags: 0, unlisted: [] });
const demo = run('--skill', 'demo');
expect(demo.code).toBe(1);
expect(demo.out).toContain('DRIFT_UNKNOWN_FLAG demo/SKILL.md.tmpl:7 bin=gstack-notes-search flag=--bad');
expect(demo.out).toContain(FIX_LINE);
expect(run('--skill', 'missing').code).toBe(2);
expect(run('--help').code).toBe(0);
});
});
+77
View File
@@ -0,0 +1,77 @@
import { describe, expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_MS, JUDGE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, runId, describeIfSelected, testConcurrentIfSelected, copyDirSync, logCost, recordE2E,
createEvalCollector, finalizeEvalCollector,
} from './helpers/e2e-helpers';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
const evalCollector = createEvalCollector('e2e-write-skill');
const DEPLOY_SKILL = `---
name: deploy
description: Deploy the web app to production.
---
# Deploy
1. Build the release image with \`make release\`.
2. Run database migrations before restarting the web servers.
3. Restart the web servers one at a time.
4. Check the health endpoint returns 200.
`;
describeIfSelected('/write-skill E2E', ['write-skill-add-lesson'], () => {
let workDir: string;
beforeAll(() => {
workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-write-skill-'));
copyDirSync(path.join(ROOT, 'write-skill'), path.join(workDir, 'write-skill'));
fs.mkdirSync(path.join(workDir, 'deploy'));
fs.writeFileSync(path.join(workDir, 'deploy', 'SKILL.md'), DEPLOY_SKILL);
});
afterAll(() => {
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
finalizeEvalCollector(evalCollector);
});
testConcurrentIfSelected('write-skill-add-lesson', async () => {
const result = await runSkillTest({
prompt: `Read the file write-skill/SKILL.md for the /write-skill skill instructions.
Add this lesson to the deploy skill in ./deploy: "Last Tuesday we restarted the web servers while migrations were still running and the site returned 500s for ten minutes. Migrations must finish before any restart."
IMPORTANT:
- The gstack bin scripts are at ${ROOT}/bin/, not at ~/.claude/skills/gstack/bin/; use that path.
- ./deploy is a skill folder the user owns, not a git repository.
- Do NOT use AskUserQuestion. Edit the file, then report what you changed and its size before and after.`.replace('${ROOT}', ROOT),
workingDirectory: workDir,
maxTurns: 15,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: JUDGE_MS,
testName: 'write-skill-add-lesson',
runId,
});
logCost('/write-skill add lesson', result);
const after = fs.readFileSync(path.join(workDir, 'deploy', 'SKILL.md'), 'utf-8');
const grewBy = after.trimEnd().split('\n').length - DEPLOY_SKILL.trimEnd().split('\n').length;
const ruleLanded = /finish|complete|wait/i.test(after.split('\n').find(l => /migration/i.test(l)) ?? '');
const storyKept = /tuesday|500s?|ten minutes/i.test(after);
const reportedSize = /\d+\s*(B|bytes)?\s*(→|->|to)\s*\d+/i.test(result.output);
const exitOk = ['success', 'error_max_turns'].includes(result.exitReason);
recordE2E(evalCollector, '/write-skill', 'Add a lesson by editing the existing line', result, {
passed: exitOk && grewBy <= 1 && ruleLanded && !storyKept && reportedSize,
});
expect(exitOk).toBe(true);
expect(ruleLanded).toBe(true);
expect(storyKept).toBe(false);
expect(grewBy).toBeLessThanOrEqual(1);
expect(reportedSize).toBe(true);
}, CAPTURE_MS);
});