Files
gstack/test/skill-bill.test.ts
Garry Tanandcapy-ai[bot] 7204a39a27 v1.91.287.0 feat(write-skill): write, change and diet skills; /learn at half its template size (#3524)
* test(skill-evals): /learn before/after tasks, written before any cut (GSTA-133)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(learnings): --help, unknown-flag errors and --stats on the learnings bins; cross-project search no longer drops every project when the state root path contains the slug

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* refactor(learn): diet /learn onto the levels: tool flags replace the inline stats program, one mode table, prune rules kept

Skill-Diet: learn
Skill-Size: 5797→2586 bytes (SKILL.md.tmpl)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* feat(write-skill): /write-skill skill, gstack-skill-bill, static skill-to-bin drift ratchet, shared skill eval contract

Skill-Size: 0→3245 bytes (write-skill/SKILL.md.tmpl)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(learn): guard the state-root lookup in Prune; re-measure the learn ceiling

Skill-Size: 2586→2707 bytes (learn/SKILL.md.tmpl)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix: pre-landing review fixes (drift scanner, learnings flags, skill-bill baselines, eval fixtures)

- skill-bin-drift: a redirect no longer ends a command, so `bin/x 2>/dev/null --bogus` is checked
- gstack-learnings-search: a missing value or a non-positive --limit exits 2 like an unknown flag
- gstack-skill-bill: the gstack root skill is a checkout skill; Skill-Diet trailers match the exact name; the baseline counts the committed bytes untrimmed
- skill-evals task files get a free contract test and are registered in FREE_FIXTURES, with the drift allowlist
- learn eval: the no-code-change safety task no longer asks the model to fix code (its check penalised obeying the user)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(learn,write-skill): full bin paths in the mode table; prune asks per entry unless the request names the action

Skill-Diet: learn
Skill-Size: 2707→2955 bytes (learn/SKILL.md.tmpl)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* docs(changelog): /write-skill, gstack-skill-bill, drift ratchet, eval contract, /learn diet

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* docs: drift ratchet in CONTRIBUTING's static tests; /learn prune example asks per entry

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(skill-bill): report a checkout skill's eval task file and check it against the contract

The skill-evals task-file test now exercises lib/skill-bill.ts's checkEvalTasks, with a negative control, instead of a test-only validator (test-of-test ratchet).

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(skill-bill): EVAL TASKS path uses forward slashes on every platform; a non-object task entry is a problem, not a crash

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(learn): BASH_COMPAT guard on in-window heredocs; count skill-e2e-write-skill in the paid census

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

---------

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
2026-10-28 02:58:29 +00:00

149 lines
7.1 KiB
TypeScript

import { describe, test, expect, afterAll } from 'bun:test';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import { spawnSync } from 'child_process';
const BIN = path.resolve(import.meta.dir, '..', 'bin', 'gstack-skill-bill');
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-bill-'));
const home = path.join(tmp, 'home');
fs.mkdirSync(home);
function sh(cwd: string, cmd: string, args: string[]) {
const r = spawnSync(cmd, args, { cwd, encoding: 'utf-8', timeout: 30_000 });
if (r.status !== 0) throw new Error(`${cmd} ${args.join(' ')} failed: ${r.stderr}`);
return r.stdout.trim();
}
function bill(cwd: string, args: string[]) {
return spawnSync(BIN, args, { cwd, encoding: 'utf-8', timeout: 30_000, env: { ...process.env, HOME: home, CODEX_HOME: path.join(home, '.codex') } });
}
function skill(dir: string, name: string, body: string, file = 'SKILL.md') {
fs.mkdirSync(dir, { recursive: true });
fs.writeFileSync(path.join(dir, file), `---\nname: ${name}\ndescription: |\n Does ${name} things.\n---\n${body}`);
}
function gstackRepo(): string {
const root = path.join(tmp, 'gstack');
fs.mkdirSync(path.join(root, 'bin'), { recursive: true });
fs.writeFileSync(path.join(root, 'SKILL.md.tmpl'), 'router');
fs.writeFileSync(path.join(root, 'VERSION'), '1.0.0');
skill(path.join(root, 'demo'), 'demo', 'x'.repeat(1000) + '\n', 'SKILL.md.tmpl');
fs.writeFileSync(path.join(root, 'demo', 'SKILL.md'), '---\nname: demo\ndescription: Does demo things.\n---\n<!-- AUTO-GENERATED from SKILL.md.tmpl -->\nbody\n');
fs.mkdirSync(path.join(root, 'demo', 'references'));
fs.writeFileSync(path.join(root, 'demo', 'references', 'deep.md'), 'deep\n');
sh(root, 'git', ['init', '-q', '-b', 'main']);
sh(root, 'git', ['config', 'user.email', 'test@example.com']);
sh(root, 'git', ['config', 'user.name', 'Test']);
sh(root, 'git', ['add', '-A']);
sh(root, 'git', ['commit', '-qm', 'diet demo\n\nSkill-Diet: demo']);
return root;
}
afterAll(() => fs.rmSync(tmp, { recursive: true, force: true }));
describe('gstack-skill-bill', () => {
const root = gstackRepo();
test('a gstack checkout skill reports every level and its last diet', () => {
const r = bill(root, ['demo']);
expect(r.status).toBe(0);
expect(r.stdout).toContain('SKILL: demo (gstack-checkout)');
expect(r.stdout).toMatch(/template\s+SKILL\.md\.tmpl/);
expect(r.stdout).toMatch(/on-demand\s+references\/deep\.md/);
expect(r.stdout).toContain('BASELINE: ');
expect(r.stdout).not.toContain('DUE');
});
test('growth over 10% since the Skill-Diet commit prints DUE, and an unrendered template is RENDER_STALE', () => {
fs.appendFileSync(path.join(root, 'demo', 'SKILL.md.tmpl'), 'y'.repeat(200));
const r = bill(root, ['demo', '--json']);
const report = JSON.parse(r.stdout);
expect(report.baseline.status).toBe('found');
expect(report.baseline.due).toBe(true);
expect(report.warnings.join('\n')).toContain('RENDER_STALE');
sh(root, 'git', ['checkout', '--', 'demo/SKILL.md.tmpl']);
});
test('a shallow clone says the baseline is unknown instead of none', () => {
const shallow = path.join(tmp, 'shallow');
sh(root, 'git', ['commit', '-q', '--allow-empty', '-m', 'later']);
sh(tmp, 'git', ['clone', '-q', '--depth', '1', 'file://' + root, shallow]);
const report = JSON.parse(bill(shallow, ['demo', '--json']).stdout);
expect(report.baseline.status).toBe('unknown');
expect(report.baseline.reason).toContain('shallow');
});
test('a standalone folder outside git is a user skill with an unknown baseline', () => {
const own = path.join(tmp, 'mine', 'deploy');
skill(own, 'deploy', 'steps\n');
const r = bill(tmp, ['./mine/deploy']);
expect(r.status).toBe(0);
expect(r.stdout).toContain('SKILL: deploy (user)');
expect(r.stdout).toContain('not a git folder');
});
test('names resolve exactly: an installed gstack copy yields to the user skill, a near miss lists the closest names', () => {
skill(path.join(home, '.claude', 'skills', 'deploy'), 'deploy', 'mine\n');
skill(path.join(home, '.agents', 'skills', 'deploy'), 'deploy', 'also mine\n');
skill(path.join(home, '.claude', 'skills', 'gstack', 'ship'), 'ship', '<!-- AUTO-GENERATED from SKILL.md.tmpl -->\n');
const amb = bill(tmp, ['deploy']);
expect(amb.status).toBe(2);
expect(amb.stderr).toContain('(SKILL_AMBIGUOUS)');
const installed = bill(tmp, ['ship']);
expect(installed.status).toBe(0);
expect(installed.stdout).toContain('(gstack-install)');
expect(installed.stdout).toContain('INSTALLED_GSTACK_SKILL');
const miss = bill(tmp, ['shp']);
expect(miss.status).toBe(2);
expect(miss.stderr).toContain('(SKILL_NOT_FOUND)');
expect(miss.stderr).toContain('closest: ship');
});
test('a folder without SKILL.md and an unknown flag are errors with a fix line', () => {
fs.mkdirSync(path.join(tmp, 'empty-dir'));
const missing = bill(tmp, ['./empty-dir']);
expect(missing.status).toBe(2);
expect(missing.stderr).toContain('(SKILL_MD_MISSING)');
expect(missing.stderr).toContain('fix:');
const flag = bill(tmp, ['demo', '--jsno']);
expect(flag.status).toBe(2);
expect(flag.stderr).toContain("unknown argument '--jsno'");
expect(bill(tmp, ['--help']).stdout).toContain('Usage: gstack-skill-bill');
});
});
describe('gstack-skill-bill baselines and the root skill', () => {
test('the gstack root skill is a checkout skill, and Skill-Diet: gstack-upgrade is not its diet', () => {
const root = path.join(tmp, 'gstack-root');
fs.mkdirSync(path.join(root, 'bin'), { recursive: true });
skill(root, 'gstack', 'router\n', 'SKILL.md.tmpl');
fs.writeFileSync(path.join(root, 'SKILL.md'), '---\nname: gstack\ndescription: Does gstack things.\n---\n<!-- AUTO-GENERATED from SKILL.md.tmpl -->\nrouter\n');
fs.writeFileSync(path.join(root, 'VERSION'), '1.0.0');
skill(path.join(root, 'gstack-upgrade'), 'gstack-upgrade', 'up\n');
sh(root, 'git', ['init', '-q', '-b', 'main']);
sh(root, 'git', ['config', 'user.email', 'test@example.com']);
sh(root, 'git', ['config', 'user.name', 'Test']);
sh(root, 'git', ['add', '-A']);
sh(root, 'git', ['commit', '-qm', 'diet upgrade\n\nSkill-Diet: gstack-upgrade']);
const report = JSON.parse(bill(root, ['.', '--json']).stdout);
expect(report.kind).toBe('gstack-checkout');
expect(report.baseline.status).toBe('none');
});
test('the baseline measures the committed bytes exactly, trailing blank lines included', () => {
const repo = path.join(tmp, 'blank-lines');
skill(path.join(repo, 'notes'), 'notes', 'body\n\n\n');
sh(repo, 'git', ['init', '-q', '-b', 'main']);
sh(repo, 'git', ['config', 'user.email', 'test@example.com']);
sh(repo, 'git', ['config', 'user.name', 'Test']);
sh(repo, 'git', ['add', '-A']);
sh(repo, 'git', ['commit', '-qm', 'diet notes\n\nSkill-Diet: notes']);
const report = JSON.parse(bill(repo, ['./notes', '--json']).stdout);
expect(report.baseline.status).toBe('found');
expect(report.baseline.bytes).toBe(fs.statSync(path.join(repo, 'notes', 'SKILL.md')).size);
expect(report.baseline.growthPct).toBe(0);
});
});