mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-31 14:06:50 +01:00
* test(skill-evals): /learn before/after tasks, written before any cut (GSTA-133) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(learnings): --help, unknown-flag errors and --stats on the learnings bins; cross-project search no longer drops every project when the state root path contains the slug Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * refactor(learn): diet /learn onto the levels: tool flags replace the inline stats program, one mode table, prune rules kept Skill-Diet: learn Skill-Size: 5797→2586 bytes (SKILL.md.tmpl) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * feat(write-skill): /write-skill skill, gstack-skill-bill, static skill-to-bin drift ratchet, shared skill eval contract Skill-Size: 0→3245 bytes (write-skill/SKILL.md.tmpl) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(learn): guard the state-root lookup in Prune; re-measure the learn ceiling Skill-Size: 2586→2707 bytes (learn/SKILL.md.tmpl) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: pre-landing review fixes (drift scanner, learnings flags, skill-bill baselines, eval fixtures) - skill-bin-drift: a redirect no longer ends a command, so `bin/x 2>/dev/null --bogus` is checked - gstack-learnings-search: a missing value or a non-positive --limit exits 2 like an unknown flag - gstack-skill-bill: the gstack root skill is a checkout skill; Skill-Diet trailers match the exact name; the baseline counts the committed bytes untrimmed - skill-evals task files get a free contract test and are registered in FREE_FIXTURES, with the drift allowlist - learn eval: the no-code-change safety task no longer asks the model to fix code (its check penalised obeying the user) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(learn,write-skill): full bin paths in the mode table; prune asks per entry unless the request names the action Skill-Diet: learn Skill-Size: 2707→2955 bytes (learn/SKILL.md.tmpl) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(changelog): /write-skill, gstack-skill-bill, drift ratchet, eval contract, /learn diet Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs: drift ratchet in CONTRIBUTING's static tests; /learn prune example asks per entry Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(skill-bill): report a checkout skill's eval task file and check it against the contract The skill-evals task-file test now exercises lib/skill-bill.ts's checkEvalTasks, with a negative control, instead of a test-only validator (test-of-test ratchet). Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(skill-bill): EVAL TASKS path uses forward slashes on every platform; a non-object task entry is a problem, not a crash Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(learn): BASH_COMPAT guard on in-window heredocs; count skill-e2e-write-skill in the paid census Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
149 lines
7.1 KiB
TypeScript
149 lines
7.1 KiB
TypeScript
import { describe, test, expect, afterAll } from 'bun:test';
|
|
import * as fs from 'fs';
|
|
import * as path from 'path';
|
|
import * as os from 'os';
|
|
import { spawnSync } from 'child_process';
|
|
|
|
const BIN = path.resolve(import.meta.dir, '..', 'bin', 'gstack-skill-bill');
|
|
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-bill-'));
|
|
const home = path.join(tmp, 'home');
|
|
fs.mkdirSync(home);
|
|
|
|
function sh(cwd: string, cmd: string, args: string[]) {
|
|
const r = spawnSync(cmd, args, { cwd, encoding: 'utf-8', timeout: 30_000 });
|
|
if (r.status !== 0) throw new Error(`${cmd} ${args.join(' ')} failed: ${r.stderr}`);
|
|
return r.stdout.trim();
|
|
}
|
|
|
|
function bill(cwd: string, args: string[]) {
|
|
return spawnSync(BIN, args, { cwd, encoding: 'utf-8', timeout: 30_000, env: { ...process.env, HOME: home, CODEX_HOME: path.join(home, '.codex') } });
|
|
}
|
|
|
|
function skill(dir: string, name: string, body: string, file = 'SKILL.md') {
|
|
fs.mkdirSync(dir, { recursive: true });
|
|
fs.writeFileSync(path.join(dir, file), `---\nname: ${name}\ndescription: |\n Does ${name} things.\n---\n${body}`);
|
|
}
|
|
|
|
function gstackRepo(): string {
|
|
const root = path.join(tmp, 'gstack');
|
|
fs.mkdirSync(path.join(root, 'bin'), { recursive: true });
|
|
fs.writeFileSync(path.join(root, 'SKILL.md.tmpl'), 'router');
|
|
fs.writeFileSync(path.join(root, 'VERSION'), '1.0.0');
|
|
skill(path.join(root, 'demo'), 'demo', 'x'.repeat(1000) + '\n', 'SKILL.md.tmpl');
|
|
fs.writeFileSync(path.join(root, 'demo', 'SKILL.md'), '---\nname: demo\ndescription: Does demo things.\n---\n<!-- AUTO-GENERATED from SKILL.md.tmpl -->\nbody\n');
|
|
fs.mkdirSync(path.join(root, 'demo', 'references'));
|
|
fs.writeFileSync(path.join(root, 'demo', 'references', 'deep.md'), 'deep\n');
|
|
sh(root, 'git', ['init', '-q', '-b', 'main']);
|
|
sh(root, 'git', ['config', 'user.email', 'test@example.com']);
|
|
sh(root, 'git', ['config', 'user.name', 'Test']);
|
|
sh(root, 'git', ['add', '-A']);
|
|
sh(root, 'git', ['commit', '-qm', 'diet demo\n\nSkill-Diet: demo']);
|
|
return root;
|
|
}
|
|
|
|
afterAll(() => fs.rmSync(tmp, { recursive: true, force: true }));
|
|
|
|
describe('gstack-skill-bill', () => {
|
|
const root = gstackRepo();
|
|
|
|
test('a gstack checkout skill reports every level and its last diet', () => {
|
|
const r = bill(root, ['demo']);
|
|
expect(r.status).toBe(0);
|
|
expect(r.stdout).toContain('SKILL: demo (gstack-checkout)');
|
|
expect(r.stdout).toMatch(/template\s+SKILL\.md\.tmpl/);
|
|
expect(r.stdout).toMatch(/on-demand\s+references\/deep\.md/);
|
|
expect(r.stdout).toContain('BASELINE: ');
|
|
expect(r.stdout).not.toContain('DUE');
|
|
});
|
|
|
|
test('growth over 10% since the Skill-Diet commit prints DUE, and an unrendered template is RENDER_STALE', () => {
|
|
fs.appendFileSync(path.join(root, 'demo', 'SKILL.md.tmpl'), 'y'.repeat(200));
|
|
const r = bill(root, ['demo', '--json']);
|
|
const report = JSON.parse(r.stdout);
|
|
expect(report.baseline.status).toBe('found');
|
|
expect(report.baseline.due).toBe(true);
|
|
expect(report.warnings.join('\n')).toContain('RENDER_STALE');
|
|
sh(root, 'git', ['checkout', '--', 'demo/SKILL.md.tmpl']);
|
|
});
|
|
|
|
test('a shallow clone says the baseline is unknown instead of none', () => {
|
|
const shallow = path.join(tmp, 'shallow');
|
|
sh(root, 'git', ['commit', '-q', '--allow-empty', '-m', 'later']);
|
|
sh(tmp, 'git', ['clone', '-q', '--depth', '1', 'file://' + root, shallow]);
|
|
const report = JSON.parse(bill(shallow, ['demo', '--json']).stdout);
|
|
expect(report.baseline.status).toBe('unknown');
|
|
expect(report.baseline.reason).toContain('shallow');
|
|
});
|
|
|
|
test('a standalone folder outside git is a user skill with an unknown baseline', () => {
|
|
const own = path.join(tmp, 'mine', 'deploy');
|
|
skill(own, 'deploy', 'steps\n');
|
|
const r = bill(tmp, ['./mine/deploy']);
|
|
expect(r.status).toBe(0);
|
|
expect(r.stdout).toContain('SKILL: deploy (user)');
|
|
expect(r.stdout).toContain('not a git folder');
|
|
});
|
|
|
|
test('names resolve exactly: an installed gstack copy yields to the user skill, a near miss lists the closest names', () => {
|
|
skill(path.join(home, '.claude', 'skills', 'deploy'), 'deploy', 'mine\n');
|
|
skill(path.join(home, '.agents', 'skills', 'deploy'), 'deploy', 'also mine\n');
|
|
skill(path.join(home, '.claude', 'skills', 'gstack', 'ship'), 'ship', '<!-- AUTO-GENERATED from SKILL.md.tmpl -->\n');
|
|
const amb = bill(tmp, ['deploy']);
|
|
expect(amb.status).toBe(2);
|
|
expect(amb.stderr).toContain('(SKILL_AMBIGUOUS)');
|
|
const installed = bill(tmp, ['ship']);
|
|
expect(installed.status).toBe(0);
|
|
expect(installed.stdout).toContain('(gstack-install)');
|
|
expect(installed.stdout).toContain('INSTALLED_GSTACK_SKILL');
|
|
const miss = bill(tmp, ['shp']);
|
|
expect(miss.status).toBe(2);
|
|
expect(miss.stderr).toContain('(SKILL_NOT_FOUND)');
|
|
expect(miss.stderr).toContain('closest: ship');
|
|
});
|
|
|
|
test('a folder without SKILL.md and an unknown flag are errors with a fix line', () => {
|
|
fs.mkdirSync(path.join(tmp, 'empty-dir'));
|
|
const missing = bill(tmp, ['./empty-dir']);
|
|
expect(missing.status).toBe(2);
|
|
expect(missing.stderr).toContain('(SKILL_MD_MISSING)');
|
|
expect(missing.stderr).toContain('fix:');
|
|
const flag = bill(tmp, ['demo', '--jsno']);
|
|
expect(flag.status).toBe(2);
|
|
expect(flag.stderr).toContain("unknown argument '--jsno'");
|
|
expect(bill(tmp, ['--help']).stdout).toContain('Usage: gstack-skill-bill');
|
|
});
|
|
});
|
|
|
|
describe('gstack-skill-bill baselines and the root skill', () => {
|
|
test('the gstack root skill is a checkout skill, and Skill-Diet: gstack-upgrade is not its diet', () => {
|
|
const root = path.join(tmp, 'gstack-root');
|
|
fs.mkdirSync(path.join(root, 'bin'), { recursive: true });
|
|
skill(root, 'gstack', 'router\n', 'SKILL.md.tmpl');
|
|
fs.writeFileSync(path.join(root, 'SKILL.md'), '---\nname: gstack\ndescription: Does gstack things.\n---\n<!-- AUTO-GENERATED from SKILL.md.tmpl -->\nrouter\n');
|
|
fs.writeFileSync(path.join(root, 'VERSION'), '1.0.0');
|
|
skill(path.join(root, 'gstack-upgrade'), 'gstack-upgrade', 'up\n');
|
|
sh(root, 'git', ['init', '-q', '-b', 'main']);
|
|
sh(root, 'git', ['config', 'user.email', 'test@example.com']);
|
|
sh(root, 'git', ['config', 'user.name', 'Test']);
|
|
sh(root, 'git', ['add', '-A']);
|
|
sh(root, 'git', ['commit', '-qm', 'diet upgrade\n\nSkill-Diet: gstack-upgrade']);
|
|
const report = JSON.parse(bill(root, ['.', '--json']).stdout);
|
|
expect(report.kind).toBe('gstack-checkout');
|
|
expect(report.baseline.status).toBe('none');
|
|
});
|
|
|
|
test('the baseline measures the committed bytes exactly, trailing blank lines included', () => {
|
|
const repo = path.join(tmp, 'blank-lines');
|
|
skill(path.join(repo, 'notes'), 'notes', 'body\n\n\n');
|
|
sh(repo, 'git', ['init', '-q', '-b', 'main']);
|
|
sh(repo, 'git', ['config', 'user.email', 'test@example.com']);
|
|
sh(repo, 'git', ['config', 'user.name', 'Test']);
|
|
sh(repo, 'git', ['add', '-A']);
|
|
sh(repo, 'git', ['commit', '-qm', 'diet notes\n\nSkill-Diet: notes']);
|
|
const report = JSON.parse(bill(repo, ['./notes', '--json']).stdout);
|
|
expect(report.baseline.status).toBe('found');
|
|
expect(report.baseline.bytes).toBe(fs.statSync(path.join(repo, 'notes', 'SKILL.md')).size);
|
|
expect(report.baseline.growthPct).toBe(0);
|
|
});
|
|
});
|