/** * GSTA-133 ratchet: every gstack bin a skill invokes exists, and every --flag a * skill passes to it appears in that bin's parser. scripts/skill-bin-drift.ts * reads templates, carved sections and Claude-host renders statically (it never * executes a bin). Today's misses live in test/fixtures/skill-bin-drift-allowlist.json, * which may only shrink: a new finding fails, and so does an entry that no * longer reproduces. PERMISSIVE and NO_HELP bins are reported, never failed. */ import { afterAll, describe, expect, test } from 'bun:test'; import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { bunIsolationArgs } from '../lib/bun-isolation'; import { ALLOWLIST_FILE, FIX_LINE, compareAllowlist, formatFinding, loadAllowlist, scanRepo, scanText, } from '../scripts/skill-bin-drift'; const ROOT = path.resolve(import.meta.dir, '..'); const SCRIPT = path.join(ROOT, 'scripts', 'skill-bin-drift.ts'); describe('skill-bin-drift: repository ratchet', () => { const report = scanRepo(ROOT); const allowlist = loadAllowlist(path.join(ROOT, ALLOWLIST_FILE)); test('reads templates, carved sections and Claude-host renders', () => { expect(report.inputs).toEqual(expect.arrayContaining(['learn/SKILL.md.tmpl', 'learn/SKILL.md', 'ship/sections/pr-body.md.tmpl', 'ship/sections/pr-body.md'])); expect(report.calls.length).toBeGreaterThan(500); expect(report.binsInvoked.length).toBeGreaterThan(50); }); test('no drift outside the shrink-only allowlist', () => { const { unlisted } = compareAllowlist(report.findings, allowlist); if (unlisted.length > 0) throw new Error(`${unlisted.length} skill-to-bin drift finding(s):\n${unlisted.map(formatFinding).join('\n')}`); }); test('every allowlist entry still reproduces, so the list only shrinks', () => { const { stale } = compareAllowlist(report.findings, allowlist); if (stale.length > 0) throw new Error(`delete these entries from ${ALLOWLIST_FILE}; they no longer reproduce:\n${stale.join('\n')}`); }); test('allowlist is sorted, unique and explains itself', () => { expect(allowlist._comment).toContain('Shrink-only'); expect(allowlist.entries).toEqual([...new Set(allowlist.entries)].sort()); }); test('help backlog and permissive parsers are listed, not failed', () => { for (const bin of [...report.noHelp, ...report.permissive]) expect(report.binsInvoked).toContain(bin); for (const bin of ['gstack-learnings-search', 'gstack-learnings-log']) { expect(report.permissive).not.toContain(bin); expect(report.noHelp).not.toContain(bin); } }); }); describe('skill-bin-drift: fixtures', () => { const roots: string[] = []; afterAll(() => { for (const root of roots) fs.rmSync(root, { recursive: true, force: true }); }); function fixture(files: Record): string { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-skill-bin-drift-')); roots.push(root); for (const [rel, text] of Object.entries(files)) { fs.mkdirSync(path.dirname(path.join(root, rel)), { recursive: true }); fs.writeFileSync(path.join(root, rel), text); } return root; } const BASH_BIN = ['#!/usr/bin/env bash', '# gstack-notes-search — read notes', '# Usage: gstack-notes-search [--type TYPE] [--limit N]', 'set -euo pipefail', 'TYPE=""; LIMIT=10', 'while [[ $# -gt 0 ]]; do', ' case "$1" in', ' --type) TYPE="$2"; shift 2 ;;', ' --limit|--max) LIMIT="$2"; shift 2 ;;', ' -h|--help) sed -n 2,3p "$0"; exit 0 ;;', ' -*) echo "gstack-notes-search: unknown flag $1" >&2; exit 2 ;;', ' *) shift ;;', ' esac', 'done', 'echo "$TYPE $LIMIT"', ''].join('\n'); const PERMISSIVE_BIN = ['#!/usr/bin/env bash', 'while [[ $# -gt 0 ]]; do', ' case "$1" in', ' --type) TYPE="$2"; shift 2 ;;', ' *) shift ;;', ' esac', 'done', ''].join('\n'); const skill = (body: string) => `---\nname: demo\n---\n# Demo\n\n${body}\n`; const fence = (cmd: string) => ['```bash', cmd, '```'].join('\n'); test('a real-shaped bash bin with case arms and a valid flag passes', () => { const root = fixture({ 'bin/gstack-notes-search': BASH_BIN, 'demo/SKILL.md.tmpl': skill(fence('~/.claude/skills/gstack/bin/gstack-notes-search --type pitfall --max 5 2>/dev/null || true')), }); const report = scanRepo(root); expect(report.calls).toEqual([{ file: 'demo/SKILL.md.tmpl', line: 7, bin: 'gstack-notes-search', flags: ['--type', '--max'] }]); expect(report.findings).toEqual([]); expect(report.permissive).toEqual([]); expect(report.noHelp).toEqual([]); }); test('negative control: an invented flag on a real bin fails with file:line and the fix line', () => { const root = fixture({ 'bin/gstack-notes-search': BASH_BIN, 'demo/SKILL.md.tmpl': skill(fence('~/.claude/skills/gstack/bin/gstack-notes-search --type pitfall --invented-flag')), }); const [finding, ...rest] = scanRepo(root).findings; expect(rest).toEqual([]); expect(finding).toMatchObject({ code: 'DRIFT_UNKNOWN_FLAG', file: 'demo/SKILL.md.tmpl', line: 7, bin: 'gstack-notes-search', flag: '--invented-flag', key: 'demo/SKILL.md.tmpl:gstack-notes-search:--invented-flag', fix: FIX_LINE }); expect(formatFinding(finding!)).toContain('DRIFT_UNKNOWN_FLAG demo/SKILL.md.tmpl:7'); }); test('flags after a pipe, ;, && or $( ) belong to the next command', () => { const cmds = [ 'bin/gstack-notes-search --type a | grep --count --fixed-strings x', 'bin/gstack-notes-search --limit 3; git log --oneline', 'bin/gstack-notes-search && jq --raw-output .x', 'bin/gstack-notes-search --limit 2 2>&1 | grep --count x', 'X=$(bin/gstack-notes-search --type a) || sort --unique', ].join('\n'); const root = fixture({ 'bin/gstack-notes-search': BASH_BIN, 'demo/SKILL.md.tmpl': skill(fence(cmds)) }); const report = scanRepo(root); expect(report.calls.map(c => c.flags)).toEqual([['--type'], ['--limit'], [], ['--limit'], ['--type']]); expect(report.findings).toEqual([]); }); test('a redirect does not end the command: flags after it are still checked', () => { const cmds = [ 'bin/gstack-notes-search --type a 2>/dev/null --bogus', 'bin/gstack-notes-search >> out.txt --max 3', 'bin/gstack-notes-search 2> err.log --limit 1', ].join('\n'); const root = fixture({ 'bin/gstack-notes-search': BASH_BIN, 'demo/SKILL.md.tmpl': skill(fence(cmds)) }); const report = scanRepo(root); expect(report.calls.map(c => c.flags)).toEqual([['--type', '--bogus'], ['--max'], ['--limit']]); expect(report.findings.map(f => f.flag)).toEqual(['--bogus']); }); test('a TS bin with string-literal flags one relative import deep passes; two deep does not count', () => { const root = fixture({ 'bin/gstack-ts-tool': `#!/usr/bin/env -S bun --no-env-file --config=/dev/null\nimport { parse } from '../lib/tool-args';\nparse(process.argv.slice(2));\n`, 'lib/tool-args.ts': `import { deeper } from './tool-deeper';\nexport function parse(argv: string[]) { return argv.includes('--dry-run') || argv.includes("--json") || deeper(argv); }\n`, 'lib/tool-deeper.ts': `export function deeper(argv: string[]) { return argv.includes('--too-deep'); }\n`, 'demo/SKILL.md.tmpl': skill('Run `bin/gstack-ts-tool --dry-run --json`, then `bin/gstack-ts-tool --too-deep`.'), }); const report = scanRepo(root); expect(report.findings.map(f => f.key)).toEqual(['demo/SKILL.md.tmpl:gstack-ts-tool:--too-deep']); }); test('a bash bin that hands off to a .ts file gains its flags', () => { const root = fixture({ 'bin/gstack-wrapper': '#!/usr/bin/env bash\n# Usage: gstack-wrapper [--fast]\nexec bun run "$(dirname "$0")/../lib/wrapper-impl.ts" "$@"\n', 'lib/wrapper-impl.ts': "const fast = process.argv.includes('--fast');\n", 'demo/SKILL.md.tmpl': skill(fence('$GSTACK_BIN/gstack-wrapper --fast')), }); expect(scanRepo(root).findings).toEqual([]); }); test('a catch-all arg loop is PERMISSIVE, and its unknown flags still fail', () => { const root = fixture({ 'bin/gstack-loose': PERMISSIVE_BIN, 'demo/SKILL.md.tmpl': skill(fence('{{BIN_DIR}}/gstack-loose --type x --ignored')), }); const report = scanRepo(root); expect(report.permissive).toEqual(['gstack-loose']); expect(report.noHelp).toEqual(['gstack-loose']); expect(report.findings.map(f => f.key)).toEqual(['demo/SKILL.md.tmpl:gstack-loose:--ignored']); }); test('a missing bin is DRIFT_UNKNOWN_BIN keyed without a flag', () => { const root = fixture({ 'bin/.keep': '', 'demo/SKILL.md.tmpl': skill(fence('~/.claude/skills/gstack/bin/gstack-gone --x')) }); expect(scanRepo(root).findings.map(f => [f.code, f.key])).toEqual([['DRIFT_UNKNOWN_BIN', 'demo/SKILL.md.tmpl:gstack-gone']]); }); test('prose and placeholders do not false-positive', () => { const text = [ 'Pass bin/gstack-notes-search --nope in prose: not code, not scanned.', '`bin/gstack-notes-search --type --limit {{LIMIT}}`', fence('bin/gstack-notes-search --type --limit "$N" # --not-a-flag in a comment'), ].join('\n'); const calls = scanText(text, 'demo/SKILL.md.tmpl'); expect(calls.map(c => [c.line, c.flags])).toEqual([[2, ['--type', '--limit']], [4, ['--type', '--limit']]]); }); test('a render finding its own template already reports is dropped; render-only findings stay', () => { const root = fixture({ 'bin/gstack-notes-search': BASH_BIN, 'demo/SKILL.md.tmpl': skill(fence('bin/gstack-notes-search --bad')), 'demo/SKILL.md': `\n${fence('bin/gstack-notes-search --bad')}\n${fence('bin/gstack-notes-search --resolver-made')}\n`, }); expect(scanRepo(root).findings.map(f => f.key)).toEqual(['demo/SKILL.md:gstack-notes-search:--resolver-made', 'demo/SKILL.md.tmpl:gstack-notes-search:--bad']); }); test('allowlist comparison reports unlisted findings and stale entries', () => { const root = fixture({ 'bin/gstack-notes-search': BASH_BIN, 'demo/SKILL.md.tmpl': skill(fence('bin/gstack-notes-search --bad')) }); const { findings } = scanRepo(root); expect(compareAllowlist(findings, { entries: [] }).unlisted.map(f => f.key)).toEqual(['demo/SKILL.md.tmpl:gstack-notes-search:--bad']); const result = compareAllowlist(findings, { entries: ['demo/SKILL.md.tmpl:gstack-notes-search:--bad', 'other/SKILL.md.tmpl:gstack-x:--y'] }); expect(result).toEqual({ unlisted: [], stale: ['other/SKILL.md.tmpl:gstack-x:--y'] }); expect(compareAllowlist(findings, { entries: ['other/SKILL.md.tmpl:gstack-x:--y'] }, ['demo/SKILL.md.tmpl']).stale).toEqual([]); }); test('CLI: --skill scans one skill, --json reports, exit codes follow the allowlist', () => { const root = fixture({ 'bin/gstack-notes-search': BASH_BIN, 'demo/SKILL.md.tmpl': skill(fence('bin/gstack-notes-search --bad')), 'other/SKILL.md.tmpl': skill(fence('bin/gstack-notes-search --type ok')), }); const run = (...args: string[]) => { const child = Bun.spawnSync([process.execPath, ...bunIsolationArgs(), SCRIPT, '--root', root, ...args], { stdout: 'pipe', stderr: 'pipe', timeout: 20_000 }); return { code: child.exitCode, out: child.stdout.toString() + child.stderr.toString() }; }; const other = run('--skill', 'other', '--json'); expect(other.code).toBe(0); expect(JSON.parse(other.out)).toMatchObject({ inputs: 1, calls: 1, unknownFlags: 0, unlisted: [] }); const demo = run('--skill', 'demo'); expect(demo.code).toBe(1); expect(demo.out).toContain('DRIFT_UNKNOWN_FLAG demo/SKILL.md.tmpl:7 bin=gstack-notes-search flag=--bad'); expect(demo.out).toContain(FIX_LINE); expect(run('--skill', 'missing').code).toBe(2); expect(run('--help').code).toBe(0); }); });