mirror of
https://github.com/garrytan/gstack.git
synced 2026-05-02 11:45:20 +02:00
test: Tier 1 static validation — 34 tests for SKILL.md command correctness
- test/helpers/skill-parser.ts: extracts $B commands from code blocks, validates against registry - test/skill-parser.test.ts: 13 parser/validator unit tests - test/skill-validation.test.ts: 13 tests validating all SKILL.md files + registry consistency - test/gen-skill-docs.test.ts: 8 generator tests (categories, sorting, freshness)
This commit is contained in:
Garry Tan
@@ -0,0 +1,73 @@
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { COMMAND_DESCRIPTIONS } from '../browse/src/commands';
|
||||
import { SNAPSHOT_FLAGS } from '../browse/src/snapshot';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
|
||||
describe('gen-skill-docs', () => {
|
||||
test('generated SKILL.md contains all command categories', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
|
||||
const categories = new Set(Object.values(COMMAND_DESCRIPTIONS).map(d => d.category));
|
||||
for (const cat of categories) {
|
||||
expect(content).toContain(`### ${cat}`);
|
||||
}
|
||||
});
|
||||
|
||||
test('generated SKILL.md contains all commands', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
|
||||
for (const [cmd, meta] of Object.entries(COMMAND_DESCRIPTIONS)) {
|
||||
const display = meta.usage || cmd;
|
||||
expect(content).toContain(display);
|
||||
}
|
||||
});
|
||||
|
||||
test('command table is sorted alphabetically within categories', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
|
||||
// Extract command names from the Navigation section as a test
|
||||
const navSection = content.match(/### Navigation\n\|.*\n\|.*\n([\s\S]*?)(?=\n###|\n## )/);
|
||||
expect(navSection).not.toBeNull();
|
||||
const rows = navSection![1].trim().split('\n');
|
||||
const commands = rows.map(r => {
|
||||
const match = r.match(/\| `(\w+)/);
|
||||
return match ? match[1] : '';
|
||||
}).filter(Boolean);
|
||||
const sorted = [...commands].sort();
|
||||
expect(commands).toEqual(sorted);
|
||||
});
|
||||
|
||||
test('generated header is present in SKILL.md', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('AUTO-GENERATED from SKILL.md.tmpl');
|
||||
expect(content).toContain('Regenerate: bun run gen:skill-docs');
|
||||
});
|
||||
|
||||
test('generated header is present in browse/SKILL.md', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('AUTO-GENERATED from SKILL.md.tmpl');
|
||||
});
|
||||
|
||||
test('snapshot flags section contains all flags', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
|
||||
for (const flag of SNAPSHOT_FLAGS) {
|
||||
expect(content).toContain(flag.short);
|
||||
expect(content).toContain(flag.description);
|
||||
}
|
||||
});
|
||||
|
||||
test('template files exist for generated SKILL.md files', () => {
|
||||
expect(fs.existsSync(path.join(ROOT, 'SKILL.md.tmpl'))).toBe(true);
|
||||
expect(fs.existsSync(path.join(ROOT, 'browse', 'SKILL.md.tmpl'))).toBe(true);
|
||||
});
|
||||
|
||||
test('templates contain placeholders', () => {
|
||||
const rootTmpl = fs.readFileSync(path.join(ROOT, 'SKILL.md.tmpl'), 'utf-8');
|
||||
expect(rootTmpl).toContain('{{COMMAND_REFERENCE}}');
|
||||
expect(rootTmpl).toContain('{{SNAPSHOT_FLAGS}}');
|
||||
|
||||
const browseTmpl = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md.tmpl'), 'utf-8');
|
||||
expect(browseTmpl).toContain('{{COMMAND_REFERENCE}}');
|
||||
expect(browseTmpl).toContain('{{SNAPSHOT_FLAGS}}');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,133 @@
|
||||
/**
|
||||
* SKILL.md parser and validator.
|
||||
*
|
||||
* Extracts $B commands from code blocks, validates them against
|
||||
* the command registry and snapshot flags.
|
||||
*
|
||||
* Used by:
|
||||
* - test/skill-validation.test.ts (Tier 1 static tests)
|
||||
* - scripts/skill-check.ts (health summary)
|
||||
* - scripts/dev-skill.ts (watch mode)
|
||||
*/
|
||||
|
||||
import { ALL_COMMANDS } from '../../browse/src/commands';
|
||||
import { parseSnapshotArgs } from '../../browse/src/snapshot';
|
||||
import * as fs from 'fs';
|
||||
|
||||
export interface BrowseCommand {
|
||||
command: string;
|
||||
args: string[];
|
||||
line: number;
|
||||
raw: string;
|
||||
}
|
||||
|
||||
export interface ValidationResult {
|
||||
valid: BrowseCommand[];
|
||||
invalid: BrowseCommand[];
|
||||
snapshotFlagErrors: Array<{ command: BrowseCommand; error: string }>;
|
||||
warnings: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract all $B invocations from bash code blocks in a SKILL.md file.
|
||||
*/
|
||||
export function extractBrowseCommands(skillPath: string): BrowseCommand[] {
|
||||
const content = fs.readFileSync(skillPath, 'utf-8');
|
||||
const lines = content.split('\n');
|
||||
const commands: BrowseCommand[] = [];
|
||||
|
||||
let inBashBlock = false;
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
|
||||
// Detect code block boundaries
|
||||
if (line.trimStart().startsWith('```')) {
|
||||
if (inBashBlock) {
|
||||
inBashBlock = false;
|
||||
} else if (line.trimStart().startsWith('```bash')) {
|
||||
inBashBlock = true;
|
||||
}
|
||||
// Non-bash code blocks (```json, ```, ```js, etc.) are skipped
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!inBashBlock) continue;
|
||||
|
||||
// Match lines with $B command invocations
|
||||
// Handle multiple $B commands on one line (e.g., "$B click @e3 $B fill @e4 "value"")
|
||||
const matches = line.matchAll(/\$B\s+(\S+)(?:\s+([^\$]*))?/g);
|
||||
for (const match of matches) {
|
||||
const command = match[1];
|
||||
let argsStr = (match[2] || '').trim();
|
||||
|
||||
// Strip inline comments (# ...) — but not inside quotes
|
||||
// Simple approach: remove everything from first unquoted # onward
|
||||
let inQuote = false;
|
||||
for (let j = 0; j < argsStr.length; j++) {
|
||||
if (argsStr[j] === '"') inQuote = !inQuote;
|
||||
if (argsStr[j] === '#' && !inQuote) {
|
||||
argsStr = argsStr.slice(0, j).trim();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Parse args — handle quoted strings
|
||||
const args: string[] = [];
|
||||
if (argsStr) {
|
||||
const argMatches = argsStr.matchAll(/"([^"]*)"|(\S+)/g);
|
||||
for (const am of argMatches) {
|
||||
args.push(am[1] ?? am[2]);
|
||||
}
|
||||
}
|
||||
|
||||
commands.push({
|
||||
command,
|
||||
args,
|
||||
line: i + 1, // 1-based
|
||||
raw: match[0].trim(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return commands;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract and validate all $B commands in a SKILL.md file.
|
||||
*/
|
||||
export function validateSkill(skillPath: string): ValidationResult {
|
||||
const commands = extractBrowseCommands(skillPath);
|
||||
const result: ValidationResult = {
|
||||
valid: [],
|
||||
invalid: [],
|
||||
snapshotFlagErrors: [],
|
||||
warnings: [],
|
||||
};
|
||||
|
||||
if (commands.length === 0) {
|
||||
result.warnings.push('no $B commands found');
|
||||
return result;
|
||||
}
|
||||
|
||||
for (const cmd of commands) {
|
||||
if (!ALL_COMMANDS.has(cmd.command)) {
|
||||
result.invalid.push(cmd);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Validate snapshot flags
|
||||
if (cmd.command === 'snapshot' && cmd.args.length > 0) {
|
||||
try {
|
||||
parseSnapshotArgs(cmd.args);
|
||||
} catch (err: any) {
|
||||
result.snapshotFlagErrors.push({ command: cmd, error: err.message });
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
result.valid.push(cmd);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { extractBrowseCommands, validateSkill } from './helpers/skill-parser';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
|
||||
const FIXTURES_DIR = path.join(os.tmpdir(), 'skill-parser-test');
|
||||
|
||||
function writeFixture(name: string, content: string): string {
|
||||
fs.mkdirSync(FIXTURES_DIR, { recursive: true });
|
||||
const p = path.join(FIXTURES_DIR, name);
|
||||
fs.writeFileSync(p, content);
|
||||
return p;
|
||||
}
|
||||
|
||||
describe('extractBrowseCommands', () => {
|
||||
test('extracts $B commands from bash code blocks', () => {
|
||||
const p = writeFixture('basic.md', [
|
||||
'# Test',
|
||||
'```bash',
|
||||
'$B goto https://example.com',
|
||||
'$B snapshot -i',
|
||||
'```',
|
||||
].join('\n'));
|
||||
const cmds = extractBrowseCommands(p);
|
||||
expect(cmds).toHaveLength(2);
|
||||
expect(cmds[0].command).toBe('goto');
|
||||
expect(cmds[0].args).toEqual(['https://example.com']);
|
||||
expect(cmds[1].command).toBe('snapshot');
|
||||
expect(cmds[1].args).toEqual(['-i']);
|
||||
});
|
||||
|
||||
test('skips non-bash code blocks', () => {
|
||||
const p = writeFixture('skip.md', [
|
||||
'```json',
|
||||
'{"key": "$B goto bad"}',
|
||||
'```',
|
||||
'```bash',
|
||||
'$B text',
|
||||
'```',
|
||||
].join('\n'));
|
||||
const cmds = extractBrowseCommands(p);
|
||||
expect(cmds).toHaveLength(1);
|
||||
expect(cmds[0].command).toBe('text');
|
||||
});
|
||||
|
||||
test('returns empty array for file with no code blocks', () => {
|
||||
const p = writeFixture('no-blocks.md', '# Just text\nSome content\n');
|
||||
const cmds = extractBrowseCommands(p);
|
||||
expect(cmds).toHaveLength(0);
|
||||
});
|
||||
|
||||
test('returns empty array for code blocks with no $B invocations', () => {
|
||||
const p = writeFixture('no-b.md', [
|
||||
'```bash',
|
||||
'echo "hello"',
|
||||
'ls -la',
|
||||
'```',
|
||||
].join('\n'));
|
||||
const cmds = extractBrowseCommands(p);
|
||||
expect(cmds).toHaveLength(0);
|
||||
});
|
||||
|
||||
test('handles multiple $B commands on one line', () => {
|
||||
const p = writeFixture('multi.md', [
|
||||
'```bash',
|
||||
'$B click @e3 $B fill @e4 "value" $B hover @e1',
|
||||
'```',
|
||||
].join('\n'));
|
||||
const cmds = extractBrowseCommands(p);
|
||||
expect(cmds).toHaveLength(3);
|
||||
expect(cmds[0].command).toBe('click');
|
||||
expect(cmds[1].command).toBe('fill');
|
||||
expect(cmds[1].args).toEqual(['@e4', 'value']);
|
||||
expect(cmds[2].command).toBe('hover');
|
||||
});
|
||||
|
||||
test('handles quoted arguments correctly', () => {
|
||||
const p = writeFixture('quoted.md', [
|
||||
'```bash',
|
||||
'$B fill @e3 "test@example.com"',
|
||||
'$B js "document.title"',
|
||||
'```',
|
||||
].join('\n'));
|
||||
const cmds = extractBrowseCommands(p);
|
||||
expect(cmds[0].args).toEqual(['@e3', 'test@example.com']);
|
||||
expect(cmds[1].args).toEqual(['document.title']);
|
||||
});
|
||||
|
||||
test('tracks correct line numbers', () => {
|
||||
const p = writeFixture('lines.md', [
|
||||
'# Header', // line 1
|
||||
'', // line 2
|
||||
'```bash', // line 3
|
||||
'$B goto x', // line 4
|
||||
'```', // line 5
|
||||
'', // line 6
|
||||
'```bash', // line 7
|
||||
'$B text', // line 8
|
||||
'```', // line 9
|
||||
].join('\n'));
|
||||
const cmds = extractBrowseCommands(p);
|
||||
expect(cmds[0].line).toBe(4);
|
||||
expect(cmds[1].line).toBe(8);
|
||||
});
|
||||
|
||||
test('skips unlabeled code blocks', () => {
|
||||
const p = writeFixture('unlabeled.md', [
|
||||
'```',
|
||||
'$B snapshot -i',
|
||||
'```',
|
||||
].join('\n'));
|
||||
const cmds = extractBrowseCommands(p);
|
||||
expect(cmds).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('validateSkill', () => {
|
||||
test('valid commands pass validation', () => {
|
||||
const p = writeFixture('valid.md', [
|
||||
'```bash',
|
||||
'$B goto https://example.com',
|
||||
'$B text',
|
||||
'$B click @e3',
|
||||
'$B snapshot -i -a',
|
||||
'```',
|
||||
].join('\n'));
|
||||
const result = validateSkill(p);
|
||||
expect(result.valid).toHaveLength(4);
|
||||
expect(result.invalid).toHaveLength(0);
|
||||
expect(result.snapshotFlagErrors).toHaveLength(0);
|
||||
});
|
||||
|
||||
test('invalid commands flagged in result', () => {
|
||||
const p = writeFixture('invalid.md', [
|
||||
'```bash',
|
||||
'$B goto https://example.com',
|
||||
'$B explode',
|
||||
'$B halp',
|
||||
'```',
|
||||
].join('\n'));
|
||||
const result = validateSkill(p);
|
||||
expect(result.valid).toHaveLength(1);
|
||||
expect(result.invalid).toHaveLength(2);
|
||||
expect(result.invalid[0].command).toBe('explode');
|
||||
expect(result.invalid[1].command).toBe('halp');
|
||||
});
|
||||
|
||||
test('snapshot flags validated via parseSnapshotArgs', () => {
|
||||
const p = writeFixture('bad-snapshot.md', [
|
||||
'```bash',
|
||||
'$B snapshot --bogus',
|
||||
'```',
|
||||
].join('\n'));
|
||||
const result = validateSkill(p);
|
||||
expect(result.snapshotFlagErrors).toHaveLength(1);
|
||||
expect(result.snapshotFlagErrors[0].error).toContain('Unknown snapshot flag');
|
||||
});
|
||||
|
||||
test('returns warning when no $B commands found', () => {
|
||||
const p = writeFixture('empty.md', '# Nothing here\n');
|
||||
const result = validateSkill(p);
|
||||
expect(result.warnings).toContain('no $B commands found');
|
||||
});
|
||||
|
||||
test('valid snapshot flags pass', () => {
|
||||
const p = writeFixture('snap-valid.md', [
|
||||
'```bash',
|
||||
'$B snapshot -i -a -C -o /tmp/out.png',
|
||||
'$B snapshot -D',
|
||||
'$B snapshot -d 3',
|
||||
'$B snapshot -s "main"',
|
||||
'```',
|
||||
].join('\n'));
|
||||
const result = validateSkill(p);
|
||||
expect(result.valid).toHaveLength(4);
|
||||
expect(result.snapshotFlagErrors).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,100 @@
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { validateSkill } from './helpers/skill-parser';
|
||||
import { ALL_COMMANDS, COMMAND_DESCRIPTIONS, READ_COMMANDS, WRITE_COMMANDS, META_COMMANDS } from '../browse/src/commands';
|
||||
import { SNAPSHOT_FLAGS } from '../browse/src/snapshot';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
|
||||
describe('SKILL.md command validation', () => {
|
||||
test('all $B commands in SKILL.md are valid browse commands', () => {
|
||||
const result = validateSkill(path.join(ROOT, 'SKILL.md'));
|
||||
expect(result.invalid).toHaveLength(0);
|
||||
expect(result.valid.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('all snapshot flags in SKILL.md are valid', () => {
|
||||
const result = validateSkill(path.join(ROOT, 'SKILL.md'));
|
||||
expect(result.snapshotFlagErrors).toHaveLength(0);
|
||||
});
|
||||
|
||||
test('all $B commands in browse/SKILL.md are valid browse commands', () => {
|
||||
const result = validateSkill(path.join(ROOT, 'browse', 'SKILL.md'));
|
||||
expect(result.invalid).toHaveLength(0);
|
||||
expect(result.valid.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('all snapshot flags in browse/SKILL.md are valid', () => {
|
||||
const result = validateSkill(path.join(ROOT, 'browse', 'SKILL.md'));
|
||||
expect(result.snapshotFlagErrors).toHaveLength(0);
|
||||
});
|
||||
|
||||
test('all $B commands in qa/SKILL.md are valid browse commands', () => {
|
||||
const qaSkill = path.join(ROOT, 'qa', 'SKILL.md');
|
||||
if (!fs.existsSync(qaSkill)) return; // skip if missing
|
||||
const result = validateSkill(qaSkill);
|
||||
expect(result.invalid).toHaveLength(0);
|
||||
});
|
||||
|
||||
test('all snapshot flags in qa/SKILL.md are valid', () => {
|
||||
const qaSkill = path.join(ROOT, 'qa', 'SKILL.md');
|
||||
if (!fs.existsSync(qaSkill)) return;
|
||||
const result = validateSkill(qaSkill);
|
||||
expect(result.snapshotFlagErrors).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Command registry consistency', () => {
|
||||
test('COMMAND_DESCRIPTIONS covers all commands in sets', () => {
|
||||
const allCmds = new Set([...READ_COMMANDS, ...WRITE_COMMANDS, ...META_COMMANDS]);
|
||||
const descKeys = new Set(Object.keys(COMMAND_DESCRIPTIONS));
|
||||
for (const cmd of allCmds) {
|
||||
expect(descKeys.has(cmd)).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
test('COMMAND_DESCRIPTIONS has no extra commands not in sets', () => {
|
||||
const allCmds = new Set([...READ_COMMANDS, ...WRITE_COMMANDS, ...META_COMMANDS]);
|
||||
for (const key of Object.keys(COMMAND_DESCRIPTIONS)) {
|
||||
expect(allCmds.has(key)).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
test('ALL_COMMANDS matches union of all sets', () => {
|
||||
const union = new Set([...READ_COMMANDS, ...WRITE_COMMANDS, ...META_COMMANDS]);
|
||||
expect(ALL_COMMANDS.size).toBe(union.size);
|
||||
for (const cmd of union) {
|
||||
expect(ALL_COMMANDS.has(cmd)).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
test('SNAPSHOT_FLAGS option keys are valid SnapshotOptions fields', () => {
|
||||
const validKeys = new Set([
|
||||
'interactive', 'compact', 'depth', 'selector',
|
||||
'diff', 'annotate', 'outputPath', 'cursorInteractive',
|
||||
]);
|
||||
for (const flag of SNAPSHOT_FLAGS) {
|
||||
expect(validKeys.has(flag.optionKey)).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('Generated SKILL.md freshness', () => {
|
||||
test('no unresolved {{placeholders}} in generated SKILL.md', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
|
||||
const unresolved = content.match(/\{\{\w+\}\}/g);
|
||||
expect(unresolved).toBeNull();
|
||||
});
|
||||
|
||||
test('no unresolved {{placeholders}} in generated browse/SKILL.md', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8');
|
||||
const unresolved = content.match(/\{\{\w+\}\}/g);
|
||||
expect(unresolved).toBeNull();
|
||||
});
|
||||
|
||||
test('generated SKILL.md has AUTO-GENERATED header', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('AUTO-GENERATED');
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user