diff --git a/test/gen-skill-docs.test.ts b/test/gen-skill-docs.test.ts new file mode 100644 index 00000000..6f45adff --- /dev/null +++ b/test/gen-skill-docs.test.ts @@ -0,0 +1,73 @@ +import { describe, test, expect } from 'bun:test'; +import { COMMAND_DESCRIPTIONS } from '../browse/src/commands'; +import { SNAPSHOT_FLAGS } from '../browse/src/snapshot'; +import * as fs from 'fs'; +import * as path from 'path'; + +const ROOT = path.resolve(import.meta.dir, '..'); + +describe('gen-skill-docs', () => { + test('generated SKILL.md contains all command categories', () => { + const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8'); + const categories = new Set(Object.values(COMMAND_DESCRIPTIONS).map(d => d.category)); + for (const cat of categories) { + expect(content).toContain(`### ${cat}`); + } + }); + + test('generated SKILL.md contains all commands', () => { + const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8'); + for (const [cmd, meta] of Object.entries(COMMAND_DESCRIPTIONS)) { + const display = meta.usage || cmd; + expect(content).toContain(display); + } + }); + + test('command table is sorted alphabetically within categories', () => { + const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8'); + // Extract command names from the Navigation section as a test + const navSection = content.match(/### Navigation\n\|.*\n\|.*\n([\s\S]*?)(?=\n###|\n## )/); + expect(navSection).not.toBeNull(); + const rows = navSection![1].trim().split('\n'); + const commands = rows.map(r => { + const match = r.match(/\| `(\w+)/); + return match ? match[1] : ''; + }).filter(Boolean); + const sorted = [...commands].sort(); + expect(commands).toEqual(sorted); + }); + + test('generated header is present in SKILL.md', () => { + const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8'); + expect(content).toContain('AUTO-GENERATED from SKILL.md.tmpl'); + expect(content).toContain('Regenerate: bun run gen:skill-docs'); + }); + + test('generated header is present in browse/SKILL.md', () => { + const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8'); + expect(content).toContain('AUTO-GENERATED from SKILL.md.tmpl'); + }); + + test('snapshot flags section contains all flags', () => { + const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8'); + for (const flag of SNAPSHOT_FLAGS) { + expect(content).toContain(flag.short); + expect(content).toContain(flag.description); + } + }); + + test('template files exist for generated SKILL.md files', () => { + expect(fs.existsSync(path.join(ROOT, 'SKILL.md.tmpl'))).toBe(true); + expect(fs.existsSync(path.join(ROOT, 'browse', 'SKILL.md.tmpl'))).toBe(true); + }); + + test('templates contain placeholders', () => { + const rootTmpl = fs.readFileSync(path.join(ROOT, 'SKILL.md.tmpl'), 'utf-8'); + expect(rootTmpl).toContain('{{COMMAND_REFERENCE}}'); + expect(rootTmpl).toContain('{{SNAPSHOT_FLAGS}}'); + + const browseTmpl = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md.tmpl'), 'utf-8'); + expect(browseTmpl).toContain('{{COMMAND_REFERENCE}}'); + expect(browseTmpl).toContain('{{SNAPSHOT_FLAGS}}'); + }); +}); diff --git a/test/helpers/skill-parser.ts b/test/helpers/skill-parser.ts new file mode 100644 index 00000000..f7fdcb30 --- /dev/null +++ b/test/helpers/skill-parser.ts @@ -0,0 +1,133 @@ +/** + * SKILL.md parser and validator. + * + * Extracts $B commands from code blocks, validates them against + * the command registry and snapshot flags. + * + * Used by: + * - test/skill-validation.test.ts (Tier 1 static tests) + * - scripts/skill-check.ts (health summary) + * - scripts/dev-skill.ts (watch mode) + */ + +import { ALL_COMMANDS } from '../../browse/src/commands'; +import { parseSnapshotArgs } from '../../browse/src/snapshot'; +import * as fs from 'fs'; + +export interface BrowseCommand { + command: string; + args: string[]; + line: number; + raw: string; +} + +export interface ValidationResult { + valid: BrowseCommand[]; + invalid: BrowseCommand[]; + snapshotFlagErrors: Array<{ command: BrowseCommand; error: string }>; + warnings: string[]; +} + +/** + * Extract all $B invocations from bash code blocks in a SKILL.md file. + */ +export function extractBrowseCommands(skillPath: string): BrowseCommand[] { + const content = fs.readFileSync(skillPath, 'utf-8'); + const lines = content.split('\n'); + const commands: BrowseCommand[] = []; + + let inBashBlock = false; + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + + // Detect code block boundaries + if (line.trimStart().startsWith('```')) { + if (inBashBlock) { + inBashBlock = false; + } else if (line.trimStart().startsWith('```bash')) { + inBashBlock = true; + } + // Non-bash code blocks (```json, ```, ```js, etc.) are skipped + continue; + } + + if (!inBashBlock) continue; + + // Match lines with $B command invocations + // Handle multiple $B commands on one line (e.g., "$B click @e3 $B fill @e4 "value"") + const matches = line.matchAll(/\$B\s+(\S+)(?:\s+([^\$]*))?/g); + for (const match of matches) { + const command = match[1]; + let argsStr = (match[2] || '').trim(); + + // Strip inline comments (# ...) — but not inside quotes + // Simple approach: remove everything from first unquoted # onward + let inQuote = false; + for (let j = 0; j < argsStr.length; j++) { + if (argsStr[j] === '"') inQuote = !inQuote; + if (argsStr[j] === '#' && !inQuote) { + argsStr = argsStr.slice(0, j).trim(); + break; + } + } + + // Parse args — handle quoted strings + const args: string[] = []; + if (argsStr) { + const argMatches = argsStr.matchAll(/"([^"]*)"|(\S+)/g); + for (const am of argMatches) { + args.push(am[1] ?? am[2]); + } + } + + commands.push({ + command, + args, + line: i + 1, // 1-based + raw: match[0].trim(), + }); + } + } + + return commands; +} + +/** + * Extract and validate all $B commands in a SKILL.md file. + */ +export function validateSkill(skillPath: string): ValidationResult { + const commands = extractBrowseCommands(skillPath); + const result: ValidationResult = { + valid: [], + invalid: [], + snapshotFlagErrors: [], + warnings: [], + }; + + if (commands.length === 0) { + result.warnings.push('no $B commands found'); + return result; + } + + for (const cmd of commands) { + if (!ALL_COMMANDS.has(cmd.command)) { + result.invalid.push(cmd); + continue; + } + + // Validate snapshot flags + if (cmd.command === 'snapshot' && cmd.args.length > 0) { + try { + parseSnapshotArgs(cmd.args); + } catch (err: any) { + result.snapshotFlagErrors.push({ command: cmd, error: err.message }); + continue; + } + } + + result.valid.push(cmd); + } + + return result; +} diff --git a/test/skill-parser.test.ts b/test/skill-parser.test.ts new file mode 100644 index 00000000..3c62c682 --- /dev/null +++ b/test/skill-parser.test.ts @@ -0,0 +1,179 @@ +import { describe, test, expect } from 'bun:test'; +import { extractBrowseCommands, validateSkill } from './helpers/skill-parser'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; + +const FIXTURES_DIR = path.join(os.tmpdir(), 'skill-parser-test'); + +function writeFixture(name: string, content: string): string { + fs.mkdirSync(FIXTURES_DIR, { recursive: true }); + const p = path.join(FIXTURES_DIR, name); + fs.writeFileSync(p, content); + return p; +} + +describe('extractBrowseCommands', () => { + test('extracts $B commands from bash code blocks', () => { + const p = writeFixture('basic.md', [ + '# Test', + '```bash', + '$B goto https://example.com', + '$B snapshot -i', + '```', + ].join('\n')); + const cmds = extractBrowseCommands(p); + expect(cmds).toHaveLength(2); + expect(cmds[0].command).toBe('goto'); + expect(cmds[0].args).toEqual(['https://example.com']); + expect(cmds[1].command).toBe('snapshot'); + expect(cmds[1].args).toEqual(['-i']); + }); + + test('skips non-bash code blocks', () => { + const p = writeFixture('skip.md', [ + '```json', + '{"key": "$B goto bad"}', + '```', + '```bash', + '$B text', + '```', + ].join('\n')); + const cmds = extractBrowseCommands(p); + expect(cmds).toHaveLength(1); + expect(cmds[0].command).toBe('text'); + }); + + test('returns empty array for file with no code blocks', () => { + const p = writeFixture('no-blocks.md', '# Just text\nSome content\n'); + const cmds = extractBrowseCommands(p); + expect(cmds).toHaveLength(0); + }); + + test('returns empty array for code blocks with no $B invocations', () => { + const p = writeFixture('no-b.md', [ + '```bash', + 'echo "hello"', + 'ls -la', + '```', + ].join('\n')); + const cmds = extractBrowseCommands(p); + expect(cmds).toHaveLength(0); + }); + + test('handles multiple $B commands on one line', () => { + const p = writeFixture('multi.md', [ + '```bash', + '$B click @e3 $B fill @e4 "value" $B hover @e1', + '```', + ].join('\n')); + const cmds = extractBrowseCommands(p); + expect(cmds).toHaveLength(3); + expect(cmds[0].command).toBe('click'); + expect(cmds[1].command).toBe('fill'); + expect(cmds[1].args).toEqual(['@e4', 'value']); + expect(cmds[2].command).toBe('hover'); + }); + + test('handles quoted arguments correctly', () => { + const p = writeFixture('quoted.md', [ + '```bash', + '$B fill @e3 "test@example.com"', + '$B js "document.title"', + '```', + ].join('\n')); + const cmds = extractBrowseCommands(p); + expect(cmds[0].args).toEqual(['@e3', 'test@example.com']); + expect(cmds[1].args).toEqual(['document.title']); + }); + + test('tracks correct line numbers', () => { + const p = writeFixture('lines.md', [ + '# Header', // line 1 + '', // line 2 + '```bash', // line 3 + '$B goto x', // line 4 + '```', // line 5 + '', // line 6 + '```bash', // line 7 + '$B text', // line 8 + '```', // line 9 + ].join('\n')); + const cmds = extractBrowseCommands(p); + expect(cmds[0].line).toBe(4); + expect(cmds[1].line).toBe(8); + }); + + test('skips unlabeled code blocks', () => { + const p = writeFixture('unlabeled.md', [ + '```', + '$B snapshot -i', + '```', + ].join('\n')); + const cmds = extractBrowseCommands(p); + expect(cmds).toHaveLength(0); + }); +}); + +describe('validateSkill', () => { + test('valid commands pass validation', () => { + const p = writeFixture('valid.md', [ + '```bash', + '$B goto https://example.com', + '$B text', + '$B click @e3', + '$B snapshot -i -a', + '```', + ].join('\n')); + const result = validateSkill(p); + expect(result.valid).toHaveLength(4); + expect(result.invalid).toHaveLength(0); + expect(result.snapshotFlagErrors).toHaveLength(0); + }); + + test('invalid commands flagged in result', () => { + const p = writeFixture('invalid.md', [ + '```bash', + '$B goto https://example.com', + '$B explode', + '$B halp', + '```', + ].join('\n')); + const result = validateSkill(p); + expect(result.valid).toHaveLength(1); + expect(result.invalid).toHaveLength(2); + expect(result.invalid[0].command).toBe('explode'); + expect(result.invalid[1].command).toBe('halp'); + }); + + test('snapshot flags validated via parseSnapshotArgs', () => { + const p = writeFixture('bad-snapshot.md', [ + '```bash', + '$B snapshot --bogus', + '```', + ].join('\n')); + const result = validateSkill(p); + expect(result.snapshotFlagErrors).toHaveLength(1); + expect(result.snapshotFlagErrors[0].error).toContain('Unknown snapshot flag'); + }); + + test('returns warning when no $B commands found', () => { + const p = writeFixture('empty.md', '# Nothing here\n'); + const result = validateSkill(p); + expect(result.warnings).toContain('no $B commands found'); + }); + + test('valid snapshot flags pass', () => { + const p = writeFixture('snap-valid.md', [ + '```bash', + '$B snapshot -i -a -C -o /tmp/out.png', + '$B snapshot -D', + '$B snapshot -d 3', + '$B snapshot -s "main"', + '```', + ].join('\n')); + const result = validateSkill(p); + expect(result.valid).toHaveLength(4); + expect(result.snapshotFlagErrors).toHaveLength(0); + }); +}); diff --git a/test/skill-validation.test.ts b/test/skill-validation.test.ts new file mode 100644 index 00000000..1c4025a2 --- /dev/null +++ b/test/skill-validation.test.ts @@ -0,0 +1,100 @@ +import { describe, test, expect } from 'bun:test'; +import { validateSkill } from './helpers/skill-parser'; +import { ALL_COMMANDS, COMMAND_DESCRIPTIONS, READ_COMMANDS, WRITE_COMMANDS, META_COMMANDS } from '../browse/src/commands'; +import { SNAPSHOT_FLAGS } from '../browse/src/snapshot'; +import * as fs from 'fs'; +import * as path from 'path'; + +const ROOT = path.resolve(import.meta.dir, '..'); + +describe('SKILL.md command validation', () => { + test('all $B commands in SKILL.md are valid browse commands', () => { + const result = validateSkill(path.join(ROOT, 'SKILL.md')); + expect(result.invalid).toHaveLength(0); + expect(result.valid.length).toBeGreaterThan(0); + }); + + test('all snapshot flags in SKILL.md are valid', () => { + const result = validateSkill(path.join(ROOT, 'SKILL.md')); + expect(result.snapshotFlagErrors).toHaveLength(0); + }); + + test('all $B commands in browse/SKILL.md are valid browse commands', () => { + const result = validateSkill(path.join(ROOT, 'browse', 'SKILL.md')); + expect(result.invalid).toHaveLength(0); + expect(result.valid.length).toBeGreaterThan(0); + }); + + test('all snapshot flags in browse/SKILL.md are valid', () => { + const result = validateSkill(path.join(ROOT, 'browse', 'SKILL.md')); + expect(result.snapshotFlagErrors).toHaveLength(0); + }); + + test('all $B commands in qa/SKILL.md are valid browse commands', () => { + const qaSkill = path.join(ROOT, 'qa', 'SKILL.md'); + if (!fs.existsSync(qaSkill)) return; // skip if missing + const result = validateSkill(qaSkill); + expect(result.invalid).toHaveLength(0); + }); + + test('all snapshot flags in qa/SKILL.md are valid', () => { + const qaSkill = path.join(ROOT, 'qa', 'SKILL.md'); + if (!fs.existsSync(qaSkill)) return; + const result = validateSkill(qaSkill); + expect(result.snapshotFlagErrors).toHaveLength(0); + }); +}); + +describe('Command registry consistency', () => { + test('COMMAND_DESCRIPTIONS covers all commands in sets', () => { + const allCmds = new Set([...READ_COMMANDS, ...WRITE_COMMANDS, ...META_COMMANDS]); + const descKeys = new Set(Object.keys(COMMAND_DESCRIPTIONS)); + for (const cmd of allCmds) { + expect(descKeys.has(cmd)).toBe(true); + } + }); + + test('COMMAND_DESCRIPTIONS has no extra commands not in sets', () => { + const allCmds = new Set([...READ_COMMANDS, ...WRITE_COMMANDS, ...META_COMMANDS]); + for (const key of Object.keys(COMMAND_DESCRIPTIONS)) { + expect(allCmds.has(key)).toBe(true); + } + }); + + test('ALL_COMMANDS matches union of all sets', () => { + const union = new Set([...READ_COMMANDS, ...WRITE_COMMANDS, ...META_COMMANDS]); + expect(ALL_COMMANDS.size).toBe(union.size); + for (const cmd of union) { + expect(ALL_COMMANDS.has(cmd)).toBe(true); + } + }); + + test('SNAPSHOT_FLAGS option keys are valid SnapshotOptions fields', () => { + const validKeys = new Set([ + 'interactive', 'compact', 'depth', 'selector', + 'diff', 'annotate', 'outputPath', 'cursorInteractive', + ]); + for (const flag of SNAPSHOT_FLAGS) { + expect(validKeys.has(flag.optionKey)).toBe(true); + } + }); +}); + +describe('Generated SKILL.md freshness', () => { + test('no unresolved {{placeholders}} in generated SKILL.md', () => { + const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8'); + const unresolved = content.match(/\{\{\w+\}\}/g); + expect(unresolved).toBeNull(); + }); + + test('no unresolved {{placeholders}} in generated browse/SKILL.md', () => { + const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8'); + const unresolved = content.match(/\{\{\w+\}\}/g); + expect(unresolved).toBeNull(); + }); + + test('generated SKILL.md has AUTO-GENERATED header', () => { + const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8'); + expect(content).toContain('AUTO-GENERATED'); + }); +});