test: Tier 1 static validation — 34 tests for SKILL.md command correctness

- test/helpers/skill-parser.ts: extracts $B commands from code blocks, validates against registry
- test/skill-parser.test.ts: 13 parser/validator unit tests
- test/skill-validation.test.ts: 13 tests validating all SKILL.md files + registry consistency
- test/gen-skill-docs.test.ts: 8 generator tests (categories, sorting, freshness)

test/gen-skill-docs.test.ts

@@ -0,0 +1,73 @@
+import { describe, test, expect } from 'bun:test';
+import { COMMAND_DESCRIPTIONS } from '../browse/src/commands';
+import { SNAPSHOT_FLAGS } from '../browse/src/snapshot';
+import * as fs from 'fs';
+import * as path from 'path';
+
+const ROOT = path.resolve(import.meta.dir, '..');
+
+describe('gen-skill-docs', () => {
+  test('generated SKILL.md contains all command categories', () => {
+    const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
+    const categories = new Set(Object.values(COMMAND_DESCRIPTIONS).map(d => d.category));
+    for (const cat of categories) {
+      expect(content).toContain(`### ${cat}`);
+    }
+  });
+
+  test('generated SKILL.md contains all commands', () => {
+    const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
+    for (const [cmd, meta] of Object.entries(COMMAND_DESCRIPTIONS)) {
+      const display = meta.usage || cmd;
+      expect(content).toContain(display);
+    }
+  });
+
+  test('command table is sorted alphabetically within categories', () => {
+    const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
+    // Extract command names from the Navigation section as a test
+    const navSection = content.match(/### Navigation\n\|.*\n\|.*\n([\s\S]*?)(?=\n###|\n## )/);
+    expect(navSection).not.toBeNull();
+    const rows = navSection![1].trim().split('\n');
+    const commands = rows.map(r => {
+      const match = r.match(/\| `(\w+)/);
+      return match ? match[1] : '';
+    }).filter(Boolean);
+    const sorted = [...commands].sort();
+    expect(commands).toEqual(sorted);
+  });
+
+  test('generated header is present in SKILL.md', () => {
+    const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
+    expect(content).toContain('AUTO-GENERATED from SKILL.md.tmpl');
+    expect(content).toContain('Regenerate: bun run gen:skill-docs');
+  });
+
+  test('generated header is present in browse/SKILL.md', () => {
+    const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8');
+    expect(content).toContain('AUTO-GENERATED from SKILL.md.tmpl');
+  });
+
+  test('snapshot flags section contains all flags', () => {
+    const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
+    for (const flag of SNAPSHOT_FLAGS) {
+      expect(content).toContain(flag.short);
+      expect(content).toContain(flag.description);
+    }
+  });
+
+  test('template files exist for generated SKILL.md files', () => {
+    expect(fs.existsSync(path.join(ROOT, 'SKILL.md.tmpl'))).toBe(true);
+    expect(fs.existsSync(path.join(ROOT, 'browse', 'SKILL.md.tmpl'))).toBe(true);
+  });
+
+  test('templates contain placeholders', () => {
+    const rootTmpl = fs.readFileSync(path.join(ROOT, 'SKILL.md.tmpl'), 'utf-8');
+    expect(rootTmpl).toContain('{{COMMAND_REFERENCE}}');
+    expect(rootTmpl).toContain('{{SNAPSHOT_FLAGS}}');
+
+    const browseTmpl = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md.tmpl'), 'utf-8');
+    expect(browseTmpl).toContain('{{COMMAND_REFERENCE}}');
+    expect(browseTmpl).toContain('{{SNAPSHOT_FLAGS}}');
+  });
+});

test/helpers/skill-parser.ts

@@ -0,0 +1,133 @@
+/**
+ * SKILL.md parser and validator.
+ *
+ * Extracts $B commands from code blocks, validates them against
+ * the command registry and snapshot flags.
+ *
+ * Used by:
+ *   - test/skill-validation.test.ts (Tier 1 static tests)
+ *   - scripts/skill-check.ts (health summary)
+ *   - scripts/dev-skill.ts (watch mode)
+ */
+
+import { ALL_COMMANDS } from '../../browse/src/commands';
+import { parseSnapshotArgs } from '../../browse/src/snapshot';
+import * as fs from 'fs';
+
+export interface BrowseCommand {
+  command: string;
+  args: string[];
+  line: number;
+  raw: string;
+}
+
+export interface ValidationResult {
+  valid: BrowseCommand[];
+  invalid: BrowseCommand[];
+  snapshotFlagErrors: Array<{ command: BrowseCommand; error: string }>;
+  warnings: string[];
+}
+
+/**
+ * Extract all $B invocations from bash code blocks in a SKILL.md file.
+ */
+export function extractBrowseCommands(skillPath: string): BrowseCommand[] {
+  const content = fs.readFileSync(skillPath, 'utf-8');
+  const lines = content.split('\n');
+  const commands: BrowseCommand[] = [];
+
+  let inBashBlock = false;
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+
+    // Detect code block boundaries
+    if (line.trimStart().startsWith('```')) {
+      if (inBashBlock) {
+        inBashBlock = false;
+      } else if (line.trimStart().startsWith('```bash')) {
+        inBashBlock = true;
+      }
+      // Non-bash code blocks (```json, ```, ```js, etc.) are skipped
+      continue;
+    }
+
+    if (!inBashBlock) continue;
+
+    // Match lines with $B command invocations
+    // Handle multiple $B commands on one line (e.g., "$B click @e3       $B fill @e4 "value"")
+    const matches = line.matchAll(/\$B\s+(\S+)(?:\s+([^\$]*))?/g);
+    for (const match of matches) {
+      const command = match[1];
+      let argsStr = (match[2] || '').trim();
+
+      // Strip inline comments (# ...) — but not inside quotes
+      // Simple approach: remove everything from first unquoted # onward
+      let inQuote = false;
+      for (let j = 0; j < argsStr.length; j++) {
+        if (argsStr[j] === '"') inQuote = !inQuote;
+        if (argsStr[j] === '#' && !inQuote) {
+          argsStr = argsStr.slice(0, j).trim();
+          break;
+        }
+      }
+
+      // Parse args — handle quoted strings
+      const args: string[] = [];
+      if (argsStr) {
+        const argMatches = argsStr.matchAll(/"([^"]*)"|(\S+)/g);
+        for (const am of argMatches) {
+          args.push(am[1] ?? am[2]);
+        }
+      }
+
+      commands.push({
+        command,
+        args,
+        line: i + 1, // 1-based
+        raw: match[0].trim(),
+      });
+    }
+  }
+
+  return commands;
+}
+
+/**
+ * Extract and validate all $B commands in a SKILL.md file.
+ */
+export function validateSkill(skillPath: string): ValidationResult {
+  const commands = extractBrowseCommands(skillPath);
+  const result: ValidationResult = {
+    valid: [],
+    invalid: [],
+    snapshotFlagErrors: [],
+    warnings: [],
+  };
+
+  if (commands.length === 0) {
+    result.warnings.push('no $B commands found');
+    return result;
+  }
+
+  for (const cmd of commands) {
+    if (!ALL_COMMANDS.has(cmd.command)) {
+      result.invalid.push(cmd);
+      continue;
+    }
+
+    // Validate snapshot flags
+    if (cmd.command === 'snapshot' && cmd.args.length > 0) {
+      try {
+        parseSnapshotArgs(cmd.args);
+      } catch (err: any) {
+        result.snapshotFlagErrors.push({ command: cmd, error: err.message });
+        continue;
+      }
+    }
+
+    result.valid.push(cmd);
+  }
+
+  return result;
+}

test/skill-parser.test.ts

@@ -0,0 +1,179 @@
+import { describe, test, expect } from 'bun:test';
+import { extractBrowseCommands, validateSkill } from './helpers/skill-parser';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as os from 'os';
+
+const FIXTURES_DIR = path.join(os.tmpdir(), 'skill-parser-test');
+
+function writeFixture(name: string, content: string): string {
+  fs.mkdirSync(FIXTURES_DIR, { recursive: true });
+  const p = path.join(FIXTURES_DIR, name);
+  fs.writeFileSync(p, content);
+  return p;
+}
+
+describe('extractBrowseCommands', () => {
+  test('extracts $B commands from bash code blocks', () => {
+    const p = writeFixture('basic.md', [
+      '# Test',
+      '```bash',
+      '$B goto https://example.com',
+      '$B snapshot -i',
+      '```',
+    ].join('\n'));
+    const cmds = extractBrowseCommands(p);
+    expect(cmds).toHaveLength(2);
+    expect(cmds[0].command).toBe('goto');
+    expect(cmds[0].args).toEqual(['https://example.com']);
+    expect(cmds[1].command).toBe('snapshot');
+    expect(cmds[1].args).toEqual(['-i']);
+  });
+
+  test('skips non-bash code blocks', () => {
+    const p = writeFixture('skip.md', [
+      '```json',
+      '{"key": "$B goto bad"}',
+      '```',
+      '```bash',
+      '$B text',
+      '```',
+    ].join('\n'));
+    const cmds = extractBrowseCommands(p);
+    expect(cmds).toHaveLength(1);
+    expect(cmds[0].command).toBe('text');
+  });
+
+  test('returns empty array for file with no code blocks', () => {
+    const p = writeFixture('no-blocks.md', '# Just text\nSome content\n');
+    const cmds = extractBrowseCommands(p);
+    expect(cmds).toHaveLength(0);
+  });
+
+  test('returns empty array for code blocks with no $B invocations', () => {
+    const p = writeFixture('no-b.md', [
+      '```bash',
+      'echo "hello"',
+      'ls -la',
+      '```',
+    ].join('\n'));
+    const cmds = extractBrowseCommands(p);
+    expect(cmds).toHaveLength(0);
+  });
+
+  test('handles multiple $B commands on one line', () => {
+    const p = writeFixture('multi.md', [
+      '```bash',
+      '$B click @e3       $B fill @e4 "value"     $B hover @e1',
+      '```',
+    ].join('\n'));
+    const cmds = extractBrowseCommands(p);
+    expect(cmds).toHaveLength(3);
+    expect(cmds[0].command).toBe('click');
+    expect(cmds[1].command).toBe('fill');
+    expect(cmds[1].args).toEqual(['@e4', 'value']);
+    expect(cmds[2].command).toBe('hover');
+  });
+
+  test('handles quoted arguments correctly', () => {
+    const p = writeFixture('quoted.md', [
+      '```bash',
+      '$B fill @e3 "test@example.com"',
+      '$B js "document.title"',
+      '```',
+    ].join('\n'));
+    const cmds = extractBrowseCommands(p);
+    expect(cmds[0].args).toEqual(['@e3', 'test@example.com']);
+    expect(cmds[1].args).toEqual(['document.title']);
+  });
+
+  test('tracks correct line numbers', () => {
+    const p = writeFixture('lines.md', [
+      '# Header',     // line 1
+      '',              // line 2
+      '```bash',       // line 3
+      '$B goto x',     // line 4
+      '```',           // line 5
+      '',              // line 6
+      '```bash',       // line 7
+      '$B text',       // line 8
+      '```',           // line 9
+    ].join('\n'));
+    const cmds = extractBrowseCommands(p);
+    expect(cmds[0].line).toBe(4);
+    expect(cmds[1].line).toBe(8);
+  });
+
+  test('skips unlabeled code blocks', () => {
+    const p = writeFixture('unlabeled.md', [
+      '```',
+      '$B snapshot -i',
+      '```',
+    ].join('\n'));
+    const cmds = extractBrowseCommands(p);
+    expect(cmds).toHaveLength(0);
+  });
+});
+
+describe('validateSkill', () => {
+  test('valid commands pass validation', () => {
+    const p = writeFixture('valid.md', [
+      '```bash',
+      '$B goto https://example.com',
+      '$B text',
+      '$B click @e3',
+      '$B snapshot -i -a',
+      '```',
+    ].join('\n'));
+    const result = validateSkill(p);
+    expect(result.valid).toHaveLength(4);
+    expect(result.invalid).toHaveLength(0);
+    expect(result.snapshotFlagErrors).toHaveLength(0);
+  });
+
+  test('invalid commands flagged in result', () => {
+    const p = writeFixture('invalid.md', [
+      '```bash',
+      '$B goto https://example.com',
+      '$B explode',
+      '$B halp',
+      '```',
+    ].join('\n'));
+    const result = validateSkill(p);
+    expect(result.valid).toHaveLength(1);
+    expect(result.invalid).toHaveLength(2);
+    expect(result.invalid[0].command).toBe('explode');
+    expect(result.invalid[1].command).toBe('halp');
+  });
+
+  test('snapshot flags validated via parseSnapshotArgs', () => {
+    const p = writeFixture('bad-snapshot.md', [
+      '```bash',
+      '$B snapshot --bogus',
+      '```',
+    ].join('\n'));
+    const result = validateSkill(p);
+    expect(result.snapshotFlagErrors).toHaveLength(1);
+    expect(result.snapshotFlagErrors[0].error).toContain('Unknown snapshot flag');
+  });
+
+  test('returns warning when no $B commands found', () => {
+    const p = writeFixture('empty.md', '# Nothing here\n');
+    const result = validateSkill(p);
+    expect(result.warnings).toContain('no $B commands found');
+  });
+
+  test('valid snapshot flags pass', () => {
+    const p = writeFixture('snap-valid.md', [
+      '```bash',
+      '$B snapshot -i -a -C -o /tmp/out.png',
+      '$B snapshot -D',
+      '$B snapshot -d 3',
+      '$B snapshot -s "main"',
+      '```',
+    ].join('\n'));
+    const result = validateSkill(p);
+    expect(result.valid).toHaveLength(4);
+    expect(result.snapshotFlagErrors).toHaveLength(0);
+  });
+});

test/skill-validation.test.ts

@@ -0,0 +1,100 @@
+import { describe, test, expect } from 'bun:test';
+import { validateSkill } from './helpers/skill-parser';
+import { ALL_COMMANDS, COMMAND_DESCRIPTIONS, READ_COMMANDS, WRITE_COMMANDS, META_COMMANDS } from '../browse/src/commands';
+import { SNAPSHOT_FLAGS } from '../browse/src/snapshot';
+import * as fs from 'fs';
+import * as path from 'path';
+
+const ROOT = path.resolve(import.meta.dir, '..');
+
+describe('SKILL.md command validation', () => {
+  test('all $B commands in SKILL.md are valid browse commands', () => {
+    const result = validateSkill(path.join(ROOT, 'SKILL.md'));
+    expect(result.invalid).toHaveLength(0);
+    expect(result.valid.length).toBeGreaterThan(0);
+  });
+
+  test('all snapshot flags in SKILL.md are valid', () => {
+    const result = validateSkill(path.join(ROOT, 'SKILL.md'));
+    expect(result.snapshotFlagErrors).toHaveLength(0);
+  });
+
+  test('all $B commands in browse/SKILL.md are valid browse commands', () => {
+    const result = validateSkill(path.join(ROOT, 'browse', 'SKILL.md'));
+    expect(result.invalid).toHaveLength(0);
+    expect(result.valid.length).toBeGreaterThan(0);
+  });
+
+  test('all snapshot flags in browse/SKILL.md are valid', () => {
+    const result = validateSkill(path.join(ROOT, 'browse', 'SKILL.md'));
+    expect(result.snapshotFlagErrors).toHaveLength(0);
+  });
+
+  test('all $B commands in qa/SKILL.md are valid browse commands', () => {
+    const qaSkill = path.join(ROOT, 'qa', 'SKILL.md');
+    if (!fs.existsSync(qaSkill)) return; // skip if missing
+    const result = validateSkill(qaSkill);
+    expect(result.invalid).toHaveLength(0);
+  });
+
+  test('all snapshot flags in qa/SKILL.md are valid', () => {
+    const qaSkill = path.join(ROOT, 'qa', 'SKILL.md');
+    if (!fs.existsSync(qaSkill)) return;
+    const result = validateSkill(qaSkill);
+    expect(result.snapshotFlagErrors).toHaveLength(0);
+  });
+});
+
+describe('Command registry consistency', () => {
+  test('COMMAND_DESCRIPTIONS covers all commands in sets', () => {
+    const allCmds = new Set([...READ_COMMANDS, ...WRITE_COMMANDS, ...META_COMMANDS]);
+    const descKeys = new Set(Object.keys(COMMAND_DESCRIPTIONS));
+    for (const cmd of allCmds) {
+      expect(descKeys.has(cmd)).toBe(true);
+    }
+  });
+
+  test('COMMAND_DESCRIPTIONS has no extra commands not in sets', () => {
+    const allCmds = new Set([...READ_COMMANDS, ...WRITE_COMMANDS, ...META_COMMANDS]);
+    for (const key of Object.keys(COMMAND_DESCRIPTIONS)) {
+      expect(allCmds.has(key)).toBe(true);
+    }
+  });
+
+  test('ALL_COMMANDS matches union of all sets', () => {
+    const union = new Set([...READ_COMMANDS, ...WRITE_COMMANDS, ...META_COMMANDS]);
+    expect(ALL_COMMANDS.size).toBe(union.size);
+    for (const cmd of union) {
+      expect(ALL_COMMANDS.has(cmd)).toBe(true);
+    }
+  });
+
+  test('SNAPSHOT_FLAGS option keys are valid SnapshotOptions fields', () => {
+    const validKeys = new Set([
+      'interactive', 'compact', 'depth', 'selector',
+      'diff', 'annotate', 'outputPath', 'cursorInteractive',
+    ]);
+    for (const flag of SNAPSHOT_FLAGS) {
+      expect(validKeys.has(flag.optionKey)).toBe(true);
+    }
+  });
+});
+
+describe('Generated SKILL.md freshness', () => {
+  test('no unresolved {{placeholders}} in generated SKILL.md', () => {
+    const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
+    const unresolved = content.match(/\{\{\w+\}\}/g);
+    expect(unresolved).toBeNull();
+  });
+
+  test('no unresolved {{placeholders}} in generated browse/SKILL.md', () => {
+    const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8');
+    const unresolved = content.match(/\{\{\w+\}\}/g);
+    expect(unresolved).toBeNull();
+  });
+
+  test('generated SKILL.md has AUTO-GENERATED header', () => {
+    const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
+    expect(content).toContain('AUTO-GENERATED');
+  });
+});