Files
gstack/test/skill-parser.test.ts
T
Garry Tan 5205070299 feat: SKILL.md template system, 3-tier testing, DX tools (v0.3.3) (#41)
* refactor: extract command registry to commands.ts, add SNAPSHOT_FLAGS metadata

- NEW: browse/src/commands.ts — command sets + COMMAND_DESCRIPTIONS + load-time validation (zero side effects)
- server.ts imports from commands.ts instead of declaring sets inline
- snapshot.ts: SNAPSHOT_FLAGS array drives parseSnapshotArgs (metadata-driven, no duplication)
- All 186 existing tests pass

* feat: SKILL.md template system with auto-generated command references

- SKILL.md.tmpl + browse/SKILL.md.tmpl with {{COMMAND_REFERENCE}} and {{SNAPSHOT_FLAGS}} placeholders
- scripts/gen-skill-docs.ts generates SKILL.md from templates (supports --dry-run)
- Build pipeline runs gen:skill-docs before binary compilation
- Generated files have AUTO-GENERATED header, committed to git

* test: Tier 1 static validation — 34 tests for SKILL.md command correctness

- test/helpers/skill-parser.ts: extracts $B commands from code blocks, validates against registry
- test/skill-parser.test.ts: 13 parser/validator unit tests
- test/skill-validation.test.ts: 13 tests validating all SKILL.md files + registry consistency
- test/gen-skill-docs.test.ts: 8 generator tests (categories, sorting, freshness)

* feat: DX tools (skill:check, dev:skill) + Tier 2 E2E test scaffolding

- scripts/skill-check.ts: health summary for all SKILL.md files (commands, templates, freshness)
- scripts/dev-skill.ts: watch mode for template development
- test/helpers/session-runner.ts: Agent SDK wrapper for E2E skill tests
- test/skill-e2e.test.ts: 2 E2E tests + 3 stubs (auto-skip inside Claude Code sessions)
- E2E tests must run from plain terminal: SKILL_E2E=1 bun test test/skill-e2e.test.ts

* ci: SKILL.md freshness check on push/PR + TODO updates

- .github/workflows/skill-docs.yml: fails if generated SKILL.md files are stale
- TODO.md: add E2E cost tracking and model pinning to future ideas

* fix: restore rich descriptions lost in auto-generation

- Snapshot flags: add back value hints (-d <N>, -s <sel>, -o <path>)
- Snapshot flags: restore parenthetical context (@e refs, @c refs, etc.)
- Commands: is → includes valid states enum
- Commands: console → notes --errors filter behavior
- Commands: press → lists common keys (Enter, Tab, Escape)
- Commands: cookie-import-browser → describes picker UI
- Commands: dialog-accept → specifies alert/confirm/prompt
- Tips: restore → arrow (was downgraded to ->)

* test: quality evals for generated SKILL.md descriptions

Catches the exact regressions we shipped and caught in review:
- Snapshot flags must include value hints (-d <N>, -s <sel>, -o <path>)
- is command must list all valid states (visible/hidden/enabled/...)
- press command must list example keys (Enter, Tab, Escape)
- console command must describe --errors behavior
- Snapshot -i must mention @e refs, -C must mention @c refs
- All descriptions must be >= 8 chars (no empty stubs)
- Tips section must use → not ->

* feat: LLM-as-judge evals for SKILL.md documentation quality

4 eval tests using Anthropic API (claude-haiku, ~$0.01-0.03/run):
- Command reference table: clarity/completeness/actionability >= 4/5
- Snapshot flags section: same thresholds
- browse/SKILL.md overall quality
- Regression: generated version must score >= hand-maintained baseline

Requires ANTHROPIC_API_KEY. Auto-skips without it.
Run: bun run test:eval (or ANTHROPIC_API_KEY=sk-... bun test test/skill-llm-eval.test.ts)

* chore: bump version to 0.3.3, update changelog

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* docs: add ARCHITECTURE.md, update CLAUDE.md and CONTRIBUTING.md

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* feat: conductor.json lifecycle hooks + .env propagation across worktrees

bin/dev-setup now copies .env from main worktree so API keys carry
over to Conductor workspaces automatically. conductor.json wires up
setup and archive hooks.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* docs: complete CHANGELOG for v0.3.3 (architecture, conductor, .env)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-13 21:08:12 -07:00

180 lines
5.3 KiB
TypeScript

import { describe, test, expect } from 'bun:test';
import { extractBrowseCommands, validateSkill } from './helpers/skill-parser';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
const FIXTURES_DIR = path.join(os.tmpdir(), 'skill-parser-test');
function writeFixture(name: string, content: string): string {
fs.mkdirSync(FIXTURES_DIR, { recursive: true });
const p = path.join(FIXTURES_DIR, name);
fs.writeFileSync(p, content);
return p;
}
describe('extractBrowseCommands', () => {
test('extracts $B commands from bash code blocks', () => {
const p = writeFixture('basic.md', [
'# Test',
'```bash',
'$B goto https://example.com',
'$B snapshot -i',
'```',
].join('\n'));
const cmds = extractBrowseCommands(p);
expect(cmds).toHaveLength(2);
expect(cmds[0].command).toBe('goto');
expect(cmds[0].args).toEqual(['https://example.com']);
expect(cmds[1].command).toBe('snapshot');
expect(cmds[1].args).toEqual(['-i']);
});
test('skips non-bash code blocks', () => {
const p = writeFixture('skip.md', [
'```json',
'{"key": "$B goto bad"}',
'```',
'```bash',
'$B text',
'```',
].join('\n'));
const cmds = extractBrowseCommands(p);
expect(cmds).toHaveLength(1);
expect(cmds[0].command).toBe('text');
});
test('returns empty array for file with no code blocks', () => {
const p = writeFixture('no-blocks.md', '# Just text\nSome content\n');
const cmds = extractBrowseCommands(p);
expect(cmds).toHaveLength(0);
});
test('returns empty array for code blocks with no $B invocations', () => {
const p = writeFixture('no-b.md', [
'```bash',
'echo "hello"',
'ls -la',
'```',
].join('\n'));
const cmds = extractBrowseCommands(p);
expect(cmds).toHaveLength(0);
});
test('handles multiple $B commands on one line', () => {
const p = writeFixture('multi.md', [
'```bash',
'$B click @e3 $B fill @e4 "value" $B hover @e1',
'```',
].join('\n'));
const cmds = extractBrowseCommands(p);
expect(cmds).toHaveLength(3);
expect(cmds[0].command).toBe('click');
expect(cmds[1].command).toBe('fill');
expect(cmds[1].args).toEqual(['@e4', 'value']);
expect(cmds[2].command).toBe('hover');
});
test('handles quoted arguments correctly', () => {
const p = writeFixture('quoted.md', [
'```bash',
'$B fill @e3 "test@example.com"',
'$B js "document.title"',
'```',
].join('\n'));
const cmds = extractBrowseCommands(p);
expect(cmds[0].args).toEqual(['@e3', 'test@example.com']);
expect(cmds[1].args).toEqual(['document.title']);
});
test('tracks correct line numbers', () => {
const p = writeFixture('lines.md', [
'# Header', // line 1
'', // line 2
'```bash', // line 3
'$B goto x', // line 4
'```', // line 5
'', // line 6
'```bash', // line 7
'$B text', // line 8
'```', // line 9
].join('\n'));
const cmds = extractBrowseCommands(p);
expect(cmds[0].line).toBe(4);
expect(cmds[1].line).toBe(8);
});
test('skips unlabeled code blocks', () => {
const p = writeFixture('unlabeled.md', [
'```',
'$B snapshot -i',
'```',
].join('\n'));
const cmds = extractBrowseCommands(p);
expect(cmds).toHaveLength(0);
});
});
describe('validateSkill', () => {
test('valid commands pass validation', () => {
const p = writeFixture('valid.md', [
'```bash',
'$B goto https://example.com',
'$B text',
'$B click @e3',
'$B snapshot -i -a',
'```',
].join('\n'));
const result = validateSkill(p);
expect(result.valid).toHaveLength(4);
expect(result.invalid).toHaveLength(0);
expect(result.snapshotFlagErrors).toHaveLength(0);
});
test('invalid commands flagged in result', () => {
const p = writeFixture('invalid.md', [
'```bash',
'$B goto https://example.com',
'$B explode',
'$B halp',
'```',
].join('\n'));
const result = validateSkill(p);
expect(result.valid).toHaveLength(1);
expect(result.invalid).toHaveLength(2);
expect(result.invalid[0].command).toBe('explode');
expect(result.invalid[1].command).toBe('halp');
});
test('snapshot flags validated via parseSnapshotArgs', () => {
const p = writeFixture('bad-snapshot.md', [
'```bash',
'$B snapshot --bogus',
'```',
].join('\n'));
const result = validateSkill(p);
expect(result.snapshotFlagErrors).toHaveLength(1);
expect(result.snapshotFlagErrors[0].error).toContain('Unknown snapshot flag');
});
test('returns warning when no $B commands found', () => {
const p = writeFixture('empty.md', '# Nothing here\n');
const result = validateSkill(p);
expect(result.warnings).toContain('no $B commands found');
});
test('valid snapshot flags pass', () => {
const p = writeFixture('snap-valid.md', [
'```bash',
'$B snapshot -i -a -C -o /tmp/out.png',
'$B snapshot -D',
'$B snapshot -d 3',
'$B snapshot -s "main"',
'```',
].join('\n'));
const result = validateSkill(p);
expect(result.valid).toHaveLength(4);
expect(result.snapshotFlagErrors).toHaveLength(0);
});
});