feat: SKILL.md template system, 3-tier testing, DX tools (v0.3.3) (#41)

* refactor: extract command registry to commands.ts, add SNAPSHOT_FLAGS metadata - NEW: browse/src/commands.ts — command sets + COMMAND_DESCRIPTIONS + load-time validation (zero side effects) - server.ts imports from commands.ts instead of declaring sets inline - snapshot.ts: SNAPSHOT_FLAGS array drives parseSnapshotArgs (metadata-driven, no duplication) - All 186 existing tests pass * feat: SKILL.md template system with auto-generated command references - SKILL.md.tmpl + browse/SKILL.md.tmpl with {{COMMAND_REFERENCE}} and {{SNAPSHOT_FLAGS}} placeholders - scripts/gen-skill-docs.ts generates SKILL.md from templates (supports --dry-run) - Build pipeline runs gen:skill-docs before binary compilation - Generated files have AUTO-GENERATED header, committed to git * test: Tier 1 static validation — 34 tests for SKILL.md command correctness - test/helpers/skill-parser.ts: extracts $B commands from code blocks, validates against registry - test/skill-parser.test.ts: 13 parser/validator unit tests - test/skill-validation.test.ts: 13 tests validating all SKILL.md files + registry consistency - test/gen-skill-docs.test.ts: 8 generator tests (categories, sorting, freshness) * feat: DX tools (skill:check, dev:skill) + Tier 2 E2E test scaffolding - scripts/skill-check.ts: health summary for all SKILL.md files (commands, templates, freshness) - scripts/dev-skill.ts: watch mode for template development - test/helpers/session-runner.ts: Agent SDK wrapper for E2E skill tests - test/skill-e2e.test.ts: 2 E2E tests + 3 stubs (auto-skip inside Claude Code sessions) - E2E tests must run from plain terminal: SKILL_E2E=1 bun test test/skill-e2e.test.ts * ci: SKILL.md freshness check on push/PR + TODO updates - .github/workflows/skill-docs.yml: fails if generated SKILL.md files are stale - TODO.md: add E2E cost tracking and model pinning to future ideas * fix: restore rich descriptions lost in auto-generation - Snapshot flags: add back value hints (-d <N>, -s <sel>, -o <path>) - Snapshot flags: restore parenthetical context (@e refs, @c refs, etc.) - Commands: is → includes valid states enum - Commands: console → notes --errors filter behavior - Commands: press → lists common keys (Enter, Tab, Escape) - Commands: cookie-import-browser → describes picker UI - Commands: dialog-accept → specifies alert/confirm/prompt - Tips: restore → arrow (was downgraded to ->) * test: quality evals for generated SKILL.md descriptions Catches the exact regressions we shipped and caught in review: - Snapshot flags must include value hints (-d <N>, -s <sel>, -o <path>) - is command must list all valid states (visible/hidden/enabled/...) - press command must list example keys (Enter, Tab, Escape) - console command must describe --errors behavior - Snapshot -i must mention @e refs, -C must mention @c refs - All descriptions must be >= 8 chars (no empty stubs) - Tips section must use → not -> * feat: LLM-as-judge evals for SKILL.md documentation quality 4 eval tests using Anthropic API (claude-haiku, ~$0.01-0.03/run): - Command reference table: clarity/completeness/actionability >= 4/5 - Snapshot flags section: same thresholds - browse/SKILL.md overall quality - Regression: generated version must score >= hand-maintained baseline Requires ANTHROPIC_API_KEY. Auto-skips without it. Run: bun run test:eval (or ANTHROPIC_API_KEY=sk-... bun test test/skill-llm-eval.test.ts) * chore: bump version to 0.3.3, update changelog Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * docs: add ARCHITECTURE.md, update CLAUDE.md and CONTRIBUTING.md Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * feat: conductor.json lifecycle hooks + .env propagation across worktrees bin/dev-setup now copies .env from main worktree so API keys carry over to Conductor workspaces automatically. conductor.json wires up setup and archive hooks. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * docs: complete CHANGELOG for v0.3.3 (architecture, conductor, .env) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-05-01 19:25:10 +02:00 · 2026-03-13 21:08:12 -07:00
parent ea0c0dad5e
commit 5205070299
29 changed files with 2479 additions and 135 deletions
@@ -0,0 +1,160 @@
+/**
+ * Agent SDK wrapper for skill E2E testing.
+ *
+ * Spawns a Claude Code session, runs a prompt, collects messages,
+ * scans tool_result messages for browse errors.
+ */
+
+import { query } from '@anthropic-ai/claude-agent-sdk';
+import * as fs from 'fs';
+import * as path from 'path';
+
+export interface SkillTestResult {
+  messages: any[];
+  toolCalls: Array<{ tool: string; input: any; output: string }>;
+  browseErrors: string[];
+  exitReason: string;
+  duration: number;
+}
+
+const BROWSE_ERROR_PATTERNS = [
+  /Unknown command: \w+/,
+  /Unknown snapshot flag: .+/,
+  /Exit code 1/,
+  /ERROR: browse binary not found/,
+  /Server failed to start/,
+];
+
+export async function runSkillTest(options: {
+  prompt: string;
+  workingDirectory: string;
+  maxTurns?: number;
+  allowedTools?: string[];
+  timeout?: number;
+}): Promise<SkillTestResult> {
+  // Fail fast if running inside an Agent SDK session — nested sessions hang
+  if (process.env.CLAUDECODE || process.env.CLAUDE_CODE_ENTRYPOINT) {
+    throw new Error(
+      'Cannot run E2E skill tests inside a Claude Code session. ' +
+      'Run from a plain terminal: SKILL_E2E=1 bun test test/skill-e2e.test.ts'
+    );
+  }
+
+  const {
+    prompt,
+    workingDirectory,
+    maxTurns = 15,
+    allowedTools = ['Bash', 'Read', 'Write'],
+    timeout = 120_000,
+  } = options;
+
+  const messages: any[] = [];
+  const toolCalls: SkillTestResult['toolCalls'] = [];
+  const browseErrors: string[] = [];
+  let exitReason = 'unknown';
+
+  const startTime = Date.now();
+
+  // Strip all Claude-related env vars to allow nested sessions.
+  // Without this, the child claude process thinks it's an SDK child
+  // and hangs waiting for parent IPC instead of running independently.
+  const env: Record<string, string | undefined> = {};
+  for (const [key] of Object.entries(process.env)) {
+    if (key.startsWith('CLAUDE') || key.startsWith('CLAUDECODE')) {
+      env[key] = undefined;
+    }
+  }
+
+  const q = query({
+    prompt,
+    options: {
+      cwd: workingDirectory,
+      allowedTools,
+      permissionMode: 'bypassPermissions',
+      allowDangerouslySkipPermissions: true,
+      maxTurns,
+      env,
+    },
+  });
+
+  const timeoutPromise = new Promise<never>((_, reject) => {
+    setTimeout(() => reject(new Error(`Skill test timed out after ${timeout}ms`)), timeout);
+  });
+
+  try {
+    const runner = (async () => {
+      for await (const msg of q) {
+        messages.push(msg);
+
+        // Extract tool calls from assistant messages
+        if (msg.type === 'assistant' && msg.message?.content) {
+          for (const block of msg.message.content) {
+            if (block.type === 'tool_use') {
+              toolCalls.push({
+                tool: block.name,
+                input: block.input,
+                output: '', // will be filled from tool_result
+              });
+            }
+            // Scan tool_result blocks for browse errors
+            if (block.type === 'tool_result' || (typeof block === 'object' && 'text' in block)) {
+              const text = typeof block === 'string' ? block : (block as any).text || '';
+              for (const pattern of BROWSE_ERROR_PATTERNS) {
+                if (pattern.test(text)) {
+                  browseErrors.push(text.slice(0, 200));
+                }
+              }
+            }
+          }
+        }
+
+        // Also scan user messages (which contain tool results)
+        if (msg.type === 'user' && msg.message?.content) {
+          const content = Array.isArray(msg.message.content) ? msg.message.content : [msg.message.content];
+          for (const block of content) {
+            const text = typeof block === 'string' ? block : (block as any)?.text || (block as any)?.content || '';
+            if (typeof text === 'string') {
+              for (const pattern of BROWSE_ERROR_PATTERNS) {
+                if (pattern.test(text)) {
+                  browseErrors.push(text.slice(0, 200));
+                }
+              }
+            }
+          }
+        }
+
+        // Capture result
+        if (msg.type === 'result') {
+          exitReason = msg.subtype || 'success';
+        }
+      }
+    })();
+
+    await Promise.race([runner, timeoutPromise]);
+  } catch (err: any) {
+    exitReason = err.message?.includes('timed out') ? 'timeout' : `error: ${err.message}`;
+  }
+
+  const duration = Date.now() - startTime;
+
+  // Save transcript on failure
+  if (browseErrors.length > 0 || exitReason !== 'success') {
+    try {
+      const transcriptDir = path.join(workingDirectory, '.gstack', 'test-transcripts');
+      fs.mkdirSync(transcriptDir, { recursive: true });
+      const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+      const transcriptPath = path.join(transcriptDir, `e2e-${timestamp}.json`);
+      fs.writeFileSync(transcriptPath, JSON.stringify({
+        prompt,
+        exitReason,
+        browseErrors,
+        duration,
+        messages: messages.map(m => ({ type: m.type, subtype: m.subtype })),
+      }, null, 2));
+    } catch {
+      // Transcript save failures are non-fatal
+    }
+  }
+
+  return { messages, toolCalls, browseErrors, exitReason, duration };
+}
@@ -0,0 +1,133 @@
+/**
+ * SKILL.md parser and validator.
+ *
+ * Extracts $B commands from code blocks, validates them against
+ * the command registry and snapshot flags.
+ *
+ * Used by:
+ *   - test/skill-validation.test.ts (Tier 1 static tests)
+ *   - scripts/skill-check.ts (health summary)
+ *   - scripts/dev-skill.ts (watch mode)
+ */
+
+import { ALL_COMMANDS } from '../../browse/src/commands';
+import { parseSnapshotArgs } from '../../browse/src/snapshot';
+import * as fs from 'fs';
+
+export interface BrowseCommand {
+  command: string;
+  args: string[];
+  line: number;
+  raw: string;
+}
+
+export interface ValidationResult {
+  valid: BrowseCommand[];
+  invalid: BrowseCommand[];
+  snapshotFlagErrors: Array<{ command: BrowseCommand; error: string }>;
+  warnings: string[];
+}
+
+/**
+ * Extract all $B invocations from bash code blocks in a SKILL.md file.
+ */
+export function extractBrowseCommands(skillPath: string): BrowseCommand[] {
+  const content = fs.readFileSync(skillPath, 'utf-8');
+  const lines = content.split('\n');
+  const commands: BrowseCommand[] = [];
+
+  let inBashBlock = false;
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+
+    // Detect code block boundaries
+    if (line.trimStart().startsWith('```')) {
+      if (inBashBlock) {
+        inBashBlock = false;
+      } else if (line.trimStart().startsWith('```bash')) {
+        inBashBlock = true;
+      }
+      // Non-bash code blocks (```json, ```, ```js, etc.) are skipped
+      continue;
+    }
+
+    if (!inBashBlock) continue;
+
+    // Match lines with $B command invocations
+    // Handle multiple $B commands on one line (e.g., "$B click @e3       $B fill @e4 "value"")
+    const matches = line.matchAll(/\$B\s+(\S+)(?:\s+([^\$]*))?/g);
+    for (const match of matches) {
+      const command = match[1];
+      let argsStr = (match[2] || '').trim();
+
+      // Strip inline comments (# ...) — but not inside quotes
+      // Simple approach: remove everything from first unquoted # onward
+      let inQuote = false;
+      for (let j = 0; j < argsStr.length; j++) {
+        if (argsStr[j] === '"') inQuote = !inQuote;
+        if (argsStr[j] === '#' && !inQuote) {
+          argsStr = argsStr.slice(0, j).trim();
+          break;
+        }
+      }
+
+      // Parse args — handle quoted strings
+      const args: string[] = [];
+      if (argsStr) {
+        const argMatches = argsStr.matchAll(/"([^"]*)"|(\S+)/g);
+        for (const am of argMatches) {
+          args.push(am[1] ?? am[2]);
+        }
+      }
+
+      commands.push({
+        command,
+        args,
+        line: i + 1, // 1-based
+        raw: match[0].trim(),
+      });
+    }
+  }
+
+  return commands;
+}
+
+/**
+ * Extract and validate all $B commands in a SKILL.md file.
+ */
+export function validateSkill(skillPath: string): ValidationResult {
+  const commands = extractBrowseCommands(skillPath);
+  const result: ValidationResult = {
+    valid: [],
+    invalid: [],
+    snapshotFlagErrors: [],
+    warnings: [],
+  };
+
+  if (commands.length === 0) {
+    result.warnings.push('no $B commands found');
+    return result;
+  }
+
+  for (const cmd of commands) {
+    if (!ALL_COMMANDS.has(cmd.command)) {
+      result.invalid.push(cmd);
+      continue;
+    }
+
+    // Validate snapshot flags
+    if (cmd.command === 'snapshot' && cmd.args.length > 0) {
+      try {
+        parseSnapshotArgs(cmd.args);
+      } catch (err: any) {
+        result.snapshotFlagErrors.push({ command: cmd, error: err.message });
+        continue;
+      }
+    }
+
+    result.valid.push(cmd);
+  }
+
+  return result;
+}