diff --git a/scripts/dev-skill.ts b/scripts/dev-skill.ts new file mode 100644 index 00000000..1842c837 --- /dev/null +++ b/scripts/dev-skill.ts @@ -0,0 +1,82 @@ +#!/usr/bin/env bun +/** + * dev:skill — Watch mode for SKILL.md template development. + * + * Watches .tmpl files, regenerates SKILL.md files on change, + * validates all $B commands immediately. + */ + +import { validateSkill } from '../test/helpers/skill-parser'; +import { execSync } from 'child_process'; +import * as fs from 'fs'; +import * as path from 'path'; + +const ROOT = path.resolve(import.meta.dir, '..'); + +const TEMPLATES = [ + { tmpl: path.join(ROOT, 'SKILL.md.tmpl'), output: 'SKILL.md' }, + { tmpl: path.join(ROOT, 'browse', 'SKILL.md.tmpl'), output: 'browse/SKILL.md' }, +]; + +function regenerateAndValidate() { + // Regenerate + try { + execSync('bun run scripts/gen-skill-docs.ts', { cwd: ROOT, stdio: 'pipe' }); + } catch (err: any) { + console.log(` [gen] ERROR: ${err.stderr?.toString().trim() || err.message}`); + return; + } + + // Validate each generated file + for (const { output } of TEMPLATES) { + const fullPath = path.join(ROOT, output); + if (!fs.existsSync(fullPath)) continue; + + const result = validateSkill(fullPath); + const totalValid = result.valid.length; + const totalInvalid = result.invalid.length; + const totalSnapErrors = result.snapshotFlagErrors.length; + + if (totalInvalid > 0 || totalSnapErrors > 0) { + console.log(` [check] \u274c ${output} (${totalValid} valid)`); + for (const inv of result.invalid) { + console.log(` Unknown command: '${inv.command}' at line ${inv.line}`); + } + for (const se of result.snapshotFlagErrors) { + console.log(` ${se.error} at line ${se.command.line}`); + } + } else { + console.log(` [check] \u2705 ${output} — ${totalValid} commands, all valid`); + } + } +} + +// Initial run +console.log(' [watch] Watching *.md.tmpl files...'); +regenerateAndValidate(); + +// Watch for changes +for (const { tmpl } of TEMPLATES) { + if (!fs.existsSync(tmpl)) continue; + fs.watch(tmpl, () => { + console.log(`\n [watch] ${path.relative(ROOT, tmpl)} changed`); + regenerateAndValidate(); + }); +} + +// Also watch commands.ts and snapshot.ts (source of truth changes) +const SOURCE_FILES = [ + path.join(ROOT, 'browse', 'src', 'commands.ts'), + path.join(ROOT, 'browse', 'src', 'snapshot.ts'), +]; + +for (const src of SOURCE_FILES) { + if (!fs.existsSync(src)) continue; + fs.watch(src, () => { + console.log(`\n [watch] ${path.relative(ROOT, src)} changed`); + regenerateAndValidate(); + }); +} + +// Keep alive +console.log(' [watch] Press Ctrl+C to stop\n'); diff --git a/scripts/skill-check.ts b/scripts/skill-check.ts new file mode 100644 index 00000000..fd10b529 --- /dev/null +++ b/scripts/skill-check.ts @@ -0,0 +1,111 @@ +#!/usr/bin/env bun +/** + * skill:check — Health summary for all SKILL.md files. + * + * Reports: + * - Command validation (valid/invalid/snapshot errors) + * - Template coverage (which SKILL.md files have .tmpl sources) + * - Freshness check (generated files match committed files) + */ + +import { validateSkill } from '../test/helpers/skill-parser'; +import * as fs from 'fs'; +import * as path from 'path'; +import { execSync } from 'child_process'; + +const ROOT = path.resolve(import.meta.dir, '..'); + +// Find all SKILL.md files +const SKILL_FILES = [ + 'SKILL.md', + 'browse/SKILL.md', + 'qa/SKILL.md', + 'ship/SKILL.md', + 'review/SKILL.md', + 'retro/SKILL.md', + 'plan-ceo-review/SKILL.md', + 'plan-eng-review/SKILL.md', + 'setup-browser-cookies/SKILL.md', +].filter(f => fs.existsSync(path.join(ROOT, f))); + +let hasErrors = false; + +// ─── Skills ───────────────────────────────────────────────── + +console.log(' Skills:'); +for (const file of SKILL_FILES) { + const fullPath = path.join(ROOT, file); + const result = validateSkill(fullPath); + + if (result.warnings.length > 0) { + console.log(` \u26a0\ufe0f ${file.padEnd(30)} — ${result.warnings.join(', ')}`); + continue; + } + + const totalValid = result.valid.length; + const totalInvalid = result.invalid.length; + const totalSnapErrors = result.snapshotFlagErrors.length; + + if (totalInvalid > 0 || totalSnapErrors > 0) { + hasErrors = true; + console.log(` \u274c ${file.padEnd(30)} — ${totalValid} valid, ${totalInvalid} invalid, ${totalSnapErrors} snapshot errors`); + for (const inv of result.invalid) { + console.log(` line ${inv.line}: unknown command '${inv.command}'`); + } + for (const se of result.snapshotFlagErrors) { + console.log(` line ${se.command.line}: ${se.error}`); + } + } else { + console.log(` \u2705 ${file.padEnd(30)} — ${totalValid} commands, all valid`); + } +} + +// ─── Templates ────────────────────────────────────────────── + +console.log('\n Templates:'); +const TEMPLATES = [ + { tmpl: 'SKILL.md.tmpl', output: 'SKILL.md' }, + { tmpl: 'browse/SKILL.md.tmpl', output: 'browse/SKILL.md' }, +]; + +for (const { tmpl, output } of TEMPLATES) { + const tmplPath = path.join(ROOT, tmpl); + const outPath = path.join(ROOT, output); + if (!fs.existsSync(tmplPath)) { + console.log(` \u26a0\ufe0f ${output.padEnd(30)} — no template`); + continue; + } + if (!fs.existsSync(outPath)) { + hasErrors = true; + console.log(` \u274c ${output.padEnd(30)} — generated file missing! Run: bun run gen:skill-docs`); + continue; + } + console.log(` \u2705 ${tmpl.padEnd(30)} \u2192 ${output}`); +} + +// Skills without templates +for (const file of SKILL_FILES) { + const tmplPath = path.join(ROOT, file + '.tmpl'); + if (!fs.existsSync(tmplPath) && !TEMPLATES.some(t => t.output === file)) { + console.log(` \u26a0\ufe0f ${file.padEnd(30)} — no template (OK if no $B commands)`); + } +} + +// ─── Freshness ────────────────────────────────────────────── + +console.log('\n Freshness:'); +try { + execSync('bun run scripts/gen-skill-docs.ts --dry-run', { cwd: ROOT, stdio: 'pipe' }); + console.log(' \u2705 All generated files are fresh'); +} catch (err: any) { + hasErrors = true; + const output = err.stdout?.toString() || ''; + console.log(' \u274c Generated files are stale:'); + for (const line of output.split('\n').filter((l: string) => l.startsWith('STALE'))) { + console.log(` ${line}`); + } + console.log(' Run: bun run gen:skill-docs'); +} + +console.log(''); +process.exit(hasErrors ? 1 : 0); diff --git a/test/helpers/session-runner.ts b/test/helpers/session-runner.ts new file mode 100644 index 00000000..13e0b7eb --- /dev/null +++ b/test/helpers/session-runner.ts @@ -0,0 +1,160 @@ +/** + * Agent SDK wrapper for skill E2E testing. + * + * Spawns a Claude Code session, runs a prompt, collects messages, + * scans tool_result messages for browse errors. + */ + +import { query } from '@anthropic-ai/claude-agent-sdk'; +import * as fs from 'fs'; +import * as path from 'path'; + +export interface SkillTestResult { + messages: any[]; + toolCalls: Array<{ tool: string; input: any; output: string }>; + browseErrors: string[]; + exitReason: string; + duration: number; +} + +const BROWSE_ERROR_PATTERNS = [ + /Unknown command: \w+/, + /Unknown snapshot flag: .+/, + /Exit code 1/, + /ERROR: browse binary not found/, + /Server failed to start/, +]; + +export async function runSkillTest(options: { + prompt: string; + workingDirectory: string; + maxTurns?: number; + allowedTools?: string[]; + timeout?: number; +}): Promise { + // Fail fast if running inside an Agent SDK session — nested sessions hang + if (process.env.CLAUDECODE || process.env.CLAUDE_CODE_ENTRYPOINT) { + throw new Error( + 'Cannot run E2E skill tests inside a Claude Code session. ' + + 'Run from a plain terminal: SKILL_E2E=1 bun test test/skill-e2e.test.ts' + ); + } + + const { + prompt, + workingDirectory, + maxTurns = 15, + allowedTools = ['Bash', 'Read', 'Write'], + timeout = 120_000, + } = options; + + const messages: any[] = []; + const toolCalls: SkillTestResult['toolCalls'] = []; + const browseErrors: string[] = []; + let exitReason = 'unknown'; + + const startTime = Date.now(); + + // Strip all Claude-related env vars to allow nested sessions. + // Without this, the child claude process thinks it's an SDK child + // and hangs waiting for parent IPC instead of running independently. + const env: Record = {}; + for (const [key] of Object.entries(process.env)) { + if (key.startsWith('CLAUDE') || key.startsWith('CLAUDECODE')) { + env[key] = undefined; + } + } + + const q = query({ + prompt, + options: { + cwd: workingDirectory, + allowedTools, + permissionMode: 'bypassPermissions', + allowDangerouslySkipPermissions: true, + maxTurns, + env, + }, + }); + + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => reject(new Error(`Skill test timed out after ${timeout}ms`)), timeout); + }); + + try { + const runner = (async () => { + for await (const msg of q) { + messages.push(msg); + + // Extract tool calls from assistant messages + if (msg.type === 'assistant' && msg.message?.content) { + for (const block of msg.message.content) { + if (block.type === 'tool_use') { + toolCalls.push({ + tool: block.name, + input: block.input, + output: '', // will be filled from tool_result + }); + } + // Scan tool_result blocks for browse errors + if (block.type === 'tool_result' || (typeof block === 'object' && 'text' in block)) { + const text = typeof block === 'string' ? block : (block as any).text || ''; + for (const pattern of BROWSE_ERROR_PATTERNS) { + if (pattern.test(text)) { + browseErrors.push(text.slice(0, 200)); + } + } + } + } + } + + // Also scan user messages (which contain tool results) + if (msg.type === 'user' && msg.message?.content) { + const content = Array.isArray(msg.message.content) ? msg.message.content : [msg.message.content]; + for (const block of content) { + const text = typeof block === 'string' ? block : (block as any)?.text || (block as any)?.content || ''; + if (typeof text === 'string') { + for (const pattern of BROWSE_ERROR_PATTERNS) { + if (pattern.test(text)) { + browseErrors.push(text.slice(0, 200)); + } + } + } + } + } + + // Capture result + if (msg.type === 'result') { + exitReason = msg.subtype || 'success'; + } + } + })(); + + await Promise.race([runner, timeoutPromise]); + } catch (err: any) { + exitReason = err.message?.includes('timed out') ? 'timeout' : `error: ${err.message}`; + } + + const duration = Date.now() - startTime; + + // Save transcript on failure + if (browseErrors.length > 0 || exitReason !== 'success') { + try { + const transcriptDir = path.join(workingDirectory, '.gstack', 'test-transcripts'); + fs.mkdirSync(transcriptDir, { recursive: true }); + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const transcriptPath = path.join(transcriptDir, `e2e-${timestamp}.json`); + fs.writeFileSync(transcriptPath, JSON.stringify({ + prompt, + exitReason, + browseErrors, + duration, + messages: messages.map(m => ({ type: m.type, subtype: m.subtype })), + }, null, 2)); + } catch { + // Transcript save failures are non-fatal + } + } + + return { messages, toolCalls, browseErrors, exitReason, duration }; +} diff --git a/test/skill-e2e.test.ts b/test/skill-e2e.test.ts new file mode 100644 index 00000000..d395fe15 --- /dev/null +++ b/test/skill-e2e.test.ts @@ -0,0 +1,79 @@ +import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { runSkillTest } from './helpers/session-runner'; +import { startTestServer } from '../browse/test/test-server'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; + +// Skip if SKILL_E2E not set, or if running inside a Claude Code / Agent SDK session +// (nested Agent SDK sessions hang because the parent intercepts child claude subprocesses) +const isInsideAgentSDK = !!process.env.CLAUDECODE || !!process.env.CLAUDE_CODE_ENTRYPOINT; +const describeE2E = (process.env.SKILL_E2E && !isInsideAgentSDK) ? describe : describe.skip; + +let testServer: ReturnType; +let tmpDir: string; + +describeE2E('Skill E2E tests', () => { + beforeAll(() => { + testServer = startTestServer(); + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-')); + + // Symlink browse binary into tmpdir for the skill to find + const browseBin = path.resolve(import.meta.dir, '..', 'browse', 'dist', 'browse'); + const binDir = path.join(tmpDir, 'browse', 'dist'); + fs.mkdirSync(binDir, { recursive: true }); + if (fs.existsSync(browseBin)) { + fs.symlinkSync(browseBin, path.join(binDir, 'browse')); + } + + // Also create browse/bin/find-browse so the SKILL.md setup works + const findBrowseDir = path.join(tmpDir, 'browse', 'bin'); + fs.mkdirSync(findBrowseDir, { recursive: true }); + fs.writeFileSync(path.join(findBrowseDir, 'find-browse'), `#!/bin/bash\necho "${browseBin}"\n`, { mode: 0o755 }); + }); + + afterAll(() => { + testServer?.server?.stop(); + // Clean up tmpdir + try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {} + }); + + test('browse basic commands work without errors', async () => { + const result = await runSkillTest({ + prompt: `You have a browse binary at ${path.resolve(import.meta.dir, '..', 'browse', 'dist', 'browse')}. Assign it to B variable and run these commands in sequence: +1. $B goto ${testServer.url} +2. $B snapshot -i +3. $B text +4. $B screenshot /tmp/skill-e2e-test.png +Report the results of each command.`, + workingDirectory: tmpDir, + maxTurns: 10, + timeout: 60_000, + }); + + expect(result.browseErrors).toHaveLength(0); + expect(result.exitReason).toBe('success'); + }, 90_000); + + test('browse snapshot flags all work', async () => { + const result = await runSkillTest({ + prompt: `You have a browse binary at ${path.resolve(import.meta.dir, '..', 'browse', 'dist', 'browse')}. Assign it to B variable and run: +1. $B goto ${testServer.url} +2. $B snapshot -i +3. $B snapshot -c +4. $B snapshot -D +5. $B snapshot -i -a -o /tmp/skill-e2e-annotated.png +Report what each command returned.`, + workingDirectory: tmpDir, + maxTurns: 10, + timeout: 60_000, + }); + + expect(result.browseErrors).toHaveLength(0); + expect(result.exitReason).toBe('success'); + }, 90_000); + + test.todo('/qa quick completes without browse errors'); + test.todo('/ship completes without browse errors'); + test.todo('/review completes without browse errors'); +});