mirror of
https://github.com/garrytan/gstack.git
synced 2026-05-05 05:05:08 +02:00
feat: DX tools (skill:check, dev:skill) + Tier 2 E2E test scaffolding
- scripts/skill-check.ts: health summary for all SKILL.md files (commands, templates, freshness) - scripts/dev-skill.ts: watch mode for template development - test/helpers/session-runner.ts: Agent SDK wrapper for E2E skill tests - test/skill-e2e.test.ts: 2 E2E tests + 3 stubs (auto-skip inside Claude Code sessions) - E2E tests must run from plain terminal: SKILL_E2E=1 bun test test/skill-e2e.test.ts
This commit is contained in:
Garry Tan
@@ -0,0 +1,82 @@
|
||||
#!/usr/bin/env bun
|
||||
/**
|
||||
* dev:skill — Watch mode for SKILL.md template development.
|
||||
*
|
||||
* Watches .tmpl files, regenerates SKILL.md files on change,
|
||||
* validates all $B commands immediately.
|
||||
*/
|
||||
|
||||
import { validateSkill } from '../test/helpers/skill-parser';
|
||||
import { execSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
|
||||
const TEMPLATES = [
|
||||
{ tmpl: path.join(ROOT, 'SKILL.md.tmpl'), output: 'SKILL.md' },
|
||||
{ tmpl: path.join(ROOT, 'browse', 'SKILL.md.tmpl'), output: 'browse/SKILL.md' },
|
||||
];
|
||||
|
||||
function regenerateAndValidate() {
|
||||
// Regenerate
|
||||
try {
|
||||
execSync('bun run scripts/gen-skill-docs.ts', { cwd: ROOT, stdio: 'pipe' });
|
||||
} catch (err: any) {
|
||||
console.log(` [gen] ERROR: ${err.stderr?.toString().trim() || err.message}`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Validate each generated file
|
||||
for (const { output } of TEMPLATES) {
|
||||
const fullPath = path.join(ROOT, output);
|
||||
if (!fs.existsSync(fullPath)) continue;
|
||||
|
||||
const result = validateSkill(fullPath);
|
||||
const totalValid = result.valid.length;
|
||||
const totalInvalid = result.invalid.length;
|
||||
const totalSnapErrors = result.snapshotFlagErrors.length;
|
||||
|
||||
if (totalInvalid > 0 || totalSnapErrors > 0) {
|
||||
console.log(` [check] \u274c ${output} (${totalValid} valid)`);
|
||||
for (const inv of result.invalid) {
|
||||
console.log(` Unknown command: '${inv.command}' at line ${inv.line}`);
|
||||
}
|
||||
for (const se of result.snapshotFlagErrors) {
|
||||
console.log(` ${se.error} at line ${se.command.line}`);
|
||||
}
|
||||
} else {
|
||||
console.log(` [check] \u2705 ${output} — ${totalValid} commands, all valid`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Initial run
|
||||
console.log(' [watch] Watching *.md.tmpl files...');
|
||||
regenerateAndValidate();
|
||||
|
||||
// Watch for changes
|
||||
for (const { tmpl } of TEMPLATES) {
|
||||
if (!fs.existsSync(tmpl)) continue;
|
||||
fs.watch(tmpl, () => {
|
||||
console.log(`\n [watch] ${path.relative(ROOT, tmpl)} changed`);
|
||||
regenerateAndValidate();
|
||||
});
|
||||
}
|
||||
|
||||
// Also watch commands.ts and snapshot.ts (source of truth changes)
|
||||
const SOURCE_FILES = [
|
||||
path.join(ROOT, 'browse', 'src', 'commands.ts'),
|
||||
path.join(ROOT, 'browse', 'src', 'snapshot.ts'),
|
||||
];
|
||||
|
||||
for (const src of SOURCE_FILES) {
|
||||
if (!fs.existsSync(src)) continue;
|
||||
fs.watch(src, () => {
|
||||
console.log(`\n [watch] ${path.relative(ROOT, src)} changed`);
|
||||
regenerateAndValidate();
|
||||
});
|
||||
}
|
||||
|
||||
// Keep alive
|
||||
console.log(' [watch] Press Ctrl+C to stop\n');
|
||||
@@ -0,0 +1,111 @@
|
||||
#!/usr/bin/env bun
|
||||
/**
|
||||
* skill:check — Health summary for all SKILL.md files.
|
||||
*
|
||||
* Reports:
|
||||
* - Command validation (valid/invalid/snapshot errors)
|
||||
* - Template coverage (which SKILL.md files have .tmpl sources)
|
||||
* - Freshness check (generated files match committed files)
|
||||
*/
|
||||
|
||||
import { validateSkill } from '../test/helpers/skill-parser';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { execSync } from 'child_process';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
|
||||
// Find all SKILL.md files
|
||||
const SKILL_FILES = [
|
||||
'SKILL.md',
|
||||
'browse/SKILL.md',
|
||||
'qa/SKILL.md',
|
||||
'ship/SKILL.md',
|
||||
'review/SKILL.md',
|
||||
'retro/SKILL.md',
|
||||
'plan-ceo-review/SKILL.md',
|
||||
'plan-eng-review/SKILL.md',
|
||||
'setup-browser-cookies/SKILL.md',
|
||||
].filter(f => fs.existsSync(path.join(ROOT, f)));
|
||||
|
||||
let hasErrors = false;
|
||||
|
||||
// ─── Skills ─────────────────────────────────────────────────
|
||||
|
||||
console.log(' Skills:');
|
||||
for (const file of SKILL_FILES) {
|
||||
const fullPath = path.join(ROOT, file);
|
||||
const result = validateSkill(fullPath);
|
||||
|
||||
if (result.warnings.length > 0) {
|
||||
console.log(` \u26a0\ufe0f ${file.padEnd(30)} — ${result.warnings.join(', ')}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const totalValid = result.valid.length;
|
||||
const totalInvalid = result.invalid.length;
|
||||
const totalSnapErrors = result.snapshotFlagErrors.length;
|
||||
|
||||
if (totalInvalid > 0 || totalSnapErrors > 0) {
|
||||
hasErrors = true;
|
||||
console.log(` \u274c ${file.padEnd(30)} — ${totalValid} valid, ${totalInvalid} invalid, ${totalSnapErrors} snapshot errors`);
|
||||
for (const inv of result.invalid) {
|
||||
console.log(` line ${inv.line}: unknown command '${inv.command}'`);
|
||||
}
|
||||
for (const se of result.snapshotFlagErrors) {
|
||||
console.log(` line ${se.command.line}: ${se.error}`);
|
||||
}
|
||||
} else {
|
||||
console.log(` \u2705 ${file.padEnd(30)} — ${totalValid} commands, all valid`);
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Templates ──────────────────────────────────────────────
|
||||
|
||||
console.log('\n Templates:');
|
||||
const TEMPLATES = [
|
||||
{ tmpl: 'SKILL.md.tmpl', output: 'SKILL.md' },
|
||||
{ tmpl: 'browse/SKILL.md.tmpl', output: 'browse/SKILL.md' },
|
||||
];
|
||||
|
||||
for (const { tmpl, output } of TEMPLATES) {
|
||||
const tmplPath = path.join(ROOT, tmpl);
|
||||
const outPath = path.join(ROOT, output);
|
||||
if (!fs.existsSync(tmplPath)) {
|
||||
console.log(` \u26a0\ufe0f ${output.padEnd(30)} — no template`);
|
||||
continue;
|
||||
}
|
||||
if (!fs.existsSync(outPath)) {
|
||||
hasErrors = true;
|
||||
console.log(` \u274c ${output.padEnd(30)} — generated file missing! Run: bun run gen:skill-docs`);
|
||||
continue;
|
||||
}
|
||||
console.log(` \u2705 ${tmpl.padEnd(30)} \u2192 ${output}`);
|
||||
}
|
||||
|
||||
// Skills without templates
|
||||
for (const file of SKILL_FILES) {
|
||||
const tmplPath = path.join(ROOT, file + '.tmpl');
|
||||
if (!fs.existsSync(tmplPath) && !TEMPLATES.some(t => t.output === file)) {
|
||||
console.log(` \u26a0\ufe0f ${file.padEnd(30)} — no template (OK if no $B commands)`);
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Freshness ──────────────────────────────────────────────
|
||||
|
||||
console.log('\n Freshness:');
|
||||
try {
|
||||
execSync('bun run scripts/gen-skill-docs.ts --dry-run', { cwd: ROOT, stdio: 'pipe' });
|
||||
console.log(' \u2705 All generated files are fresh');
|
||||
} catch (err: any) {
|
||||
hasErrors = true;
|
||||
const output = err.stdout?.toString() || '';
|
||||
console.log(' \u274c Generated files are stale:');
|
||||
for (const line of output.split('\n').filter((l: string) => l.startsWith('STALE'))) {
|
||||
console.log(` ${line}`);
|
||||
}
|
||||
console.log(' Run: bun run gen:skill-docs');
|
||||
}
|
||||
|
||||
console.log('');
|
||||
process.exit(hasErrors ? 1 : 0);
|
||||
@@ -0,0 +1,160 @@
|
||||
/**
|
||||
* Agent SDK wrapper for skill E2E testing.
|
||||
*
|
||||
* Spawns a Claude Code session, runs a prompt, collects messages,
|
||||
* scans tool_result messages for browse errors.
|
||||
*/
|
||||
|
||||
import { query } from '@anthropic-ai/claude-agent-sdk';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
export interface SkillTestResult {
|
||||
messages: any[];
|
||||
toolCalls: Array<{ tool: string; input: any; output: string }>;
|
||||
browseErrors: string[];
|
||||
exitReason: string;
|
||||
duration: number;
|
||||
}
|
||||
|
||||
const BROWSE_ERROR_PATTERNS = [
|
||||
/Unknown command: \w+/,
|
||||
/Unknown snapshot flag: .+/,
|
||||
/Exit code 1/,
|
||||
/ERROR: browse binary not found/,
|
||||
/Server failed to start/,
|
||||
];
|
||||
|
||||
export async function runSkillTest(options: {
|
||||
prompt: string;
|
||||
workingDirectory: string;
|
||||
maxTurns?: number;
|
||||
allowedTools?: string[];
|
||||
timeout?: number;
|
||||
}): Promise<SkillTestResult> {
|
||||
// Fail fast if running inside an Agent SDK session — nested sessions hang
|
||||
if (process.env.CLAUDECODE || process.env.CLAUDE_CODE_ENTRYPOINT) {
|
||||
throw new Error(
|
||||
'Cannot run E2E skill tests inside a Claude Code session. ' +
|
||||
'Run from a plain terminal: SKILL_E2E=1 bun test test/skill-e2e.test.ts'
|
||||
);
|
||||
}
|
||||
|
||||
const {
|
||||
prompt,
|
||||
workingDirectory,
|
||||
maxTurns = 15,
|
||||
allowedTools = ['Bash', 'Read', 'Write'],
|
||||
timeout = 120_000,
|
||||
} = options;
|
||||
|
||||
const messages: any[] = [];
|
||||
const toolCalls: SkillTestResult['toolCalls'] = [];
|
||||
const browseErrors: string[] = [];
|
||||
let exitReason = 'unknown';
|
||||
|
||||
const startTime = Date.now();
|
||||
|
||||
// Strip all Claude-related env vars to allow nested sessions.
|
||||
// Without this, the child claude process thinks it's an SDK child
|
||||
// and hangs waiting for parent IPC instead of running independently.
|
||||
const env: Record<string, string | undefined> = {};
|
||||
for (const [key] of Object.entries(process.env)) {
|
||||
if (key.startsWith('CLAUDE') || key.startsWith('CLAUDECODE')) {
|
||||
env[key] = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
const q = query({
|
||||
prompt,
|
||||
options: {
|
||||
cwd: workingDirectory,
|
||||
allowedTools,
|
||||
permissionMode: 'bypassPermissions',
|
||||
allowDangerouslySkipPermissions: true,
|
||||
maxTurns,
|
||||
env,
|
||||
},
|
||||
});
|
||||
|
||||
const timeoutPromise = new Promise<never>((_, reject) => {
|
||||
setTimeout(() => reject(new Error(`Skill test timed out after ${timeout}ms`)), timeout);
|
||||
});
|
||||
|
||||
try {
|
||||
const runner = (async () => {
|
||||
for await (const msg of q) {
|
||||
messages.push(msg);
|
||||
|
||||
// Extract tool calls from assistant messages
|
||||
if (msg.type === 'assistant' && msg.message?.content) {
|
||||
for (const block of msg.message.content) {
|
||||
if (block.type === 'tool_use') {
|
||||
toolCalls.push({
|
||||
tool: block.name,
|
||||
input: block.input,
|
||||
output: '', // will be filled from tool_result
|
||||
});
|
||||
}
|
||||
// Scan tool_result blocks for browse errors
|
||||
if (block.type === 'tool_result' || (typeof block === 'object' && 'text' in block)) {
|
||||
const text = typeof block === 'string' ? block : (block as any).text || '';
|
||||
for (const pattern of BROWSE_ERROR_PATTERNS) {
|
||||
if (pattern.test(text)) {
|
||||
browseErrors.push(text.slice(0, 200));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Also scan user messages (which contain tool results)
|
||||
if (msg.type === 'user' && msg.message?.content) {
|
||||
const content = Array.isArray(msg.message.content) ? msg.message.content : [msg.message.content];
|
||||
for (const block of content) {
|
||||
const text = typeof block === 'string' ? block : (block as any)?.text || (block as any)?.content || '';
|
||||
if (typeof text === 'string') {
|
||||
for (const pattern of BROWSE_ERROR_PATTERNS) {
|
||||
if (pattern.test(text)) {
|
||||
browseErrors.push(text.slice(0, 200));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Capture result
|
||||
if (msg.type === 'result') {
|
||||
exitReason = msg.subtype || 'success';
|
||||
}
|
||||
}
|
||||
})();
|
||||
|
||||
await Promise.race([runner, timeoutPromise]);
|
||||
} catch (err: any) {
|
||||
exitReason = err.message?.includes('timed out') ? 'timeout' : `error: ${err.message}`;
|
||||
}
|
||||
|
||||
const duration = Date.now() - startTime;
|
||||
|
||||
// Save transcript on failure
|
||||
if (browseErrors.length > 0 || exitReason !== 'success') {
|
||||
try {
|
||||
const transcriptDir = path.join(workingDirectory, '.gstack', 'test-transcripts');
|
||||
fs.mkdirSync(transcriptDir, { recursive: true });
|
||||
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||||
const transcriptPath = path.join(transcriptDir, `e2e-${timestamp}.json`);
|
||||
fs.writeFileSync(transcriptPath, JSON.stringify({
|
||||
prompt,
|
||||
exitReason,
|
||||
browseErrors,
|
||||
duration,
|
||||
messages: messages.map(m => ({ type: m.type, subtype: m.subtype })),
|
||||
}, null, 2));
|
||||
} catch {
|
||||
// Transcript save failures are non-fatal
|
||||
}
|
||||
}
|
||||
|
||||
return { messages, toolCalls, browseErrors, exitReason, duration };
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import { startTestServer } from '../browse/test/test-server';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
|
||||
// Skip if SKILL_E2E not set, or if running inside a Claude Code / Agent SDK session
|
||||
// (nested Agent SDK sessions hang because the parent intercepts child claude subprocesses)
|
||||
const isInsideAgentSDK = !!process.env.CLAUDECODE || !!process.env.CLAUDE_CODE_ENTRYPOINT;
|
||||
const describeE2E = (process.env.SKILL_E2E && !isInsideAgentSDK) ? describe : describe.skip;
|
||||
|
||||
let testServer: ReturnType<typeof startTestServer>;
|
||||
let tmpDir: string;
|
||||
|
||||
describeE2E('Skill E2E tests', () => {
|
||||
beforeAll(() => {
|
||||
testServer = startTestServer();
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-'));
|
||||
|
||||
// Symlink browse binary into tmpdir for the skill to find
|
||||
const browseBin = path.resolve(import.meta.dir, '..', 'browse', 'dist', 'browse');
|
||||
const binDir = path.join(tmpDir, 'browse', 'dist');
|
||||
fs.mkdirSync(binDir, { recursive: true });
|
||||
if (fs.existsSync(browseBin)) {
|
||||
fs.symlinkSync(browseBin, path.join(binDir, 'browse'));
|
||||
}
|
||||
|
||||
// Also create browse/bin/find-browse so the SKILL.md setup works
|
||||
const findBrowseDir = path.join(tmpDir, 'browse', 'bin');
|
||||
fs.mkdirSync(findBrowseDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(findBrowseDir, 'find-browse'), `#!/bin/bash\necho "${browseBin}"\n`, { mode: 0o755 });
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
testServer?.server?.stop();
|
||||
// Clean up tmpdir
|
||||
try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}
|
||||
});
|
||||
|
||||
test('browse basic commands work without errors', async () => {
|
||||
const result = await runSkillTest({
|
||||
prompt: `You have a browse binary at ${path.resolve(import.meta.dir, '..', 'browse', 'dist', 'browse')}. Assign it to B variable and run these commands in sequence:
|
||||
1. $B goto ${testServer.url}
|
||||
2. $B snapshot -i
|
||||
3. $B text
|
||||
4. $B screenshot /tmp/skill-e2e-test.png
|
||||
Report the results of each command.`,
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 10,
|
||||
timeout: 60_000,
|
||||
});
|
||||
|
||||
expect(result.browseErrors).toHaveLength(0);
|
||||
expect(result.exitReason).toBe('success');
|
||||
}, 90_000);
|
||||
|
||||
test('browse snapshot flags all work', async () => {
|
||||
const result = await runSkillTest({
|
||||
prompt: `You have a browse binary at ${path.resolve(import.meta.dir, '..', 'browse', 'dist', 'browse')}. Assign it to B variable and run:
|
||||
1. $B goto ${testServer.url}
|
||||
2. $B snapshot -i
|
||||
3. $B snapshot -c
|
||||
4. $B snapshot -D
|
||||
5. $B snapshot -i -a -o /tmp/skill-e2e-annotated.png
|
||||
Report what each command returned.`,
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 10,
|
||||
timeout: 60_000,
|
||||
});
|
||||
|
||||
expect(result.browseErrors).toHaveLength(0);
|
||||
expect(result.exitReason).toBe('success');
|
||||
}, 90_000);
|
||||
|
||||
test.todo('/qa quick completes without browse errors');
|
||||
test.todo('/ship completes without browse errors');
|
||||
test.todo('/review completes without browse errors');
|
||||
});
|
||||
Reference in New Issue
Block a user