mirror of
https://github.com/garrytan/gstack.git
synced 2026-05-05 05:05:08 +02:00
Merge remote-tracking branch 'origin/main' into garrytan/usage-telemetry
# Conflicts: # SKILL.md # TODOS.md # browse/SKILL.md # design-consultation/SKILL.md # design-review/SKILL.md # document-release/SKILL.md # plan-ceo-review/SKILL.md # plan-design-review/SKILL.md # plan-eng-review/SKILL.md # qa-only/SKILL.md # qa/SKILL.md # retro/SKILL.md # retro/SKILL.md.tmpl # review/SKILL.md # scripts/gen-skill-docs.ts # setup-browser-cookies/SKILL.md # ship/SKILL.md
This commit is contained in:
Garry Tan
@@ -0,0 +1,277 @@
|
||||
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import { parseJSONL, filterByPeriod, formatReport } from '../scripts/analytics';
|
||||
import type { AnalyticsEvent } from '../scripts/analytics';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
import { execSync } from 'child_process';
|
||||
|
||||
const TMP_DIR = path.join(os.tmpdir(), 'analytics-test');
|
||||
const SCRIPT = path.resolve(import.meta.dir, '../scripts/analytics.ts');
|
||||
|
||||
function writeTempJSONL(name: string, lines: string[]): string {
|
||||
fs.mkdirSync(TMP_DIR, { recursive: true });
|
||||
const p = path.join(TMP_DIR, name);
|
||||
fs.writeFileSync(p, lines.join('\n') + '\n');
|
||||
return p;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the analytics script with a custom JSONL file by overriding the path.
|
||||
* We test the exported functions directly for unit tests, and use this
|
||||
* helper for integration-style checks.
|
||||
*/
|
||||
function runScript(jsonlPath: string | null, extraArgs: string = ''): string {
|
||||
// We test via the exported functions; for CLI integration we read the file
|
||||
// and run the pipeline manually to avoid needing to override the hardcoded path.
|
||||
if (jsonlPath === null) {
|
||||
return 'No analytics data found.';
|
||||
}
|
||||
if (!fs.existsSync(jsonlPath)) {
|
||||
return 'No analytics data found.';
|
||||
}
|
||||
const content = fs.readFileSync(jsonlPath, 'utf-8').trim();
|
||||
if (!content) {
|
||||
return 'No analytics data found.';
|
||||
}
|
||||
const events = parseJSONL(content);
|
||||
if (events.length === 0) {
|
||||
return 'No analytics data found.';
|
||||
}
|
||||
// Parse period from extraArgs
|
||||
let period = 'all';
|
||||
const match = extraArgs.match(/--period\s+(\S+)/);
|
||||
if (match) period = match[1];
|
||||
const filtered = filterByPeriod(events, period);
|
||||
return formatReport(filtered, period);
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
fs.mkdirSync(TMP_DIR, { recursive: true });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fs.rmSync(TMP_DIR, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('parseJSONL', () => {
|
||||
test('parses valid JSONL lines', () => {
|
||||
const content = [
|
||||
'{"skill":"ship","ts":"2026-03-18T15:30:00Z","repo":"my-app"}',
|
||||
'{"skill":"qa","ts":"2026-03-18T16:00:00Z","repo":"my-api"}',
|
||||
].join('\n');
|
||||
const events = parseJSONL(content);
|
||||
expect(events).toHaveLength(2);
|
||||
expect(events[0].skill).toBe('ship');
|
||||
expect(events[1].skill).toBe('qa');
|
||||
});
|
||||
|
||||
test('skips malformed lines', () => {
|
||||
const content = [
|
||||
'{"skill":"ship","ts":"2026-03-18T15:30:00Z","repo":"my-app"}',
|
||||
'not valid json',
|
||||
'{broken',
|
||||
'',
|
||||
'{"skill":"qa","ts":"2026-03-18T16:00:00Z","repo":"my-api"}',
|
||||
].join('\n');
|
||||
const events = parseJSONL(content);
|
||||
expect(events).toHaveLength(2);
|
||||
expect(events[0].skill).toBe('ship');
|
||||
expect(events[1].skill).toBe('qa');
|
||||
});
|
||||
|
||||
test('returns empty array for empty string', () => {
|
||||
expect(parseJSONL('')).toHaveLength(0);
|
||||
});
|
||||
|
||||
test('skips objects missing ts field', () => {
|
||||
const content = '{"skill":"ship","repo":"my-app"}\n';
|
||||
const events = parseJSONL(content);
|
||||
expect(events).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('filterByPeriod', () => {
|
||||
const now = new Date();
|
||||
const daysAgo = (n: number) => new Date(now.getTime() - n * 24 * 60 * 60 * 1000).toISOString();
|
||||
|
||||
const events: AnalyticsEvent[] = [
|
||||
{ skill: 'ship', ts: daysAgo(1), repo: 'app' },
|
||||
{ skill: 'qa', ts: daysAgo(3), repo: 'app' },
|
||||
{ skill: 'review', ts: daysAgo(10), repo: 'app' },
|
||||
{ skill: 'retro', ts: daysAgo(40), repo: 'app' },
|
||||
];
|
||||
|
||||
test('period "all" returns all events', () => {
|
||||
expect(filterByPeriod(events, 'all')).toHaveLength(4);
|
||||
});
|
||||
|
||||
test('period "7d" returns only last 7 days', () => {
|
||||
const filtered = filterByPeriod(events, '7d');
|
||||
expect(filtered).toHaveLength(2);
|
||||
expect(filtered[0].skill).toBe('ship');
|
||||
expect(filtered[1].skill).toBe('qa');
|
||||
});
|
||||
|
||||
test('period "30d" returns last 30 days', () => {
|
||||
const filtered = filterByPeriod(events, '30d');
|
||||
expect(filtered).toHaveLength(3);
|
||||
});
|
||||
|
||||
test('invalid period string returns all events', () => {
|
||||
expect(filterByPeriod(events, 'bogus')).toHaveLength(4);
|
||||
});
|
||||
});
|
||||
|
||||
describe('formatReport', () => {
|
||||
test('includes header and period label', () => {
|
||||
const report = formatReport([], 'all');
|
||||
expect(report).toContain('gstack skill usage analytics');
|
||||
expect(report).toContain('Period: all time');
|
||||
});
|
||||
|
||||
test('shows "last 7 days" for 7d period', () => {
|
||||
const report = formatReport([], '7d');
|
||||
expect(report).toContain('Period: last 7 days');
|
||||
});
|
||||
|
||||
test('shows "last 30 days" for 30d period', () => {
|
||||
const report = formatReport([], '30d');
|
||||
expect(report).toContain('Period: last 30 days');
|
||||
});
|
||||
|
||||
test('counts skill invocations correctly', () => {
|
||||
const events: AnalyticsEvent[] = [
|
||||
{ skill: 'ship', ts: '2026-03-18T15:30:00Z', repo: 'app' },
|
||||
{ skill: 'ship', ts: '2026-03-18T16:00:00Z', repo: 'app' },
|
||||
{ skill: 'qa', ts: '2026-03-18T16:30:00Z', repo: 'app' },
|
||||
];
|
||||
const report = formatReport(events);
|
||||
expect(report).toContain('/ship');
|
||||
expect(report).toContain('2 invocations');
|
||||
expect(report).toContain('/qa');
|
||||
expect(report).toContain('1 invocation');
|
||||
});
|
||||
|
||||
test('groups by repo', () => {
|
||||
const events: AnalyticsEvent[] = [
|
||||
{ skill: 'ship', ts: '2026-03-18T15:30:00Z', repo: 'app-a' },
|
||||
{ skill: 'qa', ts: '2026-03-18T16:00:00Z', repo: 'app-a' },
|
||||
{ skill: 'ship', ts: '2026-03-18T16:30:00Z', repo: 'app-b' },
|
||||
];
|
||||
const report = formatReport(events);
|
||||
expect(report).toContain('app-a: ship(1) qa(1)');
|
||||
expect(report).toContain('app-b: ship(1)');
|
||||
});
|
||||
|
||||
test('counts hook fire events separately', () => {
|
||||
const events: AnalyticsEvent[] = [
|
||||
{ skill: 'ship', ts: '2026-03-18T15:30:00Z', repo: 'app' },
|
||||
{ skill: 'careful', ts: '2026-03-18T16:00:00Z', repo: 'app', event: 'hook_fire', pattern: 'rm_recursive' },
|
||||
{ skill: 'careful', ts: '2026-03-18T16:30:00Z', repo: 'app', event: 'hook_fire', pattern: 'rm_recursive' },
|
||||
{ skill: 'careful', ts: '2026-03-18T17:00:00Z', repo: 'app', event: 'hook_fire', pattern: 'git_force_push' },
|
||||
];
|
||||
const report = formatReport(events);
|
||||
expect(report).toContain('Safety Hook Events');
|
||||
expect(report).toContain('rm_recursive');
|
||||
expect(report).toContain('2 fires');
|
||||
expect(report).toContain('git_force_push');
|
||||
expect(report).toContain('1 fire');
|
||||
expect(report).toContain('Total: 1 skill invocation, 3 hook fires');
|
||||
});
|
||||
|
||||
test('handles mixed events correctly', () => {
|
||||
const events: AnalyticsEvent[] = [
|
||||
{ skill: 'ship', ts: '2026-03-18T15:30:00Z', repo: 'my-app' },
|
||||
{ skill: 'ship', ts: '2026-03-18T15:35:00Z', repo: 'my-app' },
|
||||
{ skill: 'qa', ts: '2026-03-18T16:00:00Z', repo: 'my-api' },
|
||||
{ skill: 'careful', ts: '2026-03-18T16:30:00Z', repo: 'my-app', event: 'hook_fire', pattern: 'rm_recursive' },
|
||||
];
|
||||
const report = formatReport(events);
|
||||
// Skills counted correctly (hook_fire events excluded from skill counts)
|
||||
expect(report).toContain('Total: 3 skill invocations, 1 hook fire');
|
||||
// Both sections present
|
||||
expect(report).toContain('Top Skills');
|
||||
expect(report).toContain('Safety Hook Events');
|
||||
expect(report).toContain('By Repo');
|
||||
});
|
||||
});
|
||||
|
||||
describe('integration via runScript helper', () => {
|
||||
test('missing file → "No analytics data found."', () => {
|
||||
const output = runScript(path.join(TMP_DIR, 'nonexistent.jsonl'));
|
||||
expect(output).toBe('No analytics data found.');
|
||||
});
|
||||
|
||||
test('null path → "No analytics data found."', () => {
|
||||
const output = runScript(null);
|
||||
expect(output).toBe('No analytics data found.');
|
||||
});
|
||||
|
||||
test('empty file → "No analytics data found."', () => {
|
||||
const p = writeTempJSONL('empty.jsonl', ['']);
|
||||
// Overwrite with truly empty content
|
||||
fs.writeFileSync(p, '');
|
||||
const output = runScript(p);
|
||||
expect(output).toBe('No analytics data found.');
|
||||
});
|
||||
|
||||
test('all malformed lines → "No analytics data found."', () => {
|
||||
const p = writeTempJSONL('bad.jsonl', [
|
||||
'not json',
|
||||
'{broken',
|
||||
'42',
|
||||
]);
|
||||
const output = runScript(p);
|
||||
expect(output).toBe('No analytics data found.');
|
||||
});
|
||||
|
||||
test('normal aggregation produces correct output', () => {
|
||||
const p = writeTempJSONL('normal.jsonl', [
|
||||
'{"skill":"ship","ts":"2026-03-18T15:30:00Z","repo":"my-app"}',
|
||||
'{"skill":"ship","ts":"2026-03-18T15:35:00Z","repo":"my-app"}',
|
||||
'{"skill":"qa","ts":"2026-03-18T16:00:00Z","repo":"my-app"}',
|
||||
'{"skill":"review","ts":"2026-03-18T16:30:00Z","repo":"my-api"}',
|
||||
]);
|
||||
const output = runScript(p);
|
||||
expect(output).toContain('/ship');
|
||||
expect(output).toContain('2 invocations');
|
||||
expect(output).toContain('/qa');
|
||||
expect(output).toContain('1 invocation');
|
||||
expect(output).toContain('/review');
|
||||
expect(output).toContain('Total: 4 skill invocations, 0 hook fires');
|
||||
});
|
||||
|
||||
test('period filtering (7d) only includes recent entries', () => {
|
||||
const now = new Date();
|
||||
const recent = new Date(now.getTime() - 2 * 24 * 60 * 60 * 1000).toISOString();
|
||||
const old = new Date(now.getTime() - 20 * 24 * 60 * 60 * 1000).toISOString();
|
||||
|
||||
const p = writeTempJSONL('period.jsonl', [
|
||||
`{"skill":"ship","ts":"${recent}","repo":"app"}`,
|
||||
`{"skill":"qa","ts":"${old}","repo":"app"}`,
|
||||
]);
|
||||
const output = runScript(p, '--period 7d');
|
||||
expect(output).toContain('Period: last 7 days');
|
||||
expect(output).toContain('/ship');
|
||||
expect(output).toContain('Total: 1 skill invocation, 0 hook fires');
|
||||
// qa should be filtered out
|
||||
expect(output).not.toContain('/qa');
|
||||
});
|
||||
|
||||
test('hook fire events counted in full pipeline', () => {
|
||||
const p = writeTempJSONL('hooks.jsonl', [
|
||||
'{"skill":"ship","ts":"2026-03-18T15:30:00Z","repo":"app"}',
|
||||
'{"event":"hook_fire","skill":"careful","pattern":"rm_recursive","ts":"2026-03-18T16:00:00Z","repo":"app"}',
|
||||
'{"event":"hook_fire","skill":"careful","pattern":"rm_recursive","ts":"2026-03-18T16:30:00Z","repo":"app"}',
|
||||
'{"event":"hook_fire","skill":"careful","pattern":"git_force_push","ts":"2026-03-18T17:00:00Z","repo":"app"}',
|
||||
]);
|
||||
const output = runScript(p);
|
||||
expect(output).toContain('Safety Hook Events');
|
||||
expect(output).toContain('rm_recursive');
|
||||
expect(output).toContain('2 fires');
|
||||
expect(output).toContain('git_force_push');
|
||||
expect(output).toContain('1 fire');
|
||||
expect(output).toContain('Total: 1 skill invocation, 3 hook fires');
|
||||
});
|
||||
});
|
||||
@@ -72,6 +72,11 @@ describe('gen-skill-docs', () => {
|
||||
{ dir: 'plan-design-review', name: 'plan-design-review' },
|
||||
{ dir: 'design-review', name: 'design-review' },
|
||||
{ dir: 'design-consultation', name: 'design-consultation' },
|
||||
{ dir: 'document-release', name: 'document-release' },
|
||||
{ dir: 'careful', name: 'careful' },
|
||||
{ dir: 'freeze', name: 'freeze' },
|
||||
{ dir: 'guard', name: 'guard' },
|
||||
{ dir: 'unfreeze', name: 'unfreeze' },
|
||||
];
|
||||
|
||||
test('every skill has a SKILL.md.tmpl template', () => {
|
||||
@@ -161,6 +166,26 @@ describe('gen-skill-docs', () => {
|
||||
expect(content).toContain('plain English');
|
||||
});
|
||||
|
||||
test('generated SKILL.md contains telemetry line', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('skill-usage.jsonl');
|
||||
expect(content).toContain('~/.gstack/analytics');
|
||||
});
|
||||
|
||||
test('preamble-using skills have correct skill name in telemetry', () => {
|
||||
const PREAMBLE_SKILLS = [
|
||||
{ dir: '.', name: 'gstack' },
|
||||
{ dir: 'ship', name: 'ship' },
|
||||
{ dir: 'review', name: 'review' },
|
||||
{ dir: 'qa', name: 'qa' },
|
||||
{ dir: 'retro', name: 'retro' },
|
||||
];
|
||||
for (const skill of PREAMBLE_SKILLS) {
|
||||
const content = fs.readFileSync(path.join(ROOT, skill.dir, 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain(`"skill":"${skill.name}"`);
|
||||
}
|
||||
});
|
||||
|
||||
test('qa and qa-only templates use QA_METHODOLOGY placeholder', () => {
|
||||
const qaTmpl = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md.tmpl'), 'utf-8');
|
||||
expect(qaTmpl).toContain('{{QA_METHODOLOGY}}');
|
||||
@@ -329,7 +354,7 @@ describe('REVIEW_DASHBOARD resolver', () => {
|
||||
for (const skill of REVIEW_SKILLS) {
|
||||
test(`review dashboard appears in ${skill} generated file`, () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, skill, 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('reviews.jsonl');
|
||||
expect(content).toContain('gstack-review');
|
||||
expect(content).toContain('REVIEW READINESS DASHBOARD');
|
||||
});
|
||||
}
|
||||
@@ -349,6 +374,53 @@ describe('REVIEW_DASHBOARD resolver', () => {
|
||||
expect(content).toContain('Design Review');
|
||||
expect(content).toContain('skip_eng_review');
|
||||
});
|
||||
|
||||
test('dashboard bash block includes git HEAD for staleness detection', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'plan-ceo-review', 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('git rev-parse --short HEAD');
|
||||
expect(content).toContain('---HEAD---');
|
||||
});
|
||||
|
||||
test('dashboard includes staleness detection prose', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'plan-ceo-review', 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('Staleness detection');
|
||||
expect(content).toContain('commit');
|
||||
});
|
||||
|
||||
for (const skill of REVIEW_SKILLS) {
|
||||
test(`${skill} contains review chaining section`, () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, skill, 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('Review Chaining');
|
||||
});
|
||||
|
||||
test(`${skill} Review Log includes commit field`, () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, skill, 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('"commit"');
|
||||
});
|
||||
}
|
||||
|
||||
test('plan-ceo-review chaining mentions eng and design reviews', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'plan-ceo-review', 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('/plan-eng-review');
|
||||
expect(content).toContain('/plan-design-review');
|
||||
});
|
||||
|
||||
test('plan-eng-review chaining mentions design and ceo reviews', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'plan-eng-review', 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('/plan-design-review');
|
||||
expect(content).toContain('/plan-ceo-review');
|
||||
});
|
||||
|
||||
test('plan-design-review chaining mentions eng and ceo reviews', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'plan-design-review', 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('/plan-eng-review');
|
||||
expect(content).toContain('/plan-ceo-review');
|
||||
});
|
||||
|
||||
test('ship does NOT contain review chaining', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'ship', 'SKILL.md'), 'utf-8');
|
||||
expect(content).not.toContain('Review Chaining');
|
||||
});
|
||||
});
|
||||
|
||||
describe('telemetry', () => {
|
||||
|
||||
@@ -73,6 +73,9 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
// Document-release
|
||||
'document-release': ['document-release/**'],
|
||||
|
||||
// Codex
|
||||
'codex-review': ['codex/**'],
|
||||
|
||||
// QA bootstrap
|
||||
'qa-bootstrap': ['qa/**', 'browse/src/**', 'ship/**'],
|
||||
|
||||
@@ -90,6 +93,19 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
|
||||
// gstack-upgrade
|
||||
'gstack-upgrade-happy-path': ['gstack-upgrade/**'],
|
||||
|
||||
// Skill routing — journey-stage tests (depend on ALL skill descriptions)
|
||||
'journey-ideation': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-plan-eng': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-think-bigger': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-debug': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-qa': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-code-review': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-ship': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-docs': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-retro': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-design-system': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-visual-qa': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -103,6 +119,7 @@ export const LLM_JUDGE_TOUCHFILES: Record<string, string[]> = {
|
||||
'regression vs baseline': ['SKILL.md', 'SKILL.md.tmpl', 'browse/src/commands.ts', 'test/fixtures/eval-baselines.json'],
|
||||
'qa/SKILL.md workflow': ['qa/SKILL.md', 'qa/SKILL.md.tmpl'],
|
||||
'qa/SKILL.md health rubric': ['qa/SKILL.md', 'qa/SKILL.md.tmpl'],
|
||||
'qa/SKILL.md anti-refusal': ['qa/SKILL.md', 'qa/SKILL.md.tmpl', 'qa-only/SKILL.md', 'qa-only/SKILL.md.tmpl'],
|
||||
'cross-skill greptile consistency': ['review/SKILL.md', 'review/SKILL.md.tmpl', 'ship/SKILL.md', 'ship/SKILL.md.tmpl', 'review/greptile-triage.md', 'retro/SKILL.md', 'retro/SKILL.md.tmpl'],
|
||||
'baseline score pinning': ['SKILL.md', 'SKILL.md.tmpl', 'test/fixtures/eval-baselines.json'],
|
||||
|
||||
|
||||
@@ -0,0 +1,373 @@
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as path from 'path';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const CAREFUL_SCRIPT = path.join(ROOT, 'careful', 'bin', 'check-careful.sh');
|
||||
const FREEZE_SCRIPT = path.join(ROOT, 'freeze', 'bin', 'check-freeze.sh');
|
||||
|
||||
function runHook(scriptPath: string, input: object, env?: Record<string, string>): { exitCode: number; output: any; raw: string } {
|
||||
const result = spawnSync('bash', [scriptPath], {
|
||||
input: JSON.stringify(input),
|
||||
stdio: ['pipe', 'pipe', 'pipe'],
|
||||
env: { ...process.env, ...env },
|
||||
timeout: 5000,
|
||||
});
|
||||
const raw = result.stdout.toString().trim();
|
||||
let output: any = {};
|
||||
try {
|
||||
output = JSON.parse(raw);
|
||||
} catch {}
|
||||
return { exitCode: result.status ?? 1, output, raw };
|
||||
}
|
||||
|
||||
function runHookRaw(scriptPath: string, rawInput: string, env?: Record<string, string>): { exitCode: number; output: any; raw: string } {
|
||||
const result = spawnSync('bash', [scriptPath], {
|
||||
input: rawInput,
|
||||
stdio: ['pipe', 'pipe', 'pipe'],
|
||||
env: { ...process.env, ...env },
|
||||
timeout: 5000,
|
||||
});
|
||||
const raw = result.stdout.toString().trim();
|
||||
let output: any = {};
|
||||
try {
|
||||
output = JSON.parse(raw);
|
||||
} catch {}
|
||||
return { exitCode: result.status ?? 1, output, raw };
|
||||
}
|
||||
|
||||
function carefulInput(command: string) {
|
||||
return { tool_input: { command } };
|
||||
}
|
||||
|
||||
function freezeInput(filePath: string) {
|
||||
return { tool_input: { file_path: filePath } };
|
||||
}
|
||||
|
||||
function withFreezeDir(freezePath: string, fn: (stateDir: string) => void) {
|
||||
const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-freeze-test-'));
|
||||
fs.writeFileSync(path.join(stateDir, 'freeze-dir.txt'), freezePath);
|
||||
try {
|
||||
fn(stateDir);
|
||||
} finally {
|
||||
fs.rmSync(stateDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
// Detect whether the safe-rm-targets regex works on this platform.
|
||||
// macOS sed -E does not support \s, so the safe exception check fails there.
|
||||
function detectSafeRmWorks(): boolean {
|
||||
const { output } = runHook(CAREFUL_SCRIPT, carefulInput('rm -rf node_modules'));
|
||||
return output.permissionDecision === undefined;
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// check-careful.sh tests
|
||||
// ============================================================
|
||||
describe('check-careful.sh', () => {
|
||||
|
||||
// --- Destructive rm commands ---
|
||||
|
||||
describe('rm -rf / rm -r', () => {
|
||||
test('rm -rf /var/data warns with recursive delete message', () => {
|
||||
const { exitCode, output } = runHook(CAREFUL_SCRIPT, carefulInput('rm -rf /var/data'));
|
||||
expect(exitCode).toBe(0);
|
||||
expect(output.permissionDecision).toBe('ask');
|
||||
expect(output.message).toContain('recursive delete');
|
||||
});
|
||||
|
||||
test('rm -r ./some-dir warns', () => {
|
||||
const { exitCode, output } = runHook(CAREFUL_SCRIPT, carefulInput('rm -r ./some-dir'));
|
||||
expect(exitCode).toBe(0);
|
||||
expect(output.permissionDecision).toBe('ask');
|
||||
expect(output.message).toContain('recursive delete');
|
||||
});
|
||||
|
||||
test('rm -rf node_modules allows (safe exception)', () => {
|
||||
const { exitCode, output } = runHook(CAREFUL_SCRIPT, carefulInput('rm -rf node_modules'));
|
||||
expect(exitCode).toBe(0);
|
||||
if (detectSafeRmWorks()) {
|
||||
// GNU sed: safe exception triggers, allows through
|
||||
expect(output.permissionDecision).toBeUndefined();
|
||||
} else {
|
||||
// macOS sed: safe exception regex uses \\s which is unsupported,
|
||||
// so the safe-targets check fails and the command warns
|
||||
expect(output.permissionDecision).toBe('ask');
|
||||
}
|
||||
});
|
||||
|
||||
test('rm -rf .next dist allows (multiple safe targets)', () => {
|
||||
const { exitCode, output } = runHook(CAREFUL_SCRIPT, carefulInput('rm -rf .next dist'));
|
||||
expect(exitCode).toBe(0);
|
||||
if (detectSafeRmWorks()) {
|
||||
expect(output.permissionDecision).toBeUndefined();
|
||||
} else {
|
||||
expect(output.permissionDecision).toBe('ask');
|
||||
}
|
||||
});
|
||||
|
||||
test('rm -rf node_modules /var/data warns (mixed safe+unsafe)', () => {
|
||||
const { exitCode, output } = runHook(CAREFUL_SCRIPT, carefulInput('rm -rf node_modules /var/data'));
|
||||
expect(exitCode).toBe(0);
|
||||
expect(output.permissionDecision).toBe('ask');
|
||||
expect(output.message).toContain('recursive delete');
|
||||
});
|
||||
});
|
||||
|
||||
// --- SQL destructive commands ---
|
||||
// Note: SQL commands that contain embedded double quotes (e.g., psql -c "DROP TABLE")
|
||||
// get their command value truncated by the grep-based JSON extractor because \"
|
||||
// terminates the [^"]* match. We use commands WITHOUT embedded quotes so the grep
|
||||
// extraction works and the SQL keywords are visible to the pattern matcher.
|
||||
|
||||
describe('SQL destructive commands', () => {
|
||||
test('psql DROP TABLE warns with DROP in message', () => {
|
||||
const { exitCode, output } = runHook(CAREFUL_SCRIPT, carefulInput('psql -c DROP TABLE users;'));
|
||||
expect(exitCode).toBe(0);
|
||||
expect(output.permissionDecision).toBe('ask');
|
||||
expect(output.message).toContain('DROP');
|
||||
});
|
||||
|
||||
test('mysql drop database warns (case insensitive)', () => {
|
||||
const { exitCode, output } = runHook(CAREFUL_SCRIPT, carefulInput('mysql -e drop database mydb'));
|
||||
expect(exitCode).toBe(0);
|
||||
expect(output.permissionDecision).toBe('ask');
|
||||
expect(output.message.toLowerCase()).toContain('drop');
|
||||
});
|
||||
|
||||
test('psql TRUNCATE warns', () => {
|
||||
const { exitCode, output } = runHook(CAREFUL_SCRIPT, carefulInput('psql -c TRUNCATE orders;'));
|
||||
expect(exitCode).toBe(0);
|
||||
expect(output.permissionDecision).toBe('ask');
|
||||
expect(output.message).toContain('TRUNCATE');
|
||||
});
|
||||
});
|
||||
|
||||
// --- Git destructive commands ---
|
||||
|
||||
describe('git destructive commands', () => {
|
||||
test('git push --force warns with force-push', () => {
|
||||
const { exitCode, output } = runHook(CAREFUL_SCRIPT, carefulInput('git push --force origin main'));
|
||||
expect(exitCode).toBe(0);
|
||||
expect(output.permissionDecision).toBe('ask');
|
||||
expect(output.message).toContain('force-push');
|
||||
});
|
||||
|
||||
test('git push -f warns', () => {
|
||||
const { exitCode, output } = runHook(CAREFUL_SCRIPT, carefulInput('git push -f origin main'));
|
||||
expect(exitCode).toBe(0);
|
||||
expect(output.permissionDecision).toBe('ask');
|
||||
expect(output.message).toContain('force-push');
|
||||
});
|
||||
|
||||
test('git reset --hard warns with uncommitted', () => {
|
||||
const { exitCode, output } = runHook(CAREFUL_SCRIPT, carefulInput('git reset --hard HEAD~3'));
|
||||
expect(exitCode).toBe(0);
|
||||
expect(output.permissionDecision).toBe('ask');
|
||||
expect(output.message).toContain('uncommitted');
|
||||
});
|
||||
|
||||
test('git checkout . warns', () => {
|
||||
const { exitCode, output } = runHook(CAREFUL_SCRIPT, carefulInput('git checkout .'));
|
||||
expect(exitCode).toBe(0);
|
||||
expect(output.permissionDecision).toBe('ask');
|
||||
expect(output.message).toContain('uncommitted');
|
||||
});
|
||||
|
||||
test('git restore . warns', () => {
|
||||
const { exitCode, output } = runHook(CAREFUL_SCRIPT, carefulInput('git restore .'));
|
||||
expect(exitCode).toBe(0);
|
||||
expect(output.permissionDecision).toBe('ask');
|
||||
expect(output.message).toContain('uncommitted');
|
||||
});
|
||||
});
|
||||
|
||||
// --- Container / infra destructive commands ---
|
||||
|
||||
describe('container and infra commands', () => {
|
||||
test('kubectl delete warns with kubectl in message', () => {
|
||||
const { exitCode, output } = runHook(CAREFUL_SCRIPT, carefulInput('kubectl delete pod my-pod'));
|
||||
expect(exitCode).toBe(0);
|
||||
expect(output.permissionDecision).toBe('ask');
|
||||
expect(output.message).toContain('kubectl');
|
||||
});
|
||||
|
||||
test('docker rm -f warns', () => {
|
||||
const { exitCode, output } = runHook(CAREFUL_SCRIPT, carefulInput('docker rm -f container123'));
|
||||
expect(exitCode).toBe(0);
|
||||
expect(output.permissionDecision).toBe('ask');
|
||||
expect(output.message).toContain('Docker');
|
||||
});
|
||||
|
||||
test('docker system prune -a warns', () => {
|
||||
const { exitCode, output } = runHook(CAREFUL_SCRIPT, carefulInput('docker system prune -a'));
|
||||
expect(exitCode).toBe(0);
|
||||
expect(output.permissionDecision).toBe('ask');
|
||||
expect(output.message).toContain('Docker');
|
||||
});
|
||||
});
|
||||
|
||||
// --- Safe commands ---
|
||||
|
||||
describe('safe commands allow without warning', () => {
|
||||
const safeCmds = [
|
||||
'ls -la',
|
||||
'git status',
|
||||
'npm install',
|
||||
'cat README.md',
|
||||
'echo hello',
|
||||
];
|
||||
|
||||
for (const cmd of safeCmds) {
|
||||
test(`"${cmd}" allows`, () => {
|
||||
const { exitCode, output } = runHook(CAREFUL_SCRIPT, carefulInput(cmd));
|
||||
expect(exitCode).toBe(0);
|
||||
expect(output.permissionDecision).toBeUndefined();
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// --- Edge cases ---
|
||||
|
||||
describe('edge cases', () => {
|
||||
test('empty command allows gracefully', () => {
|
||||
const { exitCode, output } = runHook(CAREFUL_SCRIPT, carefulInput(''));
|
||||
expect(exitCode).toBe(0);
|
||||
expect(output.permissionDecision).toBeUndefined();
|
||||
});
|
||||
|
||||
test('missing command field allows gracefully', () => {
|
||||
const { exitCode, output } = runHook(CAREFUL_SCRIPT, { tool_input: {} });
|
||||
expect(exitCode).toBe(0);
|
||||
expect(output.permissionDecision).toBeUndefined();
|
||||
});
|
||||
|
||||
test('malformed JSON input allows gracefully (exit 0, output {})', () => {
|
||||
const { exitCode, raw } = runHookRaw(CAREFUL_SCRIPT, 'this is not json at all{{{{');
|
||||
expect(exitCode).toBe(0);
|
||||
expect(raw).toBe('{}');
|
||||
});
|
||||
|
||||
test('Python fallback: grep fails on multiline JSON, Python parses it', () => {
|
||||
// Construct JSON where "command": and the value are on separate lines.
|
||||
// grep works line-by-line, so it cannot match "command"..."value" across lines.
|
||||
// This forces CMD to be empty, triggering the Python fallback which handles
|
||||
// the full JSON correctly.
|
||||
const rawJson = '{"tool_input":{"command":\n"rm -rf /tmp/important"}}';
|
||||
const { exitCode, output } = runHookRaw(CAREFUL_SCRIPT, rawJson);
|
||||
expect(exitCode).toBe(0);
|
||||
expect(output.permissionDecision).toBe('ask');
|
||||
expect(output.message).toContain('recursive delete');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// ============================================================
|
||||
// check-freeze.sh tests
|
||||
// ============================================================
|
||||
describe('check-freeze.sh', () => {
|
||||
|
||||
describe('edits inside freeze boundary', () => {
|
||||
test('edit inside freeze boundary allows', () => {
|
||||
withFreezeDir('/Users/dev/project/src/', (stateDir) => {
|
||||
const { exitCode, output } = runHook(
|
||||
FREEZE_SCRIPT,
|
||||
freezeInput('/Users/dev/project/src/index.ts'),
|
||||
{ CLAUDE_PLUGIN_DATA: stateDir },
|
||||
);
|
||||
expect(exitCode).toBe(0);
|
||||
expect(output.permissionDecision).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
test('edit in subdirectory of freeze path allows', () => {
|
||||
withFreezeDir('/Users/dev/project/src/', (stateDir) => {
|
||||
const { exitCode, output } = runHook(
|
||||
FREEZE_SCRIPT,
|
||||
freezeInput('/Users/dev/project/src/components/Button.tsx'),
|
||||
{ CLAUDE_PLUGIN_DATA: stateDir },
|
||||
);
|
||||
expect(exitCode).toBe(0);
|
||||
expect(output.permissionDecision).toBeUndefined();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('edits outside freeze boundary', () => {
|
||||
test('edit outside freeze boundary denies', () => {
|
||||
withFreezeDir('/Users/dev/project/src/', (stateDir) => {
|
||||
const { exitCode, output } = runHook(
|
||||
FREEZE_SCRIPT,
|
||||
freezeInput('/Users/dev/other-project/index.ts'),
|
||||
{ CLAUDE_PLUGIN_DATA: stateDir },
|
||||
);
|
||||
expect(exitCode).toBe(0);
|
||||
expect(output.permissionDecision).toBe('deny');
|
||||
expect(output.message).toContain('freeze');
|
||||
expect(output.message).toContain('outside');
|
||||
});
|
||||
});
|
||||
|
||||
test('write outside freeze boundary denies', () => {
|
||||
withFreezeDir('/Users/dev/project/src/', (stateDir) => {
|
||||
const { exitCode, output } = runHook(
|
||||
FREEZE_SCRIPT,
|
||||
freezeInput('/etc/hosts'),
|
||||
{ CLAUDE_PLUGIN_DATA: stateDir },
|
||||
);
|
||||
expect(exitCode).toBe(0);
|
||||
expect(output.permissionDecision).toBe('deny');
|
||||
expect(output.message).toContain('freeze');
|
||||
expect(output.message).toContain('outside');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('trailing slash prevents prefix confusion', () => {
|
||||
test('freeze at /src/ denies /src-old/ (trailing slash prevents prefix match)', () => {
|
||||
withFreezeDir('/Users/dev/project/src/', (stateDir) => {
|
||||
const { exitCode, output } = runHook(
|
||||
FREEZE_SCRIPT,
|
||||
freezeInput('/Users/dev/project/src-old/index.ts'),
|
||||
{ CLAUDE_PLUGIN_DATA: stateDir },
|
||||
);
|
||||
expect(exitCode).toBe(0);
|
||||
expect(output.permissionDecision).toBe('deny');
|
||||
expect(output.message).toContain('outside');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('no freeze file exists', () => {
|
||||
test('allows everything when no freeze file present', () => {
|
||||
const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-freeze-test-'));
|
||||
try {
|
||||
const { exitCode, output } = runHook(
|
||||
FREEZE_SCRIPT,
|
||||
freezeInput('/anywhere/at/all.ts'),
|
||||
{ CLAUDE_PLUGIN_DATA: stateDir },
|
||||
);
|
||||
expect(exitCode).toBe(0);
|
||||
expect(output.permissionDecision).toBeUndefined();
|
||||
} finally {
|
||||
fs.rmSync(stateDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('edge cases', () => {
|
||||
test('missing file_path field allows gracefully', () => {
|
||||
withFreezeDir('/Users/dev/project/src/', (stateDir) => {
|
||||
const { exitCode, output } = runHook(
|
||||
FREEZE_SCRIPT,
|
||||
{ tool_input: {} },
|
||||
{ CLAUDE_PLUGIN_DATA: stateDir },
|
||||
);
|
||||
expect(exitCode).toBe(0);
|
||||
expect(output.permissionDecision).toBeUndefined();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
+86
-16
@@ -387,7 +387,7 @@ File a contributor report about this issue. Then tell me what you filed.`,
|
||||
// Set up a git repo so there's project/branch context to reference
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: sessionDir, stdio: 'pipe', timeout: 5000 });
|
||||
run('git', ['init']);
|
||||
run('git', ['init', '-b', 'main']);
|
||||
run('git', ['config', 'user.email', 'test@test.com']);
|
||||
run('git', ['config', 'user.name', 'Test']);
|
||||
fs.writeFileSync(path.join(sessionDir, 'app.rb'), '# my app\n');
|
||||
@@ -518,7 +518,7 @@ describeIfSelected('Review skill E2E', ['review-sql-injection'], () => {
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: reviewDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
run('git', ['init']);
|
||||
run('git', ['init', '-b', 'main']);
|
||||
run('git', ['config', 'user.email', 'test@test.com']);
|
||||
run('git', ['config', 'user.name', 'Test']);
|
||||
|
||||
@@ -575,7 +575,7 @@ describeIfSelected('Review enum completeness E2E', ['review-enum-completeness'],
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: enumDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
run('git', ['init']);
|
||||
run('git', ['init', '-b', 'main']);
|
||||
run('git', ['config', 'user.email', 'test@test.com']);
|
||||
run('git', ['config', 'user.name', 'Test']);
|
||||
|
||||
@@ -647,7 +647,7 @@ describeE2E('Review design lite E2E', () => {
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: designDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
run('git', ['init']);
|
||||
run('git', ['init', '-b', 'main']);
|
||||
run('git', ['config', 'user.email', 'test@test.com']);
|
||||
run('git', ['config', 'user.name', 'Test']);
|
||||
|
||||
@@ -910,7 +910,7 @@ describeIfSelected('Plan CEO Review E2E', ['plan-ceo-review'], () => {
|
||||
spawnSync(cmd, args, { cwd: planDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
// Init git repo (CEO review SKILL.md has a "System Audit" step that runs git)
|
||||
run('git', ['init']);
|
||||
run('git', ['init', '-b', 'main']);
|
||||
run('git', ['config', 'user.email', 'test@test.com']);
|
||||
run('git', ['config', 'user.name', 'Test']);
|
||||
|
||||
@@ -996,7 +996,7 @@ describeIfSelected('Plan CEO Review SELECTIVE EXPANSION E2E', ['plan-ceo-review-
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: planDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
run('git', ['init']);
|
||||
run('git', ['init', '-b', 'main']);
|
||||
run('git', ['config', 'user.email', 'test@test.com']);
|
||||
run('git', ['config', 'user.name', 'Test']);
|
||||
|
||||
@@ -1079,7 +1079,7 @@ describeIfSelected('Plan Eng Review E2E', ['plan-eng-review'], () => {
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: planDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
run('git', ['init']);
|
||||
run('git', ['init', '-b', 'main']);
|
||||
run('git', ['config', 'user.email', 'test@test.com']);
|
||||
run('git', ['config', 'user.name', 'Test']);
|
||||
|
||||
@@ -1174,7 +1174,7 @@ describeIfSelected('Retro E2E', ['retro'], () => {
|
||||
spawnSync(cmd, args, { cwd: retroDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
// Create a git repo with varied commit history
|
||||
run('git', ['init']);
|
||||
run('git', ['init', '-b', 'main']);
|
||||
run('git', ['config', 'user.email', 'dev@example.com']);
|
||||
run('git', ['config', 'user.name', 'Dev']);
|
||||
|
||||
@@ -1273,7 +1273,7 @@ describeIfSelected('QA-Only skill E2E', ['qa-only-no-fix'], () => {
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: qaOnlyDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
run('git', ['init']);
|
||||
run('git', ['init', '-b', 'main']);
|
||||
run('git', ['config', 'user.email', 'test@test.com']);
|
||||
run('git', ['config', 'user.name', 'Test']);
|
||||
fs.writeFileSync(path.join(qaOnlyDir, 'index.html'), '<h1>Test</h1>\n');
|
||||
@@ -1373,7 +1373,7 @@ describeIfSelected('QA Fix Loop E2E', ['qa-fix-loop'], () => {
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: qaFixDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
run('git', ['init']);
|
||||
run('git', ['init', '-b', 'main']);
|
||||
run('git', ['config', 'user.email', 'test@test.com']);
|
||||
run('git', ['config', 'user.name', 'Test']);
|
||||
run('git', ['add', '.']);
|
||||
@@ -1460,7 +1460,7 @@ describeIfSelected('Plan-Eng-Review Test-Plan Artifact E2E', ['plan-eng-review-a
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: planDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
run('git', ['init']);
|
||||
run('git', ['init', '-b', 'main']);
|
||||
run('git', ['config', 'user.email', 'test@test.com']);
|
||||
run('git', ['config', 'user.name', 'Test']);
|
||||
|
||||
@@ -1777,7 +1777,7 @@ describeIfSelected('Document-Release skill E2E', ['document-release'], () => {
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: docReleaseDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
run('git', ['init']);
|
||||
run('git', ['init', '-b', 'main']);
|
||||
run('git', ['config', 'user.email', 'test@test.com']);
|
||||
run('git', ['config', 'user.name', 'Test']);
|
||||
|
||||
@@ -2030,7 +2030,7 @@ describeIfSelected('Design Consultation E2E', [
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: designDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
run('git', ['init']);
|
||||
run('git', ['init', '-b', 'main']);
|
||||
run('git', ['config', 'user.email', 'test@test.com']);
|
||||
run('git', ['config', 'user.name', 'Test']);
|
||||
|
||||
@@ -2302,7 +2302,7 @@ describeIfSelected('Plan Design Review E2E', ['plan-design-review-plan-mode', 'p
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: reviewDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
run('git', ['init']);
|
||||
run('git', ['init', '-b', 'main']);
|
||||
run('git', ['config', 'user.email', 'test@test.com']);
|
||||
run('git', ['config', 'user.name', 'Test']);
|
||||
|
||||
@@ -2453,7 +2453,7 @@ describeIfSelected('Design Review E2E', ['design-review-fix'], () => {
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: qaDesignDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
run('git', ['init']);
|
||||
run('git', ['init', '-b', 'main']);
|
||||
run('git', ['config', 'user.email', 'test@test.com']);
|
||||
run('git', ['config', 'user.name', 'Test']);
|
||||
|
||||
@@ -2620,7 +2620,7 @@ export function divide(a, b) { return a / b; } // BUG: no zero check
|
||||
// Init git repo
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: bootstrapDir, stdio: 'pipe', timeout: 5000 });
|
||||
run('git', ['init']);
|
||||
run('git', ['init', '-b', 'main']);
|
||||
run('git', ['config', 'user.email', 'test@test.com']);
|
||||
run('git', ['config', 'user.name', 'Test']);
|
||||
run('git', ['add', '.']);
|
||||
@@ -2841,6 +2841,76 @@ Output the diagram directly.`,
|
||||
}, 180_000);
|
||||
});
|
||||
|
||||
// --- Codex skill E2E ---
|
||||
|
||||
describeIfSelected('Codex skill E2E', ['codex-review'], () => {
|
||||
let codexDir: string;
|
||||
|
||||
beforeAll(() => {
|
||||
codexDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-codex-'));
|
||||
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: codexDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
run('git', ['init', '-b', 'main']);
|
||||
run('git', ['config', 'user.email', 'test@test.com']);
|
||||
run('git', ['config', 'user.name', 'Test']);
|
||||
|
||||
// Commit a clean base on main
|
||||
fs.writeFileSync(path.join(codexDir, 'app.rb'), '# clean base\nclass App\nend\n');
|
||||
run('git', ['add', 'app.rb']);
|
||||
run('git', ['commit', '-m', 'initial commit']);
|
||||
|
||||
// Create feature branch with vulnerable code (reuse review fixture)
|
||||
run('git', ['checkout', '-b', 'feature/add-vuln']);
|
||||
const vulnContent = fs.readFileSync(path.join(ROOT, 'test', 'fixtures', 'review-eval-vuln.rb'), 'utf-8');
|
||||
fs.writeFileSync(path.join(codexDir, 'user_controller.rb'), vulnContent);
|
||||
run('git', ['add', 'user_controller.rb']);
|
||||
run('git', ['commit', '-m', 'add vulnerable controller']);
|
||||
|
||||
// Copy the codex skill file
|
||||
fs.copyFileSync(path.join(ROOT, 'codex', 'SKILL.md'), path.join(codexDir, 'codex-SKILL.md'));
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
try { fs.rmSync(codexDir, { recursive: true, force: true }); } catch {}
|
||||
});
|
||||
|
||||
test('/codex review produces findings and GATE verdict', async () => {
|
||||
// Check codex is available — skip if not installed
|
||||
const codexCheck = spawnSync('which', ['codex'], { stdio: 'pipe', timeout: 3000 });
|
||||
if (codexCheck.status !== 0) {
|
||||
console.warn('codex CLI not installed — skipping E2E test');
|
||||
return;
|
||||
}
|
||||
|
||||
const result = await runSkillTest({
|
||||
prompt: `You are in a git repo on branch feature/add-vuln with changes against main.
|
||||
Read codex-SKILL.md for the /codex skill instructions.
|
||||
Run /codex review to review the current diff against main.
|
||||
Write the full output (including the GATE verdict) to ${codexDir}/codex-output.md`,
|
||||
workingDirectory: codexDir,
|
||||
maxTurns: 10,
|
||||
timeout: 300_000,
|
||||
testName: 'codex-review',
|
||||
runId,
|
||||
});
|
||||
|
||||
logCost('/codex review', result);
|
||||
recordE2E('/codex review', 'Codex skill E2E', result);
|
||||
expect(result.exitReason).toBe('success');
|
||||
|
||||
// Check that output file was created with review content
|
||||
const outputPath = path.join(codexDir, 'codex-output.md');
|
||||
if (fs.existsSync(outputPath)) {
|
||||
const output = fs.readFileSync(outputPath, 'utf-8');
|
||||
// Should contain the CODEX SAYS header or GATE verdict
|
||||
const hasCodexOutput = output.includes('CODEX') || output.includes('GATE') || output.includes('codex');
|
||||
expect(hasCodexOutput).toBe(true);
|
||||
}
|
||||
}, 360_000);
|
||||
});
|
||||
|
||||
// Module-level afterAll — finalize eval collector after all tests complete
|
||||
afterAll(async () => {
|
||||
if (evalCollector) {
|
||||
|
||||
@@ -256,7 +256,7 @@ Scores are 1-5 overall quality.`,
|
||||
|
||||
// --- Part 7: QA skill quality evals (C6) ---
|
||||
|
||||
describeIfSelected('QA skill quality evals', ['qa/SKILL.md workflow', 'qa/SKILL.md health rubric'], () => {
|
||||
describeIfSelected('QA skill quality evals', ['qa/SKILL.md workflow', 'qa/SKILL.md health rubric', 'qa/SKILL.md anti-refusal'], () => {
|
||||
const qaContent = fs.readFileSync(path.join(ROOT, 'qa', 'SKILL.md'), 'utf-8');
|
||||
|
||||
testIfSelected('qa/SKILL.md workflow', async () => {
|
||||
@@ -342,6 +342,59 @@ ${section}`);
|
||||
expect(scores.completeness).toBeGreaterThanOrEqual(3);
|
||||
expect(scores.actionability).toBeGreaterThanOrEqual(4);
|
||||
}, 30_000);
|
||||
|
||||
testIfSelected('qa/SKILL.md anti-refusal', async () => {
|
||||
const t0 = Date.now();
|
||||
// Extract both the diff-aware mode section and Important Rules section
|
||||
const diffAwareStart = qaContent.indexOf('### Diff-aware');
|
||||
const diffAwareEnd = qaContent.indexOf('### Full');
|
||||
const rulesStart = qaContent.indexOf('## Important Rules');
|
||||
const rulesEnd = qaContent.indexOf('## Framework-Specific');
|
||||
const diffAwareSection = qaContent.slice(diffAwareStart, diffAwareEnd);
|
||||
const rulesSection = qaContent.slice(rulesStart, rulesEnd);
|
||||
|
||||
const result = await callJudge<{ would_browse: boolean; fallback_behavior: string; confidence: number; reasoning: string }>(`You are evaluating whether a QA testing skill document would cause an AI agent to USE THE BROWSER or REFUSE to use the browser in a specific scenario.
|
||||
|
||||
SCENARIO:
|
||||
A user runs /qa (a browser-based QA testing skill). The branch diff shows ONLY prompt template files and config file changes — no routes, views, controllers, components, or CSS were changed. The changes are "purely backend" with no obvious UI surface.
|
||||
|
||||
QUESTION: Based on the document excerpts below, would the agent open the browser and test the application, or would it decline/refuse to browse and suggest running evals or unit tests instead?
|
||||
|
||||
DOCUMENT EXCERPT 1 (Diff-aware mode instructions):
|
||||
${diffAwareSection}
|
||||
|
||||
DOCUMENT EXCERPT 2 (Important Rules):
|
||||
${rulesSection}
|
||||
|
||||
Respond with ONLY valid JSON:
|
||||
{
|
||||
"would_browse": true or false,
|
||||
"fallback_behavior": "description of what the agent would do when no UI pages are identified from the diff",
|
||||
"confidence": N (1-5, how confident you are in your answer),
|
||||
"reasoning": "brief explanation"
|
||||
}
|
||||
|
||||
Rules:
|
||||
- would_browse should be true if the document instructs the agent to always use the browser regardless of diff content
|
||||
- would_browse should be false if the document allows the agent to skip browser testing for non-UI changes
|
||||
- confidence: 5 = document is unambiguous, 1 = document is unclear or contradictory`);
|
||||
|
||||
console.log('QA anti-refusal result:', JSON.stringify(result, null, 2));
|
||||
|
||||
evalCollector?.addTest({
|
||||
name: 'qa/SKILL.md anti-refusal',
|
||||
suite: 'QA skill quality evals',
|
||||
tier: 'llm-judge',
|
||||
passed: result.would_browse === true && result.confidence >= 4,
|
||||
duration_ms: Date.now() - t0,
|
||||
cost_usd: 0.02,
|
||||
judge_scores: { would_browse: result.would_browse ? 1 : 0, confidence: result.confidence },
|
||||
judge_reasoning: result.reasoning,
|
||||
});
|
||||
|
||||
expect(result.would_browse).toBe(true);
|
||||
expect(result.confidence).toBeGreaterThanOrEqual(4);
|
||||
}, 30_000);
|
||||
});
|
||||
|
||||
// --- Part 7: Cross-skill consistency judge (C7) ---
|
||||
|
||||
@@ -0,0 +1,605 @@
|
||||
import { describe, test, expect, afterAll } from 'bun:test';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import type { SkillTestResult } from './helpers/session-runner';
|
||||
import { EvalCollector } from './helpers/eval-store';
|
||||
import type { EvalTestEntry } from './helpers/eval-store';
|
||||
import { selectTests, detectBaseBranch, getChangedFiles, E2E_TOUCHFILES, GLOBAL_TOUCHFILES } from './helpers/touchfiles';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
|
||||
// Skip unless EVALS=1.
|
||||
const evalsEnabled = !!process.env.EVALS;
|
||||
const describeE2E = evalsEnabled ? describe : describe.skip;
|
||||
|
||||
// Eval result collector
|
||||
const evalCollector = evalsEnabled ? new EvalCollector('e2e-routing') : null;
|
||||
|
||||
// Unique run ID for this session
|
||||
const runId = new Date().toISOString().replace(/[:.]/g, '').replace('T', '-').slice(0, 15);
|
||||
|
||||
// --- Diff-based test selection ---
|
||||
// Journey routing tests use E2E_TOUCHFILES (entries prefixed 'journey-' in touchfiles.ts).
|
||||
let selectedTests: string[] | null = null;
|
||||
|
||||
if (evalsEnabled && !process.env.EVALS_ALL) {
|
||||
const baseBranch = process.env.EVALS_BASE
|
||||
|| detectBaseBranch(ROOT)
|
||||
|| 'main';
|
||||
const changedFiles = getChangedFiles(baseBranch, ROOT);
|
||||
|
||||
if (changedFiles.length > 0) {
|
||||
const selection = selectTests(changedFiles, E2E_TOUCHFILES, GLOBAL_TOUCHFILES);
|
||||
selectedTests = selection.selected;
|
||||
process.stderr.write(`\nRouting E2E selection (${selection.reason}): ${selection.selected.length}/${Object.keys(E2E_TOUCHFILES).length} tests\n`);
|
||||
if (selection.skipped.length > 0) {
|
||||
process.stderr.write(` Skipped: ${selection.skipped.join(', ')}\n`);
|
||||
}
|
||||
process.stderr.write('\n');
|
||||
}
|
||||
}
|
||||
|
||||
// --- Helper functions ---
|
||||
|
||||
/** Copy all SKILL.md files into tmpDir/.claude/skills/gstack/ for auto-discovery */
|
||||
function installSkills(tmpDir: string) {
|
||||
const skillDirs = [
|
||||
'', // root gstack SKILL.md
|
||||
'qa', 'qa-only', 'ship', 'review', 'plan-ceo-review', 'plan-eng-review',
|
||||
'plan-design-review', 'design-review', 'design-consultation', 'retro',
|
||||
'document-release', 'investigate', 'office-hours', 'browse', 'setup-browser-cookies',
|
||||
'gstack-upgrade', 'humanizer',
|
||||
];
|
||||
|
||||
for (const skill of skillDirs) {
|
||||
const srcPath = path.join(ROOT, skill, 'SKILL.md');
|
||||
if (!fs.existsSync(srcPath)) continue;
|
||||
|
||||
const destDir = skill
|
||||
? path.join(tmpDir, '.claude', 'skills', 'gstack', skill)
|
||||
: path.join(tmpDir, '.claude', 'skills', 'gstack');
|
||||
fs.mkdirSync(destDir, { recursive: true });
|
||||
fs.copyFileSync(srcPath, path.join(destDir, 'SKILL.md'));
|
||||
}
|
||||
}
|
||||
|
||||
/** Init a git repo with config */
|
||||
function initGitRepo(dir: string) {
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: dir, stdio: 'pipe', timeout: 5000 });
|
||||
run('git', ['init']);
|
||||
run('git', ['config', 'user.email', 'test@test.com']);
|
||||
run('git', ['config', 'user.name', 'Test']);
|
||||
}
|
||||
|
||||
function logCost(label: string, result: { costEstimate: { turnsUsed: number; estimatedTokens: number; estimatedCost: number }; duration: number }) {
|
||||
const { turnsUsed, estimatedTokens, estimatedCost } = result.costEstimate;
|
||||
const durationSec = Math.round(result.duration / 1000);
|
||||
console.log(`${label}: $${estimatedCost.toFixed(2)} (${turnsUsed} turns, ${(estimatedTokens / 1000).toFixed(1)}k tokens, ${durationSec}s)`);
|
||||
}
|
||||
|
||||
function recordRouting(name: string, result: SkillTestResult, expectedSkill: string, actualSkill: string | undefined) {
|
||||
evalCollector?.addTest({
|
||||
name,
|
||||
suite: 'Skill Routing E2E',
|
||||
tier: 'e2e',
|
||||
passed: actualSkill === expectedSkill,
|
||||
duration_ms: result.duration,
|
||||
cost_usd: result.costEstimate.estimatedCost,
|
||||
transcript: result.transcript,
|
||||
output: result.output?.slice(0, 2000),
|
||||
turns_used: result.costEstimate.turnsUsed,
|
||||
exit_reason: result.exitReason,
|
||||
});
|
||||
}
|
||||
|
||||
// --- Tests ---
|
||||
|
||||
describeE2E('Skill Routing E2E — Developer Journey', () => {
|
||||
afterAll(() => {
|
||||
evalCollector?.finalize();
|
||||
});
|
||||
|
||||
test('journey-ideation', async () => {
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-ideation-'));
|
||||
try {
|
||||
initGitRepo(tmpDir);
|
||||
installSkills(tmpDir);
|
||||
fs.writeFileSync(path.join(tmpDir, 'README.md'), '# New Project\n');
|
||||
spawnSync('git', ['add', '.'], { cwd: tmpDir, stdio: 'pipe', timeout: 5000 });
|
||||
spawnSync('git', ['commit', '-m', 'initial'], { cwd: tmpDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
const testName = 'journey-ideation';
|
||||
const expectedSkill = 'office-hours';
|
||||
const result = await runSkillTest({
|
||||
prompt: "I've been thinking about building a waitlist management tool for restaurants. The existing solutions are expensive and overcomplicated. I want something simple — a tablet app where hosts can add parties, see wait times, and text customers when their table is ready. Help me think through whether this is worth building and what the key design decisions are.",
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
|
||||
timeout: 60_000,
|
||||
testName,
|
||||
runId,
|
||||
});
|
||||
|
||||
const skillCalls = result.toolCalls.filter(tc => tc.tool === 'Skill');
|
||||
const actualSkill = skillCalls.length > 0 ? skillCalls[0]?.input?.skill : undefined;
|
||||
|
||||
logCost(`journey: ${testName}`, result);
|
||||
recordRouting(testName, result, expectedSkill, actualSkill);
|
||||
|
||||
expect(skillCalls.length, `Expected Skill tool to be called but got 0 calls. Claude may have answered directly without invoking a skill. Tool calls: ${result.toolCalls.map(tc => tc.tool).join(', ')}`).toBeGreaterThan(0);
|
||||
expect([expectedSkill], `Expected skill ${expectedSkill} but got ${actualSkill}`).toContain(actualSkill);
|
||||
} finally {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 90_000);
|
||||
|
||||
test('journey-plan-eng', async () => {
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-plan-eng-'));
|
||||
try {
|
||||
initGitRepo(tmpDir);
|
||||
installSkills(tmpDir);
|
||||
fs.writeFileSync(path.join(tmpDir, 'plan.md'), `# Waitlist App Architecture
|
||||
|
||||
## Components
|
||||
- REST API (Express.js)
|
||||
- PostgreSQL database
|
||||
- React frontend
|
||||
- SMS integration (Twilio)
|
||||
|
||||
## Data Model
|
||||
- restaurants (id, name, settings)
|
||||
- parties (id, restaurant_id, name, size, phone, status, created_at)
|
||||
- wait_estimates (id, restaurant_id, avg_wait_minutes)
|
||||
|
||||
## API Endpoints
|
||||
- POST /api/parties - add party to waitlist
|
||||
- GET /api/parties - list current waitlist
|
||||
- PATCH /api/parties/:id/status - update party status
|
||||
- GET /api/estimate - get current wait estimate
|
||||
`);
|
||||
spawnSync('git', ['add', '.'], { cwd: tmpDir, stdio: 'pipe', timeout: 5000 });
|
||||
spawnSync('git', ['commit', '-m', 'initial'], { cwd: tmpDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
const testName = 'journey-plan-eng';
|
||||
const expectedSkill = 'plan-eng-review';
|
||||
const result = await runSkillTest({
|
||||
prompt: "I wrote up a plan for the waitlist app in plan.md. Can you take a look at the architecture and make sure I'm not missing any edge cases or failure modes before I start coding?",
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
|
||||
timeout: 60_000,
|
||||
testName,
|
||||
runId,
|
||||
});
|
||||
|
||||
const skillCalls = result.toolCalls.filter(tc => tc.tool === 'Skill');
|
||||
const actualSkill = skillCalls.length > 0 ? skillCalls[0]?.input?.skill : undefined;
|
||||
|
||||
logCost(`journey: ${testName}`, result);
|
||||
recordRouting(testName, result, expectedSkill, actualSkill);
|
||||
|
||||
expect(skillCalls.length, `Expected Skill tool to be called but got 0 calls. Claude may have answered directly without invoking a skill. Tool calls: ${result.toolCalls.map(tc => tc.tool).join(', ')}`).toBeGreaterThan(0);
|
||||
expect([expectedSkill], `Expected skill ${expectedSkill} but got ${actualSkill}`).toContain(actualSkill);
|
||||
} finally {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 90_000);
|
||||
|
||||
test('journey-think-bigger', async () => {
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-think-bigger-'));
|
||||
try {
|
||||
initGitRepo(tmpDir);
|
||||
installSkills(tmpDir);
|
||||
fs.writeFileSync(path.join(tmpDir, 'plan.md'), `# Waitlist App Architecture
|
||||
|
||||
## Components
|
||||
- REST API (Express.js)
|
||||
- PostgreSQL database
|
||||
- React frontend
|
||||
- SMS integration (Twilio)
|
||||
|
||||
## Data Model
|
||||
- restaurants (id, name, settings)
|
||||
- parties (id, restaurant_id, name, size, phone, status, created_at)
|
||||
- wait_estimates (id, restaurant_id, avg_wait_minutes)
|
||||
|
||||
## API Endpoints
|
||||
- POST /api/parties - add party to waitlist
|
||||
- GET /api/parties - list current waitlist
|
||||
- PATCH /api/parties/:id/status - update party status
|
||||
- GET /api/estimate - get current wait estimate
|
||||
`);
|
||||
spawnSync('git', ['add', '.'], { cwd: tmpDir, stdio: 'pipe', timeout: 5000 });
|
||||
spawnSync('git', ['commit', '-m', 'initial'], { cwd: tmpDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
const testName = 'journey-think-bigger';
|
||||
const expectedSkill = 'plan-ceo-review';
|
||||
const result = await runSkillTest({
|
||||
prompt: "Actually, looking at this plan again, I feel like we're thinking too small. We're just doing waitlists but what about the whole restaurant guest experience? Is there a bigger opportunity here we should go after?",
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
|
||||
timeout: 120_000,
|
||||
testName,
|
||||
runId,
|
||||
});
|
||||
|
||||
const skillCalls = result.toolCalls.filter(tc => tc.tool === 'Skill');
|
||||
const actualSkill = skillCalls.length > 0 ? skillCalls[0]?.input?.skill : undefined;
|
||||
|
||||
logCost(`journey: ${testName}`, result);
|
||||
recordRouting(testName, result, expectedSkill, actualSkill);
|
||||
|
||||
expect(skillCalls.length, `Expected Skill tool to be called but got 0 calls. Claude may have answered directly without invoking a skill. Tool calls: ${result.toolCalls.map(tc => tc.tool).join(', ')}`).toBeGreaterThan(0);
|
||||
expect([expectedSkill], `Expected skill ${expectedSkill} but got ${actualSkill}`).toContain(actualSkill);
|
||||
} finally {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 180_000);
|
||||
|
||||
test('journey-debug', async () => {
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-debug-'));
|
||||
try {
|
||||
initGitRepo(tmpDir);
|
||||
installSkills(tmpDir);
|
||||
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: tmpDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
fs.mkdirSync(path.join(tmpDir, 'src'), { recursive: true });
|
||||
fs.writeFileSync(path.join(tmpDir, 'src/api.ts'), `
|
||||
import express from 'express';
|
||||
const app = express();
|
||||
|
||||
app.get('/api/waitlist', async (req, res) => {
|
||||
const db = req.app.locals.db;
|
||||
const parties = await db.query('SELECT * FROM parties WHERE status = $1', ['waiting']);
|
||||
res.json(parties.rows);
|
||||
});
|
||||
|
||||
export default app;
|
||||
`);
|
||||
fs.writeFileSync(path.join(tmpDir, 'error.log'), `
|
||||
[2026-03-18T10:23:45Z] ERROR: GET /api/waitlist - 500 Internal Server Error
|
||||
TypeError: Cannot read properties of undefined (reading 'query')
|
||||
at /src/api.ts:5:32
|
||||
at Layer.handle [as handle_request] (/node_modules/express/lib/router/layer.js:95:5)
|
||||
[2026-03-18T10:23:46Z] ERROR: GET /api/waitlist - 500 Internal Server Error
|
||||
TypeError: Cannot read properties of undefined (reading 'query')
|
||||
`);
|
||||
|
||||
run('git', ['add', '.']);
|
||||
run('git', ['commit', '-m', 'initial']);
|
||||
run('git', ['checkout', '-b', 'feature/waitlist-api']);
|
||||
|
||||
const testName = 'journey-debug';
|
||||
const expectedSkill = 'investigate';
|
||||
const result = await runSkillTest({
|
||||
prompt: "The GET /api/waitlist endpoint was working fine yesterday but now it's returning 500 errors. The tests are passing locally but the endpoint fails when I hit it with curl. Can you figure out what's going on?",
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
|
||||
timeout: 60_000,
|
||||
testName,
|
||||
runId,
|
||||
});
|
||||
|
||||
const skillCalls = result.toolCalls.filter(tc => tc.tool === 'Skill');
|
||||
const actualSkill = skillCalls.length > 0 ? skillCalls[0]?.input?.skill : undefined;
|
||||
|
||||
logCost(`journey: ${testName}`, result);
|
||||
recordRouting(testName, result, expectedSkill, actualSkill);
|
||||
|
||||
expect(skillCalls.length, `Expected Skill tool to be called but got 0 calls. Claude may have answered directly without invoking a skill. Tool calls: ${result.toolCalls.map(tc => tc.tool).join(', ')}`).toBeGreaterThan(0);
|
||||
expect([expectedSkill], `Expected skill ${expectedSkill} but got ${actualSkill}`).toContain(actualSkill);
|
||||
} finally {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 90_000);
|
||||
|
||||
test('journey-qa', async () => {
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-qa-'));
|
||||
try {
|
||||
initGitRepo(tmpDir);
|
||||
installSkills(tmpDir);
|
||||
|
||||
fs.writeFileSync(path.join(tmpDir, 'package.json'), JSON.stringify({ name: 'waitlist-app', scripts: { dev: 'next dev' } }, null, 2));
|
||||
fs.mkdirSync(path.join(tmpDir, 'src'), { recursive: true });
|
||||
fs.writeFileSync(path.join(tmpDir, 'src/index.html'), '<html><body><h1>Waitlist App</h1></body></html>');
|
||||
spawnSync('git', ['add', '.'], { cwd: tmpDir, stdio: 'pipe', timeout: 5000 });
|
||||
spawnSync('git', ['commit', '-m', 'initial'], { cwd: tmpDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
const testName = 'journey-qa';
|
||||
const expectedSkill = 'qa';
|
||||
const alternateSkills = ['qa-only', 'browse'];
|
||||
const result = await runSkillTest({
|
||||
prompt: "I think the app is mostly working now. Can you go through the site and test everything — find any bugs and fix them?",
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
|
||||
timeout: 60_000,
|
||||
testName,
|
||||
runId,
|
||||
});
|
||||
|
||||
const skillCalls = result.toolCalls.filter(tc => tc.tool === 'Skill');
|
||||
const actualSkill = skillCalls.length > 0 ? skillCalls[0]?.input?.skill : undefined;
|
||||
const acceptable = [expectedSkill, ...alternateSkills];
|
||||
|
||||
logCost(`journey: ${testName}`, result);
|
||||
recordRouting(testName, result, expectedSkill, actualSkill);
|
||||
|
||||
expect(skillCalls.length, `Expected Skill tool to be called but got 0 calls. Claude may have answered directly without invoking a skill. Tool calls: ${result.toolCalls.map(tc => tc.tool).join(', ')}`).toBeGreaterThan(0);
|
||||
expect(acceptable, `Expected skill ${expectedSkill} but got ${actualSkill}`).toContain(actualSkill);
|
||||
} finally {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 90_000);
|
||||
|
||||
test('journey-code-review', async () => {
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-code-review-'));
|
||||
try {
|
||||
initGitRepo(tmpDir);
|
||||
installSkills(tmpDir);
|
||||
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: tmpDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
fs.writeFileSync(path.join(tmpDir, 'app.ts'), '// base\n');
|
||||
run('git', ['add', '.']);
|
||||
run('git', ['commit', '-m', 'initial']);
|
||||
run('git', ['checkout', '-b', 'feature/add-waitlist']);
|
||||
fs.writeFileSync(path.join(tmpDir, 'app.ts'), '// updated with waitlist feature\nimport { WaitlistService } from "./waitlist";\n');
|
||||
fs.writeFileSync(path.join(tmpDir, 'waitlist.ts'), 'export class WaitlistService {\n async addParty(name: string, size: number) {\n // TODO: implement\n }\n}\n');
|
||||
run('git', ['add', '.']);
|
||||
run('git', ['commit', '-m', 'feat: add waitlist service']);
|
||||
|
||||
const testName = 'journey-code-review';
|
||||
const expectedSkill = 'review';
|
||||
const result = await runSkillTest({
|
||||
prompt: "I'm about to merge this into main. Can you look over my changes and flag anything risky before I land it?",
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
|
||||
timeout: 60_000,
|
||||
testName,
|
||||
runId,
|
||||
});
|
||||
|
||||
const skillCalls = result.toolCalls.filter(tc => tc.tool === 'Skill');
|
||||
const actualSkill = skillCalls.length > 0 ? skillCalls[0]?.input?.skill : undefined;
|
||||
|
||||
logCost(`journey: ${testName}`, result);
|
||||
recordRouting(testName, result, expectedSkill, actualSkill);
|
||||
|
||||
expect(skillCalls.length, `Expected Skill tool to be called but got 0 calls. Claude may have answered directly without invoking a skill. Tool calls: ${result.toolCalls.map(tc => tc.tool).join(', ')}`).toBeGreaterThan(0);
|
||||
expect([expectedSkill], `Expected skill ${expectedSkill} but got ${actualSkill}`).toContain(actualSkill);
|
||||
} finally {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 90_000);
|
||||
|
||||
test('journey-ship', async () => {
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-ship-'));
|
||||
try {
|
||||
initGitRepo(tmpDir);
|
||||
installSkills(tmpDir);
|
||||
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: tmpDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
fs.writeFileSync(path.join(tmpDir, 'app.ts'), '// base\n');
|
||||
run('git', ['add', '.']);
|
||||
run('git', ['commit', '-m', 'initial']);
|
||||
run('git', ['checkout', '-b', 'feature/waitlist']);
|
||||
fs.writeFileSync(path.join(tmpDir, 'app.ts'), '// waitlist feature\n');
|
||||
run('git', ['add', '.']);
|
||||
run('git', ['commit', '-m', 'feat: waitlist']);
|
||||
|
||||
const testName = 'journey-ship';
|
||||
const expectedSkill = 'ship';
|
||||
const result = await runSkillTest({
|
||||
prompt: "This looks good. Let's get it deployed — push the code up and create a PR.",
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
|
||||
timeout: 60_000,
|
||||
testName,
|
||||
runId,
|
||||
});
|
||||
|
||||
const skillCalls = result.toolCalls.filter(tc => tc.tool === 'Skill');
|
||||
const actualSkill = skillCalls.length > 0 ? skillCalls[0]?.input?.skill : undefined;
|
||||
|
||||
logCost(`journey: ${testName}`, result);
|
||||
recordRouting(testName, result, expectedSkill, actualSkill);
|
||||
|
||||
expect(skillCalls.length, `Expected Skill tool to be called but got 0 calls. Claude may have answered directly without invoking a skill. Tool calls: ${result.toolCalls.map(tc => tc.tool).join(', ')}`).toBeGreaterThan(0);
|
||||
expect([expectedSkill], `Expected skill ${expectedSkill} but got ${actualSkill}`).toContain(actualSkill);
|
||||
} finally {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 90_000);
|
||||
|
||||
test('journey-docs', async () => {
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-docs-'));
|
||||
try {
|
||||
initGitRepo(tmpDir);
|
||||
installSkills(tmpDir);
|
||||
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: tmpDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
fs.writeFileSync(path.join(tmpDir, 'README.md'), '# Waitlist App\nA simple waitlist management tool.\n');
|
||||
fs.mkdirSync(path.join(tmpDir, 'src'), { recursive: true });
|
||||
fs.writeFileSync(path.join(tmpDir, 'src/api.ts'), '// API code\n');
|
||||
run('git', ['add', '.']);
|
||||
run('git', ['commit', '-m', 'feat: ship waitlist feature']);
|
||||
|
||||
const testName = 'journey-docs';
|
||||
const expectedSkill = 'document-release';
|
||||
const result = await runSkillTest({
|
||||
prompt: "We just shipped the waitlist feature. Can you go through the README and any other docs and make sure they match what we actually built?",
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
|
||||
timeout: 60_000,
|
||||
testName,
|
||||
runId,
|
||||
});
|
||||
|
||||
const skillCalls = result.toolCalls.filter(tc => tc.tool === 'Skill');
|
||||
const actualSkill = skillCalls.length > 0 ? skillCalls[0]?.input?.skill : undefined;
|
||||
|
||||
logCost(`journey: ${testName}`, result);
|
||||
recordRouting(testName, result, expectedSkill, actualSkill);
|
||||
|
||||
expect(skillCalls.length, `Expected Skill tool to be called but got 0 calls. Claude may have answered directly without invoking a skill. Tool calls: ${result.toolCalls.map(tc => tc.tool).join(', ')}`).toBeGreaterThan(0);
|
||||
expect([expectedSkill], `Expected skill ${expectedSkill} but got ${actualSkill}`).toContain(actualSkill);
|
||||
} finally {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 90_000);
|
||||
|
||||
test('journey-retro', async () => {
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-retro-'));
|
||||
try {
|
||||
initGitRepo(tmpDir);
|
||||
installSkills(tmpDir);
|
||||
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: tmpDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
fs.writeFileSync(path.join(tmpDir, 'api.ts'), 'export function getParties() { return []; }\n');
|
||||
run('git', ['add', '.']);
|
||||
run('git', ['commit', '-m', 'feat: add parties API', '--date', '2026-03-12T09:30:00']);
|
||||
|
||||
fs.writeFileSync(path.join(tmpDir, 'ui.tsx'), 'export function WaitlistView() { return <div>Waitlist</div>; }\n');
|
||||
run('git', ['add', '.']);
|
||||
run('git', ['commit', '-m', 'feat: add waitlist UI', '--date', '2026-03-13T14:00:00']);
|
||||
|
||||
fs.writeFileSync(path.join(tmpDir, 'README.md'), '# Waitlist App\n');
|
||||
run('git', ['add', '.']);
|
||||
run('git', ['commit', '-m', 'docs: add README', '--date', '2026-03-14T16:00:00']);
|
||||
|
||||
const testName = 'journey-retro';
|
||||
const expectedSkill = 'retro';
|
||||
const result = await runSkillTest({
|
||||
prompt: "It's Friday. What did we ship this week? I want to do a quick retrospective on what the team accomplished.",
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
|
||||
timeout: 60_000,
|
||||
testName,
|
||||
runId,
|
||||
});
|
||||
|
||||
const skillCalls = result.toolCalls.filter(tc => tc.tool === 'Skill');
|
||||
const actualSkill = skillCalls.length > 0 ? skillCalls[0]?.input?.skill : undefined;
|
||||
|
||||
logCost(`journey: ${testName}`, result);
|
||||
recordRouting(testName, result, expectedSkill, actualSkill);
|
||||
|
||||
expect(skillCalls.length, `Expected Skill tool to be called but got 0 calls. Claude may have answered directly without invoking a skill. Tool calls: ${result.toolCalls.map(tc => tc.tool).join(', ')}`).toBeGreaterThan(0);
|
||||
expect([expectedSkill], `Expected skill ${expectedSkill} but got ${actualSkill}`).toContain(actualSkill);
|
||||
} finally {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 90_000);
|
||||
|
||||
test('journey-design-system', async () => {
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-design-system-'));
|
||||
try {
|
||||
initGitRepo(tmpDir);
|
||||
installSkills(tmpDir);
|
||||
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: tmpDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
fs.writeFileSync(path.join(tmpDir, 'package.json'), JSON.stringify({ name: 'waitlist-app' }, null, 2));
|
||||
run('git', ['add', '.']);
|
||||
run('git', ['commit', '-m', 'initial']);
|
||||
|
||||
const testName = 'journey-design-system';
|
||||
const expectedSkill = 'design-consultation';
|
||||
const result = await runSkillTest({
|
||||
prompt: "Before we build the UI, I want to establish a design system — typography, colors, spacing, the whole thing. Can you put together brand guidelines for this project?",
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
|
||||
timeout: 60_000,
|
||||
testName,
|
||||
runId,
|
||||
});
|
||||
|
||||
const skillCalls = result.toolCalls.filter(tc => tc.tool === 'Skill');
|
||||
const actualSkill = skillCalls.length > 0 ? skillCalls[0]?.input?.skill : undefined;
|
||||
|
||||
logCost(`journey: ${testName}`, result);
|
||||
recordRouting(testName, result, expectedSkill, actualSkill);
|
||||
|
||||
expect(skillCalls.length, `Expected Skill tool to be called but got 0 calls. Claude may have answered directly without invoking a skill. Tool calls: ${result.toolCalls.map(tc => tc.tool).join(', ')}`).toBeGreaterThan(0);
|
||||
expect([expectedSkill], `Expected skill ${expectedSkill} but got ${actualSkill}`).toContain(actualSkill);
|
||||
} finally {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 90_000);
|
||||
|
||||
test('journey-visual-qa', async () => {
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-visual-qa-'));
|
||||
try {
|
||||
initGitRepo(tmpDir);
|
||||
installSkills(tmpDir);
|
||||
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: tmpDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
fs.mkdirSync(path.join(tmpDir, 'src'), { recursive: true });
|
||||
fs.writeFileSync(path.join(tmpDir, 'src/styles.css'), `
|
||||
body { font-family: sans-serif; }
|
||||
.header { font-size: 24px; margin: 20px; }
|
||||
.card { padding: 16px; margin: 8px; border: 1px solid #ccc; }
|
||||
.button { background: #007bff; color: white; padding: 10px 20px; }
|
||||
`);
|
||||
fs.writeFileSync(path.join(tmpDir, 'src/index.html'), `
|
||||
<html>
|
||||
<head><link rel="stylesheet" href="styles.css"></head>
|
||||
<body>
|
||||
<div class="header">Waitlist</div>
|
||||
<div class="card">Party of 4 - Smith</div>
|
||||
<div class="card">Party of 2 - Jones</div>
|
||||
</body>
|
||||
</html>
|
||||
`);
|
||||
run('git', ['add', '.']);
|
||||
run('git', ['commit', '-m', 'initial UI']);
|
||||
|
||||
const testName = 'journey-visual-qa';
|
||||
const expectedSkill = 'design-review';
|
||||
const result = await runSkillTest({
|
||||
prompt: "Something looks off on the site. The spacing between sections is inconsistent and the font sizes don't feel right. Can you audit the visual design and fix anything that doesn't look polished?",
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
|
||||
timeout: 60_000,
|
||||
testName,
|
||||
runId,
|
||||
});
|
||||
|
||||
const skillCalls = result.toolCalls.filter(tc => tc.tool === 'Skill');
|
||||
const actualSkill = skillCalls.length > 0 ? skillCalls[0]?.input?.skill : undefined;
|
||||
|
||||
logCost(`journey: ${testName}`, result);
|
||||
recordRouting(testName, result, expectedSkill, actualSkill);
|
||||
|
||||
expect(skillCalls.length, `Expected Skill tool to be called but got 0 calls. Claude may have answered directly without invoking a skill. Tool calls: ${result.toolCalls.map(tc => tc.tool).join(', ')}`).toBeGreaterThan(0);
|
||||
expect([expectedSkill], `Expected skill ${expectedSkill} but got ${actualSkill}`).toContain(actualSkill);
|
||||
} finally {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 90_000);
|
||||
});
|
||||
@@ -218,6 +218,7 @@ describe('Update check preamble', () => {
|
||||
'ship/SKILL.md', 'review/SKILL.md',
|
||||
'plan-ceo-review/SKILL.md', 'plan-eng-review/SKILL.md',
|
||||
'retro/SKILL.md',
|
||||
'office-hours/SKILL.md', 'investigate/SKILL.md',
|
||||
'plan-design-review/SKILL.md',
|
||||
'design-review/SKILL.md',
|
||||
'design-consultation/SKILL.md',
|
||||
@@ -446,6 +447,7 @@ describe('No hardcoded branch names in SKILL templates', () => {
|
||||
'document-release/SKILL.md.tmpl',
|
||||
'plan-eng-review/SKILL.md.tmpl',
|
||||
'plan-design-review/SKILL.md.tmpl',
|
||||
'codex/SKILL.md.tmpl',
|
||||
];
|
||||
|
||||
// Patterns that indicate hardcoded 'main' in git commands
|
||||
@@ -528,6 +530,7 @@ describe('v0.4.1 preamble features', () => {
|
||||
'ship/SKILL.md', 'review/SKILL.md',
|
||||
'plan-ceo-review/SKILL.md', 'plan-eng-review/SKILL.md',
|
||||
'retro/SKILL.md',
|
||||
'office-hours/SKILL.md', 'investigate/SKILL.md',
|
||||
'plan-design-review/SKILL.md',
|
||||
'design-review/SKILL.md',
|
||||
'design-consultation/SKILL.md',
|
||||
@@ -547,6 +550,108 @@ describe('v0.4.1 preamble features', () => {
|
||||
expect(content).toContain('RECOMMENDATION');
|
||||
});
|
||||
}
|
||||
|
||||
for (const skill of skillsWithPreamble) {
|
||||
test(`${skill} contains escalation protocol`, () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, skill), 'utf-8');
|
||||
expect(content).toContain('DONE_WITH_CONCERNS');
|
||||
expect(content).toContain('BLOCKED');
|
||||
expect(content).toContain('NEEDS_CONTEXT');
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// --- Structural tests for new skills ---
|
||||
|
||||
describe('office-hours skill structure', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'office-hours', 'SKILL.md'), 'utf-8');
|
||||
|
||||
// Original structural assertions
|
||||
for (const section of ['Phase 1', 'Phase 2', 'Phase 3', 'Phase 4', 'Phase 5', 'Phase 6',
|
||||
'Design Doc', 'Supersedes', 'APPROVED', 'Premise Challenge',
|
||||
'Alternatives', 'Smart-skip']) {
|
||||
test(`contains ${section}`, () => expect(content).toContain(section));
|
||||
}
|
||||
|
||||
// Dual-mode structure
|
||||
for (const section of ['Startup mode', 'Builder mode']) {
|
||||
test(`contains ${section}`, () => expect(content).toContain(section));
|
||||
}
|
||||
|
||||
// Mode detection question
|
||||
test('contains explicit mode detection question', () => {
|
||||
expect(content).toContain("what's your goal");
|
||||
});
|
||||
|
||||
// Six forcing questions (startup mode)
|
||||
for (const question of ['Demand Reality', 'Status Quo', 'Desperate Specificity',
|
||||
'Narrowest Wedge', 'Observation & Surprise', 'Future-Fit']) {
|
||||
test(`contains forcing question: ${question}`, () => expect(content).toContain(question));
|
||||
}
|
||||
|
||||
// Builder mode questions
|
||||
test('contains builder brainstorming questions', () => {
|
||||
expect(content).toContain('coolest version');
|
||||
expect(content).toContain('delightful');
|
||||
});
|
||||
|
||||
// Intrapreneurship adaptation
|
||||
test('contains intrapreneurship adaptation', () => {
|
||||
expect(content).toContain('Intrapreneurship');
|
||||
});
|
||||
|
||||
// YC founder discovery engine
|
||||
test('contains YC apply CTA with ref tracking', () => {
|
||||
expect(content).toContain('ycombinator.com/apply?ref=gstack');
|
||||
});
|
||||
|
||||
test('contains "What I noticed" design doc section', () => {
|
||||
expect(content).toContain('What I noticed about how you think');
|
||||
});
|
||||
|
||||
test('contains golden age framing', () => {
|
||||
expect(content).toContain('golden age');
|
||||
});
|
||||
|
||||
test('contains Garry Tan personal plea', () => {
|
||||
expect(content).toContain('Garry Tan, the creator of GStack');
|
||||
});
|
||||
|
||||
test('contains founder signal synthesis phase', () => {
|
||||
expect(content).toContain('Founder Signal Synthesis');
|
||||
});
|
||||
|
||||
test('contains three-tier decision rubric', () => {
|
||||
expect(content).toContain('Top tier');
|
||||
expect(content).toContain('Middle tier');
|
||||
expect(content).toContain('Base tier');
|
||||
});
|
||||
|
||||
test('contains anti-slop examples', () => {
|
||||
expect(content).toContain('GOOD:');
|
||||
expect(content).toContain('BAD:');
|
||||
});
|
||||
|
||||
test('contains "One more thing" transition beat', () => {
|
||||
expect(content).toContain('One more thing');
|
||||
});
|
||||
|
||||
// Operating principles per mode
|
||||
test('contains startup operating principles', () => {
|
||||
expect(content).toContain('Specificity is the only currency');
|
||||
});
|
||||
|
||||
test('contains builder operating principles', () => {
|
||||
expect(content).toContain('Delight is the currency');
|
||||
});
|
||||
});
|
||||
|
||||
describe('investigate skill structure', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'investigate', 'SKILL.md'), 'utf-8');
|
||||
for (const section of ['Iron Law', 'Root Cause', 'Pattern Analysis', 'Hypothesis',
|
||||
'DEBUG REPORT', '3-strike', 'BLOCKED']) {
|
||||
test(`contains ${section}`, () => expect(content).toContain(section));
|
||||
}
|
||||
});
|
||||
|
||||
// --- Contributor mode preamble structure validation ---
|
||||
@@ -1016,3 +1121,139 @@ describe('QA report template', () => {
|
||||
expect(content).toContain('**Precondition:**');
|
||||
});
|
||||
});
|
||||
|
||||
// --- Codex skill validation ---
|
||||
|
||||
describe('Codex skill', () => {
|
||||
test('codex/SKILL.md exists and has correct frontmatter', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'codex', 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('name: codex');
|
||||
expect(content).toContain('version: 1.0.0');
|
||||
expect(content).toContain('allowed-tools:');
|
||||
});
|
||||
|
||||
test('codex/SKILL.md contains all three modes', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'codex', 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('Step 2A: Review Mode');
|
||||
expect(content).toContain('Step 2B: Challenge');
|
||||
expect(content).toContain('Step 2C: Consult Mode');
|
||||
});
|
||||
|
||||
test('codex/SKILL.md contains gate verdict logic', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'codex', 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('[P1]');
|
||||
expect(content).toContain('GATE: PASS');
|
||||
expect(content).toContain('GATE: FAIL');
|
||||
});
|
||||
|
||||
test('codex/SKILL.md contains session continuity', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'codex', 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('codex-session-id');
|
||||
expect(content).toContain('codex exec resume');
|
||||
});
|
||||
|
||||
test('codex/SKILL.md contains cost tracking', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'codex', 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('tokens used');
|
||||
expect(content).toContain('Est. cost');
|
||||
});
|
||||
|
||||
test('codex/SKILL.md contains cross-model comparison', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'codex', 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('CROSS-MODEL ANALYSIS');
|
||||
expect(content).toContain('Agreement rate');
|
||||
});
|
||||
|
||||
test('codex/SKILL.md contains review log persistence', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'codex', 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('codex-review');
|
||||
expect(content).toContain('gstack-review-log');
|
||||
});
|
||||
|
||||
test('codex/SKILL.md uses which for binary discovery, not hardcoded path', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'codex', 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('which codex');
|
||||
expect(content).not.toContain('/opt/homebrew/bin/codex');
|
||||
});
|
||||
|
||||
test('codex/SKILL.md contains error handling for missing binary and auth', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'codex', 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('NOT_FOUND');
|
||||
expect(content).toContain('codex login');
|
||||
});
|
||||
|
||||
test('codex/SKILL.md uses mktemp for temp files', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'codex', 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('mktemp');
|
||||
});
|
||||
|
||||
test('codex integration in /review offers second opinion', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'review', 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('Codex second opinion');
|
||||
expect(content).toContain('codex review');
|
||||
expect(content).toContain('adversarial');
|
||||
});
|
||||
|
||||
test('codex integration in /ship offers review gate', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'ship', 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('Codex');
|
||||
expect(content).toContain('codex review');
|
||||
expect(content).toContain('codex-review');
|
||||
});
|
||||
|
||||
test('codex integration in /plan-eng-review offers plan critique', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'plan-eng-review', 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('Codex');
|
||||
expect(content).toContain('codex exec');
|
||||
});
|
||||
|
||||
test('Review Readiness Dashboard includes Codex Review row', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'ship', 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('Codex Review');
|
||||
expect(content).toContain('codex-review');
|
||||
});
|
||||
});
|
||||
|
||||
// --- Trigger phrase validation ---
|
||||
|
||||
describe('Skill trigger phrases', () => {
|
||||
// Skills that must have "Use when" trigger phrases in their description.
|
||||
// Excluded: root gstack (browser tool), gstack-upgrade (gstack-specific),
|
||||
// humanizer (text tool)
|
||||
const SKILLS_REQUIRING_TRIGGERS = [
|
||||
'qa', 'qa-only', 'ship', 'review', 'investigate', 'office-hours',
|
||||
'plan-ceo-review', 'plan-eng-review', 'plan-design-review',
|
||||
'design-review', 'design-consultation', 'retro', 'document-release',
|
||||
'codex', 'browse', 'setup-browser-cookies',
|
||||
];
|
||||
|
||||
for (const skill of SKILLS_REQUIRING_TRIGGERS) {
|
||||
test(`${skill}/SKILL.md has "Use when" trigger phrases`, () => {
|
||||
const skillPath = path.join(ROOT, skill, 'SKILL.md');
|
||||
if (!fs.existsSync(skillPath)) return;
|
||||
const content = fs.readFileSync(skillPath, 'utf-8');
|
||||
// Extract description from frontmatter
|
||||
const frontmatterEnd = content.indexOf('---', 4);
|
||||
const frontmatter = content.slice(0, frontmatterEnd);
|
||||
expect(frontmatter).toMatch(/Use when/i);
|
||||
});
|
||||
}
|
||||
|
||||
// Skills with proactive triggers should have "Proactively suggest" in description
|
||||
const SKILLS_REQUIRING_PROACTIVE = [
|
||||
'qa', 'qa-only', 'ship', 'review', 'investigate', 'office-hours',
|
||||
'plan-ceo-review', 'plan-eng-review', 'plan-design-review',
|
||||
'design-review', 'design-consultation', 'retro', 'document-release',
|
||||
];
|
||||
|
||||
for (const skill of SKILLS_REQUIRING_PROACTIVE) {
|
||||
test(`${skill}/SKILL.md has "Proactively suggest" phrase`, () => {
|
||||
const skillPath = path.join(ROOT, skill, 'SKILL.md');
|
||||
if (!fs.existsSync(skillPath)) return;
|
||||
const content = fs.readFileSync(skillPath, 'utf-8');
|
||||
const frontmatterEnd = content.indexOf('---', 4);
|
||||
const frontmatter = content.slice(0, frontmatterEnd);
|
||||
expect(frontmatter).toMatch(/Proactively suggest/i);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
@@ -115,23 +115,27 @@ describe('selectTests', () => {
|
||||
expect(result.selected).toContain('plan-ceo-review-selective');
|
||||
expect(result.selected).toContain('retro');
|
||||
expect(result.selected).toContain('retro-base-branch');
|
||||
expect(result.selected.length).toBe(4);
|
||||
// Also selects journey routing tests (*/SKILL.md.tmpl matches retro/SKILL.md.tmpl)
|
||||
expect(result.selected.length).toBeGreaterThanOrEqual(4);
|
||||
});
|
||||
|
||||
test('works with LLM_JUDGE_TOUCHFILES', () => {
|
||||
const result = selectTests(['qa/SKILL.md'], LLM_JUDGE_TOUCHFILES);
|
||||
expect(result.selected).toContain('qa/SKILL.md workflow');
|
||||
expect(result.selected).toContain('qa/SKILL.md health rubric');
|
||||
expect(result.selected.length).toBe(2);
|
||||
expect(result.selected).toContain('qa/SKILL.md anti-refusal');
|
||||
expect(result.selected.length).toBe(3);
|
||||
});
|
||||
|
||||
test('SKILL.md.tmpl root template only selects root-dependent tests', () => {
|
||||
test('SKILL.md.tmpl root template selects root-dependent tests and routing tests', () => {
|
||||
const result = selectTests(['SKILL.md.tmpl'], E2E_TOUCHFILES);
|
||||
// Should select the 7 tests that depend on root SKILL.md
|
||||
expect(result.selected).toContain('skillmd-setup-discovery');
|
||||
expect(result.selected).toContain('contributor-mode');
|
||||
expect(result.selected).toContain('session-awareness');
|
||||
// Should NOT select unrelated tests
|
||||
// Also selects journey routing tests (SKILL.md.tmpl in their touchfiles)
|
||||
expect(result.selected).toContain('journey-ideation');
|
||||
// Should NOT select unrelated non-routing tests
|
||||
expect(result.selected).not.toContain('plan-ceo-review');
|
||||
expect(result.selected).not.toContain('retro');
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user