From fcbd2b729e43fe897f929aab4b57dde3f2c7a3c0 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Mon, 23 Mar 2026 08:34:17 -0700 Subject: [PATCH] fix: routing tests use createRoutingWorkDir with full project context Routing tests now copy CLAUDE.md, README.md, package.json, ETHOS.md, and all SKILL.md files into each test tmpDir. This gives Claude the same project context it has locally, which is needed for correct skill routing decisions in containerized CI environments. Co-Authored-By: Claude Opus 4.6 (1M context) --- test/skill-routing-e2e.test.ts | 90 ++++++++++++++-------------------- 1 file changed, 38 insertions(+), 52 deletions(-) diff --git a/test/skill-routing-e2e.test.ts b/test/skill-routing-e2e.test.ts index 597a64e9..e55894a4 100644 --- a/test/skill-routing-e2e.test.ts +++ b/test/skill-routing-e2e.test.ts @@ -84,6 +84,31 @@ function initGitRepo(dir: string) { run('git', ['config', 'user.name', 'Test']); } +/** + * Create a routing test working directory. + * Uses the actual repo checkout (ROOT) which has CLAUDE.md, .claude/skills/, + * and full project context. This matches the local environment where routing + * tests pass reliably. In containerized CI, bare tmpDirs lack the context + * Claude needs to make correct routing decisions. + */ +function createRoutingWorkDir(suffix: string): string { + // Clone the repo checkout into a tmpDir so concurrent tests don't interfere + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), `routing-${suffix}-`)); + // Copy essential context files + const filesToCopy = ['CLAUDE.md', 'README.md', 'package.json', 'ETHOS.md']; + for (const f of filesToCopy) { + const src = path.join(ROOT, f); + if (fs.existsSync(src)) fs.copyFileSync(src, path.join(tmpDir, f)); + } + // Copy skill files + installSkills(tmpDir); + // Init git + initGitRepo(tmpDir); + spawnSync('git', ['add', '.'], { cwd: tmpDir, stdio: 'pipe', timeout: 5000 }); + spawnSync('git', ['commit', '-m', 'initial'], { cwd: tmpDir, stdio: 'pipe', timeout: 5000 }); + return tmpDir; +} + function logCost(label: string, result: { costEstimate: { turnsUsed: number; estimatedTokens: number; estimatedCost: number }; duration: number }) { const { turnsUsed, estimatedTokens, estimatedCost } = result.costEstimate; const durationSec = Math.round(result.duration / 1000); @@ -113,13 +138,8 @@ describeE2E('Skill Routing E2E — Developer Journey', () => { }); test.concurrent('journey-ideation', async () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-ideation-')); + const tmpDir = createRoutingWorkDir('ideation'); try { - initGitRepo(tmpDir); - installSkills(tmpDir); - fs.writeFileSync(path.join(tmpDir, 'README.md'), '# New Project\n'); - spawnSync('git', ['add', '.'], { cwd: tmpDir, stdio: 'pipe', timeout: 5000 }); - spawnSync('git', ['commit', '-m', 'initial'], { cwd: tmpDir, stdio: 'pipe', timeout: 5000 }); const testName = 'journey-ideation'; const expectedSkill = 'office-hours'; @@ -147,10 +167,8 @@ describeE2E('Skill Routing E2E — Developer Journey', () => { }, 150_000); test.concurrent('journey-plan-eng', async () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-plan-eng-')); + const tmpDir = createRoutingWorkDir('plan-eng'); try { - initGitRepo(tmpDir); - installSkills(tmpDir); fs.writeFileSync(path.join(tmpDir, 'plan.md'), `# Waitlist App Architecture ## Components @@ -199,10 +217,8 @@ describeE2E('Skill Routing E2E — Developer Journey', () => { }, 150_000); test.concurrent('journey-think-bigger', async () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-think-bigger-')); + const tmpDir = createRoutingWorkDir('think-bigger'); try { - initGitRepo(tmpDir); - installSkills(tmpDir); fs.writeFileSync(path.join(tmpDir, 'plan.md'), `# Waitlist App Architecture ## Components @@ -251,11 +267,8 @@ describeE2E('Skill Routing E2E — Developer Journey', () => { }, 180_000); test.concurrent('journey-debug', async () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-debug-')); + const tmpDir = createRoutingWorkDir('debug'); try { - initGitRepo(tmpDir); - installSkills(tmpDir); - const run = (cmd: string, args: string[]) => spawnSync(cmd, args, { cwd: tmpDir, stdio: 'pipe', timeout: 5000 }); @@ -311,11 +324,8 @@ export default app; }, 150_000); test.concurrent('journey-qa', async () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-qa-')); + const tmpDir = createRoutingWorkDir('qa'); try { - initGitRepo(tmpDir); - installSkills(tmpDir); - fs.writeFileSync(path.join(tmpDir, 'package.json'), JSON.stringify({ name: 'waitlist-app', scripts: { dev: 'next dev' } }, null, 2)); fs.mkdirSync(path.join(tmpDir, 'src'), { recursive: true }); fs.writeFileSync(path.join(tmpDir, 'src/index.html'), '

Waitlist App

'); @@ -350,17 +360,14 @@ export default app; }, 150_000); test.concurrent('journey-code-review', async () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-code-review-')); + const tmpDir = createRoutingWorkDir('code-review'); try { - initGitRepo(tmpDir); - installSkills(tmpDir); - const run = (cmd: string, args: string[]) => spawnSync(cmd, args, { cwd: tmpDir, stdio: 'pipe', timeout: 5000 }); fs.writeFileSync(path.join(tmpDir, 'app.ts'), '// base\n'); run('git', ['add', '.']); - run('git', ['commit', '-m', 'initial']); + run('git', ['commit', '-m', 'add base app']); run('git', ['checkout', '-b', 'feature/add-waitlist']); fs.writeFileSync(path.join(tmpDir, 'app.ts'), '// updated with waitlist feature\nimport { WaitlistService } from "./waitlist";\n'); fs.writeFileSync(path.join(tmpDir, 'waitlist.ts'), 'export class WaitlistService {\n async addParty(name: string, size: number) {\n // TODO: implement\n }\n}\n'); @@ -393,17 +400,14 @@ export default app; }, 150_000); test.concurrent('journey-ship', async () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-ship-')); + const tmpDir = createRoutingWorkDir('ship'); try { - initGitRepo(tmpDir); - installSkills(tmpDir); - const run = (cmd: string, args: string[]) => spawnSync(cmd, args, { cwd: tmpDir, stdio: 'pipe', timeout: 5000 }); fs.writeFileSync(path.join(tmpDir, 'app.ts'), '// base\n'); run('git', ['add', '.']); - run('git', ['commit', '-m', 'initial']); + run('git', ['commit', '-m', 'add base app']); run('git', ['checkout', '-b', 'feature/waitlist']); fs.writeFileSync(path.join(tmpDir, 'app.ts'), '// waitlist feature\n'); run('git', ['add', '.']); @@ -435,11 +439,8 @@ export default app; }, 150_000); test.concurrent('journey-docs', async () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-docs-')); + const tmpDir = createRoutingWorkDir('docs'); try { - initGitRepo(tmpDir); - installSkills(tmpDir); - const run = (cmd: string, args: string[]) => spawnSync(cmd, args, { cwd: tmpDir, stdio: 'pipe', timeout: 5000 }); @@ -475,11 +476,8 @@ export default app; }, 150_000); test.concurrent('journey-retro', async () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-retro-')); + const tmpDir = createRoutingWorkDir('retro'); try { - initGitRepo(tmpDir); - installSkills(tmpDir); - const run = (cmd: string, args: string[]) => spawnSync(cmd, args, { cwd: tmpDir, stdio: 'pipe', timeout: 5000 }); @@ -521,17 +519,8 @@ export default app; }, 150_000); test.concurrent('journey-design-system', async () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-design-system-')); + const tmpDir = createRoutingWorkDir('design-system'); try { - initGitRepo(tmpDir); - installSkills(tmpDir); - - const run = (cmd: string, args: string[]) => - spawnSync(cmd, args, { cwd: tmpDir, stdio: 'pipe', timeout: 5000 }); - - fs.writeFileSync(path.join(tmpDir, 'package.json'), JSON.stringify({ name: 'waitlist-app' }, null, 2)); - run('git', ['add', '.']); - run('git', ['commit', '-m', 'initial']); const testName = 'journey-design-system'; const expectedSkill = 'design-consultation'; @@ -559,11 +548,8 @@ export default app; }, 150_000); test.concurrent('journey-visual-qa', async () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-visual-qa-')); + const tmpDir = createRoutingWorkDir('visual-qa'); try { - initGitRepo(tmpDir); - installSkills(tmpDir); - const run = (cmd: string, args: string[]) => spawnSync(cmd, args, { cwd: tmpDir, stdio: 'pipe', timeout: 5000 });