CalvinBackup/gstack
1
0
Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-05-05 05:05:08 +02:00
Code Issues Packages Projects Releases Wiki Activity

remove: delete journey-think-bigger routing test

Browse Source 
Never passed reliably. Tests ambiguous routing ("think bigger" →
plan-ceo-review) but Claude legitimately answers directly instead
of invoking a skill. The other 10 journey tests cover routing
with clear, actionable signals.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-03-26 22:29:03 -06:00
parent e2b9207313
commit 6a2a0a724f
2 changed files with 4 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/helpers/touchfiles.ts
-2
View File
@@ -147,7 +147,6 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
// Skill routing — journey-stage tests (depend on ALL skill descriptions)
'journey-ideation': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
'journey-plan-eng': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
'journey-think-bigger': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
'journey-debug': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
'journey-qa': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
'journey-code-review': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
@@ -268,7 +267,6 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
// Skill routing — periodic (LLM routing is non-deterministic)
'journey-ideation': 'periodic',
'journey-plan-eng': 'periodic',
'journey-think-bigger': 'periodic',
'journey-debug': 'periodic',
'journey-qa': 'periodic',
'journey-code-review': 'periodic',
test/skill-routing-e2e.test.ts
+4 -50
View File
@@ -250,56 +250,10 @@ describeE2E('Skill Routing E2E — Developer Journey', () => {
}
}, 150_000);
testIfSelected('journey-think-bigger', async () => {
const tmpDir = createRoutingWorkDir('think-bigger');
try {
fs.writeFileSync(path.join(tmpDir, 'plan.md'), `# Waitlist App Architecture
## Components
- REST API (Express.js)
- PostgreSQL database
- React frontend
- SMS integration (Twilio)
## Data Model
- restaurants (id, name, settings)
- parties (id, restaurant_id, name, size, phone, status, created_at)
- wait_estimates (id, restaurant_id, avg_wait_minutes)
## API Endpoints
- POST /api/parties - add party to waitlist
- GET /api/parties - list current waitlist
- PATCH /api/parties/:id/status - update party status
- GET /api/estimate - get current wait estimate
`);
spawnSync('git', ['add', '.'], { cwd: tmpDir, stdio: 'pipe', timeout: 5000 });
spawnSync('git', ['commit', '-m', 'initial'], { cwd: tmpDir, stdio: 'pipe', timeout: 5000 });
const testName = 'journey-think-bigger';
const expectedSkill = 'plan-ceo-review';
const result = await runSkillTest({
prompt: "I want to think bigger about this plan. We're just doing waitlists but what about the whole restaurant guest experience? Is this ambitious enough or should we expand scope?",
workingDirectory: tmpDir,
maxTurns: 3,
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
timeout: 60_000,
testName,
runId,
});
const skillCalls = result.toolCalls.filter(tc => tc.tool === 'Skill');
const actualSkill = skillCalls.length > 0 ? skillCalls[0]?.input?.skill : undefined;
logCost(`journey: ${testName}`, result);
recordRouting(testName, result, expectedSkill, actualSkill);
expect(skillCalls.length, `Expected Skill tool to be called but got 0 calls. Claude may have answered directly without invoking a skill. Tool calls: ${result.toolCalls.map(tc => tc.tool).join(', ')}`).toBeGreaterThan(0);
const validSkills = ['plan-ceo-review', 'office-hours'];
expect(validSkills, `Expected one of ${validSkills.join('/')} but got ${actualSkill}`).toContain(actualSkill);
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
}, 180_000);
// Removed: journey-think-bigger
// Tested ambiguous routing ("think bigger" → plan-ceo-review) but Claude
// legitimately answers directly instead of routing. Never passed reliably.
// The other 10 journey tests cover routing with clear signals.
testIfSelected('journey-debug', async () => {
const tmpDir = createRoutingWorkDir('debug');