mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-09 06:28:59 +02:00
fix: routing journeys lose their answer key and end at the routing decision
The journey tests exist to catch skill-DESCRIPTION regressions (touchfiles: */SKILL.md.tmpl), but the fixture CLAUDE.md shipped an explicit prompt->skill lookup table — with the answer key in context, a badly regressed frontmatter description still routed correctly, so the tests could not fail on the exact class they select for. The fixture now carries only the generic invoke-skills nudge; the frontmatter carries the routing load. Also capped all 10 journeys at maxTurns 2 / tools [Skill, Read]: only the FIRST Skill call is asserted, so 5 turns of Read/Bash/Glob/Grep was pure spend — roughly halves each journey's cost. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
011d02ac97
commit
99b5fa2e46
@@ -90,10 +90,14 @@ function installSkills(tmpDir: string) {
|
||||
fs.writeFileSync(path.join(destDir, 'SKILL.md'), extractSkillHead(srcPath));
|
||||
}
|
||||
|
||||
// Write a CLAUDE.md with explicit routing instructions.
|
||||
// The skill descriptions in system-reminder aren't strong enough to override
|
||||
// Claude's default behavior of answering directly. A CLAUDE.md instruction
|
||||
// puts routing rules in project context which Claude weighs more heavily.
|
||||
// Write a CLAUDE.md with a GENERIC invoke-skills nudge — deliberately NO
|
||||
// per-skill routing table. These journey tests exist to catch skill
|
||||
// DESCRIPTION regressions (their touchfiles key on */SKILL.md.tmpl), and
|
||||
// the old fixture shipped an explicit prompt→skill answer key: with the
|
||||
// lookup table in context, a badly regressed frontmatter description
|
||||
// still routed correctly and the tests could not fail on the regression
|
||||
// class they select for (2026-08 audit). The generic nudge keeps Claude's
|
||||
// reach-for-a-skill posture; the FRONTMATTER carries the routing load.
|
||||
fs.writeFileSync(path.join(tmpDir, 'CLAUDE.md'), `# Project Instructions
|
||||
|
||||
## Skill routing
|
||||
@@ -101,18 +105,7 @@ function installSkills(tmpDir: string) {
|
||||
When the user's request matches an available skill, ALWAYS invoke it using the Skill
|
||||
tool as your FIRST action. Do NOT answer directly, do NOT use other tools first.
|
||||
The skill has specialized workflows that produce better results than ad-hoc answers.
|
||||
|
||||
Key routing rules:
|
||||
- Product ideas, "is this worth building", brainstorming → invoke office-hours
|
||||
- Bugs, errors, "why is this broken", 500 errors → invoke investigate
|
||||
- Ship, deploy, push, create PR → invoke ship
|
||||
- QA, test the site, find bugs → invoke qa
|
||||
- Code review, check my diff → invoke review
|
||||
- Update docs after shipping → invoke document-release
|
||||
- Weekly retro → invoke retro
|
||||
- Design system, brand → invoke design-consultation
|
||||
- Visual audit, design polish → invoke design-review
|
||||
- Architecture review → invoke plan-eng-review
|
||||
Choose the skill by matching the request against each skill's description.
|
||||
`);
|
||||
}
|
||||
|
||||
@@ -196,8 +189,12 @@ describeE2E('Skill Routing E2E — Developer Journey', () => {
|
||||
const result = await runSkillTest({
|
||||
prompt: "I've been thinking about building a waitlist management tool for restaurants. The existing solutions are expensive and overcomplicated. I want something simple — a tablet app where hosts can add parties, see wait times, and text customers when their table is ready. Help me think through whether this is worth building and what the key design decisions are.",
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
|
||||
// Turn/tool cap (2026-08 audit): only the FIRST Skill call is
|
||||
// asserted, so 5 turns of Read/Bash/Glob/Grep was pure spend — the
|
||||
// session ends at the routing decision, roughly halving each
|
||||
// journey's cost.
|
||||
maxTurns: 2,
|
||||
allowedTools: ['Skill', 'Read'],
|
||||
timeout: JUDGE_MS,
|
||||
testName,
|
||||
runId,
|
||||
@@ -246,8 +243,12 @@ describeE2E('Skill Routing E2E — Developer Journey', () => {
|
||||
const result = await runSkillTest({
|
||||
prompt: "I wrote up a plan for the waitlist app in plan.md. Can you take a look at the architecture and make sure I'm not missing any edge cases or failure modes before I start coding?",
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
|
||||
// Turn/tool cap (2026-08 audit): only the FIRST Skill call is
|
||||
// asserted, so 5 turns of Read/Bash/Glob/Grep was pure spend — the
|
||||
// session ends at the routing decision, roughly halving each
|
||||
// journey's cost.
|
||||
maxTurns: 2,
|
||||
allowedTools: ['Skill', 'Read'],
|
||||
timeout: JUDGE_MS,
|
||||
testName,
|
||||
runId,
|
||||
@@ -308,8 +309,12 @@ export default app;
|
||||
const result = await runSkillTest({
|
||||
prompt: "The GET /api/waitlist endpoint was working fine yesterday but now it's returning 500 errors. The tests are passing locally but the endpoint fails when I hit it with curl. Can you figure out what's going on?",
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
|
||||
// Turn/tool cap (2026-08 audit): only the FIRST Skill call is
|
||||
// asserted, so 5 turns of Read/Bash/Glob/Grep was pure spend — the
|
||||
// session ends at the routing decision, roughly halving each
|
||||
// journey's cost.
|
||||
maxTurns: 2,
|
||||
allowedTools: ['Skill', 'Read'],
|
||||
timeout: JUDGE_MS,
|
||||
testName,
|
||||
runId,
|
||||
@@ -344,8 +349,12 @@ export default app;
|
||||
const result = await runSkillTest({
|
||||
prompt: "I think the app is mostly working now. Can you go through the site and test everything — find any bugs and fix them?",
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
|
||||
// Turn/tool cap (2026-08 audit): only the FIRST Skill call is
|
||||
// asserted, so 5 turns of Read/Bash/Glob/Grep was pure spend — the
|
||||
// session ends at the routing decision, roughly halving each
|
||||
// journey's cost.
|
||||
maxTurns: 2,
|
||||
allowedTools: ['Skill', 'Read'],
|
||||
timeout: JUDGE_MS,
|
||||
testName,
|
||||
runId,
|
||||
@@ -385,8 +394,12 @@ export default app;
|
||||
const result = await runSkillTest({
|
||||
prompt: "I'm about to merge this into main. Can you look over my changes and flag anything risky before I land it?",
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
|
||||
// Turn/tool cap (2026-08 audit): only the FIRST Skill call is
|
||||
// asserted, so 5 turns of Read/Bash/Glob/Grep was pure spend — the
|
||||
// session ends at the routing decision, roughly halving each
|
||||
// journey's cost.
|
||||
maxTurns: 2,
|
||||
allowedTools: ['Skill', 'Read'],
|
||||
timeout: JUDGE_MS,
|
||||
testName,
|
||||
runId,
|
||||
@@ -424,8 +437,12 @@ export default app;
|
||||
const result = await runSkillTest({
|
||||
prompt: "This looks good. Let's get it deployed — push the code up and create a PR.",
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
|
||||
// Turn/tool cap (2026-08 audit): only the FIRST Skill call is
|
||||
// asserted, so 5 turns of Read/Bash/Glob/Grep was pure spend — the
|
||||
// session ends at the routing decision, roughly halving each
|
||||
// journey's cost.
|
||||
maxTurns: 2,
|
||||
allowedTools: ['Skill', 'Read'],
|
||||
timeout: JUDGE_MS,
|
||||
testName,
|
||||
runId,
|
||||
@@ -461,8 +478,12 @@ export default app;
|
||||
const result = await runSkillTest({
|
||||
prompt: "We just shipped the waitlist feature. Can you go through the README and any other docs and make sure they match what we actually built?",
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
|
||||
// Turn/tool cap (2026-08 audit): only the FIRST Skill call is
|
||||
// asserted, so 5 turns of Read/Bash/Glob/Grep was pure spend — the
|
||||
// session ends at the routing decision, roughly halving each
|
||||
// journey's cost.
|
||||
maxTurns: 2,
|
||||
allowedTools: ['Skill', 'Read'],
|
||||
timeout: JUDGE_MS,
|
||||
testName,
|
||||
runId,
|
||||
@@ -504,8 +525,12 @@ export default app;
|
||||
const result = await runSkillTest({
|
||||
prompt: "It's Friday. What did we ship this week? I want to do a quick retrospective on what the team accomplished.",
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
|
||||
// Turn/tool cap (2026-08 audit): only the FIRST Skill call is
|
||||
// asserted, so 5 turns of Read/Bash/Glob/Grep was pure spend — the
|
||||
// session ends at the routing decision, roughly halving each
|
||||
// journey's cost.
|
||||
maxTurns: 2,
|
||||
allowedTools: ['Skill', 'Read'],
|
||||
timeout: JUDGE_MS,
|
||||
testName,
|
||||
runId,
|
||||
@@ -533,8 +558,12 @@ export default app;
|
||||
const result = await runSkillTest({
|
||||
prompt: "Before we build the UI, I want to establish a design system — typography, colors, spacing, the whole thing. Can you put together brand guidelines for this project?",
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
|
||||
// Turn/tool cap (2026-08 audit): only the FIRST Skill call is
|
||||
// asserted, so 5 turns of Read/Bash/Glob/Grep was pure spend — the
|
||||
// session ends at the routing decision, roughly halving each
|
||||
// journey's cost.
|
||||
maxTurns: 2,
|
||||
allowedTools: ['Skill', 'Read'],
|
||||
timeout: JUDGE_MS,
|
||||
testName,
|
||||
runId,
|
||||
@@ -584,8 +613,12 @@ body { font-family: sans-serif; }
|
||||
const result = await runSkillTest({
|
||||
prompt: "Something looks off on the site. The spacing between sections is inconsistent and the font sizes don't feel right. Can you audit the visual design and fix anything that doesn't look polished?",
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
|
||||
// Turn/tool cap (2026-08 audit): only the FIRST Skill call is
|
||||
// asserted, so 5 turns of Read/Bash/Glob/Grep was pure spend — the
|
||||
// session ends at the routing decision, roughly halving each
|
||||
// journey's cost.
|
||||
maxTurns: 2,
|
||||
allowedTools: ['Skill', 'Read'],
|
||||
timeout: JUDGE_MS,
|
||||
testName,
|
||||
runId,
|
||||
|
||||
Reference in New Issue
Block a user