diff --git a/docs/skill-routing-and-contracts.md b/docs/skill-routing-and-contracts.md index 3453ce358..24d983054 100644 --- a/docs/skill-routing-and-contracts.md +++ b/docs/skill-routing-and-contracts.md @@ -35,6 +35,7 @@ The off switch is for people. The text the Stop hook shows the model never menti The hook scores the prompt against the `triggers:` list in each installed `SKILL.md` (`~/.claude/skills/*/SKILL.md`, or `$CLAUDE_CONFIG_DIR/skills`). It makes no model call and no network request, and the same prompt always gets the same answer. - A trigger scores 1 when its whole phrase appears in the prompt. Otherwise it scores the share of its content words (common words dropped, light stemming, so "failing" matches "fails") that the prompt contains. A trigger with one content word counts only on its whole phrase, unless that word has five or more letters and is not a question word. +- Only /investigate, /review, /qa and /verify route. In the d7 check (PR #3912), scoring every installed skill's triggers sent 17 of 32 non-fit Terminal-Bench prompts to some skill at 0.66; routing only these four sends 4. The other skills wait for their own trigger review. - The skill with the best score routes when it reaches the threshold, 0.66: both words of a two-word trigger, or two of three. - Prompts that start with `/` (you already picked a skill) and prompts over 8,000 characters (pasted logs) are never routed. - The injected line names its source and its off switch, and asks the model to tell you in one sentence that gstack routed the request. diff --git a/lib/skill-route.ts b/lib/skill-route.ts index bf6b61ef2..458140b6b 100644 --- a/lib/skill-route.ts +++ b/lib/skill-route.ts @@ -176,6 +176,13 @@ export function scorePrompt(prompt: string, skills: InstalledSkill[]): RouteCand return out.sort((a, b) => b.score - a.score || b.matched - a.matched || a.skill.localeCompare(b.skill)); } +/** + * The skills the hook may route to: plan-v2 decision 4's discovery targets. Run D's prompts showed routing + * every installed skill's triggers false-routes most non-fit prompts (docs/skill-routing-and-contracts.md). + */ +export const ROUTED_SKILLS = ['investigate', 'review', 'qa', 'verify']; +const isRouted = (name: string) => ROUTED_SKILLS.includes(name.toLowerCase().replace(/^gstack[-:]/, '')); + export type RouteDecision = | { route: RouteCandidate; candidates: RouteCandidate[] } | { route: null; reason: 'slash-command' | 'prompt-too-long' | 'empty' | 'below-threshold'; candidates: RouteCandidate[] }; @@ -185,7 +192,7 @@ export function routePrompt(prompt: string, skills: InstalledSkill[], threshold if (!trimmed) return { route: null, reason: 'empty', candidates: [] }; if (trimmed.startsWith('/')) return { route: null, reason: 'slash-command', candidates: [] }; if (prompt.length > PROMPT_MAX_CHARS) return { route: null, reason: 'prompt-too-long', candidates: [] }; - const candidates = scorePrompt(prompt, skills); + const candidates = scorePrompt(prompt, skills.filter(s => isRouted(s.name))); const top = candidates[0]; return top && top.score >= threshold ? { route: top, candidates } : { route: null, reason: 'below-threshold', candidates }; } diff --git a/test/helpers/free-fixtures-data.ts b/test/helpers/free-fixtures-data.ts index 6baccf554..1cc026c28 100644 --- a/test/helpers/free-fixtures-data.ts +++ b/test/helpers/free-fixtures-data.ts @@ -240,6 +240,7 @@ export const FREE_FIXTURES: Readonly> = { 'test/fixtures/ship-queue/gbrain-stamp-pair.json': ['test/helpers/ship-queue-fixture.ts', 'test/tree-receipt-gbrain-pair.test.ts'], 'test/fixtures/ship-triage/**': ['test/ship-triage-labels.test.ts'], 'test/fixtures/skill-bin-drift-allowlist.json': ['test/skill-bin-drift.test.ts'], + 'test/fixtures/skill-contract/**': ['test/skill-contract-stop-hook.test.ts', 'test/skill-contract.test.ts'], 'test/fixtures/skill-evals/**': ['test/skill-eval-checks.test.ts', 'test/skill-eval-tasks.test.ts'], 'test/fixtures/skill-md-size-allowlist.json': ['test/skill-md-size-guard.test.ts'], 'test/fixtures/state-matrix-6792/**': ['test/state-matrix.test.ts'], diff --git a/test/skill-route.test.ts b/test/skill-route.test.ts index 1d85ecc9e..246032c21 100644 --- a/test/skill-route.test.ts +++ b/test/skill-route.test.ts @@ -8,7 +8,7 @@ import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; import { spawnSync } from 'node:child_process'; import * as fs from 'node:fs'; import * as path from 'node:path'; -import { PROMPT_MAX_CHARS, ROUTE_THRESHOLD, firstStep, hintLine, readSkillSwitch, routePrompt, scanInstalledSkills, scorePrompt, stem } from '../lib/skill-route'; +import { PROMPT_MAX_CHARS, ROUTED_SKILLS, ROUTE_THRESHOLD, firstStep, hintLine, readSkillSwitch, routePrompt, scanInstalledSkills, scorePrompt, stem } from '../lib/skill-route'; import { ROOT, ROUTE_HOOK, SKILLS, hookEnv, runHook, skillsDir, tmpDir } from './helpers/skill-contract-fixture'; let state = '', skills = ''; @@ -52,6 +52,14 @@ describe('scorer', () => { expect(routePrompt(`fix this bug ${'x'.repeat(PROMPT_MAX_CHARS)}`, fixtureSkills)).toMatchObject({ route: null, reason: 'prompt-too-long' }); }); + test('only the four discovery targets route; another installed skill with a matching trigger does not', () => { + expect(ROUTED_SKILLS).toEqual(['investigate', 'review', 'qa', 'verify']); + const ship = { name: 'ship', file: '', frontmatter: '', triggers: ['ship this branch'] }; + expect(scorePrompt('ship this branch', [ship])[0]).toMatchObject({ skill: 'ship', score: 1 }); + expect(routePrompt('ship this branch', [...fixtureSkills, ship])).toMatchObject({ route: null, reason: 'below-threshold' }); + expect(routePrompt('this test fails, ship this branch', [ship, ...fixtureSkills]).route?.skill).toBe('investigate'); + }); + test('malformed triggers are reported and skipped, never thrown', () => { const scan = scanInstalledSkills(skillsDir(['broken', 'qa'], { nofront: 'no frontmatter here\n' })); expect(scan.problems).toEqual(['broken: malformed triggers']);