mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-02 06:54:50 +01:00
test: routing E2E uses the routing instruction gstack ships (W7b routing fixture; skill-routing-e2e.test.ts:113)
readShippedSkillRouting() parses the '## Skill routing' section from the routing-injection heredoc in bin/gstack-skill-start at test time. The journey fixture now carries that instruction instead of 'ALWAYS invoke … FIRST action. Do NOT answer directly', and still withholds the per-skill rule list so a regressed description can fail. bin/gstack-skill-start and the helper join the journey, context-skills and skillify touchfiles. A free check catches heredoc drift. Runs on unchanged templates set the new baseline.
This commit is contained in:
garrytan committed
1 parent
e77cebf718
commit
4e80e41a4c
4 files changed
+85
-40
No files matched your search
@@ -0,0 +1,33 @@
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..', '..');
|
||||
|
||||
export interface ShippedSkillRouting {
|
||||
/** The whole `## Skill routing` section gstack offers to append to CLAUDE.md. */
|
||||
section: string;
|
||||
/** The heading plus the opening instruction, without the per-skill rules. */
|
||||
instruction: string;
|
||||
/** The `- request → invoke /skill` rule lines. */
|
||||
rules: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the routing section from the routing-injection heredoc in
|
||||
* bin/gstack-skill-start, so E2E fixtures use the routing text users receive.
|
||||
*/
|
||||
export function readShippedSkillRouting(root = ROOT): ShippedSkillRouting {
|
||||
const source = fs.readFileSync(path.join(root, 'bin', 'gstack-skill-start'), 'utf8');
|
||||
const heredoc = source.match(/<<'EOI' \| _emit_block routing-injection\n([\s\S]*?)\nEOI\n/)?.[1];
|
||||
if (!heredoc) throw new Error('bin/gstack-skill-start: routing-injection heredoc not found');
|
||||
const start = heredoc.search(/^## Skill routing$/m);
|
||||
const end = heredoc.search(/^If B:/m);
|
||||
if (start < 0 || end < start) throw new Error('bin/gstack-skill-start: ## Skill routing section not found');
|
||||
const section = heredoc.slice(start, end).trim();
|
||||
const rules = section.split('\n').filter(line => /^- .+→ invoke \//.test(line));
|
||||
const instruction = section.split(/\n\n/).slice(0, 2).join('\n\n');
|
||||
if (rules.length === 0 || !/Skill tool/.test(instruction)) {
|
||||
throw new Error('bin/gstack-skill-start: ## Skill routing section has an unexpected shape');
|
||||
}
|
||||
return { section, instruction, rules };
|
||||
}
|
||||
@@ -685,14 +685,14 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
// test/skill-e2e-context-skills.test.ts. These are periodic-tier because
|
||||
// each one spawns claude -p and costs ~$0.20-$0.40. Collectively they
|
||||
// verify the thing the /checkpoint → /context-save rename was for.
|
||||
'context-save-routing': [ 'context-save/**', 'scripts/resolvers/preamble.ts', 'test/skill-e2e-context-skills.test.ts'],
|
||||
'context-save-then-restore-roundtrip': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'context-save/**', 'context-restore/**', 'bin/gstack-slug', 'bin/gstack-remote-identity.sh', 'test/skill-e2e-context-skills.test.ts'],
|
||||
'context-restore-fragment-match': [ 'context-restore/**', 'test/skill-e2e-context-skills.test.ts'],
|
||||
'context-restore-empty-state': [ 'context-restore/**', 'test/skill-e2e-context-skills.test.ts'],
|
||||
'context-restore-list-delegates': [ 'context-restore/**', 'test/skill-e2e-context-skills.test.ts'],
|
||||
'context-restore-legacy-compat': [ 'context-restore/**', 'test/skill-e2e-context-skills.test.ts'],
|
||||
'context-save-list-current-branch': [ 'context-save/**', 'test/skill-e2e-context-skills.test.ts'],
|
||||
'context-save-list-all-branches': [ 'context-save/**', 'test/skill-e2e-context-skills.test.ts'],
|
||||
'context-save-routing': [ 'context-save/**', 'scripts/resolvers/preamble.ts', 'test/skill-e2e-context-skills.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts'],
|
||||
'context-save-then-restore-roundtrip': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'context-save/**', 'context-restore/**', 'bin/gstack-slug', 'bin/gstack-remote-identity.sh', 'test/skill-e2e-context-skills.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts'],
|
||||
'context-restore-fragment-match': [ 'context-restore/**', 'test/skill-e2e-context-skills.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts'],
|
||||
'context-restore-empty-state': [ 'context-restore/**', 'test/skill-e2e-context-skills.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts'],
|
||||
'context-restore-list-delegates': [ 'context-restore/**', 'test/skill-e2e-context-skills.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts'],
|
||||
'context-restore-legacy-compat': [ 'context-restore/**', 'test/skill-e2e-context-skills.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts'],
|
||||
'context-save-list-current-branch': [ 'context-save/**', 'test/skill-e2e-context-skills.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts'],
|
||||
'context-save-list-all-branches': [ 'context-save/**', 'test/skill-e2e-context-skills.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts'],
|
||||
|
||||
// Document-release
|
||||
'document-release': [ 'document-release/**', 'test/skill-e2e-workflow.test.ts', 'test/fixtures/coverage-audit-fixture.ts', 'test/helpers/coverage-audit.ts', 'test/helpers/office-hours-attempt.ts'],
|
||||
@@ -884,24 +884,24 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'scrape-match-path': [
|
||||
'scrape/**', 'browse/src/browser-skills.ts', 'browse/src/browser-skill-commands.ts',
|
||||
'browser-skills/hackernews-frontpage/**',
|
||||
'test/skill-e2e-skillify.test.ts'
|
||||
'test/skill-e2e-skillify.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts'
|
||||
],
|
||||
'scrape-prototype-path': [
|
||||
'scrape/**', 'browse/src/browser-skills.ts', 'browse/src/browser-skill-commands.ts',
|
||||
'test/skill-e2e-skillify.test.ts'
|
||||
'test/skill-e2e-skillify.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts'
|
||||
],
|
||||
'skillify-happy-path': [
|
||||
'skillify/**', 'scrape/**', 'browse/src/browser-skill-write.ts',
|
||||
'browse/src/browser-skills.ts', 'browse/src/browser-skill-commands.ts',
|
||||
'test/skill-e2e-skillify.test.ts'
|
||||
'test/skill-e2e-skillify.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts'
|
||||
],
|
||||
'skillify-provenance-refusal': [
|
||||
'skillify/**', 'browse/src/browser-skill-write.ts',
|
||||
'test/skill-e2e-skillify.test.ts'
|
||||
'test/skill-e2e-skillify.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts'
|
||||
],
|
||||
'skillify-approval-reject': [
|
||||
'skillify/**', 'scrape/**', 'browse/src/browser-skill-write.ts',
|
||||
'test/skill-e2e-skillify.test.ts'
|
||||
'test/skill-e2e-skillify.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts'
|
||||
],
|
||||
|
||||
// Skill routing — journey-stage tests (depend on ALL skill descriptions)
|
||||
@@ -909,7 +909,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
|
||||
|
||||
"test/fixtures/plan-scope-recovery-av.json",
|
||||
"test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts',
|
||||
"test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts',
|
||||
|
||||
|
||||
"scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts"
|
||||
@@ -918,7 +918,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
|
||||
|
||||
"test/fixtures/plan-scope-recovery-av.json",
|
||||
"test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts',
|
||||
"test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts',
|
||||
|
||||
|
||||
"scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts"
|
||||
@@ -927,7 +927,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
|
||||
|
||||
"test/fixtures/plan-scope-recovery-av.json",
|
||||
"test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts',
|
||||
"test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts',
|
||||
|
||||
|
||||
"scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts"
|
||||
@@ -936,7 +936,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
|
||||
|
||||
"test/fixtures/plan-scope-recovery-av.json",
|
||||
"test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts',
|
||||
"test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts',
|
||||
|
||||
|
||||
"scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts"
|
||||
@@ -945,7 +945,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
|
||||
|
||||
"test/fixtures/plan-scope-recovery-av.json",
|
||||
"test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts',
|
||||
"test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts',
|
||||
|
||||
|
||||
"scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts"
|
||||
@@ -954,7 +954,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
|
||||
|
||||
"test/fixtures/plan-scope-recovery-av.json",
|
||||
"test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts',
|
||||
"test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts',
|
||||
|
||||
|
||||
"scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts"
|
||||
@@ -963,7 +963,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
|
||||
|
||||
"test/fixtures/plan-scope-recovery-av.json",
|
||||
"test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts',
|
||||
"test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts',
|
||||
|
||||
|
||||
"scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts"
|
||||
@@ -972,7 +972,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
|
||||
|
||||
"test/fixtures/plan-scope-recovery-av.json",
|
||||
"test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts',
|
||||
"test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts',
|
||||
|
||||
|
||||
"scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts"
|
||||
@@ -981,7 +981,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
|
||||
|
||||
"test/fixtures/plan-scope-recovery-av.json",
|
||||
"test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts',
|
||||
"test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts',
|
||||
|
||||
|
||||
"scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts"
|
||||
@@ -990,7 +990,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
|
||||
|
||||
"test/fixtures/plan-scope-recovery-av.json",
|
||||
"test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts',
|
||||
"test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts',
|
||||
|
||||
|
||||
"scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts"
|
||||
@@ -1100,7 +1100,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
|
||||
|
||||
"test/fixtures/plan-scope-recovery-av.json",
|
||||
"test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts',
|
||||
"test/fixtures/design-scope-checkpoint-at.json", '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts', 'bin/gstack-skill-start', 'test/helpers/shipped-skill-routing.ts',
|
||||
|
||||
|
||||
"scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts"
|
||||
|
||||
@@ -28,6 +28,7 @@ import {
|
||||
CODEX_REVIEW_E2E_SECTIONS,
|
||||
} from './helpers/skill-fixture';
|
||||
import { E2E_TOUCHFILES, GLOBAL_TOUCHFILES, selectTests } from './helpers/touchfiles';
|
||||
import { readShippedSkillRouting } from './helpers/shipped-skill-routing';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
|
||||
@@ -449,10 +450,12 @@ test('routing catalog contains installed project names without a request-to-skil
|
||||
writeFileSync: (file: string, content: string) => writes.set(file, content),
|
||||
};
|
||||
const compiled = new Bun.Transpiler({ loader: 'ts' }).transformSync(source.slice(start, end) + '\ninstallSkills(fixtureRoot);');
|
||||
new Function('ROOT', 'fs', 'path', 'extractSkillHead', 'fixtureRoot', compiled)(
|
||||
ROOT, mockFs, path, (file: string) => installed.get(file), fixtureRoot,
|
||||
new Function('ROOT', 'fs', 'path', 'extractSkillHead', 'readShippedSkillRouting', 'fixtureRoot', compiled)(
|
||||
ROOT, mockFs, path, (file: string) => installed.get(file), readShippedSkillRouting, fixtureRoot,
|
||||
);
|
||||
const instructions = writes.get(path.join(fixtureRoot, 'CLAUDE.md'))!;
|
||||
expect(instructions).toContain(readShippedSkillRouting().instruction);
|
||||
expect(instructions).not.toContain('→ invoke');
|
||||
expect(instructions).toContain('installed gstack skills: gstack, qa, review.');
|
||||
expect(instructions).toContain("built-in skills are outside this project's workflow");
|
||||
expect(instructions).toContain('matching the request to the skill descriptions');
|
||||
@@ -464,3 +467,18 @@ test('routing catalog contains installed project names without a request-to-skil
|
||||
.toBe(installed.get(path.join(ROOT, 'review/SKILL.md')));
|
||||
expect(writes.size).toBe(4);
|
||||
});
|
||||
|
||||
test('routing fixtures read the ## Skill routing section gstack ships', () => {
|
||||
const routing = readShippedSkillRouting();
|
||||
expect(routing.instruction).toStartWith('## Skill routing\n\n');
|
||||
expect(routing.instruction).toContain('Skill tool');
|
||||
expect(routing.rules.some(rule => rule.includes('/context-save'))).toBe(true);
|
||||
expect(routing.rules.some(rule => rule.includes('/context-restore'))).toBe(true);
|
||||
expect(routing.section).not.toContain('If B:');
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'shipped-routing-drift-'));
|
||||
try {
|
||||
fs.mkdirSync(path.join(root, 'bin'));
|
||||
fs.writeFileSync(path.join(root, 'bin', 'gstack-skill-start'), '#!/bin/bash\necho no routing here\n');
|
||||
expect(() => readShippedSkillRouting(root)).toThrow('routing-injection heredoc not found');
|
||||
} finally { fs.rmSync(root, { recursive: true, force: true }); }
|
||||
});
|
||||
@@ -6,6 +6,7 @@ import { EvalCollector } from './helpers/eval-store';
|
||||
import type { EvalTestEntry } from './helpers/eval-store';
|
||||
import { selectTests, detectBaseBranch, getChangedFiles, E2E_TOUCHFILES, E2E_TIERS, GLOBAL_TOUCHFILES } from './helpers/touchfiles';
|
||||
import { extractSkillHead } from './helpers/skill-fixture';
|
||||
import { readShippedSkillRouting } from './helpers/shipped-skill-routing';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
@@ -94,26 +95,19 @@ function installSkills(tmpDir: string) {
|
||||
|
||||
// The names-only catalog keeps new CLI built-ins from changing the candidate
|
||||
// set. Descriptions still choose the skill; no request-to-skill answer key.
|
||||
// Write a CLAUDE.md with a GENERIC invoke-skills nudge — deliberately NO
|
||||
// per-skill routing table. These journey tests exist to catch skill
|
||||
// DESCRIPTION regressions (their touchfiles key on */SKILL.md.tmpl), and
|
||||
// the old fixture shipped an explicit prompt→skill answer key: with the
|
||||
// lookup table in context, a badly regressed frontmatter description
|
||||
// still routed correctly and the tests could not fail on the regression
|
||||
// class they select for (2026-08 audit). The generic nudge keeps Claude's
|
||||
// reach-for-a-skill posture; the FRONTMATTER carries the routing load.
|
||||
// The routing instruction is the one gstack ships (bin/gstack-skill-start),
|
||||
// without its per-skill rule list. These journey tests exist to catch skill
|
||||
// DESCRIPTION regressions (their touchfiles key on */SKILL.md.tmpl): with a
|
||||
// lookup table in context, a badly regressed frontmatter description still
|
||||
// routes correctly and the tests cannot fail on the regression class they
|
||||
// select for. The FRONTMATTER carries the routing load.
|
||||
fs.writeFileSync(path.join(tmpDir, 'CLAUDE.md'), `# Project Instructions
|
||||
|
||||
## Skill routing
|
||||
${readShippedSkillRouting().instruction}
|
||||
|
||||
This project uses the following installed gstack skills: ${installedSkills.join(', ')}.
|
||||
Choose among this project catalog by matching the request to the skill descriptions.
|
||||
The CLI's built-in skills are outside this project's workflow.
|
||||
|
||||
When the user's request matches an available project skill, ALWAYS invoke it using the Skill
|
||||
tool as your FIRST action. Do NOT answer directly, do NOT use other tools first.
|
||||
The skill has specialized workflows that produce better results than ad-hoc answers.
|
||||
Choose the skill by matching the request against each skill's description.
|
||||
`);
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user