Files
gstack/test/failure-routing-arm.test.ts
T
Garry Tanandcapy-ai[bot] ef99eb5a55 feat: retire /investigate after a no-gain eval (GST-1717) (#4110)
* feat: retire /investigate after a no-gain eval (GST-1717)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(skill-start): core routing offer drops every non-core rule line (GST-1717)

The retired /investigate rule became 'Bugs/errors → fix them directly', which no
longer matches '→ invoke /', so proactive=core leaked it into the offered section.
Match any '- … → ' rule line.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* revert(review): keep review/checklist.md at its no-loss-pinned bytes (GST-1717)

The pin names replay GSTA-224 H1; a reviewer checklist edit merges only after a
passing no-loss replay (crew rule 15). The /investigate flake line stays until a
replayed repin.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* test: repair free-suite fallout from retiring /investigate (GST-1717)

CI run 42340956269 at 83a8a9a: drop fixtures, ledger rows, allowlist entries and
counts for the retired skill and its deleted paid files; record the harness
change as non-behavioral; the test-of-test detector accepts path.join(ROOT, 'bin');
uncovered-prompt fixtures use a tracked prompt, since an untracked path now
reads as a deleted one.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(relink): back up a retired skill's real SKILL.md before cleaning it (GST-1717)

With the skill's source gone, the customized-file check never ran, so a
banner-carrying local copy of investigate/SKILL.md was deleted instead of moved
to the backup root (red-team review, pass 1).

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(qm): restore the review checklist's flake-line rewrite (GST-1717)

review/checklist.md keeps its pinned /investigate line, so the QM edition needs
its replaceText rule back; qm/ stays fresh against the generator.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* docs(changelog): name the setup flag break, the relink backup and the pinned checklist line (GST-1717)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(setup,relink): back up a retired skill's real SKILL.md instead of deleting it (GST-1717)

Scoped fix round under grant a-360dd62bd46a (outside-voice blocking finding and
adversarial #1, #4). setup's prune compares a copied host entry with the render
before removing that render, and backs up a copy that differs or whose render is
gone; relink no longer takes the marker's rm -rf for a skill whose source is
gone, so a real SKILL.md is backed up and the user's files stay. setup reports a
failed remove-source of the retired gate hooks on stderr.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(pr-profile): a changed prompt counts as deleted only when it is gone from disk (GST-1717)

An untracked prompt file on disk still needs prompt validation (targeted review
finding); harness change recorded as non-behavioral.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* docs(changelog): exact backup wording, the hook-removal warning and GST-1744 (GST-1717)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(setup): one backup folder per host for a retired skill's copies (GST-1717)

Under grant a-c95576b8b1f9 (fix-check#1): every copy host moved its
gstack-investigate/SKILL.md to the same backup path, so a later host's copy
overwrote an earlier host's edits. A sandboxed ./setup --host factory run showed
gen-skill-docs removes the retired render before the prune, so every real copy
is backed up; each host now gets its own folder.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* docs(changelog): say exactly what setup backs up for a retired skill (GST-1717)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* docs(changelog): end the two colon sentences prose-lint flagged (GST-1717)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* test(retired-skills): name plan-eng-review once behind a pregate-literal waiver (GST-1717)

pregate's literals check read the GSTACK_RETIRED_SKILLS change as an old
value still in the tests. plan-eng-review is still retired; investigate was
appended. The fixture names it once, with the waiver.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

---------

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
2026-11-07 06:52:04 +00:00

103 lines
5.8 KiB
TypeScript

/**
* GST-470: the router failure-routing eval's arms. GSTACK_ROUTER_ARM is
* current (the default), before or none, and anything else throws. The none arm
* builds a repository with no installed skills and a prompt that is the user's
* request alone. The current and before arms' prompts and repositories are byte
* for byte what main built (the reference copies below, from
* test/skill-e2e-failure-routing.test.ts at 0cd2b4a).
*/
import { afterAll, describe, expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { spawnSync } from 'node:child_process';
import { FIXTURES, makeRepo, plainMetrics, routerArm, routerPrompt, type Fixture } from './helpers/failure-routing-fixture';
import { extractSkillHead, sliceBetween } from './helpers/skill-fixture';
const ROOT = path.resolve(import.meta.dir, '..');
const BASE = 'HEAD';
const dirs: string[] = [];
afterAll(() => { for (const dir of dirs) fs.rmSync(dir, { recursive: true, force: true }); });
function mainRouterSection(arm: 'before' | 'current'): string {
const skill = arm === 'before'
? spawnSync('git', ['show', `${BASE}:SKILL.md`], { cwd: ROOT, encoding: 'utf8', timeout: 10_000 }).stdout
: fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf8');
return sliceBetween(skill, '## Route first', 'Best-effort, record') + sliceBetween(skill, '**Routing rules', 'If the user opts out');
}
function mainPrompt(fixture: Fixture, arm: 'before' | 'current'): string {
return `The user invoked the gstack router skill (/gstack) in this repository. The router's instructions:\n\n${mainRouterSection(arm)}\n\nThe user's request: "${fixture.request}"`;
}
function mainMakeRepo(fixture: Fixture): string {
const repo = fs.mkdtempSync(path.join(os.tmpdir(), 'failure-routing-'));
fs.writeFileSync(path.join(repo, 'package.json'), JSON.stringify({ name: 'fixture', type: 'module', scripts: { test: 'bun test' } }, null, 2));
for (const [file, body] of Object.entries(fixture.files)) {
fs.mkdirSync(path.dirname(path.join(repo, file)), { recursive: true });
fs.writeFileSync(path.join(repo, file), body);
}
for (const skill of ['qa', 'review', 'ship']) {
fs.mkdirSync(path.join(repo, '.claude', 'skills', skill), { recursive: true });
fs.writeFileSync(path.join(repo, '.claude', 'skills', skill, 'SKILL.md'), extractSkillHead(path.join(ROOT, skill, 'SKILL.md')));
}
const git = (...args: string[]) => spawnSync('git', ['-c', 'user.email=test@example.invalid', '-c', 'user.name=Test', ...args], { cwd: repo, timeout: 10_000 });
git('init', '-q', '-b', 'main');
git('add', '.');
git('commit', '-qm', 'initial');
return repo;
}
/** Every file under dir except .git, path → contents, plus the committed file list. */
function tree(dir: string): { files: Record<string, string>; committed: string } {
dirs.push(dir);
const files: Record<string, string> = {};
for (const rel of fs.readdirSync(dir, { recursive: true, encoding: 'utf8' }).sort()) {
if (rel === '.git' || rel.startsWith('.git' + path.sep)) continue;
const abs = path.join(dir, rel);
if (fs.statSync(abs).isFile()) files[rel] = fs.readFileSync(abs, 'utf8');
}
const committed = spawnSync('git', ['ls-files'], { cwd: dir, encoding: 'utf8', timeout: 10_000 }).stdout;
return { files, committed };
}
describe('router arms (GST-470)', () => {
test('GSTACK_ROUTER_ARM: unset is current; before, current and none parse; anything else throws', () => {
expect(routerArm({})).toBe('current');
for (const arm of ['before', 'current', 'none'] as const) expect(routerArm({ GSTACK_ROUTER_ARM: arm })).toBe(arm);
for (const bad of ['nonee', 'Current', 'after', '']) {
expect(() => routerArm({ GSTACK_ROUTER_ARM: bad })).toThrow('must be before, current or none, got ' + bad);
}
});
test('none: the repository has no installed skills and the prompt is the request alone', () => {
for (const fixture of FIXTURES) {
const repo = tree(makeRepo(fixture, 'none'));
const reference = tree(mainMakeRepo(fixture));
expect(Object.keys(repo.files).some(f => f.startsWith('.claude'))).toBe(false);
const withoutSkills = Object.fromEntries(Object.entries(reference.files).filter(([f]) => !f.startsWith('.claude')));
expect(repo.files).toEqual(withoutSkills);
const prompt = routerPrompt(fixture, 'none');
expect(prompt).toBe(fixture.request);
for (const word of ['## Route first', '**Routing rules', 'gstack', 'router', 'skill', '/investigate']) expect(prompt).not.toContain(word);
}
});
test('current and before: prompts and repositories are byte-identical to main', () => {
for (const fixture of FIXTURES) {
for (const arm of ['current', 'before'] as const) expect(routerPrompt(fixture, arm, BASE)).toBe(mainPrompt(fixture, arm));
for (const arm of ['current', 'before'] as const) expect(tree(makeRepo(fixture, arm))).toEqual(tree(mainMakeRepo(fixture)));
}
});
test('plain metrics count read-only calls before the first edit', () => {
const call = (tool: string, input: Record<string, unknown> = {}) => ({ tool, input, output: '' });
expect(plainMetrics([])).toMatchObject({ editedBeforeRead: false, readsBeforeEdit: 0 });
expect(plainMetrics([call('Read'), call('Grep'), call('Edit'), call('Read')])).toMatchObject({ editedBeforeRead: false, readsBeforeEdit: 2 });
expect(plainMetrics([call('Edit'), call('Read')])).toMatchObject({ editedBeforeRead: true, readsBeforeEdit: 0 });
expect(plainMetrics([call('Bash', { command: "sed -i 's/ /-/' src/slug.js" }), call('Read')])).toMatchObject({ editedBeforeRead: true, readsBeforeEdit: 0 });
expect(plainMetrics([call('Bash', { command: 'bun test' }), call('Skill', { skill: 'qa' }), call('Write')]))
.toMatchObject({ editedBeforeRead: false, readsBeforeEdit: 1, sequence: 'Bash → Skill(qa) → Write' });
});
});