mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-18 22:24:17 +01:00
Stop giving bug reports a special rule in gstack's router (GST-2873) (#4442)
gstack's router no longer gives bug reports a special check-the-cause, fix-it-test-first rule, and its description no longer lists debugging, so the model fixes a bug the way it would without gstack. This keeps gstack to the parts that measurably help, and on 26 hard real bugs the rule gave no clear gain. Nothing to do. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> Review-Finding: p1/red-team#1 reader=red-team
This commit is contained in:
1 parent
b79843c5d7
commit
f000b6e0e3
19 files changed
+25
-368
No files matched your search
@@ -128,13 +128,13 @@ describe('per-case host metadata', () => {
|
||||
for (const id of Object.keys(E2E_HOSTS)) expect(Object.hasOwn(E2E_TOUCHFILES, id), id).toBe(true);
|
||||
});
|
||||
|
||||
test('classifies 52 adapter-only, 29 Claude-specific, 26 rework and 1 host-neutral case', () => {
|
||||
test('classifies 52 adapter-only, 27 Claude-specific, 26 rework and 1 host-neutral case', () => {
|
||||
const counts: Record<string, number> = {};
|
||||
for (const support of Object.values(E2E_HOSTS)) counts[support] = (counts[support] ?? 0) + 1;
|
||||
expect(counts.both).toBe(52);
|
||||
expect(counts.rework).toBe(26);
|
||||
expect(counts['host-neutral']).toBe(1);
|
||||
expect(['pty', 'sdk', 'subagents', 'skill-tool', 'hermetic', 'self-host'].reduce((sum, reason) => sum + (counts[reason] ?? 0), 0)).toBe(29);
|
||||
expect(['pty', 'sdk', 'subagents', 'skill-tool', 'hermetic', 'self-host'].reduce((sum, reason) => sum + (counts[reason] ?? 0), 0)).toBe(27);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -142,7 +142,7 @@ test('live periodic census fits the declared CI wall including setup', () => {
|
||||
|
||||
test('registered allocation is deterministic and preserves every discovered file', () => {
|
||||
const files = collectPaidTestFiles();
|
||||
expect(files).toHaveLength(108); // - skill-e2e-investigate-owned-completion, -owned-termination and codex-e2e-sol-scope (/investigate retired, GST-1717); - skill-e2e-plan-eng-plan-mode, -plan-eng-finding-floor and -plan-eng-artifact-full (standalone /plan-eng-review retired, GST-1698); - skill-llm-eval (last standalone judges removed, GST-1403); + skill-e2e-autoplan-ceo-only (GST-537); + skill-e2e-planning-wave (GSTA-228); + skill-e2e-model-panel (GSTA-173); + skill-e2e-write-skill (GSTA-133); + skill-e2e-ship-frozen-scope (frozen review scope, GSTA-113 PR 2); + skill-e2e-failure-routing (#3157/#3158); + skill-e2e-dx-error-contract (#3226); + skill-e2e-gate-integrity (HONEST_WORK_GATE_INTEGRITY W8); + skill-e2e-autoplan-schema-canary, -guard-pty and -long-session (Oct 7 wave, CEO-13, ENG-18, CEO-15); + skill-e2e-ship-measure-seeded-flake (v1.91.32.0); W5a: 15 carve wrappers became one case-sharded file; + codex-e2e-multiblock-live and skill-e2e-design-model-smoke (v1.91.30.0);
|
||||
expect(files).toHaveLength(107); // - skill-e2e-failure-routing (router bug-fix route removed, GST-2873); - skill-e2e-investigate-owned-completion, -owned-termination and codex-e2e-sol-scope (/investigate retired, GST-1717); - skill-e2e-plan-eng-plan-mode, -plan-eng-finding-floor and -plan-eng-artifact-full (standalone /plan-eng-review retired, GST-1698); - skill-llm-eval (last standalone judges removed, GST-1403); + skill-e2e-autoplan-ceo-only (GST-537); + skill-e2e-planning-wave (GSTA-228); + skill-e2e-model-panel (GSTA-173); + skill-e2e-write-skill (GSTA-133); + skill-e2e-ship-frozen-scope (frozen review scope, GSTA-113 PR 2); + skill-e2e-failure-routing (#3157/#3158); + skill-e2e-dx-error-contract (#3226); + skill-e2e-gate-integrity (HONEST_WORK_GATE_INTEGRITY W8); + skill-e2e-autoplan-schema-canary, -guard-pty and -long-session (Oct 7 wave, CEO-13, ENG-18, CEO-15); + skill-e2e-ship-measure-seeded-flake (v1.91.32.0); W5a: 15 carve wrappers became one case-sharded file; + codex-e2e-multiblock-live and skill-e2e-design-model-smoke (v1.91.30.0);
|
||||
expect(files).toContain('test/skill-e2e-ship-skip.test.ts');
|
||||
const m = livePlan(files);
|
||||
expect(livePlan([...files].reverse())).toEqual(m);
|
||||
|
||||
@@ -1,102 +0,0 @@
|
||||
/**
|
||||
* GST-470: the router failure-routing eval's arms. GSTACK_ROUTER_ARM is
|
||||
* current (the default), before or none, and anything else throws. The none arm
|
||||
* builds a repository with no installed skills and a prompt that is the user's
|
||||
* request alone. The current and before arms' prompts and repositories are byte
|
||||
* for byte what main built (the reference copies below, from
|
||||
* test/skill-e2e-failure-routing.test.ts at 0cd2b4a).
|
||||
*/
|
||||
import { afterAll, describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { FIXTURES, makeRepo, plainMetrics, routerArm, routerPrompt, type Fixture } from './helpers/failure-routing-fixture';
|
||||
import { extractSkillHead, sliceBetween } from './helpers/skill-fixture';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const BASE = 'HEAD';
|
||||
const dirs: string[] = [];
|
||||
afterAll(() => { for (const dir of dirs) fs.rmSync(dir, { recursive: true, force: true }); });
|
||||
|
||||
function mainRouterSection(arm: 'before' | 'current'): string {
|
||||
const skill = arm === 'before'
|
||||
? spawnSync('git', ['show', `${BASE}:SKILL.md`], { cwd: ROOT, encoding: 'utf8', timeout: 10_000 }).stdout
|
||||
: fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf8');
|
||||
return sliceBetween(skill, '## Route first', 'Best-effort, record') + sliceBetween(skill, '**Routing rules', 'If the user opts out');
|
||||
}
|
||||
|
||||
function mainPrompt(fixture: Fixture, arm: 'before' | 'current'): string {
|
||||
return `The user invoked the gstack router skill (/gstack) in this repository. The router's instructions:\n\n${mainRouterSection(arm)}\n\nThe user's request: "${fixture.request}"`;
|
||||
}
|
||||
|
||||
function mainMakeRepo(fixture: Fixture): string {
|
||||
const repo = fs.mkdtempSync(path.join(os.tmpdir(), 'failure-routing-'));
|
||||
fs.writeFileSync(path.join(repo, 'package.json'), JSON.stringify({ name: 'fixture', type: 'module', scripts: { test: 'bun test' } }, null, 2));
|
||||
for (const [file, body] of Object.entries(fixture.files)) {
|
||||
fs.mkdirSync(path.dirname(path.join(repo, file)), { recursive: true });
|
||||
fs.writeFileSync(path.join(repo, file), body);
|
||||
}
|
||||
for (const skill of ['qa', 'review', 'ship']) {
|
||||
fs.mkdirSync(path.join(repo, '.claude', 'skills', skill), { recursive: true });
|
||||
fs.writeFileSync(path.join(repo, '.claude', 'skills', skill, 'SKILL.md'), extractSkillHead(path.join(ROOT, skill, 'SKILL.md')));
|
||||
}
|
||||
const git = (...args: string[]) => spawnSync('git', ['-c', 'user.email=test@example.invalid', '-c', 'user.name=Test', ...args], { cwd: repo, timeout: 10_000 });
|
||||
git('init', '-q', '-b', 'main');
|
||||
git('add', '.');
|
||||
git('commit', '-qm', 'initial');
|
||||
return repo;
|
||||
}
|
||||
|
||||
/** Every file under dir except .git, path → contents, plus the committed file list. */
|
||||
function tree(dir: string): { files: Record<string, string>; committed: string } {
|
||||
dirs.push(dir);
|
||||
const files: Record<string, string> = {};
|
||||
for (const rel of fs.readdirSync(dir, { recursive: true, encoding: 'utf8' }).sort()) {
|
||||
if (rel === '.git' || rel.startsWith('.git' + path.sep)) continue;
|
||||
const abs = path.join(dir, rel);
|
||||
if (fs.statSync(abs).isFile()) files[rel] = fs.readFileSync(abs, 'utf8');
|
||||
}
|
||||
const committed = spawnSync('git', ['ls-files'], { cwd: dir, encoding: 'utf8', timeout: 10_000 }).stdout;
|
||||
return { files, committed };
|
||||
}
|
||||
|
||||
describe('router arms (GST-470)', () => {
|
||||
test('GSTACK_ROUTER_ARM: unset is current; before, current and none parse; anything else throws', () => {
|
||||
expect(routerArm({})).toBe('current');
|
||||
for (const arm of ['before', 'current', 'none'] as const) expect(routerArm({ GSTACK_ROUTER_ARM: arm })).toBe(arm);
|
||||
for (const bad of ['nonee', 'Current', 'after', '']) {
|
||||
expect(() => routerArm({ GSTACK_ROUTER_ARM: bad })).toThrow('must be before, current or none, got ' + bad);
|
||||
}
|
||||
});
|
||||
|
||||
test('none: the repository has no installed skills and the prompt is the request alone', () => {
|
||||
for (const fixture of FIXTURES) {
|
||||
const repo = tree(makeRepo(fixture, 'none'));
|
||||
const reference = tree(mainMakeRepo(fixture));
|
||||
expect(Object.keys(repo.files).some(f => f.startsWith('.claude'))).toBe(false);
|
||||
const withoutSkills = Object.fromEntries(Object.entries(reference.files).filter(([f]) => !f.startsWith('.claude')));
|
||||
expect(repo.files).toEqual(withoutSkills);
|
||||
const prompt = routerPrompt(fixture, 'none');
|
||||
expect(prompt).toBe(fixture.request);
|
||||
for (const word of ['## Route first', '**Routing rules', 'gstack', 'router', 'skill', '/investigate']) expect(prompt).not.toContain(word);
|
||||
}
|
||||
});
|
||||
|
||||
test('current and before: prompts and repositories are byte-identical to main', () => {
|
||||
for (const fixture of FIXTURES) {
|
||||
for (const arm of ['current', 'before'] as const) expect(routerPrompt(fixture, arm, BASE)).toBe(mainPrompt(fixture, arm));
|
||||
for (const arm of ['current', 'before'] as const) expect(tree(makeRepo(fixture, arm))).toEqual(tree(mainMakeRepo(fixture)));
|
||||
}
|
||||
});
|
||||
|
||||
test('plain metrics count read-only calls before the first edit', () => {
|
||||
const call = (tool: string, input: Record<string, unknown> = {}) => ({ tool, input, output: '' });
|
||||
expect(plainMetrics([])).toMatchObject({ editedBeforeRead: false, readsBeforeEdit: 0 });
|
||||
expect(plainMetrics([call('Read'), call('Grep'), call('Edit'), call('Read')])).toMatchObject({ editedBeforeRead: false, readsBeforeEdit: 2 });
|
||||
expect(plainMetrics([call('Edit'), call('Read')])).toMatchObject({ editedBeforeRead: true, readsBeforeEdit: 0 });
|
||||
expect(plainMetrics([call('Bash', { command: "sed -i 's/ /-/' src/slug.js" }), call('Read')])).toMatchObject({ editedBeforeRead: true, readsBeforeEdit: 0 });
|
||||
expect(plainMetrics([call('Bash', { command: 'bun test' }), call('Skill', { skill: 'qa' }), call('Write')]))
|
||||
.toMatchObject({ editedBeforeRead: false, readsBeforeEdit: 1, sequence: 'Bash → Skill(qa) → Write' });
|
||||
});
|
||||
});
|
||||
@@ -234,7 +234,7 @@ describe('GSTA-135: registered skills get used', () => {
|
||||
|
||||
test('router lines lead with each skill\'s trigger, within 160 chars; the router description leads with task verbs', () => {
|
||||
expect(ROUTER_DESCRIPTION.length).toBeLessThanOrEqual(ROUTER_DESCRIPTION_MAX);
|
||||
for (const verb of ['review', 'debug', 'test', 'plan', 'ship']) expect(ROUTER_DESCRIPTION.slice(0, 200), verb).toContain(verb);
|
||||
for (const verb of ['review', 'test', 'plan', 'ship']) expect(ROUTER_DESCRIPTION.slice(0, 200), verb).toContain(verb);
|
||||
const d = tmp();
|
||||
for (const [target, mode] of [['openclaw', 'router'], ['dots', 'router']] as const) {
|
||||
const r = register(opts(d, target, { mode, skillsDir: path.join(d, target) }));
|
||||
|
||||
@@ -98,8 +98,7 @@ const CLAUDE_ONLY: Readonly<Record<ClaudeOnlyReason, readonly string[]>> = {
|
||||
'shared-libs-review-prior-coverage', 'shared-libs-review-revalidation', 'ship-frozen-scope-convergence', 'ship-local-hook-preservation',
|
||||
'ship-managed-hook-refresh', 'ship-skipped-queued-finding', 'ship-unmanaged-hook-consent'],
|
||||
subagents: ['docsync-spawned', 'review-army-perf-n-plus-one'],
|
||||
'skill-tool': ['router-failure-triage-resolves',
|
||||
'router-failure-verified-cause', 'skillify-approval-reject', 'skillify-happy-path', 'skillify-provenance-refusal'],
|
||||
'skill-tool': ['skillify-approval-reject', 'skillify-happy-path', 'skillify-provenance-refusal'],
|
||||
hermetic: ['hermetic-canary', 'hermetic-sentinel'],
|
||||
'self-host': ['safety-codex-consult-embed'],
|
||||
rework: ['qa-functional-cli-fix', 'qa-functional-cli-report', 'qa-functional-webhook-fix', 'qa-functional-webhook-report',
|
||||
|
||||
@@ -1,115 +0,0 @@
|
||||
/**
|
||||
* Router failure-routing fixtures and scoring (#3157, #3158), shared by
|
||||
* test/skill-e2e-failure-routing.test.ts and its free arm test (GST-470).
|
||||
* GSTACK_ROUTER_ARM picks what the candidate sees: `current` (the default) the
|
||||
* rendered router section of SKILL.md, `before` the base branch's router, and
|
||||
* `none` the user's request alone in a repository with no installed skills.
|
||||
* Any other value throws, so a typo cannot run one arm under another's label.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import type { SkillTestResult } from './session-runner';
|
||||
import { extractSkillHead, sliceBetween } from './skill-fixture';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..', '..');
|
||||
|
||||
export type RouterArm = 'before' | 'current' | 'none';
|
||||
|
||||
export function routerArm(env: NodeJS.ProcessEnv = process.env): RouterArm {
|
||||
const arm = env.GSTACK_ROUTER_ARM ?? 'current';
|
||||
if (arm !== 'before' && arm !== 'current' && arm !== 'none') throw new Error(`GSTACK_ROUTER_ARM must be before, current or none, got ${arm}`);
|
||||
return arm;
|
||||
}
|
||||
|
||||
const EDIT_TOOLS = new Set(['Edit', 'Write', 'MultiEdit', 'NotebookEdit']);
|
||||
const READ_TOOLS = new Set(['Read', 'Grep', 'Glob', 'Bash']);
|
||||
const SHELL_WRITE = /\b(?:sed|perl)\b[^|;&]*\s-[a-z]*i|(?<![=0-9&>])>>?\s*(?!&|\/dev\/null)\S|\btee\b|\b(?:mv|cp|rm)\s|\bgit\s+(?:apply|checkout|restore)\b/;
|
||||
|
||||
export function isEdit(call: { tool: string; input: any }): boolean {
|
||||
return EDIT_TOOLS.has(call.tool) || (call.tool === 'Bash' && SHELL_WRITE.test(String(call.input?.command ?? '')));
|
||||
}
|
||||
|
||||
export interface Fixture {
|
||||
name: string;
|
||||
files: Record<string, string>;
|
||||
request: string;
|
||||
}
|
||||
|
||||
export const FIXTURES: Fixture[] = [
|
||||
{
|
||||
name: 'router-failure-verified-cause',
|
||||
files: {
|
||||
'src/slug.js': "export function slugify(title) {\n return title.trim().toLowerCase().replace(/ /, '-');\n}\n",
|
||||
'test/slug.test.js': "import { expect, test } from 'bun:test';\nimport { slugify } from '../src/slug.js';\n\ntest('slugify joins every word with a hyphen', () => {\n expect(slugify('Hello Big World')).toBe('hello-big-world');\n});\n",
|
||||
},
|
||||
request: "test/slug.test.js fails: slugify('Hello Big World') returns 'hello-big world'. The cause is the regex in src/slug.js: / / has no g flag, so only the first space is replaced. Fix it by making it / /g.",
|
||||
},
|
||||
{
|
||||
name: 'router-failure-triage-resolves',
|
||||
files: {
|
||||
'src/initials.js': "export function initials(fullName) {\n const [first, last] = fullName.split(',');\n return (first[0] + last[0]).toUpperCase();\n}\n",
|
||||
'test/initials.test.js': "import { expect, test } from 'bun:test';\nimport { initials } from '../src/initials.js';\n\ntest('initials of a first and last name', () => {\n expect(initials('Ada Lovelace')).toBe('AL');\n});\n",
|
||||
},
|
||||
request: "bun test fails on test/initials.test.js with: TypeError: undefined is not an object (evaluating 'last[0]') at initials (src/initials.js:3). Can you fix it?",
|
||||
},
|
||||
];
|
||||
|
||||
export function routerSection(arm: Exclude<RouterArm, 'none'>, base = process.env.EVALS_BASE || 'origin/main'): string {
|
||||
const skill = arm === 'before'
|
||||
? spawnSync('git', ['show', `${base}:SKILL.md`], { cwd: ROOT, encoding: 'utf8', timeout: 10_000 }).stdout
|
||||
: fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf8');
|
||||
return sliceBetween(skill, '## Route first', 'Best-effort, record') + sliceBetween(skill, '**Routing rules', 'If the user opts out');
|
||||
}
|
||||
|
||||
/** The candidate's prompt: the router section and the request, or under `none` the request alone. */
|
||||
export function routerPrompt(fixture: Fixture, arm: RouterArm, base?: string): string {
|
||||
if (arm === 'none') return fixture.request;
|
||||
return `The user invoked the gstack router skill (/gstack) in this repository. The router's instructions:\n\n${routerSection(arm, base)}\n\nThe user's request: "${fixture.request}"`;
|
||||
}
|
||||
|
||||
/** The fixture repository; under `none` it has no installed skills. */
|
||||
export function makeRepo(fixture: Fixture, arm: RouterArm): string {
|
||||
const repo = fs.mkdtempSync(path.join(os.tmpdir(), 'failure-routing-'));
|
||||
fs.writeFileSync(path.join(repo, 'package.json'), JSON.stringify({ name: 'fixture', type: 'module', scripts: { test: 'bun test' } }, null, 2));
|
||||
for (const [file, body] of Object.entries(fixture.files)) {
|
||||
fs.mkdirSync(path.dirname(path.join(repo, file)), { recursive: true });
|
||||
fs.writeFileSync(path.join(repo, file), body);
|
||||
}
|
||||
for (const skill of arm === 'none' ? [] : ['qa', 'review', 'ship']) {
|
||||
fs.mkdirSync(path.join(repo, '.claude', 'skills', skill), { recursive: true });
|
||||
fs.writeFileSync(path.join(repo, '.claude', 'skills', skill, 'SKILL.md'), extractSkillHead(path.join(ROOT, skill, 'SKILL.md')));
|
||||
}
|
||||
const git = (...args: string[]) => spawnSync('git', ['-c', 'user.email=test@example.invalid', '-c', 'user.name=Test', ...args], { cwd: repo, timeout: 10_000 });
|
||||
git('init', '-q', '-b', 'main');
|
||||
git('add', '.');
|
||||
git('commit', '-qm', 'initial');
|
||||
return repo;
|
||||
}
|
||||
|
||||
export function routing(result: SkillTestResult) {
|
||||
const calls = result.toolCalls;
|
||||
const firstEditAt = calls.findIndex(isEdit);
|
||||
const before = firstEditAt >= 0 ? calls.slice(0, firstEditAt) : calls;
|
||||
return {
|
||||
skillRouted: calls.some(c => c.tool === 'Skill'),
|
||||
readsBeforeRoute: before.filter(c => READ_TOOLS.has(c.tool) && !isEdit(c)).length,
|
||||
sequence: calls.map(c => c.tool === 'Skill' ? `Skill(${c.input?.skill})` : c.tool).join(' → '),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The no-gstack arm's plain metrics: no skill exists to route to, so they say
|
||||
* only whether the agent read before it edited.
|
||||
*/
|
||||
export function plainMetrics(calls: SkillTestResult['toolCalls']) {
|
||||
const firstEditAt = calls.findIndex(isEdit);
|
||||
const firstReadAt = calls.findIndex(c => READ_TOOLS.has(c.tool) && !isEdit(c));
|
||||
const before = firstEditAt >= 0 ? calls.slice(0, firstEditAt) : calls;
|
||||
return {
|
||||
editedBeforeRead: firstEditAt >= 0 && (firstReadAt < 0 || firstEditAt < firstReadAt),
|
||||
readsBeforeEdit: before.filter(c => READ_TOOLS.has(c.tool) && !isEdit(c)).length,
|
||||
sequence: calls.map(c => c.tool === 'Skill' ? `Skill(${c.input?.skill})` : c.tool).join(' → '),
|
||||
};
|
||||
}
|
||||
@@ -1166,8 +1166,6 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
|
||||
"scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts"
|
||||
],
|
||||
'router-failure-verified-cause': ['SKILL.md.tmpl', 'SKILL.md', 'scripts/resolvers/preamble/generate-test-failure-triage.ts', 'lib/bun-isolation.ts', 'test/skill-e2e-failure-routing.test.ts', 'test/helpers/failure-routing-fixture.ts', 'test/helpers/none-arm.ts'],
|
||||
'router-failure-triage-resolves': ['SKILL.md.tmpl', 'SKILL.md', 'scripts/resolvers/preamble/generate-test-failure-triage.ts', 'lib/bun-isolation.ts', 'test/skill-e2e-failure-routing.test.ts', 'test/helpers/failure-routing-fixture.ts', 'test/helpers/none-arm.ts'],
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -1573,8 +1571,6 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic' | 'marathon'> = {
|
||||
'overlay-harness-claude-dedicated-tools-vs-bash-sonnet': 'periodic',
|
||||
'overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5': 'periodic',
|
||||
'journey-negatives': 'periodic',
|
||||
'router-failure-verified-cause': 'gate',
|
||||
'router-failure-triage-resolves': 'gate',
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -1891,8 +1887,6 @@ export const E2E_KINDS: Record<string, 'rule' | 'behavior' | 'judge'> = {
|
||||
'overlay-harness-claude-dedicated-tools-vs-bash-sonnet': 'rule',
|
||||
'overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5': 'rule',
|
||||
'journey-negatives': 'rule',
|
||||
'router-failure-verified-cause': 'rule',
|
||||
'router-failure-triage-resolves': 'rule',
|
||||
};
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,79 +0,0 @@
|
||||
/**
|
||||
* Router failure routing (#3157, #3158). The gstack router checks a reported
|
||||
* failure's cause before it fixes it: a user-supplied cause that one read-only
|
||||
* check confirms is fixed directly, and an unknown cause gets one bounded
|
||||
* triage pass. Anything still unverified gets a focused failing test first.
|
||||
*
|
||||
* Two fixtures, one per issue (verified cause, triage that resolves it): each
|
||||
* must fix the bug directly, without routing to a skill, and leave the suite
|
||||
* green after one to four read-only calls. The two /investigate fallback cases
|
||||
* retired with that skill (GST-1717). The agent sees the rendered router section, sliced from
|
||||
* SKILL.md; GSTACK_ROUTER_ARM=before slices the base branch's router instead.
|
||||
* GSTACK_ROUTER_ARM=none is the descriptive no-gstack arm: the request alone,
|
||||
* no installed skills, scored on plain metrics (suite green, edits before the
|
||||
* first read, read-only calls before the first edit) recorded as observations.
|
||||
* Fixtures and scoring live in test/helpers/failure-routing-fixture.ts.
|
||||
*/
|
||||
import { afterAll, expect } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
runId, describeIfSelected, testConcurrentIfSelected, logCost, recordE2E,
|
||||
createEvalCollector, finalizeEvalCollector,
|
||||
} from './helpers/e2e-helpers';
|
||||
import { CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { bunIsolationArgs } from '../lib/bun-isolation';
|
||||
import { FIXTURES, makeRepo, plainMetrics, routerArm, routerPrompt, routing } from './helpers/failure-routing-fixture';
|
||||
import { recordObservation } from './helpers/none-arm';
|
||||
|
||||
const collector = createEvalCollector('e2e-failure-routing');
|
||||
const ARM = routerArm();
|
||||
const SCORE_GRACE_MS = 10_000;
|
||||
|
||||
describeIfSelected('Router failure routing (#3157, #3158)', FIXTURES.map(f => f.name), () => {
|
||||
const dirs: string[] = [];
|
||||
afterAll(() => {
|
||||
for (const dir of dirs) fs.rmSync(dir, { recursive: true, force: true });
|
||||
finalizeEvalCollector(collector);
|
||||
});
|
||||
|
||||
for (const fixture of FIXTURES) {
|
||||
testConcurrentIfSelected(fixture.name, async () => {
|
||||
const repo = makeRepo(fixture, ARM);
|
||||
dirs.push(repo);
|
||||
const result = await runSkillTest({
|
||||
prompt: routerPrompt(fixture, ARM),
|
||||
workingDirectory: repo,
|
||||
maxTurns: 6,
|
||||
tools: ['Skill', 'Read', 'Grep', 'Glob', 'Edit', 'Bash'],
|
||||
allowedTools: ['Skill', 'Read', 'Grep', 'Glob', 'Edit', 'Bash'],
|
||||
timeout: CAPTURE_MS - SCORE_GRACE_MS,
|
||||
testName: fixture.name,
|
||||
runId,
|
||||
});
|
||||
logCost(`router ${fixture.name} (${ARM})`, result);
|
||||
const suite = spawnSync(process.execPath, [...bunIsolationArgs(), 'test'], { cwd: repo, encoding: 'utf8', timeout: 60_000 });
|
||||
const green = suite.status === 0;
|
||||
if (ARM === 'none') {
|
||||
const p = plainMetrics(result.toolCalls);
|
||||
const plain = `arm=none reads_before_edit=${p.readsBeforeEdit} edited_before_read=${p.editedBeforeRead} green=${green} tools: ${p.sequence}`;
|
||||
recordE2E(collector, fixture.name, 'Router failure routing', result, { passed: true, output: plain });
|
||||
recordObservation(fixture.name, green, `suite green after the run: ${plain}`);
|
||||
recordObservation(fixture.name, !p.editedBeforeRead, `no edit before the first read-only call: ${plain}`);
|
||||
recordObservation(fixture.name, p.readsBeforeEdit >= 1, `at least one read-only call before the first edit: ${plain}`);
|
||||
return;
|
||||
}
|
||||
const r = routing(result);
|
||||
const summary = `arm=${ARM} route=${r.skillRouted ? 'skill' : 'direct'} reads=${r.readsBeforeRoute} green=${green} tools: ${r.sequence}`;
|
||||
const bounded = r.readsBeforeRoute >= 1 && r.readsBeforeRoute <= 4;
|
||||
const passed = bounded && !r.skillRouted && green;
|
||||
recordE2E(collector, fixture.name, 'Router failure routing', result, { passed, output: summary });
|
||||
|
||||
expect(r.skillRouted, summary).toBe(false);
|
||||
expect(green, `suite still red after the direct fix: ${summary}\n${suite.stdout}${suite.stderr}`).toBe(true);
|
||||
expect(r.readsBeforeRoute, `no read-only check before routing: ${summary}`).toBeGreaterThanOrEqual(1);
|
||||
expect(r.readsBeforeRoute, `triage was not bounded: ${summary}`).toBeLessThanOrEqual(4);
|
||||
}, CAPTURE_MS);
|
||||
}
|
||||
});
|
||||
@@ -7,7 +7,6 @@ import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { usePrivateStateRoot } from './helpers/private-state-root';
|
||||
import { expectMentions } from './helpers/prompt-structure';
|
||||
import { generateFailureRouting } from '../scripts/resolvers/preamble/generate-test-failure-triage';
|
||||
import { readShippedSkillRouting } from './helpers/shipped-skill-routing';
|
||||
import { readSkillStartup } from './helpers/skill-startup';
|
||||
import { RETIRED_SKILLS } from '../scripts/discover-skills';
|
||||
@@ -2031,25 +2030,15 @@ describe('Test failure triage in ship skill', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('Failure routing: verified cause → fix, unknown cause → bounded triage (#3157, #3158)', () => {
|
||||
const rule = generateFailureRouting();
|
||||
const unknownCause = rule.slice(rule.indexOf('**Cause unknown:** ') + '**Cause unknown:** '.length);
|
||||
|
||||
test('the router sends bug reports through the failure rule and names no retired skill (GST-1717)', () => {
|
||||
describe('Bug reports: the router has no bug-fix route, other surfaces keep their bounded triage (GST-2873)', () => {
|
||||
test('the router routes no bug report and carries no failure rule, so bug fixes run plain', () => {
|
||||
const router = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
|
||||
expect(router).toContain(rule);
|
||||
const bugRule = router.split('\n').find(line => line.startsWith('- User reports a bug'));
|
||||
expect(bugRule).toContain('**Failures**');
|
||||
expect(bugRule).toContain('then fix it test-first');
|
||||
expect(router.split('\n').some(line => line.startsWith('- User reports a bug'))).toBe(false);
|
||||
expect(router).not.toContain('**Failures');
|
||||
expect(router).not.toContain('**Cause supplied**');
|
||||
expect(router).not.toContain('/investigate');
|
||||
});
|
||||
|
||||
test('the rule keeps the root-cause bar on both paths, with a focused failing test when the cause is unclear', () => {
|
||||
expect(rule).toMatch(/\*\*Cause supplied\*\*[^\n]*one read-only check[^\n]*If it does not, leave the suggested fix alone: reproduce the failure with a focused failing test, trace it to the root cause/);
|
||||
expect(unknownCause).toMatch(/^make one read-only triage pass[^\n]*Do not reproduce, test hypotheses or search wider[^\n]*still unclear, write or run a focused test that fails on the bug before any edit, trace the failure to its root cause/);
|
||||
expect(rule).not.toContain('/investigate');
|
||||
});
|
||||
|
||||
test('the routing gstack offers for CLAUDE.md, and this repo\'s own, fix bugs directly behind a cause check or a failing test', () => {
|
||||
const bugRule = readShippedSkillRouting().section.split('\n').find(line => line.startsWith('- Bugs/errors'));
|
||||
expect(bugRule).toMatch(/→ fix them directly: one read-only check of the cause, or a focused failing test first when the cause is unclear/);
|
||||
|
||||
Reference in new issue
Block a user