Files
gstack/test/outside-voice-failure-policy.test.ts
garrytan 54c2434a9c refactor(resolvers): one outside-voice failure policy (deliberate prose unification)
outsideVoiceFailurePolicy(ctx, opts) in outside-voice.ts now renders the
auth / timeout / empty-response bullets for all four call sites that
hand-typed them (Codex second opinion, adversarial step, Codex plan
review, design outside voices). Options are explicit per site
(timeoutMinutes, onTimeout, stderrOnEmpty, fallback, escape) with no
defaults.

Deliberate generated-prose changes (every host):
- office-hours: 'Fall back to <native> subagent.' becomes
  'Fall back to the <native> subagent below.'
- plan-devex-review: the plain 'Auth failure (stderr contains ...)'
  bullets become the canonical bold bullets; auth also triggers on
  'API key'; 'auth failed' becomes 'authentication failed'.
- review/ship adversarial: 'exceeded 9 minutes and was terminated'
  becomes 'timed out after 9 minutes and was terminated'; the timeout
  is still MISSING COVERAGE.
- design outside voices: unchanged.

Adds ratchet (d) (test/outside-voice-failure-policy.test.ts) with a
reasoned allowlist for /codex's own CLI errors, the MISSING COVERAGE
retention test, refreshed codex/factory ship goldens, and outside-voice.ts
in every touchfile entry of review.ts and design.ts (selection golden
extended).
2026-09-30 05:54:21 +00:00

198 lines
10 KiB
TypeScript

/**
* outsideVoiceFailurePolicy + ratchet (d).
*
* The auth / timeout / empty-response bullets that follow every outside-voice
* invocation used to be hand-typed at four call sites in review.ts and
* design.ts, and they drifted ("auth failed" vs "authentication failed", a
* missing "API key" trigger, differing fallback wording). One owner now
* renders them; the ratchet below keeps new copies from appearing.
*/
import { describe, expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { ALL_HOST_CONFIGS } from '../hosts/index';
import { outsideVoiceFailurePolicy, outsideVoiceFor, type OutsideVoiceFailurePolicyOptions } from '../scripts/resolvers/outside-voice';
import { generateAdversarialStep } from '../scripts/resolvers/outside-voice-steps';
import { HOST_PATHS, type TemplateContext } from '../scripts/resolvers/types';
const ROOT = path.resolve(import.meta.dir, '..');
const ALLOWLIST = 'test/fixtures/outside-voice-failure-prose-allowlist.json';
const OWNER_FILE = 'scripts/resolvers/outside-voice.ts';
const ctxFor = (host: string, skillName = 'review'): TemplateContext =>
({ host, skillName, tmplPath: `${skillName}/SKILL.md.tmpl`, paths: HOST_PATHS[host] }) as TemplateContext;
const base: OutsideVoiceFailurePolicyOptions = {
timeoutMinutes: 5, onTimeout: 'fallback', stderrOnEmpty: false, fallback: 'native', escape: 0,
};
describe('outsideVoiceFailurePolicy', () => {
test('native fallback: every failure names the provider message and falls back to the native subagent', () => {
expect(outsideVoiceFailurePolicy(ctxFor('claude'), base)).toBe([
'- **Auth failure:** If stderr contains "auth", "login", "unauthorized", or "API key": "Codex authentication failed. Run `codex login` to authenticate." Fall back to the Claude subagent below.',
'- **Timeout:** "Codex timed out after 5 minutes." Fall back to the Claude subagent below.',
'- **Empty response:** "Codex returned no response." Fall back to the Claude subagent below.',
].join('\n'));
});
test('missing-coverage timeout, stderr on empty, no fallback, escaped backticks', () => {
const out = outsideVoiceFailurePolicy(ctxFor('claude'), {
timeoutMinutes: 9, onTimeout: 'missing-coverage', stderrOnEmpty: true, fallback: 'none', escape: 1,
});
expect(out).toContain('Run \\`codex login\\` to authenticate."\n');
expect(out).toContain('- **Timeout:** "Codex timed out after 9 minutes and was terminated; this pass produced NO findings." A timed-out pass is MISSING COVERAGE, not a clean bill');
expect(out).toContain('- **Empty response:** "Codex returned no response. Stderr: <paste relevant error>."');
expect(out).not.toContain('Fall back');
});
test('the Codex harness names Claude Code and its login command', () => {
const out = outsideVoiceFailurePolicy(ctxFor('codex'), base);
expect(out).toContain('"Claude Code authentication failed. Run `claude auth login` to authenticate."');
expect(out).toContain(`Fall back to the ${outsideVoiceFor(ctxFor('codex')).nativeLabel} subagent below.`);
});
});
// ─── MISSING COVERAGE retention ────────────────────────────────────
// At 96764e80, exactly the adversarial step (review + ship, every host) told
// the agent that a timed-out outside pass is MISSING COVERAGE. The prose
// unification must not turn any of those into a silent timeout.
describe('timeout MISSING COVERAGE wording is retained', () => {
const timeoutLine = (text: string) => text.split('\n').find((l) => l.startsWith('- **Timeout:**')) ?? '';
test.each(ALL_HOST_CONFIGS.flatMap((h) => ['review', 'ship'].map((s) => [h.name, s])))(
'adversarial step on %s /%s', (host, skill) => {
expect(timeoutLine(generateAdversarialStep(ctxFor(host, skill)))).toContain('MISSING COVERAGE');
});
test.each(['review/sections/adversarial.md', 'ship/sections/adversarial.md'])('generated %s', (rel) => {
expect(timeoutLine(fs.readFileSync(path.join(ROOT, rel), 'utf8'))).toContain('MISSING COVERAGE');
});
});
// ─── Ratchet (d) ────────────────────────────────────────────────────
// Signature: every auth-failure, timeout and empty-response phrasing used by
// the hand-written copies at 96764e80 (review.ts:746-748, :904-906,
// :1171-1173; design.ts:855-857).
const SIGNATURE = [
/authentication failed\. Run/,
/\bauth failed\. Run/,
/timed out after \d+ minutes/,
/exceeded \d+ minutes and was terminated/,
/returned no response/,
];
interface SourceFile { file: string; text: string }
interface AllowEntry { file: string; text: string; reason?: string }
function walk(dir: string, keep: (rel: string) => boolean, out: SourceFile[] = []): SourceFile[] {
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
if (entry.name === 'node_modules' || entry.name.startsWith('.')) continue;
const full = path.join(dir, entry.name);
const rel = path.relative(ROOT, full).split(path.sep).join('/');
if (entry.isDirectory()) walk(full, keep, out);
else if (keep(rel)) out.push({ file: rel, text: fs.readFileSync(full, 'utf8') });
}
return out;
}
function scannedFiles(): SourceFile[] {
return walk(ROOT, (rel) => rel.endsWith('.tmpl') || (rel.startsWith('scripts/resolvers/') && rel.endsWith('.ts')));
}
/** Lines of the owner function itself are the one legitimate source. */
function ownerLines(text: string): Set<number> {
const lines = text.split('\n');
const start = lines.findIndex((l) => l.startsWith('export function outsideVoiceFailurePolicy('));
const owned = new Set<number>();
if (start < 0) return owned;
for (let i = start; i < lines.length; i += 1) {
owned.add(i);
if (lines[i] === '}') break;
}
return owned;
}
export function scanFailureProse(files: SourceFile[], allowlist: AllowEntry[]): string[] {
const problems: string[] = [];
for (const entry of allowlist) {
if (!entry.reason?.trim()) problems.push(`${ALLOWLIST} entry for ${entry.file} has no reason: ${entry.text}`);
}
const used = new Set<AllowEntry>();
for (const { file, text } of files) {
const owned = file === OWNER_FILE ? ownerLines(text) : new Set<number>();
text.split('\n').forEach((line, i) => {
if (owned.has(i) || !SIGNATURE.some((re) => re.test(line))) return;
const trimmed = line.trim();
const allowed = allowlist.find((e) => e.file === file && e.text === trimmed);
if (allowed) { used.add(allowed); return; }
problems.push(`${file}:${i + 1} ${trimmed.slice(0, 160)}`);
});
}
for (const entry of allowlist) {
if (!used.has(entry)) problems.push(`${ALLOWLIST} stale entry matches nothing in ${entry.file}: ${entry.text}`);
}
return problems;
}
export function formatFailureProse(problems: string[]): string {
return [
'Outside-voice failure prose found outside outsideVoiceFailurePolicy (ratchet d):',
...problems.map((p) => ` ${p}`),
'Rule: outside-voice auth/timeout/empty-response fallback bullets come only from outsideVoiceFailurePolicy(), so every skill falls back the same way instead of drifting.',
`Fix: call outsideVoiceFailurePolicy(ctx, { timeoutMinutes, onTimeout, stderrOnEmpty, fallback, escape }) from ${OWNER_FILE} instead of writing the bullets by hand.`,
`Allowlist: ${ALLOWLIST}. Add an entry (file, exact trimmed line, reason) only for prose that is not an outside-voice fallback, such as a skill's own direct CLI error handling.`,
].join('\n');
}
const readAllowlist = (): AllowEntry[] => JSON.parse(fs.readFileSync(path.join(ROOT, ALLOWLIST), 'utf8'));
describe('ratchet (d): outside-voice failure prose has one owner', () => {
test('no resolver or template hand-writes the failure bullets', () => {
const problems = scanFailureProse(scannedFiles(), readAllowlist());
if (problems.length) throw new Error(formatFailureProse(problems));
});
test('the scan is not vacuous: it reaches the owner, the call sites and the allowlisted template', () => {
const files = scannedFiles().map((f) => f.file);
for (const rel of [OWNER_FILE, 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/design.ts', 'codex/SKILL.md.tmpl']) {
expect(files).toContain(rel);
}
});
test('a planted copy is reported with file:line, the fix and the allowlist path', () => {
const planted = [
{ file: 'scripts/resolvers/planted.ts', text: 'const a = 1;\n- **Timeout:** "${label} exceeded 9 minutes and was terminated."\n' },
{ file: 'planted/SKILL.md.tmpl', text: '- Auth failure: "Codex auth failed. Run `codex login`."\n' },
];
const problems = scanFailureProse(planted, []);
expect(problems).toEqual([
'scripts/resolvers/planted.ts:2 - **Timeout:** "${label} exceeded 9 minutes and was terminated."',
'planted/SKILL.md.tmpl:1 - Auth failure: "Codex auth failed. Run `codex login`."',
]);
const message = formatFailureProse(problems);
expect(message).toContain('scripts/resolvers/planted.ts:2');
expect(message).toContain('Fix: call outsideVoiceFailurePolicy(');
expect(message).toContain(ALLOWLIST);
});
test('a copy pasted into the owner file outside the owner function is still reported', () => {
const owner = scannedFiles().find((f) => f.file === OWNER_FILE)!;
const text = `${owner.text}\nconst copy = '"Codex returned no response."';\n`;
const problems = scanFailureProse([{ file: OWNER_FILE, text }], []);
expect(problems).toHaveLength(1);
expect(problems[0]).toStartWith(`${OWNER_FILE}:`);
});
test('allowlist entries are keyed on file and text, so inserting a line above still passes', () => {
const allowlist = readAllowlist();
const files = scannedFiles().map((f) => (f.file === 'codex/SKILL.md.tmpl' ? { ...f, text: `inserted line\n${f.text}` } : f));
expect(scanFailureProse(files, allowlist)).toEqual([]);
});
test('an allowlist entry without a reason fails', () => {
const allowlist = readAllowlist().map((e, i) => (i === 0 ? { file: e.file, text: e.text } : e));
const problems = scanFailureProse(scannedFiles(), allowlist);
expect(problems).toEqual([`${ALLOWLIST} entry for codex/SKILL.md.tmpl has no reason: ${allowlist[0].text}`]);
});
});