feat(review): carve adversarial, plan-completion, and review-army into sections

The three resolver macros ship already carves as siblings now load on demand
for /review too: skeleton 100.2KB -> 55.0KB (-45%), union 93.4KB. Resolvers
stay the single source of truth (sections wrap the macros). Step 0/1, scope
drift, critical pass, confidence calibration, and fix-first stay always-loaded.
Fixtures and pins follow the moved content (codex-hardening wrapped-sites,
review-army E2E fixture builds skeleton+sections with an empty-fixture guard).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-08-25 16:36:02 +00:00
co-authored by Claude Fable 5
parent c7488f7e38
commit 3a78bf7c1d
13 changed files with 731 additions and 603 deletions
+35 -11
View File
@@ -391,10 +391,12 @@ describe('gstack-codex-probe: telemetry event emission', () => {
// ── Step 2A argv guard ─────────────────────────────────────────────────────
// Regression test for #1428: Codex CLI >=0.130.0 rejects passing a quoted
// prompt argument together with `--base <branch>`. Step 2A must never combine
// the two on the same line. Asserts across both the .tmpl source and the
// generated SKILL.md so template drift can't silently re-introduce the bug.
// the two on the same line. Step 2A lives in the carved review-mode section
// (codex/sections/review-mode.md, generated from its .md.tmpl) — asserts
// across both the .tmpl source and the generated section so template drift
// can't silently re-introduce the bug.
describe('codex SKILL.md.tmpl Step 2A: PROMPT + --base mutual exclusion guard', () => {
describe('codex review-mode section Step 2A: PROMPT + --base mutual exclusion guard', () => {
function extractStep2A(filePath: string): string {
const content = fs.readFileSync(filePath, 'utf-8');
const startIdx = content.indexOf('## Step 2A: Review Mode');
@@ -402,10 +404,14 @@ describe('codex SKILL.md.tmpl Step 2A: PROMPT + --base mutual exclusion guard',
// End at next `## ` heading (skill section boundary).
const tail = content.slice(startIdx);
const nextHeading = tail.slice(2).search(/\n## /);
return nextHeading === -1 ? tail : tail.slice(0, nextHeading + 2);
const section = nextHeading === -1 ? tail : tail.slice(0, nextHeading + 2);
// Non-empty extraction: a carve/regen that leaves only the heading behind
// must fail here, not silently pass a vacuous scan.
expect(section.length).toBeGreaterThan(1000);
return section;
}
for (const relPath of ['codex/SKILL.md.tmpl', 'codex/SKILL.md']) {
for (const relPath of ['codex/sections/review-mode.md.tmpl', 'codex/sections/review-mode.md']) {
test(`${relPath}: no \`codex review\` line combines a quoted prompt argument with --base`, () => {
const section = extractStep2A(path.join(ROOT, relPath));
// Find all lines invoking `codex review` (any prefix wrapper allowed).
@@ -460,7 +466,7 @@ describe('codex SKILL.md.tmpl Step 2A: PROMPT + --base mutual exclusion guard',
describe('codex timeout wrapper: /review + /ship diff passes', () => {
const WRAPPED_SITES = [
'scripts/resolvers/review.ts', // generator (source of truth)
'review/SKILL.md', // generated
'review/sections/adversarial.md', // review section (Step 5.7 carved out of the skeleton)
'ship/sections/adversarial.md', // ship section source
];
@@ -511,11 +517,29 @@ describe('codex timeout wrapper: /review + /ship diff passes', () => {
// budget, so the harness killed the call before the wrapper could emit
// its diagnosable exit-124 message — the same inversion #1036 fixed for
// /review and /ship.
// Asserted across both the .tmpl source and the generated SKILL.md so a regen
// or hand-edit of one but not the other can't silently reopen any of them.
describe('codex SKILL.md.tmpl: review sandbox + fail-closed gate + timeout ordering', () => {
for (const relPath of ['codex/SKILL.md.tmpl', 'codex/SKILL.md']) {
const read = () => fs.readFileSync(path.join(ROOT, relPath), 'utf-8');
// The three mode bodies are carved into codex/sections/*-mode.md (T9), so the
// sweep reads the skeleton+sections UNION on both the .tmpl side and the
// generated side — a regen or hand-edit of one but not the other can't
// silently reopen any of them. Each mode section starts with its own `## `
// heading, so the per-`## `-section split in check (c) still isolates each
// mode's gate/wrapper pair.
function readCodexUnion(kind: 'tmpl' | 'rendered'): string {
const sectionsDir = path.join(ROOT, 'codex', 'sections');
const skeleton = fs.readFileSync(
path.join(ROOT, 'codex', kind === 'tmpl' ? 'SKILL.md.tmpl' : 'SKILL.md'),
'utf-8',
);
const suffix = kind === 'tmpl' ? '.md.tmpl' : '.md';
const sections = fs.readdirSync(sectionsDir).sort()
.filter((f) => (kind === 'tmpl' ? f.endsWith('.md.tmpl') : f.endsWith('.md') && !f.endsWith('.md.tmpl')))
.map((f) => fs.readFileSync(path.join(sectionsDir, f), 'utf-8'));
expect(sections.length, `codex sections (*${suffix}) missing`).toBeGreaterThanOrEqual(3);
return [skeleton, ...sections].join('\n');
}
describe('codex skeleton+sections union: review sandbox + fail-closed gate + timeout ordering', () => {
for (const relPath of ['codex tmpl union', 'codex rendered union'] as const) {
const read = () => readCodexUnion(relPath === 'codex tmpl union' ? 'tmpl' : 'rendered');
test(`${relPath}: (a) every scoped codex review invocation pins sandbox_mode="read-only"`, () => {
const invocations = read()
+3 -1
View File
@@ -17,7 +17,9 @@ import * as path from 'path';
const ROOT = path.resolve(import.meta.dir, '..');
const GENERATED_WITH_GUIDANCE = ['review/SKILL.md', 'autoplan/SKILL.md'];
// review's specialist-dispatch guidance lives in its carved Review Army section
// (Step 4.5 moved out of the skeleton), so the pin follows it there.
const GENERATED_WITH_GUIDANCE = ['review/sections/review-army.md', 'autoplan/SKILL.md'];
// The inverted, post-2.1.198-inert phrasings. Checked across every generated
// SKILL.md so the regression can't migrate to another skill unnoticed.
+26 -1
View File
@@ -27,10 +27,35 @@ function setupRepo(prefix: string): { dir: string; run: (cmd: string, args: stri
// EXTRACTED (CLAUDE.md: "E2E test fixtures: extract, don't copy") — core
// review workflow + Step 1.5 (Plan Completion Audit) + Step 4.5 (Review Army
// dispatch: quality score, JSON schema, consensus, Red Team).
//
// /review is carved (T9): the Step 4.5 dispatch body lives in
// review/sections/review-army.md and the Plan Completion Audit in
// review/sections/plan-completion.md — the skeleton keeps only STOP-Read
// pointers, so the '## Step 4.5' H2 no longer exists in review/SKILL.md.
// Extract the skeleton H2s minus Step 4.5, then append both section files
// (still an extraction: sections ARE the minimal on-demand units).
const REVIEW_ARMY_SKELETON_SECTIONS = REVIEW_ARMY_E2E_SECTIONS.filter(
(s) => !s.startsWith('Step 4.5'),
);
function readReviewSection(file: string): string {
const p = path.join(ROOT, 'review', 'sections', file);
const content = fs.readFileSync(p, 'utf-8');
// Failure polarity: a fixture is never silently staged empty (regen missing).
if (content.trim().length < 500) {
throw new Error(`review section ${file} is unexpectedly small — was gen-skill-docs run after the carve?`);
}
return content;
}
function copyReviewFiles(dir: string) {
fs.writeFileSync(
path.join(dir, 'review-SKILL.md'),
extractSkillSections(path.join(ROOT, 'review'), REVIEW_ARMY_E2E_SECTIONS),
[
extractSkillSections(path.join(ROOT, 'review'), REVIEW_ARMY_SKELETON_SECTIONS),
readReviewSection('plan-completion.md'),
readReviewSection('review-army.md'),
].join('\n'),
);
fs.copyFileSync(path.join(ROOT, 'review', 'checklist.md'), path.join(dir, 'review-checklist.md'));
fs.copyFileSync(path.join(ROOT, 'review', 'greptile-triage.md'), path.join(dir, 'review-greptile-triage.md'));
+23 -8
View File
@@ -169,15 +169,30 @@ describe('real-skill pins: section lists used by E2E fixtures', () => {
expect(out.length).toBeLessThan(full.length * 0.5);
});
test('REVIEW_ARMY_E2E_SECTIONS extracts from review/SKILL.md', () => {
const out = extractSkillSections(path.join(ROOT, 'review'), REVIEW_ARMY_E2E_SECTIONS);
// The army tests reference the Plan Completion Audit (inside Step 1.5)
// and the Step 4.5 merge machinery (quality score, JSON schema, consensus).
expect(out).toContain('PLAN COMPLETION AUDIT');
expect(out).toContain('## Step 4.5: Review Army — Specialist Dispatch');
expect(out).toContain('quality_score');
expect(out).toContain('MULTI-SPECIALIST CONFIRMED');
test('REVIEW_ARMY_E2E_SECTIONS extracts from review/SKILL.md + carved sections', () => {
// /review is carved (T9): the Step 4.5 dispatch body lives in
// sections/review-army.md and the Plan Completion Audit in
// sections/plan-completion.md — the skeleton keeps only STOP-Read pointers.
// The E2E fixture (test/skill-e2e-review-army.test.ts) extracts the
// skeleton H2s minus Step 4.5, then appends both section files; this pin
// mirrors that exact construction so a section rename or an empty carve
// still fails FREE before a paid E2E setup throw.
const skeletonSections = REVIEW_ARMY_E2E_SECTIONS.filter(
(s) => !s.startsWith('Step 4.5'),
);
const out = extractSkillSections(path.join(ROOT, 'review'), skeletonSections);
expect(out).toContain('## Step 1.5: Scope Drift Detection');
expect(out).not.toContain('## Telemetry (run last)');
const planCompletion = fs.readFileSync(
path.join(ROOT, 'review', 'sections', 'plan-completion.md'), 'utf-8');
expect(planCompletion).toContain('PLAN COMPLETION AUDIT');
const army = fs.readFileSync(
path.join(ROOT, 'review', 'sections', 'review-army.md'), 'utf-8');
expect(army).toContain('## Step 4.5: Review Army — Specialist Dispatch');
expect(army).toContain('quality_score');
expect(army).toContain('MULTI-SPECIALIST CONFIRMED');
});
test('RETRO_E2E_SECTIONS extracts from retro/SKILL.md', () => {