diff --git a/docs/evals/judge-controls-2026-10.md b/docs/evals/judge-controls-2026-10.md index a929338ad..584eb2d73 100644 --- a/docs/evals/judge-controls-2026-10.md +++ b/docs/evals/judge-controls-2026-10.md @@ -67,3 +67,17 @@ sample and rationale). Means are clarity / completeness / actionability. bundles of 70-160 KB these judges do not detect a missing deliverable or input step. A rubric proposal (anchor 4 and 5 to named deliverables and inputs) is a pre-registered EVAL_POLICY change for Garry, not part of this change. + +## Missing-step coverage moved to a free test + +Because the qa and review workflow judges passed the targeted defects above, +missing-step coverage no longer rests on the judges. +`test/workflow-required-steps.test.ts` checks that every step and phase heading +of the judged workflow skills (qa, qa-only, review, ship, plan-ceo-review, +plan-eng-review, setup-deploy) is still in the generated SKILL.md or section +file, in order, and names the missing step when one is removed (the two control +defects above are its negative cases). The pinned list, +`test/fixtures/workflow-required-steps.json`, was derived from the templates' +step headings; a new step heading must be added to it. The judges keep scoring +clarity, completeness and actionability; the rubric revision stays a proposal +for Garry (TODOS.md). diff --git a/test/fixtures/workflow-required-steps.json b/test/fixtures/workflow-required-steps.json new file mode 100644 index 000000000..7dfaf8cb7 --- /dev/null +++ b/test/fixtures/workflow-required-steps.json @@ -0,0 +1,214 @@ +{ + "source": "Step and phase headings of the judged workflow skills, derived on 2026-10-04 from the generated Claude SKILL.md and sections (templates at the integration head). qa-only lists its template's own top-level headings. C1 calibration showed the workflow judges pass a bundle with a required step removed, so missing-step coverage lives here.", + "skills": { + "qa": { + "SKILL.md": [ + "## Step 0: Detect platform and base branch", + "## Phases 1-6: QA Baseline", + "## Phase 7: Triage", + "## Phase 8: Fix Loop", + "### 8a. Diagnose and reproduce", + "### 8a.5. Regression test before repair", + "### 8b. Fix", + "### 8c. Re-test", + "### 8d. Commit verified work", + "### 8e. Classify", + "### 8e.5. Regression Test record", + "### 8f. Self-Regulation (STOP AND EVALUATE)", + "## Phase 9: Final QA", + "## Phase 10: Report", + "## Phase 11: TODOS.md Update" + ], + "sections/exploratory.md": [ + "## 1. Charter and preflight", + "## 2. Probe loop", + "## 3. Parent handoff", + "## 4. Final report" + ], + "sections/qa-patterns.md": [ + "### Phase 1: Initialize", + "### Phase 2: Authenticate (if needed)", + "### Phase 3: Orient", + "### Phase 4: Explore", + "### Phase 5: Document", + "### Phase 6: Wrap Up" + ] + }, + "qa-only": { + "SKILL.md": [ + "## Request Parameters", + "## Test Plan Context", + "## Select Surfaces and Isolation", + "## Prepare Report Artifacts", + "## Browser Setup (conditional)", + "## Run the Selected Checks", + "## Output", + "## Additional Rules (qa-only specific)" + ], + "sections/exploratory.md": [ + "## 0. Preparation gate", + "## 1. Charter and preflight", + "## 2. Probe loop", + "## 3. Parent handoff", + "## 4. Final report" + ], + "sections/reporting.md": [ + "## 1. Establish each finding once", + "## 2. Fill timing fields from their actual boundaries", + "## 3. Assemble and check every repetition before writing", + "## 4. Capture permitted notes, then write the report" + ] + }, + "review": { + "SKILL.md": [ + "## Step 0: Detect platform and base branch", + "## Step 1: Check branch", + "## Step 1.5: Scope Drift Detection", + "## Step 2: Read the checklist", + "## Step 2.5: Check for Greptile review comments", + "## Step 3: Get the diff", + "## Step 3.4: Workspace-aware queue status (advisory)", + "## Step 3.5: Slop scan (advisory)", + "## Step 3.6: Gather review context", + "## Step 4: Critical pass (core review)", + "### Step 4.7: Exploratory QA (before Fix-First)", + "## Step 5: Fix-First Review", + "### Step 5.0: Cross-review finding dedup", + "### Step 5a: Classify each finding", + "### Step 5b: Auto-fix all AUTO-FIX items", + "### Step 5c: Batch-ask about ASK items", + "### Step 5d: Apply user-approved fixes", + "## Step 5.8: Persist Eng Review result", + "### 1. Re-review after edits", + "### 2. Fill the record" + ], + "sections/adversarial.md": [ + "## Step 4.8: Adversarial review (always-on)" + ], + "sections/review-army.md": [ + "## Step 4.5: Review Army — Specialist Dispatch", + "### Step 4.6: Collect and merge findings", + "#### 1. Parse outputs", + "#### 2. Validate severity", + "#### 3. Identify and merge", + "#### 4. Apply specialist confidence gates", + "#### 5. Score and present specialists", + "#### 6. Save specialist activity", + "#### 7. Hand off to Fix-First" + ] + }, + "ship": { + "SKILL.md": [ + "## Step 0: Detect platform and base branch", + "## Step 0.9: Apple target detection", + "## Step 1: Pre-flight", + "## Step 2: Distribution Pipeline Check", + "## Step 3: Merge the base branch (BEFORE tests)", + "## Step 11.5: Bind the reviews", + "## Step 12: Version bump (auto-decide)", + "## Step 14: TODOS.md (auto-update)", + "## Step 14.5: Documentation audit (every ship)", + "## Step 15: Commit (bisectable chunks)", + "## Step 16: Verification Gate", + "### 1. Finish writers and prepare outputs", + "### 2. Choose the change route", + "### 3. Resolve documentation freshness", + "### 4. Verify the frozen candidate", + "### 5. Report, then push", + "## Step 17: Push", + "## Step 18: Prepare publication metadata", + "## Step 20: Persist ship metrics", + "## Step 21: Plan-tune discoverability nudge (first-successful-ship only)" + ], + "sections/adversarial.md": [ + "## Step 11: Adversarial review (always-on)" + ], + "sections/changelog.md": [ + "## Step 13: CHANGELOG (auto-generate)" + ], + "sections/greptile.md": [ + "## Step 10: Address Greptile review comments (if PR exists)" + ], + "sections/plan-completion.md": [ + "## Step 8: Plan Completion Audit", + "## Step 8.1: Plan Verification", + "## Step 8.2: Scope Drift Detection" + ], + "sections/pr-body.md": [ + "## Step 19: Create PR/MR" + ], + "sections/review-army.md": [ + "## Step 9: Pre-Landing Review", + "## Step 9.1: Review Army — Specialist Dispatch", + "### Step 9.2: Collect and merge findings", + "#### 1. Parse outputs", + "#### 2. Validate severity", + "#### 3. Identify and merge", + "#### 4. Apply specialist confidence gates", + "#### 5. Score and present specialists", + "#### 6. Save specialist activity", + "#### 7. Hand off to Fix-First", + "### Step 9.2.1: Exploratory QA (before Fix-First)", + "### Step 9.3: Cross-review finding dedup", + "## Step 9.4: Fix-First and persistence" + ], + "sections/test-coverage.md": [ + "## Step 7: Test Coverage Audit" + ], + "sections/tests.md": [ + "## Step 4: Test Framework Bootstrap", + "## Step 5: Run tests (on merged code)", + "## Step 6: Eval Suites (conditional)" + ] + }, + "plan-ceo-review": { + "SKILL.md": [ + "## Step 0: Detect platform and base branch", + "## Step 0: Nuclear Scope Challenge + Mode Selection", + "### 0A. Premise Challenge", + "### 0B. Existing Code Leverage", + "### 0C. Dream State Mapping", + "### 0D. Alternatives (reusable decision procedure)", + "### 0E. Mode Selection", + "### 0F. Expansion Framing (shared by EXPANSION and SELECTIVE EXPANSION)", + "### 0G. Mode-Specific Analysis", + "### 0H. Persist CEO Plan (EXPANSION and SELECTIVE EXPANSION only)", + "### 0I. Temporal Interrogation (EXPANSION, SELECTIVE EXPANSION, and HOLD modes)" + ], + "sections/review-sections.md": [ + "### Section 1: Architecture Review", + "### Section 2: Error & Rescue Map", + "### Section 3: Security & Threat Model", + "### Section 4: Data Flow & Interaction Edge Cases", + "### Section 5: Code Quality Review", + "### Section 6: Test Review", + "### Section 7: Performance Review", + "### Section 8: Observability & Debuggability Review", + "### Section 9: Deployment & Rollout Review", + "### Section 10: Long-Term Trajectory Review", + "### Section 11: Design & UX Review (skip if no UI scope detected)" + ] + }, + "plan-eng-review": { + "SKILL.md": [ + "### Step 0: Scope Challenge" + ], + "sections/review-sections.md": [ + "### 1. Architecture review", + "### 2. Code quality review", + "### 3. Test review", + "### 4. Performance review" + ] + }, + "setup-deploy": { + "SKILL.md": [ + "### Step 1: Check existing configuration", + "### Step 2: Detect platform", + "### Step 3: Platform-specific setup", + "### Step 4: Write configuration", + "### Step 5: Verify", + "### Step 6: Summary" + ] + } + } +} diff --git a/test/workflow-required-steps.test.ts b/test/workflow-required-steps.test.ts new file mode 100644 index 000000000..9fcba45f8 --- /dev/null +++ b/test/workflow-required-steps.test.ts @@ -0,0 +1,72 @@ +/** + * Required workflow steps (C1 follow-up, 2026-10-04). The qa workflow judge + * passed a /qa bundle with "Phase 10: Report" removed and the review judge + * passed /review with "Step 3: Get the diff" removed, so the judges do not + * cover a missing step. This deterministic check does: every pinned step or + * phase heading of the judged workflow skills must still be in the generated + * SKILL.md or section file, in order. The pinned list was derived from the + * templates' step headings; a new step heading must be added to it. + */ +import { describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const PINNED = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures', 'workflow-required-steps.json'), 'utf8')) as { + skills: Record>; +}; +const STEP = /^#{2,4} (?:(?:Step|Phase|Section) \d+[A-Za-z0-9.]*|Phases \d+-\d+|\d+[A-Za-z]?(?:\.\d+)?\.)[ :].*$/; + +/** Heading lines outside code fences. */ +function headings(text: string): string[] { + let fence = false; + return text.split('\n').flatMap(line => { + if (/^\s*(?:```|~~~)/.test(line)) { fence = !fence; return []; } + return !fence && /^#{2,4} /.test(line) ? [line.trim()] : []; + }); +} + +/** Missing or out-of-order required steps, each named. */ +function missingSteps(skill: string, file: string, text: string, required: readonly string[]): string[] { + const found = headings(text); + const problems: string[] = []; + let last = -1; + for (const step of required) { + const at = found.indexOf(step, last + 1); + if (at < 0) problems.push(found.includes(step) ? `${skill}/${file}: step out of order: "${step}"` : `${skill}/${file}: missing required step "${step}"`); + else last = at; + } + return problems; +} + +const read = (skill: string, file: string) => fs.readFileSync(path.join(ROOT, skill, file), 'utf8'); + +describe('judged workflow skills keep their required steps', () => { + for (const [skill, files] of Object.entries(PINNED.skills)) { + test(`${skill}: every pinned step is present in order`, () => { + const problems = Object.entries(files).flatMap(([file, steps]) => missingSteps(skill, file, read(skill, file), steps)); + expect(problems).toEqual([]); + }); + + test(`${skill}: every step heading is pinned`, () => { + const pinnedFiles = new Set(Object.keys(files)); + const sectionFiles = fs.existsSync(path.join(ROOT, skill, 'sections')) + ? fs.readdirSync(path.join(ROOT, skill, 'sections')).filter(f => f.endsWith('.md')).map(f => `sections/${f}`) : []; + const unpinned = ['SKILL.md', ...sectionFiles].flatMap(file => headings(read(skill, file)) + .filter(h => STEP.test(h) && !(pinnedFiles.has(file) && files[file]!.includes(h))) + .map(h => `${skill}/${file}: "${h}" (add it to test/fixtures/workflow-required-steps.json)`)); + expect(unpinned).toEqual([]); + }); + } + + test('the C1 control defects are reported by step name', () => { + const qa = read('qa', 'SKILL.md'); + expect(missingSteps('qa', 'SKILL.md', qa.replace('## Phase 10: Report', '## Report'), PINNED.skills.qa!['SKILL.md']!)) + .toEqual(['qa/SKILL.md: missing required step "## Phase 10: Report"']); + const review = read('review', 'SKILL.md'); + expect(missingSteps('review', 'SKILL.md', review.replace(/^## Step 3: Get the diff$/m, ''), PINNED.skills.review!['SKILL.md']!)) + .toEqual(['review/SKILL.md: missing required step "## Step 3: Get the diff"']); + const fenced = review.replace(/^(## Step 3: Get the diff)$/m, '```\n$1\n```'); + expect(missingSteps('review', 'SKILL.md', fenced, PINNED.skills.review!['SKILL.md']!)).toHaveLength(1); + }); +});