mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-31 22:14:25 +01:00
C1: a free test catches a removed workflow step, which the judges missed
C1 controls showed the qa workflow judge passing /qa without Phase 10 and the review judge passing /review without Step 3. The new test pins every step/phase heading of qa, qa-only, review, ship, plan-ceo-review, plan-eng-review and setup-deploy (derived from the templates' step headings) and names any that goes missing or out of order; new step headings must be pinned. The judge rubric is unchanged (EVAL_POLICY).
This commit is contained in:
garrytan committed
1 parent
1c105acb31
commit
7bd137d636
3 files changed
+300
No files matched your search
@@ -67,3 +67,17 @@ sample and rationale). Means are clarity / completeness / actionability.
|
||||
bundles of 70-160 KB these judges do not detect a missing deliverable or input
|
||||
step. A rubric proposal (anchor 4 and 5 to named deliverables and inputs) is a
|
||||
pre-registered EVAL_POLICY change for Garry, not part of this change.
|
||||
|
||||
## Missing-step coverage moved to a free test
|
||||
|
||||
Because the qa and review workflow judges passed the targeted defects above,
|
||||
missing-step coverage no longer rests on the judges.
|
||||
`test/workflow-required-steps.test.ts` checks that every step and phase heading
|
||||
of the judged workflow skills (qa, qa-only, review, ship, plan-ceo-review,
|
||||
plan-eng-review, setup-deploy) is still in the generated SKILL.md or section
|
||||
file, in order, and names the missing step when one is removed (the two control
|
||||
defects above are its negative cases). The pinned list,
|
||||
`test/fixtures/workflow-required-steps.json`, was derived from the templates'
|
||||
step headings; a new step heading must be added to it. The judges keep scoring
|
||||
clarity, completeness and actionability; the rubric revision stays a proposal
|
||||
for Garry (TODOS.md).
|
||||
+214
@@ -0,0 +1,214 @@
|
||||
{
|
||||
"source": "Step and phase headings of the judged workflow skills, derived on 2026-10-04 from the generated Claude SKILL.md and sections (templates at the integration head). qa-only lists its template's own top-level headings. C1 calibration showed the workflow judges pass a bundle with a required step removed, so missing-step coverage lives here.",
|
||||
"skills": {
|
||||
"qa": {
|
||||
"SKILL.md": [
|
||||
"## Step 0: Detect platform and base branch",
|
||||
"## Phases 1-6: QA Baseline",
|
||||
"## Phase 7: Triage",
|
||||
"## Phase 8: Fix Loop",
|
||||
"### 8a. Diagnose and reproduce",
|
||||
"### 8a.5. Regression test before repair",
|
||||
"### 8b. Fix",
|
||||
"### 8c. Re-test",
|
||||
"### 8d. Commit verified work",
|
||||
"### 8e. Classify",
|
||||
"### 8e.5. Regression Test record",
|
||||
"### 8f. Self-Regulation (STOP AND EVALUATE)",
|
||||
"## Phase 9: Final QA",
|
||||
"## Phase 10: Report",
|
||||
"## Phase 11: TODOS.md Update"
|
||||
],
|
||||
"sections/exploratory.md": [
|
||||
"## 1. Charter and preflight",
|
||||
"## 2. Probe loop",
|
||||
"## 3. Parent handoff",
|
||||
"## 4. Final report"
|
||||
],
|
||||
"sections/qa-patterns.md": [
|
||||
"### Phase 1: Initialize",
|
||||
"### Phase 2: Authenticate (if needed)",
|
||||
"### Phase 3: Orient",
|
||||
"### Phase 4: Explore",
|
||||
"### Phase 5: Document",
|
||||
"### Phase 6: Wrap Up"
|
||||
]
|
||||
},
|
||||
"qa-only": {
|
||||
"SKILL.md": [
|
||||
"## Request Parameters",
|
||||
"## Test Plan Context",
|
||||
"## Select Surfaces and Isolation",
|
||||
"## Prepare Report Artifacts",
|
||||
"## Browser Setup (conditional)",
|
||||
"## Run the Selected Checks",
|
||||
"## Output",
|
||||
"## Additional Rules (qa-only specific)"
|
||||
],
|
||||
"sections/exploratory.md": [
|
||||
"## 0. Preparation gate",
|
||||
"## 1. Charter and preflight",
|
||||
"## 2. Probe loop",
|
||||
"## 3. Parent handoff",
|
||||
"## 4. Final report"
|
||||
],
|
||||
"sections/reporting.md": [
|
||||
"## 1. Establish each finding once",
|
||||
"## 2. Fill timing fields from their actual boundaries",
|
||||
"## 3. Assemble and check every repetition before writing",
|
||||
"## 4. Capture permitted notes, then write the report"
|
||||
]
|
||||
},
|
||||
"review": {
|
||||
"SKILL.md": [
|
||||
"## Step 0: Detect platform and base branch",
|
||||
"## Step 1: Check branch",
|
||||
"## Step 1.5: Scope Drift Detection",
|
||||
"## Step 2: Read the checklist",
|
||||
"## Step 2.5: Check for Greptile review comments",
|
||||
"## Step 3: Get the diff",
|
||||
"## Step 3.4: Workspace-aware queue status (advisory)",
|
||||
"## Step 3.5: Slop scan (advisory)",
|
||||
"## Step 3.6: Gather review context",
|
||||
"## Step 4: Critical pass (core review)",
|
||||
"### Step 4.7: Exploratory QA (before Fix-First)",
|
||||
"## Step 5: Fix-First Review",
|
||||
"### Step 5.0: Cross-review finding dedup",
|
||||
"### Step 5a: Classify each finding",
|
||||
"### Step 5b: Auto-fix all AUTO-FIX items",
|
||||
"### Step 5c: Batch-ask about ASK items",
|
||||
"### Step 5d: Apply user-approved fixes",
|
||||
"## Step 5.8: Persist Eng Review result",
|
||||
"### 1. Re-review after edits",
|
||||
"### 2. Fill the record"
|
||||
],
|
||||
"sections/adversarial.md": [
|
||||
"## Step 4.8: Adversarial review (always-on)"
|
||||
],
|
||||
"sections/review-army.md": [
|
||||
"## Step 4.5: Review Army — Specialist Dispatch",
|
||||
"### Step 4.6: Collect and merge findings",
|
||||
"#### 1. Parse outputs",
|
||||
"#### 2. Validate severity",
|
||||
"#### 3. Identify and merge",
|
||||
"#### 4. Apply specialist confidence gates",
|
||||
"#### 5. Score and present specialists",
|
||||
"#### 6. Save specialist activity",
|
||||
"#### 7. Hand off to Fix-First"
|
||||
]
|
||||
},
|
||||
"ship": {
|
||||
"SKILL.md": [
|
||||
"## Step 0: Detect platform and base branch",
|
||||
"## Step 0.9: Apple target detection",
|
||||
"## Step 1: Pre-flight",
|
||||
"## Step 2: Distribution Pipeline Check",
|
||||
"## Step 3: Merge the base branch (BEFORE tests)",
|
||||
"## Step 11.5: Bind the reviews",
|
||||
"## Step 12: Version bump (auto-decide)",
|
||||
"## Step 14: TODOS.md (auto-update)",
|
||||
"## Step 14.5: Documentation audit (every ship)",
|
||||
"## Step 15: Commit (bisectable chunks)",
|
||||
"## Step 16: Verification Gate",
|
||||
"### 1. Finish writers and prepare outputs",
|
||||
"### 2. Choose the change route",
|
||||
"### 3. Resolve documentation freshness",
|
||||
"### 4. Verify the frozen candidate",
|
||||
"### 5. Report, then push",
|
||||
"## Step 17: Push",
|
||||
"## Step 18: Prepare publication metadata",
|
||||
"## Step 20: Persist ship metrics",
|
||||
"## Step 21: Plan-tune discoverability nudge (first-successful-ship only)"
|
||||
],
|
||||
"sections/adversarial.md": [
|
||||
"## Step 11: Adversarial review (always-on)"
|
||||
],
|
||||
"sections/changelog.md": [
|
||||
"## Step 13: CHANGELOG (auto-generate)"
|
||||
],
|
||||
"sections/greptile.md": [
|
||||
"## Step 10: Address Greptile review comments (if PR exists)"
|
||||
],
|
||||
"sections/plan-completion.md": [
|
||||
"## Step 8: Plan Completion Audit",
|
||||
"## Step 8.1: Plan Verification",
|
||||
"## Step 8.2: Scope Drift Detection"
|
||||
],
|
||||
"sections/pr-body.md": [
|
||||
"## Step 19: Create PR/MR"
|
||||
],
|
||||
"sections/review-army.md": [
|
||||
"## Step 9: Pre-Landing Review",
|
||||
"## Step 9.1: Review Army — Specialist Dispatch",
|
||||
"### Step 9.2: Collect and merge findings",
|
||||
"#### 1. Parse outputs",
|
||||
"#### 2. Validate severity",
|
||||
"#### 3. Identify and merge",
|
||||
"#### 4. Apply specialist confidence gates",
|
||||
"#### 5. Score and present specialists",
|
||||
"#### 6. Save specialist activity",
|
||||
"#### 7. Hand off to Fix-First",
|
||||
"### Step 9.2.1: Exploratory QA (before Fix-First)",
|
||||
"### Step 9.3: Cross-review finding dedup",
|
||||
"## Step 9.4: Fix-First and persistence"
|
||||
],
|
||||
"sections/test-coverage.md": [
|
||||
"## Step 7: Test Coverage Audit"
|
||||
],
|
||||
"sections/tests.md": [
|
||||
"## Step 4: Test Framework Bootstrap",
|
||||
"## Step 5: Run tests (on merged code)",
|
||||
"## Step 6: Eval Suites (conditional)"
|
||||
]
|
||||
},
|
||||
"plan-ceo-review": {
|
||||
"SKILL.md": [
|
||||
"## Step 0: Detect platform and base branch",
|
||||
"## Step 0: Nuclear Scope Challenge + Mode Selection",
|
||||
"### 0A. Premise Challenge",
|
||||
"### 0B. Existing Code Leverage",
|
||||
"### 0C. Dream State Mapping",
|
||||
"### 0D. Alternatives (reusable decision procedure)",
|
||||
"### 0E. Mode Selection",
|
||||
"### 0F. Expansion Framing (shared by EXPANSION and SELECTIVE EXPANSION)",
|
||||
"### 0G. Mode-Specific Analysis",
|
||||
"### 0H. Persist CEO Plan (EXPANSION and SELECTIVE EXPANSION only)",
|
||||
"### 0I. Temporal Interrogation (EXPANSION, SELECTIVE EXPANSION, and HOLD modes)"
|
||||
],
|
||||
"sections/review-sections.md": [
|
||||
"### Section 1: Architecture Review",
|
||||
"### Section 2: Error & Rescue Map",
|
||||
"### Section 3: Security & Threat Model",
|
||||
"### Section 4: Data Flow & Interaction Edge Cases",
|
||||
"### Section 5: Code Quality Review",
|
||||
"### Section 6: Test Review",
|
||||
"### Section 7: Performance Review",
|
||||
"### Section 8: Observability & Debuggability Review",
|
||||
"### Section 9: Deployment & Rollout Review",
|
||||
"### Section 10: Long-Term Trajectory Review",
|
||||
"### Section 11: Design & UX Review (skip if no UI scope detected)"
|
||||
]
|
||||
},
|
||||
"plan-eng-review": {
|
||||
"SKILL.md": [
|
||||
"### Step 0: Scope Challenge"
|
||||
],
|
||||
"sections/review-sections.md": [
|
||||
"### 1. Architecture review",
|
||||
"### 2. Code quality review",
|
||||
"### 3. Test review",
|
||||
"### 4. Performance review"
|
||||
]
|
||||
},
|
||||
"setup-deploy": {
|
||||
"SKILL.md": [
|
||||
"### Step 1: Check existing configuration",
|
||||
"### Step 2: Detect platform",
|
||||
"### Step 3: Platform-specific setup",
|
||||
"### Step 4: Write configuration",
|
||||
"### Step 5: Verify",
|
||||
"### Step 6: Summary"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
/**
|
||||
* Required workflow steps (C1 follow-up, 2026-10-04). The qa workflow judge
|
||||
* passed a /qa bundle with "Phase 10: Report" removed and the review judge
|
||||
* passed /review with "Step 3: Get the diff" removed, so the judges do not
|
||||
* cover a missing step. This deterministic check does: every pinned step or
|
||||
* phase heading of the judged workflow skills must still be in the generated
|
||||
* SKILL.md or section file, in order. The pinned list was derived from the
|
||||
* templates' step headings; a new step heading must be added to it.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const PINNED = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures', 'workflow-required-steps.json'), 'utf8')) as {
|
||||
skills: Record<string, Record<string, string[]>>;
|
||||
};
|
||||
const STEP = /^#{2,4} (?:(?:Step|Phase|Section) \d+[A-Za-z0-9.]*|Phases \d+-\d+|\d+[A-Za-z]?(?:\.\d+)?\.)[ :].*$/;
|
||||
|
||||
/** Heading lines outside code fences. */
|
||||
function headings(text: string): string[] {
|
||||
let fence = false;
|
||||
return text.split('\n').flatMap(line => {
|
||||
if (/^\s*(?:```|~~~)/.test(line)) { fence = !fence; return []; }
|
||||
return !fence && /^#{2,4} /.test(line) ? [line.trim()] : [];
|
||||
});
|
||||
}
|
||||
|
||||
/** Missing or out-of-order required steps, each named. */
|
||||
function missingSteps(skill: string, file: string, text: string, required: readonly string[]): string[] {
|
||||
const found = headings(text);
|
||||
const problems: string[] = [];
|
||||
let last = -1;
|
||||
for (const step of required) {
|
||||
const at = found.indexOf(step, last + 1);
|
||||
if (at < 0) problems.push(found.includes(step) ? `${skill}/${file}: step out of order: "${step}"` : `${skill}/${file}: missing required step "${step}"`);
|
||||
else last = at;
|
||||
}
|
||||
return problems;
|
||||
}
|
||||
|
||||
const read = (skill: string, file: string) => fs.readFileSync(path.join(ROOT, skill, file), 'utf8');
|
||||
|
||||
describe('judged workflow skills keep their required steps', () => {
|
||||
for (const [skill, files] of Object.entries(PINNED.skills)) {
|
||||
test(`${skill}: every pinned step is present in order`, () => {
|
||||
const problems = Object.entries(files).flatMap(([file, steps]) => missingSteps(skill, file, read(skill, file), steps));
|
||||
expect(problems).toEqual([]);
|
||||
});
|
||||
|
||||
test(`${skill}: every step heading is pinned`, () => {
|
||||
const pinnedFiles = new Set(Object.keys(files));
|
||||
const sectionFiles = fs.existsSync(path.join(ROOT, skill, 'sections'))
|
||||
? fs.readdirSync(path.join(ROOT, skill, 'sections')).filter(f => f.endsWith('.md')).map(f => `sections/${f}`) : [];
|
||||
const unpinned = ['SKILL.md', ...sectionFiles].flatMap(file => headings(read(skill, file))
|
||||
.filter(h => STEP.test(h) && !(pinnedFiles.has(file) && files[file]!.includes(h)))
|
||||
.map(h => `${skill}/${file}: "${h}" (add it to test/fixtures/workflow-required-steps.json)`));
|
||||
expect(unpinned).toEqual([]);
|
||||
});
|
||||
}
|
||||
|
||||
test('the C1 control defects are reported by step name', () => {
|
||||
const qa = read('qa', 'SKILL.md');
|
||||
expect(missingSteps('qa', 'SKILL.md', qa.replace('## Phase 10: Report', '## Report'), PINNED.skills.qa!['SKILL.md']!))
|
||||
.toEqual(['qa/SKILL.md: missing required step "## Phase 10: Report"']);
|
||||
const review = read('review', 'SKILL.md');
|
||||
expect(missingSteps('review', 'SKILL.md', review.replace(/^## Step 3: Get the diff$/m, ''), PINNED.skills.review!['SKILL.md']!))
|
||||
.toEqual(['review/SKILL.md: missing required step "## Step 3: Get the diff"']);
|
||||
const fenced = review.replace(/^(## Step 3: Get the diff)$/m, '```\n$1\n```');
|
||||
expect(missingSteps('review', 'SKILL.md', fenced, PINNED.skills.review!['SKILL.md']!)).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user