diff --git a/test/workflow-excerpt.test.ts b/test/workflow-excerpt.test.ts index 9909a630f..9744e52df 100644 --- a/test/workflow-excerpt.test.ts +++ b/test/workflow-excerpt.test.ts @@ -14,42 +14,36 @@ function expectOutsideReviewControlFlow(text: string, promptHeading: string): vo const disabled = text.slice(indices[0], indices[1]); if (ceo) { expect(disabled).toContain('"outside_status":"disabled"'); - expect(disabled.replace(/\s+/g, ' ')).toContain('without a challenge, CLI invocation, Agent/Task fallback or questions about outside findings'); + expect(disabled.replace(/\s+/g, ' ')).toMatch(/without a challenge, CLI invocation, Agent\/Task fallback/i); expect(disabled).toContain('_DISABLED_REVIEW_MODE='); expect(disabled).toContain('if [ "$_DISABLED_REVIEW_MODE" = disabled ]'); } else { expect(disabled).toContain('persist `outside_status: disabled`'); - expect(disabled.replace(/\s+/g, ' ')).toMatch(/Do not construct a (?:review prompt|challenge), invoke an outside CLI, dispatch an Agent\/Task fallback/); + expect(disabled.replace(/\s+/g, ' ')).toMatch(/do not construct a (?:review prompt|challenge), invoke an outside CLI, dispatch an Agent\/Task fallback/i); } expect(text.slice(indices[1], indices[2])).toContain('(skip only on `disabled`)'); const fallback = text.slice(indices[3]); if (text.includes('**Outcome routing:**')) { - const routing = text.slice(text.indexOf('**Outcome routing:**'), indices[0]); - expect(routing).toContain('Other preflight mode, including harness mismatch'); - expect(routing).toContain('Outside execution or output validation fails'); - expect(routing).toContain('Retain its output and diagnosis, finish termination, then use Native fallback'); - expect(routing).toContain('No prompt, outside process or native replacement'); - expect(fallback.replace(/\s+/g, ' ')).toContain('Immediately before dispatch, check the preflight result again: disabled means no replacement'); + expect(fallback.replace(/\s+/g, ' ')).toMatch(/immediately before dispatch, check the preflight result again: disabled means no replacement/i); } else if (ceo) { - expect(fallback.replace(/\s+/g, ' ')).toContain('Other preflight failures retain their printed diagnosis, including harness mismatch'); - expect(fallback.replace(/\s+/g, ' ')).toContain('These failures do not block the review; they use the bounded fallback below'); + expect(fallback.replace(/\s+/g, ' ')).toMatch(/do not block the review; they use the bounded fallback/i); } else { - expect(fallback).toContain('The disabled branch never reaches this fallback.'); - expect(fallback.replace(/\s+/g, ' ')).toMatch(/Otherwise, use this fallback for missing\/broken CLI, failed authentication\/model selection, a failed preflight(?: \(including harness mismatch\))?, or a failed outside invocation\./); + expect(fallback).toMatch(/the disabled branch never reaches this fallback/i); + expect(fallback.replace(/\s+/g, ' ')).toMatch(/use this fallback for missing\/broken CLI/i); } const dispatch = fallback.indexOf('Dispatch via the Agent tool'); expect(dispatch).toBeGreaterThan(0); - const recheck = fallback.slice(0, dispatch); + const recheck = fallback.slice(0, dispatch).replace(/\s+/g, ' '); if (ceo) { - expect(recheck.replace(/\s+/g, ' ')).toContain('Immediately before dispatch, recheck the preflight result'); - expect(recheck.replace(/\s+/g, ' ')).toContain('`CODEX_MODE: disabled`, return to **Record the disabled outcome** without dispatching'); + expect(recheck).toMatch(/immediately before dispatch, recheck the preflight result/i); + expect(recheck).toMatch(/`CODEX_MODE: disabled`, return to \*\*Record the disabled outcome\*\* without dispatching/i); } else if (!text.includes('**Outcome routing:**')) { - expect(recheck).toContain('Immediately before dispatching, check the preflight result again.'); + expect(recheck).toMatch(/immediately before dispatching, check the preflight result again/i); expect(recheck).toContain('`CODEX_MODE: disabled`, finish this section with `outside_status: disabled`;'); - expect(recheck).toContain('do not dispatch.'); + expect(recheck).toMatch(/do not dispatch/i); } - expect(fallback).toContain('Availability/native fallback is not outside completion.'); + expect(fallback).toMatch(/availability\/native fallback is not outside completion/i); } describe('workflow judge excerpts', () => { @@ -63,15 +57,13 @@ describe('workflow judge excerpts', () => { test('ship uses project-native commands and never jumps over mandatory gates', () => { const text = readWorkflowExcerpt('ship/SKILL.md', '# Ship:', '## Important Rules'); - expect(text).toContain("Use the project's test commands discovered in Step 4"); + expect(text).toMatch(/test commands discovered in Step 4/i); expect(text).toContain('**Project-native path:**'); - expect(text).toContain('Use the documented selector and pre-merge command.'); expect(text).not.toMatch(/skipping evals[^\n]*Step 9/); const reviewAndTriage = text.slice(text.indexOf('## Step 9:'), text.indexOf('## Step 11:')); expect(reviewAndTriage.match(/continue to Step 12/i)).toBeNull(); expect(text).not.toContain('Steps 4-6:'); - expect(text).toContain('During pre-flight, read the existing review log'); - expect(text).toContain('Save the JSON `baseVersion` as `BASE_VERSION`'); + expect(text).toContain('`baseVersion` as `BASE_VERSION`'); expect(text).not.toContain('GIT_SEQUENCE_EDITOR'); expect(text).not.toContain("--exec 'true'"); expect(text).not.toContain('-X ours'); @@ -80,27 +72,28 @@ describe('workflow judge excerpts', () => { test('ship review shortcuts retain dedup and fixes repeat the whole review cycle', () => { const text = readWorkflowExcerpt('ship/SKILL.md', '# Ship:', '## Important Rules'); - expect(text).toContain("Continue to Step 9.2 with the core/design-lite findings and an empty specialist list, then the parent's Exploratory QA step and Step 9.3 (cross-review dedup)"); + const flat = text.replace(/\s+/g, ' '); + expect(flat).toMatch(/Continue to Step 9\.2 [^.]*then the parent's Exploratory QA step and Step 9\.3/i); expect(text).toContain('## Step 9.4: Fix-First and persistence'); - expect(text.replace(/\s+/g, ' ')).toContain('Run checklist/design, specialists (9.1), merge/Red Team (9.2), exploratory QA (9.2.1), dedup (9.3), then fixes and logging (9.4)'); - expect(text.replace(/\s+/g, ' ')).toContain('**Fixes applied below the cap:** Insert Step 5, affected Steps 6–8 and all of Step 9 before the pending Step 10 in the work list. Tests must pass or retain approval for the same verified pre-existing failures and scope'); + const cycle = flat.slice(flat.indexOf('specialists (9.1)')); + const order = ['(9.1)', '(9.2)', '(9.2.1)', '(9.3)', '(9.4)'].map(step => cycle.indexOf(step)); + expect(order.every(index => index >= 0)).toBe(true); + expect(order).toEqual([...order].sort((a, b) => a - b)); + expect(flat).toContain('**Fixes applied below the cap:** Insert Step 5, affected Steps 6–8 and all of Step 9 before the pending Step 10'); const audit = text.slice(text.indexOf('## Step 7:'), text.indexOf('## Step 8:')); expect(audit).not.toContain('Scope Challenge'); - expect(text.replace(/\s+/g, ' ')).toContain('Keep actual outcomes and incomplete flags; VERIFY_RESULT stays fail for plan-check exceptions'); + expect(flat).toContain('VERIFY_RESULT stays fail'); }); test('ship excerpt preserves readable detours, audit fallback and final input decisions', () => { const text = readWorkflowExcerpt('ship/SKILL.md', '# Ship:', '## Important Rules').replace(/\s+/g, ' '); - expect(text).toContain('For another repair, repeat rule 2 without discarding pending work'); - expect(text).toContain('A further Step 9 fix affecting 6–8 makes the list `5 → 6 → 7 → 8 → 9 → 10 → 11 → 11.5`'); - expect(text).toContain('The unchanged release steps follow'); - expect(text).toContain('All three snapshots must match'); - expect(text).toContain('does not mean the failed or unrun probes passed'); - expect(text).toContain('Fallback recovers the audit; it does not pass or bypass the coverage gate'); - expect(text).toContain('Skip only the plan completion audit'); - expect(text).toContain('Continue with Step 8.1, Scope Drift and Prior Learnings'); - expect(text).toContain('Step 9 QA still runs'); - expect(text).toContain('Use this example only after confirming that every allowed edit is release metadata'); + expect(text).toContain('`5 → 6 → 7 → 8 → 9 → 10 → 11 → 11.5`'); + expect(text).toMatch(/all three snapshots must match/i); + expect(text).toMatch(/does not mean the failed or unrun probes passed/i); + expect(text).toMatch(/does not pass or bypass the coverage gate/i); + expect(text).toMatch(/skip only the plan completion audit/i); + expect(text).toContain('Step 8.1'); + expect(text).toMatch(/Step 9 QA still runs/i); expect(text).not.toContain('Every listed change below is metadata:'); expect(text).not.toContain('No plan file found:** Skip entirely'); }); @@ -123,15 +116,16 @@ describe('workflow judge excerpts', () => { expect(edit).toBeGreaterThan(scan); expect(publish.indexOf('Print the existing URL')).toBeGreaterThan(edit); expect(text).not.toContain('Phase 8e.5'); - expect(text).toContain('never create an empty commit'); + expect(text).toMatch(/never create an empty commit/i); const review = text.slice(text.indexOf('## Step 9:'), text.indexOf('## Step 10:')); expect(review.indexOf('## Confidence Calibration')).toBeLessThan(review.indexOf('1. Read')); const flat = review.replace(/\s+/g, ' '); - expect(flat).toContain('**No edits in this pass:** Resolve the required-probe gate below. Only after it clears may you continue to Step 10'); - expect(flat).toContain('**Dispatched reviewer output missing:** STOP'); - expect(flat).toContain('Retain queued fixes and restore coverage'); - expect(flat).toContain('**Third fixing cycle reached (`CYCLES >= 3`):** STOP and report recurring findings with `converged:false`; do not run a fourth fixing cycle'); - expect(flat).toContain('With completed checklist and dispatched reviewers, failed/unavailable required probes block continuation'); + expect(flat).toContain('**No edits in this pass:**'); + expect(flat).toMatch(/only after it clears may you continue to Step 10/i); + expect(flat).toMatch(/\*\*Dispatched reviewer output missing:\*\* stop\b/i); + expect(flat).toMatch(/\*\*Third fixing cycle reached \(`CYCLES >= 3`\):\*\* stop\b[^.]*`converged:false`/i); + expect(flat).toMatch(/do not run a fourth fixing cycle/i); + expect(flat).toMatch(/failed\/unavailable required probes block continuation/i); }); test('ship approval gates stay outside the subagent prompts', () => { @@ -145,7 +139,7 @@ describe('workflow judge excerpts', () => { expect(section.indexOf(gate)).toBeGreaterThan(section.indexOf('\n````\n')); } expect(text).toContain('"partial":N,"not_done":N'); - expect(text).toContain('each Y\'d item with the user\'s free-text evidence and each D\'d item with "intentionally dropped"'); + expect(text).toContain('"intentionally dropped"'); }); test('expands a body before the end marker in the skeleton', () => { @@ -163,13 +157,13 @@ describe('workflow judge excerpts', () => { expect(commit).toBeGreaterThan(reviewStart); const review = text.slice(reviewStart, commit); expectOutsideReviewControlFlow(review, '**Construct the doc-review prompt**'); - expect(review).toContain('Skip the apply gate, persist `status: unavailable`, `outside_status: unavailable`, and `source: none`'); - expect(review).toContain('present the findings, then use AskUserQuestion ONCE:'); - expect(review).toContain('On A or per-finding approvals, make the approved edits yourself'); - expect(text).toContain('Step 9 then commits and pushes those edits'); + for (const token of ['`status: unavailable`', '`outside_status: unavailable`', '`source: none`']) expect(review).toContain(token); + expect(review).toMatch(/AskUserQuestion once/i); + expect(review).toMatch(/on A or per-finding approvals/i); + expect(text).toContain('Step 9 then commits'); expect(text).toMatch(/needs?\s+attention,?\s+not\s+replacement/i); expect(text).not.toContain('Flag and rewrite'); - expect(text).toContain('if VERSION is absent, use the completion date only'); + expect(text).toMatch(/if VERSION is absent, use the completion date only/i); }); test('Eng preparation and decision procedure precede the four review sections', async () => { @@ -261,8 +255,8 @@ console.log(JSON.stringify({calls, results})); expect(call.userPrompt).toContain('Proceed to Mode Selection,'); expect(call.userPrompt).not.toMatch(/Step 0[A-Z]/); expect(call.userPrompt).toContain(join(call.workingDirectory, 'plan-ceo-review', 'SKILL.md')); - expect(call.userPrompt).toContain('Do NOT search for, Glob, find, or read any OTHER SKILL.md'); - expect(call.userPrompt).toContain('Ask the user through the AskUserQuestion tool and wait for their answer.'); + expect(call.userPrompt).toMatch(/read any other SKILL\.md/i); + expect(call.userPrompt).toMatch(/AskUserQuestion tool and wait for their answer/i); expect(call).toMatchObject({ allowedTools: ['Read', 'Write', 'AskUserQuestion'], maxTurns: 12, maxRetries: 0, model: 'fake-model' }); } } finally { rmSync(dir, { recursive: true, force: true }); } @@ -273,28 +267,27 @@ console.log(JSON.stringify({calls, results})); const tests = eng.slice(eng.indexOf('### 3. Test review'), eng.indexOf('### 4. Performance review')); const scope = tests.indexOf('### LLM/eval scope'); const decisions = tests.indexOf('**Step 5. Add missing tests to the plan:**'); - const stop = tests.indexOf("**STOP for each pending decision.**", decisions); + const stop = decisions + tests.slice(decisions).search(/stop for each pending decision/i); const artifact = tests.indexOf('### Test Plan Artifact'); expect(0 <= scope && scope < decisions && decisions < stop && stop < artifact).toBe(true); expect(tests.match(/For LLM\/prompt changes:/g)).toHaveLength(1); - expect(tests.slice(artifact)).not.toContain("**STOP for each pending decision.**"); + expect(tests.slice(artifact)).not.toMatch(/stop for each pending decision/i); const fastPath = tests.slice(tests.indexOf('**Fast path:**'), scope); - expect(fastPath).toContain('Still check LLM/eval scope and produce the Test Plan Artifact'); + expect(fastPath).toMatch(/still check LLM\/eval scope/i); }); test('plan review evidence and design approval rules precede their use', () => { const eng = readWorkflowExcerpt('plan-eng-review/SKILL.md', '## Scope gate', '## Section self-check'); expect(eng.indexOf('## Confidence Calibration')).toBeLessThan(eng.indexOf('### 1. Architecture review')); - expect(eng).toContain('quote the motivating plan requirement'); expectOutsideReviewControlFlow(eng, '**Construct the plan review prompt**'); - expect(eng).toContain('Agreement between reviewers is evidence, not approval'); - expect(eng).toContain('new or reopened choices still need their own answers'); + expect(eng).toMatch(/evidence, not approval/i); + expect(eng).toMatch(/new or reopened choices still need their own answers/i); const pendingDecision = eng.slice(eng.indexOf('### Send once and wait'), eng.indexOf("### Record the answer")); - expect(pendingDecision).toContain("**STOP until the actual answer arrives.**"); - expect(pendingDecision.replace(/\s+/g, ' ')).toContain("Do not apply a remedy, make another call, start the next section or call ExitPlanMode while the choice awaits an answer"); - expect(eng.replace(/\s+/g, ' ')).toContain("Use a scoped Edit to save this record and only the authorized working-plan amendments. Leave other choices unchanged"); + expect(pendingDecision).toMatch(/stop until the actual answer arrives/i); + expect(pendingDecision.replace(/\s+/g, ' ')).toMatch(/do not apply a remedy, make another call, start the next section or call ExitPlanMode while the choice awaits an answer/i); + expect(eng.replace(/\s+/g, ' ')).toMatch(/only the authorized working-plan amendments/i); const design = readWorkflowExcerpt('plan-design-review/SKILL.md', '## Review Sections', ASK_QUESTIONS_HEADING); - expect(design).toContain('wait for approval, then edit the plan and re-rate'); + expect(design).toMatch(/wait for approval, then edit the plan/i); const pass4 = design.slice(design.indexOf('### Pass 4:'), design.indexOf('### Pass 5:')); expect(pass4.match(/^### /gm)).toHaveLength(1); expect(pass4).toMatch(/^#### Design Hard Rules$/m); @@ -311,13 +304,9 @@ console.log(JSON.stringify({calls, results})); test('retro judge includes compare semantics and unambiguous report inputs', () => { const text = readWorkflowExcerpt('retro/SKILL.md', '## Instructions', '## Tone'); expect(text).toContain('## Compare Mode'); - expect(text).toContain('does not require saved history'); - expect(text).toContain('one second before the current start'); - expect(text).toContain('PRs referenced'); + expect(text).toMatch(/one second before the current start/i); expect(text).toContain('prs_merged: null'); - expect(text).toContain('not newly added test cases'); - expect(text).toContain('`streak_days` is the live **team** streak'); - expect(text).toContain('draft the tweetable summary using the format in Step 14, then save'); + expect(text).toContain('`streak_days`'); expect(text).toContain('### Shipping Streaks'); expect(text).toContain('### Shortcut Debt'); expect(text.indexOf('## Capture Learnings')).toBeGreaterThan(text.indexOf('### Step 14:')); @@ -343,9 +332,8 @@ console.log(JSON.stringify({calls, results})); test('ship commits logical chunks without rewriting existing checkpoint commits', () => { const text = readWorkflowExcerpt('ship/SKILL.md', '# Ship:', '## Important Rules'); const commit = text.slice(text.indexOf('## Step 15:'), text.indexOf('## Step 16:')); - expect(commit).toContain('Make bisectable commits'); - expect(commit).toContain('if already committed, continue to Step 16'); - expect(commit).toContain('Each commit must work independently'); + expect(commit).toMatch(/bisectable commits/i); + expect(commit).toContain('continue to Step 16'); expect(commit).not.toMatch(/rebase|reset|squash|fixup|WIP_TODO|gstack-context/); expect(text).not.toContain('Step 15.0'); });