From ff54f3c9555cdffaffee336f12f64c122177aca4 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 01:02:23 +0000 Subject: [PATCH 01/13] test(W7h): drop issue-number pins in land-and-deploy-postfail, match guards case-insensitively W7h (t09 F12, F13). One test file. Removed assertion groups: - `cli/cli#3442` / `cli/cli#13380` pins (t09 F12). Invariant: none of its own (upstream archaeology; the IDs stay in the template, only the pin goes). The behavior they decorated, "after any non-zero `gh pr merge` exit, query authoritative state before retrying", is now a kept free meaning check. - `Universal invariant` label pin. Invariant: query-before-retry. Remaining protection: the new case-insensitive check `/non-zero exit from `gh pr merge`, query authoritative PR state before retrying/i` (stronger: the mutation that inverts the rule now goes red; the old test stayed green). Loosened to case-insensitive meaning checks (kept, W7 keep list, data-loss and destructive-op guards): STOP worktree cleanup without removing, no `--force`, primary working tree never removed, CLOSED -> stop, never replay a merge after MERGED, one direct fallback, no fallback from a direct attempt, the OPEN/no-auto/no-queue fallback precondition, branch-absent vs check-failed messages. Gate-tier E2E also covering the flow: land-and-deploy-workflow (gate), judge `land-and-deploy/SKILL.md workflow` (gate lane). Cosmetic pins dropped (no control needed): capitals on STOP / Do NOT. Mutation log: .context/mutation-land-and-deploy-postfail.md (12 mutations, 0 unexpected: reorder, field delete and 7 safety deletes red; 3 rewordings green). --- test/land-and-deploy-postfail.test.ts | 38 ++++++++++++--------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/test/land-and-deploy-postfail.test.ts b/test/land-and-deploy-postfail.test.ts index 840c303e4..8446921b2 100644 --- a/test/land-and-deploy-postfail.test.ts +++ b/test/land-and-deploy-postfail.test.ts @@ -8,12 +8,11 @@ * only the STOP-Read pointer). After ANY non-zero `gh pr merge`, the skill * must query authoritative PR state via * GraphQL (including queue membership) and - * branch on the result instead of blindly retrying `gh pr merge` (cli/cli#3442, - * cli/cli#13380). + * branch on the result instead of blindly retrying `gh pr merge`. * * Static invariants pin: * - §4a-postfail header present - * - Universal invariant text + reference to upstream gh bugs + * - Query-before-retry invariant after any non-zero merge exit * - All three state branches (MERGED, OPEN, CLOSED) named explicitly * - MERGED branch: capture merge SHA via mergeCommit.oid * - MERGED branch: non-destructive worktree cleanup with uncommitted-work guard @@ -52,12 +51,9 @@ describe("PR #1620 §4a-postfail in land-and-deploy template", () => { expect(postfail).toBeLessThan(queue); }); - test("Universal invariant + upstream gh bug references", () => { - const body = readTmpl(); - expect(body).toMatch(/Universal invariant/); - expect(body).toMatch(/non-zero exit from `gh pr merge`/); - expect(body).toMatch(/cli\/cli#3442/); - expect(body).toMatch(/cli\/cli#13380/); + test("any non-zero merge exit queries authoritative state before retrying", () => { + const body = readTmpl().replace(/\s+/g, " "); + expect(body).toMatch(/non-zero exit from `gh pr merge`, query authoritative PR state before retrying/i); }); test("Authoritative state query includes auto request and queue membership", () => { @@ -82,9 +78,9 @@ describe("PR #1620 §4a-postfail in land-and-deploy template", () => { test("MERGED worktree cleanup is non-destructive (uncommitted-work guard)", () => { const body = readTmpl(); expect(body).toMatch(/uncommitted work/); - expect(body).toMatch(/STOP worktree cleanup without removing/); - expect(body).toMatch(/Do NOT use `--force`/); - expect(body).toMatch(/Do NOT remove the user's primary working tree/); + expect(body).toMatch(/stop worktree cleanup without removing/i); + expect(body).toMatch(/do not use `--force`/i); + expect(body).toMatch(/do not remove the user's primary working tree/i); }); test("MERGED branch continues to §4b CI auto-deploy detection", () => { @@ -114,10 +110,10 @@ describe("PR #1620 §4a-postfail in land-and-deploy template", () => { test("MERGED branch reconciliation distinguishes branch-absent from check-failed", () => { const body = readTmpl(); // exit 0 + empty output = already clean (idempotent re-runs)... - expect(body).toMatch(/already been cleaned up/); + expect(body).toMatch(/already been cleaned up/i); // ...non-zero exit = unknown state, never read as a clean branch. - expect(body).toMatch(/Couldn't verify remote branch state/); - expect(body).toMatch(/never read a failed check as a clean branch/); + expect(body).toMatch(/couldn't verify remote branch state/i); + expect(body).toMatch(/never read a failed check as a clean branch/i); }); test("OPEN branch checks autoMergeRequest before treating as failure", () => { @@ -128,15 +124,15 @@ describe("PR #1620 §4a-postfail in land-and-deploy template", () => { test("CLOSED branch STOPs", () => { const body = readTmpl(); - expect(body).toMatch(/state == "CLOSED".*[\s\S]{0,200}STOP/); + expect(body).toMatch(/state == "CLOSED"[\s\S]{0,200}\bstop\b/i); }); test("Hard rule: no replay after MERGED and only one guarded direct fallback", () => { - const body = readTmpl(); - expect(body).toMatch(/never\s+replay a merge after MERGED/); - expect(body).toContain('one direct fallback'); - expect(body).toContain('There is no fallback from a direct attempt'); - expect(body).toContain('readback has confirmed OPEN, no auto request and no queue entry'); + const body = readTmpl().replace(/\s+/g, " "); + expect(body).toMatch(/never replay a merge after MERGED/i); + expect(body).toMatch(/one direct fallback/i); + expect(body).toMatch(/no fallback from a direct attempt/i); + expect(body).toMatch(/confirmed OPEN, no auto request and no queue entry/i); }); test("Generated merge-and-deploy.md carries the §4a-postfail section (atomic regen per T-Codex-3)", () => { From 8b1e3423a7bd81ea0d01ec971e34de0483a71b2a Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 01:03:14 +0000 Subject: [PATCH 02/13] test(W7c): ship-apple-gate matches its gates on meaning, not capitals W7c (t06 #19). One test file. Removed assertion groups: - "A genuine blocker may require a safety or named documentation-risk decision" and "routine interactions and blocking decisions above" sentence pins. Invariant: routine release authorization never waives a blocking safety/docs decision. Remaining protection: kept free meaning check `/stop for that decision rather than treating release authorization as a waiver/i` plus `/named documentation-risk exception before distribution/i`. - Full-sentence "in `read-only` mode against the selected release source". Invariant: the docs preflight is read-only. Remaining protection: kept free check on "`read-only` mode". Loosened (case-insensitive, kept): apply-only-to-repository-landing scope line, CLASSIFY-before-credentials (credential boundary), never abort a store release over branch topology (demonstrated failure), two routine interactions (numeric limit), stop-for-decision (consent/stop line). Kept unchanged: manifest trigger routing, Apple read before the branch gate, byte-exact branch gate appearing once, docs preflight before the authorization moment, fastlane/iris/appPriceSchedules command anchors, stale-wording bans. No gate-tier E2E covers the Apple adapter, so every invariant keeps a free check here. Cosmetic pins dropped: capitals on ONLY / CLASSIFY / STOP. Mutation log: .context/mutation-ship-apple-gate.md (14 mutations, 0 unexpected: 2 reorders, 1 marker and 6 safety mutations red; 5 rewordings green). --- test/ship-apple-gate.test.ts | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/test/ship-apple-gate.test.ts b/test/ship-apple-gate.test.ts index 38a60b4ec..fc0766609 100644 --- a/test/ship-apple-gate.test.ts +++ b/test/ship-apple-gate.test.ts @@ -39,7 +39,7 @@ describe("ship Apple gate ordering (R2)", () => { test("store distribution explicitly bypasses the branch/PR ceremony", () => { expect(SKELETON).toContain("Store distribution proceeds"); - expect(SKELETON).toMatch(/branch gate and repository-landing pipeline below apply ONLY to\s*\n?repository-landing asks/); + expect(SKELETON).toMatch(/branch gate and repository-landing pipeline below apply only to\s+repository-landing asks/i); }); test("the non-Apple branch gate is byte-unchanged and appears exactly once", () => { @@ -50,30 +50,23 @@ describe("ship Apple gate ordering (R2)", () => { test("the adapter section exists in the union with its battle-tested spine", () => { const section = readFileSync(join(ROOT, "ship", "sections", "apple-release.md"), "utf-8"); - for (const anchor of [ - "one authorization moment", - "fastlane spaceauth", - "iris/v1/apiKeys", - "appPriceSchedules", - "CLASSIFY the error before touching credentials", - "Never abort an App Store release over branch topology", - ]) { + for (const anchor of ["one authorization moment", "fastlane spaceauth", "iris/v1/apiKeys", "appPriceSchedules"]) { expect(section).toContain(anchor); } + expect(section).toMatch(/classify the error before touching credentials/i); + expect(section).toMatch(/never abort an App Store release over branch topology/i); }); test("routine interaction limits cannot waive blocking documentation or safety decisions", () => { for (const name of ["apple-release.md.tmpl", "apple-release.md"]) { - const section = readFileSync(join(ROOT, "ship", "sections", name), "utf-8"); - expect(section).toContain("Plan for two routine interactions"); - expect(section).toContain("A genuine blocker may require a safety or named documentation-risk decision"); - expect(section).toContain("STOP for that decision rather than treating release authorization as a waiver"); - expect(section).toContain("routine interactions and blocking decisions above"); + const section = readFileSync(join(ROOT, "ship", "sections", name), "utf-8").replace(/\s+/g, " "); + expect(section).toMatch(/two routine interactions/i); + expect(section).toMatch(/stop for that decision rather than treating release authorization as a waiver/i); expect(section).not.toContain("exactly two interactions, and no others"); expect(section).not.toContain("two permitted interactions"); expect(section.indexOf("**Documentation preflight:**")).toBeLessThan(section.indexOf("## The one authorization moment")); - expect(section).toContain("in `read-only` mode against the selected release source"); - expect(section).toContain("Resolve blockers or obtain an explicit named documentation-risk exception before distribution"); + expect(section).toContain("`read-only` mode"); + expect(section).toMatch(/named documentation-risk exception before distribution/i); } }); }); From 356f7cb6ce28d8d5fbdffcda76c5291c697337bc Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 01:05:00 +0000 Subject: [PATCH 03/13] test(W7c): ship-reentry-gates keeps stop/consent meaning, drops sentence pins MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit W7c (t08 F3). One test file. Removed assertion groups (invariant -> remaining protection): - "Retain queued fixes and restore coverage" -> a missing reviewer keeps queued fixes: gate E2E ship-skipped-queued-finding (gate) and judge `ship/SKILL.md workflow` (gate lane); the stop itself stays a free check. - qa decline print line ("Test bootstrap previously declined — skipping.") -> qa skips bootstrap after a saved decline: kept free meaning check `/BOOTSTRAP_DECLINED ... skip the rest of bootstrap/i`. - "with the user's free-text evidence" -> UNVERIFIABLE items are confirmed one by one: kept free `/per-item confirmation/i` here plus the blanket-confirm check in ship-plan-completion-invariants. - Docs reentry sentences ("On reentry, reuse only ...", its duplicate, "base/ input hashes still match", "retain its actual status and scope", "Otherwise use Blocked recovery ...") -> reentry reuses only this invocation's validated evidence: kept free checks "never reuse an audit across invocations" and "reentry never resets the count or authorizes a launch"; gate E2Es ship-docsync-stale-before, ship-docsync-stale-after, ship-docsync-recovery, ship-docsync-late-result (all gate). - "left uncommitted after Step 15" -> late commits take only remaining changes: kept free "never create an empty commit" and "preserve unrelated user files"; gate E2E ship-local-workflow (gate). Loosened to case-insensitive meaning checks (stop/consent/approval lines, always kept): STOP blocks advancement, answer each AskUserQuestion, routine authorization never waives gates, dispatched-reviewer-missing -> stop, CYCLES >= 3 -> stop, Any N -> stop as NOT DONE, resume only after verified, no second deferral, per-item confirmation, explicit Add-tests choice overrides the decline for this invocation only, framework approval, first entry launches the audit, no third attempt, unwaivable redaction/security gates. Kept unchanged: bootstrap branch order, no `rm` in the decline branch, the Step 14 -> 14.5 routing line, stale-string bans, `NEW_TITLE` field and `--title "$NEW_TITLE"`. Cosmetic pins dropped: STOP capitals, "mandatory". Mutation log: .context/mutation-ship-reentry-gates.md (17 mutations, 0 unexpected: 1 reorder, 2 marker and 9 safety mutations red; 5 rewordings green). --- test/ship-reentry-gates.test.ts | 54 ++++++++++++++------------------- 1 file changed, 23 insertions(+), 31 deletions(-) diff --git a/test/ship-reentry-gates.test.ts b/test/ship-reentry-gates.test.ts index 3a729a2bb..6f6c5cb8e 100644 --- a/test/ship-reentry-gates.test.ts +++ b/test/ship-reentry-gates.test.ts @@ -8,69 +8,61 @@ const read = (file: string) => readFileSync(new URL(`../${file}`, import.meta.ur const compact = (text: string) => text.replace(/\s+/g, ' '); const ctx = { host: 'claude' as const, skillName: 'ship', tmplPath: '', paths: HOST_PATHS.claude }; -test('ship STOP blocks advancement while retaining the stated repair route', () => { +test('ship stop lines block advancement while retaining the stated repair route', () => { const entry = compact(read('ship/SKILL.md.tmpl')); - expect(entry).toContain('STOP blocks advancement until the stated repair/resume route clears; without one, end this attempt'); - expect(entry).toContain('Answer each AskUserQuestion before continuing'); + expect(entry).toMatch(/stop blocks advancement until the stated repair\/resume route clears; without one, end this attempt/i); + expect(entry).toMatch(/answer each AskUserQuestion before continuing/i); + expect(entry).toMatch(/routine authorization never waives those gates or their required user decisions/i); const review = compact(read('ship/sections/review-army.md.tmpl')); - expect(review).toContain('**Dispatched reviewer output missing:** STOP'); - expect(review).toContain('Retain queued fixes and restore coverage'); - expect(review).toContain('**Third fixing cycle reached (`CYCLES >= 3`):** STOP'); - expect(entry).toContain('Routine authorization never waives those gates or their required user decisions'); + expect(review).toMatch(/\*\*Dispatched reviewer output missing:\*\* stop\b/i); + expect(review).toMatch(/\*\*Third fixing cycle reached \(`CYCLES >= 3`\):\*\* stop\b/i); }); test('a new ship bootstrap choice overrides only the saved decline, not framework selection', () => { const ship = generateTestBootstrap(ctx); const decline = ship.slice(ship.indexOf('**If BOOTSTRAP_DECLINED**'), ship.indexOf('**If NO ecosystem marker matched:**')); - expect(compact(decline)).toContain("Step 5's explicit Add tests choice overrides that marker for this invocation only"); - expect(compact(decline)).toContain('continue to runtime detection and B2–B3, including framework approval'); + expect(compact(decline)).toMatch(/explicit Add tests choice overrides that marker for this invocation only/i); + expect(compact(decline)).toMatch(/including framework approval/i); expect(decline).not.toContain('rm '); expect(ship.indexOf('**If ANY existing-test evidence appears**')).toBeLessThan(ship.indexOf('**If BOOTSTRAP_DECLINED**')); const qa = generateTestBootstrap({ ...ctx, skillName: 'qa' }); - expect(qa).toContain('**If BOOTSTRAP_DECLINED** appears: Print "Test bootstrap previously declined — skipping." **Skip the rest of bootstrap.**'); + expect(compact(qa)).toMatch(/BOOTSTRAP_DECLINED[^\n]{0,120}skip the rest of bootstrap/i); expect(qa).not.toContain("Step 5's explicit Add tests choice"); }); test('an unverified item answered not done enters the existing decision before continuing', () => { const gate = generatePlanCompletionGateShip(ctx); const exits = compact(gate.slice(gate.indexOf('**Exit conditions:**'), gate.indexOf('**Cap.**'))); - expect(exits).toContain('Any N: STOP and report that item as NOT DONE'); - expect(exits).toContain('Resume only after its required work is verified'); - expect(exits).toContain('no second deferral choice'); + expect(exits).toMatch(/any N: stop and report that item as NOT DONE/i); + expect(exits).toMatch(/resume only after its required work is verified/i); + expect(exits).toMatch(/no second deferral choice/i); expect(exits).not.toContain('re-running /ship'); - expect(gate).toContain('Per-item confirmation is mandatory'); - expect(gate).toContain('with the user\'s free-text evidence'); + expect(gate).toMatch(/per-item confirmation/i); }); test('docs reentry distinguishes the initial audit from same-invocation accepted evidence', () => { const docs = compact(read('ship/sections/documentation.md.tmpl')); const entry = docs.slice(0, docs.indexOf('## Prepare the candidate')); - expect(entry).toContain('First entry always launches the initial audit'); - expect(entry).toContain('On reentry, reuse only this invocation\'s validated audit or named-risk decision'); - expect(entry).toContain('Reentry never resets the count or authorizes a launch'); - expect(entry).toContain("this invocation's validated audit or named-risk decision"); - expect(entry).toContain('base/input hashes still match'); - expect(entry).toContain('retain its actual status and scope'); - expect(entry).toContain('Otherwise use Blocked recovery, not an unconditional launch'); - expect(docs).toContain('never a third attempt, even after Step 16 changes'); - expect(docs).toContain('never reuse an audit across invocations'); - expect(docs).toContain('Unconfirmed writers, ownership violations, unauthorized Git mutation and redaction/security gates cannot be waived'); + expect(entry).toMatch(/first entry always launches the initial audit/i); + expect(entry).toMatch(/reentry never resets the count or authorizes a launch/i); + expect(docs).toMatch(/never a third attempt/i); + expect(docs).toMatch(/never reuse an audit across invocations/i); + expect(docs).toMatch(/unauthorized Git mutation and redaction\/security gates cannot be waived/i); }); test('late behavioral repairs rebuild before the docs decision and commit only remaining changes', () => { const entry = compact(read('ship/SKILL.md.tmpl')); expect(entry).toContain('A range ending at Step 14 does not enter Step 14.5'); - expect(entry).toContain('rebuild and compare again before stage 3 decides documentation freshness'); + expect(entry).toMatch(/rebuild and compare again before stage 3/i); const commit = entry.slice(entry.indexOf('### 5. Report, then push'), entry.indexOf('## Step 17:')); - expect(commit).toContain('left uncommitted after Step 15'); - expect(commit).toContain('never create an empty commit'); - expect(commit).toContain('Preserve unrelated user files'); + expect(commit).toMatch(/never create an empty commit/i); + expect(commit).toMatch(/preserve unrelated user files/i); }); test('the title is prefixed once before the exact scanned value is published', () => { const body = compact(read('ship/sections/pr-body.md.tmpl')); - expect(body).toContain("Use Step 18's `NEW_TITLE` unchanged; its version prefix is already present"); + expect(body).toContain("Step 18's `NEW_TITLE` unchanged"); expect(body).not.toContain('`NEW_TITLE`, prefixed with'); expect(body).toContain('--title "$NEW_TITLE"'); - expect(body).toContain('the same scanned `NEW_TITLE`'); + expect(body).toContain('scanned `NEW_TITLE`'); }); From 842fceba78124522aef8dc7ff24071067cdc727f Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 01:07:59 +0000 Subject: [PATCH 04/13] test(W7c): ship-plan-completion-invariants keeps tokens and gates, drops prose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit W7c (t06 #9, #10, #11; #7/#8 already landed with W7a). One test file. Removed assertion groups (invariant -> remaining protection): - Execution-only extraction sentences ("implementation and test-creation requirements", "retain its command, expected outcome and source verbatim", "not EXTERNAL-STATE merely because ...", "Keep genuine external-state ...") -> execution-only checks route to Step 8.1/9 outside counts and are never DONE statically: kept free checks `Step 8.1/9`, outside implementation counts, `never DONE from static inspection`, do-not-waive; gate E2E ship-exploratory-plan-checks (gate). - Plan-verification prep sentences (~17: "Clarify unknown outcomes", browser vs functional items, "An API URL is not automatically a page", "Only browser evidence needs screenshots", smoke-budget and command-limit sentences, risk acceptance wording) -> plan checks are collected now, executed by the Step 9.2.1 owner before Fix-First, and never silently waived: kept free checks on the heading list, `Collect now; execute in Step 9`, `Step 8.2`, `Handoff to Step 9.2.1` order, `before Fix-First`, never silently waive, noninteractive -> blocked, the `VERIFY_RESULT=` line and `## Verification Results`; gate E2Es ship-exploratory-plan-checks, ship-exploratory-small-cli (gate). - Parent-processing duplicates on the generated section (same pins as the template test) -> kept on the template test; the generated copy keeps section order + seven-field anchor; freshness via gen-skill-docs. - "Check the task's terminal status", "recording the reason in the PR body and Step 20 metrics" -> failed audits use the fallback: kept free LAST-line JSON -> audit-failure fallback check and `(recommended/default)` marker. - Step 16 evidence sentences ("Independently prove ...", "exact command, exit, timestamp and log", "This recovery also applies ... stage 5") -> storage failure is not stale input: kept free table-row structure, storage column must not rerun `gstack-evidence run`, STALE/MISSING without proof, ledger unavailable never FRESH; judge `ship/SKILL.md workflow` (gate lane). - "Each commit must work independently" -> bisectable commits: kept `bisectable commits`, `continue to Step 16`, no empty commit, history- rewrite bans; judge `ship/SKILL.md workflow` (gate lane). Loosened to case-insensitive meaning checks (kept): per-item confirmation, blanket-confirm ban, classified DONE/NOT DONE by `[ -f` (MUST dropped), stop publication on new/changed/unwaived test failure + `Run Steps 5–15`, waiver reuse limited to verified pre-existing failures, fetch/push failure -> stop, never rewrite history, only a successful push, stop live child before inline audit, never race a late result, AskUserQuestion on second failure. Kept unchanged: seven-field JSON contract parse, review-mode negatives, CONTENT-SHAPE/validator tokens, Step 14 deferral routing, CHANGELOG negatives, the executed push-idempotency shell test. Cosmetic: MUST / mandatory / Do NOT capitals; "VAS-449" dropped from the describe title. Mutation log: .context/mutation-ship-plan-completion-invariants.md (19 mutations, 0 unexpected: 3 reorders, 4 markers, 7 safety red; 5 rewordings green). --- test/ship-plan-completion-invariants.test.ts | 126 +++++++------------ 1 file changed, 47 insertions(+), 79 deletions(-) diff --git a/test/ship-plan-completion-invariants.test.ts b/test/ship-plan-completion-invariants.test.ts index 48d65d446..c93fc409d 100644 --- a/test/ship-plan-completion-invariants.test.ts +++ b/test/ship-plan-completion-invariants.test.ts @@ -12,79 +12,60 @@ describe('authored ship-only plan verification handoff', () => { const ctx = { skillName: 'ship', tmplPath: 'ship/SKILL.md.tmpl', host: 'claude' as const, paths: HOST_PATHS.claude }; test('local execution-only checks remain required in Step 8.1/9 outside implementation counts', () => { const audit = generatePlanCompletionAuditShip(ctx); - const extraction = audit.slice(audit.indexOf('### Actionable Item Extraction'), audit.indexOf('### Verification Mode')); - expect(extraction).toContain('Separate deliverables from execution-only verification'); - expect(extraction).toContain('implementation and test-creation requirements'); - expect(extraction).toContain('retain its command, expected outcome and source verbatim'); + const extraction = audit.slice(audit.indexOf('### Actionable Item Extraction'), audit.indexOf('### Verification Mode')).replace(/\s+/g, ' '); + expect(extraction).toMatch(/execution-only verification/i); expect(extraction).toContain('Step 8.1/9'); - expect(extraction).toContain('outside implementation counts'); - expect(extraction).toContain('never DONE from static inspection'); - expect(extraction).toContain('not EXTERNAL-STATE merely because it has not run'); - expect(extraction).toContain('Keep genuine external-state and human-only checks in this audit'); - expect(extraction).toContain('zero implementation counts do not waive those checks'); + expect(extraction).toMatch(/outside implementation counts/i); + expect(extraction).toMatch(/never DONE from static inspection/); + expect(extraction).toMatch(/do not waive those checks/i); expect(fs.readFileSync(path.join(SHIP_DIR, 'sections/plan-completion.md.tmpl'), 'utf8')).toContain('exactly these seven fields'); const gate = generatePlanCompletionGateShip(ctx); expect(gate).toContain('Any NOT DONE items'); - expect(gate).toContain('Per-item confirmation is mandatory'); + expect(gate).toMatch(/per-item confirmation/i); }); test('review-mode extraction retains its existing categories and has no ship-only routing', () => { const audit = generatePlanCompletionAuditReview({ ...ctx, skillName: 'review' }); - expect(audit).toContain('**Test requirements:** "Test that X", "Add test for Y", "Verify Z"'); - expect(audit).not.toContain('Separate deliverables from execution-only verification'); + expect(audit).toContain('**Test requirements:**'); + expect(audit).not.toMatch(/execution-only verification/i); expect(audit).not.toContain('Step 8.1/9'); expect(audit).toContain('EXTERNAL-STATE'); }); test('preparation hands every explicit check to the report-only execution owner before Fix-First', () => { const text = generatePlanVerificationExec(ctx).replace(/\s+/g, ' '); expect(text).toContain('Collect now; execute in Step 9'); - expect(text).toContain('Do not invoke an entire QA skill or start probes here'); + expect(text).toMatch(/do not invoke an entire QA skill or start probes here/i); for (const heading of ['Verification', 'Test plan', 'Testing', 'How to test', 'Manual testing']) { expect(text).toContain(`\`${heading}\``); } - expect(text).toContain('any other explicit checks, including execution-only items retained by Step 8'); - expect(text).toContain('each exact expected outcome, source, surface, probe and safe prerequisites'); - expect(text).toContain('Clarify unknown outcomes'); - expect(text).toContain('Browser items use the declared project/plan dev URL and browser setup at execution'); - expect(text).toContain('functional items use native tools without discovering a web server'); - expect(text).toContain('An API URL is not automatically a page'); - expect(text).toContain('Only browser evidence needs screenshots'); - expect(text).toContain('If no verification section or no plan file exists, record no plan-specific items'); - expect(text).toContain('Automatic diff-scoped QA still runs. Continue to Step 8.2 Scope Drift below'); + expect(text).toContain('Step 8.2'); expect(text).toContain('Handoff to Step 9.2.1'); - expect(text).toContain('parent-owned report-only explorer must execute this complete list before Fix-First'); - expect(text).toContain('prerequisite, permission, evidence and changed-input revalidation rules'); - expect(text).toContain('Share current-input proof for overlapping smoke probes'); - expect(text).toContain('plan checks beyond that smoke budget remain required'); - expect(text).toContain('At command/time limits, mark remaining checks not run'); - expect(text).toContain("Send failed, blocked or unrun checks through Step 9's required-probe gate, never silently waive them"); - expect(text).toContain('Noninteractive runs return blocked'); + expect(text).toMatch(/before Fix-First/); + expect(text).toMatch(/never silently waive/i); + expect(text).toMatch(/noninteractive runs return blocked/i); expect(text).toContain('VERIFY_RESULT=pass only if all selected items pass, skipped only if none exist, otherwise fail'); - expect(text).toContain('Risk acceptance keeps the actual failed, blocked and unrun outcomes'); - expect(text).toContain("Report per-status counts, evidence and accepted risks in Step 19's `## Verification Results`, separately from automatic QA"); + expect(text).toContain('`## Verification Results`'); expect(text.indexOf('Collect now')).toBeLessThan(text.indexOf('Handoff to Step 9.2.1')); expect(text.indexOf('Handoff to Step 9.2.1')).toBeLessThan(text.indexOf('After execution')); }); test('the authored parent validates all seven fields and settles failed children before fallback', () => { const audit = fs.readFileSync(path.join(SHIP_DIR, 'sections/plan-completion.md.tmpl'), 'utf8'); const parent = audit.slice(audit.indexOf('**Parent processing:**')).replace(/\s+/g, ' '); - expect(parent).toContain("Check the task's terminal status"); - expect(parent).toContain('Without successful completion and valid LAST-line JSON, use the audit-failure fallback'); + expect(parent).toMatch(/valid LAST-line JSON, use the audit-failure fallback/i); expect(parent).toContain('exactly the seven declared fields'); - expect(parent).toContain('nonnegative integer counts whose classification sum equals `total_items`'); + expect(parent).toContain('classification sum equals `total_items`'); expect(parent).toContain('a string `summary`'); - expect(parent).toContain('Missing, extra or invalid fields fail'); - expect(parent).toContain('Valid no-plan/no-actionable reports retain zero counts and their summary'); - expect(parent).toContain('no final output after ~10 minutes'); - expect(parent).toContain('stop any live child and confirm it stopped before an inline audit'); - expect(parent).toContain('same extraction/classification logic; never race a late result'); - expect(parent).toContain('If that also fails, AskUserQuestion'); - expect(parent).toContain('recording the reason in the PR body and Step 20 metrics'); + expect(parent).toMatch(/missing, extra or invalid fields fail/i); + expect(parent).toMatch(/no-plan\/no-actionable reports retain zero counts/i); + expect(parent).toContain('~10 minutes'); + expect(parent).toMatch(/stop any live child and confirm it stopped before an inline audit/i); + expect(parent).toMatch(/never race a late result/i); + expect(parent).toMatch(/if that also fails, AskUserQuestion/i); expect(parent).toContain('Stop and fix the audit (recommended/default)'); const contract = audit.split('\n').find(line => line.startsWith('{"total_items":N,'))!; expect(Object.keys(JSON.parse(contract.replace(/:N([,}])/g, ':0$1'))).sort()) .toEqual(['total_items', 'done', 'changed', 'partial', 'not_done', 'unverifiable', 'summary'].sort()); expect(generatePlanCompletionGateShip(ctx)).toContain('Only PARTIAL items (no NOT DONE, no UNVERIFIABLE)'); - expect(generatePlanCompletionGateShip(ctx)).toContain('Continue with a note in the PR body. Not blocking'); + expect(generatePlanCompletionGateShip(ctx)).toMatch(/not blocking/i); }); }); @@ -101,13 +82,13 @@ function readShipUnion(): string { return t; } -describe('ship/SKILL.md — Plan Completion gate invariants (VAS-449 remediation)', () => { +describe('ship/SKILL.md — Plan Completion gate invariants', () => { const skill = readShipUnion(); test('Path concreteness rule: filesystem-pathed items must be test -f checked', () => { expect(skill).toContain('**Path concreteness rule.**'); expect(skill).toMatch(/concrete filesystem path/); - expect(skill).toMatch(/MUST be classified DONE or NOT DONE based on `\[ -f/); + expect(skill).toMatch(/classified DONE or NOT DONE based on `\[ -f/); }); test('Validator detection: project package.json validate-* scripts are auto-run', () => { @@ -117,8 +98,8 @@ describe('ship/SKILL.md — Plan Completion gate invariants (VAS-449 remediation }); test('Per-item UNVERIFIABLE confirmation: blanket-confirm is forbidden', () => { - expect(skill).toContain('**Per-item confirmation is mandatory.**'); - expect(skill).toMatch(/Do NOT use a single AskUserQuestion to blanket-confirm/); + expect(skill).toMatch(/per-item confirmation/i); + expect(skill).toMatch(/do not use a single AskUserQuestion to blanket-confirm/i); }); test('Subagent failure: fail-closed, not silent fail-open', () => { @@ -131,16 +112,11 @@ describe('ship/SKILL.md — Plan Completion gate invariants (VAS-449 remediation test('parent rejects audit errors and malformed counts instead of treating them as no plan', () => { const audit = fs.readFileSync(path.join(SHIP_DIR, 'sections/plan-completion.md'), 'utf8'); + expect(audit.indexOf('**Parent processing:**')).toBeGreaterThan(-1); + expect(audit.indexOf('**Parent processing:**')).toBeLessThan(audit.indexOf('**Audit-failure fallback:**')); const parent = audit.slice(audit.indexOf('**Parent processing:**'), audit.indexOf('**Audit-failure fallback:**')).replace(/\s+/g, ' '); - expect(parent).toContain("Check the task's terminal status"); - expect(parent).toContain('Without successful completion and valid LAST-line JSON, use the audit-failure fallback'); - expect(parent).toContain('Require exactly the seven declared fields'); - expect(parent).toContain('Missing, extra or invalid fields fail'); - expect(parent).toContain('nonnegative integer'); - expect(parent).toContain('classification sum equals `total_items`'); - expect(parent).toContain('a string `summary`'); - expect(parent).toContain('audit-failure fallback'); - expect(parent).toContain('Valid no-plan/no-actionable reports retain zero counts'); + expect(parent).toContain('seven declared fields'); + expect(parent).toMatch(/audit-failure fallback/i); }); test('successful plan audits use the declared seven-field contract without an error field', () => { @@ -167,7 +143,7 @@ describe('ship/SKILL.md — Plan Completion gate invariants (VAS-449 remediation expect(todos).toMatch(/Step 8[^\n]+P1[^\n]+plan/); expect(todos).toMatch(/Step 5[^\n]+P0[^\n]+deduplicate/); expect(todos.indexOf('Add approved deferrals')).toBeLessThan(todos.indexOf('Detect completed TODOs')); - expect(todos.replace(/\s+/g, ' ')).toContain("If creation was declined or a write failed, warn and retain unsaved follow-ups in Step 19's PR summary"); + expect(todos.replace(/\s+/g, ' ')).toMatch(/retain unsaved follow-ups in Step 19's PR summary/i); }); test('CHANGELOG uses the normal workflow without checkpoint context or squash prerequisites', () => { @@ -180,23 +156,15 @@ describe('ship/SKILL.md — Plan Completion gate invariants (VAS-449 remediation const entry = fs.readFileSync(path.join(SHIP_DIR, 'SKILL.md'), 'utf8'); const gate = entry.slice(entry.indexOf('## Step 16:'), entry.indexOf('## Step 17:')); const text = gate.replace(/\s+/g, ' '); - expect(text).toContain('STALE/MISSING: changed content, command or age, or no proven run'); - expect(text).toContain('Only receipt storage/readback failed'); - expect(text).toContain('Independently prove unchanged final content, the same command and valid age'); - expect(text).toContain('from the successful run\'s evidence'); - expect(text).toContain('exact command, exit, timestamp and log'); - expect(text).toContain('as **ledger unavailable**'); + expect(text).toContain('STALE/MISSING'); + expect(text).toContain('| Only receipt storage/readback failed |'); expect(text).toContain('**ledger unavailable**, never FRESH'); const storage = text.slice(text.indexOf('| Only receipt storage/readback failed |')).split('|')[2]; expect(storage).not.toContain('gstack-evidence run'); - expect(storage).toContain('from the successful run\'s evidence'); - expect(text).toContain("**New, changed or unwaived test failure:** STOP publication. Run Steps 5–15, starting with Step 5's triage, then return to Step 16 stage 1"); - expect(text).toContain('Reuse waivers only for the same verified pre-existing failures and approved scope'); - expect(text).toContain('This recovery also applies if a failure appears while reporting in stage 5'); - expect(text).toContain('Docs, TODO edits, new/generated tests and fixes make evidence STALE'); - expect(storage).toContain('Independently prove unchanged final content, the same command and valid age'); - expect(storage).toContain('Without that proof, use STALE/MISSING'); - expect(text).toContain('Without that proof, use STALE/MISSING'); + expect(storage).toMatch(/without that proof, use STALE\/MISSING/i); + expect(text).toMatch(/\*\*New, changed or unwaived test failure:\*\* stop publication\. Run Steps 5–15/i); + expect(text).toMatch(/reuse waivers only for the same verified pre-existing failures/i); + expect(text).toMatch(/make evidence STALE/); }); test('ship contract precedes base detection and fresh remote facts precede distribution decisions', () => { @@ -204,7 +172,7 @@ describe('ship/SKILL.md — Plan Completion gate invariants (VAS-449 remediation expect(entry.indexOf('# Ship:')).toBeLessThan(entry.indexOf('## Step 0:')); const preflight = entry.slice(entry.indexOf('## Step 1:'), entry.indexOf('## Step 2:')); expect(preflight).toContain('git fetch origin '); - expect(preflight).toMatch(/fetch fails[^\n]+STOP/); + expect(preflight).toMatch(/fetch fails[^\n]+\bstop\b/i); expect(entry).not.toContain('auto-generate and commit, or flag'); expect(entry).toContain('Step 15 commits those tests'); }); @@ -212,10 +180,9 @@ describe('ship/SKILL.md — Plan Completion gate invariants (VAS-449 remediation test('bisectable commits proceed directly to verification without rewriting existing history', () => { const entry = fs.readFileSync(path.join(SHIP_DIR, 'SKILL.md'), 'utf8'); const commit = entry.slice(entry.indexOf('## Step 15:'), entry.indexOf('## Step 16:')); - expect(commit).toContain('Make bisectable commits'); - expect(commit).toContain('if already committed, continue to Step 16'); - expect(commit).toMatch(/Never create an empty commit/i); - expect(commit).toContain('Each commit must work independently'); + expect(commit).toMatch(/bisectable commits/i); + expect(commit).toContain('continue to Step 16'); + expect(commit).toMatch(/never create an empty commit/i); expect(commit).not.toMatch(/checkpoint|WIP|squash|git rebase|git reset/); expect(entry).not.toMatch(/Step 15\.[012]/); }); @@ -224,13 +191,14 @@ describe('ship/SKILL.md — Plan Completion gate invariants (VAS-449 remediation const entry = fs.readFileSync(path.join(SHIP_DIR, 'SKILL.md'), 'utf8'); const push = entry.slice(entry.indexOf('## Step 17:'), entry.indexOf('## Step 20:')); const recovery = push.replace(/\s+/g, ' '); - expect(push).toMatch(/push fails[^\n]+STOP/); + expect(push).toMatch(/push fails[^\n]+\bstop\b/i); expect(recovery).toContain('**Non-fast-forward push:**'); - expect(recovery).toContain('Run Steps 5–16 before returning to Step 17. Never rewrite history'); + expect(recovery).toContain('Run Steps 5–16 before returning to Step 17'); + expect(recovery).toMatch(/never rewrite history/i); expect(recovery).toContain('**Authentication, hook or network failure:**'); - expect(recovery).toContain('repeat Step 16 even if content is unchanged before returning to Step 17'); + expect(recovery).toMatch(/repeat Step 16 even if content is unchanged/i); expect(push).toMatch(/never force.push/i); - expect(push).toContain('Only a successful push'); + expect(push).toMatch(/only a successful push/i); }); }); From a7819d5875aff47f29c1e316648e8bdd3c471f7d Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 01:09:24 +0000 Subject: [PATCH 05/13] test(W7c): skill-validation matches three capital-letter pins on meaning W7c (t08 F2, F4, F5). One test file; only these three pins change. Removed assertion groups: none. Loosened, invariant kept as a free case-insensitive meaning check: - `MUTUALLY EXCLUSIVE` (codex skeleton) -> one codex mode per invocation: `/mutually exclusive/i` (t08 F4). - `skip the Codex passes ONLY` (review adversarial) -> `codex_reviews=disabled` skips only the Codex passes, Claude adversarial still runs: `/disabled[^\n]*skip the Codex passes only/i` (t08 F5). - `If any in-branch failures remain unfixed, **STOP**. Do not proceed` (ship tests section) -> unfixed in-branch failures block advancement: `/in-branch failures remain unfixed ... stop ... do not proceed/i` (t08 F2). Also covered by gate E2E ship-triage (gate). Cosmetic pins dropped: capitals and bold only. Mutation log: .context/mutation-skill-validation.md (6 mutations, 0 unexpected: 3 rule deletions red; 3 lower-case rewordings green). --- test/skill-validation.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/skill-validation.test.ts b/test/skill-validation.test.ts index ff1faccc7..9561d4433 100644 --- a/test/skill-validation.test.ts +++ b/test/skill-validation.test.ts @@ -1487,7 +1487,7 @@ describe('Codex skill', () => { expect(content).toContain('Step 2C: Consult Mode'); const skeleton = fs.readFileSync(path.join(ROOT, 'codex', 'SKILL.md'), 'utf-8'); expect(skeleton).toContain('## Step 1: Detect mode'); - expect(skeleton).toContain('MUTUALLY EXCLUSIVE'); + expect(skeleton).toMatch(/mutually exclusive/i); }); test('codex union contains gate verdict logic', () => { @@ -1587,7 +1587,7 @@ describe('Codex skill', () => { expect(content).toContain('CODEX_MODE'); expect(content).toContain('command -v codex'); // install check kept literal // codex_reviews=disabled gates Codex passes only; Claude adversarial still runs - expect(content).toContain('skip the Codex passes ONLY'); + expect(content).toMatch(/disabled[^\n]*skip the Codex passes only/i); // Review log expect(content).toContain('adversarial-review'); expect(content).toContain('reasoning_effort="high"'); @@ -2023,7 +2023,7 @@ describe('Test failure triage in ship skill', () => { test('ship/SKILL.md uses in-branch language for stop condition', () => { const content = readShipUnion(); - expect(content).toContain('If any in-branch failures remain unfixed, **STOP**. Do not proceed'); + expect(content).toMatch(/in-branch failures remain unfixed[\s\S]{0,40}\bstop\b[\s\S]{0,20}do not proceed/i); }); }); From 43f394671a9c6e8b8c1930cdad14a19bd1e16c90 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 01:12:22 +0000 Subject: [PATCH 06/13] test(W7c): qa-probe-gates keeps loop order, bounds and the CI-proven Read gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit W7c (t07 F3, F4, F5). One test file. This file stays the single owner of shared QA loop ordering (qa-functional-prompt defers to it). Removed assertion groups (invariant -> remaining protection): - `> **STOP.** Before any probe ...` prefix pin (t07 F3, retire) -> parent QA loads methods before any probe: kept free meaning check on "before any probe, including plan checks, complete the ordered scope/method Reads" and its order before the required-checks step. - Preparation prose in the stage list ("in full and select the surfaces", "2. Read the selected surface methods below in full", "Write a **charter**", "Start once before baseline:") -> stage order scope -> methods -> charter -> deadline start -> loop: kept as structural anchors (section Reads, surface labels, `## 1. Charter and preflight`, `bun G start D SECONDS`, step 1 title); added a reorder control moving a method Read after the charter. - Bounds/layout sentences ("an unlimited mode uses the caller's bound", "announce finite command timeouts", "EARLIER_UTC = caller's absolute deadline, if set", "in the probe directory, beside its deadline if bounded", duplicate SECONDS pins in the mode-bounds test, "charters as Markdown in the report", the full "observed: its exact decoded child JSON ..." sentence) -> numeric bounds and field names are kept free checks; the observed-field shape is enforced by the native checkpoint validator (qa-functional-prompt validator tests, free) and gate E2Es qa-functional-cli-report, qa-functional-webhook-report (gate). - Publication prose ("No drafts/placeholders", "and replay limits in the report", "An absolute state path is not itself a secret", "If capture is incomplete, report that limit instead of reconstructing it") -> notes are immutable and unsafe values withheld: kept free checks on "corrections cannot repair published notes", never invent a substitute, withhold secrets/unsafe values and stop the chain; validator-backed in qa-functional-prompt; gate E2Es review-exploratory-small-cli, ship-exploratory-small-cli (gate). - Loop-step prose ("Guard if bounded; await completion", "Run that exact probe; G enforces ..." duplicate, "then minimize via those gates", "Expiry leaves confirmation/minimization incomplete", "for each affected revalidation", "If bounded, run", "Reuse Step 4's ... completed Reads", "Finish missing methods before charters") -> loop has 5 steps, guarded by G, replays via steps 2–3, revalidation returns to step 2: kept free checks on step count, `G enforces the deadline`, `bun G status D`, `via steps 2–3`, `return to step 2`, refusals as not-run, do not repeat completed Reads. - CI-fixture clause "description, section index or remembered method is not a completed instruction Read" -> covered by the kept CI-proven clauses (`## 0. Preparation gate`, await each Read, complete missing Reads before Charter) and gate E2E ship-exploratory-small-cli (gate; the CI case). Kept (meaning checks, case-insensitive where prose): CI-proven "await their results before the first probe" rule (comment + mutation kept), Reads in order, no repeated Reads, SECONDS=30/900, shorter mode/caller limit, no D without a total limit, finite timeouts, REPORT_DIR layout and caller paths win, plan checks after smoke with no smoke guard and no clock reset, caller cap, not-run on expiry, await clock results, never reset D/bypass G, invalid/missing D stops probes, expiry -> write the report, Step 4 read-only, isolation/permission preflight before setup, field names, QA_DEADLINE receipts, all semantic-flip mutation controls, method/report resource paths, helper commands. Cosmetic pins dropped: STOP / AND capitals, bold STOP prefix. Mutation log: .context/mutation-qa-probe-gates.md (22 mutations, 0 unexpected: 3 reorders, 4 markers, 8 safety red; 7 rewordings green). --- test/qa-probe-gates.test.ts | 140 ++++++++++++++++-------------------- 1 file changed, 60 insertions(+), 80 deletions(-) diff --git a/test/qa-probe-gates.test.ts b/test/qa-probe-gates.test.ts index 362225875..59ece4a61 100644 --- a/test/qa-probe-gates.test.ts +++ b/test/qa-probe-gates.test.ts @@ -7,14 +7,12 @@ import { generatePlanVerificationExec } from '../scripts/resolvers/plan-gates'; import { HOST_PATHS } from '../scripts/resolvers/types'; function assertPreparation(text: string) { - expect(text).toContain('Complete these Reads in order before writing charters or probing'); - expect(text).toMatch(/Await their results before the first probe, never in the same response\.|Await each successful Read result before continuing\./); - expect(text).toContain('Do not repeat a Read already completed in this invocation'); - const stages = ['1. Read `sections/scope.md`', 'in full and select the surfaces', - '2. Read the selected surface methods below in full', '**Functional surfaces:**', - 'Read `sections/system-functional.md` in full.', '**Browser surfaces only:**', - 'Read `sections/qa-patterns.md` in full.', '## 1. Charter and preflight', - 'Write a **charter**', 'Start once before baseline:', '1. First demonstrate success']; + expect(text).toMatch(/reads in order before writing charters or probing/i); + expect(text).toMatch(/Await their results before the first probe, never in the same response\.|Await each successful Read result before continuing\./i); + expect(text).toMatch(/do not repeat a Read already completed/i); + const stages = ['Read `sections/scope.md`', '**Functional surfaces:**', 'Read `sections/system-functional.md`', + '**Browser surfaces only:**', 'Read `sections/qa-patterns.md`', '## 1. Charter and preflight', + 'bun G start D SECONDS', '1. First demonstrate success']; const positions = stages.map(stage => text.indexOf(stage)); expect(positions.every(position => position >= 0)).toBe(true); expect(positions).toEqual([...positions].sort((a, b) => a - b)); @@ -24,36 +22,35 @@ function assertPreparation(text: string) { } function assertBoundsAndLayout(text: string) { - for (const contract of [ - 'Browser Quick: SECONDS=30', 'Browser Full/Regression: SECONDS=900', - 'Functional Full, Quick and Regression have no default total timer', - "Set SECONDS to the shorter mode/caller limit", - "an unlimited mode uses the caller\'s bound", - 'Without a total time limit, do not start D', - 'announce finite command timeouts', - "EARLIER_UTC = caller\'s absolute deadline, if set", - 'Clocks/checkpoints use REPORT_DIR; mixed standalone runs use REPORT_DIR/browser and REPORT_DIR/functional, with one final report at REPORT_DIR. Caller paths win.', - 'in the probe directory, beside its deadline if bounded', - ]) expect(text).toContain(contract); - expect(text.indexOf('Caller paths win.')).toBeLessThan(text.indexOf('Start once before baseline:')); + for (const contract of ['Browser Quick: SECONDS=30', 'Browser Full/Regression: SECONDS=900']) expect(text).toContain(contract); + for (const rule of [ + /functional full, quick and regression have no default total timer/i, + /shorter mode\/caller limit/i, + /without a total time limit, do not start D/i, + /finite command timeouts/i, + /mixed standalone runs use REPORT_DIR\/browser and REPORT_DIR\/functional/i, + /one final report at REPORT_DIR/i, + /caller paths win/i, + ]) expect(text).toMatch(rule); + expect(text.search(/caller paths win/i)).toBeLessThan(text.indexOf('bun G start D SECONDS')); } function assertPlanExecution(text: string, shared = generateQAExploratory({ host: 'claude', skillName: 'qa', tmplPath: '', paths: HOST_PATHS.claude })) { const step = text.slice(text.indexOf('**3. Run smoke and plan checks.**'), text.indexOf('**4. Check freshness before reporting.**')).replace(/\s+/g, ' '); - for (const contract of [ - 'Follow the shared Probe loop for smoke checks and replays until the smoke limit', - 'Then run required plan checks and revalidation, even after smoke expires', - 'using the same procedure but no smoke guard; never reset the clock', - "Use finite command timeouts, capped at the caller\'s remaining time if it has a deadline", - 'When the caller\'s deadline expires, mark unfinished checks not-run', - 'Await clock/guard results before acting', - ]) expect(step).toContain(contract); + for (const rule of [ + /shared Probe loop for smoke checks and replays until the smoke limit/i, + /required plan checks and revalidation, even after smoke expires/i, + /same procedure but no smoke guard; never reset the clock/i, + /capped at the caller's remaining time/i, + /deadline expires, mark unfinished checks not-run/i, + /await clock\/guard results before acting/i, + ]) expect(step).toMatch(rule); expect(step.indexOf('Follow the shared Probe loop')).toBeLessThan(step.indexOf('Then run required plan checks')); expect(step.indexOf('Then run required plan checks')).toBeLessThan(step.indexOf('same procedure')); - for (const contract of ['First demonstrate success: output AND durable effects', - 'Wait for successful checkpoint publication before dispatch', - 'Replay the exact failing command/request from the same initial fixture state']) { - expect(shared).toContain(contract); + for (const rule of [/demonstrate success: output and durable effects/i, + /wait for successful checkpoint publication before dispatch/i, + /replay the exact failing command\/request from the same initial fixture state/i]) { + expect(shared).toMatch(rule); } } @@ -67,8 +64,7 @@ describe('QA probe entry and checkpoint gates', () => { expect(old).toContain('Read `sections/system-functional.md`'); const current = generateQAExploratory({ host: 'claude', skillName: 'qa-only', tmplPath: '', paths: HOST_PATHS.claude }); for (const clause of ['## 0. Preparation gate', 'Await each successful Read result before continuing', - 'description, section index or remembered method is not a completed instruction Read', - 'If either required Read is missing, complete it now before Charter and preflight']) { + 'complete it now before Charter and preflight']) { expect(old).not.toContain(clause); expect(current).toContain(clause); } @@ -88,13 +84,13 @@ describe('QA probe entry and checkpoint gates', () => { const ctx = { host: host.name, skillName, tmplPath: '', paths: HOST_PATHS[host.name] }; const preflight = generateQAReviewPreflight(ctx).replace(/\s+/g, ' '); const body = generateQAReview(ctx).replace(/\s+/g, ' '); - expect(preflight).toContain('Resolve QA\'s `sections/...` and `templates/...` paths from that installed QA SKILL.md directory, not the caller or product directory'); + expect(preflight).toMatch(/`sections\/\.\.\.` and `templates\/\.\.\.` paths from that installed QA SKILL\.md directory, not the caller or product directory/i); expect(body).toContain('Read QA\'s `sections/browser-setup.md`'); expect(body).toContain('Read QA\'s `templates/functional-report-template.md`'); if (skillName === 'review') { expect(body).toContain('Read QA\'s `templates/qa-report-template.md`'); - expect(body).toContain('Reuse Step 4\'s surfaces and completed Reads'); - expect(body).toContain('Finish missing methods before charters; do not repeat completed Reads'); + expect(body).toContain('Step 4\'s surfaces'); + expect(body).toMatch(/do not repeat completed Reads/i); } } } @@ -110,19 +106,14 @@ describe('QA probe entry and checkpoint gates', () => { const positions = stages.map(stage => text.indexOf(stage)); expect(positions.every(position => position >= 0)).toBe(true); expect(positions).toEqual([...positions].sort((a, b) => a - b)); - expect(text).toContain('No drafts/placeholders'); - expect(text).toContain('corrections cannot repair published notes'); + expect(text).toMatch(/corrections cannot repair published notes/i); if (skillName === 'qa-only') { - expect(text).toContain('never invent a substitute path, identity or state'); - expect(text).toContain('For actual secrets/private payloads, withhold those values'); - expect(text).toContain('and replay limits in the report'); - expect(text).toContain('stop the affected probe chain'); - expect(text).toContain('An absolute state path is not itself a secret'); + expect(text).toMatch(/never invent a substitute path, identity or state/i); + expect(text).toMatch(/secrets\/private payloads, withhold those values/i); + expect(text).toMatch(/stop the affected probe chain/i); } else { - expect(text).toContain('No drafts/placeholders or invented safe-path redactions'); - expect(text).toContain('Withhold unsafe values, disclose limits and stop that chain'); + expect(text).toMatch(/withhold unsafe values, disclose limits and stop that chain/i); } - if (skillName === 'qa-only') expect(text).toContain('If capture is incomplete, report that limit instead of reconstructing it'); } } }); @@ -132,13 +123,7 @@ describe('QA probe entry and checkpoint gates', () => { const text = generateQAExploratory({ host: 'claude', skillName, tmplPath: '', paths: HOST_PATHS.claude }); for (const contract of [ 'Reuse resolved REPORT_DIR', - 'charters as Markdown in the report', - "Set SECONDS to the shorter mode/caller limit", - 'Without a total time limit, do not start D', - 'Browser Quick: SECONDS=30', - 'Browser Full/Regression: SECONDS=900', 'exactly four top-level fields:', 'observationCommand:', 'observed:', 'hypothesis:', 'nextCommand:', - 'observed: its exact decoded child JSON (no wrapper/extra keys), or its full non-JSON text', 'QA_DEADLINE receipts are not observations', ]) expect(text).toContain(contract); assertBoundsAndLayout(text); @@ -159,17 +144,15 @@ describe('QA probe entry and checkpoint gates', () => { const write = text.indexOf('**Publish before probing.**'); expect(decision).toBeGreaterThan(-1); expect(decision).toBeLessThan(write); - expect(text.slice(decision, write)).toContain('write the report, not a checkpoint'); - expect(text.slice(decision, write)).toContain('If expired or no safe next probe remains'); + expect(text.slice(decision, write)).toMatch(/write the report, not a checkpoint/i); + expect(text.slice(decision, write)).toMatch(/if expired/i); expect(text.slice(decision, write)).not.toContain('If done or blocked'); if (skillName === 'qa-only') { - expect(text).toContain('Check fields before publication'); - expect(text).toContain('retain the entire result unchanged, including owned fixture paths, IDs, hashes'); + expect(text).toMatch(/retain the entire result unchanged/i); } else { - expect(text).toContain('Preserve every safe program-JSON key/value'); - expect(text).toContain('Preserve every safe program-JSON key/value and identity hash unchanged'); + expect(text).toMatch(/preserve every safe program-JSON key\/value and identity hash unchanged/i); } - expect(text).toContain('Q supplies observed; never transcribe it'); + expect(text).toMatch(/Q supplies observed; never transcribe it/i); } } }); @@ -182,20 +165,16 @@ describe('QA probe entry and checkpoint gates', () => { expect(text).toContain('/gstack-qa-deadline'); expect(text).toContain('bun G start D SECONDS [EARLIER_UTC]'); expect(text).toContain('Bounded browsers: `bun G run D -- COMMAND ARGS`'); - expect(text).toContain('Never reset D/bypass G'); - expect(text).toContain('invalid/missing D stops probes'); - expect(steps[0]).toContain('demonstrate success: output AND durable effects'); - expect(steps[0]).toContain('Guard if bounded; await completion'); - expect(steps[1]).toContain('If bounded, run `bun G status D`'); + expect(text).toMatch(/never reset D\/bypass G/i); + expect(text).toMatch(/invalid\/missing D stops probes/i); + expect(steps[0]).toMatch(/demonstrate success: output and durable effects/i); + expect(steps[1]).toContain('`bun G status D`'); expect(steps[1].indexOf('If expired')).toBeLessThan(steps[1].indexOf('**Publish before probing.**')); - expect(steps[1]).toContain('STOP exploration; write the report, not a checkpoint'); - expect(steps[2]).toContain('Run that exact probe; G enforces the deadline when bounded'); + expect(steps[1]).toMatch(/\bstop exploration\b[^.]*write the report/i); expect(steps[2]).toContain('G enforces the deadline'); - expect(steps[2]).toContain('Report refusals as not-run'); + expect(steps[2]).toMatch(/refusals as not-run/i); expect(steps[3]).toContain('via steps 2–3'); - expect(steps[3]).toContain('then minimize via those gates'); - expect(steps[3]).toContain('Expiry leaves confirmation/minimization incomplete'); - expect(steps[4]).toContain('return to step 2 for each affected revalidation'); + expect(steps[4]).toContain('return to step 2'); } }); @@ -204,25 +183,25 @@ describe('QA probe entry and checkpoint gates', () => { for (const skillName of ['review', 'ship']) { const ctx = { host: host.name, skillName, tmplPath: '', paths: HOST_PATHS[host.name] }; const text = (skillName === 'review' ? generateQAReviewPreflight(ctx) : '') + generateQAReview(ctx); - expect(text).toContain('> **STOP.** Before any probe, including plan checks, complete the ordered scope/method Reads below'); + const gate = text.search(/before any probe, including plan checks, complete the ordered scope\/method Reads/i); + expect(gate).toBeGreaterThan(-1); expect(text).toContain('{{QA_RESOURCE:exploratory}}'); expect(text).not.toContain('{{QA_RESOURCE:scope}}'); expect(text).not.toContain('Read `sections/system-functional.md`'); const shared = generateQAExploratory({ ...ctx, skillName: 'qa' }); assertPreparation(shared); - expect(text).toContain('Before any probe, including plan checks'); assertPlanExecution(text); const required = text.indexOf(skillName === 'review' ? '**2. Check readiness and list required checks.**' : '**2. List required checks'); expect(required).toBeGreaterThan(-1); - expect(text.indexOf('> **STOP.**')).toBeLessThan(required); + expect(gate).toBeLessThan(required); if (skillName === 'review') { const isolation = text.indexOf('**1. Set the charter and isolation.**'); const setup = text.indexOf("Read QA's `sections/browser-setup.md`"); - expect(isolation).toBeGreaterThan(text.indexOf('> **STOP.**')); + expect(isolation).toBeGreaterThan(gate); expect(setup).toBeGreaterThan(required); - expect(text.slice(isolation, required).replace(/\s+/g, ' ')).toContain('complete the shared isolation/permission preflight before setup'); - expect(text).toContain('Step 4 is read-only: defer charters, setup and probes to Step 4.7'); + expect(text.slice(isolation, required).replace(/\s+/g, ' ')).toMatch(/isolation\/permission preflight before setup/i); + expect(text).toMatch(/Step 4 is read-only: defer charters, setup and probes to Step 4\.7/i); } else { const setup = text.indexOf("For browsers, Read QA's `sections/browser-setup.md`"); expect(setup).toBeGreaterThan(required); @@ -239,8 +218,9 @@ describe('QA probe entry and checkpoint gates', () => { for (const changed of [ method + '\n' + text.replace(method, ''), text.replace(method, method + '\n' + method), - 'Write a **charter**\n' + text, + '## 1. Charter and preflight\n' + text, text.replace('Do not repeat a Read already completed in this invocation', 'Repeat all Reads'), + text.replace('Read `sections/qa-patterns.md` in full.', '').replace('## 1. Charter and preflight', '## 1. Charter and preflight\nRead `sections/qa-patterns.md` in full.'), // ci-36641820398-1-gate-census-7 ship-exploratory-small-cli dispatched its first probe with the resource Reads. text.replace(' Await their results before the first probe, never in the same response.', ''), ]) expect(() => assertPreparation(changed)).toThrow(); @@ -303,8 +283,8 @@ describe('QA probe entry and checkpoint gates', () => { test('plan execution waits for the actual shared method reads, not just collection', () => { const text = generatePlanVerificationExec({ host: 'claude', skillName: 'ship', tmplPath: '', paths: HOST_PATHS.claude }); - expect(text).toContain('Do not invoke an entire QA skill or start probes here'); + expect(text).toMatch(/do not invoke an entire QA skill or start probes here/i); expect(text).toContain('Before the first plan command, complete Step 9.2.1'); - expect(text).toContain('method Reads and the shared probe loop'); + expect(text).toMatch(/method Reads and the shared probe loop/i); }); }); From 3ab469198e66447030080ee58bad4813c71193fe Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 01:14:44 +0000 Subject: [PATCH 07/13] test(W7c): qa-functional-prompt keeps the verifier contract, drops driver prose W7c (t07 F6, F7, F8). One test file. The eval prompt itself is unchanged. Removed assertion groups (invariant -> remaining protection): - ~40 driver-prompt sentence pins in the disclosure, artifact and fix-budget tests ("All four reads are required before probing ...", "command is the exact full outer capture invocation, including ...", "different later command", "not the required same-command replay", "Only the helper writes observed fields", "Link the evidence and checkpoint files ...", "Evidence rows contain ONLY complete JSON ...", "aim under 400 words", "one causal sentence per checkpoint hypothesis ...", "a reproduced in-tier defect requires ...", duplicates of "retain its headings ..." etc.) -> the run produces verifiable evidence and a linked report: free validator tests in this file (qaFunctionalVerdict replay, validateQACheckpoints on CI, R29 and R85 captures) and gate E2Es qa-functional-cli-report, qa-functional-webhook-report, qa-functional-cli-fix, qa-functional-webhook-fix (gate). Kept the machine-read contract: read list, `"command":""`, `annotations.learning`, the three copied field names, the >20-character limit, evidence.json before report.md. - `maxTurns: 40` / `completionReserveMs: timeout / 4` exact source pins -> replaced by a `maxTurns >= 40` floor and presence of `completionReserveMs`. - Fix-stage provenance sentences ("most recent completed native probe", "Tests, source edits and clock reads ...", "write no checkpoint when there is no next probe") -> test output never becomes `observed`: kept one free meaning check plus the R29 validator test (free). - Duplicate loop-order checks on qa-only/sections/exploratory.md and grep of `scripts/resolvers/qa.ts` source (t07 F6, F8) -> loop ordering has one owner, qa-probe-gates (free, every host). Here the source grep becomes a check on `generateQAExploratory` output (one probe per native command, never batch probes), and the caller test keeps resource paths, field names and structural stage anchors. Loosened to case-insensitive meaning checks (kept): never synthesize JSON, never shorten native JSON, existing tests read-only, freeze tests after red, repair scope `src/.ts`, green suite does not replace native probes, report incomplete honestly, report-only proposes test stubs, completion reserve does not end coverage, all eight scenarios required, never backfill notes, owned probe directory, only complete emitted JSON, wrapper/entrypoint boundary lines. Kept unchanged: paid-case selection, CI/R29/R85 validator replays, actor boundary and command allowlist, fix-stage order, verification probe list, scenario lists. Cosmetic pins dropped: ONLY / NOT / Never capitals. Mutation log: .context/mutation-qa-functional-prompt.md (19 mutations, 0 unexpected: 3 reorders, 4 markers, 7 safety red; 5 rewordings green). --- test/qa-functional-prompt.test.ts | 114 ++++++++++-------------------- 1 file changed, 37 insertions(+), 77 deletions(-) diff --git a/test/qa-functional-prompt.test.ts b/test/qa-functional-prompt.test.ts index 109498d44..c5bb2d047 100644 --- a/test/qa-functional-prompt.test.ts +++ b/test/qa-functional-prompt.test.ts @@ -9,6 +9,8 @@ import { qaFunctionalVerdict, qaNativeProbes } from './helpers/qa-functional-evi import { createQAFunctionalFixture, ownedPath } from './helpers/qa-functional-fixture'; import { validateQACheckpoints } from './helpers/qa-checkpoint-evidence'; import { computePaidCaseSelection } from '../scripts/test-paid-shards'; +import { generateQAExploratory } from '../scripts/resolvers/qa'; +import { HOST_PATHS } from '../scripts/resolvers/types'; test.each(['full', 'pr'] as const)('%s selection assigns the captured webhook regression to its native owner', profile => { for (const file of ['qa-webhook-r85-checkpoints.json', 'qa-functional-ci-36505065023.json']) { @@ -81,22 +83,15 @@ test('functional driver discloses its learning, CLI coverage and repair acceptan for (const entry of QA_FUNCTIONAL_CASES) { const prompt = qaFunctionalPrompt(entry); expect(prompt).toContain(`Read ${entry.mode}/SKILL.md, qa/sections/scope.md, ${entry.mode}/sections/exploratory.md and qa/sections/system-functional.md in full`); - expect(prompt).toContain('All four reads are required before probing in this fixture, even when its surfaces and isolation are already established'); - expect(prompt).toContain('command is the exact full outer capture invocation, including that ID and all wrapper options, not just the native child command after --'); expect(prompt).toContain('"command":""'); expect(prompt).not.toContain(''); - expect(prompt).toContain('different later command'); - expect(prompt).toContain('not the required same-command replay'); - expect(prompt).toContain('Select that checkpoint ID in annotations.learning'); - expect(prompt).toContain('the production helper copies its observationCommand, hypothesis and nextCommand'); - expect(prompt).toContain('different native child commands'); - expect(prompt).toContain('Only the helper writes observed fields'); - expect(prompt).toContain('English, more than 20 characters'); - if (entry.family === 'cli') expect(prompt).toContain('a successful apply; balance alone is not enough'); + expect(prompt).toContain('annotations.learning'); + expect(prompt).toContain('observationCommand, hypothesis and nextCommand'); + expect(prompt).toMatch(/more than 20 characters/); if (entry.mode === 'qa') { expect(prompt).toContain(`repair only src/${entry.family === 'cli' ? 'cli' : 'worker'}.ts`); - expect(prompt).toContain('existing tests remain read-only'); - expect(prompt).toContain('Freeze all test files after red'); + expect(prompt).toMatch(/existing tests remain read-only/i); + expect(prompt).toMatch(/freeze all test files after red/i); } } }); @@ -164,12 +159,12 @@ test('the native launcher consumes the family-specific actor boundary', () => { expect(prompt).not.toContain('at Standard depth'); expect(prompt).toContain('successful checkpoint publication before the next probe'); expect(prompt).toContain('no shell composition, scripts or added path operands'); - expect(prompt).toContain('ONLY complete JSON actually emitted'); + expect(prompt).toMatch(/only complete JSON actually emitted/i); expect(prompt).toContain('never a combined command list'); expect(prompt).toContain('Put tests, raw CLI diagnostics, launch failures and timeouts in Markdown'); - expect(prompt).toContain(entry.family === 'cli' - ? 'The generic wrapper does NOT support wait' - : 'bun cancel.ts is a CLI-only entrypoint, not part of this fixture'); + expect(prompt).toMatch(entry.family === 'cli' + ? /the generic wrapper does not support wait/i + : /bun cancel\.ts is a CLI-only entrypoint, not part of this fixture/i); expect(prompt).not.toContain('parseInt'); if (entry.family === 'webhook') expect(prompt).toContain('Choose their order from observations after the happy path'); if (entry.family === 'webhook' && entry.mode === 'qa-only') { @@ -198,51 +193,30 @@ test('declared examples respect the existing closed native grammar', () => { test('artifact completion preserves exact evidence before concise linked reporting', () => { for (const entry of QA_FUNCTIONAL_CASES) { const prompt = qaFunctionalPrompt(entry); - expect(prompt).toContain('Materialize qa-reports/evidence.json first, then write a concise qa-reports/report.md'); - expect(prompt).toContain('using the functional report structure'); - expect(prompt).toContain('Link the evidence and checkpoint files rather than repeating full probe payloads in Markdown'); - expect(prompt).toContain('Both artifacts are required before completion'); - expect(prompt).toContain('The learning array is a summary: choose one completed checkpoint'); - expect(prompt).toContain('not another probe or a duplicate of the complete checkpoint ledger'); - expect(prompt).toContain('Both commands must name exact captured probes with different native child commands'); - expect(prompt).toContain('Preserve every checkpoint and link every checkpoint in Markdown'); - expect(prompt).toContain('keep every executed probe and its complete JSON in evidence'); - expect(prompt).toContain('Evidence rows contain ONLY complete JSON actually emitted by native probes, including failures and repeats'); - expect(prompt).toContain('retain pre-repair results alongside green results'); - expect(prompt).toContain('Never synthesize JSON'); - expect(prompt).toContain('one causal sentence per checkpoint hypothesis (English, more than 20 characters) and compact JSON formatting, preserving every field and value'); - expect(prompt).toContain('retain its headings and required fields'); - expect(prompt).toContain('link to evidence.json and checkpoints for details already recorded there'); - expect(prompt).toContain('After saving both artifacts, return only their paths and the actual completion status'); - expect(prompt).toContain('Never shorten native JSON or omit a required probe, check or field'); - expect(prompt).toContain('aim under 400 words'); + expect(prompt.indexOf('qa-reports/evidence.json')).toBeGreaterThan(-1); + expect(prompt.indexOf('qa-reports/evidence.json')).toBeLessThan(prompt.indexOf('qa-reports/report.md')); + expect(prompt).toMatch(/both artifacts are required before completion/i); + expect(prompt).toMatch(/never synthesize JSON/i); + expect(prompt).toMatch(/never shorten native JSON or omit a required probe, check or field/i); } const source = readFileSync(join(import.meta.dir, 'helpers/qa-functional-eval.ts'), 'utf8'); - expect(source).toContain('maxTurns: 40'); - expect(source).toContain('completionReserveMs: timeout / 4'); + expect(Number(source.match(/maxTurns: (\d+)/)?.[1])).toBeGreaterThanOrEqual(40); + expect(source).toContain('completionReserveMs:'); }); test('fix completion budgets for required repair and avoids duplicating preserved evidence', () => { for (const entry of QA_FUNCTIONAL_CASES) { const prompt = qaFunctionalPrompt(entry); if (entry.mode === 'qa') { - expect(prompt).toContain('a reproduced in-tier defect requires the authorized native regression, repair and verification'); - expect(prompt).toContain('retain its headings and required fields'); - expect(prompt).toContain('link to evidence.json and checkpoints for details already recorded there'); - expect(prompt).toContain('Include the diagnosis, red/green test results and coverage limits'); - expect(prompt).toContain('After saving both artifacts, return only their paths and the actual completion status'); - expect(prompt).toContain('Never shorten native JSON or omit a required probe, check or field'); const stages = ['1. Prove the regression red', '2. On the repaired source', '3. Save the evidence and Markdown artifacts']; const positions = stages.map(stage => prompt.indexOf(stage)); expect(positions.every(position => position >= 0)).toBe(true); expect(positions).toEqual([...positions].sort((a, b) => a - b)); - expect(prompt).toContain('A green test suite does not substitute for these native probes'); - expect(prompt).toContain('not a signal to stop stage 2'); - expect(prompt).toContain('report incomplete; do not call it complete with a caveat'); - expect(prompt).toContain('one causal sentence per checkpoint hypothesis (English, more than 20 characters) and compact JSON formatting, preserving every field and value'); + expect(prompt).toMatch(/green test suite does not substitute for these native probes/i); + expect(prompt).toMatch(/report incomplete; do not call it complete/i); } else { expect(prompt).not.toContain('This is a fix run'); - expect(prompt).toContain('Include the diagnosis, proposed test stubs and coverage limits'); + expect(prompt).toMatch(/proposed test stubs/i); expect(prompt).not.toContain('Include the diagnosis, red/green test results'); } } @@ -265,22 +239,19 @@ test('webhook fix stage asks for the fix-loop probes; the R29 scenario omissions } const fix = qaFunctionalPrompt({ family: 'webhook', mode: 'qa' }); const verification = fix.slice(fix.indexOf('2. On the repaired source'), fix.indexOf('3. Save the evidence')); - expect(verification).toContain('run the original failing probe, an adjacent happy-path probe, cancellation and the unavailable-dependency probe'); + for (const probe of ['original failing probe', 'happy-path probe', 'cancellation', 'unavailable-dependency probe']) expect(verification).toContain(probe); expect(verification).not.toContain('All eight scenarios'); - expect(fix).toContain('the completion reserve does not end required coverage'); + expect(fix).toMatch(/completion reserve does not end required coverage/i); expect(verification).toBe(((prompt: string) => prompt.slice(prompt.indexOf('2. On the repaired source'), prompt.indexOf('3. Save the evidence')))(qaFunctionalPrompt({ family: 'cli', mode: 'qa' }))); const report = qaFunctionalPrompt({ family: 'webhook', mode: 'qa-only' }); - expect(report).toContain('All eight scenarios are required coverage; a replay does not replace another scenario'); + expect(report).toMatch(/all eight scenarios are required coverage/i); for (const scenario of required) expect(report).toContain(scenario); }); test('fix-stage checkpoint provenance survives intervening native regression tests', () => { const prompt = qaFunctionalPrompt({ family: 'webhook', mode: 'qa' }); - expect(prompt).toContain('most recent completed native probe'); - expect(prompt).toContain('Tests, source edits and clock reads do not replace that observation'); - expect(prompt).toContain('put red/green test output in the report, not in observed'); - expect(prompt).toContain('write no checkpoint when there is no next probe'); + expect(prompt).toMatch(/red\/green test output in the report, not in observed/i); }); test('R29 captured webhook bytes bind across a green test; test summaries and altered JSON do not', () => { @@ -376,32 +347,21 @@ test.each(JSON.parse(readFileSync(join(import.meta.dir, 'fixtures/qa-webhook-r85 test('report-only exploration requires a completed written checkpoint before the next probe', () => { const section = readFileSync(join(import.meta.dir, '../qa-only/sections/exploratory.md'), 'utf8'); - const positions = ['1. First demonstrate success', '2. **Decide whether another probe is needed.**', '**Publish before probing.**', '3. Run that exact probe; G enforces the deadline when bounded'] - .map(marker => section.indexOf(marker)); - expect(positions.every(position => position >= 0)).toBe(true); - expect(positions).toEqual([...positions].sort((a, b) => a - b)); expect(section).toContain('exploration-NNN.json'); - expect(section).toContain("Reuse resolved REPORT_DIR"); - expect(section).toContain('own a fresh'); - expect(section).toContain('owned probe directory'); + expect(section).toContain('Reuse resolved REPORT_DIR'); + expect(section).toMatch(/owned probe directory/i); for (const field of ['observationCommand', 'observed', 'hypothesis', 'nextCommand']) expect(section).toContain(`${field}:`); - expect(section).toContain('Wait for successful checkpoint publication'); - expect(section).toContain('Never backfill or overwrite notes'); - expect(section).toContain('link each checkpoint'); + expect(section).toMatch(/wait for successful checkpoint publication/i); + expect(section).toMatch(/never backfill or overwrite notes/i); expect(section).not.toContain('a separate assistant text message'); }); test('surface evidence checks defer to one exploratory execution sequence', () => { - const source = readFileSync(join(import.meta.dir, '../scripts/resolvers/qa.ts'), 'utf8'); - expect(source).toContain('Each probe is one native command/interaction plus checks, excluding bookkeeping'); - const positions = ['2. **Decide whether another probe is needed.**', '**Publish before probing.**', '3. Run that exact probe; G enforces the deadline when bounded'] - .map(marker => source.indexOf(marker)); - expect(positions.every(position => position >= 0)).toBe(true); - expect(positions).toEqual([...positions].sort((a, b) => a - b)); - expect(source).toContain('Never batch probes'); - expect(source).toContain('Follow the shared exploratory loop\'s order and written checkpoints'); - expect(source).toContain('Replay the exact failing command/request from the same initial fixture state'); - expect(source).toContain('Another input or a regression test is not that replay'); + const shared = generateQAExploratory({ host: 'claude', skillName: 'qa', tmplPath: '', paths: HOST_PATHS.claude }); + expect(shared).toMatch(/each probe is one native command\/interaction/i); + expect(shared).toMatch(/never batch probes/i); + const functional = readFileSync(join(import.meta.dir, '../qa/sections/system-functional.md'), 'utf8'); + expect(functional).toMatch(/follow the shared exploratory loop/i); }); test('all public callers directly require the functional method before exploration', () => { @@ -413,13 +373,13 @@ test('all public callers directly require the functional method before explorati const explorer = readFileSync(join(import.meta.dir, '..', skill === 'qa-only' ? 'qa-only' : 'qa', 'sections/exploratory.md'), 'utf8'); expect(explorer).toMatch(/Functional surfaces[^\n]*\n[^\n]*Read[^\n]*system-functional\.md/); expect(explorer).toContain('Browser surfaces only'); - const stages = ['Read `sections/scope.md`', 'in full and select the surfaces', - 'Read `sections/system-functional.md`', 'Write a **charter**', '1. First demonstrate success']; + const stages = ['Read `sections/scope.md`', 'Read `sections/system-functional.md`', + '## 1. Charter and preflight', '1. First demonstrate success']; const positions = stages.map(stage => explorer.indexOf(stage)); expect(positions.every(position => position >= 0)).toBe(true); expect(positions).toEqual([...positions].sort((a, b) => a - b)); const functional = readFileSync(join(import.meta.dir, '../qa/sections/system-functional.md'), 'utf8'); expect(functional).toContain('## Contract map'); - expect(functional).toContain("Follow the shared exploratory loop's order and written checkpoints"); + expect(functional).toMatch(/follow the shared exploratory loop/i); } }); From 08f53f3bc3d4bf153daab68b525bc06206e71cf8 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 01:18:33 +0000 Subject: [PATCH 08/13] test(W7c): qa-caller-authority keeps authority and safety lines, drops report prose W7c (t10 F12, F13). One test file; still renders every host x caller. Removed assertion groups (invariant -> remaining protection): - `STOP and wait` / `STOP for the answer` / `AskUserQuestion and WAIT` capitals (t10 F12) -> approval before `./setup` or install: now an order check (explicit approval < "only after approval" < `cd && ./setup`; OK to proceed? < setup; AskUserQuestion-and-wait < install only the actual choice), plus case-insensitive "stop for the answer". Stronger: the mutation that moves setup ahead of the approval ask now goes red (the old test stayed green). New negative control removes the approval ask. - Report-layout prose for /review and /ship (keep metadata/outcome tables, demote headings one/two levels, fields as subsections, keep scores separate, save baseline normally) -> one final linked report section: kept free checks on the `## Exploratory QA and Verification Results` / `### Browser results` / `## Exploratory QA` headings, link every checkpoint and no second report; gate E2Es review-exploratory-small-cli, ship-exploratory-small-cli (gate). - Smoke-pacing prose ("one success and the riskiest changed failure/edge", "Required even for small diffs ...", "Required: plan commands/assertions, listed separately", "Other ideas are optional, untested") and duplicate report-status pins -> kept the numeric `5 minutes/12 probes` bound and the report-clean-only-when-all-required-pass gate; gate E2Es ship-exploratory-small-cli, ship-exploratory-plan-checks (gate). - Dedup prose ("Combine saved `findings` ...", "If both history ... classify normally", "Report the suppressed count once if nonzero", "If N > 0, print once:", "Otherwise skip the summary") -> only user `skipped` findings are suppressed: kept free order check (read decisions before matching) and the `skipped` / never `fixed` / `auto-fixed` lines; gate E2E ship-skipped-queued-finding (gate). - Preparation sentence pins duplicated from qa-probe-gates (Reads in order, no repeated Reads, "in full and select the surfaces", "Write a **charter**", "charter** per behavior", "announce finite command timeouts") -> owned by qa-probe-gates (free, every host); this file keeps scope < methods < charter structural order for every caller. - Learnings prose ("Otherwise skip this optional lookup", "Reading old notes never requires writing new ones") -> report-only lookup never writes: kept free checks (directory already known, caller permits, do not run the search helper, no bash/gstack-config/AskUserQuestion, injection throw); the query interpolation stays checked via `"webhook retries"`. - Misc prose ("Re-run affected contracts and adjacent happy paths ...", "Run only for selected browser surfaces", "Confirm each issue by retrying once", "parent applies queued fixes", "No test framework detected", "Report QA setup blockers", "required QA") -> judge `qa/SKILL.md workflow` and `review/SKILL.md workflow` (gate lane); the adjacent kept gates remain free. Kept as case-insensitive meaning checks (authority, consent, safety): probe- local blocker rule, report-only never bootstraps/invokes skills, unknown caller -> report-only, declined -> blocked, credentials/passwords/other tabs, local-target ownership, no unit-test/curl substitute, functional-only reads nothing browser-side, never-pass list, named-risk acceptance, noninteractive -> blocked, test_stub ASK approval, ship waiver cannot complete review, caller-required rechecks, blocked rechecks never verify repairs, no edits during Step 4.8, no inner repair loop, REVIEW_START/token-once, native pass required for 5.8, coverage-cap stop lines, report-only never commits or dispatches, CI/test-weakening ban, never delete a valid red regression. Kept unchanged: machine tokens (`bun G start D`, `$B snapshot -i`, `DIFF_START`, `CONSOLE_ERRORS=`, `$B handoff/resume`), resource paths, the `If A and allowance remains:` count, `Minimum = 60%, Target = 80%`, all `.not.toContain` stale-workflow bans, existing negative controls. Cosmetic pins dropped: STOP / WAIT / LOCAL / Never capitals. Mutation log: .context/mutation-qa-caller-authority.md (23 mutations, 0 unexpected: 3 reorders, 3 markers, 11 safety red; 6 rewordings green). --- test/qa-caller-authority.test.ts | 208 +++++++++++++------------------ 1 file changed, 87 insertions(+), 121 deletions(-) diff --git a/test/qa-caller-authority.test.ts b/test/qa-caller-authority.test.ts index 668fcb571..2273b63f5 100644 --- a/test/qa-caller-authority.test.ts +++ b/test/qa-caller-authority.test.ts @@ -26,35 +26,34 @@ function render(file: string, ctx: TemplateContext): string { } function assertProbeLocalBlocker(body: string): void { - expect(body).toContain('QA setup blocker'); - expect(body).toContain('affected probes as blocked'); - expect(body).toContain('continue other safe probes'); - expect(body).toContain('required QA'); - expect(body).toContain('independent functional/static checks'); + expect(body).toMatch(/QA setup blocker/i); + expect(body).toMatch(/affected probes as blocked/i); + expect(body).toMatch(/continue other safe probes/i); + expect(body).toMatch(/independent functional\/static checks/i); expect(body).not.toContain('stop that workflow'); expect(body).not.toContain('stop all checks'); } function assertSharedBrowserAuthority(body: string): void { const decision = body.slice(body.indexOf('## Browser access decision'), body.indexOf('## BROWSER SETUP')); - expect(decision).toContain('invoking workflow, not this file'); + expect(decision).toMatch(/invoking workflow, not this file/i); const reportOnly = decision.slice(decision.indexOf('**Report-only'), decision.indexOf('**Standalone /qa')); expect(reportOnly).toContain('/qa-only, /review and /ship'); - expect(reportOnly).toContain("do not run the fallback's setup/install or cookie-import workflow"); - expect(reportOnly).toContain('Never bootstrap or invoke another skill'); - expect(reportOnly).toContain('block only the affected browser probes'); - expect(reportOnly).toContain('continue independent functional/static checks'); + expect(reportOnly).toMatch(/do not run the fallback's setup\/install or cookie-import workflow/i); + expect(reportOnly).toMatch(/never bootstrap or invoke another skill/i); + expect(reportOnly).toMatch(/block only the affected browser probes/i); + expect(reportOnly).toMatch(/continue independent functional\/static checks/i); expect(reportOnly).not.toContain('run `cd'); const standalone = decision.slice(decision.indexOf('**Standalone /qa')); - expect(standalone).toContain('explicit approval'); - expect(standalone).toContain('STOP and wait'); - expect(standalone).toContain('Only after approval'); - expect(standalone).toContain('`cd && ./setup`'); + const ask = standalone.search(/explicit approval/i); + const gate = standalone.search(/only after approval/i); + const setup = standalone.indexOf('`cd && ./setup`'); + expect(ask).toBeGreaterThan(-1); + expect(gate).toBeGreaterThan(ask); + expect(setup).toBeGreaterThan(gate); expect(standalone).toContain('/setup-browser-cookies'); - expect(standalone).toContain('declined, unavailable or unsuccessful'); - expect(standalone).toContain('blocked'); - expect(decision).toContain('Unknown caller'); - expect(decision).toContain('report-only'); + expect(standalone).toMatch(/declined, unavailable or unsuccessful[\s\S]{0,40}blocked/i); + expect(decision).toMatch(/unknown caller[^\n]*report-only/i); expect(body).not.toContain('If `NEEDS_SETUP`: tell the user'); expect(body).not.toContain('An authenticated page needs /setup-browser-cookies'); } @@ -64,16 +63,11 @@ describe('QA caller authority in pure host renders', () => { test(`${host.name}: a scope handoff requires the actual prior method read`, () => { for (const caller of callers) { const body = RESOLVERS.QA_EXPLORATORY(context(host.name, caller)); - expect(body).toContain('Read `sections/scope.md`'); - expect(body).toContain('Do not repeat a Read already completed in this invocation'); - expect(body).toContain('Complete these Reads in order before writing charters or probing'); - const scope = body.indexOf('1. Read `sections/scope.md`'); - const selection = body.indexOf('in full and select the surfaces'); - const methods = body.indexOf('2. Read the selected surface methods below in full'); + const scope = body.indexOf('Read `sections/scope.md`'); + const methods = body.indexOf('**Functional surfaces:**'); expect(scope).toBeGreaterThan(-1); - expect(selection).toBeGreaterThan(scope); - expect(methods).toBeGreaterThan(selection); - expect(body.indexOf('Write a **charter**')).toBeGreaterThan(methods); + expect(methods).toBeGreaterThan(scope); + expect(body.indexOf('## 1. Charter and preflight')).toBeGreaterThan(methods); expect(body).not.toContain('If the caller has not selected surfaces and established isolation'); } }); @@ -82,12 +76,10 @@ describe('QA caller authority in pure host renders', () => { for (const caller of callers) { const body = RESOLVERS.QA_EXPLORATORY(context(host.name, caller)); expect(body).toContain('The **caller** (/qa, /qa-only, /review or /ship)'); - expect(body).toContain('charter** per behavior'); expect(body).toContain('bun G start D SECONDS [EARLIER_UTC]'); expect(body).toContain('G enforces the deadline'); - expect(body).toContain('announce finite command timeouts'); expect(body.indexOf('bun G start D')).toBeLessThan(body.indexOf('1. First demonstrate success')); - expect(body).toContain('scoped contracts are tested or blocked'); + expect(body).toMatch(/stop when scoped contracts are tested or blocked/i); } }); @@ -96,9 +88,8 @@ describe('QA caller authority in pure host renders', () => { ...context(host.name, 'qa'), tmplPath: 'qa/SKILL.md.tmpl', preambleTier: 4, }); const final = body.slice(body.indexOf('## Phase 9: Final QA'), body.indexOf('## Phase 10: Report')); - expect(final).toContain('Re-run affected contracts and adjacent happy paths on the final inputs'); - expect(final).toContain('Caller-required rechecks cannot be skipped as unaffected'); - expect(final).toContain('blocked/inconclusive rechecks never verify repairs'); + expect(final).toMatch(/caller-required rechecks cannot be skipped as unaffected/i); + expect(final).toMatch(/blocked\/inconclusive rechecks never verify repairs/i); }); test(`${host.name}: prior learnings name QA findings without changing review callers`, () => { @@ -116,12 +107,10 @@ describe('QA caller authority in pure host renders', () => { test(`${host.name}: report-only learning lookup cannot configure or initialize stores`, () => { const ctx = context(host.name, 'qa-only'); const search = RESOLVERS.LEARNINGS_SEARCH(ctx, ['query=webhook retries']); - expect(search).toContain('Read this project\'s existing learnings.jsonl only if its directory is already known'); - expect(search).toContain('the caller permits that Read'); - expect(search).toContain('Otherwise skip this optional lookup'); - expect(search).toContain('Look for notes matching "webhook retries"'); - expect(search).toContain('Do not run gstack-learnings-search here'); - expect(search).toContain('Reading old notes never requires writing new ones'); + expect(search).toMatch(/learnings\.jsonl only if its directory is already known/i); + expect(search).toMatch(/the caller permits that Read/i); + expect(search).toContain('"webhook retries"'); + expect(search).toMatch(/do not run gstack-learnings-search here/i); expect(search).not.toContain('```bash'); expect(search).not.toContain('gstack-config'); expect(search).not.toContain('AskUserQuestion'); @@ -137,7 +126,7 @@ describe('QA caller authority in pure host renders', () => { assertProbeLocalBlocker(pointer); expect(pointer).toContain(`\`sections/${id}.md\``); expect(pointer).toContain('SKILL.md directory'); - expect(pointer).toContain(sharedSetup ? 'No product-directory or cross-host substitutes' : 'never the product working directory'); + expect(pointer).toMatch(sharedSetup ? /no product-directory or cross-host substitutes/i : /never the product working directory/i); expect(pointer).not.toContain('## BROWSER SETUP'); expect(pointer).not.toContain('aside repl'); } @@ -164,12 +153,11 @@ describe('QA caller authority in pure host renders', () => { assertSharedBrowserAuthority(body); expect(body.indexOf('## Browser access decision')).toBeLessThan(body.indexOf('## BROWSER SETUP')); expect(body.indexOf('## BROWSER SETUP')).toBeLessThan(body.indexOf('## Browser fallback')); - expect(body).toContain('Functional-only'); - expect(body).toContain('do not probe Aside'); - expect(body).toContain("scope section's ownership rules apply even to LOCAL browser targets"); - expect(body).toContain('never substitute unit tests or curl for the browser step'); - expect(body).toContain('Never type passwords, one-time codes, or payment details'); - expect(body).toContain('Never read, screenshot, navigate, or close any other tab'); + expect(body.replace(/\s+/g, ' ')).toMatch(/functional-only targets do not probe Aside/i); + expect(body).toMatch(/ownership rules apply even to local browser targets/i); + expect(body).toMatch(/never substitute unit tests or curl for the browser step/i); + expect(body).toMatch(/never type passwords, one-time codes, or payment details/i); + expect(body).toMatch(/never read, screenshot, navigate, or close any other tab/i); expect((body.match(/cd && \.\/setup/g) ?? [])).toHaveLength(1); }); @@ -191,22 +179,19 @@ describe('QA caller authority in pure host renders', () => { const auth = body.slice(body.indexOf('### Phase 2:'), body.indexOf('### Phase 3:')); expect(auth).toContain('Browser access decision'); expect(auth).not.toContain('Fallback: /setup-browser-cookies or'); - expect(auth).toContain('Never handle credentials'); - expect(body).toContain('Run only for selected browser surfaces'); - expect(body).toContain('Confirm each issue by retrying once'); + expect(auth).toMatch(/never handle credentials/i); }); test(`${host.name}: functional-only and report-only paths preserve independent checks and gates`, () => { const scope = RESOLVERS.QA_SCOPE(context(host.name, 'qa')); - expect(scope).toContain('Functional-only runs must not read browser setup, methodology, verification or bootstrap'); + expect(scope).toMatch(/functional-only runs must not read browser setup/i); expect(scope).not.toContain('command -v aside'); expect(scope).not.toContain('curl -sI'); for (const caller of callers) { const ctx = context(host.name, caller); const body = RESOLVERS.QA_EXPLORATORY(ctx); const compact = body.replace(/\s+/g, ' '); - expect(compact).toContain('Missing or unreadable assets, prerequisites or permission block affected probes, not independent safe checks'); - expect(compact).toContain('Report QA setup blockers'); + expect(compact).toMatch(/block affected probes, not independent safe checks/i); expect(compact).not.toContain('stop all checks'); expect(body).toContain('2. Read the selected surface methods below in full'); expect(body.indexOf('2. Read the selected surface methods below in full')).toBeLessThan(body.indexOf('1. First demonstrate success')); @@ -215,29 +200,29 @@ describe('QA caller authority in pure host renders', () => { expect(reads).toContain('sections/system-functional.md'); expect(reads).toContain('**Browser surfaces only:**'); expect(reads).toContain('sections/qa-patterns.md'); - expect(body).toContain('no workflows, framework installs or publication'); - expect(body).toContain('owns decisions, tests, fixes and publication'); - expect(body).toContain('Missing prerequisites/expectations/observations, timeouts and refusal never pass'); - expect(body).toContain('Pass requires all required current-input contracts to pass with no required remainder'); + expect(body).toMatch(/no workflows, framework installs or publication/i); + expect(body).toMatch(/owns decisions, tests, fixes and publication/i); + expect(body).toMatch(/timeouts and refusal never pass/i); + expect(body).toMatch(/pass requires all required current-input contracts to pass/i); if (caller !== 'qa-only') { expect(body).toContain('leaves /review incomplete'); - expect(body).toContain('unless the user explicitly accepts that named risk'); - expect(body).toContain('noninteractive runs return blocked'); - expect(body).toContain('test_stub proposals require ASK approval'); + expect(body).toMatch(/unless the user explicitly accepts that named risk/i); + expect(body).toMatch(/noninteractive runs return blocked/i); + expect(body).toMatch(/test_stub proposals require ASK approval/i); } } const review = RESOLVERS.QA_REVIEW(context(host.name, 'review')); const ship = RESOLVERS.QA_REVIEW(context(host.name, 'ship')); - expect(review).toContain('a ship waiver cannot complete it'); - expect(ship).toContain('explicit named-risk acceptance'); - expect(ship.replace(/\s+/g, ' ')).toContain('Report clean/completed only when all required checks pass on current inputs'); - expect(ship).toContain('List failed, blocked, inconclusive and not-run checks'); + expect(review).toMatch(/a ship waiver cannot complete it/i); + expect(ship).toMatch(/explicit named-risk acceptance/i); + expect(ship.replace(/\s+/g, ' ')).toMatch(/report clean\/completed only when all required checks pass on current inputs/i); }); test(`${host.name}: non-QA fallback retains its existing setup and human sign-in flow`, () => { const fallback = RESOLVERS.BROWSE_FALLBACK(context(host.name, 'browse')); expect(fallback).toContain('OK to proceed?'); - expect(fallback).toContain('STOP for the answer, then run `cd && ./setup`'); + expect(fallback).toMatch(/stop for the answer, then run `cd && \.\/setup`/i); + expect(fallback.indexOf('OK to proceed?')).toBeLessThan(fallback.indexOf('cd && ./setup')); expect(fallback).toContain('An authenticated page needs /setup-browser-cookies'); expect(fallback).toContain('$B handoff'); expect(fallback).toContain('$B resume'); @@ -253,23 +238,17 @@ describe('QA caller authority in pure host renders', () => { const resource = RESOLVERS.QA_RESOURCE(ctx, ['exploratory']); expect(resource).toContain(`../${host.name === 'claude' ? 'qa' : 'gstack-qa'}/sections/exploratory.md`); const shared = render('qa/sections/exploratory.md.tmpl', context(host.name, 'qa')); - expect(shared).toContain('Read `sections/system-functional.md` in full'); - expect(shared.indexOf('in full and select the surfaces')).toBeLessThan(shared.indexOf('Read `sections/system-functional.md`')); + expect(shared.indexOf('Read `sections/scope.md`')).toBeLessThan(shared.indexOf('Read `sections/system-functional.md`')); const required = body.indexOf(caller === 'review' ? '2. Check readiness and list required checks' : '2. List required checks'); expect(required).toBeGreaterThan(-1); expect(exploration).toBeLessThan(required); - expect(body).toContain('one success and the riskiest changed failure/edge'); - expect(body).toContain('Smoke: 5 minutes/12 probes'); - expect(body).toContain('Required even for small diffs or missing plans/servers'); - expect(body).toContain('Required: plan commands/assertions, listed separately'); - expect(body).toContain('Other ideas are optional, untested'); - expect(body.replace(/\s+/g, ' ')).toContain('Report clean/completed only when all required checks pass on current inputs'); - expect(body).toContain('List failed, blocked, inconclusive and not-run checks'); + expect(body).toContain('5 minutes/12 probes'); + expect(body.replace(/\s+/g, ' ')).toMatch(/report clean\/completed only when all required checks pass on current inputs/i); } const ship = RESOLVERS.QA_REVIEW(context(host.name, 'ship')); - expect(ship).toContain('Step 9.4 asks: permission/repair'); - expect(ship).toContain('explicit named-risk acceptance; otherwise blocked'); + expect(ship).toContain('Step 9.4'); + expect(ship).toMatch(/explicit named-risk acceptance; otherwise blocked/i); }); test(`${host.name}: caller QA selects surfaces directly and links checkpoints in one final section`, () => { @@ -279,33 +258,25 @@ describe('QA caller authority in pure host renders', () => { const exploration = body.indexOf('{{QA_RESOURCE:exploratory}}'); const shared = render('qa/sections/exploratory.md.tmpl', context(host.name, 'qa')); const scope = shared.indexOf('Read `sections/scope.md`'); - const selection = shared.indexOf('in full and select the surfaces'); const methods = shared.indexOf('**Functional surfaces:**'); const probes = body.indexOf(caller === 'review' ? '2. Check readiness and list required checks' : '2. List required checks'); expect(scope).toBeGreaterThan(-1); expect(exploration).toBeGreaterThan(-1); - expect(selection).toBeGreaterThan(scope); - expect(methods).toBeGreaterThan(selection); - expect(shared.indexOf('Write a **charter**')).toBeGreaterThan(methods); + expect(methods).toBeGreaterThan(scope); + expect(shared.indexOf('## 1. Charter and preflight')).toBeGreaterThan(methods); expect(exploration).toBeLessThan(probes); expect(body).not.toContain('**Functional surfaces:**'); expect(RESOLVERS.QA_RESOURCE(ctx, ['exploratory'])).toContain(`../${host.name === 'claude' ? 'qa' : 'gstack-qa'}/sections/exploratory.md`); if (caller === 'review') { const flat = body.replace(/\s+/g, ' '); - expect(flat).toContain('Title it `## Exploratory QA and Verification Results`'); - expect(flat).toContain('keep metadata/outcome tables'); - expect(flat).toContain('demote other headings one level'); - expect(flat).toContain('include it here under `### Browser results`'); - expect(flat).toContain('other headings demoted two levels'); - expect(flat).toContain('Link every checkpoint'); - expect(flat).toContain('No second report'); - expect(flat).toContain('Keep browser/functional scores and outcomes separate'); - expect(flat).toContain('save browser baseline/evidence normally'); + expect(flat).toContain('`## Exploratory QA and Verification Results`'); + expect(flat).toContain('`### Browser results`'); + expect(flat).toMatch(/link every checkpoint/i); + expect(flat).toMatch(/no second report/i); } else { - expect(body.replace(/\s+/g, ' ')).toContain('PR section `## Exploratory QA'); - expect(body).toContain('fields as subsections'); - expect(body).toContain('Link every checkpoint; no second report'); + expect(body.replace(/\s+/g, ' ')).toContain('`## Exploratory QA'); + expect(body).toMatch(/link every checkpoint; no second report/i); } expect(body).toContain('templates/functional-report-template.md'); expect(body).not.toContain('not a second report'); @@ -314,14 +285,14 @@ describe('QA caller authority in pure host renders', () => { test(`${host.name}: orchestration logs reviewer attempts before parent-owned edits`, () => { const body = RESOLVERS.ADVERSARIAL_STEP(context(host.name, 'review')); - expect(body).toContain("queued for the parent's Fix-First handling at Step 5; do not edit during Step 4.8"); - expect(body).toContain('do not start an inner repair loop'); - expect(body.replace(/\s+/g, ' ')).toContain("Keep each token with that attempt; do not overwrite the parent's REVIEW_START"); - expect(body.replace(/\s+/g, ' ')).toContain('save one record per source, phase and attempt, before the'); - expect(body).toContain('parent applies queued fixes'); - expect(body).toContain('Each token is consumed once'); + const flat = body.replace(/\s+/g, ' '); + expect(flat).toMatch(/Fix-First handling at Step 5; do not edit during Step 4\.8/i); + expect(flat).toMatch(/do not start an inner repair loop/i); + expect(flat).toMatch(/do not overwrite the parent's REVIEW_START/i); + expect(flat).toMatch(/one record per source, phase and attempt/i); + expect(flat).toMatch(/each token is consumed once/i); expect(body).not.toContain('address the findings. Re-run the same shared structured invocation'); - expect(body).toContain('The native pass is required for Step 5.8 completion'); + expect(flat).toMatch(/native pass is required for Step 5\.8 completion/i); }); test(`${host.name}: orchestration skips only history matching without user skips`, () => { @@ -329,17 +300,12 @@ describe('QA caller authority in pure host renders', () => { const body = RESOLVERS.CROSS_REVIEW_DEDUP(context(host.name, caller)); if (caller === 'ship') { const flat = body.replace(/\s+/g, ' '); - expect(flat).toContain('Combine saved `findings` with the invocation action list, honoring later user decisions'); - expect(flat).toContain('If both history and the invocation action list lack decisions, classify normally'); expect(body.indexOf('2. **Read decisions.**')).toBeLessThan(body.indexOf('3. **Match evidence.**')); - expect(flat).toContain('Only explicit `skipped` actions qualify, never `fixed`, `auto-fixed` or unanswered questions'); - expect(flat).toContain('Report the suppressed count once if nonzero'); + expect(flat).toMatch(/only explicit `skipped` actions qualify, never `fixed`, `auto-fixed` or unanswered questions/i); } else { - expect(body).toContain('skip history matching silently; still classify current findings'); + expect(body).toMatch(/still classify current findings/i); expect(body.indexOf('If no prior reviews exist')).toBeLessThan(body.indexOf('For each JSONL entry')); - expect(body).toContain('If N > 0, print once:'); - expect(body).toContain('Otherwise skip the summary'); - expect(body).toContain('Only suppress `skipped` findings — never `fixed` or `auto-fixed`'); + expect(body).toMatch(/only suppress `skipped` findings — never `fixed` or `auto-fixed`/i); } expect(body).not.toContain('skip this step silently'); } @@ -347,11 +313,11 @@ describe('QA caller authority in pure host renders', () => { test(`${host.name}: orchestration consumes the existing generation allowance at both coverage gates`, () => { const body = RESOLVERS.TEST_COVERAGE_GATE_SHIP(context(host.name, 'ship')); - expect(body).toContain("Use Step 7's remaining generation allowance"); + expect(body).toContain("Step 7's remaining generation allowance"); expect(body.match(/If A and allowance remains:/g)).toHaveLength(2); - expect(body).toContain('At the cap, offer only B/C or stop'); - expect(body).toContain('At the cap, offer only B or stop'); - expect(body).toContain('At the cap, omit A\'s generation pass and recommend stopping'); + expect(body).toMatch(/at the cap, offer only B\/C or stop/i); + expect(body).toMatch(/at the cap, offer only B or stop/i); + expect(body).toMatch(/at the cap, omit A's generation pass and recommend stopping/i); expect(body).toContain('Minimum = 60%, Target = 80%'); expect(body).not.toContain('Maximum 2 passes total'); }); @@ -359,24 +325,23 @@ describe('QA caller authority in pure host renders', () => { test('report-only recommendations cannot dispatch a repair skill during discovery', () => { const skill = readFileSync(join(root, 'qa-only/SKILL.md.tmpl'), 'utf8'); - expect(skill).toContain('Never invoke /qa or another skill from this report-only run'); - expect(skill).toContain('separate, user-authorized'); - expect(skill).toContain('No test framework detected'); - expect(skill).toContain('Never commit, stash or bootstrap'); + expect(skill).toMatch(/never invoke \/qa or another skill from this report-only run/i); + expect(skill).toMatch(/separate, user-authorized/i); + expect(skill).toMatch(/never commit, stash or bootstrap/i); }); test('standalone QA scopes its general test rule around the approved browser bootstrap', () => { const skill = readFileSync(join(root, 'qa/SKILL.md.tmpl'), 'utf8'); const rule = skill.split('\n').find(line => line.startsWith('**Outside an explicitly approved browser bootstrap:**')); expect(skill).not.toMatch(/^13\. /m); - expect(rule).toContain('Outside an explicitly approved browser bootstrap'); - expect(rule).toContain('Only create tests through authorized codification in Phase 8a.5'); - expect(rule).toContain('Never modify CI configuration or weaken existing tests'); + expect(rule).toMatch(/authorized codification in Phase 8a\.5/i); + expect(rule).toMatch(/never modify CI configuration or weaken existing tests/i); const bootstrap = readFileSync(join(root, 'qa/sections/test-bootstrap.md.tmpl'), 'utf8'); - expect(bootstrap).toContain('Browser /qa only, never functional/report-only'); - expect(bootstrap).toContain('AskUserQuestion and WAIT'); - expect(bootstrap).toContain('install only the actual choice'); - expect(bootstrap).toContain('Never silently delete a valid red regression'); + expect(bootstrap).toMatch(/browser \/qa only, never functional\/report-only/i); + const ask = bootstrap.search(/AskUserQuestion and wait/i); + expect(ask).toBeGreaterThan(-1); + expect(ask).toBeLessThan(bootstrap.search(/install only the actual choice/i)); + expect(bootstrap).toMatch(/never silently delete a valid red regression/i); expect(bootstrap).toContain('create/extend `.github/workflows/test.yml`'); }); @@ -387,6 +352,7 @@ describe('QA caller authority in pure host renders', () => { expect(() => assertProbeLocalBlocker(pointer.replace('continue other safe probes', 'stop all checks'))).toThrow(); const setup = render('qa/sections/browser-setup.md.tmpl', ctx); expect(() => assertSharedBrowserAuthority(setup.replace('Only after approval', 'Immediately'))).toThrow(); + expect(() => assertSharedBrowserAuthority(setup.replace('explicit approval', 'a status update'))).toThrow(); expect(() => assertSharedBrowserAuthority(setup.replace('Never bootstrap or invoke another skill', 'Invoke another skill'))).toThrow(); expect(() => assertSharedBrowserAuthority(setup + '\nIf `NEEDS_SETUP`: tell the user')).toThrow(); expect(() => assertSharedBrowserAuthority(setup + '\nAn authenticated page needs /setup-browser-cookies')).toThrow(); From 851328b35f6b185d1b2b5b231b63ee6902daf384 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 01:24:11 +0000 Subject: [PATCH 09/13] test(W7c): workflow-excerpt keeps heading order and tokens, drops judged prose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit W7c (t04 F5, F6, F7, F15; F1/F2 already landed with W7a). One test file. This file guards judge-input fidelity; the judges that read these excerpts grade the prose itself. Removed assertion groups (invariant -> remaining protection): - Ship rule sentences ("Use the documented selector and pre-merge command.", "During pre-flight, read the existing review log", "Tests must pass or retain approval ...", "For another repair, repeat rule 2 ...", "The unchanged release steps follow", "Use this example only after confirming ...", "Retain queued fixes and restore coverage", "Each commit must work independently", "Keep actual outcomes and incomplete flags", the long "Continue to Step 9.2 with the core/design-lite findings ..." form, the Y'd/D'd evidence sentence) -> judge `ship/SKILL.md workflow` (gate lane), which reads this exact excerpt; kept free checks on step routing (`Step 4`, `Step 8.1`, 9.2 -> 9.2.1 QA -> 9.3, `Insert Step 5, affected Steps 6–8 and all of Step 9 before the pending Step 10`, `5 → 6 → … → 11.5`, `continue to Step 16`), tokens (`BASE_VERSION`, `VERIFY_RESULT stays fail`, `"partial":N,"not_done":N`, "intentionally dropped"), and the review cycle order (9.1 < 9.2 < 9.2.1 < 9.3 < 9.4) as a structural check. - Outside-review routing sentences ("Other preflight mode, including harness mismatch", "Outside execution or output validation fails", "Retain its output and diagnosis ...", "No prompt, outside process or native replacement", "Other preflight failures retain their printed diagnosis", the full fallback-list regex) -> disabled means no outside call: kept free order of disabled/prompt/ready/fallback blocks, `outside_status` tokens, `_DISABLED_REVIEW_MODE`, disabled recheck before dispatch, "disabled means no replacement", fallback-not-completion; judges `document-release/SKILL.md workflow`, `plan-eng-review/SKILL.md sections` (gate lane). - Eng/design prose ("quote the motivating plan requirement", "Use a scoped Edit to save this record ... Leave other choices unchanged" full form, "edit the plan and re-rate") -> judge `plan-eng-review/SKILL.md sections` and `plan-design-review/SKILL.md passes` (gate lane); approval meaning checks kept. - Retro prose ("does not require saved history", "PRs referenced", "not newly added test cases", "is the live **team** streak", "draft the tweetable summary using the format in Step 14, then save") -> judge `retro/SKILL.md instructions` (gate lane); kept headings, `prs_merged: null`, `streak_days`, the one-second compare window, Capture Learnings after Step 14, no `$(date`, two `today=""`. Loosened to case-insensitive meaning checks (stop/consent/approval and safety lines, always kept): STOP for each pending decision (now a regex anchor between Step 5 and the Test Plan Artifact), STOP until the actual answer arrives, no remedy/next section/ExitPlanMode while a choice awaits an answer, evidence-not-approval, reopened choices need answers, wait for approval (design), AskUserQuestion once (t04 F6), per-finding approvals, dispatched-reviewer-missing -> stop, CYCLES >= 3 -> stop with `converged:false`, no fourth cycle, required-probe gate before Step 10, failed probes block continuation, coverage-gate not bypassed, failed/unrun probes not passed, three snapshots match, document-release VERSION-absent rule, never create an empty commit, `read any other SKILL.md` (t04 F15), AskUserQuestion-and-wait in the capture prompt. Kept unchanged: heading/step order checks, `## Review Sections` single-copy check, fail-closed marker tests, redact-before-edit ordering, approval gates outside subagent prompts, plan-tune auto-decide output lines, CEO capture harness contract, deploy formula execution, all stale-string bans. Cosmetic pins dropped: STOP / ONCE / NOT / OTHER capitals. Mutation log: .context/mutation-workflow-excerpt.md (21 mutations, 0 unexpected: 3 reorders, 3 markers, 8 safety red; 7 rewordings green). --- test/workflow-excerpt.test.ts | 126 +++++++++++++++------------------- 1 file changed, 57 insertions(+), 69 deletions(-) diff --git a/test/workflow-excerpt.test.ts b/test/workflow-excerpt.test.ts index 9909a630f..9744e52df 100644 --- a/test/workflow-excerpt.test.ts +++ b/test/workflow-excerpt.test.ts @@ -14,42 +14,36 @@ function expectOutsideReviewControlFlow(text: string, promptHeading: string): vo const disabled = text.slice(indices[0], indices[1]); if (ceo) { expect(disabled).toContain('"outside_status":"disabled"'); - expect(disabled.replace(/\s+/g, ' ')).toContain('without a challenge, CLI invocation, Agent/Task fallback or questions about outside findings'); + expect(disabled.replace(/\s+/g, ' ')).toMatch(/without a challenge, CLI invocation, Agent\/Task fallback/i); expect(disabled).toContain('_DISABLED_REVIEW_MODE='); expect(disabled).toContain('if [ "$_DISABLED_REVIEW_MODE" = disabled ]'); } else { expect(disabled).toContain('persist `outside_status: disabled`'); - expect(disabled.replace(/\s+/g, ' ')).toMatch(/Do not construct a (?:review prompt|challenge), invoke an outside CLI, dispatch an Agent\/Task fallback/); + expect(disabled.replace(/\s+/g, ' ')).toMatch(/do not construct a (?:review prompt|challenge), invoke an outside CLI, dispatch an Agent\/Task fallback/i); } expect(text.slice(indices[1], indices[2])).toContain('(skip only on `disabled`)'); const fallback = text.slice(indices[3]); if (text.includes('**Outcome routing:**')) { - const routing = text.slice(text.indexOf('**Outcome routing:**'), indices[0]); - expect(routing).toContain('Other preflight mode, including harness mismatch'); - expect(routing).toContain('Outside execution or output validation fails'); - expect(routing).toContain('Retain its output and diagnosis, finish termination, then use Native fallback'); - expect(routing).toContain('No prompt, outside process or native replacement'); - expect(fallback.replace(/\s+/g, ' ')).toContain('Immediately before dispatch, check the preflight result again: disabled means no replacement'); + expect(fallback.replace(/\s+/g, ' ')).toMatch(/immediately before dispatch, check the preflight result again: disabled means no replacement/i); } else if (ceo) { - expect(fallback.replace(/\s+/g, ' ')).toContain('Other preflight failures retain their printed diagnosis, including harness mismatch'); - expect(fallback.replace(/\s+/g, ' ')).toContain('These failures do not block the review; they use the bounded fallback below'); + expect(fallback.replace(/\s+/g, ' ')).toMatch(/do not block the review; they use the bounded fallback/i); } else { - expect(fallback).toContain('The disabled branch never reaches this fallback.'); - expect(fallback.replace(/\s+/g, ' ')).toMatch(/Otherwise, use this fallback for missing\/broken CLI, failed authentication\/model selection, a failed preflight(?: \(including harness mismatch\))?, or a failed outside invocation\./); + expect(fallback).toMatch(/the disabled branch never reaches this fallback/i); + expect(fallback.replace(/\s+/g, ' ')).toMatch(/use this fallback for missing\/broken CLI/i); } const dispatch = fallback.indexOf('Dispatch via the Agent tool'); expect(dispatch).toBeGreaterThan(0); - const recheck = fallback.slice(0, dispatch); + const recheck = fallback.slice(0, dispatch).replace(/\s+/g, ' '); if (ceo) { - expect(recheck.replace(/\s+/g, ' ')).toContain('Immediately before dispatch, recheck the preflight result'); - expect(recheck.replace(/\s+/g, ' ')).toContain('`CODEX_MODE: disabled`, return to **Record the disabled outcome** without dispatching'); + expect(recheck).toMatch(/immediately before dispatch, recheck the preflight result/i); + expect(recheck).toMatch(/`CODEX_MODE: disabled`, return to \*\*Record the disabled outcome\*\* without dispatching/i); } else if (!text.includes('**Outcome routing:**')) { - expect(recheck).toContain('Immediately before dispatching, check the preflight result again.'); + expect(recheck).toMatch(/immediately before dispatching, check the preflight result again/i); expect(recheck).toContain('`CODEX_MODE: disabled`, finish this section with `outside_status: disabled`;'); - expect(recheck).toContain('do not dispatch.'); + expect(recheck).toMatch(/do not dispatch/i); } - expect(fallback).toContain('Availability/native fallback is not outside completion.'); + expect(fallback).toMatch(/availability\/native fallback is not outside completion/i); } describe('workflow judge excerpts', () => { @@ -63,15 +57,13 @@ describe('workflow judge excerpts', () => { test('ship uses project-native commands and never jumps over mandatory gates', () => { const text = readWorkflowExcerpt('ship/SKILL.md', '# Ship:', '## Important Rules'); - expect(text).toContain("Use the project's test commands discovered in Step 4"); + expect(text).toMatch(/test commands discovered in Step 4/i); expect(text).toContain('**Project-native path:**'); - expect(text).toContain('Use the documented selector and pre-merge command.'); expect(text).not.toMatch(/skipping evals[^\n]*Step 9/); const reviewAndTriage = text.slice(text.indexOf('## Step 9:'), text.indexOf('## Step 11:')); expect(reviewAndTriage.match(/continue to Step 12/i)).toBeNull(); expect(text).not.toContain('Steps 4-6:'); - expect(text).toContain('During pre-flight, read the existing review log'); - expect(text).toContain('Save the JSON `baseVersion` as `BASE_VERSION`'); + expect(text).toContain('`baseVersion` as `BASE_VERSION`'); expect(text).not.toContain('GIT_SEQUENCE_EDITOR'); expect(text).not.toContain("--exec 'true'"); expect(text).not.toContain('-X ours'); @@ -80,27 +72,28 @@ describe('workflow judge excerpts', () => { test('ship review shortcuts retain dedup and fixes repeat the whole review cycle', () => { const text = readWorkflowExcerpt('ship/SKILL.md', '# Ship:', '## Important Rules'); - expect(text).toContain("Continue to Step 9.2 with the core/design-lite findings and an empty specialist list, then the parent's Exploratory QA step and Step 9.3 (cross-review dedup)"); + const flat = text.replace(/\s+/g, ' '); + expect(flat).toMatch(/Continue to Step 9\.2 [^.]*then the parent's Exploratory QA step and Step 9\.3/i); expect(text).toContain('## Step 9.4: Fix-First and persistence'); - expect(text.replace(/\s+/g, ' ')).toContain('Run checklist/design, specialists (9.1), merge/Red Team (9.2), exploratory QA (9.2.1), dedup (9.3), then fixes and logging (9.4)'); - expect(text.replace(/\s+/g, ' ')).toContain('**Fixes applied below the cap:** Insert Step 5, affected Steps 6–8 and all of Step 9 before the pending Step 10 in the work list. Tests must pass or retain approval for the same verified pre-existing failures and scope'); + const cycle = flat.slice(flat.indexOf('specialists (9.1)')); + const order = ['(9.1)', '(9.2)', '(9.2.1)', '(9.3)', '(9.4)'].map(step => cycle.indexOf(step)); + expect(order.every(index => index >= 0)).toBe(true); + expect(order).toEqual([...order].sort((a, b) => a - b)); + expect(flat).toContain('**Fixes applied below the cap:** Insert Step 5, affected Steps 6–8 and all of Step 9 before the pending Step 10'); const audit = text.slice(text.indexOf('## Step 7:'), text.indexOf('## Step 8:')); expect(audit).not.toContain('Scope Challenge'); - expect(text.replace(/\s+/g, ' ')).toContain('Keep actual outcomes and incomplete flags; VERIFY_RESULT stays fail for plan-check exceptions'); + expect(flat).toContain('VERIFY_RESULT stays fail'); }); test('ship excerpt preserves readable detours, audit fallback and final input decisions', () => { const text = readWorkflowExcerpt('ship/SKILL.md', '# Ship:', '## Important Rules').replace(/\s+/g, ' '); - expect(text).toContain('For another repair, repeat rule 2 without discarding pending work'); - expect(text).toContain('A further Step 9 fix affecting 6–8 makes the list `5 → 6 → 7 → 8 → 9 → 10 → 11 → 11.5`'); - expect(text).toContain('The unchanged release steps follow'); - expect(text).toContain('All three snapshots must match'); - expect(text).toContain('does not mean the failed or unrun probes passed'); - expect(text).toContain('Fallback recovers the audit; it does not pass or bypass the coverage gate'); - expect(text).toContain('Skip only the plan completion audit'); - expect(text).toContain('Continue with Step 8.1, Scope Drift and Prior Learnings'); - expect(text).toContain('Step 9 QA still runs'); - expect(text).toContain('Use this example only after confirming that every allowed edit is release metadata'); + expect(text).toContain('`5 → 6 → 7 → 8 → 9 → 10 → 11 → 11.5`'); + expect(text).toMatch(/all three snapshots must match/i); + expect(text).toMatch(/does not mean the failed or unrun probes passed/i); + expect(text).toMatch(/does not pass or bypass the coverage gate/i); + expect(text).toMatch(/skip only the plan completion audit/i); + expect(text).toContain('Step 8.1'); + expect(text).toMatch(/Step 9 QA still runs/i); expect(text).not.toContain('Every listed change below is metadata:'); expect(text).not.toContain('No plan file found:** Skip entirely'); }); @@ -123,15 +116,16 @@ describe('workflow judge excerpts', () => { expect(edit).toBeGreaterThan(scan); expect(publish.indexOf('Print the existing URL')).toBeGreaterThan(edit); expect(text).not.toContain('Phase 8e.5'); - expect(text).toContain('never create an empty commit'); + expect(text).toMatch(/never create an empty commit/i); const review = text.slice(text.indexOf('## Step 9:'), text.indexOf('## Step 10:')); expect(review.indexOf('## Confidence Calibration')).toBeLessThan(review.indexOf('1. Read')); const flat = review.replace(/\s+/g, ' '); - expect(flat).toContain('**No edits in this pass:** Resolve the required-probe gate below. Only after it clears may you continue to Step 10'); - expect(flat).toContain('**Dispatched reviewer output missing:** STOP'); - expect(flat).toContain('Retain queued fixes and restore coverage'); - expect(flat).toContain('**Third fixing cycle reached (`CYCLES >= 3`):** STOP and report recurring findings with `converged:false`; do not run a fourth fixing cycle'); - expect(flat).toContain('With completed checklist and dispatched reviewers, failed/unavailable required probes block continuation'); + expect(flat).toContain('**No edits in this pass:**'); + expect(flat).toMatch(/only after it clears may you continue to Step 10/i); + expect(flat).toMatch(/\*\*Dispatched reviewer output missing:\*\* stop\b/i); + expect(flat).toMatch(/\*\*Third fixing cycle reached \(`CYCLES >= 3`\):\*\* stop\b[^.]*`converged:false`/i); + expect(flat).toMatch(/do not run a fourth fixing cycle/i); + expect(flat).toMatch(/failed\/unavailable required probes block continuation/i); }); test('ship approval gates stay outside the subagent prompts', () => { @@ -145,7 +139,7 @@ describe('workflow judge excerpts', () => { expect(section.indexOf(gate)).toBeGreaterThan(section.indexOf('\n````\n')); } expect(text).toContain('"partial":N,"not_done":N'); - expect(text).toContain('each Y\'d item with the user\'s free-text evidence and each D\'d item with "intentionally dropped"'); + expect(text).toContain('"intentionally dropped"'); }); test('expands a body before the end marker in the skeleton', () => { @@ -163,13 +157,13 @@ describe('workflow judge excerpts', () => { expect(commit).toBeGreaterThan(reviewStart); const review = text.slice(reviewStart, commit); expectOutsideReviewControlFlow(review, '**Construct the doc-review prompt**'); - expect(review).toContain('Skip the apply gate, persist `status: unavailable`, `outside_status: unavailable`, and `source: none`'); - expect(review).toContain('present the findings, then use AskUserQuestion ONCE:'); - expect(review).toContain('On A or per-finding approvals, make the approved edits yourself'); - expect(text).toContain('Step 9 then commits and pushes those edits'); + for (const token of ['`status: unavailable`', '`outside_status: unavailable`', '`source: none`']) expect(review).toContain(token); + expect(review).toMatch(/AskUserQuestion once/i); + expect(review).toMatch(/on A or per-finding approvals/i); + expect(text).toContain('Step 9 then commits'); expect(text).toMatch(/needs?\s+attention,?\s+not\s+replacement/i); expect(text).not.toContain('Flag and rewrite'); - expect(text).toContain('if VERSION is absent, use the completion date only'); + expect(text).toMatch(/if VERSION is absent, use the completion date only/i); }); test('Eng preparation and decision procedure precede the four review sections', async () => { @@ -261,8 +255,8 @@ console.log(JSON.stringify({calls, results})); expect(call.userPrompt).toContain('Proceed to Mode Selection,'); expect(call.userPrompt).not.toMatch(/Step 0[A-Z]/); expect(call.userPrompt).toContain(join(call.workingDirectory, 'plan-ceo-review', 'SKILL.md')); - expect(call.userPrompt).toContain('Do NOT search for, Glob, find, or read any OTHER SKILL.md'); - expect(call.userPrompt).toContain('Ask the user through the AskUserQuestion tool and wait for their answer.'); + expect(call.userPrompt).toMatch(/read any other SKILL\.md/i); + expect(call.userPrompt).toMatch(/AskUserQuestion tool and wait for their answer/i); expect(call).toMatchObject({ allowedTools: ['Read', 'Write', 'AskUserQuestion'], maxTurns: 12, maxRetries: 0, model: 'fake-model' }); } } finally { rmSync(dir, { recursive: true, force: true }); } @@ -273,28 +267,27 @@ console.log(JSON.stringify({calls, results})); const tests = eng.slice(eng.indexOf('### 3. Test review'), eng.indexOf('### 4. Performance review')); const scope = tests.indexOf('### LLM/eval scope'); const decisions = tests.indexOf('**Step 5. Add missing tests to the plan:**'); - const stop = tests.indexOf("**STOP for each pending decision.**", decisions); + const stop = decisions + tests.slice(decisions).search(/stop for each pending decision/i); const artifact = tests.indexOf('### Test Plan Artifact'); expect(0 <= scope && scope < decisions && decisions < stop && stop < artifact).toBe(true); expect(tests.match(/For LLM\/prompt changes:/g)).toHaveLength(1); - expect(tests.slice(artifact)).not.toContain("**STOP for each pending decision.**"); + expect(tests.slice(artifact)).not.toMatch(/stop for each pending decision/i); const fastPath = tests.slice(tests.indexOf('**Fast path:**'), scope); - expect(fastPath).toContain('Still check LLM/eval scope and produce the Test Plan Artifact'); + expect(fastPath).toMatch(/still check LLM\/eval scope/i); }); test('plan review evidence and design approval rules precede their use', () => { const eng = readWorkflowExcerpt('plan-eng-review/SKILL.md', '## Scope gate', '## Section self-check'); expect(eng.indexOf('## Confidence Calibration')).toBeLessThan(eng.indexOf('### 1. Architecture review')); - expect(eng).toContain('quote the motivating plan requirement'); expectOutsideReviewControlFlow(eng, '**Construct the plan review prompt**'); - expect(eng).toContain('Agreement between reviewers is evidence, not approval'); - expect(eng).toContain('new or reopened choices still need their own answers'); + expect(eng).toMatch(/evidence, not approval/i); + expect(eng).toMatch(/new or reopened choices still need their own answers/i); const pendingDecision = eng.slice(eng.indexOf('### Send once and wait'), eng.indexOf("### Record the answer")); - expect(pendingDecision).toContain("**STOP until the actual answer arrives.**"); - expect(pendingDecision.replace(/\s+/g, ' ')).toContain("Do not apply a remedy, make another call, start the next section or call ExitPlanMode while the choice awaits an answer"); - expect(eng.replace(/\s+/g, ' ')).toContain("Use a scoped Edit to save this record and only the authorized working-plan amendments. Leave other choices unchanged"); + expect(pendingDecision).toMatch(/stop until the actual answer arrives/i); + expect(pendingDecision.replace(/\s+/g, ' ')).toMatch(/do not apply a remedy, make another call, start the next section or call ExitPlanMode while the choice awaits an answer/i); + expect(eng.replace(/\s+/g, ' ')).toMatch(/only the authorized working-plan amendments/i); const design = readWorkflowExcerpt('plan-design-review/SKILL.md', '## Review Sections', ASK_QUESTIONS_HEADING); - expect(design).toContain('wait for approval, then edit the plan and re-rate'); + expect(design).toMatch(/wait for approval, then edit the plan/i); const pass4 = design.slice(design.indexOf('### Pass 4:'), design.indexOf('### Pass 5:')); expect(pass4.match(/^### /gm)).toHaveLength(1); expect(pass4).toMatch(/^#### Design Hard Rules$/m); @@ -311,13 +304,9 @@ console.log(JSON.stringify({calls, results})); test('retro judge includes compare semantics and unambiguous report inputs', () => { const text = readWorkflowExcerpt('retro/SKILL.md', '## Instructions', '## Tone'); expect(text).toContain('## Compare Mode'); - expect(text).toContain('does not require saved history'); - expect(text).toContain('one second before the current start'); - expect(text).toContain('PRs referenced'); + expect(text).toMatch(/one second before the current start/i); expect(text).toContain('prs_merged: null'); - expect(text).toContain('not newly added test cases'); - expect(text).toContain('`streak_days` is the live **team** streak'); - expect(text).toContain('draft the tweetable summary using the format in Step 14, then save'); + expect(text).toContain('`streak_days`'); expect(text).toContain('### Shipping Streaks'); expect(text).toContain('### Shortcut Debt'); expect(text.indexOf('## Capture Learnings')).toBeGreaterThan(text.indexOf('### Step 14:')); @@ -343,9 +332,8 @@ console.log(JSON.stringify({calls, results})); test('ship commits logical chunks without rewriting existing checkpoint commits', () => { const text = readWorkflowExcerpt('ship/SKILL.md', '# Ship:', '## Important Rules'); const commit = text.slice(text.indexOf('## Step 15:'), text.indexOf('## Step 16:')); - expect(commit).toContain('Make bisectable commits'); - expect(commit).toContain('if already committed, continue to Step 16'); - expect(commit).toContain('Each commit must work independently'); + expect(commit).toMatch(/bisectable commits/i); + expect(commit).toContain('continue to Step 16'); expect(commit).not.toMatch(/rebase|reset|squash|fixup|WIP_TODO|gstack-context/); expect(text).not.toContain('Step 15.0'); }); From 9c5aedea1237d92e52e7a3680366733606497a7b Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 01:29:35 +0000 Subject: [PATCH 10/13] test(W7c): review-workflow-clarity keeps order, tokens and gates, drops judged prose W7c (t06 #12). One test file. Each removed pin was the text fix for a past `review/SKILL.md workflow` judge finding; that judge (gate lane) still grades the prose, so the free test keeps only structure, tokens and gates. Removed assertion groups (invariant -> remaining protection): - Plan-audit sentences ("retain the exact command ...", "A mixed item contributes to both lists", "Keep external-state ...", "If only behavioral checks remain ...", "File-existence checks ... are static audit checks") -> audited deliverables vs deferred behavioral checks: kept free checks on the separation, never DONE from a diff, zero-deliverable no-waive, UNVERIFIABLE -> Step 4.7, read-only audit (no apps/APIs/state) and inspect -before-invoke order; gate E2E review-exploratory-small-cli (gate). - QA loop/report duplicates of qa-probe-gates / qa-caller-authority (probe loop sentences, shared contracts, report layout prose, "Only the parent runs report-only discovery", "Read agent/user updates ...", "Prepare one provisional QA section", smoke composition) -> owned there (free, every host); here only `5 minutes/12 probes`, headings, link-every-checkpoint, no second report, never overwrite another run, freshness vs current inputs. - Persistence/finalization prose (~45: "final zero-edit pass, reconcile", "by structural identity", "without requiring deleted pre-extraction blocks", "raw bytes equal the bound snapshot blobs", "Keep both through re-reviews", "Earlier core tokens remain unused", "Re-read its final- snapshot supporting source ...", "Fixed advice needs no skip coverage", "The required in-host adversarial result controls native completion", "Optional outside attempts keep their own incomplete records ...", etc.) -> kept free token list (REVIEW_START/PASS_START, COMPLETED, CONVERGED, CYCLES=0 once, `review_binding`, `snapshot_covered_paths`, sharedLibsFingerprint, `completed:false`, evidence_paths/helper_target), cycle limit (at most 3, Below 3 repeat, At 3 persist `converged:false`), section order, and the integrity gates below; judge `review/SKILL.md workflow` (gate lane); gate E2Es review-army-delivery-audit, review-army-json-findings, review-army-quality-score (gate). - Confidence-calibration prose (data loss/outage examples, missing field / nullable / race quoting, metaclass/ORM sentence, missing-literal sentence, learning-log sentence) -> kept the score table as a routing table (each band's action), `[CRITICAL]` / `[CRITICAL|INFORMATIONAL]`, no P0-2, `file:line`, force 4-5, never invent 7+, unverified-without-quote, gate before table; judge `review/SKILL.md workflow` (gate lane). - Shared-code reuse safeguard enumeration (assume-unchanged/skip-worktree, sparse index, symlinks, submodules, Git filters, encodings, fsmonitor, textconv, provenance, binding wording) -> enforced in code and covered by free tests test/shared-libs-snapshot-check.test.ts, test/shared-libs-evidence.test.ts, test/shared-libs-checker-interface- evidence.test.ts; kept the checker command/heredoc/JSON tokens and the fail-closed lines. - Misc definitions ("A probe is affected when ...", "An invocation is this /review run ...", "Step 1.5's provisional notes ...", "Otherwise record each considered specialist", "Core findings use the confidence gates below", "highest confidence", "distinct specialists", "all source names", "active or unknown reader/writer", Greptile "Before edits, inspect ...") -> judge `review/SKILL.md workflow` (gate lane). Kept as case-insensitive meaning checks (stop/consent/approval/safety): no edits before Step 5, readers returned or confirmed stopped, no edits in Step 4.8, stop without edits on unconfirmed readers (review and ship), stop at 3 cycles with no clean summary or fourth pass, failed/blocked probes mean incomplete, ship named-risk cannot complete review, native vs outside cannot substitute, structured-review gate, never carry coverage forward, never invent a binding, skipped advisory cannot suppress a defect, user-skip-only suppression, Greptile reply != code approval and wait for approval before editing, HIGH-impact gate options (A ends, read-only audit, no shipping authorization), never downgrade severity, never trust a supplied hash, missing metadata cannot dedupe, test creation needs approval, report-only review never runs setup/installs/cookie import. Kept unchanged: all marker-order arrays, manifest order, Pre-Landing summary after the logger, finish-token count and negatives, stale-wording bans (incl. `git diff origin/main`), ship/review routing negatives. Cosmetic pins dropped: STOP / NON / COUNT capitals; the test titled "resolves the judged ... ambiguities" is renamed to say what it keeps. Mutation log: .context/mutation-review-workflow-clarity.md (23 mutations, 0 unexpected: 3 reorders, 4 markers, 11 safety red; 5 rewordings green). --- test/review-workflow-clarity.test.ts | 426 ++++++++++----------------- 1 file changed, 158 insertions(+), 268 deletions(-) diff --git a/test/review-workflow-clarity.test.ts b/test/review-workflow-clarity.test.ts index 1bb6b30e2..e3a63df17 100644 --- a/test/review-workflow-clarity.test.ts +++ b/test/review-workflow-clarity.test.ts @@ -4,7 +4,7 @@ import { join } from 'node:path'; import { generateReviewArmy } from '../scripts/resolvers/review-army'; import { generateCrossReviewDedup, generateSharedCodeReuse, generateScopeDrift } from '../scripts/resolvers/review-scope'; import { generatePlanCompletionAuditReview, generatePlanCompletionAuditShip } from '../scripts/resolvers/plan-gates'; -import { generateQAExploratory, generateQAReview } from '../scripts/resolvers/qa'; +import { generateQAReview } from '../scripts/resolvers/qa'; import { generateConfidenceCalibration } from '../scripts/resolvers/confidence'; import { HOST_PATHS, type TemplateContext } from '../scripts/resolvers/types'; @@ -15,36 +15,27 @@ const adversarial = readFileSync(join(root, 'review/sections/adversarial.md.tmpl test('review audits deliverables before deferring behavioral plan checks to the QA preflight', () => { const ctx: TemplateContext = { skillName: 'review', tmplPath: '', host: 'claude', paths: HOST_PATHS.claude }; const audit = generatePlanCompletionAuditReview(ctx).replace(/\s+/g, ' '); - for (const contract of [ - 'Separate static audit evidence from behavioral checks', - 'retain the exact command, expected outcome and source for Step 4.7', - 'They remain pending execution, never DONE from a diff', - 'A mixed item contributes to both lists', - 'Zero audited deliverables do not waive these checks', - 'Keep external-state and human-only checks under the existing audit rules', - 'If only behavioral checks remain, report zero audited deliverables and retain their pending Step 4.7 list', - 'For each audited deliverable, run the verification dispatch', - 'File-existence checks and verified read-only content validators are static audit checks, not behavioral probes', - 'Inspect the validator and its hooks before running it; verify read-only effects and access to the target', - 'leave the item UNVERIFIABLE and defer the command to Step 4.7', - 'Do not start applications, exercise APIs or mutate state during this audit', - 'If found and verified safe above, invoke it', - ]) expect(audit).toContain(contract); - expect(audit.indexOf('Inspect the validator and its hooks')).toBeLessThan(audit.indexOf('If found and verified safe above, invoke it')); + for (const rule of [ + /separate static audit evidence from behavioral checks/i, + /never DONE from a diff/, + /zero audited deliverables do not waive these checks/i, + /for each audited deliverable/i, + /leave the item UNVERIFIABLE and defer the command to Step 4\.7/, + /do not start applications, exercise APIs or mutate state during this audit/i, + ]) expect(audit).toMatch(rule); + const inspect = audit.search(/inspect the validator and its hooks before running it/i); + expect(inspect).toBeGreaterThan(-1); + expect(inspect).toBeLessThan(audit.search(/if found and verified safe above, invoke it/i)); expect(audit).not.toContain('For each extracted plan item, run the verification dispatch'); - const qa = generateQAReview(ctx); - expect(qa).toContain('Then run required plan checks and revalidation, even after smoke expires'); - expect(qa).toContain('Report clean/completed only when all required checks pass on current inputs'); }); test('review prior-Skip matching includes adversarial and Greptile findings without relaxing eligibility', () => { const dedup = generateCrossReviewDedup({ skillName: 'review', tmplPath: '', host: 'claude', paths: HOST_PATHS.claude }); - expect(dedup).toContain('For every combined finding, including core, specialist, exploratory QA, adversarial and valid actionable Greptile findings, check:'); - expect(dedup).toContain('Suppress only when all conditions hold: the user skipped the same unchanged finding'); - expect(dedup).toContain('same advisory/defect kind'); - expect(dedup).toContain('Never use a skipped advisory to suppress a real defect'); - expect(dedup).toContain('Only suppress `skipped` findings — never `fixed` or `auto-fixed`'); - expect(skill).toContain('Run Step 5.0 severity/prior-skip dedup on all'); + expect(dedup).toMatch(/for every combined finding, including core, specialist, exploratory QA, adversarial and valid actionable Greptile findings/i); + expect(dedup).toMatch(/suppress only when all conditions hold: the user skipped the same unchanged finding/i); + expect(dedup).toMatch(/never use a skipped advisory to suppress a real defect/i); + expect(dedup).toMatch(/only suppress `skipped` findings — never `fixed` or `auto-fixed`/i); + expect(skill).toContain('Step 5.0 severity/prior-skip dedup'); }); test('Review audit and prior-Skip clarifications do not route Ship through Review steps', () => { @@ -74,42 +65,40 @@ test('review collects every source before its single parent fix phase', () => { expect(positions.every(position => position >= 0)).toBe(true); expect(positions).toEqual([...positions].sort((a, b) => a - b)); expect(skill.match(/## Step 5: Fix-First Review/g)).toHaveLength(1); - expect(skill.replace(/\s+/g, ' ')).toContain('Do not edit reviewed source until Step 5'); - expect(skill.replace(/\s+/g, ' ')).toContain('every dispatched reader has returned or is confirmed stopped'); + expect(skill.replace(/\s+/g, ' ')).toMatch(/do not edit reviewed source until Step 5/i); + expect(skill.replace(/\s+/g, ' ')).toMatch(/every dispatched reader has returned or is confirmed stopped/i); }); test('review settles adversarial attempts before fixing and has one full-pass back edge', () => { const generated = readFileSync(join(root, 'review/sections/adversarial.md'), 'utf8'); const decision = skill.slice(skill.indexOf('## Step 5.8: Persist Eng Review result')).replace(/\s+/g, ' '); expect(generated).toContain('## Step 4.8: Adversarial review'); - expect(generated).toContain("queued for the parent's Fix-First handling at Step 5; do not edit during Step 4.8"); - expect(generated.replace(/\s+/g, ' ')).toContain('before the parent applies queued fixes'); - expect(generated).toContain('Return all findings and structured-review decisions to Step 5'); - expect(decision).toContain('A pass covers Steps 3–5, including all reviewers before fixes'); + expect(generated).toMatch(/do not edit during Step 4\.8/i); + expect(generated).toMatch(/return all findings and structured-review decisions to Step 5/i); + expect(decision).toMatch(/a pass covers Steps 3–5/i); expect(decision).toContain('Below 3, repeat Steps 3–5 with a new REVIEW_START'); expect(decision).not.toContain('Route Step'); expect(decision).not.toContain('Steps 5.0–5d'); - expect(decision).toContain('without a clean summary or a fourth pass'); + expect(decision).toMatch(/without a clean summary or a fourth pass/i); }); test('review small-diff and failed-reader paths retain QA and the required adversarial pass', () => { const army = generateReviewArmy({ skillName: 'review', tmplPath: 'review/SKILL.md.tmpl', host: 'claude', paths: HOST_PATHS.claude }); - expect(army).toContain("Continue to Step 4.6 with the core findings and an empty specialist list, then the parent's Exploratory QA step and Step 4.8 (adversarial review), then Step 5"); + expect(army.replace(/\s+/g, ' ')).toMatch(/Continue to Step 4\.6 [^.]*Exploratory QA step and Step 4\.8 \(adversarial review\), then Step 5/i); expect(army).not.toContain('design-lite'); - expect(army).toContain('Missing dispatched coverage remains incomplete, never completed or clean'); - expect(army).toContain('Continue independent Step 4.7 QA and Step 4.8 adversarial review'); + expect(army.replace(/\s+/g, ' ')).toMatch(/missing dispatched coverage remains incomplete, never completed or clean/i); + expect(army.replace(/\s+/g, ' ')).toMatch(/independent Step 4\.7 QA and Step 4\.8 adversarial review/i); expect(army).not.toContain("Exploratory QA step, then continue to Step 5."); expect(army).not.toContain('If the Red Team subagent fails or times out, skip silently'); }); test('review defines QA confidence, severity, impact selection and numeric version comparison', () => { const flat = skill.replace(/\s+/g, ' '); - expect(flat).toContain('For QA findings, assign confidence (1–10) from replay/code evidence'); - expect(flat).toContain("retain Step 4.7's severity, not a severity inferred from confidence"); - expect(flat).toContain('A probe is affected when its entrypoint, dependencies, contract or replay inputs change'); - expect(flat).toContain('If impact is uncertain, rerun it'); - expect(flat).toContain('Compare dotted version components as integers from left to right'); + expect(flat).toContain('confidence (1–10)'); + expect(flat).toMatch(/retain Step 4\.7's severity/i); + expect(flat).toMatch(/if impact is uncertain, rerun it/i); + expect(flat).toMatch(/compare dotted version components as integers/i); }); test('review emits scope check after the plan audit and before the checklist', () => { @@ -121,36 +110,30 @@ test('review emits scope check after the plan audit and before the checklist', ( expect(positions.every(position => position >= 0)).toBe(true); expect(positions).toEqual([...positions].sort((a, b) => a - b)); const audit = readFileSync(join(root, 'review/sections/plan-completion.md'), 'utf8').replace(/\s+/g, ' '); - expect(audit).toContain('When continuing after the audit (no HIGH-impact gate, or option B/C), emit the single final Scope Check'); - expect(audit).toContain("Step 1.5's provisional notes and this plan context"); - expect(audit).toContain('Emit Step 1.5\'s Scope Check once without plan fields'); + expect(audit).toMatch(/emit the single final Scope Check/i); expect(skill).not.toContain('Finish Step 1.5 here'); }); test('review composes confidence-tagged findings into one final report with explicit incomplete coverage', () => { const flat = skill.replace(/\s+/g, ' '); - expect(flat).toContain('Use CRITICAL/INFORMATIONAL labels in the finding format'); - expect(flat).toContain("Step 5.8 combines these finding lines with the checklist's action groups"); + expect(flat).toContain('CRITICAL/INFORMATIONAL'); const report = flat.slice(flat.indexOf('### Report the final review'), flat.indexOf('{{LEARNINGS_LOG}}')); - expect(report).toContain('Emit one final report, merging all reviewers rather than concatenating their reports'); - expect(report).toContain('counts final unresolved non-advisory defects'); - expect(report).toContain('State INCOMPLETE if `COMPLETED` is false, even when N=0'); - expect(report).toContain("Use the checklist's action groups with confidence-tagged finding lines"); - expect(report).toContain('Keep fixed, skipped and advisory items separate from unresolved defects; retain their dispositions'); - expect(report).toContain("Append Step 4.7's single `## Exploratory QA and Verification Results` section"); - expect(report).toContain('Neither coverage gaps nor advice are defects'); + expect(report).toMatch(/one final report/i); + expect(report).toMatch(/unresolved non-advisory defects/i); + expect(report).toMatch(/[Ss]tate INCOMPLETE if `COMPLETED` is false/); + expect(report).toContain('`## Exploratory QA and Verification Results`'); + expect(report).toMatch(/neither coverage gaps nor advice are defects/i); }); test('small-diff persistence uses an empty specialist map without manufacturing skipped coverage', () => { const army = generateReviewArmy({ skillName: 'review', tmplPath: 'review/SKILL.md.tmpl', host: 'claude', paths: HOST_PATHS.claude }); - expect(army).toContain('For DIFF_LINES < 50, keep `specialists: {}`; do not manufacture per-specialist scope records'); - expect(army).toContain('Otherwise record each considered specialist'); - expect(skill).toContain("Use Step 4.6's `specialists` object unchanged, including its empty small-diff map"); + expect(army).toContain('DIFF_LINES < 50, keep `specialists: {}`'); + expect(skill.replace(/\s+/g, ' ')).toMatch(/Step 4\.6's `specialists` object unchanged/i); const ship = readFileSync(join(root, 'ship/sections/review-army.md.tmpl'), 'utf8'); expect(ship).toContain('`specialists`: `{}` for a small-diff skip'); - expect(skill).toContain('every required Step 4.7 probe passes'); - expect(ship).toContain('all required probes pass'); + expect(skill.replace(/\s+/g, ' ')).toMatch(/every required Step 4\.7 probe passes/i); + expect(ship.replace(/\s+/g, ' ')).toMatch(/all required probes pass/i); }); test('caller QA runs charter and setup after resource loading and has a severity for unmatched functional failures', () => { @@ -165,13 +148,13 @@ test('caller QA runs charter and setup after resource loading and has a severity const readiness = body.indexOf('**2. Check readiness and list required checks.**'); expect(readiness).toBeGreaterThan(preparation); expect(probes).toBeGreaterThan(readiness); - expect(body.slice(preparation, readiness).replace(/\s+/g, ' ')).toContain('complete the shared isolation/permission preflight before setup'); + expect(body.slice(preparation, readiness).replace(/\s+/g, ' ')).toMatch(/isolation\/permission preflight before setup/i); } else expect(preparation).toBeGreaterThan(body.indexOf('**2. List required checks.**')); expect(probes).toBeGreaterThan(preparation); - expect(body.replace(/\s+/g, ' ')).toContain('unmatched functional failures are `functional-contract`, `CRITICAL`'); - expect(body).toContain('Setup/permission blockers are not defects'); - expect(body).toContain('Test creation needs user approval'); - expect(body).toContain('Return verified defects to Fix-First'); + expect(body.replace(/\s+/g, ' ')).toContain('`functional-contract`, `CRITICAL`'); + expect(body).toMatch(/setup\/permission blockers are not defects/i); + expect(body).toMatch(/test creation needs user approval/i); + expect(body).toMatch(/return verified defects to Fix-First/i); expect(body).not.toContain('for parent approval'); } }); @@ -185,34 +168,24 @@ test('review prepares context and deduplicates before classifying findings', () ].map(marker => skill.indexOf(marker)); expect(positions.every(position => position >= 0)).toBe(true); expect(positions).toEqual([...positions].sort((a, b) => a - b)); - expect(skill).toContain('findings before Step 5a classification'); + expect(skill).toMatch(/before Step 5a classification/i); }); test('review owns the complete persistence contract after the adversarial read', () => { const step = skill.slice(skill.indexOf('## Step 5.8: Persist Eng Review result')); expect(skill.indexOf('{{SECTION:adversarial}}')).toBeLessThan(skill.indexOf('## Step 5.8: Persist Eng Review result')); expect(adversarial).not.toContain('### Before persisting Eng Review (Step 5.8)'); - for (const contract of [ - 'repeat Steps 3–5', 'at most 3 fix cycles', 'final zero-edit pass, reconcile', - 'by structural identity', 'advisory/defect kind', - 'original `evidence_paths`/`helper_target`', 'without requiring deleted pre-extraction blocks', - 'Current findings determine recurring defects and unresolved counts', 'earlier fixes do not suppress them', - 'snapshot_covered_paths', - 'raw bytes equal the bound snapshot blobs', 'prior-cycle, supplied or prior-record coverage', - 'REVIEW_START', 'COMPLETED', 'CONVERGED', 'CYCLES', 'Step 4.7', - 'native Step 4.8 adversarial pass', 'means false, as does a failed native review', - 'optional outside', 'their own incomplete records when unavailable', 'named-risk', - 'zero counts', '`completed:false`', '`specialists`', '`findings`', - 'verified exploratory QA findings', - 'approved **and completed**', 'explicit Skip', 'sharedLibsFingerprint', - '`review_binding`', 'validated captured branch', - ]) expect(step.toLowerCase().replace(/\s+/g, ' ')).toContain(contract.toLowerCase()); + for (const token of [ + 'repeat Steps 3–5', 'at most 3 fix cycles', 'original `evidence_paths`/`helper_target`', + 'snapshot_covered_paths', 'REVIEW_START', 'COMPLETED', 'CONVERGED', 'CYCLES', 'Step 4.7', + 'native Step 4.8 adversarial pass', 'named-risk', 'zero counts', '`completed:false`', + '`specialists`', '`findings`', 'explicit Skip', 'sharedLibsFingerprint', '`review_binding`', + ]) expect(step.toLowerCase().replace(/\s+/g, ' ')).toContain(token.toLowerCase()); expect(step.indexOf('### 1. Re-review after edits')) .toBeLessThan(step.indexOf('### 2. Fill the record')); expect(step.indexOf('### 2. Fill the record')) .toBeLessThan(step.indexOf('~/.claude/skills/gstack/bin/gstack-review-log')); - expect(step).toContain('`quality_score` is Step 4.6\'s specialist score'); - expect(step).toContain('unresolved non-advisory core defects still count'); + expect(step).toContain('`quality_score`'); expect(step.indexOf('Pre-Landing Review: N issues (X critical, Y informational)')) .toBeGreaterThan(step.indexOf('~/.claude/skills/gstack/bin/gstack-review-log')); expect(step).toContain('`## Exploratory QA and Verification Results`'); @@ -220,26 +193,20 @@ test('review owns the complete persistence contract after the adversarial read', test('review distinguishes required native coverage from optional outside coverage', () => { const section = readFileSync(join(root, 'review/sections/adversarial.md'), 'utf8'); - expect(section).toContain('Only this optional outside adversarial pass is non-blocking'); + expect(section).toMatch(/only this optional outside adversarial pass is non-blocking/i); expect(section).not.toContain('All errors are non-blocking'); - expect(section).toContain('The native pass is required for Step 5.8 completion'); - expect(skill).toContain('Core findings use the confidence gates below'); - expect(skill).toContain('Step 4.6 applies its specialist gates'); + expect(section).toMatch(/native pass is required for Step 5\.8 completion/i); }); test('review identifies probe selection, report assets and the detected diff base', () => { const generated = generateQAReview({ skillName: 'review', tmplPath: 'review/SKILL.md.tmpl', host: 'claude', paths: HOST_PATHS.claude }); const checklist = readFileSync(join(root, 'review/checklist.md'), 'utf8'); - expect(generated).toContain('Smoke: 5 minutes/12 probes, one success and the riskiest changed failure/edge'); - expect(generated).toContain('Required even for small diffs or missing plans/servers'); - expect(generated.replace(/\s+/g, ' ')).toContain("Use checklist severity; unmatched functional failures are `functional-contract`, `CRITICAL`"); - expect(generated).toContain('Setup/permission blockers are not defects'); - expect(generated).toContain('Test creation needs user approval'); - expect(generated).toContain("Read QA\'s `templates/functional-report-template.md`. Title it"); - expect(generated).toContain('Link every checkpoint'); - expect(generated).toContain('No second report'); - expect(checklist).toContain('merge-base diff from the caller'); + expect(generated).toContain('5 minutes/12 probes'); + expect(generated).toContain("Read QA\'s `templates/functional-report-template.md`"); + expect(generated).toMatch(/link every checkpoint/i); + expect(generated).toMatch(/no second report/i); + expect(checklist).toMatch(/merge-base diff from the caller/i); expect(checklist).not.toContain('git diff origin/main'); }); @@ -247,37 +214,16 @@ test('caller QA defines execution, evidence ownership and report adaptation befo for (const skillName of ['review', 'ship']) { const generated = generateQAReview({ skillName, tmplPath: `${skillName}/SKILL.md.tmpl`, host: 'claude', paths: HOST_PATHS.claude }).replace(/\s+/g, ' '); - for (const contract of [ - 'Only the parent runs report-only discovery', - 'Never overwrite another run', - 'Follow the shared Probe loop for smoke checks and replays until the smoke limit', - 'using the same procedure but no smoke guard; never reset the clock', - 'Read agent/user updates and await results without batching them with reporting/logging', - 'Compare each probe\'s recorded source, tests, contracts, commands and fixtures (or input fingerprint) with current inputs, even without updates', - 'Re-review changed or uncertain coverage', - "Use checklist severity", - ]) expect(generated).toContain(contract); - const shared = generateQAExploratory({ skillName: 'qa', tmplPath: '', host: 'claude', paths: HOST_PATHS.claude }); - for (const contract of ['First demonstrate success: output AND durable effects', - 'Wait for successful checkpoint publication before dispatch', - 'Replay the exact failing command/request from the same initial fixture state']) { - expect(shared).toContain(contract); - } + expect(generated).toMatch(/never overwrite another run/i); + expect(generated).toMatch(/with current inputs, even without updates/i); + expect(generated).toContain('Use checklist severity'); if (skillName === 'review') { - expect(generated).toContain('Title it `## Exploratory QA and Verification Results`'); - expect(generated).toContain('keep metadata/outcome tables'); - expect(generated).toContain('demote other headings one level'); - expect(generated).toContain('include it here under `### Browser results`'); - expect(generated).toContain('other headings demoted two levels'); - expect(generated).toContain('Keep browser/functional scores and outcomes separate'); - expect(generated).toContain('save browser baseline/evidence normally'); - expect(generated).toContain('Prepare one provisional QA section'); - expect(generated).toContain('Update affected outcomes/checkpoint links through repairs/revalidation'); - expect(generated).toContain('Continue to Step 4.8 even if blocked'); - expect(generated).toContain('Step 5.8 appends this section once after final findings and decides completion'); + expect(generated).toContain('`## Exploratory QA and Verification Results`'); + expect(generated).toContain('`### Browser results`'); + expect(generated).toMatch(/continue to Step 4\.8 even if blocked/i); + expect(generated).toMatch(/Step 5\.8 appends this section once/i); } else { - expect(generated).toContain('PR section `## Exploratory QA`'); - expect(generated).toContain('fields as subsections'); + expect(generated).toContain('`## Exploratory QA`'); } } }); @@ -291,15 +237,14 @@ test('review section index follows the actual pre-fix execution order', () => { test('review finalization ownership: initialize invocation state and capture the core token before reading', () => { const start = skill.slice(skill.indexOf('## Step 3: Get the diff'), skill.indexOf('## Step 3.4:')); - expect(start).toContain('one invocation action list and CYCLES=0'); - expect(start).toContain('Keep both through re-reviews'); + expect(start).toContain('CYCLES=0'); expect(skill.match(/CYCLES=0/g)).toHaveLength(1); expect(start).toContain('gstack-review-log --start review\ngit diff "$DIFF_BASE"'); - expect(start).toContain('Save the printed REVIEW_START for this core candidate before reading its diff'); - expect(start).toContain('Each re-review captures a new token before reading, never at log time'); - expect(start.replace(/\s+/g, ' ')).toContain('Earlier core tokens remain unused'); - expect(start).toContain('Native/outside reviewer attempts own separate PASS_START tokens, not REVIEW_START'); - expect(start).toContain('Step 5.8 finishes only the final core token'); + const flat = start.replace(/\s+/g, ' '); + expect(flat).toMatch(/REVIEW_START for this core candidate before reading its diff/i); + expect(flat).toMatch(/new token before reading, never at log time/i); + expect(flat).toMatch(/separate PASS_START tokens, not REVIEW_START/); + expect(flat).toMatch(/finishes only the final core token/i); }); test('review finalization ownership: late findings use Fix-First before the bounded parent transition', () => { @@ -317,46 +262,31 @@ test('review finalization ownership: late findings use Fix-First before the boun const positions = markers.map(marker => flat.indexOf(marker)); expect(positions.every(position => position >= 0)).toBe(true); expect(positions).toEqual([...positions].sort((a, b) => a - b)); - expect(flat).toContain('Structured approval does not waive advisory/test_stub ASK gates'); - expect(flat).toContain('A pass covers Steps 3–5, including all reviewers before fixes'); expect(flat).toContain('at most 3 fix cycles'); - expect(flat).toContain('Below 3, repeat Steps 3–5 with a new REVIEW_START'); - expect(flat).toContain('At 3, persist `converged:false` and remaining findings'); + expect(flat).toContain('At 3, persist `converged:false`'); const limit = flat.slice(flat.indexOf('At 3,'), flat.indexOf('No edits:')); - expect(limit).toContain('by filling and saving the record below'); - expect(limit).toContain('Report nonconvergence and coverage gaps, then STOP this invocation'); - expect(limit).toContain('without a clean summary or a fourth pass'); + expect(limit).toMatch(/then stop this invocation/i); + expect(limit).toMatch(/without a clean summary or a fourth pass/i); }); test('review finalization ownership: affected QA reuse cannot replace a full review or erase decisions', () => { const step = skill.slice(skill.indexOf('## Step 5.8: Persist Eng Review result')); const flat = step.replace(/\s+/g, ' '); - expect(flat).toContain('On a repeat, execute Steps 3–5 in order'); - expect(flat).toContain('rerun affected probes after source, test, contract, command or fixture changes'); - expect(flat).toContain("At Step 4.7, reuse only this invocation's unchanged-input QA evidence"); - expect(flat).toContain('Reusing a probe never skips a review step'); - expect(flat).toContain("final zero-edit pass, reconcile this invocation's actions with current findings"); - expect(flat).toContain('original `evidence_paths`/`helper_target`'); - expect(flat).toContain('Current findings determine recurring defects and unresolved counts; earlier fixes do not suppress them'); - expect(flat).toContain('Re-read its final-snapshot supporting source and reconfirm the decision'); - expect(flat).toContain('otherwise report its history without a reusable skip'); - expect(flat).toContain('The logger computes `snapshot_covered_paths` from eligible paths whose raw bytes equal the bound snapshot blobs'); - expect(flat).toContain('Never carry prior-cycle, supplied or prior-record coverage forward or build this proof yourself'); - expect(flat).toContain('Fixed advice needs no skip coverage'); - expect(flat).toContain('include this invocation\'s revalidated decisions'); + expect(flat).toContain('execute Steps 3–5 in order'); + expect(flat).toMatch(/reuse only this invocation's unchanged-input QA evidence/i); + expect(flat).toMatch(/reusing a probe never skips a review step/i); + expect(flat).toMatch(/earlier fixes do not suppress them/i); + expect(flat).toContain('logger computes `snapshot_covered_paths`'); + expect(flat).toMatch(/never carry prior-cycle, supplied or prior-record coverage forward or build this proof yourself/i); }); test('review finalization ownership: required native completion and optional outside records stay separate', () => { const step = skill.slice(skill.indexOf('### 2. Fill the record')); const flat = step.replace(/\s+/g, ' '); - expect(flat).toContain('native Step 4.8 adversarial pass finish'); - expect(flat).toContain('every required Step 4.7 probe passes'); - expect(flat).toContain('Any failed, blocked, inconclusive or not-run required probe means false, as does a failed native review'); - expect(flat).toContain('`/ship` named-risk acceptance cannot complete `/review`'); - expect(flat).toContain('The required in-host adversarial result controls native completion'); - expect(flat).toContain('Optional outside attempts keep their own incomplete records when unavailable'); - expect(flat).toContain('cannot substitute for the native result, or vice versa'); - expect(flat).toContain('structured-review gate still applies'); + expect(flat).toMatch(/any failed, blocked, inconclusive or not-run required probe means false, as does a failed native review/i); + expect(flat).toMatch(/`\/ship` named-risk acceptance cannot complete `\/review`/i); + expect(flat).toMatch(/cannot substitute for the native result, or vice versa/i); + expect(flat).toMatch(/structured-review gate still applies/i); expect(flat).toContain('zero counts and `completed:false`'); expect(flat).toContain('`CONVERGED`: true only for a completed zero-edit pass'); }); @@ -367,35 +297,33 @@ test('review finalization ownership: finish only the final core token without lo expect(step.match(/--finish REVIEW_START/g)).toHaveLength(1); expect(step).not.toContain('--start review'); expect(step).not.toContain('--finish PASS_START'); - expect(flat).toContain('Never invent a binding or replace REVIEW_START at log time'); - expect(flat).toContain('finish only the final core token'); + expect(flat).toMatch(/never invent a binding or replace REVIEW_START at log time/i); + expect(flat).toMatch(/finish only the final core token/i); expect(step).toContain('"completed":COMPLETED,"converged":CONVERGED,"cycles":CYCLES'); }); test('review finalization ownership: the plan audit retains its high-impact gate before the final scope check', () => { const plan = readFileSync(join(root, 'review/sections/plan-completion.md.tmpl'), 'utf8'); - expect(plan).toContain('INFORMATIONAL except for the HIGH-impact discrepancy question below'); - expect(plan).toContain('resolve that gate before the final Scope Check'); + expect(plan).toContain('HIGH-impact discrepancy question'); + expect(plan).toMatch(/resolve that gate before the final Scope Check/i); expect(plan).not.toContain('never blocks the review'); expect(plan).toContain('{{PLAN_COMPLETION_AUDIT_REVIEW}}'); const audit = readFileSync(join(root, 'review/sections/plan-completion.md'), 'utf8'); const gate = audit.indexOf('**HIGH-impact plan-file discrepancies** trigger AskUserQuestion'); expect(gate).toBeGreaterThan(-1); expect(gate).toBeLessThan(audit.indexOf('When continuing after the audit (no HIGH-impact gate, or option B/C)')); - expect(audit).toContain('then it gates via AskUserQuestion'); - expect(audit).toContain('A ends this invocation before code review or implementation'); - expect(audit).toContain('after implementation, start a fresh /review'); - expect(audit).toContain('B queues the approved TODO changes for Step 5, not this read-only audit'); - expect(audit).toContain('B/C continue to the final Scope Check and Step 2'); - expect(audit).toContain('None of these choices authorizes shipping or waives required verification'); - expect(skill).toContain("including Step 1.5's approved TODO changes"); + const flatAudit = audit.replace(/\s+/g, ' '); + expect(flatAudit).toMatch(/A ends this invocation before code review or implementation/i); + expect(flatAudit).toMatch(/not this read-only audit/i); + expect(flatAudit).toContain('B/C continue to the final Scope Check and Step 2'); + expect(flatAudit).toMatch(/none of these choices authorizes shipping or waives required verification/i); }); test('review scope notes remain provisional until the plan section emits the only final scope check', () => { const scope = generateScopeDrift({ skillName: 'review', tmplPath: 'review/SKILL.md.tmpl', host: 'claude', paths: HOST_PATHS.claude }).replace(/\s+/g, ' '); - expect(scope).toContain('Keep these notes provisional. Next, execute the plan-completion section'); - expect(scope).toContain('it resolves the HIGH-impact decision and emits the single final Scope Check before Step 2'); + expect(scope).toMatch(/keep these notes provisional/i); + expect(scope).toMatch(/single final Scope Check before Step 2/i); expect(scope).not.toContain('Scope Check: [CLEAN'); expect(scope).not.toContain('available plan-audit results'); }); @@ -404,29 +332,20 @@ test('review confidence uses its severity labels without an undefined P0 excepti const ctx: TemplateContext = { skillName: 'review', tmplPath: 'review/SKILL.md.tmpl', host: 'claude', paths: HOST_PATHS.claude }; const confidence = generateConfidenceCalibration(ctx); - expect(confidence).toContain('Only report a suspected release-blocking catastrophe'); - expect(confidence).toContain('widespread data loss, total outage or system-wide compromise'); - expect(confidence).toContain('label it CRITICAL and explicitly speculative'); expect(confidence).toContain('[CRITICAL]'); expect(confidence).toContain('[CRITICAL|INFORMATIONAL]'); expect(confidence).not.toMatch(/\bP[012]\b/); - expect(confidence).toContain('If you cannot quote the motivating line(s), the finding is unverified'); const flat = confidence.replace(/\s+/g, ' '); - for (const rule of [ - '| 9-10 | Specific code verifies a concrete bug or exploit. | Show normally |', - '| 7-8 | High-confidence pattern match; very likely correct. | Show normally |', - '| 5-6 | Moderate; could be a false positive. | Show with caveat:', - 'Medium confidence, verify this is actually an issue', - '| 3-4 | Suspicious but may be fine. | Suppress from main report. Include in appendix only. |', - '| 1-2 | Speculation. | Only report a suspected release-blocking catastrophe', - 'Quote the specific code line', 'file:line and verbatim text', - 'For a missing field, quote its class definition; for a nullable value, its initialization; for a race, both sides', - 'Force its confidence to 4-5: use 4 for appendix-only reporting, or 5 only when the finding belongs in the main report with the medium-confidence caveat below', - 'Never invent speculative confidence 7+', - 'read and quote their generating metaclass, descriptor, ORM Meta block, migration, decorator or schema', - 'Missing literal names in the class body or grep results do not prove absence', - 'If the user confirms a reported finding scored < 7 is real, log the corrected pattern as a learning', - ]) expect(flat).toContain(rule); + for (const row of [ + /\| 9-10 \|[^|]*\| Show normally \|/, /\| 7-8 \|[^|]*\| Show normally \|/, + /\| 5-6 \|[^|]*\| Show with caveat/, /\| 3-4 \|[^|]*\| Suppress from main report/, + /\| 1-2 \|[^|]*\| Only report a suspected release-blocking catastrophe/, + ]) expect(flat).toMatch(row); + expect(flat).toMatch(/explicitly speculative/i); + expect(flat).toMatch(/the finding is unverified/i); + expect(flat).toContain('file:line'); + expect(flat).toMatch(/force its confidence to 4-5/i); + expect(flat).toMatch(/never invent speculative confidence 7\+/i); expect(confidence.indexOf('Pre-emit verification gate')).toBeLessThan(confidence.indexOf('| Score |')); expect(confidence).not.toContain('FP classes the gate kills'); expect(confidence).not.toContain('1539-framework-aware-review.md'); @@ -435,28 +354,20 @@ test('review confidence uses its severity labels without an undefined P0 excepti test('review names the lifecycle and record owners before using their persistence rules', () => { const start = skill.slice(skill.indexOf('## Step 3: Get the diff'), skill.indexOf('## Step 3.4:')).replace(/\s+/g, ' '); - expect(start).toContain('An invocation is this /review run; a pass reviews one candidate before any fixes'); - expect(start).toContain('REVIEW_START / PASS_START | Opaque start receipts from the logger'); - expect(start).toContain('a matching key alone never proves a prior Skip is reusable'); - expect(start).toContain("`review_binding` | The logger's proof tying a finished review to its captured candidate"); - expect(start).toContain('`snapshot_covered_paths` | Supporting advice files the logger proved byte-identical to that candidate'); - expect(start).toContain('Used by the prior-Skip checker, never supplied by the reviewer'); + expect(start).toContain('REVIEW_START / PASS_START |'); + expect(start).toContain('`review_binding` |'); + expect(start).toContain('`snapshot_covered_paths` |'); + expect(start).toMatch(/a matching key alone never proves a prior Skip is reusable/i); + expect(start).toMatch(/never supplied by the reviewer/i); }); test('review invocation-local advice reuse keeps raw-source and changed-decision gates', () => { const decisions = skill.slice(skill.indexOf('**Keep decisions through fix cycles:**'), skill.indexOf('### Step 5a:')).replace(/\s+/g, ' '); - for (const contract of [ - 'Immediately save completed AUTO-FIX/fix and explicit Skip actions', - 'keeping defects separate from advice', - "retain the helper's fingerprint, `advisory`, `evidence_paths` and `helper_target`", - 're-read every supporting caller and helper destination', - 'including secondary callers and transformed/indirect paths', - 'Compare their raw source with the decision evidence', - 'Unrelated auto-fixes do not reopen unchanged identity, contract and tradeoffs', - 'Material proposal, behavior, migration or risk changes require a new question', - "cannot suppress new/recurring defects or replace Step 5.0's prior-review checker", - ]) expect(decisions).toContain(contract); + expect(decisions).toMatch(/save completed AUTO-FIX\/fix and explicit Skip actions/i); + expect(decisions).toContain("`advisory`, `evidence_paths` and `helper_target`"); + expect(decisions).toMatch(/require a new question/i); + expect(decisions).toMatch(/cannot suppress new\/recurring defects/i); }); for (const skillName of ['review', 'ship']) { @@ -466,16 +377,14 @@ for (const skillName of ['review', 'ship']) { const flat = army.replace(/\s+/g, ' '); test(`${skillName} clarity: terminal failure permits independent work but never certifies coverage`, () => { - expect(flat).toContain('Confirm that each task has finished or is stopped'); - expect(flat).toContain('A timeout alone does not prove termination'); - expect(flat).toContain("If a reader or writer is still active, wait; if its state is unknown, inspect its task/process status"); - expect(flat).toContain("If you cannot confirm it stopped, use the parent's Fix-First stop path without edits"); - expect(flat).toContain('Continue independent evidence collection after a terminal failure'); - expect(flat).toContain('Missing dispatched coverage remains incomplete, never completed or clean'); + expect(flat).toMatch(/confirm that each task has finished or is stopped/i); + expect(flat).toMatch(/a timeout alone does not prove termination/i); + expect(flat).toMatch(/stop path without edits/i); + expect(flat).toMatch(/missing dispatched coverage remains incomplete, never completed or clean/i); expect(flat).not.toContain('Specialists are additive — partial results are better than no results'); const redTeam = flat.slice(flat.indexOf('### Red Team dispatch')); - expect(redTeam).toContain('confirm it stopped and record its review as incomplete, just as for other specialists'); - expect(redTeam).toContain('original specialist outputs and rerun stages 1–7'); + expect(redTeam).toMatch(/confirm it stopped and record its review as incomplete/i); + expect(redTeam).toContain('stages 1–7'); }); test(`${skillName} clarity: ordered specialist merge separates validation from scoring and provenance`, () => { @@ -487,26 +396,21 @@ for (const skillName of ['review', 'ship']) { expect(positions.every(position => position >= 0)).toBe(true); expect(positions).toEqual([...positions].sort((a, b) => a - b)); const validation = flat.slice(flat.indexOf('#### 2.'), flat.indexOf('#### 3.')); - expect(validation).toContain('core and specialist findings'); expect(validation).toContain('remove `advisory` and retain its `CRITICAL` severity'); - expect(validation).toContain('Never downgrade severity'); - expect(validation).toContain('Valid INFORMATIONAL advisories remain advisory'); + expect(validation).toMatch(/never downgrade severity/i); const merge = flat.slice(flat.indexOf('#### 3.'), flat.indexOf('#### 4.')); expect(merge.indexOf('Partition defects and advisories')).toBeLessThan(merge.indexOf('grouping by fingerprint')); - for (const gate of ['sharedLibsFingerprint', 'literal JSON on stdin', 'never trust a supplied hash', - 'Missing/malformed metadata cannot deduplicate', 'highest confidence', '+1 (cap at 10)', - 'distinct specialists', 'all source names', 'Core findings never earn a specialist confidence boost']) { - expect(merge).toContain(gate); - } + for (const token of ['sharedLibsFingerprint', 'literal JSON on stdin', '+1 (cap at 10)']) expect(merge).toContain(token); + for (const rule of [/never trust a supplied hash/i, /missing\/malformed metadata cannot deduplicate/i, + /core findings never earn a specialist confidence boost/i]) expect(merge).toMatch(rule); for (const gate of ['Confidence 7+', 'Confidence 5-6', 'Confidence 3-4', 'Confidence 1-2']) expect(army).toContain(gate); const scoring = flat.slice(flat.indexOf('#### 5.'), flat.indexOf('#### 6.')); - expect(scoring).toContain('Only specialist findings enter this header and `quality_score`; core findings do not'); - expect(scoring).toContain('NON-advisory'); + expect(scoring).toMatch(/only specialist findings enter this header and `quality_score`/i); + expect(scoring).toMatch(/non-advisory/i); expect(scoring).toContain('quality_score = max(0, 10 - (critical_count * 2 + informational_count * 0.5))'); - expect(scoring).toContain('unresolved-defect totals'); - expect(flat).toContain('Advisory findings COUNT in the stats `findings` field'); - expect(flat).toContain('Count only findings that specialist actually returned'); - expect(flat).toContain('core-only advice must not create a specialist dispatch or finding'); + expect(flat).toMatch(/advisory findings count in the stats `findings` field/i); + expect(flat).toMatch(/count only findings that specialist actually returned/i); + expect(flat).toMatch(/core-only advice must not create a specialist dispatch or finding/i); expect(flat).toContain('ASK-only'); }); @@ -517,29 +421,20 @@ for (const skillName of ['review', 'ship']) { const positions = markers.map(marker => reuse.indexOf(marker)); expect(positions.every(position => position >= 0)).toBe(true); expect(positions).toEqual([...positions].sort((a, b) => a - b)); - for (const gate of ['first-party authored provenance', 'all supporting callers and the helper destination', - '--check-shared-libs REVIEW_START', "<<'GSTACK_SHARED_LIBS_REUSE_JSON'", 'literal JSON on stdin', - '`reusable: true`', 'False, command failure or unreadable output', 'never suppression', - 'Do not supply your own snapshot, prior record or coverage', 'sharedLibsFingerprint', - 'without consuming/replacing it', 'actual repo, raw branch and current snapshot', - 'completed/converged', 'verified binding', 'explicit Skip', 'logger-versioned `snapshot_covered_paths`', - 'older unversioned coverage', 'Sanitized branch names are not identity', 'canReuseSharedLibsAdvisory', - 'byte-for-byte with its blob', 'assume-unchanged, skip-worktree', 'sparse index', 'symlinks/ancestors', - 'submodules', 'ignored/outside or unreadable', 'active/unknown Git filters', 'encodings and line conversion', - 'fsmonitor and optional locks', 'never uses external diff/textconv', 'Unknown evidence fails closed', - 'logger recomputes final coverage', 'Real defects retain normal Fix-First handling independently']) { - expect(reuse).toContain(gate); - } + for (const token of ['--check-shared-libs REVIEW_START', "<<'GSTACK_SHARED_LIBS_REUSE_JSON'", 'literal JSON on stdin', + '`reusable: true`', 'sharedLibsFingerprint', 'explicit Skip', 'logger-versioned `snapshot_covered_paths`', + 'canReuseSharedLibsAdvisory']) expect(reuse).toContain(token); + for (const rule of [/false, command failure or unreadable output[^.]*never suppression/i, + /do not supply your own snapshot, prior record or coverage/i, /unknown evidence fails closed/i, + /real defects retain normal Fix-First handling/i]) expect(reuse).toMatch(rule); }); } test('review clarity: settlement gates edits separately from incomplete required coverage', () => { const fix = skill.slice(skill.indexOf('## Step 5: Fix-First Review'), skill.indexOf('{{CROSS_REVIEW_DEDUP}}')).replace(/\s+/g, ' '); - expect(fix).toContain('every dispatched reader has returned or is confirmed stopped'); - expect(fix).toContain('active or unknown reader/writer'); - expect(fix).toContain('persist incomplete at Step 5.8 and STOP without edits'); - expect(fix).toContain('Terminal failure does not block fixes from independent evidence'); - expect(fix).toContain('Missing required output still makes the pass incomplete'); + expect(fix).toMatch(/every dispatched reader has returned or is confirmed stopped/i); + expect(fix).toMatch(/persist incomplete at Step 5\.8 and stop without edits/i); + expect(fix).toMatch(/missing required output still makes the pass incomplete/i); }); test('review clarity: Greptile reply choices never substitute for Fix-First approval', () => { @@ -547,43 +442,38 @@ test('review clarity: Greptile reply choices never substitute for Fix-First appr expect(fix).toContain('VALID & ACTIONABLE Greptile findings'); const greptile = skill.slice(skill.indexOf('### Greptile comment resolution'), skill.indexOf('## Step 5.8:')); const flat = greptile.replace(/\s+/g, ' '); - expect(flat).toContain('Step 5c alone supplies A) Fix / B) Skip'); + expect(flat).toMatch(/Step 5c alone supplies A\) Fix \/ B\) Skip/i); expect(flat).not.toContain('A: Fix it now, B: Acknowledge, C: False positive'); - expect(flat).toContain('reply decisions, not code approval'); + expect(flat).toMatch(/reply decisions, not code approval/i); expect(flat).toContain('B) Propose a code change'); - expect(flat).toContain('return to Steps 5c–5d with an ASK proposal'); - expect(flat).toContain('Show the exact change and any `test_stub`; wait for approval before editing'); - expect(flat).toContain('no new fix permission'); + expect(flat).toContain('Steps 5c–5d'); + expect(flat).toMatch(/wait for approval before editing/i); + expect(flat).toMatch(/no new fix permission/i); }); test('ship review clarity: parent settlement gate precedes classification and cannot waive coverage', () => { const ship = readFileSync(join(root, 'ship/sections/review-army.md.tmpl'), 'utf8'); const gate = ship.slice(ship.indexOf('## Step 9.4:'), ship.indexOf('1. **Classify')).replace(/\s+/g, ' '); - expect(gate).toContain("Before edits, inspect every dispatched reader/writer's handle"); - expect(gate).toContain('Wait for return or confirm termination'); - expect(gate).toContain('otherwise log incomplete through items 5–6 and STOP without edits'); - expect(gate).toContain('After terminal failure, independent evidence may support fixes'); - expect(gate).toContain('missing dispatched output still blocks continuation, even with a QA exception'); + expect(gate).toMatch(/wait for return or confirm termination/i); + expect(gate).toMatch(/log incomplete through items 5–6 and stop without edits/i); + expect(gate).toMatch(/missing dispatched output still blocks continuation, even with a QA exception/i); }); -test('review resolves the judged smoke-clock, setup-authority, plan-gate and findings-source ambiguities', () => { +test('review keeps its smoke-clock, setup-authority, plan-gate and findings-source rules', () => { const ctx = { skillName: 'review', tmplPath: 'review/SKILL.md.tmpl', host: 'claude', paths: HOST_PATHS.claude } as TemplateContext; const qa = generateQAReview(ctx).replace(/\s+/g, ' '); - expect(qa).toContain('Follow the shared Probe loop for smoke checks and replays until the smoke limit'); - expect(qa).toContain('Then run required plan checks and revalidation, even after smoke expires, using the same procedure but no smoke guard'); - expect(qa).toContain('/review sets none; only an invoker-supplied EARLIER_UTC counts'); - expect(qa).toContain('report-only /review never runs setup, installs or cookie import'); - expect(qa).toContain('After a grant, recheck readiness and run affected checks; otherwise they stay blocked'); + expect(qa).toMatch(/\/review sets none; only an invoker-supplied EARLIER_UTC counts/); + expect(qa).toMatch(/report-only \/review never runs setup, installs or cookie import/i); + expect(qa).toMatch(/otherwise they stay blocked/i); expect(qa).not.toContain('Ask for setup/permission'); expect(generateQAReview({ ...ctx, skillName: 'ship', tmplPath: 'ship/SKILL.md.tmpl' })).not.toContain('/review sets none'); - const shared = generateQAExploratory({ ...ctx, skillName: 'qa' }).replace(/\s+/g, ' '); - expect(shared).toContain('Explicit plan checks and revalidation remain required beyond this smoke budget'); const audit = generatePlanCompletionAuditReview(ctx).replace(/\s+/g, ' '); - expect(audit).toContain('No plan file found → say "No plan file detected." and use the Fallback Intent Sources below'); + expect(audit).toContain('"No plan file detected."'); + expect(audit).toContain('Fallback Intent Sources'); expect(audit).not.toContain('skip with "No plan file detected — skipping."'); expect(audit).toContain('**HIGH-impact plan-file discrepancies** trigger AskUserQuestion'); - expect(audit).toContain('Discrepancies derived only from fallback sources (commit messages, TODOS.md, PR description) never trigger this question'); + expect(audit).toMatch(/derived only from fallback sources[\s\S]{0,80}never trigger this question/i); expect(generatePlanCompletionAuditShip({ ...ctx, skillName: 'ship', tmplPath: 'ship/SKILL.md.tmpl' })).toContain('skip with "No plan file detected — skipping."'); const persist = skill.slice(skill.indexOf('### 2. Fill the record')).replace(/\s+/g, ' '); - expect(persist).toContain("Step 5's combined final-pass findings (core, specialist, adversarial, actionable Greptile, verified exploratory QA findings)"); + expect(persist).toMatch(/combined final-pass findings \(core, specialist, adversarial, actionable Greptile, verified exploratory QA findings\)/i); }); From e1495214140c2929524e3dd45e1f725f3d70c6f1 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 01:32:48 +0000 Subject: [PATCH 11/13] test(W7c): ship-control-flow keeps step routing, tokens and gates, drops rule prose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit W7c (t09 F1, F2, F3, F4). One test file. Removed assertion groups (invariant -> remaining protection): - Work-list prose ("Expand a repair into individual steps ...", "This replaces the current item ...", "Add its destination only if ...", "The saved list takes precedence ...", "For another repair, repeat rule 2 ...", "The unchanged release steps follow", "Follow the saved work list", "Keep CYCLES and scoped approvals across this repeat", "Keep the findings and their approvals", "any further repair inserts its checks ...") -> repairs insert ordered steps ahead of pending work: kept free checks on the example sequences (`5 → … → 11.5`, `9 → 10 → 11` before 11.5, `5–11.5 → 12–14 → 16`, `14.5 → 15 → 16`, `9 → 10 → 11 → 11.5`), insert-before- pending routing lines, no unlisted steps, same attempt counts, no reset of the one-repair limit; judge `ship/SKILL.md workflow` (gate lane); gate E2Es ship-docsync-recovery, ship-docsync-stale-after (gate). - Queue-qualification sentence pins duplicated in ship-version-sync (t09 F4: "require successful utility output ...", "`offline:false` qualifies ...", "Offline output without that fallback ...") -> kept in test/ship-version-sync.test.ts next to its script-behavior tests (free); here the Qualify < Usable < No-usable order, `NEW_VERSION=CANDIDATE_VERSION`, `>= NEW_VERSION`, `BUMP_LEVEL`, `currentVersion` and the approval line stay. - Distribution prose ("List candidate distribution paths", untracked files, manifest sentences) -> kept the New-artifact branch label, AskUserQuestion and the no-release-during-/ship safety line; gate E2E ship-local-workflow (gate). - Dispatch/dashboard/coverage prose ("Omitting the flag runs the subagent in the background", "keeping a fresh context", "Use one row for each entry in step 1", "including its undetermined-percentage and test-only rules") -> kept the `run_in_background: false` contract at all three sites, the dashboard header/verdict tokens, no inline Skill fallback, coverage gate not bypassed; gate E2E ship-coverage-audit (gate). - Recovery/docs/test sentences ("If an attempt remains and either ...", "a concrete launch/input/permission correction ...", "restart Step 16 stage 1 to regenerate ...", "This recovery also applies ... stage 5", "Content- preserving commits keep valid evidence", "Retain queued fixes and restore coverage", "If this pass made edits, resume at the next decision ...", "Undispatched host-unsupported/gated specialists do not block", "count it in the invocation record before launch", "Keep Step 9.4's incomplete flags", "saved handle, original token and source", "unknown or missing status is unavailable", "Finish the saved replies ...", "fetch and inspect the remote, then merge ...") -> judge `ship/SKILL.md workflow` (gate lane); gate E2Es ship-docsync-* (gate) and ship-skipped-queued-finding (gate); the adjacent stop/approval lines stay free. Kept as case-insensitive meaning checks (t09 F1/F3; stop/consent/approval and safety lines): STOP and AskUserQuestion gates apply during repairs, a repair never resets or expands approvals, children return evidence not permission, prior-invocation audits never qualify, one repair/re-audit and no third attempt (initial-plus-one, once), Otherwise -> stop unless the user accepts the named docs risk, never silently refresh hashes, never run a third audit, dispatched-reviewer-missing -> stop, CYCLES >= 3 -> stop with `converged:false`, no fourth cycle, missing coverage cannot be replaced, failed probes never clean, required-probe gate before Step 10, cannot waive reviewer/test/security gates, native recovery stops the task first and remains blocked, outside output cannot replace native, never attach new tokens to old work, failed build stops shipping, never edit the child's counts, test failure stops publication, push failure stops with no Step 19, never rewrite history, never bypass failed guards, both redaction scans before publishing, no release publish during /ship, only approval changes the version, declining bootstrap is not test approval. Kept unchanged: the executed plan-tune nudge shell test (3 roots), section/ step ordering checks, Step 11.5 placement and `review_binding.*` fields, branch-label order in the adversarial finish (every host), all stale- wording bans, `(recommended)` marker on Add tests. Cosmetic pins dropped: STOP / ONE capitals (t09 F1, F3). Mutation log: .context/mutation-ship-control-flow.md (21 mutations, 0 unexpected: 3 reorders, 3 markers, 9 safety red; 6 rewordings green). --- test/ship-control-flow.test.ts | 239 ++++++++++++++------------------- 1 file changed, 101 insertions(+), 138 deletions(-) diff --git a/test/ship-control-flow.test.ts b/test/ship-control-flow.test.ts index 5c5350197..129a0ce63 100644 --- a/test/ship-control-flow.test.ts +++ b/test/ship-control-flow.test.ts @@ -22,49 +22,40 @@ const gate = compact(entry.slice(entry.indexOf('## Step 16:'), entry.indexOf('## describe('ship source controller', () => { test('documentation freshness covers selected code paths as well as release metadata', () => { const docs = compact(read('ship/sections/documentation.md.tmpl')); - expect(docs).toContain('hashes of the selected release paths, generated outputs and docs/templates'); + expect(docs).toContain('selected release paths, generated outputs and docs/templates'); const freshness = gate.slice(gate.indexOf('### 3.'), gate.indexOf('### 4.')); expect(freshness).toContain('selected release paths, generated outputs and docs/templates'); - expect(freshness).toContain("A prior invocation's audit or risk decision never qualifies"); - expect(freshness).toContain('the same approved scope and exact content'); + expect(freshness).toMatch(/a prior invocation's audit or risk decision never qualifies/i); }); test('changed documentation inputs permit only the remaining bounded re-audit', () => { const docs = compact(read('ship/sections/documentation.md.tmpl')); - const recovery = docs.slice(docs.indexOf('## Blocked recovery')); - expect(recovery).toContain('If an attempt remains and either the audited inputs changed'); - expect(recovery).toContain('a concrete launch/input/permission correction or reviewed patch repair is available'); - expect(docs).toContain('an initial audit plus ONE repair/re-audit'); - expect(docs).toContain('never a third attempt, even after Step 16 changes'); + expect(docs.indexOf('## Blocked recovery')).toBeGreaterThan(-1); + expect(docs).toMatch(/initial[- ]audit plus one repair\/re-audit/i); + expect(docs).toMatch(/never a third attempt/i); const freshness = gate.slice(gate.indexOf('### 3.'), gate.indexOf('### 4.')); expect(freshness).toContain('**An attempt remains, with changed inputs or an available repair:**'); - expect(freshness).toContain('**Otherwise:** STOP unless the user accepts'); + expect(freshness).toMatch(/\*\*Otherwise:\*\* stop unless the user accepts/i); expect(freshness).not.toContain('**No attempt remains or no repair is available:**'); }); test('repairs use one ordered work list without a return stack', () => { const text = compact(controller); - expect(compact(entry)).toContain('**Next steps:** one ordered work list, with the current step marked'); - expect(text).toContain('Expand a repair into individual steps and insert them before the still-pending work'); - expect(text).toContain('This replaces the current item, whose actual result stays in the record'); - expect(text).toContain('Add its destination only if not already the next pending step'); - expect(text).toContain('The saved list takes precedence over ordinary next-step sentences inside a repair'); + expect(compact(entry)).toContain('**Next steps:**'); expect(text).toContain('5 → 6 → 7 → 8 → 9 → 10 → 11 → 11.5'); expect(text).not.toContain('finish the inner repair, then resume the unfinished outer range'); - expect(text).toContain('Keep the same attempt counts throughout the invocation'); - expect(text).toContain('initial-plus-ONE limit never resets'); + expect(text).toMatch(/initial-plus-one limit never resets/i); }); test('the roadmap and recovery groups define ownership and zero-edit eligibility', () => { const text = compact(entry); expect(text).toContain('integrate (1–3) → test and review (4–11.5) → prepare the release (12–15) → verify frozen content (16) → push and publish (17–21)'); - expect(text).toContain('children return evidence, not permission to proceed'); - expect(review).toContain('**No edits in this pass:** Resolve the required-probe gate below'); - expect(review).toContain('Only after it clears may you continue to Step 10'); - expect(review).toContain('With completed checklist and dispatched reviewers, failed/unavailable required probes block continuation'); - expect(review).toContain('This cannot waive missing reviewer output, recurring fixes or independent test/security gates'); - expect(review).toContain('Undispatched gated/unsupported specialists do not block independently'); - expect(text).toContain('Reuse it only for that same scope; a repair never resets approvals or expands them'); + expect(text).toMatch(/children return evidence, not permission to proceed/i); + expect(review).toContain('**No edits in this pass:**'); + expect(review).toMatch(/only after it clears may you continue to Step 10/i); + expect(review).toMatch(/failed\/unavailable required probes block continuation/i); + expect(review).toMatch(/cannot waive missing reviewer output, recurring fixes or independent test\/security gates/i); + expect(text).toMatch(/a repair never resets approvals or expands them/i); expect(text).not.toContain('eligible zero-edit pass'); expect(text).not.toContain('the same waiver'); expect(text).not.toContain("the controller's detour"); @@ -72,13 +63,9 @@ describe('ship source controller', () => { test('distribution discovery is a shortlist, not a manifest-only artifact decision', () => { const distribution = compact(entry.slice(entry.indexOf('## Step 2:'), entry.indexOf('## Step 3:'))); - expect(distribution).toContain('List candidate distribution paths'); - expect(distribution).toContain("Also inspect matching untracked files from Step 1's status"); - expect(distribution).toContain('a new `package.json` or `Cargo.toml` alone does not establish a publishable artifact'); - expect(distribution).toContain('inspect existing manifests for newly declared binaries or package exports'); - expect(distribution).toContain('New artifact without a pipeline'); + expect(distribution).toContain('**New artifact without a pipeline'); expect(distribution).toContain('AskUserQuestion'); - expect(distribution).toContain('Do not publish a release during `/ship`'); + expect(distribution).toMatch(/do not publish a release during `\/ship`/i); }); test('queue qualification exhaustively distinguishes online, git fallback and unusable output', () => { @@ -89,13 +76,11 @@ describe('ship source controller', () => { expect(qualify).toBeGreaterThan(0); expect(usable).toBeGreaterThan(qualify); expect(missing).toBeGreaterThan(usable); - expect(version).toContain('require successful utility output and a nonempty valid version'); - expect(version).toContain('`offline:false` qualifies; `offline:true` qualifies only with `fallback:"git"`'); - expect(version).toContain('Offline output without that fallback, failure, malformed output or an empty version is unusable, even if it contains a version-looking string'); - expect(version).toContain('FRESH sets `NEW_VERSION=CANDIDATE_VERSION`'); - expect(version).toContain('Only approval changes the existing version'); - expect(version).toContain('a sibling holding `>= NEW_VERSION` requires a choice: advance past it, or stop this attempt and sync'); - expect(version.slice(missing)).toContain('FRESH uses local `BUMP_LEVEL` arithmetic; ALREADY_BUMPED keeps `currentVersion`'); + expect(version).toContain('`NEW_VERSION=CANDIDATE_VERSION`'); + expect(version).toMatch(/only approval changes the existing version/i); + expect(version).toContain('`>= NEW_VERSION`'); + expect(version.slice(missing)).toContain('`BUMP_LEVEL`'); + expect(version.slice(missing)).toContain('`currentVersion`'); }); test.each(['home', 'override', 'plugin'])('the actual nudge shell honors %s roots, repeat suppression and enabled tuning', mode => { @@ -142,11 +127,8 @@ describe('ship source controller', () => { test('nested repairs resume the unfinished outer range before its destination', () => { const text = compact(controller); - expect(text).toContain('Expand a repair into individual steps and insert them before the still-pending work'); - expect(text).toContain('For another repair, repeat rule 2 without discarding pending work'); - expect(text).toContain('Step 11 fixes insert `9 → 10 → 11` before 11.5'); - expect(text).toContain('A further Step 9 fix affecting 6–8 makes the list `5 → 6 → 7 → 8 → 9 → 10 → 11 → 11.5`'); - expect(text).toContain('The unchanged release steps follow'); + expect(text).toContain('`9 → 10 → 11` before 11.5'); + expect(text).toContain('`5 → 6 → 7 → 8 → 9 → 10 → 11 → 11.5`'); expect(text).not.toContain('Enter Step 9 before REVIEW_START capture and full review'); }); @@ -157,10 +139,8 @@ describe('ship source controller', () => { expect(shared).toBeLessThan(coverage.indexOf('Dispatch the audit through Agent')); const contract = compact(coverage.slice(shared, coverage.indexOf('Dispatch the audit through Agent'))); expect(contract).toContain('use the Agent tool with `run_in_background: false`'); - expect(contract).toContain('Omitting the flag runs the subagent in the background'); - expect(contract).toContain('keeping a fresh context'); - expect(contract).toContain('Do not invoke the target as a Skill or run it inline instead'); - expect(contract).toContain("only under that section's documented fallback, after a failed subagent has stopped"); + expect(contract).toMatch(/do not invoke the target as a Skill or run it inline instead/i); + expect(contract).toMatch(/after a failed subagent has stopped/i); for (const section of ['test-coverage', 'plan-completion', 'greptile']) { const text = read(`ship/sections/${section}.md.tmpl`); expect(text).toContain('shared foreground-dispatch rule'); @@ -178,8 +158,7 @@ describe('ship source controller', () => { const ship = generateReviewDashboard(ctx); expect(ship).toContain('REVIEW READINESS DASHBOARD'); expect(ship).toContain('| Review | Runs | Last run | Status | Required |'); - expect(ship).toContain('Use one row for each entry in step 1'); - expect(ship).toContain('Only Eng Review is marked required'); + expect(ship).toMatch(/only Eng Review is marked required/i); expect(ship).toContain('{actual status and reason}'); expect(ship).toContain('VERDICT: {CLEARED or NOT CLEARED} — {reason}'); expect(ship).not.toContain('+====================================================================+'); @@ -190,18 +169,17 @@ describe('ship source controller', () => { test('coverage fallback settles the child and still applies the coverage gate', () => { const text = compact(read('ship/sections/test-coverage.md.tmpl')); const fallback = text.slice(text.indexOf('**Audit failure:**'), text.indexOf('{{TEST_COVERAGE_GATE_SHIP}}')); - expect(fallback).toContain('confirm it stopped before running the same audit inline'); - expect(fallback).toContain('does not pass or bypass the coverage gate'); - expect(fallback).toContain('including its undetermined-percentage and test-only rules'); + expect(fallback).toMatch(/confirm it stopped before running the same audit inline/i); + expect(fallback).toMatch(/does not pass or bypass the coverage gate/i); expect(text).not.toContain('partial results are better than none'); }); test('no-plan gate skips only the audit and retains verification and the remaining section', () => { const text = generatePlanCompletionGateShip({ host: 'claude', skillName: 'ship', tmplPath: '', paths: HOST_PATHS.claude }); const noPlan = compact(text.slice(text.indexOf('**No plan file found:**'))); - expect(noPlan).toContain('Skip only the plan completion audit'); - expect(noPlan).toContain('Continue with Step 8.1, Scope Drift and Prior Learnings'); - expect(noPlan).toContain('Step 9 QA still runs'); + expect(noPlan).toMatch(/skip only the plan completion audit/i); + expect(noPlan).toContain('Step 8.1'); + expect(noPlan).toMatch(/Step 9 QA still runs/i); expect(noPlan).not.toContain('Skip entirely'); }); @@ -211,8 +189,8 @@ describe('ship source controller', () => { const positions = steps.map(step => text.indexOf(step)); expect(positions.every(position => position >= 0)).toBe(true); expect(positions).toEqual([...positions].sort((a,b) => a-b)); - expect(text).toContain('All three snapshots must match'); - expect(text).toContain('does not mean the failed or unrun probes passed'); + expect(text).toMatch(/all three snapshots must match/i); + expect(text).toMatch(/does not mean the failed or unrun probes passed/i); expect(text).not.toContain('equal snapshots do not pass waived probes'); }); @@ -221,9 +199,8 @@ describe('ship source controller', () => { expect(text).toContain('**Behavior, tests or build inputs changed:**'); expect(text).toContain('**Only authored docs or release metadata changed:**'); expect(text).toContain('**No changes, or the docs-only checks still support the plan:**'); - expect(text).toContain('accepted audit matches all inputs | Continue to stage 4'); - expect(text).toContain("Use Blocked recovery with the existing count"); - expect(text).toContain('Use this example only after confirming that every allowed edit is release metadata'); + expect(text).toContain('| Continue to stage 4'); + expect(text).toMatch(/Blocked recovery with the existing count/i); expect(text).not.toContain('Every listed change below is metadata:'); }); @@ -233,26 +210,24 @@ describe('ship source controller', () => { expect(end).toBeLessThan(entry.indexOf('{{BASE_BRANCH_DETECT}}')); expect(entry.match(/### Ship control flow/g)).toHaveLength(1); const text = compact(controller); - expect(text).toContain('You, the **parent** running /ship, own advancement'); - expect(text).toContain('Follow the saved work list'); - expect(text).toContain('STOP and AskUserQuestion gates still apply during repairs'); - expect(text).toContain('The saved list takes precedence over ordinary next-step sentences inside a repair. A range never adds unlisted steps'); - expect(text).toContain('Keep the same attempt counts throughout the invocation'); + expect(text).toMatch(/the \*\*parent\*\* running \/ship, own advancement/i); + expect(text).toMatch(/stop and AskUserQuestion gates still apply during repairs/i); + expect(text).toMatch(/a range never adds unlisted steps/i); + expect(text).toMatch(/keep the same attempt counts throughout the invocation/i); expect(text).toContain('A range ending at Step 14 does not enter Step 14.5'); - expect(text).toContain('its initial-plus-ONE limit never resets'); expect(text).not.toContain('| At step |'); expect(entry.match(/Ship control flow/g)).toHaveLength(1); - expect(review).toContain('Every repeat starts before the checklist read and captures a fresh REVIEW_START'); + expect(review).toMatch(/captures a fresh REVIEW_START/); }); test('distribution and bootstrap detours end at their exact forward resume', () => { const merge = compact(entry.slice(entry.indexOf('## Step 3:'), entry.indexOf('{{SECTION:tests}}'))); - expect(merge).toContain('repeat Step 2 on the merged content, including its decisions, then continue to Step 4'); - expect(merge).toContain('Otherwise continue to Step 4 directly'); + expect(merge).toMatch(/repeat Step 2 on the merged content[^.]*continue to Step 4/i); + expect(merge).toMatch(/otherwise continue to Step 4 directly/i); const tests = compact(read('ship/sections/tests.md.tmpl')); - expect(tests).toContain('A runs Step 4 with this new bootstrap choice, then returns here to run the tests'); + expect(tests).toMatch(/A runs Step 4 with this new bootstrap choice, then returns here/i); expect(tests).toContain('A) Add tests (recommended)'); - expect(tests).toContain('declining bootstrap alone is not that approval'); + expect(tests).toMatch(/declining bootstrap alone is not that approval/i); }); test('missing dispatched output outranks the cycle cap, fixes and zero-edit continuation', () => { @@ -262,49 +237,40 @@ describe('ship source controller', () => { expect(positions.every(position => position >= 0)).toBe(true); expect(positions).toEqual([...positions].sort((a, b) => a - b)); const missing = decisions.slice(positions[0], positions[1]); - expect(missing).toContain('STOP and name each failed specialist or Red Team'); - expect(missing).toContain('Retain queued fixes and restore coverage'); - expect(missing).toContain('If this pass made edits, resume at the next decision; otherwise run a fresh complete Step 9'); - expect(missing).toContain('A successful peer or a QA exception cannot replace missing dispatched coverage'); + expect(missing).toMatch(/^1\. \*\*Dispatched reviewer output missing:\*\* stop\b/i); + expect(missing).toMatch(/cannot replace missing dispatched coverage/i); const cap = decisions.slice(positions[1], positions[2]); - expect(cap).toContain('STOP'); - expect(cap).toContain('`converged:false`; do not run a fourth fixing cycle'); + expect(cap).toMatch(/\bstop\b/i); + expect(cap).toContain('`converged:false`'); + expect(cap).toMatch(/do not run a fourth fixing cycle/i); const fixes = decisions.slice(positions[2], positions[3]); - expect(fixes).toContain('Insert Step 5, affected Steps 6–8 and all of Step 9 before the pending Step 10 in the work list'); - expect(fixes).toContain('Tests must pass or retain approval for the same verified pre-existing failures and scope'); - expect(fixes).toContain('Keep CYCLES and scoped approvals across this repeat'); - expect(decisions.slice(positions[3])).toContain('Only after it clears may you continue to Step 10'); - expect(review).toContain('Undispatched host-unsupported/gated specialists do not block'); - expect(review).toContain('Failed, blocked, inconclusive or not-run required probes mean false, never clean'); - expect(review).toContain('This cannot waive missing reviewer output, recurring fixes or independent test/security gates'); + expect(fixes).toContain('Insert Step 5, affected Steps 6–8 and all of Step 9 before the pending Step 10'); + expect(fixes).toMatch(/retain approval for the same verified pre-existing failures and scope/i); + expect(decisions.slice(positions[3])).toMatch(/only after it clears may you continue to Step 10/i); + expect(review).toMatch(/failed, blocked, inconclusive or not-run required probes mean false, never clean/i); }); test('comment fixes resume triage and preserve settled approvals and replies', () => { const section = compact(read('ship/sections/greptile.md.tmpl')); - expect(section).toContain('If fixes were approved, save their approvals and comment references'); - expect(section).toContain("Run Step 9's full review/fix loop, then return here"); - expect(review).toContain('Finish the complete review and QA before applying any fix in Step 9.4'); - expect(section).toContain('Finish the saved replies without asking again about completed fixes, and classify new comments'); - expect(section).toContain('With no queued fixes, continue to Step 11'); - expect(section).toContain('This optional triage does not block ship'); - expect(section).toContain('unknown or missing status is unavailable'); + expect(section).toMatch(/save their approvals and comment references/i); + expect(section).toMatch(/Run Step 9's full review\/fix loop, then return here/i); + expect(review).toMatch(/finish the complete review and QA before applying any fix in Step 9\.4/i); + expect(section).toMatch(/with no queued fixes, continue to Step 11/i); + expect(section).toMatch(/this optional triage does not block ship/i); expect(section).not.toContain('return to Step 9'); }); test('native recovery has one corrected attempt and cannot borrow outside completion', () => { const finish = adversarial.slice(adversarial.indexOf('### Finish the adversarial phase')); const recovery = finish.slice(finish.indexOf('1. **Required native'), finish.indexOf('2. **Fixes queued')); - expect(recovery).toContain('STOP and confirm the native task stopped'); - expect(recovery).toContain('Outside-provider output cannot replace this pass'); - expect(recovery).toContain('One recovery retry is allowed only after a concrete prerequisite correction and restored access'); - expect(recovery).toContain('count it in the invocation record before launch'); - expect(recovery).toContain('Capture a fresh PASS_START and persist the new attempt separately'); - expect(recovery).toContain('Without that correction, or if the recovery fails, ask for repair and remain blocked'); + expect(recovery).toMatch(/stop and confirm the native task stopped/i); + expect(recovery).toMatch(/outside-provider output cannot replace this pass/i); + expect(recovery).toMatch(/one recovery retry is allowed only after a concrete prerequisite correction/i); + expect(recovery).toContain('PASS_START'); + expect(recovery).toMatch(/ask for repair and remain blocked/i); const queued = finish.slice(finish.indexOf('2. **Fixes queued'), finish.indexOf('3. **Native complete')); - expect(queued).toContain('Keep the findings and their approvals'); - expect(queued).toContain('Insert Steps 9, 10 and 11 before the pending Step 11.5 in the work list. Step 9 completes full review before fixes'); - expect(queued).toContain('any further repair inserts its checks ahead of the remaining items'); - expect(queued).toContain('not recovery retries'); + expect(queued).toContain('Insert Steps 9, 10 and 11 before the pending Step 11.5'); + expect(queued).toMatch(/not recovery retries/i); expect(finish.slice(finish.indexOf('3. **Native complete'))).toContain('then continue to Step 11.5'); }); @@ -312,19 +278,19 @@ describe('ship source controller', () => { const ctx = { host, skillName: 'ship', tmplPath: '', paths: HOST_PATHS[host] }; const ship = generateAdversarialStep(ctx); const finish = compact(ship.slice(ship.indexOf('### Finish the adversarial phase'))); - expect(finish).toContain('Apply these decisions in order before leaving Step 11'); - expect(finish).toContain('Required native review incomplete'); - expect(finish).toContain('Outside-provider output cannot replace this pass'); - expect(finish).toContain('Fixes queued after native completion'); - expect(finish).toContain('Native complete with no queued fixes'); + const branches = ['Required native review incomplete', 'Fixes queued after native completion', 'Native complete with no queued fixes'] + .map(label => finish.indexOf(label)); + expect(branches.every(index => index >= 0)).toBe(true); + expect(branches).toEqual([...branches].sort((a, b) => a - b)); + expect(finish).toMatch(/outside-provider output cannot replace this pass/i); expect(finish).toContain('Step 11.5'); expect(finish).not.toContain('proceed to Step 12'); expect(finish).not.toContain('return to Step 9'); const review = generateAdversarialStep({ ...ctx, skillName: 'review' }); expect(review).not.toContain('Ship control flow'); expect(review).not.toContain('Step 11.5'); - expect(review).toContain('Return all findings and structured-review decisions to Step 5'); - expect(review).toContain('do not start an inner repair loop'); + expect(review).toMatch(/return all findings and structured-review decisions to Step 5/i); + expect(review).toMatch(/do not start an inner repair loop/i); }); test('Step 11.5 verifies original record identity before any release write', () => { @@ -332,59 +298,56 @@ describe('ship source controller', () => { expect(bindingStart).toBeGreaterThan(entry.indexOf('{{SECTION:adversarial}}')); expect(bindingStart).toBeLessThan(entry.indexOf('## Step 12:')); const binding = compact(entry.slice(bindingStart, entry.indexOf('## Step 12:'))); - for (const field of ['saved handle, original token and source', 'skill:"review"', 'via:"ship"', 'skill:"adversarial-review"', + for (const field of ['skill:"review"', 'via:"ship"', 'skill:"adversarial-review"', 'review_binding.state', 'verified', 'review_binding.start_wtree', 'review_binding.end_wtree']) expect(binding).toContain(field); - expect(binding).toContain('Never attach new tokens to old work'); - expect(binding).toContain('Keep Step 9.4\'s incomplete flags'); - expect(binding).toContain('does not mean the failed or unrun probes passed'); - expect(binding).toContain('insert `9 → 10 → 11 → 11.5` before Step 12. Bind the new records at 11.5'); + expect(binding).toMatch(/never attach new tokens to old work/i); + expect(binding).toMatch(/does not mean the failed or unrun probes passed/i); + expect(binding).toContain('`9 → 10 → 11 → 11.5` before Step 12'); }); test('late build and behavior changes complete bounded ranges before docs', () => { const build = gate.slice(gate.indexOf('### 1.'), gate.indexOf('### 2.')); - expect(build).toContain('A missing prerequisite or failed build stops shipping'); - expect(build).toContain('Repair the prerequisite or build, then repeat stage 1'); - expect(build).toContain('After it passes, continue to stage 2; treat any content repair as a behavioral change there'); + expect(build).toMatch(/a missing prerequisite or failed build stops shipping/i); + expect(build).toMatch(/repeat stage 1/i); + expect(build).toMatch(/continue to stage 2/i); const behavior = gate.slice(gate.indexOf('1. **Behavior'), gate.indexOf('2. **Only authored')); expect(behavior).toContain('Insert `5–11.5 → 12–14 → 16` before the pending Step 17'); - expect(behavior).toContain('before the pending Step 17, then stop this step. This repair excludes Step 14.5'); - expect(behavior).toContain('rebuild and compare again before stage 3 decides documentation freshness'); + expect(behavior).toMatch(/excludes Step 14\.5/i); + expect(behavior).toMatch(/rebuild and compare again before stage 3/i); const plan = gate.slice(gate.indexOf('2. **Only authored'), gate.indexOf('3. **No changes')); - expect(plan).toContain("run Step 8's audit and decision gates only, then return to Step 16 stage 1"); - expect(plan).toContain("Never edit the child's counts yourself"); + expect(plan).toMatch(/run Step 8's audit and decision gates only, then return to Step 16 stage 1/i); + expect(plan).toMatch(/never edit the child's counts yourself/i); }); test('docs freshness has bounded re-audit and exact-content exception routes', () => { const docs = gate.slice(gate.indexOf('### 3.'), gate.indexOf('### 4.')); - expect(docs).toContain("Use Blocked recovery with the existing count"); + expect(docs).toMatch(/Blocked recovery with the existing count/i); expect(docs).toContain('**An attempt remains, with changed inputs or an available repair:** insert `14.5 → 15 → 16` before Step 17'); - expect(docs).toContain('restart Step 16 stage 1 to regenerate and compare again'); - expect(docs).toContain('STOP unless the user accepts the specific named documentation risk and all unwaivable gates clear'); - expect(docs).toContain('Never run a third audit'); - expect(docs).toContain('Unchanged approved content goes to stage 4; repaired content goes to stage 1'); - expect(docs).toContain('retain `Documentation: blocked`'); - expect(docs).toContain('Missing, stale or blocked | Use recovery below. Never silently refresh hashes'); + expect(docs).toMatch(/stop unless the user accepts the specific named documentation risk and all unwaivable gates clear/i); + expect(docs).toMatch(/never run a third audit/i); + expect(docs).toMatch(/unchanged approved content goes to stage 4; repaired content goes to stage 1/i); + expect(docs).toContain('`Documentation: blocked`'); + expect(docs).toMatch(/never silently refresh hashes/i); }); test('tests, absent test approval and new writes reopen final verification', () => { const tests = gate.slice(gate.indexOf('### 4.'), gate.indexOf('### 5.')); - expect(tests).toContain("**New, changed or unwaived test failure:** STOP publication. Run Steps 5–15, starting with Step 5's triage, then return to Step 16 stage 1"); - expect(tests).toContain('This recovery also applies if a failure appears while reporting in stage 5'); - expect(tests).toContain('run Steps 5–15, including the no-tests decision, then return to Step 16 stage 1'); - expect(tests).toContain('it does not authorize a third attempt'); - expect(gate).toContain('If content changes during or after verification, restart at stage 1 and complete all five stages before Step 17'); - expect(gate).toContain('Content-preserving commits keep valid evidence'); + expect(tests).toMatch(/\*\*New, changed or unwaived test failure:\*\* stop publication\. Run Steps 5–15/i); + expect(tests).toMatch(/run Steps 5–15, including the no-tests decision, then return to Step 16 stage 1/i); + expect(tests).toMatch(/does not authorize a third attempt/i); + expect(gate).toMatch(/restart at stage 1 and complete all five stages before Step 17/i); }); test('push failure kinds and publication races have distinct resumes', () => { const push = compact(entry.slice(entry.indexOf('## Step 17:'), entry.indexOf('## Step 18:'))); - expect(push).toContain('**If the push fails, STOP.** No Step 19 or publication claim'); - expect(push).toContain("fetch and inspect the remote, then merge under Step 3's conflict rules"); - expect(push).toContain('Run Steps 5–16 before returning to Step 17. Never rewrite history'); - expect(push).toContain('repeat Step 16 even if content is unchanged before returning to Step 17'); - expect(push).toContain('Never bypass failed guards'); + expect(push).toMatch(/push fails[^.]*\bstop\b[\s\S]{0,20}no Step 19 or publication claim/i); + expect(push).toMatch(/merge under Step 3's conflict rules/i); + expect(push).toContain('Run Steps 5–16 before returning to Step 17'); + expect(push).toMatch(/never rewrite history/i); + expect(push).toMatch(/repeat Step 16 even if content is unchanged/i); + expect(push).toMatch(/never bypass failed guards/i); const publication = compact(read('ship/sections/pr-body.md.tmpl')); - expect(publication).toContain("If the open PR/MR or title changed, repeat Step 18's identity/title preparation"); - expect(publication).toContain('then return here for a new lookup, fresh body and both redaction scans before publishing'); + expect(publication).toMatch(/repeat Step 18's identity\/title preparation/i); + expect(publication).toMatch(/both redaction scans before publishing/i); }); }); From 3a4c34f184d29bc8ff5f41da83c8f7318b9a5208 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 01:38:09 +0000 Subject: [PATCH 12/13] test(W7c): ship-workflow-clarity becomes a structural check of ship-only invariants W7c (t07 F1, F2, F15). One test file: 59 tests / ~438 pins -> 33 tests. Round-tagged titles (r6/r12/r22) are gone. Every invariant this file shared with the other converted ship files is now checked once, in its owner: ship-control-flow (work list, repair ranges, docs/test/push recovery, adversarial finish order, Step 11.5 binding), ship-plan-completion- invariants (evidence ledger, push/commit, waivers), ship-reentry-gates (stop lines, docs reentry, NEW_TITLE), workflow-excerpt (review cycle order, coverage/probe gates), qa-probe-gates / qa-caller-authority (QA loop). Removed assertion groups (invariant -> remaining protection): - Duplicated literals (45 repeated 2-7x, e.g. "Keep the same attempt counts throughout the invocation" x7, "Docs, TODO edits ... make evidence STALE" x5, the CYCLES >= 3 / fixes-below-cap / no-edits sentences, push-failure and native-recovery sentences) -> one free check each in the owners above. - Smoke/plan-check loop sentences and "Required even for small diffs ..." -> qa-probe-gates (free, every host); gate E2E ship-exploratory-small-cli (gate). Kept here: smoke guard started once before step 3, every host. - Invocation-record field lists (receipt/handle/attempt field prose, "reuse this invocation's saved level", FRESH/ALREADY_BUMPED level derivation, receipt definition, start-token definition) -> kept `BUMP_LEVEL`, record outside the product tree, approval scope, never borrow/replace a token, REVIEW_START/PASS_START/DESIGN_START, `gstack-review-log --start`, `gstack-wtree`; version derivation also in test/ship-version-sync.test.ts (free); judge `ship/SKILL.md workflow` (gate lane). - Reuse/judge-evidence prose ("original command, result/counts ...", "Compare hashes or complete bytes", "consumed files, fixtures ...", "complete expanded request, rubric ...") -> kept never resample, changed/ unknown dependencies rerun, mandatory reviews still run, freeze inputs; judge `ship/SKILL.md workflow` (gate lane). - Build/route prose ("If none exists, record not applicable ...", "declared generation/build commands ...", "Run them and save results", "Explain why other changes cannot affect it", "Inspect writer handles, including the docs child", "Keep Step 8's original child report and counts", "Recheck affected plan items ...", "append current evidence ...") -> kept stage order, route-label order, prompts/templates count as behavior, no-change -> stage 3, writer termination before another writer, stop until confirmed, build-failed label, no substitute command; gate E2Es ship-docsync-* (gate). - Docs-collector prose ("Terminal completion or confirmed termination is sufficient", "Require every field/type ...", "Compare saved base and input hashes ...", "Otherwise save post-child hashes ...", "A stale snapshot is neither ...", "Reconcile those before proceeding") -> kept attempt counting incl. failed launches, request alone insufficient, exited child blocked, HEAD/index unchanged + `files_updated`, protected-file exclusions, `documentation_section`, failed/blocked -> recovery, actual user exception, blocked status retained, same approved scope and exact content, retry restarts stage 1; gate E2Es ship-docsync-missing-marker, ship-docsync-launch-failure, ship-docsync-late-result (gate). - Release/commit prose ("Record the release decision after a version was actually written", "Skip unchanged versions and manifest-only repairs", "Best-effort, non-interactive, non-blocking", "Group VERSION + CHANGELOG + TODOS.md ...", "Only the final VERSION/CHANGELOG commit gets the release version and co-author trailer", "use the first changed component ... (major/minor/patch/micro") -> kept ALREADY_BUMPED, the stale "skip if ALREADY_BUMPED" ban, `baseVersion`/`currentVersion`, absent fourth component is zero, no automatic rebump, digest tokens and pre-push digest check, no Git tag, commit-before-push order; gate E2E ship-local-workflow (gate). - Coverage/design prose ("Count each generation-authorized attempt ...", "including the initial audit, failures and zero-test results", "Two passes already used ...", "read-only reassessment uses no pass", design-lite ownership, same path/line merge, specialist stats, distinct defects) -> kept the 2-pass and 30/5/2-minute limits, re-entry never resets, missing permission is not approval, audit-only header/override order/no commit/ return decisions, child JSON, one item with both sources and stricter ASK, neither pass substitutes; gate E2Es ship-coverage-audit, ship-coverage-value (gate). Design-review-lite E2E is periodic, so the merge rule keeps its free meaning check here. - Spec/title/comment prose ("existing open PR/MR", "one match supplies the existing title/identity", "Save the result for Step 19's recheck", "In a new shell, restore ...", "Partial, deferred, failed, dropped or unverified scope uses ...", "newest", "Try to auto-resolve if they are simple", "distinguish an app from a library/CLI", "If the ask is App Store/ TestFlight distribution", "a new package.json or Cargo.toml alone ...", "After edits, the full re-review repeats ...") -> judge `ship/SKILL.md workflow` (gate lane); ship-apple-gate (free) for the Apple route. Kept as case-insensitive meaning checks (stop/consent/approval/safety, always kept): never invent credentials, ask for unknown targets first, include the workflow in tests/review, frontmatter read as data never shell source, exact spec_branch match, positive-integer issue number, Closes #N only for fully completed scope, lookup failure/ambiguous match -> stop never "no PR", never publish an unprefixed title, ambiguous conflicts -> stop, AskUserQuestion and wait, unclear Apple target -> AskUserQuestion and wait, confirmed app -> stop and Read, stop for repair (recommended) or accept each named probe risk, skipping a fix is not risk acceptance, accepted risk is not passing verification, queue without editing in Steps 10/11, never credit outside coverage, do not report a clean review, unavailable outside challenge never resets the three-cycle limit, records mismatch blocks release, missing snapshots block regardless of HEAD, uncertain edits cannot be exempted, scripts/dependencies/runtime config need live tests, named testing gap A/B/C (recommended), reuse prior B only for same scope, declared but unavailable suite blocks, explicit untested-scope approval, report the gap never FRESH, no documentation writer after push, do not run create commands for an existing PR, Greptile status truthfulness. Kept unchanged: all stale-string bans, machine tokens and JSON contracts (`gh pr list ... --json number,title,url`, glab list, `v$NEW_VERSION `, `${NEW_TITLE:?...}` guard, exit 3, `--allow-paths ...`, `| FRESH (exit 0) |`, `--label --expect-cmd`, `gstack-review-read`, `git diff `, `"status":"complete|no_pr|unavailable"`, specialist and coverage JSON, `status:"unavailable"`/`completed:false`/`converged:false`, `Pre-Landing Review: INCOMPLETE`, `ALREADY_PUSHED`), the threshold/branch/ installed-asset reference test, the section-index row check. Cosmetic pins dropped: STOP / ONE / ONLY capitals and bold (t07 F2). Mutation log: .context/mutation-ship-workflow-clarity.md (23 mutations, 0 unexpected: 4 reorders, 4 markers, 9 safety red; 6 rewordings green, all of which turned the old test red). --- test/ship-workflow-clarity.test.ts | 977 ++++++++--------------------- 1 file changed, 255 insertions(+), 722 deletions(-) diff --git a/test/ship-workflow-clarity.test.ts b/test/ship-workflow-clarity.test.ts index 0021b5fb3..ec654fb2e 100644 --- a/test/ship-workflow-clarity.test.ts +++ b/test/ship-workflow-clarity.test.ts @@ -2,792 +2,325 @@ import { expect, test } from 'bun:test'; import { readFileSync } from 'node:fs'; import { ALL_HOST_CONFIGS } from '../hosts'; import { generateAdversarialStep } from '../scripts/resolvers/outside-voice-steps'; -import { generatePlanCompletionGateShip } from '../scripts/resolvers/plan-gates'; import { generateQAReview } from '../scripts/resolvers/qa'; import { HOST_PATHS } from '../scripts/resolvers/types'; import { readWorkflowExcerpt } from './helpers/workflow-excerpt'; const readShip = () => readWorkflowExcerpt('ship/SKILL.md', '# Ship:', '## Important Rules'); - -test('Ship initializes and applies its smoke guard independently of required plan checks', () => { - for (const host of ALL_HOST_CONFIGS) { - const body = generateQAReview({ host: host.name, skillName: 'ship', tmplPath: '', paths: HOST_PATHS[host.name] }); - expect(body).toContain('Run the shared preflight; start its smoke guard once. Guard every smoke probe.'); - expect(body.indexOf('start its smoke guard once')).toBeLessThan(body.indexOf('**3. Run smoke and plan checks.**')); - expect(body).toContain('Required even for small diffs or missing plans/servers'); - expect(body).toContain('Then run required plan checks and revalidation, even after smoke expires'); - expect(body).toContain('using the same procedure but no smoke guard; never reset the clock'); - } -}); - -test('ship uses the PR template headings instead of a competing combined QA report', () => { - const ship = readShip().replace(/\s+/g, ' '); - expect(ship).toContain('PR section `## Exploratory QA`'); - expect(ship).toContain('plans in `## Verification Results`'); - expect(ship).toContain('Read QA\'s `templates/functional-report-template.md`: PR section'); - expect(ship).toContain('Link every checkpoint; no second report'); - expect(ship).not.toContain('## Exploratory QA and Verification Results'); -}); - -test('late adversarial fixes use the same bounded review cycle before release steps', () => { - const ship = readShip(); - const controller = compact(ship.slice(ship.indexOf('### Ship control flow'), ship.indexOf('## Step 0:'))); - const review = ship.slice(ship.indexOf('## Step 9:'), ship.indexOf('## Step 10:')); - const adversarial = compact(ship.slice(ship.indexOf('## Step 11:'), ship.indexOf('## Step 12:'))); - expect(review).toContain('queued Steps 10–11 findings'); - expect(compact(review)).toContain('do not run a fourth fixing cycle'); - expect(controller).toContain('Keep the same attempt counts throughout the invocation'); - expect(adversarial).toContain('queued for the parent; do not edit during Step 11'); - expect(compact(review)).toContain("Every repeat starts before the checklist read and captures a fresh REVIEW_START. Finish the complete review and QA before applying any fix in Step 9.4"); - expect(adversarial).toContain('**Required native review incomplete:** STOP'); - expect(adversarial).toContain('Optional outside failures retain their own incomplete records'); - expect(adversarial).toContain('Insert Steps 9, 10 and 11 before the pending Step 11.5 in the work list. Step 9 completes full review before fixes'); - expect(adversarial).toContain('**Native complete with no queued fixes:**'); - expect(adversarial).toContain('no queued fixes'); - expect(adversarial).toContain('then continue to Step 11.5'); - const greptile = ship.slice(ship.indexOf('## Step 10:'), ship.indexOf('## Step 11:')); - expect(greptile).toContain('queue the approved fix without editing here'); - expect(compact(greptile)).toContain('Finish the saved replies without asking again about completed fixes'); -}); - -test('late source changes repeat affected gates without resetting either allowance', () => { - const ship = readShip(); - const gate = ship.slice(ship.indexOf('## Step 16:'), ship.indexOf('## Step 17:')); - const controller = ship.slice(ship.indexOf('### Ship control flow'), ship.indexOf('## Step 0:')); - const text = compact(controller + gate); - expect(text).toContain("**Behavior, tests or build inputs changed:** Prompts/templates count as behavior"); - expect(text).toContain("Keep the same attempt counts throughout the invocation"); - expect(text).toContain('Validate the outcome before Step 15'); - const workflow = ship.replace(/\s+/g, ' '); - expect(workflow).toContain('Increment before each launch or inline takeover'); - expect(workflow).toContain('Increment before each launch or inline takeover, including failed launches'); - expect(workflow).toContain('A stale snapshot is neither a new attempt nor a current audit'); - expect(workflow).toContain('never a third attempt, even after Step 16 changes'); -}); - const readTemplate = (name: string) => readFileSync(new URL(`../${name}`, import.meta.url), 'utf8'); const compact = (text: string) => text.replace(/\s+/g, ' '); +const ordered = (text: string, markers: (string | RegExp)[]) => { + const positions = markers.map(marker => typeof marker === 'string' ? text.indexOf(marker) : text.search(marker)); + expect(positions.every(position => position >= 0)).toBe(true); + expect(positions).toEqual([...positions].sort((a, b) => a - b)); +}; const reviewTemplate = readTemplate('ship/sections/review-army.md.tmpl'); const coverageTemplate = readTemplate('ship/sections/test-coverage.md.tmpl'); const entryTemplate = readTemplate('ship/SKILL.md.tmpl'); const controlTemplate = entryTemplate.slice(entryTemplate.indexOf('### Ship control flow'), entryTemplate.indexOf('{{SECTION_INDEX:ship}}')); -const control = compact(controlTemplate); +const entry = compact(entryTemplate); const reviewFlow = compact(reviewTemplate); -const nativeFlow = compact(generateAdversarialStep({ host: 'claude', skillName: 'ship', tmplPath: '', paths: HOST_PATHS.claude })); -const finalGate = compact(entryTemplate.slice(entryTemplate.indexOf('## Step 16:'), entryTemplate.indexOf('## Step 17:'))); +const finalGate = compact(controlTemplate + entryTemplate.slice(entryTemplate.indexOf('## Step 16:'), entryTemplate.indexOf('## Step 17:'))); const pushFlow = compact(entryTemplate.slice(entryTemplate.indexOf('## Step 17:'), entryTemplate.indexOf('## Step 18:'))); -const commentsFlow = compact(readTemplate('ship/sections/greptile.md.tmpl')); const docsTemplate = readTemplate('ship/sections/documentation.md.tmpl'); +const docs = compact(docsTemplate); const prTemplate = readTemplate('ship/sections/pr-body.md.tmpl'); -test('ship r12 template: the invocation note locates release, attempts and receipts', () => { - const entry = compact(entryTemplate); - expect(entry).toContain('outside the product tree and save its absolute path'); - for (const state of [ - 'versions, `BUMP_LEVEL`, reviewed tree and attempt counts', - 'Reuse it only for that same scope; a repair never resets approvals or expands them', - 'each approval\'s finding, files and authorized action', - 'handles, original start tokens, terminal states, outputs and queued fixes', - 'command/label, result/counts, timestamp, log and consumed inputs', - 'candidate/id, attempts used, accepted hashes or named blocked exception', - ]) { - expect(entry).toContain(state); +test('ship QA starts its smoke guard once before probes on every host', () => { + for (const host of ALL_HOST_CONFIGS) { + const body = generateQAReview({ host: host.name, skillName: 'ship', tmplPath: '', paths: HOST_PATHS[host.name] }); + ordered(body, [/start its smoke guard once/i, '**3. Run smoke and plan checks.**']); } - expect(entry).toContain('handles, original start tokens, terminal states, outputs and queued fixes'); - expect(entry).toContain("reuse this invocation's saved level"); - expect(entry).toContain('Otherwise FRESH chooses it in item 2 and ALREADY_BUMPED derives it in item 1'); }); -test('ship r12 template: reuse compares recorded observations and actual inputs without resampling judges', () => { - const gate = compact(controlTemplate + entryTemplate.slice(entryTemplate.indexOf('## Step 16:'), entryTemplate.indexOf('## Step 17:'))); - expect(gate).toContain('original command, result/counts, timestamp and log'); - expect(gate).toContain('Compare hashes or complete bytes'); - expect(gate).toContain('consumed files, fixtures, dependencies and execution parameters'); - expect(gate).toContain('complete expanded request, rubric, parameters and builder/runtime dependencies'); - expect(gate).toContain('never resample it'); - expect(gate).toContain('Mandatory reviews still run'); - expect(gate).toContain('Check each test lane\'s receipt as well'); +test('ship reports QA in the PR template headings, not a second combined report', () => { + const ship = compact(readShip()); + expect(ship).toContain('`## Exploratory QA`'); + expect(ship).toContain('`## Verification Results`'); + expect(ship).toContain("Read QA's `templates/functional-report-template.md`"); + expect(ship).toMatch(/link every checkpoint; no second report/i); + expect(ship).not.toContain('## Exploratory QA and Verification Results'); }); -test('ship r12 template: undeclared builds differ from unavailable declared prerequisites', () => { - const gate = compact(controlTemplate + entryTemplate.slice(entryTemplate.indexOf('## Step 16:'), entryTemplate.indexOf('## Step 17:'))); - expect(gate).toContain('If none exists, record not applicable and the inspected sources'); - expect(gate).toContain('A missing prerequisite or failed build stops shipping'); - expect(gate).toContain('Confirm terminal completion or termination before another writer runs'); - expect(gate).toContain('Timeout or cancellation acknowledgment alone means STOP until confirmed'); - expect(gate).toContain("**No changes, or the docs-only checks still support the plan:** Continue to stage 3 without a new code review"); - expect(gate).toContain("**Only authored docs or release metadata changed:** Keep Step 8's original child report and counts"); - expect(gate).toContain("**Behavior, tests or build inputs changed:** Prompts/templates count as behavior"); +test('the invocation record keeps state outside the product tree and scopes approvals', () => { + expect(entry).toContain('**invocation record**'); + expect(entry).toMatch(/outside the product tree/i); + expect(entry).toContain('`BUMP_LEVEL`'); + expect(entry).toMatch(/each approval's finding, files and authorized action/i); + expect(entry).toMatch(/never borrow or replace a token/i); + const state = compact(entryTemplate.slice(entryTemplate.indexOf('### Keep state'), entryTemplate.indexOf('{{SECTION_INDEX:ship}}'))); + expect(state).toContain('`gstack-review-log --start`'); + expect(state).toContain('`REVIEW_START` for Step 9, a separate `PASS_START` for each Step 11 attempt, and `DESIGN_START` for design'); + expect(state).toContain('`gstack-wtree`'); + expect(entry).toMatch(/every new invocation repeats Steps 1–16, including both reviews and the docs audit/i); + expect(entry).toContain('Start with Steps 1–21 in order, including 11.5 and 14.5'); }); -test('ship r12 template: docs collection separates stopped work from acceptable coverage', () => { - const docs = compact(docsTemplate); - expect(docs).toContain('Terminal completion or confirmed termination is sufficient'); - expect(docs).toContain('Require every field/type, exact audit id, schema, status invariant and actual spawned marker'); - expect(docs).toContain('enforcing prompt/audit-scope permissions and protected-file exclusions'); - expect(docs).toContain('Compare saved base and input hashes with current content'); - expect(docs).toContain('An exited child with missing output is stopped, but its audit is blocked'); - expect(docs).toContain('HEAD and index must be unchanged, existing dirty/untracked user content preserved, and changed paths exactly `files_updated`'); - expect(docs).toContain('Otherwise save post-child hashes, status and `documentation_section` for Step 16'); - expect(docs).toContain('A failed check or `blocked` result goes to recovery, even with valid JSON'); - expect(docsTemplate).toContain('**Subagent prompt:**'); - expect(docsTemplate).toContain('**Parent processing:**'); -}); - -test('ship r12 template: review finalization precedes one prioritized return table', () => { - const persist = reviewTemplate.indexOf('6. Persist the review result'); - const route = reviewTemplate.indexOf('### Decide whether to repeat Step 9'); - expect(persist).toBeGreaterThanOrEqual(0); - expect(route).toBeGreaterThan(persist); - const exits = reviewFlow.slice(reviewFlow.indexOf("### Decide whether to repeat Step 9")); - const missing = exits.indexOf('**Dispatched reviewer output missing:** STOP'); - const cap = exits.indexOf('**Third fixing cycle reached (`CYCLES >= 3`):** STOP'); - const fixing = exits.indexOf('**Fixes applied below the cap:**'); - const zeroFix = exits.indexOf('**No edits in this pass:** Resolve the required-probe gate below. Only after it clears may you continue to Step 10'); - for (const position of [missing, cap, fixing, zeroFix]) expect(position).toBeGreaterThanOrEqual(0); - expect(missing).toBeLessThan(cap); - expect(cap).toBeLessThan(fixing); - expect(fixing).toBeLessThan(zeroFix); - expect(compact(reviewTemplate)).toContain('Finish and log this pass before choosing the next step'); - expect(compact(reviewTemplate)).toContain('Complete items 5–6 exactly once with the original REVIEW_START'); -}); - -test('ship r6 template: one progress note defines surviving state and per-pass content tokens', () => { - const entry = compact(entryTemplate); - expect(entry).toContain('one private Markdown **invocation record**'); - expect(entry).toContain('**Next steps:** one ordered work list, with the current step marked'); - expect(entry).toContain('tracked and non-ignored untracked files'); - expect(entry).toContain('`gstack-wtree` prints a Git tree hash'); - expect(entry).toContain('**start token** is the opaque value returned by `gstack-review-log --start` before it reads the diff'); - expect(entry).toContain('Never borrow or replace a token'); - expect(entry).toContain('each approval\'s finding, files and authorized action'); -}); - -test('ship r6 template: plan obligations precede learnings and scope drift even without a plan', () => { +test('plan obligations precede scope drift, learnings and review', () => { const plan = readTemplate('ship/sections/plan-completion.md.tmpl'); const route = compact(plan.slice(0, plan.indexOf('**Dispatch this step'))); - const steps = ['1. Dispatch the audit, validate its result and resolve its Gate Logic', - "2. Collect the plan's executable checks in Step 8.1; do not run them yet", - '3. Run Step 8.2 Scope Drift', - '4. Run Prior Learnings, including its setting question when offered, then proceed to Step 9 for review and QA']; - const positions = steps.map(step => route.indexOf(step)); - expect(positions.every(position => position >= 0)).toBe(true); - expect(positions).toEqual([...positions].sort((a, b) => a - b)); - const noPlan = generatePlanCompletionGateShip({ host: 'claude', skillName: 'ship', tmplPath: '', paths: HOST_PATHS.claude }); - expect(noPlan).toContain('Skip only the plan completion audit'); - expect(noPlan).toContain('Continue with Step 8.1, Scope Drift and Prior Learnings; Step 9 QA still runs'); - expect(noPlan).not.toContain('Skip entirely'); - const sections = ['{{PLAN_COMPLETION_GATE_SHIP}}', '{{PLAN_VERIFICATION_EXEC}}', - '{{SCOPE_DRIFT}}', '{{LEARNINGS_SEARCH:query=release ship version changelog merge pr}}']; - const actual = sections.map(section => plan.indexOf(section)); - expect(actual.every(position => position >= 0)).toBe(true); - expect(actual).toEqual([...actual].sort((a, b) => a - b)); + ordered(route, ['1. Dispatch the audit', '2. Collect', '3. Run Step 8.2 Scope Drift', '4. Run Prior Learnings']); + ordered(plan, ['{{PLAN_COMPLETION_GATE_SHIP}}', '{{PLAN_VERIFICATION_EXEC}}', '{{SCOPE_DRIFT}}', + '{{LEARNINGS_SEARCH:query=release ship version changelog merge pr}}']); expect(entryTemplate.indexOf('{{SECTION:plan-completion}}')).toBeLessThan(entryTemplate.indexOf('{{SECTION:review-army}}')); + const section = readTemplate('ship/sections/plan-completion.md'); + ordered(section, ['## Step 8.1:', '## Step 8.2:', '## Prior Learnings']); + expect(section).toMatch(/N\) Not done — block ship/i); + expect(section).not.toContain('re-enter the priority-1 gate'); }); -test('ship r6 template: an approved rebump logs the written version rather than its initial state', () => { - const bump = compact(entryTemplate.slice(entryTemplate.indexOf('## Step 12:'), entryTemplate.indexOf('{{SECTION:changelog}}'))); - expect(bump).toContain('Record the release decision after a version was actually written'); - expect(bump).toContain('including an approved ALREADY_BUMPED rebump'); - expect(bump).toContain('Skip unchanged versions and manifest-only repairs'); - expect(bump).not.toContain('skip if ALREADY_BUMPED'); - expect(bump).toContain('Only approval changes the existing version'); - expect(bump).toContain('Best-effort, non-interactive, non-blocking'); +test('the parent owns one ordered review phase that persists each pass once', () => { + expect(reviewFlow).toMatch(/set CYCLES to 0 on first entry only/i); + expect(reviewFlow).toMatch(/steps 10–11 queue findings without editing/i); + expect(reviewFlow).toMatch(/changed finding scope needs a new decision/i); + expect(reviewFlow).toMatch(/gated\/unsupported specialists skip only their dispatch, never QA or Step 11/i); + expect(reviewTemplate.indexOf('6. Persist the review result')).toBeGreaterThan(-1); + expect(reviewTemplate.indexOf('6. Persist the review result')).toBeLessThan(reviewTemplate.indexOf('### Decide whether to repeat Step 9')); + const finalize = compact(reviewTemplate.slice(reviewTemplate.indexOf('4. **'), reviewTemplate.indexOf('5. Output summary:'))); + expect(finalize).toMatch(/increment CYCLES once if fixes were applied/i); + expect(finalize).toContain('items 5–6 exactly once with the original REVIEW_START'); + expect(finalize).toContain('`converged:false`'); + expect(reviewTemplate).toContain('--finish REVIEW_START'); + expect(reviewFlow).toMatch(/never recapture at persistence to certify unreviewed fixes/i); + expect(reviewTemplate).toContain('`CONVERGED`: completed with zero fixes'); + expect(reviewFlow).toContain('`status:"unavailable"`, `completed:false` and `converged:false`'); + expect(reviewFlow).toContain('Pre-Landing Review: INCOMPLETE'); + expect(reviewFlow).toContain('`STATUS`: `unavailable` for missing dispatched reviewer output'); }); -test('ship r6 template: late-change routing runs prerequisites before final evidence without a circular gate', () => { - const gate = compact(controlTemplate + entryTemplate.slice(entryTemplate.indexOf('## Step 16:'), entryTemplate.indexOf('## Step 17:'))); - const build = gate.indexOf('### 1. Finish writers and prepare outputs'); - const route = gate.indexOf('### 2. Choose the change route'); - const docs = gate.indexOf('### 3. Resolve documentation freshness'); - const verify = gate.indexOf('### 4. Verify the frozen candidate'); - expect(build).toBeGreaterThan(0); - expect(route).toBeGreaterThan(build); - expect(docs).toBeGreaterThan(route); - expect(verify).toBeGreaterThan(docs); - expect(gate).toContain('This repair excludes Step 14.5 because the rebuild can change generated docs. Step 16 restarts at stage 1'); - expect(gate).toContain('Insert `5–11.5 → 12–14 → 16` before the pending Step 17, then stop this step. This repair excludes Step 14.5'); - expect(gate).toContain('Compare hashes or complete bytes of its saved and current consumed files, fixtures, dependencies and execution parameters'); - expect(gate).toContain('changed or unknown dependencies require a rerun'); - expect(gate).toContain('Freeze inputs through verification and push'); - expect(gate).toContain('If content changes during or after verification, restart at stage 1 and complete all five stages before Step 17'); +test('named QA risks need explicit acceptance and never count as passing verification', () => { + const gate = compact(reviewTemplate.slice(reviewTemplate.indexOf('**Required-probe parent gate:**'))); + expect(gate).toMatch(/AskUserQuestion: stop for repair \(recommended\), or explicitly accept each named probe's concrete risk/i); + expect(gate).toMatch(/skipping a fix is not risk acceptance or a passing probe/i); + expect(reviewFlow).toMatch(/record accepted untested risk separately, not as passing verification/i); }); -test('ship r6 template: docs attempts count at launch and exhausted late changes never open a third attempt', () => { - const docs = compact(docsTemplate); - expect(docs).toContain('an initial audit plus ONE repair/re-audit in the invocation record'); - expect(docs).toContain('an initial audit plus ONE repair/re-audit'); - expect(docs).toContain('never a third attempt, even after Step 16 changes'); - expect(docs).toContain('Increment before each launch or inline takeover'); - expect(docs).toContain('Increment before each launch or inline takeover, including failed launches'); - expect(docs).toContain('A stale snapshot is neither a new attempt nor a current audit'); - expect(docs).toContain('Terminal completion or confirmed termination is sufficient'); - expect(docs).toContain('the request alone is insufficient'); - expect(docs).toContain('Unconfirmed writers, ownership violations, unauthorized Git mutation and redaction/security gates cannot be waived'); - expect(docs).toContain('Only an actual user exception counts'); - expect(docs).toContain('reports and PRs retain blocked status'); -}); - -test('ship r6 template: evidence exemptions inspect metadata content rather than trusting filenames', () => { - const gate = compact(controlTemplate + entryTemplate.slice(entryTemplate.indexOf('## Step 16:'), entryTemplate.indexOf('## Step 17:'))); - expect(gate).toContain('Inspect changes since the run; `--allow-paths` exempts only release metadata'); - expect(gate).toContain('scripts, dependencies and runtime configuration require live tests'); - expect(gate).toContain('A `package.json` version-only edit can qualify; scripts, dependencies and runtime configuration require live tests'); - expect(gate).toContain('Docs, TODO edits, new/generated tests and fixes make evidence STALE'); - expect(gate).toContain('Only receipt storage/readback failed | Independently prove unchanged final content, the same command and valid age from the successful run\'s evidence'); - expect(gate).toContain('Only receipt storage/readback failed'); -}); - -test('ship r6 template: linked spec discovery is ordered outside the body and cannot exit publication', () => { - const instructions = compact(prTemplate.slice(0, prTemplate.indexOf('The PR/MR body should contain'))); - expect(instructions).toContain('Read archive frontmatter as data, never shell source'); - expect(instructions).toContain('exact `spec_branch` match'); - expect(instructions).toContain('newest `spec_filed_at`'); - expect(instructions).toContain('positive integer `spec_issue_number`'); - expect(instructions).toContain('omit only `## Linked Spec` and continue composing the PR'); - expect(instructions).toContain('Only fully completed Step 8 plan scope permits `Closes #N`'); - expect(instructions).toContain('Partial, deferred, failed, dropped or unverified scope uses `Linked to #N`'); - const body = prTemplate.slice(prTemplate.indexOf('The PR/MR body should contain'), prTemplate.indexOf('#### Redaction scan')); - expect(body).not.toContain('CURRENT_BRANCH='); - expect(body).not.toContain('SPEC_ARCHIVES='); - expect(body).not.toContain('SPEC_FILE=$(grep'); - expect(prTemplate).not.toContain('[ -z "$SPEC_FILE" ] && exit'); -}); - -test('ship r6 template: Step 18 prepares the exact title Step 19 scans and publishes', () => { - const title = compact(entryTemplate.slice(entryTemplate.indexOf('## Step 18:'), entryTemplate.indexOf('{{SECTION:pr-body}}'))); - expect(title).toContain('Save the result as `NEW_TITLE` for Step 19'); - expect(title).toContain('existing open PR/MR'); - expect(title).toContain('For a new PR/MR, compose `v : `'); - expect(prTemplate).toContain('Use Step 18\'s `NEW_TITLE` unchanged; its version prefix is already present'); - expect(prTemplate).not.toContain('`NEW_TITLE`, prefixed with'); - expect(prTemplate).toContain('In a new shell, restore the saved literal title before this block'); - expect(prTemplate).toContain(': "${NEW_TITLE:?Restore the saved Step 18 title before scanning}"'); - expect(prTemplate).not.toContain('NEW_TITLE=""'); - expect(prTemplate).toContain('Update the title with the same scanned `NEW_TITLE`'); - expect(compact(prTemplate)).toContain('exit 3 blocks for HIGH findings'); -}); - -test('ship template consolidation: initial, failed and inline generation attempts share one allowance', () => { +test('coverage generation has one bounded allowance and the child stays audit-only when told', () => { const allowance = compact(coverageTemplate.slice(0, coverageTemplate.indexOf('````text'))); expect(allowance).toContain('Maximum 2 generation passes total per invocation'); - expect(allowance).toContain('Count each generation-authorized attempt before dispatch/inline execution'); - expect(allowance).toContain('including the initial audit, failures and zero-test results'); - expect(allowance).toContain('Re-entry never resets it'); - expect(allowance).toContain('Two passes already used means no further generation'); - expect(allowance).toContain('read-only reassessment uses no pass'); - expect(allowance).toContain('30-path/5-tests-per-pass/2-minute per-test caps'); - expect(allowance).toContain('missing permission is not approval'); - expect(coverageTemplate).toContain("confirm it stopped before running the same audit inline"); -}); - -test('ship template consolidation: audit-only authority reaches the actual coverage child prompt', () => { + expect(allowance).toContain('30-path/5-tests-per-pass/2-minute'); + expect(allowance).toMatch(/re-entry never resets it/i); + expect(allowance).toMatch(/missing permission is not approval/i); const prompt = coverageTemplate.split('````text\n')[1]?.split('\n````')[0] ?? ''; expect(prompt).toContain('Generation: ; passes used: of 2.'); - expect(prompt).toContain('Audit-only overrides every generation instruction below.'); - expect(prompt.indexOf('Audit-only overrides')).toBeLessThan(prompt.indexOf('{{TEST_COVERAGE_AUDIT_SHIP}}')); - expect(prompt).toContain('Do not commit or push'); - expect(prompt).toContain('return unresolved user decisions to the parent'); + ordered(prompt, [/audit-only overrides every generation instruction/i, '{{TEST_COVERAGE_AUDIT_SHIP}}']); + expect(prompt).toMatch(/do not commit or push/i); + expect(prompt).toMatch(/return unresolved user decisions to the parent/i); expect(prompt).toContain('"coverage_pct":N,"gaps":N'); expect(prompt).toContain('"tests_added":["path",...]'); }); -test('ship template consolidation: duplicate design defects have one action without losing independent coverage', () => { +test('design-lite and the Design specialist merge duplicates under the stricter gate', () => { const design = compact(reviewTemplate.slice(reviewTemplate.indexOf('{{DESIGN_REVIEW_LITE}}'), reviewTemplate.indexOf('{{REVIEW_ARMY}}'))); - expect(design).toContain('The parent owns design-lite; the Design specialist is an independent read'); - expect(design).toContain('Before final counting/Fix-First'); - expect(design).toContain('same evidenced design defect at the same path/line'); - expect(design).toContain('one item with both sources and stricter ASK'); - expect(design).toContain('Retain actual specialist stats'); - expect(design).toContain('distinct defects stay separate'); - expect(design).toContain('neither pass substitutes for the other'); - expect(reviewTemplate).toContain('{{DESIGN_REVIEW_LITE}}'); - expect(reviewTemplate).toContain('{{REVIEW_ARMY}}'); + expect(design).toMatch(/one item with both sources and stricter ASK/i); + expect(design).toMatch(/neither pass substitutes for the other/i); + ordered(reviewTemplate, ['{{DESIGN_REVIEW_LITE}}', '{{REVIEW_ARMY}}']); expect(reviewTemplate).toContain('"dispatched":true,"findings":N,"critical":N,"informational":N'); }); -test('ship template consolidation: the parent owns one ordered review phase', () => { - const intro = compact(controlTemplate + reviewTemplate); - expect(intro).toContain("You, the **parent** running /ship, own advancement"); - expect(intro).toContain('Set CYCLES to 0 on first entry only'); - expect(intro).toContain('**No edits in this pass:**'); - expect(intro).toContain('Start with Steps 1–21 in order, including 11.5 and 14.5'); - expect(intro).toContain('Steps 10–11 queue findings without editing'); - expect(intro).toContain("Finish the complete review and QA before applying any fix in Step 9.4"); - expect(intro).toContain('Keep the same attempt counts throughout the invocation'); - expect(intro).toContain('changed finding scope needs a new decision'); - expect(intro).toContain('Gated/unsupported specialists skip only their dispatch, never QA or Step 11'); - expect(intro).toContain("Apply these decisions in order"); - for (const step of ['specialists (9.1)', 'exploratory QA (9.2.1)', 'fixes and logging (9.4)']) { - expect(intro.indexOf(step)).toBeGreaterThanOrEqual(0); - } - expect(intro.indexOf('specialists (9.1)')).toBeLessThan(intro.indexOf('exploratory QA (9.2.1)')); - expect(intro.indexOf('exploratory QA (9.2.1)')).toBeLessThan(intro.indexOf('fixes and logging (9.4)')); +test('late adversarial and comment fixes queue without editing outside Step 9.4', () => { + const ship = readShip(); + expect(compact(ship.slice(ship.indexOf('## Step 11:'), ship.indexOf('## Step 12:')))).toMatch(/queued for the parent; do not edit during Step 11/i); + expect(compact(ship.slice(ship.indexOf('## Step 10:'), ship.indexOf('## Step 11:')))).toMatch(/queue the approved fix without editing here/i); }); -test('ship template consolidation: every fixing pass persists once before looping or stopping at cycle three', () => { - const finalize = compact(reviewTemplate.slice(reviewTemplate.indexOf('4. **'), reviewTemplate.indexOf('5. Output summary:'))); - expect(finalize).toContain('Increment CYCLES once if fixes were applied'); - expect(finalize).toContain('Finish and log this pass before choosing the next step'); - expect(finalize).toContain('Complete items 5–6 exactly once with the original REVIEW_START'); - expect(reviewTemplate.indexOf('6. Persist the review result')).toBeLessThan(reviewTemplate.indexOf('### Decide whether to repeat Step 9')); - expect(finalize).toContain('Then commit named fixed files'); - expect(finalize).toContain('fixes also require `converged:false`'); - expect(reviewFlow).toContain('**Third fixing cycle reached (`CYCLES >= 3`):** STOP and report recurring findings with `converged:false`; do not run a fourth fixing cycle'); - expect(control).toContain('Keep the same attempt counts throughout the invocation'); - expect(reviewFlow).toContain('**Fixes applied below the cap:**'); - expect(reviewFlow).toContain("Every repeat starts before the checklist read and captures a fresh REVIEW_START"); - expect(compact(entryTemplate)).toContain('Reuse waivers only for the same verified pre-existing failures and approved scope'); - expect(reviewTemplate).toContain('--finish REVIEW_START'); - expect(compact(reviewTemplate)).toContain('never recapture at persistence to certify unreviewed fixes'); - expect(reviewTemplate).toContain('`CONVERGED`: completed with zero fixes'); +test.each(ALL_HOST_CONFIGS.map(({ name }) => name))('%s: ship adversarial approvals queue without editing and never credit outside coverage', host => { + const ctx = { host, skillName: 'ship', tmplPath: '', paths: HOST_PATHS[host] }; + const text = compact(generateAdversarialStep(ctx)); + expect(text).toMatch(/if A: queue the approved findings without editing here/i); + expect(text).toMatch(/retain the acknowledged findings and failed gate/i); + expect(text).toMatch(/do not report a clean review/i); + expect(text).toMatch(/native completion never credits outside coverage/i); + const standalone = generateAdversarialStep({ ...ctx, skillName: 'review' }); + expect(standalone).not.toContain('Before Step 12:'); + expect(standalone).not.toContain('### Finish the adversarial phase'); + expect(compact(standalone)).toMatch(/queue the findings and this approval for Step 5's Fix-First handling/i); }); -test('ship template consolidation: named QA risks remain failed or incomplete and cannot waive other gates', () => { - const intro = compact(reviewTemplate.slice(reviewTemplate.indexOf('**Required-probe parent gate:**'))); - expect(intro).toContain('explicitly accept each named probe\'s concrete risk. Skipping a fix is not risk acceptance or a passing probe'); - expect(intro).toContain('With completed checklist and dispatched reviewers, failed/unavailable required probes block continuation'); - expect(intro).toContain('Keep actual outcomes and incomplete flags; VERIFY_RESULT stays fail'); - expect(intro).toContain('VERIFY_RESULT stays fail for plan-check exceptions'); - expect(compact(reviewTemplate)).toContain('Record accepted untested risk separately, not as passing verification'); - expect(intro).toContain('cannot waive missing reviewer output, recurring fixes or independent test/security gates'); - expect(compact(reviewTemplate)).toContain('Skipping a fix is not risk acceptance or a passing probe'); - expect(compact(reviewTemplate)).toContain('all required probes pass'); - expect(compact(reviewTemplate)).toContain('Failed, blocked, inconclusive or not-run required probes mean false, never clean'); - expect(compact(reviewTemplate)).toContain('Record accepted untested risk separately, not as passing verification'); +test('outside challenge availability never resets fix limits or bypasses blocking owners', () => { + const adversarial = compact(readTemplate('ship/sections/adversarial.md')); + expect(adversarial).toMatch(/an unavailable outside challenge does not block shipping by itself/i); + expect(adversarial).toMatch(/structured P1 and non-convergence gates still apply/i); + expect(adversarial).toMatch(/returning here never resets Step 9's three-cycle fix limit/i); + const standaloneReview = readTemplate('review/sections/adversarial.md'); + expect(standaloneReview).toContain('Step 5 Fix-First'); + expect(standaloneReview).not.toContain('supported findings still enter Step 11'); }); -test('ship template consolidation: late behavioral inputs revisit named gates while docs still get freshness checks', () => { - const gate = compact(controlTemplate + entryTemplate.slice(entryTemplate.indexOf('## Step 16:'), entryTemplate.indexOf('## Step 17:'))); - expect(gate).toContain('Insert `5–11.5 → 12–14 → 16` before the pending Step 17'); - expect(gate).toContain('`5–11.5 → 12–14 → 16`'); - expect(gate).toContain("**Behavior, tests or build inputs changed:** Prompts/templates count as behavior"); - expect(gate).toContain('**Behavior, tests or build inputs changed:** Prompts/templates count as behavior. Insert `5–11.5 → 12–14 → 16` before the pending Step 17'); - expect(gate).toContain('Explain why other changes cannot affect it; changed or unknown dependencies require a rerun'); - expect(gate).toContain("**Only authored docs or release metadata changed:** Keep Step 8's original child report and counts"); - expect(gate).toContain("**No changes, or the docs-only checks still support the plan:** Continue to stage 3 without a new code review"); - expect(gate).toContain("Keep the same attempt counts throughout the invocation"); - expect(gate).toContain('Validate the outcome before Step 15'); - expect(gate).toContain("retain `Documentation: blocked`, its reason and incomplete scope"); - expect(gate).toContain("Validate the outcome before Step 15, then restart Step 16 stage 1"); - expect(gate).toContain('Inspect writer handles, including the docs child'); - expect(gate).toContain('STOP until confirmed'); +test('Step 11.5 rejects foreign records and blocks release on any mismatch', () => { + const receipt = compact(entryTemplate.slice(entryTemplate.indexOf('## Step 11.5:'), entryTemplate.indexOf('## Step 12:'))); + expect(receipt).toContain('~/.claude/skills/gstack/bin/gstack-review-read'); + expect(receipt).toMatch(/reject outside-provider or older invocation records/i); + expect(receipt).toMatch(/blocks release preparation/i); + expect(receipt).toContain('**Review records missing or mismatched**'); + for (const stale of ['phase:"core"', 'Step 9.5', 'source:"in-host"', 'status:"clean"']) expect(receipt).not.toContain(stale); }); -test('ship template consolidation: ledger recovery never waives stale content or failed verification', () => { - const gate = compact(controlTemplate + entryTemplate.slice(entryTemplate.indexOf('## Step 16:'), entryTemplate.indexOf('## Step 17:'))); - expect(gate).toContain('Only receipt storage/readback failed | Independently prove unchanged final content, the same command and valid age from the successful run\'s evidence'); - expect(gate).toContain('Only receipt storage/readback failed'); - expect(gate).toContain('Independently prove unchanged final content, the same command and valid age'); - expect(gate).toContain('Cite its exact command, exit, timestamp and log as **ledger unavailable**, never FRESH. Without that proof, use STALE/MISSING'); - expect(gate).toContain('--allow-paths CHANGELOG.md,VERSION,package.json,agents-digest/gstack-AGENTS.md'); - expect(gate).toContain('Docs, TODO edits, new/generated tests and fixes make evidence STALE'); - expect(gate).toContain('Docs, TODO edits, new/generated tests and fixes make evidence STALE'); - expect(gate).toContain("**New, changed or unwaived test failure:** STOP publication. Run Steps 5–15, starting with Step 5's triage"); - expect(gate).toContain('**New, changed or unwaived test failure:** STOP publication'); +test('version recovery and digest generation stay explicit', () => { + const version = compact(entryTemplate.slice(entryTemplate.indexOf('## Step 12:'), entryTemplate.indexOf('## Step 14:'))); + expect(version).toContain('`baseVersion` to `currentVersion`'); + expect(version).toMatch(/an absent fourth component is zero/i); + expect(version).toMatch(/not another automatic bump/i); + expect(version).toContain('ALREADY_BUMPED'); + expect(version).not.toContain('skip if ALREADY_BUMPED'); + expect(version).toContain('`agents-digest/gstack-AGENTS.md`'); + expect(version).toContain('`agentsDigest`'); + expect(version).toContain('`bun scripts/gen-agents-digest.ts`'); + expect(version).toMatch(/before push, verify the committed digest matches generation/i); }); -test('ship has one invocation route and retains each independently bounded allowance', () => { - const entry = compact(entryTemplate); - expect(entry).toContain('Every new invocation repeats Steps 1–16, including both reviews and the docs audit'); - expect(entry).toContain('prepare the release (12–15) → verify frozen content (16)'); - expect(entry).toContain('## Step 16: Verification Gate'); - expect(compact(coverageTemplate)).toContain('Maximum 2 generation passes total per invocation'); - expect(reviewFlow).toContain('do not run a fourth fixing cycle'); - expect(compact(docsTemplate)).toContain('an initial audit plus ONE repair/re-audit'); - expect(entry).toContain('Keep the same attempt counts throughout the invocation'); - const gate = compact(controlTemplate + entryTemplate.slice(entryTemplate.indexOf('## Step 16:'), entryTemplate.indexOf('## Step 17:'))); - expect(gate).toContain('against the snapshot saved before Step 12'); - expect(gate).toContain("Content-preserving commits keep valid evidence"); - expect(gate).toContain('Use its actual Step 5 label/command'); - expect(gate).toContain("--label --expect-cmd ''"); +test('distribution setup asks for unknown targets and never invents credentials', () => { + const distribution = compact(entryTemplate.slice(entryTemplate.indexOf('## Step 2:'), entryTemplate.indexOf('## Step 3:'))); + expect(distribution).toContain('git diff origin/ --diff-filter=A --name-only'); + expect(distribution).toMatch(/ask for unknown targets, registries or access first/i); + expect(distribution).toMatch(/never invent credentials/i); + expect(distribution).toMatch(/include the workflow in tests and review/i); +}); + +test('ambiguous Apple targets and merge conflicts wait for the user', () => { + const apple = compact(entryTemplate.slice(entryTemplate.indexOf('## Step 0.9:'), entryTemplate.indexOf('## Step 1:'))); + expect(apple).toContain('`Package.swift`'); + expect(apple).toMatch(/AskUserQuestion to identify the target and wait before choosing a release path/i); + expect(apple).toMatch(/confirmed app, \*{0,2}stop and read/i); + const merge = compact(entryTemplate.slice(entryTemplate.indexOf('## Step 3:'), entryTemplate.indexOf('{{SECTION:tests}}'))); + expect(merge).toMatch(/ambiguous conflicts, \*{0,2}stop\b/i); + expect(merge).toMatch(/AskUserQuestion for the needed resolution decision/i); + expect(merge).toMatch(/wait for the answer before editing or continuing/i); +}); + +test('missing test suites need a named gap decision rather than a fabricated fresh receipt', () => { + const tests = compact(readTemplate('ship/sections/tests.md.tmpl')); + expect(tests).toContain('A) Add tests (recommended), B) Ship with this named testing gap, or C) Stop'); + expect(tests).toMatch(/reuse an actual prior B answer only for the same scope and content/i); + expect(tests).toMatch(/independent build, eval, review and QA gates still apply/i); + expect(tests).toMatch(/a declared but unavailable suite is a blocker/i); + expect(finalGate).toMatch(/explicit untested-scope approval/i); + expect(finalGate).toMatch(/report the gap, never FRESH/i); +}); + +test('Step 16 stages run in order and freeze inputs through push', () => { + ordered(finalGate, ['### 1. Finish writers and prepare outputs', '### 2. Choose the change route', + '### 3. Resolve documentation freshness', '### 4. Verify the frozen candidate']); + const route = compact(entryTemplate.slice(entryTemplate.indexOf('### 2. Choose the change route'), entryTemplate.indexOf('### 3. Resolve documentation freshness'))); + ordered(route, ['**Behavior, tests or build inputs changed:**', '**Only authored docs or release metadata changed:**', + '**No changes, or the docs-only checks still support the plan:**']); + expect(route).toMatch(/prompts\/templates count as behavior/i); + expect(route).toMatch(/continue to stage 3 without a new code review/i); + expect(finalGate).toMatch(/confirm terminal completion or termination before another writer runs/i); + expect(finalGate).toMatch(/timeout or cancellation acknowledgment alone means stop until confirmed/i); + expect(finalGate).toContain('**Build failed or prerequisite missing**'); + expect(finalGate).toMatch(/never invent a substitute command/i); + expect(finalGate).toMatch(/freeze inputs through verification and push/i); + expect(finalGate).toMatch(/mandatory reviews still run/i); + expect(finalGate).toMatch(/never resample it/i); + expect(finalGate).toMatch(/changed or unknown dependencies require a rerun/i); + expect(finalGate.indexOf('**Reuse a check when its inputs match.**')).toBeGreaterThan(finalGate.indexOf('### 4. Verify the frozen candidate')); + expect(finalGate).toContain("**Check each test lane's receipt as well.**"); +}); + +test('evidence exemptions cover only release metadata and receipts name their command', () => { + expect(finalGate).toContain('--allow-paths CHANGELOG.md,VERSION,package.json,agents-digest/gstack-AGENTS.md'); + expect(finalGate).toMatch(/scripts, dependencies and runtime configuration require live tests/i); + expect(finalGate).toMatch(/uncertain edits cannot be exempted/i); + expect(finalGate).toContain('| FRESH (exit 0) |'); + expect(finalGate).toContain("--label --expect-cmd ''"); + expect(finalGate).toContain('~/.claude/skills/gstack/bin/gstack-wtree'); + expect(finalGate).toContain('git diff '); + expect(finalGate).toContain('snapshot saved before Step 12'); + expect(finalGate).toMatch(/missing snapshots block this comparison, regardless of HEAD equality/i); +}); + +test('docs attempts count at launch and the collector checks ownership before accepting output', () => { + expect(docs).toMatch(/increment before each launch or inline takeover, including failed launches/i); + expect(docs).toMatch(/the request alone is insufficient/i); + expect(docs).toMatch(/an exited child with missing output is stopped, but its audit is blocked/i); + expect(docs).toMatch(/HEAD and index must be unchanged/i); + expect(docs).toContain('`files_updated`'); + expect(docs).toMatch(/protected-file exclusions/i); + expect(docs).toContain('`documentation_section`'); + expect(docs).toMatch(/a failed check or `blocked` result goes to recovery, even with valid JSON/i); + expect(docs).toMatch(/only an actual user exception counts/i); + expect(docs).toMatch(/reports and PRs retain blocked status/i); + ordered(docsTemplate, ['**Subagent prompt:**', '**Parent processing:**']); + const freshness = compact(entryTemplate.slice(entryTemplate.indexOf('### 3. Resolve documentation freshness'), entryTemplate.indexOf('### 4. Verify the frozen candidate'))); + expect(freshness).toContain("Step 14.5's saved values"); + const retry = freshness.slice(freshness.indexOf('**An attempt remains, with changed inputs or an available repair:**'), freshness.indexOf('**Otherwise:**')); + expect(retry.length).toBeGreaterThan(0); + expect(retry).not.toContain('Continue to stage 4'); + expect(retry).toMatch(/restart Step 16 stage 1/i); + expect(freshness).toMatch(/covers the same approved scope and exact content/i); + expect(entryTemplate.indexOf('## Step 14.5:')).toBeLessThan(entryTemplate.indexOf('## Step 17:')); +}); + +test('late verified outputs are committed before push, without tags or other writers after it', () => { + const finish = compact(entryTemplate.slice(entryTemplate.indexOf('### 5. Report, then push'), entryTemplate.indexOf('## Step 17:'))); + ordered(finish, [/commit only approved, verified release changes/i, 'continue to Step 17']); + expect(entry).toMatch(/do not create a Git tag/i); + expect(entry).toContain('encode null/undetermined as -1'); + ordered(pushFlow, ['**Non-fast-forward push:**', '**Authentication, hook or network failure:**']); + expect(pushFlow).toContain('`ALREADY_PUSHED`'); + expect(pushFlow).toMatch(/no documentation writer runs after push/i); +}); + +test('Step 18 resolves open PR state before preparing the exact title Step 19 scans', () => { + const title = compact(entryTemplate.slice(entryTemplate.indexOf('## Step 18:'), entryTemplate.indexOf('{{SECTION:pr-body}}'))); + ordered(title, ['gh pr list --head --state open --json number,title,url', 'Prepare the title from that result']); + expect(title).toContain('glab mr list --source-branch --output json'); + expect(title).toMatch(/a successful empty array means new/i); + expect(title).toMatch(/lookup failure or ambiguous matches \*{0,2}stop\*{0,2} for resolution, never mean no PR/i); + expect(title).toContain('`NEW_TITLE` for Step 19'); + expect(title).toContain('`v : `'); + expect(title).toContain('`v$NEW_VERSION `'); + expect(title).toMatch(/never publish an unprefixed title/i); + expect(prTemplate).toContain(': "${NEW_TITLE:?Restore the saved Step 18 title before scanning}"'); + expect(prTemplate).not.toContain('NEW_TITLE=""'); + expect(compact(prTemplate)).toContain('exit 3 blocks for HIGH'); + const generated = readTemplate('ship/sections/pr-body.md'); + expect(generated).toContain('**Existing open PR/MR:** update'); + expect(generated).toMatch(/do not run the create commands below/i); + expect(generated).not.toContain('no PR exists yet'); +}); + +test('linked spec discovery reads frontmatter as data and claims closure only for completed scope', () => { + const instructions = compact(prTemplate.slice(0, prTemplate.indexOf('The PR/MR body should contain'))); + expect(instructions).toMatch(/read archive frontmatter as data, never shell source/i); + expect(instructions).toMatch(/exact `spec_branch` match/i); + expect(instructions).toContain('`spec_filed_at`'); + expect(instructions).toMatch(/positive integer `spec_issue_number`/i); + expect(instructions).toContain('`## Linked Spec`'); + expect(instructions).toMatch(/continue composing the PR/i); + expect(instructions).toMatch(/only fully completed Step 8 plan scope permits `Closes #N`/i); + expect(instructions).toContain('`Linked to #N`'); + const body = prTemplate.slice(prTemplate.indexOf('The PR/MR body should contain'), prTemplate.indexOf('#### Redaction scan')); + for (const stale of ['CURRENT_BRANCH=', 'SPEC_ARCHIVES=', 'SPEC_FILE=$(grep']) expect(body).not.toContain(stale); + expect(prTemplate).not.toContain('[ -z "$SPEC_FILE" ] && exit'); +}); + +test('comment triage separates absent PRs, empty fetches and unavailable evidence', () => { + const section = compact(readTemplate('ship/sections/greptile.md.tmpl')); + expect(section).toContain('"status":"complete|no_pr|unavailable"'); + expect(section).toMatch(/use `complete` only after a successful fetch, including zero comments/i); + expect(section).toMatch(/`no_pr` only after confirming no PR exists/i); + expect(section).toMatch(/`unavailable` for `gh`\/API errors or incomplete classification/i); + expect(section).toContain('a nonnegative integer total matching the comments array'); + expect(section).toMatch(/an unknown or missing status is unavailable, never an empty successful review/i); + expect(section).toContain('"Greptile: no PR exists"'); + expect(section).toContain('"Greptile: fetched, zero comments"'); + expect(section).toContain('`Greptile triage: UNAVAILABLE (dispatch failed)`'); + expect(section).toMatch(/stop a running child and confirm it stopped before continuing/i); + expect(section).not.toContain('If no PR exists, `gh` fails, the API errors, or there are zero comments'); }); test('ship resolves threshold, branch and installed-asset references without competing instructions', () => { - expect(entryTemplate).toContain("When Step 7 coverage meets its target"); + expect(entryTemplate).toContain('When Step 7 coverage meets its target'); expect(entryTemplate).not.toContain('Test coverage gaps within target threshold'); expect(entryTemplate.indexOf('Save the current branch as ``')).toBeGreaterThan(0); expect(entryTemplate.indexOf('Save the current branch as ``')).toBeLessThan(entryTemplate.indexOf('refs/heads/')); expect(entryTemplate).toContain('~/.claude/skills/gstack/review/TODOS-format.md'); expect(entryTemplate).not.toContain('`.claude/skills/review/TODOS-format.md`'); + expect(entry).toContain('## Step 16: Verification Gate'); const skeleton = readTemplate('ship/SKILL.md'); const row = skeleton.split('\n').find(line => line.startsWith('| exploratory QA before Fix-First'))!; expect(row).toContain('`sections/review-army.md`'); expect(row).not.toContain('below'); }); - -test('ship distinguishes a changed verified tree from failed test-receipt storage', () => { - const gate = compact(controlTemplate + entryTemplate.slice(entryTemplate.indexOf('## Step 16:'), entryTemplate.indexOf('## Step 17:'))); - expect(gate).toContain('declared generation/build commands in project instructions, manifests, build files and CI'); - expect(gate).toContain("Run declared docs/link/generated-file checks"); - expect(gate).toContain('Docs, TODO edits, new/generated tests and fixes make evidence STALE even without a new code review'); - expect(gate).toContain('Docs, TODO edits, new/generated tests and fixes make evidence STALE'); - expect(gate).toContain('| FRESH (exit 0) | Cite the label, exit, timestamp and log. |'); - expect(gate).toContain('Only receipt storage/readback failed'); - expect(gate).toContain('Without that proof, use STALE/MISSING'); - expect(gate).toContain('Docs, TODO edits, new/generated tests and fixes make evidence STALE'); -}); - -test('ship names the approval scope, probe-risk decision, and native-review recovery', () => { - const entry = compact(entryTemplate); - const review = compact(controlTemplate + reviewTemplate); - const adversarial = compact(generateAdversarialStep({ host: 'claude', skillName: 'ship', tmplPath: '', paths: HOST_PATHS.claude })); - expect(entry).toContain('one private Markdown **invocation record**'); - expect(entry).toContain('each approval\'s finding, files and authorized action'); - expect(entry).toContain('Never borrow or replace a token'); - expect(review).toContain('Use AskUserQuestion: stop for repair (recommended), or explicitly accept each named probe\'s concrete risk'); - expect(review).toContain('explicitly accept each named probe\'s concrete risk. Skipping a fix is not risk acceptance or a passing probe'); - expect(nativeFlow).toContain("One recovery retry is allowed only after a concrete prerequisite correction and restored access"); - expect(nativeFlow).toContain('count it in the invocation record before launch'); - expect(control).toContain('Keep the same attempt counts throughout the invocation'); - expect(nativeFlow).toContain('if the recovery fails, ask for repair and remain blocked'); - expect(adversarial).toContain('Outside-provider output cannot replace this pass'); - expect(entry).toContain('Only the final VERSION/CHANGELOG commit gets the release version and co-author trailer. Do not create a Git tag'); - expect(entry).toContain('encode null/undetermined as -1'); -}); - -test('native recovery has its own bounded retry without resetting fixing or documentation limits', () => { - const adversarial = compact(generateAdversarialStep({ host: 'claude', skillName: 'ship', tmplPath: '', paths: HOST_PATHS.claude })); - expect(nativeFlow).toContain('One recovery retry is allowed only after a concrete prerequisite correction and restored access'); - expect(reviewFlow).toContain('do not run a fourth fixing cycle'); - expect(compact(docsTemplate)).toContain('an initial audit plus ONE repair/re-audit'); - expect(nativeFlow).toContain('**Required native review incomplete:** STOP'); - expect(adversarial).toContain('STOP and confirm the native task stopped'); - expect(adversarial).toContain('Capture a fresh PASS_START and persist the new attempt separately'); - expect(nativeFlow).toContain('confirm the native task stopped'); - expect(nativeFlow).toContain('One recovery retry is allowed only after a concrete prerequisite correction and restored access'); - expect(nativeFlow).toContain('count it in the invocation record before launch'); - expect(control).toContain('Keep the same attempt counts throughout the invocation'); - expect(nativeFlow).toContain('Without that correction, or if the recovery fails, ask for repair and remain blocked'); - expect(nativeFlow).toContain('if the recovery fails, ask for repair and remain blocked'); - expect(nativeFlow).toContain('not recovery retries'); - expect(reviewFlow).toContain("Finish the complete review and QA before applying any fix in Step 9.4"); -}); - -test('late verified generated outputs are committed before publication without absorbing user files', () => { - const commit = compact(entryTemplate.slice(entryTemplate.indexOf('## Step 15:'), entryTemplate.indexOf('## Step 16:'))); - const finish = compact(entryTemplate.slice(entryTemplate.indexOf('### 5. Report, then push'), entryTemplate.indexOf('## Step 17:'))); - const gate = compact(controlTemplate + entryTemplate.slice(entryTemplate.indexOf('## Step 16:'), entryTemplate.indexOf('## Step 17:'))); - expect(commit).toContain('Group VERSION + CHANGELOG + TODOS.md after the feature commits'); - expect(finish).toContain('Commit only approved, verified release changes left uncommitted after Step 15'); - expect(finish).toContain('including generated outputs; use its grouping rules and never create an empty commit'); - expect(finish).toContain('Preserve unrelated user files'); - expect(gate).toContain('Content-preserving commits keep valid evidence'); - expect(gate).toContain('If content changes during or after verification, restart at stage 1 and complete all five stages before Step 17'); - const commitPosition = finish.indexOf('Commit only approved, verified'); - const pushPosition = finish.indexOf('continue to Step 17'); - expect(commitPosition).toBeGreaterThanOrEqual(0); - expect(pushPosition).toBeGreaterThan(commitPosition); -}); - -test('missing test suites need a named gap decision rather than a fabricated fresh receipt', () => { - const tests = compact(readTemplate('ship/sections/tests.md.tmpl')); - const gate = compact(controlTemplate + entryTemplate.slice(entryTemplate.indexOf('## Step 16:'), entryTemplate.indexOf('## Step 17:'))); - expect(tests).toContain('If no applicable test suite exists'); - expect(tests).toContain('A) Add tests (recommended), B) Ship with this named testing gap, or C) Stop'); - expect(tests).toContain('Reuse an actual prior B answer only for the same scope and content'); - expect(tests).toContain('declining bootstrap alone is not that approval'); - expect(tests).toContain('Independent build, eval, review and QA gates still apply'); - expect(tests).toContain('A declared but unavailable suite is a blocker, not an absent suite'); - expect(gate).toContain('No test lanes: require Step 5\'s explicit untested-scope approval for final content'); - expect(gate).toContain("or run Steps 5–15, including the no-tests decision, then return to Step 16 stage 1"); - expect(gate).toContain('Report the gap, never FRESH'); -}); - -test('ship template consolidation: remote integration retains all allowances and cannot bypass publication guards', () => { - const push = compact(entryTemplate.slice(entryTemplate.indexOf('## Step 17:'), entryTemplate.indexOf('## Step 18:'))); - const recovery = pushFlow; - expect(push).toContain('**If the push fails, STOP.** No Step 19 or publication claim'); - expect(recovery).toContain('Run Steps 5–16 before returning to Step 17. Never rewrite history'); - expect(control).toContain("Keep the same attempt counts throughout the invocation"); - expect(recovery).toContain("fetch and inspect the remote, then merge under Step 3's conflict rules"); - expect(push).toContain('Never force-push'); - expect(recovery).toContain('repeat Step 16 even if content is unchanged before returning to Step 17'); - expect(recovery).toContain('Never bypass failed guards'); - expect(push).toContain('Only a successful push or verified `ALREADY_PUSHED` proceeds'); - expect(push).toContain('No documentation writer runs after push'); -}); - -test('missing dispatched coverage is persisted and stopped before any zero-fix completion', () => { - const review = compact(controlTemplate + reviewTemplate); - const branches = reviewFlow.slice(reviewFlow.indexOf("### Decide whether to repeat Step 9")); - expect(branches.indexOf('Dispatched reviewer output missing')).toBeGreaterThanOrEqual(0); - expect(branches.indexOf('**Dispatched reviewer output missing:** STOP')).toBeGreaterThanOrEqual(0); - expect(branches.indexOf('**Fixes applied below the cap:**')).toBeGreaterThanOrEqual(0); - expect(branches.indexOf('**No edits in this pass:** Resolve the required-probe gate below. Only after it clears may you continue to Step 10')).toBeGreaterThanOrEqual(0); - expect(branches.indexOf('**Fixes applied below the cap:**')).toBeGreaterThan(branches.indexOf('**Dispatched reviewer output missing:** STOP')); - expect(branches.indexOf('**No edits in this pass:** Resolve the required-probe gate below. Only after it clears may you continue to Step 10')).toBeGreaterThan(branches.indexOf('**Dispatched reviewer output missing:** STOP')); - expect(review).toContain('Missing dispatched output uses `status:"unavailable"`, `completed:false` and `converged:false`'); - expect(review).toContain('Pre-Landing Review: INCOMPLETE'); - expect(branches).toContain('Retain queued fixes'); - expect(branches).toContain("If this pass made edits, resume at the next decision"); - expect(branches).toContain("otherwise run a fresh complete Step 9"); - expect(review).toContain('Undispatched host-unsupported/gated specialists do not block'); - expect(review).toContain("Apply these decisions in order"); - expect(review).toContain('**No edits in this pass:** Resolve the required-probe gate below. Only after it clears may you continue to Step 10'); - expect(review).toContain('This cannot waive missing reviewer output'); - expect(review).toContain('`STATUS`: `unavailable` for missing dispatched reviewer output'); - const settlement = review.slice(review.indexOf('## Step 9.4:'), review.indexOf('1. **Classify')); - expect(settlement).toContain('every dispatched reader/writer\'s handle. Wait for return or confirm termination'); - expect(settlement).toContain('log incomplete through items 5–6 and STOP without edits'); - expect(settlement).toContain('After terminal failure, independent evidence may support fixes'); -}); - -test('external-comment fixes refresh tests and mandatory review without repeating prior decisions', () => { - const section = compact(readTemplate('ship/sections/greptile.md.tmpl')); - const finish = section.slice(section.indexOf('**After triage:**')); - expect(section).toContain('queue the approved fix without editing here'); - expect(finish).toContain('If fixes were approved, save their approvals and comment references'); - expect(reviewFlow).toContain("Finish the complete review and QA before applying any fix in Step 9.4"); - expect(commentsFlow).toContain("Run Step 9's full review/fix loop, then return here"); - const repair = reviewFlow; - expect(repair).toContain('**Fixes applied below the cap:** Insert Step 5, affected Steps 6–8 and all of Step 9 before the pending Step 10 in the work list'); - expect(repair).toContain('**No edits in this pass:** Resolve the required-probe gate below. Only after it clears may you continue to Step 10'); - expect(finish).not.toContain('before continuing to Step 11'); - expect(finish).toContain('Finish the saved replies without asking again about completed fixes'); - expect(finish).toContain('With no queued fixes, continue to Step 11'); -}); - -test('triage distinguishes absent PRs, successful empty fetches and unavailable evidence', () => { - const section = compact(readTemplate('ship/sections/greptile.md.tmpl')); - expect(section).toContain('"status":"complete|no_pr|unavailable"'); - expect(section).toContain('Use `complete` only after a successful fetch, including zero comments'); - expect(section).toContain('`no_pr` only after confirming no PR exists'); - expect(section).toContain('`unavailable` for `gh`/API errors or incomplete classification'); - expect(section).toContain('a nonnegative integer total matching the comments array'); - expect(section).toContain('An unknown or missing status is unavailable, never an empty successful review'); - expect(section).toContain('"Greptile: no PR exists"'); - expect(section).toContain('"Greptile: fetched, zero comments"'); - expect(section).toContain('Include `Greptile triage: UNAVAILABLE (dispatch failed)` and the actual reason'); - expect(section).toContain('Stop a running child and confirm it stopped before continuing'); - expect(section).not.toContain('If no PR exists, `gh` fails, the API errors, or there are zero comments'); -}); - -test('late-change routing compares a saved reviewed snapshot without waiving probe outcomes', () => { - const receipt = compact(entryTemplate.slice(entryTemplate.indexOf('## Step 11.5:'), entryTemplate.indexOf('## Step 12:'))); - expect(receipt).toContain('~/.claude/skills/gstack/bin/gstack-review-read'); - expect(receipt).toContain("Match each to its saved handle, original token and source"); - expect(receipt).toContain('(`skill:"review"`, `via:"ship"`)'); - expect(receipt).toContain('Step 11 native record (`skill:"adversarial-review"`)'); - expect(receipt).toContain("reject outside-provider or older invocation records"); - expect(receipt).toContain("Require the native record's `review_binding.state` to be `verified`"); - expect(receipt).toContain("All three snapshots must match: its `wtree`, Step 9.4's `review_binding.start_wtree` and `review_binding.end_wtree`"); - expect(receipt).toContain("A mismatch or missing record/field blocks release preparation: report **Review records missing or mismatched**"); - expect(receipt).toContain('Never attach new tokens to old work'); - expect(receipt).toContain("Keep Step 9.4's incomplete flags and the user's exception"); - expect(receipt).toContain("Matching content does not mean the failed or unrun probes passed"); - expect(receipt).toContain("A named probe-risk exception may leave Step 9.4's root `wtree` absent; item 2 still compares its start/end snapshots"); - expect(receipt).not.toContain('phase:"core"'); - expect(receipt).not.toContain('Step 9.5'); - expect(receipt).not.toContain('source:"in-host"'); - expect(receipt).not.toContain('status:"clean"'); - const gate = compact(controlTemplate + entryTemplate.slice(entryTemplate.indexOf('## Step 16:'), entryTemplate.indexOf('## Step 17:'))); - expect(gate).toContain('~/.claude/skills/gstack/bin/gstack-wtree'); - expect(gate).toContain('git diff '); - expect(gate).toContain('against the snapshot saved before Step 12'); - expect(gate).toContain('Missing snapshots block this comparison'); - expect(gate).toContain('Missing snapshots block this comparison, regardless of HEAD equality'); - expect(gate.indexOf('**Reuse a check when its inputs match.**')).toBeGreaterThan(gate.indexOf('### 4. Verify the frozen candidate')); - expect(gate).toContain('**Check each test lane\'s receipt as well.**'); -}); - -test.each(ALL_HOST_CONFIGS.map(({ name }) => name))('%s: late adversarial fixes have a bounded return path and preserve approvals', host => { - const ctx = { host, skillName: 'ship', tmplPath: '', paths: HOST_PATHS[host] }; - const text = generateAdversarialStep(ctx); - const finishPosition = text.indexOf('### Finish the adversarial phase'); - expect(finishPosition).toBeGreaterThanOrEqual(0); - const finish = compact(text.slice(finishPosition)); - const outcomes = [ - '**Required native review incomplete:**', - '**Fixes queued after native completion:**', - '**Native complete with no queued fixes:**', - ].map(outcome => finish.indexOf(outcome)); - expect(outcomes.every(position => position >= 0)).toBe(true); - expect(outcomes).toEqual([...outcomes].sort((a, b) => a - b)); - expect(text).toContain('queued for the parent; do not edit during Step 11'); - expect(text).toContain('If A: queue the approved findings without editing here'); - expect(reviewFlow).toContain("Every repeat starts before the checklist read and captures a fresh REVIEW_START"); - expect(reviewFlow).toContain("Finish the complete review and QA before applying any fix in Step 9.4"); - expect(control).toContain('Keep the same attempt counts throughout the invocation'); - expect(finish).toContain('Insert Steps 9, 10 and 11 before the pending Step 11.5 in the work list. Step 9 completes full review before fixes'); - expect(compact(entryTemplate)).toContain('each approval\'s finding, files and authorized action'); - expect(reviewFlow).toContain('do not run a fourth fixing cycle'); - expect(reviewFlow).toContain('**Third fixing cycle reached (`CYCLES >= 3`):** STOP and report recurring findings with `converged:false`; do not run a fourth fixing cycle'); - expect(reviewFlow).toContain('**Fixes applied below the cap:** Insert Step 5, affected Steps 6–8 and all of Step 9 before the pending Step 10 in the work list'); - expect(finish).toContain('**Native complete with no queued fixes:**'); - expect(finish).toContain('**Native complete with no queued fixes:** Finish the memory updates below, then continue to Step 11.5'); - expect(text).toContain('retain the acknowledged findings and failed gate'); - expect(text).toContain('do not report a clean review'); - expect(finish).toContain('**Required native review incomplete:** STOP'); - expect(finish).toContain('Optional outside failures retain their own incomplete records'); - expect(compact(text)).toContain('native completion never credits outside coverage'); - const standalone = generateAdversarialStep({ ...ctx, skillName: 'review' }); - expect(standalone).not.toContain('Before Step 12:'); - expect(standalone).not.toContain('### Finish the adversarial phase'); - expect(standalone).toContain("queue the findings and this approval for Step 5's Fix-First handling"); - expect(standalone).toContain('After edits, the full re-review repeats this same structured invocation and diff scope'); - expect(standalone).toContain('do not start an inner repair loop'); -}); - -test('existing release levels have an explicit recovery rule, not implicit rebump approval', () => { - const root = entryTemplate; - const version = compact(root.slice(root.indexOf('## Step 12:'), root.indexOf('## Step 14:'))); - expect(version).toContain('use the first changed component from `baseVersion` to `currentVersion` (major/minor/patch/micro'); - expect(version).toContain('an absent fourth component is zero'); - expect(version).toContain('Continue at item 3, not another automatic bump'); - expect(version).toContain('Only approval changes the existing version'); -}); - -test('distribution setup asks for unknown targets and cannot release before review', () => { - const root = entryTemplate; - const distribution = root.slice(root.indexOf('## Step 2:'), root.indexOf('## Step 3:')); - expect(distribution).toContain('git diff origin/ --diff-filter=A --name-only'); - expect(distribution).toContain('a new `package.json` or `Cargo.toml` alone does not establish a publishable'); - expect(distribution).toContain('Ask for unknown targets, registries or access first'); - expect(distribution).toContain('never invent credentials'); - expect(distribution).toContain('include the workflow in tests and review'); - expect(distribution).toContain('Do not publish a release during `/ship`'); -}); - -test('ship entry decisions identify Swift app products and wait on ambiguous merge resolutions', () => { - const apple = compact(entryTemplate.slice(entryTemplate.indexOf('## Step 0.9:'), entryTemplate.indexOf('## Step 1:'))); - expect(apple).toContain('If the ask is App Store/TestFlight distribution'); - expect(apple).toContain("Read `Package.swift` and its entrypoint"); - expect(apple).toContain('distinguish an app from a library/CLI'); - expect(apple).toContain('If unclear, use AskUserQuestion'); - expect(apple).toContain('AskUserQuestion to identify the target and wait before choosing a release path'); - expect(apple).toContain('For a confirmed app, **STOP and Read'); - const merge = compact(entryTemplate.slice(entryTemplate.indexOf('## Step 3:'), entryTemplate.indexOf('{{SECTION:tests}}'))); - expect(merge).toContain('Try to auto-resolve if they are simple'); - expect(merge).toContain('For complex or ambiguous conflicts, **STOP**, show the conflicting choices'); - expect(merge).toContain('use AskUserQuestion for the needed resolution decision'); - expect(merge).toContain('wait for the answer before editing or continuing'); -}); - -test('ship final preparation discovers declared commands and verifies the versioned digest output', () => { - const gate = compact(controlTemplate + entryTemplate.slice(entryTemplate.indexOf('## Step 16:'), entryTemplate.indexOf('## Step 17:'))); - expect(gate).toContain('declared generation/build commands in project instructions, manifests, build files and CI'); - expect(gate).toContain('Run them and save results'); - expect(gate).toContain('If none exists, record not applicable and the inspected sources'); - expect(gate).toContain('A missing prerequisite or failed build stops shipping'); - const version = compact(entryTemplate.slice(entryTemplate.indexOf('## Step 12:'), entryTemplate.indexOf('{{SECTION:changelog}}'))); - expect(version).toContain('only when it and committed `agents-digest/gstack-AGENTS.md` exist'); - expect(version).toContain('If `agentsDigest` is false, run `bun scripts/gen-agents-digest.ts` and stage the digest with the bump'); - expect(version).toContain("Before push, verify the committed digest matches generation for the selected VERSION"); -}); - -test('ship publication metadata resolves open state before composing a title', () => { - const title = compact(entryTemplate.slice(entryTemplate.indexOf('## Step 18:'), entryTemplate.indexOf('{{SECTION:pr-body}}'))); - const query = title.indexOf('gh pr list --head --state open --json number,title,url'); - const prepare = title.indexOf('Prepare the title from that result'); - expect(query).toBeGreaterThanOrEqual(0); - expect(prepare).toBeGreaterThan(query); - expect(title).toContain('glab mr list --source-branch --output json` (defaults to open)'); - expect(title).toContain('A successful empty array means new'); - expect(title).toContain('one match supplies the existing title/identity'); - expect(title).toContain('Lookup failure or ambiguous matches **STOP** for resolution, never mean no PR'); - expect(title).toContain("Save the result for Step 19's recheck"); - expect(title).toContain('Save the result as `NEW_TITLE` for Step 19'); - expect(title).toContain('start with `v$NEW_VERSION `; never publish an unprefixed title'); -}); - -test('ship explains receipts and genuine review tokens before selecting current invocation records', () => { - const state = compact(entryTemplate.slice(entryTemplate.indexOf('### Keep state'), entryTemplate.indexOf('{{SECTION_INDEX:ship}}'))); - expect(state).toContain('A **receipt** is saved evidence of a check\'s command, result and consumed content'); - expect(state).toContain('**start token** is the opaque value returned by `gstack-review-log --start` before it reads the diff'); - expect(state).toContain('`REVIEW_START` for Step 9, a separate `PASS_START` for each Step 11 attempt, and `DESIGN_START` for design'); - expect(state).toContain('Finish each pass with its original token'); - expect(state).toContain('`gstack-wtree` prints a Git tree hash covering tracked and non-ignored untracked files'); -}); - -test('ship documentation-only plan refresh preserves the original audit and routes unsupported classifications back to its gates', () => { - const route = compact(entryTemplate.slice(entryTemplate.indexOf('### 2. Choose the change route'), entryTemplate.indexOf('### 3. Resolve documentation freshness'))); - expect(route).toContain("Keep Step 8's original child report and counts"); - expect(route).toContain("Recheck affected plan items using their recorded verification"); - expect(route).toContain("append current evidence to the invocation record"); - expect(route).toContain("run Step 8's audit and decision gates only, then return to Step 16 stage 1. Never edit the child's counts yourself"); - expect(route).toContain("**No changes, or the docs-only checks still support the plan:** Continue to stage 3 without a new code review"); -}); - -test('ship recovery map preserves ordered review, build and push transitions without new allowances', () => { - const gate = compact(controlTemplate + entryTemplate.slice(entryTemplate.indexOf('## Step 16:'), entryTemplate.indexOf('## Step 17:'))); - const rows = ['**Non-fast-forward push:**', '**Authentication, hook or network failure:**'].map(outcome => pushFlow.indexOf(outcome)); - expect(rows.every(position => position >= 0)).toBe(true); - expect(rows).toEqual([...rows].sort((a, b) => a - b)); - expect(reviewFlow).toContain("Every repeat starts before the checklist read and captures a fresh REVIEW_START. Finish the complete review and QA before applying any fix in Step 9.4"); - expect(reviewFlow).toContain('Insert Step 5, affected Steps 6–8 and all of Step 9 before the pending Step 10 in the work list'); - expect(commentsFlow).toContain("Run Step 9's full review/fix loop, then return here"); - expect(gate).toContain('Insert `5–11.5 → 12–14 → 16` before the pending Step 17'); - expect(gate).toContain('Insert `5–11.5 → 12–14 → 16` before the pending Step 17, then stop this step. This repair excludes Step 14.5'); - expect(gate).toContain('A missing prerequisite or failed build stops shipping: report **Build failed or prerequisite missing**, with the command, error and needed repair'); - expect(gate).toContain('Repair the prerequisite or build, then repeat stage 1'); - expect(gate).toContain("After it passes, continue to stage 2; treat any content repair as a behavioral change there"); - expect(gate).toContain('Never invent a substitute command'); - expect(gate).toContain('A `package.json` version-only edit can qualify; scripts, dependencies and runtime configuration require live tests'); - expect(gate).toContain('Uncertain edits cannot be exempted'); - const commits = compact(entryTemplate.slice(entryTemplate.indexOf('## Step 15:'), entryTemplate.indexOf('## Step 16:'))); - expect(commits).toContain('Only the final VERSION/CHANGELOG commit gets the release version and co-author trailer'); -}); - -test('ship r22 routes each late change to one restart point before final checks', () => { - const route = compact(entryTemplate.slice(entryTemplate.indexOf('### 2. Choose the change route'), entryTemplate.indexOf('### 3. Resolve documentation freshness'))); - const cases = ['**Behavior, tests or build inputs changed:**', - '**Only authored docs or release metadata changed:**', '**No changes, or the docs-only checks still support the plan:**']; - const positions = cases.map(label => route.indexOf(label)); - expect(route).toContain("Classify the comparison in this order"); - expect(positions.every(position => position >= 0)).toBe(true); - expect(positions).toEqual([...positions].sort((a, b) => a - b)); - const behavior = route.slice(positions[0], positions[1]); - expect(behavior).toContain('Insert `5–11.5 → 12–14 → 16` before the pending Step 17'); - expect(behavior).toContain('Insert `5–11.5 → 12–14 → 16` before the pending Step 17, then stop this step. This repair excludes Step 14.5'); - expect(behavior).toContain('This repair excludes Step 14.5 because the rebuild can change generated docs. Step 16 restarts at stage 1'); - expect(behavior).toContain('rebuild and compare again before stage 3 decides documentation freshness'); - const docs = route.slice(positions[1], positions[2]); - expect(docs).toContain("run Step 8's audit and decision gates only, then return to Step 16 stage 1"); - expect(docs).toContain('Never edit the child\'s counts'); - expect(route.slice(positions[2])).toContain("**No changes, or the docs-only checks still support the plan:** Continue to stage 3 without a new code review"); -}); - -test('ship r22 re-audits return through commit and regeneration rather than bypassing freshness', () => { - const docs = compact(entryTemplate.slice(entryTemplate.indexOf('### 3. Resolve documentation freshness'), entryTemplate.indexOf('### 4. Verify the frozen candidate'))); - expect(docs).toContain("Compare the base and hashes of the selected release paths, generated outputs and docs/templates with Step 14.5\'s saved values"); - expect(docs).toContain("accepted audit matches all inputs | Continue to stage 4"); - const retryStart = docs.indexOf('**An attempt remains, with changed inputs or an available repair:**'); - const retryEnd = docs.indexOf('**Otherwise:**'); - expect(retryStart).toBeGreaterThanOrEqual(0); - expect(retryEnd).toBeGreaterThan(retryStart); - const retry = docs.slice(retryStart, retryEnd); - expect(retry).toContain('Validate the outcome before Step 15'); - expect(compact(docsTemplate)).toContain('Only an actual user exception counts'); - expect(compact(docsTemplate)).toContain('Reconcile those before proceeding'); - expect(retry).toContain("Validate the outcome before Step 15, then restart Step 16 stage 1 to regenerate and compare again"); - expect(retry).not.toContain('Continue to stage 4'); - expect(finalGate).toContain("STOP unless the user accepts the specific named documentation risk and all unwaivable gates clear"); - expect(docs).toContain("retain `Documentation: blocked`, its reason and incomplete scope"); - expect(docs).toContain("covers the same approved scope and exact content"); - expect(docs).toContain('Never run a third audit'); -}); - -test('ship plan audit resolves scope drift before learnings and stops on an unverified N', () => { - const section = readTemplate('ship/sections/plan-completion.md'); - expect(section.indexOf('## Step 8.1:')).toBeLessThan(section.indexOf('## Step 8.2:')); - expect(section.indexOf('## Step 8.2:')).toBeLessThan(section.indexOf('## Prior Learnings')); - expect(section).toContain('N) Not done — block ship and report the item as NOT DONE; do not offer a second deferral choice'); - expect(section).toContain('Any N: STOP'); - expect(section).not.toContain('re-enter the priority-1 gate'); -}); - -test('outside challenge and documentation reruns preserve their actual blocking owners', () => { - const adversarial = readTemplate('ship/sections/adversarial.md'); - expect(adversarial).toContain('An unavailable outside challenge does not block shipping by itself'); - expect(adversarial).toContain('structured P1 and non-convergence gates still apply'); - expect(adversarial).toContain("Returning here never resets Step 9's three-cycle fix limit"); - const standaloneReview = readTemplate('review/sections/adversarial.md'); - expect(standaloneReview).toContain('supported findings still enter Step 5 Fix-First'); - expect(standaloneReview).not.toContain('supported findings still enter Step 11'); - const docs = readTemplate('ship/sections/pr-body.md'); - expect(docs).toContain('**Existing open PR/MR:** update'); - expect(docs).toContain('do not run the create commands below'); - expect(docs).not.toContain('no PR exists yet'); - expect(entryTemplate.indexOf('## Step 14.5:')).toBeLessThan(entryTemplate.indexOf('## Step 17:')); -}); From f8043bd8fd2c5cf213852f6db85e2520b3e487c8 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 01:39:06 +0000 Subject: [PATCH 13/13] test(W7c): qa-exploratory-callers keeps phase order and authority, drops duplicated prose W7c (t04 F8, F16). One test file; only the generated-parent-path and runner-prompt pins change. The validator-backed caller tests (observer, checkpoint causality, read-only diff grammar, write watcher, deadline authority) are unchanged. Removed assertion groups (invariant -> remaining protection): - Parent-path loop and preparation pins duplicated from qa-probe-gates (Reads in order, "Reuse Step 4's surfaces ...", "Finish missing methods before charters", isolation preflight, shared preparation order, the six plan-execution sentences, the three shared loop contracts) -> owned by qa-probe-gates (free, every host, with semantic mutation controls). - Parent prose ("Templates cannot replace them", "Only the parent runs report-only discovery", "Read agent/user updates and await results ...", report-clean and not-run list duplicates of qa-caller-authority) -> caller observer attribution tests in this file (free) and gate E2Es review-exploratory-small-cli, ship-exploratory-small-cli (gate); report gates stay in qa-caller-authority (free, every host). - Shared-loop duplicates (preserve key/value, Q supplies observed, write the report, return to step 2, pass requires) -> qa-probe-gates / qa-caller-authority; kept the observationCommand/observed field definitions and identity-hash rule here. - Runner harness prompt sentences (t04 F16: receipt-interface prose, appendSystemPrompt rules, scheduling prompt) -> reduced to one anchor per rule; `at most ${maxTurns} assistant turns`, maxTurns/timeout/reserve values, `(probe-...)`, `reports/review.md`, and every `not.toMatch` scenario-answer ban stay. validateCallerEvidence remains the behavioral oracle. Kept as case-insensitive meaning checks: phase order 1-4 per caller, review readiness follows report-only rules and never installs/imports cookies/bootstraps, smoke stop at 5 minutes or 12 probes, G enforces the deadline, never reset D/bypass G, plan checks remain required past the smoke budget, /review incomplete and /ship blocked unless the user accepts the named risk, repeat step 3 for affected checks, harness keeps parent decision gates, turn limit never authorizes skipping work, never group probes, Hard deadline UTC rule, HANDOFF.md read before completed records, no checkpoint for deferred ideas. Cosmetic pins dropped: AND / Never capitals. Mutation log: .context/mutation-qa-exploratory-callers.md (15 mutations, 0 unexpected: 2 reorders, 2 markers, 6 safety red; 5 rewordings green). --- test/qa-exploratory-callers.test.ts | 87 ++++++++++------------------- 1 file changed, 28 insertions(+), 59 deletions(-) diff --git a/test/qa-exploratory-callers.test.ts b/test/qa-exploratory-callers.test.ts index ca33aa284..b1ed75eb3 100644 --- a/test/qa-exploratory-callers.test.ts +++ b/test/qa-exploratory-callers.test.ts @@ -649,13 +649,9 @@ describe('generated actual parent paths', () => { expect(positions).toEqual([...positions].sort((a, b) => a - b)); const load = skillName === 'review' ? generateQAReviewPreflight(ctx) : parent.slice(positions[0], positions[1]); if (skillName === 'review') { - const charter = parent.slice(positions[0], positions[1]).replace(/\s+/g, ' '); - expect(charter).toContain("Reuse Step 4's surfaces and completed Reads"); - expect(charter).toContain('Finish missing methods before charters'); - expect(charter).toContain('complete the shared isolation/permission preflight before setup'); const readiness = parent.slice(positions[1], positions[2]).replace(/\s+/g, ' '); - expect(readiness).toContain('Read QA\'s `sections/browser-setup.md` and follow its report-only rules'); - expect(readiness).toContain('Never install, import cookies or bootstrap tests'); + expect(readiness).toMatch(/Read QA's `sections\/browser-setup\.md` and follow its report-only rules/i); + expect(readiness).toMatch(/never install, import cookies or bootstrap tests/i); expect(load).not.toContain('sections/browser-setup.md'); } expect(load).toContain('{{QA_RESOURCE:exploratory}}'); @@ -664,50 +660,26 @@ describe('generated actual parent paths', () => { const resource = generateQAResource(ctx, ['exploratory']); expect(resource).toContain(`installed /${skillName} SKILL.md's directory`); expect(resource).toContain('`../qa/sections/exploratory.md`'); - const shared = generateQAExploratory({ ...ctx, skillName: 'qa' }); - const preparation = ['1. Read `sections/scope.md`', 'in full and select the surfaces', - 'Read `sections/system-functional.md` in full.', 'Read `sections/qa-patterns.md` in full.', - 'Write a **charter**', '1. First demonstrate success'].map(marker => shared.indexOf(marker)); - expect(preparation.every(position => position >= 0)).toBe(true); - expect(preparation).toEqual([...preparation].sort((a, b) => a - b)); - expect(load).toContain('Templates cannot replace them'); const flat = parent.replace(/\s+/g, ' '); - expect(flat).toContain('Only the parent runs report-only discovery'); - expect(flat).toContain('Follow the shared Probe loop for smoke checks and replays until the smoke limit'); - expect(flat).toContain('Then run required plan checks and revalidation, even after smoke expires'); - expect(flat).toContain('using the same procedure but no smoke guard; never reset the clock'); - expect(flat).toContain("Use finite command timeouts, capped at the caller\'s remaining time if it has a deadline"); - expect(flat).toContain('When the caller\'s deadline expires, mark unfinished checks not-run'); - for (const contract of ['First demonstrate success: output AND durable effects', - 'Wait for successful checkpoint publication before dispatch', - 'Replay the exact failing command/request from the same initial fixture state']) { - expect(shared).toContain(contract); - } - expect(flat).toContain('Read agent/user updates and await results without batching them with reporting/logging'); - expect(flat).toContain('Re-review changed or uncertain coverage and repeat step 3 for affected checks'); - expect(flat).toContain('Report clean/completed only when all required checks pass on current inputs'); - expect(flat).toContain('List failed, blocked, inconclusive and not-run checks'); + expect(flat).toMatch(/repeat step 3 for affected checks/i); } }); test('authored shared loop preserves complete safe observations and re-enters checkpoints after input changes', () => { const text = generateQAExploratory({ skillName: 'qa', tmplPath: 'qa/SKILL.md.tmpl', host: 'claude', paths: HOST_PATHS.claude }).replace(/\s+/g, ' '); - for (const contract of ["last completed probe's full outer command", 'Preserve every safe program-JSON key/value', 'identity hash unchanged', 'Q supplies observed; never transcribe it', 'write the report, not a checkpoint', 'return to step 2 for each affected revalidation', 'Pass requires all required current-input contracts to pass with no required remainder']) { - expect(text).toContain(contract); - } - expect(text).toContain("observationCommand: last completed probe's full outer command, including guard"); - expect(text).toContain('observed: its exact decoded child JSON (no wrapper/extra keys)'); - expect(text).toContain('or its full non-JSON text'); + expect(text).toMatch(/observationCommand: last completed probe's full outer command/i); + expect(text).toMatch(/observed: its exact decoded child JSON/i); + expect(text).toMatch(/identity hash unchanged/i); expect(text).not.toContain('nest unchanged child JSON'); }); test('the shared smoke has explicit limits without waiving required plan checks', () => { const body = fs.readFileSync(path.join(import.meta.dir, '../qa/sections/exploratory.md'), 'utf8').replace(/\s+/g, ' '); - expect(body).toContain('Stop after 5 minutes or 12 probes, whichever comes first'); + expect(body).toMatch(/stop after 5 minutes or 12 probes/i); expect(body).toContain('G enforces the deadline'); - expect(body).toContain('Never reset D/bypass G'); - expect(body).toContain('Explicit plan checks and revalidation remain required beyond this smoke budget'); + expect(body).toMatch(/never reset D\/bypass G/i); + expect(body).toMatch(/plan checks and revalidation remain required beyond this smoke budget/i); expect(body).toContain('leaves /review incomplete'); - expect(body).toContain('/ship blocked unless the user explicitly accepts that named risk'); + expect(body).toMatch(/\/ship blocked unless the user explicitly accepts that named risk/i); }); test('excerpt extraction fails loudly instead of producing an empty passing fixture', () => { @@ -1152,16 +1124,15 @@ describe('real caller-specific native fixture and capture boundary', () => { try { fixture.config = path.join(fixture.root, 'callback-config'); await runQaCaller(fixture, 'free-receipt-interface', async options => { - expect(options.prompt).toContain('status is the overall supplied phase gate, not whether some probes passed'); - expect(options.prompt).toContain('Pass requires no remaining required contracts or gates'); - expect(options.prompt).toContain('Optional unavailable providers and later stages outside this excerpt are not required remainder'); - expect(options.prompt).toContain('exact id values from the captured child JSON (probe-...)'); - expect(options.prompt).toContain("never the helper's three-digit capture IDs or QA_EVIDENCE.id"); - expect(options.prompt).toContain('Capture IDs select stored observations for checkpoint/materialize'); + expect(options.prompt).toMatch(/overall supplied phase gate/i); + expect(options.prompt).toMatch(/no remaining required contracts or gates/i); + expect(options.prompt).toMatch(/not required remainder/i); + expect(options.prompt).toContain('(probe-...)'); + expect(options.prompt).toMatch(/never the helper's three-digit capture IDs/i); expect(options.prompt).not.toMatch(/bun scripts\/probe\.ts \d|exploration-[0-9]{3}|snapshot.*must|plan:nine|adverse/i); const readme = fs.readFileSync(path.join(fixture.cwd, 'README.md'), 'utf8'); - expect(readme).toContain('Every diagnostic receipt field is synthetic, nonsecret evidence'); - expect(readme).toContain('snapshot identifies the owned source and fixture inputs'); + expect(readme).toMatch(/synthetic, nonsecret/i); + expect(readme).toMatch(/snapshot identifies the owned source/i); expect(readme).not.toMatch(/checkpoint|exploration-NNN|hypothesis|nextCommand/); return { exitReason: 'success', transcript: [] } as unknown as SkillTestResult; }); @@ -1361,20 +1332,18 @@ describe('real caller-specific native fixture and capture boundary', () => { expect(options.timeout).toBe(300_000); expect(options.completionReserveMs).toBe(75_000); expect(options.appendSystemPrompt).toContain(`at most ${options.maxTurns} assistant turns`); - expect(options.appendSystemPrompt).toContain('independent source Reads and read-only discovery together as separate native tool calls'); - expect(options.appendSystemPrompt).toContain('After required clock and approval prerequisites settle'); - expect(options.appendSystemPrompt).toContain('The completion reserve is for required verification, affected-input revalidation and artifacts, not an earlier deadline'); - expect(options.appendSystemPrompt).toContain("Use each native probe's snapshot to distinguish current from superseded evidence"); - expect(options.appendSystemPrompt).toContain('Never group diagnostic probes, checkpoint publication with its next probe'); - expect(options.appendSystemPrompt).toContain('the turn limit does not authorize skipping work or reporting incomplete work as passed'); + for (const rule of [/separate native tool calls/i, /approval prerequisites settle/i, + /completion reserve is for required verification/i, /superseded evidence/i, + /never group diagnostic probes/i, /turn limit does not authorize skipping work/i]) { + expect(options.appendSystemPrompt).toMatch(rule); + } expect(options.appendSystemPrompt).not.toMatch(/bun scripts\/probe\.ts \d|invalid input|highest.risk/i); - expect(options.prompt).toContain('Keep normal parent decision gates.'); - expect(options.prompt).toContain("use the section clock's Hard deadline UTC, never its Runner entry UTC, reserve-start time or a clock-read time"); - expect(options.prompt).toContain('Before the completion report and each completed:true review record, read HANDOFF.md'); - expect(options.prompt).toContain('a later handoff read cannot validate an earlier completion'); - expect(options.prompt).toContain('If you defer an optional idea or stop exploration, do not publish a checkpoint for it'); - expect(options.prompt).toContain('An unused checkpoint requires an actual authenticated expired-capture result; nearing the deadline or choosing to stop is not enough'); - expect(options.prompt).toContain('Write the phase report to reports/review.md.'); + expect(options.prompt).toMatch(/keep normal parent decision gates/i); + expect(options.prompt).toMatch(/Hard deadline UTC, never its Runner entry UTC/i); + expect(options.prompt).toMatch(/completed:true review record, read HANDOFF\.md/i); + expect(options.prompt).toMatch(/do not publish a checkpoint for it/i); + expect(options.prompt).toMatch(/authenticated expired-capture result/i); + expect(options.prompt).toContain('reports/review.md'); return sentinel; }); expect(calls).toBe(1);