Merge remote-tracking branch 'origin/garrytan/opus55-wip-F1' into garrytan/opus-5-5-prompt-audit

This commit is contained in:
garrytan committed 2026-10-03 01:43:14 +00:00
commit 6d7935eae8
13 files changed
+882 -1684

No files matched your search

+17 -21
View File
@@ -8,12 +8,11 @@
* only the STOP-Read pointer). After ANY non-zero `gh pr merge`, the skill
* must query authoritative PR state via
* GraphQL (including queue membership) and
* branch on the result instead of blindly retrying `gh pr merge` (cli/cli#3442,
* cli/cli#13380).
* branch on the result instead of blindly retrying `gh pr merge`.
*
* Static invariants pin:
* - §4a-postfail header present
* - Universal invariant text + reference to upstream gh bugs
* - Query-before-retry invariant after any non-zero merge exit
* - All three state branches (MERGED, OPEN, CLOSED) named explicitly
* - MERGED branch: capture merge SHA via mergeCommit.oid
* - MERGED branch: non-destructive worktree cleanup with uncommitted-work guard
@@ -52,12 +51,9 @@ describe("PR #1620 §4a-postfail in land-and-deploy template", () => {
expect(postfail).toBeLessThan(queue);
});
test("Universal invariant + upstream gh bug references", () => {
const body = readTmpl();
expect(body).toMatch(/Universal invariant/);
expect(body).toMatch(/non-zero exit from `gh pr merge`/);
expect(body).toMatch(/cli\/cli#3442/);
expect(body).toMatch(/cli\/cli#13380/);
test("any non-zero merge exit queries authoritative state before retrying", () => {
const body = readTmpl().replace(/\s+/g, " ");
expect(body).toMatch(/non-zero exit from `gh pr merge`, query authoritative PR state before retrying/i);
});
test("Authoritative state query includes auto request and queue membership", () => {
@@ -82,9 +78,9 @@ describe("PR #1620 §4a-postfail in land-and-deploy template", () => {
test("MERGED worktree cleanup is non-destructive (uncommitted-work guard)", () => {
const body = readTmpl();
expect(body).toMatch(/uncommitted work/);
expect(body).toMatch(/STOP worktree cleanup without removing/);
expect(body).toMatch(/Do NOT use `--force`/);
expect(body).toMatch(/Do NOT remove the user's primary working tree/);
expect(body).toMatch(/stop worktree cleanup without removing/i);
expect(body).toMatch(/do not use `--force`/i);
expect(body).toMatch(/do not remove the user's primary working tree/i);
});
test("MERGED branch continues to §4b CI auto-deploy detection", () => {
@@ -114,10 +110,10 @@ describe("PR #1620 §4a-postfail in land-and-deploy template", () => {
test("MERGED branch reconciliation distinguishes branch-absent from check-failed", () => {
const body = readTmpl();
// exit 0 + empty output = already clean (idempotent re-runs)...
expect(body).toMatch(/already been cleaned up/);
expect(body).toMatch(/already been cleaned up/i);
// ...non-zero exit = unknown state, never read as a clean branch.
expect(body).toMatch(/Couldn't verify remote branch state/);
expect(body).toMatch(/never read a failed check as a clean branch/);
expect(body).toMatch(/couldn't verify remote branch state/i);
expect(body).toMatch(/never read a failed check as a clean branch/i);
});
test("OPEN branch checks autoMergeRequest before treating as failure", () => {
@@ -128,15 +124,15 @@ describe("PR #1620 §4a-postfail in land-and-deploy template", () => {
test("CLOSED branch STOPs", () => {
const body = readTmpl();
expect(body).toMatch(/state == "CLOSED".*[\s\S]{0,200}STOP/);
expect(body).toMatch(/state == "CLOSED"[\s\S]{0,200}\bstop\b/i);
});
test("Hard rule: no replay after MERGED and only one guarded direct fallback", () => {
const body = readTmpl();
expect(body).toMatch(/never\s+replay a merge after MERGED/);
expect(body).toContain('one direct fallback');
expect(body).toContain('There is no fallback from a direct attempt');
expect(body).toContain('readback has confirmed OPEN, no auto request and no queue entry');
const body = readTmpl().replace(/\s+/g, " ");
expect(body).toMatch(/never replay a merge after MERGED/i);
expect(body).toMatch(/one direct fallback/i);
expect(body).toMatch(/no fallback from a direct attempt/i);
expect(body).toMatch(/confirmed OPEN, no auto request and no queue entry/i);
});
test("Generated merge-and-deploy.md carries the §4a-postfail section (atomic regen per T-Codex-3)", () => {
+87 -121
View File
@@ -26,35 +26,34 @@ function render(file: string, ctx: TemplateContext): string {
}
function assertProbeLocalBlocker(body: string): void {
expect(body).toContain('QA setup blocker');
expect(body).toContain('affected probes as blocked');
expect(body).toContain('continue other safe probes');
expect(body).toContain('required QA');
expect(body).toContain('independent functional/static checks');
expect(body).toMatch(/QA setup blocker/i);
expect(body).toMatch(/affected probes as blocked/i);
expect(body).toMatch(/continue other safe probes/i);
expect(body).toMatch(/independent functional\/static checks/i);
expect(body).not.toContain('stop that workflow');
expect(body).not.toContain('stop all checks');
}
function assertSharedBrowserAuthority(body: string): void {
const decision = body.slice(body.indexOf('## Browser access decision'), body.indexOf('## BROWSER SETUP'));
expect(decision).toContain('invoking workflow, not this file');
expect(decision).toMatch(/invoking workflow, not this file/i);
const reportOnly = decision.slice(decision.indexOf('**Report-only'), decision.indexOf('**Standalone /qa'));
expect(reportOnly).toContain('/qa-only, /review and /ship');
expect(reportOnly).toContain("do not run the fallback's setup/install or cookie-import workflow");
expect(reportOnly).toContain('Never bootstrap or invoke another skill');
expect(reportOnly).toContain('block only the affected browser probes');
expect(reportOnly).toContain('continue independent functional/static checks');
expect(reportOnly).toMatch(/do not run the fallback's setup\/install or cookie-import workflow/i);
expect(reportOnly).toMatch(/never bootstrap or invoke another skill/i);
expect(reportOnly).toMatch(/block only the affected browser probes/i);
expect(reportOnly).toMatch(/continue independent functional\/static checks/i);
expect(reportOnly).not.toContain('run `cd');
const standalone = decision.slice(decision.indexOf('**Standalone /qa'));
expect(standalone).toContain('explicit approval');
expect(standalone).toContain('STOP and wait');
expect(standalone).toContain('Only after approval');
expect(standalone).toContain('`cd <SKILL_DIR> && ./setup`');
const ask = standalone.search(/explicit approval/i);
const gate = standalone.search(/only after approval/i);
const setup = standalone.indexOf('`cd <SKILL_DIR> && ./setup`');
expect(ask).toBeGreaterThan(-1);
expect(gate).toBeGreaterThan(ask);
expect(setup).toBeGreaterThan(gate);
expect(standalone).toContain('/setup-browser-cookies');
expect(standalone).toContain('declined, unavailable or unsuccessful');
expect(standalone).toContain('blocked');
expect(decision).toContain('Unknown caller');
expect(decision).toContain('report-only');
expect(standalone).toMatch(/declined, unavailable or unsuccessful[\s\S]{0,40}blocked/i);
expect(decision).toMatch(/unknown caller[^\n]*report-only/i);
expect(body).not.toContain('If `NEEDS_SETUP`: tell the user');
expect(body).not.toContain('An authenticated page needs /setup-browser-cookies');
}
@@ -64,16 +63,11 @@ describe('QA caller authority in pure host renders', () => {
test(`${host.name}: a scope handoff requires the actual prior method read`, () => {
for (const caller of callers) {
const body = RESOLVERS.QA_EXPLORATORY(context(host.name, caller));
expect(body).toContain('Read `sections/scope.md`');
expect(body).toContain('Do not repeat a Read already completed in this invocation');
expect(body).toContain('Complete these Reads in order before writing charters or probing');
const scope = body.indexOf('1. Read `sections/scope.md`');
const selection = body.indexOf('in full and select the surfaces');
const methods = body.indexOf('2. Read the selected surface methods below in full');
const scope = body.indexOf('Read `sections/scope.md`');
const methods = body.indexOf('**Functional surfaces:**');
expect(scope).toBeGreaterThan(-1);
expect(selection).toBeGreaterThan(scope);
expect(methods).toBeGreaterThan(selection);
expect(body.indexOf('Write a **charter**')).toBeGreaterThan(methods);
expect(methods).toBeGreaterThan(scope);
expect(body.indexOf('## 1. Charter and preflight')).toBeGreaterThan(methods);
expect(body).not.toContain('If the caller has not selected surfaces and established isolation');
}
});
@@ -82,12 +76,10 @@ describe('QA caller authority in pure host renders', () => {
for (const caller of callers) {
const body = RESOLVERS.QA_EXPLORATORY(context(host.name, caller));
expect(body).toContain('The **caller** (/qa, /qa-only, /review or /ship)');
expect(body).toContain('charter** per behavior');
expect(body).toContain('bun G start D SECONDS [EARLIER_UTC]');
expect(body).toContain('G enforces the deadline');
expect(body).toContain('announce finite command timeouts');
expect(body.indexOf('bun G start D')).toBeLessThan(body.indexOf('1. First demonstrate success'));
expect(body).toContain('scoped contracts are tested or blocked');
expect(body).toMatch(/stop when scoped contracts are tested or blocked/i);
}
});
@@ -96,9 +88,8 @@ describe('QA caller authority in pure host renders', () => {
...context(host.name, 'qa'), tmplPath: 'qa/SKILL.md.tmpl', preambleTier: 4,
});
const final = body.slice(body.indexOf('## Phase 9: Final QA'), body.indexOf('## Phase 10: Report'));
expect(final).toContain('Re-run affected contracts and adjacent happy paths on the final inputs');
expect(final).toContain('Caller-required rechecks cannot be skipped as unaffected');
expect(final).toContain('blocked/inconclusive rechecks never verify repairs');
expect(final).toMatch(/caller-required rechecks cannot be skipped as unaffected/i);
expect(final).toMatch(/blocked\/inconclusive rechecks never verify repairs/i);
});
test(`${host.name}: prior learnings name QA findings without changing review callers`, () => {
@@ -116,12 +107,10 @@ describe('QA caller authority in pure host renders', () => {
test(`${host.name}: report-only learning lookup cannot configure or initialize stores`, () => {
const ctx = context(host.name, 'qa-only');
const search = RESOLVERS.LEARNINGS_SEARCH(ctx, ['query=webhook retries']);
expect(search).toContain('Read this project\'s existing learnings.jsonl only if its directory is already known');
expect(search).toContain('the caller permits that Read');
expect(search).toContain('Otherwise skip this optional lookup');
expect(search).toContain('Look for notes matching "webhook retries"');
expect(search).toContain('Do not run gstack-learnings-search here');
expect(search).toContain('Reading old notes never requires writing new ones');
expect(search).toMatch(/learnings\.jsonl only if its directory is already known/i);
expect(search).toMatch(/the caller permits that Read/i);
expect(search).toContain('"webhook retries"');
expect(search).toMatch(/do not run gstack-learnings-search here/i);
expect(search).not.toContain('```bash');
expect(search).not.toContain('gstack-config');
expect(search).not.toContain('AskUserQuestion');
@@ -137,7 +126,7 @@ describe('QA caller authority in pure host renders', () => {
assertProbeLocalBlocker(pointer);
expect(pointer).toContain(`\`sections/${id}.md\``);
expect(pointer).toContain('SKILL.md directory');
expect(pointer).toContain(sharedSetup ? 'No product-directory or cross-host substitutes' : 'never the product working directory');
expect(pointer).toMatch(sharedSetup ? /no product-directory or cross-host substitutes/i : /never the product working directory/i);
expect(pointer).not.toContain('## BROWSER SETUP');
expect(pointer).not.toContain('aside repl');
}
@@ -164,12 +153,11 @@ describe('QA caller authority in pure host renders', () => {
assertSharedBrowserAuthority(body);
expect(body.indexOf('## Browser access decision')).toBeLessThan(body.indexOf('## BROWSER SETUP'));
expect(body.indexOf('## BROWSER SETUP')).toBeLessThan(body.indexOf('## Browser fallback'));
expect(body).toContain('Functional-only');
expect(body).toContain('do not probe Aside');
expect(body).toContain("scope section's ownership rules apply even to LOCAL browser targets");
expect(body).toContain('never substitute unit tests or curl for the browser step');
expect(body).toContain('Never type passwords, one-time codes, or payment details');
expect(body).toContain('Never read, screenshot, navigate, or close any other tab');
expect(body.replace(/\s+/g, ' ')).toMatch(/functional-only targets do not probe Aside/i);
expect(body).toMatch(/ownership rules apply even to local browser targets/i);
expect(body).toMatch(/never substitute unit tests or curl for the browser step/i);
expect(body).toMatch(/never type passwords, one-time codes, or payment details/i);
expect(body).toMatch(/never read, screenshot, navigate, or close any other tab/i);
expect((body.match(/cd <SKILL_DIR> && \.\/setup/g) ?? [])).toHaveLength(1);
});
@@ -191,22 +179,19 @@ describe('QA caller authority in pure host renders', () => {
const auth = body.slice(body.indexOf('### Phase 2:'), body.indexOf('### Phase 3:'));
expect(auth).toContain('Browser access decision');
expect(auth).not.toContain('Fallback: /setup-browser-cookies or');
expect(auth).toContain('Never handle credentials');
expect(body).toContain('Run only for selected browser surfaces');
expect(body).toContain('Confirm each issue by retrying once');
expect(auth).toMatch(/never handle credentials/i);
});
test(`${host.name}: functional-only and report-only paths preserve independent checks and gates`, () => {
const scope = RESOLVERS.QA_SCOPE(context(host.name, 'qa'));
expect(scope).toContain('Functional-only runs must not read browser setup, methodology, verification or bootstrap');
expect(scope).toMatch(/functional-only runs must not read browser setup/i);
expect(scope).not.toContain('command -v aside');
expect(scope).not.toContain('curl -sI');
for (const caller of callers) {
const ctx = context(host.name, caller);
const body = RESOLVERS.QA_EXPLORATORY(ctx);
const compact = body.replace(/\s+/g, ' ');
expect(compact).toContain('Missing or unreadable assets, prerequisites or permission block affected probes, not independent safe checks');
expect(compact).toContain('Report QA setup blockers');
expect(compact).toMatch(/block affected probes, not independent safe checks/i);
expect(compact).not.toContain('stop all checks');
expect(body).toContain('2. Read the selected surface methods below in full');
expect(body.indexOf('2. Read the selected surface methods below in full')).toBeLessThan(body.indexOf('1. First demonstrate success'));
@@ -215,29 +200,29 @@ describe('QA caller authority in pure host renders', () => {
expect(reads).toContain('sections/system-functional.md');
expect(reads).toContain('**Browser surfaces only:**');
expect(reads).toContain('sections/qa-patterns.md');
expect(body).toContain('no workflows, framework installs or publication');
expect(body).toContain('owns decisions, tests, fixes and publication');
expect(body).toContain('Missing prerequisites/expectations/observations, timeouts and refusal never pass');
expect(body).toContain('Pass requires all required current-input contracts to pass with no required remainder');
expect(body).toMatch(/no workflows, framework installs or publication/i);
expect(body).toMatch(/owns decisions, tests, fixes and publication/i);
expect(body).toMatch(/timeouts and refusal never pass/i);
expect(body).toMatch(/pass requires all required current-input contracts to pass/i);
if (caller !== 'qa-only') {
expect(body).toContain('leaves /review incomplete');
expect(body).toContain('unless the user explicitly accepts that named risk');
expect(body).toContain('noninteractive runs return blocked');
expect(body).toContain('test_stub proposals require ASK approval');
expect(body).toMatch(/unless the user explicitly accepts that named risk/i);
expect(body).toMatch(/noninteractive runs return blocked/i);
expect(body).toMatch(/test_stub proposals require ASK approval/i);
}
}
const review = RESOLVERS.QA_REVIEW(context(host.name, 'review'));
const ship = RESOLVERS.QA_REVIEW(context(host.name, 'ship'));
expect(review).toContain('a ship waiver cannot complete it');
expect(ship).toContain('explicit named-risk acceptance');
expect(ship.replace(/\s+/g, ' ')).toContain('Report clean/completed only when all required checks pass on current inputs');
expect(ship).toContain('List failed, blocked, inconclusive and not-run checks');
expect(review).toMatch(/a ship waiver cannot complete it/i);
expect(ship).toMatch(/explicit named-risk acceptance/i);
expect(ship.replace(/\s+/g, ' ')).toMatch(/report clean\/completed only when all required checks pass on current inputs/i);
});
test(`${host.name}: non-QA fallback retains its existing setup and human sign-in flow`, () => {
const fallback = RESOLVERS.BROWSE_FALLBACK(context(host.name, 'browse'));
expect(fallback).toContain('OK to proceed?');
expect(fallback).toContain('STOP for the answer, then run `cd <SKILL_DIR> && ./setup`');
expect(fallback).toMatch(/stop for the answer, then run `cd <SKILL_DIR> && \.\/setup`/i);
expect(fallback.indexOf('OK to proceed?')).toBeLessThan(fallback.indexOf('cd <SKILL_DIR> && ./setup'));
expect(fallback).toContain('An authenticated page needs /setup-browser-cookies');
expect(fallback).toContain('$B handoff');
expect(fallback).toContain('$B resume');
@@ -253,23 +238,17 @@ describe('QA caller authority in pure host renders', () => {
const resource = RESOLVERS.QA_RESOURCE(ctx, ['exploratory']);
expect(resource).toContain(`../${host.name === 'claude' ? 'qa' : 'gstack-qa'}/sections/exploratory.md`);
const shared = render('qa/sections/exploratory.md.tmpl', context(host.name, 'qa'));
expect(shared).toContain('Read `sections/system-functional.md` in full');
expect(shared.indexOf('in full and select the surfaces')).toBeLessThan(shared.indexOf('Read `sections/system-functional.md`'));
expect(shared.indexOf('Read `sections/scope.md`')).toBeLessThan(shared.indexOf('Read `sections/system-functional.md`'));
const required = body.indexOf(caller === 'review'
? '2. Check readiness and list required checks' : '2. List required checks');
expect(required).toBeGreaterThan(-1);
expect(exploration).toBeLessThan(required);
expect(body).toContain('one success and the riskiest changed failure/edge');
expect(body).toContain('Smoke: 5 minutes/12 probes');
expect(body).toContain('Required even for small diffs or missing plans/servers');
expect(body).toContain('Required: plan commands/assertions, listed separately');
expect(body).toContain('Other ideas are optional, untested');
expect(body.replace(/\s+/g, ' ')).toContain('Report clean/completed only when all required checks pass on current inputs');
expect(body).toContain('List failed, blocked, inconclusive and not-run checks');
expect(body).toContain('5 minutes/12 probes');
expect(body.replace(/\s+/g, ' ')).toMatch(/report clean\/completed only when all required checks pass on current inputs/i);
}
const ship = RESOLVERS.QA_REVIEW(context(host.name, 'ship'));
expect(ship).toContain('Step 9.4 asks: permission/repair');
expect(ship).toContain('explicit named-risk acceptance; otherwise blocked');
expect(ship).toContain('Step 9.4');
expect(ship).toMatch(/explicit named-risk acceptance; otherwise blocked/i);
});
test(`${host.name}: caller QA selects surfaces directly and links checkpoints in one final section`, () => {
@@ -279,33 +258,25 @@ describe('QA caller authority in pure host renders', () => {
const exploration = body.indexOf('{{QA_RESOURCE:exploratory}}');
const shared = render('qa/sections/exploratory.md.tmpl', context(host.name, 'qa'));
const scope = shared.indexOf('Read `sections/scope.md`');
const selection = shared.indexOf('in full and select the surfaces');
const methods = shared.indexOf('**Functional surfaces:**');
const probes = body.indexOf(caller === 'review'
? '2. Check readiness and list required checks' : '2. List required checks');
expect(scope).toBeGreaterThan(-1);
expect(exploration).toBeGreaterThan(-1);
expect(selection).toBeGreaterThan(scope);
expect(methods).toBeGreaterThan(selection);
expect(shared.indexOf('Write a **charter**')).toBeGreaterThan(methods);
expect(methods).toBeGreaterThan(scope);
expect(shared.indexOf('## 1. Charter and preflight')).toBeGreaterThan(methods);
expect(exploration).toBeLessThan(probes);
expect(body).not.toContain('**Functional surfaces:**');
expect(RESOLVERS.QA_RESOURCE(ctx, ['exploratory'])).toContain(`../${host.name === 'claude' ? 'qa' : 'gstack-qa'}/sections/exploratory.md`);
if (caller === 'review') {
const flat = body.replace(/\s+/g, ' ');
expect(flat).toContain('Title it `## Exploratory QA and Verification Results`');
expect(flat).toContain('keep metadata/outcome tables');
expect(flat).toContain('demote other headings one level');
expect(flat).toContain('include it here under `### Browser results`');
expect(flat).toContain('other headings demoted two levels');
expect(flat).toContain('Link every checkpoint');
expect(flat).toContain('No second report');
expect(flat).toContain('Keep browser/functional scores and outcomes separate');
expect(flat).toContain('save browser baseline/evidence normally');
expect(flat).toContain('`## Exploratory QA and Verification Results`');
expect(flat).toContain('`### Browser results`');
expect(flat).toMatch(/link every checkpoint/i);
expect(flat).toMatch(/no second report/i);
} else {
expect(body.replace(/\s+/g, ' ')).toContain('PR section `## Exploratory QA');
expect(body).toContain('fields as subsections');
expect(body).toContain('Link every checkpoint; no second report');
expect(body.replace(/\s+/g, ' ')).toContain('`## Exploratory QA');
expect(body).toMatch(/link every checkpoint; no second report/i);
}
expect(body).toContain('templates/functional-report-template.md');
expect(body).not.toContain('not a second report');
@@ -314,14 +285,14 @@ describe('QA caller authority in pure host renders', () => {
test(`${host.name}: orchestration logs reviewer attempts before parent-owned edits`, () => {
const body = RESOLVERS.ADVERSARIAL_STEP(context(host.name, 'review'));
expect(body).toContain("queued for the parent's Fix-First handling at Step 5; do not edit during Step 4.8");
expect(body).toContain('do not start an inner repair loop');
expect(body.replace(/\s+/g, ' ')).toContain("Keep each token with that attempt; do not overwrite the parent's REVIEW_START");
expect(body.replace(/\s+/g, ' ')).toContain('save one record per source, phase and attempt, before the');
expect(body).toContain('parent applies queued fixes');
expect(body).toContain('Each token is consumed once');
const flat = body.replace(/\s+/g, ' ');
expect(flat).toMatch(/Fix-First handling at Step 5; do not edit during Step 4\.8/i);
expect(flat).toMatch(/do not start an inner repair loop/i);
expect(flat).toMatch(/do not overwrite the parent's REVIEW_START/i);
expect(flat).toMatch(/one record per source, phase and attempt/i);
expect(flat).toMatch(/each token is consumed once/i);
expect(body).not.toContain('address the findings. Re-run the same shared structured invocation');
expect(body).toContain('The native pass is required for Step 5.8 completion');
expect(flat).toMatch(/native pass is required for Step 5\.8 completion/i);
});
test(`${host.name}: orchestration skips only history matching without user skips`, () => {
@@ -329,17 +300,12 @@ describe('QA caller authority in pure host renders', () => {
const body = RESOLVERS.CROSS_REVIEW_DEDUP(context(host.name, caller));
if (caller === 'ship') {
const flat = body.replace(/\s+/g, ' ');
expect(flat).toContain('Combine saved `findings` with the invocation action list, honoring later user decisions');
expect(flat).toContain('If both history and the invocation action list lack decisions, classify normally');
expect(body.indexOf('2. **Read decisions.**')).toBeLessThan(body.indexOf('3. **Match evidence.**'));
expect(flat).toContain('Only explicit `skipped` actions qualify, never `fixed`, `auto-fixed` or unanswered questions');
expect(flat).toContain('Report the suppressed count once if nonzero');
expect(flat).toMatch(/only explicit `skipped` actions qualify, never `fixed`, `auto-fixed` or unanswered questions/i);
} else {
expect(body).toContain('skip history matching silently; still classify current findings');
expect(body).toMatch(/still classify current findings/i);
expect(body.indexOf('If no prior reviews exist')).toBeLessThan(body.indexOf('For each JSONL entry'));
expect(body).toContain('If N > 0, print once:');
expect(body).toContain('Otherwise skip the summary');
expect(body).toContain('Only suppress `skipped` findings — never `fixed` or `auto-fixed`');
expect(body).toMatch(/only suppress `skipped` findings — never `fixed` or `auto-fixed`/i);
}
expect(body).not.toContain('skip this step silently');
}
@@ -347,11 +313,11 @@ describe('QA caller authority in pure host renders', () => {
test(`${host.name}: orchestration consumes the existing generation allowance at both coverage gates`, () => {
const body = RESOLVERS.TEST_COVERAGE_GATE_SHIP(context(host.name, 'ship'));
expect(body).toContain("Use Step 7's remaining generation allowance");
expect(body).toContain("Step 7's remaining generation allowance");
expect(body.match(/If A and allowance remains:/g)).toHaveLength(2);
expect(body).toContain('At the cap, offer only B/C or stop');
expect(body).toContain('At the cap, offer only B or stop');
expect(body).toContain('At the cap, omit A\'s generation pass and recommend stopping');
expect(body).toMatch(/at the cap, offer only B\/C or stop/i);
expect(body).toMatch(/at the cap, offer only B or stop/i);
expect(body).toMatch(/at the cap, omit A's generation pass and recommend stopping/i);
expect(body).toContain('Minimum = 60%, Target = 80%');
expect(body).not.toContain('Maximum 2 passes total');
});
@@ -359,24 +325,23 @@ describe('QA caller authority in pure host renders', () => {
test('report-only recommendations cannot dispatch a repair skill during discovery', () => {
const skill = readFileSync(join(root, 'qa-only/SKILL.md.tmpl'), 'utf8');
expect(skill).toContain('Never invoke /qa or another skill from this report-only run');
expect(skill).toContain('separate, user-authorized');
expect(skill).toContain('No test framework detected');
expect(skill).toContain('Never commit, stash or bootstrap');
expect(skill).toMatch(/never invoke \/qa or another skill from this report-only run/i);
expect(skill).toMatch(/separate, user-authorized/i);
expect(skill).toMatch(/never commit, stash or bootstrap/i);
});
test('standalone QA scopes its general test rule around the approved browser bootstrap', () => {
const skill = readFileSync(join(root, 'qa/SKILL.md.tmpl'), 'utf8');
const rule = skill.split('\n').find(line => line.startsWith('**Outside an explicitly approved browser bootstrap:**'));
expect(skill).not.toMatch(/^13\. /m);
expect(rule).toContain('Outside an explicitly approved browser bootstrap');
expect(rule).toContain('Only create tests through authorized codification in Phase 8a.5');
expect(rule).toContain('Never modify CI configuration or weaken existing tests');
expect(rule).toMatch(/authorized codification in Phase 8a\.5/i);
expect(rule).toMatch(/never modify CI configuration or weaken existing tests/i);
const bootstrap = readFileSync(join(root, 'qa/sections/test-bootstrap.md.tmpl'), 'utf8');
expect(bootstrap).toContain('Browser /qa only, never functional/report-only');
expect(bootstrap).toContain('AskUserQuestion and WAIT');
expect(bootstrap).toContain('install only the actual choice');
expect(bootstrap).toContain('Never silently delete a valid red regression');
expect(bootstrap).toMatch(/browser \/qa only, never functional\/report-only/i);
const ask = bootstrap.search(/AskUserQuestion and wait/i);
expect(ask).toBeGreaterThan(-1);
expect(ask).toBeLessThan(bootstrap.search(/install only the actual choice/i));
expect(bootstrap).toMatch(/never silently delete a valid red regression/i);
expect(bootstrap).toContain('create/extend `.github/workflows/test.yml`');
});
@@ -387,6 +352,7 @@ describe('QA caller authority in pure host renders', () => {
expect(() => assertProbeLocalBlocker(pointer.replace('continue other safe probes', 'stop all checks'))).toThrow();
const setup = render('qa/sections/browser-setup.md.tmpl', ctx);
expect(() => assertSharedBrowserAuthority(setup.replace('Only after approval', 'Immediately'))).toThrow();
expect(() => assertSharedBrowserAuthority(setup.replace('explicit approval', 'a status update'))).toThrow();
expect(() => assertSharedBrowserAuthority(setup.replace('Never bootstrap or invoke another skill', 'Invoke another skill'))).toThrow();
expect(() => assertSharedBrowserAuthority(setup + '\nIf `NEEDS_SETUP`: tell the user')).toThrow();
expect(() => assertSharedBrowserAuthority(setup + '\nAn authenticated page needs /setup-browser-cookies')).toThrow();
+28 -59
View File
@@ -649,13 +649,9 @@ describe('generated actual parent paths', () => {
expect(positions).toEqual([...positions].sort((a, b) => a - b));
const load = skillName === 'review' ? generateQAReviewPreflight(ctx) : parent.slice(positions[0], positions[1]);
if (skillName === 'review') {
const charter = parent.slice(positions[0], positions[1]).replace(/\s+/g, ' ');
expect(charter).toContain("Reuse Step 4's surfaces and completed Reads");
expect(charter).toContain('Finish missing methods before charters');
expect(charter).toContain('complete the shared isolation/permission preflight before setup');
const readiness = parent.slice(positions[1], positions[2]).replace(/\s+/g, ' ');
expect(readiness).toContain('Read QA\'s `sections/browser-setup.md` and follow its report-only rules');
expect(readiness).toContain('Never install, import cookies or bootstrap tests');
expect(readiness).toMatch(/Read QA's `sections\/browser-setup\.md` and follow its report-only rules/i);
expect(readiness).toMatch(/never install, import cookies or bootstrap tests/i);
expect(load).not.toContain('sections/browser-setup.md');
}
expect(load).toContain('{{QA_RESOURCE:exploratory}}');
@@ -664,50 +660,26 @@ describe('generated actual parent paths', () => {
const resource = generateQAResource(ctx, ['exploratory']);
expect(resource).toContain(`installed /${skillName} SKILL.md's directory`);
expect(resource).toContain('`../qa/sections/exploratory.md`');
const shared = generateQAExploratory({ ...ctx, skillName: 'qa' });
const preparation = ['1. Read `sections/scope.md`', 'in full and select the surfaces',
'Read `sections/system-functional.md` in full.', 'Read `sections/qa-patterns.md` in full.',
'Write a **charter**', '1. First demonstrate success'].map(marker => shared.indexOf(marker));
expect(preparation.every(position => position >= 0)).toBe(true);
expect(preparation).toEqual([...preparation].sort((a, b) => a - b));
expect(load).toContain('Templates cannot replace them');
const flat = parent.replace(/\s+/g, ' ');
expect(flat).toContain('Only the parent runs report-only discovery');
expect(flat).toContain('Follow the shared Probe loop for smoke checks and replays until the smoke limit');
expect(flat).toContain('Then run required plan checks and revalidation, even after smoke expires');
expect(flat).toContain('using the same procedure but no smoke guard; never reset the clock');
expect(flat).toContain("Use finite command timeouts, capped at the caller\'s remaining time if it has a deadline");
expect(flat).toContain('When the caller\'s deadline expires, mark unfinished checks not-run');
for (const contract of ['First demonstrate success: output AND durable effects',
'Wait for successful checkpoint publication before dispatch',
'Replay the exact failing command/request from the same initial fixture state']) {
expect(shared).toContain(contract);
}
expect(flat).toContain('Read agent/user updates and await results without batching them with reporting/logging');
expect(flat).toContain('Re-review changed or uncertain coverage and repeat step 3 for affected checks');
expect(flat).toContain('Report clean/completed only when all required checks pass on current inputs');
expect(flat).toContain('List failed, blocked, inconclusive and not-run checks');
expect(flat).toMatch(/repeat step 3 for affected checks/i);
}
});
test('authored shared loop preserves complete safe observations and re-enters checkpoints after input changes', () => {
const text = generateQAExploratory({ skillName: 'qa', tmplPath: 'qa/SKILL.md.tmpl', host: 'claude', paths: HOST_PATHS.claude }).replace(/\s+/g, ' ');
for (const contract of ["last completed probe's full outer command", 'Preserve every safe program-JSON key/value', 'identity hash unchanged', 'Q supplies observed; never transcribe it', 'write the report, not a checkpoint', 'return to step 2 for each affected revalidation', 'Pass requires all required current-input contracts to pass with no required remainder']) {
expect(text).toContain(contract);
}
expect(text).toContain("observationCommand: last completed probe's full outer command, including guard");
expect(text).toContain('observed: its exact decoded child JSON (no wrapper/extra keys)');
expect(text).toContain('or its full non-JSON text');
expect(text).toMatch(/observationCommand: last completed probe's full outer command/i);
expect(text).toMatch(/observed: its exact decoded child JSON/i);
expect(text).toMatch(/identity hash unchanged/i);
expect(text).not.toContain('nest unchanged child JSON');
});
test('the shared smoke has explicit limits without waiving required plan checks', () => {
const body = fs.readFileSync(path.join(import.meta.dir, '../qa/sections/exploratory.md'), 'utf8').replace(/\s+/g, ' ');
expect(body).toContain('Stop after 5 minutes or 12 probes, whichever comes first');
expect(body).toMatch(/stop after 5 minutes or 12 probes/i);
expect(body).toContain('G enforces the deadline');
expect(body).toContain('Never reset D/bypass G');
expect(body).toContain('Explicit plan checks and revalidation remain required beyond this smoke budget');
expect(body).toMatch(/never reset D\/bypass G/i);
expect(body).toMatch(/plan checks and revalidation remain required beyond this smoke budget/i);
expect(body).toContain('leaves /review incomplete');
expect(body).toContain('/ship blocked unless the user explicitly accepts that named risk');
expect(body).toMatch(/\/ship blocked unless the user explicitly accepts that named risk/i);
});
test('excerpt extraction fails loudly instead of producing an empty passing fixture', () => {
@@ -1152,16 +1124,15 @@ describe('real caller-specific native fixture and capture boundary', () => {
try {
fixture.config = path.join(fixture.root, 'callback-config');
await runQaCaller(fixture, 'free-receipt-interface', async options => {
expect(options.prompt).toContain('status is the overall supplied phase gate, not whether some probes passed');
expect(options.prompt).toContain('Pass requires no remaining required contracts or gates');
expect(options.prompt).toContain('Optional unavailable providers and later stages outside this excerpt are not required remainder');
expect(options.prompt).toContain('exact id values from the captured child JSON (probe-...)');
expect(options.prompt).toContain("never the helper's three-digit capture IDs or QA_EVIDENCE.id");
expect(options.prompt).toContain('Capture IDs select stored observations for checkpoint/materialize');
expect(options.prompt).toMatch(/overall supplied phase gate/i);
expect(options.prompt).toMatch(/no remaining required contracts or gates/i);
expect(options.prompt).toMatch(/not required remainder/i);
expect(options.prompt).toContain('(probe-...)');
expect(options.prompt).toMatch(/never the helper's three-digit capture IDs/i);
expect(options.prompt).not.toMatch(/bun scripts\/probe\.ts \d|exploration-[0-9]{3}|snapshot.*must|plan:nine|adverse/i);
const readme = fs.readFileSync(path.join(fixture.cwd, 'README.md'), 'utf8');
expect(readme).toContain('Every diagnostic receipt field is synthetic, nonsecret evidence');
expect(readme).toContain('snapshot identifies the owned source and fixture inputs');
expect(readme).toMatch(/synthetic, nonsecret/i);
expect(readme).toMatch(/snapshot identifies the owned source/i);
expect(readme).not.toMatch(/checkpoint|exploration-NNN|hypothesis|nextCommand/);
return { exitReason: 'success', transcript: [] } as unknown as SkillTestResult;
});
@@ -1361,20 +1332,18 @@ describe('real caller-specific native fixture and capture boundary', () => {
expect(options.timeout).toBe(300_000);
expect(options.completionReserveMs).toBe(75_000);
expect(options.appendSystemPrompt).toContain(`at most ${options.maxTurns} assistant turns`);
expect(options.appendSystemPrompt).toContain('independent source Reads and read-only discovery together as separate native tool calls');
expect(options.appendSystemPrompt).toContain('After required clock and approval prerequisites settle');
expect(options.appendSystemPrompt).toContain('The completion reserve is for required verification, affected-input revalidation and artifacts, not an earlier deadline');
expect(options.appendSystemPrompt).toContain("Use each native probe's snapshot to distinguish current from superseded evidence");
expect(options.appendSystemPrompt).toContain('Never group diagnostic probes, checkpoint publication with its next probe');
expect(options.appendSystemPrompt).toContain('the turn limit does not authorize skipping work or reporting incomplete work as passed');
for (const rule of [/separate native tool calls/i, /approval prerequisites settle/i,
/completion reserve is for required verification/i, /superseded evidence/i,
/never group diagnostic probes/i, /turn limit does not authorize skipping work/i]) {
expect(options.appendSystemPrompt).toMatch(rule);
}
expect(options.appendSystemPrompt).not.toMatch(/bun scripts\/probe\.ts \d|invalid input|highest.risk/i);
expect(options.prompt).toContain('Keep normal parent decision gates.');
expect(options.prompt).toContain("use the section clock's Hard deadline UTC, never its Runner entry UTC, reserve-start time or a clock-read time");
expect(options.prompt).toContain('Before the completion report and each completed:true review record, read HANDOFF.md');
expect(options.prompt).toContain('a later handoff read cannot validate an earlier completion');
expect(options.prompt).toContain('If you defer an optional idea or stop exploration, do not publish a checkpoint for it');
expect(options.prompt).toContain('An unused checkpoint requires an actual authenticated expired-capture result; nearing the deadline or choosing to stop is not enough');
expect(options.prompt).toContain('Write the phase report to reports/review.md.');
expect(options.prompt).toMatch(/keep normal parent decision gates/i);
expect(options.prompt).toMatch(/Hard deadline UTC, never its Runner entry UTC/i);
expect(options.prompt).toMatch(/completed:true review record, read HANDOFF\.md/i);
expect(options.prompt).toMatch(/do not publish a checkpoint for it/i);
expect(options.prompt).toMatch(/authenticated expired-capture result/i);
expect(options.prompt).toContain('reports/review.md');
return sentinel;
});
expect(calls).toBe(1);
+37 -77
View File
@@ -9,6 +9,8 @@ import { qaFunctionalVerdict, qaNativeProbes } from './helpers/qa-functional-evi
import { createQAFunctionalFixture, ownedPath } from './helpers/qa-functional-fixture';
import { validateQACheckpoints } from './helpers/qa-checkpoint-evidence';
import { computePaidCaseSelection } from '../scripts/test-paid-shards';
import { generateQAExploratory } from '../scripts/resolvers/qa';
import { HOST_PATHS } from '../scripts/resolvers/types';
test.each(['full', 'pr'] as const)('%s selection assigns the captured webhook regression to its native owner', profile => {
for (const file of ['qa-webhook-r85-checkpoints.json', 'qa-functional-ci-36505065023.json']) {
@@ -81,22 +83,15 @@ test('functional driver discloses its learning, CLI coverage and repair acceptan
for (const entry of QA_FUNCTIONAL_CASES) {
const prompt = qaFunctionalPrompt(entry);
expect(prompt).toContain(`Read ${entry.mode}/SKILL.md, qa/sections/scope.md, ${entry.mode}/sections/exploratory.md and qa/sections/system-functional.md in full`);
expect(prompt).toContain('All four reads are required before probing in this fixture, even when its surfaces and isolation are already established');
expect(prompt).toContain('command is the exact full outer capture invocation, including that ID and all wrapper options, not just the native child command after --');
expect(prompt).toContain('"command":"<exact full outer capture invocation>"');
expect(prompt).not.toContain('<exact executed native probe command>');
expect(prompt).toContain('different later command');
expect(prompt).toContain('not the required same-command replay');
expect(prompt).toContain('Select that checkpoint ID in annotations.learning');
expect(prompt).toContain('the production helper copies its observationCommand, hypothesis and nextCommand');
expect(prompt).toContain('different native child commands');
expect(prompt).toContain('Only the helper writes observed fields');
expect(prompt).toContain('English, more than 20 characters');
if (entry.family === 'cli') expect(prompt).toContain('a successful apply; balance alone is not enough');
expect(prompt).toContain('annotations.learning');
expect(prompt).toContain('observationCommand, hypothesis and nextCommand');
expect(prompt).toMatch(/more than 20 characters/);
if (entry.mode === 'qa') {
expect(prompt).toContain(`repair only src/${entry.family === 'cli' ? 'cli' : 'worker'}.ts`);
expect(prompt).toContain('existing tests remain read-only');
expect(prompt).toContain('Freeze all test files after red');
expect(prompt).toMatch(/existing tests remain read-only/i);
expect(prompt).toMatch(/freeze all test files after red/i);
}
}
});
@@ -164,12 +159,12 @@ test('the native launcher consumes the family-specific actor boundary', () => {
expect(prompt).not.toContain('at Standard depth');
expect(prompt).toContain('successful checkpoint publication before the next probe');
expect(prompt).toContain('no shell composition, scripts or added path operands');
expect(prompt).toContain('ONLY complete JSON actually emitted');
expect(prompt).toMatch(/only complete JSON actually emitted/i);
expect(prompt).toContain('never a combined command list');
expect(prompt).toContain('Put tests, raw CLI diagnostics, launch failures and timeouts in Markdown');
expect(prompt).toContain(entry.family === 'cli'
? 'The generic wrapper does NOT support wait'
: 'bun cancel.ts is a CLI-only entrypoint, not part of this fixture');
expect(prompt).toMatch(entry.family === 'cli'
? /the generic wrapper does not support wait/i
: /bun cancel\.ts is a CLI-only entrypoint, not part of this fixture/i);
expect(prompt).not.toContain('parseInt');
if (entry.family === 'webhook') expect(prompt).toContain('Choose their order from observations after the happy path');
if (entry.family === 'webhook' && entry.mode === 'qa-only') {
@@ -198,51 +193,30 @@ test('declared examples respect the existing closed native grammar', () => {
test('artifact completion preserves exact evidence before concise linked reporting', () => {
for (const entry of QA_FUNCTIONAL_CASES) {
const prompt = qaFunctionalPrompt(entry);
expect(prompt).toContain('Materialize qa-reports/evidence.json first, then write a concise qa-reports/report.md');
expect(prompt).toContain('using the functional report structure');
expect(prompt).toContain('Link the evidence and checkpoint files rather than repeating full probe payloads in Markdown');
expect(prompt).toContain('Both artifacts are required before completion');
expect(prompt).toContain('The learning array is a summary: choose one completed checkpoint');
expect(prompt).toContain('not another probe or a duplicate of the complete checkpoint ledger');
expect(prompt).toContain('Both commands must name exact captured probes with different native child commands');
expect(prompt).toContain('Preserve every checkpoint and link every checkpoint in Markdown');
expect(prompt).toContain('keep every executed probe and its complete JSON in evidence');
expect(prompt).toContain('Evidence rows contain ONLY complete JSON actually emitted by native probes, including failures and repeats');
expect(prompt).toContain('retain pre-repair results alongside green results');
expect(prompt).toContain('Never synthesize JSON');
expect(prompt).toContain('one causal sentence per checkpoint hypothesis (English, more than 20 characters) and compact JSON formatting, preserving every field and value');
expect(prompt).toContain('retain its headings and required fields');
expect(prompt).toContain('link to evidence.json and checkpoints for details already recorded there');
expect(prompt).toContain('After saving both artifacts, return only their paths and the actual completion status');
expect(prompt).toContain('Never shorten native JSON or omit a required probe, check or field');
expect(prompt).toContain('aim under 400 words');
expect(prompt.indexOf('qa-reports/evidence.json')).toBeGreaterThan(-1);
expect(prompt.indexOf('qa-reports/evidence.json')).toBeLessThan(prompt.indexOf('qa-reports/report.md'));
expect(prompt).toMatch(/both artifacts are required before completion/i);
expect(prompt).toMatch(/never synthesize JSON/i);
expect(prompt).toMatch(/never shorten native JSON or omit a required probe, check or field/i);
}
const source = readFileSync(join(import.meta.dir, 'helpers/qa-functional-eval.ts'), 'utf8');
expect(source).toContain('maxTurns: 40');
expect(source).toContain('completionReserveMs: timeout / 4');
expect(Number(source.match(/maxTurns: (\d+)/)?.[1])).toBeGreaterThanOrEqual(40);
expect(source).toContain('completionReserveMs:');
});
test('fix completion budgets for required repair and avoids duplicating preserved evidence', () => {
for (const entry of QA_FUNCTIONAL_CASES) {
const prompt = qaFunctionalPrompt(entry);
if (entry.mode === 'qa') {
expect(prompt).toContain('a reproduced in-tier defect requires the authorized native regression, repair and verification');
expect(prompt).toContain('retain its headings and required fields');
expect(prompt).toContain('link to evidence.json and checkpoints for details already recorded there');
expect(prompt).toContain('Include the diagnosis, red/green test results and coverage limits');
expect(prompt).toContain('After saving both artifacts, return only their paths and the actual completion status');
expect(prompt).toContain('Never shorten native JSON or omit a required probe, check or field');
const stages = ['1. Prove the regression red', '2. On the repaired source', '3. Save the evidence and Markdown artifacts'];
const positions = stages.map(stage => prompt.indexOf(stage));
expect(positions.every(position => position >= 0)).toBe(true);
expect(positions).toEqual([...positions].sort((a, b) => a - b));
expect(prompt).toContain('A green test suite does not substitute for these native probes');
expect(prompt).toContain('not a signal to stop stage 2');
expect(prompt).toContain('report incomplete; do not call it complete with a caveat');
expect(prompt).toContain('one causal sentence per checkpoint hypothesis (English, more than 20 characters) and compact JSON formatting, preserving every field and value');
expect(prompt).toMatch(/green test suite does not substitute for these native probes/i);
expect(prompt).toMatch(/report incomplete; do not call it complete/i);
} else {
expect(prompt).not.toContain('This is a fix run');
expect(prompt).toContain('Include the diagnosis, proposed test stubs and coverage limits');
expect(prompt).toMatch(/proposed test stubs/i);
expect(prompt).not.toContain('Include the diagnosis, red/green test results');
}
}
@@ -265,22 +239,19 @@ test('webhook fix stage asks for the fix-loop probes; the R29 scenario omissions
}
const fix = qaFunctionalPrompt({ family: 'webhook', mode: 'qa' });
const verification = fix.slice(fix.indexOf('2. On the repaired source'), fix.indexOf('3. Save the evidence'));
expect(verification).toContain('run the original failing probe, an adjacent happy-path probe, cancellation and the unavailable-dependency probe');
for (const probe of ['original failing probe', 'happy-path probe', 'cancellation', 'unavailable-dependency probe']) expect(verification).toContain(probe);
expect(verification).not.toContain('All eight scenarios');
expect(fix).toContain('the completion reserve does not end required coverage');
expect(fix).toMatch(/completion reserve does not end required coverage/i);
expect(verification).toBe(((prompt: string) => prompt.slice(prompt.indexOf('2. On the repaired source'), prompt.indexOf('3. Save the evidence')))(qaFunctionalPrompt({ family: 'cli', mode: 'qa' })));
const report = qaFunctionalPrompt({ family: 'webhook', mode: 'qa-only' });
expect(report).toContain('All eight scenarios are required coverage; a replay does not replace another scenario');
expect(report).toMatch(/all eight scenarios are required coverage/i);
for (const scenario of required) expect(report).toContain(scenario);
});
test('fix-stage checkpoint provenance survives intervening native regression tests', () => {
const prompt = qaFunctionalPrompt({ family: 'webhook', mode: 'qa' });
expect(prompt).toContain('most recent completed native probe');
expect(prompt).toContain('Tests, source edits and clock reads do not replace that observation');
expect(prompt).toContain('put red/green test output in the report, not in observed');
expect(prompt).toContain('write no checkpoint when there is no next probe');
expect(prompt).toMatch(/red\/green test output in the report, not in observed/i);
});
test('R29 captured webhook bytes bind across a green test; test summaries and altered JSON do not', () => {
@@ -376,32 +347,21 @@ test.each(JSON.parse(readFileSync(join(import.meta.dir, 'fixtures/qa-webhook-r85
test('report-only exploration requires a completed written checkpoint before the next probe', () => {
const section = readFileSync(join(import.meta.dir, '../qa-only/sections/exploratory.md'), 'utf8');
const positions = ['1. First demonstrate success', '2. **Decide whether another probe is needed.**', '**Publish before probing.**', '3. Run that exact probe; G enforces the deadline when bounded']
.map(marker => section.indexOf(marker));
expect(positions.every(position => position >= 0)).toBe(true);
expect(positions).toEqual([...positions].sort((a, b) => a - b));
expect(section).toContain('exploration-NNN.json');
expect(section).toContain("Reuse resolved REPORT_DIR");
expect(section).toContain('own a fresh');
expect(section).toContain('owned probe directory');
expect(section).toContain('Reuse resolved REPORT_DIR');
expect(section).toMatch(/owned probe directory/i);
for (const field of ['observationCommand', 'observed', 'hypothesis', 'nextCommand']) expect(section).toContain(`${field}:`);
expect(section).toContain('Wait for successful checkpoint publication');
expect(section).toContain('Never backfill or overwrite notes');
expect(section).toContain('link each checkpoint');
expect(section).toMatch(/wait for successful checkpoint publication/i);
expect(section).toMatch(/never backfill or overwrite notes/i);
expect(section).not.toContain('a separate assistant text message');
});
test('surface evidence checks defer to one exploratory execution sequence', () => {
const source = readFileSync(join(import.meta.dir, '../scripts/resolvers/qa.ts'), 'utf8');
expect(source).toContain('Each probe is one native command/interaction plus checks, excluding bookkeeping');
const positions = ['2. **Decide whether another probe is needed.**', '**Publish before probing.**', '3. Run that exact probe; G enforces the deadline when bounded']
.map(marker => source.indexOf(marker));
expect(positions.every(position => position >= 0)).toBe(true);
expect(positions).toEqual([...positions].sort((a, b) => a - b));
expect(source).toContain('Never batch probes');
expect(source).toContain('Follow the shared exploratory loop\'s order and written checkpoints');
expect(source).toContain('Replay the exact failing command/request from the same initial fixture state');
expect(source).toContain('Another input or a regression test is not that replay');
const shared = generateQAExploratory({ host: 'claude', skillName: 'qa', tmplPath: '', paths: HOST_PATHS.claude });
expect(shared).toMatch(/each probe is one native command\/interaction/i);
expect(shared).toMatch(/never batch probes/i);
const functional = readFileSync(join(import.meta.dir, '../qa/sections/system-functional.md'), 'utf8');
expect(functional).toMatch(/follow the shared exploratory loop/i);
});
test('all public callers directly require the functional method before exploration', () => {
@@ -413,13 +373,13 @@ test('all public callers directly require the functional method before explorati
const explorer = readFileSync(join(import.meta.dir, '..', skill === 'qa-only' ? 'qa-only' : 'qa', 'sections/exploratory.md'), 'utf8');
expect(explorer).toMatch(/Functional surfaces[^\n]*\n[^\n]*Read[^\n]*system-functional\.md/);
expect(explorer).toContain('Browser surfaces only');
const stages = ['Read `sections/scope.md`', 'in full and select the surfaces',
'Read `sections/system-functional.md`', 'Write a **charter**', '1. First demonstrate success'];
const stages = ['Read `sections/scope.md`', 'Read `sections/system-functional.md`',
'## 1. Charter and preflight', '1. First demonstrate success'];
const positions = stages.map(stage => explorer.indexOf(stage));
expect(positions.every(position => position >= 0)).toBe(true);
expect(positions).toEqual([...positions].sort((a, b) => a - b));
const functional = readFileSync(join(import.meta.dir, '../qa/sections/system-functional.md'), 'utf8');
expect(functional).toContain('## Contract map');
expect(functional).toContain("Follow the shared exploratory loop's order and written checkpoints");
expect(functional).toMatch(/follow the shared exploratory loop/i);
}
});
+60 -80
View File
@@ -7,14 +7,12 @@ import { generatePlanVerificationExec } from '../scripts/resolvers/plan-gates';
import { HOST_PATHS } from '../scripts/resolvers/types';
function assertPreparation(text: string) {
expect(text).toContain('Complete these Reads in order before writing charters or probing');
expect(text).toMatch(/Await their results before the first probe, never in the same response\.|Await each successful Read result before continuing\./);
expect(text).toContain('Do not repeat a Read already completed in this invocation');
const stages = ['1. Read `sections/scope.md`', 'in full and select the surfaces',
'2. Read the selected surface methods below in full', '**Functional surfaces:**',
'Read `sections/system-functional.md` in full.', '**Browser surfaces only:**',
'Read `sections/qa-patterns.md` in full.', '## 1. Charter and preflight',
'Write a **charter**', 'Start once before baseline:', '1. First demonstrate success'];
expect(text).toMatch(/reads in order before writing charters or probing/i);
expect(text).toMatch(/Await their results before the first probe, never in the same response\.|Await each successful Read result before continuing\./i);
expect(text).toMatch(/do not repeat a Read already completed/i);
const stages = ['Read `sections/scope.md`', '**Functional surfaces:**', 'Read `sections/system-functional.md`',
'**Browser surfaces only:**', 'Read `sections/qa-patterns.md`', '## 1. Charter and preflight',
'bun G start D SECONDS', '1. First demonstrate success'];
const positions = stages.map(stage => text.indexOf(stage));
expect(positions.every(position => position >= 0)).toBe(true);
expect(positions).toEqual([...positions].sort((a, b) => a - b));
@@ -24,36 +22,35 @@ function assertPreparation(text: string) {
}
function assertBoundsAndLayout(text: string) {
for (const contract of [
'Browser Quick: SECONDS=30', 'Browser Full/Regression: SECONDS=900',
'Functional Full, Quick and Regression have no default total timer',
"Set SECONDS to the shorter mode/caller limit",
"an unlimited mode uses the caller\'s bound",
'Without a total time limit, do not start D',
'announce finite command timeouts',
"EARLIER_UTC = caller\'s absolute deadline, if set",
'Clocks/checkpoints use REPORT_DIR; mixed standalone runs use REPORT_DIR/browser and REPORT_DIR/functional, with one final report at REPORT_DIR. Caller paths win.',
'in the probe directory, beside its deadline if bounded',
]) expect(text).toContain(contract);
expect(text.indexOf('Caller paths win.')).toBeLessThan(text.indexOf('Start once before baseline:'));
for (const contract of ['Browser Quick: SECONDS=30', 'Browser Full/Regression: SECONDS=900']) expect(text).toContain(contract);
for (const rule of [
/functional full, quick and regression have no default total timer/i,
/shorter mode\/caller limit/i,
/without a total time limit, do not start D/i,
/finite command timeouts/i,
/mixed standalone runs use REPORT_DIR\/browser and REPORT_DIR\/functional/i,
/one final report at REPORT_DIR/i,
/caller paths win/i,
]) expect(text).toMatch(rule);
expect(text.search(/caller paths win/i)).toBeLessThan(text.indexOf('bun G start D SECONDS'));
}
function assertPlanExecution(text: string, shared = generateQAExploratory({ host: 'claude', skillName: 'qa', tmplPath: '', paths: HOST_PATHS.claude })) {
const step = text.slice(text.indexOf('**3. Run smoke and plan checks.**'), text.indexOf('**4. Check freshness before reporting.**')).replace(/\s+/g, ' ');
for (const contract of [
'Follow the shared Probe loop for smoke checks and replays until the smoke limit',
'Then run required plan checks and revalidation, even after smoke expires',
'using the same procedure but no smoke guard; never reset the clock',
"Use finite command timeouts, capped at the caller\'s remaining time if it has a deadline",
'When the caller\'s deadline expires, mark unfinished checks not-run',
'Await clock/guard results before acting',
]) expect(step).toContain(contract);
for (const rule of [
/shared Probe loop for smoke checks and replays until the smoke limit/i,
/required plan checks and revalidation, even after smoke expires/i,
/same procedure but no smoke guard; never reset the clock/i,
/capped at the caller's remaining time/i,
/deadline expires, mark unfinished checks not-run/i,
/await clock\/guard results before acting/i,
]) expect(step).toMatch(rule);
expect(step.indexOf('Follow the shared Probe loop')).toBeLessThan(step.indexOf('Then run required plan checks'));
expect(step.indexOf('Then run required plan checks')).toBeLessThan(step.indexOf('same procedure'));
for (const contract of ['First demonstrate success: output AND durable effects',
'Wait for successful checkpoint publication before dispatch',
'Replay the exact failing command/request from the same initial fixture state']) {
expect(shared).toContain(contract);
for (const rule of [/demonstrate success: output and durable effects/i,
/wait for successful checkpoint publication before dispatch/i,
/replay the exact failing command\/request from the same initial fixture state/i]) {
expect(shared).toMatch(rule);
}
}
@@ -67,8 +64,7 @@ describe('QA probe entry and checkpoint gates', () => {
expect(old).toContain('Read `sections/system-functional.md`');
const current = generateQAExploratory({ host: 'claude', skillName: 'qa-only', tmplPath: '', paths: HOST_PATHS.claude });
for (const clause of ['## 0. Preparation gate', 'Await each successful Read result before continuing',
'description, section index or remembered method is not a completed instruction Read',
'If either required Read is missing, complete it now before Charter and preflight']) {
'complete it now before Charter and preflight']) {
expect(old).not.toContain(clause);
expect(current).toContain(clause);
}
@@ -88,13 +84,13 @@ describe('QA probe entry and checkpoint gates', () => {
const ctx = { host: host.name, skillName, tmplPath: '', paths: HOST_PATHS[host.name] };
const preflight = generateQAReviewPreflight(ctx).replace(/\s+/g, ' ');
const body = generateQAReview(ctx).replace(/\s+/g, ' ');
expect(preflight).toContain('Resolve QA\'s `sections/...` and `templates/...` paths from that installed QA SKILL.md directory, not the caller or product directory');
expect(preflight).toMatch(/`sections\/\.\.\.` and `templates\/\.\.\.` paths from that installed QA SKILL\.md directory, not the caller or product directory/i);
expect(body).toContain('Read QA\'s `sections/browser-setup.md`');
expect(body).toContain('Read QA\'s `templates/functional-report-template.md`');
if (skillName === 'review') {
expect(body).toContain('Read QA\'s `templates/qa-report-template.md`');
expect(body).toContain('Reuse Step 4\'s surfaces and completed Reads');
expect(body).toContain('Finish missing methods before charters; do not repeat completed Reads');
expect(body).toContain('Step 4\'s surfaces');
expect(body).toMatch(/do not repeat completed Reads/i);
}
}
}
@@ -110,19 +106,14 @@ describe('QA probe entry and checkpoint gates', () => {
const positions = stages.map(stage => text.indexOf(stage));
expect(positions.every(position => position >= 0)).toBe(true);
expect(positions).toEqual([...positions].sort((a, b) => a - b));
expect(text).toContain('No drafts/placeholders');
expect(text).toContain('corrections cannot repair published notes');
expect(text).toMatch(/corrections cannot repair published notes/i);
if (skillName === 'qa-only') {
expect(text).toContain('never invent a substitute path, identity or state');
expect(text).toContain('For actual secrets/private payloads, withhold those values');
expect(text).toContain('and replay limits in the report');
expect(text).toContain('stop the affected probe chain');
expect(text).toContain('An absolute state path is not itself a secret');
expect(text).toMatch(/never invent a substitute path, identity or state/i);
expect(text).toMatch(/secrets\/private payloads, withhold those values/i);
expect(text).toMatch(/stop the affected probe chain/i);
} else {
expect(text).toContain('No drafts/placeholders or invented safe-path redactions');
expect(text).toContain('Withhold unsafe values, disclose limits and stop that chain');
expect(text).toMatch(/withhold unsafe values, disclose limits and stop that chain/i);
}
if (skillName === 'qa-only') expect(text).toContain('If capture is incomplete, report that limit instead of reconstructing it');
}
}
});
@@ -132,13 +123,7 @@ describe('QA probe entry and checkpoint gates', () => {
const text = generateQAExploratory({ host: 'claude', skillName, tmplPath: '', paths: HOST_PATHS.claude });
for (const contract of [
'Reuse resolved REPORT_DIR',
'charters as Markdown in the report',
"Set SECONDS to the shorter mode/caller limit",
'Without a total time limit, do not start D',
'Browser Quick: SECONDS=30',
'Browser Full/Regression: SECONDS=900',
'exactly four top-level fields:', 'observationCommand:', 'observed:', 'hypothesis:', 'nextCommand:',
'observed: its exact decoded child JSON (no wrapper/extra keys), or its full non-JSON text',
'QA_DEADLINE receipts are not observations',
]) expect(text).toContain(contract);
assertBoundsAndLayout(text);
@@ -159,17 +144,15 @@ describe('QA probe entry and checkpoint gates', () => {
const write = text.indexOf('**Publish before probing.**');
expect(decision).toBeGreaterThan(-1);
expect(decision).toBeLessThan(write);
expect(text.slice(decision, write)).toContain('write the report, not a checkpoint');
expect(text.slice(decision, write)).toContain('If expired or no safe next probe remains');
expect(text.slice(decision, write)).toMatch(/write the report, not a checkpoint/i);
expect(text.slice(decision, write)).toMatch(/if expired/i);
expect(text.slice(decision, write)).not.toContain('If done or blocked');
if (skillName === 'qa-only') {
expect(text).toContain('Check fields before publication');
expect(text).toContain('retain the entire result unchanged, including owned fixture paths, IDs, hashes');
expect(text).toMatch(/retain the entire result unchanged/i);
} else {
expect(text).toContain('Preserve every safe program-JSON key/value');
expect(text).toContain('Preserve every safe program-JSON key/value and identity hash unchanged');
expect(text).toMatch(/preserve every safe program-JSON key\/value and identity hash unchanged/i);
}
expect(text).toContain('Q supplies observed; never transcribe it');
expect(text).toMatch(/Q supplies observed; never transcribe it/i);
}
}
});
@@ -182,20 +165,16 @@ describe('QA probe entry and checkpoint gates', () => {
expect(text).toContain('/gstack-qa-deadline');
expect(text).toContain('bun G start D SECONDS [EARLIER_UTC]');
expect(text).toContain('Bounded browsers: `bun G run D -- COMMAND ARGS`');
expect(text).toContain('Never reset D/bypass G');
expect(text).toContain('invalid/missing D stops probes');
expect(steps[0]).toContain('demonstrate success: output AND durable effects');
expect(steps[0]).toContain('Guard if bounded; await completion');
expect(steps[1]).toContain('If bounded, run `bun G status D`');
expect(text).toMatch(/never reset D\/bypass G/i);
expect(text).toMatch(/invalid\/missing D stops probes/i);
expect(steps[0]).toMatch(/demonstrate success: output and durable effects/i);
expect(steps[1]).toContain('`bun G status D`');
expect(steps[1].indexOf('If expired')).toBeLessThan(steps[1].indexOf('**Publish before probing.**'));
expect(steps[1]).toContain('STOP exploration; write the report, not a checkpoint');
expect(steps[2]).toContain('Run that exact probe; G enforces the deadline when bounded');
expect(steps[1]).toMatch(/\bstop exploration\b[^.]*write the report/i);
expect(steps[2]).toContain('G enforces the deadline');
expect(steps[2]).toContain('Report refusals as not-run');
expect(steps[2]).toMatch(/refusals as not-run/i);
expect(steps[3]).toContain('via steps 2–3');
expect(steps[3]).toContain('then minimize via those gates');
expect(steps[3]).toContain('Expiry leaves confirmation/minimization incomplete');
expect(steps[4]).toContain('return to step 2 for each affected revalidation');
expect(steps[4]).toContain('return to step 2');
}
});
@@ -204,25 +183,25 @@ describe('QA probe entry and checkpoint gates', () => {
for (const skillName of ['review', 'ship']) {
const ctx = { host: host.name, skillName, tmplPath: '', paths: HOST_PATHS[host.name] };
const text = (skillName === 'review' ? generateQAReviewPreflight(ctx) : '') + generateQAReview(ctx);
expect(text).toContain('> **STOP.** Before any probe, including plan checks, complete the ordered scope/method Reads below');
const gate = text.search(/before any probe, including plan checks, complete the ordered scope\/method Reads/i);
expect(gate).toBeGreaterThan(-1);
expect(text).toContain('{{QA_RESOURCE:exploratory}}');
expect(text).not.toContain('{{QA_RESOURCE:scope}}');
expect(text).not.toContain('Read `sections/system-functional.md`');
const shared = generateQAExploratory({ ...ctx, skillName: 'qa' });
assertPreparation(shared);
expect(text).toContain('Before any probe, including plan checks');
assertPlanExecution(text);
const required = text.indexOf(skillName === 'review'
? '**2. Check readiness and list required checks.**' : '**2. List required checks');
expect(required).toBeGreaterThan(-1);
expect(text.indexOf('> **STOP.**')).toBeLessThan(required);
expect(gate).toBeLessThan(required);
if (skillName === 'review') {
const isolation = text.indexOf('**1. Set the charter and isolation.**');
const setup = text.indexOf("Read QA's `sections/browser-setup.md`");
expect(isolation).toBeGreaterThan(text.indexOf('> **STOP.**'));
expect(isolation).toBeGreaterThan(gate);
expect(setup).toBeGreaterThan(required);
expect(text.slice(isolation, required).replace(/\s+/g, ' ')).toContain('complete the shared isolation/permission preflight before setup');
expect(text).toContain('Step 4 is read-only: defer charters, setup and probes to Step 4.7');
expect(text.slice(isolation, required).replace(/\s+/g, ' ')).toMatch(/isolation\/permission preflight before setup/i);
expect(text).toMatch(/Step 4 is read-only: defer charters, setup and probes to Step 4\.7/i);
} else {
const setup = text.indexOf("For browsers, Read QA's `sections/browser-setup.md`");
expect(setup).toBeGreaterThan(required);
@@ -239,8 +218,9 @@ describe('QA probe entry and checkpoint gates', () => {
for (const changed of [
method + '\n' + text.replace(method, ''),
text.replace(method, method + '\n' + method),
'Write a **charter**\n' + text,
'## 1. Charter and preflight\n' + text,
text.replace('Do not repeat a Read already completed in this invocation', 'Repeat all Reads'),
text.replace('Read `sections/qa-patterns.md` in full.', '').replace('## 1. Charter and preflight', '## 1. Charter and preflight\nRead `sections/qa-patterns.md` in full.'),
// ci-36641820398-1-gate-census-7 ship-exploratory-small-cli dispatched its first probe with the resource Reads.
text.replace(' Await their results before the first probe, never in the same response.', ''),
]) expect(() => assertPreparation(changed)).toThrow();
@@ -303,8 +283,8 @@ describe('QA probe entry and checkpoint gates', () => {
test('plan execution waits for the actual shared method reads, not just collection', () => {
const text = generatePlanVerificationExec({ host: 'claude', skillName: 'ship', tmplPath: '', paths: HOST_PATHS.claude });
expect(text).toContain('Do not invoke an entire QA skill or start probes here');
expect(text).toMatch(/do not invoke an entire QA skill or start probes here/i);
expect(text).toContain('Before the first plan command, complete Step 9.2.1');
expect(text).toContain('method Reads and the shared probe loop');
expect(text).toMatch(/method Reads and the shared probe loop/i);
});
});
+158 -268
View File
@@ -4,7 +4,7 @@ import { join } from 'node:path';
import { generateReviewArmy } from '../scripts/resolvers/review-army';
import { generateCrossReviewDedup, generateSharedCodeReuse, generateScopeDrift } from '../scripts/resolvers/review-scope';
import { generatePlanCompletionAuditReview, generatePlanCompletionAuditShip } from '../scripts/resolvers/plan-gates';
import { generateQAExploratory, generateQAReview } from '../scripts/resolvers/qa';
import { generateQAReview } from '../scripts/resolvers/qa';
import { generateConfidenceCalibration } from '../scripts/resolvers/confidence';
import { HOST_PATHS, type TemplateContext } from '../scripts/resolvers/types';
@@ -15,36 +15,27 @@ const adversarial = readFileSync(join(root, 'review/sections/adversarial.md.tmpl
test('review audits deliverables before deferring behavioral plan checks to the QA preflight', () => {
const ctx: TemplateContext = { skillName: 'review', tmplPath: '', host: 'claude', paths: HOST_PATHS.claude };
const audit = generatePlanCompletionAuditReview(ctx).replace(/\s+/g, ' ');
for (const contract of [
'Separate static audit evidence from behavioral checks',
'retain the exact command, expected outcome and source for Step 4.7',
'They remain pending execution, never DONE from a diff',
'A mixed item contributes to both lists',
'Zero audited deliverables do not waive these checks',
'Keep external-state and human-only checks under the existing audit rules',
'If only behavioral checks remain, report zero audited deliverables and retain their pending Step 4.7 list',
'For each audited deliverable, run the verification dispatch',
'File-existence checks and verified read-only content validators are static audit checks, not behavioral probes',
'Inspect the validator and its hooks before running it; verify read-only effects and access to the target',
'leave the item UNVERIFIABLE and defer the command to Step 4.7',
'Do not start applications, exercise APIs or mutate state during this audit',
'If found and verified safe above, invoke it',
]) expect(audit).toContain(contract);
expect(audit.indexOf('Inspect the validator and its hooks')).toBeLessThan(audit.indexOf('If found and verified safe above, invoke it'));
for (const rule of [
/separate static audit evidence from behavioral checks/i,
/never DONE from a diff/,
/zero audited deliverables do not waive these checks/i,
/for each audited deliverable/i,
/leave the item UNVERIFIABLE and defer the command to Step 4\.7/,
/do not start applications, exercise APIs or mutate state during this audit/i,
]) expect(audit).toMatch(rule);
const inspect = audit.search(/inspect the validator and its hooks before running it/i);
expect(inspect).toBeGreaterThan(-1);
expect(inspect).toBeLessThan(audit.search(/if found and verified safe above, invoke it/i));
expect(audit).not.toContain('For each extracted plan item, run the verification dispatch');
const qa = generateQAReview(ctx);
expect(qa).toContain('Then run required plan checks and revalidation, even after smoke expires');
expect(qa).toContain('Report clean/completed only when all required checks pass on current inputs');
});
test('review prior-Skip matching includes adversarial and Greptile findings without relaxing eligibility', () => {
const dedup = generateCrossReviewDedup({ skillName: 'review', tmplPath: '', host: 'claude', paths: HOST_PATHS.claude });
expect(dedup).toContain('For every combined finding, including core, specialist, exploratory QA, adversarial and valid actionable Greptile findings, check:');
expect(dedup).toContain('Suppress only when all conditions hold: the user skipped the same unchanged finding');
expect(dedup).toContain('same advisory/defect kind');
expect(dedup).toContain('Never use a skipped advisory to suppress a real defect');
expect(dedup).toContain('Only suppress `skipped` findings — never `fixed` or `auto-fixed`');
expect(skill).toContain('Run Step 5.0 severity/prior-skip dedup on all');
expect(dedup).toMatch(/for every combined finding, including core, specialist, exploratory QA, adversarial and valid actionable Greptile findings/i);
expect(dedup).toMatch(/suppress only when all conditions hold: the user skipped the same unchanged finding/i);
expect(dedup).toMatch(/never use a skipped advisory to suppress a real defect/i);
expect(dedup).toMatch(/only suppress `skipped` findings — never `fixed` or `auto-fixed`/i);
expect(skill).toContain('Step 5.0 severity/prior-skip dedup');
});
test('Review audit and prior-Skip clarifications do not route Ship through Review steps', () => {
@@ -74,42 +65,40 @@ test('review collects every source before its single parent fix phase', () => {
expect(positions.every(position => position >= 0)).toBe(true);
expect(positions).toEqual([...positions].sort((a, b) => a - b));
expect(skill.match(/## Step 5: Fix-First Review/g)).toHaveLength(1);
expect(skill.replace(/\s+/g, ' ')).toContain('Do not edit reviewed source until Step 5');
expect(skill.replace(/\s+/g, ' ')).toContain('every dispatched reader has returned or is confirmed stopped');
expect(skill.replace(/\s+/g, ' ')).toMatch(/do not edit reviewed source until Step 5/i);
expect(skill.replace(/\s+/g, ' ')).toMatch(/every dispatched reader has returned or is confirmed stopped/i);
});
test('review settles adversarial attempts before fixing and has one full-pass back edge', () => {
const generated = readFileSync(join(root, 'review/sections/adversarial.md'), 'utf8');
const decision = skill.slice(skill.indexOf('## Step 5.8: Persist Eng Review result')).replace(/\s+/g, ' ');
expect(generated).toContain('## Step 4.8: Adversarial review');
expect(generated).toContain("queued for the parent's Fix-First handling at Step 5; do not edit during Step 4.8");
expect(generated.replace(/\s+/g, ' ')).toContain('before the parent applies queued fixes');
expect(generated).toContain('Return all findings and structured-review decisions to Step 5');
expect(decision).toContain('A pass covers Steps 3–5, including all reviewers before fixes');
expect(generated).toMatch(/do not edit during Step 4\.8/i);
expect(generated).toMatch(/return all findings and structured-review decisions to Step 5/i);
expect(decision).toMatch(/a pass covers Steps 3–5/i);
expect(decision).toContain('Below 3, repeat Steps 3–5 with a new REVIEW_START');
expect(decision).not.toContain('Route Step');
expect(decision).not.toContain('Steps 5.0–5d');
expect(decision).toContain('without a clean summary or a fourth pass');
expect(decision).toMatch(/without a clean summary or a fourth pass/i);
});
test('review small-diff and failed-reader paths retain QA and the required adversarial pass', () => {
const army = generateReviewArmy({ skillName: 'review', tmplPath: 'review/SKILL.md.tmpl',
host: 'claude', paths: HOST_PATHS.claude });
expect(army).toContain("Continue to Step 4.6 with the core findings and an empty specialist list, then the parent's Exploratory QA step and Step 4.8 (adversarial review), then Step 5");
expect(army.replace(/\s+/g, ' ')).toMatch(/Continue to Step 4\.6 [^.]*Exploratory QA step and Step 4\.8 \(adversarial review\), then Step 5/i);
expect(army).not.toContain('design-lite');
expect(army).toContain('Missing dispatched coverage remains incomplete, never completed or clean');
expect(army).toContain('Continue independent Step 4.7 QA and Step 4.8 adversarial review');
expect(army.replace(/\s+/g, ' ')).toMatch(/missing dispatched coverage remains incomplete, never completed or clean/i);
expect(army.replace(/\s+/g, ' ')).toMatch(/independent Step 4\.7 QA and Step 4\.8 adversarial review/i);
expect(army).not.toContain("Exploratory QA step, then continue to Step 5.");
expect(army).not.toContain('If the Red Team subagent fails or times out, skip silently');
});
test('review defines QA confidence, severity, impact selection and numeric version comparison', () => {
const flat = skill.replace(/\s+/g, ' ');
expect(flat).toContain('For QA findings, assign confidence (1–10) from replay/code evidence');
expect(flat).toContain("retain Step 4.7's severity, not a severity inferred from confidence");
expect(flat).toContain('A probe is affected when its entrypoint, dependencies, contract or replay inputs change');
expect(flat).toContain('If impact is uncertain, rerun it');
expect(flat).toContain('Compare dotted version components as integers from left to right');
expect(flat).toContain('confidence (1–10)');
expect(flat).toMatch(/retain Step 4\.7's severity/i);
expect(flat).toMatch(/if impact is uncertain, rerun it/i);
expect(flat).toMatch(/compare dotted version components as integers/i);
});
test('review emits scope check after the plan audit and before the checklist', () => {
@@ -121,36 +110,30 @@ test('review emits scope check after the plan audit and before the checklist', (
expect(positions.every(position => position >= 0)).toBe(true);
expect(positions).toEqual([...positions].sort((a, b) => a - b));
const audit = readFileSync(join(root, 'review/sections/plan-completion.md'), 'utf8').replace(/\s+/g, ' ');
expect(audit).toContain('When continuing after the audit (no HIGH-impact gate, or option B/C), emit the single final Scope Check');
expect(audit).toContain("Step 1.5's provisional notes and this plan context");
expect(audit).toContain('Emit Step 1.5\'s Scope Check once without plan fields');
expect(audit).toMatch(/emit the single final Scope Check/i);
expect(skill).not.toContain('Finish Step 1.5 here');
});
test('review composes confidence-tagged findings into one final report with explicit incomplete coverage', () => {
const flat = skill.replace(/\s+/g, ' ');
expect(flat).toContain('Use CRITICAL/INFORMATIONAL labels in the finding format');
expect(flat).toContain("Step 5.8 combines these finding lines with the checklist's action groups");
expect(flat).toContain('CRITICAL/INFORMATIONAL');
const report = flat.slice(flat.indexOf('### Report the final review'), flat.indexOf('{{LEARNINGS_LOG}}'));
expect(report).toContain('Emit one final report, merging all reviewers rather than concatenating their reports');
expect(report).toContain('counts final unresolved non-advisory defects');
expect(report).toContain('State INCOMPLETE if `COMPLETED` is false, even when N=0');
expect(report).toContain("Use the checklist's action groups with confidence-tagged finding lines");
expect(report).toContain('Keep fixed, skipped and advisory items separate from unresolved defects; retain their dispositions');
expect(report).toContain("Append Step 4.7's single `## Exploratory QA and Verification Results` section");
expect(report).toContain('Neither coverage gaps nor advice are defects');
expect(report).toMatch(/one final report/i);
expect(report).toMatch(/unresolved non-advisory defects/i);
expect(report).toMatch(/[Ss]tate INCOMPLETE if `COMPLETED` is false/);
expect(report).toContain('`## Exploratory QA and Verification Results`');
expect(report).toMatch(/neither coverage gaps nor advice are defects/i);
});
test('small-diff persistence uses an empty specialist map without manufacturing skipped coverage', () => {
const army = generateReviewArmy({ skillName: 'review', tmplPath: 'review/SKILL.md.tmpl',
host: 'claude', paths: HOST_PATHS.claude });
expect(army).toContain('For DIFF_LINES < 50, keep `specialists: {}`; do not manufacture per-specialist scope records');
expect(army).toContain('Otherwise record each considered specialist');
expect(skill).toContain("Use Step 4.6's `specialists` object unchanged, including its empty small-diff map");
expect(army).toContain('DIFF_LINES < 50, keep `specialists: {}`');
expect(skill.replace(/\s+/g, ' ')).toMatch(/Step 4\.6's `specialists` object unchanged/i);
const ship = readFileSync(join(root, 'ship/sections/review-army.md.tmpl'), 'utf8');
expect(ship).toContain('`specialists`: `{}` for a small-diff skip');
expect(skill).toContain('every required Step 4.7 probe passes');
expect(ship).toContain('all required probes pass');
expect(skill.replace(/\s+/g, ' ')).toMatch(/every required Step 4\.7 probe passes/i);
expect(ship.replace(/\s+/g, ' ')).toMatch(/all required probes pass/i);
});
test('caller QA runs charter and setup after resource loading and has a severity for unmatched functional failures', () => {
@@ -165,13 +148,13 @@ test('caller QA runs charter and setup after resource loading and has a severity
const readiness = body.indexOf('**2. Check readiness and list required checks.**');
expect(readiness).toBeGreaterThan(preparation);
expect(probes).toBeGreaterThan(readiness);
expect(body.slice(preparation, readiness).replace(/\s+/g, ' ')).toContain('complete the shared isolation/permission preflight before setup');
expect(body.slice(preparation, readiness).replace(/\s+/g, ' ')).toMatch(/isolation\/permission preflight before setup/i);
} else expect(preparation).toBeGreaterThan(body.indexOf('**2. List required checks.**'));
expect(probes).toBeGreaterThan(preparation);
expect(body.replace(/\s+/g, ' ')).toContain('unmatched functional failures are `functional-contract`, `CRITICAL`');
expect(body).toContain('Setup/permission blockers are not defects');
expect(body).toContain('Test creation needs user approval');
expect(body).toContain('Return verified defects to Fix-First');
expect(body.replace(/\s+/g, ' ')).toContain('`functional-contract`, `CRITICAL`');
expect(body).toMatch(/setup\/permission blockers are not defects/i);
expect(body).toMatch(/test creation needs user approval/i);
expect(body).toMatch(/return verified defects to Fix-First/i);
expect(body).not.toContain('for parent approval');
}
});
@@ -185,34 +168,24 @@ test('review prepares context and deduplicates before classifying findings', ()
].map(marker => skill.indexOf(marker));
expect(positions.every(position => position >= 0)).toBe(true);
expect(positions).toEqual([...positions].sort((a, b) => a - b));
expect(skill).toContain('findings before Step 5a classification');
expect(skill).toMatch(/before Step 5a classification/i);
});
test('review owns the complete persistence contract after the adversarial read', () => {
const step = skill.slice(skill.indexOf('## Step 5.8: Persist Eng Review result'));
expect(skill.indexOf('{{SECTION:adversarial}}')).toBeLessThan(skill.indexOf('## Step 5.8: Persist Eng Review result'));
expect(adversarial).not.toContain('### Before persisting Eng Review (Step 5.8)');
for (const contract of [
'repeat Steps 3–5', 'at most 3 fix cycles', 'final zero-edit pass, reconcile',
'by structural identity', 'advisory/defect kind',
'original `evidence_paths`/`helper_target`', 'without requiring deleted pre-extraction blocks',
'Current findings determine recurring defects and unresolved counts', 'earlier fixes do not suppress them',
'snapshot_covered_paths',
'raw bytes equal the bound snapshot blobs', 'prior-cycle, supplied or prior-record coverage',
'REVIEW_START', 'COMPLETED', 'CONVERGED', 'CYCLES', 'Step 4.7',
'native Step 4.8 adversarial pass', 'means false, as does a failed native review',
'optional outside', 'their own incomplete records when unavailable', 'named-risk',
'zero counts', '`completed:false`', '`specialists`', '`findings`',
'verified exploratory QA findings',
'approved **and completed**', 'explicit Skip', 'sharedLibsFingerprint',
'`review_binding`', 'validated captured branch',
]) expect(step.toLowerCase().replace(/\s+/g, ' ')).toContain(contract.toLowerCase());
for (const token of [
'repeat Steps 3–5', 'at most 3 fix cycles', 'original `evidence_paths`/`helper_target`',
'snapshot_covered_paths', 'REVIEW_START', 'COMPLETED', 'CONVERGED', 'CYCLES', 'Step 4.7',
'native Step 4.8 adversarial pass', 'named-risk', 'zero counts', '`completed:false`',
'`specialists`', '`findings`', 'explicit Skip', 'sharedLibsFingerprint', '`review_binding`',
]) expect(step.toLowerCase().replace(/\s+/g, ' ')).toContain(token.toLowerCase());
expect(step.indexOf('### 1. Re-review after edits'))
.toBeLessThan(step.indexOf('### 2. Fill the record'));
expect(step.indexOf('### 2. Fill the record'))
.toBeLessThan(step.indexOf('~/.claude/skills/gstack/bin/gstack-review-log'));
expect(step).toContain('`quality_score` is Step 4.6\'s specialist score');
expect(step).toContain('unresolved non-advisory core defects still count');
expect(step).toContain('`quality_score`');
expect(step.indexOf('Pre-Landing Review: N issues (X critical, Y informational)'))
.toBeGreaterThan(step.indexOf('~/.claude/skills/gstack/bin/gstack-review-log'));
expect(step).toContain('`## Exploratory QA and Verification Results`');
@@ -220,26 +193,20 @@ test('review owns the complete persistence contract after the adversarial read',
test('review distinguishes required native coverage from optional outside coverage', () => {
const section = readFileSync(join(root, 'review/sections/adversarial.md'), 'utf8');
expect(section).toContain('Only this optional outside adversarial pass is non-blocking');
expect(section).toMatch(/only this optional outside adversarial pass is non-blocking/i);
expect(section).not.toContain('All errors are non-blocking');
expect(section).toContain('The native pass is required for Step 5.8 completion');
expect(skill).toContain('Core findings use the confidence gates below');
expect(skill).toContain('Step 4.6 applies its specialist gates');
expect(section).toMatch(/native pass is required for Step 5\.8 completion/i);
});
test('review identifies probe selection, report assets and the detected diff base', () => {
const generated = generateQAReview({ skillName: 'review', tmplPath: 'review/SKILL.md.tmpl',
host: 'claude', paths: HOST_PATHS.claude });
const checklist = readFileSync(join(root, 'review/checklist.md'), 'utf8');
expect(generated).toContain('Smoke: 5 minutes/12 probes, one success and the riskiest changed failure/edge');
expect(generated).toContain('Required even for small diffs or missing plans/servers');
expect(generated.replace(/\s+/g, ' ')).toContain("Use checklist severity; unmatched functional failures are `functional-contract`, `CRITICAL`");
expect(generated).toContain('Setup/permission blockers are not defects');
expect(generated).toContain('Test creation needs user approval');
expect(generated).toContain("Read QA\'s `templates/functional-report-template.md`. Title it");
expect(generated).toContain('Link every checkpoint');
expect(generated).toContain('No second report');
expect(checklist).toContain('merge-base diff from the caller');
expect(generated).toContain('5 minutes/12 probes');
expect(generated).toContain("Read QA\'s `templates/functional-report-template.md`");
expect(generated).toMatch(/link every checkpoint/i);
expect(generated).toMatch(/no second report/i);
expect(checklist).toMatch(/merge-base diff from the caller/i);
expect(checklist).not.toContain('git diff origin/main');
});
@@ -247,37 +214,16 @@ test('caller QA defines execution, evidence ownership and report adaptation befo
for (const skillName of ['review', 'ship']) {
const generated = generateQAReview({ skillName, tmplPath: `${skillName}/SKILL.md.tmpl`,
host: 'claude', paths: HOST_PATHS.claude }).replace(/\s+/g, ' ');
for (const contract of [
'Only the parent runs report-only discovery',
'Never overwrite another run',
'Follow the shared Probe loop for smoke checks and replays until the smoke limit',
'using the same procedure but no smoke guard; never reset the clock',
'Read agent/user updates and await results without batching them with reporting/logging',
'Compare each probe\'s recorded source, tests, contracts, commands and fixtures (or input fingerprint) with current inputs, even without updates',
'Re-review changed or uncertain coverage',
"Use checklist severity",
]) expect(generated).toContain(contract);
const shared = generateQAExploratory({ skillName: 'qa', tmplPath: '', host: 'claude', paths: HOST_PATHS.claude });
for (const contract of ['First demonstrate success: output AND durable effects',
'Wait for successful checkpoint publication before dispatch',
'Replay the exact failing command/request from the same initial fixture state']) {
expect(shared).toContain(contract);
}
expect(generated).toMatch(/never overwrite another run/i);
expect(generated).toMatch(/with current inputs, even without updates/i);
expect(generated).toContain('Use checklist severity');
if (skillName === 'review') {
expect(generated).toContain('Title it `## Exploratory QA and Verification Results`');
expect(generated).toContain('keep metadata/outcome tables');
expect(generated).toContain('demote other headings one level');
expect(generated).toContain('include it here under `### Browser results`');
expect(generated).toContain('other headings demoted two levels');
expect(generated).toContain('Keep browser/functional scores and outcomes separate');
expect(generated).toContain('save browser baseline/evidence normally');
expect(generated).toContain('Prepare one provisional QA section');
expect(generated).toContain('Update affected outcomes/checkpoint links through repairs/revalidation');
expect(generated).toContain('Continue to Step 4.8 even if blocked');
expect(generated).toContain('Step 5.8 appends this section once after final findings and decides completion');
expect(generated).toContain('`## Exploratory QA and Verification Results`');
expect(generated).toContain('`### Browser results`');
expect(generated).toMatch(/continue to Step 4\.8 even if blocked/i);
expect(generated).toMatch(/Step 5\.8 appends this section once/i);
} else {
expect(generated).toContain('PR section `## Exploratory QA`');
expect(generated).toContain('fields as subsections');
expect(generated).toContain('`## Exploratory QA`');
}
}
});
@@ -291,15 +237,14 @@ test('review section index follows the actual pre-fix execution order', () => {
test('review finalization ownership: initialize invocation state and capture the core token before reading', () => {
const start = skill.slice(skill.indexOf('## Step 3: Get the diff'), skill.indexOf('## Step 3.4:'));
expect(start).toContain('one invocation action list and CYCLES=0');
expect(start).toContain('Keep both through re-reviews');
expect(start).toContain('CYCLES=0');
expect(skill.match(/CYCLES=0/g)).toHaveLength(1);
expect(start).toContain('gstack-review-log --start review\ngit diff "$DIFF_BASE"');
expect(start).toContain('Save the printed REVIEW_START for this core candidate before reading its diff');
expect(start).toContain('Each re-review captures a new token before reading, never at log time');
expect(start.replace(/\s+/g, ' ')).toContain('Earlier core tokens remain unused');
expect(start).toContain('Native/outside reviewer attempts own separate PASS_START tokens, not REVIEW_START');
expect(start).toContain('Step 5.8 finishes only the final core token');
const flat = start.replace(/\s+/g, ' ');
expect(flat).toMatch(/REVIEW_START for this core candidate before reading its diff/i);
expect(flat).toMatch(/new token before reading, never at log time/i);
expect(flat).toMatch(/separate PASS_START tokens, not REVIEW_START/);
expect(flat).toMatch(/finishes only the final core token/i);
});
test('review finalization ownership: late findings use Fix-First before the bounded parent transition', () => {
@@ -317,46 +262,31 @@ test('review finalization ownership: late findings use Fix-First before the boun
const positions = markers.map(marker => flat.indexOf(marker));
expect(positions.every(position => position >= 0)).toBe(true);
expect(positions).toEqual([...positions].sort((a, b) => a - b));
expect(flat).toContain('Structured approval does not waive advisory/test_stub ASK gates');
expect(flat).toContain('A pass covers Steps 3–5, including all reviewers before fixes');
expect(flat).toContain('at most 3 fix cycles');
expect(flat).toContain('Below 3, repeat Steps 3–5 with a new REVIEW_START');
expect(flat).toContain('At 3, persist `converged:false` and remaining findings');
expect(flat).toContain('At 3, persist `converged:false`');
const limit = flat.slice(flat.indexOf('At 3,'), flat.indexOf('No edits:'));
expect(limit).toContain('by filling and saving the record below');
expect(limit).toContain('Report nonconvergence and coverage gaps, then STOP this invocation');
expect(limit).toContain('without a clean summary or a fourth pass');
expect(limit).toMatch(/then stop this invocation/i);
expect(limit).toMatch(/without a clean summary or a fourth pass/i);
});
test('review finalization ownership: affected QA reuse cannot replace a full review or erase decisions', () => {
const step = skill.slice(skill.indexOf('## Step 5.8: Persist Eng Review result'));
const flat = step.replace(/\s+/g, ' ');
expect(flat).toContain('On a repeat, execute Steps 3–5 in order');
expect(flat).toContain('rerun affected probes after source, test, contract, command or fixture changes');
expect(flat).toContain("At Step 4.7, reuse only this invocation's unchanged-input QA evidence");
expect(flat).toContain('Reusing a probe never skips a review step');
expect(flat).toContain("final zero-edit pass, reconcile this invocation's actions with current findings");
expect(flat).toContain('original `evidence_paths`/`helper_target`');
expect(flat).toContain('Current findings determine recurring defects and unresolved counts; earlier fixes do not suppress them');
expect(flat).toContain('Re-read its final-snapshot supporting source and reconfirm the decision');
expect(flat).toContain('otherwise report its history without a reusable skip');
expect(flat).toContain('The logger computes `snapshot_covered_paths` from eligible paths whose raw bytes equal the bound snapshot blobs');
expect(flat).toContain('Never carry prior-cycle, supplied or prior-record coverage forward or build this proof yourself');
expect(flat).toContain('Fixed advice needs no skip coverage');
expect(flat).toContain('include this invocation\'s revalidated decisions');
expect(flat).toContain('execute Steps 3–5 in order');
expect(flat).toMatch(/reuse only this invocation's unchanged-input QA evidence/i);
expect(flat).toMatch(/reusing a probe never skips a review step/i);
expect(flat).toMatch(/earlier fixes do not suppress them/i);
expect(flat).toContain('logger computes `snapshot_covered_paths`');
expect(flat).toMatch(/never carry prior-cycle, supplied or prior-record coverage forward or build this proof yourself/i);
});
test('review finalization ownership: required native completion and optional outside records stay separate', () => {
const step = skill.slice(skill.indexOf('### 2. Fill the record'));
const flat = step.replace(/\s+/g, ' ');
expect(flat).toContain('native Step 4.8 adversarial pass finish');
expect(flat).toContain('every required Step 4.7 probe passes');
expect(flat).toContain('Any failed, blocked, inconclusive or not-run required probe means false, as does a failed native review');
expect(flat).toContain('`/ship` named-risk acceptance cannot complete `/review`');
expect(flat).toContain('The required in-host adversarial result controls native completion');
expect(flat).toContain('Optional outside attempts keep their own incomplete records when unavailable');
expect(flat).toContain('cannot substitute for the native result, or vice versa');
expect(flat).toContain('structured-review gate still applies');
expect(flat).toMatch(/any failed, blocked, inconclusive or not-run required probe means false, as does a failed native review/i);
expect(flat).toMatch(/`\/ship` named-risk acceptance cannot complete `\/review`/i);
expect(flat).toMatch(/cannot substitute for the native result, or vice versa/i);
expect(flat).toMatch(/structured-review gate still applies/i);
expect(flat).toContain('zero counts and `completed:false`');
expect(flat).toContain('`CONVERGED`: true only for a completed zero-edit pass');
});
@@ -367,35 +297,33 @@ test('review finalization ownership: finish only the final core token without lo
expect(step.match(/--finish REVIEW_START/g)).toHaveLength(1);
expect(step).not.toContain('--start review');
expect(step).not.toContain('--finish PASS_START');
expect(flat).toContain('Never invent a binding or replace REVIEW_START at log time');
expect(flat).toContain('finish only the final core token');
expect(flat).toMatch(/never invent a binding or replace REVIEW_START at log time/i);
expect(flat).toMatch(/finish only the final core token/i);
expect(step).toContain('"completed":COMPLETED,"converged":CONVERGED,"cycles":CYCLES');
});
test('review finalization ownership: the plan audit retains its high-impact gate before the final scope check', () => {
const plan = readFileSync(join(root, 'review/sections/plan-completion.md.tmpl'), 'utf8');
expect(plan).toContain('INFORMATIONAL except for the HIGH-impact discrepancy question below');
expect(plan).toContain('resolve that gate before the final Scope Check');
expect(plan).toContain('HIGH-impact discrepancy question');
expect(plan).toMatch(/resolve that gate before the final Scope Check/i);
expect(plan).not.toContain('never blocks the review');
expect(plan).toContain('{{PLAN_COMPLETION_AUDIT_REVIEW}}');
const audit = readFileSync(join(root, 'review/sections/plan-completion.md'), 'utf8');
const gate = audit.indexOf('**HIGH-impact plan-file discrepancies** trigger AskUserQuestion');
expect(gate).toBeGreaterThan(-1);
expect(gate).toBeLessThan(audit.indexOf('When continuing after the audit (no HIGH-impact gate, or option B/C)'));
expect(audit).toContain('then it gates via AskUserQuestion');
expect(audit).toContain('A ends this invocation before code review or implementation');
expect(audit).toContain('after implementation, start a fresh /review');
expect(audit).toContain('B queues the approved TODO changes for Step 5, not this read-only audit');
expect(audit).toContain('B/C continue to the final Scope Check and Step 2');
expect(audit).toContain('None of these choices authorizes shipping or waives required verification');
expect(skill).toContain("including Step 1.5's approved TODO changes");
const flatAudit = audit.replace(/\s+/g, ' ');
expect(flatAudit).toMatch(/A ends this invocation before code review or implementation/i);
expect(flatAudit).toMatch(/not this read-only audit/i);
expect(flatAudit).toContain('B/C continue to the final Scope Check and Step 2');
expect(flatAudit).toMatch(/none of these choices authorizes shipping or waives required verification/i);
});
test('review scope notes remain provisional until the plan section emits the only final scope check', () => {
const scope = generateScopeDrift({ skillName: 'review', tmplPath: 'review/SKILL.md.tmpl',
host: 'claude', paths: HOST_PATHS.claude }).replace(/\s+/g, ' ');
expect(scope).toContain('Keep these notes provisional. Next, execute the plan-completion section');
expect(scope).toContain('it resolves the HIGH-impact decision and emits the single final Scope Check before Step 2');
expect(scope).toMatch(/keep these notes provisional/i);
expect(scope).toMatch(/single final Scope Check before Step 2/i);
expect(scope).not.toContain('Scope Check: [CLEAN');
expect(scope).not.toContain('available plan-audit results');
});
@@ -404,29 +332,20 @@ test('review confidence uses its severity labels without an undefined P0 excepti
const ctx: TemplateContext = { skillName: 'review', tmplPath: 'review/SKILL.md.tmpl',
host: 'claude', paths: HOST_PATHS.claude };
const confidence = generateConfidenceCalibration(ctx);
expect(confidence).toContain('Only report a suspected release-blocking catastrophe');
expect(confidence).toContain('widespread data loss, total outage or system-wide compromise');
expect(confidence).toContain('label it CRITICAL and explicitly speculative');
expect(confidence).toContain('[CRITICAL]');
expect(confidence).toContain('[CRITICAL|INFORMATIONAL]');
expect(confidence).not.toMatch(/\bP[012]\b/);
expect(confidence).toContain('If you cannot quote the motivating line(s), the finding is unverified');
const flat = confidence.replace(/\s+/g, ' ');
for (const rule of [
'| 9-10 | Specific code verifies a concrete bug or exploit. | Show normally |',
'| 7-8 | High-confidence pattern match; very likely correct. | Show normally |',
'| 5-6 | Moderate; could be a false positive. | Show with caveat:',
'Medium confidence, verify this is actually an issue',
'| 3-4 | Suspicious but may be fine. | Suppress from main report. Include in appendix only. |',
'| 1-2 | Speculation. | Only report a suspected release-blocking catastrophe',
'Quote the specific code line', 'file:line and verbatim text',
'For a missing field, quote its class definition; for a nullable value, its initialization; for a race, both sides',
'Force its confidence to 4-5: use 4 for appendix-only reporting, or 5 only when the finding belongs in the main report with the medium-confidence caveat below',
'Never invent speculative confidence 7+',
'read and quote their generating metaclass, descriptor, ORM Meta block, migration, decorator or schema',
'Missing literal names in the class body or grep results do not prove absence',
'If the user confirms a reported finding scored < 7 is real, log the corrected pattern as a learning',
]) expect(flat).toContain(rule);
for (const row of [
/\| 9-10 \|[^|]*\| Show normally \|/, /\| 7-8 \|[^|]*\| Show normally \|/,
/\| 5-6 \|[^|]*\| Show with caveat/, /\| 3-4 \|[^|]*\| Suppress from main report/,
/\| 1-2 \|[^|]*\| Only report a suspected release-blocking catastrophe/,
]) expect(flat).toMatch(row);
expect(flat).toMatch(/explicitly speculative/i);
expect(flat).toMatch(/the finding is unverified/i);
expect(flat).toContain('file:line');
expect(flat).toMatch(/force its confidence to 4-5/i);
expect(flat).toMatch(/never invent speculative confidence 7\+/i);
expect(confidence.indexOf('Pre-emit verification gate')).toBeLessThan(confidence.indexOf('| Score |'));
expect(confidence).not.toContain('FP classes the gate kills');
expect(confidence).not.toContain('1539-framework-aware-review.md');
@@ -435,28 +354,20 @@ test('review confidence uses its severity labels without an undefined P0 excepti
test('review names the lifecycle and record owners before using their persistence rules', () => {
const start = skill.slice(skill.indexOf('## Step 3: Get the diff'), skill.indexOf('## Step 3.4:')).replace(/\s+/g, ' ');
expect(start).toContain('An invocation is this /review run; a pass reviews one candidate before any fixes');
expect(start).toContain('REVIEW_START / PASS_START | Opaque start receipts from the logger');
expect(start).toContain('a matching key alone never proves a prior Skip is reusable');
expect(start).toContain("`review_binding` | The logger's proof tying a finished review to its captured candidate");
expect(start).toContain('`snapshot_covered_paths` | Supporting advice files the logger proved byte-identical to that candidate');
expect(start).toContain('Used by the prior-Skip checker, never supplied by the reviewer');
expect(start).toContain('REVIEW_START / PASS_START |');
expect(start).toContain('`review_binding` |');
expect(start).toContain('`snapshot_covered_paths` |');
expect(start).toMatch(/a matching key alone never proves a prior Skip is reusable/i);
expect(start).toMatch(/never supplied by the reviewer/i);
});
test('review invocation-local advice reuse keeps raw-source and changed-decision gates', () => {
const decisions = skill.slice(skill.indexOf('**Keep decisions through fix cycles:**'),
skill.indexOf('### Step 5a:')).replace(/\s+/g, ' ');
for (const contract of [
'Immediately save completed AUTO-FIX/fix and explicit Skip actions',
'keeping defects separate from advice',
"retain the helper's fingerprint, `advisory`, `evidence_paths` and `helper_target`",
're-read every supporting caller and helper destination',
'including secondary callers and transformed/indirect paths',
'Compare their raw source with the decision evidence',
'Unrelated auto-fixes do not reopen unchanged identity, contract and tradeoffs',
'Material proposal, behavior, migration or risk changes require a new question',
"cannot suppress new/recurring defects or replace Step 5.0's prior-review checker",
]) expect(decisions).toContain(contract);
expect(decisions).toMatch(/save completed AUTO-FIX\/fix and explicit Skip actions/i);
expect(decisions).toContain("`advisory`, `evidence_paths` and `helper_target`");
expect(decisions).toMatch(/require a new question/i);
expect(decisions).toMatch(/cannot suppress new\/recurring defects/i);
});
for (const skillName of ['review', 'ship']) {
@@ -466,16 +377,14 @@ for (const skillName of ['review', 'ship']) {
const flat = army.replace(/\s+/g, ' ');
test(`${skillName} clarity: terminal failure permits independent work but never certifies coverage`, () => {
expect(flat).toContain('Confirm that each task has finished or is stopped');
expect(flat).toContain('A timeout alone does not prove termination');
expect(flat).toContain("If a reader or writer is still active, wait; if its state is unknown, inspect its task/process status");
expect(flat).toContain("If you cannot confirm it stopped, use the parent's Fix-First stop path without edits");
expect(flat).toContain('Continue independent evidence collection after a terminal failure');
expect(flat).toContain('Missing dispatched coverage remains incomplete, never completed or clean');
expect(flat).toMatch(/confirm that each task has finished or is stopped/i);
expect(flat).toMatch(/a timeout alone does not prove termination/i);
expect(flat).toMatch(/stop path without edits/i);
expect(flat).toMatch(/missing dispatched coverage remains incomplete, never completed or clean/i);
expect(flat).not.toContain('Specialists are additive — partial results are better than no results');
const redTeam = flat.slice(flat.indexOf('### Red Team dispatch'));
expect(redTeam).toContain('confirm it stopped and record its review as incomplete, just as for other specialists');
expect(redTeam).toContain('original specialist outputs and rerun stages 1–7');
expect(redTeam).toMatch(/confirm it stopped and record its review as incomplete/i);
expect(redTeam).toContain('stages 1–7');
});
test(`${skillName} clarity: ordered specialist merge separates validation from scoring and provenance`, () => {
@@ -487,26 +396,21 @@ for (const skillName of ['review', 'ship']) {
expect(positions.every(position => position >= 0)).toBe(true);
expect(positions).toEqual([...positions].sort((a, b) => a - b));
const validation = flat.slice(flat.indexOf('#### 2.'), flat.indexOf('#### 3.'));
expect(validation).toContain('core and specialist findings');
expect(validation).toContain('remove `advisory` and retain its `CRITICAL` severity');
expect(validation).toContain('Never downgrade severity');
expect(validation).toContain('Valid INFORMATIONAL advisories remain advisory');
expect(validation).toMatch(/never downgrade severity/i);
const merge = flat.slice(flat.indexOf('#### 3.'), flat.indexOf('#### 4.'));
expect(merge.indexOf('Partition defects and advisories')).toBeLessThan(merge.indexOf('grouping by fingerprint'));
for (const gate of ['sharedLibsFingerprint', 'literal JSON on stdin', 'never trust a supplied hash',
'Missing/malformed metadata cannot deduplicate', 'highest confidence', '+1 (cap at 10)',
'distinct specialists', 'all source names', 'Core findings never earn a specialist confidence boost']) {
expect(merge).toContain(gate);
}
for (const token of ['sharedLibsFingerprint', 'literal JSON on stdin', '+1 (cap at 10)']) expect(merge).toContain(token);
for (const rule of [/never trust a supplied hash/i, /missing\/malformed metadata cannot deduplicate/i,
/core findings never earn a specialist confidence boost/i]) expect(merge).toMatch(rule);
for (const gate of ['Confidence 7+', 'Confidence 5-6', 'Confidence 3-4', 'Confidence 1-2']) expect(army).toContain(gate);
const scoring = flat.slice(flat.indexOf('#### 5.'), flat.indexOf('#### 6.'));
expect(scoring).toContain('Only specialist findings enter this header and `quality_score`; core findings do not');
expect(scoring).toContain('NON-advisory');
expect(scoring).toMatch(/only specialist findings enter this header and `quality_score`/i);
expect(scoring).toMatch(/non-advisory/i);
expect(scoring).toContain('quality_score = max(0, 10 - (critical_count * 2 + informational_count * 0.5))');
expect(scoring).toContain('unresolved-defect totals');
expect(flat).toContain('Advisory findings COUNT in the stats `findings` field');
expect(flat).toContain('Count only findings that specialist actually returned');
expect(flat).toContain('core-only advice must not create a specialist dispatch or finding');
expect(flat).toMatch(/advisory findings count in the stats `findings` field/i);
expect(flat).toMatch(/count only findings that specialist actually returned/i);
expect(flat).toMatch(/core-only advice must not create a specialist dispatch or finding/i);
expect(flat).toContain('ASK-only');
});
@@ -517,29 +421,20 @@ for (const skillName of ['review', 'ship']) {
const positions = markers.map(marker => reuse.indexOf(marker));
expect(positions.every(position => position >= 0)).toBe(true);
expect(positions).toEqual([...positions].sort((a, b) => a - b));
for (const gate of ['first-party authored provenance', 'all supporting callers and the helper destination',
'--check-shared-libs REVIEW_START', "<<'GSTACK_SHARED_LIBS_REUSE_JSON'", 'literal JSON on stdin',
'`reusable: true`', 'False, command failure or unreadable output', 'never suppression',
'Do not supply your own snapshot, prior record or coverage', 'sharedLibsFingerprint',
'without consuming/replacing it', 'actual repo, raw branch and current snapshot',
'completed/converged', 'verified binding', 'explicit Skip', 'logger-versioned `snapshot_covered_paths`',
'older unversioned coverage', 'Sanitized branch names are not identity', 'canReuseSharedLibsAdvisory',
'byte-for-byte with its blob', 'assume-unchanged, skip-worktree', 'sparse index', 'symlinks/ancestors',
'submodules', 'ignored/outside or unreadable', 'active/unknown Git filters', 'encodings and line conversion',
'fsmonitor and optional locks', 'never uses external diff/textconv', 'Unknown evidence fails closed',
'logger recomputes final coverage', 'Real defects retain normal Fix-First handling independently']) {
expect(reuse).toContain(gate);
}
for (const token of ['--check-shared-libs REVIEW_START', "<<'GSTACK_SHARED_LIBS_REUSE_JSON'", 'literal JSON on stdin',
'`reusable: true`', 'sharedLibsFingerprint', 'explicit Skip', 'logger-versioned `snapshot_covered_paths`',
'canReuseSharedLibsAdvisory']) expect(reuse).toContain(token);
for (const rule of [/false, command failure or unreadable output[^.]*never suppression/i,
/do not supply your own snapshot, prior record or coverage/i, /unknown evidence fails closed/i,
/real defects retain normal Fix-First handling/i]) expect(reuse).toMatch(rule);
});
}
test('review clarity: settlement gates edits separately from incomplete required coverage', () => {
const fix = skill.slice(skill.indexOf('## Step 5: Fix-First Review'), skill.indexOf('{{CROSS_REVIEW_DEDUP}}')).replace(/\s+/g, ' ');
expect(fix).toContain('every dispatched reader has returned or is confirmed stopped');
expect(fix).toContain('active or unknown reader/writer');
expect(fix).toContain('persist incomplete at Step 5.8 and STOP without edits');
expect(fix).toContain('Terminal failure does not block fixes from independent evidence');
expect(fix).toContain('Missing required output still makes the pass incomplete');
expect(fix).toMatch(/every dispatched reader has returned or is confirmed stopped/i);
expect(fix).toMatch(/persist incomplete at Step 5\.8 and stop without edits/i);
expect(fix).toMatch(/missing required output still makes the pass incomplete/i);
});
test('review clarity: Greptile reply choices never substitute for Fix-First approval', () => {
@@ -547,43 +442,38 @@ test('review clarity: Greptile reply choices never substitute for Fix-First appr
expect(fix).toContain('VALID & ACTIONABLE Greptile findings');
const greptile = skill.slice(skill.indexOf('### Greptile comment resolution'), skill.indexOf('## Step 5.8:'));
const flat = greptile.replace(/\s+/g, ' ');
expect(flat).toContain('Step 5c alone supplies A) Fix / B) Skip');
expect(flat).toMatch(/Step 5c alone supplies A\) Fix \/ B\) Skip/i);
expect(flat).not.toContain('A: Fix it now, B: Acknowledge, C: False positive');
expect(flat).toContain('reply decisions, not code approval');
expect(flat).toMatch(/reply decisions, not code approval/i);
expect(flat).toContain('B) Propose a code change');
expect(flat).toContain('return to Steps 5c–5d with an ASK proposal');
expect(flat).toContain('Show the exact change and any `test_stub`; wait for approval before editing');
expect(flat).toContain('no new fix permission');
expect(flat).toContain('Steps 5c–5d');
expect(flat).toMatch(/wait for approval before editing/i);
expect(flat).toMatch(/no new fix permission/i);
});
test('ship review clarity: parent settlement gate precedes classification and cannot waive coverage', () => {
const ship = readFileSync(join(root, 'ship/sections/review-army.md.tmpl'), 'utf8');
const gate = ship.slice(ship.indexOf('## Step 9.4:'), ship.indexOf('1. **Classify')).replace(/\s+/g, ' ');
expect(gate).toContain("Before edits, inspect every dispatched reader/writer's handle");
expect(gate).toContain('Wait for return or confirm termination');
expect(gate).toContain('otherwise log incomplete through items 5–6 and STOP without edits');
expect(gate).toContain('After terminal failure, independent evidence may support fixes');
expect(gate).toContain('missing dispatched output still blocks continuation, even with a QA exception');
expect(gate).toMatch(/wait for return or confirm termination/i);
expect(gate).toMatch(/log incomplete through items 5–6 and stop without edits/i);
expect(gate).toMatch(/missing dispatched output still blocks continuation, even with a QA exception/i);
});
test('review resolves the judged smoke-clock, setup-authority, plan-gate and findings-source ambiguities', () => {
test('review keeps its smoke-clock, setup-authority, plan-gate and findings-source rules', () => {
const ctx = { skillName: 'review', tmplPath: 'review/SKILL.md.tmpl', host: 'claude', paths: HOST_PATHS.claude } as TemplateContext;
const qa = generateQAReview(ctx).replace(/\s+/g, ' ');
expect(qa).toContain('Follow the shared Probe loop for smoke checks and replays until the smoke limit');
expect(qa).toContain('Then run required plan checks and revalidation, even after smoke expires, using the same procedure but no smoke guard');
expect(qa).toContain('/review sets none; only an invoker-supplied EARLIER_UTC counts');
expect(qa).toContain('report-only /review never runs setup, installs or cookie import');
expect(qa).toContain('After a grant, recheck readiness and run affected checks; otherwise they stay blocked');
expect(qa).toMatch(/\/review sets none; only an invoker-supplied EARLIER_UTC counts/);
expect(qa).toMatch(/report-only \/review never runs setup, installs or cookie import/i);
expect(qa).toMatch(/otherwise they stay blocked/i);
expect(qa).not.toContain('Ask for setup/permission');
expect(generateQAReview({ ...ctx, skillName: 'ship', tmplPath: 'ship/SKILL.md.tmpl' })).not.toContain('/review sets none');
const shared = generateQAExploratory({ ...ctx, skillName: 'qa' }).replace(/\s+/g, ' ');
expect(shared).toContain('Explicit plan checks and revalidation remain required beyond this smoke budget');
const audit = generatePlanCompletionAuditReview(ctx).replace(/\s+/g, ' ');
expect(audit).toContain('No plan file found → say "No plan file detected." and use the Fallback Intent Sources below');
expect(audit).toContain('"No plan file detected."');
expect(audit).toContain('Fallback Intent Sources');
expect(audit).not.toContain('skip with "No plan file detected — skipping."');
expect(audit).toContain('**HIGH-impact plan-file discrepancies** trigger AskUserQuestion');
expect(audit).toContain('Discrepancies derived only from fallback sources (commit messages, TODOS.md, PR description) never trigger this question');
expect(audit).toMatch(/derived only from fallback sources[\s\S]{0,80}never trigger this question/i);
expect(generatePlanCompletionAuditShip({ ...ctx, skillName: 'ship', tmplPath: 'ship/SKILL.md.tmpl' })).toContain('skip with "No plan file detected — skipping."');
const persist = skill.slice(skill.indexOf('### 2. Fill the record')).replace(/\s+/g, ' ');
expect(persist).toContain("Step 5's combined final-pass findings (core, specialist, adversarial, actionable Greptile, verified exploratory QA findings)");
expect(persist).toMatch(/combined final-pass findings \(core, specialist, adversarial, actionable Greptile, verified exploratory QA findings\)/i);
});
+9 -16
View File
@@ -39,7 +39,7 @@ describe("ship Apple gate ordering (R2)", () => {
test("store distribution explicitly bypasses the branch/PR ceremony", () => {
expect(SKELETON).toContain("Store distribution proceeds");
expect(SKELETON).toMatch(/branch gate and repository-landing pipeline below apply ONLY to\s*\n?repository-landing asks/);
expect(SKELETON).toMatch(/branch gate and repository-landing pipeline below apply only to\s+repository-landing asks/i);
});
test("the non-Apple branch gate is byte-unchanged and appears exactly once", () => {
@@ -50,30 +50,23 @@ describe("ship Apple gate ordering (R2)", () => {
test("the adapter section exists in the union with its battle-tested spine", () => {
const section = readFileSync(join(ROOT, "ship", "sections", "apple-release.md"), "utf-8");
for (const anchor of [
"one authorization moment",
"fastlane spaceauth",
"iris/v1/apiKeys",
"appPriceSchedules",
"CLASSIFY the error before touching credentials",
"Never abort an App Store release over branch topology",
]) {
for (const anchor of ["one authorization moment", "fastlane spaceauth", "iris/v1/apiKeys", "appPriceSchedules"]) {
expect(section).toContain(anchor);
}
expect(section).toMatch(/classify the error before touching credentials/i);
expect(section).toMatch(/never abort an App Store release over branch topology/i);
});
test("routine interaction limits cannot waive blocking documentation or safety decisions", () => {
for (const name of ["apple-release.md.tmpl", "apple-release.md"]) {
const section = readFileSync(join(ROOT, "ship", "sections", name), "utf-8");
expect(section).toContain("Plan for two routine interactions");
expect(section).toContain("A genuine blocker may require a safety or named documentation-risk decision");
expect(section).toContain("STOP for that decision rather than treating release authorization as a waiver");
expect(section).toContain("routine interactions and blocking decisions above");
const section = readFileSync(join(ROOT, "ship", "sections", name), "utf-8").replace(/\s+/g, " ");
expect(section).toMatch(/two routine interactions/i);
expect(section).toMatch(/stop for that decision rather than treating release authorization as a waiver/i);
expect(section).not.toContain("exactly two interactions, and no others");
expect(section).not.toContain("two permitted interactions");
expect(section.indexOf("**Documentation preflight:**")).toBeLessThan(section.indexOf("## The one authorization moment"));
expect(section).toContain("in `read-only` mode against the selected release source");
expect(section).toContain("Resolve blockers or obtain an explicit named documentation-risk exception before distribution");
expect(section).toContain("`read-only` mode");
expect(section).toMatch(/named documentation-risk exception before distribution/i);
}
});
});
+101 -138
View File
@@ -22,49 +22,40 @@ const gate = compact(entry.slice(entry.indexOf('## Step 16:'), entry.indexOf('##
describe('ship source controller', () => {
test('documentation freshness covers selected code paths as well as release metadata', () => {
const docs = compact(read('ship/sections/documentation.md.tmpl'));
expect(docs).toContain('hashes of the selected release paths, generated outputs and docs/templates');
expect(docs).toContain('selected release paths, generated outputs and docs/templates');
const freshness = gate.slice(gate.indexOf('### 3.'), gate.indexOf('### 4.'));
expect(freshness).toContain('selected release paths, generated outputs and docs/templates');
expect(freshness).toContain("A prior invocation's audit or risk decision never qualifies");
expect(freshness).toContain('the same approved scope and exact content');
expect(freshness).toMatch(/a prior invocation's audit or risk decision never qualifies/i);
});
test('changed documentation inputs permit only the remaining bounded re-audit', () => {
const docs = compact(read('ship/sections/documentation.md.tmpl'));
const recovery = docs.slice(docs.indexOf('## Blocked recovery'));
expect(recovery).toContain('If an attempt remains and either the audited inputs changed');
expect(recovery).toContain('a concrete launch/input/permission correction or reviewed patch repair is available');
expect(docs).toContain('an initial audit plus ONE repair/re-audit');
expect(docs).toContain('never a third attempt, even after Step 16 changes');
expect(docs.indexOf('## Blocked recovery')).toBeGreaterThan(-1);
expect(docs).toMatch(/initial[- ]audit plus one repair\/re-audit/i);
expect(docs).toMatch(/never a third attempt/i);
const freshness = gate.slice(gate.indexOf('### 3.'), gate.indexOf('### 4.'));
expect(freshness).toContain('**An attempt remains, with changed inputs or an available repair:**');
expect(freshness).toContain('**Otherwise:** STOP unless the user accepts');
expect(freshness).toMatch(/\*\*Otherwise:\*\* stop unless the user accepts/i);
expect(freshness).not.toContain('**No attempt remains or no repair is available:**');
});
test('repairs use one ordered work list without a return stack', () => {
const text = compact(controller);
expect(compact(entry)).toContain('**Next steps:** one ordered work list, with the current step marked');
expect(text).toContain('Expand a repair into individual steps and insert them before the still-pending work');
expect(text).toContain('This replaces the current item, whose actual result stays in the record');
expect(text).toContain('Add its destination only if not already the next pending step');
expect(text).toContain('The saved list takes precedence over ordinary next-step sentences inside a repair');
expect(compact(entry)).toContain('**Next steps:**');
expect(text).toContain('5 → 6 → 7 → 8 → 9 → 10 → 11 → 11.5');
expect(text).not.toContain('finish the inner repair, then resume the unfinished outer range');
expect(text).toContain('Keep the same attempt counts throughout the invocation');
expect(text).toContain('initial-plus-ONE limit never resets');
expect(text).toMatch(/initial-plus-one limit never resets/i);
});
test('the roadmap and recovery groups define ownership and zero-edit eligibility', () => {
const text = compact(entry);
expect(text).toContain('integrate (1–3) → test and review (4–11.5) → prepare the release (12–15) → verify frozen content (16) → push and publish (17–21)');
expect(text).toContain('children return evidence, not permission to proceed');
expect(review).toContain('**No edits in this pass:** Resolve the required-probe gate below');
expect(review).toContain('Only after it clears may you continue to Step 10');
expect(review).toContain('With completed checklist and dispatched reviewers, failed/unavailable required probes block continuation');
expect(review).toContain('This cannot waive missing reviewer output, recurring fixes or independent test/security gates');
expect(review).toContain('Undispatched gated/unsupported specialists do not block independently');
expect(text).toContain('Reuse it only for that same scope; a repair never resets approvals or expands them');
expect(text).toMatch(/children return evidence, not permission to proceed/i);
expect(review).toContain('**No edits in this pass:**');
expect(review).toMatch(/only after it clears may you continue to Step 10/i);
expect(review).toMatch(/failed\/unavailable required probes block continuation/i);
expect(review).toMatch(/cannot waive missing reviewer output, recurring fixes or independent test\/security gates/i);
expect(text).toMatch(/a repair never resets approvals or expands them/i);
expect(text).not.toContain('eligible zero-edit pass');
expect(text).not.toContain('the same waiver');
expect(text).not.toContain("the controller's detour");
@@ -72,13 +63,9 @@ describe('ship source controller', () => {
test('distribution discovery is a shortlist, not a manifest-only artifact decision', () => {
const distribution = compact(entry.slice(entry.indexOf('## Step 2:'), entry.indexOf('## Step 3:')));
expect(distribution).toContain('List candidate distribution paths');
expect(distribution).toContain("Also inspect matching untracked files from Step 1's status");
expect(distribution).toContain('a new `package.json` or `Cargo.toml` alone does not establish a publishable artifact');
expect(distribution).toContain('inspect existing manifests for newly declared binaries or package exports');
expect(distribution).toContain('New artifact without a pipeline');
expect(distribution).toContain('**New artifact without a pipeline');
expect(distribution).toContain('AskUserQuestion');
expect(distribution).toContain('Do not publish a release during `/ship`');
expect(distribution).toMatch(/do not publish a release during `\/ship`/i);
});
test('queue qualification exhaustively distinguishes online, git fallback and unusable output', () => {
@@ -89,13 +76,11 @@ describe('ship source controller', () => {
expect(qualify).toBeGreaterThan(0);
expect(usable).toBeGreaterThan(qualify);
expect(missing).toBeGreaterThan(usable);
expect(version).toContain('require successful utility output and a nonempty valid version');
expect(version).toContain('`offline:false` qualifies; `offline:true` qualifies only with `fallback:"git"`');
expect(version).toContain('Offline output without that fallback, failure, malformed output or an empty version is unusable, even if it contains a version-looking string');
expect(version).toContain('FRESH sets `NEW_VERSION=CANDIDATE_VERSION`');
expect(version).toContain('Only approval changes the existing version');
expect(version).toContain('a sibling holding `>= NEW_VERSION` requires a choice: advance past it, or stop this attempt and sync');
expect(version.slice(missing)).toContain('FRESH uses local `BUMP_LEVEL` arithmetic; ALREADY_BUMPED keeps `currentVersion`');
expect(version).toContain('`NEW_VERSION=CANDIDATE_VERSION`');
expect(version).toMatch(/only approval changes the existing version/i);
expect(version).toContain('`>= NEW_VERSION`');
expect(version.slice(missing)).toContain('`BUMP_LEVEL`');
expect(version.slice(missing)).toContain('`currentVersion`');
});
test.each(['home', 'override', 'plugin'])('the actual nudge shell honors %s roots, repeat suppression and enabled tuning', mode => {
@@ -142,11 +127,8 @@ describe('ship source controller', () => {
test('nested repairs resume the unfinished outer range before its destination', () => {
const text = compact(controller);
expect(text).toContain('Expand a repair into individual steps and insert them before the still-pending work');
expect(text).toContain('For another repair, repeat rule 2 without discarding pending work');
expect(text).toContain('Step 11 fixes insert `9 → 10 → 11` before 11.5');
expect(text).toContain('A further Step 9 fix affecting 6–8 makes the list `5 → 6 → 7 → 8 → 9 → 10 → 11 → 11.5`');
expect(text).toContain('The unchanged release steps follow');
expect(text).toContain('`9 → 10 → 11` before 11.5');
expect(text).toContain('`5 → 6 → 7 → 8 → 9 → 10 → 11 → 11.5`');
expect(text).not.toContain('Enter Step 9 before REVIEW_START capture and full review');
});
@@ -157,10 +139,8 @@ describe('ship source controller', () => {
expect(shared).toBeLessThan(coverage.indexOf('Dispatch the audit through Agent'));
const contract = compact(coverage.slice(shared, coverage.indexOf('Dispatch the audit through Agent')));
expect(contract).toContain('use the Agent tool with `run_in_background: false`');
expect(contract).toContain('Omitting the flag runs the subagent in the background');
expect(contract).toContain('keeping a fresh context');
expect(contract).toContain('Do not invoke the target as a Skill or run it inline instead');
expect(contract).toContain("only under that section's documented fallback, after a failed subagent has stopped");
expect(contract).toMatch(/do not invoke the target as a Skill or run it inline instead/i);
expect(contract).toMatch(/after a failed subagent has stopped/i);
for (const section of ['test-coverage', 'plan-completion', 'greptile']) {
const text = read(`ship/sections/${section}.md.tmpl`);
expect(text).toContain('shared foreground-dispatch rule');
@@ -178,8 +158,7 @@ describe('ship source controller', () => {
const ship = generateReviewDashboard(ctx);
expect(ship).toContain('REVIEW READINESS DASHBOARD');
expect(ship).toContain('| Review | Runs | Last run | Status | Required |');
expect(ship).toContain('Use one row for each entry in step 1');
expect(ship).toContain('Only Eng Review is marked required');
expect(ship).toMatch(/only Eng Review is marked required/i);
expect(ship).toContain('{actual status and reason}');
expect(ship).toContain('VERDICT: {CLEARED or NOT CLEARED} — {reason}');
expect(ship).not.toContain('+====================================================================+');
@@ -190,18 +169,17 @@ describe('ship source controller', () => {
test('coverage fallback settles the child and still applies the coverage gate', () => {
const text = compact(read('ship/sections/test-coverage.md.tmpl'));
const fallback = text.slice(text.indexOf('**Audit failure:**'), text.indexOf('{{TEST_COVERAGE_GATE_SHIP}}'));
expect(fallback).toContain('confirm it stopped before running the same audit inline');
expect(fallback).toContain('does not pass or bypass the coverage gate');
expect(fallback).toContain('including its undetermined-percentage and test-only rules');
expect(fallback).toMatch(/confirm it stopped before running the same audit inline/i);
expect(fallback).toMatch(/does not pass or bypass the coverage gate/i);
expect(text).not.toContain('partial results are better than none');
});
test('no-plan gate skips only the audit and retains verification and the remaining section', () => {
const text = generatePlanCompletionGateShip({ host: 'claude', skillName: 'ship', tmplPath: '', paths: HOST_PATHS.claude });
const noPlan = compact(text.slice(text.indexOf('**No plan file found:**')));
expect(noPlan).toContain('Skip only the plan completion audit');
expect(noPlan).toContain('Continue with Step 8.1, Scope Drift and Prior Learnings');
expect(noPlan).toContain('Step 9 QA still runs');
expect(noPlan).toMatch(/skip only the plan completion audit/i);
expect(noPlan).toContain('Step 8.1');
expect(noPlan).toMatch(/Step 9 QA still runs/i);
expect(noPlan).not.toContain('Skip entirely');
});
@@ -211,8 +189,8 @@ describe('ship source controller', () => {
const positions = steps.map(step => text.indexOf(step));
expect(positions.every(position => position >= 0)).toBe(true);
expect(positions).toEqual([...positions].sort((a,b) => a-b));
expect(text).toContain('All three snapshots must match');
expect(text).toContain('does not mean the failed or unrun probes passed');
expect(text).toMatch(/all three snapshots must match/i);
expect(text).toMatch(/does not mean the failed or unrun probes passed/i);
expect(text).not.toContain('equal snapshots do not pass waived probes');
});
@@ -221,9 +199,8 @@ describe('ship source controller', () => {
expect(text).toContain('**Behavior, tests or build inputs changed:**');
expect(text).toContain('**Only authored docs or release metadata changed:**');
expect(text).toContain('**No changes, or the docs-only checks still support the plan:**');
expect(text).toContain('accepted audit matches all inputs | Continue to stage 4');
expect(text).toContain("Use Blocked recovery with the existing count");
expect(text).toContain('Use this example only after confirming that every allowed edit is release metadata');
expect(text).toContain('| Continue to stage 4');
expect(text).toMatch(/Blocked recovery with the existing count/i);
expect(text).not.toContain('Every listed change below is metadata:');
});
@@ -233,26 +210,24 @@ describe('ship source controller', () => {
expect(end).toBeLessThan(entry.indexOf('{{BASE_BRANCH_DETECT}}'));
expect(entry.match(/### Ship control flow/g)).toHaveLength(1);
const text = compact(controller);
expect(text).toContain('You, the **parent** running /ship, own advancement');
expect(text).toContain('Follow the saved work list');
expect(text).toContain('STOP and AskUserQuestion gates still apply during repairs');
expect(text).toContain('The saved list takes precedence over ordinary next-step sentences inside a repair. A range never adds unlisted steps');
expect(text).toContain('Keep the same attempt counts throughout the invocation');
expect(text).toMatch(/the \*\*parent\*\* running \/ship, own advancement/i);
expect(text).toMatch(/stop and AskUserQuestion gates still apply during repairs/i);
expect(text).toMatch(/a range never adds unlisted steps/i);
expect(text).toMatch(/keep the same attempt counts throughout the invocation/i);
expect(text).toContain('A range ending at Step 14 does not enter Step 14.5');
expect(text).toContain('its initial-plus-ONE limit never resets');
expect(text).not.toContain('| At step |');
expect(entry.match(/Ship control flow/g)).toHaveLength(1);
expect(review).toContain('Every repeat starts before the checklist read and captures a fresh REVIEW_START');
expect(review).toMatch(/captures a fresh REVIEW_START/);
});
test('distribution and bootstrap detours end at their exact forward resume', () => {
const merge = compact(entry.slice(entry.indexOf('## Step 3:'), entry.indexOf('{{SECTION:tests}}')));
expect(merge).toContain('repeat Step 2 on the merged content, including its decisions, then continue to Step 4');
expect(merge).toContain('Otherwise continue to Step 4 directly');
expect(merge).toMatch(/repeat Step 2 on the merged content[^.]*continue to Step 4/i);
expect(merge).toMatch(/otherwise continue to Step 4 directly/i);
const tests = compact(read('ship/sections/tests.md.tmpl'));
expect(tests).toContain('A runs Step 4 with this new bootstrap choice, then returns here to run the tests');
expect(tests).toMatch(/A runs Step 4 with this new bootstrap choice, then returns here/i);
expect(tests).toContain('A) Add tests (recommended)');
expect(tests).toContain('declining bootstrap alone is not that approval');
expect(tests).toMatch(/declining bootstrap alone is not that approval/i);
});
test('missing dispatched output outranks the cycle cap, fixes and zero-edit continuation', () => {
@@ -262,49 +237,40 @@ describe('ship source controller', () => {
expect(positions.every(position => position >= 0)).toBe(true);
expect(positions).toEqual([...positions].sort((a, b) => a - b));
const missing = decisions.slice(positions[0], positions[1]);
expect(missing).toContain('STOP and name each failed specialist or Red Team');
expect(missing).toContain('Retain queued fixes and restore coverage');
expect(missing).toContain('If this pass made edits, resume at the next decision; otherwise run a fresh complete Step 9');
expect(missing).toContain('A successful peer or a QA exception cannot replace missing dispatched coverage');
expect(missing).toMatch(/^1\. \*\*Dispatched reviewer output missing:\*\* stop\b/i);
expect(missing).toMatch(/cannot replace missing dispatched coverage/i);
const cap = decisions.slice(positions[1], positions[2]);
expect(cap).toContain('STOP');
expect(cap).toContain('`converged:false`; do not run a fourth fixing cycle');
expect(cap).toMatch(/\bstop\b/i);
expect(cap).toContain('`converged:false`');
expect(cap).toMatch(/do not run a fourth fixing cycle/i);
const fixes = decisions.slice(positions[2], positions[3]);
expect(fixes).toContain('Insert Step 5, affected Steps 6–8 and all of Step 9 before the pending Step 10 in the work list');
expect(fixes).toContain('Tests must pass or retain approval for the same verified pre-existing failures and scope');
expect(fixes).toContain('Keep CYCLES and scoped approvals across this repeat');
expect(decisions.slice(positions[3])).toContain('Only after it clears may you continue to Step 10');
expect(review).toContain('Undispatched host-unsupported/gated specialists do not block');
expect(review).toContain('Failed, blocked, inconclusive or not-run required probes mean false, never clean');
expect(review).toContain('This cannot waive missing reviewer output, recurring fixes or independent test/security gates');
expect(fixes).toContain('Insert Step 5, affected Steps 6–8 and all of Step 9 before the pending Step 10');
expect(fixes).toMatch(/retain approval for the same verified pre-existing failures and scope/i);
expect(decisions.slice(positions[3])).toMatch(/only after it clears may you continue to Step 10/i);
expect(review).toMatch(/failed, blocked, inconclusive or not-run required probes mean false, never clean/i);
});
test('comment fixes resume triage and preserve settled approvals and replies', () => {
const section = compact(read('ship/sections/greptile.md.tmpl'));
expect(section).toContain('If fixes were approved, save their approvals and comment references');
expect(section).toContain("Run Step 9's full review/fix loop, then return here");
expect(review).toContain('Finish the complete review and QA before applying any fix in Step 9.4');
expect(section).toContain('Finish the saved replies without asking again about completed fixes, and classify new comments');
expect(section).toContain('With no queued fixes, continue to Step 11');
expect(section).toContain('This optional triage does not block ship');
expect(section).toContain('unknown or missing status is unavailable');
expect(section).toMatch(/save their approvals and comment references/i);
expect(section).toMatch(/Run Step 9's full review\/fix loop, then return here/i);
expect(review).toMatch(/finish the complete review and QA before applying any fix in Step 9\.4/i);
expect(section).toMatch(/with no queued fixes, continue to Step 11/i);
expect(section).toMatch(/this optional triage does not block ship/i);
expect(section).not.toContain('return to Step 9');
});
test('native recovery has one corrected attempt and cannot borrow outside completion', () => {
const finish = adversarial.slice(adversarial.indexOf('### Finish the adversarial phase'));
const recovery = finish.slice(finish.indexOf('1. **Required native'), finish.indexOf('2. **Fixes queued'));
expect(recovery).toContain('STOP and confirm the native task stopped');
expect(recovery).toContain('Outside-provider output cannot replace this pass');
expect(recovery).toContain('One recovery retry is allowed only after a concrete prerequisite correction and restored access');
expect(recovery).toContain('count it in the invocation record before launch');
expect(recovery).toContain('Capture a fresh PASS_START and persist the new attempt separately');
expect(recovery).toContain('Without that correction, or if the recovery fails, ask for repair and remain blocked');
expect(recovery).toMatch(/stop and confirm the native task stopped/i);
expect(recovery).toMatch(/outside-provider output cannot replace this pass/i);
expect(recovery).toMatch(/one recovery retry is allowed only after a concrete prerequisite correction/i);
expect(recovery).toContain('PASS_START');
expect(recovery).toMatch(/ask for repair and remain blocked/i);
const queued = finish.slice(finish.indexOf('2. **Fixes queued'), finish.indexOf('3. **Native complete'));
expect(queued).toContain('Keep the findings and their approvals');
expect(queued).toContain('Insert Steps 9, 10 and 11 before the pending Step 11.5 in the work list. Step 9 completes full review before fixes');
expect(queued).toContain('any further repair inserts its checks ahead of the remaining items');
expect(queued).toContain('not recovery retries');
expect(queued).toContain('Insert Steps 9, 10 and 11 before the pending Step 11.5');
expect(queued).toMatch(/not recovery retries/i);
expect(finish.slice(finish.indexOf('3. **Native complete'))).toContain('then continue to Step 11.5');
});
@@ -312,19 +278,19 @@ describe('ship source controller', () => {
const ctx = { host, skillName: 'ship', tmplPath: '', paths: HOST_PATHS[host] };
const ship = generateAdversarialStep(ctx);
const finish = compact(ship.slice(ship.indexOf('### Finish the adversarial phase')));
expect(finish).toContain('Apply these decisions in order before leaving Step 11');
expect(finish).toContain('Required native review incomplete');
expect(finish).toContain('Outside-provider output cannot replace this pass');
expect(finish).toContain('Fixes queued after native completion');
expect(finish).toContain('Native complete with no queued fixes');
const branches = ['Required native review incomplete', 'Fixes queued after native completion', 'Native complete with no queued fixes']
.map(label => finish.indexOf(label));
expect(branches.every(index => index >= 0)).toBe(true);
expect(branches).toEqual([...branches].sort((a, b) => a - b));
expect(finish).toMatch(/outside-provider output cannot replace this pass/i);
expect(finish).toContain('Step 11.5');
expect(finish).not.toContain('proceed to Step 12');
expect(finish).not.toContain('return to Step 9');
const review = generateAdversarialStep({ ...ctx, skillName: 'review' });
expect(review).not.toContain('Ship control flow');
expect(review).not.toContain('Step 11.5');
expect(review).toContain('Return all findings and structured-review decisions to Step 5');
expect(review).toContain('do not start an inner repair loop');
expect(review).toMatch(/return all findings and structured-review decisions to Step 5/i);
expect(review).toMatch(/do not start an inner repair loop/i);
});
test('Step 11.5 verifies original record identity before any release write', () => {
@@ -332,59 +298,56 @@ describe('ship source controller', () => {
expect(bindingStart).toBeGreaterThan(entry.indexOf('{{SECTION:adversarial}}'));
expect(bindingStart).toBeLessThan(entry.indexOf('## Step 12:'));
const binding = compact(entry.slice(bindingStart, entry.indexOf('## Step 12:')));
for (const field of ['saved handle, original token and source', 'skill:"review"', 'via:"ship"', 'skill:"adversarial-review"',
for (const field of ['skill:"review"', 'via:"ship"', 'skill:"adversarial-review"',
'review_binding.state', 'verified', 'review_binding.start_wtree', 'review_binding.end_wtree']) expect(binding).toContain(field);
expect(binding).toContain('Never attach new tokens to old work');
expect(binding).toContain('Keep Step 9.4\'s incomplete flags');
expect(binding).toContain('does not mean the failed or unrun probes passed');
expect(binding).toContain('insert `9 → 10 → 11 → 11.5` before Step 12. Bind the new records at 11.5');
expect(binding).toMatch(/never attach new tokens to old work/i);
expect(binding).toMatch(/does not mean the failed or unrun probes passed/i);
expect(binding).toContain('`9 → 10 → 11 → 11.5` before Step 12');
});
test('late build and behavior changes complete bounded ranges before docs', () => {
const build = gate.slice(gate.indexOf('### 1.'), gate.indexOf('### 2.'));
expect(build).toContain('A missing prerequisite or failed build stops shipping');
expect(build).toContain('Repair the prerequisite or build, then repeat stage 1');
expect(build).toContain('After it passes, continue to stage 2; treat any content repair as a behavioral change there');
expect(build).toMatch(/a missing prerequisite or failed build stops shipping/i);
expect(build).toMatch(/repeat stage 1/i);
expect(build).toMatch(/continue to stage 2/i);
const behavior = gate.slice(gate.indexOf('1. **Behavior'), gate.indexOf('2. **Only authored'));
expect(behavior).toContain('Insert `5–11.5 → 12–14 → 16` before the pending Step 17');
expect(behavior).toContain('before the pending Step 17, then stop this step. This repair excludes Step 14.5');
expect(behavior).toContain('rebuild and compare again before stage 3 decides documentation freshness');
expect(behavior).toMatch(/excludes Step 14\.5/i);
expect(behavior).toMatch(/rebuild and compare again before stage 3/i);
const plan = gate.slice(gate.indexOf('2. **Only authored'), gate.indexOf('3. **No changes'));
expect(plan).toContain("run Step 8's audit and decision gates only, then return to Step 16 stage 1");
expect(plan).toContain("Never edit the child's counts yourself");
expect(plan).toMatch(/run Step 8's audit and decision gates only, then return to Step 16 stage 1/i);
expect(plan).toMatch(/never edit the child's counts yourself/i);
});
test('docs freshness has bounded re-audit and exact-content exception routes', () => {
const docs = gate.slice(gate.indexOf('### 3.'), gate.indexOf('### 4.'));
expect(docs).toContain("Use Blocked recovery with the existing count");
expect(docs).toMatch(/Blocked recovery with the existing count/i);
expect(docs).toContain('**An attempt remains, with changed inputs or an available repair:** insert `14.5 → 15 → 16` before Step 17');
expect(docs).toContain('restart Step 16 stage 1 to regenerate and compare again');
expect(docs).toContain('STOP unless the user accepts the specific named documentation risk and all unwaivable gates clear');
expect(docs).toContain('Never run a third audit');
expect(docs).toContain('Unchanged approved content goes to stage 4; repaired content goes to stage 1');
expect(docs).toContain('retain `Documentation: blocked`');
expect(docs).toContain('Missing, stale or blocked | Use recovery below. Never silently refresh hashes');
expect(docs).toMatch(/stop unless the user accepts the specific named documentation risk and all unwaivable gates clear/i);
expect(docs).toMatch(/never run a third audit/i);
expect(docs).toMatch(/unchanged approved content goes to stage 4; repaired content goes to stage 1/i);
expect(docs).toContain('`Documentation: blocked`');
expect(docs).toMatch(/never silently refresh hashes/i);
});
test('tests, absent test approval and new writes reopen final verification', () => {
const tests = gate.slice(gate.indexOf('### 4.'), gate.indexOf('### 5.'));
expect(tests).toContain("**New, changed or unwaived test failure:** STOP publication. Run Steps 5–15, starting with Step 5's triage, then return to Step 16 stage 1");
expect(tests).toContain('This recovery also applies if a failure appears while reporting in stage 5');
expect(tests).toContain('run Steps 5–15, including the no-tests decision, then return to Step 16 stage 1');
expect(tests).toContain('it does not authorize a third attempt');
expect(gate).toContain('If content changes during or after verification, restart at stage 1 and complete all five stages before Step 17');
expect(gate).toContain('Content-preserving commits keep valid evidence');
expect(tests).toMatch(/\*\*New, changed or unwaived test failure:\*\* stop publication\. Run Steps 5–15/i);
expect(tests).toMatch(/run Steps 5–15, including the no-tests decision, then return to Step 16 stage 1/i);
expect(tests).toMatch(/does not authorize a third attempt/i);
expect(gate).toMatch(/restart at stage 1 and complete all five stages before Step 17/i);
});
test('push failure kinds and publication races have distinct resumes', () => {
const push = compact(entry.slice(entry.indexOf('## Step 17:'), entry.indexOf('## Step 18:')));
expect(push).toContain('**If the push fails, STOP.** No Step 19 or publication claim');
expect(push).toContain("fetch and inspect the remote, then merge under Step 3's conflict rules");
expect(push).toContain('Run Steps 5–16 before returning to Step 17. Never rewrite history');
expect(push).toContain('repeat Step 16 even if content is unchanged before returning to Step 17');
expect(push).toContain('Never bypass failed guards');
expect(push).toMatch(/push fails[^.]*\bstop\b[\s\S]{0,20}no Step 19 or publication claim/i);
expect(push).toMatch(/merge under Step 3's conflict rules/i);
expect(push).toContain('Run Steps 5–16 before returning to Step 17');
expect(push).toMatch(/never rewrite history/i);
expect(push).toMatch(/repeat Step 16 even if content is unchanged/i);
expect(push).toMatch(/never bypass failed guards/i);
const publication = compact(read('ship/sections/pr-body.md.tmpl'));
expect(publication).toContain("If the open PR/MR or title changed, repeat Step 18's identity/title preparation");
expect(publication).toContain('then return here for a new lookup, fresh body and both redaction scans before publishing');
expect(publication).toMatch(/repeat Step 18's identity\/title preparation/i);
expect(publication).toMatch(/both redaction scans before publishing/i);
});
});
+47 -79
View File
@@ -12,79 +12,60 @@ describe('authored ship-only plan verification handoff', () => {
const ctx = { skillName: 'ship', tmplPath: 'ship/SKILL.md.tmpl', host: 'claude' as const, paths: HOST_PATHS.claude };
test('local execution-only checks remain required in Step 8.1/9 outside implementation counts', () => {
const audit = generatePlanCompletionAuditShip(ctx);
const extraction = audit.slice(audit.indexOf('### Actionable Item Extraction'), audit.indexOf('### Verification Mode'));
expect(extraction).toContain('Separate deliverables from execution-only verification');
expect(extraction).toContain('implementation and test-creation requirements');
expect(extraction).toContain('retain its command, expected outcome and source verbatim');
const extraction = audit.slice(audit.indexOf('### Actionable Item Extraction'), audit.indexOf('### Verification Mode')).replace(/\s+/g, ' ');
expect(extraction).toMatch(/execution-only verification/i);
expect(extraction).toContain('Step 8.1/9');
expect(extraction).toContain('outside implementation counts');
expect(extraction).toContain('never DONE from static inspection');
expect(extraction).toContain('not EXTERNAL-STATE merely because it has not run');
expect(extraction).toContain('Keep genuine external-state and human-only checks in this audit');
expect(extraction).toContain('zero implementation counts do not waive those checks');
expect(extraction).toMatch(/outside implementation counts/i);
expect(extraction).toMatch(/never DONE from static inspection/);
expect(extraction).toMatch(/do not waive those checks/i);
expect(fs.readFileSync(path.join(SHIP_DIR, 'sections/plan-completion.md.tmpl'), 'utf8')).toContain('exactly these seven fields');
const gate = generatePlanCompletionGateShip(ctx);
expect(gate).toContain('Any NOT DONE items');
expect(gate).toContain('Per-item confirmation is mandatory');
expect(gate).toMatch(/per-item confirmation/i);
});
test('review-mode extraction retains its existing categories and has no ship-only routing', () => {
const audit = generatePlanCompletionAuditReview({ ...ctx, skillName: 'review' });
expect(audit).toContain('**Test requirements:** "Test that X", "Add test for Y", "Verify Z"');
expect(audit).not.toContain('Separate deliverables from execution-only verification');
expect(audit).toContain('**Test requirements:**');
expect(audit).not.toMatch(/execution-only verification/i);
expect(audit).not.toContain('Step 8.1/9');
expect(audit).toContain('EXTERNAL-STATE');
});
test('preparation hands every explicit check to the report-only execution owner before Fix-First', () => {
const text = generatePlanVerificationExec(ctx).replace(/\s+/g, ' ');
expect(text).toContain('Collect now; execute in Step 9');
expect(text).toContain('Do not invoke an entire QA skill or start probes here');
expect(text).toMatch(/do not invoke an entire QA skill or start probes here/i);
for (const heading of ['Verification', 'Test plan', 'Testing', 'How to test', 'Manual testing']) {
expect(text).toContain(`\`${heading}\``);
}
expect(text).toContain('any other explicit checks, including execution-only items retained by Step 8');
expect(text).toContain('each exact expected outcome, source, surface, probe and safe prerequisites');
expect(text).toContain('Clarify unknown outcomes');
expect(text).toContain('Browser items use the declared project/plan dev URL and browser setup at execution');
expect(text).toContain('functional items use native tools without discovering a web server');
expect(text).toContain('An API URL is not automatically a page');
expect(text).toContain('Only browser evidence needs screenshots');
expect(text).toContain('If no verification section or no plan file exists, record no plan-specific items');
expect(text).toContain('Automatic diff-scoped QA still runs. Continue to Step 8.2 Scope Drift below');
expect(text).toContain('Step 8.2');
expect(text).toContain('Handoff to Step 9.2.1');
expect(text).toContain('parent-owned report-only explorer must execute this complete list before Fix-First');
expect(text).toContain('prerequisite, permission, evidence and changed-input revalidation rules');
expect(text).toContain('Share current-input proof for overlapping smoke probes');
expect(text).toContain('plan checks beyond that smoke budget remain required');
expect(text).toContain('At command/time limits, mark remaining checks not run');
expect(text).toContain("Send failed, blocked or unrun checks through Step 9's required-probe gate, never silently waive them");
expect(text).toContain('Noninteractive runs return blocked');
expect(text).toMatch(/before Fix-First/);
expect(text).toMatch(/never silently waive/i);
expect(text).toMatch(/noninteractive runs return blocked/i);
expect(text).toContain('VERIFY_RESULT=pass only if all selected items pass, skipped only if none exist, otherwise fail');
expect(text).toContain('Risk acceptance keeps the actual failed, blocked and unrun outcomes');
expect(text).toContain("Report per-status counts, evidence and accepted risks in Step 19's `## Verification Results`, separately from automatic QA");
expect(text).toContain('`## Verification Results`');
expect(text.indexOf('Collect now')).toBeLessThan(text.indexOf('Handoff to Step 9.2.1'));
expect(text.indexOf('Handoff to Step 9.2.1')).toBeLessThan(text.indexOf('After execution'));
});
test('the authored parent validates all seven fields and settles failed children before fallback', () => {
const audit = fs.readFileSync(path.join(SHIP_DIR, 'sections/plan-completion.md.tmpl'), 'utf8');
const parent = audit.slice(audit.indexOf('**Parent processing:**')).replace(/\s+/g, ' ');
expect(parent).toContain("Check the task's terminal status");
expect(parent).toContain('Without successful completion and valid LAST-line JSON, use the audit-failure fallback');
expect(parent).toMatch(/valid LAST-line JSON, use the audit-failure fallback/i);
expect(parent).toContain('exactly the seven declared fields');
expect(parent).toContain('nonnegative integer counts whose classification sum equals `total_items`');
expect(parent).toContain('classification sum equals `total_items`');
expect(parent).toContain('a string `summary`');
expect(parent).toContain('Missing, extra or invalid fields fail');
expect(parent).toContain('Valid no-plan/no-actionable reports retain zero counts and their summary');
expect(parent).toContain('no final output after ~10 minutes');
expect(parent).toContain('stop any live child and confirm it stopped before an inline audit');
expect(parent).toContain('same extraction/classification logic; never race a late result');
expect(parent).toContain('If that also fails, AskUserQuestion');
expect(parent).toContain('recording the reason in the PR body and Step 20 metrics');
expect(parent).toMatch(/missing, extra or invalid fields fail/i);
expect(parent).toMatch(/no-plan\/no-actionable reports retain zero counts/i);
expect(parent).toContain('~10 minutes');
expect(parent).toMatch(/stop any live child and confirm it stopped before an inline audit/i);
expect(parent).toMatch(/never race a late result/i);
expect(parent).toMatch(/if that also fails, AskUserQuestion/i);
expect(parent).toContain('Stop and fix the audit (recommended/default)');
const contract = audit.split('\n').find(line => line.startsWith('{"total_items":N,'))!;
expect(Object.keys(JSON.parse(contract.replace(/:N([,}])/g, ':0$1'))).sort())
.toEqual(['total_items', 'done', 'changed', 'partial', 'not_done', 'unverifiable', 'summary'].sort());
expect(generatePlanCompletionGateShip(ctx)).toContain('Only PARTIAL items (no NOT DONE, no UNVERIFIABLE)');
expect(generatePlanCompletionGateShip(ctx)).toContain('Continue with a note in the PR body. Not blocking');
expect(generatePlanCompletionGateShip(ctx)).toMatch(/not blocking/i);
});
});
@@ -101,13 +82,13 @@ function readShipUnion(): string {
return t;
}
describe('ship/SKILL.md — Plan Completion gate invariants (VAS-449 remediation)', () => {
describe('ship/SKILL.md — Plan Completion gate invariants', () => {
const skill = readShipUnion();
test('Path concreteness rule: filesystem-pathed items must be test -f checked', () => {
expect(skill).toContain('**Path concreteness rule.**');
expect(skill).toMatch(/concrete filesystem path/);
expect(skill).toMatch(/MUST be classified DONE or NOT DONE based on `\[ -f/);
expect(skill).toMatch(/classified DONE or NOT DONE based on `\[ -f/);
});
test('Validator detection: project package.json validate-* scripts are auto-run', () => {
@@ -117,8 +98,8 @@ describe('ship/SKILL.md — Plan Completion gate invariants (VAS-449 remediation
});
test('Per-item UNVERIFIABLE confirmation: blanket-confirm is forbidden', () => {
expect(skill).toContain('**Per-item confirmation is mandatory.**');
expect(skill).toMatch(/Do NOT use a single AskUserQuestion to blanket-confirm/);
expect(skill).toMatch(/per-item confirmation/i);
expect(skill).toMatch(/do not use a single AskUserQuestion to blanket-confirm/i);
});
test('Subagent failure: fail-closed, not silent fail-open', () => {
@@ -131,16 +112,11 @@ describe('ship/SKILL.md — Plan Completion gate invariants (VAS-449 remediation
test('parent rejects audit errors and malformed counts instead of treating them as no plan', () => {
const audit = fs.readFileSync(path.join(SHIP_DIR, 'sections/plan-completion.md'), 'utf8');
expect(audit.indexOf('**Parent processing:**')).toBeGreaterThan(-1);
expect(audit.indexOf('**Parent processing:**')).toBeLessThan(audit.indexOf('**Audit-failure fallback:**'));
const parent = audit.slice(audit.indexOf('**Parent processing:**'), audit.indexOf('**Audit-failure fallback:**')).replace(/\s+/g, ' ');
expect(parent).toContain("Check the task's terminal status");
expect(parent).toContain('Without successful completion and valid LAST-line JSON, use the audit-failure fallback');
expect(parent).toContain('Require exactly the seven declared fields');
expect(parent).toContain('Missing, extra or invalid fields fail');
expect(parent).toContain('nonnegative integer');
expect(parent).toContain('classification sum equals `total_items`');
expect(parent).toContain('a string `summary`');
expect(parent).toContain('audit-failure fallback');
expect(parent).toContain('Valid no-plan/no-actionable reports retain zero counts');
expect(parent).toContain('seven declared fields');
expect(parent).toMatch(/audit-failure fallback/i);
});
test('successful plan audits use the declared seven-field contract without an error field', () => {
@@ -167,7 +143,7 @@ describe('ship/SKILL.md — Plan Completion gate invariants (VAS-449 remediation
expect(todos).toMatch(/Step 8[^\n]+P1[^\n]+plan/);
expect(todos).toMatch(/Step 5[^\n]+P0[^\n]+deduplicate/);
expect(todos.indexOf('Add approved deferrals')).toBeLessThan(todos.indexOf('Detect completed TODOs'));
expect(todos.replace(/\s+/g, ' ')).toContain("If creation was declined or a write failed, warn and retain unsaved follow-ups in Step 19's PR summary");
expect(todos.replace(/\s+/g, ' ')).toMatch(/retain unsaved follow-ups in Step 19's PR summary/i);
});
test('CHANGELOG uses the normal workflow without checkpoint context or squash prerequisites', () => {
@@ -180,23 +156,15 @@ describe('ship/SKILL.md — Plan Completion gate invariants (VAS-449 remediation
const entry = fs.readFileSync(path.join(SHIP_DIR, 'SKILL.md'), 'utf8');
const gate = entry.slice(entry.indexOf('## Step 16:'), entry.indexOf('## Step 17:'));
const text = gate.replace(/\s+/g, ' ');
expect(text).toContain('STALE/MISSING: changed content, command or age, or no proven run');
expect(text).toContain('Only receipt storage/readback failed');
expect(text).toContain('Independently prove unchanged final content, the same command and valid age');
expect(text).toContain('from the successful run\'s evidence');
expect(text).toContain('exact command, exit, timestamp and log');
expect(text).toContain('as **ledger unavailable**');
expect(text).toContain('STALE/MISSING');
expect(text).toContain('| Only receipt storage/readback failed |');
expect(text).toContain('**ledger unavailable**, never FRESH');
const storage = text.slice(text.indexOf('| Only receipt storage/readback failed |')).split('|')[2];
expect(storage).not.toContain('gstack-evidence run');
expect(storage).toContain('from the successful run\'s evidence');
expect(text).toContain("**New, changed or unwaived test failure:** STOP publication. Run Steps 5–15, starting with Step 5's triage, then return to Step 16 stage 1");
expect(text).toContain('Reuse waivers only for the same verified pre-existing failures and approved scope');
expect(text).toContain('This recovery also applies if a failure appears while reporting in stage 5');
expect(text).toContain('Docs, TODO edits, new/generated tests and fixes make evidence STALE');
expect(storage).toContain('Independently prove unchanged final content, the same command and valid age');
expect(storage).toContain('Without that proof, use STALE/MISSING');
expect(text).toContain('Without that proof, use STALE/MISSING');
expect(storage).toMatch(/without that proof, use STALE\/MISSING/i);
expect(text).toMatch(/\*\*New, changed or unwaived test failure:\*\* stop publication\. Run Steps 5–15/i);
expect(text).toMatch(/reuse waivers only for the same verified pre-existing failures/i);
expect(text).toMatch(/make evidence STALE/);
});
test('ship contract precedes base detection and fresh remote facts precede distribution decisions', () => {
@@ -204,7 +172,7 @@ describe('ship/SKILL.md — Plan Completion gate invariants (VAS-449 remediation
expect(entry.indexOf('# Ship:')).toBeLessThan(entry.indexOf('## Step 0:'));
const preflight = entry.slice(entry.indexOf('## Step 1:'), entry.indexOf('## Step 2:'));
expect(preflight).toContain('git fetch origin <base>');
expect(preflight).toMatch(/fetch fails[^\n]+STOP/);
expect(preflight).toMatch(/fetch fails[^\n]+\bstop\b/i);
expect(entry).not.toContain('auto-generate and commit, or flag');
expect(entry).toContain('Step 15 commits those tests');
});
@@ -212,10 +180,9 @@ describe('ship/SKILL.md — Plan Completion gate invariants (VAS-449 remediation
test('bisectable commits proceed directly to verification without rewriting existing history', () => {
const entry = fs.readFileSync(path.join(SHIP_DIR, 'SKILL.md'), 'utf8');
const commit = entry.slice(entry.indexOf('## Step 15:'), entry.indexOf('## Step 16:'));
expect(commit).toContain('Make bisectable commits');
expect(commit).toContain('if already committed, continue to Step 16');
expect(commit).toMatch(/Never create an empty commit/i);
expect(commit).toContain('Each commit must work independently');
expect(commit).toMatch(/bisectable commits/i);
expect(commit).toContain('continue to Step 16');
expect(commit).toMatch(/never create an empty commit/i);
expect(commit).not.toMatch(/checkpoint|WIP|squash|git rebase|git reset/);
expect(entry).not.toMatch(/Step 15\.[012]/);
});
@@ -224,13 +191,14 @@ describe('ship/SKILL.md — Plan Completion gate invariants (VAS-449 remediation
const entry = fs.readFileSync(path.join(SHIP_DIR, 'SKILL.md'), 'utf8');
const push = entry.slice(entry.indexOf('## Step 17:'), entry.indexOf('## Step 20:'));
const recovery = push.replace(/\s+/g, ' ');
expect(push).toMatch(/push fails[^\n]+STOP/);
expect(push).toMatch(/push fails[^\n]+\bstop\b/i);
expect(recovery).toContain('**Non-fast-forward push:**');
expect(recovery).toContain('Run Steps 5–16 before returning to Step 17. Never rewrite history');
expect(recovery).toContain('Run Steps 5–16 before returning to Step 17');
expect(recovery).toMatch(/never rewrite history/i);
expect(recovery).toContain('**Authentication, hook or network failure:**');
expect(recovery).toContain('repeat Step 16 even if content is unchanged before returning to Step 17');
expect(recovery).toMatch(/repeat Step 16 even if content is unchanged/i);
expect(push).toMatch(/never force.push/i);
expect(push).toContain('Only a successful push');
expect(push).toMatch(/only a successful push/i);
});
});
+23 -31
View File
@@ -8,69 +8,61 @@ const read = (file: string) => readFileSync(new URL(`../${file}`, import.meta.ur
const compact = (text: string) => text.replace(/\s+/g, ' ');
const ctx = { host: 'claude' as const, skillName: 'ship', tmplPath: '', paths: HOST_PATHS.claude };
test('ship STOP blocks advancement while retaining the stated repair route', () => {
test('ship stop lines block advancement while retaining the stated repair route', () => {
const entry = compact(read('ship/SKILL.md.tmpl'));
expect(entry).toContain('STOP blocks advancement until the stated repair/resume route clears; without one, end this attempt');
expect(entry).toContain('Answer each AskUserQuestion before continuing');
expect(entry).toMatch(/stop blocks advancement until the stated repair\/resume route clears; without one, end this attempt/i);
expect(entry).toMatch(/answer each AskUserQuestion before continuing/i);
expect(entry).toMatch(/routine authorization never waives those gates or their required user decisions/i);
const review = compact(read('ship/sections/review-army.md.tmpl'));
expect(review).toContain('**Dispatched reviewer output missing:** STOP');
expect(review).toContain('Retain queued fixes and restore coverage');
expect(review).toContain('**Third fixing cycle reached (`CYCLES >= 3`):** STOP');
expect(entry).toContain('Routine authorization never waives those gates or their required user decisions');
expect(review).toMatch(/\*\*Dispatched reviewer output missing:\*\* stop\b/i);
expect(review).toMatch(/\*\*Third fixing cycle reached \(`CYCLES >= 3`\):\*\* stop\b/i);
});
test('a new ship bootstrap choice overrides only the saved decline, not framework selection', () => {
const ship = generateTestBootstrap(ctx);
const decline = ship.slice(ship.indexOf('**If BOOTSTRAP_DECLINED**'), ship.indexOf('**If NO ecosystem marker matched:**'));
expect(compact(decline)).toContain("Step 5's explicit Add tests choice overrides that marker for this invocation only");
expect(compact(decline)).toContain('continue to runtime detection and B2–B3, including framework approval');
expect(compact(decline)).toMatch(/explicit Add tests choice overrides that marker for this invocation only/i);
expect(compact(decline)).toMatch(/including framework approval/i);
expect(decline).not.toContain('rm ');
expect(ship.indexOf('**If ANY existing-test evidence appears**')).toBeLessThan(ship.indexOf('**If BOOTSTRAP_DECLINED**'));
const qa = generateTestBootstrap({ ...ctx, skillName: 'qa' });
expect(qa).toContain('**If BOOTSTRAP_DECLINED** appears: Print "Test bootstrap previously declined — skipping." **Skip the rest of bootstrap.**');
expect(compact(qa)).toMatch(/BOOTSTRAP_DECLINED[^\n]{0,120}skip the rest of bootstrap/i);
expect(qa).not.toContain("Step 5's explicit Add tests choice");
});
test('an unverified item answered not done enters the existing decision before continuing', () => {
const gate = generatePlanCompletionGateShip(ctx);
const exits = compact(gate.slice(gate.indexOf('**Exit conditions:**'), gate.indexOf('**Cap.**')));
expect(exits).toContain('Any N: STOP and report that item as NOT DONE');
expect(exits).toContain('Resume only after its required work is verified');
expect(exits).toContain('no second deferral choice');
expect(exits).toMatch(/any N: stop and report that item as NOT DONE/i);
expect(exits).toMatch(/resume only after its required work is verified/i);
expect(exits).toMatch(/no second deferral choice/i);
expect(exits).not.toContain('re-running /ship');
expect(gate).toContain('Per-item confirmation is mandatory');
expect(gate).toContain('with the user\'s free-text evidence');
expect(gate).toMatch(/per-item confirmation/i);
});
test('docs reentry distinguishes the initial audit from same-invocation accepted evidence', () => {
const docs = compact(read('ship/sections/documentation.md.tmpl'));
const entry = docs.slice(0, docs.indexOf('## Prepare the candidate'));
expect(entry).toContain('First entry always launches the initial audit');
expect(entry).toContain('On reentry, reuse only this invocation\'s validated audit or named-risk decision');
expect(entry).toContain('Reentry never resets the count or authorizes a launch');
expect(entry).toContain("this invocation's validated audit or named-risk decision");
expect(entry).toContain('base/input hashes still match');
expect(entry).toContain('retain its actual status and scope');
expect(entry).toContain('Otherwise use Blocked recovery, not an unconditional launch');
expect(docs).toContain('never a third attempt, even after Step 16 changes');
expect(docs).toContain('never reuse an audit across invocations');
expect(docs).toContain('Unconfirmed writers, ownership violations, unauthorized Git mutation and redaction/security gates cannot be waived');
expect(entry).toMatch(/first entry always launches the initial audit/i);
expect(entry).toMatch(/reentry never resets the count or authorizes a launch/i);
expect(docs).toMatch(/never a third attempt/i);
expect(docs).toMatch(/never reuse an audit across invocations/i);
expect(docs).toMatch(/unauthorized Git mutation and redaction\/security gates cannot be waived/i);
});
test('late behavioral repairs rebuild before the docs decision and commit only remaining changes', () => {
const entry = compact(read('ship/SKILL.md.tmpl'));
expect(entry).toContain('A range ending at Step 14 does not enter Step 14.5');
expect(entry).toContain('rebuild and compare again before stage 3 decides documentation freshness');
expect(entry).toMatch(/rebuild and compare again before stage 3/i);
const commit = entry.slice(entry.indexOf('### 5. Report, then push'), entry.indexOf('## Step 17:'));
expect(commit).toContain('left uncommitted after Step 15');
expect(commit).toContain('never create an empty commit');
expect(commit).toContain('Preserve unrelated user files');
expect(commit).toMatch(/never create an empty commit/i);
expect(commit).toMatch(/preserve unrelated user files/i);
});
test('the title is prefixed once before the exact scanned value is published', () => {
const body = compact(read('ship/sections/pr-body.md.tmpl'));
expect(body).toContain("Use Step 18's `NEW_TITLE` unchanged; its version prefix is already present");
expect(body).toContain("Step 18's `NEW_TITLE` unchanged");
expect(body).not.toContain('`NEW_TITLE`, prefixed with');
expect(body).toContain('--title "$NEW_TITLE"');
expect(body).toContain('the same scanned `NEW_TITLE`');
expect(body).toContain('scanned `NEW_TITLE`');
});
File diff suppressed because it is too large. Load diff
+3 -3
View File
@@ -1487,7 +1487,7 @@ describe('Codex skill', () => {
expect(content).toContain('Step 2C: Consult Mode');
const skeleton = fs.readFileSync(path.join(ROOT, 'codex', 'SKILL.md'), 'utf-8');
expect(skeleton).toContain('## Step 1: Detect mode');
expect(skeleton).toContain('MUTUALLY EXCLUSIVE');
expect(skeleton).toMatch(/mutually exclusive/i);
});
test('codex union contains gate verdict logic', () => {
@@ -1587,7 +1587,7 @@ describe('Codex skill', () => {
expect(content).toContain('CODEX_MODE');
expect(content).toContain('command -v codex'); // install check kept literal
// codex_reviews=disabled gates Codex passes only; Claude adversarial still runs
expect(content).toContain('skip the Codex passes ONLY');
expect(content).toMatch(/disabled[^\n]*skip the Codex passes only/i);
// Review log
expect(content).toContain('adversarial-review');
expect(content).toContain('reasoning_effort="high"');
@@ -2023,7 +2023,7 @@ describe('Test failure triage in ship skill', () => {
test('ship/SKILL.md uses in-branch language for stop condition', () => {
const content = readShipUnion();
expect(content).toContain('If any in-branch failures remain unfixed, **STOP**. Do not proceed');
expect(content).toMatch(/in-branch failures remain unfixed[\s\S]{0,40}\bstop\b[\s\S]{0,20}do not proceed/i);
});
});
+57 -69
View File
@@ -14,42 +14,36 @@ function expectOutsideReviewControlFlow(text: string, promptHeading: string): vo
const disabled = text.slice(indices[0], indices[1]);
if (ceo) {
expect(disabled).toContain('"outside_status":"disabled"');
expect(disabled.replace(/\s+/g, ' ')).toContain('without a challenge, CLI invocation, Agent/Task fallback or questions about outside findings');
expect(disabled.replace(/\s+/g, ' ')).toMatch(/without a challenge, CLI invocation, Agent\/Task fallback/i);
expect(disabled).toContain('_DISABLED_REVIEW_MODE=');
expect(disabled).toContain('if [ "$_DISABLED_REVIEW_MODE" = disabled ]');
} else {
expect(disabled).toContain('persist `outside_status: disabled`');
expect(disabled.replace(/\s+/g, ' ')).toMatch(/Do not construct a (?:review prompt|challenge), invoke an outside CLI, dispatch an Agent\/Task fallback/);
expect(disabled.replace(/\s+/g, ' ')).toMatch(/do not construct a (?:review prompt|challenge), invoke an outside CLI, dispatch an Agent\/Task fallback/i);
}
expect(text.slice(indices[1], indices[2])).toContain('(skip only on `disabled`)');
const fallback = text.slice(indices[3]);
if (text.includes('**Outcome routing:**')) {
const routing = text.slice(text.indexOf('**Outcome routing:**'), indices[0]);
expect(routing).toContain('Other preflight mode, including harness mismatch');
expect(routing).toContain('Outside execution or output validation fails');
expect(routing).toContain('Retain its output and diagnosis, finish termination, then use Native fallback');
expect(routing).toContain('No prompt, outside process or native replacement');
expect(fallback.replace(/\s+/g, ' ')).toContain('Immediately before dispatch, check the preflight result again: disabled means no replacement');
expect(fallback.replace(/\s+/g, ' ')).toMatch(/immediately before dispatch, check the preflight result again: disabled means no replacement/i);
} else if (ceo) {
expect(fallback.replace(/\s+/g, ' ')).toContain('Other preflight failures retain their printed diagnosis, including harness mismatch');
expect(fallback.replace(/\s+/g, ' ')).toContain('These failures do not block the review; they use the bounded fallback below');
expect(fallback.replace(/\s+/g, ' ')).toMatch(/do not block the review; they use the bounded fallback/i);
} else {
expect(fallback).toContain('The disabled branch never reaches this fallback.');
expect(fallback.replace(/\s+/g, ' ')).toMatch(/Otherwise, use this fallback for missing\/broken CLI, failed authentication\/model selection, a failed preflight(?: \(including harness mismatch\))?, or a failed outside invocation\./);
expect(fallback).toMatch(/the disabled branch never reaches this fallback/i);
expect(fallback.replace(/\s+/g, ' ')).toMatch(/use this fallback for missing\/broken CLI/i);
}
const dispatch = fallback.indexOf('Dispatch via the Agent tool');
expect(dispatch).toBeGreaterThan(0);
const recheck = fallback.slice(0, dispatch);
const recheck = fallback.slice(0, dispatch).replace(/\s+/g, ' ');
if (ceo) {
expect(recheck.replace(/\s+/g, ' ')).toContain('Immediately before dispatch, recheck the preflight result');
expect(recheck.replace(/\s+/g, ' ')).toContain('`CODEX_MODE: disabled`, return to **Record the disabled outcome** without dispatching');
expect(recheck).toMatch(/immediately before dispatch, recheck the preflight result/i);
expect(recheck).toMatch(/`CODEX_MODE: disabled`, return to \*\*Record the disabled outcome\*\* without dispatching/i);
} else if (!text.includes('**Outcome routing:**')) {
expect(recheck).toContain('Immediately before dispatching, check the preflight result again.');
expect(recheck).toMatch(/immediately before dispatching, check the preflight result again/i);
expect(recheck).toContain('`CODEX_MODE: disabled`, finish this section with `outside_status: disabled`;');
expect(recheck).toContain('do not dispatch.');
expect(recheck).toMatch(/do not dispatch/i);
}
expect(fallback).toContain('Availability/native fallback is not outside completion.');
expect(fallback).toMatch(/availability\/native fallback is not outside completion/i);
}
describe('workflow judge excerpts', () => {
@@ -63,15 +57,13 @@ describe('workflow judge excerpts', () => {
test('ship uses project-native commands and never jumps over mandatory gates', () => {
const text = readWorkflowExcerpt('ship/SKILL.md', '# Ship:', '## Important Rules');
expect(text).toContain("Use the project's test commands discovered in Step 4");
expect(text).toMatch(/test commands discovered in Step 4/i);
expect(text).toContain('**Project-native path:**');
expect(text).toContain('Use the documented selector and pre-merge command.');
expect(text).not.toMatch(/skipping evals[^\n]*Step 9/);
const reviewAndTriage = text.slice(text.indexOf('## Step 9:'), text.indexOf('## Step 11:'));
expect(reviewAndTriage.match(/continue to Step 12/i)).toBeNull();
expect(text).not.toContain('Steps 4-6:');
expect(text).toContain('During pre-flight, read the existing review log');
expect(text).toContain('Save the JSON `baseVersion` as `BASE_VERSION`');
expect(text).toContain('`baseVersion` as `BASE_VERSION`');
expect(text).not.toContain('GIT_SEQUENCE_EDITOR');
expect(text).not.toContain("--exec 'true'");
expect(text).not.toContain('-X ours');
@@ -80,27 +72,28 @@ describe('workflow judge excerpts', () => {
test('ship review shortcuts retain dedup and fixes repeat the whole review cycle', () => {
const text = readWorkflowExcerpt('ship/SKILL.md', '# Ship:', '## Important Rules');
expect(text).toContain("Continue to Step 9.2 with the core/design-lite findings and an empty specialist list, then the parent's Exploratory QA step and Step 9.3 (cross-review dedup)");
const flat = text.replace(/\s+/g, ' ');
expect(flat).toMatch(/Continue to Step 9\.2 [^.]*then the parent's Exploratory QA step and Step 9\.3/i);
expect(text).toContain('## Step 9.4: Fix-First and persistence');
expect(text.replace(/\s+/g, ' ')).toContain('Run checklist/design, specialists (9.1), merge/Red Team (9.2), exploratory QA (9.2.1), dedup (9.3), then fixes and logging (9.4)');
expect(text.replace(/\s+/g, ' ')).toContain('**Fixes applied below the cap:** Insert Step 5, affected Steps 6–8 and all of Step 9 before the pending Step 10 in the work list. Tests must pass or retain approval for the same verified pre-existing failures and scope');
const cycle = flat.slice(flat.indexOf('specialists (9.1)'));
const order = ['(9.1)', '(9.2)', '(9.2.1)', '(9.3)', '(9.4)'].map(step => cycle.indexOf(step));
expect(order.every(index => index >= 0)).toBe(true);
expect(order).toEqual([...order].sort((a, b) => a - b));
expect(flat).toContain('**Fixes applied below the cap:** Insert Step 5, affected Steps 6–8 and all of Step 9 before the pending Step 10');
const audit = text.slice(text.indexOf('## Step 7:'), text.indexOf('## Step 8:'));
expect(audit).not.toContain('Scope Challenge');
expect(text.replace(/\s+/g, ' ')).toContain('Keep actual outcomes and incomplete flags; VERIFY_RESULT stays fail for plan-check exceptions');
expect(flat).toContain('VERIFY_RESULT stays fail');
});
test('ship excerpt preserves readable detours, audit fallback and final input decisions', () => {
const text = readWorkflowExcerpt('ship/SKILL.md', '# Ship:', '## Important Rules').replace(/\s+/g, ' ');
expect(text).toContain('For another repair, repeat rule 2 without discarding pending work');
expect(text).toContain('A further Step 9 fix affecting 6–8 makes the list `5 → 6 → 7 → 8 → 9 → 10 → 11 → 11.5`');
expect(text).toContain('The unchanged release steps follow');
expect(text).toContain('All three snapshots must match');
expect(text).toContain('does not mean the failed or unrun probes passed');
expect(text).toContain('Fallback recovers the audit; it does not pass or bypass the coverage gate');
expect(text).toContain('Skip only the plan completion audit');
expect(text).toContain('Continue with Step 8.1, Scope Drift and Prior Learnings');
expect(text).toContain('Step 9 QA still runs');
expect(text).toContain('Use this example only after confirming that every allowed edit is release metadata');
expect(text).toContain('`5 → 6 → 7 → 8 → 9 → 10 → 11 → 11.5`');
expect(text).toMatch(/all three snapshots must match/i);
expect(text).toMatch(/does not mean the failed or unrun probes passed/i);
expect(text).toMatch(/does not pass or bypass the coverage gate/i);
expect(text).toMatch(/skip only the plan completion audit/i);
expect(text).toContain('Step 8.1');
expect(text).toMatch(/Step 9 QA still runs/i);
expect(text).not.toContain('Every listed change below is metadata:');
expect(text).not.toContain('No plan file found:** Skip entirely');
});
@@ -123,15 +116,16 @@ describe('workflow judge excerpts', () => {
expect(edit).toBeGreaterThan(scan);
expect(publish.indexOf('Print the existing URL')).toBeGreaterThan(edit);
expect(text).not.toContain('Phase 8e.5');
expect(text).toContain('never create an empty commit');
expect(text).toMatch(/never create an empty commit/i);
const review = text.slice(text.indexOf('## Step 9:'), text.indexOf('## Step 10:'));
expect(review.indexOf('## Confidence Calibration')).toBeLessThan(review.indexOf('1. Read'));
const flat = review.replace(/\s+/g, ' ');
expect(flat).toContain('**No edits in this pass:** Resolve the required-probe gate below. Only after it clears may you continue to Step 10');
expect(flat).toContain('**Dispatched reviewer output missing:** STOP');
expect(flat).toContain('Retain queued fixes and restore coverage');
expect(flat).toContain('**Third fixing cycle reached (`CYCLES >= 3`):** STOP and report recurring findings with `converged:false`; do not run a fourth fixing cycle');
expect(flat).toContain('With completed checklist and dispatched reviewers, failed/unavailable required probes block continuation');
expect(flat).toContain('**No edits in this pass:**');
expect(flat).toMatch(/only after it clears may you continue to Step 10/i);
expect(flat).toMatch(/\*\*Dispatched reviewer output missing:\*\* stop\b/i);
expect(flat).toMatch(/\*\*Third fixing cycle reached \(`CYCLES >= 3`\):\*\* stop\b[^.]*`converged:false`/i);
expect(flat).toMatch(/do not run a fourth fixing cycle/i);
expect(flat).toMatch(/failed\/unavailable required probes block continuation/i);
});
test('ship approval gates stay outside the subagent prompts', () => {
@@ -145,7 +139,7 @@ describe('workflow judge excerpts', () => {
expect(section.indexOf(gate)).toBeGreaterThan(section.indexOf('\n````\n'));
}
expect(text).toContain('"partial":N,"not_done":N');
expect(text).toContain('each Y\'d item with the user\'s free-text evidence and each D\'d item with "intentionally dropped"');
expect(text).toContain('"intentionally dropped"');
});
test('expands a body before the end marker in the skeleton', () => {
@@ -163,13 +157,13 @@ describe('workflow judge excerpts', () => {
expect(commit).toBeGreaterThan(reviewStart);
const review = text.slice(reviewStart, commit);
expectOutsideReviewControlFlow(review, '**Construct the doc-review prompt**');
expect(review).toContain('Skip the apply gate, persist `status: unavailable`, `outside_status: unavailable`, and `source: none`');
expect(review).toContain('present the findings, then use AskUserQuestion ONCE:');
expect(review).toContain('On A or per-finding approvals, make the approved edits yourself');
expect(text).toContain('Step 9 then commits and pushes those edits');
for (const token of ['`status: unavailable`', '`outside_status: unavailable`', '`source: none`']) expect(review).toContain(token);
expect(review).toMatch(/AskUserQuestion once/i);
expect(review).toMatch(/on A or per-finding approvals/i);
expect(text).toContain('Step 9 then commits');
expect(text).toMatch(/needs?\s+attention,?\s+not\s+replacement/i);
expect(text).not.toContain('Flag and rewrite');
expect(text).toContain('if VERSION is absent, use the completion date only');
expect(text).toMatch(/if VERSION is absent, use the completion date only/i);
});
test('Eng preparation and decision procedure precede the four review sections', async () => {
@@ -261,8 +255,8 @@ console.log(JSON.stringify({calls, results}));
expect(call.userPrompt).toContain('Proceed to Mode Selection,');
expect(call.userPrompt).not.toMatch(/Step 0[A-Z]/);
expect(call.userPrompt).toContain(join(call.workingDirectory, 'plan-ceo-review', 'SKILL.md'));
expect(call.userPrompt).toContain('Do NOT search for, Glob, find, or read any OTHER SKILL.md');
expect(call.userPrompt).toContain('Ask the user through the AskUserQuestion tool and wait for their answer.');
expect(call.userPrompt).toMatch(/read any other SKILL\.md/i);
expect(call.userPrompt).toMatch(/AskUserQuestion tool and wait for their answer/i);
expect(call).toMatchObject({ allowedTools: ['Read', 'Write', 'AskUserQuestion'], maxTurns: 12, maxRetries: 0, model: 'fake-model' });
}
} finally { rmSync(dir, { recursive: true, force: true }); }
@@ -273,28 +267,27 @@ console.log(JSON.stringify({calls, results}));
const tests = eng.slice(eng.indexOf('### 3. Test review'), eng.indexOf('### 4. Performance review'));
const scope = tests.indexOf('### LLM/eval scope');
const decisions = tests.indexOf('**Step 5. Add missing tests to the plan:**');
const stop = tests.indexOf("**STOP for each pending decision.**", decisions);
const stop = decisions + tests.slice(decisions).search(/stop for each pending decision/i);
const artifact = tests.indexOf('### Test Plan Artifact');
expect(0 <= scope && scope < decisions && decisions < stop && stop < artifact).toBe(true);
expect(tests.match(/For LLM\/prompt changes:/g)).toHaveLength(1);
expect(tests.slice(artifact)).not.toContain("**STOP for each pending decision.**");
expect(tests.slice(artifact)).not.toMatch(/stop for each pending decision/i);
const fastPath = tests.slice(tests.indexOf('**Fast path:**'), scope);
expect(fastPath).toContain('Still check LLM/eval scope and produce the Test Plan Artifact');
expect(fastPath).toMatch(/still check LLM\/eval scope/i);
});
test('plan review evidence and design approval rules precede their use', () => {
const eng = readWorkflowExcerpt('plan-eng-review/SKILL.md', '## Scope gate', '## Section self-check');
expect(eng.indexOf('## Confidence Calibration')).toBeLessThan(eng.indexOf('### 1. Architecture review'));
expect(eng).toContain('quote the motivating plan requirement');
expectOutsideReviewControlFlow(eng, '**Construct the plan review prompt**');
expect(eng).toContain('Agreement between reviewers is evidence, not approval');
expect(eng).toContain('new or reopened choices still need their own answers');
expect(eng).toMatch(/evidence, not approval/i);
expect(eng).toMatch(/new or reopened choices still need their own answers/i);
const pendingDecision = eng.slice(eng.indexOf('### Send once and wait'), eng.indexOf("### Record the answer"));
expect(pendingDecision).toContain("**STOP until the actual answer arrives.**");
expect(pendingDecision.replace(/\s+/g, ' ')).toContain("Do not apply a remedy, make another call, start the next section or call ExitPlanMode while the choice awaits an answer");
expect(eng.replace(/\s+/g, ' ')).toContain("Use a scoped Edit to save this record and only the authorized working-plan amendments. Leave other choices unchanged");
expect(pendingDecision).toMatch(/stop until the actual answer arrives/i);
expect(pendingDecision.replace(/\s+/g, ' ')).toMatch(/do not apply a remedy, make another call, start the next section or call ExitPlanMode while the choice awaits an answer/i);
expect(eng.replace(/\s+/g, ' ')).toMatch(/only the authorized working-plan amendments/i);
const design = readWorkflowExcerpt('plan-design-review/SKILL.md', '## Review Sections', ASK_QUESTIONS_HEADING);
expect(design).toContain('wait for approval, then edit the plan and re-rate');
expect(design).toMatch(/wait for approval, then edit the plan/i);
const pass4 = design.slice(design.indexOf('### Pass 4:'), design.indexOf('### Pass 5:'));
expect(pass4.match(/^### /gm)).toHaveLength(1);
expect(pass4).toMatch(/^#### Design Hard Rules$/m);
@@ -311,13 +304,9 @@ console.log(JSON.stringify({calls, results}));
test('retro judge includes compare semantics and unambiguous report inputs', () => {
const text = readWorkflowExcerpt('retro/SKILL.md', '## Instructions', '## Tone');
expect(text).toContain('## Compare Mode');
expect(text).toContain('does not require saved history');
expect(text).toContain('one second before the current start');
expect(text).toContain('PRs referenced');
expect(text).toMatch(/one second before the current start/i);
expect(text).toContain('prs_merged: null');
expect(text).toContain('not newly added test cases');
expect(text).toContain('`streak_days` is the live **team** streak');
expect(text).toContain('draft the tweetable summary using the format in Step 14, then save');
expect(text).toContain('`streak_days`');
expect(text).toContain('### Shipping Streaks');
expect(text).toContain('### Shortcut Debt');
expect(text.indexOf('## Capture Learnings')).toBeGreaterThan(text.indexOf('### Step 14:'));
@@ -343,9 +332,8 @@ console.log(JSON.stringify({calls, results}));
test('ship commits logical chunks without rewriting existing checkpoint commits', () => {
const text = readWorkflowExcerpt('ship/SKILL.md', '# Ship:', '## Important Rules');
const commit = text.slice(text.indexOf('## Step 15:'), text.indexOf('## Step 16:'));
expect(commit).toContain('Make bisectable commits');
expect(commit).toContain('if already committed, continue to Step 16');
expect(commit).toContain('Each commit must work independently');
expect(commit).toMatch(/bisectable commits/i);
expect(commit).toContain('continue to Step 16');
expect(commit).not.toMatch(/rebase|reset|squash|fixup|WIP_TODO|gstack-context/);
expect(text).not.toContain('Step 15.0');
});