mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-12 19:44:48 +01:00
Merge remote-tracking branch 'origin/garrytan/opus55-wip-F1' into garrytan/opus-5-5-prompt-audit
This commit is contained in:
commit
6d7935eae8
13 files changed
+882
-1684
No files matched your search
@@ -8,12 +8,11 @@
|
||||
* only the STOP-Read pointer). After ANY non-zero `gh pr merge`, the skill
|
||||
* must query authoritative PR state via
|
||||
* GraphQL (including queue membership) and
|
||||
* branch on the result instead of blindly retrying `gh pr merge` (cli/cli#3442,
|
||||
* cli/cli#13380).
|
||||
* branch on the result instead of blindly retrying `gh pr merge`.
|
||||
*
|
||||
* Static invariants pin:
|
||||
* - §4a-postfail header present
|
||||
* - Universal invariant text + reference to upstream gh bugs
|
||||
* - Query-before-retry invariant after any non-zero merge exit
|
||||
* - All three state branches (MERGED, OPEN, CLOSED) named explicitly
|
||||
* - MERGED branch: capture merge SHA via mergeCommit.oid
|
||||
* - MERGED branch: non-destructive worktree cleanup with uncommitted-work guard
|
||||
@@ -52,12 +51,9 @@ describe("PR #1620 §4a-postfail in land-and-deploy template", () => {
|
||||
expect(postfail).toBeLessThan(queue);
|
||||
});
|
||||
|
||||
test("Universal invariant + upstream gh bug references", () => {
|
||||
const body = readTmpl();
|
||||
expect(body).toMatch(/Universal invariant/);
|
||||
expect(body).toMatch(/non-zero exit from `gh pr merge`/);
|
||||
expect(body).toMatch(/cli\/cli#3442/);
|
||||
expect(body).toMatch(/cli\/cli#13380/);
|
||||
test("any non-zero merge exit queries authoritative state before retrying", () => {
|
||||
const body = readTmpl().replace(/\s+/g, " ");
|
||||
expect(body).toMatch(/non-zero exit from `gh pr merge`, query authoritative PR state before retrying/i);
|
||||
});
|
||||
|
||||
test("Authoritative state query includes auto request and queue membership", () => {
|
||||
@@ -82,9 +78,9 @@ describe("PR #1620 §4a-postfail in land-and-deploy template", () => {
|
||||
test("MERGED worktree cleanup is non-destructive (uncommitted-work guard)", () => {
|
||||
const body = readTmpl();
|
||||
expect(body).toMatch(/uncommitted work/);
|
||||
expect(body).toMatch(/STOP worktree cleanup without removing/);
|
||||
expect(body).toMatch(/Do NOT use `--force`/);
|
||||
expect(body).toMatch(/Do NOT remove the user's primary working tree/);
|
||||
expect(body).toMatch(/stop worktree cleanup without removing/i);
|
||||
expect(body).toMatch(/do not use `--force`/i);
|
||||
expect(body).toMatch(/do not remove the user's primary working tree/i);
|
||||
});
|
||||
|
||||
test("MERGED branch continues to §4b CI auto-deploy detection", () => {
|
||||
@@ -114,10 +110,10 @@ describe("PR #1620 §4a-postfail in land-and-deploy template", () => {
|
||||
test("MERGED branch reconciliation distinguishes branch-absent from check-failed", () => {
|
||||
const body = readTmpl();
|
||||
// exit 0 + empty output = already clean (idempotent re-runs)...
|
||||
expect(body).toMatch(/already been cleaned up/);
|
||||
expect(body).toMatch(/already been cleaned up/i);
|
||||
// ...non-zero exit = unknown state, never read as a clean branch.
|
||||
expect(body).toMatch(/Couldn't verify remote branch state/);
|
||||
expect(body).toMatch(/never read a failed check as a clean branch/);
|
||||
expect(body).toMatch(/couldn't verify remote branch state/i);
|
||||
expect(body).toMatch(/never read a failed check as a clean branch/i);
|
||||
});
|
||||
|
||||
test("OPEN branch checks autoMergeRequest before treating as failure", () => {
|
||||
@@ -128,15 +124,15 @@ describe("PR #1620 §4a-postfail in land-and-deploy template", () => {
|
||||
|
||||
test("CLOSED branch STOPs", () => {
|
||||
const body = readTmpl();
|
||||
expect(body).toMatch(/state == "CLOSED".*[\s\S]{0,200}STOP/);
|
||||
expect(body).toMatch(/state == "CLOSED"[\s\S]{0,200}\bstop\b/i);
|
||||
});
|
||||
|
||||
test("Hard rule: no replay after MERGED and only one guarded direct fallback", () => {
|
||||
const body = readTmpl();
|
||||
expect(body).toMatch(/never\s+replay a merge after MERGED/);
|
||||
expect(body).toContain('one direct fallback');
|
||||
expect(body).toContain('There is no fallback from a direct attempt');
|
||||
expect(body).toContain('readback has confirmed OPEN, no auto request and no queue entry');
|
||||
const body = readTmpl().replace(/\s+/g, " ");
|
||||
expect(body).toMatch(/never replay a merge after MERGED/i);
|
||||
expect(body).toMatch(/one direct fallback/i);
|
||||
expect(body).toMatch(/no fallback from a direct attempt/i);
|
||||
expect(body).toMatch(/confirmed OPEN, no auto request and no queue entry/i);
|
||||
});
|
||||
|
||||
test("Generated merge-and-deploy.md carries the §4a-postfail section (atomic regen per T-Codex-3)", () => {
|
||||
|
||||
@@ -26,35 +26,34 @@ function render(file: string, ctx: TemplateContext): string {
|
||||
}
|
||||
|
||||
function assertProbeLocalBlocker(body: string): void {
|
||||
expect(body).toContain('QA setup blocker');
|
||||
expect(body).toContain('affected probes as blocked');
|
||||
expect(body).toContain('continue other safe probes');
|
||||
expect(body).toContain('required QA');
|
||||
expect(body).toContain('independent functional/static checks');
|
||||
expect(body).toMatch(/QA setup blocker/i);
|
||||
expect(body).toMatch(/affected probes as blocked/i);
|
||||
expect(body).toMatch(/continue other safe probes/i);
|
||||
expect(body).toMatch(/independent functional\/static checks/i);
|
||||
expect(body).not.toContain('stop that workflow');
|
||||
expect(body).not.toContain('stop all checks');
|
||||
}
|
||||
|
||||
function assertSharedBrowserAuthority(body: string): void {
|
||||
const decision = body.slice(body.indexOf('## Browser access decision'), body.indexOf('## BROWSER SETUP'));
|
||||
expect(decision).toContain('invoking workflow, not this file');
|
||||
expect(decision).toMatch(/invoking workflow, not this file/i);
|
||||
const reportOnly = decision.slice(decision.indexOf('**Report-only'), decision.indexOf('**Standalone /qa'));
|
||||
expect(reportOnly).toContain('/qa-only, /review and /ship');
|
||||
expect(reportOnly).toContain("do not run the fallback's setup/install or cookie-import workflow");
|
||||
expect(reportOnly).toContain('Never bootstrap or invoke another skill');
|
||||
expect(reportOnly).toContain('block only the affected browser probes');
|
||||
expect(reportOnly).toContain('continue independent functional/static checks');
|
||||
expect(reportOnly).toMatch(/do not run the fallback's setup\/install or cookie-import workflow/i);
|
||||
expect(reportOnly).toMatch(/never bootstrap or invoke another skill/i);
|
||||
expect(reportOnly).toMatch(/block only the affected browser probes/i);
|
||||
expect(reportOnly).toMatch(/continue independent functional\/static checks/i);
|
||||
expect(reportOnly).not.toContain('run `cd');
|
||||
const standalone = decision.slice(decision.indexOf('**Standalone /qa'));
|
||||
expect(standalone).toContain('explicit approval');
|
||||
expect(standalone).toContain('STOP and wait');
|
||||
expect(standalone).toContain('Only after approval');
|
||||
expect(standalone).toContain('`cd <SKILL_DIR> && ./setup`');
|
||||
const ask = standalone.search(/explicit approval/i);
|
||||
const gate = standalone.search(/only after approval/i);
|
||||
const setup = standalone.indexOf('`cd <SKILL_DIR> && ./setup`');
|
||||
expect(ask).toBeGreaterThan(-1);
|
||||
expect(gate).toBeGreaterThan(ask);
|
||||
expect(setup).toBeGreaterThan(gate);
|
||||
expect(standalone).toContain('/setup-browser-cookies');
|
||||
expect(standalone).toContain('declined, unavailable or unsuccessful');
|
||||
expect(standalone).toContain('blocked');
|
||||
expect(decision).toContain('Unknown caller');
|
||||
expect(decision).toContain('report-only');
|
||||
expect(standalone).toMatch(/declined, unavailable or unsuccessful[\s\S]{0,40}blocked/i);
|
||||
expect(decision).toMatch(/unknown caller[^\n]*report-only/i);
|
||||
expect(body).not.toContain('If `NEEDS_SETUP`: tell the user');
|
||||
expect(body).not.toContain('An authenticated page needs /setup-browser-cookies');
|
||||
}
|
||||
@@ -64,16 +63,11 @@ describe('QA caller authority in pure host renders', () => {
|
||||
test(`${host.name}: a scope handoff requires the actual prior method read`, () => {
|
||||
for (const caller of callers) {
|
||||
const body = RESOLVERS.QA_EXPLORATORY(context(host.name, caller));
|
||||
expect(body).toContain('Read `sections/scope.md`');
|
||||
expect(body).toContain('Do not repeat a Read already completed in this invocation');
|
||||
expect(body).toContain('Complete these Reads in order before writing charters or probing');
|
||||
const scope = body.indexOf('1. Read `sections/scope.md`');
|
||||
const selection = body.indexOf('in full and select the surfaces');
|
||||
const methods = body.indexOf('2. Read the selected surface methods below in full');
|
||||
const scope = body.indexOf('Read `sections/scope.md`');
|
||||
const methods = body.indexOf('**Functional surfaces:**');
|
||||
expect(scope).toBeGreaterThan(-1);
|
||||
expect(selection).toBeGreaterThan(scope);
|
||||
expect(methods).toBeGreaterThan(selection);
|
||||
expect(body.indexOf('Write a **charter**')).toBeGreaterThan(methods);
|
||||
expect(methods).toBeGreaterThan(scope);
|
||||
expect(body.indexOf('## 1. Charter and preflight')).toBeGreaterThan(methods);
|
||||
expect(body).not.toContain('If the caller has not selected surfaces and established isolation');
|
||||
}
|
||||
});
|
||||
@@ -82,12 +76,10 @@ describe('QA caller authority in pure host renders', () => {
|
||||
for (const caller of callers) {
|
||||
const body = RESOLVERS.QA_EXPLORATORY(context(host.name, caller));
|
||||
expect(body).toContain('The **caller** (/qa, /qa-only, /review or /ship)');
|
||||
expect(body).toContain('charter** per behavior');
|
||||
expect(body).toContain('bun G start D SECONDS [EARLIER_UTC]');
|
||||
expect(body).toContain('G enforces the deadline');
|
||||
expect(body).toContain('announce finite command timeouts');
|
||||
expect(body.indexOf('bun G start D')).toBeLessThan(body.indexOf('1. First demonstrate success'));
|
||||
expect(body).toContain('scoped contracts are tested or blocked');
|
||||
expect(body).toMatch(/stop when scoped contracts are tested or blocked/i);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -96,9 +88,8 @@ describe('QA caller authority in pure host renders', () => {
|
||||
...context(host.name, 'qa'), tmplPath: 'qa/SKILL.md.tmpl', preambleTier: 4,
|
||||
});
|
||||
const final = body.slice(body.indexOf('## Phase 9: Final QA'), body.indexOf('## Phase 10: Report'));
|
||||
expect(final).toContain('Re-run affected contracts and adjacent happy paths on the final inputs');
|
||||
expect(final).toContain('Caller-required rechecks cannot be skipped as unaffected');
|
||||
expect(final).toContain('blocked/inconclusive rechecks never verify repairs');
|
||||
expect(final).toMatch(/caller-required rechecks cannot be skipped as unaffected/i);
|
||||
expect(final).toMatch(/blocked\/inconclusive rechecks never verify repairs/i);
|
||||
});
|
||||
|
||||
test(`${host.name}: prior learnings name QA findings without changing review callers`, () => {
|
||||
@@ -116,12 +107,10 @@ describe('QA caller authority in pure host renders', () => {
|
||||
test(`${host.name}: report-only learning lookup cannot configure or initialize stores`, () => {
|
||||
const ctx = context(host.name, 'qa-only');
|
||||
const search = RESOLVERS.LEARNINGS_SEARCH(ctx, ['query=webhook retries']);
|
||||
expect(search).toContain('Read this project\'s existing learnings.jsonl only if its directory is already known');
|
||||
expect(search).toContain('the caller permits that Read');
|
||||
expect(search).toContain('Otherwise skip this optional lookup');
|
||||
expect(search).toContain('Look for notes matching "webhook retries"');
|
||||
expect(search).toContain('Do not run gstack-learnings-search here');
|
||||
expect(search).toContain('Reading old notes never requires writing new ones');
|
||||
expect(search).toMatch(/learnings\.jsonl only if its directory is already known/i);
|
||||
expect(search).toMatch(/the caller permits that Read/i);
|
||||
expect(search).toContain('"webhook retries"');
|
||||
expect(search).toMatch(/do not run gstack-learnings-search here/i);
|
||||
expect(search).not.toContain('```bash');
|
||||
expect(search).not.toContain('gstack-config');
|
||||
expect(search).not.toContain('AskUserQuestion');
|
||||
@@ -137,7 +126,7 @@ describe('QA caller authority in pure host renders', () => {
|
||||
assertProbeLocalBlocker(pointer);
|
||||
expect(pointer).toContain(`\`sections/${id}.md\``);
|
||||
expect(pointer).toContain('SKILL.md directory');
|
||||
expect(pointer).toContain(sharedSetup ? 'No product-directory or cross-host substitutes' : 'never the product working directory');
|
||||
expect(pointer).toMatch(sharedSetup ? /no product-directory or cross-host substitutes/i : /never the product working directory/i);
|
||||
expect(pointer).not.toContain('## BROWSER SETUP');
|
||||
expect(pointer).not.toContain('aside repl');
|
||||
}
|
||||
@@ -164,12 +153,11 @@ describe('QA caller authority in pure host renders', () => {
|
||||
assertSharedBrowserAuthority(body);
|
||||
expect(body.indexOf('## Browser access decision')).toBeLessThan(body.indexOf('## BROWSER SETUP'));
|
||||
expect(body.indexOf('## BROWSER SETUP')).toBeLessThan(body.indexOf('## Browser fallback'));
|
||||
expect(body).toContain('Functional-only');
|
||||
expect(body).toContain('do not probe Aside');
|
||||
expect(body).toContain("scope section's ownership rules apply even to LOCAL browser targets");
|
||||
expect(body).toContain('never substitute unit tests or curl for the browser step');
|
||||
expect(body).toContain('Never type passwords, one-time codes, or payment details');
|
||||
expect(body).toContain('Never read, screenshot, navigate, or close any other tab');
|
||||
expect(body.replace(/\s+/g, ' ')).toMatch(/functional-only targets do not probe Aside/i);
|
||||
expect(body).toMatch(/ownership rules apply even to local browser targets/i);
|
||||
expect(body).toMatch(/never substitute unit tests or curl for the browser step/i);
|
||||
expect(body).toMatch(/never type passwords, one-time codes, or payment details/i);
|
||||
expect(body).toMatch(/never read, screenshot, navigate, or close any other tab/i);
|
||||
expect((body.match(/cd <SKILL_DIR> && \.\/setup/g) ?? [])).toHaveLength(1);
|
||||
});
|
||||
|
||||
@@ -191,22 +179,19 @@ describe('QA caller authority in pure host renders', () => {
|
||||
const auth = body.slice(body.indexOf('### Phase 2:'), body.indexOf('### Phase 3:'));
|
||||
expect(auth).toContain('Browser access decision');
|
||||
expect(auth).not.toContain('Fallback: /setup-browser-cookies or');
|
||||
expect(auth).toContain('Never handle credentials');
|
||||
expect(body).toContain('Run only for selected browser surfaces');
|
||||
expect(body).toContain('Confirm each issue by retrying once');
|
||||
expect(auth).toMatch(/never handle credentials/i);
|
||||
});
|
||||
|
||||
test(`${host.name}: functional-only and report-only paths preserve independent checks and gates`, () => {
|
||||
const scope = RESOLVERS.QA_SCOPE(context(host.name, 'qa'));
|
||||
expect(scope).toContain('Functional-only runs must not read browser setup, methodology, verification or bootstrap');
|
||||
expect(scope).toMatch(/functional-only runs must not read browser setup/i);
|
||||
expect(scope).not.toContain('command -v aside');
|
||||
expect(scope).not.toContain('curl -sI');
|
||||
for (const caller of callers) {
|
||||
const ctx = context(host.name, caller);
|
||||
const body = RESOLVERS.QA_EXPLORATORY(ctx);
|
||||
const compact = body.replace(/\s+/g, ' ');
|
||||
expect(compact).toContain('Missing or unreadable assets, prerequisites or permission block affected probes, not independent safe checks');
|
||||
expect(compact).toContain('Report QA setup blockers');
|
||||
expect(compact).toMatch(/block affected probes, not independent safe checks/i);
|
||||
expect(compact).not.toContain('stop all checks');
|
||||
expect(body).toContain('2. Read the selected surface methods below in full');
|
||||
expect(body.indexOf('2. Read the selected surface methods below in full')).toBeLessThan(body.indexOf('1. First demonstrate success'));
|
||||
@@ -215,29 +200,29 @@ describe('QA caller authority in pure host renders', () => {
|
||||
expect(reads).toContain('sections/system-functional.md');
|
||||
expect(reads).toContain('**Browser surfaces only:**');
|
||||
expect(reads).toContain('sections/qa-patterns.md');
|
||||
expect(body).toContain('no workflows, framework installs or publication');
|
||||
expect(body).toContain('owns decisions, tests, fixes and publication');
|
||||
expect(body).toContain('Missing prerequisites/expectations/observations, timeouts and refusal never pass');
|
||||
expect(body).toContain('Pass requires all required current-input contracts to pass with no required remainder');
|
||||
expect(body).toMatch(/no workflows, framework installs or publication/i);
|
||||
expect(body).toMatch(/owns decisions, tests, fixes and publication/i);
|
||||
expect(body).toMatch(/timeouts and refusal never pass/i);
|
||||
expect(body).toMatch(/pass requires all required current-input contracts to pass/i);
|
||||
if (caller !== 'qa-only') {
|
||||
expect(body).toContain('leaves /review incomplete');
|
||||
expect(body).toContain('unless the user explicitly accepts that named risk');
|
||||
expect(body).toContain('noninteractive runs return blocked');
|
||||
expect(body).toContain('test_stub proposals require ASK approval');
|
||||
expect(body).toMatch(/unless the user explicitly accepts that named risk/i);
|
||||
expect(body).toMatch(/noninteractive runs return blocked/i);
|
||||
expect(body).toMatch(/test_stub proposals require ASK approval/i);
|
||||
}
|
||||
}
|
||||
const review = RESOLVERS.QA_REVIEW(context(host.name, 'review'));
|
||||
const ship = RESOLVERS.QA_REVIEW(context(host.name, 'ship'));
|
||||
expect(review).toContain('a ship waiver cannot complete it');
|
||||
expect(ship).toContain('explicit named-risk acceptance');
|
||||
expect(ship.replace(/\s+/g, ' ')).toContain('Report clean/completed only when all required checks pass on current inputs');
|
||||
expect(ship).toContain('List failed, blocked, inconclusive and not-run checks');
|
||||
expect(review).toMatch(/a ship waiver cannot complete it/i);
|
||||
expect(ship).toMatch(/explicit named-risk acceptance/i);
|
||||
expect(ship.replace(/\s+/g, ' ')).toMatch(/report clean\/completed only when all required checks pass on current inputs/i);
|
||||
});
|
||||
|
||||
test(`${host.name}: non-QA fallback retains its existing setup and human sign-in flow`, () => {
|
||||
const fallback = RESOLVERS.BROWSE_FALLBACK(context(host.name, 'browse'));
|
||||
expect(fallback).toContain('OK to proceed?');
|
||||
expect(fallback).toContain('STOP for the answer, then run `cd <SKILL_DIR> && ./setup`');
|
||||
expect(fallback).toMatch(/stop for the answer, then run `cd <SKILL_DIR> && \.\/setup`/i);
|
||||
expect(fallback.indexOf('OK to proceed?')).toBeLessThan(fallback.indexOf('cd <SKILL_DIR> && ./setup'));
|
||||
expect(fallback).toContain('An authenticated page needs /setup-browser-cookies');
|
||||
expect(fallback).toContain('$B handoff');
|
||||
expect(fallback).toContain('$B resume');
|
||||
@@ -253,23 +238,17 @@ describe('QA caller authority in pure host renders', () => {
|
||||
const resource = RESOLVERS.QA_RESOURCE(ctx, ['exploratory']);
|
||||
expect(resource).toContain(`../${host.name === 'claude' ? 'qa' : 'gstack-qa'}/sections/exploratory.md`);
|
||||
const shared = render('qa/sections/exploratory.md.tmpl', context(host.name, 'qa'));
|
||||
expect(shared).toContain('Read `sections/system-functional.md` in full');
|
||||
expect(shared.indexOf('in full and select the surfaces')).toBeLessThan(shared.indexOf('Read `sections/system-functional.md`'));
|
||||
expect(shared.indexOf('Read `sections/scope.md`')).toBeLessThan(shared.indexOf('Read `sections/system-functional.md`'));
|
||||
const required = body.indexOf(caller === 'review'
|
||||
? '2. Check readiness and list required checks' : '2. List required checks');
|
||||
expect(required).toBeGreaterThan(-1);
|
||||
expect(exploration).toBeLessThan(required);
|
||||
expect(body).toContain('one success and the riskiest changed failure/edge');
|
||||
expect(body).toContain('Smoke: 5 minutes/12 probes');
|
||||
expect(body).toContain('Required even for small diffs or missing plans/servers');
|
||||
expect(body).toContain('Required: plan commands/assertions, listed separately');
|
||||
expect(body).toContain('Other ideas are optional, untested');
|
||||
expect(body.replace(/\s+/g, ' ')).toContain('Report clean/completed only when all required checks pass on current inputs');
|
||||
expect(body).toContain('List failed, blocked, inconclusive and not-run checks');
|
||||
expect(body).toContain('5 minutes/12 probes');
|
||||
expect(body.replace(/\s+/g, ' ')).toMatch(/report clean\/completed only when all required checks pass on current inputs/i);
|
||||
}
|
||||
const ship = RESOLVERS.QA_REVIEW(context(host.name, 'ship'));
|
||||
expect(ship).toContain('Step 9.4 asks: permission/repair');
|
||||
expect(ship).toContain('explicit named-risk acceptance; otherwise blocked');
|
||||
expect(ship).toContain('Step 9.4');
|
||||
expect(ship).toMatch(/explicit named-risk acceptance; otherwise blocked/i);
|
||||
});
|
||||
|
||||
test(`${host.name}: caller QA selects surfaces directly and links checkpoints in one final section`, () => {
|
||||
@@ -279,33 +258,25 @@ describe('QA caller authority in pure host renders', () => {
|
||||
const exploration = body.indexOf('{{QA_RESOURCE:exploratory}}');
|
||||
const shared = render('qa/sections/exploratory.md.tmpl', context(host.name, 'qa'));
|
||||
const scope = shared.indexOf('Read `sections/scope.md`');
|
||||
const selection = shared.indexOf('in full and select the surfaces');
|
||||
const methods = shared.indexOf('**Functional surfaces:**');
|
||||
const probes = body.indexOf(caller === 'review'
|
||||
? '2. Check readiness and list required checks' : '2. List required checks');
|
||||
expect(scope).toBeGreaterThan(-1);
|
||||
expect(exploration).toBeGreaterThan(-1);
|
||||
expect(selection).toBeGreaterThan(scope);
|
||||
expect(methods).toBeGreaterThan(selection);
|
||||
expect(shared.indexOf('Write a **charter**')).toBeGreaterThan(methods);
|
||||
expect(methods).toBeGreaterThan(scope);
|
||||
expect(shared.indexOf('## 1. Charter and preflight')).toBeGreaterThan(methods);
|
||||
expect(exploration).toBeLessThan(probes);
|
||||
expect(body).not.toContain('**Functional surfaces:**');
|
||||
expect(RESOLVERS.QA_RESOURCE(ctx, ['exploratory'])).toContain(`../${host.name === 'claude' ? 'qa' : 'gstack-qa'}/sections/exploratory.md`);
|
||||
if (caller === 'review') {
|
||||
const flat = body.replace(/\s+/g, ' ');
|
||||
expect(flat).toContain('Title it `## Exploratory QA and Verification Results`');
|
||||
expect(flat).toContain('keep metadata/outcome tables');
|
||||
expect(flat).toContain('demote other headings one level');
|
||||
expect(flat).toContain('include it here under `### Browser results`');
|
||||
expect(flat).toContain('other headings demoted two levels');
|
||||
expect(flat).toContain('Link every checkpoint');
|
||||
expect(flat).toContain('No second report');
|
||||
expect(flat).toContain('Keep browser/functional scores and outcomes separate');
|
||||
expect(flat).toContain('save browser baseline/evidence normally');
|
||||
expect(flat).toContain('`## Exploratory QA and Verification Results`');
|
||||
expect(flat).toContain('`### Browser results`');
|
||||
expect(flat).toMatch(/link every checkpoint/i);
|
||||
expect(flat).toMatch(/no second report/i);
|
||||
} else {
|
||||
expect(body.replace(/\s+/g, ' ')).toContain('PR section `## Exploratory QA');
|
||||
expect(body).toContain('fields as subsections');
|
||||
expect(body).toContain('Link every checkpoint; no second report');
|
||||
expect(body.replace(/\s+/g, ' ')).toContain('`## Exploratory QA');
|
||||
expect(body).toMatch(/link every checkpoint; no second report/i);
|
||||
}
|
||||
expect(body).toContain('templates/functional-report-template.md');
|
||||
expect(body).not.toContain('not a second report');
|
||||
@@ -314,14 +285,14 @@ describe('QA caller authority in pure host renders', () => {
|
||||
|
||||
test(`${host.name}: orchestration logs reviewer attempts before parent-owned edits`, () => {
|
||||
const body = RESOLVERS.ADVERSARIAL_STEP(context(host.name, 'review'));
|
||||
expect(body).toContain("queued for the parent's Fix-First handling at Step 5; do not edit during Step 4.8");
|
||||
expect(body).toContain('do not start an inner repair loop');
|
||||
expect(body.replace(/\s+/g, ' ')).toContain("Keep each token with that attempt; do not overwrite the parent's REVIEW_START");
|
||||
expect(body.replace(/\s+/g, ' ')).toContain('save one record per source, phase and attempt, before the');
|
||||
expect(body).toContain('parent applies queued fixes');
|
||||
expect(body).toContain('Each token is consumed once');
|
||||
const flat = body.replace(/\s+/g, ' ');
|
||||
expect(flat).toMatch(/Fix-First handling at Step 5; do not edit during Step 4\.8/i);
|
||||
expect(flat).toMatch(/do not start an inner repair loop/i);
|
||||
expect(flat).toMatch(/do not overwrite the parent's REVIEW_START/i);
|
||||
expect(flat).toMatch(/one record per source, phase and attempt/i);
|
||||
expect(flat).toMatch(/each token is consumed once/i);
|
||||
expect(body).not.toContain('address the findings. Re-run the same shared structured invocation');
|
||||
expect(body).toContain('The native pass is required for Step 5.8 completion');
|
||||
expect(flat).toMatch(/native pass is required for Step 5\.8 completion/i);
|
||||
});
|
||||
|
||||
test(`${host.name}: orchestration skips only history matching without user skips`, () => {
|
||||
@@ -329,17 +300,12 @@ describe('QA caller authority in pure host renders', () => {
|
||||
const body = RESOLVERS.CROSS_REVIEW_DEDUP(context(host.name, caller));
|
||||
if (caller === 'ship') {
|
||||
const flat = body.replace(/\s+/g, ' ');
|
||||
expect(flat).toContain('Combine saved `findings` with the invocation action list, honoring later user decisions');
|
||||
expect(flat).toContain('If both history and the invocation action list lack decisions, classify normally');
|
||||
expect(body.indexOf('2. **Read decisions.**')).toBeLessThan(body.indexOf('3. **Match evidence.**'));
|
||||
expect(flat).toContain('Only explicit `skipped` actions qualify, never `fixed`, `auto-fixed` or unanswered questions');
|
||||
expect(flat).toContain('Report the suppressed count once if nonzero');
|
||||
expect(flat).toMatch(/only explicit `skipped` actions qualify, never `fixed`, `auto-fixed` or unanswered questions/i);
|
||||
} else {
|
||||
expect(body).toContain('skip history matching silently; still classify current findings');
|
||||
expect(body).toMatch(/still classify current findings/i);
|
||||
expect(body.indexOf('If no prior reviews exist')).toBeLessThan(body.indexOf('For each JSONL entry'));
|
||||
expect(body).toContain('If N > 0, print once:');
|
||||
expect(body).toContain('Otherwise skip the summary');
|
||||
expect(body).toContain('Only suppress `skipped` findings — never `fixed` or `auto-fixed`');
|
||||
expect(body).toMatch(/only suppress `skipped` findings — never `fixed` or `auto-fixed`/i);
|
||||
}
|
||||
expect(body).not.toContain('skip this step silently');
|
||||
}
|
||||
@@ -347,11 +313,11 @@ describe('QA caller authority in pure host renders', () => {
|
||||
|
||||
test(`${host.name}: orchestration consumes the existing generation allowance at both coverage gates`, () => {
|
||||
const body = RESOLVERS.TEST_COVERAGE_GATE_SHIP(context(host.name, 'ship'));
|
||||
expect(body).toContain("Use Step 7's remaining generation allowance");
|
||||
expect(body).toContain("Step 7's remaining generation allowance");
|
||||
expect(body.match(/If A and allowance remains:/g)).toHaveLength(2);
|
||||
expect(body).toContain('At the cap, offer only B/C or stop');
|
||||
expect(body).toContain('At the cap, offer only B or stop');
|
||||
expect(body).toContain('At the cap, omit A\'s generation pass and recommend stopping');
|
||||
expect(body).toMatch(/at the cap, offer only B\/C or stop/i);
|
||||
expect(body).toMatch(/at the cap, offer only B or stop/i);
|
||||
expect(body).toMatch(/at the cap, omit A's generation pass and recommend stopping/i);
|
||||
expect(body).toContain('Minimum = 60%, Target = 80%');
|
||||
expect(body).not.toContain('Maximum 2 passes total');
|
||||
});
|
||||
@@ -359,24 +325,23 @@ describe('QA caller authority in pure host renders', () => {
|
||||
|
||||
test('report-only recommendations cannot dispatch a repair skill during discovery', () => {
|
||||
const skill = readFileSync(join(root, 'qa-only/SKILL.md.tmpl'), 'utf8');
|
||||
expect(skill).toContain('Never invoke /qa or another skill from this report-only run');
|
||||
expect(skill).toContain('separate, user-authorized');
|
||||
expect(skill).toContain('No test framework detected');
|
||||
expect(skill).toContain('Never commit, stash or bootstrap');
|
||||
expect(skill).toMatch(/never invoke \/qa or another skill from this report-only run/i);
|
||||
expect(skill).toMatch(/separate, user-authorized/i);
|
||||
expect(skill).toMatch(/never commit, stash or bootstrap/i);
|
||||
});
|
||||
|
||||
test('standalone QA scopes its general test rule around the approved browser bootstrap', () => {
|
||||
const skill = readFileSync(join(root, 'qa/SKILL.md.tmpl'), 'utf8');
|
||||
const rule = skill.split('\n').find(line => line.startsWith('**Outside an explicitly approved browser bootstrap:**'));
|
||||
expect(skill).not.toMatch(/^13\. /m);
|
||||
expect(rule).toContain('Outside an explicitly approved browser bootstrap');
|
||||
expect(rule).toContain('Only create tests through authorized codification in Phase 8a.5');
|
||||
expect(rule).toContain('Never modify CI configuration or weaken existing tests');
|
||||
expect(rule).toMatch(/authorized codification in Phase 8a\.5/i);
|
||||
expect(rule).toMatch(/never modify CI configuration or weaken existing tests/i);
|
||||
const bootstrap = readFileSync(join(root, 'qa/sections/test-bootstrap.md.tmpl'), 'utf8');
|
||||
expect(bootstrap).toContain('Browser /qa only, never functional/report-only');
|
||||
expect(bootstrap).toContain('AskUserQuestion and WAIT');
|
||||
expect(bootstrap).toContain('install only the actual choice');
|
||||
expect(bootstrap).toContain('Never silently delete a valid red regression');
|
||||
expect(bootstrap).toMatch(/browser \/qa only, never functional\/report-only/i);
|
||||
const ask = bootstrap.search(/AskUserQuestion and wait/i);
|
||||
expect(ask).toBeGreaterThan(-1);
|
||||
expect(ask).toBeLessThan(bootstrap.search(/install only the actual choice/i));
|
||||
expect(bootstrap).toMatch(/never silently delete a valid red regression/i);
|
||||
expect(bootstrap).toContain('create/extend `.github/workflows/test.yml`');
|
||||
});
|
||||
|
||||
@@ -387,6 +352,7 @@ describe('QA caller authority in pure host renders', () => {
|
||||
expect(() => assertProbeLocalBlocker(pointer.replace('continue other safe probes', 'stop all checks'))).toThrow();
|
||||
const setup = render('qa/sections/browser-setup.md.tmpl', ctx);
|
||||
expect(() => assertSharedBrowserAuthority(setup.replace('Only after approval', 'Immediately'))).toThrow();
|
||||
expect(() => assertSharedBrowserAuthority(setup.replace('explicit approval', 'a status update'))).toThrow();
|
||||
expect(() => assertSharedBrowserAuthority(setup.replace('Never bootstrap or invoke another skill', 'Invoke another skill'))).toThrow();
|
||||
expect(() => assertSharedBrowserAuthority(setup + '\nIf `NEEDS_SETUP`: tell the user')).toThrow();
|
||||
expect(() => assertSharedBrowserAuthority(setup + '\nAn authenticated page needs /setup-browser-cookies')).toThrow();
|
||||
|
||||
@@ -649,13 +649,9 @@ describe('generated actual parent paths', () => {
|
||||
expect(positions).toEqual([...positions].sort((a, b) => a - b));
|
||||
const load = skillName === 'review' ? generateQAReviewPreflight(ctx) : parent.slice(positions[0], positions[1]);
|
||||
if (skillName === 'review') {
|
||||
const charter = parent.slice(positions[0], positions[1]).replace(/\s+/g, ' ');
|
||||
expect(charter).toContain("Reuse Step 4's surfaces and completed Reads");
|
||||
expect(charter).toContain('Finish missing methods before charters');
|
||||
expect(charter).toContain('complete the shared isolation/permission preflight before setup');
|
||||
const readiness = parent.slice(positions[1], positions[2]).replace(/\s+/g, ' ');
|
||||
expect(readiness).toContain('Read QA\'s `sections/browser-setup.md` and follow its report-only rules');
|
||||
expect(readiness).toContain('Never install, import cookies or bootstrap tests');
|
||||
expect(readiness).toMatch(/Read QA's `sections\/browser-setup\.md` and follow its report-only rules/i);
|
||||
expect(readiness).toMatch(/never install, import cookies or bootstrap tests/i);
|
||||
expect(load).not.toContain('sections/browser-setup.md');
|
||||
}
|
||||
expect(load).toContain('{{QA_RESOURCE:exploratory}}');
|
||||
@@ -664,50 +660,26 @@ describe('generated actual parent paths', () => {
|
||||
const resource = generateQAResource(ctx, ['exploratory']);
|
||||
expect(resource).toContain(`installed /${skillName} SKILL.md's directory`);
|
||||
expect(resource).toContain('`../qa/sections/exploratory.md`');
|
||||
const shared = generateQAExploratory({ ...ctx, skillName: 'qa' });
|
||||
const preparation = ['1. Read `sections/scope.md`', 'in full and select the surfaces',
|
||||
'Read `sections/system-functional.md` in full.', 'Read `sections/qa-patterns.md` in full.',
|
||||
'Write a **charter**', '1. First demonstrate success'].map(marker => shared.indexOf(marker));
|
||||
expect(preparation.every(position => position >= 0)).toBe(true);
|
||||
expect(preparation).toEqual([...preparation].sort((a, b) => a - b));
|
||||
expect(load).toContain('Templates cannot replace them');
|
||||
const flat = parent.replace(/\s+/g, ' ');
|
||||
expect(flat).toContain('Only the parent runs report-only discovery');
|
||||
expect(flat).toContain('Follow the shared Probe loop for smoke checks and replays until the smoke limit');
|
||||
expect(flat).toContain('Then run required plan checks and revalidation, even after smoke expires');
|
||||
expect(flat).toContain('using the same procedure but no smoke guard; never reset the clock');
|
||||
expect(flat).toContain("Use finite command timeouts, capped at the caller\'s remaining time if it has a deadline");
|
||||
expect(flat).toContain('When the caller\'s deadline expires, mark unfinished checks not-run');
|
||||
for (const contract of ['First demonstrate success: output AND durable effects',
|
||||
'Wait for successful checkpoint publication before dispatch',
|
||||
'Replay the exact failing command/request from the same initial fixture state']) {
|
||||
expect(shared).toContain(contract);
|
||||
}
|
||||
expect(flat).toContain('Read agent/user updates and await results without batching them with reporting/logging');
|
||||
expect(flat).toContain('Re-review changed or uncertain coverage and repeat step 3 for affected checks');
|
||||
expect(flat).toContain('Report clean/completed only when all required checks pass on current inputs');
|
||||
expect(flat).toContain('List failed, blocked, inconclusive and not-run checks');
|
||||
expect(flat).toMatch(/repeat step 3 for affected checks/i);
|
||||
}
|
||||
});
|
||||
|
||||
test('authored shared loop preserves complete safe observations and re-enters checkpoints after input changes', () => {
|
||||
const text = generateQAExploratory({ skillName: 'qa', tmplPath: 'qa/SKILL.md.tmpl', host: 'claude', paths: HOST_PATHS.claude }).replace(/\s+/g, ' ');
|
||||
for (const contract of ["last completed probe's full outer command", 'Preserve every safe program-JSON key/value', 'identity hash unchanged', 'Q supplies observed; never transcribe it', 'write the report, not a checkpoint', 'return to step 2 for each affected revalidation', 'Pass requires all required current-input contracts to pass with no required remainder']) {
|
||||
expect(text).toContain(contract);
|
||||
}
|
||||
expect(text).toContain("observationCommand: last completed probe's full outer command, including guard");
|
||||
expect(text).toContain('observed: its exact decoded child JSON (no wrapper/extra keys)');
|
||||
expect(text).toContain('or its full non-JSON text');
|
||||
expect(text).toMatch(/observationCommand: last completed probe's full outer command/i);
|
||||
expect(text).toMatch(/observed: its exact decoded child JSON/i);
|
||||
expect(text).toMatch(/identity hash unchanged/i);
|
||||
expect(text).not.toContain('nest unchanged child JSON');
|
||||
});
|
||||
test('the shared smoke has explicit limits without waiving required plan checks', () => {
|
||||
const body = fs.readFileSync(path.join(import.meta.dir, '../qa/sections/exploratory.md'), 'utf8').replace(/\s+/g, ' ');
|
||||
expect(body).toContain('Stop after 5 minutes or 12 probes, whichever comes first');
|
||||
expect(body).toMatch(/stop after 5 minutes or 12 probes/i);
|
||||
expect(body).toContain('G enforces the deadline');
|
||||
expect(body).toContain('Never reset D/bypass G');
|
||||
expect(body).toContain('Explicit plan checks and revalidation remain required beyond this smoke budget');
|
||||
expect(body).toMatch(/never reset D\/bypass G/i);
|
||||
expect(body).toMatch(/plan checks and revalidation remain required beyond this smoke budget/i);
|
||||
expect(body).toContain('leaves /review incomplete');
|
||||
expect(body).toContain('/ship blocked unless the user explicitly accepts that named risk');
|
||||
expect(body).toMatch(/\/ship blocked unless the user explicitly accepts that named risk/i);
|
||||
});
|
||||
|
||||
test('excerpt extraction fails loudly instead of producing an empty passing fixture', () => {
|
||||
@@ -1152,16 +1124,15 @@ describe('real caller-specific native fixture and capture boundary', () => {
|
||||
try {
|
||||
fixture.config = path.join(fixture.root, 'callback-config');
|
||||
await runQaCaller(fixture, 'free-receipt-interface', async options => {
|
||||
expect(options.prompt).toContain('status is the overall supplied phase gate, not whether some probes passed');
|
||||
expect(options.prompt).toContain('Pass requires no remaining required contracts or gates');
|
||||
expect(options.prompt).toContain('Optional unavailable providers and later stages outside this excerpt are not required remainder');
|
||||
expect(options.prompt).toContain('exact id values from the captured child JSON (probe-...)');
|
||||
expect(options.prompt).toContain("never the helper's three-digit capture IDs or QA_EVIDENCE.id");
|
||||
expect(options.prompt).toContain('Capture IDs select stored observations for checkpoint/materialize');
|
||||
expect(options.prompt).toMatch(/overall supplied phase gate/i);
|
||||
expect(options.prompt).toMatch(/no remaining required contracts or gates/i);
|
||||
expect(options.prompt).toMatch(/not required remainder/i);
|
||||
expect(options.prompt).toContain('(probe-...)');
|
||||
expect(options.prompt).toMatch(/never the helper's three-digit capture IDs/i);
|
||||
expect(options.prompt).not.toMatch(/bun scripts\/probe\.ts \d|exploration-[0-9]{3}|snapshot.*must|plan:nine|adverse/i);
|
||||
const readme = fs.readFileSync(path.join(fixture.cwd, 'README.md'), 'utf8');
|
||||
expect(readme).toContain('Every diagnostic receipt field is synthetic, nonsecret evidence');
|
||||
expect(readme).toContain('snapshot identifies the owned source and fixture inputs');
|
||||
expect(readme).toMatch(/synthetic, nonsecret/i);
|
||||
expect(readme).toMatch(/snapshot identifies the owned source/i);
|
||||
expect(readme).not.toMatch(/checkpoint|exploration-NNN|hypothesis|nextCommand/);
|
||||
return { exitReason: 'success', transcript: [] } as unknown as SkillTestResult;
|
||||
});
|
||||
@@ -1361,20 +1332,18 @@ describe('real caller-specific native fixture and capture boundary', () => {
|
||||
expect(options.timeout).toBe(300_000);
|
||||
expect(options.completionReserveMs).toBe(75_000);
|
||||
expect(options.appendSystemPrompt).toContain(`at most ${options.maxTurns} assistant turns`);
|
||||
expect(options.appendSystemPrompt).toContain('independent source Reads and read-only discovery together as separate native tool calls');
|
||||
expect(options.appendSystemPrompt).toContain('After required clock and approval prerequisites settle');
|
||||
expect(options.appendSystemPrompt).toContain('The completion reserve is for required verification, affected-input revalidation and artifacts, not an earlier deadline');
|
||||
expect(options.appendSystemPrompt).toContain("Use each native probe's snapshot to distinguish current from superseded evidence");
|
||||
expect(options.appendSystemPrompt).toContain('Never group diagnostic probes, checkpoint publication with its next probe');
|
||||
expect(options.appendSystemPrompt).toContain('the turn limit does not authorize skipping work or reporting incomplete work as passed');
|
||||
for (const rule of [/separate native tool calls/i, /approval prerequisites settle/i,
|
||||
/completion reserve is for required verification/i, /superseded evidence/i,
|
||||
/never group diagnostic probes/i, /turn limit does not authorize skipping work/i]) {
|
||||
expect(options.appendSystemPrompt).toMatch(rule);
|
||||
}
|
||||
expect(options.appendSystemPrompt).not.toMatch(/bun scripts\/probe\.ts \d|invalid input|highest.risk/i);
|
||||
expect(options.prompt).toContain('Keep normal parent decision gates.');
|
||||
expect(options.prompt).toContain("use the section clock's Hard deadline UTC, never its Runner entry UTC, reserve-start time or a clock-read time");
|
||||
expect(options.prompt).toContain('Before the completion report and each completed:true review record, read HANDOFF.md');
|
||||
expect(options.prompt).toContain('a later handoff read cannot validate an earlier completion');
|
||||
expect(options.prompt).toContain('If you defer an optional idea or stop exploration, do not publish a checkpoint for it');
|
||||
expect(options.prompt).toContain('An unused checkpoint requires an actual authenticated expired-capture result; nearing the deadline or choosing to stop is not enough');
|
||||
expect(options.prompt).toContain('Write the phase report to reports/review.md.');
|
||||
expect(options.prompt).toMatch(/keep normal parent decision gates/i);
|
||||
expect(options.prompt).toMatch(/Hard deadline UTC, never its Runner entry UTC/i);
|
||||
expect(options.prompt).toMatch(/completed:true review record, read HANDOFF\.md/i);
|
||||
expect(options.prompt).toMatch(/do not publish a checkpoint for it/i);
|
||||
expect(options.prompt).toMatch(/authenticated expired-capture result/i);
|
||||
expect(options.prompt).toContain('reports/review.md');
|
||||
return sentinel;
|
||||
});
|
||||
expect(calls).toBe(1);
|
||||
|
||||
@@ -9,6 +9,8 @@ import { qaFunctionalVerdict, qaNativeProbes } from './helpers/qa-functional-evi
|
||||
import { createQAFunctionalFixture, ownedPath } from './helpers/qa-functional-fixture';
|
||||
import { validateQACheckpoints } from './helpers/qa-checkpoint-evidence';
|
||||
import { computePaidCaseSelection } from '../scripts/test-paid-shards';
|
||||
import { generateQAExploratory } from '../scripts/resolvers/qa';
|
||||
import { HOST_PATHS } from '../scripts/resolvers/types';
|
||||
|
||||
test.each(['full', 'pr'] as const)('%s selection assigns the captured webhook regression to its native owner', profile => {
|
||||
for (const file of ['qa-webhook-r85-checkpoints.json', 'qa-functional-ci-36505065023.json']) {
|
||||
@@ -81,22 +83,15 @@ test('functional driver discloses its learning, CLI coverage and repair acceptan
|
||||
for (const entry of QA_FUNCTIONAL_CASES) {
|
||||
const prompt = qaFunctionalPrompt(entry);
|
||||
expect(prompt).toContain(`Read ${entry.mode}/SKILL.md, qa/sections/scope.md, ${entry.mode}/sections/exploratory.md and qa/sections/system-functional.md in full`);
|
||||
expect(prompt).toContain('All four reads are required before probing in this fixture, even when its surfaces and isolation are already established');
|
||||
expect(prompt).toContain('command is the exact full outer capture invocation, including that ID and all wrapper options, not just the native child command after --');
|
||||
expect(prompt).toContain('"command":"<exact full outer capture invocation>"');
|
||||
expect(prompt).not.toContain('<exact executed native probe command>');
|
||||
expect(prompt).toContain('different later command');
|
||||
expect(prompt).toContain('not the required same-command replay');
|
||||
expect(prompt).toContain('Select that checkpoint ID in annotations.learning');
|
||||
expect(prompt).toContain('the production helper copies its observationCommand, hypothesis and nextCommand');
|
||||
expect(prompt).toContain('different native child commands');
|
||||
expect(prompt).toContain('Only the helper writes observed fields');
|
||||
expect(prompt).toContain('English, more than 20 characters');
|
||||
if (entry.family === 'cli') expect(prompt).toContain('a successful apply; balance alone is not enough');
|
||||
expect(prompt).toContain('annotations.learning');
|
||||
expect(prompt).toContain('observationCommand, hypothesis and nextCommand');
|
||||
expect(prompt).toMatch(/more than 20 characters/);
|
||||
if (entry.mode === 'qa') {
|
||||
expect(prompt).toContain(`repair only src/${entry.family === 'cli' ? 'cli' : 'worker'}.ts`);
|
||||
expect(prompt).toContain('existing tests remain read-only');
|
||||
expect(prompt).toContain('Freeze all test files after red');
|
||||
expect(prompt).toMatch(/existing tests remain read-only/i);
|
||||
expect(prompt).toMatch(/freeze all test files after red/i);
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -164,12 +159,12 @@ test('the native launcher consumes the family-specific actor boundary', () => {
|
||||
expect(prompt).not.toContain('at Standard depth');
|
||||
expect(prompt).toContain('successful checkpoint publication before the next probe');
|
||||
expect(prompt).toContain('no shell composition, scripts or added path operands');
|
||||
expect(prompt).toContain('ONLY complete JSON actually emitted');
|
||||
expect(prompt).toMatch(/only complete JSON actually emitted/i);
|
||||
expect(prompt).toContain('never a combined command list');
|
||||
expect(prompt).toContain('Put tests, raw CLI diagnostics, launch failures and timeouts in Markdown');
|
||||
expect(prompt).toContain(entry.family === 'cli'
|
||||
? 'The generic wrapper does NOT support wait'
|
||||
: 'bun cancel.ts is a CLI-only entrypoint, not part of this fixture');
|
||||
expect(prompt).toMatch(entry.family === 'cli'
|
||||
? /the generic wrapper does not support wait/i
|
||||
: /bun cancel\.ts is a CLI-only entrypoint, not part of this fixture/i);
|
||||
expect(prompt).not.toContain('parseInt');
|
||||
if (entry.family === 'webhook') expect(prompt).toContain('Choose their order from observations after the happy path');
|
||||
if (entry.family === 'webhook' && entry.mode === 'qa-only') {
|
||||
@@ -198,51 +193,30 @@ test('declared examples respect the existing closed native grammar', () => {
|
||||
test('artifact completion preserves exact evidence before concise linked reporting', () => {
|
||||
for (const entry of QA_FUNCTIONAL_CASES) {
|
||||
const prompt = qaFunctionalPrompt(entry);
|
||||
expect(prompt).toContain('Materialize qa-reports/evidence.json first, then write a concise qa-reports/report.md');
|
||||
expect(prompt).toContain('using the functional report structure');
|
||||
expect(prompt).toContain('Link the evidence and checkpoint files rather than repeating full probe payloads in Markdown');
|
||||
expect(prompt).toContain('Both artifacts are required before completion');
|
||||
expect(prompt).toContain('The learning array is a summary: choose one completed checkpoint');
|
||||
expect(prompt).toContain('not another probe or a duplicate of the complete checkpoint ledger');
|
||||
expect(prompt).toContain('Both commands must name exact captured probes with different native child commands');
|
||||
expect(prompt).toContain('Preserve every checkpoint and link every checkpoint in Markdown');
|
||||
expect(prompt).toContain('keep every executed probe and its complete JSON in evidence');
|
||||
expect(prompt).toContain('Evidence rows contain ONLY complete JSON actually emitted by native probes, including failures and repeats');
|
||||
expect(prompt).toContain('retain pre-repair results alongside green results');
|
||||
expect(prompt).toContain('Never synthesize JSON');
|
||||
expect(prompt).toContain('one causal sentence per checkpoint hypothesis (English, more than 20 characters) and compact JSON formatting, preserving every field and value');
|
||||
expect(prompt).toContain('retain its headings and required fields');
|
||||
expect(prompt).toContain('link to evidence.json and checkpoints for details already recorded there');
|
||||
expect(prompt).toContain('After saving both artifacts, return only their paths and the actual completion status');
|
||||
expect(prompt).toContain('Never shorten native JSON or omit a required probe, check or field');
|
||||
expect(prompt).toContain('aim under 400 words');
|
||||
expect(prompt.indexOf('qa-reports/evidence.json')).toBeGreaterThan(-1);
|
||||
expect(prompt.indexOf('qa-reports/evidence.json')).toBeLessThan(prompt.indexOf('qa-reports/report.md'));
|
||||
expect(prompt).toMatch(/both artifacts are required before completion/i);
|
||||
expect(prompt).toMatch(/never synthesize JSON/i);
|
||||
expect(prompt).toMatch(/never shorten native JSON or omit a required probe, check or field/i);
|
||||
}
|
||||
const source = readFileSync(join(import.meta.dir, 'helpers/qa-functional-eval.ts'), 'utf8');
|
||||
expect(source).toContain('maxTurns: 40');
|
||||
expect(source).toContain('completionReserveMs: timeout / 4');
|
||||
expect(Number(source.match(/maxTurns: (\d+)/)?.[1])).toBeGreaterThanOrEqual(40);
|
||||
expect(source).toContain('completionReserveMs:');
|
||||
});
|
||||
|
||||
test('fix completion budgets for required repair and avoids duplicating preserved evidence', () => {
|
||||
for (const entry of QA_FUNCTIONAL_CASES) {
|
||||
const prompt = qaFunctionalPrompt(entry);
|
||||
if (entry.mode === 'qa') {
|
||||
expect(prompt).toContain('a reproduced in-tier defect requires the authorized native regression, repair and verification');
|
||||
expect(prompt).toContain('retain its headings and required fields');
|
||||
expect(prompt).toContain('link to evidence.json and checkpoints for details already recorded there');
|
||||
expect(prompt).toContain('Include the diagnosis, red/green test results and coverage limits');
|
||||
expect(prompt).toContain('After saving both artifacts, return only their paths and the actual completion status');
|
||||
expect(prompt).toContain('Never shorten native JSON or omit a required probe, check or field');
|
||||
const stages = ['1. Prove the regression red', '2. On the repaired source', '3. Save the evidence and Markdown artifacts'];
|
||||
const positions = stages.map(stage => prompt.indexOf(stage));
|
||||
expect(positions.every(position => position >= 0)).toBe(true);
|
||||
expect(positions).toEqual([...positions].sort((a, b) => a - b));
|
||||
expect(prompt).toContain('A green test suite does not substitute for these native probes');
|
||||
expect(prompt).toContain('not a signal to stop stage 2');
|
||||
expect(prompt).toContain('report incomplete; do not call it complete with a caveat');
|
||||
expect(prompt).toContain('one causal sentence per checkpoint hypothesis (English, more than 20 characters) and compact JSON formatting, preserving every field and value');
|
||||
expect(prompt).toMatch(/green test suite does not substitute for these native probes/i);
|
||||
expect(prompt).toMatch(/report incomplete; do not call it complete/i);
|
||||
} else {
|
||||
expect(prompt).not.toContain('This is a fix run');
|
||||
expect(prompt).toContain('Include the diagnosis, proposed test stubs and coverage limits');
|
||||
expect(prompt).toMatch(/proposed test stubs/i);
|
||||
expect(prompt).not.toContain('Include the diagnosis, red/green test results');
|
||||
}
|
||||
}
|
||||
@@ -265,22 +239,19 @@ test('webhook fix stage asks for the fix-loop probes; the R29 scenario omissions
|
||||
}
|
||||
const fix = qaFunctionalPrompt({ family: 'webhook', mode: 'qa' });
|
||||
const verification = fix.slice(fix.indexOf('2. On the repaired source'), fix.indexOf('3. Save the evidence'));
|
||||
expect(verification).toContain('run the original failing probe, an adjacent happy-path probe, cancellation and the unavailable-dependency probe');
|
||||
for (const probe of ['original failing probe', 'happy-path probe', 'cancellation', 'unavailable-dependency probe']) expect(verification).toContain(probe);
|
||||
expect(verification).not.toContain('All eight scenarios');
|
||||
expect(fix).toContain('the completion reserve does not end required coverage');
|
||||
expect(fix).toMatch(/completion reserve does not end required coverage/i);
|
||||
expect(verification).toBe(((prompt: string) => prompt.slice(prompt.indexOf('2. On the repaired source'), prompt.indexOf('3. Save the evidence')))(qaFunctionalPrompt({ family: 'cli', mode: 'qa' })));
|
||||
const report = qaFunctionalPrompt({ family: 'webhook', mode: 'qa-only' });
|
||||
expect(report).toContain('All eight scenarios are required coverage; a replay does not replace another scenario');
|
||||
expect(report).toMatch(/all eight scenarios are required coverage/i);
|
||||
for (const scenario of required) expect(report).toContain(scenario);
|
||||
});
|
||||
|
||||
|
||||
test('fix-stage checkpoint provenance survives intervening native regression tests', () => {
|
||||
const prompt = qaFunctionalPrompt({ family: 'webhook', mode: 'qa' });
|
||||
expect(prompt).toContain('most recent completed native probe');
|
||||
expect(prompt).toContain('Tests, source edits and clock reads do not replace that observation');
|
||||
expect(prompt).toContain('put red/green test output in the report, not in observed');
|
||||
expect(prompt).toContain('write no checkpoint when there is no next probe');
|
||||
expect(prompt).toMatch(/red\/green test output in the report, not in observed/i);
|
||||
});
|
||||
|
||||
test('R29 captured webhook bytes bind across a green test; test summaries and altered JSON do not', () => {
|
||||
@@ -376,32 +347,21 @@ test.each(JSON.parse(readFileSync(join(import.meta.dir, 'fixtures/qa-webhook-r85
|
||||
|
||||
test('report-only exploration requires a completed written checkpoint before the next probe', () => {
|
||||
const section = readFileSync(join(import.meta.dir, '../qa-only/sections/exploratory.md'), 'utf8');
|
||||
const positions = ['1. First demonstrate success', '2. **Decide whether another probe is needed.**', '**Publish before probing.**', '3. Run that exact probe; G enforces the deadline when bounded']
|
||||
.map(marker => section.indexOf(marker));
|
||||
expect(positions.every(position => position >= 0)).toBe(true);
|
||||
expect(positions).toEqual([...positions].sort((a, b) => a - b));
|
||||
expect(section).toContain('exploration-NNN.json');
|
||||
expect(section).toContain("Reuse resolved REPORT_DIR");
|
||||
expect(section).toContain('own a fresh');
|
||||
expect(section).toContain('owned probe directory');
|
||||
expect(section).toContain('Reuse resolved REPORT_DIR');
|
||||
expect(section).toMatch(/owned probe directory/i);
|
||||
for (const field of ['observationCommand', 'observed', 'hypothesis', 'nextCommand']) expect(section).toContain(`${field}:`);
|
||||
expect(section).toContain('Wait for successful checkpoint publication');
|
||||
expect(section).toContain('Never backfill or overwrite notes');
|
||||
expect(section).toContain('link each checkpoint');
|
||||
expect(section).toMatch(/wait for successful checkpoint publication/i);
|
||||
expect(section).toMatch(/never backfill or overwrite notes/i);
|
||||
expect(section).not.toContain('a separate assistant text message');
|
||||
});
|
||||
|
||||
test('surface evidence checks defer to one exploratory execution sequence', () => {
|
||||
const source = readFileSync(join(import.meta.dir, '../scripts/resolvers/qa.ts'), 'utf8');
|
||||
expect(source).toContain('Each probe is one native command/interaction plus checks, excluding bookkeeping');
|
||||
const positions = ['2. **Decide whether another probe is needed.**', '**Publish before probing.**', '3. Run that exact probe; G enforces the deadline when bounded']
|
||||
.map(marker => source.indexOf(marker));
|
||||
expect(positions.every(position => position >= 0)).toBe(true);
|
||||
expect(positions).toEqual([...positions].sort((a, b) => a - b));
|
||||
expect(source).toContain('Never batch probes');
|
||||
expect(source).toContain('Follow the shared exploratory loop\'s order and written checkpoints');
|
||||
expect(source).toContain('Replay the exact failing command/request from the same initial fixture state');
|
||||
expect(source).toContain('Another input or a regression test is not that replay');
|
||||
const shared = generateQAExploratory({ host: 'claude', skillName: 'qa', tmplPath: '', paths: HOST_PATHS.claude });
|
||||
expect(shared).toMatch(/each probe is one native command\/interaction/i);
|
||||
expect(shared).toMatch(/never batch probes/i);
|
||||
const functional = readFileSync(join(import.meta.dir, '../qa/sections/system-functional.md'), 'utf8');
|
||||
expect(functional).toMatch(/follow the shared exploratory loop/i);
|
||||
});
|
||||
|
||||
test('all public callers directly require the functional method before exploration', () => {
|
||||
@@ -413,13 +373,13 @@ test('all public callers directly require the functional method before explorati
|
||||
const explorer = readFileSync(join(import.meta.dir, '..', skill === 'qa-only' ? 'qa-only' : 'qa', 'sections/exploratory.md'), 'utf8');
|
||||
expect(explorer).toMatch(/Functional surfaces[^\n]*\n[^\n]*Read[^\n]*system-functional\.md/);
|
||||
expect(explorer).toContain('Browser surfaces only');
|
||||
const stages = ['Read `sections/scope.md`', 'in full and select the surfaces',
|
||||
'Read `sections/system-functional.md`', 'Write a **charter**', '1. First demonstrate success'];
|
||||
const stages = ['Read `sections/scope.md`', 'Read `sections/system-functional.md`',
|
||||
'## 1. Charter and preflight', '1. First demonstrate success'];
|
||||
const positions = stages.map(stage => explorer.indexOf(stage));
|
||||
expect(positions.every(position => position >= 0)).toBe(true);
|
||||
expect(positions).toEqual([...positions].sort((a, b) => a - b));
|
||||
const functional = readFileSync(join(import.meta.dir, '../qa/sections/system-functional.md'), 'utf8');
|
||||
expect(functional).toContain('## Contract map');
|
||||
expect(functional).toContain("Follow the shared exploratory loop's order and written checkpoints");
|
||||
expect(functional).toMatch(/follow the shared exploratory loop/i);
|
||||
}
|
||||
});
|
||||
+60
-80
@@ -7,14 +7,12 @@ import { generatePlanVerificationExec } from '../scripts/resolvers/plan-gates';
|
||||
import { HOST_PATHS } from '../scripts/resolvers/types';
|
||||
|
||||
function assertPreparation(text: string) {
|
||||
expect(text).toContain('Complete these Reads in order before writing charters or probing');
|
||||
expect(text).toMatch(/Await their results before the first probe, never in the same response\.|Await each successful Read result before continuing\./);
|
||||
expect(text).toContain('Do not repeat a Read already completed in this invocation');
|
||||
const stages = ['1. Read `sections/scope.md`', 'in full and select the surfaces',
|
||||
'2. Read the selected surface methods below in full', '**Functional surfaces:**',
|
||||
'Read `sections/system-functional.md` in full.', '**Browser surfaces only:**',
|
||||
'Read `sections/qa-patterns.md` in full.', '## 1. Charter and preflight',
|
||||
'Write a **charter**', 'Start once before baseline:', '1. First demonstrate success'];
|
||||
expect(text).toMatch(/reads in order before writing charters or probing/i);
|
||||
expect(text).toMatch(/Await their results before the first probe, never in the same response\.|Await each successful Read result before continuing\./i);
|
||||
expect(text).toMatch(/do not repeat a Read already completed/i);
|
||||
const stages = ['Read `sections/scope.md`', '**Functional surfaces:**', 'Read `sections/system-functional.md`',
|
||||
'**Browser surfaces only:**', 'Read `sections/qa-patterns.md`', '## 1. Charter and preflight',
|
||||
'bun G start D SECONDS', '1. First demonstrate success'];
|
||||
const positions = stages.map(stage => text.indexOf(stage));
|
||||
expect(positions.every(position => position >= 0)).toBe(true);
|
||||
expect(positions).toEqual([...positions].sort((a, b) => a - b));
|
||||
@@ -24,36 +22,35 @@ function assertPreparation(text: string) {
|
||||
}
|
||||
|
||||
function assertBoundsAndLayout(text: string) {
|
||||
for (const contract of [
|
||||
'Browser Quick: SECONDS=30', 'Browser Full/Regression: SECONDS=900',
|
||||
'Functional Full, Quick and Regression have no default total timer',
|
||||
"Set SECONDS to the shorter mode/caller limit",
|
||||
"an unlimited mode uses the caller\'s bound",
|
||||
'Without a total time limit, do not start D',
|
||||
'announce finite command timeouts',
|
||||
"EARLIER_UTC = caller\'s absolute deadline, if set",
|
||||
'Clocks/checkpoints use REPORT_DIR; mixed standalone runs use REPORT_DIR/browser and REPORT_DIR/functional, with one final report at REPORT_DIR. Caller paths win.',
|
||||
'in the probe directory, beside its deadline if bounded',
|
||||
]) expect(text).toContain(contract);
|
||||
expect(text.indexOf('Caller paths win.')).toBeLessThan(text.indexOf('Start once before baseline:'));
|
||||
for (const contract of ['Browser Quick: SECONDS=30', 'Browser Full/Regression: SECONDS=900']) expect(text).toContain(contract);
|
||||
for (const rule of [
|
||||
/functional full, quick and regression have no default total timer/i,
|
||||
/shorter mode\/caller limit/i,
|
||||
/without a total time limit, do not start D/i,
|
||||
/finite command timeouts/i,
|
||||
/mixed standalone runs use REPORT_DIR\/browser and REPORT_DIR\/functional/i,
|
||||
/one final report at REPORT_DIR/i,
|
||||
/caller paths win/i,
|
||||
]) expect(text).toMatch(rule);
|
||||
expect(text.search(/caller paths win/i)).toBeLessThan(text.indexOf('bun G start D SECONDS'));
|
||||
}
|
||||
|
||||
function assertPlanExecution(text: string, shared = generateQAExploratory({ host: 'claude', skillName: 'qa', tmplPath: '', paths: HOST_PATHS.claude })) {
|
||||
const step = text.slice(text.indexOf('**3. Run smoke and plan checks.**'), text.indexOf('**4. Check freshness before reporting.**')).replace(/\s+/g, ' ');
|
||||
for (const contract of [
|
||||
'Follow the shared Probe loop for smoke checks and replays until the smoke limit',
|
||||
'Then run required plan checks and revalidation, even after smoke expires',
|
||||
'using the same procedure but no smoke guard; never reset the clock',
|
||||
"Use finite command timeouts, capped at the caller\'s remaining time if it has a deadline",
|
||||
'When the caller\'s deadline expires, mark unfinished checks not-run',
|
||||
'Await clock/guard results before acting',
|
||||
]) expect(step).toContain(contract);
|
||||
for (const rule of [
|
||||
/shared Probe loop for smoke checks and replays until the smoke limit/i,
|
||||
/required plan checks and revalidation, even after smoke expires/i,
|
||||
/same procedure but no smoke guard; never reset the clock/i,
|
||||
/capped at the caller's remaining time/i,
|
||||
/deadline expires, mark unfinished checks not-run/i,
|
||||
/await clock\/guard results before acting/i,
|
||||
]) expect(step).toMatch(rule);
|
||||
expect(step.indexOf('Follow the shared Probe loop')).toBeLessThan(step.indexOf('Then run required plan checks'));
|
||||
expect(step.indexOf('Then run required plan checks')).toBeLessThan(step.indexOf('same procedure'));
|
||||
for (const contract of ['First demonstrate success: output AND durable effects',
|
||||
'Wait for successful checkpoint publication before dispatch',
|
||||
'Replay the exact failing command/request from the same initial fixture state']) {
|
||||
expect(shared).toContain(contract);
|
||||
for (const rule of [/demonstrate success: output and durable effects/i,
|
||||
/wait for successful checkpoint publication before dispatch/i,
|
||||
/replay the exact failing command\/request from the same initial fixture state/i]) {
|
||||
expect(shared).toMatch(rule);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,8 +64,7 @@ describe('QA probe entry and checkpoint gates', () => {
|
||||
expect(old).toContain('Read `sections/system-functional.md`');
|
||||
const current = generateQAExploratory({ host: 'claude', skillName: 'qa-only', tmplPath: '', paths: HOST_PATHS.claude });
|
||||
for (const clause of ['## 0. Preparation gate', 'Await each successful Read result before continuing',
|
||||
'description, section index or remembered method is not a completed instruction Read',
|
||||
'If either required Read is missing, complete it now before Charter and preflight']) {
|
||||
'complete it now before Charter and preflight']) {
|
||||
expect(old).not.toContain(clause);
|
||||
expect(current).toContain(clause);
|
||||
}
|
||||
@@ -88,13 +84,13 @@ describe('QA probe entry and checkpoint gates', () => {
|
||||
const ctx = { host: host.name, skillName, tmplPath: '', paths: HOST_PATHS[host.name] };
|
||||
const preflight = generateQAReviewPreflight(ctx).replace(/\s+/g, ' ');
|
||||
const body = generateQAReview(ctx).replace(/\s+/g, ' ');
|
||||
expect(preflight).toContain('Resolve QA\'s `sections/...` and `templates/...` paths from that installed QA SKILL.md directory, not the caller or product directory');
|
||||
expect(preflight).toMatch(/`sections\/\.\.\.` and `templates\/\.\.\.` paths from that installed QA SKILL\.md directory, not the caller or product directory/i);
|
||||
expect(body).toContain('Read QA\'s `sections/browser-setup.md`');
|
||||
expect(body).toContain('Read QA\'s `templates/functional-report-template.md`');
|
||||
if (skillName === 'review') {
|
||||
expect(body).toContain('Read QA\'s `templates/qa-report-template.md`');
|
||||
expect(body).toContain('Reuse Step 4\'s surfaces and completed Reads');
|
||||
expect(body).toContain('Finish missing methods before charters; do not repeat completed Reads');
|
||||
expect(body).toContain('Step 4\'s surfaces');
|
||||
expect(body).toMatch(/do not repeat completed Reads/i);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -110,19 +106,14 @@ describe('QA probe entry and checkpoint gates', () => {
|
||||
const positions = stages.map(stage => text.indexOf(stage));
|
||||
expect(positions.every(position => position >= 0)).toBe(true);
|
||||
expect(positions).toEqual([...positions].sort((a, b) => a - b));
|
||||
expect(text).toContain('No drafts/placeholders');
|
||||
expect(text).toContain('corrections cannot repair published notes');
|
||||
expect(text).toMatch(/corrections cannot repair published notes/i);
|
||||
if (skillName === 'qa-only') {
|
||||
expect(text).toContain('never invent a substitute path, identity or state');
|
||||
expect(text).toContain('For actual secrets/private payloads, withhold those values');
|
||||
expect(text).toContain('and replay limits in the report');
|
||||
expect(text).toContain('stop the affected probe chain');
|
||||
expect(text).toContain('An absolute state path is not itself a secret');
|
||||
expect(text).toMatch(/never invent a substitute path, identity or state/i);
|
||||
expect(text).toMatch(/secrets\/private payloads, withhold those values/i);
|
||||
expect(text).toMatch(/stop the affected probe chain/i);
|
||||
} else {
|
||||
expect(text).toContain('No drafts/placeholders or invented safe-path redactions');
|
||||
expect(text).toContain('Withhold unsafe values, disclose limits and stop that chain');
|
||||
expect(text).toMatch(/withhold unsafe values, disclose limits and stop that chain/i);
|
||||
}
|
||||
if (skillName === 'qa-only') expect(text).toContain('If capture is incomplete, report that limit instead of reconstructing it');
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -132,13 +123,7 @@ describe('QA probe entry and checkpoint gates', () => {
|
||||
const text = generateQAExploratory({ host: 'claude', skillName, tmplPath: '', paths: HOST_PATHS.claude });
|
||||
for (const contract of [
|
||||
'Reuse resolved REPORT_DIR',
|
||||
'charters as Markdown in the report',
|
||||
"Set SECONDS to the shorter mode/caller limit",
|
||||
'Without a total time limit, do not start D',
|
||||
'Browser Quick: SECONDS=30',
|
||||
'Browser Full/Regression: SECONDS=900',
|
||||
'exactly four top-level fields:', 'observationCommand:', 'observed:', 'hypothesis:', 'nextCommand:',
|
||||
'observed: its exact decoded child JSON (no wrapper/extra keys), or its full non-JSON text',
|
||||
'QA_DEADLINE receipts are not observations',
|
||||
]) expect(text).toContain(contract);
|
||||
assertBoundsAndLayout(text);
|
||||
@@ -159,17 +144,15 @@ describe('QA probe entry and checkpoint gates', () => {
|
||||
const write = text.indexOf('**Publish before probing.**');
|
||||
expect(decision).toBeGreaterThan(-1);
|
||||
expect(decision).toBeLessThan(write);
|
||||
expect(text.slice(decision, write)).toContain('write the report, not a checkpoint');
|
||||
expect(text.slice(decision, write)).toContain('If expired or no safe next probe remains');
|
||||
expect(text.slice(decision, write)).toMatch(/write the report, not a checkpoint/i);
|
||||
expect(text.slice(decision, write)).toMatch(/if expired/i);
|
||||
expect(text.slice(decision, write)).not.toContain('If done or blocked');
|
||||
if (skillName === 'qa-only') {
|
||||
expect(text).toContain('Check fields before publication');
|
||||
expect(text).toContain('retain the entire result unchanged, including owned fixture paths, IDs, hashes');
|
||||
expect(text).toMatch(/retain the entire result unchanged/i);
|
||||
} else {
|
||||
expect(text).toContain('Preserve every safe program-JSON key/value');
|
||||
expect(text).toContain('Preserve every safe program-JSON key/value and identity hash unchanged');
|
||||
expect(text).toMatch(/preserve every safe program-JSON key\/value and identity hash unchanged/i);
|
||||
}
|
||||
expect(text).toContain('Q supplies observed; never transcribe it');
|
||||
expect(text).toMatch(/Q supplies observed; never transcribe it/i);
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -182,20 +165,16 @@ describe('QA probe entry and checkpoint gates', () => {
|
||||
expect(text).toContain('/gstack-qa-deadline');
|
||||
expect(text).toContain('bun G start D SECONDS [EARLIER_UTC]');
|
||||
expect(text).toContain('Bounded browsers: `bun G run D -- COMMAND ARGS`');
|
||||
expect(text).toContain('Never reset D/bypass G');
|
||||
expect(text).toContain('invalid/missing D stops probes');
|
||||
expect(steps[0]).toContain('demonstrate success: output AND durable effects');
|
||||
expect(steps[0]).toContain('Guard if bounded; await completion');
|
||||
expect(steps[1]).toContain('If bounded, run `bun G status D`');
|
||||
expect(text).toMatch(/never reset D\/bypass G/i);
|
||||
expect(text).toMatch(/invalid\/missing D stops probes/i);
|
||||
expect(steps[0]).toMatch(/demonstrate success: output and durable effects/i);
|
||||
expect(steps[1]).toContain('`bun G status D`');
|
||||
expect(steps[1].indexOf('If expired')).toBeLessThan(steps[1].indexOf('**Publish before probing.**'));
|
||||
expect(steps[1]).toContain('STOP exploration; write the report, not a checkpoint');
|
||||
expect(steps[2]).toContain('Run that exact probe; G enforces the deadline when bounded');
|
||||
expect(steps[1]).toMatch(/\bstop exploration\b[^.]*write the report/i);
|
||||
expect(steps[2]).toContain('G enforces the deadline');
|
||||
expect(steps[2]).toContain('Report refusals as not-run');
|
||||
expect(steps[2]).toMatch(/refusals as not-run/i);
|
||||
expect(steps[3]).toContain('via steps 2–3');
|
||||
expect(steps[3]).toContain('then minimize via those gates');
|
||||
expect(steps[3]).toContain('Expiry leaves confirmation/minimization incomplete');
|
||||
expect(steps[4]).toContain('return to step 2 for each affected revalidation');
|
||||
expect(steps[4]).toContain('return to step 2');
|
||||
}
|
||||
});
|
||||
|
||||
@@ -204,25 +183,25 @@ describe('QA probe entry and checkpoint gates', () => {
|
||||
for (const skillName of ['review', 'ship']) {
|
||||
const ctx = { host: host.name, skillName, tmplPath: '', paths: HOST_PATHS[host.name] };
|
||||
const text = (skillName === 'review' ? generateQAReviewPreflight(ctx) : '') + generateQAReview(ctx);
|
||||
expect(text).toContain('> **STOP.** Before any probe, including plan checks, complete the ordered scope/method Reads below');
|
||||
const gate = text.search(/before any probe, including plan checks, complete the ordered scope\/method Reads/i);
|
||||
expect(gate).toBeGreaterThan(-1);
|
||||
expect(text).toContain('{{QA_RESOURCE:exploratory}}');
|
||||
expect(text).not.toContain('{{QA_RESOURCE:scope}}');
|
||||
expect(text).not.toContain('Read `sections/system-functional.md`');
|
||||
const shared = generateQAExploratory({ ...ctx, skillName: 'qa' });
|
||||
assertPreparation(shared);
|
||||
expect(text).toContain('Before any probe, including plan checks');
|
||||
assertPlanExecution(text);
|
||||
const required = text.indexOf(skillName === 'review'
|
||||
? '**2. Check readiness and list required checks.**' : '**2. List required checks');
|
||||
expect(required).toBeGreaterThan(-1);
|
||||
expect(text.indexOf('> **STOP.**')).toBeLessThan(required);
|
||||
expect(gate).toBeLessThan(required);
|
||||
if (skillName === 'review') {
|
||||
const isolation = text.indexOf('**1. Set the charter and isolation.**');
|
||||
const setup = text.indexOf("Read QA's `sections/browser-setup.md`");
|
||||
expect(isolation).toBeGreaterThan(text.indexOf('> **STOP.**'));
|
||||
expect(isolation).toBeGreaterThan(gate);
|
||||
expect(setup).toBeGreaterThan(required);
|
||||
expect(text.slice(isolation, required).replace(/\s+/g, ' ')).toContain('complete the shared isolation/permission preflight before setup');
|
||||
expect(text).toContain('Step 4 is read-only: defer charters, setup and probes to Step 4.7');
|
||||
expect(text.slice(isolation, required).replace(/\s+/g, ' ')).toMatch(/isolation\/permission preflight before setup/i);
|
||||
expect(text).toMatch(/Step 4 is read-only: defer charters, setup and probes to Step 4\.7/i);
|
||||
} else {
|
||||
const setup = text.indexOf("For browsers, Read QA's `sections/browser-setup.md`");
|
||||
expect(setup).toBeGreaterThan(required);
|
||||
@@ -239,8 +218,9 @@ describe('QA probe entry and checkpoint gates', () => {
|
||||
for (const changed of [
|
||||
method + '\n' + text.replace(method, ''),
|
||||
text.replace(method, method + '\n' + method),
|
||||
'Write a **charter**\n' + text,
|
||||
'## 1. Charter and preflight\n' + text,
|
||||
text.replace('Do not repeat a Read already completed in this invocation', 'Repeat all Reads'),
|
||||
text.replace('Read `sections/qa-patterns.md` in full.', '').replace('## 1. Charter and preflight', '## 1. Charter and preflight\nRead `sections/qa-patterns.md` in full.'),
|
||||
// ci-36641820398-1-gate-census-7 ship-exploratory-small-cli dispatched its first probe with the resource Reads.
|
||||
text.replace(' Await their results before the first probe, never in the same response.', ''),
|
||||
]) expect(() => assertPreparation(changed)).toThrow();
|
||||
@@ -303,8 +283,8 @@ describe('QA probe entry and checkpoint gates', () => {
|
||||
|
||||
test('plan execution waits for the actual shared method reads, not just collection', () => {
|
||||
const text = generatePlanVerificationExec({ host: 'claude', skillName: 'ship', tmplPath: '', paths: HOST_PATHS.claude });
|
||||
expect(text).toContain('Do not invoke an entire QA skill or start probes here');
|
||||
expect(text).toMatch(/do not invoke an entire QA skill or start probes here/i);
|
||||
expect(text).toContain('Before the first plan command, complete Step 9.2.1');
|
||||
expect(text).toContain('method Reads and the shared probe loop');
|
||||
expect(text).toMatch(/method Reads and the shared probe loop/i);
|
||||
});
|
||||
});
|
||||
@@ -4,7 +4,7 @@ import { join } from 'node:path';
|
||||
import { generateReviewArmy } from '../scripts/resolvers/review-army';
|
||||
import { generateCrossReviewDedup, generateSharedCodeReuse, generateScopeDrift } from '../scripts/resolvers/review-scope';
|
||||
import { generatePlanCompletionAuditReview, generatePlanCompletionAuditShip } from '../scripts/resolvers/plan-gates';
|
||||
import { generateQAExploratory, generateQAReview } from '../scripts/resolvers/qa';
|
||||
import { generateQAReview } from '../scripts/resolvers/qa';
|
||||
import { generateConfidenceCalibration } from '../scripts/resolvers/confidence';
|
||||
import { HOST_PATHS, type TemplateContext } from '../scripts/resolvers/types';
|
||||
|
||||
@@ -15,36 +15,27 @@ const adversarial = readFileSync(join(root, 'review/sections/adversarial.md.tmpl
|
||||
test('review audits deliverables before deferring behavioral plan checks to the QA preflight', () => {
|
||||
const ctx: TemplateContext = { skillName: 'review', tmplPath: '', host: 'claude', paths: HOST_PATHS.claude };
|
||||
const audit = generatePlanCompletionAuditReview(ctx).replace(/\s+/g, ' ');
|
||||
for (const contract of [
|
||||
'Separate static audit evidence from behavioral checks',
|
||||
'retain the exact command, expected outcome and source for Step 4.7',
|
||||
'They remain pending execution, never DONE from a diff',
|
||||
'A mixed item contributes to both lists',
|
||||
'Zero audited deliverables do not waive these checks',
|
||||
'Keep external-state and human-only checks under the existing audit rules',
|
||||
'If only behavioral checks remain, report zero audited deliverables and retain their pending Step 4.7 list',
|
||||
'For each audited deliverable, run the verification dispatch',
|
||||
'File-existence checks and verified read-only content validators are static audit checks, not behavioral probes',
|
||||
'Inspect the validator and its hooks before running it; verify read-only effects and access to the target',
|
||||
'leave the item UNVERIFIABLE and defer the command to Step 4.7',
|
||||
'Do not start applications, exercise APIs or mutate state during this audit',
|
||||
'If found and verified safe above, invoke it',
|
||||
]) expect(audit).toContain(contract);
|
||||
expect(audit.indexOf('Inspect the validator and its hooks')).toBeLessThan(audit.indexOf('If found and verified safe above, invoke it'));
|
||||
for (const rule of [
|
||||
/separate static audit evidence from behavioral checks/i,
|
||||
/never DONE from a diff/,
|
||||
/zero audited deliverables do not waive these checks/i,
|
||||
/for each audited deliverable/i,
|
||||
/leave the item UNVERIFIABLE and defer the command to Step 4\.7/,
|
||||
/do not start applications, exercise APIs or mutate state during this audit/i,
|
||||
]) expect(audit).toMatch(rule);
|
||||
const inspect = audit.search(/inspect the validator and its hooks before running it/i);
|
||||
expect(inspect).toBeGreaterThan(-1);
|
||||
expect(inspect).toBeLessThan(audit.search(/if found and verified safe above, invoke it/i));
|
||||
expect(audit).not.toContain('For each extracted plan item, run the verification dispatch');
|
||||
const qa = generateQAReview(ctx);
|
||||
expect(qa).toContain('Then run required plan checks and revalidation, even after smoke expires');
|
||||
expect(qa).toContain('Report clean/completed only when all required checks pass on current inputs');
|
||||
});
|
||||
|
||||
test('review prior-Skip matching includes adversarial and Greptile findings without relaxing eligibility', () => {
|
||||
const dedup = generateCrossReviewDedup({ skillName: 'review', tmplPath: '', host: 'claude', paths: HOST_PATHS.claude });
|
||||
expect(dedup).toContain('For every combined finding, including core, specialist, exploratory QA, adversarial and valid actionable Greptile findings, check:');
|
||||
expect(dedup).toContain('Suppress only when all conditions hold: the user skipped the same unchanged finding');
|
||||
expect(dedup).toContain('same advisory/defect kind');
|
||||
expect(dedup).toContain('Never use a skipped advisory to suppress a real defect');
|
||||
expect(dedup).toContain('Only suppress `skipped` findings — never `fixed` or `auto-fixed`');
|
||||
expect(skill).toContain('Run Step 5.0 severity/prior-skip dedup on all');
|
||||
expect(dedup).toMatch(/for every combined finding, including core, specialist, exploratory QA, adversarial and valid actionable Greptile findings/i);
|
||||
expect(dedup).toMatch(/suppress only when all conditions hold: the user skipped the same unchanged finding/i);
|
||||
expect(dedup).toMatch(/never use a skipped advisory to suppress a real defect/i);
|
||||
expect(dedup).toMatch(/only suppress `skipped` findings — never `fixed` or `auto-fixed`/i);
|
||||
expect(skill).toContain('Step 5.0 severity/prior-skip dedup');
|
||||
});
|
||||
|
||||
test('Review audit and prior-Skip clarifications do not route Ship through Review steps', () => {
|
||||
@@ -74,42 +65,40 @@ test('review collects every source before its single parent fix phase', () => {
|
||||
expect(positions.every(position => position >= 0)).toBe(true);
|
||||
expect(positions).toEqual([...positions].sort((a, b) => a - b));
|
||||
expect(skill.match(/## Step 5: Fix-First Review/g)).toHaveLength(1);
|
||||
expect(skill.replace(/\s+/g, ' ')).toContain('Do not edit reviewed source until Step 5');
|
||||
expect(skill.replace(/\s+/g, ' ')).toContain('every dispatched reader has returned or is confirmed stopped');
|
||||
expect(skill.replace(/\s+/g, ' ')).toMatch(/do not edit reviewed source until Step 5/i);
|
||||
expect(skill.replace(/\s+/g, ' ')).toMatch(/every dispatched reader has returned or is confirmed stopped/i);
|
||||
});
|
||||
|
||||
test('review settles adversarial attempts before fixing and has one full-pass back edge', () => {
|
||||
const generated = readFileSync(join(root, 'review/sections/adversarial.md'), 'utf8');
|
||||
const decision = skill.slice(skill.indexOf('## Step 5.8: Persist Eng Review result')).replace(/\s+/g, ' ');
|
||||
expect(generated).toContain('## Step 4.8: Adversarial review');
|
||||
expect(generated).toContain("queued for the parent's Fix-First handling at Step 5; do not edit during Step 4.8");
|
||||
expect(generated.replace(/\s+/g, ' ')).toContain('before the parent applies queued fixes');
|
||||
expect(generated).toContain('Return all findings and structured-review decisions to Step 5');
|
||||
expect(decision).toContain('A pass covers Steps 3–5, including all reviewers before fixes');
|
||||
expect(generated).toMatch(/do not edit during Step 4\.8/i);
|
||||
expect(generated).toMatch(/return all findings and structured-review decisions to Step 5/i);
|
||||
expect(decision).toMatch(/a pass covers Steps 3–5/i);
|
||||
expect(decision).toContain('Below 3, repeat Steps 3–5 with a new REVIEW_START');
|
||||
expect(decision).not.toContain('Route Step');
|
||||
expect(decision).not.toContain('Steps 5.0–5d');
|
||||
expect(decision).toContain('without a clean summary or a fourth pass');
|
||||
expect(decision).toMatch(/without a clean summary or a fourth pass/i);
|
||||
});
|
||||
|
||||
test('review small-diff and failed-reader paths retain QA and the required adversarial pass', () => {
|
||||
const army = generateReviewArmy({ skillName: 'review', tmplPath: 'review/SKILL.md.tmpl',
|
||||
host: 'claude', paths: HOST_PATHS.claude });
|
||||
expect(army).toContain("Continue to Step 4.6 with the core findings and an empty specialist list, then the parent's Exploratory QA step and Step 4.8 (adversarial review), then Step 5");
|
||||
expect(army.replace(/\s+/g, ' ')).toMatch(/Continue to Step 4\.6 [^.]*Exploratory QA step and Step 4\.8 \(adversarial review\), then Step 5/i);
|
||||
expect(army).not.toContain('design-lite');
|
||||
expect(army).toContain('Missing dispatched coverage remains incomplete, never completed or clean');
|
||||
expect(army).toContain('Continue independent Step 4.7 QA and Step 4.8 adversarial review');
|
||||
expect(army.replace(/\s+/g, ' ')).toMatch(/missing dispatched coverage remains incomplete, never completed or clean/i);
|
||||
expect(army.replace(/\s+/g, ' ')).toMatch(/independent Step 4\.7 QA and Step 4\.8 adversarial review/i);
|
||||
expect(army).not.toContain("Exploratory QA step, then continue to Step 5.");
|
||||
expect(army).not.toContain('If the Red Team subagent fails or times out, skip silently');
|
||||
});
|
||||
|
||||
test('review defines QA confidence, severity, impact selection and numeric version comparison', () => {
|
||||
const flat = skill.replace(/\s+/g, ' ');
|
||||
expect(flat).toContain('For QA findings, assign confidence (1–10) from replay/code evidence');
|
||||
expect(flat).toContain("retain Step 4.7's severity, not a severity inferred from confidence");
|
||||
expect(flat).toContain('A probe is affected when its entrypoint, dependencies, contract or replay inputs change');
|
||||
expect(flat).toContain('If impact is uncertain, rerun it');
|
||||
expect(flat).toContain('Compare dotted version components as integers from left to right');
|
||||
expect(flat).toContain('confidence (1–10)');
|
||||
expect(flat).toMatch(/retain Step 4\.7's severity/i);
|
||||
expect(flat).toMatch(/if impact is uncertain, rerun it/i);
|
||||
expect(flat).toMatch(/compare dotted version components as integers/i);
|
||||
});
|
||||
|
||||
test('review emits scope check after the plan audit and before the checklist', () => {
|
||||
@@ -121,36 +110,30 @@ test('review emits scope check after the plan audit and before the checklist', (
|
||||
expect(positions.every(position => position >= 0)).toBe(true);
|
||||
expect(positions).toEqual([...positions].sort((a, b) => a - b));
|
||||
const audit = readFileSync(join(root, 'review/sections/plan-completion.md'), 'utf8').replace(/\s+/g, ' ');
|
||||
expect(audit).toContain('When continuing after the audit (no HIGH-impact gate, or option B/C), emit the single final Scope Check');
|
||||
expect(audit).toContain("Step 1.5's provisional notes and this plan context");
|
||||
expect(audit).toContain('Emit Step 1.5\'s Scope Check once without plan fields');
|
||||
expect(audit).toMatch(/emit the single final Scope Check/i);
|
||||
expect(skill).not.toContain('Finish Step 1.5 here');
|
||||
});
|
||||
|
||||
test('review composes confidence-tagged findings into one final report with explicit incomplete coverage', () => {
|
||||
const flat = skill.replace(/\s+/g, ' ');
|
||||
expect(flat).toContain('Use CRITICAL/INFORMATIONAL labels in the finding format');
|
||||
expect(flat).toContain("Step 5.8 combines these finding lines with the checklist's action groups");
|
||||
expect(flat).toContain('CRITICAL/INFORMATIONAL');
|
||||
const report = flat.slice(flat.indexOf('### Report the final review'), flat.indexOf('{{LEARNINGS_LOG}}'));
|
||||
expect(report).toContain('Emit one final report, merging all reviewers rather than concatenating their reports');
|
||||
expect(report).toContain('counts final unresolved non-advisory defects');
|
||||
expect(report).toContain('State INCOMPLETE if `COMPLETED` is false, even when N=0');
|
||||
expect(report).toContain("Use the checklist's action groups with confidence-tagged finding lines");
|
||||
expect(report).toContain('Keep fixed, skipped and advisory items separate from unresolved defects; retain their dispositions');
|
||||
expect(report).toContain("Append Step 4.7's single `## Exploratory QA and Verification Results` section");
|
||||
expect(report).toContain('Neither coverage gaps nor advice are defects');
|
||||
expect(report).toMatch(/one final report/i);
|
||||
expect(report).toMatch(/unresolved non-advisory defects/i);
|
||||
expect(report).toMatch(/[Ss]tate INCOMPLETE if `COMPLETED` is false/);
|
||||
expect(report).toContain('`## Exploratory QA and Verification Results`');
|
||||
expect(report).toMatch(/neither coverage gaps nor advice are defects/i);
|
||||
});
|
||||
|
||||
test('small-diff persistence uses an empty specialist map without manufacturing skipped coverage', () => {
|
||||
const army = generateReviewArmy({ skillName: 'review', tmplPath: 'review/SKILL.md.tmpl',
|
||||
host: 'claude', paths: HOST_PATHS.claude });
|
||||
expect(army).toContain('For DIFF_LINES < 50, keep `specialists: {}`; do not manufacture per-specialist scope records');
|
||||
expect(army).toContain('Otherwise record each considered specialist');
|
||||
expect(skill).toContain("Use Step 4.6's `specialists` object unchanged, including its empty small-diff map");
|
||||
expect(army).toContain('DIFF_LINES < 50, keep `specialists: {}`');
|
||||
expect(skill.replace(/\s+/g, ' ')).toMatch(/Step 4\.6's `specialists` object unchanged/i);
|
||||
const ship = readFileSync(join(root, 'ship/sections/review-army.md.tmpl'), 'utf8');
|
||||
expect(ship).toContain('`specialists`: `{}` for a small-diff skip');
|
||||
expect(skill).toContain('every required Step 4.7 probe passes');
|
||||
expect(ship).toContain('all required probes pass');
|
||||
expect(skill.replace(/\s+/g, ' ')).toMatch(/every required Step 4\.7 probe passes/i);
|
||||
expect(ship.replace(/\s+/g, ' ')).toMatch(/all required probes pass/i);
|
||||
});
|
||||
|
||||
test('caller QA runs charter and setup after resource loading and has a severity for unmatched functional failures', () => {
|
||||
@@ -165,13 +148,13 @@ test('caller QA runs charter and setup after resource loading and has a severity
|
||||
const readiness = body.indexOf('**2. Check readiness and list required checks.**');
|
||||
expect(readiness).toBeGreaterThan(preparation);
|
||||
expect(probes).toBeGreaterThan(readiness);
|
||||
expect(body.slice(preparation, readiness).replace(/\s+/g, ' ')).toContain('complete the shared isolation/permission preflight before setup');
|
||||
expect(body.slice(preparation, readiness).replace(/\s+/g, ' ')).toMatch(/isolation\/permission preflight before setup/i);
|
||||
} else expect(preparation).toBeGreaterThan(body.indexOf('**2. List required checks.**'));
|
||||
expect(probes).toBeGreaterThan(preparation);
|
||||
expect(body.replace(/\s+/g, ' ')).toContain('unmatched functional failures are `functional-contract`, `CRITICAL`');
|
||||
expect(body).toContain('Setup/permission blockers are not defects');
|
||||
expect(body).toContain('Test creation needs user approval');
|
||||
expect(body).toContain('Return verified defects to Fix-First');
|
||||
expect(body.replace(/\s+/g, ' ')).toContain('`functional-contract`, `CRITICAL`');
|
||||
expect(body).toMatch(/setup\/permission blockers are not defects/i);
|
||||
expect(body).toMatch(/test creation needs user approval/i);
|
||||
expect(body).toMatch(/return verified defects to Fix-First/i);
|
||||
expect(body).not.toContain('for parent approval');
|
||||
}
|
||||
});
|
||||
@@ -185,34 +168,24 @@ test('review prepares context and deduplicates before classifying findings', ()
|
||||
].map(marker => skill.indexOf(marker));
|
||||
expect(positions.every(position => position >= 0)).toBe(true);
|
||||
expect(positions).toEqual([...positions].sort((a, b) => a - b));
|
||||
expect(skill).toContain('findings before Step 5a classification');
|
||||
expect(skill).toMatch(/before Step 5a classification/i);
|
||||
});
|
||||
|
||||
test('review owns the complete persistence contract after the adversarial read', () => {
|
||||
const step = skill.slice(skill.indexOf('## Step 5.8: Persist Eng Review result'));
|
||||
expect(skill.indexOf('{{SECTION:adversarial}}')).toBeLessThan(skill.indexOf('## Step 5.8: Persist Eng Review result'));
|
||||
expect(adversarial).not.toContain('### Before persisting Eng Review (Step 5.8)');
|
||||
for (const contract of [
|
||||
'repeat Steps 3–5', 'at most 3 fix cycles', 'final zero-edit pass, reconcile',
|
||||
'by structural identity', 'advisory/defect kind',
|
||||
'original `evidence_paths`/`helper_target`', 'without requiring deleted pre-extraction blocks',
|
||||
'Current findings determine recurring defects and unresolved counts', 'earlier fixes do not suppress them',
|
||||
'snapshot_covered_paths',
|
||||
'raw bytes equal the bound snapshot blobs', 'prior-cycle, supplied or prior-record coverage',
|
||||
'REVIEW_START', 'COMPLETED', 'CONVERGED', 'CYCLES', 'Step 4.7',
|
||||
'native Step 4.8 adversarial pass', 'means false, as does a failed native review',
|
||||
'optional outside', 'their own incomplete records when unavailable', 'named-risk',
|
||||
'zero counts', '`completed:false`', '`specialists`', '`findings`',
|
||||
'verified exploratory QA findings',
|
||||
'approved **and completed**', 'explicit Skip', 'sharedLibsFingerprint',
|
||||
'`review_binding`', 'validated captured branch',
|
||||
]) expect(step.toLowerCase().replace(/\s+/g, ' ')).toContain(contract.toLowerCase());
|
||||
for (const token of [
|
||||
'repeat Steps 3–5', 'at most 3 fix cycles', 'original `evidence_paths`/`helper_target`',
|
||||
'snapshot_covered_paths', 'REVIEW_START', 'COMPLETED', 'CONVERGED', 'CYCLES', 'Step 4.7',
|
||||
'native Step 4.8 adversarial pass', 'named-risk', 'zero counts', '`completed:false`',
|
||||
'`specialists`', '`findings`', 'explicit Skip', 'sharedLibsFingerprint', '`review_binding`',
|
||||
]) expect(step.toLowerCase().replace(/\s+/g, ' ')).toContain(token.toLowerCase());
|
||||
expect(step.indexOf('### 1. Re-review after edits'))
|
||||
.toBeLessThan(step.indexOf('### 2. Fill the record'));
|
||||
expect(step.indexOf('### 2. Fill the record'))
|
||||
.toBeLessThan(step.indexOf('~/.claude/skills/gstack/bin/gstack-review-log'));
|
||||
expect(step).toContain('`quality_score` is Step 4.6\'s specialist score');
|
||||
expect(step).toContain('unresolved non-advisory core defects still count');
|
||||
expect(step).toContain('`quality_score`');
|
||||
expect(step.indexOf('Pre-Landing Review: N issues (X critical, Y informational)'))
|
||||
.toBeGreaterThan(step.indexOf('~/.claude/skills/gstack/bin/gstack-review-log'));
|
||||
expect(step).toContain('`## Exploratory QA and Verification Results`');
|
||||
@@ -220,26 +193,20 @@ test('review owns the complete persistence contract after the adversarial read',
|
||||
|
||||
test('review distinguishes required native coverage from optional outside coverage', () => {
|
||||
const section = readFileSync(join(root, 'review/sections/adversarial.md'), 'utf8');
|
||||
expect(section).toContain('Only this optional outside adversarial pass is non-blocking');
|
||||
expect(section).toMatch(/only this optional outside adversarial pass is non-blocking/i);
|
||||
expect(section).not.toContain('All errors are non-blocking');
|
||||
expect(section).toContain('The native pass is required for Step 5.8 completion');
|
||||
expect(skill).toContain('Core findings use the confidence gates below');
|
||||
expect(skill).toContain('Step 4.6 applies its specialist gates');
|
||||
expect(section).toMatch(/native pass is required for Step 5\.8 completion/i);
|
||||
});
|
||||
|
||||
test('review identifies probe selection, report assets and the detected diff base', () => {
|
||||
const generated = generateQAReview({ skillName: 'review', tmplPath: 'review/SKILL.md.tmpl',
|
||||
host: 'claude', paths: HOST_PATHS.claude });
|
||||
const checklist = readFileSync(join(root, 'review/checklist.md'), 'utf8');
|
||||
expect(generated).toContain('Smoke: 5 minutes/12 probes, one success and the riskiest changed failure/edge');
|
||||
expect(generated).toContain('Required even for small diffs or missing plans/servers');
|
||||
expect(generated.replace(/\s+/g, ' ')).toContain("Use checklist severity; unmatched functional failures are `functional-contract`, `CRITICAL`");
|
||||
expect(generated).toContain('Setup/permission blockers are not defects');
|
||||
expect(generated).toContain('Test creation needs user approval');
|
||||
expect(generated).toContain("Read QA\'s `templates/functional-report-template.md`. Title it");
|
||||
expect(generated).toContain('Link every checkpoint');
|
||||
expect(generated).toContain('No second report');
|
||||
expect(checklist).toContain('merge-base diff from the caller');
|
||||
expect(generated).toContain('5 minutes/12 probes');
|
||||
expect(generated).toContain("Read QA\'s `templates/functional-report-template.md`");
|
||||
expect(generated).toMatch(/link every checkpoint/i);
|
||||
expect(generated).toMatch(/no second report/i);
|
||||
expect(checklist).toMatch(/merge-base diff from the caller/i);
|
||||
expect(checklist).not.toContain('git diff origin/main');
|
||||
});
|
||||
|
||||
@@ -247,37 +214,16 @@ test('caller QA defines execution, evidence ownership and report adaptation befo
|
||||
for (const skillName of ['review', 'ship']) {
|
||||
const generated = generateQAReview({ skillName, tmplPath: `${skillName}/SKILL.md.tmpl`,
|
||||
host: 'claude', paths: HOST_PATHS.claude }).replace(/\s+/g, ' ');
|
||||
for (const contract of [
|
||||
'Only the parent runs report-only discovery',
|
||||
'Never overwrite another run',
|
||||
'Follow the shared Probe loop for smoke checks and replays until the smoke limit',
|
||||
'using the same procedure but no smoke guard; never reset the clock',
|
||||
'Read agent/user updates and await results without batching them with reporting/logging',
|
||||
'Compare each probe\'s recorded source, tests, contracts, commands and fixtures (or input fingerprint) with current inputs, even without updates',
|
||||
'Re-review changed or uncertain coverage',
|
||||
"Use checklist severity",
|
||||
]) expect(generated).toContain(contract);
|
||||
const shared = generateQAExploratory({ skillName: 'qa', tmplPath: '', host: 'claude', paths: HOST_PATHS.claude });
|
||||
for (const contract of ['First demonstrate success: output AND durable effects',
|
||||
'Wait for successful checkpoint publication before dispatch',
|
||||
'Replay the exact failing command/request from the same initial fixture state']) {
|
||||
expect(shared).toContain(contract);
|
||||
}
|
||||
expect(generated).toMatch(/never overwrite another run/i);
|
||||
expect(generated).toMatch(/with current inputs, even without updates/i);
|
||||
expect(generated).toContain('Use checklist severity');
|
||||
if (skillName === 'review') {
|
||||
expect(generated).toContain('Title it `## Exploratory QA and Verification Results`');
|
||||
expect(generated).toContain('keep metadata/outcome tables');
|
||||
expect(generated).toContain('demote other headings one level');
|
||||
expect(generated).toContain('include it here under `### Browser results`');
|
||||
expect(generated).toContain('other headings demoted two levels');
|
||||
expect(generated).toContain('Keep browser/functional scores and outcomes separate');
|
||||
expect(generated).toContain('save browser baseline/evidence normally');
|
||||
expect(generated).toContain('Prepare one provisional QA section');
|
||||
expect(generated).toContain('Update affected outcomes/checkpoint links through repairs/revalidation');
|
||||
expect(generated).toContain('Continue to Step 4.8 even if blocked');
|
||||
expect(generated).toContain('Step 5.8 appends this section once after final findings and decides completion');
|
||||
expect(generated).toContain('`## Exploratory QA and Verification Results`');
|
||||
expect(generated).toContain('`### Browser results`');
|
||||
expect(generated).toMatch(/continue to Step 4\.8 even if blocked/i);
|
||||
expect(generated).toMatch(/Step 5\.8 appends this section once/i);
|
||||
} else {
|
||||
expect(generated).toContain('PR section `## Exploratory QA`');
|
||||
expect(generated).toContain('fields as subsections');
|
||||
expect(generated).toContain('`## Exploratory QA`');
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -291,15 +237,14 @@ test('review section index follows the actual pre-fix execution order', () => {
|
||||
|
||||
test('review finalization ownership: initialize invocation state and capture the core token before reading', () => {
|
||||
const start = skill.slice(skill.indexOf('## Step 3: Get the diff'), skill.indexOf('## Step 3.4:'));
|
||||
expect(start).toContain('one invocation action list and CYCLES=0');
|
||||
expect(start).toContain('Keep both through re-reviews');
|
||||
expect(start).toContain('CYCLES=0');
|
||||
expect(skill.match(/CYCLES=0/g)).toHaveLength(1);
|
||||
expect(start).toContain('gstack-review-log --start review\ngit diff "$DIFF_BASE"');
|
||||
expect(start).toContain('Save the printed REVIEW_START for this core candidate before reading its diff');
|
||||
expect(start).toContain('Each re-review captures a new token before reading, never at log time');
|
||||
expect(start.replace(/\s+/g, ' ')).toContain('Earlier core tokens remain unused');
|
||||
expect(start).toContain('Native/outside reviewer attempts own separate PASS_START tokens, not REVIEW_START');
|
||||
expect(start).toContain('Step 5.8 finishes only the final core token');
|
||||
const flat = start.replace(/\s+/g, ' ');
|
||||
expect(flat).toMatch(/REVIEW_START for this core candidate before reading its diff/i);
|
||||
expect(flat).toMatch(/new token before reading, never at log time/i);
|
||||
expect(flat).toMatch(/separate PASS_START tokens, not REVIEW_START/);
|
||||
expect(flat).toMatch(/finishes only the final core token/i);
|
||||
});
|
||||
|
||||
test('review finalization ownership: late findings use Fix-First before the bounded parent transition', () => {
|
||||
@@ -317,46 +262,31 @@ test('review finalization ownership: late findings use Fix-First before the boun
|
||||
const positions = markers.map(marker => flat.indexOf(marker));
|
||||
expect(positions.every(position => position >= 0)).toBe(true);
|
||||
expect(positions).toEqual([...positions].sort((a, b) => a - b));
|
||||
expect(flat).toContain('Structured approval does not waive advisory/test_stub ASK gates');
|
||||
expect(flat).toContain('A pass covers Steps 3–5, including all reviewers before fixes');
|
||||
expect(flat).toContain('at most 3 fix cycles');
|
||||
expect(flat).toContain('Below 3, repeat Steps 3–5 with a new REVIEW_START');
|
||||
expect(flat).toContain('At 3, persist `converged:false` and remaining findings');
|
||||
expect(flat).toContain('At 3, persist `converged:false`');
|
||||
const limit = flat.slice(flat.indexOf('At 3,'), flat.indexOf('No edits:'));
|
||||
expect(limit).toContain('by filling and saving the record below');
|
||||
expect(limit).toContain('Report nonconvergence and coverage gaps, then STOP this invocation');
|
||||
expect(limit).toContain('without a clean summary or a fourth pass');
|
||||
expect(limit).toMatch(/then stop this invocation/i);
|
||||
expect(limit).toMatch(/without a clean summary or a fourth pass/i);
|
||||
});
|
||||
|
||||
test('review finalization ownership: affected QA reuse cannot replace a full review or erase decisions', () => {
|
||||
const step = skill.slice(skill.indexOf('## Step 5.8: Persist Eng Review result'));
|
||||
const flat = step.replace(/\s+/g, ' ');
|
||||
expect(flat).toContain('On a repeat, execute Steps 3–5 in order');
|
||||
expect(flat).toContain('rerun affected probes after source, test, contract, command or fixture changes');
|
||||
expect(flat).toContain("At Step 4.7, reuse only this invocation's unchanged-input QA evidence");
|
||||
expect(flat).toContain('Reusing a probe never skips a review step');
|
||||
expect(flat).toContain("final zero-edit pass, reconcile this invocation's actions with current findings");
|
||||
expect(flat).toContain('original `evidence_paths`/`helper_target`');
|
||||
expect(flat).toContain('Current findings determine recurring defects and unresolved counts; earlier fixes do not suppress them');
|
||||
expect(flat).toContain('Re-read its final-snapshot supporting source and reconfirm the decision');
|
||||
expect(flat).toContain('otherwise report its history without a reusable skip');
|
||||
expect(flat).toContain('The logger computes `snapshot_covered_paths` from eligible paths whose raw bytes equal the bound snapshot blobs');
|
||||
expect(flat).toContain('Never carry prior-cycle, supplied or prior-record coverage forward or build this proof yourself');
|
||||
expect(flat).toContain('Fixed advice needs no skip coverage');
|
||||
expect(flat).toContain('include this invocation\'s revalidated decisions');
|
||||
expect(flat).toContain('execute Steps 3–5 in order');
|
||||
expect(flat).toMatch(/reuse only this invocation's unchanged-input QA evidence/i);
|
||||
expect(flat).toMatch(/reusing a probe never skips a review step/i);
|
||||
expect(flat).toMatch(/earlier fixes do not suppress them/i);
|
||||
expect(flat).toContain('logger computes `snapshot_covered_paths`');
|
||||
expect(flat).toMatch(/never carry prior-cycle, supplied or prior-record coverage forward or build this proof yourself/i);
|
||||
});
|
||||
|
||||
test('review finalization ownership: required native completion and optional outside records stay separate', () => {
|
||||
const step = skill.slice(skill.indexOf('### 2. Fill the record'));
|
||||
const flat = step.replace(/\s+/g, ' ');
|
||||
expect(flat).toContain('native Step 4.8 adversarial pass finish');
|
||||
expect(flat).toContain('every required Step 4.7 probe passes');
|
||||
expect(flat).toContain('Any failed, blocked, inconclusive or not-run required probe means false, as does a failed native review');
|
||||
expect(flat).toContain('`/ship` named-risk acceptance cannot complete `/review`');
|
||||
expect(flat).toContain('The required in-host adversarial result controls native completion');
|
||||
expect(flat).toContain('Optional outside attempts keep their own incomplete records when unavailable');
|
||||
expect(flat).toContain('cannot substitute for the native result, or vice versa');
|
||||
expect(flat).toContain('structured-review gate still applies');
|
||||
expect(flat).toMatch(/any failed, blocked, inconclusive or not-run required probe means false, as does a failed native review/i);
|
||||
expect(flat).toMatch(/`\/ship` named-risk acceptance cannot complete `\/review`/i);
|
||||
expect(flat).toMatch(/cannot substitute for the native result, or vice versa/i);
|
||||
expect(flat).toMatch(/structured-review gate still applies/i);
|
||||
expect(flat).toContain('zero counts and `completed:false`');
|
||||
expect(flat).toContain('`CONVERGED`: true only for a completed zero-edit pass');
|
||||
});
|
||||
@@ -367,35 +297,33 @@ test('review finalization ownership: finish only the final core token without lo
|
||||
expect(step.match(/--finish REVIEW_START/g)).toHaveLength(1);
|
||||
expect(step).not.toContain('--start review');
|
||||
expect(step).not.toContain('--finish PASS_START');
|
||||
expect(flat).toContain('Never invent a binding or replace REVIEW_START at log time');
|
||||
expect(flat).toContain('finish only the final core token');
|
||||
expect(flat).toMatch(/never invent a binding or replace REVIEW_START at log time/i);
|
||||
expect(flat).toMatch(/finish only the final core token/i);
|
||||
expect(step).toContain('"completed":COMPLETED,"converged":CONVERGED,"cycles":CYCLES');
|
||||
});
|
||||
|
||||
test('review finalization ownership: the plan audit retains its high-impact gate before the final scope check', () => {
|
||||
const plan = readFileSync(join(root, 'review/sections/plan-completion.md.tmpl'), 'utf8');
|
||||
expect(plan).toContain('INFORMATIONAL except for the HIGH-impact discrepancy question below');
|
||||
expect(plan).toContain('resolve that gate before the final Scope Check');
|
||||
expect(plan).toContain('HIGH-impact discrepancy question');
|
||||
expect(plan).toMatch(/resolve that gate before the final Scope Check/i);
|
||||
expect(plan).not.toContain('never blocks the review');
|
||||
expect(plan).toContain('{{PLAN_COMPLETION_AUDIT_REVIEW}}');
|
||||
const audit = readFileSync(join(root, 'review/sections/plan-completion.md'), 'utf8');
|
||||
const gate = audit.indexOf('**HIGH-impact plan-file discrepancies** trigger AskUserQuestion');
|
||||
expect(gate).toBeGreaterThan(-1);
|
||||
expect(gate).toBeLessThan(audit.indexOf('When continuing after the audit (no HIGH-impact gate, or option B/C)'));
|
||||
expect(audit).toContain('then it gates via AskUserQuestion');
|
||||
expect(audit).toContain('A ends this invocation before code review or implementation');
|
||||
expect(audit).toContain('after implementation, start a fresh /review');
|
||||
expect(audit).toContain('B queues the approved TODO changes for Step 5, not this read-only audit');
|
||||
expect(audit).toContain('B/C continue to the final Scope Check and Step 2');
|
||||
expect(audit).toContain('None of these choices authorizes shipping or waives required verification');
|
||||
expect(skill).toContain("including Step 1.5's approved TODO changes");
|
||||
const flatAudit = audit.replace(/\s+/g, ' ');
|
||||
expect(flatAudit).toMatch(/A ends this invocation before code review or implementation/i);
|
||||
expect(flatAudit).toMatch(/not this read-only audit/i);
|
||||
expect(flatAudit).toContain('B/C continue to the final Scope Check and Step 2');
|
||||
expect(flatAudit).toMatch(/none of these choices authorizes shipping or waives required verification/i);
|
||||
});
|
||||
|
||||
test('review scope notes remain provisional until the plan section emits the only final scope check', () => {
|
||||
const scope = generateScopeDrift({ skillName: 'review', tmplPath: 'review/SKILL.md.tmpl',
|
||||
host: 'claude', paths: HOST_PATHS.claude }).replace(/\s+/g, ' ');
|
||||
expect(scope).toContain('Keep these notes provisional. Next, execute the plan-completion section');
|
||||
expect(scope).toContain('it resolves the HIGH-impact decision and emits the single final Scope Check before Step 2');
|
||||
expect(scope).toMatch(/keep these notes provisional/i);
|
||||
expect(scope).toMatch(/single final Scope Check before Step 2/i);
|
||||
expect(scope).not.toContain('Scope Check: [CLEAN');
|
||||
expect(scope).not.toContain('available plan-audit results');
|
||||
});
|
||||
@@ -404,29 +332,20 @@ test('review confidence uses its severity labels without an undefined P0 excepti
|
||||
const ctx: TemplateContext = { skillName: 'review', tmplPath: 'review/SKILL.md.tmpl',
|
||||
host: 'claude', paths: HOST_PATHS.claude };
|
||||
const confidence = generateConfidenceCalibration(ctx);
|
||||
expect(confidence).toContain('Only report a suspected release-blocking catastrophe');
|
||||
expect(confidence).toContain('widespread data loss, total outage or system-wide compromise');
|
||||
expect(confidence).toContain('label it CRITICAL and explicitly speculative');
|
||||
expect(confidence).toContain('[CRITICAL]');
|
||||
expect(confidence).toContain('[CRITICAL|INFORMATIONAL]');
|
||||
expect(confidence).not.toMatch(/\bP[012]\b/);
|
||||
expect(confidence).toContain('If you cannot quote the motivating line(s), the finding is unverified');
|
||||
const flat = confidence.replace(/\s+/g, ' ');
|
||||
for (const rule of [
|
||||
'| 9-10 | Specific code verifies a concrete bug or exploit. | Show normally |',
|
||||
'| 7-8 | High-confidence pattern match; very likely correct. | Show normally |',
|
||||
'| 5-6 | Moderate; could be a false positive. | Show with caveat:',
|
||||
'Medium confidence, verify this is actually an issue',
|
||||
'| 3-4 | Suspicious but may be fine. | Suppress from main report. Include in appendix only. |',
|
||||
'| 1-2 | Speculation. | Only report a suspected release-blocking catastrophe',
|
||||
'Quote the specific code line', 'file:line and verbatim text',
|
||||
'For a missing field, quote its class definition; for a nullable value, its initialization; for a race, both sides',
|
||||
'Force its confidence to 4-5: use 4 for appendix-only reporting, or 5 only when the finding belongs in the main report with the medium-confidence caveat below',
|
||||
'Never invent speculative confidence 7+',
|
||||
'read and quote their generating metaclass, descriptor, ORM Meta block, migration, decorator or schema',
|
||||
'Missing literal names in the class body or grep results do not prove absence',
|
||||
'If the user confirms a reported finding scored < 7 is real, log the corrected pattern as a learning',
|
||||
]) expect(flat).toContain(rule);
|
||||
for (const row of [
|
||||
/\| 9-10 \|[^|]*\| Show normally \|/, /\| 7-8 \|[^|]*\| Show normally \|/,
|
||||
/\| 5-6 \|[^|]*\| Show with caveat/, /\| 3-4 \|[^|]*\| Suppress from main report/,
|
||||
/\| 1-2 \|[^|]*\| Only report a suspected release-blocking catastrophe/,
|
||||
]) expect(flat).toMatch(row);
|
||||
expect(flat).toMatch(/explicitly speculative/i);
|
||||
expect(flat).toMatch(/the finding is unverified/i);
|
||||
expect(flat).toContain('file:line');
|
||||
expect(flat).toMatch(/force its confidence to 4-5/i);
|
||||
expect(flat).toMatch(/never invent speculative confidence 7\+/i);
|
||||
expect(confidence.indexOf('Pre-emit verification gate')).toBeLessThan(confidence.indexOf('| Score |'));
|
||||
expect(confidence).not.toContain('FP classes the gate kills');
|
||||
expect(confidence).not.toContain('1539-framework-aware-review.md');
|
||||
@@ -435,28 +354,20 @@ test('review confidence uses its severity labels without an undefined P0 excepti
|
||||
|
||||
test('review names the lifecycle and record owners before using their persistence rules', () => {
|
||||
const start = skill.slice(skill.indexOf('## Step 3: Get the diff'), skill.indexOf('## Step 3.4:')).replace(/\s+/g, ' ');
|
||||
expect(start).toContain('An invocation is this /review run; a pass reviews one candidate before any fixes');
|
||||
expect(start).toContain('REVIEW_START / PASS_START | Opaque start receipts from the logger');
|
||||
expect(start).toContain('a matching key alone never proves a prior Skip is reusable');
|
||||
expect(start).toContain("`review_binding` | The logger's proof tying a finished review to its captured candidate");
|
||||
expect(start).toContain('`snapshot_covered_paths` | Supporting advice files the logger proved byte-identical to that candidate');
|
||||
expect(start).toContain('Used by the prior-Skip checker, never supplied by the reviewer');
|
||||
expect(start).toContain('REVIEW_START / PASS_START |');
|
||||
expect(start).toContain('`review_binding` |');
|
||||
expect(start).toContain('`snapshot_covered_paths` |');
|
||||
expect(start).toMatch(/a matching key alone never proves a prior Skip is reusable/i);
|
||||
expect(start).toMatch(/never supplied by the reviewer/i);
|
||||
});
|
||||
|
||||
test('review invocation-local advice reuse keeps raw-source and changed-decision gates', () => {
|
||||
const decisions = skill.slice(skill.indexOf('**Keep decisions through fix cycles:**'),
|
||||
skill.indexOf('### Step 5a:')).replace(/\s+/g, ' ');
|
||||
for (const contract of [
|
||||
'Immediately save completed AUTO-FIX/fix and explicit Skip actions',
|
||||
'keeping defects separate from advice',
|
||||
"retain the helper's fingerprint, `advisory`, `evidence_paths` and `helper_target`",
|
||||
're-read every supporting caller and helper destination',
|
||||
'including secondary callers and transformed/indirect paths',
|
||||
'Compare their raw source with the decision evidence',
|
||||
'Unrelated auto-fixes do not reopen unchanged identity, contract and tradeoffs',
|
||||
'Material proposal, behavior, migration or risk changes require a new question',
|
||||
"cannot suppress new/recurring defects or replace Step 5.0's prior-review checker",
|
||||
]) expect(decisions).toContain(contract);
|
||||
expect(decisions).toMatch(/save completed AUTO-FIX\/fix and explicit Skip actions/i);
|
||||
expect(decisions).toContain("`advisory`, `evidence_paths` and `helper_target`");
|
||||
expect(decisions).toMatch(/require a new question/i);
|
||||
expect(decisions).toMatch(/cannot suppress new\/recurring defects/i);
|
||||
});
|
||||
|
||||
for (const skillName of ['review', 'ship']) {
|
||||
@@ -466,16 +377,14 @@ for (const skillName of ['review', 'ship']) {
|
||||
const flat = army.replace(/\s+/g, ' ');
|
||||
|
||||
test(`${skillName} clarity: terminal failure permits independent work but never certifies coverage`, () => {
|
||||
expect(flat).toContain('Confirm that each task has finished or is stopped');
|
||||
expect(flat).toContain('A timeout alone does not prove termination');
|
||||
expect(flat).toContain("If a reader or writer is still active, wait; if its state is unknown, inspect its task/process status");
|
||||
expect(flat).toContain("If you cannot confirm it stopped, use the parent's Fix-First stop path without edits");
|
||||
expect(flat).toContain('Continue independent evidence collection after a terminal failure');
|
||||
expect(flat).toContain('Missing dispatched coverage remains incomplete, never completed or clean');
|
||||
expect(flat).toMatch(/confirm that each task has finished or is stopped/i);
|
||||
expect(flat).toMatch(/a timeout alone does not prove termination/i);
|
||||
expect(flat).toMatch(/stop path without edits/i);
|
||||
expect(flat).toMatch(/missing dispatched coverage remains incomplete, never completed or clean/i);
|
||||
expect(flat).not.toContain('Specialists are additive — partial results are better than no results');
|
||||
const redTeam = flat.slice(flat.indexOf('### Red Team dispatch'));
|
||||
expect(redTeam).toContain('confirm it stopped and record its review as incomplete, just as for other specialists');
|
||||
expect(redTeam).toContain('original specialist outputs and rerun stages 1–7');
|
||||
expect(redTeam).toMatch(/confirm it stopped and record its review as incomplete/i);
|
||||
expect(redTeam).toContain('stages 1–7');
|
||||
});
|
||||
|
||||
test(`${skillName} clarity: ordered specialist merge separates validation from scoring and provenance`, () => {
|
||||
@@ -487,26 +396,21 @@ for (const skillName of ['review', 'ship']) {
|
||||
expect(positions.every(position => position >= 0)).toBe(true);
|
||||
expect(positions).toEqual([...positions].sort((a, b) => a - b));
|
||||
const validation = flat.slice(flat.indexOf('#### 2.'), flat.indexOf('#### 3.'));
|
||||
expect(validation).toContain('core and specialist findings');
|
||||
expect(validation).toContain('remove `advisory` and retain its `CRITICAL` severity');
|
||||
expect(validation).toContain('Never downgrade severity');
|
||||
expect(validation).toContain('Valid INFORMATIONAL advisories remain advisory');
|
||||
expect(validation).toMatch(/never downgrade severity/i);
|
||||
const merge = flat.slice(flat.indexOf('#### 3.'), flat.indexOf('#### 4.'));
|
||||
expect(merge.indexOf('Partition defects and advisories')).toBeLessThan(merge.indexOf('grouping by fingerprint'));
|
||||
for (const gate of ['sharedLibsFingerprint', 'literal JSON on stdin', 'never trust a supplied hash',
|
||||
'Missing/malformed metadata cannot deduplicate', 'highest confidence', '+1 (cap at 10)',
|
||||
'distinct specialists', 'all source names', 'Core findings never earn a specialist confidence boost']) {
|
||||
expect(merge).toContain(gate);
|
||||
}
|
||||
for (const token of ['sharedLibsFingerprint', 'literal JSON on stdin', '+1 (cap at 10)']) expect(merge).toContain(token);
|
||||
for (const rule of [/never trust a supplied hash/i, /missing\/malformed metadata cannot deduplicate/i,
|
||||
/core findings never earn a specialist confidence boost/i]) expect(merge).toMatch(rule);
|
||||
for (const gate of ['Confidence 7+', 'Confidence 5-6', 'Confidence 3-4', 'Confidence 1-2']) expect(army).toContain(gate);
|
||||
const scoring = flat.slice(flat.indexOf('#### 5.'), flat.indexOf('#### 6.'));
|
||||
expect(scoring).toContain('Only specialist findings enter this header and `quality_score`; core findings do not');
|
||||
expect(scoring).toContain('NON-advisory');
|
||||
expect(scoring).toMatch(/only specialist findings enter this header and `quality_score`/i);
|
||||
expect(scoring).toMatch(/non-advisory/i);
|
||||
expect(scoring).toContain('quality_score = max(0, 10 - (critical_count * 2 + informational_count * 0.5))');
|
||||
expect(scoring).toContain('unresolved-defect totals');
|
||||
expect(flat).toContain('Advisory findings COUNT in the stats `findings` field');
|
||||
expect(flat).toContain('Count only findings that specialist actually returned');
|
||||
expect(flat).toContain('core-only advice must not create a specialist dispatch or finding');
|
||||
expect(flat).toMatch(/advisory findings count in the stats `findings` field/i);
|
||||
expect(flat).toMatch(/count only findings that specialist actually returned/i);
|
||||
expect(flat).toMatch(/core-only advice must not create a specialist dispatch or finding/i);
|
||||
expect(flat).toContain('ASK-only');
|
||||
});
|
||||
|
||||
@@ -517,29 +421,20 @@ for (const skillName of ['review', 'ship']) {
|
||||
const positions = markers.map(marker => reuse.indexOf(marker));
|
||||
expect(positions.every(position => position >= 0)).toBe(true);
|
||||
expect(positions).toEqual([...positions].sort((a, b) => a - b));
|
||||
for (const gate of ['first-party authored provenance', 'all supporting callers and the helper destination',
|
||||
'--check-shared-libs REVIEW_START', "<<'GSTACK_SHARED_LIBS_REUSE_JSON'", 'literal JSON on stdin',
|
||||
'`reusable: true`', 'False, command failure or unreadable output', 'never suppression',
|
||||
'Do not supply your own snapshot, prior record or coverage', 'sharedLibsFingerprint',
|
||||
'without consuming/replacing it', 'actual repo, raw branch and current snapshot',
|
||||
'completed/converged', 'verified binding', 'explicit Skip', 'logger-versioned `snapshot_covered_paths`',
|
||||
'older unversioned coverage', 'Sanitized branch names are not identity', 'canReuseSharedLibsAdvisory',
|
||||
'byte-for-byte with its blob', 'assume-unchanged, skip-worktree', 'sparse index', 'symlinks/ancestors',
|
||||
'submodules', 'ignored/outside or unreadable', 'active/unknown Git filters', 'encodings and line conversion',
|
||||
'fsmonitor and optional locks', 'never uses external diff/textconv', 'Unknown evidence fails closed',
|
||||
'logger recomputes final coverage', 'Real defects retain normal Fix-First handling independently']) {
|
||||
expect(reuse).toContain(gate);
|
||||
}
|
||||
for (const token of ['--check-shared-libs REVIEW_START', "<<'GSTACK_SHARED_LIBS_REUSE_JSON'", 'literal JSON on stdin',
|
||||
'`reusable: true`', 'sharedLibsFingerprint', 'explicit Skip', 'logger-versioned `snapshot_covered_paths`',
|
||||
'canReuseSharedLibsAdvisory']) expect(reuse).toContain(token);
|
||||
for (const rule of [/false, command failure or unreadable output[^.]*never suppression/i,
|
||||
/do not supply your own snapshot, prior record or coverage/i, /unknown evidence fails closed/i,
|
||||
/real defects retain normal Fix-First handling/i]) expect(reuse).toMatch(rule);
|
||||
});
|
||||
}
|
||||
|
||||
test('review clarity: settlement gates edits separately from incomplete required coverage', () => {
|
||||
const fix = skill.slice(skill.indexOf('## Step 5: Fix-First Review'), skill.indexOf('{{CROSS_REVIEW_DEDUP}}')).replace(/\s+/g, ' ');
|
||||
expect(fix).toContain('every dispatched reader has returned or is confirmed stopped');
|
||||
expect(fix).toContain('active or unknown reader/writer');
|
||||
expect(fix).toContain('persist incomplete at Step 5.8 and STOP without edits');
|
||||
expect(fix).toContain('Terminal failure does not block fixes from independent evidence');
|
||||
expect(fix).toContain('Missing required output still makes the pass incomplete');
|
||||
expect(fix).toMatch(/every dispatched reader has returned or is confirmed stopped/i);
|
||||
expect(fix).toMatch(/persist incomplete at Step 5\.8 and stop without edits/i);
|
||||
expect(fix).toMatch(/missing required output still makes the pass incomplete/i);
|
||||
});
|
||||
|
||||
test('review clarity: Greptile reply choices never substitute for Fix-First approval', () => {
|
||||
@@ -547,43 +442,38 @@ test('review clarity: Greptile reply choices never substitute for Fix-First appr
|
||||
expect(fix).toContain('VALID & ACTIONABLE Greptile findings');
|
||||
const greptile = skill.slice(skill.indexOf('### Greptile comment resolution'), skill.indexOf('## Step 5.8:'));
|
||||
const flat = greptile.replace(/\s+/g, ' ');
|
||||
expect(flat).toContain('Step 5c alone supplies A) Fix / B) Skip');
|
||||
expect(flat).toMatch(/Step 5c alone supplies A\) Fix \/ B\) Skip/i);
|
||||
expect(flat).not.toContain('A: Fix it now, B: Acknowledge, C: False positive');
|
||||
expect(flat).toContain('reply decisions, not code approval');
|
||||
expect(flat).toMatch(/reply decisions, not code approval/i);
|
||||
expect(flat).toContain('B) Propose a code change');
|
||||
expect(flat).toContain('return to Steps 5c–5d with an ASK proposal');
|
||||
expect(flat).toContain('Show the exact change and any `test_stub`; wait for approval before editing');
|
||||
expect(flat).toContain('no new fix permission');
|
||||
expect(flat).toContain('Steps 5c–5d');
|
||||
expect(flat).toMatch(/wait for approval before editing/i);
|
||||
expect(flat).toMatch(/no new fix permission/i);
|
||||
});
|
||||
|
||||
test('ship review clarity: parent settlement gate precedes classification and cannot waive coverage', () => {
|
||||
const ship = readFileSync(join(root, 'ship/sections/review-army.md.tmpl'), 'utf8');
|
||||
const gate = ship.slice(ship.indexOf('## Step 9.4:'), ship.indexOf('1. **Classify')).replace(/\s+/g, ' ');
|
||||
expect(gate).toContain("Before edits, inspect every dispatched reader/writer's handle");
|
||||
expect(gate).toContain('Wait for return or confirm termination');
|
||||
expect(gate).toContain('otherwise log incomplete through items 5–6 and STOP without edits');
|
||||
expect(gate).toContain('After terminal failure, independent evidence may support fixes');
|
||||
expect(gate).toContain('missing dispatched output still blocks continuation, even with a QA exception');
|
||||
expect(gate).toMatch(/wait for return or confirm termination/i);
|
||||
expect(gate).toMatch(/log incomplete through items 5–6 and stop without edits/i);
|
||||
expect(gate).toMatch(/missing dispatched output still blocks continuation, even with a QA exception/i);
|
||||
});
|
||||
|
||||
test('review resolves the judged smoke-clock, setup-authority, plan-gate and findings-source ambiguities', () => {
|
||||
test('review keeps its smoke-clock, setup-authority, plan-gate and findings-source rules', () => {
|
||||
const ctx = { skillName: 'review', tmplPath: 'review/SKILL.md.tmpl', host: 'claude', paths: HOST_PATHS.claude } as TemplateContext;
|
||||
const qa = generateQAReview(ctx).replace(/\s+/g, ' ');
|
||||
expect(qa).toContain('Follow the shared Probe loop for smoke checks and replays until the smoke limit');
|
||||
expect(qa).toContain('Then run required plan checks and revalidation, even after smoke expires, using the same procedure but no smoke guard');
|
||||
expect(qa).toContain('/review sets none; only an invoker-supplied EARLIER_UTC counts');
|
||||
expect(qa).toContain('report-only /review never runs setup, installs or cookie import');
|
||||
expect(qa).toContain('After a grant, recheck readiness and run affected checks; otherwise they stay blocked');
|
||||
expect(qa).toMatch(/\/review sets none; only an invoker-supplied EARLIER_UTC counts/);
|
||||
expect(qa).toMatch(/report-only \/review never runs setup, installs or cookie import/i);
|
||||
expect(qa).toMatch(/otherwise they stay blocked/i);
|
||||
expect(qa).not.toContain('Ask for setup/permission');
|
||||
expect(generateQAReview({ ...ctx, skillName: 'ship', tmplPath: 'ship/SKILL.md.tmpl' })).not.toContain('/review sets none');
|
||||
const shared = generateQAExploratory({ ...ctx, skillName: 'qa' }).replace(/\s+/g, ' ');
|
||||
expect(shared).toContain('Explicit plan checks and revalidation remain required beyond this smoke budget');
|
||||
const audit = generatePlanCompletionAuditReview(ctx).replace(/\s+/g, ' ');
|
||||
expect(audit).toContain('No plan file found → say "No plan file detected." and use the Fallback Intent Sources below');
|
||||
expect(audit).toContain('"No plan file detected."');
|
||||
expect(audit).toContain('Fallback Intent Sources');
|
||||
expect(audit).not.toContain('skip with "No plan file detected — skipping."');
|
||||
expect(audit).toContain('**HIGH-impact plan-file discrepancies** trigger AskUserQuestion');
|
||||
expect(audit).toContain('Discrepancies derived only from fallback sources (commit messages, TODOS.md, PR description) never trigger this question');
|
||||
expect(audit).toMatch(/derived only from fallback sources[\s\S]{0,80}never trigger this question/i);
|
||||
expect(generatePlanCompletionAuditShip({ ...ctx, skillName: 'ship', tmplPath: 'ship/SKILL.md.tmpl' })).toContain('skip with "No plan file detected — skipping."');
|
||||
const persist = skill.slice(skill.indexOf('### 2. Fill the record')).replace(/\s+/g, ' ');
|
||||
expect(persist).toContain("Step 5's combined final-pass findings (core, specialist, adversarial, actionable Greptile, verified exploratory QA findings)");
|
||||
expect(persist).toMatch(/combined final-pass findings \(core, specialist, adversarial, actionable Greptile, verified exploratory QA findings\)/i);
|
||||
});
|
||||
@@ -39,7 +39,7 @@ describe("ship Apple gate ordering (R2)", () => {
|
||||
|
||||
test("store distribution explicitly bypasses the branch/PR ceremony", () => {
|
||||
expect(SKELETON).toContain("Store distribution proceeds");
|
||||
expect(SKELETON).toMatch(/branch gate and repository-landing pipeline below apply ONLY to\s*\n?repository-landing asks/);
|
||||
expect(SKELETON).toMatch(/branch gate and repository-landing pipeline below apply only to\s+repository-landing asks/i);
|
||||
});
|
||||
|
||||
test("the non-Apple branch gate is byte-unchanged and appears exactly once", () => {
|
||||
@@ -50,30 +50,23 @@ describe("ship Apple gate ordering (R2)", () => {
|
||||
|
||||
test("the adapter section exists in the union with its battle-tested spine", () => {
|
||||
const section = readFileSync(join(ROOT, "ship", "sections", "apple-release.md"), "utf-8");
|
||||
for (const anchor of [
|
||||
"one authorization moment",
|
||||
"fastlane spaceauth",
|
||||
"iris/v1/apiKeys",
|
||||
"appPriceSchedules",
|
||||
"CLASSIFY the error before touching credentials",
|
||||
"Never abort an App Store release over branch topology",
|
||||
]) {
|
||||
for (const anchor of ["one authorization moment", "fastlane spaceauth", "iris/v1/apiKeys", "appPriceSchedules"]) {
|
||||
expect(section).toContain(anchor);
|
||||
}
|
||||
expect(section).toMatch(/classify the error before touching credentials/i);
|
||||
expect(section).toMatch(/never abort an App Store release over branch topology/i);
|
||||
});
|
||||
|
||||
test("routine interaction limits cannot waive blocking documentation or safety decisions", () => {
|
||||
for (const name of ["apple-release.md.tmpl", "apple-release.md"]) {
|
||||
const section = readFileSync(join(ROOT, "ship", "sections", name), "utf-8");
|
||||
expect(section).toContain("Plan for two routine interactions");
|
||||
expect(section).toContain("A genuine blocker may require a safety or named documentation-risk decision");
|
||||
expect(section).toContain("STOP for that decision rather than treating release authorization as a waiver");
|
||||
expect(section).toContain("routine interactions and blocking decisions above");
|
||||
const section = readFileSync(join(ROOT, "ship", "sections", name), "utf-8").replace(/\s+/g, " ");
|
||||
expect(section).toMatch(/two routine interactions/i);
|
||||
expect(section).toMatch(/stop for that decision rather than treating release authorization as a waiver/i);
|
||||
expect(section).not.toContain("exactly two interactions, and no others");
|
||||
expect(section).not.toContain("two permitted interactions");
|
||||
expect(section.indexOf("**Documentation preflight:**")).toBeLessThan(section.indexOf("## The one authorization moment"));
|
||||
expect(section).toContain("in `read-only` mode against the selected release source");
|
||||
expect(section).toContain("Resolve blockers or obtain an explicit named documentation-risk exception before distribution");
|
||||
expect(section).toContain("`read-only` mode");
|
||||
expect(section).toMatch(/named documentation-risk exception before distribution/i);
|
||||
}
|
||||
});
|
||||
});
|
||||
+101
-138
@@ -22,49 +22,40 @@ const gate = compact(entry.slice(entry.indexOf('## Step 16:'), entry.indexOf('##
|
||||
describe('ship source controller', () => {
|
||||
test('documentation freshness covers selected code paths as well as release metadata', () => {
|
||||
const docs = compact(read('ship/sections/documentation.md.tmpl'));
|
||||
expect(docs).toContain('hashes of the selected release paths, generated outputs and docs/templates');
|
||||
expect(docs).toContain('selected release paths, generated outputs and docs/templates');
|
||||
const freshness = gate.slice(gate.indexOf('### 3.'), gate.indexOf('### 4.'));
|
||||
expect(freshness).toContain('selected release paths, generated outputs and docs/templates');
|
||||
expect(freshness).toContain("A prior invocation's audit or risk decision never qualifies");
|
||||
expect(freshness).toContain('the same approved scope and exact content');
|
||||
expect(freshness).toMatch(/a prior invocation's audit or risk decision never qualifies/i);
|
||||
});
|
||||
|
||||
test('changed documentation inputs permit only the remaining bounded re-audit', () => {
|
||||
const docs = compact(read('ship/sections/documentation.md.tmpl'));
|
||||
const recovery = docs.slice(docs.indexOf('## Blocked recovery'));
|
||||
expect(recovery).toContain('If an attempt remains and either the audited inputs changed');
|
||||
expect(recovery).toContain('a concrete launch/input/permission correction or reviewed patch repair is available');
|
||||
expect(docs).toContain('an initial audit plus ONE repair/re-audit');
|
||||
expect(docs).toContain('never a third attempt, even after Step 16 changes');
|
||||
expect(docs.indexOf('## Blocked recovery')).toBeGreaterThan(-1);
|
||||
expect(docs).toMatch(/initial[- ]audit plus one repair\/re-audit/i);
|
||||
expect(docs).toMatch(/never a third attempt/i);
|
||||
const freshness = gate.slice(gate.indexOf('### 3.'), gate.indexOf('### 4.'));
|
||||
expect(freshness).toContain('**An attempt remains, with changed inputs or an available repair:**');
|
||||
expect(freshness).toContain('**Otherwise:** STOP unless the user accepts');
|
||||
expect(freshness).toMatch(/\*\*Otherwise:\*\* stop unless the user accepts/i);
|
||||
expect(freshness).not.toContain('**No attempt remains or no repair is available:**');
|
||||
});
|
||||
|
||||
test('repairs use one ordered work list without a return stack', () => {
|
||||
const text = compact(controller);
|
||||
expect(compact(entry)).toContain('**Next steps:** one ordered work list, with the current step marked');
|
||||
expect(text).toContain('Expand a repair into individual steps and insert them before the still-pending work');
|
||||
expect(text).toContain('This replaces the current item, whose actual result stays in the record');
|
||||
expect(text).toContain('Add its destination only if not already the next pending step');
|
||||
expect(text).toContain('The saved list takes precedence over ordinary next-step sentences inside a repair');
|
||||
expect(compact(entry)).toContain('**Next steps:**');
|
||||
expect(text).toContain('5 → 6 → 7 → 8 → 9 → 10 → 11 → 11.5');
|
||||
expect(text).not.toContain('finish the inner repair, then resume the unfinished outer range');
|
||||
expect(text).toContain('Keep the same attempt counts throughout the invocation');
|
||||
expect(text).toContain('initial-plus-ONE limit never resets');
|
||||
expect(text).toMatch(/initial-plus-one limit never resets/i);
|
||||
});
|
||||
|
||||
test('the roadmap and recovery groups define ownership and zero-edit eligibility', () => {
|
||||
const text = compact(entry);
|
||||
expect(text).toContain('integrate (1–3) → test and review (4–11.5) → prepare the release (12–15) → verify frozen content (16) → push and publish (17–21)');
|
||||
expect(text).toContain('children return evidence, not permission to proceed');
|
||||
expect(review).toContain('**No edits in this pass:** Resolve the required-probe gate below');
|
||||
expect(review).toContain('Only after it clears may you continue to Step 10');
|
||||
expect(review).toContain('With completed checklist and dispatched reviewers, failed/unavailable required probes block continuation');
|
||||
expect(review).toContain('This cannot waive missing reviewer output, recurring fixes or independent test/security gates');
|
||||
expect(review).toContain('Undispatched gated/unsupported specialists do not block independently');
|
||||
expect(text).toContain('Reuse it only for that same scope; a repair never resets approvals or expands them');
|
||||
expect(text).toMatch(/children return evidence, not permission to proceed/i);
|
||||
expect(review).toContain('**No edits in this pass:**');
|
||||
expect(review).toMatch(/only after it clears may you continue to Step 10/i);
|
||||
expect(review).toMatch(/failed\/unavailable required probes block continuation/i);
|
||||
expect(review).toMatch(/cannot waive missing reviewer output, recurring fixes or independent test\/security gates/i);
|
||||
expect(text).toMatch(/a repair never resets approvals or expands them/i);
|
||||
expect(text).not.toContain('eligible zero-edit pass');
|
||||
expect(text).not.toContain('the same waiver');
|
||||
expect(text).not.toContain("the controller's detour");
|
||||
@@ -72,13 +63,9 @@ describe('ship source controller', () => {
|
||||
|
||||
test('distribution discovery is a shortlist, not a manifest-only artifact decision', () => {
|
||||
const distribution = compact(entry.slice(entry.indexOf('## Step 2:'), entry.indexOf('## Step 3:')));
|
||||
expect(distribution).toContain('List candidate distribution paths');
|
||||
expect(distribution).toContain("Also inspect matching untracked files from Step 1's status");
|
||||
expect(distribution).toContain('a new `package.json` or `Cargo.toml` alone does not establish a publishable artifact');
|
||||
expect(distribution).toContain('inspect existing manifests for newly declared binaries or package exports');
|
||||
expect(distribution).toContain('New artifact without a pipeline');
|
||||
expect(distribution).toContain('**New artifact without a pipeline');
|
||||
expect(distribution).toContain('AskUserQuestion');
|
||||
expect(distribution).toContain('Do not publish a release during `/ship`');
|
||||
expect(distribution).toMatch(/do not publish a release during `\/ship`/i);
|
||||
});
|
||||
|
||||
test('queue qualification exhaustively distinguishes online, git fallback and unusable output', () => {
|
||||
@@ -89,13 +76,11 @@ describe('ship source controller', () => {
|
||||
expect(qualify).toBeGreaterThan(0);
|
||||
expect(usable).toBeGreaterThan(qualify);
|
||||
expect(missing).toBeGreaterThan(usable);
|
||||
expect(version).toContain('require successful utility output and a nonempty valid version');
|
||||
expect(version).toContain('`offline:false` qualifies; `offline:true` qualifies only with `fallback:"git"`');
|
||||
expect(version).toContain('Offline output without that fallback, failure, malformed output or an empty version is unusable, even if it contains a version-looking string');
|
||||
expect(version).toContain('FRESH sets `NEW_VERSION=CANDIDATE_VERSION`');
|
||||
expect(version).toContain('Only approval changes the existing version');
|
||||
expect(version).toContain('a sibling holding `>= NEW_VERSION` requires a choice: advance past it, or stop this attempt and sync');
|
||||
expect(version.slice(missing)).toContain('FRESH uses local `BUMP_LEVEL` arithmetic; ALREADY_BUMPED keeps `currentVersion`');
|
||||
expect(version).toContain('`NEW_VERSION=CANDIDATE_VERSION`');
|
||||
expect(version).toMatch(/only approval changes the existing version/i);
|
||||
expect(version).toContain('`>= NEW_VERSION`');
|
||||
expect(version.slice(missing)).toContain('`BUMP_LEVEL`');
|
||||
expect(version.slice(missing)).toContain('`currentVersion`');
|
||||
});
|
||||
|
||||
test.each(['home', 'override', 'plugin'])('the actual nudge shell honors %s roots, repeat suppression and enabled tuning', mode => {
|
||||
@@ -142,11 +127,8 @@ describe('ship source controller', () => {
|
||||
|
||||
test('nested repairs resume the unfinished outer range before its destination', () => {
|
||||
const text = compact(controller);
|
||||
expect(text).toContain('Expand a repair into individual steps and insert them before the still-pending work');
|
||||
expect(text).toContain('For another repair, repeat rule 2 without discarding pending work');
|
||||
expect(text).toContain('Step 11 fixes insert `9 → 10 → 11` before 11.5');
|
||||
expect(text).toContain('A further Step 9 fix affecting 6–8 makes the list `5 → 6 → 7 → 8 → 9 → 10 → 11 → 11.5`');
|
||||
expect(text).toContain('The unchanged release steps follow');
|
||||
expect(text).toContain('`9 → 10 → 11` before 11.5');
|
||||
expect(text).toContain('`5 → 6 → 7 → 8 → 9 → 10 → 11 → 11.5`');
|
||||
expect(text).not.toContain('Enter Step 9 before REVIEW_START capture and full review');
|
||||
});
|
||||
|
||||
@@ -157,10 +139,8 @@ describe('ship source controller', () => {
|
||||
expect(shared).toBeLessThan(coverage.indexOf('Dispatch the audit through Agent'));
|
||||
const contract = compact(coverage.slice(shared, coverage.indexOf('Dispatch the audit through Agent')));
|
||||
expect(contract).toContain('use the Agent tool with `run_in_background: false`');
|
||||
expect(contract).toContain('Omitting the flag runs the subagent in the background');
|
||||
expect(contract).toContain('keeping a fresh context');
|
||||
expect(contract).toContain('Do not invoke the target as a Skill or run it inline instead');
|
||||
expect(contract).toContain("only under that section's documented fallback, after a failed subagent has stopped");
|
||||
expect(contract).toMatch(/do not invoke the target as a Skill or run it inline instead/i);
|
||||
expect(contract).toMatch(/after a failed subagent has stopped/i);
|
||||
for (const section of ['test-coverage', 'plan-completion', 'greptile']) {
|
||||
const text = read(`ship/sections/${section}.md.tmpl`);
|
||||
expect(text).toContain('shared foreground-dispatch rule');
|
||||
@@ -178,8 +158,7 @@ describe('ship source controller', () => {
|
||||
const ship = generateReviewDashboard(ctx);
|
||||
expect(ship).toContain('REVIEW READINESS DASHBOARD');
|
||||
expect(ship).toContain('| Review | Runs | Last run | Status | Required |');
|
||||
expect(ship).toContain('Use one row for each entry in step 1');
|
||||
expect(ship).toContain('Only Eng Review is marked required');
|
||||
expect(ship).toMatch(/only Eng Review is marked required/i);
|
||||
expect(ship).toContain('{actual status and reason}');
|
||||
expect(ship).toContain('VERDICT: {CLEARED or NOT CLEARED} — {reason}');
|
||||
expect(ship).not.toContain('+====================================================================+');
|
||||
@@ -190,18 +169,17 @@ describe('ship source controller', () => {
|
||||
test('coverage fallback settles the child and still applies the coverage gate', () => {
|
||||
const text = compact(read('ship/sections/test-coverage.md.tmpl'));
|
||||
const fallback = text.slice(text.indexOf('**Audit failure:**'), text.indexOf('{{TEST_COVERAGE_GATE_SHIP}}'));
|
||||
expect(fallback).toContain('confirm it stopped before running the same audit inline');
|
||||
expect(fallback).toContain('does not pass or bypass the coverage gate');
|
||||
expect(fallback).toContain('including its undetermined-percentage and test-only rules');
|
||||
expect(fallback).toMatch(/confirm it stopped before running the same audit inline/i);
|
||||
expect(fallback).toMatch(/does not pass or bypass the coverage gate/i);
|
||||
expect(text).not.toContain('partial results are better than none');
|
||||
});
|
||||
|
||||
test('no-plan gate skips only the audit and retains verification and the remaining section', () => {
|
||||
const text = generatePlanCompletionGateShip({ host: 'claude', skillName: 'ship', tmplPath: '', paths: HOST_PATHS.claude });
|
||||
const noPlan = compact(text.slice(text.indexOf('**No plan file found:**')));
|
||||
expect(noPlan).toContain('Skip only the plan completion audit');
|
||||
expect(noPlan).toContain('Continue with Step 8.1, Scope Drift and Prior Learnings');
|
||||
expect(noPlan).toContain('Step 9 QA still runs');
|
||||
expect(noPlan).toMatch(/skip only the plan completion audit/i);
|
||||
expect(noPlan).toContain('Step 8.1');
|
||||
expect(noPlan).toMatch(/Step 9 QA still runs/i);
|
||||
expect(noPlan).not.toContain('Skip entirely');
|
||||
});
|
||||
|
||||
@@ -211,8 +189,8 @@ describe('ship source controller', () => {
|
||||
const positions = steps.map(step => text.indexOf(step));
|
||||
expect(positions.every(position => position >= 0)).toBe(true);
|
||||
expect(positions).toEqual([...positions].sort((a,b) => a-b));
|
||||
expect(text).toContain('All three snapshots must match');
|
||||
expect(text).toContain('does not mean the failed or unrun probes passed');
|
||||
expect(text).toMatch(/all three snapshots must match/i);
|
||||
expect(text).toMatch(/does not mean the failed or unrun probes passed/i);
|
||||
expect(text).not.toContain('equal snapshots do not pass waived probes');
|
||||
});
|
||||
|
||||
@@ -221,9 +199,8 @@ describe('ship source controller', () => {
|
||||
expect(text).toContain('**Behavior, tests or build inputs changed:**');
|
||||
expect(text).toContain('**Only authored docs or release metadata changed:**');
|
||||
expect(text).toContain('**No changes, or the docs-only checks still support the plan:**');
|
||||
expect(text).toContain('accepted audit matches all inputs | Continue to stage 4');
|
||||
expect(text).toContain("Use Blocked recovery with the existing count");
|
||||
expect(text).toContain('Use this example only after confirming that every allowed edit is release metadata');
|
||||
expect(text).toContain('| Continue to stage 4');
|
||||
expect(text).toMatch(/Blocked recovery with the existing count/i);
|
||||
expect(text).not.toContain('Every listed change below is metadata:');
|
||||
});
|
||||
|
||||
@@ -233,26 +210,24 @@ describe('ship source controller', () => {
|
||||
expect(end).toBeLessThan(entry.indexOf('{{BASE_BRANCH_DETECT}}'));
|
||||
expect(entry.match(/### Ship control flow/g)).toHaveLength(1);
|
||||
const text = compact(controller);
|
||||
expect(text).toContain('You, the **parent** running /ship, own advancement');
|
||||
expect(text).toContain('Follow the saved work list');
|
||||
expect(text).toContain('STOP and AskUserQuestion gates still apply during repairs');
|
||||
expect(text).toContain('The saved list takes precedence over ordinary next-step sentences inside a repair. A range never adds unlisted steps');
|
||||
expect(text).toContain('Keep the same attempt counts throughout the invocation');
|
||||
expect(text).toMatch(/the \*\*parent\*\* running \/ship, own advancement/i);
|
||||
expect(text).toMatch(/stop and AskUserQuestion gates still apply during repairs/i);
|
||||
expect(text).toMatch(/a range never adds unlisted steps/i);
|
||||
expect(text).toMatch(/keep the same attempt counts throughout the invocation/i);
|
||||
expect(text).toContain('A range ending at Step 14 does not enter Step 14.5');
|
||||
expect(text).toContain('its initial-plus-ONE limit never resets');
|
||||
expect(text).not.toContain('| At step |');
|
||||
expect(entry.match(/Ship control flow/g)).toHaveLength(1);
|
||||
expect(review).toContain('Every repeat starts before the checklist read and captures a fresh REVIEW_START');
|
||||
expect(review).toMatch(/captures a fresh REVIEW_START/);
|
||||
});
|
||||
|
||||
test('distribution and bootstrap detours end at their exact forward resume', () => {
|
||||
const merge = compact(entry.slice(entry.indexOf('## Step 3:'), entry.indexOf('{{SECTION:tests}}')));
|
||||
expect(merge).toContain('repeat Step 2 on the merged content, including its decisions, then continue to Step 4');
|
||||
expect(merge).toContain('Otherwise continue to Step 4 directly');
|
||||
expect(merge).toMatch(/repeat Step 2 on the merged content[^.]*continue to Step 4/i);
|
||||
expect(merge).toMatch(/otherwise continue to Step 4 directly/i);
|
||||
const tests = compact(read('ship/sections/tests.md.tmpl'));
|
||||
expect(tests).toContain('A runs Step 4 with this new bootstrap choice, then returns here to run the tests');
|
||||
expect(tests).toMatch(/A runs Step 4 with this new bootstrap choice, then returns here/i);
|
||||
expect(tests).toContain('A) Add tests (recommended)');
|
||||
expect(tests).toContain('declining bootstrap alone is not that approval');
|
||||
expect(tests).toMatch(/declining bootstrap alone is not that approval/i);
|
||||
});
|
||||
|
||||
test('missing dispatched output outranks the cycle cap, fixes and zero-edit continuation', () => {
|
||||
@@ -262,49 +237,40 @@ describe('ship source controller', () => {
|
||||
expect(positions.every(position => position >= 0)).toBe(true);
|
||||
expect(positions).toEqual([...positions].sort((a, b) => a - b));
|
||||
const missing = decisions.slice(positions[0], positions[1]);
|
||||
expect(missing).toContain('STOP and name each failed specialist or Red Team');
|
||||
expect(missing).toContain('Retain queued fixes and restore coverage');
|
||||
expect(missing).toContain('If this pass made edits, resume at the next decision; otherwise run a fresh complete Step 9');
|
||||
expect(missing).toContain('A successful peer or a QA exception cannot replace missing dispatched coverage');
|
||||
expect(missing).toMatch(/^1\. \*\*Dispatched reviewer output missing:\*\* stop\b/i);
|
||||
expect(missing).toMatch(/cannot replace missing dispatched coverage/i);
|
||||
const cap = decisions.slice(positions[1], positions[2]);
|
||||
expect(cap).toContain('STOP');
|
||||
expect(cap).toContain('`converged:false`; do not run a fourth fixing cycle');
|
||||
expect(cap).toMatch(/\bstop\b/i);
|
||||
expect(cap).toContain('`converged:false`');
|
||||
expect(cap).toMatch(/do not run a fourth fixing cycle/i);
|
||||
const fixes = decisions.slice(positions[2], positions[3]);
|
||||
expect(fixes).toContain('Insert Step 5, affected Steps 6–8 and all of Step 9 before the pending Step 10 in the work list');
|
||||
expect(fixes).toContain('Tests must pass or retain approval for the same verified pre-existing failures and scope');
|
||||
expect(fixes).toContain('Keep CYCLES and scoped approvals across this repeat');
|
||||
expect(decisions.slice(positions[3])).toContain('Only after it clears may you continue to Step 10');
|
||||
expect(review).toContain('Undispatched host-unsupported/gated specialists do not block');
|
||||
expect(review).toContain('Failed, blocked, inconclusive or not-run required probes mean false, never clean');
|
||||
expect(review).toContain('This cannot waive missing reviewer output, recurring fixes or independent test/security gates');
|
||||
expect(fixes).toContain('Insert Step 5, affected Steps 6–8 and all of Step 9 before the pending Step 10');
|
||||
expect(fixes).toMatch(/retain approval for the same verified pre-existing failures and scope/i);
|
||||
expect(decisions.slice(positions[3])).toMatch(/only after it clears may you continue to Step 10/i);
|
||||
expect(review).toMatch(/failed, blocked, inconclusive or not-run required probes mean false, never clean/i);
|
||||
});
|
||||
|
||||
test('comment fixes resume triage and preserve settled approvals and replies', () => {
|
||||
const section = compact(read('ship/sections/greptile.md.tmpl'));
|
||||
expect(section).toContain('If fixes were approved, save their approvals and comment references');
|
||||
expect(section).toContain("Run Step 9's full review/fix loop, then return here");
|
||||
expect(review).toContain('Finish the complete review and QA before applying any fix in Step 9.4');
|
||||
expect(section).toContain('Finish the saved replies without asking again about completed fixes, and classify new comments');
|
||||
expect(section).toContain('With no queued fixes, continue to Step 11');
|
||||
expect(section).toContain('This optional triage does not block ship');
|
||||
expect(section).toContain('unknown or missing status is unavailable');
|
||||
expect(section).toMatch(/save their approvals and comment references/i);
|
||||
expect(section).toMatch(/Run Step 9's full review\/fix loop, then return here/i);
|
||||
expect(review).toMatch(/finish the complete review and QA before applying any fix in Step 9\.4/i);
|
||||
expect(section).toMatch(/with no queued fixes, continue to Step 11/i);
|
||||
expect(section).toMatch(/this optional triage does not block ship/i);
|
||||
expect(section).not.toContain('return to Step 9');
|
||||
});
|
||||
|
||||
test('native recovery has one corrected attempt and cannot borrow outside completion', () => {
|
||||
const finish = adversarial.slice(adversarial.indexOf('### Finish the adversarial phase'));
|
||||
const recovery = finish.slice(finish.indexOf('1. **Required native'), finish.indexOf('2. **Fixes queued'));
|
||||
expect(recovery).toContain('STOP and confirm the native task stopped');
|
||||
expect(recovery).toContain('Outside-provider output cannot replace this pass');
|
||||
expect(recovery).toContain('One recovery retry is allowed only after a concrete prerequisite correction and restored access');
|
||||
expect(recovery).toContain('count it in the invocation record before launch');
|
||||
expect(recovery).toContain('Capture a fresh PASS_START and persist the new attempt separately');
|
||||
expect(recovery).toContain('Without that correction, or if the recovery fails, ask for repair and remain blocked');
|
||||
expect(recovery).toMatch(/stop and confirm the native task stopped/i);
|
||||
expect(recovery).toMatch(/outside-provider output cannot replace this pass/i);
|
||||
expect(recovery).toMatch(/one recovery retry is allowed only after a concrete prerequisite correction/i);
|
||||
expect(recovery).toContain('PASS_START');
|
||||
expect(recovery).toMatch(/ask for repair and remain blocked/i);
|
||||
const queued = finish.slice(finish.indexOf('2. **Fixes queued'), finish.indexOf('3. **Native complete'));
|
||||
expect(queued).toContain('Keep the findings and their approvals');
|
||||
expect(queued).toContain('Insert Steps 9, 10 and 11 before the pending Step 11.5 in the work list. Step 9 completes full review before fixes');
|
||||
expect(queued).toContain('any further repair inserts its checks ahead of the remaining items');
|
||||
expect(queued).toContain('not recovery retries');
|
||||
expect(queued).toContain('Insert Steps 9, 10 and 11 before the pending Step 11.5');
|
||||
expect(queued).toMatch(/not recovery retries/i);
|
||||
expect(finish.slice(finish.indexOf('3. **Native complete'))).toContain('then continue to Step 11.5');
|
||||
});
|
||||
|
||||
@@ -312,19 +278,19 @@ describe('ship source controller', () => {
|
||||
const ctx = { host, skillName: 'ship', tmplPath: '', paths: HOST_PATHS[host] };
|
||||
const ship = generateAdversarialStep(ctx);
|
||||
const finish = compact(ship.slice(ship.indexOf('### Finish the adversarial phase')));
|
||||
expect(finish).toContain('Apply these decisions in order before leaving Step 11');
|
||||
expect(finish).toContain('Required native review incomplete');
|
||||
expect(finish).toContain('Outside-provider output cannot replace this pass');
|
||||
expect(finish).toContain('Fixes queued after native completion');
|
||||
expect(finish).toContain('Native complete with no queued fixes');
|
||||
const branches = ['Required native review incomplete', 'Fixes queued after native completion', 'Native complete with no queued fixes']
|
||||
.map(label => finish.indexOf(label));
|
||||
expect(branches.every(index => index >= 0)).toBe(true);
|
||||
expect(branches).toEqual([...branches].sort((a, b) => a - b));
|
||||
expect(finish).toMatch(/outside-provider output cannot replace this pass/i);
|
||||
expect(finish).toContain('Step 11.5');
|
||||
expect(finish).not.toContain('proceed to Step 12');
|
||||
expect(finish).not.toContain('return to Step 9');
|
||||
const review = generateAdversarialStep({ ...ctx, skillName: 'review' });
|
||||
expect(review).not.toContain('Ship control flow');
|
||||
expect(review).not.toContain('Step 11.5');
|
||||
expect(review).toContain('Return all findings and structured-review decisions to Step 5');
|
||||
expect(review).toContain('do not start an inner repair loop');
|
||||
expect(review).toMatch(/return all findings and structured-review decisions to Step 5/i);
|
||||
expect(review).toMatch(/do not start an inner repair loop/i);
|
||||
});
|
||||
|
||||
test('Step 11.5 verifies original record identity before any release write', () => {
|
||||
@@ -332,59 +298,56 @@ describe('ship source controller', () => {
|
||||
expect(bindingStart).toBeGreaterThan(entry.indexOf('{{SECTION:adversarial}}'));
|
||||
expect(bindingStart).toBeLessThan(entry.indexOf('## Step 12:'));
|
||||
const binding = compact(entry.slice(bindingStart, entry.indexOf('## Step 12:')));
|
||||
for (const field of ['saved handle, original token and source', 'skill:"review"', 'via:"ship"', 'skill:"adversarial-review"',
|
||||
for (const field of ['skill:"review"', 'via:"ship"', 'skill:"adversarial-review"',
|
||||
'review_binding.state', 'verified', 'review_binding.start_wtree', 'review_binding.end_wtree']) expect(binding).toContain(field);
|
||||
expect(binding).toContain('Never attach new tokens to old work');
|
||||
expect(binding).toContain('Keep Step 9.4\'s incomplete flags');
|
||||
expect(binding).toContain('does not mean the failed or unrun probes passed');
|
||||
expect(binding).toContain('insert `9 → 10 → 11 → 11.5` before Step 12. Bind the new records at 11.5');
|
||||
expect(binding).toMatch(/never attach new tokens to old work/i);
|
||||
expect(binding).toMatch(/does not mean the failed or unrun probes passed/i);
|
||||
expect(binding).toContain('`9 → 10 → 11 → 11.5` before Step 12');
|
||||
});
|
||||
|
||||
test('late build and behavior changes complete bounded ranges before docs', () => {
|
||||
const build = gate.slice(gate.indexOf('### 1.'), gate.indexOf('### 2.'));
|
||||
expect(build).toContain('A missing prerequisite or failed build stops shipping');
|
||||
expect(build).toContain('Repair the prerequisite or build, then repeat stage 1');
|
||||
expect(build).toContain('After it passes, continue to stage 2; treat any content repair as a behavioral change there');
|
||||
expect(build).toMatch(/a missing prerequisite or failed build stops shipping/i);
|
||||
expect(build).toMatch(/repeat stage 1/i);
|
||||
expect(build).toMatch(/continue to stage 2/i);
|
||||
const behavior = gate.slice(gate.indexOf('1. **Behavior'), gate.indexOf('2. **Only authored'));
|
||||
expect(behavior).toContain('Insert `5–11.5 → 12–14 → 16` before the pending Step 17');
|
||||
expect(behavior).toContain('before the pending Step 17, then stop this step. This repair excludes Step 14.5');
|
||||
expect(behavior).toContain('rebuild and compare again before stage 3 decides documentation freshness');
|
||||
expect(behavior).toMatch(/excludes Step 14\.5/i);
|
||||
expect(behavior).toMatch(/rebuild and compare again before stage 3/i);
|
||||
const plan = gate.slice(gate.indexOf('2. **Only authored'), gate.indexOf('3. **No changes'));
|
||||
expect(plan).toContain("run Step 8's audit and decision gates only, then return to Step 16 stage 1");
|
||||
expect(plan).toContain("Never edit the child's counts yourself");
|
||||
expect(plan).toMatch(/run Step 8's audit and decision gates only, then return to Step 16 stage 1/i);
|
||||
expect(plan).toMatch(/never edit the child's counts yourself/i);
|
||||
});
|
||||
|
||||
test('docs freshness has bounded re-audit and exact-content exception routes', () => {
|
||||
const docs = gate.slice(gate.indexOf('### 3.'), gate.indexOf('### 4.'));
|
||||
expect(docs).toContain("Use Blocked recovery with the existing count");
|
||||
expect(docs).toMatch(/Blocked recovery with the existing count/i);
|
||||
expect(docs).toContain('**An attempt remains, with changed inputs or an available repair:** insert `14.5 → 15 → 16` before Step 17');
|
||||
expect(docs).toContain('restart Step 16 stage 1 to regenerate and compare again');
|
||||
expect(docs).toContain('STOP unless the user accepts the specific named documentation risk and all unwaivable gates clear');
|
||||
expect(docs).toContain('Never run a third audit');
|
||||
expect(docs).toContain('Unchanged approved content goes to stage 4; repaired content goes to stage 1');
|
||||
expect(docs).toContain('retain `Documentation: blocked`');
|
||||
expect(docs).toContain('Missing, stale or blocked | Use recovery below. Never silently refresh hashes');
|
||||
expect(docs).toMatch(/stop unless the user accepts the specific named documentation risk and all unwaivable gates clear/i);
|
||||
expect(docs).toMatch(/never run a third audit/i);
|
||||
expect(docs).toMatch(/unchanged approved content goes to stage 4; repaired content goes to stage 1/i);
|
||||
expect(docs).toContain('`Documentation: blocked`');
|
||||
expect(docs).toMatch(/never silently refresh hashes/i);
|
||||
});
|
||||
|
||||
test('tests, absent test approval and new writes reopen final verification', () => {
|
||||
const tests = gate.slice(gate.indexOf('### 4.'), gate.indexOf('### 5.'));
|
||||
expect(tests).toContain("**New, changed or unwaived test failure:** STOP publication. Run Steps 5–15, starting with Step 5's triage, then return to Step 16 stage 1");
|
||||
expect(tests).toContain('This recovery also applies if a failure appears while reporting in stage 5');
|
||||
expect(tests).toContain('run Steps 5–15, including the no-tests decision, then return to Step 16 stage 1');
|
||||
expect(tests).toContain('it does not authorize a third attempt');
|
||||
expect(gate).toContain('If content changes during or after verification, restart at stage 1 and complete all five stages before Step 17');
|
||||
expect(gate).toContain('Content-preserving commits keep valid evidence');
|
||||
expect(tests).toMatch(/\*\*New, changed or unwaived test failure:\*\* stop publication\. Run Steps 5–15/i);
|
||||
expect(tests).toMatch(/run Steps 5–15, including the no-tests decision, then return to Step 16 stage 1/i);
|
||||
expect(tests).toMatch(/does not authorize a third attempt/i);
|
||||
expect(gate).toMatch(/restart at stage 1 and complete all five stages before Step 17/i);
|
||||
});
|
||||
|
||||
test('push failure kinds and publication races have distinct resumes', () => {
|
||||
const push = compact(entry.slice(entry.indexOf('## Step 17:'), entry.indexOf('## Step 18:')));
|
||||
expect(push).toContain('**If the push fails, STOP.** No Step 19 or publication claim');
|
||||
expect(push).toContain("fetch and inspect the remote, then merge under Step 3's conflict rules");
|
||||
expect(push).toContain('Run Steps 5–16 before returning to Step 17. Never rewrite history');
|
||||
expect(push).toContain('repeat Step 16 even if content is unchanged before returning to Step 17');
|
||||
expect(push).toContain('Never bypass failed guards');
|
||||
expect(push).toMatch(/push fails[^.]*\bstop\b[\s\S]{0,20}no Step 19 or publication claim/i);
|
||||
expect(push).toMatch(/merge under Step 3's conflict rules/i);
|
||||
expect(push).toContain('Run Steps 5–16 before returning to Step 17');
|
||||
expect(push).toMatch(/never rewrite history/i);
|
||||
expect(push).toMatch(/repeat Step 16 even if content is unchanged/i);
|
||||
expect(push).toMatch(/never bypass failed guards/i);
|
||||
const publication = compact(read('ship/sections/pr-body.md.tmpl'));
|
||||
expect(publication).toContain("If the open PR/MR or title changed, repeat Step 18's identity/title preparation");
|
||||
expect(publication).toContain('then return here for a new lookup, fresh body and both redaction scans before publishing');
|
||||
expect(publication).toMatch(/repeat Step 18's identity\/title preparation/i);
|
||||
expect(publication).toMatch(/both redaction scans before publishing/i);
|
||||
});
|
||||
});
|
||||
@@ -12,79 +12,60 @@ describe('authored ship-only plan verification handoff', () => {
|
||||
const ctx = { skillName: 'ship', tmplPath: 'ship/SKILL.md.tmpl', host: 'claude' as const, paths: HOST_PATHS.claude };
|
||||
test('local execution-only checks remain required in Step 8.1/9 outside implementation counts', () => {
|
||||
const audit = generatePlanCompletionAuditShip(ctx);
|
||||
const extraction = audit.slice(audit.indexOf('### Actionable Item Extraction'), audit.indexOf('### Verification Mode'));
|
||||
expect(extraction).toContain('Separate deliverables from execution-only verification');
|
||||
expect(extraction).toContain('implementation and test-creation requirements');
|
||||
expect(extraction).toContain('retain its command, expected outcome and source verbatim');
|
||||
const extraction = audit.slice(audit.indexOf('### Actionable Item Extraction'), audit.indexOf('### Verification Mode')).replace(/\s+/g, ' ');
|
||||
expect(extraction).toMatch(/execution-only verification/i);
|
||||
expect(extraction).toContain('Step 8.1/9');
|
||||
expect(extraction).toContain('outside implementation counts');
|
||||
expect(extraction).toContain('never DONE from static inspection');
|
||||
expect(extraction).toContain('not EXTERNAL-STATE merely because it has not run');
|
||||
expect(extraction).toContain('Keep genuine external-state and human-only checks in this audit');
|
||||
expect(extraction).toContain('zero implementation counts do not waive those checks');
|
||||
expect(extraction).toMatch(/outside implementation counts/i);
|
||||
expect(extraction).toMatch(/never DONE from static inspection/);
|
||||
expect(extraction).toMatch(/do not waive those checks/i);
|
||||
expect(fs.readFileSync(path.join(SHIP_DIR, 'sections/plan-completion.md.tmpl'), 'utf8')).toContain('exactly these seven fields');
|
||||
const gate = generatePlanCompletionGateShip(ctx);
|
||||
expect(gate).toContain('Any NOT DONE items');
|
||||
expect(gate).toContain('Per-item confirmation is mandatory');
|
||||
expect(gate).toMatch(/per-item confirmation/i);
|
||||
});
|
||||
test('review-mode extraction retains its existing categories and has no ship-only routing', () => {
|
||||
const audit = generatePlanCompletionAuditReview({ ...ctx, skillName: 'review' });
|
||||
expect(audit).toContain('**Test requirements:** "Test that X", "Add test for Y", "Verify Z"');
|
||||
expect(audit).not.toContain('Separate deliverables from execution-only verification');
|
||||
expect(audit).toContain('**Test requirements:**');
|
||||
expect(audit).not.toMatch(/execution-only verification/i);
|
||||
expect(audit).not.toContain('Step 8.1/9');
|
||||
expect(audit).toContain('EXTERNAL-STATE');
|
||||
});
|
||||
test('preparation hands every explicit check to the report-only execution owner before Fix-First', () => {
|
||||
const text = generatePlanVerificationExec(ctx).replace(/\s+/g, ' ');
|
||||
expect(text).toContain('Collect now; execute in Step 9');
|
||||
expect(text).toContain('Do not invoke an entire QA skill or start probes here');
|
||||
expect(text).toMatch(/do not invoke an entire QA skill or start probes here/i);
|
||||
for (const heading of ['Verification', 'Test plan', 'Testing', 'How to test', 'Manual testing']) {
|
||||
expect(text).toContain(`\`${heading}\``);
|
||||
}
|
||||
expect(text).toContain('any other explicit checks, including execution-only items retained by Step 8');
|
||||
expect(text).toContain('each exact expected outcome, source, surface, probe and safe prerequisites');
|
||||
expect(text).toContain('Clarify unknown outcomes');
|
||||
expect(text).toContain('Browser items use the declared project/plan dev URL and browser setup at execution');
|
||||
expect(text).toContain('functional items use native tools without discovering a web server');
|
||||
expect(text).toContain('An API URL is not automatically a page');
|
||||
expect(text).toContain('Only browser evidence needs screenshots');
|
||||
expect(text).toContain('If no verification section or no plan file exists, record no plan-specific items');
|
||||
expect(text).toContain('Automatic diff-scoped QA still runs. Continue to Step 8.2 Scope Drift below');
|
||||
expect(text).toContain('Step 8.2');
|
||||
expect(text).toContain('Handoff to Step 9.2.1');
|
||||
expect(text).toContain('parent-owned report-only explorer must execute this complete list before Fix-First');
|
||||
expect(text).toContain('prerequisite, permission, evidence and changed-input revalidation rules');
|
||||
expect(text).toContain('Share current-input proof for overlapping smoke probes');
|
||||
expect(text).toContain('plan checks beyond that smoke budget remain required');
|
||||
expect(text).toContain('At command/time limits, mark remaining checks not run');
|
||||
expect(text).toContain("Send failed, blocked or unrun checks through Step 9's required-probe gate, never silently waive them");
|
||||
expect(text).toContain('Noninteractive runs return blocked');
|
||||
expect(text).toMatch(/before Fix-First/);
|
||||
expect(text).toMatch(/never silently waive/i);
|
||||
expect(text).toMatch(/noninteractive runs return blocked/i);
|
||||
expect(text).toContain('VERIFY_RESULT=pass only if all selected items pass, skipped only if none exist, otherwise fail');
|
||||
expect(text).toContain('Risk acceptance keeps the actual failed, blocked and unrun outcomes');
|
||||
expect(text).toContain("Report per-status counts, evidence and accepted risks in Step 19's `## Verification Results`, separately from automatic QA");
|
||||
expect(text).toContain('`## Verification Results`');
|
||||
expect(text.indexOf('Collect now')).toBeLessThan(text.indexOf('Handoff to Step 9.2.1'));
|
||||
expect(text.indexOf('Handoff to Step 9.2.1')).toBeLessThan(text.indexOf('After execution'));
|
||||
});
|
||||
test('the authored parent validates all seven fields and settles failed children before fallback', () => {
|
||||
const audit = fs.readFileSync(path.join(SHIP_DIR, 'sections/plan-completion.md.tmpl'), 'utf8');
|
||||
const parent = audit.slice(audit.indexOf('**Parent processing:**')).replace(/\s+/g, ' ');
|
||||
expect(parent).toContain("Check the task's terminal status");
|
||||
expect(parent).toContain('Without successful completion and valid LAST-line JSON, use the audit-failure fallback');
|
||||
expect(parent).toMatch(/valid LAST-line JSON, use the audit-failure fallback/i);
|
||||
expect(parent).toContain('exactly the seven declared fields');
|
||||
expect(parent).toContain('nonnegative integer counts whose classification sum equals `total_items`');
|
||||
expect(parent).toContain('classification sum equals `total_items`');
|
||||
expect(parent).toContain('a string `summary`');
|
||||
expect(parent).toContain('Missing, extra or invalid fields fail');
|
||||
expect(parent).toContain('Valid no-plan/no-actionable reports retain zero counts and their summary');
|
||||
expect(parent).toContain('no final output after ~10 minutes');
|
||||
expect(parent).toContain('stop any live child and confirm it stopped before an inline audit');
|
||||
expect(parent).toContain('same extraction/classification logic; never race a late result');
|
||||
expect(parent).toContain('If that also fails, AskUserQuestion');
|
||||
expect(parent).toContain('recording the reason in the PR body and Step 20 metrics');
|
||||
expect(parent).toMatch(/missing, extra or invalid fields fail/i);
|
||||
expect(parent).toMatch(/no-plan\/no-actionable reports retain zero counts/i);
|
||||
expect(parent).toContain('~10 minutes');
|
||||
expect(parent).toMatch(/stop any live child and confirm it stopped before an inline audit/i);
|
||||
expect(parent).toMatch(/never race a late result/i);
|
||||
expect(parent).toMatch(/if that also fails, AskUserQuestion/i);
|
||||
expect(parent).toContain('Stop and fix the audit (recommended/default)');
|
||||
const contract = audit.split('\n').find(line => line.startsWith('{"total_items":N,'))!;
|
||||
expect(Object.keys(JSON.parse(contract.replace(/:N([,}])/g, ':0$1'))).sort())
|
||||
.toEqual(['total_items', 'done', 'changed', 'partial', 'not_done', 'unverifiable', 'summary'].sort());
|
||||
expect(generatePlanCompletionGateShip(ctx)).toContain('Only PARTIAL items (no NOT DONE, no UNVERIFIABLE)');
|
||||
expect(generatePlanCompletionGateShip(ctx)).toContain('Continue with a note in the PR body. Not blocking');
|
||||
expect(generatePlanCompletionGateShip(ctx)).toMatch(/not blocking/i);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -101,13 +82,13 @@ function readShipUnion(): string {
|
||||
return t;
|
||||
}
|
||||
|
||||
describe('ship/SKILL.md — Plan Completion gate invariants (VAS-449 remediation)', () => {
|
||||
describe('ship/SKILL.md — Plan Completion gate invariants', () => {
|
||||
const skill = readShipUnion();
|
||||
|
||||
test('Path concreteness rule: filesystem-pathed items must be test -f checked', () => {
|
||||
expect(skill).toContain('**Path concreteness rule.**');
|
||||
expect(skill).toMatch(/concrete filesystem path/);
|
||||
expect(skill).toMatch(/MUST be classified DONE or NOT DONE based on `\[ -f/);
|
||||
expect(skill).toMatch(/classified DONE or NOT DONE based on `\[ -f/);
|
||||
});
|
||||
|
||||
test('Validator detection: project package.json validate-* scripts are auto-run', () => {
|
||||
@@ -117,8 +98,8 @@ describe('ship/SKILL.md — Plan Completion gate invariants (VAS-449 remediation
|
||||
});
|
||||
|
||||
test('Per-item UNVERIFIABLE confirmation: blanket-confirm is forbidden', () => {
|
||||
expect(skill).toContain('**Per-item confirmation is mandatory.**');
|
||||
expect(skill).toMatch(/Do NOT use a single AskUserQuestion to blanket-confirm/);
|
||||
expect(skill).toMatch(/per-item confirmation/i);
|
||||
expect(skill).toMatch(/do not use a single AskUserQuestion to blanket-confirm/i);
|
||||
});
|
||||
|
||||
test('Subagent failure: fail-closed, not silent fail-open', () => {
|
||||
@@ -131,16 +112,11 @@ describe('ship/SKILL.md — Plan Completion gate invariants (VAS-449 remediation
|
||||
|
||||
test('parent rejects audit errors and malformed counts instead of treating them as no plan', () => {
|
||||
const audit = fs.readFileSync(path.join(SHIP_DIR, 'sections/plan-completion.md'), 'utf8');
|
||||
expect(audit.indexOf('**Parent processing:**')).toBeGreaterThan(-1);
|
||||
expect(audit.indexOf('**Parent processing:**')).toBeLessThan(audit.indexOf('**Audit-failure fallback:**'));
|
||||
const parent = audit.slice(audit.indexOf('**Parent processing:**'), audit.indexOf('**Audit-failure fallback:**')).replace(/\s+/g, ' ');
|
||||
expect(parent).toContain("Check the task's terminal status");
|
||||
expect(parent).toContain('Without successful completion and valid LAST-line JSON, use the audit-failure fallback');
|
||||
expect(parent).toContain('Require exactly the seven declared fields');
|
||||
expect(parent).toContain('Missing, extra or invalid fields fail');
|
||||
expect(parent).toContain('nonnegative integer');
|
||||
expect(parent).toContain('classification sum equals `total_items`');
|
||||
expect(parent).toContain('a string `summary`');
|
||||
expect(parent).toContain('audit-failure fallback');
|
||||
expect(parent).toContain('Valid no-plan/no-actionable reports retain zero counts');
|
||||
expect(parent).toContain('seven declared fields');
|
||||
expect(parent).toMatch(/audit-failure fallback/i);
|
||||
});
|
||||
|
||||
test('successful plan audits use the declared seven-field contract without an error field', () => {
|
||||
@@ -167,7 +143,7 @@ describe('ship/SKILL.md — Plan Completion gate invariants (VAS-449 remediation
|
||||
expect(todos).toMatch(/Step 8[^\n]+P1[^\n]+plan/);
|
||||
expect(todos).toMatch(/Step 5[^\n]+P0[^\n]+deduplicate/);
|
||||
expect(todos.indexOf('Add approved deferrals')).toBeLessThan(todos.indexOf('Detect completed TODOs'));
|
||||
expect(todos.replace(/\s+/g, ' ')).toContain("If creation was declined or a write failed, warn and retain unsaved follow-ups in Step 19's PR summary");
|
||||
expect(todos.replace(/\s+/g, ' ')).toMatch(/retain unsaved follow-ups in Step 19's PR summary/i);
|
||||
});
|
||||
|
||||
test('CHANGELOG uses the normal workflow without checkpoint context or squash prerequisites', () => {
|
||||
@@ -180,23 +156,15 @@ describe('ship/SKILL.md — Plan Completion gate invariants (VAS-449 remediation
|
||||
const entry = fs.readFileSync(path.join(SHIP_DIR, 'SKILL.md'), 'utf8');
|
||||
const gate = entry.slice(entry.indexOf('## Step 16:'), entry.indexOf('## Step 17:'));
|
||||
const text = gate.replace(/\s+/g, ' ');
|
||||
expect(text).toContain('STALE/MISSING: changed content, command or age, or no proven run');
|
||||
expect(text).toContain('Only receipt storage/readback failed');
|
||||
expect(text).toContain('Independently prove unchanged final content, the same command and valid age');
|
||||
expect(text).toContain('from the successful run\'s evidence');
|
||||
expect(text).toContain('exact command, exit, timestamp and log');
|
||||
expect(text).toContain('as **ledger unavailable**');
|
||||
expect(text).toContain('STALE/MISSING');
|
||||
expect(text).toContain('| Only receipt storage/readback failed |');
|
||||
expect(text).toContain('**ledger unavailable**, never FRESH');
|
||||
const storage = text.slice(text.indexOf('| Only receipt storage/readback failed |')).split('|')[2];
|
||||
expect(storage).not.toContain('gstack-evidence run');
|
||||
expect(storage).toContain('from the successful run\'s evidence');
|
||||
expect(text).toContain("**New, changed or unwaived test failure:** STOP publication. Run Steps 5–15, starting with Step 5's triage, then return to Step 16 stage 1");
|
||||
expect(text).toContain('Reuse waivers only for the same verified pre-existing failures and approved scope');
|
||||
expect(text).toContain('This recovery also applies if a failure appears while reporting in stage 5');
|
||||
expect(text).toContain('Docs, TODO edits, new/generated tests and fixes make evidence STALE');
|
||||
expect(storage).toContain('Independently prove unchanged final content, the same command and valid age');
|
||||
expect(storage).toContain('Without that proof, use STALE/MISSING');
|
||||
expect(text).toContain('Without that proof, use STALE/MISSING');
|
||||
expect(storage).toMatch(/without that proof, use STALE\/MISSING/i);
|
||||
expect(text).toMatch(/\*\*New, changed or unwaived test failure:\*\* stop publication\. Run Steps 5–15/i);
|
||||
expect(text).toMatch(/reuse waivers only for the same verified pre-existing failures/i);
|
||||
expect(text).toMatch(/make evidence STALE/);
|
||||
});
|
||||
|
||||
test('ship contract precedes base detection and fresh remote facts precede distribution decisions', () => {
|
||||
@@ -204,7 +172,7 @@ describe('ship/SKILL.md — Plan Completion gate invariants (VAS-449 remediation
|
||||
expect(entry.indexOf('# Ship:')).toBeLessThan(entry.indexOf('## Step 0:'));
|
||||
const preflight = entry.slice(entry.indexOf('## Step 1:'), entry.indexOf('## Step 2:'));
|
||||
expect(preflight).toContain('git fetch origin <base>');
|
||||
expect(preflight).toMatch(/fetch fails[^\n]+STOP/);
|
||||
expect(preflight).toMatch(/fetch fails[^\n]+\bstop\b/i);
|
||||
expect(entry).not.toContain('auto-generate and commit, or flag');
|
||||
expect(entry).toContain('Step 15 commits those tests');
|
||||
});
|
||||
@@ -212,10 +180,9 @@ describe('ship/SKILL.md — Plan Completion gate invariants (VAS-449 remediation
|
||||
test('bisectable commits proceed directly to verification without rewriting existing history', () => {
|
||||
const entry = fs.readFileSync(path.join(SHIP_DIR, 'SKILL.md'), 'utf8');
|
||||
const commit = entry.slice(entry.indexOf('## Step 15:'), entry.indexOf('## Step 16:'));
|
||||
expect(commit).toContain('Make bisectable commits');
|
||||
expect(commit).toContain('if already committed, continue to Step 16');
|
||||
expect(commit).toMatch(/Never create an empty commit/i);
|
||||
expect(commit).toContain('Each commit must work independently');
|
||||
expect(commit).toMatch(/bisectable commits/i);
|
||||
expect(commit).toContain('continue to Step 16');
|
||||
expect(commit).toMatch(/never create an empty commit/i);
|
||||
expect(commit).not.toMatch(/checkpoint|WIP|squash|git rebase|git reset/);
|
||||
expect(entry).not.toMatch(/Step 15\.[012]/);
|
||||
});
|
||||
@@ -224,13 +191,14 @@ describe('ship/SKILL.md — Plan Completion gate invariants (VAS-449 remediation
|
||||
const entry = fs.readFileSync(path.join(SHIP_DIR, 'SKILL.md'), 'utf8');
|
||||
const push = entry.slice(entry.indexOf('## Step 17:'), entry.indexOf('## Step 20:'));
|
||||
const recovery = push.replace(/\s+/g, ' ');
|
||||
expect(push).toMatch(/push fails[^\n]+STOP/);
|
||||
expect(push).toMatch(/push fails[^\n]+\bstop\b/i);
|
||||
expect(recovery).toContain('**Non-fast-forward push:**');
|
||||
expect(recovery).toContain('Run Steps 5–16 before returning to Step 17. Never rewrite history');
|
||||
expect(recovery).toContain('Run Steps 5–16 before returning to Step 17');
|
||||
expect(recovery).toMatch(/never rewrite history/i);
|
||||
expect(recovery).toContain('**Authentication, hook or network failure:**');
|
||||
expect(recovery).toContain('repeat Step 16 even if content is unchanged before returning to Step 17');
|
||||
expect(recovery).toMatch(/repeat Step 16 even if content is unchanged/i);
|
||||
expect(push).toMatch(/never force.push/i);
|
||||
expect(push).toContain('Only a successful push');
|
||||
expect(push).toMatch(/only a successful push/i);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -8,69 +8,61 @@ const read = (file: string) => readFileSync(new URL(`../${file}`, import.meta.ur
|
||||
const compact = (text: string) => text.replace(/\s+/g, ' ');
|
||||
const ctx = { host: 'claude' as const, skillName: 'ship', tmplPath: '', paths: HOST_PATHS.claude };
|
||||
|
||||
test('ship STOP blocks advancement while retaining the stated repair route', () => {
|
||||
test('ship stop lines block advancement while retaining the stated repair route', () => {
|
||||
const entry = compact(read('ship/SKILL.md.tmpl'));
|
||||
expect(entry).toContain('STOP blocks advancement until the stated repair/resume route clears; without one, end this attempt');
|
||||
expect(entry).toContain('Answer each AskUserQuestion before continuing');
|
||||
expect(entry).toMatch(/stop blocks advancement until the stated repair\/resume route clears; without one, end this attempt/i);
|
||||
expect(entry).toMatch(/answer each AskUserQuestion before continuing/i);
|
||||
expect(entry).toMatch(/routine authorization never waives those gates or their required user decisions/i);
|
||||
const review = compact(read('ship/sections/review-army.md.tmpl'));
|
||||
expect(review).toContain('**Dispatched reviewer output missing:** STOP');
|
||||
expect(review).toContain('Retain queued fixes and restore coverage');
|
||||
expect(review).toContain('**Third fixing cycle reached (`CYCLES >= 3`):** STOP');
|
||||
expect(entry).toContain('Routine authorization never waives those gates or their required user decisions');
|
||||
expect(review).toMatch(/\*\*Dispatched reviewer output missing:\*\* stop\b/i);
|
||||
expect(review).toMatch(/\*\*Third fixing cycle reached \(`CYCLES >= 3`\):\*\* stop\b/i);
|
||||
});
|
||||
|
||||
test('a new ship bootstrap choice overrides only the saved decline, not framework selection', () => {
|
||||
const ship = generateTestBootstrap(ctx);
|
||||
const decline = ship.slice(ship.indexOf('**If BOOTSTRAP_DECLINED**'), ship.indexOf('**If NO ecosystem marker matched:**'));
|
||||
expect(compact(decline)).toContain("Step 5's explicit Add tests choice overrides that marker for this invocation only");
|
||||
expect(compact(decline)).toContain('continue to runtime detection and B2–B3, including framework approval');
|
||||
expect(compact(decline)).toMatch(/explicit Add tests choice overrides that marker for this invocation only/i);
|
||||
expect(compact(decline)).toMatch(/including framework approval/i);
|
||||
expect(decline).not.toContain('rm ');
|
||||
expect(ship.indexOf('**If ANY existing-test evidence appears**')).toBeLessThan(ship.indexOf('**If BOOTSTRAP_DECLINED**'));
|
||||
const qa = generateTestBootstrap({ ...ctx, skillName: 'qa' });
|
||||
expect(qa).toContain('**If BOOTSTRAP_DECLINED** appears: Print "Test bootstrap previously declined — skipping." **Skip the rest of bootstrap.**');
|
||||
expect(compact(qa)).toMatch(/BOOTSTRAP_DECLINED[^\n]{0,120}skip the rest of bootstrap/i);
|
||||
expect(qa).not.toContain("Step 5's explicit Add tests choice");
|
||||
});
|
||||
|
||||
test('an unverified item answered not done enters the existing decision before continuing', () => {
|
||||
const gate = generatePlanCompletionGateShip(ctx);
|
||||
const exits = compact(gate.slice(gate.indexOf('**Exit conditions:**'), gate.indexOf('**Cap.**')));
|
||||
expect(exits).toContain('Any N: STOP and report that item as NOT DONE');
|
||||
expect(exits).toContain('Resume only after its required work is verified');
|
||||
expect(exits).toContain('no second deferral choice');
|
||||
expect(exits).toMatch(/any N: stop and report that item as NOT DONE/i);
|
||||
expect(exits).toMatch(/resume only after its required work is verified/i);
|
||||
expect(exits).toMatch(/no second deferral choice/i);
|
||||
expect(exits).not.toContain('re-running /ship');
|
||||
expect(gate).toContain('Per-item confirmation is mandatory');
|
||||
expect(gate).toContain('with the user\'s free-text evidence');
|
||||
expect(gate).toMatch(/per-item confirmation/i);
|
||||
});
|
||||
|
||||
test('docs reentry distinguishes the initial audit from same-invocation accepted evidence', () => {
|
||||
const docs = compact(read('ship/sections/documentation.md.tmpl'));
|
||||
const entry = docs.slice(0, docs.indexOf('## Prepare the candidate'));
|
||||
expect(entry).toContain('First entry always launches the initial audit');
|
||||
expect(entry).toContain('On reentry, reuse only this invocation\'s validated audit or named-risk decision');
|
||||
expect(entry).toContain('Reentry never resets the count or authorizes a launch');
|
||||
expect(entry).toContain("this invocation's validated audit or named-risk decision");
|
||||
expect(entry).toContain('base/input hashes still match');
|
||||
expect(entry).toContain('retain its actual status and scope');
|
||||
expect(entry).toContain('Otherwise use Blocked recovery, not an unconditional launch');
|
||||
expect(docs).toContain('never a third attempt, even after Step 16 changes');
|
||||
expect(docs).toContain('never reuse an audit across invocations');
|
||||
expect(docs).toContain('Unconfirmed writers, ownership violations, unauthorized Git mutation and redaction/security gates cannot be waived');
|
||||
expect(entry).toMatch(/first entry always launches the initial audit/i);
|
||||
expect(entry).toMatch(/reentry never resets the count or authorizes a launch/i);
|
||||
expect(docs).toMatch(/never a third attempt/i);
|
||||
expect(docs).toMatch(/never reuse an audit across invocations/i);
|
||||
expect(docs).toMatch(/unauthorized Git mutation and redaction\/security gates cannot be waived/i);
|
||||
});
|
||||
|
||||
test('late behavioral repairs rebuild before the docs decision and commit only remaining changes', () => {
|
||||
const entry = compact(read('ship/SKILL.md.tmpl'));
|
||||
expect(entry).toContain('A range ending at Step 14 does not enter Step 14.5');
|
||||
expect(entry).toContain('rebuild and compare again before stage 3 decides documentation freshness');
|
||||
expect(entry).toMatch(/rebuild and compare again before stage 3/i);
|
||||
const commit = entry.slice(entry.indexOf('### 5. Report, then push'), entry.indexOf('## Step 17:'));
|
||||
expect(commit).toContain('left uncommitted after Step 15');
|
||||
expect(commit).toContain('never create an empty commit');
|
||||
expect(commit).toContain('Preserve unrelated user files');
|
||||
expect(commit).toMatch(/never create an empty commit/i);
|
||||
expect(commit).toMatch(/preserve unrelated user files/i);
|
||||
});
|
||||
|
||||
test('the title is prefixed once before the exact scanned value is published', () => {
|
||||
const body = compact(read('ship/sections/pr-body.md.tmpl'));
|
||||
expect(body).toContain("Use Step 18's `NEW_TITLE` unchanged; its version prefix is already present");
|
||||
expect(body).toContain("Step 18's `NEW_TITLE` unchanged");
|
||||
expect(body).not.toContain('`NEW_TITLE`, prefixed with');
|
||||
expect(body).toContain('--title "$NEW_TITLE"');
|
||||
expect(body).toContain('the same scanned `NEW_TITLE`');
|
||||
expect(body).toContain('scanned `NEW_TITLE`');
|
||||
});
|
||||
+255
-722
File diff suppressed because it is too large.
Load diff
@@ -1487,7 +1487,7 @@ describe('Codex skill', () => {
|
||||
expect(content).toContain('Step 2C: Consult Mode');
|
||||
const skeleton = fs.readFileSync(path.join(ROOT, 'codex', 'SKILL.md'), 'utf-8');
|
||||
expect(skeleton).toContain('## Step 1: Detect mode');
|
||||
expect(skeleton).toContain('MUTUALLY EXCLUSIVE');
|
||||
expect(skeleton).toMatch(/mutually exclusive/i);
|
||||
});
|
||||
|
||||
test('codex union contains gate verdict logic', () => {
|
||||
@@ -1587,7 +1587,7 @@ describe('Codex skill', () => {
|
||||
expect(content).toContain('CODEX_MODE');
|
||||
expect(content).toContain('command -v codex'); // install check kept literal
|
||||
// codex_reviews=disabled gates Codex passes only; Claude adversarial still runs
|
||||
expect(content).toContain('skip the Codex passes ONLY');
|
||||
expect(content).toMatch(/disabled[^\n]*skip the Codex passes only/i);
|
||||
// Review log
|
||||
expect(content).toContain('adversarial-review');
|
||||
expect(content).toContain('reasoning_effort="high"');
|
||||
@@ -2023,7 +2023,7 @@ describe('Test failure triage in ship skill', () => {
|
||||
|
||||
test('ship/SKILL.md uses in-branch language for stop condition', () => {
|
||||
const content = readShipUnion();
|
||||
expect(content).toContain('If any in-branch failures remain unfixed, **STOP**. Do not proceed');
|
||||
expect(content).toMatch(/in-branch failures remain unfixed[\s\S]{0,40}\bstop\b[\s\S]{0,20}do not proceed/i);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -14,42 +14,36 @@ function expectOutsideReviewControlFlow(text: string, promptHeading: string): vo
|
||||
const disabled = text.slice(indices[0], indices[1]);
|
||||
if (ceo) {
|
||||
expect(disabled).toContain('"outside_status":"disabled"');
|
||||
expect(disabled.replace(/\s+/g, ' ')).toContain('without a challenge, CLI invocation, Agent/Task fallback or questions about outside findings');
|
||||
expect(disabled.replace(/\s+/g, ' ')).toMatch(/without a challenge, CLI invocation, Agent\/Task fallback/i);
|
||||
expect(disabled).toContain('_DISABLED_REVIEW_MODE=');
|
||||
expect(disabled).toContain('if [ "$_DISABLED_REVIEW_MODE" = disabled ]');
|
||||
} else {
|
||||
expect(disabled).toContain('persist `outside_status: disabled`');
|
||||
expect(disabled.replace(/\s+/g, ' ')).toMatch(/Do not construct a (?:review prompt|challenge), invoke an outside CLI, dispatch an Agent\/Task fallback/);
|
||||
expect(disabled.replace(/\s+/g, ' ')).toMatch(/do not construct a (?:review prompt|challenge), invoke an outside CLI, dispatch an Agent\/Task fallback/i);
|
||||
}
|
||||
expect(text.slice(indices[1], indices[2])).toContain('(skip only on `disabled`)');
|
||||
|
||||
const fallback = text.slice(indices[3]);
|
||||
if (text.includes('**Outcome routing:**')) {
|
||||
const routing = text.slice(text.indexOf('**Outcome routing:**'), indices[0]);
|
||||
expect(routing).toContain('Other preflight mode, including harness mismatch');
|
||||
expect(routing).toContain('Outside execution or output validation fails');
|
||||
expect(routing).toContain('Retain its output and diagnosis, finish termination, then use Native fallback');
|
||||
expect(routing).toContain('No prompt, outside process or native replacement');
|
||||
expect(fallback.replace(/\s+/g, ' ')).toContain('Immediately before dispatch, check the preflight result again: disabled means no replacement');
|
||||
expect(fallback.replace(/\s+/g, ' ')).toMatch(/immediately before dispatch, check the preflight result again: disabled means no replacement/i);
|
||||
} else if (ceo) {
|
||||
expect(fallback.replace(/\s+/g, ' ')).toContain('Other preflight failures retain their printed diagnosis, including harness mismatch');
|
||||
expect(fallback.replace(/\s+/g, ' ')).toContain('These failures do not block the review; they use the bounded fallback below');
|
||||
expect(fallback.replace(/\s+/g, ' ')).toMatch(/do not block the review; they use the bounded fallback/i);
|
||||
} else {
|
||||
expect(fallback).toContain('The disabled branch never reaches this fallback.');
|
||||
expect(fallback.replace(/\s+/g, ' ')).toMatch(/Otherwise, use this fallback for missing\/broken CLI, failed authentication\/model selection, a failed preflight(?: \(including harness mismatch\))?, or a failed outside invocation\./);
|
||||
expect(fallback).toMatch(/the disabled branch never reaches this fallback/i);
|
||||
expect(fallback.replace(/\s+/g, ' ')).toMatch(/use this fallback for missing\/broken CLI/i);
|
||||
}
|
||||
const dispatch = fallback.indexOf('Dispatch via the Agent tool');
|
||||
expect(dispatch).toBeGreaterThan(0);
|
||||
const recheck = fallback.slice(0, dispatch);
|
||||
const recheck = fallback.slice(0, dispatch).replace(/\s+/g, ' ');
|
||||
if (ceo) {
|
||||
expect(recheck.replace(/\s+/g, ' ')).toContain('Immediately before dispatch, recheck the preflight result');
|
||||
expect(recheck.replace(/\s+/g, ' ')).toContain('`CODEX_MODE: disabled`, return to **Record the disabled outcome** without dispatching');
|
||||
expect(recheck).toMatch(/immediately before dispatch, recheck the preflight result/i);
|
||||
expect(recheck).toMatch(/`CODEX_MODE: disabled`, return to \*\*Record the disabled outcome\*\* without dispatching/i);
|
||||
} else if (!text.includes('**Outcome routing:**')) {
|
||||
expect(recheck).toContain('Immediately before dispatching, check the preflight result again.');
|
||||
expect(recheck).toMatch(/immediately before dispatching, check the preflight result again/i);
|
||||
expect(recheck).toContain('`CODEX_MODE: disabled`, finish this section with `outside_status: disabled`;');
|
||||
expect(recheck).toContain('do not dispatch.');
|
||||
expect(recheck).toMatch(/do not dispatch/i);
|
||||
}
|
||||
expect(fallback).toContain('Availability/native fallback is not outside completion.');
|
||||
expect(fallback).toMatch(/availability\/native fallback is not outside completion/i);
|
||||
}
|
||||
|
||||
describe('workflow judge excerpts', () => {
|
||||
@@ -63,15 +57,13 @@ describe('workflow judge excerpts', () => {
|
||||
|
||||
test('ship uses project-native commands and never jumps over mandatory gates', () => {
|
||||
const text = readWorkflowExcerpt('ship/SKILL.md', '# Ship:', '## Important Rules');
|
||||
expect(text).toContain("Use the project's test commands discovered in Step 4");
|
||||
expect(text).toMatch(/test commands discovered in Step 4/i);
|
||||
expect(text).toContain('**Project-native path:**');
|
||||
expect(text).toContain('Use the documented selector and pre-merge command.');
|
||||
expect(text).not.toMatch(/skipping evals[^\n]*Step 9/);
|
||||
const reviewAndTriage = text.slice(text.indexOf('## Step 9:'), text.indexOf('## Step 11:'));
|
||||
expect(reviewAndTriage.match(/continue to Step 12/i)).toBeNull();
|
||||
expect(text).not.toContain('Steps 4-6:');
|
||||
expect(text).toContain('During pre-flight, read the existing review log');
|
||||
expect(text).toContain('Save the JSON `baseVersion` as `BASE_VERSION`');
|
||||
expect(text).toContain('`baseVersion` as `BASE_VERSION`');
|
||||
expect(text).not.toContain('GIT_SEQUENCE_EDITOR');
|
||||
expect(text).not.toContain("--exec 'true'");
|
||||
expect(text).not.toContain('-X ours');
|
||||
@@ -80,27 +72,28 @@ describe('workflow judge excerpts', () => {
|
||||
|
||||
test('ship review shortcuts retain dedup and fixes repeat the whole review cycle', () => {
|
||||
const text = readWorkflowExcerpt('ship/SKILL.md', '# Ship:', '## Important Rules');
|
||||
expect(text).toContain("Continue to Step 9.2 with the core/design-lite findings and an empty specialist list, then the parent's Exploratory QA step and Step 9.3 (cross-review dedup)");
|
||||
const flat = text.replace(/\s+/g, ' ');
|
||||
expect(flat).toMatch(/Continue to Step 9\.2 [^.]*then the parent's Exploratory QA step and Step 9\.3/i);
|
||||
expect(text).toContain('## Step 9.4: Fix-First and persistence');
|
||||
expect(text.replace(/\s+/g, ' ')).toContain('Run checklist/design, specialists (9.1), merge/Red Team (9.2), exploratory QA (9.2.1), dedup (9.3), then fixes and logging (9.4)');
|
||||
expect(text.replace(/\s+/g, ' ')).toContain('**Fixes applied below the cap:** Insert Step 5, affected Steps 6–8 and all of Step 9 before the pending Step 10 in the work list. Tests must pass or retain approval for the same verified pre-existing failures and scope');
|
||||
const cycle = flat.slice(flat.indexOf('specialists (9.1)'));
|
||||
const order = ['(9.1)', '(9.2)', '(9.2.1)', '(9.3)', '(9.4)'].map(step => cycle.indexOf(step));
|
||||
expect(order.every(index => index >= 0)).toBe(true);
|
||||
expect(order).toEqual([...order].sort((a, b) => a - b));
|
||||
expect(flat).toContain('**Fixes applied below the cap:** Insert Step 5, affected Steps 6–8 and all of Step 9 before the pending Step 10');
|
||||
const audit = text.slice(text.indexOf('## Step 7:'), text.indexOf('## Step 8:'));
|
||||
expect(audit).not.toContain('Scope Challenge');
|
||||
expect(text.replace(/\s+/g, ' ')).toContain('Keep actual outcomes and incomplete flags; VERIFY_RESULT stays fail for plan-check exceptions');
|
||||
expect(flat).toContain('VERIFY_RESULT stays fail');
|
||||
});
|
||||
|
||||
test('ship excerpt preserves readable detours, audit fallback and final input decisions', () => {
|
||||
const text = readWorkflowExcerpt('ship/SKILL.md', '# Ship:', '## Important Rules').replace(/\s+/g, ' ');
|
||||
expect(text).toContain('For another repair, repeat rule 2 without discarding pending work');
|
||||
expect(text).toContain('A further Step 9 fix affecting 6–8 makes the list `5 → 6 → 7 → 8 → 9 → 10 → 11 → 11.5`');
|
||||
expect(text).toContain('The unchanged release steps follow');
|
||||
expect(text).toContain('All three snapshots must match');
|
||||
expect(text).toContain('does not mean the failed or unrun probes passed');
|
||||
expect(text).toContain('Fallback recovers the audit; it does not pass or bypass the coverage gate');
|
||||
expect(text).toContain('Skip only the plan completion audit');
|
||||
expect(text).toContain('Continue with Step 8.1, Scope Drift and Prior Learnings');
|
||||
expect(text).toContain('Step 9 QA still runs');
|
||||
expect(text).toContain('Use this example only after confirming that every allowed edit is release metadata');
|
||||
expect(text).toContain('`5 → 6 → 7 → 8 → 9 → 10 → 11 → 11.5`');
|
||||
expect(text).toMatch(/all three snapshots must match/i);
|
||||
expect(text).toMatch(/does not mean the failed or unrun probes passed/i);
|
||||
expect(text).toMatch(/does not pass or bypass the coverage gate/i);
|
||||
expect(text).toMatch(/skip only the plan completion audit/i);
|
||||
expect(text).toContain('Step 8.1');
|
||||
expect(text).toMatch(/Step 9 QA still runs/i);
|
||||
expect(text).not.toContain('Every listed change below is metadata:');
|
||||
expect(text).not.toContain('No plan file found:** Skip entirely');
|
||||
});
|
||||
@@ -123,15 +116,16 @@ describe('workflow judge excerpts', () => {
|
||||
expect(edit).toBeGreaterThan(scan);
|
||||
expect(publish.indexOf('Print the existing URL')).toBeGreaterThan(edit);
|
||||
expect(text).not.toContain('Phase 8e.5');
|
||||
expect(text).toContain('never create an empty commit');
|
||||
expect(text).toMatch(/never create an empty commit/i);
|
||||
const review = text.slice(text.indexOf('## Step 9:'), text.indexOf('## Step 10:'));
|
||||
expect(review.indexOf('## Confidence Calibration')).toBeLessThan(review.indexOf('1. Read'));
|
||||
const flat = review.replace(/\s+/g, ' ');
|
||||
expect(flat).toContain('**No edits in this pass:** Resolve the required-probe gate below. Only after it clears may you continue to Step 10');
|
||||
expect(flat).toContain('**Dispatched reviewer output missing:** STOP');
|
||||
expect(flat).toContain('Retain queued fixes and restore coverage');
|
||||
expect(flat).toContain('**Third fixing cycle reached (`CYCLES >= 3`):** STOP and report recurring findings with `converged:false`; do not run a fourth fixing cycle');
|
||||
expect(flat).toContain('With completed checklist and dispatched reviewers, failed/unavailable required probes block continuation');
|
||||
expect(flat).toContain('**No edits in this pass:**');
|
||||
expect(flat).toMatch(/only after it clears may you continue to Step 10/i);
|
||||
expect(flat).toMatch(/\*\*Dispatched reviewer output missing:\*\* stop\b/i);
|
||||
expect(flat).toMatch(/\*\*Third fixing cycle reached \(`CYCLES >= 3`\):\*\* stop\b[^.]*`converged:false`/i);
|
||||
expect(flat).toMatch(/do not run a fourth fixing cycle/i);
|
||||
expect(flat).toMatch(/failed\/unavailable required probes block continuation/i);
|
||||
});
|
||||
|
||||
test('ship approval gates stay outside the subagent prompts', () => {
|
||||
@@ -145,7 +139,7 @@ describe('workflow judge excerpts', () => {
|
||||
expect(section.indexOf(gate)).toBeGreaterThan(section.indexOf('\n````\n'));
|
||||
}
|
||||
expect(text).toContain('"partial":N,"not_done":N');
|
||||
expect(text).toContain('each Y\'d item with the user\'s free-text evidence and each D\'d item with "intentionally dropped"');
|
||||
expect(text).toContain('"intentionally dropped"');
|
||||
});
|
||||
|
||||
test('expands a body before the end marker in the skeleton', () => {
|
||||
@@ -163,13 +157,13 @@ describe('workflow judge excerpts', () => {
|
||||
expect(commit).toBeGreaterThan(reviewStart);
|
||||
const review = text.slice(reviewStart, commit);
|
||||
expectOutsideReviewControlFlow(review, '**Construct the doc-review prompt**');
|
||||
expect(review).toContain('Skip the apply gate, persist `status: unavailable`, `outside_status: unavailable`, and `source: none`');
|
||||
expect(review).toContain('present the findings, then use AskUserQuestion ONCE:');
|
||||
expect(review).toContain('On A or per-finding approvals, make the approved edits yourself');
|
||||
expect(text).toContain('Step 9 then commits and pushes those edits');
|
||||
for (const token of ['`status: unavailable`', '`outside_status: unavailable`', '`source: none`']) expect(review).toContain(token);
|
||||
expect(review).toMatch(/AskUserQuestion once/i);
|
||||
expect(review).toMatch(/on A or per-finding approvals/i);
|
||||
expect(text).toContain('Step 9 then commits');
|
||||
expect(text).toMatch(/needs?\s+attention,?\s+not\s+replacement/i);
|
||||
expect(text).not.toContain('Flag and rewrite');
|
||||
expect(text).toContain('if VERSION is absent, use the completion date only');
|
||||
expect(text).toMatch(/if VERSION is absent, use the completion date only/i);
|
||||
});
|
||||
|
||||
test('Eng preparation and decision procedure precede the four review sections', async () => {
|
||||
@@ -261,8 +255,8 @@ console.log(JSON.stringify({calls, results}));
|
||||
expect(call.userPrompt).toContain('Proceed to Mode Selection,');
|
||||
expect(call.userPrompt).not.toMatch(/Step 0[A-Z]/);
|
||||
expect(call.userPrompt).toContain(join(call.workingDirectory, 'plan-ceo-review', 'SKILL.md'));
|
||||
expect(call.userPrompt).toContain('Do NOT search for, Glob, find, or read any OTHER SKILL.md');
|
||||
expect(call.userPrompt).toContain('Ask the user through the AskUserQuestion tool and wait for their answer.');
|
||||
expect(call.userPrompt).toMatch(/read any other SKILL\.md/i);
|
||||
expect(call.userPrompt).toMatch(/AskUserQuestion tool and wait for their answer/i);
|
||||
expect(call).toMatchObject({ allowedTools: ['Read', 'Write', 'AskUserQuestion'], maxTurns: 12, maxRetries: 0, model: 'fake-model' });
|
||||
}
|
||||
} finally { rmSync(dir, { recursive: true, force: true }); }
|
||||
@@ -273,28 +267,27 @@ console.log(JSON.stringify({calls, results}));
|
||||
const tests = eng.slice(eng.indexOf('### 3. Test review'), eng.indexOf('### 4. Performance review'));
|
||||
const scope = tests.indexOf('### LLM/eval scope');
|
||||
const decisions = tests.indexOf('**Step 5. Add missing tests to the plan:**');
|
||||
const stop = tests.indexOf("**STOP for each pending decision.**", decisions);
|
||||
const stop = decisions + tests.slice(decisions).search(/stop for each pending decision/i);
|
||||
const artifact = tests.indexOf('### Test Plan Artifact');
|
||||
expect(0 <= scope && scope < decisions && decisions < stop && stop < artifact).toBe(true);
|
||||
expect(tests.match(/For LLM\/prompt changes:/g)).toHaveLength(1);
|
||||
expect(tests.slice(artifact)).not.toContain("**STOP for each pending decision.**");
|
||||
expect(tests.slice(artifact)).not.toMatch(/stop for each pending decision/i);
|
||||
const fastPath = tests.slice(tests.indexOf('**Fast path:**'), scope);
|
||||
expect(fastPath).toContain('Still check LLM/eval scope and produce the Test Plan Artifact');
|
||||
expect(fastPath).toMatch(/still check LLM\/eval scope/i);
|
||||
});
|
||||
|
||||
test('plan review evidence and design approval rules precede their use', () => {
|
||||
const eng = readWorkflowExcerpt('plan-eng-review/SKILL.md', '## Scope gate', '## Section self-check');
|
||||
expect(eng.indexOf('## Confidence Calibration')).toBeLessThan(eng.indexOf('### 1. Architecture review'));
|
||||
expect(eng).toContain('quote the motivating plan requirement');
|
||||
expectOutsideReviewControlFlow(eng, '**Construct the plan review prompt**');
|
||||
expect(eng).toContain('Agreement between reviewers is evidence, not approval');
|
||||
expect(eng).toContain('new or reopened choices still need their own answers');
|
||||
expect(eng).toMatch(/evidence, not approval/i);
|
||||
expect(eng).toMatch(/new or reopened choices still need their own answers/i);
|
||||
const pendingDecision = eng.slice(eng.indexOf('### Send once and wait'), eng.indexOf("### Record the answer"));
|
||||
expect(pendingDecision).toContain("**STOP until the actual answer arrives.**");
|
||||
expect(pendingDecision.replace(/\s+/g, ' ')).toContain("Do not apply a remedy, make another call, start the next section or call ExitPlanMode while the choice awaits an answer");
|
||||
expect(eng.replace(/\s+/g, ' ')).toContain("Use a scoped Edit to save this record and only the authorized working-plan amendments. Leave other choices unchanged");
|
||||
expect(pendingDecision).toMatch(/stop until the actual answer arrives/i);
|
||||
expect(pendingDecision.replace(/\s+/g, ' ')).toMatch(/do not apply a remedy, make another call, start the next section or call ExitPlanMode while the choice awaits an answer/i);
|
||||
expect(eng.replace(/\s+/g, ' ')).toMatch(/only the authorized working-plan amendments/i);
|
||||
const design = readWorkflowExcerpt('plan-design-review/SKILL.md', '## Review Sections', ASK_QUESTIONS_HEADING);
|
||||
expect(design).toContain('wait for approval, then edit the plan and re-rate');
|
||||
expect(design).toMatch(/wait for approval, then edit the plan/i);
|
||||
const pass4 = design.slice(design.indexOf('### Pass 4:'), design.indexOf('### Pass 5:'));
|
||||
expect(pass4.match(/^### /gm)).toHaveLength(1);
|
||||
expect(pass4).toMatch(/^#### Design Hard Rules$/m);
|
||||
@@ -311,13 +304,9 @@ console.log(JSON.stringify({calls, results}));
|
||||
test('retro judge includes compare semantics and unambiguous report inputs', () => {
|
||||
const text = readWorkflowExcerpt('retro/SKILL.md', '## Instructions', '## Tone');
|
||||
expect(text).toContain('## Compare Mode');
|
||||
expect(text).toContain('does not require saved history');
|
||||
expect(text).toContain('one second before the current start');
|
||||
expect(text).toContain('PRs referenced');
|
||||
expect(text).toMatch(/one second before the current start/i);
|
||||
expect(text).toContain('prs_merged: null');
|
||||
expect(text).toContain('not newly added test cases');
|
||||
expect(text).toContain('`streak_days` is the live **team** streak');
|
||||
expect(text).toContain('draft the tweetable summary using the format in Step 14, then save');
|
||||
expect(text).toContain('`streak_days`');
|
||||
expect(text).toContain('### Shipping Streaks');
|
||||
expect(text).toContain('### Shortcut Debt');
|
||||
expect(text.indexOf('## Capture Learnings')).toBeGreaterThan(text.indexOf('### Step 14:'));
|
||||
@@ -343,9 +332,8 @@ console.log(JSON.stringify({calls, results}));
|
||||
test('ship commits logical chunks without rewriting existing checkpoint commits', () => {
|
||||
const text = readWorkflowExcerpt('ship/SKILL.md', '# Ship:', '## Important Rules');
|
||||
const commit = text.slice(text.indexOf('## Step 15:'), text.indexOf('## Step 16:'));
|
||||
expect(commit).toContain('Make bisectable commits');
|
||||
expect(commit).toContain('if already committed, continue to Step 16');
|
||||
expect(commit).toContain('Each commit must work independently');
|
||||
expect(commit).toMatch(/bisectable commits/i);
|
||||
expect(commit).toContain('continue to Step 16');
|
||||
expect(commit).not.toMatch(/rebase|reset|squash|fixup|WIP_TODO|gstack-context/);
|
||||
expect(text).not.toContain('Step 15.0');
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user