From 77cce3bec4752beb1696d57c1f7eba9aef490589 Mon Sep 17 00:00:00 2001 From: garrytan Date: Wed, 30 Sep 2026 12:11:34 +0000 Subject: [PATCH] fix(ship,qa,document-release): repair proof-run regressions and fixture gaps - ship-docsync-completion: yesterday's audit-scope result dropped the section's status, so /ship spliced one in; the section now opens with **Status:**. - ship-docsync-missing-asset: a missing section or old Ship-owned mode blocks before launch. - ship-docsync-late-result: the invocation record says prepare already saves the candidate selection (no extra Read; budget unchanged). - qa exploratory: await the method Reads before the first probe. - qa-callers fixture: quote the real review-log record template; allow the git log command plan-completion prescribes. - qa functional observer: a receipt caught mid-link(2) is checked at stop instead of failing with ENOENT (reproduced from CI). Each repaired case passed a focused paid run. --- document-release/sections/audit-scope.md | 7 ++-- document-release/sections/audit-scope.md.tmpl | 7 ++-- qa/sections/exploratory.md | 2 +- scripts/resolvers/qa.ts | 2 +- ship/sections/documentation.md | 4 +- ship/sections/documentation.md.tmpl | 4 +- test/docsync-fault-interface.test.ts | 3 ++ test/fixtures/golden/codex-ship-SKILL.md | 4 +- test/fixtures/golden/factory-ship-SKILL.md | 4 +- test/helpers/docsync-fault-actor.ts | 2 +- test/helpers/qa-callers-fixture.ts | 11 ++++- test/helpers/qa-functional-observer.ts | 20 +++++++++ test/qa-exploratory-callers.test.ts | 41 ++++++++++++++++++- test/qa-functional-observer.test.ts | 27 ++++++++++++ test/qa-probe-gates.test.ts | 3 ++ test/ship-document-release-dispatch.test.ts | 16 ++++++++ 16 files changed, 136 insertions(+), 21 deletions(-) diff --git a/document-release/sections/audit-scope.md b/document-release/sections/audit-scope.md index 7760b5c7c..18da0168c 100644 --- a/document-release/sections/audit-scope.md +++ b/document-release/sections/audit-scope.md @@ -32,9 +32,10 @@ on the LAST nonempty line, without fences or trailing prose: - `files_updated`, `files_reviewed`: unique repo-relative file paths actually edited and actually read; `blockers`, `decisions`: strings. Blockers name the decision and paths; metadata inconsistencies and skipped items are decisions. -- `documentation_section`: nonempty Markdown without a `## Documentation` heading: - audited scope, per-file status in Step 9's `Documentation health` form (no VERSION - row), and Step 1.5's coverage debt and diagram drift. Describe scope even without docs. +- `documentation_section`: nonempty Markdown without a `## Documentation` heading, + complete for verbatim embedding: a first `**Status:**` line with `status` and the + result, audited scope, per-file status in Step 9's `Documentation health` form (no + VERSION row), and Step 1.5's coverage debt and diagram drift. Describe scope even without docs. ## Discovery (both modes) diff --git a/document-release/sections/audit-scope.md.tmpl b/document-release/sections/audit-scope.md.tmpl index f9c865310..fb58ff26d 100644 --- a/document-release/sections/audit-scope.md.tmpl +++ b/document-release/sections/audit-scope.md.tmpl @@ -30,9 +30,10 @@ on the LAST nonempty line, without fences or trailing prose: - `files_updated`, `files_reviewed`: unique repo-relative file paths actually edited and actually read; `blockers`, `decisions`: strings. Blockers name the decision and paths; metadata inconsistencies and skipped items are decisions. -- `documentation_section`: nonempty Markdown without a `## Documentation` heading: - audited scope, per-file status in Step 9's `Documentation health` form (no VERSION - row), and Step 1.5's coverage debt and diagram drift. Describe scope even without docs. +- `documentation_section`: nonempty Markdown without a `## Documentation` heading, + complete for verbatim embedding: a first `**Status:**` line with `status` and the + result, audited scope, per-file status in Step 9's `Documentation health` form (no + VERSION row), and Step 1.5's coverage debt and diagram drift. Describe scope even without docs. ## Discovery (both modes) diff --git a/qa/sections/exploratory.md b/qa/sections/exploratory.md index 961713199..c769f8ebc 100644 --- a/qa/sections/exploratory.md +++ b/qa/sections/exploratory.md @@ -5,7 +5,7 @@ The **caller** (/qa, /qa-only, /review or /ship) owns decisions, tests, fixes and publication. Discovery writes only reports/evidence and owned fixture state; no workflows, framework installs or publication. -Complete these Reads in order before writing charters or probing. Do not repeat a Read already completed in this invocation. +Complete these Reads in order before writing charters or probing. Await their results before the first probe, never in the same response. Do not repeat a Read already completed in this invocation. 1. Read `sections/scope.md` relative to the installed `qa`/`gstack-qa` SKILL.md directory in full and select the surfaces. 2. Read the selected surface methods below in full. diff --git a/scripts/resolvers/qa.ts b/scripts/resolvers/qa.ts index 55841180e..ff8816c3c 100644 --- a/scripts/resolvers/qa.ts +++ b/scripts/resolvers/qa.ts @@ -52,7 +52,7 @@ and owned fixture state; no workflows, framework installs or publication. ${reportOnly ? `## 0. Preparation gate -Complete these Reads in order before writing charters or probing:` : 'Complete these Reads in order before writing charters or probing. Do not repeat a Read already completed in this invocation.'} +Complete these Reads in order before writing charters or probing:` : 'Complete these Reads in order before writing charters or probing. Await their results before the first probe, never in the same response. Do not repeat a Read already completed in this invocation.'} 1. Read ${sectionPath(ctx, 'qa', 'scope')} in full and select the surfaces. 2. Read the selected surface methods below in full. diff --git a/ship/sections/documentation.md b/ship/sections/documentation.md index 24c05b016..a67e8e7f4 100644 --- a/ship/sections/documentation.md +++ b/ship/sections/documentation.md @@ -19,8 +19,8 @@ Reentry never resets the count or authorizes a launch. ## Prepare the candidate 1. Read installed document-release SKILL.md and its full audit-scope/release-body - content, linked as sections or inlined for external hosts. Missing/old - `Ship-owned documentation mode` blocks; never substitute. + content, linked as sections or inlined for external hosts. A missing section + or old `Ship-owned documentation mode` blocks before launch; never substitute. 2. Select release paths and base SHA. Inspect committed changes (`git diff HEAD`), staged (`git diff --cached`), unstaged (`git diff`) and selected new files (`git ls-files --others --exclude-standard`; read contents). Store-only audits diff --git a/ship/sections/documentation.md.tmpl b/ship/sections/documentation.md.tmpl index 745d6fe18..70697554c 100644 --- a/ship/sections/documentation.md.tmpl +++ b/ship/sections/documentation.md.tmpl @@ -17,8 +17,8 @@ Reentry never resets the count or authorizes a launch. ## Prepare the candidate 1. Read installed document-release SKILL.md and its full audit-scope/release-body - content, linked as sections or inlined for external hosts. Missing/old - `Ship-owned documentation mode` blocks; never substitute. + content, linked as sections or inlined for external hosts. A missing section + or old `Ship-owned documentation mode` blocks before launch; never substitute. 2. Select release paths and base SHA. Inspect committed changes (`git diff HEAD`), staged (`git diff --cached`), unstaged (`git diff`) and selected new files (`git ls-files --others --exclude-standard`; read contents). Store-only audits diff --git a/test/docsync-fault-interface.test.ts b/test/docsync-fault-interface.test.ts index a9e35d00c..62c43720b 100644 --- a/test/docsync-fault-interface.test.ts +++ b/test/docsync-fault-interface.test.ts @@ -19,6 +19,9 @@ test('prepare copies the exact generated prompt and snapshots actual inputs with const prepared = JSON.parse(response.text); const candidate = JSON.parse(fs.readFileSync(prepared.candidate, 'utf8')); expect(candidate).toEqual(docsCandidate(fixture.repo, 'first', 'edit', candidate.base_sha)); + const supplied = JSON.parse(fs.readFileSync(path.join(fixture.home, 'candidate.json'), 'utf8')); + expect(candidate.selected_paths).toEqual(supplied.selected_paths); + expect(fs.readFileSync(fixture.invocation, 'utf8')).toContain('prepare saves the same selection with current hashes, so it needs no separate Read.'); const source = extractDocsDispatch(fs.readFileSync(path.join(fixture.skills, 'ship/sections/documentation.md'), 'utf8')); expect(fs.readFileSync(prepared.prompt, 'utf8')).toBe(source.replaceAll('${HOME}', fixture.home) .replaceAll('', 'feature/docs').replaceAll('', 'main') diff --git a/test/fixtures/golden/codex-ship-SKILL.md b/test/fixtures/golden/codex-ship-SKILL.md index fdfb983ff..69d65ce74 100644 --- a/test/fixtures/golden/codex-ship-SKILL.md +++ b/test/fixtures/golden/codex-ship-SKILL.md @@ -2903,8 +2903,8 @@ Reentry never resets the count or authorizes a launch. ## Prepare the candidate 1. Read installed document-release SKILL.md and its full audit-scope/release-body - content, linked as sections or inlined for external hosts. Missing/old - `Ship-owned documentation mode` blocks; never substitute. + content, linked as sections or inlined for external hosts. A missing section + or old `Ship-owned documentation mode` blocks before launch; never substitute. 2. Select release paths and base SHA. Inspect committed changes (`git diff HEAD`), staged (`git diff --cached`), unstaged (`git diff`) and selected new files (`git ls-files --others --exclude-standard`; read contents). Store-only audits diff --git a/test/fixtures/golden/factory-ship-SKILL.md b/test/fixtures/golden/factory-ship-SKILL.md index 2bb5ddbdd..c882d1736 100644 --- a/test/fixtures/golden/factory-ship-SKILL.md +++ b/test/fixtures/golden/factory-ship-SKILL.md @@ -3167,8 +3167,8 @@ Reentry never resets the count or authorizes a launch. ## Prepare the candidate 1. Read installed document-release SKILL.md and its full audit-scope/release-body - content, linked as sections or inlined for external hosts. Missing/old - `Ship-owned documentation mode` blocks; never substitute. + content, linked as sections or inlined for external hosts. A missing section + or old `Ship-owned documentation mode` blocks before launch; never substitute. 2. Select release paths and base SHA. Inspect committed changes (`git diff HEAD`), staged (`git diff --cached`), unstaged (`git diff`) and selected new files (`git ls-files --others --exclude-standard`; read contents). Store-only audits diff --git a/test/helpers/docsync-fault-actor.ts b/test/helpers/docsync-fault-actor.ts index 5ce9cfdc3..6b9c3acbd 100644 --- a/test/helpers/docsync-fault-actor.ts +++ b/test/helpers/docsync-fault-actor.ts @@ -232,7 +232,7 @@ Earlier review stages are synthetic and outside this fixture. No live review han ## Checks Earlier check stages are synthetic and outside this fixture. No test receipts are asserted. ## Initial documentation state -Attempts used: 0. No accepted audit, hashes, exception or child handle. The supplied candidate.json is initial fixture input, not an accepted audit. +Attempts used: 0. No accepted audit, hashes, exception or child handle. The supplied candidate.json is initial fixture input, not an accepted audit; prepare saves the same selection with current hashes, so it needs no separate Read. ## Initial next steps 1. CURRENT: documentation phase (Step 14.5, or store documentation preflight). 2. Save the result and optionally execute the authorized local publication stand-in if the actual documentation gate permits it. diff --git a/test/helpers/qa-callers-fixture.ts b/test/helpers/qa-callers-fixture.ts index 98fcad9de..ba87320d4 100644 --- a/test/helpers/qa-callers-fixture.ts +++ b/test/helpers/qa-callers-fixture.ts @@ -243,7 +243,7 @@ export function qaCallerCommandAllowed(command: string, workflowCommands: string } if (/[\n\r;&|<>`$\\(){}]/.test(text)) return false; return /^(?:pwd|ls(?: -la)?|bun --version|date -u \+%Y-%m-%dT%H:%M:%SZ|bun (?:run test|test(?: cli\.test\.ts)?))$/.test(text) - || /^git (?:status --(?:short|porcelain)|branch --show-current|rev-parse (?:--short )?HEAD|merge-base origin\/main HEAD|ls-files(?: --others --exclude-standard)?)$/.test(text) + || /^git (?:status --(?:short|porcelain)|branch --show-current|rev-parse (?:--short )?HEAD|merge-base origin\/main HEAD|log origin\/main\.\.HEAD --oneline|ls-files(?: --others --exclude-standard)?)$/.test(text) || /^\/?[\w./-]+\/bin\/gstack-review-log --start (?:review|adversarial-review)$/.test(text) || /^\/?[\w./-]+\/bin\/gstack-(?:review-read|specialist-stats)$/.test(text); } @@ -638,9 +638,16 @@ process.exit(exit ?? 127); } } +export function callerReviewRecordTemplate(fixture: Pick): string { + const source = fs.readFileSync(path.join(QA_CALLER_ROOT, fixture.caller === 'review' ? 'review/SKILL.md' : 'ship/sections/review-army.md'), 'utf8'); + const templates = source.match(/^~\/\.claude\/skills\/gstack\/bin\/gstack-review-log '\{"skill":"review",.*$/gm) ?? []; + if (templates.length !== 1) throw new Error(`Expected one installed /${fixture.caller} review record template, found ${templates.length}`); + return templates[0].replace('~/.claude/skills/gstack', fixture.runtime); +} + export function qaCallerSessionOptions(fixture: QaCallerFixture, runId: string): Parameters[0] { return { - prompt: `Load gstack's /${fixture.caller} supplied parent phase from caller-${fixture.caller}.md and resume it on the selected working-tree diff against origin/main. This excerpt comes from ${fixture.runtime}/${fixture.caller}/SKILL.md; resolve installed-relative references there, not from the excerpt file or product directory. That path identifies the asset base, not another entrypoint: do not read or invoke the full parent SKILL.md or rerun its preamble. Earlier preamble/branch/base setup is complete; use the existing local origin/main ref without fetch. Earlier-phase asset locators are ${fixture.runtime}/review/checklist.md and ${fixture.runtime}/qa/templates/functional-report-template.md. Read those files directly when referenced; recursive Glob does not follow the installed asset symlinks. Cross-project learnings are configured off in this owned fixture. ${fixture.reviewStart ? `The actual review-start helper already captured REVIEW_START=${fixture.reviewStart} for this unchanged core pass; retain that token. ` : ''} This fixture evaluates only the supplied parent phase, not later publication stages. Read README.md for the project contract and commands. Use diagnostic-client commands such as \`bun scripts/probe.ts \` for exploratory discoveries and their checkpoint evidence. A required \`bun run test\` is separate suite verification: report it as verification, never as a diagnostic observation or checkpoint anchor/target. Use the production evidence helper to publish each diagnostic checkpoint as \`reports/exploration-NNN.json\`, not inside a nested directory; do not transcribe its observed payload. ${fixture.caseId === 'ship-exploratory-plan-checks' ? 'The previously discovered plan is PLAN.md.' : 'No plan file was found.'} There is no remote service and no release publication is authorized. There is no interactive approver; do not invent answers or permission. Keep normal parent decision gates. Before every completion report or bookkeeping log, read HANDOFF.md and reports/HANDOFF.md if present for any concurrent collaborator update, await the results, and compare evidence with current inputs. A gstack-review-log completed:true record is a completion, not preliminary bookkeeping; a later handoff read cannot validate an earlier completion.\n\nDeadline bookkeeping additionally permits \`bun ${fixture.runtime}/bin/gstack-qa-deadline start ${fixture.cwd}/reports/deadline.json SECONDS [EARLIER_UTC]\`, \`bun ${fixture.runtime}/bin/gstack-qa-deadline status ${fixture.cwd}/reports/deadline.json\`, and \`bun ${fixture.runtime}/bin/gstack-qa-deadline run ${fixture.cwd}/reports/deadline.json -- bun scripts/probe.ts [literal]\`. These are closed literal forms: SECONDS must be positive and at most 300, EARLIER_UTC is the optional caller absolute deadline: use the section clock's Hard deadline UTC, never its Runner entry UTC, reserve-start time or a clock-read time. The child is only the existing diagnostic client with zero or one literal argument. Resolve these exact helper and state paths; do not use variables, another helper, another state file, nested wrappers, scripts, operators or substitutions. Only this helper may create or change reports/deadline.json and its .qa-deadline- temporary files; never use Write/Edit/MultiEdit on those paths. Record the full outer run command in checkpoints and evidence; keep the unchanged child JSON as observed, separate from prefixed guard diagnostics. A completed expired guard-run is not a probe or a pass: retain its unused checkpoint, report not-run coverage and do not restart the deadline. Keep the 12-probe smoke limit. Required suites and explicit plan checks are outside the bounded smoke budget, not permission to reset it.\n\nFunctional evidence uses the same production helper and existing diagnostic client: \`bun ${fixture.runtime}/bin/gstack-qa-evidence capture ${fixture.cwd}/reports NNN --public --deadline ${fixture.cwd}/reports/deadline.json -- bun scripts/probe.ts [literal]\`. These diagnostic receipts are declared public/synthetic, so --public is approved; a fresh three-digit ID is required each time. Explicit plan probes outside the smoke budget may replace --deadline with --timeout-ms 10000; this does not reset or bypass the smoke deadline. Publish causal intent with \`bun ${fixture.runtime}/bin/gstack-qa-evidence checkpoint ${fixture.cwd}/reports NNN CAPTURE_ID 'full prior capture command' 'causal hypothesis' 'full next capture command'\`; quote arguments literally. For complex quoting, Write only capture, observationCommand, hypothesis and nextCommand to reports/intent.json; publish with the same helper: checkpoint REPORT_ROOT NNN intent.json. Materialize is supported when evidence.json is required. Sources stay inside reports. Decide to execute the next probe before publishing its checkpoint, then await successful publication and dispatch that exact probe. If you defer an optional idea or stop exploration, do not publish a checkpoint for it; describe it as not run in Markdown. An unused checkpoint requires an actual authenticated expired-capture result; nearing the deadline or choosing to stop is not enough. A complete capture can truthfully retain a nonzero domain exit (including expected rejection or unavailable dependency); it is not automatically a pass. Wrapper-failed, interrupted or incomplete captures are not observations or passes; an authenticated expired capture can retain an unused checkpoint as not-run evidence. No new native child command or shell authority is granted.\n\nThe supported Bash interface is one literal documented native command or one exact generated workflow shell block with main substituted for ; even read-only commands must not be chained except for the exact DIFF_BASE preface below. Native CLI commands accept no argument or one literal argument: an unquoted shell-safe word, single-quoted text, or double-quoted text without expansion or escapes; no multiline arguments or shell composition. Inventory commands are pwd, bun --version, and ls with an optional combined -l/-a flag, optional -- separator, and literal path operands using the same quoting rules; operands beginning with - require --. Listing never permits other options, glob expansion, substitution, redirection or composition. The supported Git forms are git status --short, git status --porcelain, git branch --show-current, git rev-parse HEAD, git rev-parse --short HEAD, git merge-base origin/main HEAD, git diff (optional origin/main base and at most one output mode: --stat, --numstat, --name-only, or --name-status, before or after the base; optional literal pathspec arguments after --), git ls-files, and git ls-files --others --exclude-standard. Diff pathspecs use the same literal argument syntax as the CLI; quote globs so Git, not the shell, interprets them. The only variable-base form is DIFF_BASE=$(git merge-base origin/main HEAD) && git diff with exactly the double-quoted "$DIFF_BASE" base, the same optional output mode and literal pathspecs. That exact preface recomputes the base in the same command; other assignments and variable expansions are forbidden; the only additional command substitutions are the two installed bookkeeping fields described below. No other Git options, configuration overrides, --output, --no-index, --ext-diff, --textconv, external helpers or mutations are authorized. Exact generated workflow blocks remain allowed. Review bookkeeping permits the installed gstack-review-log start commands and single-quoted JSON records for review/adversarial-review (the installed quote-spliced double-quoted $(date -u +%Y-%m-%dT%H:%M:%SZ) is permitted only as the complete top-level timestamp value, and $(git rev-parse --short HEAD) only as the complete top-level commit value; no other substitutions, field placements or shell evaluation), gstack-review-read, gstack-specialist-stats, plus date -u +%Y-%m-%dT%H:%M:%SZ; use an original start token for a started attempt and completed:false/converged:false without a token for an unstarted one. These installed helpers are shell scripts: run them by exact installed path, as in \`${fixture.runtime}/bin/gstack-review-log '' --finish \`, never through bun or another interpreter. A review record's status is clean only for a completed review with zero unresolved defects, otherwise issues_found${fixture.caller === 'ship' ? ' (unavailable for missing dispatched reviewer output)' : ''}; a review stopped at a gate records completed:false, never a receipt status such as blocked. Bookkeeping has an owned state directory and private Git object store; the real product, index, refs and config must stay unchanged. Outside providers are unavailable in this isolated fixture; the native reviewer is still required. Other interpreters, shell composition outside those generated blocks or the exact DIFF_BASE preface, global setup and external operations are not authorized. Use Read/Glob/Grep for discovery instead of cat/head/tail; use native Write/Edit tools for any authorized file output. Pass these same command and write boundaries to any child; generated shell fragments remain verbatim apart from the declared substitutions.\n\nWrite the phase report to reports/review.md. Also save reports/receipt.json with this machine-readable result shape: {\"status\":\"pass|fail|blocked|inconclusive\",\"probes\":[\"observed diagnostic receipt ids\"],\"remaining\":[\"names of incomplete contracts\"]}. The status is the overall supplied phase gate, not whether some probes passed. The probes array contains the exact id values from the captured child JSON (probe-...), never the helper's three-digit capture IDs or QA_EVIDENCE.id. Capture IDs select stored observations for checkpoint/materialize; child diagnostic IDs identify the actual probes in this final receipt. Read the child JSON's id for each probe used as evidence; do not derive it from a filename or execution counter. remaining names unresolved required contracts or gates in this phase. Pass requires no remaining required contracts or gates. Optional unavailable providers and later stages outside this excerpt are not required remainder. This format does not establish that any work succeeded. Stop at the end of this phase, or at its first unresolved approval gate.`, + prompt: `Load gstack's /${fixture.caller} supplied parent phase from caller-${fixture.caller}.md and resume it on the selected working-tree diff against origin/main. This excerpt comes from ${fixture.runtime}/${fixture.caller}/SKILL.md; resolve installed-relative references there, not from the excerpt file or product directory. That path identifies the asset base, not another entrypoint: do not read or invoke the full parent SKILL.md or rerun its preamble. Earlier preamble/branch/base setup is complete; use the existing local origin/main ref without fetch. Earlier-phase asset locators are ${fixture.runtime}/review/checklist.md and ${fixture.runtime}/qa/templates/functional-report-template.md. Read those files directly when referenced; recursive Glob does not follow the installed asset symlinks. Cross-project learnings are configured off in this owned fixture. ${fixture.reviewStart ? `The actual review-start helper already captured REVIEW_START=${fixture.reviewStart} for this unchanged core pass; retain that token. ` : ''} This fixture evaluates only the supplied parent phase, not later publication stages. Read README.md for the project contract and commands. Use diagnostic-client commands such as \`bun scripts/probe.ts \` for exploratory discoveries and their checkpoint evidence. A required \`bun run test\` is separate suite verification: report it as verification, never as a diagnostic observation or checkpoint anchor/target. Use the production evidence helper to publish each diagnostic checkpoint as \`reports/exploration-NNN.json\`, not inside a nested directory; do not transcribe its observed payload. ${fixture.caseId === 'ship-exploratory-plan-checks' ? 'The previously discovered plan is PLAN.md.' : 'No plan file was found.'} There is no remote service and no release publication is authorized. There is no interactive approver; do not invent answers or permission. Keep normal parent decision gates. Before every completion report or bookkeeping log, read HANDOFF.md and reports/HANDOFF.md if present for any concurrent collaborator update, await the results, and compare evidence with current inputs. A gstack-review-log completed:true record is a completion, not preliminary bookkeeping; a later handoff read cannot validate an earlier completion.\n\nDeadline bookkeeping additionally permits \`bun ${fixture.runtime}/bin/gstack-qa-deadline start ${fixture.cwd}/reports/deadline.json SECONDS [EARLIER_UTC]\`, \`bun ${fixture.runtime}/bin/gstack-qa-deadline status ${fixture.cwd}/reports/deadline.json\`, and \`bun ${fixture.runtime}/bin/gstack-qa-deadline run ${fixture.cwd}/reports/deadline.json -- bun scripts/probe.ts [literal]\`. These are closed literal forms: SECONDS must be positive and at most 300, EARLIER_UTC is the optional caller absolute deadline: use the section clock's Hard deadline UTC, never its Runner entry UTC, reserve-start time or a clock-read time. The child is only the existing diagnostic client with zero or one literal argument. Resolve these exact helper and state paths; do not use variables, another helper, another state file, nested wrappers, scripts, operators or substitutions. Only this helper may create or change reports/deadline.json and its .qa-deadline- temporary files; never use Write/Edit/MultiEdit on those paths. Record the full outer run command in checkpoints and evidence; keep the unchanged child JSON as observed, separate from prefixed guard diagnostics. A completed expired guard-run is not a probe or a pass: retain its unused checkpoint, report not-run coverage and do not restart the deadline. Keep the 12-probe smoke limit. Required suites and explicit plan checks are outside the bounded smoke budget, not permission to reset it.\n\nFunctional evidence uses the same production helper and existing diagnostic client: \`bun ${fixture.runtime}/bin/gstack-qa-evidence capture ${fixture.cwd}/reports NNN --public --deadline ${fixture.cwd}/reports/deadline.json -- bun scripts/probe.ts [literal]\`. These diagnostic receipts are declared public/synthetic, so --public is approved; a fresh three-digit ID is required each time. Explicit plan probes outside the smoke budget may replace --deadline with --timeout-ms 10000; this does not reset or bypass the smoke deadline. Publish causal intent with \`bun ${fixture.runtime}/bin/gstack-qa-evidence checkpoint ${fixture.cwd}/reports NNN CAPTURE_ID 'full prior capture command' 'causal hypothesis' 'full next capture command'\`; quote arguments literally. For complex quoting, Write only capture, observationCommand, hypothesis and nextCommand to reports/intent.json; publish with the same helper: checkpoint REPORT_ROOT NNN intent.json. Materialize is supported when evidence.json is required. Sources stay inside reports. Decide to execute the next probe before publishing its checkpoint, then await successful publication and dispatch that exact probe. If you defer an optional idea or stop exploration, do not publish a checkpoint for it; describe it as not run in Markdown. An unused checkpoint requires an actual authenticated expired-capture result; nearing the deadline or choosing to stop is not enough. A complete capture can truthfully retain a nonzero domain exit (including expected rejection or unavailable dependency); it is not automatically a pass. Wrapper-failed, interrupted or incomplete captures are not observations or passes; an authenticated expired capture can retain an unused checkpoint as not-run evidence. No new native child command or shell authority is granted.\n\nThe supported Bash interface is one literal documented native command or one exact generated workflow shell block with main substituted for ; even read-only commands must not be chained except for the exact DIFF_BASE preface below. Native CLI commands accept no argument or one literal argument: an unquoted shell-safe word, single-quoted text, or double-quoted text without expansion or escapes; no multiline arguments or shell composition. Inventory commands are pwd, bun --version, and ls with an optional combined -l/-a flag, optional -- separator, and literal path operands using the same quoting rules; operands beginning with - require --. Listing never permits other options, glob expansion, substitution, redirection or composition. The supported Git forms are git status --short, git status --porcelain, git branch --show-current, git rev-parse HEAD, git rev-parse --short HEAD, git merge-base origin/main HEAD, the plan-completion section's git log origin/main..HEAD --oneline, git diff (optional origin/main base and at most one output mode: --stat, --numstat, --name-only, or --name-status, before or after the base; optional literal pathspec arguments after --), git ls-files, and git ls-files --others --exclude-standard. Diff pathspecs use the same literal argument syntax as the CLI; quote globs so Git, not the shell, interprets them. The only variable-base form is DIFF_BASE=$(git merge-base origin/main HEAD) && git diff with exactly the double-quoted "$DIFF_BASE" base, the same optional output mode and literal pathspecs. That exact preface recomputes the base in the same command; other assignments and variable expansions are forbidden; the only additional command substitutions are the two installed bookkeeping fields described below. No other Git options, configuration overrides, --output, --no-index, --ext-diff, --textconv, external helpers or mutations are authorized. Exact generated workflow blocks remain allowed. Review bookkeeping permits the installed gstack-review-log start commands and single-quoted JSON records for review/adversarial-review (the installed quote-spliced double-quoted $(date -u +%Y-%m-%dT%H:%M:%SZ) is permitted only as the complete top-level timestamp value, and $(git rev-parse --short HEAD) only as the complete top-level commit value; no other substitutions, field placements or shell evaluation), gstack-review-read, gstack-specialist-stats, plus date -u +%Y-%m-%dT%H:%M:%SZ; use an original start token for a started attempt and completed:false/converged:false without a token for an unstarted one. These installed helpers are shell scripts: run them by exact installed path, never through bun or another interpreter. A review record fills this installed template, keeping its keys and adding none: \`${callerReviewRecordTemplate(fixture)}\`. A review record's status is clean only for a completed review with zero unresolved defects, otherwise issues_found${fixture.caller === 'ship' ? ' (unavailable for missing dispatched reviewer output)' : ''}; a review stopped at a gate records completed:false, never a receipt status such as blocked. Bookkeeping has an owned state directory and private Git object store; the real product, index, refs and config must stay unchanged. Outside providers are unavailable in this isolated fixture; the native reviewer is still required. Other interpreters, shell composition outside those generated blocks or the exact DIFF_BASE preface, global setup and external operations are not authorized. Use Read/Glob/Grep for discovery instead of cat/head/tail; use native Write/Edit tools for any authorized file output. Pass these same command and write boundaries to any child; generated shell fragments remain verbatim apart from the declared substitutions.\n\nWrite the phase report to reports/review.md. Also save reports/receipt.json with this machine-readable result shape: {\"status\":\"pass|fail|blocked|inconclusive\",\"probes\":[\"observed diagnostic receipt ids\"],\"remaining\":[\"names of incomplete contracts\"]}. The status is the overall supplied phase gate, not whether some probes passed. The probes array contains the exact id values from the captured child JSON (probe-...), never the helper's three-digit capture IDs or QA_EVIDENCE.id. Capture IDs select stored observations for checkpoint/materialize; child diagnostic IDs identify the actual probes in this final receipt. Read the child JSON's id for each probe used as evidence; do not derive it from a filename or execution counter. remaining names unresolved required contracts or gates in this phase. Pass requires no remaining required contracts or gates. Optional unavailable providers and later stages outside this excerpt are not required remainder. This format does not establish that any work succeeded. Stop at the end of this phase, or at its first unresolved approval gate.`, appendSystemPrompt: `Caller execution scheduling (fixture contract): This session has at most 25 assistant turns, including required verification and final artifacts. The command boundary applies to each Bash call, not to the number of independent tool calls in an assistant turn. After required clock and approval prerequisites settle, issue independent source Reads and read-only discovery together as separate native tool calls once their paths and inputs are known. Wait for their results before decisions that depend on them. diff --git a/test/helpers/qa-functional-observer.ts b/test/helpers/qa-functional-observer.ts index 229c4a9c0..3230b9cba 100644 --- a/test/helpers/qa-functional-observer.ts +++ b/test/helpers/qa-functional-observer.ts @@ -95,6 +95,7 @@ export async function observeQAWrites(root: string, options: { reportDirectory?: const events: QAWriteObservation['events'] = []; const failures: string[] = []; const publications = new Map(); + const pendingLinks = new Map(); let stopped = false; const observedPath = (relative: string, knownPair = false): string => { try { @@ -150,6 +151,18 @@ export async function observeQAWrites(root: string, options: { reportDirectory?: publications.set(relativeTarget, publication); return target; } catch (publicationError) { + if (receipt === undefined && (publicationError as NodeJS.ErrnoException).code === 'ENOENT' && temporaryName.test(basename)) { + let linking: number | undefined; + try { linking = fs.openSync(path.join(parent, basename), fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK); } + catch (openError) { if ((openError as NodeJS.ErrnoException).code === 'ENOENT') return path.join(parent, basename); } + if (linking !== undefined) try { + const entry = fs.fstatSync(linking); + if (entry.isFile() && entry.nlink === 2 && entry.uid === parentStat.uid && (entry.mode & 0o777) === mode) { + pendingLinks.set(relative, { target: path.relative(root, target), dev: entry.dev, ino: entry.ino, mode, bytes: fs.readFileSync(linking, 'utf8') }); + return path.join(parent, basename); + } + } finally { fs.closeSync(linking); } + } try { return ownedPath(root, relative); } catch { throw publicationError; } } finally { if (receipt !== undefined) fs.closeSync(receipt); } } @@ -241,6 +254,13 @@ export async function observeQAWrites(root: string, options: { reportDirectory?: || fs.readFileSync(target, 'utf8') !== publication.bytes) throw new Error('Evidence publication did not settle unchanged'); } catch (error) { failures.push(String(pathFailure(root, relative, error))); } } + for (const [temporary, pending] of pendingLinks) { + try { + const entry = fs.lstatSync(ownedPath(root, pending.target)); + if (fs.existsSync(ownedPath(root, temporary)) || entry.dev !== pending.dev || entry.ino !== pending.ino || entry.nlink !== 1 + || (entry.mode & 0o777) !== pending.mode || fs.readFileSync(ownedPath(root, pending.target), 'utf8') !== pending.bytes) throw new Error('Evidence publication did not settle unchanged'); + } catch (error) { failures.push(String(pathFailure(root, temporary, error))); } + } let after: Record = {}; try { after = qaTreeSnapshot(root); } catch (error) { failures.push(String(error)); } fs.closeSync(fd); diff --git a/test/qa-exploratory-callers.test.ts b/test/qa-exploratory-callers.test.ts index ecc0da975..a22b6cb92 100644 --- a/test/qa-exploratory-callers.test.ts +++ b/test/qa-exploratory-callers.test.ts @@ -4,7 +4,7 @@ import * as path from 'node:path'; import * as os from 'node:os'; import { spawnSync } from 'node:child_process'; import { - callerExcerpt, callerSnapshot, callerTools, createQaCallerFixture, qaCallerInstructions, + callerExcerpt, callerReviewRecordTemplate, callerSnapshot, callerTools, createQaCallerFixture, qaCallerInstructions, QA_CALLER_CASES, QA_CALLER_TEST_MS, qaCallerSessionOptions, qaCallerCommandAllowed, readCallerReceipt, retainQaCallerEvidence, runQaCaller, validateCallerEvidence, type CallerProbe, type CallerReceipt, type QaCallerFixture, @@ -136,11 +136,48 @@ describe('caller native-event observer controls', () => { const fixture = createQaCallerFixture(id, { installRuntime: false }); try { return qaCallerSessionOptions(fixture, 'free-control').prompt; } finally { fs.rmSync(fixture.root, { recursive: true, force: true }); } }; - expect(prompt('review-exploratory-small-cli')).toContain("/bin/gstack-review-log '' --finish `, never through bun"); + expect(prompt('review-exploratory-small-cli')).toContain('never through bun or another interpreter. A review record fills this installed template, keeping its keys and adding none: `'); expect(prompt('review-exploratory-small-cli')).toContain('otherwise issues_found; a review stopped at a gate records completed:false'); expect(prompt('ship-exploratory-small-cli')).toContain('otherwise issues_found (unavailable for missing dispatched reviewer output)'); }); + test('captured PR-lane review record: an invented shape without the installed template is rejected; the template is quoted', () => { + // ci-36641824710-1-eval-slices-6 review-exploratory-small-cli, native event 8jewsM (fixture paths shortened). + const invented = `/runtime/bin/gstack-review-log '{"timestamp":"'"$(date -u +%Y-%m-%dT%H:%M:%SZ)"'","commit":"'"$(git rev-parse --short HEAD)"'","branch":"caller-change","status":"issues_found","completed":false,"gate":"fix-first-ask","findings":[{"path":"scale.ts","line":3,"category":"functional-contract","severity":"CRITICAL","fingerprint":"scale.ts:3:functional-contract","confidence":10,"action":"ask","summary":"new !n guard rejects documented lower bound 0 (exit 2 instead of 0)"}],"qa":{"probes":["probe-167de79c-2645-4b60-b06b-3d0d6eebf13f"],"checkpoints":["exploration-001.json","exploration-002.json"],"suite":"bun run test 1 pass 0 fail"},"remaining":["fix-first-ask-approval"]}' --finish 37b567b3-c4bf-4469-9bad-29096740c234`; + const errorsFor = (command: string) => { + const observed = evidence(); + observed.result.transcript.push(...nativeCall('record', 'Bash', { command }, '')); + return validateCallerEvidence(observed); + }; + expect(errorsFor(invented)).toEqual(['command outside declared caller observation interface', 'QA checkpoint: Unsupported checkpoint Bash interaction']); + for (const caller of ['review', 'ship'] as const) { + const template = callerReviewRecordTemplate({ caller, runtime: '/runtime' }); + expect(template.startsWith(`/runtime/bin/gstack-review-log '{"skill":"review","timestamp":`)).toBe(true); + expect(template).toEndWith(`}' --finish REVIEW_START`); + const filled = template.replace('"timestamp":"TIMESTAMP"', `"timestamp":"'"$(date -u +%Y-%m-%dT%H:%M:%SZ)"'"`) + .replace('"commit":"COMMIT"', `"commit":"'"$(git rev-parse --short HEAD)"'"`) + .replace('"STATUS"', '"issues_found"').replace(/"issues_found":N/, '"issues_found":1').replace('"critical":N', '"critical":1').replace('"informational":N', '"informational":0') + .replace('SCORE', '10.0').replace('SPECIALISTS_JSON', '{}').replace('FINDINGS_JSON', '[{"fingerprint":"scale.ts:3:functional-contract","severity":"CRITICAL","action":"ask-pending"}]') + .replace('COMPLETED', 'false').replace('CONVERGED', 'false').replace('CYCLES', '0').replace('REVIEW_START', 'native-token'); + expect(errorsFor(filled)).toEqual([]); + const fixture = createQaCallerFixture(caller === 'review' ? 'review-exploratory-small-cli' : 'ship-exploratory-small-cli', { installRuntime: false }); + try { expect(qaCallerSessionOptions(fixture, 'free-control').prompt).toContain(callerReviewRecordTemplate(fixture)); } finally { fs.rmSync(fixture.root, { recursive: true, force: true }); } + } + }); + + test('captured PR-lane plan-completion git log is in the caller interface; other log forms stay outside', () => { + // ci-36641824710-1-eval-slices-7 ship-exploratory-plan-checks, native event 3Pvdmu: ship/sections/plan-completion.md requires this read. + expect(fs.readFileSync(path.join(import.meta.dir, '../ship/sections/plan-completion.md'), 'utf8')).toContain('`git log origin/..HEAD --oneline`'); + expect(qaCallerCommandAllowed('git log origin/main..HEAD --oneline')).toBe(true); + for (const command of ['git log', 'git log --oneline', 'git log origin/main..HEAD', 'git log origin/main..HEAD --oneline -p', + 'git log origin/main..HEAD --oneline --output=/tmp/x', 'git log --all --oneline', 'git log origin/main..HEAD --oneline; git push', + 'git log origin/main..HEAD --oneline && git commit -am x', 'git -c core.pager=x log origin/main..HEAD --oneline', 'git log origin/main...HEAD --oneline']) { + expect(qaCallerCommandAllowed(command)).toBe(false); + } + const fixture = createQaCallerFixture('ship-exploratory-plan-checks', { installRuntime: false }); + try { expect(qaCallerSessionOptions(fixture, 'free-control').prompt).toContain("git log origin/main..HEAD --oneline, git diff"); } finally { fs.rmSync(fixture.root, { recursive: true, force: true }); } + }); + test('a later unchanged handoff reread does not invalidate an already completed freshness decision', () => { const observed = evidence(); observed.result.transcript.push( diff --git a/test/qa-functional-observer.test.ts b/test/qa-functional-observer.test.ts index 809863ebf..dcb648714 100644 --- a/test/qa-functional-observer.test.ts +++ b/test/qa-functional-observer.test.ts @@ -110,6 +110,33 @@ describe('QA command-observation boundary', () => { }); } + // ci-36709485593-1-eval-slices-6 qa-functional-cli-fix: link(2) raised the temporary receipt's nlink to 2 before the + // receipt.json name resolved, so the observer's open failed with ENOENT during an ordinary atomic publication. + for (const variant of ['published', 'foreign-link-kept', 'foreign-link-dropped', 'replaced-target'] as const) { + test(`evidence publication observed mid-link: ${variant}`, async () => { + const fixture = createQAFunctionalFixture('cli'); + const outside = fs.mkdtempSync(path.join(path.dirname(fixture.root), 'qa-link-')); + const observer = await observeQAWrites(fixture.root, { evidenceProducer: true }); + try { + const directory = path.join(fixture.root, 'qa-reports/.qa-evidence/002'); + fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); + observer.drain(); + const temporary = path.join(directory, 'receipt.json.tmp.2282.fd4f6b4d'); + const target = path.join(directory, 'receipt.json'); + const foreign = path.join(outside, 'second-name'); + fs.writeFileSync(temporary, JSON.stringify({ version: 1, id: '002', status: 'complete' }), { mode: 0o600 }); + fs.linkSync(temporary, foreign); + observer.drain(); + if (variant === 'published') { fs.unlinkSync(foreign); fs.linkSync(temporary, target); observer.drain(); fs.unlinkSync(temporary); } + if (variant === 'foreign-link-dropped') fs.unlinkSync(temporary); + if (variant === 'replaced-target') { fs.unlinkSync(foreign); fs.unlinkSync(temporary); fs.writeFileSync(target, '{}', { mode: 0o600 }); } + const verdict = qaWriteVerdict(observer.stop(), 'qa-only'); + if (variant === 'published') expect(verdict).toEqual([]); + else expect(verdict).toContain('incomplete write observation'); + } finally { fixture.cleanup(); fs.rmSync(outside, { recursive: true, force: true }); } + }); + } + test('lost directory watches and allowed-directory link substitutions fail closed', async () => { const fixture = createQAFunctionalFixture('cli'); const observer = await observeQAWrites(fixture.root); diff --git a/test/qa-probe-gates.test.ts b/test/qa-probe-gates.test.ts index c959a920e..934505e20 100644 --- a/test/qa-probe-gates.test.ts +++ b/test/qa-probe-gates.test.ts @@ -8,6 +8,7 @@ import { HOST_PATHS } from '../scripts/resolvers/types'; function assertPreparation(text: string) { expect(text).toContain('Complete these Reads in order before writing charters or probing'); + expect(text).toMatch(/Await their results before the first probe, never in the same response\.|Await each successful Read result before continuing\./); expect(text).toContain('Do not repeat a Read already completed in this invocation'); const stages = ['1. Read `sections/scope.md`', 'in full and select the surfaces', '2. Read the selected surface methods below in full', '**Functional surfaces:**', @@ -240,6 +241,8 @@ describe('QA probe entry and checkpoint gates', () => { text.replace(method, method + '\n' + method), 'Write a **charter**\n' + text, text.replace('Do not repeat a Read already completed in this invocation', 'Repeat all Reads'), + // ci-36641820398-1-gate-census-7 ship-exploratory-small-cli dispatched its first probe with the resource Reads. + text.replace(' Await their results before the first probe, never in the same response.', ''), ]) expect(() => assertPreparation(changed)).toThrow(); }); diff --git a/test/ship-document-release-dispatch.test.ts b/test/ship-document-release-dispatch.test.ts index d294cd284..02583cead 100644 --- a/test/ship-document-release-dispatch.test.ts +++ b/test/ship-document-release-dispatch.test.ts @@ -138,6 +138,22 @@ describe('pre-publication documentation lifecycle', () => { expect(read('ship/sections/apple-release.md.tmpl')).toContain('ship/sections/documentation.md'); }); + test('ship-owned documentation_section carries its status so /ship embeds it unchanged', () => { + // ci-36641820398-1-gate-census-3 ship-docsync-completion: the section had scope, health and debt but no result, + // so the parent spliced a Status line into it and the report no longer contained the returned section. + const scope = read('document-release/sections/audit-scope.md.tmpl').replace(/\s+/g, ' '); + expect(scope).toContain('complete for verbatim embedding: a first `**Status:**` line with `status` and the result, audited scope'); + expect(read('ship/sections/documentation.md.tmpl')).toContain('nonempty Markdown with scope, result and debt'); + expect(read('ship/sections/pr-body.md.tmpl')).toContain("Embed Step 14.5's vetted nonempty `documentation_section`"); + }); + + test('a missing installed document-release section blocks before launch', () => { + // ci-36709485593-1-eval-slices-2 ship-docsync-missing-asset: audit-scope.md was absent, but the parent saw the + // SKILL.md "Ship-owned documentation mode" heading, read the gate as satisfied and dispatched. + const gate = read('ship/sections/documentation.md.tmpl').replace(/\s+/g, ' '); + expect(gate).toContain('full audit-scope/release-body content, linked as sections or inlined for external hosts. A missing section or old `Ship-owned documentation mode` blocks before launch; never substitute.'); + }); + test('nested authored discovery and standalone protections survive', () => { const skill = read('document-release/SKILL.md.tmpl') + read('document-release/sections/audit-scope.md.tmpl'); expect(skill).not.toContain('find . -maxdepth 2');