From 8aae497d2b3a6faff79ce4242681c32ac9957fc2 Mon Sep 17 00:00:00 2001 From: garrytan Date: Wed, 7 Oct 2026 20:05:53 +0000 Subject: [PATCH] fix(outside-review): severity words in label position count as findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Oct 7 /autoplan run's Codex CEO and DX reviews labeled seven and eight findings High/Medium without [Pn] tags, and the classifier printed VERDICT: clean / FINDINGS: none for both. lib/outside-review-result.ts now reads severity words in label position (Severity: High, a leading High: / High — / [High], **High**, a table cell): critical/high block like P0/P1, medium/low are advisory. Words in prose (high-level, low-risk, no critical findings) never count. A review or structured response with neither tags, labels nor an explicit no-findings statement is unverified, never clean (DX-11, ENG-10). Design-direction proposals move to a new proposal gate so they keep needing only their Recommendation line, and review prompts ask for a severity per finding. Fixtures reproduce the CEO/DX label shapes, the eng [P1] output, a table and prose negative controls; the stored Codex panel keeps its verdicts. docs/troubleshooting.md explains how the verdict is read. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- autoplan/sections/ceo-phase.md | 2 +- autoplan/sections/design-phase.md | 2 +- autoplan/sections/dx-phase.md | 2 +- autoplan/sections/eng-phase.md | 2 +- .../sections/proposal-and-preview.md | 2 +- design-review/SKILL.md | 2 +- docs/troubleshooting.md | 27 ++++- document-release/sections/release-body.md | 2 +- lib/outside-review-result.ts | 38 ++++-- office-hours/SKILL.md | 4 +- plan-ceo-review/sections/review-sections.md | 2 +- plan-design-review/SKILL.md | 2 +- plan-devex-review/sections/review-sections.md | 2 +- plan-eng-review/sections/review-sections.md | 2 +- review/sections/adversarial.md | 2 +- scripts/resolvers/outside-voice.ts | 4 +- ship/sections/adversarial.md | 2 +- ship/sections/review-army.md | 2 +- test/design-consultation-contract.test.ts | 2 +- .../ceo-codex-shape.txt | 17 +++ .../dx-codex-shape.txt | 27 +++++ .../eng-codex-tagged.txt | 16 +++ .../prose-negative.txt | 5 + .../prose-unlabeled.txt | 3 + .../outside-review-severity/table-shape.txt | 6 + test/helpers/carve-guards.ts | 14 +-- test/helpers/free-fixtures-data.ts | 3 +- test/outside-review-result.test.ts | 5 +- test/outside-review-severity.test.ts | 110 ++++++++++++++++++ test/outside-voice-invocation.test.ts | 4 +- test/outside-voice-preflight.test.ts | 2 +- test/outside-voice-receipt.test.ts | 2 +- test/run-in-background-guidance.test.ts | 4 +- 33 files changed, 276 insertions(+), 45 deletions(-) create mode 100644 test/fixtures/outside-review-severity/ceo-codex-shape.txt create mode 100644 test/fixtures/outside-review-severity/dx-codex-shape.txt create mode 100644 test/fixtures/outside-review-severity/eng-codex-tagged.txt create mode 100644 test/fixtures/outside-review-severity/prose-negative.txt create mode 100644 test/fixtures/outside-review-severity/prose-unlabeled.txt create mode 100644 test/fixtures/outside-review-severity/table-shape.txt create mode 100644 test/outside-review-severity.test.ts diff --git a/autoplan/sections/ceo-phase.md b/autoplan/sections/ceo-phase.md index 8f8e36c2e..71930c0df 100644 --- a/autoplan/sections/ceo-phase.md +++ b/autoplan/sections/ceo-phase.md @@ -91,7 +91,7 @@ IMPORTANT: Do NOT read or execute any SKILL.md files or paths containing skills/ No compliments. Just the strategic blind spots. File: -Write the **complete prompt and context**, including actual plan/spec/source, to a private file. Substitute its shell-quoted path for ``; never interpolate user text into shell source. Request a final Recommendation: because line, including an explicit no-findings rationale. +Write the **complete prompt and context**, including actual plan/spec/source, to a private file. Substitute its shell-quoted path for ``; never interpolate user text into shell source. Request a severity (Critical, High, Medium or Low) per finding and a final Recommendation: because line, including an explicit no-findings rationale. ```bash # GSTACK_ACTIVE_HOST names the harness, never the model. diff --git a/autoplan/sections/design-phase.md b/autoplan/sections/design-phase.md index 5b72890ed..d0ce10572 100644 --- a/autoplan/sections/design-phase.md +++ b/autoplan/sections/design-phase.md @@ -50,7 +50,7 @@ IMPORTANT: Do NOT read or execute any SKILL.md files or paths containing skills/ What design decisions will haunt the implementer if left ambiguous? Be opinionated. No hedging. -Write the **complete prompt and context**, including actual plan/spec/source, to a private file. Substitute its shell-quoted path for ``; never interpolate user text into shell source. Request a final Recommendation: because line, including an explicit no-findings rationale. +Write the **complete prompt and context**, including actual plan/spec/source, to a private file. Substitute its shell-quoted path for ``; never interpolate user text into shell source. Request a severity (Critical, High, Medium or Low) per finding and a final Recommendation: because line, including an explicit no-findings rationale. ```bash # GSTACK_ACTIVE_HOST names the harness, never the model. diff --git a/autoplan/sections/dx-phase.md b/autoplan/sections/dx-phase.md index 63c1a4fe0..7d7d34656 100644 --- a/autoplan/sections/dx-phase.md +++ b/autoplan/sections/dx-phase.md @@ -54,7 +54,7 @@ IMPORTANT: Do NOT read or execute any SKILL.md files or paths containing skills/ 5. Upgrade path: can devs upgrade without fear? Migration guides? Deprecation warnings? Be adversarial. Think like a developer who is evaluating this against 3 competitors. -Write the **complete prompt and context**, including actual plan/spec/source, to a private file. Substitute its shell-quoted path for ``; never interpolate user text into shell source. Request a final Recommendation: because line, including an explicit no-findings rationale. +Write the **complete prompt and context**, including actual plan/spec/source, to a private file. Substitute its shell-quoted path for ``; never interpolate user text into shell source. Request a severity (Critical, High, Medium or Low) per finding and a final Recommendation: because line, including an explicit no-findings rationale. ```bash # GSTACK_ACTIVE_HOST names the harness, never the model. diff --git a/autoplan/sections/eng-phase.md b/autoplan/sections/eng-phase.md index 76cba4ea9..c52eec2b1 100644 --- a/autoplan/sections/eng-phase.md +++ b/autoplan/sections/eng-phase.md @@ -43,7 +43,7 @@ IMPORTANT: Do NOT read or execute any SKILL.md files or paths containing skills/ File: -Write the **complete prompt and context**, including actual plan/spec/source, to a private file. Substitute its shell-quoted path for ``; never interpolate user text into shell source. Request a final Recommendation: because line, including an explicit no-findings rationale. +Write the **complete prompt and context**, including actual plan/spec/source, to a private file. Substitute its shell-quoted path for ``; never interpolate user text into shell source. Request a severity (Critical, High, Medium or Low) per finding and a final Recommendation: because line, including an explicit no-findings rationale. ```bash # GSTACK_ACTIVE_HOST names the harness, never the model. diff --git a/design-consultation/sections/proposal-and-preview.md b/design-consultation/sections/proposal-and-preview.md index 1b9d39584..fc05fb2a1 100644 --- a/design-consultation/sections/proposal-and-preview.md +++ b/design-consultation/sections/proposal-and-preview.md @@ -190,7 +190,7 @@ cat "$_OUTSIDE_TMP/text" 2>/dev/null || tail -n 20 "$_OUTSIDE_TMP/events" cat "$_OUTSIDE_TMP/stderr" >&2 || { [ "$_OUTSIDE_EXIT" -ne 0 ] || _OUTSIDE_EXIT=1; } _OUTSIDE_RC=0 -bun "$HOME/.claude/skills/gstack/lib/outside-review-result.ts" --label 'Codex outside review' --exit "$_OUTSIDE_EXIT" --stderr "$_OUTSIDE_TMP/stderr" --events "$_OUTSIDE_TMP/events" review "$_OUTSIDE_TMP/text" || _OUTSIDE_RC=$? +bun "$HOME/.claude/skills/gstack/lib/outside-review-result.ts" --label 'Codex outside review' --exit "$_OUTSIDE_EXIT" --stderr "$_OUTSIDE_TMP/stderr" --events "$_OUTSIDE_TMP/events" proposal "$_OUTSIDE_TMP/text" || _OUTSIDE_RC=$? case "$_OUTSIDE_RC" in 0|3) ;; 4) echo 'OUTSIDE_STATUS: unverified provider=codex host=claude'; exit 4 ;; diff --git a/design-review/SKILL.md b/design-review/SKILL.md index 2332b5a83..20349c45b 100644 --- a/design-review/SKILL.md +++ b/design-review/SKILL.md @@ -1658,7 +1658,7 @@ HARD REJECTION — flag if ANY apply: Be specific. Reference file:line for every finding." -Write the **complete prompt and context**, including actual plan/spec/source, to a private file. Substitute its shell-quoted path for ``; never interpolate user text into shell source. Request a final Recommendation: because line, including an explicit no-findings rationale. +Write the **complete prompt and context**, including actual plan/spec/source, to a private file. Substitute its shell-quoted path for ``; never interpolate user text into shell source. Request a severity (Critical, High, Medium or Low) per finding and a final Recommendation: because line, including an explicit no-findings rationale. ```bash # GSTACK_ACTIVE_HOST names the harness, never the model. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index ed3e8a224..f11d2fa06 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -31,6 +31,28 @@ P0 or P1 finding blocks exactly like a native P0/P1. `unverified` and `unavailable` are missing coverage: /ship and /review continue, show the gap in the readiness dashboard and the PR body, and never count it as a pass. + +### How the verdict is read from a review + +- **Severity tags.** `[P0]`-`[P3]` (or Codex's native `P1:` labels). P0 and P1 + block; P2 and P3 are advisory. +- **Severity words in label position.** `Severity: High`, `Priority: low`, a + line that starts with `High:`, `High —`, `[High]` or `**High**` (after an + optional heading, bullet or number), a bold `**High**` anywhere, or a table + cell `| high |`. Critical and High block like P0 and P1; Medium and Low are + advisory like P2 and P3. Words inside prose do not count: "high-level", + "low-risk", "a medium-term follow-up" and "no critical findings" are not + findings. +- **No findings.** A review with no tag and no label is `clean` only when it + says so explicitly (`NO_FINDINGS`, "no issues", "did not find any bugs"). + Otherwise it is `unverified` (see below), never `clean`. +- **Design proposals** (the design-direction voices in /design-consultation + and /office-hours) are read with the `proposal` gate: a completed proposal + needs only its `Recommendation: ... because ...` line. + +Outside-review prompts ask the reviewer to label each finding Critical, High, +Medium or Low, so most reviews land on `findings` or `clean`. + ### `gstack: cannot load gstack-codex-probe` / `gstack: cannot locate (shell: ...)` / `CODEX_MODE: helper_unavailable` @@ -161,8 +183,9 @@ severity tags. ### `... outside review: ran, verdict unverified (...)` / `OUTSIDE_STATUS: unverified` / `GATE: UNVERIFIED` -**Meaning.** The review completed but tagged nothing and gave no explicit -no-findings conclusion, so no pass or fail can be read from it. +**Meaning.** The review completed but had no severity tag or label and gave +no explicit no-findings conclusion, so no pass or fail can be read from it +([how the verdict is read](#outside-review-verdict)). **What is kept.** The full answer is shown above. diff --git a/document-release/sections/release-body.md b/document-release/sections/release-body.md index e5b298117..5e5579f40 100644 --- a/document-release/sections/release-body.md +++ b/document-release/sections/release-body.md @@ -330,7 +330,7 @@ THE DOCS AND DIFF: `; never interpolate user text into shell source. Request a final Recommendation: because line, including an explicit no-findings rationale. +Write the **complete prompt and context**, including actual plan/spec/source, to a private file. Substitute its shell-quoted path for ``; never interpolate user text into shell source. Request a severity (Critical, High, Medium or Low) per finding and a final Recommendation: because line, including an explicit no-findings rationale. ```bash # GSTACK_ACTIVE_HOST names the harness, never the model. diff --git a/lib/outside-review-result.ts b/lib/outside-review-result.ts index f9a353341..530474bf8 100644 --- a/lib/outside-review-result.ts +++ b/lib/outside-review-result.ts @@ -9,7 +9,7 @@ */ import { GATE_OUTCOMES, gateOutcomeLine, type GateReason } from './gate-outcomes'; -export type OutsideGate = 'review' | 'structured' | 'spec' | 'execution'; +export type OutsideGate = 'review' | 'structured' | 'spec' | 'execution' | 'proposal'; export type OutsideVerdict = 'clean' | 'findings' | 'unverified' | 'unavailable'; export type Severity = 'P0' | 'P1' | 'P2' | 'P3'; @@ -112,6 +112,27 @@ function execution(input: OutsideReviewInput): OutsideReviewClassification['exec return { state: 'ran' }; } +/** + * Severity words count as findings only in label position, never inside prose + * ("high-level", "a low-risk change", "no critical findings"): `Severity: High` + * or `Priority: high`, a leading `High:` / `High —` / `[High]` (after an + * optional heading, bullet or number), a bold `**High**`, or a table cell. + * Critical and high block like P0/P1; medium and low are advisory like P2/P3. + */ +const SEVERITY_WORDS: Record = { critical: 'P0', high: 'P1', medium: 'P2', low: 'P3' }; +const WORD = '(critical|high|medium|low)'; +const SEVERITY_LABELS = [ + new RegExp(`\\b(?:severity|priority)\\b[\\t ]*[:=][\\t ]*(?:\\*\\*|__|\\[|\`)*${WORD}\\b`, 'gim'), + new RegExp(`^[\\t ]*(?:>[\\t ]*)?(?:#{1,6}[\\t ]+|[-+*][\\t ]+|\\(?\\d{1,3}[.)][\\t ]+)?(?:\\*\\*|__)?\\[?${WORD}\\]?(?:\\*\\*|__)?[\\t ]*(?::|\u2014|\u2013|-[\\t ]|\\]|\\(|\\*\\*[\\t ]*(?:\u2014|\u2013|-[\\t ]))`, 'gim'), + new RegExp(`(?:\\*\\*|__)\\[?${WORD}\\]?:?(?:\\*\\*|__)`, 'gi'), + new RegExp(`\\|[\\t ]*(?:\\*\\*)?${WORD}(?:\\*\\*)?[\\t ]*(?=\\|)`, 'gi'), +]; +function severityWords(text: string): Severity[] { + return SEVERITY_LABELS.flatMap(re => [...text.matchAll(re)].map(m => SEVERITY_WORDS[m[1]!.toLowerCase()]!)); +} +/** An explicit conclusion that the review found nothing to report. */ +const NO_FINDINGS = /\bNO_FINDINGS\b|\bno (?:actionable |significant |new |concrete )?(?:bugs?|issues?|findings?|problems?)\b|\b(?:did not|didn't|cannot|can't|could not|couldn't) (?:find|identify) any (?:actionable |new |concrete )?(?:bugs|issues|findings|problems)\b/i; + /** Formatting the requested marker in bold, inline code, or a list does not invalidate a completed review. */ function plainReview(text: string): string { return text.split(/\r?\n/).map(line => line.replace(/^[\t ]*(?:#{1,6}[\t ]+|[-+*][\t ]+)?/, '').replace(/[*_`]/g, '')).join('\n'); @@ -119,7 +140,8 @@ function plainReview(text: string): string { export function classifyOutsideReview(input: OutsideReviewInput): OutsideReviewClassification { const plain = plainReview(input.text); - const levels = [...plain.matchAll(/\[(P[0-3])\]|^(P[0-3]):/gm)].map(m => (m[1] ?? m[2]) as Severity); + const levels = [...[...plain.matchAll(/\[(P[0-3])\]|^(P[0-3]):/gm)].map(m => (m[1] ?? m[2]) as Severity), + ...(input.gate === 'review' || input.gate === 'structured' ? severityWords(input.text) : [])]; const findings = { highest: levels.length ? levels.sort()[0]! : null }; const ran = execution(input); if (ran.state === 'unavailable') return { execution: ran, findings, verdict: 'unavailable', reason: ran.reason, detail: ran.detail }; @@ -133,14 +155,12 @@ export function classifyOutsideReview(input: OutsideReviewInput): OutsideReviewC const score = Number(scores[0]![1]); return { ...result(score >= 7 ? 'clean' : 'findings'), score }; } - if (input.gate === 'structured') { - const clear = /\bNO_FINDINGS\b|\bno (?:actionable |significant |new |concrete )?(?:bugs|issues|findings|problems)\b|\b(?:did not|didn't) (?:find|identify) any (?:actionable |new |concrete )?(?:bugs|issues|findings|problems)\b/i.test(input.text); - if (!findings.highest && !clear) return result('unverified', 'untagged_review', 'missing severity or explicit no-findings conclusion'); - return result(blocking ? 'findings' : 'clean'); - } - if (input.gate === 'review' && !/^Recommendation:[\t ]*[^\r\n]+\bbecause\b[\t ]*\S[^\r\n]+$/im.test(plain)) { + if ((input.gate === 'review' || input.gate === 'proposal') && !/^Recommendation:[\t ]*[^\r\n]+\bbecause\b[\t ]*\S[^\r\n]+$/im.test(plain)) { return result('unavailable', 'missing_markers', 'missing review completion recommendation'); } + if ((input.gate === 'structured' || input.gate === 'review') && !findings.highest && !NO_FINDINGS.test(input.text)) { + return result('unverified', 'untagged_review', 'missing severity or explicit no-findings conclusion'); + } return result(blocking ? 'findings' : 'clean'); } @@ -162,7 +182,7 @@ export function validateOutsideReview(text: string, gate: OutsideGate): { comple } const VERDICT_EXIT: Record = { clean: 0, findings: 3, unverified: 4, unavailable: 1 }; -const GATES = ['review', 'structured', 'spec', 'execution']; +const GATES = ['review', 'structured', 'spec', 'execution', 'proposal']; const USAGE = `Usage: outside-review-result.ts outside-review-result.ts --verdict [--stderr ] [--exit ] [--events ] [--label ] Gates: ${GATES.join('|')}. Two-argument form exits 0 completed, 1 unavailable, 2 usage. diff --git a/office-hours/SKILL.md b/office-hours/SKILL.md index 3daa945f6..7f33cbc80 100644 --- a/office-hours/SKILL.md +++ b/office-hours/SKILL.md @@ -820,7 +820,7 @@ Then add the context block and mode-appropriate instructions: 3. Run Codex with the assembled prompt: -Write the **complete prompt and context**, including actual plan/spec/source, to a private file. Substitute its shell-quoted path for ``; never interpolate user text into shell source. Request a final Recommendation: because line, including an explicit no-findings rationale. +Write the **complete prompt and context**, including actual plan/spec/source, to a private file. Substitute its shell-quoted path for ``; never interpolate user text into shell source. Request a severity (Critical, High, Medium or Low) per finding and a final Recommendation: because line, including an explicit no-findings rationale. ```bash # GSTACK_ACTIVE_HOST names the harness, never the model. @@ -1215,7 +1215,7 @@ cat "$_OUTSIDE_TMP/text" 2>/dev/null || tail -n 20 "$_OUTSIDE_TMP/events" cat "$_OUTSIDE_TMP/stderr" >&2 || { [ "$_OUTSIDE_EXIT" -ne 0 ] || _OUTSIDE_EXIT=1; } _OUTSIDE_RC=0 -bun "$HOME/.claude/skills/gstack/lib/outside-review-result.ts" --label 'Codex outside review' --exit "$_OUTSIDE_EXIT" --stderr "$_OUTSIDE_TMP/stderr" --events "$_OUTSIDE_TMP/events" review "$_OUTSIDE_TMP/text" || _OUTSIDE_RC=$? +bun "$HOME/.claude/skills/gstack/lib/outside-review-result.ts" --label 'Codex outside review' --exit "$_OUTSIDE_EXIT" --stderr "$_OUTSIDE_TMP/stderr" --events "$_OUTSIDE_TMP/events" proposal "$_OUTSIDE_TMP/text" || _OUTSIDE_RC=$? case "$_OUTSIDE_RC" in 0|3) ;; 4) echo 'OUTSIDE_STATUS: unverified provider=codex host=claude'; exit 4 ;; diff --git a/plan-ceo-review/sections/review-sections.md b/plan-ceo-review/sections/review-sections.md index 0f869cf3b..cccc7e8f3 100644 --- a/plan-ceo-review/sections/review-sections.md +++ b/plan-ceo-review/sections/review-sections.md @@ -489,7 +489,7 @@ Its opening harness guard rechecks the fresh shell: exit 78 uses the same Native fallback below, never a replacement provider. Finish termination before fallback and consume only completed output. Use private temporary paths, with no background jobs. -Create a private prompt file: run `umask 077; mktemp "${TMPDIR:-/tmp}/gstack-plan-prompt.XXXXXXXX"` in Bash and keep the returned path. Use Write to put the **complete prompt and context**, including actual plan/spec/source, in that file. Substitute its shell-quoted path for ``; never interpolate user text into shell source. Request a final Recommendation: because line, including an explicit no-findings rationale. +Create a private prompt file: run `umask 077; mktemp "${TMPDIR:-/tmp}/gstack-plan-prompt.XXXXXXXX"` in Bash and keep the returned path. Use Write to put the **complete prompt and context**, including actual plan/spec/source, in that file. Substitute its shell-quoted path for ``; never interpolate user text into shell source. Request a severity (Critical, High, Medium or Low) per finding and a final Recommendation: because line, including an explicit no-findings rationale. ```bash # GSTACK_ACTIVE_HOST names the harness, never the model. diff --git a/plan-design-review/SKILL.md b/plan-design-review/SKILL.md index 14d0dd717..e0f69f890 100644 --- a/plan-design-review/SKILL.md +++ b/plan-design-review/SKILL.md @@ -1030,7 +1030,7 @@ HARD RULES — first classify as MARKETING/LANDING PAGE vs APP UI vs HYBRID, the For each finding: what's wrong, what will happen if it ships unresolved, and the specific fix. Be opinionated. No hedging." -Write the **complete prompt and context**, including actual plan/spec/source, to a private file. Substitute its shell-quoted path for ``; never interpolate user text into shell source. Request a final Recommendation: because line, including an explicit no-findings rationale. +Write the **complete prompt and context**, including actual plan/spec/source, to a private file. Substitute its shell-quoted path for ``; never interpolate user text into shell source. Request a severity (Critical, High, Medium or Low) per finding and a final Recommendation: because line, including an explicit no-findings rationale. ```bash # GSTACK_ACTIVE_HOST names the harness, never the model. diff --git a/plan-devex-review/sections/review-sections.md b/plan-devex-review/sections/review-sections.md index 037429cb9..9e5980a03 100644 --- a/plan-devex-review/sections/review-sections.md +++ b/plan-devex-review/sections/review-sections.md @@ -423,7 +423,7 @@ Run the selected backend in the one foreground Bash invocation described below. Finish a failed attempt's termination before fallback; consume only its completed output. No background jobs or shared temporary paths. -Write the **complete prompt and context**, including actual plan/spec/source, to a private file. Substitute its shell-quoted path for ``; never interpolate user text into shell source. Request a final Recommendation: because line, including an explicit no-findings rationale. +Write the **complete prompt and context**, including actual plan/spec/source, to a private file. Substitute its shell-quoted path for ``; never interpolate user text into shell source. Request a severity (Critical, High, Medium or Low) per finding and a final Recommendation: because line, including an explicit no-findings rationale. ```bash # GSTACK_ACTIVE_HOST names the harness, never the model. diff --git a/plan-eng-review/sections/review-sections.md b/plan-eng-review/sections/review-sections.md index 394a44d12..a0a56b1fc 100644 --- a/plan-eng-review/sections/review-sections.md +++ b/plan-eng-review/sections/review-sections.md @@ -928,7 +928,7 @@ Its opening harness guard rechecks the fresh shell: exit 78 uses the same Native fallback below, never a replacement provider. Finish termination before fallback and consume only completed output. Use private temporary paths, with no background jobs. -Create a private prompt file: run `umask 077; mktemp "${TMPDIR:-/tmp}/gstack-plan-prompt.XXXXXXXX"` in Bash and keep the returned path. Use Write to put the **complete prompt and context**, including actual plan/spec/source, in that file. Substitute its shell-quoted path for ``; never interpolate user text into shell source. Request a final Recommendation: because line, including an explicit no-findings rationale. +Create a private prompt file: run `umask 077; mktemp "${TMPDIR:-/tmp}/gstack-plan-prompt.XXXXXXXX"` in Bash and keep the returned path. Use Write to put the **complete prompt and context**, including actual plan/spec/source, in that file. Substitute its shell-quoted path for ``; never interpolate user text into shell source. Request a severity (Critical, High, Medium or Low) per finding and a final Recommendation: because line, including an explicit no-findings rationale. ```bash # GSTACK_ACTIVE_HOST names the harness, never the model. diff --git a/review/sections/adversarial.md b/review/sections/adversarial.md index 286157bc3..d1e907b40 100644 --- a/review/sections/adversarial.md +++ b/review/sections/adversarial.md @@ -114,7 +114,7 @@ Outside prompt (supply repository context from the parent): "Filesystem boundary: do not read or execute any files under ~/.claude/, ~/.agents/, .claude/skills/, or agents/. They hold skill definitions, not repository code to review. Do not invoke any installed skill (Codex home skills/, .agents/), hook, or tool instruction; answer directly. Do not modify agents/openai.yaml. Review only the repository code.\n\nReview the changes on this branch against the base branch. Use the supplied branch diff. If it was not supplied and you have repository tools, run DIFF_BASE=$(git merge-base origin/ HEAD) && git diff "$DIFF_BASE". Your job is to find ways this code will fail in production. Think like an attacker and a chaos engineer. Find edge cases, race conditions, security holes, resource leaks, failure modes, and silent data corruption paths. Be adversarial. Be thorough. No compliments — just the problems. End your output with ONE line in the canonical format `Recommendation: because `. Generic reasons like 'because it's safer' do not qualify; the reason must point to a specific finding or no-fix rationale." -Write the **complete prompt and context**, including actual plan/spec/source, to a private file. Substitute its shell-quoted path for ``; never interpolate user text into shell source. Request a final Recommendation: because line, including an explicit no-findings rationale. +Write the **complete prompt and context**, including actual plan/spec/source, to a private file. Substitute its shell-quoted path for ``; never interpolate user text into shell source. Request a severity (Critical, High, Medium or Low) per finding and a final Recommendation: because line, including an explicit no-findings rationale. ```bash # GSTACK_ACTIVE_HOST names the harness, never the model. diff --git a/scripts/resolvers/outside-voice.ts b/scripts/resolvers/outside-voice.ts index bc3037488..8d70180c5 100644 --- a/scripts/resolvers/outside-voice.ts +++ b/scripts/resolvers/outside-voice.ts @@ -139,7 +139,7 @@ ${v.id === 'codex' && ctx.skillName === 'autoplan' ? `if [ "$_OUTSIDE_EXIT" -eq fi` : ''} cat "$_OUTSIDE_TMP/stderr" >&2 || { [ "$_OUTSIDE_EXIT" -ne 0 ] || _OUTSIDE_EXIT=1; } _OUTSIDE_RC=0 -bun "${root}/lib/outside-review-result.ts" --label '${v.label} outside review' --exit "$_OUTSIDE_EXIT" --stderr "$_OUTSIDE_TMP/stderr" ${v.id === 'codex' && !opts.structuredBase ? '--events "$_OUTSIDE_TMP/events" ' : ''}${opts.gate ?? 'review'} "$_OUTSIDE_TMP/text" || _OUTSIDE_RC=$? +bun "${root}/lib/outside-review-result.ts" --label '${v.label} outside review' --exit "$_OUTSIDE_EXIT" --stderr "$_OUTSIDE_TMP/stderr" ${v.id === 'codex' && !opts.structuredBase ? '--events "$_OUTSIDE_TMP/events" ' : ''}${opts.purpose === 'design-direction' ? 'proposal' : opts.gate ?? 'review'} "$_OUTSIDE_TMP/text" || _OUTSIDE_RC=$? ${v.id === 'claude-code' ? '[ "$_OUTSIDE_RC" -eq 1 ] || cat "$_OUTSIDE_TMP/text" || exit 1\n' : ''}case "$_OUTSIDE_RC" in 0|3) ;; 4) echo 'OUTSIDE_STATUS: unverified provider=${v.id} host=${ctx.host}'; exit 4 ;; @@ -166,7 +166,7 @@ export function outsideVoiceInvocation(ctx: TemplateContext, requested: OutsideC ? 'Request severity-tagged findings or an explicit NO_FINDINGS conclusion.' : opts.purpose === 'design-direction' ? 'Request a complete design proposal ending with Recommendation: because .' - : 'Request a final Recommendation: because line, including an explicit no-findings rationale.'; + : 'Request a severity (Critical, High, Medium or Low) per finding and a final Recommendation: because line, including an explicit no-findings rationale.'; const preparation = nativeStructured ? 'Run Codex’s built-in structured review with the selected base. It supplies its own prompt and accepts no custom prompt file with --base. Require severity-tagged findings (including native P1:/P2: labels) or an explicit no-findings conclusion; arbitrary prose or a refusal is missing coverage.' : `${['plan-ceo-review', 'plan-eng-review'].includes(ctx.skillName) diff --git a/ship/sections/adversarial.md b/ship/sections/adversarial.md index fd15fa6a4..fcb652699 100644 --- a/ship/sections/adversarial.md +++ b/ship/sections/adversarial.md @@ -114,7 +114,7 @@ Outside prompt (supply repository context from the parent): "Filesystem boundary: do not read or execute any files under ~/.claude/, ~/.agents/, .claude/skills/, or agents/. They hold skill definitions, not repository code to review. Do not invoke any installed skill (Codex home skills/, .agents/), hook, or tool instruction; answer directly. Do not modify agents/openai.yaml. Review only the repository code.\n\nReview the changes on this branch against the base branch. Use the supplied branch diff. If it was not supplied and you have repository tools, run DIFF_BASE=$(git merge-base origin/ HEAD) && git diff "$DIFF_BASE". Your job is to find ways this code will fail in production. Think like an attacker and a chaos engineer. Find edge cases, race conditions, security holes, resource leaks, failure modes, and silent data corruption paths. Be adversarial. Be thorough. No compliments — just the problems. End your output with ONE line in the canonical format `Recommendation: because `. Generic reasons like 'because it's safer' do not qualify; the reason must point to a specific finding or no-fix rationale." -Write the **complete prompt and context**, including actual plan/spec/source, to a private file. Substitute its shell-quoted path for ``; never interpolate user text into shell source. Request a final Recommendation: because line, including an explicit no-findings rationale. +Write the **complete prompt and context**, including actual plan/spec/source, to a private file. Substitute its shell-quoted path for ``; never interpolate user text into shell source. Request a severity (Critical, High, Medium or Low) per finding and a final Recommendation: because line, including an explicit no-findings rationale. ```bash # GSTACK_ACTIVE_HOST names the harness, never the model. diff --git a/ship/sections/review-army.md b/ship/sections/review-army.md index 9564d545f..e1cff613c 100644 --- a/ship/sections/review-army.md +++ b/ship/sections/review-army.md @@ -163,7 +163,7 @@ If Codex is available, run a lightweight design check on the diff: Prompt: "Review the git diff on this branch. Run 7 litmus checks (YES/NO each): 1. Brand/product unmistakable in first screen? 2. One strong visual anchor present? 3. Page understandable by scanning headlines only? 4. Each section has one job? 5. Are cards actually necessary? 6. Does motion improve hierarchy or atmosphere? 7. Would design feel premium with all decorative shadows removed? Flag any hard rejections: 1. Generic SaaS card grid as first impression 2. Beautiful image with weak brand 3. Strong headline with no clear action 4. Busy imagery behind text 5. Sections repeating same mood statement 6. Carousel with no narrative purpose 7. App UI made of stacked cards instead of layout 5 most important design findings only. Reference file:line." -Write the **complete prompt and context**, including actual plan/spec/source, to a private file. Substitute its shell-quoted path for ``; never interpolate user text into shell source. Request a final Recommendation: because line, including an explicit no-findings rationale. +Write the **complete prompt and context**, including actual plan/spec/source, to a private file. Substitute its shell-quoted path for ``; never interpolate user text into shell source. Request a severity (Critical, High, Medium or Low) per finding and a final Recommendation: because line, including an explicit no-findings rationale. ```bash # GSTACK_ACTIVE_HOST names the harness, never the model. diff --git a/test/design-consultation-contract.test.ts b/test/design-consultation-contract.test.ts index 2572c29d4..1081aa327 100644 --- a/test/design-consultation-contract.test.ts +++ b/test/design-consultation-contract.test.ts @@ -16,7 +16,7 @@ for (const { name: host } of ALL_HOST_CONFIGS) { const prompt = text.match(/"(Given this product context, propose a complete design direction:[\s\S]*?)"\n/)!; expect(prompt).not.toBeNull(); expect(prompt[1]).toContain('Recommendation: because '); - expect(validateOutsideReview('Recommendation: use a compact triage table because operators compare many incident rows.', 'review').completed).toBe(true); + expect(validateOutsideReview('Recommendation: use a compact triage table because operators compare many incident rows.', 'proposal').completed).toBe(true); expect(validateOutsideReview('A compact table sounds nice.', 'review').completed).toBe(false); const preparation = outsideVoiceInvocation(context(host), { timeoutMs: 300000, purpose: 'design-direction' }); expect(preparation).toContain('missing Recommendation marker'); diff --git a/test/fixtures/outside-review-severity/ceo-codex-shape.txt b/test/fixtures/outside-review-severity/ceo-codex-shape.txt new file mode 100644 index 000000000..f951d79e1 --- /dev/null +++ b/test/fixtures/outside-review-severity/ceo-codex-shape.txt @@ -0,0 +1,17 @@ +Reviewed the CEO implementation snapshot end to end, including the release promises, Tier 1 items and the validation plan. + +1. **High** — P1 promises more than its acceptance proves. The plan declares /autoplan recovered on current Claude Code, but the only live proof is a single foreground phase entry. Require live phase transitions in the foreground and through `claude attach`, or narrow P1 and leave the issue open. + +2. **High** — The plan deepens the private-journal dependency. The payload path, flush rules and allowlist all read Claude Code's private journal. Time-box a supported-hook-event alternative, prove the canary catches a deliberate incompatibility, and test a working fallback. + +3. **High** — A4 can publish inconsistent artifact groups. Holding one flagged file while pushing its siblings can leave a decision log without its active snapshot on the remote. Hold coupled groups and restore onto a second machine in acceptance. + +4. **High** — Optional work can hold urgent recovery. Tier 2 rides in the same PR as the P0 fix. Keep one PR, but name the mandatory set and an admission cutoff, and produce the eval schedule and its cost before Tier 2 is admitted. + +5. **Medium** — B2 may not deliver faster review. Pushing early only helps when Greptile finishes before /ship's other passes. Measure it and keep it optional. + +6. **Medium** — There is no denominator to show the guards stopped crying wolf. Log every guarded attempt and its outcome, not only denials. + +7. **Medium** — The blanket contributor rewrite loses tested edge cases. Reuse contributors' regression tests when they pass review. + +Recommendation: Revise before implementation because the plan can declare recovery without proving its release promises and can publish partially usable state while optional work delays the fixes. diff --git a/test/fixtures/outside-review-severity/dx-codex-shape.txt b/test/fixtures/outside-review-severity/dx-codex-shape.txt new file mode 100644 index 000000000..0dd99a333 --- /dev/null +++ b/test/fixtures/outside-review-severity/dx-codex-shape.txt @@ -0,0 +1,27 @@ +Developer experience challenge for the follow-up wave plan. + +### Finding 1: Post-after-approval has no contract +Severity: High +The posting helper refuses a MEDIUM finding but says nothing about how the caller resumes after the user approves. Define the confirmation and its exit codes. + +### Finding 2: select-model drops the per-request model precedence +Severity: High +The subcommand list names `select-model ` with no `--model`, so a model named for one request silently falls back to the environment or config. + +### Finding 3: One-release sourcing window is unsafe +Severity: High +Skills rendered before the upgrade keep sourcing the probe; a single release can be skipped by users who upgrade monthly. Keep sourcing for at least one release and 14 days, with a deprecation line. + +### Finding 4: The early PR is still automatic +Severity: High +B2 pushes and opens a draft whenever Greptile is detected, while the challenge to make it opt-in is still open. + +Medium: Doctor cannot tell an intentionally absent Codex from a broken one; give each row ok, warn, not configured or fail. + +Medium: Recovery docs are not an acceptance requirement, and docs/troubleshooting.md still teaches sourcing the probe. + +Medium: --skip-file is permanent with no undo; add an unskip command. + +Medium: The novice hello-world journey under five minutes is unproven. + +Recommendation: revise before implementation because approval resumption, model-selection compatibility, upgrade continuity, and early-publication defaults lack complete developer-facing contracts. diff --git a/test/fixtures/outside-review-severity/eng-codex-tagged.txt b/test/fixtures/outside-review-severity/eng-codex-tagged.txt new file mode 100644 index 000000000..cb7f52b4b --- /dev/null +++ b/test/fixtures/outside-review-severity/eng-codex-tagged.txt @@ -0,0 +1,16 @@ +Architecture challenge. + +- [P1] Same-phase batches are still timing-dependent on the journal path — autoplan/bin/phase-publication-hook.ts:476 + A batch whose second call is journaled before the first is evaluated differently depending on flush order. +- [P1] A 10-minute age does not prove an index lock is abandoned — bin/gstack-skill-start:326 + The skill-start merge and artifacts init run git outside the drain lock. +- [P1] Rule-id confirmation cannot bind exact bytes — scripts/resolvers/redact-doc.ts:79 + Replacing one flagged email with another keeps the same rule ids and lines. +- [P1] Holding coupled files does not make publication atomic — bin/gstack-decision-log:84 + Only the log is enqueued; the snapshot follows later. +- [P1] A failed commit still finalizes the queue as idle — bin/gstack-brain-sync:763 +- [P2] A2 bounds record size, not total memory or evaluation cost — autoplan/bin/phase-publication-hook.ts:441 +- [P1] The long-session PTY case would be observed by the 32 MiB counting reader — lib/claude-public-transcript.ts:349 +- [P2] A3 line numbers are slice-relative — bin/gstack-redact-prepush:310 + +Recommendation: Revise before implementation because batch handling, sync transactions, posting confirmations, and long-session validation still contain concrete correctness gaps. diff --git a/test/fixtures/outside-review-severity/prose-negative.txt b/test/fixtures/outside-review-severity/prose-negative.txt new file mode 100644 index 000000000..67268279b --- /dev/null +++ b/test/fixtures/outside-review-severity/prose-negative.txt @@ -0,0 +1,5 @@ +This is a high-level plan with low-risk changes and a medium-term follow-up. The design is highly consistent, and there are no critical findings. A high number of the steps are already covered by tests, and the low cost of the rollback keeps the change safe. + +No issues found in the changed helpers. + +Recommendation: proceed because the high-quality tests cover every changed path. diff --git a/test/fixtures/outside-review-severity/prose-unlabeled.txt b/test/fixtures/outside-review-severity/prose-unlabeled.txt new file mode 100644 index 000000000..75ed98f2c --- /dev/null +++ b/test/fixtures/outside-review-severity/prose-unlabeled.txt @@ -0,0 +1,3 @@ +The plan's high-level structure is sound, but the rollback path writes the queue before the lock is held, and the docs still describe the old flag. The medium-term cleanup can wait. + +Recommendation: fix the rollback ordering because a concurrent drain can lose records. diff --git a/test/fixtures/outside-review-severity/table-shape.txt b/test/fixtures/outside-review-severity/table-shape.txt new file mode 100644 index 000000000..9bd9dfc06 --- /dev/null +++ b/test/fixtures/outside-review-severity/table-shape.txt @@ -0,0 +1,6 @@ +| # | Severity | Finding | Fix | +|---|---|---|---| +| 1 | high | The retry loop ignores the configured cap. | Check the cap before each attempt. | +| 2 | medium | The backoff constant is duplicated. | Import it from config. | + +Recommendation: fix the retry cap because callers can loop forever. diff --git a/test/helpers/carve-guards.ts b/test/helpers/carve-guards.ts index 4a5cad011..acc749573 100644 --- a/test/helpers/carve-guards.ts +++ b/test/helpers/carve-guards.ts @@ -166,7 +166,7 @@ export const CARVE_GUARDS: Record = { // wave's headline capability) grows the union to 1.195x. Deliberate: // the section is on-demand (loads only for Apple store targets), so // per-invocation cost for non-iOS ships is one manifest line. - maxSizeRatio: 1.4995, // Shared advisory identity/dedup + critical-severity validation: 248,065 union bytes / 187,706 baseline = 1.3216 (2026-09-17). + test value bar in the lazy Step 7 section (value cards, weak paths, gate table, base control, machine checks; ~13.6KB): measured 1.396 (2026-09-29). + W1 guarded state-root resolution (`eval gstack-paths; : "${GSTACK_STATE_ROOT:?…}"`) in the Context Recovery preamble, the eureka log and each state-writing bash block; measured 1.401 (2026-09-30). + the shared QA review step's plan-check timing rule (plan checks and their revalidation run on --timeout-ms after smoke expiry); measured 1.4022 (2026-09-30). + the shared TEST_BOOTSTRAP owned-change undo rules (s03 H1-H3, M1) and the value-bar coverage wording (s03 M2/M3); measured 1.4044 (2026-10-02). + severe fix wave ship-review lane: G1 NO_VERSION dispatch, title and CHANGELOG arms (versionless repos ship without inventing a version), F1 parent-side plan binding before the audit dispatch (binding, docs/designs candidates, exact not-run line) and D3's data-not-instructions line; +2,126 union bytes, measured 1.4135 (2026-10-03). + INV-1 consumer lines (outside review `unverified`/`unavailable` is missing coverage in the readiness note and PR body); measured 1.4148 (2026-10-03). + v1.91.19.0 severe fix wave: verdict-form outside-review callers with sandbox preflight and first-use notice (INV-1/B1/Q2), P0/P1 gate and unverified wording, bound-plan discovery (F1), visible LEARNINGS failures (B5), carved external pointers; measured 267,752 / 187,706 = 1.4264 (2026-10-03). + v1.91.18.0 (#3023) merged into the v1.91.19.0 wave; measured 267,874 / 187,706 = 1.4271 (2026-10-04). + Oct 6 fix wave: quota_exhausted preflight branch (#3051) and quoted-heredoc issue template (#3046); measured 268,199 / 187,706 = 1.4288 (2026-10-06). + Step 10's inline no-PR check before the Greptile dispatch (#3020); measured 268,655 / 187,706 = 1.4313 (2026-10-06). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); measured 275,422 / 187,706 = 1.4673 (2026-10-06). Combined Oct 6 wave head (B free-text files + D measure loop + A rework): measured 280,363 / 187,706 = 1.4936 (2026-10-06). + gate-once push rule (no second paid gate after a CI push); measured 280,573 / 187,706 = 1.4947 (2026-10-06). + the measurement bar's decision step (exit-code branches for MEETS, MEETS-qualified, needs-classify with the qualifying and fix-round classes, EXTEND once; the measure prose was tightened to offset most of it); measured 281,058 / 187,706 = 1.4973 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 281,438 / 187,706 = 1.4994 (2026-10-07). + maxSizeRatio: 1.5001, // Shared advisory identity/dedup + critical-severity validation: 248,065 union bytes / 187,706 baseline = 1.3216 (2026-09-17). + test value bar in the lazy Step 7 section (value cards, weak paths, gate table, base control, machine checks; ~13.6KB): measured 1.396 (2026-09-29). + W1 guarded state-root resolution (`eval gstack-paths; : "${GSTACK_STATE_ROOT:?…}"`) in the Context Recovery preamble, the eureka log and each state-writing bash block; measured 1.401 (2026-09-30). + the shared QA review step's plan-check timing rule (plan checks and their revalidation run on --timeout-ms after smoke expiry); measured 1.4022 (2026-09-30). + the shared TEST_BOOTSTRAP owned-change undo rules (s03 H1-H3, M1) and the value-bar coverage wording (s03 M2/M3); measured 1.4044 (2026-10-02). + severe fix wave ship-review lane: G1 NO_VERSION dispatch, title and CHANGELOG arms (versionless repos ship without inventing a version), F1 parent-side plan binding before the audit dispatch (binding, docs/designs candidates, exact not-run line) and D3's data-not-instructions line; +2,126 union bytes, measured 1.4135 (2026-10-03). + INV-1 consumer lines (outside review `unverified`/`unavailable` is missing coverage in the readiness note and PR body); measured 1.4148 (2026-10-03). + v1.91.19.0 severe fix wave: verdict-form outside-review callers with sandbox preflight and first-use notice (INV-1/B1/Q2), P0/P1 gate and unverified wording, bound-plan discovery (F1), visible LEARNINGS failures (B5), carved external pointers; measured 267,752 / 187,706 = 1.4264 (2026-10-03). + v1.91.18.0 (#3023) merged into the v1.91.19.0 wave; measured 267,874 / 187,706 = 1.4271 (2026-10-04). + Oct 6 fix wave: quota_exhausted preflight branch (#3051) and quoted-heredoc issue template (#3046); measured 268,199 / 187,706 = 1.4288 (2026-10-06). + Step 10's inline no-PR check before the Greptile dispatch (#3020); measured 268,655 / 187,706 = 1.4313 (2026-10-06). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); measured 275,422 / 187,706 = 1.4673 (2026-10-06). Combined Oct 6 wave head (B free-text files + D measure loop + A rework): measured 280,363 / 187,706 = 1.4936 (2026-10-06). + gate-once push rule (no second paid gate after a CI push); measured 280,573 / 187,706 = 1.4947 (2026-10-06). + the measurement bar's decision step (exit-code branches for MEETS, MEETS-qualified, needs-classify with the qualifying and fix-round classes, EXTEND once; the measure prose was tightened to offset most of it); measured 281,058 / 187,706 = 1.4973 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 281,438 / 187,706 = 1.4994 (2026-10-07). + Oct 7 wave CEO-28/DX-11: review prompts ask for a severity per finding and design proposals use the proposal gate; measured 281,556 / 187,706 = 1.5000 (2026-10-07). }, 'plan-ceo-review': { skill: 'plan-ceo-review', @@ -221,7 +221,7 @@ export const CARVE_GUARDS: Record = { // 1.08 → 1.10: the scope-gate exceptions block (+ its adversarial-review // hardening: host-anchored mode signal, precedence, passing-mention // guards) and the plan-mode preamble reword land the union at 1.092. - maxSizeRatio: 1.1855, // + clarity rules for saved decisions/setup gates + the Aside probe's failure reason; measured 1.1504. + test value bar and Tests to Retire in the lazy Test review section (~2.6KB); measured 1.168 + W1 guarded state-root resolution (`eval gstack-paths; : "${GSTACK_STATE_ROOT:?…}"`) in the Context Recovery preamble, the eureka log and each state-writing bash block; measured 1.173 (2026-09-30). + v1.91.12.0 merge of #2999 (headless rule: a disallowed question tool never qualifies) with #3002; measured 1.1741 (2026-10-01). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); measured 146,905 / 124,597 = 1.1790 (2026-10-06). Combined Oct 6 wave head (A and B): measured 147,524 / 124,597 = 1.1840 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 147,693 / 124,597 = 1.1854 (2026-10-07). + maxSizeRatio: 1.1859, // + clarity rules for saved decisions/setup gates + the Aside probe's failure reason; measured 1.1504. + test value bar and Tests to Retire in the lazy Test review section (~2.6KB); measured 1.168 + W1 guarded state-root resolution (`eval gstack-paths; : "${GSTACK_STATE_ROOT:?…}"`) in the Context Recovery preamble, the eureka log and each state-writing bash block; measured 1.173 (2026-09-30). + v1.91.12.0 merge of #2999 (headless rule: a disallowed question tool never qualifies) with #3002; measured 1.1741 (2026-10-01). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); measured 146,905 / 124,597 = 1.1790 (2026-10-06). Combined Oct 6 wave head (A and B): measured 147,524 / 124,597 = 1.1840 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 147,693 / 124,597 = 1.1854 (2026-10-07). + Oct 7 wave CEO-28/DX-11: review prompts ask for a severity per finding and design proposals use the proposal gate; measured 147,752 / 124,597 = 1.1858 (2026-10-07). }, 'plan-design-review': { skill: 'plan-design-review', @@ -243,7 +243,7 @@ export const CARVE_GUARDS: Record = { // tier-2+ skeleton (measured 89,184). Main's v1.64.0.0 adds ~340 B more // (telemetry --error-message/--failed-step preamble prose, PR #769). // Budget covers the sum of both waves. - maxSkeletonBytes: 84_850, // Harness-aware outside voice: validated dispatch and provenance. + v1.91.19.0 wave: design --version probe (B4) and verdict-form outside voice (INV-1) on top of v1.91.17.0's round accounting (#3022); measured 80,443 (2026-10-03). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); skeleton measured 84,669 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; skeleton measured 84,804 (2026-10-07). + maxSkeletonBytes: 84_900, // Harness-aware outside voice: validated dispatch and provenance. + v1.91.19.0 wave: design --version probe (B4) and verdict-form outside voice (INV-1) on top of v1.91.17.0's round accounting (#3022); measured 80,443 (2026-10-03). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); skeleton measured 84,669 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; skeleton measured 84,804 (2026-10-07). + Oct 7 wave CEO-28/DX-11: review prompts ask for a severity per finding and design proposals use the proposal gate; skeleton measured 84,863 (2026-10-07). minUnionBytes: 99_200, // token-reduction Phases 1-2 (v1.69.x branch); measured union 110,293 mustContain: ['design', 'visual'], maxSizeRatio: 1.12, // D1 1.104 + main's ~0.008 @@ -348,10 +348,10 @@ do not launch the downstream skill or open a browser.`, // the #538 opt-out + D1 evidence directive — ratio 1.104 measured. // #2499 project-scope MCP jq in the brain-sync block grew every tier-2+ // skeleton ~1.5KB (entry resolution emitted once per SKILL.md). - maxSkeletonBytes: 92_900, // Office-hours + sketch outside voices include host guards and completion checks. + v1.91.19.0 wave: Q open-question prose form (#2729, approved Q3), DESIGN_READY --version probe (B4), Aside ~/.local/bin fallback (E7), verdict-form outside voice (INV-1), plus v1.91.17.0's round accounting (#3022) merged in; measured 89,684 (2026-10-03). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); skeleton measured 92,590 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; skeleton measured 92,869 (2026-10-07). + maxSkeletonBytes: 92_950, // Office-hours + sketch outside voices include host guards and completion checks. + v1.91.19.0 wave: Q open-question prose form (#2729, approved Q3), DESIGN_READY --version probe (B4), Aside ~/.local/bin fallback (E7), verdict-form outside voice (INV-1), plus v1.91.17.0's round accounting (#3022) merged in; measured 89,684 (2026-10-03). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); skeleton measured 92,590 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; skeleton measured 92,869 (2026-10-07). + Oct 7 wave CEO-28/DX-11: review prompts ask for a severity per finding and design proposals use the proposal gate; skeleton measured 92,930 (2026-10-07). minUnionBytes: 115_800, // Phase 4 wave 4; measured union 118,175 mustContain: ['design doc', 'problem statement'], - maxSizeRatio: 1.1445, // + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); measured 149,758 / 131,110 = 1.1422 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 150,037 / 131,110 = 1.1444 (2026-10-07). + maxSizeRatio: 1.1449, // + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); measured 149,758 / 131,110 = 1.1422 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 150,037 / 131,110 = 1.1444 (2026-10-07). + Oct 7 wave CEO-28/DX-11: review prompts ask for a severity per finding and design proposals use the proposal gate; measured 150,098 / 131,110 = 1.1448 (2026-10-07). }, 'document-release': { skill: 'document-release', @@ -485,7 +485,7 @@ do not launch the downstream skill or open a browser.`, maxSkeletonBytes: 74_881, // Shared-code identity/skip/action rules + critical-severity validation; measured 74,493 (2026-09-17). + v1.91.12.0 merge of #2999 (review clarity repairs: await reads, research alongside dispatch, /review deadline and setup authority, findings sources) with #3002 (guarded state-root lines, plan-check checkpoints); each fit alone; measured 74,881 (2026-10-01). minUnionBytes: 89_000, // Phase 4 wave 1; measured union 93,357 mustContain: ['confidence', 'P1', 'P2', 'Review Army', 'adversarial'], - maxSizeRatio: 1.2183, // Shared-code feature + critical-severity validation: 128,042 union bytes / 108,523 baseline = 1.1799; preserves content floors. + v1.91.12.0 merge of #2999 (above, plus plan-completion fallback intent and specialist checklist-by-path) with #3002; measured 1.1843 (2026-10-01). + v1.91.19.0 wave: verdict-form Codex callers with sandbox preflight (INV-1/B1), P0/P1 structured gate (B1b), bound-plan discovery and data-not-instructions guard (F1/D3), visible LEARNINGS failures (B5); measured 129,941 / 108,523 = 1.1974 (2026-10-03). + v1.91.18.0 (#3023) merged into the v1.91.19.0 wave; measured 130,025 / 108,523 = 1.1981 (2026-10-04). + Oct 6 fix wave: quota_exhausted preflight branch (#3051); measured 130,359 / 108,523 = 1.2012 (2026-10-06). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); measured 131,314 / 108,523 = 1.2100 (2026-10-06). Combined Oct 6 wave head (A and B): measured 131,966 / 108,523 = 1.2160 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 132,199 / 108,523 = 1.2182 (2026-10-07). + maxSizeRatio: 1.2188, // Shared-code feature + critical-severity validation: 128,042 union bytes / 108,523 baseline = 1.1799; preserves content floors. + v1.91.12.0 merge of #2999 (above, plus plan-completion fallback intent and specialist checklist-by-path) with #3002; measured 1.1843 (2026-10-01). + v1.91.19.0 wave: verdict-form Codex callers with sandbox preflight (INV-1/B1), P0/P1 structured gate (B1b), bound-plan discovery and data-not-instructions guard (F1/D3), visible LEARNINGS failures (B5); measured 129,941 / 108,523 = 1.1974 (2026-10-03). + v1.91.18.0 (#3023) merged into the v1.91.19.0 wave; measured 130,025 / 108,523 = 1.1981 (2026-10-04). + Oct 6 fix wave: quota_exhausted preflight branch (#3051); measured 130,359 / 108,523 = 1.2012 (2026-10-06). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); measured 131,314 / 108,523 = 1.2100 (2026-10-06). Combined Oct 6 wave head (A and B): measured 131,966 / 108,523 = 1.2160 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 132,199 / 108,523 = 1.2182 (2026-10-07). + Oct 7 wave CEO-28/DX-11: review prompts ask for a severity per finding and design proposals use the proposal gate; measured 132,258 / 108,523 = 1.2187 (2026-10-07). }, codex: { skill: 'codex', @@ -575,7 +575,7 @@ do not launch the downstream skill or open a browser.`, maxSkeletonBytes: 70_000, // Phase-specific outside coverage, native fallback, and harness guard. minUnionBytes: 85_000, // measured union 86,926 mustContain: ['6 Decision Principles', 'TASTE DECISION', 'USER CHALLENGE', 'consensus', 'Restore Point'], - maxSizeRatio: 1.121, // Four validated outside invocations replace raw CLI calls; phases keep independent coverage. + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 114,297 / 101,979 = 1.1208 (2026-10-07). + maxSizeRatio: 1.1232, // Four validated outside invocations replace raw CLI calls; phases keep independent coverage. + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 114,297 / 101,979 = 1.1208 (2026-10-07). + Oct 7 wave CEO-28/DX-11: review prompts ask for a severity per finding and design proposals use the proposal gate; measured 114,533 / 101,979 = 1.1231 (2026-10-07). }, spec: { skill: 'spec', diff --git a/test/helpers/free-fixtures-data.ts b/test/helpers/free-fixtures-data.ts index 0e45678e6..ee86098c1 100644 --- a/test/helpers/free-fixtures-data.ts +++ b/test/helpers/free-fixtures-data.ts @@ -52,7 +52,7 @@ export const FREE_FIXTURES: Readonly> = { 'test/fixtures/claude-native-journal-roots-2.1.284.json': ['test/autoplan-publication-hook.test.ts', 'test/plan-count-transcript.test.ts'], 'test/fixtures/codex-boundary-commands.json': ['test/codex-boundary-evidence.test.ts'], 'test/fixtures/codex-review-sandbox-unavailable-37158847998.json': ['test/outside-voice-evidence.test.ts'], - 'test/fixtures/codex-review/**': ['test/codex-hardening.test.ts', 'test/outside-voice-evidence.test.ts', 'test/skill-validation.test.ts'], + 'test/fixtures/codex-review/**': ['test/codex-hardening.test.ts', 'test/outside-review-severity.test.ts', 'test/outside-voice-evidence.test.ts', 'test/skill-validation.test.ts'], 'test/fixtures/codex-sandbox/**': ['test/codex-eval-recording.test.ts', 'test/codex-model-probe.test.ts', 'test/outside-review-result.test.ts', 'test/outside-voice-invocation.test.ts'], 'test/fixtures/context-bill/**': ['test/context-bill.test.ts', 'test/context-budget-ratchet.test.ts'], 'test/fixtures/context-budget.json': ['test/helpers/capture-context-budget.ts', 'test/test-value-bar.test.ts'], @@ -128,6 +128,7 @@ export const FREE_FIXTURES: Readonly> = { 'test/fixtures/native-auq-devex-clipped-elided-sep21.json': ['test/auq-native-capture.test.ts'], 'test/fixtures/native-auq-margin-elided-sep21.json': ['test/auq-native-capture.test.ts'], 'test/fixtures/native-auq-packet-clipped-elided-sep21.json': ['test/auq-native-capture.test.ts'], + 'test/fixtures/outside-review-severity/**': ['test/outside-review-severity.test.ts'], 'test/fixtures/outside-voice-failure-prose-allowlist.json': ['test/outside-voice-failure-policy.test.ts'], 'test/fixtures/overlay-v4-replay-37198445662.json': ['test/overlay-measurement.test.ts'], 'test/fixtures/paid-report-census/**': ['test/paid-report-census.test.ts', 'test/paid-shards.test.ts'], diff --git a/test/outside-review-result.test.ts b/test/outside-review-result.test.ts index 908e182d0..5d3a2ab7e 100644 --- a/test/outside-review-result.test.ts +++ b/test/outside-review-result.test.ts @@ -27,11 +27,14 @@ function cli(args: string[]) { return { status: r.status, stdout: r.stdout, stderr: r.stderr }; } -const RECOMMEND = 'Recommendation: fix the guard because changed.ts loses data.'; +// A clean review states that it found nothing; an untagged finding is unverified (DX-11). +const RECOMMEND = 'No issues found in changed.ts.\nRecommendation: approve because changed.ts keeps its data.'; +const UNTAGGED = 'Recommendation: fix the guard because changed.ts loses data.'; describe('classifyOutsideReview: separate execution, findings and verdict', () => { const cases: Array<[string, { text: string; gate: OutsideGate; stderr?: string; exit?: number }, string, string | null, string | undefined]> = [ ['clean review', { text: RECOMMEND, gate: 'review' }, 'clean', null, undefined], + ['review with an untagged finding', { text: UNTAGGED, gate: 'review' }, 'unverified', null, 'untagged_review'], ['review with P1', { text: `[P1] data loss\n${RECOMMEND}`, gate: 'review' }, 'findings', 'P1', undefined], ['review with only P2', { text: `[P2] naming\n${RECOMMEND}`, gate: 'review' }, 'clean', 'P2', undefined], ['review missing recommendation', { text: 'A few observations.', gate: 'review' }, 'unavailable', null, 'missing_markers'], diff --git a/test/outside-review-severity.test.ts b/test/outside-review-severity.test.ts new file mode 100644 index 000000000..82f6c2b0d --- /dev/null +++ b/test/outside-review-severity.test.ts @@ -0,0 +1,110 @@ +/** + * CEO-28 / DX-11 / ENG-10: a review that lists high-severity findings is never + * `VERDICT: clean`. The Oct 7 /autoplan run's Codex CEO and DX outputs labeled + * seven and eight findings "High"/"Medium" without [Pn] tags, and the installed + * classifier printed `VERDICT: clean` / `FINDINGS: none` for both. The fixtures + * in test/fixtures/outside-review-severity/ reproduce those shapes (the + * findings come from the run's review record; the raw outputs were not kept), + * plus the eng output's [P1] tags, a table, and prose negative controls. + */ +import { describe, expect, test } from 'bun:test'; +import { spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { classifyOutsideReview, type OutsideGate } from '../lib/outside-review-result'; +import { HOST_PATHS } from '../scripts/resolvers/types'; +import { outsideVoiceCommand } from '../scripts/resolvers/outside-voice'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const LIB = path.join(ROOT, 'lib', 'outside-review-result.ts'); +const FIX = path.join(ROOT, 'test', 'fixtures', 'outside-review-severity'); +const fixture = (name: string) => path.join(FIX, name); +const read = (name: string) => fs.readFileSync(fixture(name), 'utf8'); +const REC = 'Recommendation: fix it because the change loses data.'; + +describe('severity words count as findings in label position', () => { + for (const [name, highest] of [ + ['ceo-codex-shape.txt', 'P1'], + ['dx-codex-shape.txt', 'P1'], + ['eng-codex-tagged.txt', 'P1'], + ['table-shape.txt', 'P1'], + ] as const) { + test(`${name}: the CLI prints VERDICT: findings, FINDINGS: ${highest}, exit 3`, () => { + const r = spawnSync(process.execPath, [LIB, '--verdict', 'review', fixture(name)], { encoding: 'utf8', timeout: 10000 }); + expect([r.status, r.stdout]).toEqual([3, `VERDICT: findings\nFINDINGS: ${highest}\n`]); + }); + } + + test('every label form ENG-10 names is read, with critical/high blocking and medium/low advisory', () => { + const cases: Array<[string, string | null, string]> = [ + ['Severity: High\nThe cap is ignored.', 'P1', 'findings'], + ['Priority: critical — data loss', 'P0', 'findings'], + ['Severity: **medium**', 'P2', 'clean'], + ['High: the retry loop never stops.', 'P1', 'findings'], + ['1. **High** — the cap is ignored.', 'P1', 'findings'], + ['- [High] the cap is ignored.', 'P1', 'findings'], + ['### Critical - secrets in logs', 'P0', 'findings'], + ['High — the cap is ignored.', 'P1', 'findings'], + ['The cap is ignored (**High**).', 'P1', 'findings'], + ['| 1 | high | cap ignored |', 'P1', 'findings'], + ['Low: rename the helper.', 'P3', 'clean'], + ['Medium: the docs lag.\nLow: a typo.', 'P2', 'clean'], + ]; + for (const [text, highest, verdict] of cases) { + const r = classifyOutsideReview({ text: `${text}\n${REC}`, gate: 'review' }); + expect({ text, highest: r.findings.highest as string | null, verdict: r.verdict as string }).toEqual({ text, highest, verdict }); + } + }); + + test('negative controls: severity words inside prose never count', () => { + const r = classifyOutsideReview({ text: read('prose-negative.txt'), gate: 'review' }); + expect([r.verdict, r.findings.highest]).toEqual(['clean', null]); + for (const text of ['A high-level plan.', 'Highly consistent.', 'This is low-risk.', 'A medium-term follow-up.', + 'There are no critical findings.', 'High availability: kept.', 'The high number of retries is fine.', 'Lower the limit.']) { + expect({ text, highest: classifyOutsideReview({ text: `${text}\nNo issues found.\n${REC}`, gate: 'review' }).findings.highest }) + .toEqual({ text, highest: null }); + } + }); + + test('a review with neither tags, severity labels nor a no-findings statement is unverified, never clean', () => { + const r = classifyOutsideReview({ text: read('prose-unlabeled.txt'), gate: 'review' }); + expect([r.verdict, r.reason]).toEqual(['unverified', 'untagged_review']); + const cli = spawnSync(process.execPath, [LIB, '--verdict', 'review', fixture('prose-unlabeled.txt')], { encoding: 'utf8', timeout: 10000 }); + expect(cli.status).toBe(4); + expect(cli.stdout).toContain('VERDICT: unverified\n'); + expect(classifyOutsideReview({ text: `No issues found.\n${REC}`, gate: 'review' }).verdict).toBe('clean'); + expect(classifyOutsideReview({ text: `I did not find any issues.\n${REC}`, gate: 'review' }).verdict).toBe('clean'); + }); + + test('the structured gate reads severity labels too', () => { + expect(classifyOutsideReview({ text: 'High: race in cleanup.', gate: 'structured' }).verdict).toBe('findings'); + expect(classifyOutsideReview({ text: 'Medium: stale comment.', gate: 'structured' }).verdict).toBe('clean'); + }); + + test('a stored panel of real Codex outputs keeps its verdicts', () => { + const panel = path.join(ROOT, 'test', 'fixtures', 'codex-review'); + const verdict = (name: string, gate: OutsideGate) => { + const r = classifyOutsideReview({ text: fs.readFileSync(path.join(panel, name), 'utf8'), gate }); + return [r.verdict, r.findings.highest, r.reason ?? null]; + }; + expect(verdict('clean-review.stdout.txt', 'structured')).toEqual(['unverified', null, 'untagged_review']); + expect(verdict('p2-review.stdout.txt', 'structured')).toEqual(['clean', 'P2', null]); + expect(verdict('clean-review.stdout.txt', 'review')).toEqual(['unavailable', null, 'missing_markers']); + expect(verdict('p2-review.stdout.txt', 'review')).toEqual(['unavailable', 'P2', 'missing_markers']); + }); +}); + +describe('design proposals use the proposal gate', () => { + test('a completed proposal needs only its Recommendation line; severity rules do not apply', () => { + const proposal = 'Use a dense triage table with sticky filters.\nRecommendation: table because operators compare many rows.'; + expect(classifyOutsideReview({ text: proposal, gate: 'proposal' }).verdict).toBe('clean'); + expect(classifyOutsideReview({ text: 'A table.', gate: 'proposal' }).reason).toBe('missing_markers'); + expect(classifyOutsideReview({ text: proposal, gate: 'review' }).verdict).toBe('unverified'); + }); + + test('design-direction invocations render the proposal gate; reviews keep the review gate', () => { + const ctx = { skillName: 'design-consultation', tmplPath: 'design-consultation/SKILL.md.tmpl', host: 'claude' as const, paths: HOST_PATHS.claude }; + expect(outsideVoiceCommand(ctx, { timeoutMs: 300000, purpose: 'design-direction' })).toMatch(/ proposal "\$_OUTSIDE_TMP\/text"/); + expect(outsideVoiceCommand({ ...ctx, skillName: 'plan-ceo-review' }, { timeoutMs: 300000 })).toMatch(/ review "\$_OUTSIDE_TMP\/text"/); + }); +}); diff --git a/test/outside-voice-invocation.test.ts b/test/outside-voice-invocation.test.ts index 08d7589fb..f9f35b286 100644 --- a/test/outside-voice-invocation.test.ts +++ b/test/outside-voice-invocation.test.ts @@ -34,7 +34,7 @@ if (process.env.FAKE_MODE === 'timeout') { await new Promise(() => {}); } if (process.env.FAKE_MODE === 'auth') { console.error('authentication_error: please log in'); process.exit(1); } -const response = process.env.FAKE_RESPONSE || 'Recommendation: fix the seeded defect because changed.ts loses data.'; +const response = process.env.FAKE_RESPONSE || 'Medium: changed.ts loses data on retry.\\nRecommendation: fix the seeded defect because changed.ts loses data.'; if (claude) { if (process.env.FAKE_MODE === 'malformed') {console.log('{broken');process.exit(0);} console.log(JSON.stringify({result:response,session_id:'outside-session',modelUsage:{'model-a':{inputTokens:4},'model-b':{inputTokens:8}}})); @@ -359,7 +359,7 @@ esac '### Recommendation: fix the guard because it loses data.', ]) { test(`formatted completion remains valid: ${response}`, () => { - expect(invoke('claude', {}, { FAKE_RESPONSE: response }).status).toBe(0); + expect(invoke('claude', {}, { FAKE_RESPONSE: `[P2] the guard drops one write.\n${response}` }).status).toBe(0); }); } diff --git a/test/outside-voice-preflight.test.ts b/test/outside-voice-preflight.test.ts index 8a67371d3..7cfb40807 100644 --- a/test/outside-voice-preflight.test.ts +++ b/test/outside-voice-preflight.test.ts @@ -145,7 +145,7 @@ function fixture() { if (git.status !== 0) throw new Error(git.stderr); const capture = path.join(dir, 'capture.json'); const fake = path.join(dir, 'fake-claude.ts'); - fs.writeFileSync(fake, `const prompt=await Bun.stdin.text(); await Bun.write(process.env.FIXTURE_CAPTURE!,JSON.stringify({prompt,args:process.argv.slice(2)})); console.log(JSON.stringify({result:'Recommendation: ship because the isolated fixture completed its review.'}));`); + fs.writeFileSync(fake, `const prompt=await Bun.stdin.text(); await Bun.write(process.env.FIXTURE_CAPTURE!,JSON.stringify({prompt,args:process.argv.slice(2)})); console.log(JSON.stringify({result:'No issues found.\\nRecommendation: ship because the isolated fixture completed its review.'}));`); const env = { ...process.env, HOME: home, CODEX_HOME: '', GSTACK_HOME: path.join(home, '.gstack'), GSTACK_ROOT: '', GSTACK_BIN: '', GSTACK_ACTIVE_HOST: 'codex', CODEX_THREAD_ID: 'fixture', CODEX_SANDBOX: '', CLAUDECODE: '', GSTACK_CLAUDE_BIN: process.execPath, GSTACK_CLAUDE_BIN_ARGS: JSON.stringify([fake]), FIXTURE_CAPTURE: capture }; diff --git a/test/outside-voice-receipt.test.ts b/test/outside-voice-receipt.test.ts index d13a234c7..b9b260c3b 100644 --- a/test/outside-voice-receipt.test.ts +++ b/test/outside-voice-receipt.test.ts @@ -5,7 +5,7 @@ import * as path from 'node:path'; import { createOutsideReceiptRuntime } from './helpers/outside-voice-receipt'; import { codexExecutionTranscript, foundInvoiceAuthorizationDefect } from './helpers/outside-voice-evidence'; -const finding = 'invoice.ts removes the owner check and exposes another user’s bankAccount.\nRecommendation: Restore authorization because invoice financial data crosses ownership boundaries.'; +const finding = 'High: invoice.ts removes the owner check and exposes another user’s bankAccount.\nRecommendation: Restore authorization because invoice financial data crosses ownership boundaries.'; function fixture() { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'outside-receipt-free-')); const real = path.join(dir, 'real runtime'), repo = path.join(dir, 'repo'), observed = path.join(dir, 'observed.json'); diff --git a/test/run-in-background-guidance.test.ts b/test/run-in-background-guidance.test.ts index 0e0870b1e..641b81daf 100644 --- a/test/run-in-background-guidance.test.ts +++ b/test/run-in-background-guidance.test.ts @@ -70,7 +70,7 @@ printf '%s\\n' "$p" printf '%s\\n' "$FAKE_REVIEW_ID" >> "$FAKE_CALLS" out=; prev=; for a in "$@"; do [ "$prev" = -o ] && out=$a; prev=$a; done cat > /dev/null -printf '%s\\n' "$FAKE_REVIEW_ID: current findings" "Recommendation: fix $FAKE_REVIEW_ID because this is the current finding." > "$out" +printf '%s\\n' "$FAKE_REVIEW_ID: current findings" "No issues found." "Recommendation: fix $FAKE_REVIEW_ID because this is the current finding." > "$out" printf '%s\\n' '{"type":"turn.completed"}' printf '%s\\n' "$FAKE_REVIEW_ID: current stderr" >&2 exit "$FAKE_CODEX_STATUS" @@ -101,7 +101,7 @@ exec ${quote(Bun.which('cat')!)} "$@" cleanup: () => fs.rmSync(dir, { recursive: true, force: true }) }; } - const findings = (id: string) => `${id}: current findings\nRecommendation: fix ${id} because this is the current finding.\n`; + const findings = (id: string) => `${id}: current findings\nNo issues found.\nRecommendation: fix ${id} because this is the current finding.\n`; // INV-1: the verdict-form validator prints its verdict before the completed status. const unavailable = (id: string) => `${findings(id)}VERDICT: unavailable\nFINDINGS: none\nREASON: execution_failed\n`; const completed = (id: string) => `${findings(id)}VERDICT: clean\nFINDINGS: none\nOUTSIDE_STATUS: completed provider=codex host=claude\n`;