diff --git a/changelog.d/capy-gsta-283-recommendation-heading.md b/changelog.d/capy-gsta-283-recommendation-heading.md new file mode 100644 index 000000000..0acbed625 --- /dev/null +++ b/changelog.d/capy-gsta-283-recommendation-heading.md @@ -0,0 +1 @@ +- **A Codex outside review that puts its recommendation under a `## Recommendation` heading now counts as finished.** The outside-review parser accepted only an inline `Recommendation: because ` line. When the model wrote the recommendation as a Markdown heading with the sentence on the next line, /ship and /codex reported the finished review as unavailable (`missing_markers`). A heading followed by a sentence with a `because` reason now reads the same as the inline line. A heading with nothing under it, a sentence with no `because`, a bullet or quote line under the heading, or a heading inside a code fence still reports the recommendation as missing. Severity labels and `Verdict:` lines under the heading are read exactly as before, so a blocking review still blocks. (#3980, thanks @nfitzsimmons) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index e675e7f7b..55232733a 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -350,8 +350,9 @@ it; `gstack-outside-voice run` sends that default to Anthropic only. ### `... outside review unavailable: the response lacks the markers this gate requires (...)` **Meaning.** The answer did not contain what the gate checks: a -`Recommendation: ... because ...` line, a `SCORE:`/`AMBIGUITIES:` pair, or -severity tags. +`Recommendation: ... because ...` line (or a `## Recommendation` heading whose +next non-blank line is a plain sentence with a `because` reason), a +`SCORE:`/`AMBIGUITIES:` pair, or severity tags. **Fix.** Re-run the review. diff --git a/lib/outside-review-result.ts b/lib/outside-review-result.ts index f982422d3..e2db1b4ac 100644 --- a/lib/outside-review-result.ts +++ b/lib/outside-review-result.ts @@ -192,6 +192,15 @@ function plainReview(text: string): string { return text.split(/\r?\n/).map(line => line.replace(/^[\t ]*(?:#{1,6}[\t ]+|[-+*][\t ]+)?/, '').replace(/[*_`]/g, '')).join('\n'); } +/** + * A `## Recommendation` heading and the prose line under it are the same marker as an inline `Recommendation:` line (#3980). + * Only the completion check and the no-findings conclusion read this view: a `P1:` or `Verdict:` line under the heading stays + * at line start for the severity and verdict parsers. + */ +function recommendationHeading(text: string): string { + return text.replace(/^[\t ]*#{1,6}[\t ]+Recommendation[\t ]*:?[\t ]*\r?\n(?:[\t ]*\r?\n)*(?![\t ]*(?:[#>]|[-+*][\t ]))(?=[^\r\n]*\bbecause\b)[\t ]*(\S[^\r\n]*)/gim, 'Recommendation: $1'); +} + /** * Code quotes text, so a severity inside a fence or an inline code span is not * a label. An inline span holding only a severity token in label position @@ -266,11 +275,12 @@ export function classifyOutsideReview(input: OutsideReviewInput): OutsideReviewC const reviewed = input.gate !== 'proposal'; const code = withoutCode(input.text), plainCode = plainReview(code); const read = withoutCode(input.text, true), plainRead = plainReview(read); + const marked = recommendationHeading(code); const levelsOf = (c: string, p: string) => [...[...p.matchAll(/\[(P[0-3])\]|^(P[0-3]):/gm)].map(m => (m[1] ?? m[2]) as Severity), ...severityTokens(c), ...(reviewed ? severityWords(c) : [])]; const levels = levelsOf(read, plainRead); const verdicts = [lastVerdict(plainCode), lastVerdict(plainRead)]; - const unlabeled = reviewed && verdicts[1] !== undefined && !concludesNoFindings(code) + const unlabeled = reviewed && verdicts[1] !== undefined && !concludesNoFindings(marked) ? [...read.matchAll(NUMBERED_ITEM)].filter(m => !levelsOf(m[0], plainReview(m[0])).length).length : 0; const findings = { highest: levels.length ? levels.sort()[0]! : null, ...(unlabeled ? { unlabeled } : {}) }; const cleanEvidence = levelsOf(code, plainCode).length > 0; @@ -286,7 +296,7 @@ export function classifyOutsideReview(input: OutsideReviewInput): OutsideReviewC const score = Number(scores[0]![1]); return { ...result(score >= 7 ? 'clean' : 'findings'), score }; } - if ((input.gate === 'review' || input.gate === 'proposal') && !/^Recommendation:[\t ]*[^\r\n]+\bbecause\b[\t ]*\S[^\r\n]+$/im.test(plain)) { + if ((input.gate === 'review' || input.gate === 'proposal') && ![plain, plainReview(marked)].some(text => /^Recommendation:[\t ]*[^\r\n]+\bbecause\b[\t ]*\S[^\r\n]+$/im.test(text))) { return result('unavailable', 'missing_markers', 'missing review completion recommendation'); } if (!reviewed) return result(blocking ? 'findings' : 'clean'); @@ -297,7 +307,7 @@ export function classifyOutsideReview(input: OutsideReviewInput): OutsideReviewC if (unparsed) return { ...result('unverified', 'unparsed_severity', unparsed), unparsed }; const unready = verdicts.find(v => v !== undefined && !/^NO_?FINDINGS$/.test(v) && (!READY_VERDICT.test(v) || v.includes('?') || UNVERIFIED_CLAIM.test(v))); if (unready !== undefined) return result('unverified', 'unverified_verdict', `Verdict: ${unready}`.trim().slice(0, 160)); - if ((input.gate === 'structured' || input.gate === 'review') && !cleanEvidence && !concludesNoFindings(code)) { + if ((input.gate === 'structured' || input.gate === 'review') && !cleanEvidence && !concludesNoFindings(marked)) { return result('unverified', 'untagged_review', 'missing severity or explicit no-findings conclusion'); } return result('clean'); diff --git a/test/outside-review-result.test.ts b/test/outside-review-result.test.ts index e9550285b..0e2d37b1e 100644 --- a/test/outside-review-result.test.ts +++ b/test/outside-review-result.test.ts @@ -31,6 +31,65 @@ function cli(args: string[]) { const RECOMMEND = 'No issues found in changed.ts.\nRecommendation: approve because changed.ts keeps its data.'; const UNTAGGED = 'Recommendation: fix the guard because changed.ts loses data.'; +// A model may write the completion recommendation as a Markdown section instead of an inline line (#3980). +describe('a ## Recommendation heading carries the completion recommendation (#3980)', () => { + const review = (tail: string, head = 'Severity: Low. A future caller may need a guard.') => + classifyOutsideReview({ text: `${head}\n\n${tail}`, gate: 'review' }); + + test('an advisory review with a heading is clean and keeps its severity', () => { + const result = review('## Recommendation\n\nShip as-is because current callers already enforce the invariant.'); + expect(result.verdict).toBe('clean'); + expect(result.findings.highest).toBe('P3'); + expect(review('## Recommendation\n\n**Ship as-is** because current callers already enforce the invariant.').verdict).toBe('clean'); + }); + + test('a blocker under the heading still blocks', () => { + const result = review('## Recommendation\n\nFix the write because it loses data.', 'Severity: High. The write drops rows.'); + expect(result.verdict).toBe('findings'); + expect(result.findings.highest).toBe('P1'); + }); + + test('a label or verdict on the line under the heading still blocks, with or without an inline line', () => { + for (const [under, highest] of [['P1: Fix the race because it loses data.', 'P1'], ['Critical: the migration drops rows because it skips the copy.', 'P0'], + ['Verdict: not ready', 'P2']] as const) { + for (const inline of ['', '\n\nRecommendation: fix it because it loses data.']) { + const result = classifyOutsideReview({ text: `[P2] nit\n\n## Recommendation\n\n${under}${inline}`, gate: 'review' }); + expect(result.findings.highest).toBe(highest); + expect(result.verdict).toBe(inline || highest !== 'P2' ? 'findings' : 'unavailable'); + } + } + }); + + test('a no-findings rationale under the heading concludes the review', () => { + const result = classifyOutsideReview({ text: '### Recommendation:\nApprove because no issues were found.', gate: 'proposal' }); + expect(result.verdict).toBe('clean'); + expect(classifyOutsideReview({ text: '## Recommendation\n\nApprove because I found no issues.', gate: 'review' }).verdict).toBe('clean'); + }); + + test('a heading over a no-findings line with no because-clause reads as it did before the heading fold', () => { + for (const text of ['## Recommendation\n\nNo issues found.', 'Reviewed the diff.\n\n## Recommendation\n\nNo issues found in the change.']) { + expect(classifyOutsideReview({ text, gate: 'structured' }).verdict).toBe('clean'); + expect(classifyOutsideReview({ text, gate: 'review' }).reason).toBe('missing_markers'); + } + }); + + test('the inline form still works', () => { + const result = review('Recommendation: Ship as-is because current callers already enforce the invariant.'); + expect(result.verdict).toBe('clean'); + expect(result.findings.highest).toBe('P3'); + }); + + test('a reply with no recommendation, an empty section, no rationale or a fenced example is still missing', () => { + for (const tail of ['No recommendation here.', '## Recommendation', '## Recommendation\n\nShip as-is.', + '## Recommendation\n\n- Ship as-is because callers enforce it.', + '```text\n## Recommendation\n\nShip as-is because this is an example.\n```']) { + const result = review(tail); + expect(result.verdict).toBe('unavailable'); + expect(result.reason).toBe('missing_markers'); + } + }); +}); + describe('classifyOutsideReview: separate execution, findings and verdict', () => { const cases: Array<[string, { text: string; gate: OutsideGate; stderr?: string; exit?: number }, string, string | null, string | undefined]> = [ ['clean review', { text: RECOMMEND, gate: 'review' }, 'clean', null, undefined],