mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-16 05:17:24 +01:00
fix(outside-review): a ## Recommendation heading counts as the completion marker (#4000)
* fix(outside-review): a ## Recommendation heading counts as the completion marker (#3980) A completed Codex review whose model wrote its recommendation as a Markdown heading with the prose underneath was classified unavailable/missing_markers, because only an inline "Recommendation: <action> because <reason>" line passed. The fence-blanked views now fold the heading and its next prose line into the inline marker. Empty sections, no because-rationale, list or quote lines under the heading, and fenced examples still report missing. Co-authored-by: nfitzsimmons <2198211+nfitzsimmons@users.noreply.github.com> Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(outside-review): keep labels and verdicts under the heading visible to the severity parsers Review p1 (specialist + adversarial) found that folding the heading into the fence-blanked views moved a P1:, Critical: or Verdict: line under the heading off line start, so a blocking review read clean. The folded view now feeds only the completion check and the no-findings conclusion. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(outside-review): fold only a heading line that carries a because-clause Pass 2's fix-check found that a structured reply ending '## Recommendation' over 'No issues found.' read clean at base and unverified after the fold. A line with no because can never meet the completion marker, so it is left as prose. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(changelog): fragment for the Recommendation heading fix Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(troubleshooting): missing-markers entry names the Recommendation heading form Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: nfitzsimmons <2198211+nfitzsimmons@users.noreply.github.com> Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
4 files changed
+76
-5
No files matched your search
@@ -0,0 +1 @@
|
||||
- **A Codex outside review that puts its recommendation under a `## Recommendation` heading now counts as finished.** The outside-review parser accepted only an inline `Recommendation: <action> because <reason>` line. When the model wrote the recommendation as a Markdown heading with the sentence on the next line, /ship and /codex reported the finished review as unavailable (`missing_markers`). A heading followed by a sentence with a `because` reason now reads the same as the inline line. A heading with nothing under it, a sentence with no `because`, a bullet or quote line under the heading, or a heading inside a code fence still reports the recommendation as missing. Severity labels and `Verdict:` lines under the heading are read exactly as before, so a blocking review still blocks. (#3980, thanks @nfitzsimmons)
|
||||
@@ -350,8 +350,9 @@ it; `gstack-outside-voice run` sends that default to Anthropic only.
|
||||
### `... outside review unavailable: the response lacks the markers this gate requires (...)`
|
||||
|
||||
**Meaning.** The answer did not contain what the gate checks: a
|
||||
`Recommendation: ... because ...` line, a `SCORE:`/`AMBIGUITIES:` pair, or
|
||||
severity tags.
|
||||
`Recommendation: ... because ...` line (or a `## Recommendation` heading whose
|
||||
next non-blank line is a plain sentence with a `because` reason), a
|
||||
`SCORE:`/`AMBIGUITIES:` pair, or severity tags.
|
||||
|
||||
**Fix.** Re-run the review.
|
||||
|
||||
|
||||
@@ -192,6 +192,15 @@ function plainReview(text: string): string {
|
||||
return text.split(/\r?\n/).map(line => line.replace(/^[\t ]*(?:#{1,6}[\t ]+|[-+*][\t ]+)?/, '').replace(/[*_`]/g, '')).join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* A `## Recommendation` heading and the prose line under it are the same marker as an inline `Recommendation:` line (#3980).
|
||||
* Only the completion check and the no-findings conclusion read this view: a `P1:` or `Verdict:` line under the heading stays
|
||||
* at line start for the severity and verdict parsers.
|
||||
*/
|
||||
function recommendationHeading(text: string): string {
|
||||
return text.replace(/^[\t ]*#{1,6}[\t ]+Recommendation[\t ]*:?[\t ]*\r?\n(?:[\t ]*\r?\n)*(?![\t ]*(?:[#>]|[-+*][\t ]))(?=[^\r\n]*\bbecause\b)[\t ]*(\S[^\r\n]*)/gim, 'Recommendation: $1');
|
||||
}
|
||||
|
||||
/**
|
||||
* Code quotes text, so a severity inside a fence or an inline code span is not
|
||||
* a label. An inline span holding only a severity token in label position
|
||||
@@ -266,11 +275,12 @@ export function classifyOutsideReview(input: OutsideReviewInput): OutsideReviewC
|
||||
const reviewed = input.gate !== 'proposal';
|
||||
const code = withoutCode(input.text), plainCode = plainReview(code);
|
||||
const read = withoutCode(input.text, true), plainRead = plainReview(read);
|
||||
const marked = recommendationHeading(code);
|
||||
const levelsOf = (c: string, p: string) => [...[...p.matchAll(/\[(P[0-3])\]|^(P[0-3]):/gm)].map(m => (m[1] ?? m[2]) as Severity),
|
||||
...severityTokens(c), ...(reviewed ? severityWords(c) : [])];
|
||||
const levels = levelsOf(read, plainRead);
|
||||
const verdicts = [lastVerdict(plainCode), lastVerdict(plainRead)];
|
||||
const unlabeled = reviewed && verdicts[1] !== undefined && !concludesNoFindings(code)
|
||||
const unlabeled = reviewed && verdicts[1] !== undefined && !concludesNoFindings(marked)
|
||||
? [...read.matchAll(NUMBERED_ITEM)].filter(m => !levelsOf(m[0], plainReview(m[0])).length).length : 0;
|
||||
const findings = { highest: levels.length ? levels.sort()[0]! : null, ...(unlabeled ? { unlabeled } : {}) };
|
||||
const cleanEvidence = levelsOf(code, plainCode).length > 0;
|
||||
@@ -286,7 +296,7 @@ export function classifyOutsideReview(input: OutsideReviewInput): OutsideReviewC
|
||||
const score = Number(scores[0]![1]);
|
||||
return { ...result(score >= 7 ? 'clean' : 'findings'), score };
|
||||
}
|
||||
if ((input.gate === 'review' || input.gate === 'proposal') && !/^Recommendation:[\t ]*[^\r\n]+\bbecause\b[\t ]*\S[^\r\n]+$/im.test(plain)) {
|
||||
if ((input.gate === 'review' || input.gate === 'proposal') && ![plain, plainReview(marked)].some(text => /^Recommendation:[\t ]*[^\r\n]+\bbecause\b[\t ]*\S[^\r\n]+$/im.test(text))) {
|
||||
return result('unavailable', 'missing_markers', 'missing review completion recommendation');
|
||||
}
|
||||
if (!reviewed) return result(blocking ? 'findings' : 'clean');
|
||||
@@ -297,7 +307,7 @@ export function classifyOutsideReview(input: OutsideReviewInput): OutsideReviewC
|
||||
if (unparsed) return { ...result('unverified', 'unparsed_severity', unparsed), unparsed };
|
||||
const unready = verdicts.find(v => v !== undefined && !/^NO_?FINDINGS$/.test(v) && (!READY_VERDICT.test(v) || v.includes('?') || UNVERIFIED_CLAIM.test(v)));
|
||||
if (unready !== undefined) return result('unverified', 'unverified_verdict', `Verdict: ${unready}`.trim().slice(0, 160));
|
||||
if ((input.gate === 'structured' || input.gate === 'review') && !cleanEvidence && !concludesNoFindings(code)) {
|
||||
if ((input.gate === 'structured' || input.gate === 'review') && !cleanEvidence && !concludesNoFindings(marked)) {
|
||||
return result('unverified', 'untagged_review', 'missing severity or explicit no-findings conclusion');
|
||||
}
|
||||
return result('clean');
|
||||
|
||||
@@ -31,6 +31,65 @@ function cli(args: string[]) {
|
||||
const RECOMMEND = 'No issues found in changed.ts.\nRecommendation: approve because changed.ts keeps its data.';
|
||||
const UNTAGGED = 'Recommendation: fix the guard because changed.ts loses data.';
|
||||
|
||||
// A model may write the completion recommendation as a Markdown section instead of an inline line (#3980).
|
||||
describe('a ## Recommendation heading carries the completion recommendation (#3980)', () => {
|
||||
const review = (tail: string, head = 'Severity: Low. A future caller may need a guard.') =>
|
||||
classifyOutsideReview({ text: `${head}\n\n${tail}`, gate: 'review' });
|
||||
|
||||
test('an advisory review with a heading is clean and keeps its severity', () => {
|
||||
const result = review('## Recommendation\n\nShip as-is because current callers already enforce the invariant.');
|
||||
expect(result.verdict).toBe('clean');
|
||||
expect(result.findings.highest).toBe('P3');
|
||||
expect(review('## Recommendation\n\n**Ship as-is** because current callers already enforce the invariant.').verdict).toBe('clean');
|
||||
});
|
||||
|
||||
test('a blocker under the heading still blocks', () => {
|
||||
const result = review('## Recommendation\n\nFix the write because it loses data.', 'Severity: High. The write drops rows.');
|
||||
expect(result.verdict).toBe('findings');
|
||||
expect(result.findings.highest).toBe('P1');
|
||||
});
|
||||
|
||||
test('a label or verdict on the line under the heading still blocks, with or without an inline line', () => {
|
||||
for (const [under, highest] of [['P1: Fix the race because it loses data.', 'P1'], ['Critical: the migration drops rows because it skips the copy.', 'P0'],
|
||||
['Verdict: not ready', 'P2']] as const) {
|
||||
for (const inline of ['', '\n\nRecommendation: fix it because it loses data.']) {
|
||||
const result = classifyOutsideReview({ text: `[P2] nit\n\n## Recommendation\n\n${under}${inline}`, gate: 'review' });
|
||||
expect(result.findings.highest).toBe(highest);
|
||||
expect(result.verdict).toBe(inline || highest !== 'P2' ? 'findings' : 'unavailable');
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('a no-findings rationale under the heading concludes the review', () => {
|
||||
const result = classifyOutsideReview({ text: '### Recommendation:\nApprove because no issues were found.', gate: 'proposal' });
|
||||
expect(result.verdict).toBe('clean');
|
||||
expect(classifyOutsideReview({ text: '## Recommendation\n\nApprove because I found no issues.', gate: 'review' }).verdict).toBe('clean');
|
||||
});
|
||||
|
||||
test('a heading over a no-findings line with no because-clause reads as it did before the heading fold', () => {
|
||||
for (const text of ['## Recommendation\n\nNo issues found.', 'Reviewed the diff.\n\n## Recommendation\n\nNo issues found in the change.']) {
|
||||
expect(classifyOutsideReview({ text, gate: 'structured' }).verdict).toBe('clean');
|
||||
expect(classifyOutsideReview({ text, gate: 'review' }).reason).toBe('missing_markers');
|
||||
}
|
||||
});
|
||||
|
||||
test('the inline form still works', () => {
|
||||
const result = review('Recommendation: Ship as-is because current callers already enforce the invariant.');
|
||||
expect(result.verdict).toBe('clean');
|
||||
expect(result.findings.highest).toBe('P3');
|
||||
});
|
||||
|
||||
test('a reply with no recommendation, an empty section, no rationale or a fenced example is still missing', () => {
|
||||
for (const tail of ['No recommendation here.', '## Recommendation', '## Recommendation\n\nShip as-is.',
|
||||
'## Recommendation\n\n- Ship as-is because callers enforce it.',
|
||||
'```text\n## Recommendation\n\nShip as-is because this is an example.\n```']) {
|
||||
const result = review(tail);
|
||||
expect(result.verdict).toBe('unavailable');
|
||||
expect(result.reason).toBe('missing_markers');
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('classifyOutsideReview: separate execution, findings and verdict', () => {
|
||||
const cases: Array<[string, { text: string; gate: OutsideGate; stderr?: string; exit?: number }, string, string | null, string | undefined]> = [
|
||||
['clean review', { text: RECOMMEND, gate: 'review' }, 'clean', null, undefined],
|
||||
|
||||
Reference in new issue
Block a user