diff --git a/docs/review-policy.md b/docs/review-policy.md index 4d7f1e250..f31ce846c 100644 --- a/docs/review-policy.md +++ b/docs/review-policy.md @@ -712,8 +712,16 @@ ones, and the arm lost blockers. `bin/gstack-review-dispatch` closes both ends: `Severity` or `Blocks` line comes, so those fields never split off as a second record (GST-1031). An unnumbered, unlabeled bullet with no file:line under a paragraph finding whose Severity or Blocks line has not come yet is that finding's detail, so the field line after it lands on the finding; a numbered item is its own finding. - A negated severity word still collects at its label ("found nothing CRITICAL", "No P0/P1 issues"), so a reader's - closing summary is a known phantom: it collects at that label. When it is a summary candidate (no lead label, no + A reader's own no-defect closing sentence is no finding at the label it denies (GST-435): a prose line with no + list, heading, bold or lead label, no `Severity` or `Blocks` field, no FIXABLE or INVESTIGATE, no file:line or path + and no exception clause, whose every labelled sentence is wholly a denial ("I found nothing CRITICAL.", "I found no + defect that is CRITICAL.", "No P0/P1 issues.", "Neither finding is CRITICAL."), a count ("I found three + INFORMATIONAL issues and nothing CRITICAL" collects at INFORMATIONAL) or "nothing blocks". A quantifier that points + at a finding elsewhere ("nothing else", "no other", "remaining", "new"), a named defect ("No auth check guards the + admin route, so nothing CRITICAL") or any other labelled sentence on the line keeps the whole line at its label, and a + reply whose only text is such a sentence, with no clean marker, is still no review. Any other negated severity word + still collects at its label, so a closing summary with a clause of its own ("I checked these against the post and + found nothing CRITICAL") is a known phantom: it collects at that label. When it is a summary candidate (no lead label, no `Severity` or `Blocks` field, and no file:line or exception clause in any sentence naming a label: "so nothing is CRITICAL", "the test covers the CRITICAL fix"), collect marks it `summary_candidate` and lead judgment may lower it with a reason, which judge records as `dismissed` (GST-1031); otherwise lead judgment can fold it only into a finding diff --git a/lib/review-ledger.ts b/lib/review-ledger.ts index 483a18ddc..723243f52 100644 --- a/lib/review-ledger.ts +++ b/lib/review-ledger.ts @@ -101,6 +101,35 @@ const DENIED_OR_FIXED = new RegExp(`(?:(? ANY_LABEL.test(t) && !ANY_LABEL.test(t.replace(DENIED_OR_FIXED, ' ')) && !t.startsWith('{') && !t.startsWith('**') && !LIST_ITEM.test(t) && !HEADING.test(t) && !LEAD_LABEL.test(t) && !t.match(SEVERITY_FIELD) && !t.match(STATED_BLOCKS) && [...t.matchAll(ALL_LABELS)].every(m => !!rank(m[1])) && t.split(SENTENCE).filter(s => ANY_LABEL.test(s)).every(s => !PATH_LOCUS.test(s) && !EXCEPTION.test(s)); +/** + * A reader's own no-defect closing sentence (GST-435: gstack-evals #194 "I found nothing CRITICAL.", #202 "I found no defect that is + * CRITICAL..."): a prose line with no list, heading, bold or lead label, no Severity or Blocks field, no FIXABLE or INVESTIGATE, no + * file:line or path and no exception clause, whose every labelled sentence is wholly one of these clauses: a denial (`no CRITICAL + * issues`, `no defect that is CRITICAL`, `nothing at CRITICAL`, `there are no High findings`, `neither finding is CRITICAL`), a + * count (`I found three INFORMATIONAL issues`) or `nothing blocks`. It yields the labels its counts name (none for a pure denial), so + * the denied label is no finding. A quantifier that points at a finding elsewhere (`nothing else`, `no other`, `remaining`), a + * named defect (`No auth check guards the admin route, so nothing CRITICAL`) or any other labelled sentence keeps the line as it was. + */ +const SEV = `[*\`]*(?:CRITICAL|INFORMATIONAL|Critical|Informational|High|HIGH|Medium|MEDIUM|Low|LOW|P[0-3])[*\`]*`; +const SEV_LIST = `${SEV}(?:\\s*(?:,|/|\\bor\\b|\\band\\b|\\bnor\\b)\\s*${SEV})*(?![A-Za-z0-9])`; +const NOUN = '(?:issues?|findings?|defects?|bugs?|problems?|regressions?|blockers?|concerns?|items?|notes?|ones?)'; +const DENY = `(? { + if (t.startsWith('{') || t.startsWith('**') || LIST_ITEM.test(t) || HEADING.test(t) || LEAD_LABEL.test(t) || t.match(SEVERITY_FIELD) || t.match(STATED_BLOCKS) || BLOCKS_YES.test(t) + || LOCUS.test(t) || PATH_LOCUS.test(t) || EXCEPTION.test(t) || /(? ANY_LABEL.test(s)); + if (!labelled.length || !labelled.every(s => SUMMARY_SENTENCE.test(s.trim()))) return undefined; + return labelled.flatMap(s => [...s.replace(DENIED, ' ').matchAll(ALL_LABELS)].map(m => m[1]!)); +}; /** * The native reader's own verdict line (GSTA-258 L4: #3901 "There are three low-severity findings, all FIXABLE, and none blocks."): a prose * line, no list item, heading, lead label, file:line, Severity or Blocks field, whose only labels are FIXABLE or INVESTIGATE and which says @@ -153,6 +182,8 @@ export function parseFindings(reader: string, output: string, fixableRaises = tr const leadOf = (t: string) => { const l = LEAD_LABEL.exec(t)?.[1]; return l === undefined || !fixableRaises ? l : named(l); }; const labelOf = (t: string) => { if (role === 'adversarial-native' && quietSummary(t)) return 'FIXABLE'; + const summary = closingSummary(t); + if (summary) return strongestOf(summary.map(named)); const field = fieldOf(t); if (field === undefined) return strongest(t); return critical(t) ? 'CRITICAL' : strongestOf([leadOf(t), field])!; diff --git a/test/fixtures/review-ledger/gst-435-closing-no-defect.txt b/test/fixtures/review-ledger/gst-435-closing-no-defect.txt new file mode 100644 index 000000000..ae8ae8533 --- /dev/null +++ b/test/fixtures/review-ledger/gst-435-closing-no-defect.txt @@ -0,0 +1,7 @@ +{"severity":"INFORMATIONAL","path":"wiki/plans/p/README.md","line":91,"summary":"The run-order step still says the run goes on when every linked log holds only no-contract-skill lines, where the readout stops it."} +{"severity":"INFORMATIONAL","path":"wiki/plans/p/hooks.ts.txt","line":46,"summary":"ls() now swallows every readdirSync error, so an unreadable arm folder is skipped without a word."} + +I found no defect that is CRITICAL. Here is what I checked: +- **The silent-hook rule.** All five outcomes the hook can trace are in the logged count, so a working hook can't be read as silent. +- **The lost rule.** Its result feeds only the tally and the exposure label, and no other caller reads it. +- **Tests weakened.** None. diff --git a/test/fixtures/review-ledger/gst-435-closing-nothing-critical.txt b/test/fixtures/review-ledger/gst-435-closing-nothing-critical.txt new file mode 100644 index 000000000..032a76047 --- /dev/null +++ b/test/fixtures/review-ledger/gst-435-closing-nothing-critical.txt @@ -0,0 +1,9 @@ +{"severity":"INFORMATIONAL","path":"wiki/plans/p/hooks.ts.txt","line":97,"summary":"The watch still checks a path and then opens it, so a folder swapped between the two can race the check."} +{"severity":"INFORMATIONAL","path":"wiki/plans/p/hooks.ts.txt","line":186,"summary":"lost only looks at sessions that have a delta.json, so a stopped session whose log was missed reads log_lost=no."} +{"severity":"INFORMATIONAL","path":"wiki/plans/p/README.md","line":173,"summary":"The new sentence says a delta.json listing is enough, which does not hold when the agent can write its own state root."} + +I found nothing CRITICAL. I checked these and found them sound: +- **Attachment hashes:** they match the README front matter, and the wiki check passes on this tree. +- **Missing-list parsing:** the parse at line 187 matches the hook's trace format (stop-hook.ts:134). +- **State root:** it's a real directory, so the realpath check doesn't refuse real logs. +- **Dry run:** the planted-link case does catch the old behaviour. diff --git a/test/review-closing-summary.test.ts b/test/review-closing-summary.test.ts new file mode 100644 index 000000000..985d7ddc4 --- /dev/null +++ b/test/review-closing-summary.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { collect, judge, ledgerBlockers, parseFindings, readJsonl, recordJudgment, summaryCandidate, type MergedLine } from '../lib/review-ledger'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const fixture = (name: string) => fs.readFileSync(path.join(ROOT, 'test/fixtures/review-ledger', name), 'utf8'); +const labels = (output: string, reader = 'fix-check') => parseFindings(reader, output).map(f => f.label); + +describe("a reader's no-defect closing sentence is not a finding (GST-435)", () => { + // Value: protects=a fix-check's own closing sentence that denies every severity it names ("I found nothing CRITICAL", + // "I found no defect that is CRITICAL") collects nothing, so the review loop stops counting a blocker that names no defect; + // fails_when=any of these sentences collects a finding; why_new=gstack-evals #194 (blocking_found=4 from "I found nothing + // CRITICAL") and #202 (fix-check#3 from "I found no defect that is CRITICAL...") each needed a Captain exception because judge + // refuses to lower a collected CRITICAL; seam=parseFindings and collect; no_claim=that every summary shape is recognized (one + // that opens with a defect clause, "None of them breaks a caller, so there's nothing at CRITICAL", still collects as a candidate) + test('the quoted closing sentences and plain negations collect nothing', () => { + for (const line of ['I found nothing CRITICAL.', 'I found nothing CRITICAL', 'I found no defect that is CRITICAL.', 'I found no defect that is CRITICAL...', + 'I found no CRITICAL defects. Here is what I checked:', 'I found nothing CRITICAL. I checked these and found them sound:', 'Neither finding is CRITICAL.', + 'No CRITICAL issues.', 'no CRITICAL issues', 'Nothing blocking, nothing CRITICAL.', 'Nothing blocking; no CRITICAL issues.', 'There are no CRITICAL findings.', + 'No CRITICAL or High findings to report.', "So there's nothing at CRITICAL.", 'Nothing here is CRITICAL.', 'Neither fix is CRITICAL.', 'Nothing in this diff is CRITICAL.', + 'No CRITICAL findings remain.', 'We found no P0 or P1 issues.', 'None of the fixes blocks landing, and nothing is CRITICAL.']) + expect([line, labels(line)]).toEqual([line, []]); + }); + + test('the #194 and #202 fix-check replies collect only their structured findings, and nothing blocks', () => { + for (const [name, json] of [['gst-435-closing-nothing-critical.txt', 3], ['gst-435-closing-no-defect.txt', 2]] as const) { + const d = fs.mkdtempSync(path.join(os.tmpdir(), 'gst435-')); + const ledger = path.join(d, 'l.jsonl'); + const merged = path.join(d, 'm.jsonl'); + fs.writeFileSync(path.join(d, 'out.txt'), fixture(name)); + const c = collect(ledger, 'fix-check', fixture(name), merged, path.join(d, 'out.txt')); + expect([name, c.review, c.lines.map(l => l.label)]).toEqual([name, true, Array(json).fill('INFORMATIONAL')]); + fs.writeFileSync(path.join(d, 'j.jsonl'), c.lines.map(l => JSON.stringify({ id: l.id, severity: l.label })).join('\n') + '\n'); + expect(recordJudgment(ledger, c.lines, readJsonl(path.join(d, 'j.jsonl')), path.join(d, 'j.jsonl'), merged).ok).toBe(true); + expect(ledgerBlockers(ledger)).toEqual([]); + fs.rmSync(d, { recursive: true, force: true }); + } + const d = fs.mkdtempSync(path.join(os.tmpdir(), 'gst435-')); + const merged = path.join(d, 'm.jsonl'); + const r = collect(path.join(d, 'l.jsonl'), 'fix-check', fixture('geva-77-fix-check.txt'), merged); + expect(r.lines.map(l => [l.id, l.label])).toEqual([['fix-check#1', 'INFORMATIONAL'], ['fix-check#2', 'INFORMATIONAL']]); + expect(judge(readJsonl(merged), r.lines.map(l => ({ id: l.id, severity: l.label }))).ok).toBe(true); + fs.rmSync(d, { recursive: true, force: true }); + }); + + test('a count summary keeps the label it counts and drops only the one it denies', () => { + expect(labels('I found three INFORMATIONAL issues and nothing CRITICAL.')).toEqual(['INFORMATIONAL']); + expect(labels('I found one High issue and nothing CRITICAL.')).toEqual(['High']); + expect(labels('None of the fixes blocks landing; I found four INFORMATIONAL defects and no CRITICAL.')).toEqual(['INFORMATIONAL']); + }); + + test('a summary-only reply with no clean marker is still no review', () => { + const d = fs.mkdtempSync(path.join(os.tmpdir(), 'gst435-')); + const r = collect(path.join(d, 'l.jsonl'), 'fix-check', 'Both fixes hold. I found nothing CRITICAL.\n', path.join(d, 'm.jsonl')); + expect([r.review, r.lines]).toEqual([false, []]); + fs.rmSync(d, { recursive: true, force: true }); + }); + + // Value: protects=a real blocker keeps collecting at its label: a sentence that names a file:line or a defect, an exception clause + // (GST-796), an "other"/"else" quantifier that implies a blocker named elsewhere, a list item, heading, bold opener, field or + // Blocks: yes, or any second labelled sentence that is not itself a denial; fails_when=any of them collects nothing or a weaker + // label; why_new=GST-766's negation drop lost real blockers and was reverted; seam=parseFindings; no_claim=that each is a real finding + test('negative controls: a locus, a named defect, an exception, a quantifier or a structured line keeps its label', () => { + const kept: Array<[string, string]> = [['The token leak in log.ts:3 is CRITICAL.', 'CRITICAL'], ['I found nothing CRITICAL in auth.ts:9.', 'CRITICAL'], + ['No CRITICAL issues (a.ts:3).', 'CRITICAL'], ['I found nothing CRITICAL. The token leak in log.ts:3 is CRITICAL.', 'CRITICAL'], + ['I found nothing CRITICAL. The retry loop never ends, which is High.', 'CRITICAL'], ['No CRITICAL issues except the token leak in log.ts:3', 'CRITICAL'], + ['There is no CRITICAL issue other than the race in a.ts:3.', 'CRITICAL'], ['Nothing CRITICAL apart from the unescaped query.', 'CRITICAL'], + ['No new CRITICAL findings, but the race still blocks.', 'CRITICAL'], ['I found no defect that is CRITICAL, but the retry loop never ends.', 'CRITICAL'], + ['No auth check guards the admin route, so nothing CRITICAL.', 'CRITICAL'], ['There is no CRITICAL lock around the refund write, so two workers double-pay.', 'CRITICAL'], + ['No input validation is CRITICAL.', 'CRITICAL'], ['Nothing guards the write path the failure of which is CRITICAL', 'CRITICAL'], ['Nothing else is CRITICAL.', 'CRITICAL'], + ['No other issue is CRITICAL.', 'CRITICAL'], ['No remaining finding is CRITICAL.', 'CRITICAL'], ['None of the blockers are CRITICAL.', 'CRITICAL'], ['This is not CRITICAL.', 'CRITICAL'], + ['Otherwise nothing CRITICAL.', 'CRITICAL'], ['- No CRITICAL issues.', 'CRITICAL'], ['1. No CRITICAL issues.', 'CRITICAL'], ['**No CRITICAL issues.**', 'CRITICAL'], + ['CRITICAL: none of the writes are atomic.', 'CRITICAL'], ['No CRITICAL issues. Severity: High', 'High'], ['No CRITICAL issues. Blocks: yes', 'CRITICAL'], + ['{"severity":"CRITICAL","summary":"I found nothing CRITICAL"}', 'CRITICAL'], ['No CRITICAL — the upload handler drops auth.', 'CRITICAL']]; + for (const [line, label] of kept) expect([line, labels(line)]).toEqual([line, [label]]); + expect(labels('1. Token leak in log.ts:3 must be fixed before landing.\n\nNothing else is CRITICAL.')).toEqual(['CRITICAL']); + expect(labels('The token leak blocks the merge.\nNo other issue is CRITICAL.')).toEqual(['CRITICAL']); + expect(labels('## CRITICAL\n- No CRITICAL issues in the fix.')).toEqual(['CRITICAL']); + expect(labels('I found nothing CRITICAL. Here is what I checked:\n- **Auth:** the token check at auth.ts:9 is CRITICAL and missing.')).toEqual(['CRITICAL']); + expect(labels('No FIXABLE findings.', 'adversarial-native')).toEqual(['CRITICAL']); + expect(summaryCandidate('No auth check guards the admin route, so nothing CRITICAL.')).toBe(true); + }); +}); diff --git a/test/review-dispatch.test.ts b/test/review-dispatch.test.ts index 662d1ddc5..eb933e1fa 100644 --- a/test/review-dispatch.test.ts +++ b/test/review-dispatch.test.ts @@ -339,11 +339,11 @@ describe('gstack-review-dispatch sent, collect, judge and verify: nothing change expect([prose, reader, parseFindings(reader, prose).map(f => f.label)]).toEqual([prose, reader, ['CRITICAL']]); }); - // Value: protects=a negated severity word still collects at its label, so a reader's closing prose ("found nothing CRITICAL") is a - // known phantom lead judgment folds and never a dropped line (GSTA-167 S1 reverts GST-766's negation drop); fails_when=any negated - // form collects nothing or a weaker label, except under the finding's own Severity field, which decides it (GST-976); - // why_new=GST-766's drop lost blockers named in exception clauses; seam=parseFindings; no_claim=that a negated line is a real - // finding (judge folds phantoms) + // Value: protects=a negated severity word still collects at its label, so a reader's closing prose ("found nothing CRITICAL" after a + // clause of its own) is a known phantom lead judgment folds and never a dropped line (GSTA-167 S1 reverts GST-766's negation drop), + // except a sentence that is wholly a no-defect summary (GST-435), which is no finding; fails_when=any other negated form collects + // nothing or a weaker label, except under the finding's own Severity field, which decides it (GST-976); why_new=GST-766's drop lost + // blockers named in exception clauses; seam=parseFindings; no_claim=that a negated line is a real finding (judge folds phantoms) test('a negated severity word still collects at its label: GEVA-92 fix-check#5 and its negated forms are known phantoms', () => { const fixCheck = [ 'REVIEWED_TREE: bedd511efa0fa16c9277821073c68e438dc760d2 /workspace/gstack-evals', @@ -355,20 +355,22 @@ describe('gstack-review-dispatch sent, collect, judge and verify: nothing change '- The minimum p of 2^-15.', ].join('\n'); expect(parseFindings('fix-check', fixCheck).map(f => f.label)).toEqual(['INFORMATIONAL', 'INFORMATIONAL', 'CRITICAL']); - const kept: Array<[string, string]> = [['Found no CRITICAL issues in the fix range.', 'CRITICAL'], ['Nothing CRITICAL or HIGH remains.', 'CRITICAL'], ['This is not CRITICAL.', 'CRITICAL'], + const kept: Array<[string, string]> = [['Found no CRITICAL issues in the fix range.', 'CRITICAL'], ['This is not CRITICAL.', 'CRITICAL'], ['I found zero Critical findings, and none High.', 'Critical'], ['The retry path is safe; without CRITICAL impact.', 'CRITICAL'], ['No new CRITICAL issues introduced by the fix.', 'CRITICAL'], - ['No remaining CRITICAL or HIGH findings.', 'CRITICAL'], ['**No** CRITICAL issues found.', 'CRITICAL'], ['No **CRITICAL** issues found.', 'CRITICAL'], ['No `CRITICAL` issue.', 'CRITICAL'], - ['No [P1] issues remain.', 'P1'], ['Nothing at CRITICAL severity remains.', 'CRITICAL'], ['No CRITICAL regressions introduced by the fix.', 'CRITICAL'], ['Nothing CRITICAL to report.', 'CRITICAL'], + ['No remaining CRITICAL or HIGH findings.', 'CRITICAL'], ['**No** CRITICAL issues found.', 'CRITICAL'], + ['No [P1] issues remain.', 'P1'], ['No CRITICAL regressions introduced by the fix.', 'CRITICAL'], ['There is nothing CRITICAL in the fix range.', 'CRITICAL'], ['No CRITICAL — the fix is sound.', 'CRITICAL'], ['Found no CRITICAL or HIGH regressions.', 'CRITICAL'], ['No HIGH-risk changes.', 'HIGH'], - ['No CRITICAL/HIGH findings.', 'CRITICAL'], ['No P0/P1 issues.', 'P0'], ['No CRITICAL, HIGH or MEDIUM issues.', 'CRITICAL'], ['Checked collect() at lib/review-ledger.ts:189; no CRITICAL issues remain.', 'CRITICAL'], ['No CRITICAL issues; served on localhost:3000.', 'CRITICAL']]; for (const [prose, label] of kept) expect([prose, parseFindings('fix-check', prose).map(f => f.label)]).toEqual([prose, [label]]); + for (const prose of ['Nothing CRITICAL or HIGH remains.', 'No **CRITICAL** issues found.', 'No `CRITICAL` issue.', 'Nothing at CRITICAL severity remains.', 'Nothing CRITICAL to report.', + 'No CRITICAL/HIGH findings.', 'No P0/P1 issues.', 'No CRITICAL, HIGH or MEDIUM issues.']) + expect([prose, parseFindings('fix-check', prose).map(f => f.label)]).toEqual([prose, []]); expect(parseFindings('r', 'Not CRITICAL, but High: the cache drops a row (c.ts:4)').map(f => f.label)).toEqual(['CRITICAL']); expect(parseFindings('r', 'No CRITICAL, High: the cache drops a row (c.ts:4)').map(f => f.label)).toEqual(['CRITICAL']); expect(parseFindings('r', 'Nothing CRITICAL / HIGH: race in q.ts:7').map(f => f.label)).toEqual(['CRITICAL']); expect(parseFindings('r', 'No CRITICAL lock guards the write in Makefile:7\nSeverity: CRITICAL').map(f => f.label)).toEqual(['CRITICAL']); expect(parseFindings('r', '1. auth.ts:3 LOW: log line too verbose\nNo CRITICAL lock guards the write in Makefile:7\nSeverity: CRITICAL').map(f => [f.label, f.raw])).toEqual([['LOW', '1. auth.ts:3 LOW: log line too verbose'], ['CRITICAL', 'No CRITICAL lock guards the write in Makefile:7 | Severity: CRITICAL']]); - expect(parseFindings('r', '1. auth.ts:3 LOW: log line too verbose\nNo CRITICAL issues remain.').map(f => f.label)).toEqual(['LOW', 'CRITICAL']); + expect(parseFindings('r', '1. auth.ts:3 LOW: log line too verbose\nNo CRITICAL issues remain.').map(f => [f.label, f.raw])).toEqual([['LOW', '1. auth.ts:3 LOW: log line too verbose']]); expect(parseFindings('outside-voice', '- a.ts:3 race. Severity: Low. Blocks: no\n Separately, nothing CRITICAL about b.ts:9, but it leaks fds.\n Severity: High').map(f => [f.label, f.raw])).toEqual([['Low', '- a.ts:3 race. Severity: Low. Blocks: no'], ['High', 'Separately, nothing CRITICAL about b.ts:9, but it leaks fds. | Severity: High']]); expect(parseFindings('r', '1. A CRITICAL race in queue.ts:78 that no lock covers').map(f => f.label)).toEqual(['CRITICAL']); expect(parseFindings('r', '- A divide by zero CRITICAL bug in parse()').map(f => f.label)).toEqual(['CRITICAL']); @@ -618,32 +620,34 @@ describe('gstack-review-dispatch sent, collect, judge and verify: nothing change expect(parseFindings('adversarial-native', 'No rate limit is FIXABLE (login.ts:9)').map(f => f.label)).toEqual(['CRITICAL']); }); - // Value: protects=a reader's own closing summary is a known phantom that keeps its label (HARN-15's Red Team and fix-check replies, - // GEVA-77's "Neither finding is CRITICAL.", "found nothing CRITICAL"): it collects, and judge refuses to drop or lower it; the - // follow-up tags such lines summary_candidate so judge may dismiss a tagged one on record; fails_when=a summary line collects nothing - // or a weaker label, which would mean a drop path is back; why_new=GST-435 removed the drop after it lost blockers in three review - // rounds; seam=parseFindings, collect and judge; no_claim=that a phantom is free (it stays CRITICAL in the judged set) - test('known phantoms: closing summaries keep their label until the summary_candidate follow-up', () => { + // Value: protects=a reader's own closing summary that opens with a clause of its own is a known phantom that keeps its label (HARN-15's + // ship2 Red Team "Red-team pass is done: …", harness-evals #152's "…, so there's nothing at CRITICAL"): it collects, and judge refuses + // to drop or lower it unless it is a summary_candidate lowered with a reason; a sentence that is wholly a no-defect summary (GEVA-77's + // "Neither finding is CRITICAL.", HARN-15 ship2 p2's count summaries) is no finding at the label it denies (GST-435); + // fails_when=a phantom collects nothing or a weaker label, or a wholly no-defect sentence still collects its denied label; + // why_new=GST-435 removed the negation drop after it lost blockers in three review rounds, then narrowed it to whole summary + // sentences; seam=parseFindings, collect and judge; no_claim=that a phantom is free (it stays CRITICAL in the judged set) + test('known phantoms: closing summaries with a clause of their own keep their label; wholly no-defect sentences collect none', () => { const fixture = (name: string) => fs.readFileSync(path.join(ROOT, 'test/fixtures/review-ledger', name), 'utf8'); - const replies: Array<[string, string, number]> = [['red-team', 'harn15-ship2-red-team.txt', 3], ['red-team', 'harn15-ship2-p2-red-team.txt', 3], - ['fix-check', 'harn15-ship2-p2-fix-check.txt', 4], ['fix-check', 'harn15-152-p2-fix-check.txt', 2]]; - for (const [reader, name, json] of replies) { + const replies: Array<[string, string, string, number]> = [['red-team', 'harn15-ship2-red-team.txt', 'CRITICAL', 3], ['red-team', 'harn15-ship2-p2-red-team.txt', 'INFORMATIONAL', 3], + ['fix-check', 'harn15-ship2-p2-fix-check.txt', 'INFORMATIONAL', 4], ['fix-check', 'harn15-152-p2-fix-check.txt', 'CRITICAL', 2]]; + for (const [reader, name, summary, json] of replies) { const found = parseFindings(reader, fixture(name)); - expect([name, found.map(f => f.label)]).toEqual([name, ['CRITICAL', ...Array(json).fill('INFORMATIONAL')]]); + expect([name, found.map(f => f.label)]).toEqual([name, [summary, ...Array(json).fill('INFORMATIONAL')]]); expect(found[0]!.raw).toMatch(/(no|nothing( at)?) CRITICAL/); } const fc = fixture('geva-77-fix-check.txt'); const d = tmp(); const merged = path.join(d, 'merged.jsonl'); const r = collect(path.join(d, 'l.jsonl'), 'fix-check', fc, merged); - expect([r.review, r.lines.map(l => [l.id, l.label])]).toEqual([true, [['fix-check#1', 'INFORMATIONAL'], ['fix-check#2', 'INFORMATIONAL'], ['fix-check#3', 'CRITICAL']]]); - expect(r.lines[2]!.raw).toStartWith('Neither finding is CRITICAL.'); - const judged = path.join(d, 'judged.jsonl'); - fs.writeFileSync(judged, r.lines.map(l => JSON.stringify({ id: l.id, severity: 'INFORMATIONAL' })).join('\n') + '\n'); - expect(judge(readJsonl(merged), readJsonl(judged)).ok).toBe(false); + expect([r.review, r.lines.map(l => [l.id, l.label])]).toEqual([true, [['fix-check#1', 'INFORMATIONAL'], ['fix-check#2', 'INFORMATIONAL']]]); + const phantom = collect(path.join(d, 'l2.jsonl'), 'fix-check', fixture('harn15-152-p2-fix-check.txt'), path.join(d, 'm2.jsonl')); + fs.writeFileSync(path.join(d, 'j2.jsonl'), phantom.lines.map(l => JSON.stringify({ id: l.id, severity: 'INFORMATIONAL' })).join('\n') + '\n'); + expect(judge(readJsonl(path.join(d, 'm2.jsonl')), readJsonl(path.join(d, 'j2.jsonl'))).ok).toBe(false); for (const line of ['There are no CRITICAL findings.', 'No CRITICAL or High findings to report.', 'I found nothing CRITICAL.', "So there's nothing at CRITICAL.", - 'Neither finding is CRITICAL.', 'Nothing here is CRITICAL.', 'Neither fix is CRITICAL.', 'Nothing in this diff is CRITICAL.', 'This is not CRITICAL.']) - expect([line, parseFindings('outside-voice', line).map(f => f.label)]).toEqual([line, ['CRITICAL']]); + 'Neither finding is CRITICAL.', 'Nothing here is CRITICAL.', 'Neither fix is CRITICAL.', 'Nothing in this diff is CRITICAL.']) + expect([line, parseFindings('outside-voice', line).map(f => f.label)]).toEqual([line, []]); + expect(parseFindings('outside-voice', 'This is not CRITICAL.').map(f => f.label)).toEqual(['CRITICAL']); expect(parseFindings('outside-voice', 'Severity: Low (not High)').map(f => f.label)).toEqual(['Low']); }); diff --git a/test/review-ledger-invariant.test.ts b/test/review-ledger-invariant.test.ts index 13496e1c2..95df302f5 100644 --- a/test/review-ledger-invariant.test.ts +++ b/test/review-ledger-invariant.test.ts @@ -12,6 +12,18 @@ const LEAD = new RegExp(`(?:^|\\| )[^\\p{L}\\p{N}]*(?:\\d+[.)]\\s*[^\\p{L}\\p{N} const HEADING = new RegExp(`^(?:#{1,6}\\s*|\\*\\*)(?:${LABELS})\\b`, 'm'); const LABEL = new RegExp(`${LABELS}`, 'gi'); const LEAD_NO_FIXABLE = new RegExp(LEAD.source.replace('|FIXABLE|INVESTIGATE', ''), 'u'); +/** + * GST-435: a reader's own no-defect closing sentence is no finding at the label it denies, so these pinned inputs collect fewer or + * weaker records than main; each is held to its exact new labels on both classes. Every other input keeps main as its floor. + */ +const NO_DEFECT_SUMMARIES: Record = { + 'Nothing CRITICAL or HIGH remains.': [], 'No **CRITICAL** issues found.': [], 'No `CRITICAL` issue.': [], 'Nothing at CRITICAL severity remains.': [], + 'Nothing CRITICAL to report.': [], 'No CRITICAL/HIGH findings.': [], 'No P0/P1 issues.': [], 'No CRITICAL, HIGH or MEDIUM issues.': [], + '1. auth.ts:3 LOW: log line too verbose\nNo CRITICAL issues remain.': ['LOW'], 'There are no CRITICAL findings.': [], 'No CRITICAL or High findings to report.': [], + 'I found nothing CRITICAL.': [], "So there's nothing at CRITICAL.": [], 'Neither finding is CRITICAL.': [], 'Nothing here is CRITICAL.': [], 'Neither fix is CRITICAL.': [], + 'Nothing in this diff is CRITICAL.': [], 'review-ledger/geva-77-fix-check.txt': ['INFORMATIONAL', 'INFORMATIONAL'], + 'review-ledger/harn15-ship2-p2-red-team.txt': Array(4).fill('INFORMATIONAL'), 'review-ledger/harn15-ship2-p2-fix-check.txt': Array(5).fill('INFORMATIONAL'), +}; describe('parseFindings is never weaker than main (GST-976 invariant)', () => { // Value: protects=no input the suite pins, no reader fixture and no Severity field shape collects below the label main (82e9d7ec) @@ -32,6 +44,8 @@ describe('parseFindings is never weaker than main (GST-976 invariant)', () => { for (const e of entries as Array<{ reader: string; output?: string; fixture?: string; main: string[] }>) { const output = e.output ?? fs.readFileSync(path.join(ROOT, 'test/fixtures', e.fixture!), 'utf8'); const got = parseFindings(e.reader, output, raises); + const summary = NO_DEFECT_SUMMARIES[e.fixture ?? output]; + if (summary) { if (JSON.stringify(got.map(f => f.label)) !== JSON.stringify(summary)) bad.push(`GST-435 ${JSON.stringify(got.map(f => f.label))} ${JSON.stringify(output.slice(0, 80))}`); continue; } if (!e.main.length) continue; if (got.length !== e.main.length) { bad.push(`count ${got.length}/${e.main.length} ${JSON.stringify(output.slice(0, 80))}`); continue; } got.forEach((f, i) => {