mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-03 07:18:03 +01:00
test(review-ledger): never weaker than main invariant over pinned inputs, fixtures and generated field shapes; a multi-label Severity field counts as none, lowercase fields keep their label (GST-976)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
a0adf6e3bc
commit
c9e3cd9bf1
4 files changed
+52
-4
No files matched your search
@@ -79,6 +79,7 @@ const SEVERITY_FIELD = new RegExp(`(?:^(?:[-*+>]\\s+|\\d+[.)]\\s+)?\\**\\s*Sever
|
||||
* "1. **High**: …", "### CRITICAL: …", "1. api.ts:10 CRITICAL: …", "1. CRITICAL missing auth"): the reader's own label, which a
|
||||
* Severity field never lowers. "P0 pilot …" is not one.
|
||||
*/
|
||||
const FIELD_LABELS = new RegExp(ANY_LABEL.source, 'gi');
|
||||
const LEAD_LABEL = new RegExp(`^(?:[^\\p{L}\\p{N}\\s]\\s*|\\d+[.)]\\s*)*(?:\\w[\\w./-]*:\\d+(?:[-–:,]\\s*\\d+)*[^\\p{L}\\p{N}\\s]*\\s+(?:[^\\p{L}\\p{N}\\s]\\s*)*)?(${LABELS})(?=[*_\`\\])]|\\s*[:|—–]|\\s+-\\s|\\s*$|(?<=[A-Z]{3})[\\s,(.])`, 'u');
|
||||
const STATED_BLOCKS = /(?<![A-Za-z0-9])Blocks\**\s*:?\s*\**\s*(yes|no)(?![A-Za-z0-9])/gi;
|
||||
function stated(raw: string): Stated | undefined {
|
||||
@@ -93,7 +94,7 @@ export interface MergedLine { id: string; reader: string; label: string; stated?
|
||||
/**
|
||||
* One reader's findings as written: a JSON object per line (severity kept as written); else each prose line carrying a severity or
|
||||
* FIXABLE/INVESTIGATE label, at the most severe label on it, and each top-level list item under a label-only heading (the stronger
|
||||
* of the two labels). A line with a `Severity` field takes that field's label instead (the strongest, if it states two), whatever its
|
||||
* of the two labels). A line with a `Severity` field takes that field's label instead (a field stating more than one label, "High/Critical", counts as none), whatever its
|
||||
* body names (GST-976: "the P0 pilot" collected as P0 under `Severity: Low`); a label leading the line and an earlier field of the same
|
||||
* finding are its floor. A `Severity` or `Blocks` field line continues the open finding, once per field: `Severity` sets its label, `Blocks: yes` makes it CRITICAL, `Blocks: no` never lowers it; with no finding open it is one, carrying
|
||||
* the paragraph's subject.
|
||||
@@ -114,7 +115,7 @@ export function parseFindings(reader: string, output: string): Array<{ label: st
|
||||
const named = (l: string) => (role === 'adversarial-native' && l.toUpperCase() === 'FIXABLE' ? 'CRITICAL' : l);
|
||||
const strongestOf = (labels: Array<string | undefined>) => labels.reduce<string | undefined>((a, b) => (a === undefined ? b : b === undefined ? a : stronger(a, b)), undefined);
|
||||
const strongest = (t: string) => strongestOf([...t.matchAll(ALL_LABELS)].map(m => named(m[1]!)));
|
||||
const fieldOf = (t: string) => strongestOf([...t.matchAll(SEVERITY_FIELD)].flatMap(m => [...(m[1] ?? m[2]!).matchAll(ALL_LABELS)].map(l => l[1]!)));
|
||||
const fieldOf = (t: string) => { const ls = [...t.matchAll(SEVERITY_FIELD)].flatMap(m => (m[1] ?? m[2]!).match(FIELD_LABELS)!); return ls.length === 1 ? ls[0] : undefined; };
|
||||
const critical = (t: string) => BLOCKS_YES.test(t) || (role === 'adversarial-native' && /(?<![A-Za-z0-9])FIXABLE(?![A-Za-z0-9])/.test(t));
|
||||
const leadOf = (t: string) => { const l = LEAD_LABEL.exec(t)?.[1]; return l === undefined ? undefined : named(l); };
|
||||
const labelOf = (t: string) => {
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
{"note":"parseFindings labels at main 82e9d7ec (GST-976): every input the review-dispatch and prose-voice-floor tests pass it, every reader fixture here, and generated Severity field shapes. Regenerate only from a frozen main.","main":"82e9d7ec7a2c222f20b04aaf5b965fa50d7b6bdb","entries":[{"source":"pinned","reader":"adversarial-native","output":"FIXABLE: race in a.ts:3\nSeverity: Medium\nBlocks: no","main":["CRITICAL"]},{"source":"pinned","reader":"adversarial-native","output":"INVESTIGATE: odd lock in b.ts:9\nSeverity: Low\nBlocks: yes","main":["CRITICAL"]},{"source":"pinned","reader":"adversarial-native","output":"INVESTIGATE: odd lock in b.ts:9\nBlocks: no","main":["INVESTIGATE"]},{"source":"pinned","reader":"adversarial-native","output":"FIXABLE: race in a.ts:3. Severity: Medium. Blocks: no","main":["CRITICAL"]},{"source":"pinned","reader":"adversarial-native","output":"1. FIXABLE: race in a.ts:3 — Severity: Low, Blocks: no","main":["CRITICAL"]},{"source":"pinned","reader":"adversarial-native","output":"- FIXABLE: race in a.ts:3\n- Severity: Medium\n- Blocks: no","main":["CRITICAL"]},{"source":"pinned","reader":"outside-voice","output":"1. Drops the last row (q.ts:7). Severity: Critical. Blocks: no","main":["Critical"]},{"source":"pinned","reader":"outside-voice","output":"1. Cache evicts sessions (c.ts:4)\n Severity: High","main":["High"]},{"source":"pinned","reader":"outside-voice","output":"1. Retry storm (r.ts:2)\n Severity: Medium\n Blocks: yes","main":["CRITICAL"]},{"source":"pinned","reader":"outside-voice","output":"1. Index drift (i.ts:5). Severity: Low. Blocks: Yes, it loses the row","main":["CRITICAL"]},{"source":"pinned","reader":"outside-voice","output":"1. Naming (n.ts:1)\n Severity: Low\n Blocks: no","main":["Low"]},{"source":"pinned","reader":"specialist/testing","output":"REVIEWED_TREE: d0dd42d6a88fbc2a6abca4f6b2dfde7e35520c5a /home/user/.capy/work/gst976/wt\nNO FINDINGS\n","main":[]},{"source":"pinned","reader":"specialist/testing","output":"REVIEWED_TREE: d0dd42d6a88fbc2a6abca4f6b2dfde7e35520c5a /home/user/.capy/work/gst976/wt\n{\"severity\":\"CRITICAL\",\"path\":\"a.ts\",\"line\":3,\"category\":\"logic\",\"summary\":\"off by one\",\"specialist\":\"testing\"}\n{\"severity\":\"INFORMATIONAL\",\"path\":\"a.ts\",\"line\":9,\"category\":\"style\",\"summary\":\"name\",\"specialist\":\"testing\"}\n","main":["CRITICAL","INFORMATIONAL"]},{"source":"pinned","reader":"outside-voice","output":"REVIEWED_TREE: d0dd42d6a88fbc2a6abca4f6b2dfde7e35520c5a /home/user/.capy/work/gst976/wt\nFindings:\n1. **High**: retry loop never backs off (q.ts:78)\n2. Low - log line is noisy\nRecommendation: Fix the retry because it DoSes the pool\n","main":["High","Low"]},{"source":"pinned","reader":"specialist/testing","output":"REVIEWED_TREE: d0dd42d6a88fbc2a6abca4f6b2dfde7e35520c5a /home/user/.capy/work/gst976/wt\n{\"severity\":\"CRITICAL\",\"path\":\"a.ts\",\"line\":3,\"category\":\"logic\",\"summary\":\"x\",\"specialist\":\"testing\"}\n","main":["CRITICAL"]},{"source":"pinned","reader":"red-team","output":"REVIEWED_TREE: d0dd42d6a88fbc2a6abca4f6b2dfde7e35520c5a /home/user/.capy/work/gst976/wt\n{\"severity\":\"CRITICAL\",\"path\":\"b.ts\",\"line\":1,\"category\":\"x\",\"summary\":\"y\",\"specialist\":\"red-team\"}\n","main":["CRITICAL"]},{"source":"pinned","reader":"specialist/testing","output":"REVIEWED_TREE: d0dd42d6a88fbc2a6abca4f6b2dfde7e35520c5a /home/user/.capy/work/gst976/wt\n{\"severity\":\"CRITICAL\",\"path\":\"a.ts\",\"line\":3,\"category\":\"logic\",\"summary\":\"x\",\"specialist\":\"testing\"}\n{\"severity\":\"INFORMATIONAL\",\"path\":\"b.ts\",\"line\":1,\"category\":\"n\",\"summary\":\"y\",\"specialist\":\"testing\"}\n","main":["CRITICAL","INFORMATIONAL"]},{"source":"pinned","reader":"r","output":"NO FINDINGS\n","main":[]},{"source":"pinned","reader":"r","output":"## High\n1. The auth check skips webhooks (a.ts:4)\n - Impact: nested evidence\n **Fix:**\n- second one\n## Notes\n- not a finding\n**Critical findings**\n- real one\n**Recommendations**\n- consider renaming foo\n","main":["High","High","Critical"]},{"source":"pinned","reader":"r","output":"1. **Unbounded retry in queue.ts:78** (FIXABLE): loops on 429\n2. `auth.ts:42` skips token expiry check — INVESTIGATE\n### CRITICAL: missing auth check in api.ts:10\n#### [P1] token not expired\n### High: authorization bypass (auth.ts:14)\n- **HIGH**: leak\n| High | race in queue.ts:4 |\nClassification: INVESTIGATE\n7. **Informational: reads every file**","main":["FIXABLE","INVESTIGATE","CRITICAL","P1","High","HIGH","High","INVESTIGATE","Informational"]},{"source":"pinned","reader":"r","output":"FIXABLE: race in a.ts:3\nINVESTIGATE: odd lock\nRecommendation: Fix the race because x\n","main":["FIXABLE","INVESTIGATE"]},{"source":"pinned","reader":"outside-voice","output":"No CRITICAL issues except the token leak in log.ts:3","main":["CRITICAL"]},{"source":"pinned","reader":"fix-check","output":"No CRITICAL issues except the token leak in lLine truncated
|
||||
@@ -690,8 +690,8 @@ describe('gstack-review-dispatch sent, collect, judge and verify: nothing change
|
||||
['1. lib/x.ts:42-58 CRITICAL: drops rows. Severity: Low', 'CRITICAL'], ['1. lib/x.ts:42:7 CRITICAL: drops rows. Severity: Low', 'CRITICAL'],
|
||||
['1. (lib/x.ts:42, 50) CRITICAL: drops rows. Severity: Low', 'CRITICAL'],
|
||||
['1. The P0 pilot table drops a row\nSeverity: Low: only the pilot', 'Low'], ['1. The P0 pilot table drops a row\nSeverity: Low because it is internal', 'Low'],
|
||||
['1. The P0 pilot table drops a row\nSeverity: Low/Medium', 'Medium'], ['Severity: Low: only the P0 pilot table', 'Low'],
|
||||
['1. The P0 pilot drops a row. Severity: Low/P3', 'Low'], ['1. drops rows\nSeverity: High/Critical', 'Critical'],
|
||||
['1. The P0 pilot table drops a row\nSeverity: Low/Medium', 'P0'], ['Severity: Low: only the P0 pilot table', 'Low'],
|
||||
['1. The P0 pilot drops a row. Severity: Low/P3', 'P0'], ['1. drops rows\nSeverity: High/Critical', 'Critical'],
|
||||
['Severity: Medium/CRITICAL', 'CRITICAL'], ['1. x.ts:3 drops prod rows. Severity: Low/CRITICAL', 'CRITICAL'],
|
||||
['1. x.ts:3 is CRITICAL (Severity: Low/Medium in pass 1)', 'CRITICAL'], ['1. x.ts:3 is CRITICAL (Severity: Low: pass 1)', 'CRITICAL'],
|
||||
['Still open: x.ts:3 is P0, Severity: Low: pass 1 understated it', 'P0']];
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { parseFindings, rank } from '../lib/review-ledger';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const LABELS = 'CRITICAL|INFORMATIONAL|Critical|Informational|High|HIGH|Medium|MEDIUM|Low|LOW|P[0-3]|FIXABLE|INVESTIGATE';
|
||||
const CHAIN = `(?:${LABELS})(?:\\s*[/→]\\s*\\**\\s*(?:${LABELS}))*`;
|
||||
/** A finding's own Severity field as GST-976 defines it: at a line's start (colon optional), or after a separator with its colon and a closed clause. */
|
||||
const FIELD = new RegExp(`(?:(?:^|\\| )(?:[-*+>]\\s+|\\d+[.)]\\s+)?\\**\\s*Severity\\**\\s*:?\\s*\\**\\s*(${CHAIN})(?![A-Za-z0-9])|(?:[.;,|()]\\s*|\\s[—–-]\\s*)\\**\\s*Severity\\**\\s*:\\s*\\**\\s*(${CHAIN})(?![A-Za-z0-9])(?=\\**\\s*(?:$|[.;,|()*—–]|-\\s|Blocks)))`, 'gi');
|
||||
const LEAD = new RegExp(`(?:^|\\| )[^\\p{L}\\p{N}]*(?:\\d+[.)]\\s*[^\\p{L}\\p{N}]*)?(?:\\w[\\w./-]*:\\d+(?:[-–:,]\\s*\\d+)*\\S*\\s+[^\\p{L}\\p{N}]*)?(?:${LABELS})(?![a-z])`, 'u');
|
||||
const HEADING = new RegExp(`^(?:#{1,6}\\s*|\\*\\*)(?:${LABELS})\\b`, 'm');
|
||||
const LABEL = new RegExp(`${LABELS}`, 'gi');
|
||||
|
||||
describe('parseFindings is never weaker than main (GST-976 invariant)', () => {
|
||||
// Value: protects=no input the suite pins, no reader fixture and no Severity field shape collects below the label main (82e9d7ec)
|
||||
// collected, unless the finding's own Severity field states exactly one label, and such a finding collects at that label unless a
|
||||
// leading label, a label heading, Blocks: yes or the native voice's FIXABLE raises it; fails_when=a field chain (High/Critical) keeps
|
||||
// its weaker label (b3ddd6c) or a field whose clause runs on stops counting and a body P0 raises the finding (f3810d4);
|
||||
// why_new=three GST-976 review rounds each found a shape the row tables missed; seam=parseFindings; no_claim=that main's label was
|
||||
// right (it is the floor), or coverage of stored private reader outputs (checked outside the repo)
|
||||
test('every pinned input, reader fixture and generated field shape', () => {
|
||||
const { entries, shapes } = JSON.parse(fs.readFileSync(path.join(ROOT, 'test/fixtures/review-ledger/main-labels.json'), 'utf8'));
|
||||
const { bodies, fields: tails, joins, readers, generated } = shapes;
|
||||
for (const [b, f, j, r, main] of generated) entries.push({ reader: readers[r], output: bodies[b] + joins[j] + tails[f], main });
|
||||
const bad: string[] = [];
|
||||
let findings = 0;
|
||||
for (const e of entries as Array<{ reader: string; output?: string; fixture?: string; main: string[] }>) {
|
||||
const output = e.output ?? fs.readFileSync(path.join(ROOT, 'test/fixtures', e.fixture!), 'utf8');
|
||||
const got = parseFindings(e.reader, output);
|
||||
if (!e.main.length) continue;
|
||||
if (got.length !== e.main.length) { bad.push(`count ${got.length}/${e.main.length} ${JSON.stringify(output.slice(0, 80))}`); continue; }
|
||||
got.forEach((f, i) => {
|
||||
findings++;
|
||||
const fields = [...f.raw.matchAll(FIELD)].map(m => (m[1] ?? m[2])!);
|
||||
const single = fields.length === 1 && fields[0]!.match(LABEL)!.length === 1 ? fields[0]!.match(LABEL)![0] : undefined;
|
||||
const raised = /Blocks:?\**:?\s*\**\s*yes/i.test(f.raw) || (e.reader.startsWith('adversarial-native') && /FIXABLE/.test(f.raw)) || LEAD.test(f.raw) || HEADING.test(output);
|
||||
const ok = single === undefined ? (rank(f.label) ?? 0) >= (rank(e.main[i]) ?? 0)
|
||||
: (rank(f.label) ?? 0) >= (rank(single) ?? 0) && (raised || (rank(f.label) ?? 0) === (rank(single) ?? 0));
|
||||
if (!ok) bad.push(`${e.reader} main=${e.main[i]} got=${f.label} field=${single ?? '-'} ${JSON.stringify(f.raw.slice(0, 100))}`);
|
||||
});
|
||||
}
|
||||
expect(bad).toEqual([]);
|
||||
expect(findings).toBeGreaterThan(1000);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user