CalvinBackup/gstack
1
0 Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-11-02 15:04:37 +01:00
Code Issues Packages Projects Releases Wiki Activity

v1.91.367.0 fix(ship): collect never counts a non-review as clean, and a severity word used as a word is no finding (GST-434, GST-435) (#3677)

Browse Source 
* fix(ship): collect counts a reply with no finding and no clean marker as a non-review, never clean (GSTA-167)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(ship): a label word used as a word is no finding, and a paragraph finding keeps its detail bullets (GST-435)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(ship): a negation counts only when it applies to the label; compact table rows keep their label; register the review-ledger fixtures (pre-landing review)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(ship): negation stops at a conjunction, a summary line with a blocker stays, detail bullets need no gap and no locus; a re-sent review voids a refusal and the outside voice can be collected unavailable (pre-landing review)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(ship): bold and code marks keep a negated label a word, a subject phrase with and/or stays negated, table negation counts, --unavailable refuses an output with findings, VOIDED needs a re-send (fix-check)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(ship): a quoted label counts at the end of its line and in table cells, which ends a negated subject, and a negated subject never cancels a label on a file:line line (fix-check)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(ship): a line naming a file:line or a table row always keeps a finding; "of which" stays inside a negated subject (fix-check)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* test(ship): the review-pass fake outside voice ends with its Recommendation: line, the marker its brief prescribes (#3665 integration)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* revert(ship): parseFindings labels and negation back to main; GST-435 moves to its own PR (seed branch gsta-435-seed). Keeps the non-review rule and the detail-bullet merge (split)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

---------

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
Garry Tanandcapy-ai[bot] authored and GitHub committed 2026-10-31 13:03:38 +00:00
1 parent 12dc6782c2
commit 41a67b996d
12 files changed
+307 -32
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

lib/review-ledger.ts
+66 -17
View File
@@ -4,7 +4,7 @@
*
* rendered a brief gstack-review-dispatch render wrote (role, name, file, sha256)
* sent the prompt a dispatch program actually passed, by hash; a mismatch fails the pass
* output a reader's raw output (sha256) and the merged.jsonl ids collect wrote from it
* output a reader's raw output (sha256), the merged.jsonl ids collect wrote from it, and whether it was a review at all
* judgment each lead-judgment change: an added or promoted finding, with its reason
* judge the verdict on the parent's judged set (ok, or the dropped and downgraded ids)
*
@@ -72,11 +72,12 @@ export interface MergedLine { id: string; reader: string; label: string; raw: st
* of the two labels). A `Severity` or `Blocks` field line continues the open finding, once per field: `Severity` raises its label,
* `Blocks: yes` makes it CRITICAL, `Blocks: no` never lowers it; with no finding open it is one, carrying the paragraph's subject.
* A list item, a heading, a bold or post-blank paragraph at the margin, or a line naming a file:line after the open finding's fields
* starts the next finding. An inline `Blocks: yes` is CRITICAL too. The native voice's FIXABLE collects as CRITICAL; other labels
* stay as written for lead judgment. When in doubt the parser merges rather than drops: a lost blocker costs the gate.
* starts the next finding, except an unlabeled, unnumbered bullet with no file:line under a paragraph finding whose fields have not
* come yet, which is its detail and joins its text (GSTA-196: 4 findings with detail bullets collected as 8). An inline
* `Blocks: yes` is CRITICAL too. The native voice's FIXABLE collects as CRITICAL; other labels stay as written for lead judgment. When in doubt the parser merges rather than drops: a lost blocker costs the gate.
*/
export function parseFindings(reader: string, output: string): Array<{ label: string; raw: string }> {
type Finding = { label: string; raw: string; fields: Set<string> };
type Finding = { label: string; raw: string; fields: Set<string>; list: boolean };
const out: Finding[] = [];
let section: string | null = null;
let current: Finding | null = null;
@@ -85,7 +86,7 @@ export function parseFindings(reader: string, output: string): Array<{ label: st
const role = reader.split('/')[0];
const named = (l: string) => (role === 'adversarial-native' && l.toUpperCase() === 'FIXABLE' ? 'CRITICAL' : l);
const strongest = (t: string) => [...t.matchAll(ALL_LABELS)].map(m => named(m[1]!)).reduce<string | undefined>((a, b) => (a === undefined ? b : stronger(a, b)), undefined);
const push = (label: string, raw: string, fields: string[] = []): Finding => { const f = { label, raw, fields: new Set(fields) }; out.push(f); return f; };
const push = (label: string, raw: string, fields: string[] = []): Finding => { const f = { label, raw, fields: new Set(fields), list: LIST_ITEM.test(raw) }; out.push(f); return f; };
for (const line of output.replace(/\r\n/g, '\n').split('\n')) {
const t = line.trim();
if (!t) { gap = true; continue; }
@@ -118,7 +119,9 @@ export function parseFindings(reader: string, output: string): Array<{ label: st
if (typeof o.severity === 'string') { current = push(o.severity, t); continue; }
} catch { /* prose that starts with a brace */ }
}
const fresh = listItem || (!indented && (t.startsWith('**') || afterGap)) || (!!current?.fields.size && locus);
const detail = listItem && /^[-*+]\s/.test(t) && !locus && !label && !section && !!current && !current.list && !current.fields.size;
if (detail && !blocks) { current!.raw += ` | ${t}`; continue; }
const fresh = (listItem && !detail) || (!indented && !detail && (t.startsWith('**') || afterGap)) || (!!current?.fields.size && locus);
if (fresh) { current = null; head = t; }
const inherited = section && listItem ? section : undefined;
const found = label && inherited ? stronger(inherited, label) : label ?? inherited;
@@ -133,15 +136,39 @@ export function parseFindings(reader: string, output: string): Array<{ label: st
return out.map(({ label, raw }) => ({ label, raw }));
}
/** collect: hash the raw output, append its findings to merged.jsonl and record both. */
export function collect(ledger: string, reader: string, output: string, mergedFile: string, outputFile?: string): { sha: string; lines: MergedLine[] } {
/**
* The clean marker each reader's own brief prescribes for a review that found nothing: `NO FINDINGS` alone on a line for the
* specialists, the batched specialist and the targeted reader (ship/sections/review-army.md, the specialist brief; review/specialists/*.md),
* the Red Team (its brief in ship/sections/review-army.md) and fix-check (scripts/resolvers/fix-check.ts); the `Recommendation:` line
* for the outside voice (ship/sections/adversarial.md, the outside prompt); either for the native adversarial voice, whose brief asks
* for both. A reply with no finding and no marker (a refusal, an empty reply, a reader that never saw the diff) is not a review.
*/
const NO_FINDINGS: [string, RegExp] = ['NO FINDINGS', /^[`*]*NO FINDINGS\.?[`*]*$/];
const RECOMMENDATION: [string, RegExp] = ['Recommendation:', /^[`*]*Recommendation:/i];
const CLEAN_MARKERS: Record<string, Array<[string, RegExp]>> = {
specialist: [NO_FINDINGS], 'specialist-batched': [NO_FINDINGS], targeted: [NO_FINDINGS], 'red-team': [NO_FINDINGS], 'fix-check': [NO_FINDINGS],
'adversarial-native': [NO_FINDINGS, RECOMMENDATION], 'outside-voice': [RECOMMENDATION],
};
/** The markers a reader's zero-finding reply may carry; a role outside the brief roles takes either. */
export const cleanMarkers = (reader: string) => CLEAN_MARKERS[reader.split('/')[0]!] ?? [NO_FINDINGS, RECOMMENDATION];
const hasCleanMarker = (reader: string, output: string) => output.replace(/\r\n/g, '\n').split('\n').some(l => cleanMarkers(reader).some(([, re]) => re.test(l.trim())));
/**
* collect: hash the raw output, append its findings to merged.jsonl and record both; a reply with no finding and no clean marker records
* review: false. With unavailable (the outside voice only) the output is its failure text, recorded as missing coverage.
*/
export function collect(ledger: string, reader: string, output: string, mergedFile: string, outputFile?: string, unavailable?: string): { sha: string; lines: MergedLine[]; review: boolean } {
const sha = promptHash(output);
const prior = readLedger(ledger).filter(e => e.event === 'output' && e.reader === reader).length;
const lines = parseFindings(reader, output).map((f, i) => ({ id: `${reader}#${prior ? `${prior + 1}.` : ''}${i + 1}`, reader, label: f.label, raw: f.raw, output_sha256: sha }));
const lines = unavailable ? [] : parseFindings(reader, output).map((f, i) => ({ id: `${reader}#${prior ? `${prior + 1}.` : ''}${i + 1}`, reader, label: f.label, raw: f.raw, output_sha256: sha }));
const review = !unavailable && (lines.length > 0 || hasCleanMarker(reader, output));
fs.appendFileSync(mergedFile, lines.map(l => JSON.stringify(l) + '\n').join(''), { mode: 0o600 });
appendEvent(ledger, { event: 'output', reader, sha256: sha, file: outputFile ? path.resolve(outputFile) : null, merged: path.resolve(mergedFile), ids: lines.map(l => l.id) });
return { sha, lines };
appendEvent(ledger, { event: 'output', reader, sha256: sha, file: outputFile ? path.resolve(outputFile) : null, merged: path.resolve(mergedFile), ids: lines.map(l => l.id), review, ...(unavailable ? { unavailable } : {}) });
return { sha, lines, review };
}
/** The outside voice may fail without blocking (ship/sections/adversarial.md): its failure is recorded as unavailable coverage, never as a review. */
export const mayBeUnavailable = (reader: string) => reader.split('/')[0] === 'outside-voice';
const unavailableOutput = (o: LedgerEvent) => typeof o.unavailable === 'string' && mayBeUnavailable(String(o.reader));
export function readJsonl<T>(file: string): T[] {
return fs.readFileSync(file, 'utf8').split('\n').filter(l => l.trim()).map(l => JSON.parse(l) as T);
@@ -190,13 +217,24 @@ export function recordJudgment(ledger: string, merged: MergedLine[], judged: Jud
return res;
}
/**
* Whether a recorded output was a review: it has findings, or its file (unchanged since collect) carries the reader's clean marker, so
* a ledger written before collect recorded the field is judged by the same rule; with no readable file, collect's own review field.
*/
function reviewed(o: LedgerEvent): boolean {
if (Array.isArray(o.ids) && o.ids.length) return true;
let text: string | undefined;
try { text = typeof o.file === 'string' ? fs.readFileSync(o.file, 'utf8') : undefined; } catch { /* unreadable */ }
return text !== undefined && promptHash(text) === o.sha256 ? hasCleanMarker(String(o.reader), text) : o.review === true;
}
/** Every id collect recorded, in order. */
const collectedIds = (events: LedgerEvent[]) => events.filter(e => e.event === 'output').flatMap(e => (Array.isArray(e.ids) ? (e.ids as string[]) : []));
export interface VerifyRow { kind: 'BRIEF' | 'OUTPUT' | 'JUDGE'; reader: string; status: string; detail: string }
/** Replay every check from a finished run's ledger: briefs dispatched as rendered, outputs merged as returned, judgment without loss. */
export function verify(ledger: string): { rows: VerifyRow[]; clean: boolean; briefs: number; voided: number } {
export function verify(ledger: string): { rows: VerifyRow[]; clean: boolean; briefs: number; voided: number; unavailable: number } {
const events = readLedger(ledger);
const rows: VerifyRow[] = [];
const short = (s: unknown) => (typeof s === 'string' ? s.slice(0, 12) : '-');
@@ -225,10 +263,12 @@ export function verify(ledger: string): { rows: VerifyRow[]; clean: boolean; bri
}
voided += epVoided;
const lastMatch = [...sent].reverse().find(({ e }) => e.sha256 === r.sha256);
const collected = !!lastMatch && inside.some(({ e, i }) => e.event === 'output' && i > lastMatch.i);
const status = !sent.length ? 'NOT_SENT' : mismatched ? 'MISMATCH' : collected ? 'MATCH' : 'NOT_COLLECTED';
const outputs = lastMatch ? inside.filter(({ e, i }) => e.event === 'output' && i > lastMatch.i) : [];
const collected = outputs.some(({ e }) => reviewed(e));
const unavailable = !collected && outputs.some(({ e }) => unavailableOutput(e));
const status = !sent.length ? 'NOT_SENT' : mismatched ? 'MISMATCH' : collected ? 'MATCH' : unavailable ? 'UNAVAILABLE' : 'NOT_COLLECTED';
const tampered = fileNow && fileNow !== r.sha256 ? ' file_changed_since_render' : '';
rows.push({ kind: 'BRIEF', reader: p.label, status: tampered ? 'MISMATCH' : status, detail: `rendered=${short(r.sha256)} sent=${sent.map(({ e }) => short(e.sha256)).join(',') || '-'}${epVoided ? ` voided=${epVoided}` : ''}${tampered}` });
rows.push({ kind: 'BRIEF', reader: p.label, status: tampered ? 'MISMATCH' : status, detail: `rendered=${short(r.sha256)} sent=${sent.map(({ e }) => short(e.sha256)).join(',') || '-'}${epVoided ? ` voided=${epVoided}` : ''}${outputs.length && !collected && !unavailable ? ' not_reviewed' : ''}${tampered}` });
}
for (const [i, o] of events.entries()) {
if (o.event !== 'output') continue;
@@ -245,6 +285,15 @@ export function verify(ledger: string): { rows: VerifyRow[]; clean: boolean; bri
try { lines = typeof o.merged === 'string' ? readJsonl<MergedLine>(o.merged).filter(l => ids.includes(l.id)) : []; } catch { notes.push('merged_unreadable'); status = 'MISMATCH'; }
if (lines.length !== ids.length) { status = 'MISMATCH'; notes.push(`merged_lines=${lines.length}/${ids.length}`); }
if (lines.some(l => l.output_sha256 !== o.sha256)) { status = 'MISMATCH'; notes.push('merged_line_from_other_output'); }
if (status === 'MATCH' && !reviewed(o)) {
// A non-review the same dispatch later replaced with a review (the brief re-sent verbatim, then answered) is void, counted like a voided send.
const later = p ? events.slice(i + 1, p.next) : [];
const review = later.findIndex(e => e.event === 'output' && e.reader === reader && reviewed(e));
const replaced = review > 0 && later.slice(0, review).some(e => e.event === 'sent' && e.reader === reader && e.sha256 === events[p!.at]!.sha256);
if (unavailableOutput(o)) { status = 'UNAVAILABLE'; notes.push(`reason=${JSON.stringify(o.unavailable)}`); }
else if (replaced) { status = 'VOIDED'; voided++; notes.push('replaced_by_a_review'); }
else { status = 'NOT_REVIEWED'; notes.push('no_finding_no_clean_marker'); }
}
rows.push({ kind: 'OUTPUT', reader: p?.label ?? reader, status, detail: `sha=${short(o.sha256)} findings=${ids.length}${notes.length ? ' ' + notes.join(' ') : ''}` });
}
const lastJudge = events.map(e => e.event).lastIndexOf('judge');
@@ -266,6 +315,6 @@ export function verify(ledger: string): { rows: VerifyRow[]; clean: boolean; bri
const status = !v.ok ? 'REJECTED' : notes.length ? 'MISMATCH' : 'OK';
rows.push({ kind: 'JUDGE', reader: '-', status, detail: `dropped=${JSON.stringify(v.dropped)} downgraded=${JSON.stringify(v.downgraded)} added=${v.added} promoted=${v.promoted} folded=${v.folded}${notes.length ? ' ' + notes.join(' ') : ''}` });
} else if (events.some(e => e.event === 'output')) rows.push({ kind: 'JUDGE', reader: '-', status: 'MISSING', detail: 'outputs were collected but no judged set was checked' });
const clean = episodes.length > 0 && rows.every(r => r.status === 'MATCH' || r.status === 'OK');
return { rows, clean, briefs: episodes.length, voided };
const clean = episodes.length > 0 && rows.every(r => ['MATCH', 'OK', 'VOIDED', 'UNAVAILABLE'].includes(r.status));
return { rows, clean, briefs: episodes.length, voided, unavailable: rows.filter(r => r.kind === 'BRIEF' && r.status === 'UNAVAILABLE').length };
}