/ship reviewers state their blocker count in one line, and gstack checks it against their findings (GST-2495) (#4353)

* Reviewers state their blocker count in one line, and gstack checks it against the findings it parsed

Every blocking /ship reader's brief now asks for REVIEW_VERDICT: blocking=<n>. collect adds a CRITICAL row for each
declared blocker it did not parse, marks word-only CRITICAL rows as summary candidates when the count is lower, and
never lowers a row on its own except the native voice's unanchored FIXABLE raise. Ledgers rendered before this change
replay unchanged. Includes the preregistration for the no-loss replay.

Refs: GST-2495
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* Add the verdict-line no-loss replay results

Refs: GST-2495
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* Keep a finding line that mentions REVIEW_VERDICT instead of stripping it

Only a well-formed verdict line is removed before findings are parsed; prose that names the token, or a malformed attempt, stays in the text so its finding is still collected.

Review-Finding: p1/specialist-batched#1 reader=specialist-batched
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* Check a stored summary flag against the reply on every ledger, not only ones that asked for the verdict line

verify now re-derives summary_candidate for every merged row, so a flag edited onto a real CRITICAL in an older ledger can no longer let judgment lower it while verify reports clean.

Review-Finding: v2/outside-voice#1 reader=outside-voice
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* Never lower a finding automatically when a reviewer's blocker count is lower than the parse

A valid verdict line used to relabel the native voice's unanchored FIXABLE prose to its own label. Beside a
denial line such as 'No other CRITICAL issues.' that let a real blocker drop to FIXABLE while the count still
agreed. Those rows now stay CRITICAL and become summary candidates the lead lowers with a written reason.

Review-Finding: v3/adversarial-native#1 reader=adversarial-native
Refs: GST-2495
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* Split a run-on sentence in the verdict-line changelog entry

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

---------

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
Garry Tanandcapy-ai[bot] authored and GitHub committed 2026-11-14 07:31:35 +00:00
1 parent 89be362eb2
commit 3b650356f3
13 files changed
+590 -35

No files matched your search

+12 -5
View File
@@ -23,7 +23,7 @@ import * as fs from 'node:fs';
import * as path from 'node:path';
import { EXIT, exitTableHelp } from '../lib/headless-artifacts';
import { BRIEF_ROLES, BriefError, briefFile, checkReviewedTree, checkoutRoot, checkoutTree, promptHash, renderBrief, sectionTexts, sliceDiff, sliceNote, withWorkspace, type BriefRole, type Slots, type TreeCheck } from '../lib/review-brief';
import { cleanMarkers, collect, fixableRaises, mayBeUnavailable, parseFindings, readJsonl, readLedger, readerKey, recordJudgment, recordSent, appendEvent, verify, type JudgedLine, type MergedLine, type VerifyRow } from '../lib/review-ledger';
import { cleanMarkers, collect, fixableRaises, mayBeUnavailable, parseFindings, parseReply, readJsonl, readLedger, readerKey, recordJudgment, recordSent, appendEvent, verify, type JudgedLine, type MergedLine, type VerdictCheck, type VerifyRow } from '../lib/review-ledger';
import { buildManifest, chunkCount, chunkOf, manifestPath, markReader, planCyberRetry, readManifest, renderStage, withAuditModel, writeManifest, STAGES, type Extra, type Manifest, type PassReader, type ReaderState, type Stage } from '../lib/review-pass';
import { classifyOutsideReview } from '../lib/outside-review-result';
import { gateOutcomeLine } from '../lib/gate-outcomes';
@@ -192,14 +192,14 @@ function render(): number {
const workdir = checkoutRoot();
const tree = checkoutTree(workdir);
if (!tree) throw new BriefError(`cannot read HEAD^{tree} and the toplevel of ${workdir}; a brief names the checkout its reply must prove`);
const text = withWorkspace(renderBrief(role, slots, sectionTexts(ROOT)), workdir, readers);
const text = withWorkspace(renderBrief(role, slots, sectionTexts(ROOT)), workdir, readers, { verdict: true });
fs.mkdirSync(out, { recursive: true, mode: 0o700 });
const file = path.join(path.resolve(out), briefFile(role, name));
fs.writeFileSync(file, text, { mode: 0o600 });
const ledger = flag('--ledger') ?? path.join(out, 'review-ledger.jsonl');
const sha = promptHash(text);
const targeted = brief === undefined ? {} : { classes: parseClassList(flag('--classes')!), rules: TARGETED_RULES, brief_sha: targetedBriefSha(merged ? path.join(ROOT, 'review/specialists/targeted.md') : checklist), diff_base: diffBase ?? null, ...(diffBase ? targetedRead(process.cwd(), diffBase) : { patch_id: 'none', head: null }) };
appendEvent(ledger, { event: 'rendered', reader: readerKey(role, name), role, name: name ?? null, file, sha256: sha, bytes: Buffer.byteLength(text), workdir, tree, ...targeted, ...(diffFile.diff_file ? { payload: { file: diffFile.diff_file, sha256: diffFile.diff_sha256 } } : {}), ...(disclosureDigest(sel.disclosure) ? { exclude: disclosureDigest(sel.disclosure) } : {}), readers: readers.digest });
appendEvent(ledger, { event: 'rendered', reader: readerKey(role, name), role, name: name ?? null, file, sha256: sha, bytes: Buffer.byteLength(text), workdir, tree, verdict: 1, ...targeted, ...(diffFile.diff_file ? { payload: { file: diffFile.diff_file, sha256: diffFile.diff_sha256 } } : {}), ...(disclosureDigest(sel.disclosure) ? { exclude: disclosureDigest(sel.disclosure) } : {}), readers: readers.digest });
process.stdout.write(`BRIEF: reader=${readerKey(role, name)} sha256=${sha} bytes=${Buffer.byteLength(text)} file=${file}\n`);
return EXIT.ok;
}
@@ -293,6 +293,9 @@ function sweep(): number {
}
const quote = (s: string) => `'${s.replace(/'/g, "'\\''")}'`;
/** The reader's REVIEW_VERDICT line against what collect parsed (GST-2495), with the next step when they disagree or the line is malformed. */
const verdictLine = (reader: string, v: VerdictCheck, added: string[]) => `COLLECT_VERDICT: reader=${reader} declared=${v.declared ?? v.state} parsed=${v.parsed}${v.state === 'valid' ? ` agrees=${v.agrees ? 'yes' : 'no'}` : ''}${added.length ? ` added=${added.join(',')}` : ''}${v.why ? ` why=${JSON.stringify(v.why)}` : ''}`
+ (v.state === 'malformed' || v.agrees === false ? ' (next: find the declared blocker via at=, promote it and fold the added row into it; lower a summary candidate with a reason; or take the review-loop exception; docs/review-policy.md#verdict-line)\n' : '\n');
const wrongTree = (reader: string, t: TreeCheck) => `${t.moved ? 'COLLECT_TREE_MOVED' : 'COLLECT_WRONG_TREE'}: reader=${reader} reported=${JSON.stringify(t.reported)} expected=${JSON.stringify(t.expected)} (${t.moved ? 'the checkout moved after this brief was rendered' : 'the reply did not review the checkout its brief named'}; nothing merged; restore it with a fresh pass under a new --pass and --out)\n`;
const PROGRAM_FILE = path.join(ROOT, 'lib', 'capy-review-pass.js');
@@ -599,10 +602,12 @@ function main(): number {
const unavailable = flag('--unavailable');
if (unavailable !== undefined && !mayBeUnavailable(reader)) { process.stderr.write(`gstack-review-dispatch collect: --unavailable is for the outside voice only; ${reader}'s missing coverage leaves the review incomplete, never clean\n`); return EXIT.usage; }
if (unavailable !== undefined && !unavailable.trim()) { process.stderr.write('gstack-review-dispatch collect: --unavailable needs a reason\n'); return EXIT.usage; }
if (unavailable !== undefined && parseFindings(reader, output).length) { process.stderr.write(`gstack-review-dispatch collect: ${reader}'s output holds findings; collect it without --unavailable so none is lost\n`); return EXIT.usage; }
const asked = readLedger(ledger).filter(e => e.event === 'rendered' && e.reader === reader).at(-1)?.verdict === 1;
const held = () => parseReply(reader, output, true, asked).findings.length;
if (unavailable !== undefined && held()) { process.stderr.write(`gstack-review-dispatch collect: ${reader}'s output holds findings; collect it without --unavailable so none is lost\n`); return EXIT.usage; }
const rendered = readLedger(ledger).filter(e => e.event === 'rendered' && e.reader === reader).at(-1);
const tree = unavailable === undefined && rendered && 'tree' in rendered ? checkReviewedTree(output, typeof rendered.tree === 'string' ? rendered.tree : null) : undefined;
if (tree && !tree.ok && mayBeUnavailable(reader) && !parseFindings(reader, output).length) {
if (tree && !tree.ok && mayBeUnavailable(reader) && !held()) {
const why = `reviewed another tree: reported ${tree.reported}, expected ${tree.expected}`;
const r = collect(ledger, reader, output, merged, path.resolve(outFile!), why);
process.stdout.write(`COLLECT_UNAVAILABLE: reader=${reader} sha256=${r.sha} reason=${JSON.stringify(why)} (missing outside coverage: unavailable, never clean)\n`);
@@ -623,6 +628,7 @@ function main(): number {
return EXIT.fail;
}
process.stdout.write(`COLLECT: reader=${reader} findings=${r.lines.length} sha256=${r.sha} merged=${merged}\n`);
if (r.verdict) process.stdout.write(verdictLine(reader, r.verdict, r.lines.filter(l => /#(?:\d+\.)?v\d+$/.test(l.id)).map(l => l.id)));
return EXIT.ok;
}
if (cmd === 'judge') {
@@ -658,6 +664,7 @@ function main(): number {
if (sel && 'digest' in sel) process.stdout.write(`REVIEW_EXCLUDE_RECORD: digest=${sel.digest} (gstack-ship-receipt write --review-exclude ${sel.digest})\n`);
if (sel && 'mixed' in sel) process.stdout.write(`REVIEW_EXCLUDE_RECORD: mixed digests=${sel.mixed.join(',')} (readers read different review_exclude_paths selections; rerun the review so every reader reads HEAD's)\n`);
const clean = v.clean && !(sel && 'mixed' in sel);
if (v.verdicts.asked) process.stdout.write(`VERIFY_VERDICT: agree=${v.verdicts.agree} asked=${v.verdicts.asked} (informational: a disagreeing count already counts its blockers; docs/review-policy.md#verdict-line)\n`);
process.stdout.write(`VERIFY: ${clean ? 'clean' : 'dirty'} briefs=${v.briefs} outputs=${v.rows.filter(r => r.kind === 'OUTPUT').length} voided=${v.voided}${v.unavailable ? ` unavailable=${v.unavailable}` : ''}\n`);
return clean ? EXIT.ok : EXIT.fail;
}
+1
View File
@@ -0,0 +1 @@
- **/ship's reviewers now state how many findings block in one machine-readable line, and gstack checks that count instead of guessing from their prose.** Every blocking reader's brief asks for `REVIEW_VERDICT: blocking=<n>` (with optional `at=<path>:<line>` locations). When the count is higher than the findings gstack parsed, collect adds a CRITICAL row for each missing blocker, so a blocker written in a form the parser misses can no longer vanish. When it is lower, nothing is lowered automatically. Rows that block only because a sentence used the word CRITICAL or P0, or because the native adversarial voice wrote FIXABLE in plain prose (FOUN-298's 'all FIXABLE, nothing blocks'), become summary candidates the lead may lower with a written reason. A missing or malformed line changes nothing. Ledgers written before this change replay exactly as before. See docs/review-policy.md#verdict-line.
+99
View File
@@ -0,0 +1,99 @@
# Preregistration: the reviewer verdict line, no-loss and compliance replay (GST-2495)
Frozen before any panel dispatch, on 2026-11-13. This is the no-loss replay that docs/review-policy.md requires for a
change to reviewer prompts, and the evidence that the change helps on frontier models. The change under test is
[the verdict line](../review-policy.md#verdict-line): every blocking reader's brief asks for one
`REVIEW_VERDICT: blocking=<n>` line, and collect checks that count against the findings it parses.
## Question
Does asking blocking readers for the line lose any blocker, and do Claude Opus 5.5 and GPT-6.1 Sol write the line
correctly when asked?
## Panel
A small synthetic TypeScript service, built by `fixture.sh` (public-safe, no real project data), with eight branches
off `main`. Four plant a real blocker, one per failure family the old prose parser lost or invented:
| Branch | Planted defect | Tree |
|---|---|---|
| b1-log-headers | Logs every request header, including `Authorization` (secret leak) | da7d95ef |
| b2-cache-key | The review cache key drops the tree, so one tree's review is served for another | d0053de2 |
| b3-migration | Deletes the old rows before the copy, outside the transaction (data loss) | c7828803 |
| b4-admin-route | A new delete-user admin route skips `requireAdmin` | 2cff15ff |
| c1-rename | Clean: renames `plural` to `pluralize` with its test | d5716f57 |
| c2-paginate | Clean: paginates the notes list with a clamped page size | 35fd32e3 |
| c3-format | Clean: `formatBytes` learns gigabytes, with a test | 983a820e |
| c4-forbidden-text | Clean: a clearer 403 body and a doc comment | a4e91e29 |
Every branch's own tests pass.
## Readers, models and arms
- Five brief families: adversarial-native, outside-voice, specialist (rendered as specialist-batched with the
security, data-migration, testing and performance checklists; specialist and targeted share its section body),
red-team (the specialists' findings slot says they found nothing) and fix-check (the fix list names an earlier
finding that the branch's one commit fixes; on a blocker branch that fix introduces the planted defect).
- Briefs are rendered by `gstack-review-dispatch render`: treatment with this branch's binary, control with main's
(4bee369). `compare-briefs.ts` proves the 40 pairs are byte-identical apart from the two verdict sentences.
- Models under test: `claude-opus-5-5` (Anthropic Messages API, adaptive thinking, effort high) and `gpt-6.1-sol`
(OpenAI Responses API, reasoning effort high, stateless). Each dispatch is a tool loop with one bash tool run in that
branch's checkout with an environment holding no keys and no network commands, at most 40 tool calls, 16,000
output tokens per turn.
- Per cell (family x branch x model): 2 treatment trials and 1 control trial. 5 x 8 x 2 x 3 = 240 dispatches, in an
order shuffled with seed 2495, 8 at a time.
- A model under test is never swapped. A `cyber_policy` refusal is recorded as a result and is not retried on another
model. Transport errors (HTTP 429, 5xx, network) retry up to three times inside one dispatch; that is not a rerun.
- The run happens once. If the process dies, a restart completes only the dispatches with no recorded result. The
AGENTS.md single rerun applies only to a red rule case, and none of these metrics is a model judge.
## Scoring (mechanical, `analyze.ts`; no model judge, so the R78 judge check does not apply)
Every reply is parsed by this branch's `parseReply` on the gate class (FIXABLE raises), the strictest setting for
the native lowering. "Main's parse" is `parseReply` with no ask, which the corpus identity test pins to main's output.
A blocking row is a row at CRITICAL with no `Blocks: no` of its own; summary candidates still count, because they block
until lead judgment lowers them with a written reason.
Pass rule, both required:
1. **G1, mechanism no-loss:** no treatment reply that blocks under main's parse has zero blocking rows under the verdict
parse. Replies whose only blocking rows became summary candidates are listed and read by hand.
2. **G2, arm no-loss (the Captain's rule):** on the four blocker branches, no treatment trial has zero blocking rows
when the same cell's control trial has at least one. Each loss is labelled mechanism (G1 fails on it too) or reader
miss, and any loss fails the rule.
With zero losses over N = 80 blocker treatment trials, the 95% upper bound on the loss rate is 3/80, about 3.8%. That
is the smallest loss rate this panel can rule out.
Reported, not gating, each with a 95% interval (Wilson for proportions, a paired normal interval for differences):
3. Compliance: valid lines over treatment replies, per model and family (16 replies per cell, about +/-20 points) and
per model (80 replies, about +/-9 points). Under 80% on a model is a FIX-METHOD finding for the wording.
4. Agreement: lines whose count matches the parsed blocking rows, over valid lines.
5. Contradictions: valid `blocking=0` lines on replies that block under main's parse, each read by hand.
6. Clean branches: blocking rows per reply, treatment mean minus control, paired by cell (40 cells), and the share
of replies with any blocking row.
7. Secondary: G1 on the relaxed class.
Not measured: the Codex CLI outside-voice route, the targeted reader's class-specific brief, and readers on a Capy
machine (this harness calls the APIs directly with the same rendered brief bytes and a read-only shell).
R44: there is no no-gstack arm, because the line exists only inside gstack's dispatch; this compares two gstack briefs.
## Budget
Up to $260 of API spend, granted by the Captain. The runner stops starting dispatches at $245. Two smoke dispatches
on panel cells, written to a separate directory and excluded from the results, cost $0.10.
## Frozen inputs (sha256)
| File | sha256 |
|---|---|
| fixture.sh | b18009fa0b9542ca04a94407472ec616ffa68d33626e8c25ef314a7f8fa0db3a |
| inputs.sh | 52c391f3855e9e7fe5556ec2d3e171430aa63b0e0448a40661491781591393d8 |
| render.sh | 445dbd7441a3e31dc20977017aa2c4fb712c753971c42cf3168f7a135ee0c174 |
| compare-briefs.ts | 129c22082d9e16042687a50798eef7582c02b6eb9e8910097472db6b3b3c6e83 |
| run.ts | 7a5cf31b3731a14afc3a508173b9baac11abc782f84df51f70b1b69f2c95bdfe |
| analyze.ts | 536349ef42f57b75589d9ca94ed8f1ee8d8a0aa4a015e341a5752fd1300486ef |
The harness files are kept with the results.
+53
View File
@@ -0,0 +1,53 @@
# Verdict-line replay: results (GST-2495)
We learned that Claude Opus 5.5 and GPT-6.1 Sol write the requested `REVIEW_VERDICT: blocking=<n>` line every time
they are asked (160 of 160 replies), and that the count matches gstack's own parse 96.9% of the time. You should
treat the line as a cross-check on the parser, not a replacement: checking it lost no blocker in 80 paired trials,
and where it disagreed, the parser had split or invented a row.
Preregistration: [verdict-line-replay-prereg.md](verdict-line-replay-prereg.md), frozen at commit b5d0abe before any
panel dispatch. All 240 dispatches ran once on 2026-11-14, with no errors, no `cyber_policy` refusals and no reruns.
API cost: $15.61.
## What worked
- **No blocker was lost.** G1 (the same reply under main's parse and the verdict parse): 0 of 160 treatment replies
stopped blocking. G2 (each treatment trial against the same cell's control on the four blocker branches): 0 lost
blockers in 80 trials, so the 95% upper bound on the loss rate is 3.8%. Both arms flagged a blocker in 80 of 80
blocker trials (95% interval 95.4% to 100%). The planted file was named in a blocking row slightly more often
with the ask (paired difference +3.7 points, interval -5.1 to +12.6, so no real difference).
- **The line is written reliably.** Compliance was 80/80 for each model (95.4% to 100%) and 16/16 in every model and
family cell. No reply left the line out or wrote it malformed.
- **Agreement was high.** 155 of 160 valid lines matched the parsed blocking count (96.9%, interval 92.9% to 98.7%):
Opus 78/80, Sol 77/80.
## What didn't, or wasn't shown
- **The five disagreements were parser errors, not reader errors.** Read by hand:
- Three Sol native replies (b3-migration, b4-admin-route, c2-paginate) declared 1 blocker while the parser counted 2:
a `### FIXABLE — …` heading and its trailing `| Severity: … | Blocks: yes` line collected as two rows. Both
rows stay blocking, which is the safe direction.
- One Opus red-team reply on b1-log-headers declared 1; the parser also counted a bold list item that explains the
count ("**Blocking count:** this counts CRITICAL findings, so only the credential-logging issue blocks"). That
phantom stays blocking too, because a bold-led list item is anchored.
- One Opus red-team reply on clean c2-paginate declared 0 while its closing sentence ("I count only CRITICAL
findings as blocking") collected as CRITICAL. This is the one contradiction, and the CAPY-1 shape: with the
line, that row is a summary candidate the lead may lower with a reason, instead of an unexplained blocker.
- **The native FIXABLE lowering never fired in live replies, and it was removed after this run.** Today's native
brief already asks for `Severity` and `Blocks` on every finding, and no reply wrote the FOUN-298 sentence shape.
/ship's review of this PR then found a reply shape where that automatic lowering loses a real blocker, so the
shipped code lowers nothing by itself: such rows become summary candidates the lead lowers with a written reason.
Because the lowering fired 0 times in 160 replies, every figure above is the same under the shipped code.
- **No clean-diff improvement was shown.** Blocking rows per clean-branch reply: treatment 0.063, control 0.100;
paired difference -0.037 (interval -0.111 to +0.036, 40 cells). The clean-branch blockers were almost all Sol
judging that a `NaN` page size in c2-paginate blocks, in both arms; the line does not change a reader's judgment.
## Mechanics
- Panel: eight synthetic branches (four planted blockers, four clean), five brief families, two models, two treatment
trials and one control trial per cell. Briefs were rendered by `gstack-review-dispatch render`, treatment from this
branch and control from main 4bee369, and the 40 pairs are byte-identical apart from the two verdict sentences.
- Each dispatch was a direct API tool loop with one read-only bash tool in the branch's checkout (no keys in its
environment, no network commands). This is not a Capy-hosted reader; the Codex CLI route was not measured.
- Secondary: on the relaxed review class, G1 was also 0 of 160.
- Harness, inputs, per-reply outputs and the full per-reply table are kept with the crew's handoff for GSTA-526.
+32
View File
@@ -967,6 +967,38 @@ ones, and the arm lost blockers. `bin/gstack-review-dispatch` closes both ends:
line for the specialists, the batched specialist, the targeted reader, the Red Team and fix-check, the `Recommendation:`
line for the outside voice, either for the native adversarial voice. Without one (a refusal, an empty reply, a reader
that never saw the diff) collect records `review: false`, prints `COLLECT_NOT_REVIEWED` and exits 1.
<a id="verdict-line"></a>
3a. **The verdict line (GST-2495).** Every blocking reader's brief (specialists, the batched specialist, the
targeted reader, the Red Team, fix-check, the native adversarial voice and the outside voice; audit readers
are not asked) tells it to put one line in its reply:
```
REVIEW_VERDICT: blocking=<n>[ at=<path>:<line>,<path>:<line>...]
```
`<n>` is how many findings in the reply block (0 to 99), and `at=` may name up to `<n>` of their locations.
`render` records `verdict: 1` on a brief that asked for the line, and `collect` checks a reply only when its
rendered brief asked, so a ledger written before this change replays exactly as it did. A well-formed line never
collects as a finding; a line that only starts with `REVIEW_VERDICT` stays for the parser. A line inside a code fence is ignored; a line repeated byte for byte counts once, and
two different lines are malformed. Collect still parses every finding as described in step 3, then compares:
| Reply | What collect does |
|---|---|
| No line | Findings as parsed. `COLLECT_VERDICT: reader=<r> declared=missing parsed=<k>`. |
| Malformed line (bad count, bad `at=`, more locations than `<n>`, two different lines) | Findings as parsed. `declared=malformed`, with the reason in `why=`. |
| `blocking=<n>` equals the blocking rows parsed | Findings as parsed. `agrees=yes`. |
| `<n>` is more than parsed | Every parsed row stays, and collect adds CRITICAL rows (ids `<reader>#v1`, `#v2`…): one per `at=` location that no parsed blocking row names, then one more row for the declared blockers left over. The added rows count toward what blocks like any other. |
| `<n>` is less than parsed | Nothing is lowered automatically. Each blocking row with no list item, heading, field, lead label, JSON or file:line, and no `Blocks: yes` (a row that blocks only because a sentence uses the word CRITICAL or P0, or because the native voice's FIXABLE raises it) is marked `summary_candidate`, so lead judgment may lower it with a written reason, which `judge` records as `dismissed`. Anchored rows keep blocking. |
A valid line never lowers a row by itself. FOUN-298's native "nothing blocks … all FIXABLE" sentence with
`blocking=0` still collects as CRITICAL, as a summary candidate the lead lowers with a reason. An automatic relabel
was tried and removed (gstack#4353): beside a denial line such as "No other CRITICAL issues.", it let a real
unanchored FIXABLE blocker drop to FIXABLE while the count still agreed. A reader that undercounts a real blocker
therefore loses nothing without a written reason. `verify` re-parses each output the way its
render asked, prints `VERIFY_VERDICT: agree=<a> asked=<n>` and one `VERDICT <reader> AGREES|DISAGREES|MISSING|MALFORMED`
row per checked reply. These are informational: a disagreeing count has already added or marked its rows, so
they never make a ledger dirty. When collect prints `agrees=no` or `declared=malformed`, read the reply, then judge the
added rows and summary candidates as usual.
4. Lead judgment writes `judged.jsonl`; `judge` refuses a set that drops a merged
finding, lowers its severity, folds it into a less severe duplicate, or adds or
promotes one without a reason (`JUDGMENT_REJECTED`). A merged file missing any id
+12 -5
View File
@@ -212,20 +212,27 @@ const treeCommand = (dir: string) => `cd ${quoted(dir)} && echo "REVIEWED_TREE:
* follows a findings-only output format can skip a request that only closes a long brief (gpt-6.1-sol did on GST-850), and
* "the line <command> prints" was read as the command itself, so the step comes first and names the output line.
*/
export function workspaceStep(dir: string): string {
return `First step, before anything else: run \`${treeCommand(dir)}\`. It outputs one line, \`REVIEWED_TREE: <tree hash> <path>\`. Copy that output line, not the command, as the first line of your final reply, above everything else, even where this brief limits your output to findings or says how to end it; a reply without it, or from another checkout, does not count.`;
export function workspaceStep(dir: string, verdict = false): string {
return `First step, before anything else: run \`${treeCommand(dir)}\`. It outputs one line, \`REVIEWED_TREE: <tree hash> <path>\`. Copy that output line, not the command, as the first line of your final reply, above everything else, even where this brief limits your output to findings or says how to end it; a reply without it, or from another checkout, does not count.${verdict ? ` ${VERDICT_ASK}` : ''}`;
}
/**
* The verdict ask every blocking reader's brief carries right after the fingerprint step (GST-2495, lib/review-verdict.ts): one
* machine-readable count of the reply's blocking findings, which collect checks against the findings it parses.
*/
export const VERDICT_ASK = 'Put `REVIEW_VERDICT: blocking=<n>` on the second line, right under it: n is how many findings in your reply block the merge by this brief\'s own rule, 0 when none; add ` at=<path>:<line>,…` for those findings that have a location. Write it once, on its own line, outside any code block; it does not replace your findings or the ending this brief asks for.';
/**
* The closing paragraphs every dispatched brief carries: the fetch rule and the readers policy (GST-1322, lib/review-fetch.ts),
* then the workspace (GST-482): a reader on a shared machine starts in that machine's working directory, and every command runs
* in a fresh shell, so the brief names the checkout under review and asks again for the fingerprint workspaceStep opens with.
* The section's render sits between workspaceStep and these, byte for byte what renderBrief returns.
*/
export function workspaceParagraph(dir: string, readers?: ReadersPolicy): string {
return `${fetchParagraph(readers)}\n\nWorkspace: the checkout under review is ${dir}. Your shell and file tools may start in another checkout, and every command runs in a fresh shell: begin each command with \`cd ${quoted(dir)} && \`, and give every file read, search, edit or write an absolute path under ${dir}. The first line of your final reply is the \`REVIEWED_TREE: <tree hash> <path>\` line that \`${treeCommand(dir)}\` outputs (the output, not the command), even where this brief limits your output to findings; a reply without it, or from another checkout, does not count.`;
export function workspaceParagraph(dir: string, readers?: ReadersPolicy, verdict = false): string {
return `${fetchParagraph(readers)}\n\nWorkspace: the checkout under review is ${dir}. Your shell and file tools may start in another checkout, and every command runs in a fresh shell: begin each command with \`cd ${quoted(dir)} && \`, and give every file read, search, edit or write an absolute path under ${dir}. The first line of your final reply is the \`REVIEWED_TREE: <tree hash> <path>\` line that \`${treeCommand(dir)}\` outputs (the output, not the command), even where this brief limits your output to findings; a reply without it, or from another checkout, does not count.${verdict ? ' The second line is your `REVIEW_VERDICT: blocking=<n>` line; gstack checks that count against your findings, so do not repeat it.' : ''}`;
}
export const withWorkspace = (text: string, dir: string, readers?: ReadersPolicy) => `${workspaceStep(dir)}\n\n${text}\n\n${workspaceParagraph(dir, readers)}`;
/** A blocking reader's brief (one of BRIEF_ROLES) takes the verdict ask; an audit reader, which cannot block, does not. */
export const withWorkspace = (text: string, dir: string, readers?: ReadersPolicy, opts: { verdict?: boolean } = {}) => `${workspaceStep(dir, opts.verdict)}\n\n${text}\n\n${workspaceParagraph(dir, readers, opts.verdict)}`;
/** The checkout a brief names: the toplevel of the directory the bins run in, where its diff and merge base come from. */
export function checkoutRoot(): string {
const r = spawnSync('git', ['rev-parse', '--show-toplevel'], { encoding: 'utf8', timeout: 30_000 });
+68 -21
View File
@@ -18,6 +18,7 @@ import { createHash } from 'node:crypto';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { checkReviewedTree, isReviewedTreeLine, promptHash, type TreeCheck } from './review-brief';
import { parseVerdict, withoutVerdictLines } from './review-verdict';
export interface LedgerEvent { event: 'rendered' | 'sent' | 'output' | 'judgment' | 'judge' | 'program' | 'quarantined' | 'fetch-sweep'; [k: string]: unknown }
export const readerKey = (role: string, name?: string) => (name ? `${role}/${name}` : role);
@@ -191,7 +192,7 @@ export interface MergedLine { id: string; reader: string; label: string; stated?
* Severity or Blocks: yes field, the native voice's FIXABLE or INVESTIGATE, or a file:line on it or on the line it runs on from) collects
* nothing; elsewhere it collects as above. A line led by a bold, unlabelled lead ("**1. Coverage:** …") is read for a closing summary after it.
*/
function parseLabelled(reader: string, output: string, fixableRaises: boolean): Array<{ label: string; raw: string; shaped: boolean }> {
function parseLabelled(reader: string, output: string, fixableRaises: boolean): Array<{ label: string; raw: string; shaped: boolean; anchored: boolean }> {
type Finding = { label: string; raw: string; fields: Set<string>; list: boolean; lead?: string; fixable?: boolean; heading?: boolean; item?: boolean; nested?: boolean; closed?: boolean; section?: boolean; context?: string };
const out: Finding[] = [];
let section: string | null = null;
@@ -312,19 +313,51 @@ function parseLabelled(reader: string, output: string, fixableRaises: boolean):
const unblocked = (f: Finding) => fixableRaises || role !== 'adversarial-native' || !(f.fixable || f.label.toUpperCase() === 'FIXABLE' || /(?<![A-Za-z0-9])FIXABLE(?![A-Za-z0-9])/.test(f.raw)) || statesNoBlocks(f.raw);
const relaxed = fixableRaises && role === 'adversarial-native' ? parseLabelled(reader, output, false) : undefined;
const statesSeverity = (raw: string) => { const ls = [...raw.matchAll(SEVERITY_FIELD)].flatMap(m => (m[1] ?? m[2]!).match(FIELD_LABELS)!); return ls.length === 1 && !!rank(ls[0]); };
const shaped = (f: Finding) => f.section || f.heading || [...f.raw.split(' | '), f.context ?? ''].some(t => t.startsWith('{') || LEAD_LABEL.test(t) || META.test(t) || !!t.match(STATED_SEVERITY) || BLOCKS_YES.test(t)
|| LOCUS.test(t) || PATH_LOCUS.test(t) || (role === 'adversarial-native' && /(?<![A-Za-z0-9])(?:FIXABLE|INVESTIGATE)(?![A-Za-z0-9])/i.test(t)));
const shaped = (f: Finding, tag = true) => f.section || f.heading || [...f.raw.split(' | '), f.context ?? ''].some(t => t.startsWith('{') || LEAD_LABEL.test(t) || META.test(t) || !!t.match(STATED_SEVERITY) || BLOCKS_YES.test(t)
|| LOCUS.test(t) || PATH_LOCUS.test(t) || (tag && role === 'adversarial-native' && /(?<![A-Za-z0-9])(?:FIXABLE|INVESTIGATE)(?![A-Za-z0-9])/i.test(t)));
return out.map((f, i) => {
const own = relaxed?.length === out.length && relaxed[i]!.raw === f.raw && statesNoBlocks(f.raw) && statesSeverity(f.raw) ? relaxed[i]!.label : undefined;
return { label: own ?? (unblocked(f) ? f.label : 'CRITICAL'), raw: f.raw, shaped: shaped(f) };
return { label: own ?? (unblocked(f) ? f.label : 'CRITICAL'), raw: f.raw, shaped: shaped(f), anchored: f.list || shaped(f, false) };
});
}
export function parseFindings(reader: string, output: string, fixableRaises = true): Array<{ label: string; raw: string }> {
const clean = output.split('\n').some(l => /^NO FINDINGS\.?$/.test(l.trim()));
return parseLabelled(reader, output, fixableRaises).flatMap(({ label, raw, shaped }) => {
return shaped || !clean ? [{ label, raw }] : [];
});
return parseReply(reader, output, fixableRaises, false).findings.map(({ label, raw }) => ({ label, raw }));
}
export interface VerdictCheck { state: 'valid' | 'missing' | 'malformed'; parsed: number; declared?: number; agrees?: boolean; why?: string }
export interface ReplyFinding { label: string; raw: string; summary?: true; synthetic?: true }
const blockingRow = (f: { label: string; raw: string }) => (rank(f.label) ?? 0) >= 4 && !statesNoBlocks(f.raw);
/**
* A reply's findings, and with asked (its render carried the verdict ask, GST-2495) the check of its REVIEW_VERDICT line
* (lib/review-verdict.ts, docs/review-policy.md#verdict-line). A well-formed verdict line never parses as a finding, and a valid
* line lowers nothing by itself. Declared more than parsed adds CRITICAL rows, one per unmatched at= location and one for the
* rest; declared fewer marks each blocking row with no anchor (no list item, heading, field, lead label, JSON or file:line) and
* no Blocks: yes a summary candidate, whether a CRITICAL or P0 word or the native voice's FIXABLE raise made it block. It still
* blocks until lead judgment lowers it with a reason.
*/
export function parseReply(reader: string, output: string, fixableRaises = true, asked = false): { findings: ReplyFinding[]; verdict?: VerdictCheck } {
const verdict = asked ? parseVerdict(output) : undefined;
const text = asked ? withoutVerdictLines(output) : output;
const clean = text.split('\n').some(l => /^NO FINDINGS\.?$/.test(l.trim()));
const rows = parseLabelled(reader, text, fixableRaises).filter(r => r.shaped || !clean);
if (verdict?.state !== 'valid') {
const findings = rows.map(({ label, raw }) => ({ label, raw }));
return { findings, ...(verdict ? { verdict: { state: verdict.state, parsed: findings.filter(blockingRow).length, ...(verdict.state === 'malformed' ? { why: verdict.why } : {}) } } : {}) };
}
const n = verdict.blocking;
const parsed = rows.filter(blockingRow).length;
const findings: ReplyFinding[] = rows.map(f => ({ label: f.label, raw: f.raw, ...(n < parsed && blockingRow(f) && !f.anchored && !BLOCKS_YES.test(f.raw) ? { summary: true as const } : {}) }));
if (n > parsed) {
const loci = new Set(rows.filter(blockingRow).map(f => lineLocus(f.raw)));
const unmatched = verdict.at.map(a => a.replace(/^\.\//, '')).filter(a => !loci.has(a)).slice(0, n - parsed);
const rest = n - parsed - unmatched.length;
const note = `REVIEW_VERDICT declared blocking=${n}, but collect parsed ${parsed} blocking finding${parsed === 1 ? '' : 's'} in this reply`;
for (const a of unmatched) findings.push({ label: 'CRITICAL', raw: `${a}: ${note}; this declared blocker names this location`, synthetic: true });
if (rest > 0) findings.push({ label: 'CRITICAL', raw: `${note}; ${rest} declared blocker${rest === 1 ? '' : 's'} with no location named`, synthetic: true });
}
return { findings, verdict: { state: 'valid', parsed, declared: n, agrees: n === parsed } };
}
/**
@@ -361,19 +394,23 @@ const hasCleanMarker = (reader: string, output: string) => output.replace(/\r\n/
* review: false. With unavailable (the outside voice only) the output is its failure text, recorded as missing coverage. With a
* failed tree check (GST-482: the reply reviewed another checkout, or names none) nothing is merged and the output is no review.
*/
export function collect(ledger: string, reader: string, output: string, mergedFile: string, outputFile?: string, unavailable?: string, tree?: TreeCheck, raises = true): { sha: string; lines: MergedLine[]; review: boolean } {
export function collect(ledger: string, reader: string, output: string, mergedFile: string, outputFile?: string, unavailable?: string, tree?: TreeCheck, raises = true): { sha: string; lines: MergedLine[]; review: boolean; verdict?: VerdictCheck } {
const sha = promptHash(output);
const prior = readLedger(ledger).filter(e => e.event === 'output' && e.reader === reader).length;
const events = readLedger(ledger);
const prior = events.filter(e => e.event === 'output' && e.reader === reader).length;
const wrongTree = !unavailable && tree !== undefined && !tree.ok;
const lines = unavailable || wrongTree ? [] : parseFindings(reader, output, raises).map((f, i) => {
const reply = unavailable || wrongTree ? { findings: [] } : parseReply(reader, output, raises, lastRendered(events, reader)?.verdict === 1);
let parsedNo = 0, addedNo = 0;
const lines = reply.findings.map(f => {
const s = stated(f.raw);
return { id: `${reader}#${prior ? `${prior + 1}.` : ''}${i + 1}`, reader, label: f.label, ...(s ? { stated: s } : {}), ...(summaryCandidate(f.raw) ? { summary_candidate: true as const } : {}), raw: f.raw, output_sha256: sha };
const id = `${reader}#${prior ? `${prior + 1}.` : ''}${f.synthetic ? `v${++addedNo}` : ++parsedNo}`;
return { id, reader, label: f.label, ...(s ? { stated: s } : {}), ...(f.summary || summaryCandidate(f.raw) ? { summary_candidate: true as const } : {}), raw: f.raw, output_sha256: sha };
});
const review = !unavailable && !wrongTree && (lines.length > 0 || hasCleanMarker(reader, output));
fs.appendFileSync(mergedFile, lines.map(l => JSON.stringify(l) + '\n').join(''), { mode: 0o600 });
appendEvent(ledger, { event: 'output', reader, sha256: sha, file: outputFile ? path.resolve(outputFile) : null, merged: path.resolve(mergedFile), ids: lines.map(l => l.id), review, ...(unavailable ? { unavailable } : {}), ...(raises ? {} : { fixable_raises: false }),
...(tree && !unavailable ? { tree: tree.expected, reported_tree: tree.reported, tree_ok: tree.ok } : {}) });
return { sha, lines, review };
...(tree && !unavailable ? { tree: tree.expected, reported_tree: tree.reported, tree_ok: tree.ok } : {}), ...('verdict' in reply && reply.verdict ? { verdict: reply.verdict } : {}) });
return { sha, lines, review, ...('verdict' in reply && reply.verdict ? { verdict: reply.verdict } : {}) };
}
/** The outside voice may fail without blocking (ship/sections/adversarial.md): its failure is recorded as unavailable coverage, never as a review. */
export const mayBeUnavailable = (reader: string) => reader.split('/')[0] === 'outside-voice';
@@ -401,7 +438,7 @@ export function judge(merged: MergedLine[], judged: JudgedLine[]): Judgment {
const from = rank(m.label) ?? 0;
const to = rank(target.severity);
if (to === null) continue;
if (to < from) (!j.duplicate_of && j.reason?.trim() && (summaryCandidate(m.raw) || statedLowering(m.reader, m.raw, to)) ? res.dismissed : res.downgraded).push(m.id);
if (to < from) (!j.duplicate_of && j.reason?.trim() && (m.summary_candidate === true || summaryCandidate(m.raw) || statedLowering(m.reader, m.raw, to)) ? res.dismissed : res.downgraded).push(m.id);
else if (j.duplicate_of) res.folded.push(m.id);
else if (to > from) { res.promoted.push(m.id); if (!j.reason?.trim()) res.unexplained.push(m.id); }
}
@@ -472,6 +509,7 @@ export function ledgerBlockers(ledger: string): LedgerBlocker[] {
const raws = new Map<string, string>();
const readers = new Map<string, string>();
const loci = new Map<string, string | null>();
const candidates = new Set<string>();
const gone = quarantinedIds(events);
for (const o of events.filter(e => e.event === 'output' && idsOf(e).some(id => !gone.has(id)))) {
const lines = jsonLines(read(o.merged, `merged file of ${String(o.reader)}`), String(o.merged)) as MergedLine[];
@@ -480,6 +518,7 @@ export function ledgerBlockers(ledger: string): LedgerBlocker[] {
ranks.set(id, rank(line.label) ?? 0);
if (statesNoBlocks(line.raw)) unblocking.set(id, rank(line.label) ?? 0);
raws.set(id, line.raw);
if (line.summary_candidate === true) candidates.add(id);
readers.set(id, String(o.reader));
loci.set(id, lineLocus(line.raw));
}
@@ -490,7 +529,7 @@ export function ledgerBlockers(ledger: string): LedgerBlocker[] {
if (promptHash(text) !== verdict.judged_sha256) fail(`judged set ${String(verdict.judged)} changed since judge`);
for (const j of jsonLines(text, String(verdict.judged)) as JudgedLine[]) {
if (j.duplicate_of) { ranks.delete(j.id); continue; }
const dismissed = !!j.reason?.trim() && (summaryCandidate(raws.get(j.id) ?? '') || statedLowering(readers.get(j.id) ?? '', raws.get(j.id) ?? '', rank(j.severity) ?? 0));
const dismissed = !!j.reason?.trim() && (candidates.has(j.id) || summaryCandidate(raws.get(j.id) ?? '') || statedLowering(readers.get(j.id) ?? '', raws.get(j.id) ?? '', rank(j.severity) ?? 0));
ranks.set(j.id, dismissed ? rank(j.severity) ?? 0 : Math.max(ranks.get(j.id) ?? 0, rank(j.severity) ?? 0));
if (!loci.has(j.id)) loci.set(j.id, lineLocus(j.reason ?? ''));
}
@@ -521,12 +560,13 @@ const collectedIds = (events: LedgerEvent[]) => {
return events.filter(e => e.event === 'output').flatMap(idsOf).filter(id => !gone.has(id));
};
export interface VerifyRow { kind: 'BRIEF' | 'OUTPUT' | 'JUDGE' | 'FETCH'; reader: string; status: string; detail: string }
export interface VerifyRow { kind: 'BRIEF' | 'OUTPUT' | 'JUDGE' | 'FETCH' | 'VERDICT'; reader: string; status: string; detail: string }
/** Replay every check from a finished run's ledger: briefs dispatched as rendered, outputs merged as returned, judgment without loss. */
export function verify(ledger: string): { rows: VerifyRow[]; clean: boolean; briefs: number; voided: number; unavailable: number } {
export function verify(ledger: string): { rows: VerifyRow[]; clean: boolean; briefs: number; voided: number; unavailable: number; verdicts: { asked: number; agree: number } } {
const events = readLedger(ledger);
const rows: VerifyRow[] = [];
const verdicts: VerifyRow[] = [];
const short = (s: unknown) => (typeof s === 'string' ? s.slice(0, 12) : '-');
// Each render opens one dispatch of its reader (a pass); its sends and outputs run until that reader's next render.
const renders = events.flatMap((e, i) => (e.event === 'rendered' ? [i] : []));
@@ -594,8 +634,13 @@ export function verify(ledger: string): { rows: VerifyRow[]; clean: boolean; bri
notes.push(typeof o.file === 'string' ? 'output_file_missing' : 'output_file_unrecorded');
differs.push(...ids);
} else if (promptHash(text) === o.sha256) {
const parsed = parseFindings(reader, text, o.fixable_raises !== false);
const off = ids.filter((id, k) => { const m = lines.find(l => l.id === id); return !m || m.reader !== reader || parsed[k]?.label !== m.label || parsed[k]?.raw !== m.raw; });
const asked = render?.verdict === 1;
const reply = parseReply(reader, text, o.fixable_raises !== false, asked);
const parsed = reply.findings;
const off = ids.filter((id, k) => { const m = lines.find(l => l.id === id); return !m || m.reader !== reader || parsed[k]?.label !== m.label || parsed[k]?.raw !== m.raw
|| !!(parsed[k]!.summary || summaryCandidate(m.raw)) !== (m.summary_candidate === true); });
if (reply.verdict) verdicts.push({ kind: 'VERDICT', reader: p?.label ?? reader, status: reply.verdict.state === 'valid' ? (reply.verdict.agrees ? 'AGREES' : 'DISAGREES') : reply.verdict.state.toUpperCase(),
detail: `declared=${reply.verdict.declared ?? reply.verdict.state} parsed=${reply.verdict.parsed}${reply.verdict.why ? ` why=${JSON.stringify(reply.verdict.why)}` : ''}` });
if (unavailableOutput(o) && parsed.length) { status = 'MISMATCH'; notes.push('unavailable_output_has_findings'); }
else if (!unavailableOutput(o)) {
if (off.length) { status = 'MISMATCH'; notes.push(`merged_line_differs=${JSON.stringify(off)}`); differs.push(...off); }
@@ -651,5 +696,7 @@ export function verify(ledger: string): { rows: VerifyRow[]; clean: boolean; bri
rows.push({ kind: 'FETCH', reader: `${String(s.pass)}/${String(s.stage)}#${String(s.attempt)}`, status: s.verdict === 'violation' ? (resolved ? 'RESOLVED' : 'VIOLATION') : String(s.verdict).toUpperCase(), detail: `found=${found.length}${found.length ? ` paths=${JSON.stringify(found.map(x => x.path))}` : ''}${s.partial ? ` partial=${JSON.stringify(s.partial)}` : ''}` });
}
const clean = episodes.length > 0 && rows.every(r => ['MATCH', 'OK', 'VOIDED', 'UNAVAILABLE', 'SUPERSEDED', 'CLEAN', 'PARTIAL', 'RESOLVED'].includes(r.status));
return { rows, clean, briefs: episodes.length, voided, unavailable: rows.filter(r => r.kind === 'BRIEF' && r.status === 'UNAVAILABLE').length };
rows.push(...verdicts);
return { rows, clean, briefs: episodes.length, voided, unavailable: rows.filter(r => r.kind === 'BRIEF' && r.status === 'UNAVAILABLE').length,
verdicts: { asked: verdicts.length, agree: verdicts.filter(r => r.status === 'AGREES').length } };
}
+2 -2
View File
@@ -253,12 +253,12 @@ export function renderStage(m: Manifest, stage: Stage, t: SectionTexts, diffFor:
Object.assign(slots, { checklist, brief, classes: (r.classes ?? []).join(','), diff_base: r.diff_base ?? m.diff_base });
targeted = { classes: parseClassList(slots.classes!), rules: TARGETED_RULES, brief_sha: targetedBriefSha(checklist), diff_base: slots.diff_base, ...targetedRead(process.cwd(), slots.diff_base!) };
}
const text = withWorkspace(renderBrief(role, slots, t), workdir, m.fetch_policy);
const text = withWorkspace(renderBrief(role, slots, t), workdir, m.fetch_policy, { verdict: true });
const file = path.join(briefs, briefFile(role, r.name));
fs.writeFileSync(file, text, { mode: 0o600 });
const sha = promptHash(text);
const bytes = Buffer.byteLength(text);
appendEvent(m.ledger, { event: 'rendered', reader: readerKey(role, r.name), role, name: r.name ?? null, file, sha256: sha, bytes, pass: m.pass, stage, workdir, tree, ...(findingsSha ? { findings_sha256: findingsSha } : {}), ...targeted, ...slice, ...(disclosureDigest(m.disclosure) ? { exclude: disclosureDigest(m.disclosure) } : {}), ...(m.fetch_policy ? { readers: m.fetch_policy.digest } : {}) });
appendEvent(m.ledger, { event: 'rendered', reader: readerKey(role, r.name), role, name: r.name ?? null, file, sha256: sha, bytes, pass: m.pass, stage, workdir, tree, verdict: 1, ...(findingsSha ? { findings_sha256: findingsSha } : {}), ...targeted, ...slice, ...(disclosureDigest(m.disclosure) ? { exclude: disclosureDigest(m.disclosure) } : {}), ...(m.fetch_policy ? { readers: m.fetch_policy.digest } : {}) });
Object.assign(r, { file, sha256: sha, bytes, chunks: chunkCount(text), state: 'rendered', tree, ...(findingsSha ? { findings_sha256: findingsSha } : {}) });
done.push(r);
}
+51
View File
@@ -0,0 +1,51 @@
/**
* review-verdict: the one machine-readable line every blocking reader's reply carries (GST-2495), right under its
* REVIEWED_TREE line: `REVIEW_VERDICT: blocking=<n>[ at=<path>:<line>[,<path>:<line>...]]`, n being how many findings
* in the reply block the merge. collect reads it only when the reader's render asked for it (the `verdict: 1` render
* event), so a reply to an older brief, or an old ledger, parses exactly as before. Lines inside fenced code blocks are
* ignored, byte-identical repeats count once, and differing repeats are malformed. docs/review-policy.md#verdict-line.
*/
export type Verdict = { state: 'valid'; blocking: number; at: string[] } | { state: 'missing' } | { state: 'malformed'; why: string };
const FENCE = /^\s*(`{3,}|~{3,})/;
const LOCUS = /^[\w./-]+:\d+$/;
const GRAMMAR = /^REVIEW_VERDICT:\s*blocking=(\d+)(?:\s+at=(\S+))?$/;
/** A line that tries to be the verdict line, wrappers removed (list marker, heading mark, whole-line bold or backticks), or null. */
export function verdictAttempt(line: string): string | null {
let t = line.trim().replace(/^(?:[-*>]\s+|\d+[.)]\s+|#+\s+)/, '');
for (const w of ['**', '`']) if (t.length > 2 * w.length && t.startsWith(w) && t.endsWith(w)) t = t.slice(w.length, -w.length).trim();
t = t.replace(/^\*\*REVIEW_VERDICT:\*\*/, 'REVIEW_VERDICT:');
return /^REVIEW_VERDICT\s*(?:[:=]|blocking\b)/i.test(t) ? t : null;
}
/** Each line of a reply, marked when it is a verdict attempt outside a code fence. */
function scan(output: string): Array<{ line: string; attempt: string | null }> {
let fence: string | null = null;
return output.replace(/\r\n/g, '\n').split('\n').map(line => {
const f = FENCE.exec(line)?.[1];
if (f && (fence === null || f[0] === fence[0])) { fence = fence === null ? f : null; return { line, attempt: null }; }
return { line, attempt: fence === null ? verdictAttempt(line) : null };
});
}
export function parseVerdict(output: string): Verdict {
const attempts = [...new Set(scan(output).flatMap(l => (l.attempt === null ? [] : [l.attempt])))];
if (!attempts.length) return { state: 'missing' };
if (attempts.length > 1) return { state: 'malformed', why: `${attempts.length} different REVIEW_VERDICT lines` };
const m = GRAMMAR.exec(attempts[0]!);
if (!m) return { state: 'malformed', why: `not REVIEW_VERDICT: blocking=<n>[ at=<path>:<line>,...]: ${JSON.stringify(attempts[0]!.slice(0, 120))}` };
const blocking = Number(m[1]);
if (blocking > 99) return { state: 'malformed', why: `blocking=${blocking} is over 99` };
const at = m[2] ? m[2].split(',') : [];
if (at.some(a => !LOCUS.test(a))) return { state: 'malformed', why: `at= holds something other than <path>:<line>: ${m[2]}` };
if (at.length > blocking) return { state: 'malformed', why: `at= names ${at.length} locations for blocking=${blocking}` };
return { state: 'valid', blocking, at };
}
/**
* The reply with every well-formed verdict line outside a code fence removed, so the line never joins or colours a neighbouring
* finding. A malformed attempt stays in the text, so a finding written on that line is still parsed and never lost.
*/
export const withoutVerdictLines = (output: string) => scan(output).filter(l => l.attempt === null || !GRAMMAR.test(l.attempt)).map(l => l.line).join('\n');
+43
View File
@@ -0,0 +1,43 @@
{
"source": "GST-2495 negative controls: handoffs/GSTA-521/probes.md lines; FOUN-298 (founderstar#926) native output with project names genericized; #3901 pass-2 excerpt; CAPY-1's qm #3567 fix-check paragraph and qm #3589 red-team sentence with identifiers reworded",
"probes": [
"I found nothing that blocks. There are three Low findings, all FIXABLE, and none can change outcomes, draws or save replay.",
"CRITICAL: the token leak in log.ts:3 ships the API key.",
"1. **The cache key drops the tree (key.ts:3).** FIXABLE: key on the tree hash.",
"I found nothing that blocks except the token leak, and it is FIXABLE. There are two Low findings.",
"Apart from the token leak, nothing that blocks; three Low findings, all FIXABLE.",
"Nothing that blocks, but the cache key drops the tree; three Low findings, all FIXABLE.",
"I found one blocker. There are three Low findings, all FIXABLE.",
"Nothing else blocks. Blocks: the cache key drops the tree, all FIXABLE.",
"I found nothing that blocks. One finding blocks the merge: three Low findings, all FIXABLE.",
"I found nothing that blocks. The cache key drops the reviewed tree, FIXABLE.",
"I found nothing that blocks. Three Low findings, all FIXABLE; the token leak in log.ts:3 is FIXABLE.",
"I found nothing that blocks. There are two Low findings, all FIXABLE. Blocks: yes",
"I found nothing that blocks. There are three Low findings, all FIXABLE, with one exception: the migration drops user rows.",
"I found nothing that blocks. However, the API key is logged in plain text. There are two Low findings, all FIXABLE.",
"Nothing blocks, though the API key is logged; two Low findings, all FIXABLE.",
"There are two Low findings, all FIXABLE, and none of them matter though the key is logged.",
"I found nothing that blocks. The migration drops user rows on retry. There are two Low findings, all FIXABLE.",
"The migration drops user rows, but nothing blocks; two Low findings, all FIXABLE.",
"I reviewed the migration, and apart from the dropped user rows\nnothing that blocks; two Low findings, all FIXABLE.",
"Nothing blocking an unauthenticated caller from the admin route; two Low findings, all FIXABLE.",
"There are two Low findings, all FIXABLE, and nothing blocking an unauthenticated caller from the admin route.",
"Probably nothing blocks; two Low findings, all FIXABLE.",
"I think nothing should block. There are two Low findings, all FIXABLE.",
"I found one blocker, and two Low findings, all FIXABLE, none of which would block.",
"I found nothing that blocks. There are two Low findings, all FIXABLE, and one blocking issue.",
"Nothing blocks, but there is one security issue; two Low findings, all FIXABLE.",
"Nothing blocks; one critical regression and two Low findings, all FIXABLE.",
"Nothing blocks; one data-loss bug and two Low findings, all FIXABLE.",
"There are two Low findings, all FIXABLE, and nothing stops an unauthenticated caller from reading every user token.",
"All nine fixes hold under the design you gave me. I didn't find any CRITICAL defects. These are the parts I checked against the code at the reviewed head. The tree is still clean.",
"The findings are all INFORMATIONAL and none CRITICAL."
],
"outputs": {
"foun298": "REVIEWED_TREE: bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb /workspace/app\n\nI found nothing that blocks. There are three Low findings, all FIXABLE, and none can change outcomes, draws or save replay.\n\n**Scope.** I read the full source diff against 0000000a (src/content.ts, game.ts, meeting.ts, the two docs) and checked the data with scripts: ideas.json typed lines, raise_meetings.json takes, flavor.json Demo Day takes, people.json, and data/overlays/was_idea_lines.json. The four test files (tests/core_test.ts, content_lint.ts, dana_fit_check.ts, was_overlay.ts) were reviewed in summary mode only, so coverage there is reduced on purpose:\n- core_test adds tests that the new lines move no outcome (on three sets of ideas, including pivots), that typed lines are no longer than stock, and that draws still come from the stock pool.\n- content_lint now loads was_idea_lines.\n- was_overlay.ts learns path lookup for overlays.\n\n**What I checked and found correct:**\n- `typed_line` makes the same `_fresh` call as before: same pool name, key, laps, and the same default `once=false`. The callback still gets laps 0, so `used` and the rng keys don't change.\n- Below `idea_lines`, the typed lines, meeting takes and Demo Day takes are all switched off (`rule_number` check, and `take_industry()` returns \"\").\n- was_idea_lines restores the default old wants correctly. For accountants, big_cos and cities, which had no `want` before, the overlay's restored value is exactly the default own `want`, so the old screen text comes back the same.\n- No stock pool (`calls.*`, `evidence.*`, Demo Day in/out) has duplicate lines. Every typed line and take is no longer than the line it replaces, and the slots and tokens match.\n- I ran the targeted tests: `ONLY=test_idea_lines_*,test_investor_voice*` core_test PASSED with 0 failures, and content_lint was all ok.\n\n**Findings**\n\n1. **A meeting take is checked against a different line than the one it replaces at runtime.** `_validate_takes` (content.ts ~1160) compares a take's slots with the voice's `lines.open`, using a regex that allows `{Caller}`-style uppercase tokens. At runtime, `Meeting._line` (meeting.ts:36-38) runs `voiced(stock open_<arch>, take)` with `Meeting.slots`, whose regex is lowercase only. So a take can pass lint and still be dropped. When that happens the code returns the stock line, which skips the investor's own voiced `open` as well, and the player sees a generic opener with no warning. No take has slots today, so this can't happen yet. FIXABLE: validate takes against each stock `open_*` line with `Meeting.slots`, and when `voiced` rejects a take, fall through to the voice's `open` instead of returning stock. Severity: Low. Blocks: no.\n\n2. **Which typed line shows depends on the stock line's text being unique.** Both `typed_line` (game.ts ~7101) and `_demo_line` (game.ts ~12476) find the drawn line's position with `stock.find(line)`. If someone later adds a duplicate line to a stock pool, the typed line at the second copy's position never shows, and the first copy's typed line shows in its place. Nothing checks for this, so it would fail silently. There are no duplicates today. FIXABLE: reject duplicate stock lines in `_validate_shadow`, or have `_fresh` return the index it drew. Severity: Low. Blocks: no.\n\n3. **Some new support has no content behind it.** `TYPED_POOLS` and five call sites send `sideways`, `polite`, `maybe`, `paid` and `callback` through `typed_line`, but no customer type in ideas.json has lines for those pools. Only `open` and `evidence.*` are filled in. The code is harmless and is validated, but nothing uses it yet. INVESTIGATE: either fill these pools in a content follow-up or drop them from `TYPED_POOLS` until lines exist. Severity: Low. Blocks: no.\n\nRecommendation: Ship as-is because the strongest finding is the take-validation mismatch at meeting.ts:36, which can't trigger with today's slot-free takes and at worst shows a stock opener, never a wrong outcome.",
"pr3901": "REVIEWED_TREE: 7be72149fbd6d1b8e713c7c42ef4cd9ba6373e20 /workspace/gstack\n\nI found no blocking defects in this range. There are three low-severity findings, all FIXABLE, and none blocks.\n\n### Findings\n\n1. **The remote-gate route hint ignores `--no-ci-carry`** (`lib/pregate/checks/lanes.ts`, the `route` condition). The hint prints whenever `remote_gate_runs` is declared. FIXABLE. Severity: Low. Blocks: no.\n\nRecommendation: Ship as-is.",
"capy1_fix_check": "REVIEWED_TREE: cccccccccccccccccccccccccccccccccccccccc /workspace/app\n{\"severity\":\"INFORMATIONAL\",\"confidence\":7,\"path\":\"src/sync.ts\",\"line\":41,\"category\":\"fix-check\",\"summary\":\"The retry comment still names the old helper.\"}\n\nAll nine fixes hold under the design you gave me. I didn't find any CRITICAL defects. These are the parts I checked against the code at the reviewed head. The tree is still clean.",
"capy1_red_team": "REVIEWED_TREE: cccccccccccccccccccccccccccccccccccccccc /workspace/app\n{\"severity\":\"INFORMATIONAL\",\"confidence\":6,\"path\":\"src/queue.ts\",\"line\":88,\"category\":\"red-team\",\"summary\":\"A slow consumer delays the next poll by one tick.\"}\n\nThe findings are all INFORMATIONAL and none CRITICAL.",
"native_relabel_loss": "REVIEWED_TREE: bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb /workspace/app\nREVIEW_VERDICT: blocking=1\n\nThe migration deletes the old rows before the copy, so a failed copy loses every note. FIXABLE.\n\nNo other CRITICAL issues.\n\nRecommendation: Fix the migration because a failed copy loses every note."
}
}
+1 -1
View File
@@ -956,7 +956,7 @@ describe('the workspace paragraph (GST-482)', () => {
const r = run(['render', '--role', role, '--out', out, ...(role === 'fix-check' ? [] : ['--diff-base', SHA]), ...args]);
expect(r.status, role + ': ' + r.stderr).toBe(0);
const main = renderBrief(role as Parameters<typeof renderBrief>[0], { diff_base: role === 'fix-check' ? undefined : SHA, ...slots }, t);
expect(fs.readFileSync(field(r.stdout, 'file'), 'utf8'), role).toBe(workspaceStep(top) + '\n\n' + main + '\n\n' + workspaceParagraph(top));
expect(fs.readFileSync(field(r.stdout, 'file'), 'utf8'), role).toBe(workspaceStep(top, true) + '\n\n' + main + '\n\n' + workspaceParagraph(top, undefined, true));
}
expect(workspaceParagraph(top)).toContain(`begin each command with \`cd '${top}' && \``);
expect(workspaceParagraph(top)).toContain(`absolute path under ${top}`);
+4 -1
View File
@@ -1398,7 +1398,10 @@ describe('readers on a shared machine review the checkout the pass ships (GST-48
expect(res).not.toContain('STOP:');
expect(res).toContain('NEXT: judge (Phase 4)');
expect(h.creates.length).toBeGreaterThan(2);
for (const c of h.creates) expect([c.prompt.startsWith(workspaceStep(wt) + '\n\n'), c.prompt.endsWith('\n\n' + workspaceParagraph(wt))], c.title).toEqual([true, true]);
for (const c of h.creates) {
const verdict = !c.title.includes(' audit/');
expect([c.prompt.startsWith(workspaceStep(wt, verdict) + '\n\n'), c.prompt.endsWith('\n\n' + workspaceParagraph(wt, undefined, verdict))], c.title).toEqual([true, true]);
}
expect(bin(wt, env, ['verify', '--ledger', path.join(out, 'review-ledger.jsonl')]).stdout).not.toContain('wrong_tree');
}, 240_000);
+212
View File
@@ -0,0 +1,212 @@
/**
* GST-2495: every blocking reader's reply carries `REVIEW_VERDICT: blocking=<n>[ at=<path>:<line>,...]`, and collect checks it
* against the findings it parses instead of inferring "nothing blocks" from prose (docs/review-policy.md#verdict-line). The
* controls are test/fixtures/review-ledger/verdict-controls.json: the GSTA-521 probe lines, FOUN-298's native reply
* (founderstar#926), #3901's verdict line and CAPY-1's fix-check (qm #3567) and red-team (qm #3589) summaries.
*/
import { afterEach, describe, expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { BRIEF_ROLES, promptHash, VERDICT_ASK, withWorkspace } from '../lib/review-brief';
import { appendEvent, collect, judge, parseFindings, parseReply, rank, readJsonl, recordJudgment, recordSent, statesNoBlocks, verify, type MergedLine } from '../lib/review-ledger';
import { parseVerdict, withoutVerdictLines } from '../lib/review-verdict';
const ROOT = path.resolve(import.meta.dir, '..');
const CONTROLS = JSON.parse(fs.readFileSync(path.join(ROOT, 'test/fixtures/review-ledger/verdict-controls.json'), 'utf8')) as { probes: string[]; outputs: Record<string, string> };
const TREE = `REVIEWED_TREE: ${'a'.repeat(40)} /workspace/app`;
const blocking = (fs: Array<{ label: string; raw: string }>) => fs.filter(f => (rank(f.label) ?? 0) >= 4 && !statesNoBlocks(f.raw)).length;
const reply = (body: string, line?: string) => [TREE, ...(line === undefined ? [] : [line]), '', body].join('\n');
const asked = (role: string, out: string, raises = true) => parseReply(role, out, raises, true);
describe('the REVIEW_VERDICT line', () => {
test('grammar: count, optional locations, wrappers; fenced lines ignored; identical repeats count once, differing ones are malformed', () => {
expect(parseVerdict('REVIEW_VERDICT: blocking=0')).toEqual({ state: 'valid', blocking: 0, at: [] });
expect(parseVerdict('x\n**REVIEW_VERDICT: blocking=2 at=lib/a.ts:3,b.md:9**')).toEqual({ state: 'valid', blocking: 2, at: ['lib/a.ts:3', 'b.md:9'] });
expect(parseVerdict('- `REVIEW_VERDICT: blocking=1`')).toEqual({ state: 'valid', blocking: 1, at: [] });
expect(parseVerdict('REVIEW_VERDICT: blocking=1\nREVIEW_VERDICT: blocking=1')).toEqual({ state: 'valid', blocking: 1, at: [] });
expect(parseVerdict('no line here').state).toBe('missing');
expect(parseVerdict('```\nREVIEW_VERDICT: blocking=0\n```').state).toBe('missing');
for (const bad of ['REVIEW_VERDICT: blocking=two', 'REVIEW_VERDICT: blocking=0 at=lib/a.ts:3', 'REVIEW_VERDICT: blocking=100', 'REVIEW_VERDICT: none',
'REVIEW_VERDICT: blocking=1 at=somewhere', 'REVIEW_VERDICT: blocking=0\nREVIEW_VERDICT: blocking=1', 'review_verdict: blocking=0 trailing words']) expect(parseVerdict(bad).state, bad).toBe('malformed');
expect(withoutVerdictLines('a\nREVIEW_VERDICT: blocking=0\n```\nREVIEW_VERDICT: blocking=9\n```')).toBe('a\n```\nREVIEW_VERDICT: blocking=9\n```');
});
test('the ask: every blocking role\'s brief carries it after the fingerprint step and in the closing paragraph; an audit reader\'s does not', () => {
const w = withWorkspace('BODY', '/work/repo', undefined, { verdict: true });
expect(w.split('\n\n')[0]).toContain(VERDICT_ASK);
expect(w.endsWith('gstack checks that count against your findings, so do not repeat it.')).toBe(true);
expect(withWorkspace('BODY', '/work/repo')).not.toContain('REVIEW_VERDICT');
expect(BRIEF_ROLES.length).toBe(7);
});
});
describe('negative controls on every blocking role, both review classes', () => {
// A valid line lowers nothing by itself (Captain ruling D1, 2026-11-14): no probe line, role or line state stops blocking.
for (const role of BRIEF_ROLES) for (const raises of [true, false]) {
test(`${role} ${raises ? 'gate' : 'relaxed'} class`, () => {
const lowered: number[] = [];
for (const [i, p] of CONTROLS.probes.entries()) {
const main = parseFindings(role, reply(p), raises);
for (const line of [undefined, 'REVIEW_VERDICT: blocking=none', 'REVIEW_VERDICT: blocking=0\nREVIEW_VERDICT: blocking=1']) {
expect(asked(role, reply(p, line), raises).findings.map(f => [f.label, f.raw]), `${i} ${line}`).toEqual(main.map(f => [f.label, f.raw]));
}
// A declared blocker the parser missed always blocks; unlocated ones share one row, so blocking_found counts them once.
for (const n of [1, 2]) expect(blocking(asked(role, reply(p, `REVIEW_VERDICT: blocking=${n}`), raises).findings), `${i} blocking=${n}`).toBeGreaterThanOrEqual(1);
const zero = asked(role, reply(p, 'REVIEW_VERDICT: blocking=0'), raises).findings;
if (blocking(main) && !blocking(zero)) lowered.push(i);
}
expect(lowered).toEqual([]);
});
}
test('FOUN-298: blocking=0 keeps CRITICAL,Low,Low,Low and makes the summary row a candidate the lead may lower; with no line nothing is flagged', () => {
const out = CONTROLS.outputs.foun298!;
const at = out.replace(/^(REVIEWED_TREE: .*)$/m, '$1\nREVIEW_VERDICT: blocking=0');
for (const raises of [true, false]) {
const r = asked('adversarial-native', at, raises);
expect(r.findings.map(f => [f.label, f.summary ?? false])).toEqual([['CRITICAL', true], ['Low', false], ['Low', false], ['Low', false]]);
expect(r.verdict).toEqual({ state: 'valid', parsed: 1, declared: 0, agrees: false });
expect(asked('adversarial-native', out, raises).findings.some(f => f.summary)).toBe(false);
expect(asked('adversarial-native', out, raises).findings.map(f => f.label)).toEqual(['CRITICAL', 'Low', 'Low', 'Low']);
expect(asked('adversarial-native', at.replace('blocking=0', 'blocking=1'), raises).verdict).toMatchObject({ declared: 1, parsed: 1, agrees: true });
}
});
test('#3901: without a line every role parses as on main', () => {
for (const role of BRIEF_ROLES) for (const raises of [true, false]) expect(asked(role, CONTROLS.outputs.pr3901!, raises).findings).toEqual(parseFindings(role, CONTROLS.outputs.pr3901!, raises));
});
test('CAPY-1 (qm #3567 fix-check, qm #3589 red-team): blocking=0 keeps the CRITICAL row but makes it a summary candidate; no line keeps it a plain CRITICAL', () => {
for (const key of ['capy1_fix_check', 'capy1_red_team']) for (const role of BRIEF_ROLES) {
const out = CONTROLS.outputs[key]!;
const zero = asked(role, out.replace(/^(REVIEWED_TREE: .*)$/m, '$1\nREVIEW_VERDICT: blocking=0'));
expect(zero.findings.map(f => [f.label, f.summary ?? false]), `${key} ${role}`).toEqual([['INFORMATIONAL', false], ['CRITICAL', true]]);
expect(zero.verdict).toMatchObject({ declared: 0, parsed: 1, agrees: false });
expect(asked(role, out).findings.map(f => [f.label, f.summary ?? false])).toEqual([['INFORMATIONAL', false], ['CRITICAL', false]]);
expect(asked(role, out.replace(/^(REVIEWED_TREE: .*)$/m, '$1\nREVIEW_VERDICT: blocking=1')).findings.map(f => [f.label, f.summary ?? false])).toEqual([['INFORMATIONAL', false], ['CRITICAL', false]]);
}
});
test('positive phrasings under blocking=0 still block, as summary candidates the lead may lower with a reason', () => {
const one = (p: string) => asked('adversarial-native', reply(p, 'REVIEW_VERDICT: blocking=0')).findings.map(f => [f.label, f.summary ?? false]);
expect(one('Nothing that blocks: one Medium and two Low findings, all FIXABLE.')).toEqual([['CRITICAL', true]]);
expect(one('Nothing that blocks: three High findings, all FIXABLE.')).toEqual([['CRITICAL', true]]);
expect(one('Nothing that blocks: two CRITICAL findings, all FIXABLE.')).toEqual([['CRITICAL', true]]);
});
test('a real unanchored FIXABLE blocker beside a denial line keeps blocking when the count agrees (Step 11 rerun at 1acb0d5)', () => {
const out = CONTROLS.outputs.native_relabel_loss!;
for (const raises of [true, false]) {
const r = asked('adversarial-native', out, raises);
expect(r.findings[0]!.label, `raises=${raises}`).toBe(raises ? 'CRITICAL' : parseFindings('adversarial-native', out, raises)[0]!.label);
expect(r.findings.map(f => [f.label, f.raw])).toEqual(parseFindings('adversarial-native', out, raises).map(f => [f.label, f.raw]));
}
const gate = asked('adversarial-native', out);
expect(blocking(gate.findings)).toBe(2);
expect(gate.verdict).toMatchObject({ declared: 1, parsed: 2, agrees: false });
expect(gate.findings.map(f => f.summary ?? false)).toEqual([true, true]);
});
});
describe('declared more blockers than parsed', () => {
test('one row per unmatched location and one for the rest, appended after the parsed rows', () => {
const out = reply('1. **Cache key (key.ts:3).** FIXABLE: key on the tree. Severity: Low. Blocks: no.', 'REVIEW_VERDICT: blocking=3 at=key.ts:3,lib/auth.ts:10');
const r = asked('adversarial-native', out);
expect(r.findings.map(f => [f.label, !!f.synthetic])).toEqual([['Low', false], ['CRITICAL', true], ['CRITICAL', true], ['CRITICAL', true]]);
expect(r.findings[1]!.raw.startsWith('key.ts:3: REVIEW_VERDICT declared blocking=3, but collect parsed 0')).toBe(true);
expect(r.findings[2]!.raw.startsWith('lib/auth.ts:10: ')).toBe(true);
expect(r.findings[3]!.raw).toContain('1 declared blocker with no location named');
expect(asked('adversarial-native', out.replace('blocking=3 at=key.ts:3,lib/auth.ts:10', 'blocking=1 at=key.ts:3')).findings.map(f => f.raw.split(':')[0])).toEqual(['1. **Cache key (key.ts', 'key.ts']);
});
});
describe('the line never colours its neighbour', () => {
test('in an asked render it is skipped; main\'s parser (no ask) still raises the shapes it always did', () => {
const next = 'I found no Critical findings and two Low findings.';
const out = `${TREE}\nREVIEW_VERDICT: blocking=0\n${next}`;
expect(asked('fix-check', out).findings).toEqual(parseFindings('fix-check', `${TREE}\n${next}`));
expect(parseFindings('fix-check', `${TREE}\n${next}`).map(f => f.label)).toEqual(['Low']);
const pinned = parseFindings('fix-check', out).map(f => f.label);
expect(pinned).toEqual(parseFindings('fix-check', out).map(f => f.label));
for (const role of BRIEF_ROLES) for (const p of CONTROLS.probes) {
const without = parseFindings(role, reply(p));
const withLine = parseFindings(role, reply(p, 'REVIEW_VERDICT: blocking=0'));
expect(blocking(withLine), `${role} ${p}`).toBeGreaterThanOrEqual(blocking(without));
}
});
});
describe('a finding line that names REVIEW_VERDICT is never stripped', () => {
test('prose about the token is no attempt, and a malformed attempt keeps its line for the parser', () => {
const finding = '1. REVIEW_VERDICT parsing in lib/a.ts:3 drops findings. CRITICAL. Blocks: yes';
const r = asked('specialist/testing', reply(finding, 'REVIEW_VERDICT: blocking=1 at=lib/a.ts:3'));
expect(r.findings.filter(f => f.label === 'CRITICAL').length).toBeGreaterThanOrEqual(1);
expect(r.verdict?.state).toBe('valid');
expect(r.verdict?.agrees).toBe(true);
const malformed = '- REVIEW_VERDICT: lines in lib/a.ts:3 are dropped. CRITICAL. Blocks: yes';
const m = asked('specialist/testing', reply(malformed));
expect(m.verdict?.state).toBe('malformed');
expect(blocking(m.findings)).toBe(blocking(parseFindings('specialist/testing', reply(malformed))));
expect(blocking(m.findings)).toBeGreaterThanOrEqual(1);
});
});
describe('a ledger round trip', () => {
const dirs: string[] = [];
afterEach(() => { while (dirs.length) fs.rmSync(dirs.pop()!, { recursive: true, force: true }); });
const run = (verdict: boolean, outputs: string[]) => {
const d = fs.mkdtempSync(path.join(os.tmpdir(), 'review-verdict-')); dirs.push(d);
const ledger = path.join(d, 'review-ledger.jsonl'), merged = path.join(d, 'merged.jsonl');
fs.writeFileSync(merged, '');
for (const [k, out] of outputs.entries()) {
const brief = path.join(d, `brief${k}.md`), text = `brief ${k}`;
fs.writeFileSync(brief, text);
appendEvent(ledger, { event: 'rendered', reader: 'adversarial-native', role: 'adversarial-native', name: null, file: brief, sha256: promptHash(text), bytes: text.length, ...(verdict ? { verdict: 1 } : {}) });
recordSent(ledger, 'adversarial-native', text);
const file = path.join(d, `out${k}.md`);
fs.writeFileSync(file, out);
collect(ledger, 'adversarial-native', out, merged, file);
}
const rows = readJsonl<MergedLine>(merged);
const judged = rows.map(m => ({ id: m.id, severity: m.label }));
fs.writeFileSync(path.join(d, 'judged.jsonl'), judged.map(j => JSON.stringify(j)).join('\n') + '\n');
recordJudgment(ledger, rows, judged, path.join(d, 'judged.jsonl'), merged);
return { ledger, merged, rows, v: verify(ledger) };
};
const declared = reply('1. **Cache key (key.ts:3).** FIXABLE: key on the tree. Severity: Low. Blocks: no.', 'REVIEW_VERDICT: blocking=1');
test('re-rendered twice: ids keep the per-output prefix, synthetic rows survive verify, and verify counts the verdicts', () => {
const { rows, v } = run(true, [declared, declared]);
expect(rows.map(r => r.id)).toEqual(['adversarial-native#1', 'adversarial-native#v1', 'adversarial-native#2.1', 'adversarial-native#2.v1']);
expect(v.rows.filter(r => r.kind !== 'VERDICT').map(r => r.status).every(s => s === 'MATCH' || s === 'OK')).toBe(true);
expect(v.clean).toBe(true);
expect(v.verdicts).toEqual({ asked: 2, agree: 0 });
});
test('an edited summary flag or verdict makes verify dirty; a render that never asked ignores the line', () => {
const capy = CONTROLS.outputs.capy1_fix_check!.replace(/^(REVIEWED_TREE: .*)$/m, '$1\nREVIEW_VERDICT: blocking=0');
const a = run(true, [capy]);
expect(a.rows.map(r => r.summary_candidate ?? false)).toEqual([false, true]);
fs.writeFileSync(a.merged, a.rows.map(r => JSON.stringify({ ...r, summary_candidate: undefined })).join('\n') + '\n');
expect(verify(a.ledger).clean).toBe(false);
const b = run(false, [declared]);
expect(b.rows.map(r => r.id)).toEqual(['adversarial-native#1']);
expect(b.v.verdicts).toEqual({ asked: 0, agree: 0 });
});
test('a summary flag edited onto a real CRITICAL makes verify dirty even when the render never asked', () => {
const c = run(false, [reply('1. CRITICAL: token leak in auth.ts:3. Blocks: yes')]);
expect(c.v.clean).toBe(true);
expect(c.rows.map(r => [r.label, r.summary_candidate ?? false])).toEqual([['CRITICAL', false]]);
fs.writeFileSync(c.merged, c.rows.map(r => JSON.stringify({ ...r, summary_candidate: true })).join('\n') + '\n');
expect(verify(c.ledger).clean).toBe(false);
});
test('lead judgment lowers a verdict summary candidate only with a reason', () => {
const rows = run(true, [CONTROLS.outputs.capy1_red_team!.replace(/^(REVIEWED_TREE: .*)$/m, '$1\nREVIEW_VERDICT: blocking=0')]).rows;
const lower = (reason?: string) => judge(rows, rows.map(m => ({ id: m.id, severity: m.summary_candidate ? 'INFORMATIONAL' : m.label, ...(reason ? { reason } : {}) })));
expect(lower('the reader\'s own verdict line says nothing blocks').dismissed).toEqual([rows[1]!.id]);
expect(lower().downgraded).toEqual([rows[1]!.id]);
});
});