feat(ship): verbatim reviewer briefs and loss-free collection (gstack-review-dispatch render/sent/collect/judge/verify)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed 2026-10-30 18:13:59 +00:00
1 parent 8160c4517b
commit ac3bd7b671
8 files changed
+640 -26

No files matched your search

+130
View File
@@ -0,0 +1,130 @@
#!/usr/bin/env -S bun --no-env-file --config=/dev/null
/**
* gstack-review-dispatch — /ship's reviewers get their briefs byte for byte and the parent changes
* nothing they return (GSTA-167 no-loss gate; docs/review-policy.md#verbatim-dispatch).
*
* gstack-review-dispatch render --role <role> [--name <n>] --out <dir> [slots] write the brief, print its sha256
* gstack-review-dispatch sent --ledger <f> --role <role> [--name <n>] --sent <file> the prompt actually passed; mismatch fails
* gstack-review-dispatch collect --ledger <f> --role <role> [--name <n>] --output <file> --merged <file>
* gstack-review-dispatch judge --ledger <f> --merged <file> --judged <file> no drop, no downgrade
* gstack-review-dispatch verify --ledger <f> replay every hash and the judgment
*
* Logic: lib/review-brief.ts (rendering from the sections) and lib/review-ledger.ts (ledger, collect, judge, verify).
*/
import { spawnSync } from 'node:child_process';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { EXIT, exitTableHelp } from '../lib/headless-artifacts';
import { BRIEF_ROLES, BriefError, briefFile, promptHash, renderBrief, sectionTexts, type BriefRole, type Slots } from '../lib/review-brief';
import { collect, readJsonl, readerKey, recordJudgment, recordSent, appendEvent, verify, type JudgedLine, type MergedLine } from '../lib/review-ledger';
const HELP = `gstack-review-dispatch — verbatim reviewer briefs and loss-free collection for /ship
render --role <role> [--name <n>] --out <dir> [--ledger <f>] [slots]
roles: ${BRIEF_ROLES.join(', ')}
slots: --diff-base <sha> (every role but fix-check) --exclude '<DIFF_CMD tail>' --stack <s> --test-fw <s>
--learnings-file <f> --checklist <path> (default <root>/review/specialists/<name>.md; red-team.md for red-team)
--checklists <a,b> (specialist-batched) --findings-file <f> --specialists <n> (red-team)
--fix-list <f> --hunks <f> --tests <f> [--callers <f>] (fix-check)
writes <out>/<role>[-<name>].prompt.md, records it in the ledger (default <out>/review-ledger.jsonl) and prints
BRIEF: reader=<role/name> sha256=<h> bytes=<n> file=<path>. Pass that file's content as the reader's prompt, unchanged.
sent --ledger <f> --role <role> [--name <n>] --sent <file>
the exact prompt a dispatch passed: BRIEF_SENT: … match, or BRIEF_MISMATCH: … (exit 1; the pass is not reviewed)
collect --ledger <f> --role <role> [--name <n>] --output <file> --merged <merged.jsonl>
the reader's raw output: hashes it and appends each finding to merged.jsonl with its label as written
judge --ledger <f> --merged <merged.jsonl> --judged <judged.jsonl>
judged lines {"id","severity","reason"?,"duplicate_of"?}: every merged id kept at its severity or above (a
duplicate folds into one at least as severe); added and promoted findings need a reason. JUDGMENT: ok …, or
JUDGMENT_REJECTED: dropped=… downgraded=… (exit 1)
verify --ledger <f>
BRIEF, OUTPUT and JUDGE rows from the ledger alone, then VERIFY: clean (exit 0), VERIFY: dirty (exit 1) or
VERIFY: no brief records (exit 3: this run's ledger kept no prompts)
${exitTableHelp()}`;
const args = process.argv.slice(2);
const flag = (n: string) => { const i = args.indexOf(n); return i >= 0 ? args[i + 1] : undefined; };
const readFile = (f: string | undefined, what: string) => {
if (!f) return undefined;
try { return fs.readFileSync(f, 'utf8'); } catch { throw new BriefError(`cannot read ${what} file ${f}`); }
};
const ROOT = path.resolve(import.meta.dir, '..');
function render(): number {
const role = flag('--role') as BriefRole | undefined;
const out = flag('--out');
if (!role || !BRIEF_ROLES.includes(role) || !out) { process.stderr.write(`gstack-review-dispatch render: needs --role <${BRIEF_ROLES.join('|')}> and --out <dir>\n`); return EXIT.usage; }
const name = flag('--name');
const diffBase = flag('--diff-base');
const checklist = flag('--checklist') ?? (role === 'red-team' ? path.join(ROOT, 'review/specialists/red-team.md') : role === 'specialist' && name ? path.join(ROOT, 'review/specialists', `${name}.md`) : undefined);
if (checklist && (role === 'specialist' || role === 'red-team') && !fs.existsSync(checklist)) throw new BriefError(`checklist not found: ${checklist}`);
let diff: string | undefined;
if (role === 'outside-voice' && diffBase) {
const r = spawnSync('git', ['-c', 'core.quotePath=false', 'diff', '--no-color', '--no-ext-diff', diffBase], { encoding: 'utf8', timeout: 60_000, maxBuffer: 256 * 1024 * 1024 });
if (r.status !== 0) throw new BriefError(`git diff ${diffBase} failed: ${(r.stderr ?? '').trim().split('\n')[0]}`);
diff = r.stdout;
}
const slots: Slots = {
name, diff_base: diffBase, exclude: flag('--exclude'), stack: flag('--stack'), test_fw: flag('--test-fw'),
learnings: readFile(flag('--learnings-file'), 'learnings'), checklist, checklists: flag('--checklists'),
findings: readFile(flag('--findings-file'), 'findings'), specialists: flag('--specialists'), diff,
fix_list: readFile(flag('--fix-list'), 'fix list'), hunks: readFile(flag('--hunks'), 'hunks'), tests: readFile(flag('--tests'), 'tests'), callers: readFile(flag('--callers'), 'callers'),
};
const text = renderBrief(role, slots, sectionTexts(ROOT));
fs.mkdirSync(out, { recursive: true, mode: 0o700 });
const file = path.join(path.resolve(out), briefFile(role, name));
fs.writeFileSync(file, text, { mode: 0o600 });
const ledger = flag('--ledger') ?? path.join(out, 'review-ledger.jsonl');
const sha = promptHash(text);
appendEvent(ledger, { event: 'rendered', reader: readerKey(role, name), role, name: name ?? null, file, sha256: sha, bytes: Buffer.byteLength(text) });
process.stdout.write(`BRIEF: reader=${readerKey(role, name)} sha256=${sha} bytes=${Buffer.byteLength(text)} file=${file}\n`);
return EXIT.ok;
}
function main(): number {
const cmd = args[0];
if (!cmd || cmd === '--help' || cmd === '-h') { process.stdout.write(HELP + '\n'); return cmd ? EXIT.ok : EXIT.usage; }
if (cmd === 'render') return render();
const ledger = flag('--ledger');
if (!ledger) { process.stderr.write(`gstack-review-dispatch ${cmd}: --ledger <file> is required\n`); return EXIT.usage; }
const reader = flag('--role') ? readerKey(flag('--role')!, flag('--name')) : '';
if (cmd === 'sent') {
const sent = readFile(flag('--sent'), 'sent prompt');
if (!reader || sent === undefined) { process.stderr.write('gstack-review-dispatch sent: needs --role and --sent <file>\n'); return EXIT.usage; }
const r = recordSent(ledger, reader, sent);
process.stdout.write(r.ok ? `BRIEF_SENT: reader=${reader} sha256=${r.sent} match\n`
: `BRIEF_MISMATCH: reader=${reader} rendered=${r.rendered ?? 'none'} sent=${r.sent} (dispatch the rendered file unchanged; this reader's review does not count)\n`);
return r.ok ? EXIT.ok : EXIT.fail;
}
if (cmd === 'collect') {
const outFile = flag('--output');
const output = readFile(outFile, 'reader output');
const merged = flag('--merged');
if (!reader || output === undefined || !merged) { process.stderr.write('gstack-review-dispatch collect: needs --role, --output <file> and --merged <file>\n'); return EXIT.usage; }
const r = collect(ledger, reader, output, merged, path.resolve(outFile!));
process.stdout.write(`COLLECT: reader=${reader} findings=${r.lines.length} sha256=${r.sha} merged=${merged}\n`);
return EXIT.ok;
}
if (cmd === 'judge') {
const merged = flag('--merged');
const judged = flag('--judged');
if (!merged || !judged) { process.stderr.write('gstack-review-dispatch judge: needs --merged <file> and --judged <file>\n'); return EXIT.usage; }
const r = recordJudgment(ledger, readJsonl<MergedLine>(merged), readJsonl<JudgedLine>(judged), judged);
process.stdout.write(r.ok ? `JUDGMENT: ok added=${r.added.length} promoted=${r.promoted.length} folded=${r.folded.length}\n`
: `JUDGMENT_REJECTED: dropped=${JSON.stringify(r.dropped)} downgraded=${JSON.stringify(r.downgraded)} unexplained=${JSON.stringify(r.unexplained)} invalid=${JSON.stringify(r.invalid)} (lead judgment may add or promote a finding, never drop or downgrade one)\n`);
return r.ok ? EXIT.ok : EXIT.fail;
}
if (cmd === 'verify') {
const v = verify(ledger);
for (const r of v.rows) process.stdout.write(`${r.kind} ${r.reader} ${r.status} ${r.detail}\n`);
if (!v.briefs) { process.stdout.write('VERIFY: no brief records (this run\'s ledger kept no prompts; dispatched-vs-rendered cannot be checked)\n'); return EXIT.refused; }
process.stdout.write(`VERIFY: ${v.clean ? 'clean' : 'dirty'} briefs=${v.briefs} outputs=${v.rows.filter(r => r.kind === 'OUTPUT').length}\n`);
return v.clean ? EXIT.ok : EXIT.fail;
}
process.stderr.write(`gstack-review-dispatch: unknown command ${cmd}\n${HELP}\n`);
return EXIT.usage;
}
try { process.exit(main()); } catch (e) {
if (e instanceof BriefError) { process.stderr.write(`BRIEF_RENDER_FAILED: ${e.message}\n`); process.exit(EXIT.fail); }
throw e;
}
+31
View File
@@ -357,8 +357,39 @@ email case-folded, so running it twice changes nothing. The `COAUTHORS: <n> in
<range>` line on stderr is the receipt; `--json` prints `{ range, authors,
trailers, appended }`.
## Verbatim dispatch
A driven `/ship` (ship/DRIVER.md) never lets the model write a reviewer's brief
or merge a reviewer's findings. The no-loss replays showed why: with briefs the
parent filled in by hand, only 37 of 112 dispatched prompts matched the rendered
ones, and the arm lost blockers. `bin/gstack-review-dispatch` closes both ends:
1. `render --role <role> [--name <n>] --out <dir>` cuts the brief out of
`ship/sections/review-army.md` or `ship/sections/adversarial.md` (the fix-check
from `scripts/resolvers/fix-check.ts`) and fills only declared slots: the
literal diff base, the plan's `DIFF_CMD` excludes, stack, test framework,
learnings, checklist paths, the Red Team's merged findings and specialist count,
and the outside voice's branch diff. A section that lost an anchor or placeholder
refuses (`BRIEF_RENDER_FAILED`) instead of rendering something else.
2. The dispatch program passes that file's content unchanged as the reader's
prompt and runs `sent` on what it passed: a hash that differs from the rendered
one is `BRIEF_MISMATCH`, and that reader's review does not count.
3. `collect` hashes each reader's raw output and appends its findings to
`merged.jsonl` with the label exactly as written (a JSON line's `severity`, or a
prose line's Critical, High, Medium, Low, P0-P3, FIXABLE or INVESTIGATE).
4. Lead judgment writes `judged.jsonl`; `judge` refuses a set that drops a merged
finding, lowers its severity, folds it into a less severe duplicate, or adds or
promotes one without a reason (`JUDGMENT_REJECTED`). Each addition and promotion
is logged with its reason.
5. `verify --ledger <f>` replays all of it from the run's ledger: rendered against
dispatched hashes, output hashes against the merged lines they produced, and the
judgment verdict. A ledger from a run that kept no prompts prints
`VERIFY: no brief records` (exit 3), never clean.
## Where this is enforced
- `lib/review-brief.ts`, `lib/review-ledger.ts`, `bin/gstack-review-dispatch`: [verbatim dispatch](#verbatim-dispatch) (`test/review-dispatch.test.ts`).
- `scripts/resolvers/review-army.ts`: the floor and `NO FINDINGS` in every specialist and Red Team prompt (rendered into `review/sections/review-army.md` and `ship/sections/review-army.md`), and the lead judgment's "On record" line that turns a [ruled class](#rulings-below-the-floor) into a note; in `/ship` every specialist and Red Team prompt it renders carries its diff base as a literal sha the parent writes in (`DIFF_BASE=<diff-base>`: the merge base on pass 1, `RANGE_BASE` from pass 2 on) beside its plan's `DIFF_CMD` excludes, because a reviewer is its own process and never sees the parent's shell variables; the fan-out batches above.
- `ship/sections/review-army.md` Step 9: the plan's `REVIEW_DEPTH`, `GATE_SCAN` and `--depth-floor` on every run, the depth fields of the review-log row, passes, `RANGE_BASE`, classify, defer, render, the cap check before any edit, the Step 11 late-defer row, scratch files under `.gstack/tmp/`, the cap decisions; `ship/sections/adversarial.md` Step 11: the outside voice reads the same range and scopes queued fixes the same way.
- `lib/review-depth.ts`, `bin/gstack-review-plan`: the diff class, the depth (`depthFor`, `DEPTH_RULES`, `depthOfCommitInput`), each dispatch's `exclude` pathspecs and `DIFF_CMD`, the transition table, the numeric-claim scan and each role's effort tier (`test/review-depth.test.ts`, `test/review-plan-cli.test.ts`).
+130
View File
@@ -0,0 +1,130 @@
/**
* review-brief — every /ship reviewer brief rendered from its section's own bytes (GSTA-167
* no-loss gate). The text comes from ship/sections/review-army.md and ship/sections/adversarial.md
* at render time, cut out between fixed anchors; only the declared slots below are filled. The
* dispatch program passes the rendered file verbatim as the reader's prompt, and
* lib/review-ledger.ts proves it did by hash. Nothing here is composed by the model.
*
* Slots by role:
* specialist name, checklist, diff_base, exclude, stack, test_fw, learnings (security adds the defensive framing)
* specialist-batched checklists (comma list), diff_base, exclude, stack, test_fw, learnings
* red-team checklist, diff_base, findings (the merged specialist lines), specialists (their count)
* adversarial-native diff_base
* outside-voice diff_base, diff (the branch diff, which the bin reads from git)
* fix-check fix_list, hunks, tests, callers (scripts/resolvers/fix-check.ts)
*/
import { createHash } from 'node:crypto';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { renderFixCheckPrompt } from '../scripts/resolvers/fix-check';
export const BRIEF_ROLES = ['specialist', 'specialist-batched', 'red-team', 'adversarial-native', 'outside-voice', 'fix-check'] as const;
export type BriefRole = typeof BRIEF_ROLES[number];
export type Slots = Record<string, string | undefined>;
/** The hash every brief and every dispatched prompt is compared by: sha256 of the text without trailing whitespace. */
export function promptHash(text: string): string {
return createHash('sha256').update(text.replace(/\r\n/g, '\n').replace(/\s+$/, '')).digest('hex');
}
export class BriefError extends Error {}
/** The text strictly between start and end, end searched after start; both anchors are required. */
function between(text: string, start: string, end: string, what: string): string {
const i = text.indexOf(start);
if (i < 0) throw new BriefError(`${what}: start anchor not found in the section (${JSON.stringify(start.slice(0, 40))})`);
const j = text.indexOf(end, i + start.length);
if (j < 0) throw new BriefError(`${what}: end anchor not found in the section (${JSON.stringify(end.slice(0, 40))})`);
return text.slice(i + start.length, j);
}
/** A quoted prompt body as the reader receives it: the section escapes its inner quotes and line breaks. */
const unquote = (s: string) => s.replace(/\\"/g, '"').replace(/\\n/g, '\n');
export interface SectionTexts { specialist: string; framing: string; fixtures: string; redTeam: string; native: string; outside: string }
/** Cut every brief body out of the installed sections under root (the directory holding ship/). */
export function sectionTexts(root: string): SectionTexts {
const read = (rel: string) => {
try { return fs.readFileSync(path.join(root, rel), 'utf8').replace(/\r\n/g, '\n'); } catch { throw new BriefError(`cannot read ${rel} under ${root}`); }
};
const army = read('ship/sections/review-army.md');
const adv = read('ship/sections/adversarial.md');
const specialist = unquote(between(army, '\n"You are a specialist code reviewer.', 'Past learnings: {learnings or \'none\'}"', 'specialist brief'));
return {
specialist: `You are a specialist code reviewer.${specialist}Past learnings: {learnings or 'none'}`
.replace(/ \(`<diff-base>`: write the sha in before dispatch[\s\S]*?on `DIFF_EXCLUDE: none`\)/, ''),
framing: unquote(between(army, 'Begin the prompt with:\n"', '"\nand replace its full-diff command', 'defensive framing')),
fixtures: unquote(between(army, 'and replace its full-diff command with: "', '"\nEvery other specialist', 'fixture handling')),
redTeam: unquote(between(army, 'fixture handling from item 4 of the specialist dispatch: "', 'don\'t cover."', 'red-team brief')) + 'don\'t cover.',
native: unquote(between(adv, 'Subagent prompt:\n"', '"\n\nPresent findings', 'adversarial-native brief')),
outside: unquote(between(adv, 'Outside prompt (supply repository context from the parent):\n\n"', '"\n\nWrite the **complete prompt and context**', 'outside-voice brief')),
};
}
const SHA = /^[0-9a-f]{7,40}$/;
const FENCE = '`'.repeat(3);
function need(slots: Slots, key: string): string {
const v = slots[key];
if (v === undefined || v === '') throw new BriefError(`missing slot ${key}`);
return v;
}
function fill(text: string, pairs: Array<[string, string]>, what: string): string {
let out = text;
for (const [from, to] of pairs) {
if (!out.includes(from)) throw new BriefError(`${what}: placeholder ${from} is not in the section text (the section changed; update lib/review-brief.ts)`);
out = out.split(from).join(to);
}
return out;
}
const SPECIALIST_DIFF = 'run\n`DIFF_BASE=<diff-base> && git diff "$DIFF_BASE" <exclude>` to get the diff';
const PLACEHOLDERS = ['<diff-base>', '<exclude>', '<base>', '{checklist path}', '{STACK}', '{TEST_FW}', '{merged findings summary}', '{red-team checklist path}', '{learnings'];
/** Render one reader's brief. Throws BriefError on a missing slot, a section that no longer carries an anchor or placeholder, or a placeholder left unfilled. */
export function renderBrief(role: BriefRole, slots: Slots, t: SectionTexts): string {
const text = renderRole(role, slots, t);
const left = role === 'fix-check' || role === 'outside-voice' ? [] : PLACEHOLDERS.filter(p => text.includes(p));
if (left.length) throw new BriefError(`${role}: unfilled placeholder ${left.join(', ')}`);
return text;
}
function renderRole(role: BriefRole, slots: Slots, t: SectionTexts): string {
const diffBase = role === 'fix-check' ? '' : need(slots, 'diff_base');
if (diffBase && !SHA.test(diffBase)) throw new BriefError(`slot diff_base must be a literal commit sha, got ${JSON.stringify(diffBase)}`);
const common = (body: string, checklist: string) => fill(body, [
...(checklist ? [['{checklist path}', checklist] as [string, string]] : []), ['<diff-base>', diffBase],
['{TEST_FW}', slots.test_fw || 'unknown'], ['{STACK}', slots.stack || 'unknown'], ['{learnings or \'none\'}', slots.learnings?.trim() || 'none'],
], role).split(' <exclude>`').join(slots.exclude ? ' ' + slots.exclude + '`' : '`');
switch (role) {
case 'specialist': {
const name = need(slots, 'name');
const checklist = need(slots, 'checklist');
if (name !== 'security') return common(t.specialist, checklist);
const body = fill(t.specialist, [[SPECIALIST_DIFF, 'set `DIFF_BASE=<diff-base>` first. ' + t.fixtures + ' Then read the diff that way']], role);
return `${t.framing}\n\n${common(body, checklist)}`;
}
case 'specialist-batched': {
const lists = need(slots, 'checklists').split(',').map(s => s.trim()).filter(Boolean);
const body = fill(t.specialist, [['Read the checklist at {checklist path}', `Read each checklist at ${lists.join(', ')} (tag each finding's "specialist" with the name of the checklist it came from)`]], role);
return common(body, '');
}
case 'red-team': {
const body = fill(t.redTeam, [['{merged findings summary}', (slots.findings ?? '').trim() || '(none)'], ['by N specialists', `by ${need(slots, 'specialists')} specialists`],
['{red-team checklist path}', need(slots, 'checklist')], ['<diff-base>', diffBase]], role);
return `${t.framing}\n\n${t.fixtures}\n\n${body}`;
}
case 'adversarial-native':
return fill(t.native, [['DIFF_BASE=$(git merge-base origin/<base> HEAD)', `DIFF_BASE=${diffBase}`]], role);
case 'outside-voice': {
const body = fill(t.outside, [['DIFF_BASE=$(git merge-base origin/<base> HEAD)', `DIFF_BASE=${diffBase}`]], role);
return [body, '', 'Give each finding a severity: Critical, High, Medium or Low.', '', `## Branch diff (git diff ${diffBase})`, FENCE + 'diff', (slots.diff ?? '').trim(), FENCE].join('\n');
}
case 'fix-check':
return renderFixCheckPrompt({ fixList: need(slots, 'fix_list'), touchedHunks: need(slots, 'hunks'), testResults: need(slots, 'tests'), callers: slots.callers ?? '' });
}
}
/** The file a rendered brief is written to under the run's briefs directory. */
export function briefFile(role: BriefRole, name?: string): string {
return `${role}${name ? `-${name.replace(/[^a-zA-Z0-9._-]/g, '_')}` : ''}.prompt.md`;
}
+158
View File
@@ -0,0 +1,158 @@
/**
* review-ledger — the proof that /ship's reviewers got their briefs unchanged and that the
* parent changed nothing they returned (GSTA-167 no-loss gate). One JSONL ledger per run:
*
* rendered a brief gstack-review-dispatch render wrote (role, name, file, sha256)
* sent the prompt a dispatch program actually passed, by hash; a mismatch fails the pass
* output a reader's raw output (sha256) and the merged.jsonl ids collect wrote from it
* judgment each lead-judgment change: an added or promoted finding, with its reason
* judge the verdict on the parent's judged set (ok, or the dropped and downgraded ids)
*
* Findings reach merged.jsonl only through collect, with each voice's label exactly as written.
* The parent's judged set may add or promote a finding and may fold a duplicate into one at least
* as severe; it never drops or downgrades one. verify replays every check from the ledger alone.
*/
import * as fs from 'node:fs';
import { promptHash } from './review-brief';
export interface LedgerEvent { event: 'rendered' | 'sent' | 'output' | 'judgment' | 'judge'; [k: string]: unknown }
export const readerKey = (role: string, name?: string) => (name ? `${role}/${name}` : role);
export function appendEvent(ledger: string, e: LedgerEvent): void {
fs.appendFileSync(ledger, JSON.stringify({ ...e, at: new Date().toISOString() }) + '\n', { mode: 0o600 });
}
export function readLedger(ledger: string): LedgerEvent[] {
let text: string;
try { text = fs.readFileSync(ledger, 'utf8'); } catch { return []; }
return text.split('\n').filter(l => l.trim()).flatMap(l => { try { return [JSON.parse(l) as LedgerEvent]; } catch { return []; } });
}
const lastRendered = (events: LedgerEvent[], reader: string) => [...events].reverse().find(e => e.event === 'rendered' && e.reader === reader);
/** Compare a dispatched prompt with the brief rendered for that reader; records the sent event. */
export function recordSent(ledger: string, reader: string, sentText: string): { ok: boolean; rendered: string | null; sent: string } {
const r = lastRendered(readLedger(ledger), reader);
const sent = promptHash(sentText);
const rendered = typeof r?.sha256 === 'string' ? r.sha256 : null;
const ok = rendered === sent;
appendEvent(ledger, { event: 'sent', reader, sha256: sent, rendered, match: ok });
return { ok, rendered, sent };
}
/** Severity ranks across the voices' vocabularies; a label with no severity (FIXABLE, INVESTIGATE) ranks 0. */
const RANKS: Record<string, number> = { critical: 4, p0: 4, high: 3, p1: 3, medium: 2, p2: 2, informational: 2, low: 1, p3: 1, fixable: 0, investigate: 0 };
export function rank(label: string): number | null {
const r = RANKS[label.trim().replace(/^\[|\]$/g, '').toLowerCase()];
return r === undefined ? null : r;
}
const PROSE_LABEL = /(?:^|[\s[(*|:-])(CRITICAL|INFORMATIONAL|Critical|High|Medium|Low|P[0-3]|FIXABLE|INVESTIGATE)(?=$|[\s\]):*|,.-])/;
export interface MergedLine { id: string; reader: string; label: string; raw: string; output_sha256: string }
/** One reader's findings as written: a JSON object per line (severity kept as written), else each prose line carrying a severity or FIXABLE/INVESTIGATE label. */
export function parseFindings(reader: string, output: string): Array<{ label: string; raw: string }> {
const out: Array<{ label: string; raw: string }> = [];
for (const line of output.replace(/\r\n/g, '\n').split('\n')) {
const t = line.trim();
if (!t || /^NO FINDINGS\.?$/.test(t) || /^Recommendation:/i.test(t)) continue;
if (t.startsWith('{')) {
try {
const o = JSON.parse(t) as Record<string, unknown>;
if (typeof o.severity === 'string') { out.push({ label: o.severity, raw: t }); continue; }
} catch { /* prose that starts with a brace */ }
}
const m = PROSE_LABEL.exec(t);
if (m) out.push({ label: m[1]!, raw: t });
}
return out;
}
/** collect: hash the raw output, append its findings to merged.jsonl and record both. */
export function collect(ledger: string, reader: string, output: string, mergedFile: string, outputFile?: string): { sha: string; lines: MergedLine[] } {
const sha = promptHash(output);
const prior = readLedger(ledger).filter(e => e.event === 'output' && e.reader === reader).length;
const lines = parseFindings(reader, output).map((f, i) => ({ id: `${reader}#${prior ? `${prior + 1}.` : ''}${i + 1}`, reader, label: f.label, raw: f.raw, output_sha256: sha }));
fs.appendFileSync(mergedFile, lines.map(l => JSON.stringify(l) + '\n').join(''), { mode: 0o600 });
appendEvent(ledger, { event: 'output', reader, sha256: sha, file: outputFile ?? null, merged: mergedFile, ids: lines.map(l => l.id) });
return { sha, lines };
}
export function readJsonl<T>(file: string): T[] {
return fs.readFileSync(file, 'utf8').split('\n').filter(l => l.trim()).map(l => JSON.parse(l) as T);
}
export interface JudgedLine { id: string; severity: string; reason?: string; duplicate_of?: string }
export interface Judgment { ok: boolean; dropped: string[]; downgraded: string[]; unexplained: string[]; invalid: string[]; added: string[]; promoted: string[]; folded: string[] }
/** The lead-judgment rule: every merged finding survives at its severity or above; additions and promotions carry a reason. */
export function judge(merged: MergedLine[], judged: JudgedLine[]): Judgment {
const byId = new Map(judged.map(j => [j.id, j]));
const mergedIds = new Set(merged.map(m => m.id));
const res: Judgment = { ok: true, dropped: [], downgraded: [], unexplained: [], invalid: [], added: [], promoted: [], folded: [] };
for (const j of judged) if (rank(j.severity) === null) res.invalid.push(j.id);
for (const m of merged) {
const j = byId.get(m.id);
if (!j) { res.dropped.push(m.id); continue; }
const target = j.duplicate_of ? byId.get(j.duplicate_of) : j;
if (!target) { res.dropped.push(m.id); continue; }
const from = rank(m.label) ?? 0;
const to = rank(target.severity);
if (to === null) continue;
if (to < from) res.downgraded.push(m.id);
else if (j.duplicate_of) res.folded.push(m.id);
else if (to > from) { res.promoted.push(m.id); if (!j.reason?.trim()) res.unexplained.push(m.id); }
}
for (const j of judged) if (!mergedIds.has(j.id) && !j.duplicate_of) { res.added.push(j.id); if (!j.reason?.trim()) res.unexplained.push(j.id); }
res.ok = !res.dropped.length && !res.downgraded.length && !res.unexplained.length && !res.invalid.length;
return res;
}
/** judge, logged: one judgment event per addition and promotion, then the verdict. */
export function recordJudgment(ledger: string, merged: MergedLine[], judged: JudgedLine[], judgedFile: string): Judgment {
const res = judge(merged, judged);
const byId = new Map(judged.map(j => [j.id, j]));
const labels = new Map(merged.map(m => [m.id, m.label]));
if (res.ok) {
for (const id of res.added) appendEvent(ledger, { event: 'judgment', kind: 'added', id, to: byId.get(id)!.severity, reason: byId.get(id)!.reason });
for (const id of res.promoted) appendEvent(ledger, { event: 'judgment', kind: 'promoted', id, from: labels.get(id), to: byId.get(id)!.severity, reason: byId.get(id)!.reason });
}
appendEvent(ledger, { event: 'judge', ok: res.ok, judged: judgedFile, dropped: res.dropped, downgraded: res.downgraded, unexplained: res.unexplained, invalid: res.invalid, added: res.added.length, promoted: res.promoted.length, folded: res.folded.length });
return res;
}
export interface VerifyRow { kind: 'BRIEF' | 'OUTPUT' | 'JUDGE'; reader: string; status: string; detail: string }
/** Replay every check from a finished run's ledger: briefs dispatched as rendered, outputs merged as returned, judgment without loss. */
export function verify(ledger: string): { rows: VerifyRow[]; clean: boolean; briefs: number } {
const events = readLedger(ledger);
const rows: VerifyRow[] = [];
const short = (s: unknown) => (typeof s === 'string' ? s.slice(0, 12) : '-');
const readers = [...new Set(events.filter(e => e.event === 'rendered').map(e => String(e.reader)))];
for (const reader of readers) {
const r = lastRendered(events, reader)!;
const sent = events.filter(e => e.event === 'sent' && e.reader === reader);
let fileNow = '';
try { fileNow = typeof r.file === 'string' && fs.existsSync(r.file) ? promptHash(fs.readFileSync(r.file, 'utf8')) : ''; } catch { /* unreadable */ }
const status = !sent.length ? 'NOT_SENT' : sent.every(s => s.sha256 === r.sha256) ? 'MATCH' : 'MISMATCH';
const tampered = fileNow && fileNow !== r.sha256 ? ' file_changed_since_render' : '';
rows.push({ kind: 'BRIEF', reader, status: tampered ? 'MISMATCH' : status, detail: `rendered=${short(r.sha256)} sent=${sent.map(s => short(s.sha256)).join(',') || '-'}${tampered}` });
}
for (const o of events.filter(e => e.event === 'output')) {
const reader = String(o.reader);
let status = 'MATCH';
const notes: string[] = [];
if (typeof o.file === 'string' && fs.existsSync(o.file) && promptHash(fs.readFileSync(o.file, 'utf8')) !== o.sha256) { status = 'MISMATCH'; notes.push('output_file_changed'); }
const ids = Array.isArray(o.ids) ? (o.ids as string[]) : [];
let lines: MergedLine[] = [];
try { lines = typeof o.merged === 'string' ? readJsonl<MergedLine>(o.merged).filter(l => ids.includes(l.id)) : []; } catch { notes.push('merged_unreadable'); status = 'MISMATCH'; }
if (lines.length !== ids.length) { status = 'MISMATCH'; notes.push(`merged_lines=${lines.length}/${ids.length}`); }
if (lines.some(l => l.output_sha256 !== o.sha256)) { status = 'MISMATCH'; notes.push('merged_line_from_other_output'); }
rows.push({ kind: 'OUTPUT', reader, status, detail: `sha=${short(o.sha256)} findings=${ids.length}${notes.length ? ' ' + notes.join(' ') : ''}` });
}
const verdicts = events.filter(e => e.event === 'judge');
if (verdicts.length) {
const v = verdicts[verdicts.length - 1]!;
rows.push({ kind: 'JUDGE', reader: '-', status: v.ok ? 'OK' : 'REJECTED', detail: `dropped=${JSON.stringify(v.dropped)} downgraded=${JSON.stringify(v.downgraded)} added=${v.added} promoted=${v.promoted} folded=${v.folded}` });
} else if (events.some(e => e.event === 'output')) rows.push({ kind: 'JUDGE', reader: '-', status: 'MISSING', detail: 'outputs were collected but no judged set was checked' });
const clean = readers.length > 0 && rows.every(r => r.status === 'MATCH' || r.status === 'OK');
return { rows, clean, briefs: readers.length };
}
+14 -13
View File
@@ -7,7 +7,7 @@ Read this instead of `ship/SKILL.md` when a parent agent on any host runs /ship
the `GSTACK_LAUNCH` assignments your installer printed (`. <the GSTACK_LAUNCH_FILE it printed>`), never
`export $(...)`, `env`, `printenv` or a bare `set` in a tool fence.
**Rules.** Each phase below is one call: one program (an execute program on Capy, else a shell
**Rules.** Each phase below is one call: one program (Capy: an execute program; else a shell
script) that runs the bins, reads, dispatches and waits, returning only result lines and reader
outputs; print them verbatim. The model keeps only lead judgment, fixes, CHANGELOG and PR prose, the
gates and the report. `STOP:` (exit 3) means nothing after it ran; `NEXT:` names
@@ -24,32 +24,32 @@ version, CHANGELOG, commits), **Documentation** (Step 14.5) and **Next steps** (
**Reviewers are the sections' own.** Every audit, specialist, Red Team, QA, fix-check, adversarial,
outside-voice and documentation brief, its severity and confidence rules and its gates come from its
section verbatim, with only its placeholders filled. This file restates none of them and
never shortens a reviewer's brief. Audit readers (Steps 7, 8, 9.2.1, 14.5) cannot block, so each runs on a
never shortens a reviewer's brief. A blocking reader's prompt is its `gstack-review-dispatch render`
file, unchanged; its findings reach `merged.jsonl` only via `collect`; `judge` refuses a drop or downgrade. Audit readers (Steps 7, 8, 9.2.1, 14.5) cannot block, so each runs on a
small model at low effort; their findings go to the PR body.
**Calls.** One pass with no fixes is 9 calls: Phases 1-8 and the report; at most 12 with a STOP's
section read or a fix-check. A pass that fixes something repeats Phases 3-4. Waits add pings, not work.
section read or a fix-check. A pass that fixes something repeats Phases 3-4.
**Budget: observe.** `gstack-ship-budget start --mode observe` records, never admits: each program ends
with `gstack-ship-budget note --step <last step> --reading R --credits C`, R and C read in that program
(Capy: `cost.tokens` and `cost.credits` of your own jam). No check per step, no reservation, no meter-only
(Capy: your own jam's `cost.tokens` and `cost.credits`). No check per step, no reservation, no meter-only
call, no handoff; under `--token-cap` a note past the cap prints `verdict=stop`, the one stop check.
A program that dispatches readers takes `BEFORE` (R and C) right before they start and passes
A dispatching program takes `BEFORE` (R and C) as its readers start and passes
`--dispatched-at <R> --dispatched-credits <C> --readers <n>` with the reading after the last settles.
**Waits.** No call waits past 240 s: a program polls or sleeps at most 240 s and returns what still
runs; the next call resumes (Capy: `wait` 240 s on the operation). A call after a gap over 5 minutes
bills the whole context fresh (about 40x a cached call); a ping costs a few percent of that. A small
waiting child saves nothing: the parent's cache expires during its wait too. Past 100 minutes, stop.
**Waits.** No call waits past 240 s: a program returns whatever still runs at 240 s; the next call resumes (Capy: `wait` 240 s on it). A small
waiting child saves nothing. Past 100 minutes, stop.
**One program per pass.** Each review pass, Step 11 and the Step 14.5 audit is one program, and the
parent makes no other call while it runs: (1) `gstack-review-plan --base <base>` with the pass's
flags (its `DISPATCH:` lines; Step 14.5 has one reader, `docs`); (2) `gstack-review-log --start
<skill>` once per row the step logs (Step 14.5 takes none); (3) `BEFORE`; (4) one subagent per reader;
<skill>` once per row the step logs (Step 14.5 takes none); (3) `BEFORE`; (4) one subagent per reader on its rendered file, then
`sent` (`BRIEF_MISMATCH` voids it);
(5) a poll until every reader settles (status `done` or `failed`, or a final summary; a fresh `idle`
has not run); (6) `note` with the reading pair; a later batch (the Red Team,
given the specialists' collected lines) repeats (3)-(6); (7) the collect: return only each reader's
final output and status. The next call applies the section's merge, judgment, fixes and persistence;
final output and status, via `collect`. The next call applies the section's merge, judgment, fixes and persistence;
it never walks the dispatch steps call by call.
## 1. Load: Step 0
@@ -85,7 +85,8 @@ reader ran.
## 4. Judgment: Steps 8.1, 8.2, 9.3, 9.4 and Step 11
Steps 8.1 and 8.2 (`Scope Check:`) on the todos and commits files, then the core checklist, Steps
9.3, 9.4 and the repeat decision as written (`gstack-review-plan` included), fixes and persistence in
9.3, 9.4 and the repeat decision as written (`gstack-review-plan` included;
`JUDGMENT: ok` before any fix), fixes and persistence in
one program. Prints `Pre-Landing Review:`, `Gate edits:`. `NEXT: stop` stops; `full` or
`scoped` reruns Step 5 (`--tests-only`) and Phases 3-4; `done`: the same program runs Step 11 from
`sections/adversarial.md` as written (above), only the voices the last full-pass plan lists (neither
@@ -94,7 +95,7 @@ prints `ADVERSARIAL: skipped (review plan)`).
## 5. Voices, bind and version: Steps 11, 11.5, 12
Judge the voices; missing outside coverage is `unavailable`, never clean; queued fixes rerun Phases
3-4. Then `gstack-review-read`: the final Step 9 row and Step 11 native row (or `CURRENT` freshness
3-4. Then `gstack-review-dispatch verify` (`VERIFY: clean`) and `gstack-review-read`: the final Step 9 row and Step 11 native row (or `CURRENT` freshness
after a planned skip) agree on `wtree`, `start_wtree`, `end_wtree`; save that reviewed tree. Then
`gstack-ship-release version --base <base> [--bump micro|patch] --why '<line>'`; MINOR or MAJOR is a
question; `VERSION: skipped (policy: ...)`: write no version.
+14 -13
View File
@@ -5,7 +5,7 @@ Read this instead of `ship/SKILL.md` when a parent agent on any host runs /ship
the `GSTACK_LAUNCH` assignments your installer printed (`. <the GSTACK_LAUNCH_FILE it printed>`), never
`export $(...)`, `env`, `printenv` or a bare `set` in a tool fence.
**Rules.** Each phase below is one call: one program (an execute program on Capy, else a shell
**Rules.** Each phase below is one call: one program (Capy: an execute program; else a shell
script) that runs the bins, reads, dispatches and waits, returning only result lines and reader
outputs; print them verbatim. The model keeps only lead judgment, fixes, CHANGELOG and PR prose, the
gates and the report. `STOP:` (exit 3) means nothing after it ran; `NEXT:` names
@@ -22,32 +22,32 @@ version, CHANGELOG, commits), **Documentation** (Step 14.5) and **Next steps** (
**Reviewers are the sections' own.** Every audit, specialist, Red Team, QA, fix-check, adversarial,
outside-voice and documentation brief, its severity and confidence rules and its gates come from its
section verbatim, with only its placeholders filled. This file restates none of them and
never shortens a reviewer's brief. Audit readers (Steps 7, 8, 9.2.1, 14.5) cannot block, so each runs on a
never shortens a reviewer's brief. A blocking reader's prompt is its `gstack-review-dispatch render`
file, unchanged; its findings reach `merged.jsonl` only via `collect`; `judge` refuses a drop or downgrade. Audit readers (Steps 7, 8, 9.2.1, 14.5) cannot block, so each runs on a
small model at low effort; their findings go to the PR body.
**Calls.** One pass with no fixes is 9 calls: Phases 1-8 and the report; at most 12 with a STOP's
section read or a fix-check. A pass that fixes something repeats Phases 3-4. Waits add pings, not work.
section read or a fix-check. A pass that fixes something repeats Phases 3-4.
**Budget: observe.** `gstack-ship-budget start --mode observe` records, never admits: each program ends
with `gstack-ship-budget note --step <last step> --reading R --credits C`, R and C read in that program
(Capy: `cost.tokens` and `cost.credits` of your own jam). No check per step, no reservation, no meter-only
(Capy: your own jam's `cost.tokens` and `cost.credits`). No check per step, no reservation, no meter-only
call, no handoff; under `--token-cap` a note past the cap prints `verdict=stop`, the one stop check.
A program that dispatches readers takes `BEFORE` (R and C) right before they start and passes
A dispatching program takes `BEFORE` (R and C) as its readers start and passes
`--dispatched-at <R> --dispatched-credits <C> --readers <n>` with the reading after the last settles.
**Waits.** No call waits past 240 s: a program polls or sleeps at most 240 s and returns what still
runs; the next call resumes (Capy: `wait` 240 s on the operation). A call after a gap over 5 minutes
bills the whole context fresh (about 40x a cached call); a ping costs a few percent of that. A small
waiting child saves nothing: the parent's cache expires during its wait too. Past 100 minutes, stop.
**Waits.** No call waits past 240 s: a program returns whatever still runs at 240 s; the next call resumes (Capy: `wait` 240 s on it). A small
waiting child saves nothing. Past 100 minutes, stop.
**One program per pass.** Each review pass, Step 11 and the Step 14.5 audit is one program, and the
parent makes no other call while it runs: (1) `gstack-review-plan --base <base>` with the pass's
flags (its `DISPATCH:` lines; Step 14.5 has one reader, `docs`); (2) `gstack-review-log --start
<skill>` once per row the step logs (Step 14.5 takes none); (3) `BEFORE`; (4) one subagent per reader;
<skill>` once per row the step logs (Step 14.5 takes none); (3) `BEFORE`; (4) one subagent per reader on its rendered file, then
`sent` (`BRIEF_MISMATCH` voids it);
(5) a poll until every reader settles (status `done` or `failed`, or a final summary; a fresh `idle`
has not run); (6) `note` with the reading pair; a later batch (the Red Team,
given the specialists' collected lines) repeats (3)-(6); (7) the collect: return only each reader's
final output and status. The next call applies the section's merge, judgment, fixes and persistence;
final output and status, via `collect`. The next call applies the section's merge, judgment, fixes and persistence;
it never walks the dispatch steps call by call.
## 1. Load: Step 0
@@ -83,7 +83,8 @@ reader ran.
## 4. Judgment: Steps 8.1, 8.2, 9.3, 9.4 and Step 11
Steps 8.1 and 8.2 (`Scope Check:`) on the todos and commits files, then the core checklist, Steps
9.3, 9.4 and the repeat decision as written (`gstack-review-plan` included), fixes and persistence in
9.3, 9.4 and the repeat decision as written (`gstack-review-plan` included;
`JUDGMENT: ok` before any fix), fixes and persistence in
one program. Prints `Pre-Landing Review:`, `Gate edits:`. `NEXT: stop` stops; `full` or
`scoped` reruns Step 5 (`--tests-only`) and Phases 3-4; `done`: the same program runs Step 11 from
`sections/adversarial.md` as written (above), only the voices the last full-pass plan lists (neither
@@ -92,7 +93,7 @@ prints `ADVERSARIAL: skipped (review plan)`).
## 5. Voices, bind and version: Steps 11, 11.5, 12
Judge the voices; missing outside coverage is `unavailable`, never clean; queued fixes rerun Phases
3-4. Then `gstack-review-read`: the final Step 9 row and Step 11 native row (or `CURRENT` freshness
3-4. Then `gstack-review-dispatch verify` (`VERIFY: clean`) and `gstack-review-read`: the final Step 9 row and Step 11 native row (or `CURRENT` freshness
after a planned skip) agree on `wtree`, `start_wtree`, `end_wtree`; save that reviewed tree. Then
`gstack-ship-release version --base <base> [--bump micro|patch] --why '<line>'`; MINOR or MAJOR is a
question; `VERSION: skipped (policy: ...)`: write no version.
+153
View File
@@ -0,0 +1,153 @@
import { afterEach, describe, expect, test } from 'bun:test';
import { spawnSync } from 'node:child_process';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { BriefError, promptHash, renderBrief, sectionTexts } from '../lib/review-brief';
import { judge, parseFindings, type MergedLine } from '../lib/review-ledger';
const ROOT = path.resolve(import.meta.dir, '..');
const BIN = path.join(ROOT, 'bin', 'gstack-review-dispatch');
const dirs: string[] = [];
afterEach(() => { while (dirs.length) fs.rmSync(dirs.pop()!, { recursive: true, force: true }); });
const tmp = () => { const d = fs.mkdtempSync(path.join(os.tmpdir(), 'review-dispatch-')); dirs.push(d); return d; };
const run = (args: string[], cwd = ROOT) => spawnSync(BIN, args, { cwd, encoding: 'utf8', timeout: 60_000 });
const SHA = 'a'.repeat(40);
const section = (rel: string) => fs.readFileSync(path.join(ROOT, rel), 'utf8');
const field = (out: string, k: string) => new RegExp(`${k}=(\\S+)`).exec(out)?.[1] ?? '';
describe('gstack-review-dispatch render: briefs from the sections, slots only', () => {
// Value: protects=every blocking reader's brief is cut from its section's own bytes with only declared slots filled, so a driven /ship sends what ship/SKILL.md sends; fails_when=a renderer paraphrases, drops a line, leaves a placeholder, or a section edit silently stops matching (GSTA-167 no-loss round 4: 37 of 112 dispatched briefs matched; a batched brief lost two lines);
// why_new=no renderer existed; the driver relied on the model to fill placeholders; seam=lib/review-brief.ts and bin/gstack-review-dispatch; no_claim=that the reader then follows its brief
test('each role renders, carries its section lines verbatim and leaves no placeholder', () => {
const t = sectionTexts(ROOT);
const slots = { diff_base: SHA, exclude: "-- . ':(exclude)*.lock'", stack: 'TypeScript', test_fw: 'bun:test', name: 'testing', checklist: '/x/testing.md', checklists: '/x/a.md,/x/b.md',
findings: '[CRITICAL] a.ts:3 off by one', specialists: '4', diff: '+const a = 1;', fix_list: 'a.ts:3 fixed', hunks: '+x', tests: '3 pass' };
const army = section('ship/sections/review-army.md');
const adv = section('ship/sections/adversarial.md');
const spec = renderBrief('specialist', slots, t);
for (const line of ['Output nothing else: no preamble, no summary or commentary.', 'Required fields: severity, confidence, path, category, summary, specialist.',
'blocks with clear intent. Skip test_stub for architectural or design-only findings.']) { expect(army).toContain(line); expect(spec).toContain(line); }
expect(spec).toContain('Read the checklist at /x/testing.md');
expect(spec).toContain(`git diff "$DIFF_BASE" -- . ':(exclude)*.lock'`);
expect(spec).toContain(`DIFF_BASE=${SHA}`);
expect(spec).not.toContain('write the sha in before dispatch');
const batched = renderBrief('specialist-batched', slots, t);
expect(batched).toContain('Read each checklist at /x/a.md, /x/b.md');
expect(batched.split('\n').length).toBe(spec.split('\n').length);
const security = renderBrief('specialist', { ...slots, name: 'security' }, t);
expect(security.startsWith('This is an authorized defensive-security review')).toBe(true);
expect(security).toContain('reviewed in summary mode');
const red = renderBrief('red-team', slots, t);
expect(red).toContain('reviewed by 4 specialists');
expect(red).toContain('[CRITICAL] a.ts:3 off by one');
const native = renderBrief('adversarial-native', slots, t);
expect(adv).toContain('Think like an attacker and a chaos engineer. Your job is to find ways this code will fail in production.');
expect(native).toContain('Think like an attacker and a chaos engineer. Your job is to find ways this code will fail in production.');
expect(native).toContain(`DIFF_BASE=${SHA} && git diff --name-status`);
const outside = renderBrief('outside-voice', slots, t);
expect(outside).toContain('Filesystem boundary: do not read or execute any files under ~/.claude/');
expect(outside).toContain('+const a = 1;');
expect(renderBrief('fix-check', slots, t)).toContain('You are the fix-check reviewer');
for (const text of [spec, batched, security, red, native]) for (const p of ['<diff-base>', '<exclude>', '<base>', '{checklist path}', '{STACK}', '{TEST_FW}', '{learnings']) expect(text).not.toContain(p);
});
test('a missing slot, a non-sha diff base and a section that lost its anchor refuse instead of rendering', () => {
const t = sectionTexts(ROOT);
expect(() => renderBrief('specialist', { diff_base: SHA, name: 'testing' }, t)).toThrow(BriefError);
expect(() => renderBrief('adversarial-native', { diff_base: '$RANGE_BASE' }, t)).toThrow(/literal commit sha/);
const root = tmp();
fs.mkdirSync(path.join(root, 'ship/sections'), { recursive: true });
fs.writeFileSync(path.join(root, 'ship/sections/review-army.md'), section('ship/sections/review-army.md').replace('"You are a specialist code reviewer.', '"You review code.'));
fs.writeFileSync(path.join(root, 'ship/sections/adversarial.md'), section('ship/sections/adversarial.md'));
expect(() => sectionTexts(root)).toThrow(/specialist brief: start anchor not found/);
});
test('the CLI writes the brief, prints its hash and records it; the file hashes to the printed value', () => {
const out = tmp();
const r = run(['render', '--role', 'specialist', '--name', 'testing', '--out', out, '--diff-base', SHA, '--stack', 'TypeScript']);
expect(r.status).toBe(0);
const file = field(r.stdout, 'file');
expect(r.stdout).toContain('BRIEF: reader=specialist/testing');
expect(promptHash(fs.readFileSync(file, 'utf8'))).toBe(field(r.stdout, 'sha256'));
expect(fs.readFileSync(path.join(out, 'review-ledger.jsonl'), 'utf8')).toContain('"event":"rendered"');
expect(run(['render', '--role', 'specialist', '--name', 'no-such-specialist', '--out', out, '--diff-base', SHA]).stderr).toContain('BRIEF_RENDER_FAILED: checklist not found');
});
});
describe('gstack-review-dispatch sent, collect, judge and verify: nothing changed on the way in or out', () => {
function pass() {
const out = tmp();
const ledger = path.join(out, 'review-ledger.jsonl');
const merged = path.join(out, 'merged.jsonl');
const file = field(run(['render', '--role', 'specialist', '--name', 'testing', '--out', out, '--diff-base', SHA]).stdout, 'file');
return { out, ledger, merged, file, w: (n: string, t: string) => { const f = path.join(out, n); fs.writeFileSync(f, t); return f; } };
}
// Value: protects=a dispatched prompt that is not byte-for-byte its rendered brief fails the pass and shows in verify; fails_when=a paraphrased or trimmed brief is recorded as reviewed (round 4: 75 of 112 briefs differed from the rendered ones);
// why_new=no ledger recorded prompt hashes; seam=sent and verify; no_claim=that the host passed the recorded text
test('the rendered file sent unchanged matches; a brief with one line dropped is BRIEF_MISMATCH and verify is dirty', () => {
const p = pass();
const ok = run(['sent', '--ledger', p.ledger, '--role', 'specialist', '--name', 'testing', '--sent', p.file]);
expect([ok.status, ok.stdout.startsWith('BRIEF_SENT: reader=specialist/testing')]).toEqual([0, true]);
expect(run(['verify', '--ledger', p.ledger]).stdout).toContain('BRIEF specialist/testing MATCH');
const lines = fs.readFileSync(p.file, 'utf8').split('\n');
const trimmed = p.w('trimmed.txt', lines.filter((_, i) => i !== 3).join('\n'));
const bad = run(['sent', '--ledger', p.ledger, '--role', 'specialist', '--name', 'testing', '--sent', trimmed]);
expect(bad.status).toBe(1);
expect(bad.stdout).toContain('BRIEF_MISMATCH: reader=specialist/testing');
const v = run(['verify', '--ledger', p.ledger]);
expect([v.status, v.stdout.includes('BRIEF specialist/testing MISMATCH'), v.stdout.includes('VERIFY: dirty')]).toEqual([1, true, true]);
});
// Value: protects=findings reach merged.jsonl only through collect with each voice's label as written, and lead judgment can add or promote but never drop or downgrade; fails_when=the parent drops a reader's finding, lowers its severity, folds it into a weaker duplicate or adds one silently, or merged lines no longer trace to the output hash;
// why_new=merging was the model's own prose step; seam=collect, judge and verify; no_claim=that the judged set is used downstream by Step 9.4
test('collect keeps raw labels; judge rejects a drop and a downgrade, accepts a reasoned promotion and a fold into an equal duplicate', () => {
const p = pass();
run(['sent', '--ledger', p.ledger, '--role', 'specialist', '--name', 'testing', '--sent', p.file]);
const spec = p.w('spec.out', '{"severity":"CRITICAL","path":"a.ts","line":3,"category":"logic","summary":"off by one","specialist":"testing"}\n{"severity":"INFORMATIONAL","path":"a.ts","line":9,"category":"style","summary":"name","specialist":"testing"}\n');
const outside = p.w('outside.out', 'Findings:\n1. **High**: retry loop never backs off (q.ts:78)\n2. Low - log line is noisy\nRecommendation: Fix the retry because it DoSes the pool\n');
const c1 = run(['collect', '--ledger', p.ledger, '--role', 'specialist', '--name', 'testing', '--output', spec, '--merged', p.merged]);
const c2 = run(['collect', '--ledger', p.ledger, '--role', 'outside-voice', '--output', outside, '--merged', p.merged]);
expect([c1.stdout.includes('findings=2'), c2.stdout.includes('findings=2')]).toEqual([true, true]);
const merged = fs.readFileSync(p.merged, 'utf8').trim().split('\n').map(l => JSON.parse(l) as MergedLine);
expect(merged.map(m => [m.id, m.label])).toEqual([['specialist/testing#1', 'CRITICAL'], ['specialist/testing#2', 'INFORMATIONAL'], ['outside-voice#1', 'High'], ['outside-voice#2', 'Low']]);
const jl = (rows: object[]) => p.w('judged.jsonl', rows.map(r => JSON.stringify(r)).join('\n') + '\n');
const keep = [{ id: 'specialist/testing#1', severity: 'CRITICAL' }, { id: 'specialist/testing#2', severity: 'INFORMATIONAL' }, { id: 'outside-voice#1', severity: 'High' }, { id: 'outside-voice#2', severity: 'Low' }];
const dropped = run(['judge', '--ledger', p.ledger, '--merged', p.merged, '--judged', jl(keep.filter(k => k.id !== 'outside-voice#2'))]);
expect([dropped.status, dropped.stdout]).toEqual([1, expect.stringContaining('JUDGMENT_REJECTED: dropped=["outside-voice#2"]')]);
const downgraded = run(['judge', '--ledger', p.ledger, '--merged', p.merged, '--judged', jl(keep.map(k => (k.id === 'specialist/testing#1' ? { ...k, severity: 'INFORMATIONAL' } : k)))]);
expect([downgraded.status, downgraded.stdout]).toEqual([1, expect.stringContaining('downgraded=["specialist/testing#1"]')]);
const weakFold = run(['judge', '--ledger', p.ledger, '--merged', p.merged, '--judged', jl(keep.map(k => (k.id === 'outside-voice#1' ? { ...k, duplicate_of: 'specialist/testing#2' } : k)))]);
expect(weakFold.stdout).toContain('downgraded=["outside-voice#1"]');
const silentAdd = run(['judge', '--ledger', p.ledger, '--merged', p.merged, '--judged', jl([...keep, { id: 'lead#1', severity: 'CRITICAL' }])]);
expect(silentAdd.stdout).toContain('unexplained=["lead#1"]');
const ok = run(['judge', '--ledger', p.ledger, '--merged', p.merged, '--judged', jl([
...keep.map(k => (k.id === 'outside-voice#1' ? { ...k, duplicate_of: 'specialist/testing#1' } : k.id === 'outside-voice#2' ? { ...k, severity: 'INFORMATIONAL', reason: 'it hides a real error line' } : k)),
{ id: 'lead#1', severity: 'CRITICAL', reason: 'the core checklist found a broken caller' }])]);
expect([ok.status, ok.stdout.trim()]).toEqual([0, 'JUDGMENT: ok added=1 promoted=1 folded=1']);
const ledger = fs.readFileSync(p.ledger, 'utf8');
expect(ledger).toContain('"kind":"promoted","id":"outside-voice#2","from":"Low","to":"INFORMATIONAL","reason":"it hides a real error line"');
expect(ledger).toContain('"kind":"added","id":"lead#1"');
const v = run(['verify', '--ledger', p.ledger]);
expect([v.status, v.stdout.includes('OUTPUT outside-voice MATCH'), v.stdout.includes('JUDGE - OK'), v.stdout.includes('VERIFY: clean briefs=1 outputs=2')]).toEqual([0, true, true, true]);
fs.writeFileSync(p.merged, fs.readFileSync(p.merged, 'utf8').replace('"label":"Low"', '"label":"Low","x":1').replace(/"output_sha256":"[0-9a-f]+"}\n$/, '"output_sha256":"' + '0'.repeat(64) + '"}\n'));
const tampered = run(['verify', '--ledger', p.ledger]);
expect([tampered.status, tampered.stdout.includes('OUTPUT outside-voice MISMATCH')]).toEqual([1, true]);
});
test('judge and parseFindings as functions: an unknown severity is invalid, NO FINDINGS and the Recommendation line are not findings', () => {
expect(parseFindings('r', 'NO FINDINGS\n')).toEqual([]);
expect(parseFindings('r', 'FIXABLE: race in a.ts:3\nINVESTIGATE: odd lock\nRecommendation: Fix the race because x\n').map(f => f.label)).toEqual(['FIXABLE', 'INVESTIGATE']);
const m: MergedLine[] = [{ id: 'r#1', reader: 'r', label: 'FIXABLE', raw: 'x', output_sha256: 'h' }];
expect(judge(m, [{ id: 'r#1', severity: 'severe' }]).invalid).toEqual(['r#1']);
expect(judge(m, [{ id: 'r#1', severity: 'INFORMATIONAL', reason: 'a real defect' }]).ok).toBe(true);
});
test('verify on a ledger that kept no prompts says so and exits 3', () => {
const d = tmp();
const ledger = path.join(d, 'ledger.jsonl');
fs.writeFileSync(ledger, '{"event":"note","step":"9"}\n');
const v = run(['verify', '--ledger', ledger]);
expect([v.status, v.stdout.trim()]).toEqual([3, "VERIFY: no brief records (this run's ledger kept no prompts; dispatched-vs-rendered cannot be checked)"]);
});
});
+10
View File
@@ -204,6 +204,16 @@ describe('variant G: the call plan, observe budget, waits and audit readers (GST
expect(capy).toContain('model: { model_id: "anthropic/claude-sonnet-5-5", reasoning_mode: "low" }');
expect(capy).toContain('`anthropic/claude-haiku-5-5` also fits');
});
// Value: protects=a driven /ship dispatches every blocking reader on its rendered brief file unchanged, checks it with `sent`, merges findings only through `collect`, gates judgment with `judge` and verifies the run before the bind; fails_when=the driver lets the model write or trim a brief again or merge findings in prose (round 4b: arm D lost 8 blockers with 75 of 112 briefs paraphrased);
// why_new=the driver named no renderer or hash check; seam=none; no_claim=the parent's adherence, which verify checks after the fact
test('briefs go out as rendered and findings come back unchanged: render, sent, collect, judge, verify', () => {
const f = flat(doc);
for (const s of ["A blocking reader's prompt is its `gstack-review-dispatch render` file, unchanged", 'only via `collect`', '`judge` refuses a drop or downgrade',
'(4) one subagent per reader on its rendered file, then `sent` (`BRIEF_MISMATCH` voids it)', 'final output and status, via `collect`', '`JUDGMENT: ok` before any fix',
'`gstack-review-dispatch verify` (`VERIFY: clean`)']) expect(f).toContain(s);
expect(f.indexOf('gstack-review-dispatch verify')).toBeLessThan(f.indexOf('`gstack-review-read`'));
});
});
describe('STOP anchors', () => {