From ac3bd7b6714e2f9a9f2d7b8404447a7ea7582b34 Mon Sep 17 00:00:00 2001 From: garrytan Date: Fri, 30 Oct 2026 18:13:59 +0000 Subject: [PATCH] feat(ship): verbatim reviewer briefs and loss-free collection (gstack-review-dispatch render/sent/collect/judge/verify) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- bin/gstack-review-dispatch | 130 ++++++++++++++++++++++++++++ docs/review-policy.md | 31 +++++++ lib/review-brief.ts | 130 ++++++++++++++++++++++++++++ lib/review-ledger.ts | 158 +++++++++++++++++++++++++++++++++++ ship/DRIVER.md | 27 +++--- ship/DRIVER.md.tmpl | 27 +++--- test/review-dispatch.test.ts | 153 +++++++++++++++++++++++++++++++++ test/ship-driver.test.ts | 10 +++ 8 files changed, 640 insertions(+), 26 deletions(-) create mode 100755 bin/gstack-review-dispatch create mode 100644 lib/review-brief.ts create mode 100644 lib/review-ledger.ts create mode 100644 test/review-dispatch.test.ts diff --git a/bin/gstack-review-dispatch b/bin/gstack-review-dispatch new file mode 100755 index 000000000..641befef5 --- /dev/null +++ b/bin/gstack-review-dispatch @@ -0,0 +1,130 @@ +#!/usr/bin/env -S bun --no-env-file --config=/dev/null +/** + * gstack-review-dispatch — /ship's reviewers get their briefs byte for byte and the parent changes + * nothing they return (GSTA-167 no-loss gate; docs/review-policy.md#verbatim-dispatch). + * + * gstack-review-dispatch render --role [--name ] --out [slots] write the brief, print its sha256 + * gstack-review-dispatch sent --ledger --role [--name ] --sent the prompt actually passed; mismatch fails + * gstack-review-dispatch collect --ledger --role [--name ] --output --merged + * gstack-review-dispatch judge --ledger --merged --judged no drop, no downgrade + * gstack-review-dispatch verify --ledger replay every hash and the judgment + * + * Logic: lib/review-brief.ts (rendering from the sections) and lib/review-ledger.ts (ledger, collect, judge, verify). + */ +import { spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { EXIT, exitTableHelp } from '../lib/headless-artifacts'; +import { BRIEF_ROLES, BriefError, briefFile, promptHash, renderBrief, sectionTexts, type BriefRole, type Slots } from '../lib/review-brief'; +import { collect, readJsonl, readerKey, recordJudgment, recordSent, appendEvent, verify, type JudgedLine, type MergedLine } from '../lib/review-ledger'; + +const HELP = `gstack-review-dispatch — verbatim reviewer briefs and loss-free collection for /ship + + render --role [--name ] --out [--ledger ] [slots] + roles: ${BRIEF_ROLES.join(', ')} + slots: --diff-base (every role but fix-check) --exclude '' --stack --test-fw + --learnings-file --checklist (default /review/specialists/.md; red-team.md for red-team) + --checklists (specialist-batched) --findings-file --specialists (red-team) + --fix-list --hunks --tests [--callers ] (fix-check) + writes /[-].prompt.md, records it in the ledger (default /review-ledger.jsonl) and prints + BRIEF: reader= sha256= bytes= file=. Pass that file's content as the reader's prompt, unchanged. + sent --ledger --role [--name ] --sent + the exact prompt a dispatch passed: BRIEF_SENT: … match, or BRIEF_MISMATCH: … (exit 1; the pass is not reviewed) + collect --ledger --role [--name ] --output --merged + the reader's raw output: hashes it and appends each finding to merged.jsonl with its label as written + judge --ledger --merged --judged + judged lines {"id","severity","reason"?,"duplicate_of"?}: every merged id kept at its severity or above (a + duplicate folds into one at least as severe); added and promoted findings need a reason. JUDGMENT: ok …, or + JUDGMENT_REJECTED: dropped=… downgraded=… (exit 1) + verify --ledger + BRIEF, OUTPUT and JUDGE rows from the ledger alone, then VERIFY: clean (exit 0), VERIFY: dirty (exit 1) or + VERIFY: no brief records (exit 3: this run's ledger kept no prompts) +${exitTableHelp()}`; + +const args = process.argv.slice(2); +const flag = (n: string) => { const i = args.indexOf(n); return i >= 0 ? args[i + 1] : undefined; }; +const readFile = (f: string | undefined, what: string) => { + if (!f) return undefined; + try { return fs.readFileSync(f, 'utf8'); } catch { throw new BriefError(`cannot read ${what} file ${f}`); } +}; +const ROOT = path.resolve(import.meta.dir, '..'); + +function render(): number { + const role = flag('--role') as BriefRole | undefined; + const out = flag('--out'); + if (!role || !BRIEF_ROLES.includes(role) || !out) { process.stderr.write(`gstack-review-dispatch render: needs --role <${BRIEF_ROLES.join('|')}> and --out \n`); return EXIT.usage; } + const name = flag('--name'); + const diffBase = flag('--diff-base'); + const checklist = flag('--checklist') ?? (role === 'red-team' ? path.join(ROOT, 'review/specialists/red-team.md') : role === 'specialist' && name ? path.join(ROOT, 'review/specialists', `${name}.md`) : undefined); + if (checklist && (role === 'specialist' || role === 'red-team') && !fs.existsSync(checklist)) throw new BriefError(`checklist not found: ${checklist}`); + let diff: string | undefined; + if (role === 'outside-voice' && diffBase) { + const r = spawnSync('git', ['-c', 'core.quotePath=false', 'diff', '--no-color', '--no-ext-diff', diffBase], { encoding: 'utf8', timeout: 60_000, maxBuffer: 256 * 1024 * 1024 }); + if (r.status !== 0) throw new BriefError(`git diff ${diffBase} failed: ${(r.stderr ?? '').trim().split('\n')[0]}`); + diff = r.stdout; + } + const slots: Slots = { + name, diff_base: diffBase, exclude: flag('--exclude'), stack: flag('--stack'), test_fw: flag('--test-fw'), + learnings: readFile(flag('--learnings-file'), 'learnings'), checklist, checklists: flag('--checklists'), + findings: readFile(flag('--findings-file'), 'findings'), specialists: flag('--specialists'), diff, + fix_list: readFile(flag('--fix-list'), 'fix list'), hunks: readFile(flag('--hunks'), 'hunks'), tests: readFile(flag('--tests'), 'tests'), callers: readFile(flag('--callers'), 'callers'), + }; + const text = renderBrief(role, slots, sectionTexts(ROOT)); + fs.mkdirSync(out, { recursive: true, mode: 0o700 }); + const file = path.join(path.resolve(out), briefFile(role, name)); + fs.writeFileSync(file, text, { mode: 0o600 }); + const ledger = flag('--ledger') ?? path.join(out, 'review-ledger.jsonl'); + const sha = promptHash(text); + appendEvent(ledger, { event: 'rendered', reader: readerKey(role, name), role, name: name ?? null, file, sha256: sha, bytes: Buffer.byteLength(text) }); + process.stdout.write(`BRIEF: reader=${readerKey(role, name)} sha256=${sha} bytes=${Buffer.byteLength(text)} file=${file}\n`); + return EXIT.ok; +} + +function main(): number { + const cmd = args[0]; + if (!cmd || cmd === '--help' || cmd === '-h') { process.stdout.write(HELP + '\n'); return cmd ? EXIT.ok : EXIT.usage; } + if (cmd === 'render') return render(); + const ledger = flag('--ledger'); + if (!ledger) { process.stderr.write(`gstack-review-dispatch ${cmd}: --ledger is required\n`); return EXIT.usage; } + const reader = flag('--role') ? readerKey(flag('--role')!, flag('--name')) : ''; + if (cmd === 'sent') { + const sent = readFile(flag('--sent'), 'sent prompt'); + if (!reader || sent === undefined) { process.stderr.write('gstack-review-dispatch sent: needs --role and --sent \n'); return EXIT.usage; } + const r = recordSent(ledger, reader, sent); + process.stdout.write(r.ok ? `BRIEF_SENT: reader=${reader} sha256=${r.sent} match\n` + : `BRIEF_MISMATCH: reader=${reader} rendered=${r.rendered ?? 'none'} sent=${r.sent} (dispatch the rendered file unchanged; this reader's review does not count)\n`); + return r.ok ? EXIT.ok : EXIT.fail; + } + if (cmd === 'collect') { + const outFile = flag('--output'); + const output = readFile(outFile, 'reader output'); + const merged = flag('--merged'); + if (!reader || output === undefined || !merged) { process.stderr.write('gstack-review-dispatch collect: needs --role, --output and --merged \n'); return EXIT.usage; } + const r = collect(ledger, reader, output, merged, path.resolve(outFile!)); + process.stdout.write(`COLLECT: reader=${reader} findings=${r.lines.length} sha256=${r.sha} merged=${merged}\n`); + return EXIT.ok; + } + if (cmd === 'judge') { + const merged = flag('--merged'); + const judged = flag('--judged'); + if (!merged || !judged) { process.stderr.write('gstack-review-dispatch judge: needs --merged and --judged \n'); return EXIT.usage; } + const r = recordJudgment(ledger, readJsonl(merged), readJsonl(judged), judged); + process.stdout.write(r.ok ? `JUDGMENT: ok added=${r.added.length} promoted=${r.promoted.length} folded=${r.folded.length}\n` + : `JUDGMENT_REJECTED: dropped=${JSON.stringify(r.dropped)} downgraded=${JSON.stringify(r.downgraded)} unexplained=${JSON.stringify(r.unexplained)} invalid=${JSON.stringify(r.invalid)} (lead judgment may add or promote a finding, never drop or downgrade one)\n`); + return r.ok ? EXIT.ok : EXIT.fail; + } + if (cmd === 'verify') { + const v = verify(ledger); + for (const r of v.rows) process.stdout.write(`${r.kind} ${r.reader} ${r.status} ${r.detail}\n`); + if (!v.briefs) { process.stdout.write('VERIFY: no brief records (this run\'s ledger kept no prompts; dispatched-vs-rendered cannot be checked)\n'); return EXIT.refused; } + process.stdout.write(`VERIFY: ${v.clean ? 'clean' : 'dirty'} briefs=${v.briefs} outputs=${v.rows.filter(r => r.kind === 'OUTPUT').length}\n`); + return v.clean ? EXIT.ok : EXIT.fail; + } + process.stderr.write(`gstack-review-dispatch: unknown command ${cmd}\n${HELP}\n`); + return EXIT.usage; +} + +try { process.exit(main()); } catch (e) { + if (e instanceof BriefError) { process.stderr.write(`BRIEF_RENDER_FAILED: ${e.message}\n`); process.exit(EXIT.fail); } + throw e; +} diff --git a/docs/review-policy.md b/docs/review-policy.md index 34be51c04..c1e9213fd 100644 --- a/docs/review-policy.md +++ b/docs/review-policy.md @@ -357,8 +357,39 @@ email case-folded, so running it twice changes nothing. The `COAUTHORS: in ` line on stderr is the receipt; `--json` prints `{ range, authors, trailers, appended }`. +## Verbatim dispatch + +A driven `/ship` (ship/DRIVER.md) never lets the model write a reviewer's brief +or merge a reviewer's findings. The no-loss replays showed why: with briefs the +parent filled in by hand, only 37 of 112 dispatched prompts matched the rendered +ones, and the arm lost blockers. `bin/gstack-review-dispatch` closes both ends: + +1. `render --role [--name ] --out ` cuts the brief out of + `ship/sections/review-army.md` or `ship/sections/adversarial.md` (the fix-check + from `scripts/resolvers/fix-check.ts`) and fills only declared slots: the + literal diff base, the plan's `DIFF_CMD` excludes, stack, test framework, + learnings, checklist paths, the Red Team's merged findings and specialist count, + and the outside voice's branch diff. A section that lost an anchor or placeholder + refuses (`BRIEF_RENDER_FAILED`) instead of rendering something else. +2. The dispatch program passes that file's content unchanged as the reader's + prompt and runs `sent` on what it passed: a hash that differs from the rendered + one is `BRIEF_MISMATCH`, and that reader's review does not count. +3. `collect` hashes each reader's raw output and appends its findings to + `merged.jsonl` with the label exactly as written (a JSON line's `severity`, or a + prose line's Critical, High, Medium, Low, P0-P3, FIXABLE or INVESTIGATE). +4. Lead judgment writes `judged.jsonl`; `judge` refuses a set that drops a merged + finding, lowers its severity, folds it into a less severe duplicate, or adds or + promotes one without a reason (`JUDGMENT_REJECTED`). Each addition and promotion + is logged with its reason. +5. `verify --ledger ` replays all of it from the run's ledger: rendered against + dispatched hashes, output hashes against the merged lines they produced, and the + judgment verdict. A ledger from a run that kept no prompts prints + `VERIFY: no brief records` (exit 3), never clean. + ## Where this is enforced +- `lib/review-brief.ts`, `lib/review-ledger.ts`, `bin/gstack-review-dispatch`: [verbatim dispatch](#verbatim-dispatch) (`test/review-dispatch.test.ts`). + - `scripts/resolvers/review-army.ts`: the floor and `NO FINDINGS` in every specialist and Red Team prompt (rendered into `review/sections/review-army.md` and `ship/sections/review-army.md`), and the lead judgment's "On record" line that turns a [ruled class](#rulings-below-the-floor) into a note; in `/ship` every specialist and Red Team prompt it renders carries its diff base as a literal sha the parent writes in (`DIFF_BASE=`: the merge base on pass 1, `RANGE_BASE` from pass 2 on) beside its plan's `DIFF_CMD` excludes, because a reviewer is its own process and never sees the parent's shell variables; the fan-out batches above. - `ship/sections/review-army.md` Step 9: the plan's `REVIEW_DEPTH`, `GATE_SCAN` and `--depth-floor` on every run, the depth fields of the review-log row, passes, `RANGE_BASE`, classify, defer, render, the cap check before any edit, the Step 11 late-defer row, scratch files under `.gstack/tmp/`, the cap decisions; `ship/sections/adversarial.md` Step 11: the outside voice reads the same range and scopes queued fixes the same way. - `lib/review-depth.ts`, `bin/gstack-review-plan`: the diff class, the depth (`depthFor`, `DEPTH_RULES`, `depthOfCommitInput`), each dispatch's `exclude` pathspecs and `DIFF_CMD`, the transition table, the numeric-claim scan and each role's effort tier (`test/review-depth.test.ts`, `test/review-plan-cli.test.ts`). diff --git a/lib/review-brief.ts b/lib/review-brief.ts new file mode 100644 index 000000000..b3e3063c2 --- /dev/null +++ b/lib/review-brief.ts @@ -0,0 +1,130 @@ +/** + * review-brief — every /ship reviewer brief rendered from its section's own bytes (GSTA-167 + * no-loss gate). The text comes from ship/sections/review-army.md and ship/sections/adversarial.md + * at render time, cut out between fixed anchors; only the declared slots below are filled. The + * dispatch program passes the rendered file verbatim as the reader's prompt, and + * lib/review-ledger.ts proves it did by hash. Nothing here is composed by the model. + * + * Slots by role: + * specialist name, checklist, diff_base, exclude, stack, test_fw, learnings (security adds the defensive framing) + * specialist-batched checklists (comma list), diff_base, exclude, stack, test_fw, learnings + * red-team checklist, diff_base, findings (the merged specialist lines), specialists (their count) + * adversarial-native diff_base + * outside-voice diff_base, diff (the branch diff, which the bin reads from git) + * fix-check fix_list, hunks, tests, callers (scripts/resolvers/fix-check.ts) + */ +import { createHash } from 'node:crypto'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { renderFixCheckPrompt } from '../scripts/resolvers/fix-check'; + +export const BRIEF_ROLES = ['specialist', 'specialist-batched', 'red-team', 'adversarial-native', 'outside-voice', 'fix-check'] as const; +export type BriefRole = typeof BRIEF_ROLES[number]; +export type Slots = Record; + +/** The hash every brief and every dispatched prompt is compared by: sha256 of the text without trailing whitespace. */ +export function promptHash(text: string): string { + return createHash('sha256').update(text.replace(/\r\n/g, '\n').replace(/\s+$/, '')).digest('hex'); +} + +export class BriefError extends Error {} + +/** The text strictly between start and end, end searched after start; both anchors are required. */ +function between(text: string, start: string, end: string, what: string): string { + const i = text.indexOf(start); + if (i < 0) throw new BriefError(`${what}: start anchor not found in the section (${JSON.stringify(start.slice(0, 40))})`); + const j = text.indexOf(end, i + start.length); + if (j < 0) throw new BriefError(`${what}: end anchor not found in the section (${JSON.stringify(end.slice(0, 40))})`); + return text.slice(i + start.length, j); +} +/** A quoted prompt body as the reader receives it: the section escapes its inner quotes and line breaks. */ +const unquote = (s: string) => s.replace(/\\"/g, '"').replace(/\\n/g, '\n'); + +export interface SectionTexts { specialist: string; framing: string; fixtures: string; redTeam: string; native: string; outside: string } + +/** Cut every brief body out of the installed sections under root (the directory holding ship/). */ +export function sectionTexts(root: string): SectionTexts { + const read = (rel: string) => { + try { return fs.readFileSync(path.join(root, rel), 'utf8').replace(/\r\n/g, '\n'); } catch { throw new BriefError(`cannot read ${rel} under ${root}`); } + }; + const army = read('ship/sections/review-army.md'); + const adv = read('ship/sections/adversarial.md'); + const specialist = unquote(between(army, '\n"You are a specialist code reviewer.', 'Past learnings: {learnings or \'none\'}"', 'specialist brief')); + return { + specialist: `You are a specialist code reviewer.${specialist}Past learnings: {learnings or 'none'}` + .replace(/ \(``: write the sha in before dispatch[\s\S]*?on `DIFF_EXCLUDE: none`\)/, ''), + framing: unquote(between(army, 'Begin the prompt with:\n"', '"\nand replace its full-diff command', 'defensive framing')), + fixtures: unquote(between(army, 'and replace its full-diff command with: "', '"\nEvery other specialist', 'fixture handling')), + redTeam: unquote(between(army, 'fixture handling from item 4 of the specialist dispatch: "', 'don\'t cover."', 'red-team brief')) + 'don\'t cover.', + native: unquote(between(adv, 'Subagent prompt:\n"', '"\n\nPresent findings', 'adversarial-native brief')), + outside: unquote(between(adv, 'Outside prompt (supply repository context from the parent):\n\n"', '"\n\nWrite the **complete prompt and context**', 'outside-voice brief')), + }; +} + +const SHA = /^[0-9a-f]{7,40}$/; +const FENCE = '`'.repeat(3); +function need(slots: Slots, key: string): string { + const v = slots[key]; + if (v === undefined || v === '') throw new BriefError(`missing slot ${key}`); + return v; +} +function fill(text: string, pairs: Array<[string, string]>, what: string): string { + let out = text; + for (const [from, to] of pairs) { + if (!out.includes(from)) throw new BriefError(`${what}: placeholder ${from} is not in the section text (the section changed; update lib/review-brief.ts)`); + out = out.split(from).join(to); + } + return out; +} +const SPECIALIST_DIFF = 'run\n`DIFF_BASE= && git diff "$DIFF_BASE" ` to get the diff'; + +const PLACEHOLDERS = ['', '', '', '{checklist path}', '{STACK}', '{TEST_FW}', '{merged findings summary}', '{red-team checklist path}', '{learnings']; + +/** Render one reader's brief. Throws BriefError on a missing slot, a section that no longer carries an anchor or placeholder, or a placeholder left unfilled. */ +export function renderBrief(role: BriefRole, slots: Slots, t: SectionTexts): string { + const text = renderRole(role, slots, t); + const left = role === 'fix-check' || role === 'outside-voice' ? [] : PLACEHOLDERS.filter(p => text.includes(p)); + if (left.length) throw new BriefError(`${role}: unfilled placeholder ${left.join(', ')}`); + return text; +} + +function renderRole(role: BriefRole, slots: Slots, t: SectionTexts): string { + const diffBase = role === 'fix-check' ? '' : need(slots, 'diff_base'); + if (diffBase && !SHA.test(diffBase)) throw new BriefError(`slot diff_base must be a literal commit sha, got ${JSON.stringify(diffBase)}`); + const common = (body: string, checklist: string) => fill(body, [ + ...(checklist ? [['{checklist path}', checklist] as [string, string]] : []), ['', diffBase], + ['{TEST_FW}', slots.test_fw || 'unknown'], ['{STACK}', slots.stack || 'unknown'], ['{learnings or \'none\'}', slots.learnings?.trim() || 'none'], + ], role).split(' `').join(slots.exclude ? ' ' + slots.exclude + '`' : '`'); + switch (role) { + case 'specialist': { + const name = need(slots, 'name'); + const checklist = need(slots, 'checklist'); + if (name !== 'security') return common(t.specialist, checklist); + const body = fill(t.specialist, [[SPECIALIST_DIFF, 'set `DIFF_BASE=` first. ' + t.fixtures + ' Then read the diff that way']], role); + return `${t.framing}\n\n${common(body, checklist)}`; + } + case 'specialist-batched': { + const lists = need(slots, 'checklists').split(',').map(s => s.trim()).filter(Boolean); + const body = fill(t.specialist, [['Read the checklist at {checklist path}', `Read each checklist at ${lists.join(', ')} (tag each finding's "specialist" with the name of the checklist it came from)`]], role); + return common(body, ''); + } + case 'red-team': { + const body = fill(t.redTeam, [['{merged findings summary}', (slots.findings ?? '').trim() || '(none)'], ['by N specialists', `by ${need(slots, 'specialists')} specialists`], + ['{red-team checklist path}', need(slots, 'checklist')], ['', diffBase]], role); + return `${t.framing}\n\n${t.fixtures}\n\n${body}`; + } + case 'adversarial-native': + return fill(t.native, [['DIFF_BASE=$(git merge-base origin/ HEAD)', `DIFF_BASE=${diffBase}`]], role); + case 'outside-voice': { + const body = fill(t.outside, [['DIFF_BASE=$(git merge-base origin/ HEAD)', `DIFF_BASE=${diffBase}`]], role); + return [body, '', 'Give each finding a severity: Critical, High, Medium or Low.', '', `## Branch diff (git diff ${diffBase})`, FENCE + 'diff', (slots.diff ?? '').trim(), FENCE].join('\n'); + } + case 'fix-check': + return renderFixCheckPrompt({ fixList: need(slots, 'fix_list'), touchedHunks: need(slots, 'hunks'), testResults: need(slots, 'tests'), callers: slots.callers ?? '' }); + } +} + +/** The file a rendered brief is written to under the run's briefs directory. */ +export function briefFile(role: BriefRole, name?: string): string { + return `${role}${name ? `-${name.replace(/[^a-zA-Z0-9._-]/g, '_')}` : ''}.prompt.md`; +} diff --git a/lib/review-ledger.ts b/lib/review-ledger.ts new file mode 100644 index 000000000..45c06a9e9 --- /dev/null +++ b/lib/review-ledger.ts @@ -0,0 +1,158 @@ +/** + * review-ledger — the proof that /ship's reviewers got their briefs unchanged and that the + * parent changed nothing they returned (GSTA-167 no-loss gate). One JSONL ledger per run: + * + * rendered a brief gstack-review-dispatch render wrote (role, name, file, sha256) + * sent the prompt a dispatch program actually passed, by hash; a mismatch fails the pass + * output a reader's raw output (sha256) and the merged.jsonl ids collect wrote from it + * judgment each lead-judgment change: an added or promoted finding, with its reason + * judge the verdict on the parent's judged set (ok, or the dropped and downgraded ids) + * + * Findings reach merged.jsonl only through collect, with each voice's label exactly as written. + * The parent's judged set may add or promote a finding and may fold a duplicate into one at least + * as severe; it never drops or downgrades one. verify replays every check from the ledger alone. + */ +import * as fs from 'node:fs'; +import { promptHash } from './review-brief'; + +export interface LedgerEvent { event: 'rendered' | 'sent' | 'output' | 'judgment' | 'judge'; [k: string]: unknown } +export const readerKey = (role: string, name?: string) => (name ? `${role}/${name}` : role); + +export function appendEvent(ledger: string, e: LedgerEvent): void { + fs.appendFileSync(ledger, JSON.stringify({ ...e, at: new Date().toISOString() }) + '\n', { mode: 0o600 }); +} +export function readLedger(ledger: string): LedgerEvent[] { + let text: string; + try { text = fs.readFileSync(ledger, 'utf8'); } catch { return []; } + return text.split('\n').filter(l => l.trim()).flatMap(l => { try { return [JSON.parse(l) as LedgerEvent]; } catch { return []; } }); +} +const lastRendered = (events: LedgerEvent[], reader: string) => [...events].reverse().find(e => e.event === 'rendered' && e.reader === reader); + +/** Compare a dispatched prompt with the brief rendered for that reader; records the sent event. */ +export function recordSent(ledger: string, reader: string, sentText: string): { ok: boolean; rendered: string | null; sent: string } { + const r = lastRendered(readLedger(ledger), reader); + const sent = promptHash(sentText); + const rendered = typeof r?.sha256 === 'string' ? r.sha256 : null; + const ok = rendered === sent; + appendEvent(ledger, { event: 'sent', reader, sha256: sent, rendered, match: ok }); + return { ok, rendered, sent }; +} + +/** Severity ranks across the voices' vocabularies; a label with no severity (FIXABLE, INVESTIGATE) ranks 0. */ +const RANKS: Record = { critical: 4, p0: 4, high: 3, p1: 3, medium: 2, p2: 2, informational: 2, low: 1, p3: 1, fixable: 0, investigate: 0 }; +export function rank(label: string): number | null { + const r = RANKS[label.trim().replace(/^\[|\]$/g, '').toLowerCase()]; + return r === undefined ? null : r; +} +const PROSE_LABEL = /(?:^|[\s[(*|:-])(CRITICAL|INFORMATIONAL|Critical|High|Medium|Low|P[0-3]|FIXABLE|INVESTIGATE)(?=$|[\s\]):*|,.-])/; + +export interface MergedLine { id: string; reader: string; label: string; raw: string; output_sha256: string } + +/** One reader's findings as written: a JSON object per line (severity kept as written), else each prose line carrying a severity or FIXABLE/INVESTIGATE label. */ +export function parseFindings(reader: string, output: string): Array<{ label: string; raw: string }> { + const out: Array<{ label: string; raw: string }> = []; + for (const line of output.replace(/\r\n/g, '\n').split('\n')) { + const t = line.trim(); + if (!t || /^NO FINDINGS\.?$/.test(t) || /^Recommendation:/i.test(t)) continue; + if (t.startsWith('{')) { + try { + const o = JSON.parse(t) as Record; + if (typeof o.severity === 'string') { out.push({ label: o.severity, raw: t }); continue; } + } catch { /* prose that starts with a brace */ } + } + const m = PROSE_LABEL.exec(t); + if (m) out.push({ label: m[1]!, raw: t }); + } + return out; +} + +/** collect: hash the raw output, append its findings to merged.jsonl and record both. */ +export function collect(ledger: string, reader: string, output: string, mergedFile: string, outputFile?: string): { sha: string; lines: MergedLine[] } { + const sha = promptHash(output); + const prior = readLedger(ledger).filter(e => e.event === 'output' && e.reader === reader).length; + const lines = parseFindings(reader, output).map((f, i) => ({ id: `${reader}#${prior ? `${prior + 1}.` : ''}${i + 1}`, reader, label: f.label, raw: f.raw, output_sha256: sha })); + fs.appendFileSync(mergedFile, lines.map(l => JSON.stringify(l) + '\n').join(''), { mode: 0o600 }); + appendEvent(ledger, { event: 'output', reader, sha256: sha, file: outputFile ?? null, merged: mergedFile, ids: lines.map(l => l.id) }); + return { sha, lines }; +} + +export function readJsonl(file: string): T[] { + return fs.readFileSync(file, 'utf8').split('\n').filter(l => l.trim()).map(l => JSON.parse(l) as T); +} + +export interface JudgedLine { id: string; severity: string; reason?: string; duplicate_of?: string } +export interface Judgment { ok: boolean; dropped: string[]; downgraded: string[]; unexplained: string[]; invalid: string[]; added: string[]; promoted: string[]; folded: string[] } + +/** The lead-judgment rule: every merged finding survives at its severity or above; additions and promotions carry a reason. */ +export function judge(merged: MergedLine[], judged: JudgedLine[]): Judgment { + const byId = new Map(judged.map(j => [j.id, j])); + const mergedIds = new Set(merged.map(m => m.id)); + const res: Judgment = { ok: true, dropped: [], downgraded: [], unexplained: [], invalid: [], added: [], promoted: [], folded: [] }; + for (const j of judged) if (rank(j.severity) === null) res.invalid.push(j.id); + for (const m of merged) { + const j = byId.get(m.id); + if (!j) { res.dropped.push(m.id); continue; } + const target = j.duplicate_of ? byId.get(j.duplicate_of) : j; + if (!target) { res.dropped.push(m.id); continue; } + const from = rank(m.label) ?? 0; + const to = rank(target.severity); + if (to === null) continue; + if (to < from) res.downgraded.push(m.id); + else if (j.duplicate_of) res.folded.push(m.id); + else if (to > from) { res.promoted.push(m.id); if (!j.reason?.trim()) res.unexplained.push(m.id); } + } + for (const j of judged) if (!mergedIds.has(j.id) && !j.duplicate_of) { res.added.push(j.id); if (!j.reason?.trim()) res.unexplained.push(j.id); } + res.ok = !res.dropped.length && !res.downgraded.length && !res.unexplained.length && !res.invalid.length; + return res; +} + +/** judge, logged: one judgment event per addition and promotion, then the verdict. */ +export function recordJudgment(ledger: string, merged: MergedLine[], judged: JudgedLine[], judgedFile: string): Judgment { + const res = judge(merged, judged); + const byId = new Map(judged.map(j => [j.id, j])); + const labels = new Map(merged.map(m => [m.id, m.label])); + if (res.ok) { + for (const id of res.added) appendEvent(ledger, { event: 'judgment', kind: 'added', id, to: byId.get(id)!.severity, reason: byId.get(id)!.reason }); + for (const id of res.promoted) appendEvent(ledger, { event: 'judgment', kind: 'promoted', id, from: labels.get(id), to: byId.get(id)!.severity, reason: byId.get(id)!.reason }); + } + appendEvent(ledger, { event: 'judge', ok: res.ok, judged: judgedFile, dropped: res.dropped, downgraded: res.downgraded, unexplained: res.unexplained, invalid: res.invalid, added: res.added.length, promoted: res.promoted.length, folded: res.folded.length }); + return res; +} + +export interface VerifyRow { kind: 'BRIEF' | 'OUTPUT' | 'JUDGE'; reader: string; status: string; detail: string } + +/** Replay every check from a finished run's ledger: briefs dispatched as rendered, outputs merged as returned, judgment without loss. */ +export function verify(ledger: string): { rows: VerifyRow[]; clean: boolean; briefs: number } { + const events = readLedger(ledger); + const rows: VerifyRow[] = []; + const short = (s: unknown) => (typeof s === 'string' ? s.slice(0, 12) : '-'); + const readers = [...new Set(events.filter(e => e.event === 'rendered').map(e => String(e.reader)))]; + for (const reader of readers) { + const r = lastRendered(events, reader)!; + const sent = events.filter(e => e.event === 'sent' && e.reader === reader); + let fileNow = ''; + try { fileNow = typeof r.file === 'string' && fs.existsSync(r.file) ? promptHash(fs.readFileSync(r.file, 'utf8')) : ''; } catch { /* unreadable */ } + const status = !sent.length ? 'NOT_SENT' : sent.every(s => s.sha256 === r.sha256) ? 'MATCH' : 'MISMATCH'; + const tampered = fileNow && fileNow !== r.sha256 ? ' file_changed_since_render' : ''; + rows.push({ kind: 'BRIEF', reader, status: tampered ? 'MISMATCH' : status, detail: `rendered=${short(r.sha256)} sent=${sent.map(s => short(s.sha256)).join(',') || '-'}${tampered}` }); + } + for (const o of events.filter(e => e.event === 'output')) { + const reader = String(o.reader); + let status = 'MATCH'; + const notes: string[] = []; + if (typeof o.file === 'string' && fs.existsSync(o.file) && promptHash(fs.readFileSync(o.file, 'utf8')) !== o.sha256) { status = 'MISMATCH'; notes.push('output_file_changed'); } + const ids = Array.isArray(o.ids) ? (o.ids as string[]) : []; + let lines: MergedLine[] = []; + try { lines = typeof o.merged === 'string' ? readJsonl(o.merged).filter(l => ids.includes(l.id)) : []; } catch { notes.push('merged_unreadable'); status = 'MISMATCH'; } + if (lines.length !== ids.length) { status = 'MISMATCH'; notes.push(`merged_lines=${lines.length}/${ids.length}`); } + if (lines.some(l => l.output_sha256 !== o.sha256)) { status = 'MISMATCH'; notes.push('merged_line_from_other_output'); } + rows.push({ kind: 'OUTPUT', reader, status, detail: `sha=${short(o.sha256)} findings=${ids.length}${notes.length ? ' ' + notes.join(' ') : ''}` }); + } + const verdicts = events.filter(e => e.event === 'judge'); + if (verdicts.length) { + const v = verdicts[verdicts.length - 1]!; + rows.push({ kind: 'JUDGE', reader: '-', status: v.ok ? 'OK' : 'REJECTED', detail: `dropped=${JSON.stringify(v.dropped)} downgraded=${JSON.stringify(v.downgraded)} added=${v.added} promoted=${v.promoted} folded=${v.folded}` }); + } else if (events.some(e => e.event === 'output')) rows.push({ kind: 'JUDGE', reader: '-', status: 'MISSING', detail: 'outputs were collected but no judged set was checked' }); + const clean = readers.length > 0 && rows.every(r => r.status === 'MATCH' || r.status === 'OK'); + return { rows, clean, briefs: readers.length }; +} diff --git a/ship/DRIVER.md b/ship/DRIVER.md index 77395ec84..91f484786 100644 --- a/ship/DRIVER.md +++ b/ship/DRIVER.md @@ -7,7 +7,7 @@ Read this instead of `ship/SKILL.md` when a parent agent on any host runs /ship the `GSTACK_LAUNCH` assignments your installer printed (`. `), never `export $(...)`, `env`, `printenv` or a bare `set` in a tool fence. -**Rules.** Each phase below is one call: one program (an execute program on Capy, else a shell +**Rules.** Each phase below is one call: one program (Capy: an execute program; else a shell script) that runs the bins, reads, dispatches and waits, returning only result lines and reader outputs; print them verbatim. The model keeps only lead judgment, fixes, CHANGELOG and PR prose, the gates and the report. `STOP:` (exit 3) means nothing after it ran; `NEXT:` names @@ -24,32 +24,32 @@ version, CHANGELOG, commits), **Documentation** (Step 14.5) and **Next steps** ( **Reviewers are the sections' own.** Every audit, specialist, Red Team, QA, fix-check, adversarial, outside-voice and documentation brief, its severity and confidence rules and its gates come from its section verbatim, with only its placeholders filled. This file restates none of them and -never shortens a reviewer's brief. Audit readers (Steps 7, 8, 9.2.1, 14.5) cannot block, so each runs on a +never shortens a reviewer's brief. A blocking reader's prompt is its `gstack-review-dispatch render` +file, unchanged; its findings reach `merged.jsonl` only via `collect`; `judge` refuses a drop or downgrade. Audit readers (Steps 7, 8, 9.2.1, 14.5) cannot block, so each runs on a small model at low effort; their findings go to the PR body. **Calls.** One pass with no fixes is 9 calls: Phases 1-8 and the report; at most 12 with a STOP's -section read or a fix-check. A pass that fixes something repeats Phases 3-4. Waits add pings, not work. +section read or a fix-check. A pass that fixes something repeats Phases 3-4. **Budget: observe.** `gstack-ship-budget start --mode observe` records, never admits: each program ends with `gstack-ship-budget note --step --reading R --credits C`, R and C read in that program -(Capy: `cost.tokens` and `cost.credits` of your own jam). No check per step, no reservation, no meter-only +(Capy: your own jam's `cost.tokens` and `cost.credits`). No check per step, no reservation, no meter-only call, no handoff; under `--token-cap` a note past the cap prints `verdict=stop`, the one stop check. -A program that dispatches readers takes `BEFORE` (R and C) right before they start and passes +A dispatching program takes `BEFORE` (R and C) as its readers start and passes `--dispatched-at --dispatched-credits --readers ` with the reading after the last settles. -**Waits.** No call waits past 240 s: a program polls or sleeps at most 240 s and returns what still -runs; the next call resumes (Capy: `wait` 240 s on the operation). A call after a gap over 5 minutes -bills the whole context fresh (about 40x a cached call); a ping costs a few percent of that. A small -waiting child saves nothing: the parent's cache expires during its wait too. Past 100 minutes, stop. +**Waits.** No call waits past 240 s: a program returns whatever still runs at 240 s; the next call resumes (Capy: `wait` 240 s on it). A small +waiting child saves nothing. Past 100 minutes, stop. **One program per pass.** Each review pass, Step 11 and the Step 14.5 audit is one program, and the parent makes no other call while it runs: (1) `gstack-review-plan --base ` with the pass's flags (its `DISPATCH:` lines; Step 14.5 has one reader, `docs`); (2) `gstack-review-log --start -` once per row the step logs (Step 14.5 takes none); (3) `BEFORE`; (4) one subagent per reader; +` once per row the step logs (Step 14.5 takes none); (3) `BEFORE`; (4) one subagent per reader on its rendered file, then +`sent` (`BRIEF_MISMATCH` voids it); (5) a poll until every reader settles (status `done` or `failed`, or a final summary; a fresh `idle` has not run); (6) `note` with the reading pair; a later batch (the Red Team, given the specialists' collected lines) repeats (3)-(6); (7) the collect: return only each reader's -final output and status. The next call applies the section's merge, judgment, fixes and persistence; +final output and status, via `collect`. The next call applies the section's merge, judgment, fixes and persistence; it never walks the dispatch steps call by call. ## 1. Load: Step 0 @@ -85,7 +85,8 @@ reader ran. ## 4. Judgment: Steps 8.1, 8.2, 9.3, 9.4 and Step 11 Steps 8.1 and 8.2 (`Scope Check:`) on the todos and commits files, then the core checklist, Steps -9.3, 9.4 and the repeat decision as written (`gstack-review-plan` included), fixes and persistence in +9.3, 9.4 and the repeat decision as written (`gstack-review-plan` included; +`JUDGMENT: ok` before any fix), fixes and persistence in one program. Prints `Pre-Landing Review:`, `Gate edits:`. `NEXT: stop` stops; `full` or `scoped` reruns Step 5 (`--tests-only`) and Phases 3-4; `done`: the same program runs Step 11 from `sections/adversarial.md` as written (above), only the voices the last full-pass plan lists (neither @@ -94,7 +95,7 @@ prints `ADVERSARIAL: skipped (review plan)`). ## 5. Voices, bind and version: Steps 11, 11.5, 12 Judge the voices; missing outside coverage is `unavailable`, never clean; queued fixes rerun Phases -3-4. Then `gstack-review-read`: the final Step 9 row and Step 11 native row (or `CURRENT` freshness +3-4. Then `gstack-review-dispatch verify` (`VERIFY: clean`) and `gstack-review-read`: the final Step 9 row and Step 11 native row (or `CURRENT` freshness after a planned skip) agree on `wtree`, `start_wtree`, `end_wtree`; save that reviewed tree. Then `gstack-ship-release version --base [--bump micro|patch] --why ''`; MINOR or MAJOR is a question; `VERSION: skipped (policy: ...)`: write no version. diff --git a/ship/DRIVER.md.tmpl b/ship/DRIVER.md.tmpl index 5f25501b0..aaf24ed95 100644 --- a/ship/DRIVER.md.tmpl +++ b/ship/DRIVER.md.tmpl @@ -5,7 +5,7 @@ Read this instead of `ship/SKILL.md` when a parent agent on any host runs /ship the `GSTACK_LAUNCH` assignments your installer printed (`. `), never `export $(...)`, `env`, `printenv` or a bare `set` in a tool fence. -**Rules.** Each phase below is one call: one program (an execute program on Capy, else a shell +**Rules.** Each phase below is one call: one program (Capy: an execute program; else a shell script) that runs the bins, reads, dispatches and waits, returning only result lines and reader outputs; print them verbatim. The model keeps only lead judgment, fixes, CHANGELOG and PR prose, the gates and the report. `STOP:` (exit 3) means nothing after it ran; `NEXT:` names @@ -22,32 +22,32 @@ version, CHANGELOG, commits), **Documentation** (Step 14.5) and **Next steps** ( **Reviewers are the sections' own.** Every audit, specialist, Red Team, QA, fix-check, adversarial, outside-voice and documentation brief, its severity and confidence rules and its gates come from its section verbatim, with only its placeholders filled. This file restates none of them and -never shortens a reviewer's brief. Audit readers (Steps 7, 8, 9.2.1, 14.5) cannot block, so each runs on a +never shortens a reviewer's brief. A blocking reader's prompt is its `gstack-review-dispatch render` +file, unchanged; its findings reach `merged.jsonl` only via `collect`; `judge` refuses a drop or downgrade. Audit readers (Steps 7, 8, 9.2.1, 14.5) cannot block, so each runs on a small model at low effort; their findings go to the PR body. **Calls.** One pass with no fixes is 9 calls: Phases 1-8 and the report; at most 12 with a STOP's -section read or a fix-check. A pass that fixes something repeats Phases 3-4. Waits add pings, not work. +section read or a fix-check. A pass that fixes something repeats Phases 3-4. **Budget: observe.** `gstack-ship-budget start --mode observe` records, never admits: each program ends with `gstack-ship-budget note --step --reading R --credits C`, R and C read in that program -(Capy: `cost.tokens` and `cost.credits` of your own jam). No check per step, no reservation, no meter-only +(Capy: your own jam's `cost.tokens` and `cost.credits`). No check per step, no reservation, no meter-only call, no handoff; under `--token-cap` a note past the cap prints `verdict=stop`, the one stop check. -A program that dispatches readers takes `BEFORE` (R and C) right before they start and passes +A dispatching program takes `BEFORE` (R and C) as its readers start and passes `--dispatched-at --dispatched-credits --readers ` with the reading after the last settles. -**Waits.** No call waits past 240 s: a program polls or sleeps at most 240 s and returns what still -runs; the next call resumes (Capy: `wait` 240 s on the operation). A call after a gap over 5 minutes -bills the whole context fresh (about 40x a cached call); a ping costs a few percent of that. A small -waiting child saves nothing: the parent's cache expires during its wait too. Past 100 minutes, stop. +**Waits.** No call waits past 240 s: a program returns whatever still runs at 240 s; the next call resumes (Capy: `wait` 240 s on it). A small +waiting child saves nothing. Past 100 minutes, stop. **One program per pass.** Each review pass, Step 11 and the Step 14.5 audit is one program, and the parent makes no other call while it runs: (1) `gstack-review-plan --base ` with the pass's flags (its `DISPATCH:` lines; Step 14.5 has one reader, `docs`); (2) `gstack-review-log --start -` once per row the step logs (Step 14.5 takes none); (3) `BEFORE`; (4) one subagent per reader; +` once per row the step logs (Step 14.5 takes none); (3) `BEFORE`; (4) one subagent per reader on its rendered file, then +`sent` (`BRIEF_MISMATCH` voids it); (5) a poll until every reader settles (status `done` or `failed`, or a final summary; a fresh `idle` has not run); (6) `note` with the reading pair; a later batch (the Red Team, given the specialists' collected lines) repeats (3)-(6); (7) the collect: return only each reader's -final output and status. The next call applies the section's merge, judgment, fixes and persistence; +final output and status, via `collect`. The next call applies the section's merge, judgment, fixes and persistence; it never walks the dispatch steps call by call. ## 1. Load: Step 0 @@ -83,7 +83,8 @@ reader ran. ## 4. Judgment: Steps 8.1, 8.2, 9.3, 9.4 and Step 11 Steps 8.1 and 8.2 (`Scope Check:`) on the todos and commits files, then the core checklist, Steps -9.3, 9.4 and the repeat decision as written (`gstack-review-plan` included), fixes and persistence in +9.3, 9.4 and the repeat decision as written (`gstack-review-plan` included; +`JUDGMENT: ok` before any fix), fixes and persistence in one program. Prints `Pre-Landing Review:`, `Gate edits:`. `NEXT: stop` stops; `full` or `scoped` reruns Step 5 (`--tests-only`) and Phases 3-4; `done`: the same program runs Step 11 from `sections/adversarial.md` as written (above), only the voices the last full-pass plan lists (neither @@ -92,7 +93,7 @@ prints `ADVERSARIAL: skipped (review plan)`). ## 5. Voices, bind and version: Steps 11, 11.5, 12 Judge the voices; missing outside coverage is `unavailable`, never clean; queued fixes rerun Phases -3-4. Then `gstack-review-read`: the final Step 9 row and Step 11 native row (or `CURRENT` freshness +3-4. Then `gstack-review-dispatch verify` (`VERIFY: clean`) and `gstack-review-read`: the final Step 9 row and Step 11 native row (or `CURRENT` freshness after a planned skip) agree on `wtree`, `start_wtree`, `end_wtree`; save that reviewed tree. Then `gstack-ship-release version --base [--bump micro|patch] --why ''`; MINOR or MAJOR is a question; `VERSION: skipped (policy: ...)`: write no version. diff --git a/test/review-dispatch.test.ts b/test/review-dispatch.test.ts new file mode 100644 index 000000000..07959b848 --- /dev/null +++ b/test/review-dispatch.test.ts @@ -0,0 +1,153 @@ +import { afterEach, describe, expect, test } from 'bun:test'; +import { spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { BriefError, promptHash, renderBrief, sectionTexts } from '../lib/review-brief'; +import { judge, parseFindings, type MergedLine } from '../lib/review-ledger'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const BIN = path.join(ROOT, 'bin', 'gstack-review-dispatch'); +const dirs: string[] = []; +afterEach(() => { while (dirs.length) fs.rmSync(dirs.pop()!, { recursive: true, force: true }); }); +const tmp = () => { const d = fs.mkdtempSync(path.join(os.tmpdir(), 'review-dispatch-')); dirs.push(d); return d; }; +const run = (args: string[], cwd = ROOT) => spawnSync(BIN, args, { cwd, encoding: 'utf8', timeout: 60_000 }); +const SHA = 'a'.repeat(40); +const section = (rel: string) => fs.readFileSync(path.join(ROOT, rel), 'utf8'); +const field = (out: string, k: string) => new RegExp(`${k}=(\\S+)`).exec(out)?.[1] ?? ''; + +describe('gstack-review-dispatch render: briefs from the sections, slots only', () => { + // Value: protects=every blocking reader's brief is cut from its section's own bytes with only declared slots filled, so a driven /ship sends what ship/SKILL.md sends; fails_when=a renderer paraphrases, drops a line, leaves a placeholder, or a section edit silently stops matching (GSTA-167 no-loss round 4: 37 of 112 dispatched briefs matched; a batched brief lost two lines); + // why_new=no renderer existed; the driver relied on the model to fill placeholders; seam=lib/review-brief.ts and bin/gstack-review-dispatch; no_claim=that the reader then follows its brief + test('each role renders, carries its section lines verbatim and leaves no placeholder', () => { + const t = sectionTexts(ROOT); + const slots = { diff_base: SHA, exclude: "-- . ':(exclude)*.lock'", stack: 'TypeScript', test_fw: 'bun:test', name: 'testing', checklist: '/x/testing.md', checklists: '/x/a.md,/x/b.md', + findings: '[CRITICAL] a.ts:3 off by one', specialists: '4', diff: '+const a = 1;', fix_list: 'a.ts:3 fixed', hunks: '+x', tests: '3 pass' }; + const army = section('ship/sections/review-army.md'); + const adv = section('ship/sections/adversarial.md'); + const spec = renderBrief('specialist', slots, t); + for (const line of ['Output nothing else: no preamble, no summary or commentary.', 'Required fields: severity, confidence, path, category, summary, specialist.', + 'blocks with clear intent. Skip test_stub for architectural or design-only findings.']) { expect(army).toContain(line); expect(spec).toContain(line); } + expect(spec).toContain('Read the checklist at /x/testing.md'); + expect(spec).toContain(`git diff "$DIFF_BASE" -- . ':(exclude)*.lock'`); + expect(spec).toContain(`DIFF_BASE=${SHA}`); + expect(spec).not.toContain('write the sha in before dispatch'); + const batched = renderBrief('specialist-batched', slots, t); + expect(batched).toContain('Read each checklist at /x/a.md, /x/b.md'); + expect(batched.split('\n').length).toBe(spec.split('\n').length); + const security = renderBrief('specialist', { ...slots, name: 'security' }, t); + expect(security.startsWith('This is an authorized defensive-security review')).toBe(true); + expect(security).toContain('reviewed in summary mode'); + const red = renderBrief('red-team', slots, t); + expect(red).toContain('reviewed by 4 specialists'); + expect(red).toContain('[CRITICAL] a.ts:3 off by one'); + const native = renderBrief('adversarial-native', slots, t); + expect(adv).toContain('Think like an attacker and a chaos engineer. Your job is to find ways this code will fail in production.'); + expect(native).toContain('Think like an attacker and a chaos engineer. Your job is to find ways this code will fail in production.'); + expect(native).toContain(`DIFF_BASE=${SHA} && git diff --name-status`); + const outside = renderBrief('outside-voice', slots, t); + expect(outside).toContain('Filesystem boundary: do not read or execute any files under ~/.claude/'); + expect(outside).toContain('+const a = 1;'); + expect(renderBrief('fix-check', slots, t)).toContain('You are the fix-check reviewer'); + for (const text of [spec, batched, security, red, native]) for (const p of ['', '', '', '{checklist path}', '{STACK}', '{TEST_FW}', '{learnings']) expect(text).not.toContain(p); + }); + + test('a missing slot, a non-sha diff base and a section that lost its anchor refuse instead of rendering', () => { + const t = sectionTexts(ROOT); + expect(() => renderBrief('specialist', { diff_base: SHA, name: 'testing' }, t)).toThrow(BriefError); + expect(() => renderBrief('adversarial-native', { diff_base: '$RANGE_BASE' }, t)).toThrow(/literal commit sha/); + const root = tmp(); + fs.mkdirSync(path.join(root, 'ship/sections'), { recursive: true }); + fs.writeFileSync(path.join(root, 'ship/sections/review-army.md'), section('ship/sections/review-army.md').replace('"You are a specialist code reviewer.', '"You review code.')); + fs.writeFileSync(path.join(root, 'ship/sections/adversarial.md'), section('ship/sections/adversarial.md')); + expect(() => sectionTexts(root)).toThrow(/specialist brief: start anchor not found/); + }); + + test('the CLI writes the brief, prints its hash and records it; the file hashes to the printed value', () => { + const out = tmp(); + const r = run(['render', '--role', 'specialist', '--name', 'testing', '--out', out, '--diff-base', SHA, '--stack', 'TypeScript']); + expect(r.status).toBe(0); + const file = field(r.stdout, 'file'); + expect(r.stdout).toContain('BRIEF: reader=specialist/testing'); + expect(promptHash(fs.readFileSync(file, 'utf8'))).toBe(field(r.stdout, 'sha256')); + expect(fs.readFileSync(path.join(out, 'review-ledger.jsonl'), 'utf8')).toContain('"event":"rendered"'); + expect(run(['render', '--role', 'specialist', '--name', 'no-such-specialist', '--out', out, '--diff-base', SHA]).stderr).toContain('BRIEF_RENDER_FAILED: checklist not found'); + }); +}); + +describe('gstack-review-dispatch sent, collect, judge and verify: nothing changed on the way in or out', () => { + function pass() { + const out = tmp(); + const ledger = path.join(out, 'review-ledger.jsonl'); + const merged = path.join(out, 'merged.jsonl'); + const file = field(run(['render', '--role', 'specialist', '--name', 'testing', '--out', out, '--diff-base', SHA]).stdout, 'file'); + return { out, ledger, merged, file, w: (n: string, t: string) => { const f = path.join(out, n); fs.writeFileSync(f, t); return f; } }; + } + // Value: protects=a dispatched prompt that is not byte-for-byte its rendered brief fails the pass and shows in verify; fails_when=a paraphrased or trimmed brief is recorded as reviewed (round 4: 75 of 112 briefs differed from the rendered ones); + // why_new=no ledger recorded prompt hashes; seam=sent and verify; no_claim=that the host passed the recorded text + test('the rendered file sent unchanged matches; a brief with one line dropped is BRIEF_MISMATCH and verify is dirty', () => { + const p = pass(); + const ok = run(['sent', '--ledger', p.ledger, '--role', 'specialist', '--name', 'testing', '--sent', p.file]); + expect([ok.status, ok.stdout.startsWith('BRIEF_SENT: reader=specialist/testing')]).toEqual([0, true]); + expect(run(['verify', '--ledger', p.ledger]).stdout).toContain('BRIEF specialist/testing MATCH'); + const lines = fs.readFileSync(p.file, 'utf8').split('\n'); + const trimmed = p.w('trimmed.txt', lines.filter((_, i) => i !== 3).join('\n')); + const bad = run(['sent', '--ledger', p.ledger, '--role', 'specialist', '--name', 'testing', '--sent', trimmed]); + expect(bad.status).toBe(1); + expect(bad.stdout).toContain('BRIEF_MISMATCH: reader=specialist/testing'); + const v = run(['verify', '--ledger', p.ledger]); + expect([v.status, v.stdout.includes('BRIEF specialist/testing MISMATCH'), v.stdout.includes('VERIFY: dirty')]).toEqual([1, true, true]); + }); + + // Value: protects=findings reach merged.jsonl only through collect with each voice's label as written, and lead judgment can add or promote but never drop or downgrade; fails_when=the parent drops a reader's finding, lowers its severity, folds it into a weaker duplicate or adds one silently, or merged lines no longer trace to the output hash; + // why_new=merging was the model's own prose step; seam=collect, judge and verify; no_claim=that the judged set is used downstream by Step 9.4 + test('collect keeps raw labels; judge rejects a drop and a downgrade, accepts a reasoned promotion and a fold into an equal duplicate', () => { + const p = pass(); + run(['sent', '--ledger', p.ledger, '--role', 'specialist', '--name', 'testing', '--sent', p.file]); + const spec = p.w('spec.out', '{"severity":"CRITICAL","path":"a.ts","line":3,"category":"logic","summary":"off by one","specialist":"testing"}\n{"severity":"INFORMATIONAL","path":"a.ts","line":9,"category":"style","summary":"name","specialist":"testing"}\n'); + const outside = p.w('outside.out', 'Findings:\n1. **High**: retry loop never backs off (q.ts:78)\n2. Low - log line is noisy\nRecommendation: Fix the retry because it DoSes the pool\n'); + const c1 = run(['collect', '--ledger', p.ledger, '--role', 'specialist', '--name', 'testing', '--output', spec, '--merged', p.merged]); + const c2 = run(['collect', '--ledger', p.ledger, '--role', 'outside-voice', '--output', outside, '--merged', p.merged]); + expect([c1.stdout.includes('findings=2'), c2.stdout.includes('findings=2')]).toEqual([true, true]); + const merged = fs.readFileSync(p.merged, 'utf8').trim().split('\n').map(l => JSON.parse(l) as MergedLine); + expect(merged.map(m => [m.id, m.label])).toEqual([['specialist/testing#1', 'CRITICAL'], ['specialist/testing#2', 'INFORMATIONAL'], ['outside-voice#1', 'High'], ['outside-voice#2', 'Low']]); + const jl = (rows: object[]) => p.w('judged.jsonl', rows.map(r => JSON.stringify(r)).join('\n') + '\n'); + const keep = [{ id: 'specialist/testing#1', severity: 'CRITICAL' }, { id: 'specialist/testing#2', severity: 'INFORMATIONAL' }, { id: 'outside-voice#1', severity: 'High' }, { id: 'outside-voice#2', severity: 'Low' }]; + const dropped = run(['judge', '--ledger', p.ledger, '--merged', p.merged, '--judged', jl(keep.filter(k => k.id !== 'outside-voice#2'))]); + expect([dropped.status, dropped.stdout]).toEqual([1, expect.stringContaining('JUDGMENT_REJECTED: dropped=["outside-voice#2"]')]); + const downgraded = run(['judge', '--ledger', p.ledger, '--merged', p.merged, '--judged', jl(keep.map(k => (k.id === 'specialist/testing#1' ? { ...k, severity: 'INFORMATIONAL' } : k)))]); + expect([downgraded.status, downgraded.stdout]).toEqual([1, expect.stringContaining('downgraded=["specialist/testing#1"]')]); + const weakFold = run(['judge', '--ledger', p.ledger, '--merged', p.merged, '--judged', jl(keep.map(k => (k.id === 'outside-voice#1' ? { ...k, duplicate_of: 'specialist/testing#2' } : k)))]); + expect(weakFold.stdout).toContain('downgraded=["outside-voice#1"]'); + const silentAdd = run(['judge', '--ledger', p.ledger, '--merged', p.merged, '--judged', jl([...keep, { id: 'lead#1', severity: 'CRITICAL' }])]); + expect(silentAdd.stdout).toContain('unexplained=["lead#1"]'); + const ok = run(['judge', '--ledger', p.ledger, '--merged', p.merged, '--judged', jl([ + ...keep.map(k => (k.id === 'outside-voice#1' ? { ...k, duplicate_of: 'specialist/testing#1' } : k.id === 'outside-voice#2' ? { ...k, severity: 'INFORMATIONAL', reason: 'it hides a real error line' } : k)), + { id: 'lead#1', severity: 'CRITICAL', reason: 'the core checklist found a broken caller' }])]); + expect([ok.status, ok.stdout.trim()]).toEqual([0, 'JUDGMENT: ok added=1 promoted=1 folded=1']); + const ledger = fs.readFileSync(p.ledger, 'utf8'); + expect(ledger).toContain('"kind":"promoted","id":"outside-voice#2","from":"Low","to":"INFORMATIONAL","reason":"it hides a real error line"'); + expect(ledger).toContain('"kind":"added","id":"lead#1"'); + const v = run(['verify', '--ledger', p.ledger]); + expect([v.status, v.stdout.includes('OUTPUT outside-voice MATCH'), v.stdout.includes('JUDGE - OK'), v.stdout.includes('VERIFY: clean briefs=1 outputs=2')]).toEqual([0, true, true, true]); + fs.writeFileSync(p.merged, fs.readFileSync(p.merged, 'utf8').replace('"label":"Low"', '"label":"Low","x":1').replace(/"output_sha256":"[0-9a-f]+"}\n$/, '"output_sha256":"' + '0'.repeat(64) + '"}\n')); + const tampered = run(['verify', '--ledger', p.ledger]); + expect([tampered.status, tampered.stdout.includes('OUTPUT outside-voice MISMATCH')]).toEqual([1, true]); + }); + + test('judge and parseFindings as functions: an unknown severity is invalid, NO FINDINGS and the Recommendation line are not findings', () => { + expect(parseFindings('r', 'NO FINDINGS\n')).toEqual([]); + expect(parseFindings('r', 'FIXABLE: race in a.ts:3\nINVESTIGATE: odd lock\nRecommendation: Fix the race because x\n').map(f => f.label)).toEqual(['FIXABLE', 'INVESTIGATE']); + const m: MergedLine[] = [{ id: 'r#1', reader: 'r', label: 'FIXABLE', raw: 'x', output_sha256: 'h' }]; + expect(judge(m, [{ id: 'r#1', severity: 'severe' }]).invalid).toEqual(['r#1']); + expect(judge(m, [{ id: 'r#1', severity: 'INFORMATIONAL', reason: 'a real defect' }]).ok).toBe(true); + }); + + test('verify on a ledger that kept no prompts says so and exits 3', () => { + const d = tmp(); + const ledger = path.join(d, 'ledger.jsonl'); + fs.writeFileSync(ledger, '{"event":"note","step":"9"}\n'); + const v = run(['verify', '--ledger', ledger]); + expect([v.status, v.stdout.trim()]).toEqual([3, "VERIFY: no brief records (this run's ledger kept no prompts; dispatched-vs-rendered cannot be checked)"]); + }); +}); diff --git a/test/ship-driver.test.ts b/test/ship-driver.test.ts index a94632396..809bb5ccc 100644 --- a/test/ship-driver.test.ts +++ b/test/ship-driver.test.ts @@ -204,6 +204,16 @@ describe('variant G: the call plan, observe budget, waits and audit readers (GST expect(capy).toContain('model: { model_id: "anthropic/claude-sonnet-5-5", reasoning_mode: "low" }'); expect(capy).toContain('`anthropic/claude-haiku-5-5` also fits'); }); + + // Value: protects=a driven /ship dispatches every blocking reader on its rendered brief file unchanged, checks it with `sent`, merges findings only through `collect`, gates judgment with `judge` and verifies the run before the bind; fails_when=the driver lets the model write or trim a brief again or merge findings in prose (round 4b: arm D lost 8 blockers with 75 of 112 briefs paraphrased); + // why_new=the driver named no renderer or hash check; seam=none; no_claim=the parent's adherence, which verify checks after the fact + test('briefs go out as rendered and findings come back unchanged: render, sent, collect, judge, verify', () => { + const f = flat(doc); + for (const s of ["A blocking reader's prompt is its `gstack-review-dispatch render` file, unchanged", 'only via `collect`', '`judge` refuses a drop or downgrade', + '(4) one subagent per reader on its rendered file, then `sent` (`BRIEF_MISMATCH` voids it)', 'final output and status, via `collect`', '`JUDGMENT: ok` before any fix', + '`gstack-review-dispatch verify` (`VERIFY: clean`)']) expect(f).toContain(s); + expect(f.indexOf('gstack-review-dispatch verify')).toBeLessThan(f.indexOf('`gstack-review-read`')); + }); }); describe('STOP anchors', () => {