mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-01 22:45:02 +01:00
feat(ship): verbatim reviewer briefs and loss-free collection (gstack-review-dispatch render/sent/collect/judge/verify)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
8160c4517b
commit
ac3bd7b671
8 files changed
+640
-26
No files matched your search
Executable
+130
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env -S bun --no-env-file --config=/dev/null
|
||||
/**
|
||||
* gstack-review-dispatch — /ship's reviewers get their briefs byte for byte and the parent changes
|
||||
* nothing they return (GSTA-167 no-loss gate; docs/review-policy.md#verbatim-dispatch).
|
||||
*
|
||||
* gstack-review-dispatch render --role <role> [--name <n>] --out <dir> [slots] write the brief, print its sha256
|
||||
* gstack-review-dispatch sent --ledger <f> --role <role> [--name <n>] --sent <file> the prompt actually passed; mismatch fails
|
||||
* gstack-review-dispatch collect --ledger <f> --role <role> [--name <n>] --output <file> --merged <file>
|
||||
* gstack-review-dispatch judge --ledger <f> --merged <file> --judged <file> no drop, no downgrade
|
||||
* gstack-review-dispatch verify --ledger <f> replay every hash and the judgment
|
||||
*
|
||||
* Logic: lib/review-brief.ts (rendering from the sections) and lib/review-ledger.ts (ledger, collect, judge, verify).
|
||||
*/
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { EXIT, exitTableHelp } from '../lib/headless-artifacts';
|
||||
import { BRIEF_ROLES, BriefError, briefFile, promptHash, renderBrief, sectionTexts, type BriefRole, type Slots } from '../lib/review-brief';
|
||||
import { collect, readJsonl, readerKey, recordJudgment, recordSent, appendEvent, verify, type JudgedLine, type MergedLine } from '../lib/review-ledger';
|
||||
|
||||
const HELP = `gstack-review-dispatch — verbatim reviewer briefs and loss-free collection for /ship
|
||||
|
||||
render --role <role> [--name <n>] --out <dir> [--ledger <f>] [slots]
|
||||
roles: ${BRIEF_ROLES.join(', ')}
|
||||
slots: --diff-base <sha> (every role but fix-check) --exclude '<DIFF_CMD tail>' --stack <s> --test-fw <s>
|
||||
--learnings-file <f> --checklist <path> (default <root>/review/specialists/<name>.md; red-team.md for red-team)
|
||||
--checklists <a,b> (specialist-batched) --findings-file <f> --specialists <n> (red-team)
|
||||
--fix-list <f> --hunks <f> --tests <f> [--callers <f>] (fix-check)
|
||||
writes <out>/<role>[-<name>].prompt.md, records it in the ledger (default <out>/review-ledger.jsonl) and prints
|
||||
BRIEF: reader=<role/name> sha256=<h> bytes=<n> file=<path>. Pass that file's content as the reader's prompt, unchanged.
|
||||
sent --ledger <f> --role <role> [--name <n>] --sent <file>
|
||||
the exact prompt a dispatch passed: BRIEF_SENT: … match, or BRIEF_MISMATCH: … (exit 1; the pass is not reviewed)
|
||||
collect --ledger <f> --role <role> [--name <n>] --output <file> --merged <merged.jsonl>
|
||||
the reader's raw output: hashes it and appends each finding to merged.jsonl with its label as written
|
||||
judge --ledger <f> --merged <merged.jsonl> --judged <judged.jsonl>
|
||||
judged lines {"id","severity","reason"?,"duplicate_of"?}: every merged id kept at its severity or above (a
|
||||
duplicate folds into one at least as severe); added and promoted findings need a reason. JUDGMENT: ok …, or
|
||||
JUDGMENT_REJECTED: dropped=… downgraded=… (exit 1)
|
||||
verify --ledger <f>
|
||||
BRIEF, OUTPUT and JUDGE rows from the ledger alone, then VERIFY: clean (exit 0), VERIFY: dirty (exit 1) or
|
||||
VERIFY: no brief records (exit 3: this run's ledger kept no prompts)
|
||||
${exitTableHelp()}`;
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const flag = (n: string) => { const i = args.indexOf(n); return i >= 0 ? args[i + 1] : undefined; };
|
||||
const readFile = (f: string | undefined, what: string) => {
|
||||
if (!f) return undefined;
|
||||
try { return fs.readFileSync(f, 'utf8'); } catch { throw new BriefError(`cannot read ${what} file ${f}`); }
|
||||
};
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
|
||||
function render(): number {
|
||||
const role = flag('--role') as BriefRole | undefined;
|
||||
const out = flag('--out');
|
||||
if (!role || !BRIEF_ROLES.includes(role) || !out) { process.stderr.write(`gstack-review-dispatch render: needs --role <${BRIEF_ROLES.join('|')}> and --out <dir>\n`); return EXIT.usage; }
|
||||
const name = flag('--name');
|
||||
const diffBase = flag('--diff-base');
|
||||
const checklist = flag('--checklist') ?? (role === 'red-team' ? path.join(ROOT, 'review/specialists/red-team.md') : role === 'specialist' && name ? path.join(ROOT, 'review/specialists', `${name}.md`) : undefined);
|
||||
if (checklist && (role === 'specialist' || role === 'red-team') && !fs.existsSync(checklist)) throw new BriefError(`checklist not found: ${checklist}`);
|
||||
let diff: string | undefined;
|
||||
if (role === 'outside-voice' && diffBase) {
|
||||
const r = spawnSync('git', ['-c', 'core.quotePath=false', 'diff', '--no-color', '--no-ext-diff', diffBase], { encoding: 'utf8', timeout: 60_000, maxBuffer: 256 * 1024 * 1024 });
|
||||
if (r.status !== 0) throw new BriefError(`git diff ${diffBase} failed: ${(r.stderr ?? '').trim().split('\n')[0]}`);
|
||||
diff = r.stdout;
|
||||
}
|
||||
const slots: Slots = {
|
||||
name, diff_base: diffBase, exclude: flag('--exclude'), stack: flag('--stack'), test_fw: flag('--test-fw'),
|
||||
learnings: readFile(flag('--learnings-file'), 'learnings'), checklist, checklists: flag('--checklists'),
|
||||
findings: readFile(flag('--findings-file'), 'findings'), specialists: flag('--specialists'), diff,
|
||||
fix_list: readFile(flag('--fix-list'), 'fix list'), hunks: readFile(flag('--hunks'), 'hunks'), tests: readFile(flag('--tests'), 'tests'), callers: readFile(flag('--callers'), 'callers'),
|
||||
};
|
||||
const text = renderBrief(role, slots, sectionTexts(ROOT));
|
||||
fs.mkdirSync(out, { recursive: true, mode: 0o700 });
|
||||
const file = path.join(path.resolve(out), briefFile(role, name));
|
||||
fs.writeFileSync(file, text, { mode: 0o600 });
|
||||
const ledger = flag('--ledger') ?? path.join(out, 'review-ledger.jsonl');
|
||||
const sha = promptHash(text);
|
||||
appendEvent(ledger, { event: 'rendered', reader: readerKey(role, name), role, name: name ?? null, file, sha256: sha, bytes: Buffer.byteLength(text) });
|
||||
process.stdout.write(`BRIEF: reader=${readerKey(role, name)} sha256=${sha} bytes=${Buffer.byteLength(text)} file=${file}\n`);
|
||||
return EXIT.ok;
|
||||
}
|
||||
|
||||
function main(): number {
|
||||
const cmd = args[0];
|
||||
if (!cmd || cmd === '--help' || cmd === '-h') { process.stdout.write(HELP + '\n'); return cmd ? EXIT.ok : EXIT.usage; }
|
||||
if (cmd === 'render') return render();
|
||||
const ledger = flag('--ledger');
|
||||
if (!ledger) { process.stderr.write(`gstack-review-dispatch ${cmd}: --ledger <file> is required\n`); return EXIT.usage; }
|
||||
const reader = flag('--role') ? readerKey(flag('--role')!, flag('--name')) : '';
|
||||
if (cmd === 'sent') {
|
||||
const sent = readFile(flag('--sent'), 'sent prompt');
|
||||
if (!reader || sent === undefined) { process.stderr.write('gstack-review-dispatch sent: needs --role and --sent <file>\n'); return EXIT.usage; }
|
||||
const r = recordSent(ledger, reader, sent);
|
||||
process.stdout.write(r.ok ? `BRIEF_SENT: reader=${reader} sha256=${r.sent} match\n`
|
||||
: `BRIEF_MISMATCH: reader=${reader} rendered=${r.rendered ?? 'none'} sent=${r.sent} (dispatch the rendered file unchanged; this reader's review does not count)\n`);
|
||||
return r.ok ? EXIT.ok : EXIT.fail;
|
||||
}
|
||||
if (cmd === 'collect') {
|
||||
const outFile = flag('--output');
|
||||
const output = readFile(outFile, 'reader output');
|
||||
const merged = flag('--merged');
|
||||
if (!reader || output === undefined || !merged) { process.stderr.write('gstack-review-dispatch collect: needs --role, --output <file> and --merged <file>\n'); return EXIT.usage; }
|
||||
const r = collect(ledger, reader, output, merged, path.resolve(outFile!));
|
||||
process.stdout.write(`COLLECT: reader=${reader} findings=${r.lines.length} sha256=${r.sha} merged=${merged}\n`);
|
||||
return EXIT.ok;
|
||||
}
|
||||
if (cmd === 'judge') {
|
||||
const merged = flag('--merged');
|
||||
const judged = flag('--judged');
|
||||
if (!merged || !judged) { process.stderr.write('gstack-review-dispatch judge: needs --merged <file> and --judged <file>\n'); return EXIT.usage; }
|
||||
const r = recordJudgment(ledger, readJsonl<MergedLine>(merged), readJsonl<JudgedLine>(judged), judged);
|
||||
process.stdout.write(r.ok ? `JUDGMENT: ok added=${r.added.length} promoted=${r.promoted.length} folded=${r.folded.length}\n`
|
||||
: `JUDGMENT_REJECTED: dropped=${JSON.stringify(r.dropped)} downgraded=${JSON.stringify(r.downgraded)} unexplained=${JSON.stringify(r.unexplained)} invalid=${JSON.stringify(r.invalid)} (lead judgment may add or promote a finding, never drop or downgrade one)\n`);
|
||||
return r.ok ? EXIT.ok : EXIT.fail;
|
||||
}
|
||||
if (cmd === 'verify') {
|
||||
const v = verify(ledger);
|
||||
for (const r of v.rows) process.stdout.write(`${r.kind} ${r.reader} ${r.status} ${r.detail}\n`);
|
||||
if (!v.briefs) { process.stdout.write('VERIFY: no brief records (this run\'s ledger kept no prompts; dispatched-vs-rendered cannot be checked)\n'); return EXIT.refused; }
|
||||
process.stdout.write(`VERIFY: ${v.clean ? 'clean' : 'dirty'} briefs=${v.briefs} outputs=${v.rows.filter(r => r.kind === 'OUTPUT').length}\n`);
|
||||
return v.clean ? EXIT.ok : EXIT.fail;
|
||||
}
|
||||
process.stderr.write(`gstack-review-dispatch: unknown command ${cmd}\n${HELP}\n`);
|
||||
return EXIT.usage;
|
||||
}
|
||||
|
||||
try { process.exit(main()); } catch (e) {
|
||||
if (e instanceof BriefError) { process.stderr.write(`BRIEF_RENDER_FAILED: ${e.message}\n`); process.exit(EXIT.fail); }
|
||||
throw e;
|
||||
}
|
||||
@@ -357,8 +357,39 @@ email case-folded, so running it twice changes nothing. The `COAUTHORS: <n> in
|
||||
<range>` line on stderr is the receipt; `--json` prints `{ range, authors,
|
||||
trailers, appended }`.
|
||||
|
||||
## Verbatim dispatch
|
||||
|
||||
A driven `/ship` (ship/DRIVER.md) never lets the model write a reviewer's brief
|
||||
or merge a reviewer's findings. The no-loss replays showed why: with briefs the
|
||||
parent filled in by hand, only 37 of 112 dispatched prompts matched the rendered
|
||||
ones, and the arm lost blockers. `bin/gstack-review-dispatch` closes both ends:
|
||||
|
||||
1. `render --role <role> [--name <n>] --out <dir>` cuts the brief out of
|
||||
`ship/sections/review-army.md` or `ship/sections/adversarial.md` (the fix-check
|
||||
from `scripts/resolvers/fix-check.ts`) and fills only declared slots: the
|
||||
literal diff base, the plan's `DIFF_CMD` excludes, stack, test framework,
|
||||
learnings, checklist paths, the Red Team's merged findings and specialist count,
|
||||
and the outside voice's branch diff. A section that lost an anchor or placeholder
|
||||
refuses (`BRIEF_RENDER_FAILED`) instead of rendering something else.
|
||||
2. The dispatch program passes that file's content unchanged as the reader's
|
||||
prompt and runs `sent` on what it passed: a hash that differs from the rendered
|
||||
one is `BRIEF_MISMATCH`, and that reader's review does not count.
|
||||
3. `collect` hashes each reader's raw output and appends its findings to
|
||||
`merged.jsonl` with the label exactly as written (a JSON line's `severity`, or a
|
||||
prose line's Critical, High, Medium, Low, P0-P3, FIXABLE or INVESTIGATE).
|
||||
4. Lead judgment writes `judged.jsonl`; `judge` refuses a set that drops a merged
|
||||
finding, lowers its severity, folds it into a less severe duplicate, or adds or
|
||||
promotes one without a reason (`JUDGMENT_REJECTED`). Each addition and promotion
|
||||
is logged with its reason.
|
||||
5. `verify --ledger <f>` replays all of it from the run's ledger: rendered against
|
||||
dispatched hashes, output hashes against the merged lines they produced, and the
|
||||
judgment verdict. A ledger from a run that kept no prompts prints
|
||||
`VERIFY: no brief records` (exit 3), never clean.
|
||||
|
||||
## Where this is enforced
|
||||
|
||||
- `lib/review-brief.ts`, `lib/review-ledger.ts`, `bin/gstack-review-dispatch`: [verbatim dispatch](#verbatim-dispatch) (`test/review-dispatch.test.ts`).
|
||||
|
||||
- `scripts/resolvers/review-army.ts`: the floor and `NO FINDINGS` in every specialist and Red Team prompt (rendered into `review/sections/review-army.md` and `ship/sections/review-army.md`), and the lead judgment's "On record" line that turns a [ruled class](#rulings-below-the-floor) into a note; in `/ship` every specialist and Red Team prompt it renders carries its diff base as a literal sha the parent writes in (`DIFF_BASE=<diff-base>`: the merge base on pass 1, `RANGE_BASE` from pass 2 on) beside its plan's `DIFF_CMD` excludes, because a reviewer is its own process and never sees the parent's shell variables; the fan-out batches above.
|
||||
- `ship/sections/review-army.md` Step 9: the plan's `REVIEW_DEPTH`, `GATE_SCAN` and `--depth-floor` on every run, the depth fields of the review-log row, passes, `RANGE_BASE`, classify, defer, render, the cap check before any edit, the Step 11 late-defer row, scratch files under `.gstack/tmp/`, the cap decisions; `ship/sections/adversarial.md` Step 11: the outside voice reads the same range and scopes queued fixes the same way.
|
||||
- `lib/review-depth.ts`, `bin/gstack-review-plan`: the diff class, the depth (`depthFor`, `DEPTH_RULES`, `depthOfCommitInput`), each dispatch's `exclude` pathspecs and `DIFF_CMD`, the transition table, the numeric-claim scan and each role's effort tier (`test/review-depth.test.ts`, `test/review-plan-cli.test.ts`).
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
/**
|
||||
* review-brief — every /ship reviewer brief rendered from its section's own bytes (GSTA-167
|
||||
* no-loss gate). The text comes from ship/sections/review-army.md and ship/sections/adversarial.md
|
||||
* at render time, cut out between fixed anchors; only the declared slots below are filled. The
|
||||
* dispatch program passes the rendered file verbatim as the reader's prompt, and
|
||||
* lib/review-ledger.ts proves it did by hash. Nothing here is composed by the model.
|
||||
*
|
||||
* Slots by role:
|
||||
* specialist name, checklist, diff_base, exclude, stack, test_fw, learnings (security adds the defensive framing)
|
||||
* specialist-batched checklists (comma list), diff_base, exclude, stack, test_fw, learnings
|
||||
* red-team checklist, diff_base, findings (the merged specialist lines), specialists (their count)
|
||||
* adversarial-native diff_base
|
||||
* outside-voice diff_base, diff (the branch diff, which the bin reads from git)
|
||||
* fix-check fix_list, hunks, tests, callers (scripts/resolvers/fix-check.ts)
|
||||
*/
|
||||
import { createHash } from 'node:crypto';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { renderFixCheckPrompt } from '../scripts/resolvers/fix-check';
|
||||
|
||||
export const BRIEF_ROLES = ['specialist', 'specialist-batched', 'red-team', 'adversarial-native', 'outside-voice', 'fix-check'] as const;
|
||||
export type BriefRole = typeof BRIEF_ROLES[number];
|
||||
export type Slots = Record<string, string | undefined>;
|
||||
|
||||
/** The hash every brief and every dispatched prompt is compared by: sha256 of the text without trailing whitespace. */
|
||||
export function promptHash(text: string): string {
|
||||
return createHash('sha256').update(text.replace(/\r\n/g, '\n').replace(/\s+$/, '')).digest('hex');
|
||||
}
|
||||
|
||||
export class BriefError extends Error {}
|
||||
|
||||
/** The text strictly between start and end, end searched after start; both anchors are required. */
|
||||
function between(text: string, start: string, end: string, what: string): string {
|
||||
const i = text.indexOf(start);
|
||||
if (i < 0) throw new BriefError(`${what}: start anchor not found in the section (${JSON.stringify(start.slice(0, 40))})`);
|
||||
const j = text.indexOf(end, i + start.length);
|
||||
if (j < 0) throw new BriefError(`${what}: end anchor not found in the section (${JSON.stringify(end.slice(0, 40))})`);
|
||||
return text.slice(i + start.length, j);
|
||||
}
|
||||
/** A quoted prompt body as the reader receives it: the section escapes its inner quotes and line breaks. */
|
||||
const unquote = (s: string) => s.replace(/\\"/g, '"').replace(/\\n/g, '\n');
|
||||
|
||||
export interface SectionTexts { specialist: string; framing: string; fixtures: string; redTeam: string; native: string; outside: string }
|
||||
|
||||
/** Cut every brief body out of the installed sections under root (the directory holding ship/). */
|
||||
export function sectionTexts(root: string): SectionTexts {
|
||||
const read = (rel: string) => {
|
||||
try { return fs.readFileSync(path.join(root, rel), 'utf8').replace(/\r\n/g, '\n'); } catch { throw new BriefError(`cannot read ${rel} under ${root}`); }
|
||||
};
|
||||
const army = read('ship/sections/review-army.md');
|
||||
const adv = read('ship/sections/adversarial.md');
|
||||
const specialist = unquote(between(army, '\n"You are a specialist code reviewer.', 'Past learnings: {learnings or \'none\'}"', 'specialist brief'));
|
||||
return {
|
||||
specialist: `You are a specialist code reviewer.${specialist}Past learnings: {learnings or 'none'}`
|
||||
.replace(/ \(`<diff-base>`: write the sha in before dispatch[\s\S]*?on `DIFF_EXCLUDE: none`\)/, ''),
|
||||
framing: unquote(between(army, 'Begin the prompt with:\n"', '"\nand replace its full-diff command', 'defensive framing')),
|
||||
fixtures: unquote(between(army, 'and replace its full-diff command with: "', '"\nEvery other specialist', 'fixture handling')),
|
||||
redTeam: unquote(between(army, 'fixture handling from item 4 of the specialist dispatch: "', 'don\'t cover."', 'red-team brief')) + 'don\'t cover.',
|
||||
native: unquote(between(adv, 'Subagent prompt:\n"', '"\n\nPresent findings', 'adversarial-native brief')),
|
||||
outside: unquote(between(adv, 'Outside prompt (supply repository context from the parent):\n\n"', '"\n\nWrite the **complete prompt and context**', 'outside-voice brief')),
|
||||
};
|
||||
}
|
||||
|
||||
const SHA = /^[0-9a-f]{7,40}$/;
|
||||
const FENCE = '`'.repeat(3);
|
||||
function need(slots: Slots, key: string): string {
|
||||
const v = slots[key];
|
||||
if (v === undefined || v === '') throw new BriefError(`missing slot ${key}`);
|
||||
return v;
|
||||
}
|
||||
function fill(text: string, pairs: Array<[string, string]>, what: string): string {
|
||||
let out = text;
|
||||
for (const [from, to] of pairs) {
|
||||
if (!out.includes(from)) throw new BriefError(`${what}: placeholder ${from} is not in the section text (the section changed; update lib/review-brief.ts)`);
|
||||
out = out.split(from).join(to);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
const SPECIALIST_DIFF = 'run\n`DIFF_BASE=<diff-base> && git diff "$DIFF_BASE" <exclude>` to get the diff';
|
||||
|
||||
const PLACEHOLDERS = ['<diff-base>', '<exclude>', '<base>', '{checklist path}', '{STACK}', '{TEST_FW}', '{merged findings summary}', '{red-team checklist path}', '{learnings'];
|
||||
|
||||
/** Render one reader's brief. Throws BriefError on a missing slot, a section that no longer carries an anchor or placeholder, or a placeholder left unfilled. */
|
||||
export function renderBrief(role: BriefRole, slots: Slots, t: SectionTexts): string {
|
||||
const text = renderRole(role, slots, t);
|
||||
const left = role === 'fix-check' || role === 'outside-voice' ? [] : PLACEHOLDERS.filter(p => text.includes(p));
|
||||
if (left.length) throw new BriefError(`${role}: unfilled placeholder ${left.join(', ')}`);
|
||||
return text;
|
||||
}
|
||||
|
||||
function renderRole(role: BriefRole, slots: Slots, t: SectionTexts): string {
|
||||
const diffBase = role === 'fix-check' ? '' : need(slots, 'diff_base');
|
||||
if (diffBase && !SHA.test(diffBase)) throw new BriefError(`slot diff_base must be a literal commit sha, got ${JSON.stringify(diffBase)}`);
|
||||
const common = (body: string, checklist: string) => fill(body, [
|
||||
...(checklist ? [['{checklist path}', checklist] as [string, string]] : []), ['<diff-base>', diffBase],
|
||||
['{TEST_FW}', slots.test_fw || 'unknown'], ['{STACK}', slots.stack || 'unknown'], ['{learnings or \'none\'}', slots.learnings?.trim() || 'none'],
|
||||
], role).split(' <exclude>`').join(slots.exclude ? ' ' + slots.exclude + '`' : '`');
|
||||
switch (role) {
|
||||
case 'specialist': {
|
||||
const name = need(slots, 'name');
|
||||
const checklist = need(slots, 'checklist');
|
||||
if (name !== 'security') return common(t.specialist, checklist);
|
||||
const body = fill(t.specialist, [[SPECIALIST_DIFF, 'set `DIFF_BASE=<diff-base>` first. ' + t.fixtures + ' Then read the diff that way']], role);
|
||||
return `${t.framing}\n\n${common(body, checklist)}`;
|
||||
}
|
||||
case 'specialist-batched': {
|
||||
const lists = need(slots, 'checklists').split(',').map(s => s.trim()).filter(Boolean);
|
||||
const body = fill(t.specialist, [['Read the checklist at {checklist path}', `Read each checklist at ${lists.join(', ')} (tag each finding's "specialist" with the name of the checklist it came from)`]], role);
|
||||
return common(body, '');
|
||||
}
|
||||
case 'red-team': {
|
||||
const body = fill(t.redTeam, [['{merged findings summary}', (slots.findings ?? '').trim() || '(none)'], ['by N specialists', `by ${need(slots, 'specialists')} specialists`],
|
||||
['{red-team checklist path}', need(slots, 'checklist')], ['<diff-base>', diffBase]], role);
|
||||
return `${t.framing}\n\n${t.fixtures}\n\n${body}`;
|
||||
}
|
||||
case 'adversarial-native':
|
||||
return fill(t.native, [['DIFF_BASE=$(git merge-base origin/<base> HEAD)', `DIFF_BASE=${diffBase}`]], role);
|
||||
case 'outside-voice': {
|
||||
const body = fill(t.outside, [['DIFF_BASE=$(git merge-base origin/<base> HEAD)', `DIFF_BASE=${diffBase}`]], role);
|
||||
return [body, '', 'Give each finding a severity: Critical, High, Medium or Low.', '', `## Branch diff (git diff ${diffBase})`, FENCE + 'diff', (slots.diff ?? '').trim(), FENCE].join('\n');
|
||||
}
|
||||
case 'fix-check':
|
||||
return renderFixCheckPrompt({ fixList: need(slots, 'fix_list'), touchedHunks: need(slots, 'hunks'), testResults: need(slots, 'tests'), callers: slots.callers ?? '' });
|
||||
}
|
||||
}
|
||||
|
||||
/** The file a rendered brief is written to under the run's briefs directory. */
|
||||
export function briefFile(role: BriefRole, name?: string): string {
|
||||
return `${role}${name ? `-${name.replace(/[^a-zA-Z0-9._-]/g, '_')}` : ''}.prompt.md`;
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
/**
|
||||
* review-ledger — the proof that /ship's reviewers got their briefs unchanged and that the
|
||||
* parent changed nothing they returned (GSTA-167 no-loss gate). One JSONL ledger per run:
|
||||
*
|
||||
* rendered a brief gstack-review-dispatch render wrote (role, name, file, sha256)
|
||||
* sent the prompt a dispatch program actually passed, by hash; a mismatch fails the pass
|
||||
* output a reader's raw output (sha256) and the merged.jsonl ids collect wrote from it
|
||||
* judgment each lead-judgment change: an added or promoted finding, with its reason
|
||||
* judge the verdict on the parent's judged set (ok, or the dropped and downgraded ids)
|
||||
*
|
||||
* Findings reach merged.jsonl only through collect, with each voice's label exactly as written.
|
||||
* The parent's judged set may add or promote a finding and may fold a duplicate into one at least
|
||||
* as severe; it never drops or downgrades one. verify replays every check from the ledger alone.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import { promptHash } from './review-brief';
|
||||
|
||||
export interface LedgerEvent { event: 'rendered' | 'sent' | 'output' | 'judgment' | 'judge'; [k: string]: unknown }
|
||||
export const readerKey = (role: string, name?: string) => (name ? `${role}/${name}` : role);
|
||||
|
||||
export function appendEvent(ledger: string, e: LedgerEvent): void {
|
||||
fs.appendFileSync(ledger, JSON.stringify({ ...e, at: new Date().toISOString() }) + '\n', { mode: 0o600 });
|
||||
}
|
||||
export function readLedger(ledger: string): LedgerEvent[] {
|
||||
let text: string;
|
||||
try { text = fs.readFileSync(ledger, 'utf8'); } catch { return []; }
|
||||
return text.split('\n').filter(l => l.trim()).flatMap(l => { try { return [JSON.parse(l) as LedgerEvent]; } catch { return []; } });
|
||||
}
|
||||
const lastRendered = (events: LedgerEvent[], reader: string) => [...events].reverse().find(e => e.event === 'rendered' && e.reader === reader);
|
||||
|
||||
/** Compare a dispatched prompt with the brief rendered for that reader; records the sent event. */
|
||||
export function recordSent(ledger: string, reader: string, sentText: string): { ok: boolean; rendered: string | null; sent: string } {
|
||||
const r = lastRendered(readLedger(ledger), reader);
|
||||
const sent = promptHash(sentText);
|
||||
const rendered = typeof r?.sha256 === 'string' ? r.sha256 : null;
|
||||
const ok = rendered === sent;
|
||||
appendEvent(ledger, { event: 'sent', reader, sha256: sent, rendered, match: ok });
|
||||
return { ok, rendered, sent };
|
||||
}
|
||||
|
||||
/** Severity ranks across the voices' vocabularies; a label with no severity (FIXABLE, INVESTIGATE) ranks 0. */
|
||||
const RANKS: Record<string, number> = { critical: 4, p0: 4, high: 3, p1: 3, medium: 2, p2: 2, informational: 2, low: 1, p3: 1, fixable: 0, investigate: 0 };
|
||||
export function rank(label: string): number | null {
|
||||
const r = RANKS[label.trim().replace(/^\[|\]$/g, '').toLowerCase()];
|
||||
return r === undefined ? null : r;
|
||||
}
|
||||
const PROSE_LABEL = /(?:^|[\s[(*|:-])(CRITICAL|INFORMATIONAL|Critical|High|Medium|Low|P[0-3]|FIXABLE|INVESTIGATE)(?=$|[\s\]):*|,.-])/;
|
||||
|
||||
export interface MergedLine { id: string; reader: string; label: string; raw: string; output_sha256: string }
|
||||
|
||||
/** One reader's findings as written: a JSON object per line (severity kept as written), else each prose line carrying a severity or FIXABLE/INVESTIGATE label. */
|
||||
export function parseFindings(reader: string, output: string): Array<{ label: string; raw: string }> {
|
||||
const out: Array<{ label: string; raw: string }> = [];
|
||||
for (const line of output.replace(/\r\n/g, '\n').split('\n')) {
|
||||
const t = line.trim();
|
||||
if (!t || /^NO FINDINGS\.?$/.test(t) || /^Recommendation:/i.test(t)) continue;
|
||||
if (t.startsWith('{')) {
|
||||
try {
|
||||
const o = JSON.parse(t) as Record<string, unknown>;
|
||||
if (typeof o.severity === 'string') { out.push({ label: o.severity, raw: t }); continue; }
|
||||
} catch { /* prose that starts with a brace */ }
|
||||
}
|
||||
const m = PROSE_LABEL.exec(t);
|
||||
if (m) out.push({ label: m[1]!, raw: t });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** collect: hash the raw output, append its findings to merged.jsonl and record both. */
|
||||
export function collect(ledger: string, reader: string, output: string, mergedFile: string, outputFile?: string): { sha: string; lines: MergedLine[] } {
|
||||
const sha = promptHash(output);
|
||||
const prior = readLedger(ledger).filter(e => e.event === 'output' && e.reader === reader).length;
|
||||
const lines = parseFindings(reader, output).map((f, i) => ({ id: `${reader}#${prior ? `${prior + 1}.` : ''}${i + 1}`, reader, label: f.label, raw: f.raw, output_sha256: sha }));
|
||||
fs.appendFileSync(mergedFile, lines.map(l => JSON.stringify(l) + '\n').join(''), { mode: 0o600 });
|
||||
appendEvent(ledger, { event: 'output', reader, sha256: sha, file: outputFile ?? null, merged: mergedFile, ids: lines.map(l => l.id) });
|
||||
return { sha, lines };
|
||||
}
|
||||
|
||||
export function readJsonl<T>(file: string): T[] {
|
||||
return fs.readFileSync(file, 'utf8').split('\n').filter(l => l.trim()).map(l => JSON.parse(l) as T);
|
||||
}
|
||||
|
||||
export interface JudgedLine { id: string; severity: string; reason?: string; duplicate_of?: string }
|
||||
export interface Judgment { ok: boolean; dropped: string[]; downgraded: string[]; unexplained: string[]; invalid: string[]; added: string[]; promoted: string[]; folded: string[] }
|
||||
|
||||
/** The lead-judgment rule: every merged finding survives at its severity or above; additions and promotions carry a reason. */
|
||||
export function judge(merged: MergedLine[], judged: JudgedLine[]): Judgment {
|
||||
const byId = new Map(judged.map(j => [j.id, j]));
|
||||
const mergedIds = new Set(merged.map(m => m.id));
|
||||
const res: Judgment = { ok: true, dropped: [], downgraded: [], unexplained: [], invalid: [], added: [], promoted: [], folded: [] };
|
||||
for (const j of judged) if (rank(j.severity) === null) res.invalid.push(j.id);
|
||||
for (const m of merged) {
|
||||
const j = byId.get(m.id);
|
||||
if (!j) { res.dropped.push(m.id); continue; }
|
||||
const target = j.duplicate_of ? byId.get(j.duplicate_of) : j;
|
||||
if (!target) { res.dropped.push(m.id); continue; }
|
||||
const from = rank(m.label) ?? 0;
|
||||
const to = rank(target.severity);
|
||||
if (to === null) continue;
|
||||
if (to < from) res.downgraded.push(m.id);
|
||||
else if (j.duplicate_of) res.folded.push(m.id);
|
||||
else if (to > from) { res.promoted.push(m.id); if (!j.reason?.trim()) res.unexplained.push(m.id); }
|
||||
}
|
||||
for (const j of judged) if (!mergedIds.has(j.id) && !j.duplicate_of) { res.added.push(j.id); if (!j.reason?.trim()) res.unexplained.push(j.id); }
|
||||
res.ok = !res.dropped.length && !res.downgraded.length && !res.unexplained.length && !res.invalid.length;
|
||||
return res;
|
||||
}
|
||||
|
||||
/** judge, logged: one judgment event per addition and promotion, then the verdict. */
|
||||
export function recordJudgment(ledger: string, merged: MergedLine[], judged: JudgedLine[], judgedFile: string): Judgment {
|
||||
const res = judge(merged, judged);
|
||||
const byId = new Map(judged.map(j => [j.id, j]));
|
||||
const labels = new Map(merged.map(m => [m.id, m.label]));
|
||||
if (res.ok) {
|
||||
for (const id of res.added) appendEvent(ledger, { event: 'judgment', kind: 'added', id, to: byId.get(id)!.severity, reason: byId.get(id)!.reason });
|
||||
for (const id of res.promoted) appendEvent(ledger, { event: 'judgment', kind: 'promoted', id, from: labels.get(id), to: byId.get(id)!.severity, reason: byId.get(id)!.reason });
|
||||
}
|
||||
appendEvent(ledger, { event: 'judge', ok: res.ok, judged: judgedFile, dropped: res.dropped, downgraded: res.downgraded, unexplained: res.unexplained, invalid: res.invalid, added: res.added.length, promoted: res.promoted.length, folded: res.folded.length });
|
||||
return res;
|
||||
}
|
||||
|
||||
export interface VerifyRow { kind: 'BRIEF' | 'OUTPUT' | 'JUDGE'; reader: string; status: string; detail: string }
|
||||
|
||||
/** Replay every check from a finished run's ledger: briefs dispatched as rendered, outputs merged as returned, judgment without loss. */
|
||||
export function verify(ledger: string): { rows: VerifyRow[]; clean: boolean; briefs: number } {
|
||||
const events = readLedger(ledger);
|
||||
const rows: VerifyRow[] = [];
|
||||
const short = (s: unknown) => (typeof s === 'string' ? s.slice(0, 12) : '-');
|
||||
const readers = [...new Set(events.filter(e => e.event === 'rendered').map(e => String(e.reader)))];
|
||||
for (const reader of readers) {
|
||||
const r = lastRendered(events, reader)!;
|
||||
const sent = events.filter(e => e.event === 'sent' && e.reader === reader);
|
||||
let fileNow = '';
|
||||
try { fileNow = typeof r.file === 'string' && fs.existsSync(r.file) ? promptHash(fs.readFileSync(r.file, 'utf8')) : ''; } catch { /* unreadable */ }
|
||||
const status = !sent.length ? 'NOT_SENT' : sent.every(s => s.sha256 === r.sha256) ? 'MATCH' : 'MISMATCH';
|
||||
const tampered = fileNow && fileNow !== r.sha256 ? ' file_changed_since_render' : '';
|
||||
rows.push({ kind: 'BRIEF', reader, status: tampered ? 'MISMATCH' : status, detail: `rendered=${short(r.sha256)} sent=${sent.map(s => short(s.sha256)).join(',') || '-'}${tampered}` });
|
||||
}
|
||||
for (const o of events.filter(e => e.event === 'output')) {
|
||||
const reader = String(o.reader);
|
||||
let status = 'MATCH';
|
||||
const notes: string[] = [];
|
||||
if (typeof o.file === 'string' && fs.existsSync(o.file) && promptHash(fs.readFileSync(o.file, 'utf8')) !== o.sha256) { status = 'MISMATCH'; notes.push('output_file_changed'); }
|
||||
const ids = Array.isArray(o.ids) ? (o.ids as string[]) : [];
|
||||
let lines: MergedLine[] = [];
|
||||
try { lines = typeof o.merged === 'string' ? readJsonl<MergedLine>(o.merged).filter(l => ids.includes(l.id)) : []; } catch { notes.push('merged_unreadable'); status = 'MISMATCH'; }
|
||||
if (lines.length !== ids.length) { status = 'MISMATCH'; notes.push(`merged_lines=${lines.length}/${ids.length}`); }
|
||||
if (lines.some(l => l.output_sha256 !== o.sha256)) { status = 'MISMATCH'; notes.push('merged_line_from_other_output'); }
|
||||
rows.push({ kind: 'OUTPUT', reader, status, detail: `sha=${short(o.sha256)} findings=${ids.length}${notes.length ? ' ' + notes.join(' ') : ''}` });
|
||||
}
|
||||
const verdicts = events.filter(e => e.event === 'judge');
|
||||
if (verdicts.length) {
|
||||
const v = verdicts[verdicts.length - 1]!;
|
||||
rows.push({ kind: 'JUDGE', reader: '-', status: v.ok ? 'OK' : 'REJECTED', detail: `dropped=${JSON.stringify(v.dropped)} downgraded=${JSON.stringify(v.downgraded)} added=${v.added} promoted=${v.promoted} folded=${v.folded}` });
|
||||
} else if (events.some(e => e.event === 'output')) rows.push({ kind: 'JUDGE', reader: '-', status: 'MISSING', detail: 'outputs were collected but no judged set was checked' });
|
||||
const clean = readers.length > 0 && rows.every(r => r.status === 'MATCH' || r.status === 'OK');
|
||||
return { rows, clean, briefs: readers.length };
|
||||
}
|
||||
+14
-13
@@ -7,7 +7,7 @@ Read this instead of `ship/SKILL.md` when a parent agent on any host runs /ship
|
||||
the `GSTACK_LAUNCH` assignments your installer printed (`. <the GSTACK_LAUNCH_FILE it printed>`), never
|
||||
`export $(...)`, `env`, `printenv` or a bare `set` in a tool fence.
|
||||
|
||||
**Rules.** Each phase below is one call: one program (an execute program on Capy, else a shell
|
||||
**Rules.** Each phase below is one call: one program (Capy: an execute program; else a shell
|
||||
script) that runs the bins, reads, dispatches and waits, returning only result lines and reader
|
||||
outputs; print them verbatim. The model keeps only lead judgment, fixes, CHANGELOG and PR prose, the
|
||||
gates and the report. `STOP:` (exit 3) means nothing after it ran; `NEXT:` names
|
||||
@@ -24,32 +24,32 @@ version, CHANGELOG, commits), **Documentation** (Step 14.5) and **Next steps** (
|
||||
**Reviewers are the sections' own.** Every audit, specialist, Red Team, QA, fix-check, adversarial,
|
||||
outside-voice and documentation brief, its severity and confidence rules and its gates come from its
|
||||
section verbatim, with only its placeholders filled. This file restates none of them and
|
||||
never shortens a reviewer's brief. Audit readers (Steps 7, 8, 9.2.1, 14.5) cannot block, so each runs on a
|
||||
never shortens a reviewer's brief. A blocking reader's prompt is its `gstack-review-dispatch render`
|
||||
file, unchanged; its findings reach `merged.jsonl` only via `collect`; `judge` refuses a drop or downgrade. Audit readers (Steps 7, 8, 9.2.1, 14.5) cannot block, so each runs on a
|
||||
small model at low effort; their findings go to the PR body.
|
||||
|
||||
**Calls.** One pass with no fixes is 9 calls: Phases 1-8 and the report; at most 12 with a STOP's
|
||||
section read or a fix-check. A pass that fixes something repeats Phases 3-4. Waits add pings, not work.
|
||||
section read or a fix-check. A pass that fixes something repeats Phases 3-4.
|
||||
|
||||
**Budget: observe.** `gstack-ship-budget start --mode observe` records, never admits: each program ends
|
||||
with `gstack-ship-budget note --step <last step> --reading R --credits C`, R and C read in that program
|
||||
(Capy: `cost.tokens` and `cost.credits` of your own jam). No check per step, no reservation, no meter-only
|
||||
(Capy: your own jam's `cost.tokens` and `cost.credits`). No check per step, no reservation, no meter-only
|
||||
call, no handoff; under `--token-cap` a note past the cap prints `verdict=stop`, the one stop check.
|
||||
A program that dispatches readers takes `BEFORE` (R and C) right before they start and passes
|
||||
A dispatching program takes `BEFORE` (R and C) as its readers start and passes
|
||||
`--dispatched-at <R> --dispatched-credits <C> --readers <n>` with the reading after the last settles.
|
||||
|
||||
**Waits.** No call waits past 240 s: a program polls or sleeps at most 240 s and returns what still
|
||||
runs; the next call resumes (Capy: `wait` 240 s on the operation). A call after a gap over 5 minutes
|
||||
bills the whole context fresh (about 40x a cached call); a ping costs a few percent of that. A small
|
||||
waiting child saves nothing: the parent's cache expires during its wait too. Past 100 minutes, stop.
|
||||
**Waits.** No call waits past 240 s: a program returns whatever still runs at 240 s; the next call resumes (Capy: `wait` 240 s on it). A small
|
||||
waiting child saves nothing. Past 100 minutes, stop.
|
||||
|
||||
**One program per pass.** Each review pass, Step 11 and the Step 14.5 audit is one program, and the
|
||||
parent makes no other call while it runs: (1) `gstack-review-plan --base <base>` with the pass's
|
||||
flags (its `DISPATCH:` lines; Step 14.5 has one reader, `docs`); (2) `gstack-review-log --start
|
||||
<skill>` once per row the step logs (Step 14.5 takes none); (3) `BEFORE`; (4) one subagent per reader;
|
||||
<skill>` once per row the step logs (Step 14.5 takes none); (3) `BEFORE`; (4) one subagent per reader on its rendered file, then
|
||||
`sent` (`BRIEF_MISMATCH` voids it);
|
||||
(5) a poll until every reader settles (status `done` or `failed`, or a final summary; a fresh `idle`
|
||||
has not run); (6) `note` with the reading pair; a later batch (the Red Team,
|
||||
given the specialists' collected lines) repeats (3)-(6); (7) the collect: return only each reader's
|
||||
final output and status. The next call applies the section's merge, judgment, fixes and persistence;
|
||||
final output and status, via `collect`. The next call applies the section's merge, judgment, fixes and persistence;
|
||||
it never walks the dispatch steps call by call.
|
||||
|
||||
## 1. Load: Step 0
|
||||
@@ -85,7 +85,8 @@ reader ran.
|
||||
## 4. Judgment: Steps 8.1, 8.2, 9.3, 9.4 and Step 11
|
||||
|
||||
Steps 8.1 and 8.2 (`Scope Check:`) on the todos and commits files, then the core checklist, Steps
|
||||
9.3, 9.4 and the repeat decision as written (`gstack-review-plan` included), fixes and persistence in
|
||||
9.3, 9.4 and the repeat decision as written (`gstack-review-plan` included;
|
||||
`JUDGMENT: ok` before any fix), fixes and persistence in
|
||||
one program. Prints `Pre-Landing Review:`, `Gate edits:`. `NEXT: stop` stops; `full` or
|
||||
`scoped` reruns Step 5 (`--tests-only`) and Phases 3-4; `done`: the same program runs Step 11 from
|
||||
`sections/adversarial.md` as written (above), only the voices the last full-pass plan lists (neither
|
||||
@@ -94,7 +95,7 @@ prints `ADVERSARIAL: skipped (review plan)`).
|
||||
## 5. Voices, bind and version: Steps 11, 11.5, 12
|
||||
|
||||
Judge the voices; missing outside coverage is `unavailable`, never clean; queued fixes rerun Phases
|
||||
3-4. Then `gstack-review-read`: the final Step 9 row and Step 11 native row (or `CURRENT` freshness
|
||||
3-4. Then `gstack-review-dispatch verify` (`VERIFY: clean`) and `gstack-review-read`: the final Step 9 row and Step 11 native row (or `CURRENT` freshness
|
||||
after a planned skip) agree on `wtree`, `start_wtree`, `end_wtree`; save that reviewed tree. Then
|
||||
`gstack-ship-release version --base <base> [--bump micro|patch] --why '<line>'`; MINOR or MAJOR is a
|
||||
question; `VERSION: skipped (policy: ...)`: write no version.
|
||||
|
||||
+14
-13
@@ -5,7 +5,7 @@ Read this instead of `ship/SKILL.md` when a parent agent on any host runs /ship
|
||||
the `GSTACK_LAUNCH` assignments your installer printed (`. <the GSTACK_LAUNCH_FILE it printed>`), never
|
||||
`export $(...)`, `env`, `printenv` or a bare `set` in a tool fence.
|
||||
|
||||
**Rules.** Each phase below is one call: one program (an execute program on Capy, else a shell
|
||||
**Rules.** Each phase below is one call: one program (Capy: an execute program; else a shell
|
||||
script) that runs the bins, reads, dispatches and waits, returning only result lines and reader
|
||||
outputs; print them verbatim. The model keeps only lead judgment, fixes, CHANGELOG and PR prose, the
|
||||
gates and the report. `STOP:` (exit 3) means nothing after it ran; `NEXT:` names
|
||||
@@ -22,32 +22,32 @@ version, CHANGELOG, commits), **Documentation** (Step 14.5) and **Next steps** (
|
||||
**Reviewers are the sections' own.** Every audit, specialist, Red Team, QA, fix-check, adversarial,
|
||||
outside-voice and documentation brief, its severity and confidence rules and its gates come from its
|
||||
section verbatim, with only its placeholders filled. This file restates none of them and
|
||||
never shortens a reviewer's brief. Audit readers (Steps 7, 8, 9.2.1, 14.5) cannot block, so each runs on a
|
||||
never shortens a reviewer's brief. A blocking reader's prompt is its `gstack-review-dispatch render`
|
||||
file, unchanged; its findings reach `merged.jsonl` only via `collect`; `judge` refuses a drop or downgrade. Audit readers (Steps 7, 8, 9.2.1, 14.5) cannot block, so each runs on a
|
||||
small model at low effort; their findings go to the PR body.
|
||||
|
||||
**Calls.** One pass with no fixes is 9 calls: Phases 1-8 and the report; at most 12 with a STOP's
|
||||
section read or a fix-check. A pass that fixes something repeats Phases 3-4. Waits add pings, not work.
|
||||
section read or a fix-check. A pass that fixes something repeats Phases 3-4.
|
||||
|
||||
**Budget: observe.** `gstack-ship-budget start --mode observe` records, never admits: each program ends
|
||||
with `gstack-ship-budget note --step <last step> --reading R --credits C`, R and C read in that program
|
||||
(Capy: `cost.tokens` and `cost.credits` of your own jam). No check per step, no reservation, no meter-only
|
||||
(Capy: your own jam's `cost.tokens` and `cost.credits`). No check per step, no reservation, no meter-only
|
||||
call, no handoff; under `--token-cap` a note past the cap prints `verdict=stop`, the one stop check.
|
||||
A program that dispatches readers takes `BEFORE` (R and C) right before they start and passes
|
||||
A dispatching program takes `BEFORE` (R and C) as its readers start and passes
|
||||
`--dispatched-at <R> --dispatched-credits <C> --readers <n>` with the reading after the last settles.
|
||||
|
||||
**Waits.** No call waits past 240 s: a program polls or sleeps at most 240 s and returns what still
|
||||
runs; the next call resumes (Capy: `wait` 240 s on the operation). A call after a gap over 5 minutes
|
||||
bills the whole context fresh (about 40x a cached call); a ping costs a few percent of that. A small
|
||||
waiting child saves nothing: the parent's cache expires during its wait too. Past 100 minutes, stop.
|
||||
**Waits.** No call waits past 240 s: a program returns whatever still runs at 240 s; the next call resumes (Capy: `wait` 240 s on it). A small
|
||||
waiting child saves nothing. Past 100 minutes, stop.
|
||||
|
||||
**One program per pass.** Each review pass, Step 11 and the Step 14.5 audit is one program, and the
|
||||
parent makes no other call while it runs: (1) `gstack-review-plan --base <base>` with the pass's
|
||||
flags (its `DISPATCH:` lines; Step 14.5 has one reader, `docs`); (2) `gstack-review-log --start
|
||||
<skill>` once per row the step logs (Step 14.5 takes none); (3) `BEFORE`; (4) one subagent per reader;
|
||||
<skill>` once per row the step logs (Step 14.5 takes none); (3) `BEFORE`; (4) one subagent per reader on its rendered file, then
|
||||
`sent` (`BRIEF_MISMATCH` voids it);
|
||||
(5) a poll until every reader settles (status `done` or `failed`, or a final summary; a fresh `idle`
|
||||
has not run); (6) `note` with the reading pair; a later batch (the Red Team,
|
||||
given the specialists' collected lines) repeats (3)-(6); (7) the collect: return only each reader's
|
||||
final output and status. The next call applies the section's merge, judgment, fixes and persistence;
|
||||
final output and status, via `collect`. The next call applies the section's merge, judgment, fixes and persistence;
|
||||
it never walks the dispatch steps call by call.
|
||||
|
||||
## 1. Load: Step 0
|
||||
@@ -83,7 +83,8 @@ reader ran.
|
||||
## 4. Judgment: Steps 8.1, 8.2, 9.3, 9.4 and Step 11
|
||||
|
||||
Steps 8.1 and 8.2 (`Scope Check:`) on the todos and commits files, then the core checklist, Steps
|
||||
9.3, 9.4 and the repeat decision as written (`gstack-review-plan` included), fixes and persistence in
|
||||
9.3, 9.4 and the repeat decision as written (`gstack-review-plan` included;
|
||||
`JUDGMENT: ok` before any fix), fixes and persistence in
|
||||
one program. Prints `Pre-Landing Review:`, `Gate edits:`. `NEXT: stop` stops; `full` or
|
||||
`scoped` reruns Step 5 (`--tests-only`) and Phases 3-4; `done`: the same program runs Step 11 from
|
||||
`sections/adversarial.md` as written (above), only the voices the last full-pass plan lists (neither
|
||||
@@ -92,7 +93,7 @@ prints `ADVERSARIAL: skipped (review plan)`).
|
||||
## 5. Voices, bind and version: Steps 11, 11.5, 12
|
||||
|
||||
Judge the voices; missing outside coverage is `unavailable`, never clean; queued fixes rerun Phases
|
||||
3-4. Then `gstack-review-read`: the final Step 9 row and Step 11 native row (or `CURRENT` freshness
|
||||
3-4. Then `gstack-review-dispatch verify` (`VERIFY: clean`) and `gstack-review-read`: the final Step 9 row and Step 11 native row (or `CURRENT` freshness
|
||||
after a planned skip) agree on `wtree`, `start_wtree`, `end_wtree`; save that reviewed tree. Then
|
||||
`gstack-ship-release version --base <base> [--bump micro|patch] --why '<line>'`; MINOR or MAJOR is a
|
||||
question; `VERSION: skipped (policy: ...)`: write no version.
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
import { afterEach, describe, expect, test } from 'bun:test';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { BriefError, promptHash, renderBrief, sectionTexts } from '../lib/review-brief';
|
||||
import { judge, parseFindings, type MergedLine } from '../lib/review-ledger';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const BIN = path.join(ROOT, 'bin', 'gstack-review-dispatch');
|
||||
const dirs: string[] = [];
|
||||
afterEach(() => { while (dirs.length) fs.rmSync(dirs.pop()!, { recursive: true, force: true }); });
|
||||
const tmp = () => { const d = fs.mkdtempSync(path.join(os.tmpdir(), 'review-dispatch-')); dirs.push(d); return d; };
|
||||
const run = (args: string[], cwd = ROOT) => spawnSync(BIN, args, { cwd, encoding: 'utf8', timeout: 60_000 });
|
||||
const SHA = 'a'.repeat(40);
|
||||
const section = (rel: string) => fs.readFileSync(path.join(ROOT, rel), 'utf8');
|
||||
const field = (out: string, k: string) => new RegExp(`${k}=(\\S+)`).exec(out)?.[1] ?? '';
|
||||
|
||||
describe('gstack-review-dispatch render: briefs from the sections, slots only', () => {
|
||||
// Value: protects=every blocking reader's brief is cut from its section's own bytes with only declared slots filled, so a driven /ship sends what ship/SKILL.md sends; fails_when=a renderer paraphrases, drops a line, leaves a placeholder, or a section edit silently stops matching (GSTA-167 no-loss round 4: 37 of 112 dispatched briefs matched; a batched brief lost two lines);
|
||||
// why_new=no renderer existed; the driver relied on the model to fill placeholders; seam=lib/review-brief.ts and bin/gstack-review-dispatch; no_claim=that the reader then follows its brief
|
||||
test('each role renders, carries its section lines verbatim and leaves no placeholder', () => {
|
||||
const t = sectionTexts(ROOT);
|
||||
const slots = { diff_base: SHA, exclude: "-- . ':(exclude)*.lock'", stack: 'TypeScript', test_fw: 'bun:test', name: 'testing', checklist: '/x/testing.md', checklists: '/x/a.md,/x/b.md',
|
||||
findings: '[CRITICAL] a.ts:3 off by one', specialists: '4', diff: '+const a = 1;', fix_list: 'a.ts:3 fixed', hunks: '+x', tests: '3 pass' };
|
||||
const army = section('ship/sections/review-army.md');
|
||||
const adv = section('ship/sections/adversarial.md');
|
||||
const spec = renderBrief('specialist', slots, t);
|
||||
for (const line of ['Output nothing else: no preamble, no summary or commentary.', 'Required fields: severity, confidence, path, category, summary, specialist.',
|
||||
'blocks with clear intent. Skip test_stub for architectural or design-only findings.']) { expect(army).toContain(line); expect(spec).toContain(line); }
|
||||
expect(spec).toContain('Read the checklist at /x/testing.md');
|
||||
expect(spec).toContain(`git diff "$DIFF_BASE" -- . ':(exclude)*.lock'`);
|
||||
expect(spec).toContain(`DIFF_BASE=${SHA}`);
|
||||
expect(spec).not.toContain('write the sha in before dispatch');
|
||||
const batched = renderBrief('specialist-batched', slots, t);
|
||||
expect(batched).toContain('Read each checklist at /x/a.md, /x/b.md');
|
||||
expect(batched.split('\n').length).toBe(spec.split('\n').length);
|
||||
const security = renderBrief('specialist', { ...slots, name: 'security' }, t);
|
||||
expect(security.startsWith('This is an authorized defensive-security review')).toBe(true);
|
||||
expect(security).toContain('reviewed in summary mode');
|
||||
const red = renderBrief('red-team', slots, t);
|
||||
expect(red).toContain('reviewed by 4 specialists');
|
||||
expect(red).toContain('[CRITICAL] a.ts:3 off by one');
|
||||
const native = renderBrief('adversarial-native', slots, t);
|
||||
expect(adv).toContain('Think like an attacker and a chaos engineer. Your job is to find ways this code will fail in production.');
|
||||
expect(native).toContain('Think like an attacker and a chaos engineer. Your job is to find ways this code will fail in production.');
|
||||
expect(native).toContain(`DIFF_BASE=${SHA} && git diff --name-status`);
|
||||
const outside = renderBrief('outside-voice', slots, t);
|
||||
expect(outside).toContain('Filesystem boundary: do not read or execute any files under ~/.claude/');
|
||||
expect(outside).toContain('+const a = 1;');
|
||||
expect(renderBrief('fix-check', slots, t)).toContain('You are the fix-check reviewer');
|
||||
for (const text of [spec, batched, security, red, native]) for (const p of ['<diff-base>', '<exclude>', '<base>', '{checklist path}', '{STACK}', '{TEST_FW}', '{learnings']) expect(text).not.toContain(p);
|
||||
});
|
||||
|
||||
test('a missing slot, a non-sha diff base and a section that lost its anchor refuse instead of rendering', () => {
|
||||
const t = sectionTexts(ROOT);
|
||||
expect(() => renderBrief('specialist', { diff_base: SHA, name: 'testing' }, t)).toThrow(BriefError);
|
||||
expect(() => renderBrief('adversarial-native', { diff_base: '$RANGE_BASE' }, t)).toThrow(/literal commit sha/);
|
||||
const root = tmp();
|
||||
fs.mkdirSync(path.join(root, 'ship/sections'), { recursive: true });
|
||||
fs.writeFileSync(path.join(root, 'ship/sections/review-army.md'), section('ship/sections/review-army.md').replace('"You are a specialist code reviewer.', '"You review code.'));
|
||||
fs.writeFileSync(path.join(root, 'ship/sections/adversarial.md'), section('ship/sections/adversarial.md'));
|
||||
expect(() => sectionTexts(root)).toThrow(/specialist brief: start anchor not found/);
|
||||
});
|
||||
|
||||
test('the CLI writes the brief, prints its hash and records it; the file hashes to the printed value', () => {
|
||||
const out = tmp();
|
||||
const r = run(['render', '--role', 'specialist', '--name', 'testing', '--out', out, '--diff-base', SHA, '--stack', 'TypeScript']);
|
||||
expect(r.status).toBe(0);
|
||||
const file = field(r.stdout, 'file');
|
||||
expect(r.stdout).toContain('BRIEF: reader=specialist/testing');
|
||||
expect(promptHash(fs.readFileSync(file, 'utf8'))).toBe(field(r.stdout, 'sha256'));
|
||||
expect(fs.readFileSync(path.join(out, 'review-ledger.jsonl'), 'utf8')).toContain('"event":"rendered"');
|
||||
expect(run(['render', '--role', 'specialist', '--name', 'no-such-specialist', '--out', out, '--diff-base', SHA]).stderr).toContain('BRIEF_RENDER_FAILED: checklist not found');
|
||||
});
|
||||
});
|
||||
|
||||
describe('gstack-review-dispatch sent, collect, judge and verify: nothing changed on the way in or out', () => {
|
||||
function pass() {
|
||||
const out = tmp();
|
||||
const ledger = path.join(out, 'review-ledger.jsonl');
|
||||
const merged = path.join(out, 'merged.jsonl');
|
||||
const file = field(run(['render', '--role', 'specialist', '--name', 'testing', '--out', out, '--diff-base', SHA]).stdout, 'file');
|
||||
return { out, ledger, merged, file, w: (n: string, t: string) => { const f = path.join(out, n); fs.writeFileSync(f, t); return f; } };
|
||||
}
|
||||
// Value: protects=a dispatched prompt that is not byte-for-byte its rendered brief fails the pass and shows in verify; fails_when=a paraphrased or trimmed brief is recorded as reviewed (round 4: 75 of 112 briefs differed from the rendered ones);
|
||||
// why_new=no ledger recorded prompt hashes; seam=sent and verify; no_claim=that the host passed the recorded text
|
||||
test('the rendered file sent unchanged matches; a brief with one line dropped is BRIEF_MISMATCH and verify is dirty', () => {
|
||||
const p = pass();
|
||||
const ok = run(['sent', '--ledger', p.ledger, '--role', 'specialist', '--name', 'testing', '--sent', p.file]);
|
||||
expect([ok.status, ok.stdout.startsWith('BRIEF_SENT: reader=specialist/testing')]).toEqual([0, true]);
|
||||
expect(run(['verify', '--ledger', p.ledger]).stdout).toContain('BRIEF specialist/testing MATCH');
|
||||
const lines = fs.readFileSync(p.file, 'utf8').split('\n');
|
||||
const trimmed = p.w('trimmed.txt', lines.filter((_, i) => i !== 3).join('\n'));
|
||||
const bad = run(['sent', '--ledger', p.ledger, '--role', 'specialist', '--name', 'testing', '--sent', trimmed]);
|
||||
expect(bad.status).toBe(1);
|
||||
expect(bad.stdout).toContain('BRIEF_MISMATCH: reader=specialist/testing');
|
||||
const v = run(['verify', '--ledger', p.ledger]);
|
||||
expect([v.status, v.stdout.includes('BRIEF specialist/testing MISMATCH'), v.stdout.includes('VERIFY: dirty')]).toEqual([1, true, true]);
|
||||
});
|
||||
|
||||
// Value: protects=findings reach merged.jsonl only through collect with each voice's label as written, and lead judgment can add or promote but never drop or downgrade; fails_when=the parent drops a reader's finding, lowers its severity, folds it into a weaker duplicate or adds one silently, or merged lines no longer trace to the output hash;
|
||||
// why_new=merging was the model's own prose step; seam=collect, judge and verify; no_claim=that the judged set is used downstream by Step 9.4
|
||||
test('collect keeps raw labels; judge rejects a drop and a downgrade, accepts a reasoned promotion and a fold into an equal duplicate', () => {
|
||||
const p = pass();
|
||||
run(['sent', '--ledger', p.ledger, '--role', 'specialist', '--name', 'testing', '--sent', p.file]);
|
||||
const spec = p.w('spec.out', '{"severity":"CRITICAL","path":"a.ts","line":3,"category":"logic","summary":"off by one","specialist":"testing"}\n{"severity":"INFORMATIONAL","path":"a.ts","line":9,"category":"style","summary":"name","specialist":"testing"}\n');
|
||||
const outside = p.w('outside.out', 'Findings:\n1. **High**: retry loop never backs off (q.ts:78)\n2. Low - log line is noisy\nRecommendation: Fix the retry because it DoSes the pool\n');
|
||||
const c1 = run(['collect', '--ledger', p.ledger, '--role', 'specialist', '--name', 'testing', '--output', spec, '--merged', p.merged]);
|
||||
const c2 = run(['collect', '--ledger', p.ledger, '--role', 'outside-voice', '--output', outside, '--merged', p.merged]);
|
||||
expect([c1.stdout.includes('findings=2'), c2.stdout.includes('findings=2')]).toEqual([true, true]);
|
||||
const merged = fs.readFileSync(p.merged, 'utf8').trim().split('\n').map(l => JSON.parse(l) as MergedLine);
|
||||
expect(merged.map(m => [m.id, m.label])).toEqual([['specialist/testing#1', 'CRITICAL'], ['specialist/testing#2', 'INFORMATIONAL'], ['outside-voice#1', 'High'], ['outside-voice#2', 'Low']]);
|
||||
const jl = (rows: object[]) => p.w('judged.jsonl', rows.map(r => JSON.stringify(r)).join('\n') + '\n');
|
||||
const keep = [{ id: 'specialist/testing#1', severity: 'CRITICAL' }, { id: 'specialist/testing#2', severity: 'INFORMATIONAL' }, { id: 'outside-voice#1', severity: 'High' }, { id: 'outside-voice#2', severity: 'Low' }];
|
||||
const dropped = run(['judge', '--ledger', p.ledger, '--merged', p.merged, '--judged', jl(keep.filter(k => k.id !== 'outside-voice#2'))]);
|
||||
expect([dropped.status, dropped.stdout]).toEqual([1, expect.stringContaining('JUDGMENT_REJECTED: dropped=["outside-voice#2"]')]);
|
||||
const downgraded = run(['judge', '--ledger', p.ledger, '--merged', p.merged, '--judged', jl(keep.map(k => (k.id === 'specialist/testing#1' ? { ...k, severity: 'INFORMATIONAL' } : k)))]);
|
||||
expect([downgraded.status, downgraded.stdout]).toEqual([1, expect.stringContaining('downgraded=["specialist/testing#1"]')]);
|
||||
const weakFold = run(['judge', '--ledger', p.ledger, '--merged', p.merged, '--judged', jl(keep.map(k => (k.id === 'outside-voice#1' ? { ...k, duplicate_of: 'specialist/testing#2' } : k)))]);
|
||||
expect(weakFold.stdout).toContain('downgraded=["outside-voice#1"]');
|
||||
const silentAdd = run(['judge', '--ledger', p.ledger, '--merged', p.merged, '--judged', jl([...keep, { id: 'lead#1', severity: 'CRITICAL' }])]);
|
||||
expect(silentAdd.stdout).toContain('unexplained=["lead#1"]');
|
||||
const ok = run(['judge', '--ledger', p.ledger, '--merged', p.merged, '--judged', jl([
|
||||
...keep.map(k => (k.id === 'outside-voice#1' ? { ...k, duplicate_of: 'specialist/testing#1' } : k.id === 'outside-voice#2' ? { ...k, severity: 'INFORMATIONAL', reason: 'it hides a real error line' } : k)),
|
||||
{ id: 'lead#1', severity: 'CRITICAL', reason: 'the core checklist found a broken caller' }])]);
|
||||
expect([ok.status, ok.stdout.trim()]).toEqual([0, 'JUDGMENT: ok added=1 promoted=1 folded=1']);
|
||||
const ledger = fs.readFileSync(p.ledger, 'utf8');
|
||||
expect(ledger).toContain('"kind":"promoted","id":"outside-voice#2","from":"Low","to":"INFORMATIONAL","reason":"it hides a real error line"');
|
||||
expect(ledger).toContain('"kind":"added","id":"lead#1"');
|
||||
const v = run(['verify', '--ledger', p.ledger]);
|
||||
expect([v.status, v.stdout.includes('OUTPUT outside-voice MATCH'), v.stdout.includes('JUDGE - OK'), v.stdout.includes('VERIFY: clean briefs=1 outputs=2')]).toEqual([0, true, true, true]);
|
||||
fs.writeFileSync(p.merged, fs.readFileSync(p.merged, 'utf8').replace('"label":"Low"', '"label":"Low","x":1').replace(/"output_sha256":"[0-9a-f]+"}\n$/, '"output_sha256":"' + '0'.repeat(64) + '"}\n'));
|
||||
const tampered = run(['verify', '--ledger', p.ledger]);
|
||||
expect([tampered.status, tampered.stdout.includes('OUTPUT outside-voice MISMATCH')]).toEqual([1, true]);
|
||||
});
|
||||
|
||||
test('judge and parseFindings as functions: an unknown severity is invalid, NO FINDINGS and the Recommendation line are not findings', () => {
|
||||
expect(parseFindings('r', 'NO FINDINGS\n')).toEqual([]);
|
||||
expect(parseFindings('r', 'FIXABLE: race in a.ts:3\nINVESTIGATE: odd lock\nRecommendation: Fix the race because x\n').map(f => f.label)).toEqual(['FIXABLE', 'INVESTIGATE']);
|
||||
const m: MergedLine[] = [{ id: 'r#1', reader: 'r', label: 'FIXABLE', raw: 'x', output_sha256: 'h' }];
|
||||
expect(judge(m, [{ id: 'r#1', severity: 'severe' }]).invalid).toEqual(['r#1']);
|
||||
expect(judge(m, [{ id: 'r#1', severity: 'INFORMATIONAL', reason: 'a real defect' }]).ok).toBe(true);
|
||||
});
|
||||
|
||||
test('verify on a ledger that kept no prompts says so and exits 3', () => {
|
||||
const d = tmp();
|
||||
const ledger = path.join(d, 'ledger.jsonl');
|
||||
fs.writeFileSync(ledger, '{"event":"note","step":"9"}\n');
|
||||
const v = run(['verify', '--ledger', ledger]);
|
||||
expect([v.status, v.stdout.trim()]).toEqual([3, "VERIFY: no brief records (this run's ledger kept no prompts; dispatched-vs-rendered cannot be checked)"]);
|
||||
});
|
||||
});
|
||||
@@ -204,6 +204,16 @@ describe('variant G: the call plan, observe budget, waits and audit readers (GST
|
||||
expect(capy).toContain('model: { model_id: "anthropic/claude-sonnet-5-5", reasoning_mode: "low" }');
|
||||
expect(capy).toContain('`anthropic/claude-haiku-5-5` also fits');
|
||||
});
|
||||
|
||||
// Value: protects=a driven /ship dispatches every blocking reader on its rendered brief file unchanged, checks it with `sent`, merges findings only through `collect`, gates judgment with `judge` and verifies the run before the bind; fails_when=the driver lets the model write or trim a brief again or merge findings in prose (round 4b: arm D lost 8 blockers with 75 of 112 briefs paraphrased);
|
||||
// why_new=the driver named no renderer or hash check; seam=none; no_claim=the parent's adherence, which verify checks after the fact
|
||||
test('briefs go out as rendered and findings come back unchanged: render, sent, collect, judge, verify', () => {
|
||||
const f = flat(doc);
|
||||
for (const s of ["A blocking reader's prompt is its `gstack-review-dispatch render` file, unchanged", 'only via `collect`', '`judge` refuses a drop or downgrade',
|
||||
'(4) one subagent per reader on its rendered file, then `sent` (`BRIEF_MISMATCH` voids it)', 'final output and status, via `collect`', '`JUDGMENT: ok` before any fix',
|
||||
'`gstack-review-dispatch verify` (`VERIFY: clean`)']) expect(f).toContain(s);
|
||||
expect(f.indexOf('gstack-review-dispatch verify')).toBeLessThan(f.indexOf('`gstack-review-read`'));
|
||||
});
|
||||
});
|
||||
|
||||
describe('STOP anchors', () => {
|
||||
|
||||
Reference in new issue
Block a user