fix(office-hours): reviewer returns a hash-bound receipt instead of echoing its verdict

The spec reviewer now Writes its verdict once, seals it with
gstack-office-hours-review seal (validates the saved JSON and prints
OFFICE_HOURS_VERDICT round=N sha256=<hash> path=<verdict>), and returns
only that line. check --receipt recomputes the hash and fails closed on a
missing, malformed, stale-round, wrong-path or mismatched receipt. The
saved file stays the sole findings inventory; validators bind the receipt
to the saved bytes and the judge reads those bytes.
This commit is contained in:
garrytan committed 2026-10-05 18:50:21 +00:00
1 parent 43e2b89369
commit 209a61dcd3
10 files changed
+178 -62

No files matched your search

+27 -4
View File
@@ -2,17 +2,19 @@
import * as fs from 'node:fs';
import * as path from 'node:path';
import { atomicWriteSync } from '../lib/fs-atomic.ts';
import { assessOfficeHoursReviews, loadOfficeHoursReviews, renderOfficeHoursReview, renderOfficeHoursReviewerPrompt, replaceOfficeHoursReviewBlock } from '../lib/office-hours-review.ts';
import { assessOfficeHoursReviews, loadOfficeHoursReviews, officeHoursVerdictReceipt, parseOfficeHoursVerdictReceipt, renderOfficeHoursReview, renderOfficeHoursReviewerPrompt, replaceOfficeHoursReviewBlock, verifyOfficeHoursVerdictReceipt } from '../lib/office-hours-review.ts';
const shellQuote = value => `'${value.replaceAll("'", `'\\''`)}'`;
function main(args) {
const command = args.shift();
if (!['check', 'prepare', 'finalize'].includes(command)) throw new Error('usage: gstack-office-hours-review check <round.json...> | prepare --design <doc> --out-dir <reviewdir> [round.json...] | finalize --design <doc> [--report <report>] [--unreviewed <reason>] <round.json...>');
if (!['check', 'prepare', 'seal', 'finalize'].includes(command)) throw new Error('usage: gstack-office-hours-review check --receipt <reviewer response> <round.json...> | prepare --design <doc> --out-dir <reviewdir> [round.json...] | seal --design <doc> --verdict <round-N.json> [preceding round.json...] | finalize --design <doc> [--report <report>] [--unreviewed <reason>] <round.json...>');
const options = {};
const files = [];
while (args.length) {
const value = args.shift();
if (value.startsWith('--')) {
const allowed = command === 'prepare' ? ['--design', '--out-dir'] : command === 'finalize' ? ['--design', '--report', '--unreviewed'] : [];
const allowed = { check: ['--receipt'], prepare: ['--design', '--out-dir'], seal: ['--design', '--verdict'], finalize: ['--design', '--report', '--unreviewed'] }[command];
if (!allowed.includes(value)
|| options[value] !== undefined || !args.length) throw new Error(`invalid option: ${value}`);
options[value] = args.shift();
@@ -20,12 +22,32 @@ function main(args) {
}
const rounds = loadOfficeHoursReviews(files);
if (command === 'check') {
// The receipt binds the reviewer's response to the last round's exact bytes.
if (!options['--receipt'] || !files.length) throw new Error('check requires --receipt with the reviewer response and every completed round');
const last = files.at(-1);
const named = parseOfficeHoursVerdictReceipt(options['--receipt']).path;
const same = fs.existsSync(named) && fs.realpathSync(named) === fs.realpathSync(last);
verifyOfficeHoursVerdictReceipt(options['--receipt'], { round: files.length, verdictPath: same ? named : last, bytes: fs.readFileSync(last) });
console.log(JSON.stringify(assessOfficeHoursReviews(rounds)));
return;
}
if (!options['--design']) throw new Error(`${command} requires --design`);
const designPath = path.resolve(options['--design']);
if (rounds.some(review => path.resolve(review.document) !== designPath)) throw new Error('artifact document does not match --design');
if (command === 'seal') {
if (!options['--verdict']) throw new Error('seal requires --verdict');
const verdictPath = path.resolve(options['--verdict']);
const round = rounds.length + 1;
if (path.basename(verdictPath) !== `round-${round}.json`) throw new Error(`verdict must be round-${round}.json after the supplied history`);
const bytes = fs.readFileSync(verdictPath);
let value;
try { value = JSON.parse(bytes.toString('utf8')); }
catch (error) { throw new Error(`saved verdict is invalid JSON: ${error instanceof Error ? error.message : String(error)}`); }
const review = assessOfficeHoursReviews([...rounds, value]).rounds.at(-1);
if (path.resolve(review.document) !== designPath) throw new Error('verdict document does not match --design');
console.log(officeHoursVerdictReceipt(round, verdictPath, bytes));
return;
}
if (command === 'prepare') {
if (!options['--out-dir']) throw new Error('prepare requires --out-dir');
if (!fs.statSync(designPath).isFile()) throw new Error('design must be an existing file');
@@ -44,7 +66,8 @@ function main(args) {
if (existing) {
if (!existing.isFile() || fs.readFileSync(promptPath, 'utf8') !== prompt) throw new Error('existing review prompt differs from the complete saved evidence');
} else atomicWriteSync(promptPath, prompt, { mode: 0o600 });
const dispatch = `Read ${JSON.stringify(promptPath)} and execute the independent spec review as instructed.\nDocument: ${designPath}\nPrompt: ${promptPath}\nVerdict: ${verdictPath}`;
const seal = [shellQuote(fs.realpathSync(process.argv[1])), 'seal --design', shellQuote(designPath), '--verdict', shellQuote(verdictPath), ...files.map(file => shellQuote(fs.realpathSync(file)))].join(' ');
const dispatch = `Read ${JSON.stringify(promptPath)} and execute the independent spec review as instructed.\nDocument: ${designPath}\nPrompt: ${promptPath}\nVerdict: ${verdictPath}\nSeal: ${seal}`;
console.log(JSON.stringify({ round, promptPath, verdictPath, dispatch }));
return;
}
+25 -2
View File
@@ -1,4 +1,5 @@
/** Office-hours review artifacts are the verdict; prose is rendered from them. */
import { createHash } from 'node:crypto';
import * as fs from 'node:fs';
import * as path from 'node:path';
@@ -130,6 +131,27 @@ export function loadOfficeHoursReviews(paths: readonly string[]): OfficeHoursRev
return values.length ? assessOfficeHoursReviews(values).rounds : [];
}
export interface OfficeHoursVerdictReceipt { round: number; sha256: string; path: string }
/** The reviewer's entire response: one line binding its round to the exact saved bytes. */
export function officeHoursVerdictReceipt(round: number, verdictPath: string, bytes: string | Uint8Array): string {
return `OFFICE_HOURS_VERDICT round=${round} sha256=${createHash('sha256').update(bytes).digest('hex')} path=${verdictPath}`;
}
export function parseOfficeHoursVerdictReceipt(response: string): OfficeHoursVerdictReceipt {
const match = /^OFFICE_HOURS_VERDICT round=([1-3]) sha256=([0-9a-f]{64}) path=(.+)$/.exec(response.trim());
if (!match || !path.isAbsolute(match[3])) fail('reviewer response is not exactly one verdict receipt line');
return { round: Number(match[1]), sha256: match[2], path: match[3] };
}
/** Fail closed unless the receipt names this round, this path, and these bytes. */
export function verifyOfficeHoursVerdictReceipt(response: string, { round, verdictPath, bytes }: {
round: number; verdictPath: string; bytes: string | Uint8Array;
}): OfficeHoursVerdictReceipt {
const receipt = parseOfficeHoursVerdictReceipt(response);
if (receipt.round !== round) fail(`receipt names round ${receipt.round}, not round ${round}`);
if (path.resolve(receipt.path) !== path.resolve(verdictPath)) fail('receipt names a different verdict path');
if (receipt.sha256 !== createHash('sha256').update(bytes).digest('hex')) fail('receipt hash does not match the saved verdict bytes');
return receipt;
}
/** The caller validates the complete history before supplying its last verdict. */
export function renderOfficeHoursReviewerPrompt({ document, verdictPath, previous }: {
document: string; verdictPath: string; previous?: OfficeHoursReview;
@@ -154,8 +176,9 @@ export function renderOfficeHoursReviewerPrompt({ document, verdictPath, previou
Document: ${document}
Verdict: ${verdictPath}
Use only Read and Write for this review. Read the design at ${JSON.stringify(document)} with Read and review all 5 dimensions independently, including new defects. Do not use Bash or Edit, and do not change the design.
Use Write only to save your complete verdict as JSON to ${JSON.stringify(verdictPath)}, then return that identical JSON as your entire response (no Markdown fences or prose). The parent runs the formatter to validate your saved JSON.
Use only Read, Write, and the one Bash seal command for this review. Read the design at ${JSON.stringify(document)} with Read and review all 5 dimensions independently, including new defects. Do not use Edit, and do not change the design.
Use Write to save your complete verdict as JSON to ${JSON.stringify(verdictPath)}. Then run the \`Seal:\` command from your dispatch message with Bash, exactly as given; it validates the saved file and prints one receipt line. If it reports an error, correct the saved JSON with Write and run the same command again. Use Bash for nothing else.
Return only that printed \`OFFICE_HOURS_VERDICT round=${round} sha256=<hash> path=<verdict path>\` line, unchanged, as your entire response: no JSON, Markdown fences, or prose. The parent verifies the receipt against the saved bytes.
The saved JSON is your sole findings inventory: include every unresolved problem and necessary remedy, including minor findings that a short conclusion might omit.
Use one finding per distinct obligation. An exact duplicate shares a finding; a shared component does not combine separate decisions, behavior, or effort.
+7 -3
View File
@@ -199,19 +199,23 @@ string unchanged as the prompt. The reviewer must Read the entire prepared promp
file before reviewing the design. Do not recreate the prompt, copy selected fields,
or summarize prior findings. A parent Read does not deliver the file to the reviewer.
The reviewer has fresh context and cannot see the brainstorming conversation.
Its prepared contract requires a complete JSON Write and an identical JSON response.
Its prepared contract requires a complete JSON Write, sealed by the dispatch's
`Seal:` command, and a one-line `OFFICE_HOURS_VERDICT` receipt as its entire response.
It protects the required coaching and Assignment sections, distinguishes unknown
customer facts from committed behavior, and requires evidence for every prior status.
**Step 2: Check stop conditions, then fix and re-dispatch**
After each verdict, BEFORE fixing any findings or dispatching again, validate the
saved files with the helper (list every completed round in order):
saved files with the helper. Pass the reviewer's entire response unchanged as the
receipt and list every completed round in order:
```bash
~/.claude/skills/gstack/bin/gstack-office-hours-review check "<round-1.json>" "<round-2.json if present>" "<round-3.json if present>"
~/.claude/skills/gstack/bin/gstack-office-hours-review check --receipt "<reviewer response>" "<round-1.json>" "<round-2.json if present>" "<round-3.json if present>"
```
A missing, malformed, or mismatched receipt fails the check: that attempt is a failed review.
Omit absent arguments rather than passing placeholders.
**Convergence guard and stopping rules:** Read its stop reason:
- PASS: no unresolved findings; proceed to Step 3.
+7 -3
View File
@@ -57,19 +57,23 @@ string unchanged as the prompt. The reviewer must Read the entire prepared promp
file before reviewing the design. Do not recreate the prompt, copy selected fields,
or summarize prior findings. A parent Read does not deliver the file to the reviewer.
The reviewer has fresh context and cannot see the brainstorming conversation.
Its prepared contract requires a complete JSON Write and an identical JSON response.
Its prepared contract requires a complete JSON Write, sealed by the dispatch's
\`Seal:\` command, and a one-line \`OFFICE_HOURS_VERDICT\` receipt as its entire response.
It protects the required coaching and Assignment sections, distinguishes unknown
customer facts from committed behavior, and requires evidence for every prior status.
**Step 2: Check stop conditions, then fix and re-dispatch**
After each verdict, BEFORE fixing any findings or dispatching again, validate the
saved files with the helper (list every completed round in order):
saved files with the helper. Pass the reviewer's entire response unchanged as the
receipt and list every completed round in order:
\`\`\`bash
~/.claude/skills/gstack/bin/gstack-office-hours-review check "<round-1.json>" "<round-2.json if present>" "<round-3.json if present>"
~/.claude/skills/gstack/bin/gstack-office-hours-review check --receipt "<reviewer response>" "<round-1.json>" "<round-2.json if present>" "<round-3.json if present>"
\`\`\`
A missing, malformed, or mismatched receipt fails the check: that attempt is a failed review.
Omit absent arguments rather than passing placeholders.
**Convergence guard and stopping rules:** Read its stop reason:
- PASS: no unresolved findings; proceed to Step 3.
-1
View File
@@ -265,7 +265,6 @@
"test/model-overlays.test.ts\tTS2322\tType 'string' is not assignable to type '\"claude\" | \"fable-5\" | \"gemini\" | \"gpt\" | \"gpt-5.4\" | \"gpt-5.6-sol\" | \"gpt-6-astra\" | \"o-series\" | \"opus-4-7\" | \"opus-4-8\" | \"sonnet-5\" | undefined'.": 4,
"test/native-auto-decide-pty.test.ts\tTS2345\tArgument of type 'string | undefined' is not assignable to parameter of type 'string'. Type 'undefined' is not assignable to type 'string'.": 1,
"test/native-auto-decide.test.ts\tTS2345\tArgument of type '{ status: string; calls: never[]; assistantMessages: { sessionId: string; text: string; timestamp: string; }[]; }' is not assignable to parameter of type 'PlanCountTranscript'. Types of property 'status' are incompatible. Type 'string' is not assignable to type '\"error\" | \"missing\" | \"ready\"'.": 1,
"test/office-hours-completion.test.ts\tTS18046\t'reordered.findings' is of type 'unknown'.": 1,
"test/office-hours-review.test.ts\tTS18048\t'check.stderr' is possibly 'undefined'.": 1,
"test/office-hours-review.test.ts\tTS18048\t'check.stdout' is possibly 'undefined'.": 1,
"test/office-hours-review.test.ts\tTS18048\t'direct.stderr' is possibly 'undefined'.": 1,
+4 -2
View File
@@ -1544,7 +1544,9 @@ describe('SPEC_REVIEW_LOOP resolver', () => {
for (const section of ["'The Assignment'", "'What I noticed about how you think'"]) expect(prompt).toContain(section);
expect(prompt).toMatch(/never invent customer answers/i);
expect(prompt).toMatch(/including minor findings/i);
expect(prompt).toMatch(/identical JSON/i);
expect(prompt).toContain('`OFFICE_HOURS_VERDICT round=1 sha256=<hash> path=<verdict path>`');
expect(prompt).toMatch(/run the `Seal:` command/);
expect(prompt).not.toMatch(/identical JSON/i);
expect(prompt).toMatch(/classify every preceding finding/i);
ordered(prompt, ['1. **Completeness**', '2. **Consistency**', '3. **Clarity**', '4. **Scope**', '5. **Feasibility**']);
expect(prompt).toMatch(/distinguished from committed behavior/i);
@@ -1556,7 +1558,7 @@ describe('SPEC_REVIEW_LOOP resolver', () => {
const output = render('office-hours');
const step2 = output.slice(output.indexOf('**Step 2:'), output.indexOf('**Step 3:'));
expectMentions(step2, [['before','dispatching','findings']], 'step2');
ordered(step2, ['gstack-office-hours-review check', '- PASS:', '- CONVERGENCE:', '- MAX_ITERATIONS: round 3', '- CONTINUE:', 'gstack-office-hours-review finalize --design']);
ordered(step2, ['gstack-office-hours-review check --receipt "<reviewer response>"', 'mismatched receipt fails the check', '- PASS:', '- CONVERGENCE:', '- MAX_ITERATIONS: round 3', '- CONTINUE:', 'gstack-office-hours-review finalize --design']);
expectMentions(step2, [['do not','re-dispatch','again']], 'step2');
expectMentions(step2, [['do not','generated','section']], 'step2');
expect(step2.slice(step2.indexOf('finalize --design'))).toMatch(/existing user approval/i);
+25 -25
View File
@@ -1,6 +1,6 @@
import { isDeepStrictEqual } from 'node:util';
import * as path from 'node:path';
import { renderOfficeHoursReviewerPrompt, validateOfficeHoursReview, renderOfficeHoursReview, extractOfficeHoursReviewBlock, type OfficeHoursReview } from '../../lib/office-hours-review';
import { renderOfficeHoursReviewerPrompt, validateOfficeHoursReview, renderOfficeHoursReview, extractOfficeHoursReviewBlock, parseOfficeHoursVerdictReceipt, verifyOfficeHoursVerdictReceipt, type OfficeHoursReview } from '../../lib/office-hours-review';
/**
* Completion evidence for the fixed office-hours section-loading fixture.
@@ -208,7 +208,7 @@ export function validateOfficeHoursCompletion(evidence: OfficeHoursCompletionEvi
export function validateOfficeHoursReviewArtifacts(
evidence: OfficeHoursCompletionEvidence,
artifacts: Array<{ path: string; content: string | null }>,
): void {
): OfficeHoursReviewEvidence | null {
const fail = (message: string): never => { throw new Error(`Office-hours review artifacts: ${message}`); };
const reviewEvidence = validateOfficeHoursCompletion(evidence);
const normalize = (value: unknown) => String(value ?? '').replace(/\\/g, '/').replace(/^\.\//, '');
@@ -223,30 +223,30 @@ export function validateOfficeHoursReviewArtifacts(
&& [designPath, relativeDesign].some(p => normalize(call.input?.prompt).includes(p)));
if (!attempts.length || attempts.length > 3) fail('expected one to three reviewer attempts');
const rounds: OfficeHoursReview[] = [];
const verdicts: string[] = [];
let failedAttempt = false;
for (let i = 0; i < attempts.length; i++) {
let verdict: OfficeHoursReview;
try {
const response = (attempts[i].output ?? '').trim();
// One enclosing JSON fence changes presentation, not the authored data.
// Additional prose or a second payload still fails JSON parsing.
const payload = /^```(?:json)?[ \t]*\n([\s\S]*)\n```$/i.exec(response)?.[1] ?? response;
verdict = validateOfficeHoursReview(JSON.parse(payload), rounds.at(-1));
} catch (error) {
const response = attempts[i].output ?? '';
// Anything but exactly one receipt line is a failed attempt; the saved
// file, not the response, is the findings inventory.
try { parseOfficeHoursVerdictReceipt(response); }
catch {
if (i !== attempts.length - 1) fail('continued reviewing after an invalid or failed verdict');
failedAttempt = true;
verdicts.push(response);
break;
}
if (normalize(verdict!.document) !== designPath) fail(`round ${i + 1} reviewed a different document`);
const matches = artifacts.filter(artifact => normalize(artifact.path).endsWith(`/round-${i + 1}.json`));
if (matches.length !== 1 || matches[0].content === null) fail(`round ${i + 1} needs exactly one saved verdict`);
const prompt = normalize(attempts[i].input?.prompt).replaceAll('/./', '/');
const assigned = [artifactPath(matches[0].path), path.relative(fixtureRoot, artifactPath(matches[0].path))];
if (!assigned.some(reference => prompt.includes(reference))) fail(`round ${i + 1} artifact was not assigned to the reviewer`);
let saved: unknown;
try { saved = JSON.parse(matches[0].content!); }
catch { fail(`round ${i + 1} saved verdict is invalid JSON`); }
if (!isDeepStrictEqual(saved, verdict!)) fail(`round ${i + 1} saved verdict differs from the reviewer response`);
try { verifyOfficeHoursVerdictReceipt(response, { round: i + 1, verdictPath: artifactPath(matches[0].path), bytes: matches[0].content! }); }
catch (error) { fail(`round ${i + 1} saved verdict differs from the reviewer receipt: ${(error as Error).message}`); }
let verdict: OfficeHoursReview;
try { verdict = validateOfficeHoursReview(JSON.parse(matches[0].content!), rounds.at(-1)); }
catch { fail(`round ${i + 1} saved verdict is invalid`); }
if (normalize(verdict!.document) !== designPath) fail(`round ${i + 1} reviewed a different document`);
// The flattened trace proves authored content, not the writer's parent ID.
// An identical copy after dispatch is valid; a preexisting file is not proof.
const attemptIndex = evidence.toolCalls.indexOf(attempts[i]);
@@ -257,6 +257,7 @@ export function validateOfficeHoursReviewArtifacts(
catch { fail(`round ${i + 1} lacks an observed JSON Write`); }
if (!isDeepStrictEqual(written, verdict!)) fail(`round ${i + 1} artifact differs from its observed Write`);
rounds.push(verdict!);
verdicts.push(matches[0].content!);
}
// Even when the actual final tool result is absent, the existing completion
// validator requires a declared UNREVIEWED and its explanation in the report.
@@ -276,7 +277,7 @@ export function validateOfficeHoursReviewArtifacts(
try {
validateOfficeHoursReview(JSON.parse(artifact.content), rounds.at(-1));
} catch { continue; } // Preserve a malformed failed attempt as evidence.
fail('valid saved verdict has no matching reviewer response');
fail('valid saved verdict has no matching reviewer receipt');
}
}
const expected = renderOfficeHoursReview(rounds, unavailable);
@@ -290,6 +291,8 @@ export function validateOfficeHoursReviewArtifacts(
|| withoutTrailingBreaks(sectionBody(evidence.designContent!, ['reviewer concerns'], true)) !== expectedConcerns) {
fail('Reviewer Concerns does not preserve every saved problem and remedy');
}
// A receipt names the verdict; the preservation judge reads the saved bytes.
return reviewEvidence && { ...reviewEvidence, verdict: verdicts.at(-1)!, priorVerdicts: verdicts.slice(0, -1) };
}
/** Prove that each actual reviewer received the entire generated contract and prior verdict. */
@@ -364,9 +367,7 @@ export function validateOfficeHoursReviewerHandoffs(
}
let validVerdict = false;
try {
const response = (attempt.output ?? '').trim();
const payload = /^```(?:json)?[ \t]*\n([\s\S]*)\n```$/i.exec(response)?.[1] ?? response;
previous = validateOfficeHoursReview(JSON.parse(payload), previous);
parseOfficeHoursVerdictReceipt(attempt.output ?? '');
validVerdict = true;
} catch {
if (index !== attempts.length - 1) fail('another reviewer followed an invalid verdict');
@@ -374,13 +375,12 @@ export function validateOfficeHoursReviewerHandoffs(
}
if (validVerdict) {
const matches = artifacts.filter(artifact => path.resolve(artifact.path) === verdictPath);
let saved: unknown;
try { saved = matches.length === 1 ? JSON.parse(matches[0].content ?? '') : null; }
if (matches.length !== 1 || matches[0].content === null) fail(`round ${index + 1} has no valid saved verdict`);
try { verifyOfficeHoursVerdictReceipt(attempt.output!, { round: index + 1, verdictPath, bytes: matches[0].content! }); }
catch (error) { fail(`round ${index + 1} saved verdict differs from the reviewer receipt: ${(error as Error).message}`); }
// Prepare reads the saved object; its insertion order reproduces the prompt bytes.
try { previous = validateOfficeHoursReview(JSON.parse(matches[0].content!), previous); }
catch { fail(`round ${index + 1} has no valid saved verdict`); }
if (!isDeepStrictEqual(saved, previous)) fail(`round ${index + 1} saved verdict differs from the reviewer response`);
// Prepare reads the saved object. JSON object key order is immaterial to
// equality, but using that same insertion order reproduces its prompt bytes.
previous = saved as OfficeHoursReview;
}
// A native tool failure before the reviewer can Read is genuinely unavailable.
// A malformed completed verdict or a parent declaration alone cannot waive delivery.
+43 -17
View File
@@ -1,4 +1,4 @@
import { renderOfficeHoursReviewerPrompt, renderOfficeHoursReview, extractOfficeHoursReviewBlock, type OfficeHoursReview } from '../lib/office-hours-review';
import { renderOfficeHoursReviewerPrompt, renderOfficeHoursReview, extractOfficeHoursReviewBlock, officeHoursVerdictReceipt, type OfficeHoursReview } from '../lib/office-hours-review';
import { describe, expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as path from 'node:path';
@@ -722,7 +722,7 @@ describe('office-hours mechanical review evidence', () => {
const artifacts = rounds.map((r, i) => ({ path: `/tmp/office-hours-fixture/review/round-${i + 1}.json`, content: JSON.stringify(r) }));
evidence.toolCalls = evidence.toolCalls.slice(0, 2);
for (const [i, artifact] of artifacts.entries()) {
evidence.toolCalls.push({ tool: 'Agent', input: { prompt: `Review ${designPath}; write ${artifact.path}.` }, output: JSON.stringify(rounds[i]) });
evidence.toolCalls.push({ tool: 'Agent', input: { prompt: `Review ${designPath}; write ${artifact.path}.` }, output: officeHoursVerdictReceipt(i + 1, artifact.path, artifact.content) });
evidence.toolCalls.push({ tool: 'Write', input: { file_path: artifact.path, content: artifact.content } });
}
return { evidence, artifacts, rounds };
@@ -804,15 +804,34 @@ describe('office-hours mechanical review evidence', () => {
expect(() => validateOfficeHoursReviewerHandoffs(evidence, artifacts)).toThrow('round 2 did not receive the complete');
});
test('JSON object key order cannot change complete reviewer delivery', () => {
test('receipts bind the exact saved bytes in both validators', () => {
const { evidence, artifacts, attempts } = handoffs(true);
const reordered = Object.fromEntries(Object.entries(JSON.parse(attempts[0].output!)).reverse());
attempts[0].output = JSON.stringify(reordered);
expect(() => validateOfficeHoursReviewArtifacts(evidence, artifacts)).not.toThrow();
expect(() => validateOfficeHoursReviewerHandoffs(evidence, artifacts)).not.toThrow();
reordered.findings[0].remedy = 'Different obligation';
attempts[0].output = JSON.stringify(reordered);
expect(() => validateOfficeHoursReviewerHandoffs(evidence, artifacts)).toThrow('saved verdict differs');
const changed = JSON.parse(artifacts[0].content!);
changed.findings[0].remedy = 'Different obligation';
artifacts[0].content = JSON.stringify(changed);
expect(() => validateOfficeHoursReviewerHandoffs(evidence, artifacts)).toThrow('round 1 saved verdict differs from the reviewer receipt');
expect(() => validateOfficeHoursReviewArtifacts(evidence, artifacts)).toThrow('round 1 saved verdict differs from the reviewer receipt');
attempts[0].output = attempts[0].output!.replace(/sha256=[0-9a-f]{64}/, `sha256=${'0'.repeat(64)}`);
expect(() => validateOfficeHoursReviewerHandoffs(evidence, artifacts)).toThrow('receipt hash does not match');
});
test('a stale earlier-round receipt cannot satisfy a later round', () => {
const { evidence, artifacts, attempts } = handoffs(true);
attempts[1].output = attempts[0].output;
expect(() => validateOfficeHoursReviewerHandoffs(evidence, artifacts)).toThrow('round 2 saved verdict differs from the reviewer receipt');
expect(() => validateOfficeHoursReviewArtifacts(evidence, artifacts)).toThrow('round 2 saved verdict differs from the reviewer receipt');
attempts[1].output = officeHoursVerdictReceipt(2, artifacts[0].path, artifacts[0].content!);
expect(() => validateOfficeHoursReviewArtifacts(evidence, artifacts)).toThrow('different verdict path');
artifacts[1].content = artifacts[0].content;
attempts[1].output = officeHoursVerdictReceipt(2, artifacts[1].path, artifacts[1].content!);
expect(() => validateOfficeHoursReviewArtifacts(evidence, artifacts)).toThrow('round 2 saved verdict is invalid');
});
test('judge evidence comes from the saved verdict bytes, not the receipt', () => {
const { evidence, artifacts } = structured();
const judged = validateOfficeHoursReviewArtifacts(evidence, artifacts);
expect(judged?.verdict).toBe(artifacts[1].content!);
expect(judged?.priorVerdicts).toEqual([artifacts[0].content!]);
});
test('a separate coaching closing does not hide the actual Handoff recommendation', () => {
@@ -835,6 +854,9 @@ describe('office-hours mechanical review evidence', () => {
// this is not a replay of a successful native/model completion.
const { evidence, artifacts, rounds } = JSON.parse(JSON.stringify(handoffs())
.replaceAll('/tmp/office-hours-fixture', dir.replaceAll('\\', '/')));
// Relocated artifacts have new bytes, so their receipts are reissued.
evidence.toolCalls.filter((call: { output?: string }) => call.output?.startsWith('OFFICE_HOURS_VERDICT '))
.forEach((call: { output?: string }, i: number) => { call.output = officeHoursVerdictReceipt(i + 1, artifacts[i].path, artifacts[i].content); });
const reportPath = path.join(dir, 'REPORT.md');
const closing = `# Office-hours completion report
## Phase Outcomes
@@ -890,11 +912,14 @@ Next: /plan-eng-review after observing the workflow. The user declined launching
expect(() => validateOfficeHoursReviewArtifacts(evidence, artifacts)).toThrow('computed metrics');
});
test('accepts one JSON fence around the identical authored verdict', () => {
const { evidence, artifacts } = structured();
test.each(['fence', 'prose', 'echoed JSON', 'missing path'])('a %s response is a failed attempt, never a completed verdict', mode => {
const { evidence, artifacts, rounds } = structured();
const review = evidence.toolCalls.findLast(call => call.tool === 'Agent')!;
review.output = `\`\`\`json\n${review.output}\n\`\`\``;
expect(() => validateOfficeHoursReviewArtifacts(evidence, artifacts)).not.toThrow();
if (mode === 'fence') review.output = `\`\`\`\n${review.output}\n\`\`\``;
if (mode === 'prose') review.output = `Saved.\n${review.output}`;
if (mode === 'echoed JSON') review.output = JSON.stringify(rounds[1]);
if (mode === 'missing path') review.output = review.output!.replace(/ path=.*$/, '');
expect(() => validateOfficeHoursReviewArtifacts(evidence, artifacts)).toThrow('contradicts');
});
test('rejects the observed encoding obligation omission even if a judge would accept it', () => {
@@ -914,7 +939,7 @@ Next: /plan-eng-review after observing the workflow. The user declined launching
const { evidence, artifacts, rounds } = structured();
rounds[1].findings[5].remedy = 'Always silently decode with latin-1.';
artifacts[1].content = JSON.stringify(rounds[1]);
expect(() => validateOfficeHoursReviewArtifacts(evidence, artifacts)).toThrow('differs from the reviewer response');
expect(() => validateOfficeHoursReviewArtifacts(evidence, artifacts)).toThrow('differs from the reviewer receipt');
});
test('requires the actual saved file, not just a reviewer response', () => {
@@ -946,8 +971,9 @@ Next: /plan-eng-review after observing the workflow. The user declined launching
const { evidence, artifacts } = structured();
expect(() => validateOfficeHoursReviewArtifacts(evidence, [...artifacts, artifacts[1]])).toThrow('exactly one saved verdict');
const review = evidence.toolCalls.findLast(call => call.tool === 'Agent')!;
const json = JSON.parse(review.output!); json.document = '/tmp/different-design.md';
review.output = JSON.stringify(json);
const json = JSON.parse(artifacts[1].content!); json.document = '/tmp/different-design.md';
artifacts[1].content = JSON.stringify(json);
review.output = officeHoursVerdictReceipt(2, artifacts[1].path, artifacts[1].content);
expect(() => validateOfficeHoursReviewArtifacts(evidence, artifacts)).toThrow();
});
+38 -3
View File
@@ -5,7 +5,7 @@ import * as path from 'node:path';
import {
OFFICE_HOURS_DIMENSIONS, assessOfficeHoursReviews, validateOfficeHoursReview,
renderOfficeHoursReview, renderOfficeHoursReviewerPrompt, extractOfficeHoursReviewBlock, replaceOfficeHoursReviewBlock,
type OfficeHoursReview,
officeHoursVerdictReceipt, type OfficeHoursReview,
} from '../lib/office-hours-review';
import { expectMentions } from './helpers/prompt-structure';
@@ -314,6 +314,39 @@ describe('office-hours review CLI', () => {
expect(fs.readFileSync(prepared.verdictPath, 'utf8')).toBe(saved);
expect(fs.readFileSync(data.design, 'utf8')).toBe(before);
}));
test('seal validates the saved verdict and prints the receipt that check requires', () => fixture((dir, invoke) => {
const data = files(dir), outDir = path.join(dir, 'reviews');
fs.mkdirSync(outDir);
const stdout = (...args: string[]) => String(invoke(...args).stdout);
const stderr = (...args: string[]) => String(invoke(...args).stderr);
const prepared = JSON.parse(stdout('prepare', '--design', data.design, '--out-dir', outDir));
const quote = (value: string) => `'${value.replaceAll("'", `'\\''`)}'`;
const seal = `${quote(fs.realpathSync(cli))} seal --design ${quote(data.design)} --verdict ${quote(prepared.verdictPath)}`;
expect(prepared.dispatch.split('\n').at(-1)).toBe(`Seal: ${seal}`);
expect(fs.readFileSync(prepared.promptPath, 'utf8')).toMatch(/run the `Seal:` command from your dispatch message/);
fs.writeFileSync(prepared.verdictPath, '{bad');
expect(stderr('seal', '--design', data.design, '--verdict', prepared.verdictPath)).toContain('invalid JSON');
fs.writeFileSync(prepared.verdictPath, JSON.stringify(data.rounds[1]));
expect(stderr('seal', '--design', data.design, '--verdict', prepared.verdictPath)).toContain('contiguous');
fs.writeFileSync(prepared.verdictPath, JSON.stringify(data.rounds[0]));
expect(stderr('seal', '--design', data.design, '--verdict', data.artifacts[1])).toContain('round-1.json');
expect(invoke('seal', '--design', path.join(dir, 'other.md'), '--verdict', prepared.verdictPath).exitCode).toBe(1);
const sealed = process.platform === 'win32'
? invoke('seal', '--design', data.design, '--verdict', prepared.verdictPath)
: Bun.spawnSync(['bash', '-c', seal], { cwd: dir, timeout: 5000,
env: { ...process.env, PATH: `${path.dirname(process.execPath)}${path.delimiter}${process.env.PATH ?? ''}` } });
expect(sealed.exitCode, String(sealed.stderr)).toBe(0);
const receipt = String(sealed.stdout).trim();
expect(receipt).toBe(officeHoursVerdictReceipt(1, prepared.verdictPath, fs.readFileSync(prepared.verdictPath)));
expect(invoke('check', '--receipt', receipt, prepared.verdictPath).exitCode).toBe(0);
expect(stderr('check', prepared.verdictPath)).toContain('requires --receipt');
expect(stderr('check', '--receipt', receipt.replace(/sha256=\w+/, `sha256=${'0'.repeat(64)}`), prepared.verdictPath)).toContain('hash');
expect(stderr('check', '--receipt', `\`${receipt}\``, prepared.verdictPath)).toContain('receipt line');
expect(stderr('check', '--receipt', receipt, data.artifacts[0])).toContain('different verdict path');
expect(stderr('check', '--receipt', receipt, ...data.artifacts)).toContain('not round 2');
const next = JSON.parse(stdout('prepare', '--design', data.design, '--out-dir', outDir, prepared.verdictPath));
expect(next.dispatch).toContain(`--verdict ${quote(next.verdictPath)} ${quote(fs.realpathSync(prepared.verdictPath))}`);
}));
test('prepare refuses stale output, malformed or terminal history, and incomplete ordered history', () => fixture((dir, invoke) => {
const data = files(dir), outDir = path.join(dir, 'reviews');
fs.mkdirSync(outDir);
@@ -347,7 +380,8 @@ describe('office-hours review CLI', () => {
}));
test('check is read-only; finalize renders both authoritative files from the same inventory', () => fixture((dir, invoke) => {
const data = files(dir), before = fs.readFileSync(data.design, 'utf8');
const check = invoke('check', ...data.artifacts);
const receipt = officeHoursVerdictReceipt(2, data.artifacts[1], fs.readFileSync(data.artifacts[1]));
const check = invoke('check', '--receipt', receipt, ...data.artifacts);
expect(check.exitCode, check.stderr.toString()).toBe(0);
expect(JSON.parse(check.stdout.toString()).stop).toBe('CONVERGENCE');
expect(fs.readFileSync(data.design, 'utf8')).toBe(before);
@@ -378,7 +412,8 @@ describe('office-hours review CLI', () => {
fs.writeFileSync(data.report, '<!-- gstack:office-hours:report:start -->');
expect(invoke('finalize', '--design', data.design, '--report', data.report, ...data.artifacts).exitCode).toBe(1);
expect(fs.readFileSync(data.design, 'utf8')).toBe(before);
expect(invoke('check', path.join(dir, 'missing.json')).exitCode).toBe(1);
const missing = path.join(dir, 'missing.json');
expect(invoke('check', '--receipt', officeHoursVerdictReceipt(1, missing, ''), missing).exitCode).toBe(1);
const missingReport = invoke('finalize', '--design', data.design, '--report', path.join(dir, 'missing-report.md'), ...data.artifacts);
expect(missingReport.exitCode).toBe(1);
expect(missingReport.stderr.toString()).toContain('missing-report.md');
@@ -65,7 +65,7 @@ Completion delivery, after the full workflow and design approval:
maxTurns: 40,
});
const designPath = path.join(planDir, 'docs/designs/roster-check.md');
const reviewEvidence = validateOfficeHoursCompletion({
validateOfficeHoursCompletion({
...capture, designPath,
designContent: fs.existsSync(designPath) ? fs.readFileSync(designPath, 'utf-8') : null,
});
@@ -75,7 +75,7 @@ Completion delivery, after the full workflow and design approval:
const artifacts = artifactPaths.map(artifactPath => ({ path: artifactPath,
content: fs.existsSync(artifactPath) ? fs.readFileSync(artifactPath, 'utf-8') : null,
}));
validateOfficeHoursReviewArtifacts({
const reviewEvidence = validateOfficeHoursReviewArtifacts({
...capture, designPath,
designContent: fs.existsSync(designPath) ? fs.readFileSync(designPath, 'utf-8') : null,
}, artifacts);