fix(office-hours): rounds 2 and 3 are delta re-reviews of the captured design diff

prepare saves each reviewed design as round-N.design.md and gives later
reviewers the exact line diff since the previous round. A new later-round
blocking finding must cite (changed_text) an excerpt of a changed line;
seal, check and the validators verify it against the snapshots and fail
closed when a citation or snapshot is missing. Other new observations are
minor. Severity, anti-downgrade and CONVERGENCE rules are unchanged.
This commit is contained in:
garrytan committed 2026-10-05 19:59:26 +00:00
1 parent 69a71f5da2
commit b0351491e2
8 files changed
+278 -70

No files matched your search

+19 -6
View File
@@ -2,7 +2,7 @@
import * as fs from 'node:fs';
import * as path from 'node:path';
import { atomicWriteSync } from '../lib/fs-atomic.ts';
import { assessOfficeHoursReviews, loadOfficeHoursReviews, officeHoursVerdictReceipt, parseOfficeHoursVerdictReceipt, renderOfficeHoursReview, renderOfficeHoursReviewerPrompt, replaceOfficeHoursReviewBlock, verifyOfficeHoursVerdictReceipt } from '../lib/office-hours-review.ts';
import { assessOfficeHoursReviews, loadOfficeHoursReviews, officeHoursDesignChanges, officeHoursSnapshotChanges, officeHoursVerdictReceipt, parseOfficeHoursVerdictReceipt, renderOfficeHoursReview, renderOfficeHoursReviewerPrompt, replaceOfficeHoursReviewBlock, verifyOfficeHoursCitations, verifyOfficeHoursVerdictReceipt } from '../lib/office-hours-review.ts';
const shellQuote = value => `'${value.replaceAll("'", `'\\''`)}'`;
@@ -45,6 +45,7 @@ function main(args) {
catch (error) { throw new Error(`saved verdict is invalid JSON: ${error instanceof Error ? error.message : String(error)}`); }
const review = assessOfficeHoursReviews([...rounds, value]).rounds.at(-1);
if (path.resolve(review.document) !== designPath) throw new Error('verdict document does not match --design');
if (round > 1) verifyOfficeHoursCitations(review, officeHoursSnapshotChanges(verdictPath, round).changed);
console.log(officeHoursVerdictReceipt(round, verdictPath, bytes));
return;
}
@@ -56,16 +57,28 @@ function main(args) {
const round = rounds.length + 1;
const promptPath = path.join(outDir, `round-${round}.prompt.md`);
const verdictPath = path.join(outDir, `round-${round}.json`);
const prompt = renderOfficeHoursReviewerPrompt({ document: designPath, verdictPath, previous: rounds.at(-1) });
// Each round records the exact design it reviewed; later rounds see the diff since then.
const snapshotPath = path.join(outDir, `round-${round}.design.md`);
const design = fs.readFileSync(designPath, 'utf8');
let changes;
if (round > 1 && assessOfficeHoursReviews(rounds).stop !== 'CONTINUE') throw new Error('cannot prepare another round after a terminal review');
if (round > 1) {
const before = path.join(outDir, `round-${round - 1}.design.md`);
if (!fs.existsSync(before)) throw new Error(`out-dir lacks round-${round - 1}.design.md; prepare every round in the same review directory`);
changes = officeHoursDesignChanges(fs.readFileSync(before, 'utf8'), design).diff;
}
const prompt = renderOfficeHoursReviewerPrompt({ document: designPath, verdictPath, previous: rounds.at(-1), changes });
const inputs = new Set([fs.realpathSync(designPath), ...files.map(file => fs.realpathSync(file))]);
for (const destination of [promptPath, verdictPath]) {
for (const destination of [promptPath, verdictPath, snapshotPath]) {
if (fs.existsSync(destination) && inputs.has(fs.realpathSync(destination))) throw new Error('output destination overlaps input evidence');
}
if (fs.lstatSync(verdictPath, { throwIfNoEntry: false })) throw new Error('next verdict already exists; check it before preparing another attempt');
const captured = fs.lstatSync(snapshotPath, { throwIfNoEntry: false });
if (captured && (!captured.isFile() || fs.readFileSync(snapshotPath, 'utf8') !== design)) throw new Error(`the design changed after round ${round} was prepared`);
const existing = fs.lstatSync(promptPath, { throwIfNoEntry: false });
if (existing) {
if (!existing.isFile() || fs.readFileSync(promptPath, 'utf8') !== prompt) throw new Error('existing review prompt differs from the complete saved evidence');
} else atomicWriteSync(promptPath, prompt, { mode: 0o600 });
if (existing && (!existing.isFile() || fs.readFileSync(promptPath, 'utf8') !== prompt)) throw new Error('existing review prompt differs from the complete saved evidence');
if (!captured) atomicWriteSync(snapshotPath, design, { mode: 0o600 });
if (!existing) atomicWriteSync(promptPath, prompt, { mode: 0o600 });
const seal = [shellQuote(fs.realpathSync(process.argv[1])), 'seal --design', shellQuote(designPath), '--verdict', shellQuote(verdictPath), ...files.map(file => shellQuote(fs.realpathSync(file)))].join(' ');
const dispatch = `Read ${JSON.stringify(promptPath)} and execute the independent spec review as instructed.\nDocument: ${designPath}\nPrompt: ${promptPath}\nVerdict: ${verdictPath}\nSeal: ${seal}`;
console.log(JSON.stringify({ round, promptPath, verdictPath, dispatch }));
+70 -7
View File
@@ -11,6 +11,7 @@ export interface OfficeHoursFinding {
id: string;
dimension: OfficeHoursDimension;
severity: OfficeHoursSeverity;
changed_text: string | null;
problem: string;
remedy: string;
}
@@ -66,7 +67,7 @@ export function validateOfficeHoursReview(value: unknown, previous?: OfficeHours
if (!Array.isArray(review.findings) || !Array.isArray(review.prior)) fail('findings and prior must be arrays');
const ids = new Set<string>();
for (const raw of review.findings) {
const finding = object(raw, ['id', 'dimension', 'severity', 'problem', 'remedy'], 'finding');
const finding = object(raw, ['id', 'dimension', 'severity', 'changed_text', 'problem', 'remedy'], 'finding');
if (typeof finding.id !== 'string' || !new RegExp(`^R${review.round}-[1-9][0-9]*$`).test(finding.id)
|| ids.has(finding.id)) fail('finding ids must be unique R<round>-<positive integer> identifiers');
ids.add(finding.id);
@@ -74,6 +75,8 @@ export function validateOfficeHoursReview(value: unknown, previous?: OfficeHours
if (!OFFICE_HOURS_SEVERITIES.includes(finding.severity)) fail(`${finding.id} severity must be blocking or minor`);
nonempty(finding.problem, `${finding.id} problem`);
nonempty(finding.remedy, `${finding.id} remedy`);
if (finding.changed_text !== null && (review.round === 1 || typeof finding.changed_text !== 'string'
|| finding.changed_text.replace(/\s/g, '').length < 8)) fail(`${finding.id} changed_text must be null in round 1, else null or an excerpt of at least 8 characters`);
}
for (const dimension of OFFICE_HOURS_DIMENSIONS) {
const expected = review.findings.some((finding: OfficeHoursFinding) => finding.dimension === dimension) ? 'ISSUES' : 'PASS';
@@ -101,6 +104,11 @@ export function validateOfficeHoursReview(value: unknown, previous?: OfficeHours
}
}
if (covered.size !== previousIds.size) fail('prior must cover each preceding finding exactly once');
for (const finding of review.findings as OfficeHoursFinding[]) {
if (review.round > 1 && finding.severity === 'blocking' && !currentLinks.has(finding.id) && finding.changed_text === null) {
fail(`new blocking ${finding.id} must cite the changed design text that introduced or exposed it`);
}
}
return review as OfficeHoursReview;
}
@@ -137,12 +145,49 @@ export function assessOfficeHoursReviews(values: readonly unknown[]): {
}
return { rounds, stop: stopFor(rounds.at(-1)!), metrics: metricsFor(rounds) };
}
const flat = (text: string) => text.replace(/\s+/g, ' ').trim();
/** The exact line diff between two reviewed design versions, with 2 lines of context per hunk. */
export function officeHoursDesignChanges(before: string, after: string): { diff: string; changed: string[] } {
const a = before.split('\n'), b = after.split('\n');
const lcs = Array.from({ length: a.length + 1 }, () => new Uint32Array(b.length + 1));
for (let i = a.length - 1; i >= 0; i--) for (let j = b.length - 1; j >= 0; j--) {
lcs[i][j] = a[i] === b[j] ? lcs[i + 1][j + 1] + 1 : Math.max(lcs[i + 1][j], lcs[i][j + 1]);
}
const lines: Array<{ mark: ' ' | '-' | '+'; text: string }> = [];
for (let i = 0, j = 0; i < a.length || j < b.length;) {
if (i < a.length && j < b.length && a[i] === b[j]) { lines.push({ mark: ' ', text: a[i++] }); j++; }
else if (i < a.length && (j === b.length || lcs[i + 1][j] >= lcs[i][j + 1])) lines.push({ mark: '-', text: a[i++] });
else lines.push({ mark: '+', text: b[j++] });
}
const near = lines.map((_, n) => lines.slice(Math.max(0, n - 2), n + 3).some(line => line.mark !== ' '));
const diff = lines.map((line, n) => near[n] ? `${line.mark}${line.text}` : near[n - 1] ? '@@' : null)
.filter((line): line is string => line !== null).join('\n').replace(/^@@\n?|\n@@$/g, '');
return { diff: diff || '(no changes)', changed: lines.filter(line => line.mark !== ' ' && flat(line.text)).map(line => line.text) };
}
/** Fail closed unless each cited excerpt lies within one changed (+/-) line. */
export function verifyOfficeHoursCitations(review: OfficeHoursReview, changed: readonly string[]): void {
for (const finding of review.findings) {
if (finding.changed_text !== null && !changed.some(line => flat(line).includes(flat(finding.changed_text!)))) {
fail(`${finding.id} changed_text is not inside the design changes before round ${review.round}`);
}
}
}
/** Rounds 2+ need the design snapshots prepare captured beside each verdict. */
export function officeHoursSnapshotChanges(verdictPath: string, round: number): { diff: string; changed: string[] } {
const snapshot = (n: number) => {
try { return fs.readFileSync(path.join(path.dirname(verdictPath), `round-${n}.design.md`), 'utf8'); }
catch { return fail(`round ${round} lacks its captured design snapshot round-${n}.design.md`); }
};
return officeHoursDesignChanges(snapshot(round - 1), snapshot(round));
}
export function loadOfficeHoursReviews(paths: readonly string[]): OfficeHoursReview[] {
const values = paths.map(file => {
try { return JSON.parse(fs.readFileSync(file, 'utf8')); }
catch (cause) { throw new Error(`Office-hours review: cannot read artifact ${file}: ${cause instanceof Error ? cause.message : String(cause)}`, { cause }); }
});
return values.length ? assessOfficeHoursReviews(values).rounds : [];
const rounds = values.length ? assessOfficeHoursReviews(values).rounds : [];
for (const review of rounds.slice(1)) verifyOfficeHoursCitations(review, officeHoursSnapshotChanges(paths[review.round - 1], review.round).changed);
return rounds;
}
export interface OfficeHoursVerdictReceipt { round: number; sha256: string; path: string }
@@ -167,8 +212,8 @@ export function verifyOfficeHoursVerdictReceipt(response: string, { round, verdi
}
/** The caller validates the complete history before supplying its last verdict. */
export function renderOfficeHoursReviewerPrompt({ document, verdictPath, previous }: {
document: string; verdictPath: string; previous?: OfficeHoursReview;
export function renderOfficeHoursReviewerPrompt({ document, verdictPath, previous, changes }: {
document: string; verdictPath: string; previous?: OfficeHoursReview; changes?: string;
}): string {
for (const [label, value] of [['document', document], ['verdictPath', verdictPath]]) {
nonempty(value, label);
@@ -177,11 +222,14 @@ export function renderOfficeHoursReviewerPrompt({ document, verdictPath, previou
}
if (previous && path.resolve(previous.document) !== path.resolve(document)) fail('review prompt targets a different document');
if (previous && stopFor(previous) !== 'CONTINUE') fail('cannot prepare another round after a terminal review');
if ((previous === undefined) !== (changes === undefined)) fail('later rounds, and only later rounds, need the captured design changes');
const round = (previous?.round ?? 0) + 1;
const fence = '`'.repeat(Math.max(3, ...[...(changes ?? '').matchAll(/`+/g)].map(run => run[0].length + 1)));
const example = {
version: 2, round, document, quality_score: 7,
dimensions: { completeness: 'PASS', consistency: 'PASS', clarity: 'ISSUES', scope: 'PASS', feasibility: 'PASS' },
findings: [{ id: `R${round}-1`, dimension: 'clarity', severity: 'blocking', problem: "The fallback's user-visible behavior is unspecified.",
findings: [{ id: `R${round}-1`, dimension: 'clarity', severity: 'blocking',
changed_text: previous ? 'Excerpt of a changed (+/-) line that shows the defect' : null, problem: "The fallback's user-visible behavior is unspecified.",
remedy: 'Choose and document whether the fallback warns the user or is intentionally silent.' }],
prior: [],
};
@@ -190,7 +238,9 @@ export function renderOfficeHoursReviewerPrompt({ document, verdictPath, previou
Document: ${document}
Verdict: ${verdictPath}
Use only Read, Write, and the one Bash seal command for this review. Read the design at ${JSON.stringify(document)} with Read and review all 5 dimensions independently, including new defects. Do not use Edit, and do not change the design.
Use only Read, Write, and the one Bash seal command for this review. Read the design at ${JSON.stringify(document)} with Read and ${previous
? `perform a delta re-review against the preceding verdict and the exact design changes captured below.`
: 'review all 5 dimensions independently, including new defects.'} Do not use Edit, and do not change the design.
Use Write to save your complete verdict as JSON to ${JSON.stringify(verdictPath)}. Then run the \`Seal:\` command from your dispatch message with Bash, exactly as given; it validates the saved file and prints one receipt line. If it reports an error, correct the saved JSON with Write and run the same command again. Use Bash for nothing else.
Return only that printed \`OFFICE_HOURS_VERDICT round=${round} sha256=<hash> path=<verdict path>\` line, unchanged, as your entire response: no JSON, Markdown fences, or prose. The parent verifies the receipt against the saved bytes.
The saved JSON is your sole findings inventory: include every unresolved problem and necessary remedy, including minor findings that a short conclusion might omit.
@@ -202,9 +252,22 @@ Give every finding a severity. Only blocking findings send the design back for a
- **blocking**: a contradiction; a safety or correctness risk; an unsupported claim the recommendation depends on; missing behavior the committed approach needs; or a persisting blocking prior obligation. Example: the design promises the roster is never stored, yet its sync step saves it nightly.
- **minor**: clarity, wording, or polish that does not change a decision or behavior. Example: the Recommended Approach repeats the problem statement's wording and could be shorter.
When unsure whether a gap changes a decision or behavior, it is blocking.
${previous ? `
## Delta re-review scope
This is round ${round}. Round 1 already reviewed the whole design. Raise a NEW blocking finding only when the changes since round ${round - 1} (a) introduced it, as a regression, or (b) exposed it in text that changed. Set its changed_text to a verbatim excerpt of at least 8 characters from one changed (+ or -) line of the diff below. Anything else you notice is minor and never forces another round; give it changed_text null unless it also concerns changed text.
A persisting or unverified prior obligation keeps its current finding without a citation.
The helper captured this exact line diff between the design reviewed in round ${round - 1} and the current design ("-" removed, "+" added, "@@" separates hunks):
${fence}diff
${changes}
${fence}
` : `
Every round-1 finding has changed_text null.
`}
This is an /office-hours design and coaching document, produced before engineering planning. The startup-mode 'The Assignment' and both modes' 'What I noticed about how you think' sections are intentional: evaluate their evidence and usefulness; do not remove them merely because they are coaching content. Unknown customer facts may remain explicit Open Questions or assignments; do not invent answers.
Still flag unsupported claims, contradictions, safety/correctness risks, and missing behavior needed by the approach the document actually commits to. Labeling a contradiction or a required behavior an open question does not resolve it.
Still flag unsupported claims, contradictions, safety/correctness risks, and missing behavior needed by the approach the document actually commits to. Labeling a contradiction or a required behavior an open question does not resolve it.${previous ? ' In this delta round, such problems outside the changed text are minor.' : ''}
On re-review, classify EVERY preceding finding as resolved, persisting, or unverified. Cite the specific document decision/behavior proving the status or the missing evidence. Absence from the new findings list is not confirmation.
A new refinement of an accepted fix is new unless the same specific original obligation demonstrably remains unmet. For persisting/unverified issues, include that unmet obligation in the current findings and reference its current ID. Distinct prior obligations must retain distinct current findings.
+8 -2
View File
@@ -192,7 +192,11 @@ all preceding valid round files in order (omit them for round 1):
Omit absent arguments rather than passing placeholders. The helper chooses the next
round and writes `round-N.prompt.md`. It includes the full finding schema, all five
review dimensions (Completeness, Consistency, Clarity, Scope, Feasibility), the
office-hours coaching contract, and the COMPLETE preceding JSON verdict.
office-hours coaching contract, and the COMPLETE preceding JSON verdict. It also
saves the design as reviewed (`round-N.design.md`). Round 1 is a full review;
rounds 2 and 3 are delta re-reviews of the exact design diff since the last round,
so prepare every round in the same review directory and do not edit the design
between prepare and its review.
Use the Agent tool with `run_in_background: false` and its returned `dispatch`
string unchanged as the prompt. The reviewer must Read the entire prepared prompt
@@ -227,7 +231,9 @@ blocking findings require another round. Read its stop reason:
are insufficient.
- MAX_ITERATIONS: round 3 completed; stop.
- CONTINUE: fix every blocking finding in the design (you may batch minor fixes
with them), then return to Step 1 to prepare and dispatch the next review.
with them), then return to Step 1 to prepare and dispatch the next review. Its
reviewer checks every prior finding and raises new blocking findings only for
problems your changes introduced or exposed.
On a stop, do not fix again or re-dispatch. Run the finalizer before approval:
+8 -2
View File
@@ -50,7 +50,11 @@ all preceding valid round files in order (omit them for round 1):
Omit absent arguments rather than passing placeholders. The helper chooses the next
round and writes \`round-N.prompt.md\`. It includes the full finding schema, all five
review dimensions (Completeness, Consistency, Clarity, Scope, Feasibility), the
office-hours coaching contract, and the COMPLETE preceding JSON verdict.
office-hours coaching contract, and the COMPLETE preceding JSON verdict. It also
saves the design as reviewed (\`round-N.design.md\`). Round 1 is a full review;
rounds 2 and 3 are delta re-reviews of the exact design diff since the last round,
so prepare every round in the same review directory and do not edit the design
between prepare and its review.
Use the Agent tool with \`run_in_background: false\` and its returned \`dispatch\`
string unchanged as the prompt. The reviewer must Read the entire prepared prompt
@@ -85,7 +89,9 @@ blocking findings require another round. Read its stop reason:
are insufficient.
- MAX_ITERATIONS: round 3 completed; stop.
- CONTINUE: fix every blocking finding in the design (you may batch minor fixes
with them), then return to Step 1 to prepare and dispatch the next review.
with them), then return to Step 1 to prepare and dispatch the next review. Its
reviewer checks every prior finding and raises new blocking findings only for
problems your changes introduced or exposed.
On a stop, do not fix again or re-dispatch. Run the finalizer before approval:
+23 -4
View File
@@ -1,6 +1,18 @@
import { isDeepStrictEqual } from 'node:util';
import * as path from 'node:path';
import { renderOfficeHoursReviewerPrompt, validateOfficeHoursReview, renderOfficeHoursReview, extractOfficeHoursReviewBlock, parseOfficeHoursVerdictReceipt, verifyOfficeHoursVerdictReceipt, type OfficeHoursReview } from '../../lib/office-hours-review';
import { renderOfficeHoursReviewerPrompt, validateOfficeHoursReview, renderOfficeHoursReview, extractOfficeHoursReviewBlock, parseOfficeHoursVerdictReceipt, verifyOfficeHoursVerdictReceipt, officeHoursDesignChanges, verifyOfficeHoursCitations, type OfficeHoursReview } from '../../lib/office-hours-review';
export type OfficeHoursFile = { path: string; content: string | null };
/** The helper's captured diff before a later round; a missing snapshot fails closed. */
function snapshotChanges(snapshots: readonly OfficeHoursFile[], verdictPath: string, round: number) {
const read = (n: number) => {
const wanted = path.join(path.dirname(path.resolve(verdictPath)), `round-${n}.design.md`);
const content = snapshots.find(file => path.resolve(file.path) === wanted)?.content;
if (typeof content !== 'string') throw new Error(`Office-hours review artifacts: round ${round} lacks its captured design snapshot round-${n}.design.md`);
return content;
};
return officeHoursDesignChanges(read(round - 1), read(round));
}
/**
* Completion evidence for the fixed office-hours section-loading fixture.
@@ -207,7 +219,8 @@ export function validateOfficeHoursCompletion(evidence: OfficeHoursCompletionEvi
*/
export function validateOfficeHoursReviewArtifacts(
evidence: OfficeHoursCompletionEvidence,
artifacts: Array<{ path: string; content: string | null }>,
artifacts: OfficeHoursFile[],
snapshots: readonly OfficeHoursFile[] = [],
): OfficeHoursReviewEvidence | null {
const fail = (message: string): never => { throw new Error(`Office-hours review artifacts: ${message}`); };
const reviewEvidence = validateOfficeHoursCompletion(evidence);
@@ -247,6 +260,10 @@ export function validateOfficeHoursReviewArtifacts(
try { verdict = validateOfficeHoursReview(JSON.parse(matches[0].content!), rounds.at(-1)); }
catch { fail(`round ${i + 1} saved verdict is invalid`); }
if (normalize(verdict!.document) !== designPath) fail(`round ${i + 1} reviewed a different document`);
if (i > 0) {
try { verifyOfficeHoursCitations(verdict!, snapshotChanges(snapshots, artifactPath(matches[0].path), i + 1).changed); }
catch (error) { fail(`round ${i + 1} citation: ${(error as Error).message}`); }
}
// The flattened trace proves authored content, not the writer's parent ID.
// An identical copy after dispatch is valid; a preexisting file is not proof.
const attemptIndex = evidence.toolCalls.indexOf(attempts[i]);
@@ -298,7 +315,8 @@ export function validateOfficeHoursReviewArtifacts(
/** Prove that each actual reviewer received the entire generated contract and prior verdict. */
export function validateOfficeHoursReviewerHandoffs(
evidence: OfficeHoursCompletionEvidence,
artifacts: Array<{ path: string; content: string | null }>,
artifacts: OfficeHoursFile[],
snapshots: readonly OfficeHoursFile[] = [],
): void {
const fail = (message: string): never => { throw new Error(`Office-hours reviewer handoff: ${message}`); };
const calls = evidence.toolCalls;
@@ -335,7 +353,8 @@ export function validateOfficeHoursReviewerHandoffs(
|| path.basename(verdicts[0]) !== `round-${index + 1}.json`) fail(`round ${index + 1} lacks its assigned verdict path`);
const verdictPath = verdicts[0];
const promptPath = verdictPath.replace(/\.json$/, '.prompt.md');
const expected = renderOfficeHoursReviewerPrompt({ document: evidence.designPath, verdictPath, previous });
const expected = renderOfficeHoursReviewerPrompt({ document: evidence.designPath, verdictPath, previous,
changes: index > 0 ? snapshotChanges(snapshots, verdictPath, index + 1).diff : undefined });
let delivered = prompt.includes(expected.trimEnd());
let unavailableBeforeRead = false;
if (!delivered) {
+70 -42
View File
@@ -1,4 +1,4 @@
import { renderOfficeHoursReviewerPrompt, renderOfficeHoursReview, extractOfficeHoursReviewBlock, officeHoursVerdictReceipt, type OfficeHoursReview } from '../lib/office-hours-review';
import { renderOfficeHoursReviewerPrompt, renderOfficeHoursReview, extractOfficeHoursReviewBlock, officeHoursVerdictReceipt, officeHoursDesignChanges, type OfficeHoursReview } from '../lib/office-hours-review';
import { describe, expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as path from 'node:path';
@@ -696,6 +696,14 @@ describe('office-hours completion eval selection', () => {
});
describe('office-hours mechanical review evidence', () => {
// The helper's captured design versions; round 2 reviews one changed line.
const reviewed = (round: number) => round === 1 ? design : design.replace('## Reviewer Concerns', 'Status: revised after round 1.\n\n## Reviewer Concerns');
const snapshotsIn = (dir: string) => [1, 2].map(round => ({ path: `${dir}/round-${round}.design.md`, content: reviewed(round) }));
const fixtureSnapshots = snapshotsIn('/tmp/office-hours-fixture/review');
const checkArtifacts = (evidence: Parameters<typeof validateOfficeHoursReviewArtifacts>[0], artifacts: Parameters<typeof validateOfficeHoursReviewArtifacts>[1], snapshots = fixtureSnapshots) =>
validateOfficeHoursReviewArtifacts(evidence, artifacts, snapshots);
const checkHandoffs = (evidence: Parameters<typeof validateOfficeHoursReviewerHandoffs>[0], artifacts: Parameters<typeof validateOfficeHoursReviewerHandoffs>[1], snapshots = fixtureSnapshots) =>
validateOfficeHoursReviewerHandoffs(evidence, artifacts, snapshots);
// The six actual obligations from evidence-final, including Clarity 3 which
// disappeared from both the prose report and the judge's accepted inventory.
const problems = [
@@ -710,7 +718,7 @@ describe('office-hours mechanical review evidence', () => {
const round = (n: number): OfficeHoursReview => ({
version: 2, round: n, document: designPath, quality_score: 7,
dimensions: { completeness: 'ISSUES', consistency: 'ISSUES', clarity: 'ISSUES', scope: 'PASS', feasibility: 'PASS' },
findings: problems.map(([dimension, problem, remedy], i) => ({ id: `R${n}-${i + 1}`, dimension, severity: 'blocking' as const, problem, remedy })),
findings: problems.map(([dimension, problem, remedy], i) => ({ id: `R${n}-${i + 1}`, dimension, severity: 'blocking' as const, changed_text: null, problem, remedy })),
prior: n === 1 ? [] : problems.map((_, i) => ({ id: `R1-${i + 1}`, status: 'persisting',
evidence: `The Recommended Approach still omits the original obligation: ${problems[i][1]}`, current_id: `R2-${i + 1}` })),
});
@@ -735,7 +743,8 @@ describe('office-hours mechanical review evidence', () => {
for (const [i, attempt] of attempts.entries()) {
const verdictPath = state.artifacts[i].path;
const promptPath = verdictPath.replace('.json', '.prompt.md');
const prompt = renderOfficeHoursReviewerPrompt({ document: designPath, verdictPath, previous: state.rounds[i - 1] });
const prompt = renderOfficeHoursReviewerPrompt({ document: designPath, verdictPath, previous: state.rounds[i - 1],
changes: i ? officeHoursDesignChanges(reviewed(i), reviewed(i + 1)).diff : undefined });
attempt.input!.prompt = useRead
? `Read ${promptPath} for the independent review.\nDocument: ${designPath}\nPrompt: ${promptPath}\nVerdict: ${verdictPath}`
: prompt;
@@ -752,15 +761,15 @@ describe('office-hours mechanical review evidence', () => {
test('delivers the entire schema, coaching contract, and preceding JSON inline', () => {
const { evidence, artifacts, attempts } = handoffs();
expect(() => validateOfficeHoursReviewerHandoffs(evidence, artifacts)).not.toThrow();
expect(() => checkHandoffs(evidence, artifacts)).not.toThrow();
attempts[1].input!.prompt = String(attempts[1].input!.prompt).replace(problems[5][2], 'Handle encodings.');
evidence.transcript = [];
expect(() => validateOfficeHoursReviewerHandoffs(evidence, artifacts)).toThrow('round 2 did not receive the complete');
expect(() => checkHandoffs(evidence, artifacts)).toThrow('round 2 did not receive the complete');
});
test('accepts a complete numbered Read delivered to the actual child', () => {
const { evidence, artifacts } = handoffs(true);
expect(() => validateOfficeHoursReviewerHandoffs(evidence, artifacts)).not.toThrow();
expect(() => checkHandoffs(evidence, artifacts)).not.toThrow();
});
test.each(['parent', 'wrong child', 'wrong result ID', 'partial', 'after completion', 'wrong path', 'error', 'missing IDs'])
@@ -780,7 +789,7 @@ describe('office-hours mechanical review evidence', () => {
delete read.parent_tool_use_id; delete result.parent_tool_use_id;
delete read.message.content[0].id; delete result.message.content[0].tool_use_id;
}
expect(() => validateOfficeHoursReviewerHandoffs(evidence, artifacts)).toThrow('round 1 did not receive the complete');
expect(() => checkHandoffs(evidence, artifacts)).toThrow('round 1 did not receive the complete');
});
test('retains genuine native reviewer unavailability before Read, without waiving malformed completed output', () => {
@@ -790,18 +799,18 @@ describe('office-hours mechanical review evidence', () => {
const completion = transcript[7];
completion.message.content[0].is_error = true;
evidence.transcript = [...transcript.slice(0, 5), completion];
expect(() => validateOfficeHoursReviewerHandoffs(evidence, artifacts)).not.toThrow();
expect(() => checkHandoffs(evidence, artifacts)).not.toThrow();
delete completion.message.content[0].is_error;
expect(() => validateOfficeHoursReviewerHandoffs(evidence, artifacts)).toThrow('round 2 did not receive the complete');
expect(() => checkHandoffs(evidence, artifacts)).toThrow('round 2 did not receive the complete');
});
test('a shortened schema fails even when a failed verdict is declared unreviewed', () => {
const { evidence, artifacts, attempts } = handoffs();
attempts[1].output = '{invalid';
evidence.transcript = [];
expect(() => validateOfficeHoursReviewerHandoffs(evidence, artifacts)).not.toThrow();
expect(() => checkHandoffs(evidence, artifacts)).not.toThrow();
attempts[1].input!.prompt = String(attempts[1].input!.prompt).replace('"remedy":', '"summary":');
expect(() => validateOfficeHoursReviewerHandoffs(evidence, artifacts)).toThrow('round 2 did not receive the complete');
expect(() => checkHandoffs(evidence, artifacts)).toThrow('round 2 did not receive the complete');
});
test('receipts bind the exact saved bytes in both validators', () => {
@@ -809,27 +818,27 @@ describe('office-hours mechanical review evidence', () => {
const changed = JSON.parse(artifacts[0].content!);
changed.findings[0].remedy = 'Different obligation';
artifacts[0].content = JSON.stringify(changed);
expect(() => validateOfficeHoursReviewerHandoffs(evidence, artifacts)).toThrow('round 1 saved verdict differs from the reviewer receipt');
expect(() => validateOfficeHoursReviewArtifacts(evidence, artifacts)).toThrow('round 1 saved verdict differs from the reviewer receipt');
expect(() => checkHandoffs(evidence, artifacts)).toThrow('round 1 saved verdict differs from the reviewer receipt');
expect(() => checkArtifacts(evidence, artifacts)).toThrow('round 1 saved verdict differs from the reviewer receipt');
attempts[0].output = attempts[0].output!.replace(/sha256=[0-9a-f]{64}/, `sha256=${'0'.repeat(64)}`);
expect(() => validateOfficeHoursReviewerHandoffs(evidence, artifacts)).toThrow('receipt hash does not match');
expect(() => checkHandoffs(evidence, artifacts)).toThrow('receipt hash does not match');
});
test('a stale earlier-round receipt cannot satisfy a later round', () => {
const { evidence, artifacts, attempts } = handoffs(true);
attempts[1].output = attempts[0].output;
expect(() => validateOfficeHoursReviewerHandoffs(evidence, artifacts)).toThrow('round 2 saved verdict differs from the reviewer receipt');
expect(() => validateOfficeHoursReviewArtifacts(evidence, artifacts)).toThrow('round 2 saved verdict differs from the reviewer receipt');
expect(() => checkHandoffs(evidence, artifacts)).toThrow('round 2 saved verdict differs from the reviewer receipt');
expect(() => checkArtifacts(evidence, artifacts)).toThrow('round 2 saved verdict differs from the reviewer receipt');
attempts[1].output = officeHoursVerdictReceipt(2, artifacts[0].path, artifacts[0].content!);
expect(() => validateOfficeHoursReviewArtifacts(evidence, artifacts)).toThrow('different verdict path');
expect(() => checkArtifacts(evidence, artifacts)).toThrow('different verdict path');
artifacts[1].content = artifacts[0].content;
attempts[1].output = officeHoursVerdictReceipt(2, artifacts[1].path, artifacts[1].content!);
expect(() => validateOfficeHoursReviewArtifacts(evidence, artifacts)).toThrow('round 2 saved verdict is invalid');
expect(() => checkArtifacts(evidence, artifacts)).toThrow('round 2 saved verdict is invalid');
});
test('judge evidence comes from the saved verdict bytes, not the receipt', () => {
const { evidence, artifacts } = structured();
const judged = validateOfficeHoursReviewArtifacts(evidence, artifacts);
const judged = checkArtifacts(evidence, artifacts);
expect(judged?.verdict).toBe(artifacts[1].content!);
expect(judged?.priorVerdicts).toEqual([artifacts[0].content!]);
});
@@ -837,7 +846,7 @@ describe('office-hours mechanical review evidence', () => {
test('a separate coaching closing does not hide the actual Handoff recommendation', () => {
const { evidence, artifacts } = structured();
evidence.output = evidence.output.replace('<!-- gstack:office-hours:report:start -->', '## Relationship Closing\nYou prioritize clear demand evidence.\n\n<!-- gstack:office-hours:report:start -->');
expect(() => validateOfficeHoursReviewArtifacts(evidence, artifacts)).not.toThrow();
expect(() => checkArtifacts(evidence, artifacts)).not.toThrow();
evidence.output = evidence.output.replace('/plan-eng-review', 'another discussion');
expect(() => validateOfficeHoursCompletion(evidence)).toThrow('next-skill recommendation');
});
@@ -853,7 +862,7 @@ describe('office-hours mechanical review evidence', () => {
evidence.toolCalls = evidence.toolCalls.slice(0, 4);
evidence.toolCalls[2].output = officeHoursVerdictReceipt(1, saved[0].path, saved[0].content);
evidence.toolCalls[3].input!.content = saved[0].content;
expect(() => validateOfficeHoursReviewArtifacts(evidence, saved)).not.toThrow();
expect(() => checkArtifacts(evidence, saved)).not.toThrow();
expect(evidence.designContent).toContain('Disposition: COMPLETED');
for (const [, problem, remedy] of problems) {
expect(evidence.designContent).toContain(problem);
@@ -861,9 +870,26 @@ describe('office-hours mechanical review evidence', () => {
}
});
test.each([['inside', 'revised after round 1'], ['outside', 'Observe Lee reconcile an event unaided']])
('a new round-2 blocking regression citing text %s the captured diff', (where, citation) => {
const { evidence, artifacts, rounds } = structured();
rounds[1].findings.push({ id: 'R2-7', dimension: 'feasibility', severity: 'blocking', changed_text: citation,
problem: 'The revision note claims a fix the design does not contain.', remedy: 'Remove the note or make the fix.' });
rounds[1].dimensions.feasibility = 'ISSUES';
const rendered = renderOfficeHoursReview(rounds);
evidence.designContent = evidence.designContent!.replace(/<!-- gstack:office-hours:concerns:start -->[\s\S]*<!-- gstack:office-hours:concerns:end -->/, rendered.concerns);
evidence.output = evidence.output.replace(/<!-- gstack:office-hours:report:start -->[\s\S]*<!-- gstack:office-hours:report:end -->/, rendered.report);
artifacts[1].content = JSON.stringify(rounds[1]);
evidence.toolCalls.at(-2)!.output = officeHoursVerdictReceipt(2, artifacts[1].path, artifacts[1].content);
evidence.toolCalls.at(-1)!.input!.content = artifacts[1].content;
if (where === 'inside') expect(() => checkArtifacts(evidence, artifacts)).not.toThrow();
else expect(() => checkArtifacts(evidence, artifacts)).toThrow('round 2 citation');
expect(() => checkArtifacts(evidence, artifacts, [])).toThrow('lacks its captured design snapshot');
});
test('accepts a complete, reviewer-owned six-finding report without a judge', () => {
const { evidence, artifacts } = structured();
expect(() => validateOfficeHoursReviewArtifacts(evidence, artifacts)).not.toThrow();
expect(() => checkArtifacts(evidence, artifacts)).not.toThrow();
});
test('the real formatter completes a linked closing report without replaying the design or losing reviewer evidence', () => {
@@ -896,8 +922,10 @@ Next: /plan-eng-review after observing the workflow. The user declined launching
fs.mkdirSync(path.dirname(artifact.path), { recursive: true });
fs.writeFileSync(artifact.path, artifact.content);
}
const snapshots = snapshotsIn(path.dirname(artifacts[0].path));
for (const snapshot of snapshots) fs.writeFileSync(snapshot.path, snapshot.content);
const beforeDesign = fs.readFileSync(evidence.designPath);
expect(() => validateOfficeHoursReviewArtifacts({ ...evidence, output: closing }, artifacts)).toThrow('Disposition');
expect(() => checkArtifacts({ ...evidence, output: closing }, artifacts, snapshots)).toThrow('Disposition');
const result = Bun.spawnSync([process.execPath, path.resolve(import.meta.dir, '../bin/gstack-office-hours-review'),
'finalize', '--design', evidence.designPath, '--report', reportPath, ...artifacts.map((artifact: { path: string }) => artifact.path)],
{ cwd: dir, timeout: 5000 });
@@ -914,11 +942,11 @@ Next: /plan-eng-review after observing the workflow. The user declined launching
}
evidence.output = output;
expect(() => validateOfficeHoursCompletion(evidence)).not.toThrow();
expect(() => validateOfficeHoursReviewArtifacts(evidence, artifacts)).not.toThrow();
expect(() => validateOfficeHoursReviewerHandoffs(evidence, artifacts)).not.toThrow();
expect(() => checkArtifacts(evidence, artifacts, snapshots)).not.toThrow();
expect(() => checkHandoffs(evidence, artifacts, snapshots)).not.toThrow();
expect(() => validateOfficeHoursCompletion({ ...evidence, exitReason: 'timeout' })).toThrow('execution failed: timeout');
const changed = output.replace(rounds.at(-1).findings[0].remedy, 'A shorter different remedy.');
expect(() => validateOfficeHoursReviewArtifacts({ ...evidence, output: changed }, artifacts)).toThrow();
expect(() => checkArtifacts({ ...evidence, output: changed }, artifacts, snapshots)).toThrow();
} finally { fs.rmSync(dir, { recursive: true, force: true }); }
});
@@ -926,9 +954,9 @@ Next: /plan-eng-review after observing the workflow. The user declined launching
const { evidence, artifacts } = structured();
evidence.output = evidence.output.replace('## Handoff', '---\n\n## Handoff');
evidence.designContent += '\n\n* * *\n';
expect(() => validateOfficeHoursReviewArtifacts(evidence, artifacts)).not.toThrow();
expect(() => checkArtifacts(evidence, artifacts)).not.toThrow();
evidence.output = evidence.output.replace('---\n\n## Handoff', 'All issues were fixed.\n\n---\n\n## Handoff');
expect(() => validateOfficeHoursReviewArtifacts(evidence, artifacts)).toThrow('computed metrics');
expect(() => checkArtifacts(evidence, artifacts)).toThrow('computed metrics');
});
test.each(['fence', 'prose', 'echoed JSON', 'missing path'])('a %s response is a failed attempt, never a completed verdict', mode => {
@@ -938,68 +966,68 @@ Next: /plan-eng-review after observing the workflow. The user declined launching
if (mode === 'prose') review.output = `Saved.\n${review.output}`;
if (mode === 'echoed JSON') review.output = JSON.stringify(rounds[1]);
if (mode === 'missing path') review.output = review.output!.replace(/ path=.*$/, '');
expect(() => validateOfficeHoursReviewArtifacts(evidence, artifacts)).toThrow('contradicts');
expect(() => checkArtifacts(evidence, artifacts)).toThrow('contradicts');
});
test('rejects the observed encoding obligation omission even if a judge would accept it', () => {
const { evidence, artifacts } = structured();
evidence.designContent = evidence.designContent!.replace(problems[5][2], '');
expect(() => validateOfficeHoursReviewArtifacts(evidence, artifacts)).toThrow('every saved problem and remedy');
expect(() => checkArtifacts(evidence, artifacts)).toThrow('every saved problem and remedy');
});
test('rejects a five-finding count for the actual six', () => {
const { evidence, artifacts } = structured();
evidence.output = evidence.output.replace('Unresolved findings in the last completed inventory: 6 (6 blocking, 0 minor).',
'Unresolved findings in the last completed inventory: 5 (5 blocking, 0 minor).');
expect(() => validateOfficeHoursReviewArtifacts(evidence, artifacts)).toThrow('computed metrics');
expect(() => checkArtifacts(evidence, artifacts)).toThrow('computed metrics');
});
test('rejects a changed saved verdict even when the report was faithfully generated from it', () => {
const { evidence, artifacts, rounds } = structured();
rounds[1].findings[5].remedy = 'Always silently decode with latin-1.';
artifacts[1].content = JSON.stringify(rounds[1]);
expect(() => validateOfficeHoursReviewArtifacts(evidence, artifacts)).toThrow('differs from the reviewer receipt');
expect(() => checkArtifacts(evidence, artifacts)).toThrow('differs from the reviewer receipt');
});
test('requires the actual saved file, not just a reviewer response', () => {
const { evidence, artifacts } = structured();
expect(() => validateOfficeHoursReviewArtifacts(evidence, artifacts.slice(0, 1))).toThrow('exactly one saved verdict');
expect(() => checkArtifacts(evidence, artifacts.slice(0, 1))).toThrow('exactly one saved verdict');
});
test('requires an observed Write for the saved artifact', () => {
const { evidence, artifacts } = structured();
evidence.toolCalls = evidence.toolCalls.filter(call => call.input?.file_path !== artifacts[1].path);
expect(() => validateOfficeHoursReviewArtifacts(evidence, artifacts)).toThrow('observed JSON Write');
expect(() => checkArtifacts(evidence, artifacts)).toThrow('observed JSON Write');
});
test('accepts relative and dot-segment Writes to the assigned artifact', () => {
const { evidence, artifacts } = structured();
evidence.toolCalls.find(call => call.input?.file_path === artifacts[0].path)!.input!.file_path = './review/round-1.json';
evidence.toolCalls.find(call => call.input?.file_path === artifacts[1].path)!.input!.file_path = '/tmp/office-hours-fixture/review/./round-2.json';
expect(() => validateOfficeHoursReviewArtifacts(evidence, artifacts)).not.toThrow();
expect(() => checkArtifacts(evidence, artifacts)).not.toThrow();
});
test('a pre-dispatch Write cannot prove the verdict was saved after review', () => {
const { evidence, artifacts } = structured();
const write = evidence.toolCalls.pop()!;
evidence.toolCalls.splice(2, 0, write);
expect(() => validateOfficeHoursReviewArtifacts(evidence, artifacts)).toThrow('observed JSON Write');
expect(() => checkArtifacts(evidence, artifacts)).toThrow('observed JSON Write');
});
test('rejects duplicate round artifacts and a review of a different design', () => {
const { evidence, artifacts } = structured();
expect(() => validateOfficeHoursReviewArtifacts(evidence, [...artifacts, artifacts[1]])).toThrow('exactly one saved verdict');
expect(() => checkArtifacts(evidence, [...artifacts, artifacts[1]])).toThrow('exactly one saved verdict');
const review = evidence.toolCalls.findLast(call => call.tool === 'Agent')!;
const json = JSON.parse(artifacts[1].content!); json.document = '/tmp/different-design.md';
artifacts[1].content = JSON.stringify(json);
review.output = officeHoursVerdictReceipt(2, artifacts[1].path, artifacts[1].content);
expect(() => validateOfficeHoursReviewArtifacts(evidence, artifacts)).toThrow();
expect(() => checkArtifacts(evidence, artifacts)).toThrow();
});
test('cannot hide a valid completed verdict behind an unreviewed declaration', () => {
const { evidence, artifacts } = structured();
evidence.output = evidence.output.replace('Disposition: CONCERNS_RECORDED', 'Disposition: UNREVIEWED');
expect(() => validateOfficeHoursReviewArtifacts(evidence, artifacts)).toThrow('contradicts');
expect(() => checkArtifacts(evidence, artifacts)).toThrow('contradicts');
});
test.each(['', 'timeout', '{invalid'])('keeps a genuine failed first attempt explicit (%s)', output => {
@@ -1008,19 +1036,19 @@ Next: /plan-eng-review after observing the workflow. The user declined launching
evidence.toolCalls[2].output = output;
evidence.designContent = design.replace(/## Reviewer Concerns[\s\S]*/, failure.concerns);
evidence.output = report.replace(/## Spec Review[\s\S]*?(?=## Handoff)/, `${failure.report}\n\n`);
expect(() => validateOfficeHoursReviewArtifacts(evidence, [])).not.toThrow();
expect(() => checkArtifacts(evidence, [])).not.toThrow();
});
test.each(['report', 'design'])('rejects a duplicate section outside the owned %s block', target => {
const { evidence, artifacts } = structured();
if (target === 'report') evidence.output += '\n## Spec Review\nDisposition: COMPLETED\nAll fixed.\n';
else evidence.designContent += '\n## Reviewer Concerns\nNone.\n';
expect(() => validateOfficeHoursReviewArtifacts(evidence, artifacts)).toThrow();
expect(() => checkArtifacts(evidence, artifacts)).toThrow();
});
test('a missing verdict with a completed declaration fails', () => {
const { evidence, artifacts } = structured();
evidence.toolCalls.findLast(call => call.tool === 'Agent')!.output = '';
expect(() => validateOfficeHoursReviewArtifacts(evidence, artifacts)).toThrow('final spec-review output is missing');
expect(() => checkArtifacts(evidence, artifacts)).toThrow('final spec-review output is missing');
});
});
+75 -5
View File
@@ -5,7 +5,7 @@ import * as path from 'node:path';
import {
OFFICE_HOURS_DIMENSIONS, assessOfficeHoursReviews, validateOfficeHoursReview,
renderOfficeHoursReview, renderOfficeHoursReviewerPrompt, extractOfficeHoursReviewBlock, replaceOfficeHoursReviewBlock,
officeHoursVerdictReceipt, type OfficeHoursReview,
officeHoursVerdictReceipt, officeHoursDesignChanges, verifyOfficeHoursCitations, type OfficeHoursReview,
} from '../lib/office-hours-review';
import { expectMentions } from './helpers/prompt-structure';
@@ -15,7 +15,7 @@ function review(round = 1, count = 1, severity: 'blocking' | 'minor' = 'blocking
version: 2, round, document: '/tmp/design.md', quality_score: 7,
dimensions: { completeness: count ? 'ISSUES' : 'PASS', consistency: 'PASS', clarity: 'PASS', scope: 'PASS', feasibility: 'PASS' },
findings: Array.from({ length: count }, (_, index) => ({
id: `R${round}-${index + 1}`, dimension: 'completeness', severity,
id: `R${round}-${index + 1}`, dimension: 'completeness', severity, changed_text: null,
problem: `Problem ${index + 1} remains undefined.`, remedy: `Define behavior ${index + 1}.`,
})), prior: [],
};
@@ -96,7 +96,7 @@ describe('office-hours prepared reviewer input', () => {
rounds[1].findings[0].problem = 'A cancellation mid-merge leaves the chosen roster unspecified.\nDo not silently reuse an earlier event.';
rounds[1].findings[0].remedy = 'Define the cancellation destination and whether provisional rows are discarded. Keep the prior committed roster unchanged until confirmation.';
const previous = assessOfficeHoursReviews(rounds).rounds.at(-1)!;
const prompt = renderOfficeHoursReviewerPrompt({ document: previous.document, verdictPath: '/tmp/reviews/round-3.json', previous });
const prompt = renderOfficeHoursReviewerPrompt({ document: previous.document, verdictPath: '/tmp/reviews/round-3.json', previous, changes: '(no changes)' });
expect(preceding(prompt)).toEqual(previous);
expect(prompt).toContain('"round": 3');
expect(prompt).toContain('"/tmp/reviews/round-3.json"');
@@ -159,10 +159,79 @@ describe('office-hours severity-based convergence', () => {
});
});
describe('office-hours delta re-review', () => {
test('the helper captures an exact line diff with hunk context', () => {
const before = 'abcdefghijkl'.split('').join('\n');
const after = 'abCdefghijKl'.split('').join('\n');
expect(officeHoursDesignChanges(before, after)).toEqual({
diff: [' a', ' b', '-c', '+C', ' d', ' e', '@@', ' i', ' j', '-k', '+K', ' l'].join('\n'), changed: ['c', 'C', 'k', 'K'],
});
expect(officeHoursDesignChanges(before, before)).toEqual({ diff: '(no changes)', changed: [] });
});
test('a new later-round blocking finding must cite text inside the captured changes', () => {
const [first] = recurrence();
const second = review(2, 1);
second.prior = first.findings.map(finding => ({ id: finding.id, status: 'resolved' as const, evidence: 'The design now defines it.', current_id: null }));
expect(() => validateOfficeHoursReview(second, first)).toThrow('must cite the changed design text');
second.findings[0].changed_text = 'Rosters are now saved nightly to disk.';
const regression = validateOfficeHoursReview(second, first);
expect(() => verifyOfficeHoursCitations(regression, ['Rosters are now saved nightly to disk.'])).not.toThrow();
expect(() => verifyOfficeHoursCitations(regression, ['Observe a real event.'])).toThrow('not inside the design changes');
const round1 = review(1, 1);
round1.findings[0].changed_text = 'Rosters are now saved nightly to disk.';
expect(() => validateOfficeHoursReview(round1)).toThrow('changed_text must be null in round 1');
second.findings[0].changed_text = 'short';
expect(() => validateOfficeHoursReview(second, first)).toThrow('at least 8 characters');
});
test('a persisting prior keeps its finding without a citation, and later rounds with only minor findings PASS', () => {
expect(assessOfficeHoursReviews(recurrence()).stop).toBe('CONVERGENCE');
const [first] = recurrence();
const second = review(2, 2, 'minor');
second.prior = first.findings.map(finding => ({ id: finding.id, status: 'resolved' as const, evidence: 'The design now defines it.', current_id: null }));
expect(assessOfficeHoursReviews([first, second]).stop).toBe('PASS');
});
test('prepare, seal and check enforce citations against the snapshots prepare captured', () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'office-delta-'));
try {
const run = (...args: string[]) => Bun.spawnSync([process.execPath, cli, ...args], { cwd: dir, stdout: 'pipe', stderr: 'pipe', timeout: 5000 });
const design = path.join(dir, 'design.md'), outDir = path.join(dir, 'reviews');
fs.writeFileSync(design, '# Design\n\n## Approach\nRosters stay in memory.\n\n## Assignment\nObserve a real event.\n');
fs.mkdirSync(outDir);
const first = JSON.parse(String(run('prepare', '--design', design, '--out-dir', outDir).stdout));
expect(fs.readFileSync(path.join(outDir, 'round-1.design.md'), 'utf8')).toBe(fs.readFileSync(design, 'utf8'));
const round1 = { ...review(1, 1), document: fs.realpathSync(design) };
fs.appendFileSync(design, 'Rosters are now saved nightly to disk.\n');
expect(String(run('prepare', '--design', design, '--out-dir', outDir).stderr)).toContain('design changed after round 1 was prepared');
fs.writeFileSync(first.verdictPath, JSON.stringify(round1));
const second = JSON.parse(String(run('prepare', '--design', design, '--out-dir', outDir, first.verdictPath).stdout));
expect(fs.readFileSync(second.promptPath, 'utf8')).toContain('\n+Rosters are now saved nightly to disk.\n');
expect(fs.readFileSync(second.promptPath, 'utf8')).toContain('## Delta re-review scope');
const round2 = { ...review(2, 1), document: round1.document,
prior: [{ id: 'R1-1', status: 'resolved', evidence: 'The Approach now defines it.', current_id: null }] };
const seal = () => run('seal', '--design', design, '--verdict', second.verdictPath, first.verdictPath);
round2.findings[0].changed_text = 'Observe a real event.';
fs.writeFileSync(second.verdictPath, JSON.stringify(round2));
expect(String(seal().stderr)).toContain('not inside the design changes');
const receipt = (file: string) => officeHoursVerdictReceipt(2, file, fs.readFileSync(file));
expect(String(run('check', '--receipt', receipt(second.verdictPath), first.verdictPath, second.verdictPath).stderr)).toContain('not inside the design changes');
round2.findings[0].changed_text = 'saved nightly to disk';
fs.writeFileSync(second.verdictPath, JSON.stringify(round2));
expect(seal().exitCode, String(seal().stderr)).toBe(0);
const check = run('check', '--receipt', receipt(second.verdictPath), first.verdictPath, second.verdictPath);
expect(JSON.parse(String(check.stdout)).stop).toBe('CONTINUE');
fs.unlinkSync(path.join(outDir, 'round-2.design.md'));
expect(String(run('check', '--receipt', receipt(second.verdictPath), first.verdictPath, second.verdictPath).stderr)).toContain('lacks its captured design snapshot');
} finally { fs.rmSync(dir, { recursive: true, force: true }); }
});
});
describe('office-hours lifecycle and deterministic preservation', () => {
test('convergence precedes further editing even when another finding is new', () => {
const rounds = recurrence();
rounds[1].findings.push({ id: 'R2-2', dimension: 'clarity', severity: 'minor', problem: 'Encoding fallback has no user-visible policy.', remedy: 'Specify whether the latin-1 fallback emits a warning or is intentionally silent.' });
rounds[1].findings.push({ id: 'R2-2', dimension: 'clarity', severity: 'minor', changed_text: null, problem: 'Encoding fallback has no user-visible policy.', remedy: 'Specify whether the latin-1 fallback emits a warning or is intentionally silent.' });
rounds[1].dimensions.clarity = 'ISSUES';
const result = renderOfficeHoursReview(rounds);
expect(result.stop).toBe('CONVERGENCE');
@@ -328,6 +397,7 @@ describe('office-hours review CLI', () => {
item.document = design;
const file = path.join(dir, `round-${item.round}.json`);
fs.writeFileSync(file, JSON.stringify(item));
fs.writeFileSync(path.join(dir, `round-${item.round}.design.md`), fs.readFileSync(design, 'utf8'));
return file;
});
return { design, report, artifacts, rounds };
@@ -355,7 +425,7 @@ describe('office-hours review CLI', () => {
expect(second.exitCode, second.stderr.toString()).toBe(0);
const next = JSON.parse(second.stdout.toString());
expect(next.round).toBe(2);
expect(fs.readFileSync(next.promptPath, 'utf8')).toBe(renderOfficeHoursReviewerPrompt({ document: data.design, verdictPath: next.verdictPath, previous: data.rounds[0] }));
expect(fs.readFileSync(next.promptPath, 'utf8')).toBe(renderOfficeHoursReviewerPrompt({ document: data.design, verdictPath: next.verdictPath, previous: data.rounds[0], changes: '(no changes)' }));
expect(fs.readFileSync(prepared.verdictPath, 'utf8')).toBe(saved);
expect(fs.readFileSync(data.design, 'utf8')).toBe(before);
}));
@@ -75,14 +75,17 @@ Completion delivery, after the full workflow and design approval:
const artifacts = artifactPaths.map(artifactPath => ({ path: artifactPath,
content: fs.existsSync(artifactPath) ? fs.readFileSync(artifactPath, 'utf-8') : null,
}));
const snapshots = [...new Set(artifactPaths.map(artifactPath => path.dirname(artifactPath)))].filter(dir => fs.existsSync(dir)).flatMap(dir =>
fs.readdirSync(dir).filter(name => /^round-[123]\.design\.md$/.test(name))
.map(name => ({ path: path.join(dir, name), content: fs.readFileSync(path.join(dir, name), 'utf-8') })));
const reviewEvidence = validateOfficeHoursReviewArtifacts({
...capture, designPath,
designContent: fs.existsSync(designPath) ? fs.readFileSync(designPath, 'utf-8') : null,
}, artifacts);
}, artifacts, snapshots);
validateOfficeHoursReviewerHandoffs({
...capture, designPath,
designContent: fs.existsSync(designPath) ? fs.readFileSync(designPath, 'utf-8') : null,
}, artifacts);
}, artifacts, snapshots);
const missing = guard.requiredReads.filter(section => !capture.readSections.has(section));
expect({ reportProduced: capture.reportProduced, read: [...capture.readSections], missing }).toEqual({
reportProduced: true, read: expect.any(Array), missing: [],