fix(review): pass Review Army checklists by path, run research alongside dispatch, always probe the design detector; state review-log invocation and statuses in the caller fixture

- review-army-perf-n-plus-one: the parent copied full checklists into agent
  prompts and ran web research before dispatch (290 s on a 12-line diff); 212 s now.
- review-design-lite: 5 of 6 captured trials reported the detector absent
  without probing; the probe is mandatory and its first line is reported, and
  the contract credits only fake-engine rule ids the checklist never names.
- review-exploratory-small-cli: the fixture never gave review-log's direct
  invocation or status vocabulary; the model ran it through bun and wrote
  status "blocked". The prompt states both and the validator rejects
  out-of-vocabulary review statuses.
Each case passed a focused paid run after repair.
This commit is contained in:
garrytan committed 2026-09-29 22:49:12 +00:00
1 parent a18e6cf655
commit a06d22e52a
15 files changed
+155 -62

No files matched your search

+36 -16
View File
@@ -11,12 +11,17 @@ const CASES = [
['review-enum-completeness', 300, 15],
['review-design-lite', 400, 35],
] as const;
for (const [id, workMs, maxTurns] of CASES) {
test.each(['success', 'timeout'])(`${id} records late results before finalization: %s`, scenario => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'review-finalization-'));
const script = path.join(dir, 'registration.test.ts');
const facts = path.join(dir, 'events.jsonl');
fs.writeFileSync(script, `
const SYNTHETIC_REPORT = 'SQL injection. Returned enum status critical. Papyrus font family;14px font-size;outline focus;!important;purple gradient;generic hero copy;3-column feature grid;detector [low-contrast] x3.';
const DESIGN_CAPTURES = JSON.parse(fs.readFileSync(path.join(ROOT, 'test/fixtures/review-design-lite-reports-ci-36633323521.json'), 'utf8')) as {
reports: Array<{ run: string; trial: string; scanRan: boolean; report: string }>;
};
function runRegistration(id: string, scenario: string, report: string) {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'review-finalization-'));
const script = path.join(dir, 'registration.test.ts');
const facts = path.join(dir, 'events.jsonl');
const evalDir = path.join(dir, 'eval');
fs.writeFileSync(script, `
import { describe, expect, mock, test } from 'bun:test';
import * as fs from 'node:fs';
import * as path from 'node:path';
@@ -50,8 +55,7 @@ mock.module(path.join(root, 'test/helpers/session-runner.ts'), () => ({
const target = selected === 'review-enum-completeness'
? opts.prompt.match(/Write your review findings once to (\\S+)/)[1]
: path.join(opts.workingDirectory, 'review-output.md');
fs.writeFileSync(target,
'SQL injection. Returned enum status critical. Papyrus font family;14px font-size;outline focus;!important;purple gradient;generic hero copy;3-column feature grid;impeccable detector [ai-color-palette].');
fs.writeFileSync(target, ${JSON.stringify(report)});
}
return { attemptId: id, exitReason: timeout ? 'timeout' : 'success', duration: opts.timeout,
model: 'free-fixture-model', toolCalls: [], browseErrors: [], output: '', transcript: [],
@@ -60,15 +64,22 @@ mock.module(path.join(root, 'test/helpers/session-runner.ts'), () => ({
}));
await import(path.join(root, ${JSON.stringify(PAID_FILE)}));
`);
const retries = retriesForFiles([PAID_FILE]);
expect(retries).toBe(0);
const child = Bun.spawnSync([process.execPath, ...buildPaidShardArgs([script], resolvePaidShardTimeoutMs([PAID_FILE]), 2, retries)], {
cwd: ROOT, timeout: 15_000, stdout: 'pipe', stderr: 'pipe',
env: { ...process.env, EVALS: '', EVALS_ALL: '', TMPDIR: dir, TMP: dir, TEMP: dir, GSTACK_EVAL_DIR: evalDir },
});
const violations = path.join(evalDir, 'contract-violations.jsonl');
return { dir, facts, exitCode: child.exitCode, output: child.stdout.toString() + child.stderr.toString(),
violations: fs.existsSync(violations) ? fs.readFileSync(violations, 'utf8') : '' };
}
for (const [id, workMs, maxTurns] of CASES) {
test.each(['success', 'timeout'])(`${id} records late results before finalization: %s`, scenario => {
const { dir, facts, exitCode, output } = runRegistration(id, scenario, SYNTHETIC_REPORT);
try {
const retries = retriesForFiles([PAID_FILE]);
expect(retries).toBe(0);
const child = Bun.spawnSync([process.execPath, ...buildPaidShardArgs([script], resolvePaidShardTimeoutMs([PAID_FILE]), 2, retries)], {
cwd: ROOT, timeout: 15_000, stdout: 'pipe', stderr: 'pipe',
env: { ...process.env, EVALS: '', EVALS_ALL: '', TMPDIR: dir, TMP: dir, TEMP: dir },
});
const output = child.stdout.toString() + child.stderr.toString();
expect(child.exitCode, output).toBe(scenario === 'success' ? 0 : 1);
expect(exitCode, output).toBe(scenario === 'success' ? 0 : 1);
expect(output).not.toContain('Unhandled error between tests');
const events = fs.readFileSync(facts, 'utf8').trim().split('\n').map(line => JSON.parse(line));
const starts = events.filter(event => event.kind === 'start');
@@ -88,3 +99,12 @@ await import(path.join(root, ${JSON.stringify(PAID_FILE)}));
} finally { fs.rmSync(dir, { recursive: true, force: true }); }
});
}
test.each(DESIGN_CAPTURES.reports.map(capture => [`${capture.run} ${capture.trial}`, capture] as const))(
'review-design-lite credits detector rows in captured report %s only when the scan ran', (_label, capture) => {
const { dir, exitCode, output, violations } = runRegistration('review-design-lite', 'success', capture.report);
try {
expect(exitCode, output).toBe(capture.scanRan ? 0 : 1);
expect(violations.includes('the review omitted the mechanical detector rows'), output).toBe(!capture.scanRan);
} finally { fs.rmSync(dir, { recursive: true, force: true }); }
});