mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-08 04:11:18 +02:00
fix(review): pass Review Army checklists by path, run research alongside dispatch, always probe the design detector; state review-log invocation and statuses in the caller fixture
- review-army-perf-n-plus-one: the parent copied full checklists into agent prompts and ran web research before dispatch (290 s on a 12-line diff); 212 s now. - review-design-lite: 5 of 6 captured trials reported the detector absent without probing; the probe is mandatory and its first line is reported, and the contract credits only fake-engine rule ids the checklist never names. - review-exploratory-small-cli: the fixture never gave review-log's direct invocation or status vocabulary; the model ran it through bun and wrote status "blocked". The prompt states both and the validator rejects out-of-vocabulary review statuses. Each case passed a focused paid run after repair.
This commit is contained in:
1 parent
a18e6cf655
commit
a06d22e52a
15 files changed
+155
-62
No files matched your search
@@ -11,12 +11,17 @@ const CASES = [
|
||||
['review-enum-completeness', 300, 15],
|
||||
['review-design-lite', 400, 35],
|
||||
] as const;
|
||||
for (const [id, workMs, maxTurns] of CASES) {
|
||||
test.each(['success', 'timeout'])(`${id} records late results before finalization: %s`, scenario => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'review-finalization-'));
|
||||
const script = path.join(dir, 'registration.test.ts');
|
||||
const facts = path.join(dir, 'events.jsonl');
|
||||
fs.writeFileSync(script, `
|
||||
const SYNTHETIC_REPORT = 'SQL injection. Returned enum status critical. Papyrus font family;14px font-size;outline focus;!important;purple gradient;generic hero copy;3-column feature grid;detector [low-contrast] x3.';
|
||||
const DESIGN_CAPTURES = JSON.parse(fs.readFileSync(path.join(ROOT, 'test/fixtures/review-design-lite-reports-ci-36633323521.json'), 'utf8')) as {
|
||||
reports: Array<{ run: string; trial: string; scanRan: boolean; report: string }>;
|
||||
};
|
||||
|
||||
function runRegistration(id: string, scenario: string, report: string) {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'review-finalization-'));
|
||||
const script = path.join(dir, 'registration.test.ts');
|
||||
const facts = path.join(dir, 'events.jsonl');
|
||||
const evalDir = path.join(dir, 'eval');
|
||||
fs.writeFileSync(script, `
|
||||
import { describe, expect, mock, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
@@ -50,8 +55,7 @@ mock.module(path.join(root, 'test/helpers/session-runner.ts'), () => ({
|
||||
const target = selected === 'review-enum-completeness'
|
||||
? opts.prompt.match(/Write your review findings once to (\\S+)/)[1]
|
||||
: path.join(opts.workingDirectory, 'review-output.md');
|
||||
fs.writeFileSync(target,
|
||||
'SQL injection. Returned enum status critical. Papyrus font family;14px font-size;outline focus;!important;purple gradient;generic hero copy;3-column feature grid;impeccable detector [ai-color-palette].');
|
||||
fs.writeFileSync(target, ${JSON.stringify(report)});
|
||||
}
|
||||
return { attemptId: id, exitReason: timeout ? 'timeout' : 'success', duration: opts.timeout,
|
||||
model: 'free-fixture-model', toolCalls: [], browseErrors: [], output: '', transcript: [],
|
||||
@@ -60,15 +64,22 @@ mock.module(path.join(root, 'test/helpers/session-runner.ts'), () => ({
|
||||
}));
|
||||
await import(path.join(root, ${JSON.stringify(PAID_FILE)}));
|
||||
`);
|
||||
const retries = retriesForFiles([PAID_FILE]);
|
||||
expect(retries).toBe(0);
|
||||
const child = Bun.spawnSync([process.execPath, ...buildPaidShardArgs([script], resolvePaidShardTimeoutMs([PAID_FILE]), 2, retries)], {
|
||||
cwd: ROOT, timeout: 15_000, stdout: 'pipe', stderr: 'pipe',
|
||||
env: { ...process.env, EVALS: '', EVALS_ALL: '', TMPDIR: dir, TMP: dir, TEMP: dir, GSTACK_EVAL_DIR: evalDir },
|
||||
});
|
||||
const violations = path.join(evalDir, 'contract-violations.jsonl');
|
||||
return { dir, facts, exitCode: child.exitCode, output: child.stdout.toString() + child.stderr.toString(),
|
||||
violations: fs.existsSync(violations) ? fs.readFileSync(violations, 'utf8') : '' };
|
||||
}
|
||||
|
||||
for (const [id, workMs, maxTurns] of CASES) {
|
||||
test.each(['success', 'timeout'])(`${id} records late results before finalization: %s`, scenario => {
|
||||
const { dir, facts, exitCode, output } = runRegistration(id, scenario, SYNTHETIC_REPORT);
|
||||
try {
|
||||
const retries = retriesForFiles([PAID_FILE]);
|
||||
expect(retries).toBe(0);
|
||||
const child = Bun.spawnSync([process.execPath, ...buildPaidShardArgs([script], resolvePaidShardTimeoutMs([PAID_FILE]), 2, retries)], {
|
||||
cwd: ROOT, timeout: 15_000, stdout: 'pipe', stderr: 'pipe',
|
||||
env: { ...process.env, EVALS: '', EVALS_ALL: '', TMPDIR: dir, TMP: dir, TEMP: dir },
|
||||
});
|
||||
const output = child.stdout.toString() + child.stderr.toString();
|
||||
expect(child.exitCode, output).toBe(scenario === 'success' ? 0 : 1);
|
||||
expect(exitCode, output).toBe(scenario === 'success' ? 0 : 1);
|
||||
expect(output).not.toContain('Unhandled error between tests');
|
||||
const events = fs.readFileSync(facts, 'utf8').trim().split('\n').map(line => JSON.parse(line));
|
||||
const starts = events.filter(event => event.kind === 'start');
|
||||
@@ -88,3 +99,12 @@ await import(path.join(root, ${JSON.stringify(PAID_FILE)}));
|
||||
} finally { fs.rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
}
|
||||
|
||||
test.each(DESIGN_CAPTURES.reports.map(capture => [`${capture.run} ${capture.trial}`, capture] as const))(
|
||||
'review-design-lite credits detector rows in captured report %s only when the scan ran', (_label, capture) => {
|
||||
const { dir, exitCode, output, violations } = runRegistration('review-design-lite', 'success', capture.report);
|
||||
try {
|
||||
expect(exitCode, output).toBe(capture.scanRan ? 0 : 1);
|
||||
expect(violations.includes('the review omitted the mechanical detector rows'), output).toBe(!capture.scanRan);
|
||||
} finally { fs.rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
Reference in new issue
Block a user