mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-04 10:26:52 +02:00
fix(review): pass Review Army checklists by path, run research alongside dispatch, always probe the design detector; state review-log invocation and statuses in the caller fixture
- review-army-perf-n-plus-one: the parent copied full checklists into agent prompts and ran web research before dispatch (290 s on a 12-line diff); 212 s now. - review-design-lite: 5 of 6 captured trials reported the detector absent without probing; the probe is mandatory and its first line is reported, and the contract credits only fake-engine rule ids the checklist never names. - review-exploratory-small-cli: the fixture never gave review-log's direct invocation or status vocabulary; the model ran it through bun and wrote status "blocked". The prompt states both and the validator rejects out-of-vocabulary review statuses. Each case passed a focused paid run after repair.
This commit is contained in:
1 parent
a18e6cf655
commit
a06d22e52a
15 files changed
+155
-62
No files matched your search
@@ -13,7 +13,7 @@ import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
import { installFakeImpeccable } from './helpers/fake-impeccable';
|
||||
import { carriesDetectorRows, installFakeImpeccable } from './helpers/fake-impeccable';
|
||||
|
||||
const evalCollector = createEvalCollector('e2e-review');
|
||||
// Capture cleanup and recording must finish before Bun starts its retry.
|
||||
@@ -287,8 +287,9 @@ Important: The design checklist should catch issues like blacklisted fonts, smal
|
||||
if (review.includes('welcome to') || review.includes('all-in-one') || review.includes('generic') || review.includes('hero copy') || review.includes('ai slop')) detected++;
|
||||
// Issue 7: 3-column feature grid — LOW
|
||||
if (review.includes('3-column') || review.includes('three-column') || review.includes('feature grid') || review.includes('icon') || review.includes('circle')) detected++;
|
||||
// Signal 8: the mechanical pass (fake impeccable engine via IMPECCABLE_BIN) surfaced a detector row
|
||||
const detectorSeen = review.includes('detector') || review.includes('[ai-color-palette]') || review.includes('[low-contrast]') || review.includes('impeccable');
|
||||
// Signal 8: the mechanical pass (fake impeccable engine via IMPECCABLE_BIN) surfaced a detector row.
|
||||
// Only rule ids the checklist never names count; claiming the detector is absent is not a row.
|
||||
const detectorSeen = carriesDetectorRows(review, fs.readFileSync(path.join(designDir, 'review-design-checklist.md'), 'utf-8'));
|
||||
|
||||
console.log(`Design review detected ${detected}/7 planted checklist signals; detector rows surfaced: ${detectorSeen}`);
|
||||
expect(detected).toBeGreaterThanOrEqual(4); // the LLM-checklist bar, unchanged by the detector
|
||||
|
||||
Reference in new issue
Block a user