test(plan-design-review): plan-term credit needs a rated summary; adapter regression for census 37179171083

This commit is contained in:
garrytan committed 2026-10-04 06:13:58 +00:00
1 parent bb93bff5b7
commit 44ef18cd0e
2 files changed
+12 -6

No files matched your search

+9 -2
View File
@@ -14,7 +14,7 @@ const id = 'plan-design-review-plan-mode';
// Source-evaluation pattern: run the real suite registration and selected // Source-evaluation pattern: run the real suite registration and selected
// callback, without importing paid initialization. All filesystem mutations // callback, without importing paid initialization. All filesystem mutations
// stay in this standalone fixture. // stay in this standalone fixture.
async function exercise(mode: 'success' | 'max-turns' | 'first-timeout' | 'second-timeout' | 'saved-timeout' | 'empty-summary' | 'write-failure' | 'unchanged-seed' | 'no-additions' | 'short-plan' | 'api-error' | 'plan-read-failure' | 'attempt-deadline') { async function exercise(mode: 'success' | 'max-turns' | 'first-timeout' | 'second-timeout' | 'saved-timeout' | 'empty-summary' | 'rated-summary' | 'write-failure' | 'unchanged-seed' | 'no-additions' | 'short-plan' | 'api-error' | 'plan-read-failure' | 'attempt-deadline') {
const scratch = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'legacy-design-free-'))); const scratch = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'legacy-design-free-')));
const home = path.join(scratch, 'home'); fs.mkdirSync(home); const home = path.join(scratch, 'home'); fs.mkdirSync(home);
const env = { PATH: process.env.PATH ?? '', HOME: home, GIT_CONFIG_NOSYSTEM: '1', const env = { PATH: process.env.PATH ?? '', HOME: home, GIT_CONFIG_NOSYSTEM: '1',
@@ -115,7 +115,8 @@ async function exercise(mode: 'success' | 'max-turns' | 'first-timeout' | 'secon
'Interaction state table: loading, empty, error and success. Responsive and accessibility decisions recorded.\n'; 'Interaction state table: loading, empty, error and success. Responsive and accessibility decisions recorded.\n';
fs.writeFileSync(plan, reviewed); saved = fs.readFileSync(plan, 'utf8'); fs.writeFileSync(plan, reviewed); saved = fs.readFileSync(plan, 'utf8');
} }
return { output: timedOut ? '' : mode === 'empty-summary' ? 'Done.' : 'Saved the information architecture and interaction state decisions in plan.md.', return { output: timedOut ? '' : mode === 'empty-summary' ? 'Done.' :
mode === 'rated-summary' ? 'Design review saved. Overall design score 0/10 → 8/10; state table and tokens added.' : 'Saved the information architecture and interaction state decisions in plan.md.',
exitReason: timedOut ? 'timeout' : mode === 'max-turns' ? 'error_max_turns' : mode === 'api-error' ? 'error_api' : 'success', exitReason: timedOut ? 'timeout' : mode === 'max-turns' ? 'error_max_turns' : mode === 'api-error' ? 'error_api' : 'success',
duration: timedOut ? (mode === 'first-timeout' ? 300039 : 300032) : 1, duration: timedOut ? (mode === 'first-timeout' ? 300039 : 300032) : 1,
toolCalls: [], browseErrors: [], transcript: [], model: process.env.EVALS_MODEL ?? resolveEvalModel('capture'), toolCalls: [], browseErrors: [], transcript: [], model: process.env.EVALS_MODEL ?? resolveEvalModel('capture'),
@@ -157,6 +158,12 @@ test('legacy Design still requires substantive output after a saved plan', async
expect(result.resultError).toBeDefined(); expect(result.rows.map(row => row.passed)).toEqual([false]); expect(result.resultError).toBeDefined(); expect(result.rows.map(row => row.passed)).toEqual([false]);
}); });
// Census 37179171083: a rated summary whose design terms live in the plan the review wrote.
test('legacy Design credits design terms the review added to the plan behind a rated summary', async () => {
const result = await exercise('rated-summary');
expect(result.resultError).toBeUndefined(); expect(result.rows.map(row => row.passed)).toEqual([true]);
});
test('legacy Design cleans the fixture when its write fails', async () => { test('legacy Design cleans the fixture when its write fails', async () => {
const result = await exercise('write-failure'); const result = await exercise('write-failure');
expect(String(result.resultError)).toContain('controlled plan write failure'); expect(result.rows).toHaveLength(1); expect(String(result.resultError)).toContain('controlled plan write failure'); expect(result.rows).toHaveLength(1);
+3 -4
View File
@@ -490,12 +490,11 @@ IMPORTANT: Do NOT try to browse any URLs or use a browse binary. This is a plan
// Check that the plan file was edited (the core new behavior) // Check that the plan file was edited (the core new behavior)
const planAfter = fs.readFileSync(path.join(reviewDir, 'plan.md'), 'utf-8'); const planAfter = fs.readFileSync(path.join(reviewDir, 'plan.md'), 'utf-8');
// The review's design content is what it wrote into the plan; the // A rated summary may leave the design terms to the plan it wrote
// final message may only summarize it (census 37179171083). // (census 37179171083); a plan term counts only when the review added it.
// A term counts from the plan only when the review added it.
const count = (text: string, term: string) => text.toLowerCase().split(term).length - 1; const count = (text: string, term: string) => text.toLowerCase().split(term).length - 1;
const hasDesignContent = ['information architecture', 'interaction state', 'ai slop', 'hierarchy'].some(term => const hasDesignContent = ['information architecture', 'interaction state', 'ai slop', 'hierarchy'].some(term =>
output.toLowerCase().includes(term) || count(planAfter, term) > count(planBefore, term)); output.toLowerCase().includes(term) || (hasRatings && count(planAfter, term) > count(planBefore, term)));
const planWasEdited = planAfter !== planBefore && planAfter.length > 300; const planWasEdited = planAfter !== planBefore && planAfter.length > 300;
const planHasDesignAdditions = planAfter.toLowerCase().includes('empty') || const planHasDesignAdditions = planAfter.toLowerCase().includes('empty') ||
planAfter.toLowerCase().includes('loading') || planAfter.toLowerCase().includes('loading') ||