diff --git a/test/plan-design-sdk-fixture.test.ts b/test/plan-design-sdk-fixture.test.ts index c42e04e72..04f18c4e3 100644 --- a/test/plan-design-sdk-fixture.test.ts +++ b/test/plan-design-sdk-fixture.test.ts @@ -14,7 +14,7 @@ const id = 'plan-design-review-plan-mode'; // Source-evaluation pattern: run the real suite registration and selected // callback, without importing paid initialization. All filesystem mutations // stay in this standalone fixture. -async function exercise(mode: 'success' | 'max-turns' | 'first-timeout' | 'second-timeout' | 'saved-timeout' | 'empty-summary' | 'write-failure' | 'unchanged-seed' | 'no-additions' | 'short-plan' | 'api-error' | 'plan-read-failure' | 'attempt-deadline') { +async function exercise(mode: 'success' | 'max-turns' | 'first-timeout' | 'second-timeout' | 'saved-timeout' | 'empty-summary' | 'rated-summary' | 'write-failure' | 'unchanged-seed' | 'no-additions' | 'short-plan' | 'api-error' | 'plan-read-failure' | 'attempt-deadline') { const scratch = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'legacy-design-free-'))); const home = path.join(scratch, 'home'); fs.mkdirSync(home); const env = { PATH: process.env.PATH ?? '', HOME: home, GIT_CONFIG_NOSYSTEM: '1', @@ -115,7 +115,8 @@ async function exercise(mode: 'success' | 'max-turns' | 'first-timeout' | 'secon 'Interaction state table: loading, empty, error and success. Responsive and accessibility decisions recorded.\n'; fs.writeFileSync(plan, reviewed); saved = fs.readFileSync(plan, 'utf8'); } - return { output: timedOut ? '' : mode === 'empty-summary' ? 'Done.' : 'Saved the information architecture and interaction state decisions in plan.md.', + return { output: timedOut ? '' : mode === 'empty-summary' ? 'Done.' : + mode === 'rated-summary' ? 'Design review saved. Overall design score 0/10 → 8/10; state table and tokens added.' : 'Saved the information architecture and interaction state decisions in plan.md.', exitReason: timedOut ? 'timeout' : mode === 'max-turns' ? 'error_max_turns' : mode === 'api-error' ? 'error_api' : 'success', duration: timedOut ? (mode === 'first-timeout' ? 300039 : 300032) : 1, toolCalls: [], browseErrors: [], transcript: [], model: process.env.EVALS_MODEL ?? resolveEvalModel('capture'), @@ -157,6 +158,12 @@ test('legacy Design still requires substantive output after a saved plan', async expect(result.resultError).toBeDefined(); expect(result.rows.map(row => row.passed)).toEqual([false]); }); +// Census 37179171083: a rated summary whose design terms live in the plan the review wrote. +test('legacy Design credits design terms the review added to the plan behind a rated summary', async () => { + const result = await exercise('rated-summary'); + expect(result.resultError).toBeUndefined(); expect(result.rows.map(row => row.passed)).toEqual([true]); +}); + test('legacy Design cleans the fixture when its write fails', async () => { const result = await exercise('write-failure'); expect(String(result.resultError)).toContain('controlled plan write failure'); expect(result.rows).toHaveLength(1); diff --git a/test/skill-e2e-design.test.ts b/test/skill-e2e-design.test.ts index 3f86921e8..02cbb5b97 100644 --- a/test/skill-e2e-design.test.ts +++ b/test/skill-e2e-design.test.ts @@ -490,12 +490,11 @@ IMPORTANT: Do NOT try to browse any URLs or use a browse binary. This is a plan // Check that the plan file was edited (the core new behavior) const planAfter = fs.readFileSync(path.join(reviewDir, 'plan.md'), 'utf-8'); - // The review's design content is what it wrote into the plan; the - // final message may only summarize it (census 37179171083). - // A term counts from the plan only when the review added it. + // A rated summary may leave the design terms to the plan it wrote + // (census 37179171083); a plan term counts only when the review added it. const count = (text: string, term: string) => text.toLowerCase().split(term).length - 1; const hasDesignContent = ['information architecture', 'interaction state', 'ai slop', 'hierarchy'].some(term => - output.toLowerCase().includes(term) || count(planAfter, term) > count(planBefore, term)); + output.toLowerCase().includes(term) || (hasRatings && count(planAfter, term) > count(planBefore, term))); const planWasEdited = planAfter !== planBefore && planAfter.length > 300; const planHasDesignAdditions = planAfter.toLowerCase().includes('empty') || planAfter.toLowerCase().includes('loading') ||