mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-31 14:06:50 +01:00
test(plan-design-review): plan-term credit needs a rated summary; adapter regression for census 37179171083
This commit is contained in:
1 parent
bb93bff5b7
commit
44ef18cd0e
2 files changed
+12
-6
No files matched your search
@@ -14,7 +14,7 @@ const id = 'plan-design-review-plan-mode';
|
|||||||
// Source-evaluation pattern: run the real suite registration and selected
|
// Source-evaluation pattern: run the real suite registration and selected
|
||||||
// callback, without importing paid initialization. All filesystem mutations
|
// callback, without importing paid initialization. All filesystem mutations
|
||||||
// stay in this standalone fixture.
|
// stay in this standalone fixture.
|
||||||
async function exercise(mode: 'success' | 'max-turns' | 'first-timeout' | 'second-timeout' | 'saved-timeout' | 'empty-summary' | 'write-failure' | 'unchanged-seed' | 'no-additions' | 'short-plan' | 'api-error' | 'plan-read-failure' | 'attempt-deadline') {
|
async function exercise(mode: 'success' | 'max-turns' | 'first-timeout' | 'second-timeout' | 'saved-timeout' | 'empty-summary' | 'rated-summary' | 'write-failure' | 'unchanged-seed' | 'no-additions' | 'short-plan' | 'api-error' | 'plan-read-failure' | 'attempt-deadline') {
|
||||||
const scratch = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'legacy-design-free-')));
|
const scratch = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'legacy-design-free-')));
|
||||||
const home = path.join(scratch, 'home'); fs.mkdirSync(home);
|
const home = path.join(scratch, 'home'); fs.mkdirSync(home);
|
||||||
const env = { PATH: process.env.PATH ?? '', HOME: home, GIT_CONFIG_NOSYSTEM: '1',
|
const env = { PATH: process.env.PATH ?? '', HOME: home, GIT_CONFIG_NOSYSTEM: '1',
|
||||||
@@ -115,7 +115,8 @@ async function exercise(mode: 'success' | 'max-turns' | 'first-timeout' | 'secon
|
|||||||
'Interaction state table: loading, empty, error and success. Responsive and accessibility decisions recorded.\n';
|
'Interaction state table: loading, empty, error and success. Responsive and accessibility decisions recorded.\n';
|
||||||
fs.writeFileSync(plan, reviewed); saved = fs.readFileSync(plan, 'utf8');
|
fs.writeFileSync(plan, reviewed); saved = fs.readFileSync(plan, 'utf8');
|
||||||
}
|
}
|
||||||
return { output: timedOut ? '' : mode === 'empty-summary' ? 'Done.' : 'Saved the information architecture and interaction state decisions in plan.md.',
|
return { output: timedOut ? '' : mode === 'empty-summary' ? 'Done.' :
|
||||||
|
mode === 'rated-summary' ? 'Design review saved. Overall design score 0/10 → 8/10; state table and tokens added.' : 'Saved the information architecture and interaction state decisions in plan.md.',
|
||||||
exitReason: timedOut ? 'timeout' : mode === 'max-turns' ? 'error_max_turns' : mode === 'api-error' ? 'error_api' : 'success',
|
exitReason: timedOut ? 'timeout' : mode === 'max-turns' ? 'error_max_turns' : mode === 'api-error' ? 'error_api' : 'success',
|
||||||
duration: timedOut ? (mode === 'first-timeout' ? 300039 : 300032) : 1,
|
duration: timedOut ? (mode === 'first-timeout' ? 300039 : 300032) : 1,
|
||||||
toolCalls: [], browseErrors: [], transcript: [], model: process.env.EVALS_MODEL ?? resolveEvalModel('capture'),
|
toolCalls: [], browseErrors: [], transcript: [], model: process.env.EVALS_MODEL ?? resolveEvalModel('capture'),
|
||||||
@@ -157,6 +158,12 @@ test('legacy Design still requires substantive output after a saved plan', async
|
|||||||
expect(result.resultError).toBeDefined(); expect(result.rows.map(row => row.passed)).toEqual([false]);
|
expect(result.resultError).toBeDefined(); expect(result.rows.map(row => row.passed)).toEqual([false]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Census 37179171083: a rated summary whose design terms live in the plan the review wrote.
|
||||||
|
test('legacy Design credits design terms the review added to the plan behind a rated summary', async () => {
|
||||||
|
const result = await exercise('rated-summary');
|
||||||
|
expect(result.resultError).toBeUndefined(); expect(result.rows.map(row => row.passed)).toEqual([true]);
|
||||||
|
});
|
||||||
|
|
||||||
test('legacy Design cleans the fixture when its write fails', async () => {
|
test('legacy Design cleans the fixture when its write fails', async () => {
|
||||||
const result = await exercise('write-failure');
|
const result = await exercise('write-failure');
|
||||||
expect(String(result.resultError)).toContain('controlled plan write failure'); expect(result.rows).toHaveLength(1);
|
expect(String(result.resultError)).toContain('controlled plan write failure'); expect(result.rows).toHaveLength(1);
|
||||||
|
|||||||
@@ -490,12 +490,11 @@ IMPORTANT: Do NOT try to browse any URLs or use a browse binary. This is a plan
|
|||||||
|
|
||||||
// Check that the plan file was edited (the core new behavior)
|
// Check that the plan file was edited (the core new behavior)
|
||||||
const planAfter = fs.readFileSync(path.join(reviewDir, 'plan.md'), 'utf-8');
|
const planAfter = fs.readFileSync(path.join(reviewDir, 'plan.md'), 'utf-8');
|
||||||
// The review's design content is what it wrote into the plan; the
|
// A rated summary may leave the design terms to the plan it wrote
|
||||||
// final message may only summarize it (census 37179171083).
|
// (census 37179171083); a plan term counts only when the review added it.
|
||||||
// A term counts from the plan only when the review added it.
|
|
||||||
const count = (text: string, term: string) => text.toLowerCase().split(term).length - 1;
|
const count = (text: string, term: string) => text.toLowerCase().split(term).length - 1;
|
||||||
const hasDesignContent = ['information architecture', 'interaction state', 'ai slop', 'hierarchy'].some(term =>
|
const hasDesignContent = ['information architecture', 'interaction state', 'ai slop', 'hierarchy'].some(term =>
|
||||||
output.toLowerCase().includes(term) || count(planAfter, term) > count(planBefore, term));
|
output.toLowerCase().includes(term) || (hasRatings && count(planAfter, term) > count(planBefore, term)));
|
||||||
const planWasEdited = planAfter !== planBefore && planAfter.length > 300;
|
const planWasEdited = planAfter !== planBefore && planAfter.length > 300;
|
||||||
const planHasDesignAdditions = planAfter.toLowerCase().includes('empty') ||
|
const planHasDesignAdditions = planAfter.toLowerCase().includes('empty') ||
|
||||||
planAfter.toLowerCase().includes('loading') ||
|
planAfter.toLowerCase().includes('loading') ||
|
||||||
|
|||||||
Reference in new issue
Block a user