test(design-floor): answer the seed-declared all-seven 0D focus menu while it is pending

The actor declares 'Design: review all seven dimensions', but its picker
reused designReviewSetupAUQ, which only matches already-answered calls
(and a narrower header/label set), so the pending D1 focus menu was never
answered and the case waited out its 609 s deadline. The skill's Step 0D
requires asking; the fixture now answers it.
This commit is contained in:
garrytan committed 2026-09-29 17:07:38 +00:00
1 parent 3a888896db
commit 41dca7609d
3 files changed
+61 -38

No files matched your search

+22
View File
@@ -0,0 +1,22 @@
{
"provenance": {
"run": "36597762183",
"job": "109508195705",
"note": "pending Step 0D focus menu left unanswered until the 609 s timeout"
},
"question": {
"question": "D1 — Review all 7 design dimensions, or focus?\nProject/branch/task: main — plan-design-review of PLAN.md (\"Plan: Marketing landing page\").\nELI10: I've rated this plan 2/10 on design completeness. It makes three layout calls and all three hurt the page: centered body copy is hard to scan, the h1 has no room to be the headline, and the main button looks the same as the \"Learn more\" link so visitors don't know what to click. Almost everything else (type, spacing, sections, states, mobile, accessibility) is unspecified. Next I'll generate visual mockups, then review every dimension and ask you about each fix one at a time.\nStakes if we pick wrong: a narrow review ships a landing page that looks fine in a screenshot but converts poorly and reads badly on phones.\nRecommendation: A because the plan is thin everywhere, so every dimension has real gaps; skipping any leaves holes the implementer will fill by guessing.\nCompleteness: A=10/10, B=6/10\nPros / cons:\nA) All 7 dimensions (recommended)\n ✅ Covers hierarchy, states, journey, slop risk, responsive, accessibility, and copy in one pass\n ✅ Matches your standing instruction to review all seven dimensions\n ❌ More decision briefs to answer, roughly one per gap found in each pass\nB) Focus on hierarchy + CTA only\n ✅ Fastest path to fixing the three anti-patterns already named in the plan\n ✅ Fewer questions if you only want the headline problems fixed today\n ❌ Leaves responsive, accessibility, states, and copy unspecified for the implementer\nNet: A trades more questions for a plan someone can actually build from; B fixes the obvious and leaves the rest to chance.",
"header": "Review scope",
"multiSelect": false,
"options": [
{
"label": "All 7 dimensions (recommended)",
"description": "✅ Full coverage of every design dimension\n✅ Matches the standing 'review all seven' instruction\n❌ More per-gap decision briefs to answer"
},
{
"label": "Focus: hierarchy + CTA",
"description": "✅ Fastest fix for the three named anti-patterns\n✅ Fewer questions today\n❌ Responsive, a11y, states, copy stay unspecified"
}
]
}
}
+20 -38
View File
@@ -3123,45 +3123,29 @@ export const engStep0Boundary: Step0BoundaryPredicate = (fp) =>
// plan-eng-review-idempotency, plan-eng-review-todos-e2e-concurrent.
/gstack-qid:\s*(?:plan-)?eng-review-/i.test(fp.promptSnippet);
/** Completed plan-wide focus and local-learnings choices remain setup, even when asked late. */
export const designReviewSetupAUQ: Step0BoundaryPredicate = (fp) => {
const call = fp.nativeCall;
if (call?.answered !== true || call.failed !== false || !call.sessionId || !call.toolUseId ||
call.questions.length !== 1 || !Array.isArray(call.unansweredQuestionIndices) || call.unansweredQuestionIndices.length ||
!Number.isFinite(Date.parse(call.answeredAt ?? '')) || fp.signature !== `${call.sessionId}:${call.toolUseId}` ||
(fp.nativeQuestionIndex !== undefined && fp.nativeQuestionIndex !== 0)) return false;
const q = call.questions[0]!;
if (q.multiSelect || q.options.length !== 2 || new Set(q.options.map(o => o.label)).size !== 2 ||
Object.keys(call.answers ?? {}).length !== 1 || q.options.filter(o => o.label === call.answers?.[q.question]).length !== 1 ||
fp.options.length !== 2 || !fp.options.every((o, i) => o.index === i + 1 && o.label === q.options[i]!.label)) return false;
/**
* The seed declares "Design: review all seven dimensions" for the pending Step 0D
* focus menu. Pick its single all-seven option only when every other option
* narrows the review and the brief approves no product action.
*/
export function pickDesignFocusAll(q: NativePlanQuestionCall['questions'][number]): number | null {
if (q.multiSelect || q.options.length < 2 || q.options.length > 4 || new Set(q.options.map(o => o.label)).size !== q.options.length) return null;
const text = q.question.trim();
const title = text.split(/\r?\n/, 1)[0]!.replace(/^D[1-9]\d*\s*[—–:-]\s*/i, '');
const sources = [...text.matchAll(/^Project\/branch\/task:\s*([^\n]+)$/gm)];
const source = sources[0]?.[1] ?? '';
// Setup never approves another product action. Quoted examples and negative
// consequences are explanatory; current imperative clauses remain decisions.
const explanatory = [text, ...q.options.map(o => o.description ?? '')].join('\n')
.replace(/`+[^`]*`+|"[^"\n]*"|“[^”\n]*”|‘[^’\n]*’/g, '')
.replace(/[✅❌*]/g, '');
if (/(?:^|[.!?;:\n]|\b(?:and|while))\s*(?:(?:also|please|then|now)\s+)*(?:approv(?:e|ing)|deploy(?:ing)?|implement(?:ing)?|ship(?:ping)?|merg(?:e|ing)|delet(?:e|ing))\b/im.test(explanatory)) return false;
if (sources.length !== 1 || (text.match(/\?/g)?.length ?? 0) !== 1 || /```|~~~|^\s*>/m.test(text) ||
!/\bplan-design-review of PLAN\.md\b/i.test(source) ||
/\b(?:historical|archived|quoted|example|foreign|other|another|previous)\b/i.test(source)) return false;
const labels = q.options.map(o => o.label.trim().replace(/^[A-Z][).:]\s+/i, '')
.replace(/\s*\(recommended\)\s*$/i, ''));
if (/^(?:Learnings|Cross-project)$/i.test(q.header.trim()) &&
/^Enable cross[- ]project learnings(?: search)?\?$/i.test(title) &&
labels.some(label => /^Enable cross[- ]project learnings$/i.test(label)) &&
labels.some(label => /^Keep learnings project[- ]scoped(?: only)?$/i.test(label))) {
// Reuse the existing native cross-project premise/owned answer classifier.
return engSetupAUQ(fp);
}
return /^(?:Focus|Review focus)$/i.test(q.header.trim()) &&
/^Review all 7 (?:design )?(?:dimensions|passes),? or focus(?: on (?:specific areas|a subset))?\?$/i.test(title) &&
/^ELI10:\s*I['’]ve rated this plan (?:10(?:\.0+)?|[0-9](?:\.\d+)?)\/10 on design completeness\./mi.test(text) &&
labels.some(label => /^(?:Review )?All 7 (?:design )?(?:dimensions|passes)$/i.test(label)) &&
labels.some(label => /^(?:Only (?:the )?[1-6](?: listed)? (?:gaps|areas|dimensions|passes)|Focus on (?:specific areas|a subset))$/i.test(label));
};
.replace(/`+[^`]*`+|"[^"\n]*"|“[^”\n]*”|‘[^’\n]*’/g, '').replace(/[✅❌*]/g, '');
if (!/^Review all (?:7|seven) (?:design )?(?:dimensions|passes),? or focus(?: on [^?\n]+)?\?$/i.test(title) ||
sources.length !== 1 || !/\bplan-design-review of PLAN\.md\b/i.test(sources[0]![1]!) ||
/\b(?:historical|archived|quoted|example|foreign|other|another|previous)\b/i.test(sources[0]![1]!) ||
(text.match(/\?/g)?.length ?? 0) !== 1 || /```|~~~|^\s*>/m.test(text) ||
!/^ELI10:\s*I['’]ve rated this plan (?:10(?:\.0+)?|[0-9](?:\.\d+)?)\/10 on design completeness\./mi.test(text) ||
/(?:^|[.!?;:\n]|\b(?:and|while))\s*(?:(?:also|please|then|now)\s+)*(?:approv(?:e|ing)|deploy(?:ing)?|implement(?:ing)?|ship(?:ping)?|merg(?:e|ing)|delet(?:e|ing))\b/im.test(explanatory)) return null;
const labels = q.options.map(o => o.label.trim().replace(/^[A-Z][).:]\s+/i, '').replace(/\s*\(recommended\)\s*$/i, ''));
const all = labels.flatMap((label, i) => /^(?:Review )?All (?:7|seven) (?:design )?(?:dimensions|passes)$/i.test(label) ? [i + 1] : []);
if (all.length !== 1 || labels.some((label, i) => i + 1 !== all[0] && !/^(?:Only|Focus)\b/i.test(label))) return null;
return all[0]!;
}
/**
* Spawn `claude --permission-mode plan` in a real PTY and return a session
@@ -4931,11 +4915,9 @@ export async function runPlanSkillFloorCheck(opts: {
const question = pendingQuestion.questions[index]!;
const key = `${pendingQuestion.sessionId}:${pendingQuestion.toolUseId}`;
const chosen = setupChoices.get(key) ?? new Set<number>();
const allDesign = opts.skillName === 'plan-design-review' && designReviewSetupAUQ(fp)
? question.options.flatMap((option, i) => /^(?:Review )?All 7 (?:design )?(?:dimensions|passes)(?:\s*\(recommended\))?$/i.test(option.label.trim()) ? [i + 1] : []) : [];
const pick = pickPlanFloorMode(opts.skillName, question) ?? planCountPrerequisitePick(fp, fp)
?? (opts.skillName === 'plan-devex-review' ? pickPlanFloorProductType(question, opts.productType) : null)
?? (allDesign.length === 1 ? allDesign[0]! : null);
?? (opts.skillName === 'plan-design-review' ? pickDesignFocusAll(question) : null);
if (pick !== null) {
if (!chosen.has(index)) {
session.send(planCountQuestionInput(viewport, fp, pick));
@@ -57,6 +57,7 @@ import {
devexStep0Boundary,
type ClaudePtyOptions,
type AskUserQuestionFingerprint,
pickDesignFocusAll,
} from './claude-pty-runner';
describe('saved preference annotation', () => {
@@ -3523,3 +3524,21 @@ describe('explicit Step 0 complexity gate with size in native choices', () => {
}
});
});
describe('pickDesignFocusAll: the seed-declared all-seven answer for the pending 0D focus menu', () => {
const captured = require('../fixtures/design-floor-focus-36597762183.json');
const q = () => structuredClone(captured.question);
test('census 36597762183 pending menu selects the all-seven option', () => {
expect(pickDesignFocusAll(q())).toBe(1);
});
test.each([
['a different title', (x: any) => { x.question = x.question.replace('Review all 7 design dimensions, or focus?', 'Which fixes should I apply?'); }],
['a non-narrowing alternative', (x: any) => { x.options[1].label = 'Approve every fix now'; }],
['two all-seven options', (x: any) => { x.options[1].label = 'All seven dimensions'; }],
['a bundled product approval', (x: any) => { x.question = x.question.replace('Net:', 'Also approve the CTA redesign.\nNet:'); }],
['a foreign plan context', (x: any) => { x.question = x.question.replace('plan-design-review of PLAN.md', 'plan-design-review of OTHER.md'); }],
['multi select', (x: any) => { x.multiSelect = true; }],
])('%s is not answered', (_name, mutate) => {
const x = q(); mutate(x); expect(pickDesignFocusAll(x)).toBeNull();
});
});