From 6f7488a70f23bc71745bde704eda98033cc33bfa Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Fri, 23 Oct 2026 11:35:59 -0700 Subject: [PATCH] v1.91.185.0 fix(evals): plan-design focus rating in plain words, docsync own-call rule where --help is named, floor Create crop and 0D menu (GSTA-65) (#3358) * fix(evals): plan-design focus rating in plain words, docsync own-call rule at the candidate --help, floor Create menu cropped at its rule (GSTA-65) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(evals): floor actor answers Step 0D's focus menu in the template's own words (GSTA-65) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(evals): floor actor identifies Step 0D's focus menu by its structure, not one wording (GSTA-65) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(changelog): GSTA-65 paid-side flaky repairs under [Unreleased] Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(evals): the 0D focus picker moves into plan-design-finding-floor's owned fixture; no harness series reset (GSTA-65) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test(evals): the 0D picker rows join the floor case's owner test (test-of-test ratchet) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * v1.91.185.0 fix(evals): plan-design focus rating in plain words, docsync own-call rule, floor Create crop and 0D menu (GSTA-65) (#3358) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 20 +++ VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- package.json | 2 +- scripts/harness-version.json | 6 +- test/docsync-command-grammar.test.ts | 12 ++ test/fixtures/design-floor-focus-picker.ts | 35 +++++ .../design-floor-focus-review-focus.json | 21 +++ .../design-floor-focus-template-0d.json | 21 +++ .../design-ui-focus-capture-39852003387.json | 147 ++++++++++++++++++ .../plan-design-floor-create-rule-crop.json | 15 ++ .../claude-pty-runner.boundaries.unit.test.ts | 19 --- test/helpers/claude-pty-runner.ts | 2 +- test/helpers/docsync-interface.ts | 2 +- test/helpers/free-fixtures-data.ts | 5 +- test/helpers/plan-count-file-permission.ts | 4 +- test/helpers/pty/boundaries.ts | 24 --- test/helpers/pty/runners/floor.ts | 5 +- test/helpers/touchfiles-data.ts | 4 +- test/plan-create-prepublication.test.ts | 35 ++++- test/plan-design-floor-fixture.test.ts | 44 +++++- test/plan-design-with-ui-fixture.test.ts | 18 ++- ...kill-e2e-plan-design-finding-floor.test.ts | 2 + test/skill-e2e-plan-design-with-ui.test.ts | 7 +- 24 files changed, 386 insertions(+), 68 deletions(-) create mode 100644 test/fixtures/design-floor-focus-picker.ts create mode 100644 test/fixtures/design-floor-focus-review-focus.json create mode 100644 test/fixtures/design-floor-focus-template-0d.json create mode 100644 test/fixtures/design-ui-focus-capture-39852003387.json create mode 100644 test/fixtures/plan-design-floor-create-rule-crop.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a9ed0d55..8f91ed86b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## [1.91.185.0] - 2026-10-23 + +**Three flaky paid eval cases get repairs: three harness gaps closed and one fixture rule moved to where the agent reads it.** Today's daily flaky-fix pass took the two repeat flakes and one periodic timeout from the flaky ledger. plan-design-with-ui-scope failed its first attempt in train gate 39852003387 (its fourth flake) because the Opus 5.5 agent rated the plan "4/10 on design" in plain words and the focus check wanted the words "design completeness". ship-docsync-completion failed in train gate 39904966815 and in GSTA-63's focused run with the same slip both times: the agent added `gstack-docs-candidate --help` to the end of its first line of reads, which the fixture interface forbids. plan-design-finding-floor timed out at 610 s in periodic run 39209106033 because nothing answered the owned plan-file Create menu after its title scrolled off screen; two focused trials then showed a second stall, at the Step 0D focus menu the floor actor answers only when it matches one captured wording. + +### Fixed +- **`test/skill-e2e-plan-design-with-ui.test.ts`**: the Step 0A rating the focus check looks for is now any sentence that scores the plan's design out of 10, so "I rated this plan 4/10 on design." counts like "4/10 on design completeness". The check still needs a question that names the focus and an option that leads with all seven areas, and a rating split from the word "design" across sentences, an unscored design sentence, and a non-design score are still refused. +- **`test/helpers/docsync-interface.ts`**: the declared observation interface now states the own-call rule in the sentence that introduces the Preamble block, lifecycle helpers and `gstack-docs-candidate` commands ("each as its own Bash call, with nothing joined before or after it ... a --help is no exception"), instead of in a later sentence after the read grammar. The grammar is unchanged and the captured joined line is still refused. +- **`test/helpers/plan-count-file-permission.ts`**: a cropped owned Create menu whose preview starts on the native rule (the "Create file" title and file row scrolled away) now binds to its pending Write, as a cropped Edit menu already did. Every preview row must still match the owned Write's content, and the prompt must still name the owned file. +- **`test/fixtures/design-floor-focus-picker.ts`**: the floor actor's Step 0D answer moves out of the shared PTY harness into the case's own fixture, and `runPlanSkillFloorCheck` takes it through a new optional `pickSetupChoice` hook that only plan-design-finding-floor passes. The answer (`pickDesignFocusAll`, the seed's declared "review all seven dimensions") identifies the focus menu by its structure: a title or question about the focus, exactly one option that is all seven dimensions, passes or areas, every other option narrowing ("Only …", "Focus …" or "… only"), and a Project/branch/task line naming PLAN.md or the seeded plan. It used to need the exact title "Review all 7 design dimensions, or focus?", a source line reading "plan-design-review of PLAN.md" and an ELI10 that began with the rating, so the template's own 0D sentence ("I've rated this plan 2/10 on design completeness. … Want me to focus on specific areas instead?") and a menu titled "Review focus for the landing page plan" both sat unanswered until the 610 s timeout. A second question, a quoted or foreign plan, a non-narrowing alternative, two all-seven options and a bundled approval still get no answer. + +### Tests +- `test/plan-design-with-ui-fixture.test.ts` replays gate 39852003387's four native questions (`test/fixtures/design-ui-focus-capture-39852003387.json`) as a pass, and the same capture without its focus question as a failure. New controls: a rating and the word "design" in different sentences, and an unscored design sentence. +- `test/docsync-command-grammar.test.ts` checks that the captured joined line stays refused, that the candidate's `--help` alone is accepted, and that the sentence naming the candidate commands carries the own-call rule. +- `test/plan-create-prepublication.test.ts` replays the periodic run's final viewport (`test/fixtures/plan-design-floor-create-rule-crop.json`): it is granted once and fails without the fix. Controls: two rules, text above the rule, a quoted row, a changed row and a foreign file name are refused. +- `test/plan-design-floor-fixture.test.ts` (the case's owner test; the rows moved with the picker from the boundaries unit test) replays both pending focus menus from the focused trials (`test/fixtures/design-floor-focus-template-0d.json`, `test/fixtures/design-floor-focus-review-focus.json`); each selects the all-seven option, and the template-worded one failed before the change. New controls: a question not about focus, a foreign or quoted plan, an alternative that does not narrow, and a second question. + +### Gate integrity +- No threshold, budget, timeout, trial count or case kind changed. Two accepted-input changes to review: the plan-design focus rating accepts "design" without "completeness", and the floor actor answers the 0D focus menu without requiring a rating sentence or one exact title (it answers only with the seed's declared all-seven option). +- HARNESS_VERSION stays 6. `boundaries.ts`, `claude-pty-runner.ts` and `runners/floor.ts` are recorded non-behavioral: the picker left the shared harness and the new hook has one caller, so no other case's input or verdict can change, while plan-design-finding-floor's owned bytes (its test file and the new fixture) start that case's own pass-rate series. + ## [1.91.184.0] - 2026-10-23 **The office-hours attempt lifecycle test no longer races the wall clock.** Today's daily flaky-fix pass: in #3341's remote pregate, "recommendation near-success stays inside the shared work deadline" failed once with two judge starts where it expects one, then passed on the serial retry. Its fake runner and judge used real sleeps (220 ms plus 10 ms) against a mocked 300 ms deadline, so a loaded machine could push the first attempt past it. The sibling late-judge case had the same exposure from the other side. diff --git a/VERSION b/VERSION index 30f50f945..e63cb230a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.184.0 +1.91.185.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index a92a37177..26600c1d9 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.184.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.185.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/package.json b/package.json index 5a6b8285f..9a137d6a1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.184", + "version": "1.91.185", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/scripts/harness-version.json b/scripts/harness-version.json index 3ee92c6d1..d95e24d5a 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -21,7 +21,7 @@ "scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":6}, "test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":6}, "test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":6}, - "test/helpers/claude-pty-runner.ts": {"blob":"20cc5894532873aa6145fc6617fdeec8cef94a2d","since":6}, + "test/helpers/claude-pty-runner.ts": {"blob":"ce7a5eaa66e502a052ea9c53348dd2ab25db6148","since":6,"nonBehavioral":"Removes the plan-design 0D focus picker from the shared harness; floor.ts gains an optional pickSetupChoice hook that no other caller passes, so no other case's input or verdict can change. plan-design-finding-floor, the only case whose seed raises that menu, passes the picker from its owned fixture test/fixtures/design-floor-focus-picker.ts, so its owned bytes start its own new series."}, "test/helpers/codex-host-runner.ts": {"blob":"bff225017e235f0be6ece29f04971ac04b95b544","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "test/helpers/codex-session-runner.ts": {"blob":"c040ebbfa815b8cce19ef80c6133bd5a5474149a","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "test/helpers/e2e-gate.ts": {"blob":"efb7e197dd5f66c3ad0e34074b14e640754f7a45","since":6}, @@ -34,14 +34,14 @@ "test/helpers/pty-screen.ts": {"blob":"4f77569f9ce11f9cee9f347216e468cd7c956820","since":6}, "test/helpers/pty/auq.ts": {"blob":"16e5ea2bb5b9f28f44808a1a09024dbd3e6f28c4","since":6}, "test/helpers/pty/binary.ts": {"blob":"404d7515076d5c00804f89e23ea69296312c00e1","since":6}, - "test/helpers/pty/boundaries.ts": {"blob":"53eb76e57c0558c00254366d6334a1c833126575","since":6}, + "test/helpers/pty/boundaries.ts": {"blob":"8c87100c9dd0cfc28c02598f250726fd93bfbdea","since":6,"nonBehavioral":"Removes the plan-design 0D focus picker from the shared harness; floor.ts gains an optional pickSetupChoice hook that no other caller passes, so no other case's input or verdict can change. plan-design-finding-floor, the only case whose seed raises that menu, passes the picker from its owned fixture test/fixtures/design-floor-focus-picker.ts, so its owned bytes start its own new series."}, "test/helpers/pty/classify.ts": {"blob":"3f2daa12ff6484f96b8d28d910a50231c4bb3ae8","since":6}, "test/helpers/pty/fake-session.ts": {"blob":"554e064ef259b8444280eddfe9b0db93c66a8202","since":6}, "test/helpers/pty/judge.ts": {"blob":"81c5bb4c48ab928aca2f186a1ade9df7169d7fbe","since":6}, "test/helpers/pty/launch.ts": {"blob":"e4413de380b799659170e7da248d51afda3b0e3f","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, "test/helpers/pty/plan-native.ts": {"blob":"ddba9cb68c7c0da935cff7c1cea258dee2dff740","since":6}, "test/helpers/pty/runners/counting.ts": {"blob":"8c5c05e54abf961259d34d9c3d77db67138d7794","since":6}, - "test/helpers/pty/runners/floor.ts": {"blob":"57fa44c31300754b00473276cf7b972d4e9e47d8","since":6}, + "test/helpers/pty/runners/floor.ts": {"blob":"54e2145832bb98913644d10ff303792d67f9362e","since":6,"nonBehavioral":"Removes the plan-design 0D focus picker from the shared harness; floor.ts gains an optional pickSetupChoice hook that no other caller passes, so no other case's input or verdict can change. plan-design-finding-floor, the only case whose seed raises that menu, passes the picker from its owned fixture test/fixtures/design-floor-focus-picker.ts, so its owned bytes start its own new series."}, "test/helpers/pty/runners/observation.ts": {"blob":"1c2b75fbb984708725abaa8939125f7cab969f9f","since":6}, "test/helpers/pty/screen.ts": {"blob":"f3743152fc9bb7a8e8c70002f75bf45914f260d5","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, "test/helpers/pty/session.ts": {"blob":"54fe04b61d953e4e55e2fc251532c04a49fab66f","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, diff --git a/test/docsync-command-grammar.test.ts b/test/docsync-command-grammar.test.ts index d0fd8aef9..bf862d951 100644 --- a/test/docsync-command-grammar.test.ts +++ b/test/docsync-command-grammar.test.ts @@ -234,6 +234,18 @@ test('the installed docs-candidate helper is admitted only with a private .json expect(docsNativeInterface(fixture)).toContain('compare and --help commands'); }); +// Train gate 39904966815 (ship-docsync-completion, attempt 1) and GSTA-63's focused run joined the candidate's +// --help onto a line of reads. The join stays refused; the own-call rule now sits in the sentence naming --help. +test('a candidate --help joined to reads stays refused, and the interface says so where it names the command', () => { + const skills = fixture.skills.split(path.sep).join('/'); + const joined = `git diff; git diff --cached; cat options.ts; ls -la handbook handbook/reference handbook/reference/commands; ${skills}/bin/gstack-docs-candidate --help`; + expect(docsCommandAllowed(joined, fixture)).toBe(false); + expect(docsCommandAllowed(`${skills}/bin/gstack-docs-candidate --help`, fixture)).toBe(true); + const sentence = docsNativeInterface(fixture).split(/(?<=\.)\s+(?=[A-Z])/).find(part => part.includes('gstack-docs-candidate snapshot')); + expect(sentence).toContain('as its own Bash call, with nothing joined before or after it'); + expect(sentence).toContain('a --help is no exception'); +}); + test('the installed lifecycle helpers are admitted for their own --help, which starts nothing', () => { const skills = fixture.skills.split(path.sep).join('/'); for (const helper of ['gstack-skill-start', 'gstack-skill-end']) { diff --git a/test/fixtures/design-floor-focus-picker.ts b/test/fixtures/design-floor-focus-picker.ts new file mode 100644 index 000000000..b7e90c530 --- /dev/null +++ b/test/fixtures/design-floor-focus-picker.ts @@ -0,0 +1,35 @@ +/** + * Case-owned setup answer for test/skill-e2e-plan-design-finding-floor.test.ts. + * It lives with the case's fixtures so a change to it starts that case's own + * pass-rate series instead of every case's (scripts/eval-trial-series.ts). + */ +import type { NativePlanQuestionCall } from '../helpers/plan-count-transcript'; + +/** + * The seed declares "Design: review all seven dimensions" for the pending Step 0D + * focus menu. Pick its single all-seven option only when every other option + * narrows the review and the brief approves no product action. + */ +export function pickDesignFocusAll(q: NativePlanQuestionCall['questions'][number]): number | null { + if (q.multiSelect || q.options.length < 2 || q.options.length > 4 || new Set(q.options.map(o => o.label)).size !== q.options.length) return null; + const text = q.question.trim(); + const unquote = (value: string) => value.replace(/`+[^`]*`+|"[^"\n]*"|“[^”\n]*”|‘[^’\n]*’/g, ''); + const sources = [...text.matchAll(/^Project\/branch\/task:\s*([^\n]+)$/gm)].map(match => unquote(match[1]!)); + const explanatory = unquote([text, ...q.options.flatMap(o => [o.label, o.description ?? ''])].join('\n')).replace(/[✅❌*]/g, ''); + // The menu is identified by its structure, not one wording: a title or question about the + // focus ("Review all 7 design dimensions, or focus?", census 36597762183; "Want me to focus on + // specific areas instead?" and "Review focus for the landing page plan", GSTA-65 focused + // trials), one all-seven option, and only narrowing alternatives. + const sentences = text.split(/(?<=[.!?])\s+|\r?\n/); + const title = text.split(/\r?\n/, 1)[0]!.replace(/^D[1-9]\d*\s*[—–:-]\s*/i, ''); + const asks = sentences.filter(sentence => sentence.includes('?')); + if (![title, ...asks].some(line => /\bfocus\b/i.test(line)) || + sources.length !== 1 || !/\b(?:PLAN\.md|Marketing landing page)\b/i.test(sources[0]!) || + /\b(?:historical|archived|quoted|example|foreign|other|another|previous)\b/i.test(sources[0]!) || + (text.match(/\?/g)?.length ?? 0) > 1 || /```|~~~|^\s*>/m.test(text) || + /(?:^|[.!?;:\n]|\b(?:and|while))\s*(?:(?:also|please|then|now)\s+)*(?:approv(?:e|ing)|deploy(?:ing)?|implement(?:ing)?|ship(?:ping)?|merg(?:e|ing)|delet(?:e|ing))\b/im.test(explanatory)) return null; + const labels = q.options.map(o => o.label.trim().replace(/^[A-Z][).:]\s+/i, '').replace(/\s*\(recommended\)\s*$/i, '')); + const all = labels.flatMap((label, i) => /^(?:Review )?All (?:7|seven) (?:design )?(?:dimensions|passes|areas)$/i.test(label) ? [i + 1] : []); + if (all.length !== 1 || labels.some((label, i) => i + 1 !== all[0] && !/^(?:Only|Focus)\b|\bonly$/i.test(label))) return null; + return all[0]!; +} diff --git a/test/fixtures/design-floor-focus-review-focus.json b/test/fixtures/design-floor-focus-review-focus.json new file mode 100644 index 000000000..39f2c9df0 --- /dev/null +++ b/test/fixtures/design-floor-focus-review-focus.json @@ -0,0 +1,21 @@ +{ + "provenance": { + "run": "GSTA-65 focused trial (local, 2026-10-23, head 0dff819)", + "note": "pending Step 0D focus menu titled 'Review focus for the landing page plan', with no question mark and an unscored rating sentence, left unanswered until the 610 s timeout" + }, + "question": { + "question": "D1 \u2014 Review focus for the landing page plan\nProject/branch/task: PLAN.md on main, marketing landing page.\nELI10: I rated this plan 2/10. The biggest gaps: everything is centered, the headline is cramped against the subhead, and the main button looks the same as the 'Learn more' link, so visitors don't know what to click. Mockups are skipped (designer not installed). I can review all 7 dimensions, or focus on a few.\nStakes if we pick wrong: narrowing the focus leaves states, responsive and a11y unspecified, so the engineer invents them.\nRecommendation: A because your instructions ask for all seven dimensions and the plan is thin everywhere.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nPros / cons:\nA) All 7 passes (recommended)\n \u2705 Covers hierarchy, states, journey, slop, system, responsive, decisions\n \u2705 Matches the requested scope of reviewing all seven dimensions\n \u274c More individual questions, one per gap found in the plan\nB) Hero only (alignment, spacing, CTA weight)\n \u2705 Fastest path to fixing the three problems the plan names explicitly\n \u2705 Fewer questions, quicker to an implementable hero section\n \u274c Leaves responsive, a11y and states unspecified for implementation\nNet: full coverage vs. speed on the three named hero problems.", + "header": "Focus", + "multiSelect": false, + "options": [ + { + "label": "All 7 passes (recommended)", + "description": "Review every dimension; one question per gap." + }, + { + "label": "Hero only", + "description": "Only alignment, h1 spacing and CTA weight." + } + ] + } +} diff --git a/test/fixtures/design-floor-focus-template-0d.json b/test/fixtures/design-floor-focus-template-0d.json new file mode 100644 index 000000000..c6b337817 --- /dev/null +++ b/test/fixtures/design-floor-focus-template-0d.json @@ -0,0 +1,21 @@ +{ + "provenance": { + "run": "GSTA-65 focused trial t1 (local, 2026-10-23, tree 01d1d72a)", + "note": "pending Step 0D focus menu in the template's own words left unanswered until the 610 s timeout" + }, + "question": { + "question": "D1 \u2014 I've rated this plan 2/10 on design completeness. The biggest gaps are centered-everything alignment, the 8px h1-to-subhead gap, and a CTA with no more weight than 'Learn more'. Mockups are unavailable (no designer binary), so I'll review all 7 dimensions text-only. Want me to focus on specific areas instead?\nProject/branch/task: Marketing landing page plan on main.\nELI10: The review can cover every design angle or just a few. Covering all 7 catches gaps like mobile and accessibility that are cheap to fix now and expensive later.\nStakes if we pick wrong: Narrowing the scope leaves unreviewed gaps that engineers will fill with defaults.\nRecommendation: A because your instructions ask for all seven dimensions and the plan is short.\nCompleteness: A=10/10, B=6/10\nPros / cons:\nA) All 7 dimensions (recommended)\n \u2705 Every gap is caught, including states, mobile, and accessibility\n \u2705 Matches your stated request to review all seven dimensions\n \u274c More questions, roughly one per finding, before the plan is final\nB) Layout/hierarchy only\n \u2705 Faster, focused on the three issues the plan already names\n \u274c Leaves responsive, a11y, and states unspecified for engineering\nNet: Full coverage costs a few more questions and avoids default-driven gaps.", + "header": "Focus", + "multiSelect": false, + "options": [ + { + "label": "All 7 dimensions (recommended)", + "description": "Full review: IA, states, journey, AI slop, design system, responsive/a11y, unresolved decisions." + }, + { + "label": "Layout/hierarchy only", + "description": "Only the alignment, spacing, and CTA weight issues already in the plan." + } + ] + } +} diff --git a/test/fixtures/design-ui-focus-capture-39852003387.json b/test/fixtures/design-ui-focus-capture-39852003387.json new file mode 100644 index 000000000..72c1f3962 --- /dev/null +++ b/test/fixtures/design-ui-focus-capture-39852003387.json @@ -0,0 +1,147 @@ +{ + "provenance": { + "kind": "public native AskUserQuestion calls and answers only (observation.json fingerprints[].nativeCall, byte-exact)", + "runId": 39852003387, + "artifact": "paid-slice-4-a1", + "headSha": "357027f559b3a906d2665511e411747a693ae664", + "recordedOutcome": "ceiling_reached", + "recordedReviewCount": 1, + "recordedStep0Count": 3, + "recordedPassed": false, + "note": "The Step 0D question was titled 'Review focus' and its ELI10 rated the plan '4/10 on design' (plain words, without 'completeness'), with options 'All 7 passes (recommended)' and 'States + layout only'. The runner reached the review ceiling on the issue-labeled D4 finding, but the focus predicate required the words 'design completeness' in the rating sentence, so the gate failed." + }, + "calls": [ + { + "preReview": true, + "observedAtMs": 57813, + "call": { + "sessionId": "cee83a04-4dec-490f-8008-4b242391684a", + "toolUseId": "toolu_01X4KQ4iKW2R3H6XtqEXjkzc", + "questions": [ + { + "question": "D1 \u2014 Review focus\nProject/branch/task: Design review of the User Dashboard plan (review-input.md) on main.\nELI10: I rated this plan 4/10 on design. The biggest gaps: no layout order for the three panels, states named but not described, and no visual tokens. Next I'll make mockups, then go through 7 areas (layout, states, user journey, generic-look risk, design system, responsive/a11y, open decisions). You can narrow that.\nStakes if we pick wrong: If we narrow too much, gaps like mobile layout or modal focus behavior go unspecified and the engineer improvises.\nRecommendation: A because the plan has gaps in every area and a full review is cheap.\nCompleteness: A=10/10, B=6/10\nPros / cons:\nA) All 7 passes (recommended)\n \u2705 Covers layout, states, journey, slop, system, responsive and decisions\n \u2705 Every gap gets its own decision instead of an engineer guessing later\n \u274c More questions, roughly one per finding, so it takes longer overall\nB) Focus on states + layout only\n \u2705 Faster, hits the two gaps most likely to ship as broken UI\n \u274c Leaves responsive, a11y and token gaps for implementation to improvise\nNet: Full coverage costs a few extra questions; narrowing saves time but leaves known gaps open.", + "header": "Focus", + "multiSelect": false, + "options": [ + { + "label": "All 7 passes (recommended)", + "description": "\u2705 Covers every design dimension with per-issue decisions. \u2705 Nothing left for the engineer to guess. \u274c More questions overall." + }, + { + "label": "States + layout only", + "description": "\u2705 Faster, targets the gaps most likely to ship broken. \u274c Responsive, a11y and tokens stay unspecified." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D1 \u2014 Review focus\nProject/branch/task: Design review of the User Dashboard plan (review-input.md) on main.\nELI10: I rated this plan 4/10 on design. The biggest gaps: no layout order for the three panels, states named but not described, and no visual tokens. Next I'll make mockups, then go through 7 areas (layout, states, user journey, generic-look risk, design system, responsive/a11y, open decisions). You can narrow that.\nStakes if we pick wrong: If we narrow too much, gaps like mobile layout or modal focus behavior go unspecified and the engineer improvises.\nRecommendation: A because the plan has gaps in every area and a full review is cheap.\nCompleteness: A=10/10, B=6/10\nPros / cons:\nA) All 7 passes (recommended)\n \u2705 Covers layout, states, journey, slop, system, responsive and decisions\n \u2705 Every gap gets its own decision instead of an engineer guessing later\n \u274c More questions, roughly one per finding, so it takes longer overall\nB) Focus on states + layout only\n \u2705 Faster, hits the two gaps most likely to ship as broken UI\n \u274c Leaves responsive, a11y and token gaps for implementation to improvise\nNet: Full coverage costs a few extra questions; narrowing saves time but leaves known gaps open.": "All 7 passes (recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-10-22T20:51:59.856Z" + } + }, + { + "preReview": true, + "observedAtMs": 92416, + "call": { + "sessionId": "cee83a04-4dec-490f-8008-4b242391684a", + "toolUseId": "toolu_0177YBa2Hb6R4v3GuoNfcCu3", + "questions": [ + { + "question": "Want outside design voices before the detailed review?\nD2 \u2014 Optional outside design voices\nProject/branch/task: Design review of the User Dashboard plan on main.\nELI10: I can get two independent second opinions before the 7 passes: Codex scores the plan against hard design rules, and a separate Claude reviewer checks it for completeness. They catch things one reviewer misses, but they add time.\nStakes if we pick wrong: Skipping means findings rest on one reviewer; running adds minutes and may only get one voice if Codex is missing.\nRecommendation: B, a taste call with no strong preference. This plan's gaps are concrete enough that one reviewer catches the big ones.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nNet: Cross-model check versus fewer steps to a fixed plan.", + "header": "Voices", + "multiSelect": false, + "options": [ + { + "label": "No, proceed without (recommended)", + "description": "\u2705 Straight into the 7 passes with the approved mockup as anchor. \u2705 Fewest steps to a fixed plan. \u274c Findings rest on one reviewer; no cross-model check on slop risk or hierarchy." + }, + { + "label": "Yes, run outside design voices", + "description": "\u2705 Codex hard-rule + litmus scorecard plus an independent Claude completeness read. \u2705 Disagreements surface with both perspectives in the relevant pass. \u274c Adds a few minutes; Codex may be missing here, in which case only the Claude voice runs and it's tagged single-model." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "Want outside design voices before the detailed review?\nD2 \u2014 Optional outside design voices\nProject/branch/task: Design review of the User Dashboard plan on main.\nELI10: I can get two independent second opinions before the 7 passes: Codex scores the plan against hard design rules, and a separate Claude reviewer checks it for completeness. They catch things one reviewer misses, but they add time.\nStakes if we pick wrong: Skipping means findings rest on one reviewer; running adds minutes and may only get one voice if Codex is missing.\nRecommendation: B, a taste call with no strong preference. This plan's gaps are concrete enough that one reviewer catches the big ones.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nNet: Cross-model check versus fewer steps to a fixed plan.": "No, proceed without (recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-10-22T20:52:33.581Z" + } + }, + { + "preReview": true, + "observedAtMs": 105593, + "call": { + "sessionId": "cee83a04-4dec-490f-8008-4b242391684a", + "toolUseId": "toolu_01URxrnNA3oppVUTro7BMwpa", + "questions": [ + { + "question": "D3 \u2014 Cross-project learnings\nProject/branch/task: Design review of the User Dashboard plan on main.\nELI10: gstack can search learnings from your other projects on this machine for patterns that might apply here. Nothing leaves your machine. Good for solo developers; skip it if you work across client codebases that shouldn't mix.\nStakes if we pick wrong: Enabling it on mixed client work could bring one client's patterns into another's review.\nRecommendation: A because local cross-project patterns usually help a solo builder and stay on disk.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nNet: More pattern recall versus strict project isolation.", + "header": "Learnings", + "multiSelect": false, + "options": [ + { + "label": "Enable cross-project (recommended)", + "description": "\u2705 Reviews can reuse patterns found in your other local projects. \u2705 Stays on this machine; nothing is uploaded. \u274c Patterns from unrelated client codebases may leak into this review's context." + }, + { + "label": "Project-scoped only", + "description": "\u2705 Strict isolation between codebases, safe for client work. \u2705 No chance of irrelevant cross-project noise. \u274c Misses useful patterns you already learned elsewhere on this machine." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D3 \u2014 Cross-project learnings\nProject/branch/task: Design review of the User Dashboard plan on main.\nELI10: gstack can search learnings from your other projects on this machine for patterns that might apply here. Nothing leaves your machine. Good for solo developers; skip it if you work across client codebases that shouldn't mix.\nStakes if we pick wrong: Enabling it on mixed client work could bring one client's patterns into another's review.\nRecommendation: A because local cross-project patterns usually help a solo builder and stay on disk.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nNet: More pattern recall versus strict project isolation.": "Enable cross-project (recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-10-22T20:52:46.301Z" + } + }, + { + "preReview": false, + "observedAtMs": 177070, + "call": { + "sessionId": "cee83a04-4dec-490f-8008-4b242391684a", + "toolUseId": "toolu_015nA5dQ6WgNMx2BF5coGXhR", + "questions": [ + { + "question": "D4 \u2014 Issue 1: Which panel leads the dashboard?\nProject/branch/task: Design review of the User Dashboard plan on main.\nELI10: The plan has three panels but no order. Users scan from the top left, so whatever is there gets noticed. Notifications is the only panel that asks the user to do something. Activity is history. Quick actions are two links.\nStakes if we pick wrong: Unread notifications end up below the fold or tied with history, and users miss what needs their attention.\nRecommendation: 1A because Hierarchy as service: put the actionable thing first and keep two links small.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nPros / cons:\nA) Header actions + notifications-first (recommended)\n \u2705 Quick actions sit as a compact button row in the page header next to 'Dashboard'\n \u2705 Notifications is the primary, wider column; Activity is the secondary column\n \u274c Activity gets less space, which matters if users mostly come for history\nB) Quick actions as a hero strip, then panels\n \u2705 'Open workspace' gets the most prominent spot for users heading to work\n \u274c Two links take a full row of prime space and push notifications down\nC) Equal three-panel grid (current implied)\n \u2705 Simplest to build, no ranking decision needed now\n \u274c Everything competes, so nothing wins; the classic generic dashboard look\nNet: Rank by what the user must act on versus treating all panels as equals.", + "header": "Hierarchy", + "multiSelect": false, + "options": [ + { + "label": "1A Notifications-first (recommended)", + "description": "\u2705 Quick actions as a compact row in the page header. \u2705 Notifications wide primary column, Activity secondary. \u274c Activity gets less room." + }, + { + "label": "1B Quick-actions strip first", + "description": "\u2705 'Open workspace' is the most prominent element. \u274c Two links eat a full row and push notifications down." + }, + { + "label": "1C Equal grid", + "description": "\u2705 Simplest to build. \u274c No hierarchy; everything competes, generic dashboard look." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D4 \u2014 Issue 1: Which panel leads the dashboard?\nProject/branch/task: Design review of the User Dashboard plan on main.\nELI10: The plan has three panels but no order. Users scan from the top left, so whatever is there gets noticed. Notifications is the only panel that asks the user to do something. Activity is history. Quick actions are two links.\nStakes if we pick wrong: Unread notifications end up below the fold or tied with history, and users miss what needs their attention.\nRecommendation: 1A because Hierarchy as service: put the actionable thing first and keep two links small.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nPros / cons:\nA) Header actions + notifications-first (recommended)\n \u2705 Quick actions sit as a compact button row in the page header next to 'Dashboard'\n \u2705 Notifications is the primary, wider column; Activity is the secondary column\n \u274c Activity gets less space, which matters if users mostly come for history\nB) Quick actions as a hero strip, then panels\n \u2705 'Open workspace' gets the most prominent spot for users heading to work\n \u274c Two links take a full row of prime space and push notifications down\nC) Equal three-panel grid (current implied)\n \u2705 Simplest to build, no ranking decision needed now\n \u274c Everything competes, so nothing wins; the classic generic dashboard look\nNet: Rank by what the user must act on versus treating all panels as equals.": "1A Notifications-first (recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-10-22T20:53:54.578Z" + } + } + ] +} diff --git a/test/fixtures/plan-design-floor-create-rule-crop.json b/test/fixtures/plan-design-floor-create-rule-crop.json new file mode 100644 index 000000000..3362ec73d --- /dev/null +++ b/test/fixtures/plan-design-floor-create-rule-crop.json @@ -0,0 +1,15 @@ +{ + "source": "Periodic run 39209106033 slice 30 (main 2d42e4e, v1.91.143.0), plan-design-finding-floor: outcome timeout at 610 s (observer_timeout), issue #2756.", + "note": "Exact final viewport (terminal.screen.log) and the owned Write input from its own PreToolUse witness (pendingWriteInputs). The owned working-plan Create menu's 'Create file' title and file row scrolled away, so the crop starts on the native rule above the numbered preview. The harness then had no grant and the menu stayed open until the observer timed out; on main it would end as permission_unanswered after 20 s.", + "sessionId": "94631d4e-4d3b-48fc-9835-1d17d04e8f71", + "toolUseId": "toolu_01LtZoeoZvC7CMStkNyremFA", + "basename": "gstack-test-plan-design-floor.md", + "recorded": { + "outcome": "timeout", + "summary": "no qualifying finding question within 600000ms", + "elapsedMs": 610041 + }, + "screen": "\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\n 1 # Plan: Marketing landing page\n 2\n 3 ## Context\n 4\n 5 Source plan: `PLAN.md` (seed commit c2e0324). This file is the plan-mode plan\n 6 produced by `/plan-design-review`. It starts as a faithful copy of the supplied\n 7 requirements. Design gaps found during review are recorded below as PENDING\n 8 until the user approves each fix individually. Nothing in the \"Approved\n 9 implementation\" section changes until that approval lands.\n 10\n 11 Product scope is preserved as supplied: a single marketing landing page.\n 12 No DESIGN.md exists in this repo, so recommendations calibrate against\n 13 universal design principles, not a project design system.\n 14\n 15 ## Supplied requirements (verbatim from PLAN.md)\n 16\n 17 ### Layout\n 18 All headings, taglines, and body copy will be center-aligned for a\n 19 \"clean modern look.\" The hero h1 sits 8px above the subhead with no\n 20 breathing room; the CTA button is the same visual weight as a\n 21 secondary \"Learn more\" link directly beside it.\n 22\n 23 ## Approved implementation\n 24\n 25 _(Nothing approved beyond the supplied requirements yet. Items move here from\n 26 \"Pending design decisions\" only after an explicit per-issue approval.)_\n 27 \n 28 ## Pending design decisions\n 29 \n 30 _(Populated during the review passes. Each entry is a proposed fix, not a\n 31 completed decision.)_\n\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\n Do you want to create gstack-test-plan-design-floor.md?\n \u276f 1. Yes\n 2. Yes, and switch to accept edits (auto-approve file edits and common file commands) for this session (shift+tab)\n 3. No\n\n Esc to cancel \u00b7 Tab to amend\n", + "screenSha256": "ca7939cd9c82db92a4b5b8bc980777d90e4970977481687643f1c0ea3542d126", + "writeContent": "# Plan: Marketing landing page\n\n## Context\n\nSource plan: `PLAN.md` (seed commit c2e0324). This file is the plan-mode plan\nproduced by `/plan-design-review`. It starts as a faithful copy of the supplied\nrequirements. Design gaps found during review are recorded below as PENDING\nuntil the user approves each fix individually. Nothing in the \"Approved\nimplementation\" section changes until that approval lands.\n\nProduct scope is preserved as supplied: a single marketing landing page.\nNo DESIGN.md exists in this repo, so recommendations calibrate against\nuniversal design principles, not a project design system.\n\n## Supplied requirements (verbatim from PLAN.md)\n\n### Layout\nAll headings, taglines, and body copy will be center-aligned for a\n\"clean modern look.\" The hero h1 sits 8px above the subhead with no\nbreathing room; the CTA button is the same visual weight as a\nsecondary \"Learn more\" link directly beside it.\n\n## Approved implementation\n\n_(Nothing approved beyond the supplied requirements yet. Items move here from\n\"Pending design decisions\" only after an explicit per-issue approval.)_\n\n## Pending design decisions\n\n_(Populated during the review passes. Each entry is a proposed fix, not a\ncompleted decision.)_\n" +} diff --git a/test/helpers/claude-pty-runner.boundaries.unit.test.ts b/test/helpers/claude-pty-runner.boundaries.unit.test.ts index 8cb45f591..94d22c736 100644 --- a/test/helpers/claude-pty-runner.boundaries.unit.test.ts +++ b/test/helpers/claude-pty-runner.boundaries.unit.test.ts @@ -17,7 +17,6 @@ import { nativePlanCallFingerprint, devexStep0Boundary, type AskUserQuestionFingerprint, - pickDesignFocusAll, } from './claude-pty-runner'; describe('Step0BoundaryPredicate per-skill', () => { @@ -1773,21 +1772,3 @@ describe('explicit Step 0 complexity gate with size in native choices', () => { } }); }); - -describe('pickDesignFocusAll: the seed-declared all-seven answer for the pending 0D focus menu', () => { - const captured = require('../fixtures/design-floor-focus-36597762183.json'); - const q = () => structuredClone(captured.question); - test('census 36597762183 pending menu selects the all-seven option', () => { - expect(pickDesignFocusAll(q())).toBe(1); - }); - test.each([ - ['a different title', (x: any) => { x.question = x.question.replace('Review all 7 design dimensions, or focus?', 'Which fixes should I apply?'); }], - ['a non-narrowing alternative', (x: any) => { x.options[1].label = 'Approve every fix now'; }], - ['two all-seven options', (x: any) => { x.options[1].label = 'All seven dimensions'; }], - ['a bundled product approval', (x: any) => { x.question = x.question.replace('Net:', 'Also approve the CTA redesign.\nNet:'); }], - ['a foreign plan context', (x: any) => { x.question = x.question.replace('plan-design-review of PLAN.md', 'plan-design-review of OTHER.md'); }], - ['multi select', (x: any) => { x.multiSelect = true; }], - ])('%s is not answered', (_name, mutate) => { - const x = q(); mutate(x); expect(pickDesignFocusAll(x)).toBeNull(); - }); -}); diff --git a/test/helpers/claude-pty-runner.ts b/test/helpers/claude-pty-runner.ts index 20cc58945..ce7a5eaa6 100644 --- a/test/helpers/claude-pty-runner.ts +++ b/test/helpers/claude-pty-runner.ts @@ -21,7 +21,7 @@ export { nativePlanCallFingerprint, planCountQuestionPhase, parseQuestionPrompt, export type { AskUserQuestionFingerprint, Step0BoundaryPredicate } from './pty/auq'; export { assertReviewReportAtBottom, hasCompletePlanReport, hasNativePlanCompletion, isQuestionlessNativePlanExit, evaluateOwnedNativePlanTerminal, hasNativePlanTerminal, assertReportAtBottomIfPlanWritten } from './pty/plan-native'; export type { ReviewReportAtBottomResult, NativePlanTerminalReview, NativePlanTerminalAssessment, NativePlanTerminalEvaluator } from './pty/plan-native'; -export { ceoStep0Boundary, engSetupAUQ, engFirstReviewAUQ, engStep0Boundary, pickDesignFocusAll } from './pty/boundaries'; +export { ceoStep0Boundary, engSetupAUQ, engFirstReviewAUQ, engStep0Boundary } from './pty/boundaries'; export { runPlanSkillObservation } from './pty/runners/observation'; export type { PlanSkillObservation, PlanSkillObservationOptions } from './pty/runners/observation'; export { runPlanSkillCounting, countingCapture, isNativeCompletionSummary, readEngTestPlans } from './pty/runners/counting'; diff --git a/test/helpers/docsync-interface.ts b/test/helpers/docsync-interface.ts index 382508de8..66d192a1e 100644 --- a/test/helpers/docsync-interface.ts +++ b/test/helpers/docsync-interface.ts @@ -5,7 +5,7 @@ import { readOnlyInterfaceText } from './read-only-commands'; export function docsNativeInterface(fixture: Pick, 'home' | 'repo' | 'skills'>, scripts: string[] = [], transport = false, insert = false, file?: string): string { const skills = fixture.skills.split(path.sep).join('/'); - return `Fixture observation interface (applies to parent and every child; ${file ? `saved at ${file}: each child prompt tells the child to Read that file first instead of copying this text` : 'include this interface in child prompts'}): Bash may execute only separate literal pwd, ls, cat, stat, sha256sum, Git read commands (status, diff, show, log, ls-files, ls-tree, check-ignore, rev-parse, merge-base, hash-object without -w, branch --show-current, describe with only --tags, --always, --abbrev= or --exact-match and at most one revision), the exact generated Preamble block with its spawned prefix, literal installed gstack-skill-start/gstack-skill-end commands for document-release (start requires GSTACK_SESSION_KIND=spawned) or their own --help, or the literal installed ${skills}/bin/gstack-docs-candidate snapshot (its --out a private .json under ${fixture.home}), compare and --help commands. ${readOnlyInterfaceText(fixture.home)} The Git reads above may join such a line too, but not in a line with cd. A for loop over literal words (a lowercase loop variable; words without quotes, globs or $) whose body uses only such reads counts as those reads, one pass per word, wherever it sits in the line. Every read stays off actor-state.json and off ${docsConfigDir(fixture.home)} except its settings.json: that directory holds Claude's credentials and every agent's session transcript, so judge a child only by the output it returns (its echoed SESSION_KIND line included) and by the repository. The Preamble block, lifecycle and candidate commands and scripts never join such a line: run each as its own Bash call, never joined to another command with ;, &&, || or a pipe. Otherwise no shell composition, custom interpreters, arbitrary scripts, inline eval or memory-mapped writes. The only additional scripts are ${scripts.length ? scripts.join(', ') : 'none'}.${insert ? ` To insert one saved private Markdown artifact into another (a saved section file into a report), Bash may also run the single literal command cat SOURCE.md >> TARGET.md with absolute paths under ${fixture.home}.` : ''} Read/Glob/Grep remain available; Read skill and section files with Read (offset/limit for ranges), because Bash output over 30KB becomes a truncated preview. Use Write/Edit for permitted docs and private JSON/Markdown artifacts under ${fixture.home}; do not rewrite installed skills, config, actor state or scripts. No effects outside the owned fixture. The owner preserves evidence and cleans up. Missing observer coverage blocks acceptance; the Linux kernel monitor covers syscall writes in the product tree, not hostile processes or arbitrary external destinations. + return `Fixture observation interface (applies to parent and every child; ${file ? `saved at ${file}: each child prompt tells the child to Read that file first instead of copying this text` : 'include this interface in child prompts'}): Bash may execute only separate literal pwd, ls, cat, stat, sha256sum and Git read commands (status, diff, show, log, ls-files, ls-tree, check-ignore, rev-parse, merge-base, hash-object without -w, branch --show-current, describe with only --tags, --always, --abbrev= or --exact-match and at most one revision). Bash may also run each of the following as its own Bash call, with nothing joined before or after it by ;, &&, || or a pipe (a --help is no exception, so never add one to a line of reads): the exact generated Preamble block with its spawned prefix, literal installed gstack-skill-start/gstack-skill-end commands for document-release (start requires GSTACK_SESSION_KIND=spawned) or their own --help, or the literal installed ${skills}/bin/gstack-docs-candidate snapshot (its --out a private .json under ${fixture.home}), compare and --help commands. ${readOnlyInterfaceText(fixture.home)} The Git reads above may join such a line too, but not in a line with cd. A for loop over literal words (a lowercase loop variable; words without quotes, globs or $) whose body uses only such reads counts as those reads, one pass per word, wherever it sits in the line. Every read stays off actor-state.json and off ${docsConfigDir(fixture.home)} except its settings.json: that directory holds Claude's credentials and every agent's session transcript, so judge a child only by the output it returns (its echoed SESSION_KIND line included) and by the repository. The additional scripts below also run each as its own Bash call; no own-call command ever joins such a line. Otherwise no shell composition, custom interpreters, arbitrary scripts, inline eval or memory-mapped writes. The only additional scripts are ${scripts.length ? scripts.join(', ') : 'none'}.${insert ? ` To insert one saved private Markdown artifact into another (a saved section file into a report), Bash may also run the single literal command cat SOURCE.md >> TARGET.md with absolute paths under ${fixture.home}.` : ''} Read/Glob/Grep remain available; Read skill and section files with Read (offset/limit for ranges), because Bash output over 30KB becomes a truncated preview. Use Write/Edit for permitted docs and private JSON/Markdown artifacts under ${fixture.home}; do not rewrite installed skills, config, actor state or scripts. No effects outside the owned fixture. The owner preserves evidence and cleans up. Missing observer coverage blocks acceptance; the Linux kernel monitor covers syscall writes in the product tree, not hostile processes or arbitrary external destinations. The working directory for parent and child Bash calls is already ${fixture.repo}. Run Git reads directly, for example: git status, git diff --cached, git merge-base main HEAD, git rev-parse HEAD. Do not use Git global options such as -C, -c, --git-dir or --work-tree, and do not prepend cd or another shell wrapper. The literal git subcommand must immediately follow git; an absolute owned repository path does not make git -C an allowed command. diff --git a/test/helpers/free-fixtures-data.ts b/test/helpers/free-fixtures-data.ts index aaca3c551..4fcd60246 100644 --- a/test/helpers/free-fixtures-data.ts +++ b/test/helpers/free-fixtures-data.ts @@ -74,7 +74,9 @@ export const FREE_FIXTURES: Readonly> = { 'test/fixtures/design-count-native-8525.json': ['test/plan-count-completion.test.ts'], 'test/fixtures/design-count-native-issue-fields.json': ['test/plan-count-completion.test.ts'], 'test/fixtures/design-detector-source-public.json': ['test/design-detector-source-fixture.test.ts'], - 'test/fixtures/design-floor-focus-36597762183.json': ['test/helpers/claude-pty-runner.boundaries.unit.test.ts'], + 'test/fixtures/design-floor-focus-36597762183.json': ['test/plan-design-floor-fixture.test.ts'], + 'test/fixtures/design-floor-focus-review-focus.json': ['test/plan-design-floor-fixture.test.ts'], + 'test/fixtures/design-floor-focus-template-0d.json': ['test/plan-design-floor-fixture.test.ts'], 'test/fixtures/design-handoff-n-calls.json': ['test/design-completion-handoff-scored.test.ts'], 'test/fixtures/design-handoff-q-calls.json': ['test/design-completion-handoff-scored.test.ts'], 'test/fixtures/design-md-legacy.md': ['test/design-md.test.ts'], @@ -155,6 +157,7 @@ export const FREE_FIXTURES: Readonly> = { 'test/fixtures/parity-baseline-v1.81.0.0.json': ['test/skill-size-budget.test.ts'], 'test/fixtures/plan-ceo-floor-create-quote-wrap.json': ['test/plan-create-prepublication.test.ts'], 'test/fixtures/plan-ceo-floor-edit-prepublication.json': ['test/plan-edit-cropped-permission.test.ts'], + 'test/fixtures/plan-design-floor-create-rule-crop.json': ['test/plan-create-prepublication.test.ts'], 'test/fixtures/plan-ceo-floor-toolonly-create.json': ['test/plan-create-prepublication.test.ts', 'test/plan-floor-permission.test.ts'], 'test/fixtures/plan-count-design-questionless-report.md': ['test/plan-count-empty-review.test.ts'], 'test/fixtures/plan-create-cropped-title-batching.json': ['test/plan-create-combined-permission.test.ts'], diff --git a/test/helpers/plan-count-file-permission.ts b/test/helpers/plan-count-file-permission.ts index 805ff5c11..e6a534251 100644 --- a/test/helpers/plan-count-file-permission.ts +++ b/test/helpers/plan-count-file-permission.ts @@ -333,9 +333,11 @@ function currentCreatePreview(preview: string, r: any, config: string, cwd: stri Buffer.byteLength(event.input.content)>MAX_WRITE_INPUT_BYTES) return false; // A crop can keep the pane's file row and rule above the preview while its // "Create file" title scrolls away. That row must name the owned path. + // It can also start on that single rule, as croppedEditPane allows. const header=/^ {0,3}(?![1-9]\d*(?:[ \t]|\n))(\S[^\n]*)\n[╌─━]{3,}[ \t]*\n/.exec(preview); if(header && path.resolve(cwd,header[1]!.trim())!==r.expected) return false; - const source=event.input.content.split(/\r?\n/), rows=preview.slice(header?.[0].length ?? 0).split('\n'); + const body=header ? preview.slice(header[0].length) : preview.replace(/^[╌─━]{3,}[ \t]*\n/,''); + const source=event.input.content.split(/\r?\n/), rows=body.split('\n'); const numbered:Array<{line:number;text:string}>=[]; let leading=''; for(const row of rows) { diff --git a/test/helpers/pty/boundaries.ts b/test/helpers/pty/boundaries.ts index 53eb76e57..8c87100c9 100644 --- a/test/helpers/pty/boundaries.ts +++ b/test/helpers/pty/boundaries.ts @@ -637,27 +637,3 @@ export const engStep0Boundary: Step0BoundaryPredicate = (fp) => // the skill-name prefix; live qids observed: plan-eng-review-jitter, // plan-eng-review-idempotency, plan-eng-review-todos-e2e-concurrent. /gstack-qid:\s*(?:plan-)?eng-review-/i.test(fp.promptSnippet); - -/** - * The seed declares "Design: review all seven dimensions" for the pending Step 0D - * focus menu. Pick its single all-seven option only when every other option - * narrows the review and the brief approves no product action. - */ -export function pickDesignFocusAll(q: NativePlanQuestionCall['questions'][number]): number | null { - if (q.multiSelect || q.options.length < 2 || q.options.length > 4 || new Set(q.options.map(o => o.label)).size !== q.options.length) return null; - const text = q.question.trim(); - const title = text.split(/\r?\n/, 1)[0]!.replace(/^D[1-9]\d*\s*[—–:-]\s*/i, ''); - const sources = [...text.matchAll(/^Project\/branch\/task:\s*([^\n]+)$/gm)]; - const explanatory = [text, ...q.options.map(o => o.description ?? '')].join('\n') - .replace(/`+[^`]*`+|"[^"\n]*"|“[^”\n]*”|‘[^’\n]*’/g, '').replace(/[✅❌*]/g, ''); - if (!/^Review all (?:7|seven) (?:design )?(?:dimensions|passes),? or focus(?: on [^?\n]+)?\?$/i.test(title) || - sources.length !== 1 || !/\bplan-design-review of PLAN\.md\b/i.test(sources[0]![1]!) || - /\b(?:historical|archived|quoted|example|foreign|other|another|previous)\b/i.test(sources[0]![1]!) || - (text.match(/\?/g)?.length ?? 0) !== 1 || /```|~~~|^\s*>/m.test(text) || - !/^ELI10:\s*I['’]ve rated this plan (?:10(?:\.0+)?|[0-9](?:\.\d+)?)\/10 on design completeness\./mi.test(text) || - /(?:^|[.!?;:\n]|\b(?:and|while))\s*(?:(?:also|please|then|now)\s+)*(?:approv(?:e|ing)|deploy(?:ing)?|implement(?:ing)?|ship(?:ping)?|merg(?:e|ing)|delet(?:e|ing))\b/im.test(explanatory)) return null; - const labels = q.options.map(o => o.label.trim().replace(/^[A-Z][).:]\s+/i, '').replace(/\s*\(recommended\)\s*$/i, '')); - const all = labels.flatMap((label, i) => /^(?:Review )?All (?:7|seven) (?:design )?(?:dimensions|passes)$/i.test(label) ? [i + 1] : []); - if (all.length !== 1 || labels.some((label, i) => i + 1 !== all[0] && !/^(?:Only|Focus)\b/i.test(label))) return null; - return all[0]!; -} diff --git a/test/helpers/pty/runners/floor.ts b/test/helpers/pty/runners/floor.ts index 57fa44c31..54e214583 100644 --- a/test/helpers/pty/runners/floor.ts +++ b/test/helpers/pty/runners/floor.ts @@ -16,7 +16,6 @@ import { isDeepStrictEqual } from 'node:util'; import { randomUUID } from 'node:crypto'; import { capturePlanCountQuestion, createPlanCountPermissionGuard, matchesNativePlanQuestion, planCountPrerequisitePick, planCountQuestionInput } from '../auq'; import { resolveClaudeBinary } from '../binary'; -import { pickDesignFocusAll } from '../boundaries'; import { SANCTIONED_WRITE_SUBSTRINGS, isProseAUQVisible, planCountSubmissionInput } from '../classify'; import type { ClaudePtySession } from '../launch'; import { isNumberedOptionListVisible, isPermissionDialogVisible, isPlanReadyVisible, isRejectedSlashCommand, stripPtyResidue } from '../screen'; @@ -145,6 +144,8 @@ export interface PlanSkillFloorOptions { productType?: 'sdk-documentation'; /** Literal persona/journey correction declared before DX setup; approves no offered remedy. */ devexSetupContext?: string; + /** Case-owned answer to a declared setup menu (the design seed's Step 0D focus choice); null leaves the question to the floor review. */ + pickSetupChoice?: (question: NativePlanQuestionCall['questions'][number]) => number | null; /** Installation cwd retained for caller compatibility; review uses an owned seeded project. */ cwd?: string; /** Total budget. Default 600000 (10 min). Tests exit early on AUQ. */ @@ -444,7 +445,7 @@ function floorQuestion(run: FloorRun, session: ClaudePtySession, currentCalls: N const chosen = run.setupChoices.get(key) ?? new Set(); const pick = pickPlanFloorMode(opts.skillName, question) ?? planCountPrerequisitePick(fp, fp) ?? (opts.skillName === 'plan-devex-review' ? pickPlanFloorProductType(question, opts.productType) : null) - ?? (opts.skillName === 'plan-design-review' ? pickDesignFocusAll(question) : null); + ?? opts.pickSetupChoice?.(question) ?? null; if (pick !== null) { if (!chosen.has(index)) { session.send(planCountQuestionInput(viewport, fp, pick)); diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index 004b48abb..e48a40bdd 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -379,7 +379,7 @@ export const E2E_TOUCHFILES: Record = { "test/fixtures/plan-scope-recovery-av.json", - "test/fixtures/design-scope-checkpoint-at.json",'plan-design-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/fixtures/design-ui-focus-capture-39298269902-a1.json', 'test/fixtures/design-ui-focus-capture-39298269902-a2.json', 'test/fixtures/design-ui-focus-capture-39363705692.json', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty/**', 'test/fixtures/design-ui-boxed-question.json', 'test/helpers/hermetic-skill-runtime.ts', 'test/helpers/pty-trust-dialog.ts', 'test/skill-e2e-plan-design-with-ui.test.ts', 'test/fixtures/eng-d2-truncated-border-0bcd.json', 'test/fixtures/eng-d1-clipped-elision-1579.json', 'test/fixtures/eng-d2-planning-prelude-4d.json', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', + "test/fixtures/design-scope-checkpoint-at.json",'plan-design-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/fixtures/design-ui-focus-capture-39298269902-a1.json', 'test/fixtures/design-ui-focus-capture-39298269902-a2.json', 'test/fixtures/design-ui-focus-capture-39363705692.json', 'test/fixtures/design-ui-focus-capture-39852003387.json', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty/**', 'test/fixtures/design-ui-boxed-question.json', 'test/helpers/hermetic-skill-runtime.ts', 'test/helpers/pty-trust-dialog.ts', 'test/skill-e2e-plan-design-with-ui.test.ts', 'test/fixtures/eng-d2-truncated-border-0bcd.json', 'test/fixtures/eng-d1-clipped-elision-1579.json', 'test/fixtures/eng-d2-planning-prelude-4d.json', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', "scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts", @@ -477,7 +477,7 @@ export const E2E_TOUCHFILES: Record = { 'test/fixtures/ceo-report-permission-fb10.json', 'test/fixtures/plan-floor-permission-fb10.json', 'test/helpers/plan-count-file-permission.ts', 'test/fixtures/plan-edit-cropped-permission-1579.json', 'test/fixtures/plan-count-permission-target-ad-v2.json', 'test/fixtures/design-crop-gutter-ap.json', 'test/fixtures/plan-count-crop-ak.json', 'test/fixtures/plan-count-long-edit-0bcd.json', 'test/fixtures/plan-count-cropped-wrap-6714.json', "test/fixtures/plan-scope-recovery-av.json", - "test/fixtures/design-scope-checkpoint-at.json",'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty/**', 'test/fixtures/design-ui-boxed-question.json', 'test/helpers/hermetic-skill-runtime.ts', 'test/helpers/pty-trust-dialog.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-design-finding-floor.test.ts', 'test/fixtures/eng-d2-truncated-border-0bcd.json', 'test/fixtures/eng-d1-clipped-elision-1579.json', 'test/fixtures/eng-d2-planning-prelude-4d.json', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/helpers/plan-floor-target.ts', 'test/helpers/plan-count-fixture.ts', 'test/helpers/plan-count-artifacts.ts', + "test/fixtures/design-scope-checkpoint-at.json",'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty/**', 'test/fixtures/design-ui-boxed-question.json', 'test/helpers/hermetic-skill-runtime.ts', 'test/helpers/pty-trust-dialog.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/fixtures/design-floor-focus-picker.ts', 'test/skill-e2e-plan-design-finding-floor.test.ts', 'test/fixtures/eng-d2-truncated-border-0bcd.json', 'test/fixtures/eng-d1-clipped-elision-1579.json', 'test/fixtures/eng-d2-planning-prelude-4d.json', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/helpers/plan-floor-target.ts', 'test/helpers/plan-count-fixture.ts', 'test/helpers/plan-count-artifacts.ts', "scripts/resolvers/preamble/generate-preamble-bash.ts", diff --git a/test/plan-create-prepublication.test.ts b/test/plan-create-prepublication.test.ts index 96ae0ac31..08a1cd2ab 100644 --- a/test/plan-create-prepublication.test.ts +++ b/test/plan-create-prepublication.test.ts @@ -10,6 +10,7 @@ import {createPlanCountSnapshotWriter} from './helpers/plan-count-artifacts'; import captured from './fixtures/plan-create-prepublication-491.json'; import toolOnlyCreate from './fixtures/plan-ceo-floor-toolonly-create.json'; import quoteWrapCreate from './fixtures/plan-ceo-floor-create-quote-wrap.json'; +import ruleCropCreate from './fixtures/plan-design-floor-create-rule-crop.json'; function fixture(){ const root=fs.mkdtempSync(path.join(os.tmpdir(),'gstack-prewrite-'));const cwd=path.join(root,'owned'),config=path.join(root,'config');fs.mkdirSync(cwd); @@ -223,14 +224,15 @@ test('a tool-only session gets no credit for a changed preview or a foreign crea // Focused trial t3 (2026-10-20): a soft-wrapped table row continued with '>activation->paid)' and the // blockquote guard refused the owned Create menu until permission_unanswered. -function quoteWrap(){ +type CapturedCreate={basename:string;sessionId:string;toolUseId:string;writeContent:string;screen:string}; +function quoteWrap(capture:CapturedCreate=quoteWrapCreate){ const root=fs.mkdtempSync(path.join(os.tmpdir(),'gstack-quote-wrap-')),cwd=path.join(root,'owned'),config=path.join(root,'config');fs.mkdirSync(cwd); - const expected=path.join(cwd,quoteWrapCreate.basename),sid=quoteWrapCreate.sessionId,id=quoteWrapCreate.toolUseId; + const expected=path.join(cwd,capture.basename),sid=capture.sessionId,id=capture.toolUseId; const journal=path.join(config,'projects','owned',sid+'.jsonl');fs.mkdirSync(path.dirname(journal),{recursive:true}); - const startedAt=Date.now()-1000,recorder=createFilePermissionRecorder(cwd,config,expected)!,input={file_path:expected,content:quoteWrapCreate.writeContent}; + const startedAt=Date.now()-1000,recorder=createFilePermissionRecorder(cwd,config,expected)!,input={file_path:expected,content:capture.writeContent}; fs.writeFileSync(journal,JSON.stringify({cwd,sessionId:sid,isSidechain:false,timestamp:new Date().toISOString(),message:{role:'assistant',content:[{type:'text',text:'Writing the working plan.'}]}})+'\n'); recordFilePermission(JSON.stringify({hook_event_name:'PreToolUse',tool_name:'Write',session_id:sid,tool_use_id:id,cwd,transcript_path:journal,tool_input:input}),recorder.file,cwd,config,expected); - const read=(screen=quoteWrapCreate.screen)=>currentFilePermissionEpoch(recorder.file,expected,cwd,config,startedAt,readPlanCountTranscript(config,cwd),screen); + const read=(screen=capture.screen)=>currentFilePermissionEpoch(recorder.file,expected,cwd,config,startedAt,readPlanCountTranscript(config,cwd),screen); return {sid,id,read,close(){recorder.dispose();fs.rmSync(root,{recursive:true,force:true});}}; } @@ -255,3 +257,28 @@ test.each(['quoted-row','fenced-row','checkbox','changed-continuation'])('t3 neg expect(createPlanCountPermissionGuard()(changed,'',f.read(changed))).not.toBe('grant'); }finally{f.close();} }); + +// Periodic run 39209106033 (plan-design-finding-floor, issue #2756): the owned Create menu's title and file row +// scrolled away, so the crop starts on the native rule above the preview. The menu sat until the 610 s timeout. +test('39209106033: a Create crop that starts on the rule above the preview binds the owned Write once',()=>{ + const f=quoteWrap(ruleCropCreate);try{ + expect(createHash('sha256').update(ruleCropCreate.screen).digest('hex')).toBe(ruleCropCreate.screenSha256); + expect(ruleCropCreate.screen).toMatch(/^╌{3,}\n 1 # Plan: Marketing landing page\n/); + expect(f.read()?.pendingId).toBe(`${f.sid}:${f.id}`); + const guard=createPlanCountPermissionGuard();expect(guard(ruleCropCreate.screen,'',f.read())).toBe('grant');expect(guard(ruleCropCreate.screen,'',f.read())).toBe('handled'); + }finally{f.close();} +}); + +test.each(['two-rules','text-above-rule','quoted-row','changed-row','foreign-basename'])('39209106033 negative control: %s is refused',mode=>{ + const f=quoteWrap(ruleCropCreate);try{ + const screen=ruleCropCreate.screen,rule=screen.slice(0,screen.indexOf('\n')+1),row=' 12 No DESIGN.md exists in this repo'; + const changed=mode==='two-rules'?rule+screen + :mode==='text-above-rule'?'Writing the plan now\n'+screen + :mode==='quoted-row'?screen.replace(row,' > Question: D1 approve the layout?\n'+row) + :mode==='changed-row'?screen.replace(row,' 12 A DESIGN.md exists in this repo') + :screen.replace('Do you want to create gstack-test-plan-design-floor.md?','Do you want to create other-plan.md?'); + expect(changed).not.toBe(screen); + // No owned epoch, so the floor and count runners never reach the guard's grant. + expect(f.read(changed) ?? null).toBeNull(); + }finally{f.close();} +}); diff --git a/test/plan-design-floor-fixture.test.ts b/test/plan-design-floor-fixture.test.ts index 2d4edc1d3..5a87d40f3 100644 --- a/test/plan-design-floor-fixture.test.ts +++ b/test/plan-design-floor-fixture.test.ts @@ -1,7 +1,8 @@ -import { expect, test } from 'bun:test'; +import { describe, expect, test } from 'bun:test'; import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; +import { pickDesignFocusAll } from './fixtures/design-floor-focus-picker'; const ROOT = path.resolve(import.meta.dir, '..'); @@ -18,6 +19,7 @@ import * as path from 'node:path'; import { execFileSync } from 'node:child_process'; import { FORCING_FLOOR_DESIGN } from ${JSON.stringify(path.join(ROOT, 'test/fixtures/forcing-finding-seeds.ts'))}; import { CAPTURE_LONG_MS } from ${JSON.stringify(path.join(ROOT, 'test/helpers/eval-budgets.ts'))}; +import { pickDesignFocusAll } from ${JSON.stringify(path.join(ROOT, 'test/fixtures/design-floor-focus-picker.ts'))}; const outcome = ${JSON.stringify(outcome)}; let calls = 0; mock.module(${JSON.stringify(path.join(ROOT, 'test/helpers/e2e-gate.ts'))}, () => ({ @@ -35,6 +37,7 @@ mock.module(${JSON.stringify(path.join(ROOT, 'test/helpers/claude-pty-runner.ts' requestedPlanPath: '/tmp/gstack-test-plan-design-floor.md', timeoutMs: CAPTURE_LONG_MS, env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, + pickSetupChoice: pickDesignFocusAll, }); const input = fs.readFileSync(path.join(opts.cwd, 'review-input.md'), 'utf8'); expect(input).toBe(FORCING_FLOOR_DESIGN); @@ -94,3 +97,42 @@ test('design floor cleans its project when observation throws and preserves the expect(result.code, result.output).toBe(1); expect(result.output).toContain('controlled floor observation failure'); }, 20_000); + +describe('pickDesignFocusAll: the seed-declared all-seven answer for the pending 0D focus menu', () => { + const captured = require('./fixtures/design-floor-focus-36597762183.json'); + const q = () => structuredClone(captured.question); + test('census 36597762183 pending menu selects the all-seven option', () => { + expect(pickDesignFocusAll(q())).toBe(1); + }); + test.each([ + ['a different title', (x: any) => { x.question = x.question.replace('Review all 7 design dimensions, or focus?', 'Which fixes should I apply?'); }], + ['a non-narrowing alternative', (x: any) => { x.options[1].label = 'Approve every fix now'; }], + ['two all-seven options', (x: any) => { x.options[1].label = 'All seven dimensions'; }], + ['a bundled product approval', (x: any) => { x.question = x.question.replace('Net:', 'Also approve the CTA redesign.\nNet:'); }], + ['a foreign plan context', (x: any) => { x.question = x.question.replace('plan-design-review of PLAN.md', 'plan-design-review of OTHER.md'); }], + ['multi select', (x: any) => { x.multiSelect = true; }], + ])('%s is not answered', (_name, mutate) => { + const x = q(); mutate(x); expect(pickDesignFocusAll(x)).toBeNull(); + }); +}); + +describe('pickDesignFocusAll: Step 0D in the template\'s own words (GSTA-65 focused trial)', () => { + const captured = require('./fixtures/design-floor-focus-template-0d.json'); + const q = () => structuredClone(captured.question); + test('the pending template-worded menu selects the all-seven option', () => { + expect(pickDesignFocusAll(q())).toBe(1); + }); + test("a menu titled 'Review focus' with no question mark selects the all-seven option", () => { + expect(pickDesignFocusAll(structuredClone(require('./fixtures/design-floor-focus-review-focus.json').question))).toBe(1); + }); + test.each([ + ['a question not about focus', (x: any) => { x.question = x.question.replace('Want me to focus on specific areas instead?', 'Want me to apply every fix now?'); }], + ['a foreign plan', (x: any) => { x.question = x.question.replace('Marketing landing page plan on main.', 'Pricing page plan on main.'); }], + ['a quoted plan name', (x: any) => { x.question = x.question.replace('Marketing landing page plan on main.', 'OTHER.md, not "Marketing landing page".'); }], + ['an alternative that does not narrow', (x: any) => { x.options[1].label = 'Layout/hierarchy first'; }], + ['a second question', (x: any) => { x.question = x.question.replace('Net:', 'Ready?\nNet:'); }], + ['no focus in the title or question', (x: any) => { x.question = x.question.replace('Want me to focus on specific areas instead?', 'Which areas should I cover?'); }], + ])('%s is not answered', (_name, mutate) => { + const x = q(); mutate(x); expect(pickDesignFocusAll(x)).toBeNull(); + }); +}); diff --git a/test/plan-design-with-ui-fixture.test.ts b/test/plan-design-with-ui-fixture.test.ts index cd9539ff0..ddf2f5018 100644 --- a/test/plan-design-with-ui-fixture.test.ts +++ b/test/plan-design-with-ui-fixture.test.ts @@ -91,6 +91,10 @@ const captured39298269902 = [1, 2].map(attempt => JSON.parse(fs.readFileSync(${J // and offered "All 7 areas (recommended)" against "Only specific areas". const captured39363705692 = JSON.parse(fs.readFileSync(${JSON.stringify(path.join(ROOT, 'test/fixtures/design-ui-focus-capture-39363705692.json'))}, 'utf8')) .calls.map(({ preReview, observedAtMs, call }) => nativePlanCallFingerprint(call, observedAtMs, preReview)); +// Run 39852003387 slice 4 (357027f): Opus 5.5's ELI10 rated the plan "4/10 on design" (no +// "completeness") and offered "All 7 passes (recommended)" against "States + layout only". +const captured39852003387 = JSON.parse(fs.readFileSync(${JSON.stringify(path.join(ROOT, 'test/fixtures/design-ui-focus-capture-39852003387.json'))}, 'utf8')) + .calls.map(({ preReview, observedAtMs, call }) => nativePlanCallFingerprint(call, observedAtMs, preReview)); const unnumberedFinding = fp('unnumbered-finding', "Which region is the dashboard's primary anchor, and how is the page composed?", false); unnumberedFinding.nativeCall.questions[0].options = nativeFinding.nativeCall.questions[0].options; const finding = fp('finding', 'Which loading feedback should Save show?', false); @@ -215,15 +219,21 @@ mock.module(${JSON.stringify(path.join(ROOT, 'test/helpers/claude-pty-runner.ts' expect(opts.isLastStep0AUQ(captured39363705692[0])).toBe(true); expect(opts.isReviewAUQ(captured39363705692[0])).toBe(false); expect(opts.isReviewAUQ(captured39363705692[3])).toBe(true); + expect(opts.isLastStep0AUQ(captured39852003387[0])).toBe(true); + expect(opts.isReviewAUQ(captured39852003387[0])).toBe(false); + expect(captured39852003387.slice(1, 3).some(call => opts.isLastStep0AUQ(call) || opts.isReviewAUQ(call))).toBe(false); + expect(opts.isReviewAUQ(captured39852003387[3])).toBe(true); const areas = ['All 7 areas (recommended)', 'Only specific areas']; for (const body of ['ELI10: I rated this plan 4/10 for design completeness.', 'ELI10: This plan is a 3/10 on design completeness.', - 'ELI10: Design completeness: 6.5/10. The gaps are layout and states.']) { + 'ELI10: Design completeness: 6.5/10. The gaps are layout and states.', 'ELI10: I rated this plan 4/10 on design.']) { expect(opts.isLastStep0AUQ(labeled('worded-rating', 'D1 — Review focus\\nProject/branch/task: main.\\n' + body, areas))).toBe(true); } for (const [label, question, labels] of [ ['rating-split-across-sentences', 'D1 — Review focus\\nELI10: I rated this plan 4/10. Design completeness matters most here.', areas], ['rating-out-of-seven', 'D1 — Review focus\\nELI10: I rated this plan 4/7 for design completeness.', areas], ['rating-not-design', 'D1 — Review focus\\nELI10: I rated this plan 4/10 for test completeness.', areas], + ['design-split-from-rating', 'D1 — Review focus\\nELI10: I rated this plan 4/10. The design needs mockups first.', areas], + ['rating-unscored-design', 'D1 — Review focus\\nELI10: I rated this plan on design. It needs mockups first.', areas], ['areas-without-focus', 'D1 — Which plan should I review?\\nELI10: I rated this plan 4/10 for design completeness.', areas], ['areas-not-leading', 'D1 — Review focus\\nELI10: I rated this plan 4/10 for design completeness.', ['Review all 7 areas (recommended)', 'Only specific areas']], ['issue-labeled-areas', 'D5 — Issue 1: Which panel leads the review focus?\\nELI10: I rated this plan 4/10 for design completeness.', ['1A All 7 areas first', '1B Activity first']], @@ -286,6 +296,8 @@ mock.module(${JSON.stringify(path.join(ROOT, 'test/helpers/claude-pty-runner.ts' : mode === 'captured-39298269902-focus-only' ? captured39298269902[1] : mode === 'captured-39363705692' ? captured39363705692 : mode === 'captured-39363705692-no-focus' ? captured39363705692.filter((_, index) => index !== 0) + : mode === 'captured-39852003387' ? captured39852003387 + : mode === 'captured-39852003387-no-focus' ? captured39852003387.filter((_, index) => index !== 0) : mode === 'captured-focus-after-finding' ? [...captured.filter((_, index) => index !== 1), captured[1]] : mode.startsWith('target-menu') ? [target, fp('other', 'Which artifact should I inspect?', false)] : mode === 'early-exit' ? [] : mode === 'focus-only' ? [chosenFocus] @@ -330,12 +342,12 @@ await import(${JSON.stringify(path.join(ROOT, 'test/skill-e2e-plan-design-with-u } finally { fs.rmSync(directory, { recursive: true, force: true }); } } -test.each(['source', 'paraphrase', 'native-sequence', 'native-unnumbered', 'native-passes', 'captured', 'captured-rated', 'captured-39298269902', 'captured-39363705692'])('UI gate seeds the exact target and accepts Design progress with an unselected no-UI alternative (%s)', mode => { +test.each(['source', 'paraphrase', 'native-sequence', 'native-unnumbered', 'native-passes', 'captured', 'captured-rated', 'captured-39298269902', 'captured-39363705692', 'captured-39852003387'])('UI gate seeds the exact target and accepts Design progress with an unselected no-UI alternative (%s)', mode => { const result = exercise(mode); expect(result.code, result.output).toBe(0); }, 20_000); -test.each(['target-menu', 'target-menu-design-system', 'focus-only', 'captured-no-focus', 'captured-focus-after-finding', 'captured-rated-no-focus', 'captured-39298269902-no-focus', 'captured-39298269902-focus-only', 'captured-39363705692-no-focus', 'native-setup-only', 'native-voices-only', 'early-exit', 'timeout', 'exited', 'unanswered-finding', 'failed-finding', 'no-native-finding'])('UI gate rejects %s and removes its fixture', mode => { +test.each(['target-menu', 'target-menu-design-system', 'focus-only', 'captured-no-focus', 'captured-focus-after-finding', 'captured-rated-no-focus', 'captured-39298269902-no-focus', 'captured-39298269902-focus-only', 'captured-39363705692-no-focus', 'captured-39852003387-no-focus', 'native-setup-only', 'native-voices-only', 'early-exit', 'timeout', 'exited', 'unanswered-finding', 'failed-finding', 'no-native-finding'])('UI gate rejects %s and removes its fixture', mode => { const result = exercise(mode); expect(result.code, result.output).toBe(1); expect(result.output).toContain('plan-design-review with UI scope FAILED'); diff --git a/test/skill-e2e-plan-design-finding-floor.test.ts b/test/skill-e2e-plan-design-finding-floor.test.ts index 5027336a4..a6aec58fa 100644 --- a/test/skill-e2e-plan-design-finding-floor.test.ts +++ b/test/skill-e2e-plan-design-finding-floor.test.ts @@ -9,6 +9,7 @@ import { CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets'; import { describeE2ETier } from './helpers/e2e-gate'; import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner'; import { FORCING_FLOOR_DESIGN } from './fixtures/forcing-finding-seeds'; +import { pickDesignFocusAll } from './fixtures/design-floor-focus-picker'; import { seedPlanReviewProject } from './helpers/ceo-finding-fixture'; import * as fs from 'node:fs'; import * as os from 'node:os'; @@ -33,6 +34,7 @@ describeE2E('/plan-design-review AskUserQuestion floor (periodic)', () => { cwd: project, timeoutMs: CAPTURE_LONG_MS, env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' }, + pickSetupChoice: pickDesignFocusAll, }); if (obs.outcome !== 'auq_observed') { diff --git a/test/skill-e2e-plan-design-with-ui.test.ts b/test/skill-e2e-plan-design-with-ui.test.ts index 088f626b8..b2ad7437b 100644 --- a/test/skill-e2e-plan-design-with-ui.test.ts +++ b/test/skill-e2e-plan-design-with-ui.test.ts @@ -32,11 +32,12 @@ const designFocusBoundary = (fp: AskUserQuestionFingerprint): boolean => // count may be a numeral or a word; the title must name design, unless the // body carries the step's own design-completeness rating (run 39199692993). const title = text.split(/\r?\n/, 1)[0]!; - // Step 0A's rating is a sentence that scores design completeness out of 10, + // Step 0A's rating is a sentence that scores the plan's design out of 10, // however it is worded ("rated this plan 4/10 for design completeness", run - // 39363705692; "This plan is a 3/10 on design completeness", the 0A example). + // 39363705692; "This plan is a 3/10 on design completeness", the 0A example; + // the ELI10 plain-words "I rated this plan 4/10 on design.", run 39852003387). const rated = (body: string) => body.split(/(?<=[.!?])\s+|\r?\n/).some(sentence => - /\bdesign completeness\b/i.test(sentence) && /(?