Files
gstack/test/skill-e2e-plan-design-with-ui.test.ts
T
Garry Tanandcapy-ai[bot] 85b0f531f5 v1.91.162.0 feat(evals): merge-train workflow side, inert behind the train policy block (GSTA-49 T1b) (#3279)
* feat(ship-queue): merge trains with pattern bisect and train-record verify (GSTA-49 T1)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* test(windows): curate ship-queue-train.test.ts out of the Windows subset (gh shim not resolved on windows-latest)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* feat(evals): merge-train workflow side, inert behind the train policy block (GSTA-49 T1b)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* chore(evals): T1b CHANGELOG under [Unreleased]; classify evals-main-backstop.yml as free-lane only

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(evals): plan-design-with-ui focus detector accepts Opus 5.5's 'Review focus' title with the step's rating and an all-7 option; replay run 39298269902

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* v1.91.162.0 feat(evals): merge-train workflow side, inert behind the train policy block (GSTA-49 T1b)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

---------

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
2026-10-20 02:13:39 +00:00

113 lines
6.9 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* /plan-design-review with UI scope (gate, paid, real-PTY).
*
* The exact UI-heavy plan is committed before the first model turn. Observe
* an acknowledged Design focus/rating question and a subsequent review
* question using the existing native counting driver. An initial target menu
* or a no-UI early exit cannot satisfy this positive path. Option descriptions
* are proposals, so mentioning "no UI scope" there is not an exit verdict.
*/
import { test } from 'bun:test';
import { PTY_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import { createDesignReviewPicker, DESIGN_BOARD_ACTOR_PROTOCOL } from './helpers/plan-review-board-feedback';
import * as fs from 'node:fs';
import * as path from 'node:path';
import {
runPlanSkillCounting,
type AskUserQuestionFingerprint,
} from './helpers/claude-pty-runner';
const describeE2E = describeE2ETier('gate');
const ROOT = path.resolve(import.meta.dir, '..');
const FIXTURE = path.join(ROOT, 'test', 'fixtures', 'plans', 'ui-heavy-feature.md');
const designFocusBoundary = (fp: AskUserQuestionFingerprint): boolean =>
fp.nativeCall?.answered === true && !fp.nativeCall.failed && fp.nativeCall.questions.some(({ question, options }) => {
const text = question.trim().replace(/^D\d+(?:\.\d+)?\s*[—–:-]\s*/i, '');
// A target menu can mention a design system without reviewing this plan.
// The source Step 0D question, or a native title (first line) that offers all
// seven design dimensions/passes versus a focus, in any order or wording. The
// count may be a numeral or a word; the title must name design, unless the
// body carries the step's own design-completeness rating (run 39199692993).
const title = text.split(/\r?\n/, 1)[0]!;
const rated = /\bI(?:['’]ve| have)? rated this plan (?:10(?:\.0+)?|[0-9](?:\.\d+)?)\/10 on design completeness\b/i;
// Runs 39298269902 a1/a2: a title that only names the focus ("Review focus.",
// on one line before Project/branch/task:), with the step's own rating in the
// body and an option that offers all seven dimensions against a narrower one.
const lead = text.split(/\r?\n|\s+(?=Project\/branch\/task:)/, 1)[0]!;
return /^I(?:['’]ve| have) rated this plan (?:10(?:\.0+)?|[0-9](?:\.\d+)?)\/10 on design completeness\.[\s\S]*\bWant me to focus on specific areas instead(?: of all (?:7|seven))?\?/i.test(text)
|| (/\ball (?:7|seven) (?:design )?(?:dimensions|passes)\b/i.test(title) && /\bfocus\b/i.test(title)
&& (/\bdesign\b/i.test(title) || rated.test(text.slice(title.length))) && /\?\s*$/.test(title))
|| (/\bfocus\b/i.test(lead) && rated.test(text) && options.length >= 2
&& options.some(({ label }) => /^\s*all (?:7|seven) (?:design |review )?(?:dimensions|passes)\b/i.test(label)));
});
// Require a choice about the supplied UI, not a workflow offer after focus.
// The review labels each finding's options with its issue number and a letter
// ("3A", "3B"; review-sections.md), so a complete issue-labeled option set
// identifies a Design finding whatever vocabulary its title uses. Otherwise both
// the question and an offered remedy must describe concrete UI behavior.
const issueLabeled = (options: Array<{ label: string }>): boolean => {
const labels = options.map(({ label }) => /^\s*(\d+)([A-Z])(?=[):.\s]|$)/.exec(label));
return labels.length >= 2 && labels.every(Boolean)
&& new Set(labels.map(match => match![1])).size === 1
&& new Set(labels.map(match => match![2])).size === labels.length;
};
const uiChoice = /\b(?:hierarch(?:y|ies)|see first|layout|compos(?:e|ed|ition)|anchor|regions?|panels?|notifications?|activity|quick actions?|loading|skeletons?|empty|errors?|success|modals?|toasts?|buttons?|links?|copy|typography|fonts?|spacing|contrast|colors?|breakpoints?|responsive|keyboard|focus (?:order|trap|management)|aria|a11y|accessibility)\b/i;
const designReviewFinding = (fp: AskUserQuestionFingerprint): boolean =>
fp.nativeCall?.answered === true && !fp.nativeCall.failed && !designFocusBoundary(fp) && fp.nativeCall.questions.some(({ question, options }) => {
const title = question.split(/\r?\n/, 1)[0]!.trim().replace(/^D\d+(?:\.\d+)?\s*[—–:-]\s*/i, '');
const setup = /\b(?:outside (?:design )?voices|cross[ -]project learnings|review (?:target|scope|mode)|what should I (?:design[ -])?review|which (?:artifact|plan|file))\b/i;
return !setup.test(title) && (issueLabeled(options)
|| (uiChoice.test(title) && options.some(option => uiChoice.test(`${option.label} ${option.description}`))));
});
describeE2E('/plan-design-review with UI scope (gate)', () => {
test(
'reviews the supplied UI plan through an acknowledged Design finding',
async () => {
const startedAt = Date.now();
const plan = fs.readFileSync(FIXTURE, 'utf8');
let picker: ReturnType<typeof createDesignReviewPicker> | undefined;
let pickerCwd: string | undefined;
const obs = await runPlanSkillCounting({
skillName: 'plan-design-review', slashCommand: '/plan-design-review',
followUpPrompt: ['Review the supplied UI plan in review-input.md. Read it before',
'choosing review scope.', '', plan, '', DESIGN_BOARD_ACTOR_PROTOCOL].join('\n'),
fixtureFiles: {'review-input.md': plan},
isLastStep0AUQ: designFocusBoundary,
isReviewAUQ: designReviewFinding,
reviewCountCeiling: 1,
observeSetupQuestions: true,
bindDesignBoardState: true,
pickAUQ: (_routing, active, context) => {
const call = active.nativeCall;
const index = active.nativeQuestionIndex ?? (call?.questions.length === 1 ? 0 : -1);
if (!call && (/\/boards\//.test(active.promptSnippet) || active.options.some(option => /\bSubmitted\b/.test(option.label)))) {
throw new Error('Design board choice requires an owned native question');
}
if (!call || call.answered || call.failed || index < 0 || !call.questions[index]) return null;
if (pickerCwd && pickerCwd !== context.cwd) throw new Error('Design board picker fixture changed');
pickerCwd = context.cwd;
picker ??= createDesignReviewPicker(context);
const question = call.questions[index];
return picker({...question, multiSelect: question.multiSelect ?? false,
options: question.options.map(option => ({...option, description: option.description ?? ''}))});
},
timeoutMs: 600_000 - (Date.now() - startedAt),
});
const focus = obs.fingerprints.findIndex(designFocusBoundary);
const postFocus = focus >= 0 && obs.fingerprints.slice(focus + 1).some(fp => !fp.preReview && designReviewFinding(fp));
if (obs.outcome !== 'ceiling_reached' || !postFocus) {
throw new Error(
`plan-design-review with UI scope FAILED: outcome=${obs.outcome}; no acknowledged Design focus and subsequent review question\n` +
`--- evidence (last 3KB) ---\n${obs.evidence}`,
);
}
},
PTY_MS,
);
});