mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-06 08:54:04 +01:00
* fix(evals): plan-design focus rating in plain words, docsync own-call rule at the candidate --help, floor Create menu cropped at its rule (GSTA-65) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(evals): floor actor answers Step 0D's focus menu in the template's own words (GSTA-65) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(evals): floor actor identifies Step 0D's focus menu by its structure, not one wording (GSTA-65) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(changelog): GSTA-65 paid-side flaky repairs under [Unreleased] Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(evals): the 0D focus picker moves into plan-design-finding-floor's owned fixture; no harness series reset (GSTA-65) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test(evals): the 0D picker rows join the floor case's owner test (test-of-test ratchet) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * v1.91.185.0 fix(evals): plan-design focus rating in plain words, docsync own-call rule, floor Create crop and 0D menu (GSTA-65) (#3358) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
361 lines
29 KiB
TypeScript
361 lines
29 KiB
TypeScript
import { expect, test } from 'bun:test';
|
|
import * as fs from 'node:fs';
|
|
import * as os from 'node:os';
|
|
import * as path from 'node:path';
|
|
|
|
const ROOT = path.resolve(import.meta.dir, '..');
|
|
|
|
// Execute the real paid registration with native observation mocked. The actual
|
|
// actor picker, exact seeding, owned binding, assertions and cleanup execute.
|
|
function exercise(mode: string) {
|
|
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'design-ui-free-'));
|
|
const script = path.join(directory, 'registration.test.ts');
|
|
const facts = path.join(directory, 'facts.json');
|
|
fs.writeFileSync(script, `
|
|
import { describe, expect, mock, spyOn } from 'bun:test';
|
|
import * as fs from 'node:fs';
|
|
import * as path from 'node:path';
|
|
import { execFileSync } from 'node:child_process';
|
|
const mode = ${JSON.stringify(mode)};
|
|
const fixture = ${JSON.stringify(path.join(ROOT, 'test/fixtures/plans/ui-heavy-feature.md'))};
|
|
const template = fs.readFileSync(${JSON.stringify(path.join(ROOT, 'plan-design-review/SKILL.md.tmpl'))}, 'utf8');
|
|
const focus = template.match(/### 0D\\. Focus Areas\\nAskUserQuestion: "([^\\n]+)"/)![1]
|
|
.replace('{N}', '4').replace('{X, Y, Z}', 'hierarchy, spacing, contrast');
|
|
const { DESIGN_BOARD_ACTOR_PROTOCOL, createDesignReviewPicker: actualCreateDesignReviewPicker } = await import(${JSON.stringify(path.join(ROOT, 'test/helpers/plan-review-board-feedback.ts'))});
|
|
const outsideVoices = JSON.parse(fs.readFileSync(${JSON.stringify(path.join(ROOT, 'test/fixtures/design-outside-voices-question.json'))}, 'utf8'));
|
|
let pickerScope;
|
|
let pickerQuestion;
|
|
mock.module(${JSON.stringify(path.join(ROOT, 'test/helpers/plan-review-board-feedback.ts'))}, () => ({
|
|
DESIGN_BOARD_ACTOR_PROTOCOL,
|
|
createDesignReviewPicker: scope => {
|
|
pickerScope = scope;
|
|
const actualPicker = actualCreateDesignReviewPicker(scope);
|
|
return question => { pickerQuestion = question; return actualPicker(question); };
|
|
},
|
|
}));
|
|
const { createPlanCountFixture } = await import(${JSON.stringify(path.join(ROOT, 'test/helpers/plan-count-fixture.ts'))});
|
|
const { nativePlanCallFingerprint } = await import(${JSON.stringify(path.join(ROOT, 'test/helpers/claude-pty-runner.ts'))});
|
|
const fp = (id, question, preReview = true) => nativePlanCallFingerprint({
|
|
sessionId: 'owned-ui-fixture', toolUseId: id, answered: true, failed: false,
|
|
unansweredQuestionIndices: [],
|
|
questions: [{ header: 'Review', question, multiSelect: false, options: [
|
|
{ label: 'Review all 7 dimensions (recommended)', description: 'Review the supplied UI plan.' },
|
|
{ label: 'Review a different branch', description: 'That branch may have no UI scope; design review is not applicable there.' },
|
|
] }],
|
|
}, 1000, preReview);
|
|
const target = fp('target', 'D1 — What should I design-review?\\nProject/branch/task: gstack repo, no plan file drafted yet.\\nELI10: A design review needs a target. Pick what I should rate 0-10 across the 7 design dimensions.');
|
|
target.nativeCall.questions[0].header = 'Review target';
|
|
if (mode === 'target-menu-design-system') {
|
|
target.nativeCall.questions[0].question = 'What should I design-review? ELI10: Choose a plan, design system, or the current branch diff.';
|
|
target.promptSnippet = target.nativeCall.questions[0].question;
|
|
}
|
|
target.nativeCall.questions[0].options = [
|
|
{ label: 'B) A plan or design doc (recommended)', description: 'You need to paste the plan or point me to a path.' },
|
|
{ label: 'A) The current branch diff', description: 'The working tree is clean, so there may be no UI scope to review.' },
|
|
];
|
|
const paraphrase = fp('focus', 'D1 — Review all 7 design dimensions, or focus on specific areas?\\nELI10: I rated this plan 4/10.');
|
|
// Public titles/options projected from the retained Sep 11 native PostToolUse
|
|
// events: focus toolu_01XJZk6qbs3Fj3VRbm6sCNv2, setup toolu_01WkR4juMdcMxCTJe8FfRMVY,
|
|
// finding toolu_012EsyqshgBk2Ap7CfuzwhwQ. No transcript paths or private content.
|
|
const nativeFocus = fp('native-focus', mode === 'native-passes'
|
|
? 'D1 — Review all 7 design passes, or focus?'
|
|
: 'D1 — Review all 7 design dimensions, or focus?');
|
|
nativeFocus.nativeCall.questions[0].options = [
|
|
{ label: 'All 7 dimensions (recommended)', description: 'Full review: hierarchy, states, journey, specificity, AI slop, responsive, accessibility. Completeness 10/10.' },
|
|
{ label: 'States + hierarchy + a11y', description: 'Focus on the three weakest areas only. Completeness 7/10.' },
|
|
{ label: 'Visual + AI slop only', description: 'Focus on look and originality, skip state and interaction passes. Completeness 4/10.' },
|
|
];
|
|
const nativeSetup = fp('native-setup', 'D2 — Run outside design voices before the 7 passes?', false);
|
|
nativeSetup.nativeCall.questions[0].options = [
|
|
{ label: 'Yes, run outside voices (recommended)', description: 'Codex design critique + independent Claude subagent, run in parallel, synthesized into a litmus scorecard.' },
|
|
{ label: 'No, skip to the passes', description: 'Go straight to Pass 1 with my review only.' },
|
|
];
|
|
const nativeFinding = fp('native-finding', "D3 — Issue 1 [HARD REJECTION risk]: Which region is the dashboard's primary anchor, and how is the page composed?", false);
|
|
nativeFinding.nativeCall.questions[0].options = [
|
|
{ label: '1A Notifications-primary (recommended)', description: 'Status line + unread count anchors; notifications ~7/12, activity ~5/12; mobile order status, notifications, activity. Completeness 10/10.' },
|
|
{ label: '1B Activity-primary (Codex)', description: 'Activity dominant ~7/12, notifications right rail; primary action beside title. Completeness 10/10.' },
|
|
{ label: '1C Three peer regions, decide later', description: 'Keep the plan as written; hard rejection #1 stays open. Completeness 3/10.' },
|
|
];
|
|
// Run 37091743395 slice 8 (987a937b): the five answered native calls, byte-exact,
|
|
// whose comma-less Step 0D title the focus predicate once rejected.
|
|
const captured = JSON.parse(fs.readFileSync(${JSON.stringify(path.join(ROOT, 'test/fixtures/design-ui-focus-capture.json'))}, 'utf8'))
|
|
.calls.map(({ preReview, observedAtMs, call }) => nativePlanCallFingerprint(call, observedAtMs, preReview));
|
|
// Run 39199692993 slice 4 (521fd755): the Step 0D title omitted 'design'; its body carried the rating.
|
|
const capturedRated = JSON.parse(fs.readFileSync(${JSON.stringify(path.join(ROOT, 'test/fixtures/design-ui-focus-capture-39199692993.json'))}, 'utf8'))
|
|
.calls.map(({ preReview, observedAtMs, call }) => nativePlanCallFingerprint(call, observedAtMs, preReview));
|
|
// Run 39298269902 slice 4 (072d4b9), attempts 1 and 2: Opus 5.5 titled Step 0D only
|
|
// "Review focus", with the rating in the body and an all-7 option (a2 on one line).
|
|
const captured39298269902 = [1, 2].map(attempt => JSON.parse(fs.readFileSync(${JSON.stringify(ROOT)} + '/test/fixtures/design-ui-focus-capture-39298269902-a' + attempt + '.json', 'utf8'))
|
|
.calls.map(({ preReview, observedAtMs, call }) => nativePlanCallFingerprint(call, observedAtMs, preReview)));
|
|
// Run 39363705692 slice 4 (a592321): Opus 5.5 rated the plan "4/10 for design completeness"
|
|
// and offered "All 7 areas (recommended)" against "Only specific areas".
|
|
const captured39363705692 = JSON.parse(fs.readFileSync(${JSON.stringify(path.join(ROOT, 'test/fixtures/design-ui-focus-capture-39363705692.json'))}, 'utf8'))
|
|
.calls.map(({ preReview, observedAtMs, call }) => nativePlanCallFingerprint(call, observedAtMs, preReview));
|
|
// Run 39852003387 slice 4 (357027f): Opus 5.5's ELI10 rated the plan "4/10 on design" (no
|
|
// "completeness") and offered "All 7 passes (recommended)" against "States + layout only".
|
|
const captured39852003387 = JSON.parse(fs.readFileSync(${JSON.stringify(path.join(ROOT, 'test/fixtures/design-ui-focus-capture-39852003387.json'))}, 'utf8'))
|
|
.calls.map(({ preReview, observedAtMs, call }) => nativePlanCallFingerprint(call, observedAtMs, preReview));
|
|
const unnumberedFinding = fp('unnumbered-finding', "Which region is the dashboard's primary anchor, and how is the page composed?", false);
|
|
unnumberedFinding.nativeCall.questions[0].options = nativeFinding.nativeCall.questions[0].options;
|
|
const finding = fp('finding', 'Which loading feedback should Save show?', false);
|
|
finding.nativeCall.questions[0].options = [
|
|
{ label: 'Saving label and spinner (recommended)', description: 'Show in-flight feedback in the Save button.' },
|
|
{ label: 'Keep the current button', description: 'Keep the button label and current loading feedback.' },
|
|
];
|
|
let calls = 0;
|
|
fs.writeFileSync(${JSON.stringify(facts)}, JSON.stringify({ calls }));
|
|
mock.module(${JSON.stringify(path.join(ROOT, 'test/helpers/e2e-gate.ts'))}, () => ({
|
|
describeE2ETier: tier => { expect(tier).toBe('gate'); return describe; },
|
|
}));
|
|
mock.module(${JSON.stringify(path.join(ROOT, 'test/helpers/claude-pty-runner.ts'))}, () => ({
|
|
runPlanSkillCounting: async opts => {
|
|
calls++;
|
|
expect(opts.cwd).toBeUndefined();
|
|
const ownedFixture = createPlanCountFixture(opts.followUpPrompt, {nativeReviewOnly: true, files: opts.fixtureFiles});
|
|
const cwd = ownedFixture.cwd;
|
|
try {
|
|
expect(path.dirname(cwd)).toBe(${JSON.stringify(directory)});
|
|
expect(cwd).not.toBe(${JSON.stringify(ROOT)});
|
|
expect(opts.skillName).toBe('plan-design-review');
|
|
expect(opts).not.toHaveProperty('readDesignArtifacts');
|
|
expect(opts.slashCommand).toBe('/plan-design-review');
|
|
expect(opts.followUpPrompt).toContain(fs.readFileSync(${JSON.stringify(path.join(ROOT, 'test/fixtures/plans/ui-heavy-feature.md'))}, 'utf8'));
|
|
expect(opts.followUpPrompt).toContain(DESIGN_BOARD_ACTOR_PROTOCOL);
|
|
expect(opts.reviewCountCeiling).toBe(1);
|
|
expect(opts.observeSetupQuestions).toBe(true);
|
|
expect(opts.bindDesignBoardState).toBe(true);
|
|
expect(opts).not.toHaveProperty('questionPick');
|
|
expect(typeof opts.pickAUQ).toBe('function');
|
|
expect(opts).not.toHaveProperty('model');
|
|
expect(opts.timeoutMs).toBeGreaterThan(0);
|
|
expect(opts.timeoutMs).toBeLessThanOrEqual(600_000);
|
|
expect(fs.readFileSync(path.join(cwd, 'review-input.md'), 'utf8')).toBe(fs.readFileSync(fixture, 'utf8'));
|
|
const instructions = fs.readFileSync(path.join(cwd, 'CLAUDE.md'), 'utf8');
|
|
expect(instructions).toContain(DESIGN_BOARD_ACTOR_PROTOCOL);
|
|
expect(instructions).toContain('Decline the optional Design Outside Voices step');
|
|
expect(instructions).toContain('all ordinary Design review decisions still apply');
|
|
expect(execFileSync('git', ['show', 'HEAD:CLAUDE.md'], { cwd: cwd, encoding: 'utf8', timeout: 5000 })).toBe(instructions);
|
|
expect(execFileSync('git', ['diff', 'origin/main...HEAD'], { cwd: cwd, encoding: 'utf8', timeout: 5000 })).toBe('');
|
|
expect(execFileSync('git', ['show', 'HEAD:review-input.md'], { cwd: cwd, encoding: 'utf8', timeout: 5000 })).toBe(fs.readFileSync(fixture, 'utf8'));
|
|
expect(fs.readFileSync(path.join(cwd, 'CLAUDE.md'), 'utf8')).toContain('Read it before\\nchoosing review scope.');
|
|
expect(opts.isLastStep0AUQ(target)).toBe(false);
|
|
expect(opts.isLastStep0AUQ(fp('focus', focus))).toBe(true);
|
|
expect(opts.isLastStep0AUQ(paraphrase)).toBe(true);
|
|
// Census 36633323521 gate-census-7: the same Step 0D menu titled with "specific ones".
|
|
expect(opts.isLastStep0AUQ(fp('focus-ones', 'D1 — Review all 7 design dimensions, or focus on specific ones?'))).toBe(true);
|
|
if (mode.startsWith('native-')) expect(opts.isLastStep0AUQ(nativeFocus)).toBe(true);
|
|
expect(opts.isLastStep0AUQ(captured[1])).toBe(true);
|
|
expect(opts.isLastStep0AUQ(capturedRated[1])).toBe(true);
|
|
// Runs 39181697029 and 39195396438: the 0D sentence ended "specific areas instead?" (no "of all 7").
|
|
const instead = "I've rated this plan 5/10 on design completeness. The biggest gaps are panel hierarchy/responsive layout, row anatomy plus state copy, and the accessibility contract. I'll generate visual mockups next, then review all 7 dimensions. Want me to focus on specific areas instead?";
|
|
expect(opts.isLastStep0AUQ(fp('instead', 'D2 — ' + instead + '\\nProject/branch/task: main.'))).toBe(true);
|
|
expect(opts.isLastStep0AUQ(fp('instead-split', 'D2 — ' + instead.replace(' The biggest', '\\nThe biggest')))).toBe(true);
|
|
for (const absent of ["D2 — I've rated this plan 5/10 on design completeness.\\nI'll generate visual mockups next.",
|
|
"D2 — I've rated this plan 5/10 on design completeness.\\nWant me to focus on the current branch diff instead?",
|
|
"D2 — Want me to focus on specific areas instead?\\nI'll review the plan next."]) {
|
|
expect(opts.isLastStep0AUQ(fp('instead-absent', absent))).toBe(false);
|
|
}
|
|
for (const unrated of ['D2 — Review focus: all 7 dimensions or specific areas?\\nELI10: I will review every dimension unless you narrow the focus.',
|
|
'D3 — Want outside design voices before the detailed review?\\nELI10: I rated this plan 5/10 on design completeness.',
|
|
'D1 — Which file should I review against the design system?\\nELI10: I rated this plan 5/10 on design completeness.',
|
|
'D2 — Review focus: all 7 dimensions or specific areas?\\nELI10: I rated this plan 5/7 on design completeness.']) {
|
|
expect(opts.isLastStep0AUQ(fp('unrated', unrated))).toBe(false);
|
|
}
|
|
// Run on b2f729b slice 5: the rating sentence led the title and 'design' named only the completeness.
|
|
expect(opts.isLastStep0AUQ(fp('focus-rated', "D2 — I've rated this plan 5/10 on design completeness. Review all 7 dimensions, or focus on specific areas?\\nProject/branch/task: main."))).toBe(true);
|
|
// Run 37166586458 slice 5 (dfaf154): the Step 0D title spelled the count as a word.
|
|
expect(opts.isLastStep0AUQ(fp('focus-word', "D2 — Review all seven design dimensions, or focus on specific areas?\\nProject/branch/task: main branch, design review of the User Dashboard plan before implementation."))).toBe(true);
|
|
for (const unrelated of ['Review all 4 design passes, or focus?',
|
|
'Review all 7 engineering passes, or focus?', 'Review all 7 passes, or focus?',
|
|
'Review all 4 design passes or focus?', 'Review all 7 engineering passes or focus?', 'Review all 7 passes or focus?',
|
|
'Which plan should receive all 7 design passes?', 'Review all seven engineering passes, or focus?',
|
|
'Review all six design passes, or focus?', 'Review all seven passes, or focus?',
|
|
'Review all 7 dimensions, or focus on specific areas?', "I've rated this plan 5/10 on design completeness. Review all 7 dimensions?"]) {
|
|
expect(opts.isLastStep0AUQ(fp('unrelated', unrelated))).toBe(false);
|
|
}
|
|
expect(opts.isReviewAUQ(target)).toBe(false);
|
|
expect(opts.isReviewAUQ(nativeFocus)).toBe(false);
|
|
expect(opts.isReviewAUQ(nativeSetup)).toBe(false);
|
|
expect(opts.isReviewAUQ(nativeFinding)).toBe(true);
|
|
expect(opts.isReviewAUQ(unnumberedFinding)).toBe(true);
|
|
expect(opts.isReviewAUQ(finding)).toBe(true);
|
|
// Titles and option-label prefixes projected from Sep 29 local captures of the
|
|
// real skill. The first four went unrecognized, so the run timed out after
|
|
// four answered findings; issue-numbered option sets identify each finding.
|
|
const labeled = (id, question, labels) => {
|
|
const call = fp(id, question, false);
|
|
call.nativeCall.questions[0].options = labels.map(label => ({ label, description: '' }));
|
|
return call;
|
|
};
|
|
for (const [question, labels] of [
|
|
['D7 — Issue 1: Codify the page hierarchy from approved Variant A in the plan?', ['1A) Full hierarchy with caps and overflo', '1B) Layout and read order only', '1C) Leave hierarchy out']],
|
|
['D8 — Issue 2: How does the dashboard fit the existing app shell and navigation?', ['2A) Reuse existing shell, Dashboard nav ', '2B) Standalone minimal header, no nav it', '2C) Leave the shell unspecified']],
|
|
['D9 — Issue 3: Add a user-visible interaction state table for every dashboard feature?', ['3A) Full state table with copy (recommen', '3B) Structure only, copy TBD', '3C) Leave states as listed']],
|
|
['D10 — Issue 4: Confirmation pattern and content for "Mark all as read"?', ['4A) Keep modal, specify copy and labels ', '4B) No modal; immediate action plus undo', '4C) Keep modal, leave copy to implemente']],
|
|
['Issue 1 — What does the user see first on a phone?', ['1A: Notifications first on sm, feed firs', '1B: Feed first everywhere', '1C: Bell badge + drawer on sm']],
|
|
]) expect(opts.isReviewAUQ(labeled('captured-issue', question, labels))).toBe(true);
|
|
// Runs 39157502890 and 39161502626: the template's Pass 1 hierarchy question, without issue-numbered labels.
|
|
for (const question of ['D5 — Issue 1: What does the user see first, second, third on /dashboard?', 'D5 — Issue 1: What does the user see first on the dashboard?',
|
|
'D5 — Which information hierarchy should the dashboard use?']) {
|
|
expect(opts.isReviewAUQ(labeled('hierarchy', question, ['Notifications panel first (recommended)', 'Activity feed first']))).toBe(true);
|
|
}
|
|
for (const question of ['D5 — Search learnings from your other projects too?', 'D5 — Ready to implement, or run /plan-eng-review next?',
|
|
'D5 — Run /design-shotgun to explore visual variants?', 'D5 — Save the plan and review report?']) {
|
|
expect(opts.isReviewAUQ(labeled('hierarchy-absent', question, ['Notifications panel first (recommended)', 'Activity feed first']))).toBe(false);
|
|
}
|
|
for (const [question, labels] of [
|
|
['D11 — Update the approved mockups with these decisions?', ['A) Regenerate mockups (recommended)', 'B) Keep the current mockups']],
|
|
['D12 — Which follow-up should I record?', ['1A) Record a TODO', '2B) Skip it']],
|
|
['D13 — Which follow-up should I record?', ['1A) Record a TODO', '1A) Record it again']],
|
|
['D14 — Record this follow-up?', ['1A) Record a TODO']],
|
|
['D15 — Want outside design voices before the detailed review?', ['1A) Yes, run outside voices', '1B) No, proceed without']],
|
|
]) expect(opts.isReviewAUQ(labeled('not-a-finding', question, labels))).toBe(false);
|
|
const [focusA1, focusA2] = [captured39298269902[0][1], captured39298269902[1][0]];
|
|
for (const focusCall of [focusA1, focusA2]) {
|
|
expect(opts.isLastStep0AUQ(focusCall)).toBe(true);
|
|
expect(opts.isReviewAUQ(focusCall)).toBe(false);
|
|
}
|
|
expect(opts.isReviewAUQ(captured39298269902[0][4])).toBe(true);
|
|
expect(opts.isLastStep0AUQ(captured39363705692[0])).toBe(true);
|
|
expect(opts.isReviewAUQ(captured39363705692[0])).toBe(false);
|
|
expect(opts.isReviewAUQ(captured39363705692[3])).toBe(true);
|
|
expect(opts.isLastStep0AUQ(captured39852003387[0])).toBe(true);
|
|
expect(opts.isReviewAUQ(captured39852003387[0])).toBe(false);
|
|
expect(captured39852003387.slice(1, 3).some(call => opts.isLastStep0AUQ(call) || opts.isReviewAUQ(call))).toBe(false);
|
|
expect(opts.isReviewAUQ(captured39852003387[3])).toBe(true);
|
|
const areas = ['All 7 areas (recommended)', 'Only specific areas'];
|
|
for (const body of ['ELI10: I rated this plan 4/10 for design completeness.', 'ELI10: This plan is a 3/10 on design completeness.',
|
|
'ELI10: Design completeness: 6.5/10. The gaps are layout and states.', 'ELI10: I rated this plan 4/10 on design.']) {
|
|
expect(opts.isLastStep0AUQ(labeled('worded-rating', 'D1 — Review focus\\nProject/branch/task: main.\\n' + body, areas))).toBe(true);
|
|
}
|
|
for (const [label, question, labels] of [
|
|
['rating-split-across-sentences', 'D1 — Review focus\\nELI10: I rated this plan 4/10. Design completeness matters most here.', areas],
|
|
['rating-out-of-seven', 'D1 — Review focus\\nELI10: I rated this plan 4/7 for design completeness.', areas],
|
|
['rating-not-design', 'D1 — Review focus\\nELI10: I rated this plan 4/10 for test completeness.', areas],
|
|
['design-split-from-rating', 'D1 — Review focus\\nELI10: I rated this plan 4/10. The design needs mockups first.', areas],
|
|
['rating-unscored-design', 'D1 — Review focus\\nELI10: I rated this plan on design. It needs mockups first.', areas],
|
|
['areas-without-focus', 'D1 — Which plan should I review?\\nELI10: I rated this plan 4/10 for design completeness.', areas],
|
|
['areas-not-leading', 'D1 — Review focus\\nELI10: I rated this plan 4/10 for design completeness.', ['Review all 7 areas (recommended)', 'Only specific areas']],
|
|
['issue-labeled-areas', 'D5 — Issue 1: Which panel leads the review focus?\\nELI10: I rated this plan 4/10 for design completeness.', ['1A All 7 areas first', '1B Activity first']],
|
|
['bare-all-seven', 'D1 — Review focus\\nELI10: I rated this plan 4/10 for design completeness.', ['All 7', 'Only specific areas']],
|
|
]) expect(opts.isLastStep0AUQ(labeled(label, question, labels)), label).toBe(false);
|
|
const ratedBody = 'Project/branch/task: main. ELI10: I rated this plan 4/10 on design completeness.';
|
|
const ratedFinding = labeled('rated-finding', 'D5 — Issue 1: Which panel leads the review focus? ' + ratedBody, ['1A All 7 dimensions first', '1B Activity first']);
|
|
expect(opts.isLastStep0AUQ(ratedFinding)).toBe(false);
|
|
expect(opts.isReviewAUQ(ratedFinding)).toBe(true);
|
|
for (const [label, labels] of [['single-option-focus', ['All 7 dimensions (recommended)']], ['no-all-seven-option', ['Every dimension (recommended)', 'States + hierarchy only']],
|
|
['all-seven-not-leading', ['Review all 7 dimensions (recommended)', 'States + hierarchy only']]]) {
|
|
expect(opts.isLastStep0AUQ(labeled(label, 'D1 — Review focus. ' + ratedBody, labels))).toBe(false);
|
|
}
|
|
expect(opts.isLastStep0AUQ(labeled('unrated-focus', 'D1 — Review focus. Project/branch/task: main. ELI10: I rated this plan 4/7 on design completeness.',
|
|
['All 7 passes (recommended)', 'States + hierarchy only']))).toBe(false);
|
|
const chosenFocus = mode.startsWith('native-') ? nativeFocus : mode === 'paraphrase' ? paraphrase : fp('focus', focus);
|
|
const pendingCall = {...chosenFocus.nativeCall, answered: false, unansweredQuestionIndices: [0]};
|
|
const pending = nativePlanCallFingerprint(pendingCall, 1000, true);
|
|
pending.nativeQuestionIndex = 0;
|
|
const context = Object.freeze({cwd, deadlineAt: Date.now() + opts.timeoutMs - 5000});
|
|
expect(opts.pickAUQ(pending, pending, context)).toBe(1);
|
|
expect(pickerScope).toEqual(context);
|
|
expect(pickerQuestion).toEqual(pendingCall.questions[0]);
|
|
expect(opts.pickAUQ(chosenFocus, chosenFocus, context)).toBeNull();
|
|
// Replay the retained public menu; this synthetic answer is not a rewrite
|
|
// of its historical Yes. No optional outside-review answer counts as a finding.
|
|
const voicesCall = { sessionId: 'owned-ui-fixture', toolUseId: 'outside-voices-replay',
|
|
answered: false, failed: false, unansweredQuestionIndices: [0],
|
|
questions: [outsideVoices.question] };
|
|
const pendingVoices = nativePlanCallFingerprint(voicesCall, 1000, false);
|
|
pendingVoices.nativeQuestionIndex = 0;
|
|
expect(opts.pickAUQ(pendingVoices, pendingVoices, context)).toBe(2);
|
|
expect(pickerQuestion).toEqual(outsideVoices.question);
|
|
expect(opts.isReviewAUQ(pendingVoices)).toBe(false);
|
|
const declinedVoices = nativePlanCallFingerprint({ ...voicesCall, answered: true,
|
|
unansweredQuestionIndices: [], answers: { [outsideVoices.question.question]: 'No, proceed without' } }, 1000, false);
|
|
expect(opts.isReviewAUQ(declinedVoices)).toBe(false);
|
|
expect(opts.pickAUQ(declinedVoices, declinedVoices, context)).toBeNull();
|
|
const explainedQuestion = outsideVoices.sameLineExplanation77.question;
|
|
const explainedCall = { ...voicesCall, toolUseId: 'outside-voices-explanation-replay', questions: [explainedQuestion] };
|
|
const explainedPending = nativePlanCallFingerprint(explainedCall, 1000, false);
|
|
explainedPending.nativeQuestionIndex = 0;
|
|
expect(opts.pickAUQ(explainedPending, explainedPending, context)).toBe(1);
|
|
expect(pickerQuestion).toEqual(explainedQuestion);
|
|
expect(opts.isReviewAUQ(explainedPending)).toBe(false);
|
|
const explainedDeclined = nativePlanCallFingerprint({ ...explainedCall, answered: true,
|
|
unansweredQuestionIndices: [], answers: { [explainedQuestion.question]: 'No, proceed without (recommended)' } }, 1000, false);
|
|
expect(opts.isReviewAUQ(explainedDeclined)).toBe(false);
|
|
expect(opts.pickAUQ(explainedDeclined, explainedDeclined, context)).toBeNull();
|
|
const unboundBoard = {...pending, nativeCall: undefined, options: [{index: 1, label: 'Submitted'}]};
|
|
expect(() => opts.pickAUQ(unboundBoard, unboundBoard, context)).toThrow('requires an owned native question');
|
|
fs.writeFileSync(${JSON.stringify(facts)}, JSON.stringify({ calls, cwd: cwd, seeded: true }));
|
|
if (mode === 'throw') throw new Error('controlled UI observation failure');
|
|
const observed = mode === 'captured' ? captured
|
|
: mode === 'captured-rated' ? capturedRated
|
|
: mode === 'captured-rated-no-focus' ? capturedRated.filter((_, index) => index !== 1)
|
|
: mode === 'captured-no-focus' ? captured.filter((_, index) => index !== 1)
|
|
: mode === 'captured-39298269902' ? captured39298269902[0]
|
|
: mode === 'captured-39298269902-no-focus' ? captured39298269902[0].filter((_, index) => index !== 1)
|
|
: mode === 'captured-39298269902-focus-only' ? captured39298269902[1]
|
|
: mode === 'captured-39363705692' ? captured39363705692
|
|
: mode === 'captured-39363705692-no-focus' ? captured39363705692.filter((_, index) => index !== 0)
|
|
: mode === 'captured-39852003387' ? captured39852003387
|
|
: mode === 'captured-39852003387-no-focus' ? captured39852003387.filter((_, index) => index !== 0)
|
|
: mode === 'captured-focus-after-finding' ? [...captured.filter((_, index) => index !== 1), captured[1]]
|
|
: mode.startsWith('target-menu') ? [target, fp('other', 'Which artifact should I inspect?', false)]
|
|
: mode === 'early-exit' ? [] : mode === 'focus-only' ? [chosenFocus]
|
|
: mode === 'native-setup-only' ? [nativeFocus, nativeSetup]
|
|
: mode === 'native-voices-only' ? [nativeFocus, declinedVoices]
|
|
: mode.startsWith('native-') ? [nativeFocus, nativeSetup, mode === 'native-unnumbered' ? unnumberedFinding : nativeFinding]
|
|
: [chosenFocus, finding];
|
|
if (mode === 'unanswered-finding') observed.at(-1).nativeCall.answered = false;
|
|
if (mode === 'failed-finding') observed.at(-1).nativeCall.failed = true;
|
|
if (mode === 'no-native-finding') delete observed.at(-1).nativeCall;
|
|
return {
|
|
outcome: mode === 'early-exit' ? 'plan_ready' : mode === 'timeout' ? 'timeout' : mode === 'exited' ? 'exited' : 'ceiling_reached',
|
|
fingerprints: observed, step0Count: 1, reviewCount: 1, elapsedMs: 1000,
|
|
evidence: mode === 'early-exit' ? "This plan has no UI scope. A design review isn't applicable."
|
|
: 'Unselected alternative: The other branch may have no UI scope.',
|
|
};
|
|
} finally { ownedFixture.cleanup(); }
|
|
},
|
|
}));
|
|
if (mode === 'missing-fixture') {
|
|
const read = fs.readFileSync;
|
|
spyOn(fs, 'readFileSync').mockImplementation((file, ...args) => {
|
|
if (String(file) === fixture) throw Object.assign(new Error('UI fixture is missing'), { code: 'ENOENT' });
|
|
return read(file, ...args);
|
|
});
|
|
}
|
|
await import(${JSON.stringify(path.join(ROOT, 'test/skill-e2e-plan-design-with-ui.test.ts'))});
|
|
`);
|
|
try {
|
|
const child = Bun.spawnSync([process.execPath, 'test', script], {
|
|
cwd: ROOT, timeout: 10_000,
|
|
env: { PATH: process.env.PATH ?? '', HOME: directory, TMPDIR: directory, TEMP: directory, TMP: directory,
|
|
GIT_CONFIG_NOSYSTEM: '1', ...(process.env.SystemRoot ? { SystemRoot: process.env.SystemRoot } : {}) },
|
|
});
|
|
const output = child.stdout.toString() + child.stderr.toString();
|
|
expect(child.signalCode ?? null, output).toBeNull();
|
|
const observed = JSON.parse(fs.readFileSync(facts, 'utf8'));
|
|
expect(observed.calls, output).toBe(mode === 'missing-fixture' ? 0 : 1);
|
|
if (mode !== 'missing-fixture') expect(observed.seeded, output).toBe(true);
|
|
expect(fs.readdirSync(directory).filter(name => name.startsWith('gstack-plan-count-') || name.startsWith('gstack-native-review-state-'))).toEqual([]);
|
|
return { code: child.exitCode, output };
|
|
} finally { fs.rmSync(directory, { recursive: true, force: true }); }
|
|
}
|
|
|
|
test.each(['source', 'paraphrase', 'native-sequence', 'native-unnumbered', 'native-passes', 'captured', 'captured-rated', 'captured-39298269902', 'captured-39363705692', 'captured-39852003387'])('UI gate seeds the exact target and accepts Design progress with an unselected no-UI alternative (%s)', mode => {
|
|
const result = exercise(mode);
|
|
expect(result.code, result.output).toBe(0);
|
|
}, 20_000);
|
|
|
|
test.each(['target-menu', 'target-menu-design-system', 'focus-only', 'captured-no-focus', 'captured-focus-after-finding', 'captured-rated-no-focus', 'captured-39298269902-no-focus', 'captured-39298269902-focus-only', 'captured-39363705692-no-focus', 'captured-39852003387-no-focus', 'native-setup-only', 'native-voices-only', 'early-exit', 'timeout', 'exited', 'unanswered-finding', 'failed-finding', 'no-native-finding'])('UI gate rejects %s and removes its fixture', mode => {
|
|
const result = exercise(mode);
|
|
expect(result.code, result.output).toBe(1);
|
|
expect(result.output).toContain('plan-design-review with UI scope FAILED');
|
|
}, 20_000);
|
|
|
|
test.each(['throw', 'missing-fixture'])('UI gate preserves %s failure and removes its fixture', mode => {
|
|
const result = exercise(mode);
|
|
expect(result.code, result.output).toBe(1);
|
|
expect(result.output).toContain(mode === 'throw' ? 'controlled UI observation failure' : 'UI fixture is missing');
|
|
}, 20_000);
|