v1.87.5.0 perf: remove idle waits from tests and CI planning (#2897)

* v1.87.5.0 perf: remove idle waits from tests and CI planning

* fix: settle split PTY redraws before routing input

* docs: record final burst-safe test benchmarks

* fix: keep cold-setup snapshot metadata dependency-free

* fix: avoid early-reader pipe races in artifact URL parsing

* fix: preserve safety matches for multiline command payloads

* fix: recognize concurrent CSO publication removal

* test: preload the UI design-review target before invocation

* docs: record validation blocker fixes

* fix: bind plan observer rejection to the invoked command

* fix: count only native design decisions in the UI gate

* docs: clarify UI-positive eval evidence requirements

* test: recognize native UI decisions without weakening finding counts

* test: decouple native UI evidence from question punctuation

* test: recognize concrete native UI decisions independently of prose format

* fix: retain failed eval logs under the hidden CI cache

* test: await telemetry completion instead of racing disk writes

test/plan-count-fixture.test.ts

@@ -7,6 +7,10 @@ import * as path from 'node:path';
 import { pathToFileURL } from 'node:url';
 import { createPlanCountFixture } from './helpers/plan-count-fixture';
 import { getHermeticDirs } from './helpers/hermetic-env';
+import { nativePlanCallFingerprint } from './helpers/claude-pty-runner';
+import { isDesignCountFirstReview } from './helpers/design-count-review';
+import { isDesignUIScopeReview } from './helpers/design-ui-scope';
+import designUICapture from './fixtures/plan-design-ui-scope.json';
 
 const ROOT = path.resolve(import.meta.dir, '..');
 const PROMPT = '# Seeded settings plan\n\nReview each issue separately.\n' +
@@ -166,6 +170,12 @@ try {
       const cases = [
         { name: 'design', skillName: 'plan-design-review', prompt: PROMPT, mode: 'complete', files: { 'DESIGN.md': '# Approved design\nKeep the existing layout.\n' } },
         { name: 'design-direct', skillName: 'plan-design-review', prompt: PROMPT, mode: 'direct-finding' },
+        { name: 'design-named-target', skillName: 'plan-design-review', prompt: fs.readFileSync(path.join(ROOT, 'test/fixtures/plans/ui-heavy-feature.md'), 'utf8'), mode: 'direct-finding', namedTarget: true },
+        { name: 'design-tool-diagnostic', skillName: 'plan-design-review', prompt: PROMPT, mode: 'direct-finding', toolDiagnostic: true },
+        { name: 'design-gate-positive', skillName: 'plan-design-review', prompt: PROMPT, mode: 'direct-finding', gateFilter: true },
+        { name: 'design-ui-captured', skillName: 'plan-design-review', prompt: fs.readFileSync(path.join(ROOT, 'test/fixtures/plans/ui-heavy-feature.md'), 'utf8'), mode: 'direct-finding', gateFilter: true, namedTarget: true, capturedQuestions: designUICapture.calls[3]!.questions },
+        { name: 'design-ui-captured-separators', skillName: 'plan-design-review', prompt: fs.readFileSync(path.join(ROOT, 'test/fixtures/plans/ui-heavy-feature.md'), 'utf8'), mode: 'direct-finding', gateFilter: true, namedTarget: true, capturedQuestions: designUICapture.additionalCaptures[0]!.calls[2]!.questions },
+        { name: 'design-ui-captured-decision', skillName: 'plan-design-review', prompt: fs.readFileSync(path.join(ROOT, 'test/fixtures/plans/ui-heavy-feature.md'), 'utf8'), mode: 'direct-finding', gateFilter: true, namedTarget: true, capturedQuestions: [designUICapture.additionalQuestionCaptures[0]!.question] },
         { name: 'design-batched', skillName: 'plan-design-review', prompt: PROMPT, mode: 'batched-finding' },
         { name: 'failed-native', skillName: 'plan-design-review', prompt: PROMPT, mode: 'failed-call' },
         { name: 'native-permission-policy', skillName: 'plan-eng-review', prompt: PROMPT, mode: 'native-permission-policy', report: path.join(dir, 'native-policy-report.md') },
@@ -414,6 +424,13 @@ process.stdin.on('data', (data) => {
   }
   firstInput = false;
   if (process.env.FIXTURE_MODE === 'exit') process.exit(7);
+  if (process.env.FIXTURE_TOOL_DIAGNOSTIC === 'true') render('Unknown command: --help\n');
+  if (process.env.FIXTURE_GATE_FILTER === 'true') {
+    ask([questionMetadata('Focus', 'D2 — Review all 7 design dimensions, or focus on specific areas?', ['All 7 dimensions', 'Choose areas'])]);
+    answer();
+    ask([questionMetadata('Outside voices', 'D3 — Want outside design voices before the detailed review?\nA fresh reviewer checks completeness. <gstack-qid:outside-voices-design>', ['Yes, run outside voices (recommended)', 'No, proceed without'])]);
+    answer();
+  }
   if (process.env.FIXTURE_MODE === 'damaged-menu') {
     render('☐Stripe event types\nWhich event should the handler accept?\n❯1.Specify one canonical event\n2.Accept all events\n' +
       '·'.repeat(4200) + '\nMinimum required test cases:\n1.Happy path\n2.Email failure\n3.DB timeout\n4.Unknown event\n5.Unknown user\n❯1\n');
@@ -443,9 +460,16 @@ process.stdin.on('data', (data) => {
       ask([questionMetadata('Missing answer', 'Should the save retry be idempotent?', ['Yes', 'No'])]);
       native('user', [{ type: 'tool_result', tool_use_id: 'question-' + callId, is_error: true, content: 'Question rejected' }]);
     }
-    const questions = [questionMetadata('Button style', 'D1 — How should the four header buttons differ? <gstack-qid:plan-design-review-button-hierarchy>', ['Filled primary', 'Ghost buttons'])];
+    const detail = process.env.FIXTURE_GATE_FILTER === 'true' ? ' Specify the primary button treatment.'.repeat(30) : '';
+    const questions = process.env.FIXTURE_CAPTURED_QUESTIONS ? JSON.parse(process.env.FIXTURE_CAPTURED_QUESTIONS)
+      : [questionMetadata('Button style', 'D1 — How should the four header buttons differ?' + detail + ' <gstack-qid:plan-design-review-button-hierarchy>', ['Filled primary', 'Ghost buttons'])];
     if (process.env.FIXTURE_MODE === 'batched-finding') questions.push(questionMetadata('Loading', 'D2 — Define the loading state <gstack-qid:plan-design-review-loading>', ['Add spinner', 'Keep blank']));
     ask(questions);
+    if (process.env.FIXTURE_CAPTURED_QUESTIONS) {
+      const q = questions[0];
+      render('\r☐' + q.header + '\r' + q.question.split('\n')[0] + '\r' + q.options.map((o, i) => (i === 0 ? '❯' : '') + (i + 1) + '.' + o.label).join('\r') + '\r');
+      return;
+    }
     render('\r☐Buttonstyle\r│D1—Howshouldthe4headerbuttonsbedifferentiated?<gstack-qid:plan-design-review-butn-hierarchy>\r❯1.Filledprimary\r2.Ghostbuttons\r');
     return;
   }
@@ -485,10 +509,14 @@ process.stdin.resume();
       const runnerUrl = pathToFileURL(path.join(ROOT, 'test/helpers/claude-pty-runner.ts')).href;
       const hermeticUrl = pathToFileURL(path.join(ROOT, 'test/helpers/hermetic-env.ts')).href;
       const devexUrl = pathToFileURL(path.join(ROOT, 'test/helpers/devex-count-fixture.ts')).href;
+      const designUrl = pathToFileURL(path.join(ROOT, 'test/helpers/design-count-review.ts')).href;
+      const designUIUrl = pathToFileURL(path.join(ROOT, 'test/helpers/design-ui-scope.ts')).href;
       fs.writeFileSync(workerPath, `
 import { runPlanSkillCounting, designFirstReviewAUQ } from ${JSON.stringify(runnerUrl)};
 import { getHermeticDirs } from ${JSON.stringify(hermeticUrl)};
 import { devexReviewModePick } from ${JSON.stringify(devexUrl)};
+import { isDesignCountFirstReview } from ${JSON.stringify(designUrl)};
+import { isDesignUIScopeReview } from ${JSON.stringify(designUIUrl)};
 import * as fs from 'node:fs';
 import * as path from 'node:path';
 const shared = getHermeticDirs().gstackHome;
@@ -500,23 +528,25 @@ const results = await Promise.all(cases.map(async (item) => ({
   name: item.name,
   observation: await runPlanSkillCounting({
     skillName: item.skillName,
-    slashCommand: '/' + item.skillName,
+    slashCommand: '/' + item.skillName + (item.namedTarget ? ' PLAN.md' : ''),
     followUpPrompt: item.prompt,
     fixtureFiles: item.files,
     expectedPlanPath: item.report,
-    isLastStep0AUQ: () => false,
+    isLastStep0AUQ: item.gateFilter ? fp => fp.nativeCall?.questions[0]?.header === 'Focus' : () => false,
     isFirstReviewAUQ: ['direct-finding', 'batched-finding', 'failed-call'].includes(item.mode) ? designFirstReviewAUQ : undefined,
-    isReviewAUQ: item.custom ? fp => fp.promptSnippet.includes('routing-proof-after-240') : undefined,
+    isReviewAUQ: item.capturedQuestions ? isDesignUIScopeReview : item.gateFilter ? isDesignCountFirstReview : item.custom ? fp => fp.promptSnippet.includes('routing-proof-after-240') : undefined,
     pickAUQ: item.mode === 'native-permission-policy' ? () => 2
       : ['late-mode', 'batched-mode'].includes(item.mode) ? devexReviewModePick
       : item.custom ? fp => fp.promptSnippet.includes('routing-proof-after-240') ? 1 : null : undefined,
-    reviewCountCeiling: 8,
+    reviewCountCeiling: item.gateFilter ? 1 : 8,
     timeoutMs: item.mode === 'permission-lifecycle' ? 35000 : 28000,
     firstAUQPick: () => ['late-mode', 'batched-mode'].includes(item.mode) ? 1 : 2,
     env: {
       FIXTURE_RECORD: item.record, FIXTURE_SKILL: item.skillName, FIXTURE_MODE: item.mode,
       FIXTURE_EXPECTED_REPORT: item.report ?? '',
       FIXTURE_CUSTOM: String(item.custom ?? false),
+      FIXTURE_TOOL_DIAGNOSTIC: String(item.toolDiagnostic ?? false), FIXTURE_GATE_FILTER: String(item.gateFilter ?? false),
+      FIXTURE_CAPTURED_QUESTIONS: item.capturedQuestions ? JSON.stringify(item.capturedQuestions) : '',
       FIXTURE_SKIP_INDEX: String(item.skipIndex ?? ''), FIXTURE_CONFIG_BIN: ${JSON.stringify(path.join(ROOT, 'bin/gstack-config'))},
       FIXTURE_SKIP_LABEL: item.skipLabel ?? '',
       GSTACK_HOME: ${JSON.stringify(hostState)}, GSTACK_STATE_ROOT: ${JSON.stringify(hostState)},
@@ -575,7 +605,7 @@ await Bun.write(${JSON.stringify(resultPath)}, JSON.stringify({ results, onboard
           expect(startup.skill).toContain(`name: ${item.skillName}`);
           expect(startup.sections).toBe(fs.readFileSync(path.join(ROOT, item.skillName, 'sections/review-sections.md'), 'utf8'));
           expect(events.filter((event) => event.type === 'input').map((event) => event.data).join(''))
-            .toBe(`/${item.skillName}\r` + (item.mode === 'prerequisite' ? `${item.custom ? '1\r' : '2'}${item.skipIndex}`
+            .toBe(`/${item.skillName}${item.namedTarget ? ' PLAN.md' : ''}\r` + (item.mode === 'prerequisite' ? `${item.custom ? '1\r' : '2'}${item.skipIndex}`
               : item.mode === 'permission-lifecycle' ? '1\r1\r2'
               : item.mode === 'damaged-submit' ? '\x1b[Z2\r\r'
               : item.mode === 'batched-finding' ? '2\r1\r' : item.mode === 'batched-mode' ? '1\r'
@@ -586,7 +616,7 @@ await Bun.write(${JSON.stringify(resultPath)}, JSON.stringify({ results, onboard
           expect(() => process.kill(startup.pid, 0)).toThrow();
           const result = results.find((result) => result.name === item.name);
           expect(result.observation.outcome, `${item.name}: ${JSON.stringify(result.observation)}`).toBe(item.mode === 'exit' ? 'exited'
-            : ['missing-transcript', 'failed-call'].includes(item.mode) ? 'transcript_unavailable' : 'completion_summary');
+            : ['missing-transcript', 'failed-call'].includes(item.mode) ? 'transcript_unavailable' : item.gateFilter ? 'ceiling_reached' : 'completion_summary');
           const artifacts = result.observation.artifactDir;
           expect(result.observation.artifactError).toBeUndefined();
           expect(fs.existsSync(artifacts)).toBe(true); // Survives the temporary fixture's cleanup.
@@ -595,12 +625,21 @@ await Bun.write(${JSON.stringify(resultPath)}, JSON.stringify({ results, onboard
           expect(captured.capture.cwd).toBe(startup.cwd);
           expect(fs.readFileSync(path.join(artifacts, 'terminal.raw.log'), 'utf8')).toContain(item.mode === 'exit' ? 'STARTUP_DIAGNOSTIC' : 'GSTACK REVIEW REPORT');
           expect(fs.readFileSync(path.join(artifacts, 'terminal.visible.log'), 'utf8')).toContain(item.mode === 'exit' ? 'STARTUP_DIAGNOSTIC' : 'GSTACK REVIEW REPORT');
-          if (['direct-finding', 'batched-finding', 'failed-call'].includes(item.mode)) {
+          if (['direct-finding', 'batched-finding', 'failed-call'].includes(item.mode) && !item.gateFilter) {
             expect(result.observation.reviewCount).toBe(1);
             expect(result.observation.step0Count).toBe(0);
             expect(result.observation.fingerprints).toHaveLength(1);
             expect(result.observation.fingerprints[0].nativeCall.questions).toHaveLength(item.mode === 'batched-finding' ? 2 : 1);
           }
+          if (item.gateFilter) {
+            expect(result.observation.reviewCount).toBe(1);
+            expect(result.observation.step0Count).toBe(2);
+            expect(result.observation.fingerprints.map(fp => fp.preReview)).toEqual([true, true, false]);
+            expect(result.observation.fingerprints.at(-1).nativeCall.questions[0].header).toBe(item.capturedQuestions?.[0].header ?? 'Button style');
+            const finding = result.observation.fingerprints.at(-1);
+            expect(finding.promptSnippet.length).toBe(240);
+            expect((item.capturedQuestions ? isDesignUIScopeReview : isDesignCountFirstReview)(nativePlanCallFingerprint(finding.nativeCall, finding.observedAtMs, finding.preReview))).toBe(true);
+          }
           if (item.mode === 'damaged-menu') {
             expect(events.filter(event => event.type === 'input-during-prose')).toEqual([]);
             expect(events.filter(event => event.type === 'damaged-menu-answer').map(event => event.input)).toEqual(['2\r']);