v1.87.5.0 perf: remove idle waits from tests and CI planning (#2897)

* v1.87.5.0 perf: remove idle waits from tests and CI planning

* fix: settle split PTY redraws before routing input

* docs: record final burst-safe test benchmarks

* fix: keep cold-setup snapshot metadata dependency-free

* fix: avoid early-reader pipe races in artifact URL parsing

* fix: preserve safety matches for multiline command payloads

* fix: recognize concurrent CSO publication removal

* test: preload the UI design-review target before invocation

* docs: record validation blocker fixes

* fix: bind plan observer rejection to the invoked command

* fix: count only native design decisions in the UI gate

* docs: clarify UI-positive eval evidence requirements

* test: recognize native UI decisions without weakening finding counts

* test: decouple native UI evidence from question punctuation

* test: recognize concrete native UI decisions independently of prose format

* fix: retain failed eval logs under the hidden CI cache

* test: await telemetry completion instead of racing disk writes

test/helpers/claude-pty-runner.ts

@@ -143,6 +143,7 @@ export interface ClaudePtySession {
    * dialog or boot banner residue. Returns a marker handle.
    */
   mark(): number;
+  waitForOutput(since: number, timeoutMs: number): Promise<void>;
   /** Visible text since the most recent (or specific) mark. */
   visibleSince(marker?: number): string;
   /**
@@ -3637,7 +3638,7 @@ export const designFirstReviewAUQ: Step0BoundaryPredicate = (fp) => {
   // question ID as well, and exclude its scope/focus/onboarding identities.
   const id = /<gstack-qid:\s*plan-design-review-([a-z0-9-]+)/i.exec(fp.promptSnippet)?.[1];
   if (id && /(?:^|[│\s])D\s*\d+\s*[—–-]/i.test(fp.promptSnippet) &&
-      !/(?:^|-)(?:scope|focus|setup|routing|onboarding|posture|mockups?|target)(?:-|$)/i.test(id) &&
+      !/(?:^|-)(?:scope|focus|setup|routing|onboarding|posture|mockups?|target|outside(?:-design)?-voices)(?:-|$)/i.test(id) &&
       !designStep0Boundary(fp)) return true;
   // Explicit pass headings are also review evidence; an initial assessment
   // that merely mentions reviewing seven passes does not match this shape.
@@ -3674,7 +3675,10 @@ export async function launchClaudePty(
 
   let buffer = '';
   let exited = false;
+  let closing = false;
   let exitCodeCaptured: number | null = null;
+  const outputWaiters = new Set<() => void>();
+  const notifyOutput = () => { for (const done of outputWaiters) done(); };
 
   const args: string[] = [];
   // Pin the model so smokes don't inherit the operator's settings.json model
@@ -3757,6 +3761,7 @@ export async function launchClaudePty(
         const text = chunk.toString('utf-8');
         buffer += text;
         if (screen && !screenClosing) screen.write(text);
+        notifyOutput();
       },
     },
     cwd,
@@ -3770,10 +3775,12 @@ export async function launchClaudePty(
       .then(async (code: number | null) => {
         exitCodeCaptured = code;
         exited = true;
+        notifyOutput();
         await disposeScreen();
       })
       .catch(async () => {
         exited = true;
+        notifyOutput();
         await disposeScreen();
       });
   }
@@ -3848,6 +3855,19 @@ export async function launchClaudePty(
     return stripAnsi(buffer.slice(offset));
   }
 
+  async function waitForOutput(since: number, timeoutMs: number): Promise<void> {
+    if (buffer.length > since || exited || closing) return;
+    await new Promise<void>((resolve) => {
+      const done = () => {
+        clearTimeout(timer);
+        outputWaiters.delete(done);
+        resolve();
+      };
+      const timer = setTimeout(done, timeoutMs);
+      outputWaiters.add(done);
+    });
+  }
+
   async function waitForAny(
     patterns: Array<RegExp | string>,
     waitOpts?: { timeoutMs?: number; pollMs?: number; since?: number },
@@ -3890,25 +3910,28 @@ export async function launchClaudePty(
   }
 
   async function close(): Promise<void> {
+    closing = true;
+    notifyOutput();
     clearTimeout(wallTimer);
     clearTimeout(trustWatcherStop);
     clearInterval(trustWatcher);
     for (const timer of trustInputTimers) clearTimeout(timer);
     if (exited) { pendingFiles.forEach(({ recorder }) => recorder.dispose()); pendingExit?.dispose(); pendingQuestion?.dispose(); pendingArtifact?.dispose(); await disposeScreen(); return; }
-    try {
-      proc.kill?.('SIGINT');
-    } catch {
-      /* ignore */
-    }
-    // Wait up to 2s for graceful exit.
-    await Promise.race([exitedPromise, Bun.sleep(2000)]);
-    if (!exited) {
+    for (const [signal, timeout] of [['SIGINT', 2000], ['SIGKILL', 1000]] as const) {
+      if (exited) break;
       try {
-        proc.kill?.('SIGKILL');
+        proc.kill?.(signal);
       } catch {
         /* ignore */
       }
-      await Promise.race([exitedPromise, Bun.sleep(1000)]);
+      let deadline!: ReturnType<typeof setTimeout>;
+      try {
+        await Promise.race([exitedPromise, new Promise<void>((resolve) => {
+          deadline = setTimeout(resolve, timeout);
+        })]);
+      } finally {
+        clearTimeout(deadline);
+      }
     }
     pendingFiles.forEach(({ recorder }) => recorder.dispose());
     pendingExit?.dispose();
@@ -3928,6 +3951,7 @@ export async function launchClaudePty(
       return screen.read();
     },
     mark,
+    waitForOutput,
     visibleSince,
     waitForAny,
     waitFor,
@@ -4256,7 +4280,7 @@ export async function runPlanSkillObservation(opts: {
           ...highWaterFlags(),
         };
       }
-      if (visible.includes('Unknown command:')) {
+      if (isUnknownSlashCommandVisible(visible, `/${opts.skillName}`)) {
         return {
           outcome: 'exited',
           summary: `claude rejected /${opts.skillName} as unknown command (skill not registered in this cwd)`,
@@ -4440,6 +4464,12 @@ export interface PlanSkillCountObservation {
   administrativeCount: number;
 }
 
+export function isUnknownSlashCommandVisible(visible: string, slashCommand: string): boolean {
+  const command = slashCommand.trim().split(/\s+/)[0];
+  return [...visible.matchAll(/Unknown command:\s*(\/[\w-]+)(?=\s|$)/g)]
+    .some(match => match[1] === command);
+}
+
 /**
  * Drive a plan-* skill in plan mode and count distinct native review-phase
  * AskUserQuestions until a terminal signal fires. Each run disables the
@@ -4536,6 +4566,7 @@ export async function runPlanSkillCounting(opts: {
   firstAUQPick?: (fp: AskUserQuestionFingerprint) => number;
   /** Total budget including startup and cleanup. Must exceed the 5s cleanup reserve. Default 1_500_000. */
   timeoutMs?: number;
+  startupReadyMarker?: string;
   /** Extra env merged into the spawned `claude` process. */
   env?: Record<string, string>;
   /** Override the spawned model. Defaults via launchClaudePty's chain. */
@@ -4545,6 +4576,9 @@ export async function runPlanSkillCounting(opts: {
   const startedAt = Date.now();
   const defaultPick = opts.defaultPick ?? 1;
   const timeoutMs = opts.timeoutMs ?? 1_500_000;
+  if (opts.startupReadyMarker !== undefined && !opts.startupReadyMarker.length) {
+    throw new RangeError('Plan counting startup-ready marker must not be empty');
+  }
   // The caller may use this same limit as its Bun timeout. Leave room for
   // close()'s 2s graceful + 1s forced exit waits and artifact/fixture cleanup.
   // A second work window after boot lets Bun retry while this body is alive.
@@ -4642,14 +4676,29 @@ export async function runPlanSkillCounting(opts: {
     return observation;
   }
 
+  let observedOutput = session.mark();
+  let lastObservationAt = -Infinity;
   try {
-    if (await waitForWork(8000)) { // boot grace is part of the total budget
-      session.mark();
+    let startupReady: boolean;
+    if (opts.startupReadyMarker !== undefined) {
+      await session.waitFor(opts.startupReadyMarker, { timeoutMs: Math.min(8000, remainingWork()) });
+      startupReady = remainingWork() > 0;
+    } else {
+      startupReady = await waitForWork(8000);
+    }
+    if (startupReady) {
+      observedOutput = session.mark();
       session.send(`${opts.slashCommand}\r`);
     }
 
     while (remainingWork() > 0) {
-      if (!await waitForWork(2000)) break;
+      await session.waitForOutput(observedOutput, Math.min(2000, remainingWork()));
+      if (remainingWork() <= 0) break;
+      const coalesceMs = session.rawOutput().length > observedOutput
+        ? 250 : 250 - (performance.now() - lastObservationAt);
+      if (coalesceMs > 0 && !await waitForWork(coalesceMs)) break;
+      observedOutput = session.mark();
+      lastObservationAt = performance.now();
       const visible = viewport = await session.currentScreen();
       if (remainingWork() <= 0) break;
       transcript = session.hermeticConfigDir
@@ -4703,7 +4752,7 @@ export async function runPlanSkillCounting(opts: {
         );
       }
 
-      if (visible.includes('Unknown command:')) {
+      if (isUnknownSlashCommandVisible(visible, opts.slashCommand)) {
         return snapshot(
           'exited',
           `claude rejected ${opts.slashCommand} as unknown command (skill not registered in this cwd)`,
@@ -5010,7 +5059,7 @@ export async function runPlanSkillFloorCheck(opts: {
           elapsedMs: Date.now() - startedAt,
         });
       }
-      if (visible.includes('Unknown command:')) {
+      if (isUnknownSlashCommandVisible(visible, opts.slashCommand)) {
         return finish({
           auqObserved: false,
           outcome: 'exited',

test/helpers/design-count-outside.ts

@@ -17,7 +17,7 @@ export function pickDesignCountOutsideVoices(
     if (active.signature !== identity) return null;
     const q = call.questions[index]!;
     if (q.multiSelect || !/^outside(?: design)? voices$/i.test(q.header.trim()) ||
-        !/<gstack-qid:outside-voices-design>/.test(q.question)) return null;
+        !/<gstack-qid:(?:outside-voices-design|plan-design-review-outside-voices)>/.test(q.question)) return null;
     question = q.question;
     labels = q.options.map(option => option.label);
   } else {
@@ -31,11 +31,11 @@ export function pickDesignCountOutsideVoices(
     labels = active.options.map(option => option.label);
     while (labels.length > 2 && /^(?:Type something\.?|Chat about this)$/i.test(labels.at(-1)!.trim())) labels.pop();
   }
-  if (!/\b(?:want|run|include|enable)\b[^?]{0,90}\boutside design voices\b/i.test(question) ||
+  if (!/\b(?:want|run|include|enable)\b[^?]{0,90}\boutside(?: design)? voices\b/i.test(question) ||
       !/\b(?:before|for)\s+(?:the\s+)?(?:detailed\s+)?(?:design\s+)?review\b/i.test(question)) return null;
   labels = labels.map(label => label.trim().replace(/\s*\(recommended\)\s*$/i, ''));
   if (labels.length !== 2) return null;
-  const yes = labels.map(label => /^Yes,?\s+run outside design voices$/i.test(label));
+  const yes = labels.map(label => /^Yes,?\s+run outside(?: design)? voices$/i.test(label));
   const no = labels.map(label => /^No,?\s+proceed without$/i.test(label));
   if (yes.filter(Boolean).length !== 1 || no.filter(Boolean).length !== 1) return null;
   return no.findIndex(Boolean) + 1;

test/helpers/design-ui-scope.ts

@@ -0,0 +1,27 @@
+import type { AskUserQuestionFingerprint } from './claude-pty-runner';
+import { isDesignCountFirstReview } from './design-count-review';
+
+export function isDesignUIScopeReview(fp: AskUserQuestionFingerprint): boolean {
+  const call = fp.nativeCall;
+  if (!call?.answered || call.failed || !Array.isArray(call.unansweredQuestionIndices) ||
+      call.unansweredQuestionIndices.length || !call.questions.length ||
+      fp.signature !== `${call.sessionId}:${call.toolUseId}`) return false;
+  if (call.questions.some(q => q.multiSelect || q.options.length < 2 ||
+      new Set(q.options.map(option => option.label)).size !== q.options.length ||
+      !q.options.some(option => option.label === call.answers?.[q.question]))) return false;
+  if (isDesignCountFirstReview(fp)) return true;
+  const workflow = /\b(?:review(?:s|ers?)?|scope|setup|learnings|routing|mockups?|permissions?|codex|claude|outside)\b/i;
+  const ui = /\b(?:dashboard|hierarchy|panels?|layout|headers?|buttons?|navigation|notifications?|activity|actions?|spacing|colou?rs?|fonts?|typography|loading|errors?|focus|contrast|keyboard|mobile|responsive|toasts?|modals?|empty)\b/i;
+  return call.questions.some(q => {
+    if (/^(?:scope|focus|learnings|routing|next steps?|outside(?: design)? voices)$/i.test(q.header.trim())) return false;
+    const issue = /^(?:D\d+\s*[—–:-]\s*)?Issue ([1-9]\d*)\s*[:—–-]\s*([^\n]+\?)$/i.exec(q.question.split('\n')[0]!.trim());
+    if (!issue || workflow.test(issue[2]!) || !ui.test(issue[2]!) ||
+        !q.options.some(option => ui.test(`${option.label} ${option.description ?? ''}`))) return false;
+    const context = /^Project\/branch\/task:([^\n]*)/mi.exec(q.question)?.[1] ?? '';
+    const namedPlans = context.match(/\b[\w.-]+\.md\b/gi) ?? [];
+    if ((namedPlans.length && !namedPlans.some(plan => /^PLAN\.md$/i.test(plan))) ||
+        /\b(?:before|prior to)\s+Pass\b/i.test(context)) return false;
+    const choice = new RegExp(`^${issue[1]}[A-Z](?:[).:—–-]\\s*|\\s+)\\S`);
+    return q.options.every(option => choice.test(option.label) && !workflow.test(option.label));
+  });
+}

test/helpers/touchfiles-data.ts

@@ -228,6 +228,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
     "test/ceo-hold-commitment-ar.test.ts", "test/fixtures/ceo-hold-commitment-ar.json",
   ],
   'plan-design-with-ui-scope':   [
+    'test/helpers/design-ui-scope.ts', 'test/design-ui-scope.test.ts', 'test/fixtures/plan-design-ui-scope.json',
     "test/plan-scope-recovery-av.test.ts",
     "test/fixtures/plan-scope-recovery-av.json",
     "test/fixtures/design-scope-checkpoint-at.json",'plan-design-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-design-with-ui.test.ts', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt',