Merge remote-tracking branch 'origin/main' into garrytan/retention-cohorts

# Conflicts:
#	CHANGELOG.md
#	VERSION
#	package.json

test/helpers/carve-guards.ts

@@ -128,6 +128,8 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
     maxSkeletonBytes: 90_000,
     minUnionBytes: 120_000,
     mustContain: ['VERSION', 'CHANGELOG', 'review', 'merge', 'PR'],
+    // v1.58.5.0: pre-push-guard install (#2077) stacks on the shared first-run-guidance preamble.
+    maxSizeRatio: 1.08,
   },
   'plan-ceo-review': {
     skill: 'plan-ceo-review',
@@ -161,8 +163,8 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
       gateAfterStop: 'EXIT PLAN MODE GATE',
     },
     behavioral: 'plan',
-    // v1.2.0 activation lift: first-run-guidance section in the shared preamble.
-    maxSkeletonBytes: 65_000,
+    // v1.2.0 activation lift (shared first-run-guidance preamble) + #2077 ask-first scope gate.
+    maxSkeletonBytes: 67_000,
     minUnionBytes: 70_000,
     mustContain: ['Architecture', 'Code Quality', 'Test', 'Performance'],
     // Cross-cutting preamble growth (v1.57.2.0 AUQ-failure prose fallback + the
@@ -186,10 +188,11 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
     behavioral: 'plan',
     // +Conductor AUQ-default-prose rule + one-way/continuation safety in the
     // always-loaded AskUserQuestion Format section.
-    // v1.2.0 activation lift: first-run-guidance section in the shared preamble.
-    maxSkeletonBytes: 86_000,
+    // v1.2.0 activation lift (shared first-run-guidance preamble) + #2077 ask-first scope gate.
+    maxSkeletonBytes: 88_000,
     minUnionBytes: 70_000,
     mustContain: ['design', 'visual'],
+    maxSizeRatio: 1.07,
   },
   'plan-devex-review': {
     skill: 'plan-devex-review',

test/helpers/claude-pty-runner.ts

@@ -71,6 +71,15 @@ export interface ClaudePtyOptions {
   permissionMode?: 'plan' | 'default' | 'acceptEdits' | 'bypassPermissions' | 'auto' | 'dontAsk' | null;
   /** Extra args after the permission-mode flag. */
   extraArgs?: string[];
+  /**
+   * Model for the spawned interactive `claude`. Without an explicit --model the
+   * child inherits the operator's ~/.claude/settings.json model (e.g.
+   * claude-fable-5[1m]), which can spend 5+ min in extended thinking on an empty
+   * plan-mode context and blow every smoke budget. Resolution mirrors
+   * session-runner.ts:144 exactly: opts.model ?? EVALS_MODEL ?? 'claude-sonnet-4-6'.
+   * Pushed BEFORE extraArgs so a test-supplied --model still wins (last flag wins).
+   */
+  model?: string;
   /** Terminal size. Default 120x40. Plan-mode UI lays out cleanly at this size. */
   cols?: number;
   rows?: number;
@@ -406,8 +415,10 @@ export function judgePtyState(
   const prompt = `You are reading a snapshot of a terminal where Claude Code is running in plan mode for an automated test. Your job: classify the agent's current state.
 
 Pick exactly ONE:
-- WAITING — agent surfaced a question or option list and is sitting at the input prompt waiting for user reply. Signs: numbered/lettered options visible (1./2./3. or A)/B)/C)), "Recommendation:" line, cursor at empty input prompt with no recent generation activity.
+- WAITING — agent surfaced a question or option list and is sitting at the input prompt waiting for user reply. Signs: numbered/lettered options visible (1./2./3. or A)/B)/C)), "Recommendation:" line, cursor at empty input prompt with no recent generation activity, OR a fully-rendered question + reply-instruction (e.g. "Reply with A, B, or C" / "Recommendation:") is visible.
 - WORKING — agent is actively generating or running tools. Signs: spinner glyphs (✻ ✶ ✳ ✢ ✽), "Musing..." or "Churned for ..." text, recent tool-call blocks (Read/Edit/Bash/Grep), in-flight token output.
+
+PRECEDENCE OVERRIDE: if a lettered/numbered option list (A)/B)/1./2.) AND a "Recommendation:" or "Reply with"/"Reply A" instruction are BOTH visible in this snapshot, classify WAITING even when spinner glyphs (✻ ✶ ✳ ✢ ✽) are still animating — Claude Code keeps the spinner up at an idle prose decision, so a spinner alongside a fully-rendered question + reply-instruction is a residual render artifact, not active generation.
 - HUNG — agent has stopped without surfacing a question and without any spinner/work activity. Rare; usually means a crash.
 
 Respond with strict JSON ONLY (no markdown fences, no prose):
@@ -503,6 +514,16 @@ ${tail}
  *     for plan-eng / plan-design / plan-devex prose AUQ
  *   - 3+ distinct numbered options (1. 2. 3.) at line starts WITHOUT a
  *     `❯<spaces>1.` cursor — typical for autoplan / office-hours prose AUQ
+ *   - 3+ markdown bold-bullet options (`- **label**`) following an
+ *     interrogative line — office-hours renders its mode question this way
+ *     (`> - **Building a startup**`), which has no letter/number marker
+ *   - Pattern 4/5 (collapsed-form): a reply-instruction OR recommendation
+ *     marker PLUS 2+ distinct A-D letter markers each punctuated by ) : or (
+ *     anywhere in the tail. stripAnsi destroys the newlines + inter-word
+ *     spaces that the line-anchored patterns above need, so a real prose AUQ
+ *     arrives collapsed ("ReplywithA,B,orC", "A(recommended)", "-B:") and is
+ *     invisible to Patterns 1-3. This is the dominant Shape-B render mode in
+ *     the plan-design smoke + floor timeouts (verified against real run bytes).
  *
  * Used by classifyVisible and runPlanSkillFloorCheck to return outcome='asked'
  * (or auq_observed) instead of letting the harness time out when the model
@@ -546,7 +567,55 @@ export function isProseAUQVisible(visible: string): boolean {
   while ((nm = numberedRe.exec(tail)) !== null) {
     if (nm[1]) numberedHits.add(nm[1]);
   }
-  return numberedHits.size >= 2;
+  if (numberedHits.size >= 2) return true;
+
+  // Pattern 3: markdown bold-bullet option list. office-hours renders its
+  // mode question as `> - **Building a startup**` lines under
+  // --disallowedTools — no letter/number marker, so Patterns 1-2 miss it,
+  // and the model keeps a spinner up so the Haiku judge scores it 'working'
+  // and the run times out despite the question being on screen.
+  // Require both: an interrogative line (the question stem ends in '?') AND
+  // 3+ bold-bullet markers. The bold (`- **`) requirement is what separates
+  // an option list from incidental prose bullets; the line anchor is dropped
+  // because stripAnsi can collapse option lines (see Pattern 1 note), so we
+  // count markers anywhere in the tail. The `❯ 1.` cursor gate above already
+  // excludes a live native list.
+  if (/\?/.test(tail)) {
+    const boldBulletHits = (tail.match(/[-*•]\s+\*\*/g) || []).length;
+    if (boldBulletHits >= 3) return true;
+  }
+
+  // Pattern 4/5: collapsed-form prose AUQ. stripAnsi removes the
+  // cursor-positioning escapes that render option newlines + inter-word
+  // spaces, so "Reply with A, B, or C" arrives as "ReplywithA,B,orC" and
+  // "A) ..." as "A(recommended)" / "-B:" — defeating every line-anchored or
+  // ')'-anchored pattern above (Patterns 1-3 all return false on the real
+  // plan-design smoke + floor timeout bytes). Detect via two INDEPENDENT
+  // signals that must BOTH hold — the corroboration is what separates a real
+  // AUQ from incidental report prose that happens to mention a recommendation:
+  //   (1) a reply-instruction matched space-insensitively OR a recommendation
+  //       marker, AND
+  //   (2) 2+ distinct A-D letter markers each punctuated by ) : or ( anywhere
+  //       in the tail.
+  // A single 'B)' + the word "recommendation", or a comma-only collapsed
+  // "ReplywithA,B,orC" with no )/:/( punctuation on the letters, both stay
+  // false — the two-signal contract is pinned by unit tests.
+  const replyOrRec =
+    /reply\s*(?:with)?\s*[A-D]/i.test(tail) ||
+    /reply(?:with)?[A-D]/i.test(tail.replace(/\s+/g, '')) ||
+    /\bRecommendation\s*:/i.test(tail) ||
+    /\(recommended\)/i.test(tail);
+  if (replyOrRec) {
+    const collapsedLetterRe = /\b([A-D])[):(]/g;
+    const collapsedHits = new Set<string>();
+    let cm: RegExpExecArray | null;
+    while ((cm = collapsedLetterRe.exec(tail)) !== null) {
+      if (cm[1]) collapsedHits.add(cm[1]);
+    }
+    if (collapsedHits.size >= 2) return true;
+  }
+
+  return false;
 }
 
 /**
@@ -1145,9 +1214,15 @@ export async function launchClaudePty(
   let exited = false;
   let exitCodeCaptured: number | null = null;
 
+  const args: string[] = [];
+  // Pin the model so smokes don't inherit the operator's settings.json model
+  // (see ClaudePtyOptions.model). Chain mirrors session-runner.ts:144 so PTY and
+  // `claude -p` evals always agree. Pushed before extraArgs => a test-supplied
+  // --model wins (last flag wins).
+  const model = opts.model ?? process.env.EVALS_MODEL ?? 'claude-sonnet-4-6';
+  args.push('--model', model);
   // Permission mode: 'plan' default, null => omit flag entirely.
   const permissionMode = opts.permissionMode === undefined ? 'plan' : opts.permissionMode;
-  const args: string[] = [];
   if (permissionMode !== null) {
     args.push('--permission-mode', permissionMode);
   }
@@ -1498,6 +1573,9 @@ export async function runPlanSkillObservation(opts: {
    * Step 0 reads the prior conversation context so it sees the draft.
    */
   initialPlanContent?: string;
+  /** Override the spawned model. Defaults via launchClaudePty's chain
+   *  (opts.model ?? EVALS_MODEL ?? 'claude-sonnet-4-6'). */
+  model?: string;
 }): Promise<PlanSkillObservation> {
   const startedAt = Date.now();
   const session = await launchClaudePty({
@@ -1506,6 +1584,7 @@ export async function runPlanSkillObservation(opts: {
     timeoutMs: (opts.timeoutMs ?? 180_000) + 30_000,
     extraArgs: opts.extraArgs,
     env: opts.env,
+    model: opts.model,
   });
 
   try {
@@ -1762,6 +1841,8 @@ export async function runPlanSkillCounting(opts: {
   timeoutMs?: number;
   /** Extra env merged into the spawned `claude` process. */
   env?: Record<string, string>;
+  /** Override the spawned model. Defaults via launchClaudePty's chain. */
+  model?: string;
 }): Promise<PlanSkillCountObservation> {
   const startedAt = Date.now();
   const defaultPick = opts.defaultPick ?? 1;
@@ -1772,6 +1853,7 @@ export async function runPlanSkillCounting(opts: {
     cwd: opts.cwd,
     timeoutMs: timeoutMs + 60_000,
     env: opts.env,
+    model: opts.model,
   });
 
   const fingerprints: AskUserQuestionFingerprint[] = [];
@@ -1993,6 +2075,8 @@ export async function runPlanSkillFloorCheck(opts: {
   timeoutMs?: number;
   /** Extra env merged into the spawned `claude` process. */
   env?: Record<string, string>;
+  /** Override the spawned model. Defaults via launchClaudePty's chain. */
+  model?: string;
 }): Promise<PlanSkillFloorObservation> {
   const startedAt = Date.now();
   const timeoutMs = opts.timeoutMs ?? 600_000;
@@ -2002,6 +2086,7 @@ export async function runPlanSkillFloorCheck(opts: {
     cwd: opts.cwd,
     timeoutMs: timeoutMs + 60_000,
     env: opts.env,
+    model: opts.model,
   });
 
   try {

test/helpers/claude-pty-runner.unit.test.ts

@@ -23,6 +23,7 @@
  */
 
 import { describe, test, expect } from 'bun:test';
+import { readFileSync } from 'node:fs';
 import {
   isPermissionDialogVisible,
   isNumberedOptionListVisible,
@@ -290,6 +291,109 @@ This refers to (see option B) above and also to point A) earlier.
     expect(isProseAUQVisible('Just some plain text output from the model.')).toBe(false);
     expect(isProseAUQVisible('')).toBe(false);
   });
+
+  // Pattern 3: markdown bold-bullet options — office-hours renders its mode
+  // question this way under --disallowedTools, with no letter/number marker.
+  test('matches office-hours markdown bold-bullet mode question (Pattern 3)', () => {
+    const sample = `
+> Before we dig in — what's your goal with this?
+>
+> - **Building a startup** (or thinking about it)
+> - **Intrapreneurship** — internal project at a company, need to ship fast
+> - **Hackathon / demo** — time-boxed, need to impress
+> - **Open source / research** — building for a community
+> - **Learning** — teaching yourself to code
+❯
+`;
+    expect(isProseAUQVisible(sample)).toBe(true);
+  });
+
+  test('bold-bullets require a preceding interrogative — no "?" => false', () => {
+    // 3+ bold bullets but no question stem: this is a feature list, not an AUQ.
+    const sample = `
+Here is what shipped:
+- **Faster builds** via caching
+- **Smaller binaries** through tree-shaking
+- **Better errors** with source maps
+`;
+    expect(isProseAUQVisible(sample)).toBe(false);
+  });
+
+  test('a question with fewer than 3 bold bullets stays false (guard)', () => {
+    const sample = `
+Which approach do you prefer?
+- **Option one** is simpler
+- **Option two** is faster
+`;
+    expect(isProseAUQVisible(sample)).toBe(false);
+  });
+
+  test('plain (non-bold) bullets after a question do not trigger Pattern 3', () => {
+    // Only bold bullets count — plain "- text" prose lists are too common.
+    const sample = `
+What should we do about this?
+- run the tests
+- ship the fix
+- file a follow-up
+`;
+    expect(isProseAUQVisible(sample)).toBe(false);
+  });
+
+  test('Pattern 3 still defers to a live native cursor list (❯ 1.)', () => {
+    const sample = `
+> What's your goal?
+❯ 1. **Building a startup**
+  2. **Intrapreneurship**
+  3. **Hackathon**
+`;
+    // The ❯1. cursor gate fires first — native list handling owns this.
+    expect(isProseAUQVisible(sample)).toBe(false);
+  });
+
+  // Pattern 4/5: collapsed-form prose AUQ. stripAnsi destroys the newlines +
+  // inter-word spaces, so a real prose AUQ arrives collapsed and defeats the
+  // line-anchored Patterns 1-3. These are the dominant Shape-B render mode in
+  // the plan-design smoke + floor timeouts — verbatim de-spinnered bytes from
+  // the real failing runs (bdm3sucql.output).
+  test('matches the real collapsed floor render (colon-delimited, Pattern 4/5)', () => {
+    const sample =
+      'The review is blocked on D1—reply withA, B, r Cabovetocontinue:' +
+      '- A(recommended): Spec thefull P1AskUserQuestioncopy in this review' +
+      '-B:LeaveP1copytotheimplementerwithstructuralrequirements' +
+      'C: Add a placeholder template to the plan';
+    expect(isProseAUQVisible(sample)).toBe(true);
+  });
+
+  test('matches the real collapsed plan-mode render (Recommendation + collapsed A)/B), Pattern 4/5)', () => {
+    const sample =
+      'Recommendation:A—writethecopynow.(recommended)A) Writ the fullcopy in thisdesign review— now.' +
+      '(recommended) Completeness:10/10 B) Leveit to theimplemente — task spec is enough.' +
+      'Reply withA (write the copy now)orB(leavetoimplementer)';
+    expect(isProseAUQVisible(sample)).toBe(true);
+  });
+
+  test('collapsed-form requires BOTH signals — single B) + word "recommendation" stays false', () => {
+    // Only one punctuated letter marker: the two-signal contract is not met.
+    const sample =
+      'We should consider option B) here. My recommendation is to do it now.';
+    expect(isProseAUQVisible(sample)).toBe(false);
+  });
+
+  test('collapsed-form requires letter punctuation — comma-only "ReplywithA,B,orC" stays false', () => {
+    // Reply-instruction present, but the letters carry no ) : or ( punctuation,
+    // so they could be incidental enumerations in running prose. Stays false.
+    const sample = 'ReplywithA,B,orC';
+    expect(isProseAUQVisible(sample)).toBe(false);
+  });
+
+  test('collapsed-form does not regress the existing FP guard (see option B) ... point A))', () => {
+    // The classic citation FP: a model referencing prior options in prose.
+    // No reply-instruction / recommendation marker on its own line, so the
+    // collapsed-form signal does not fire either.
+    const sample =
+      'As noted (see option B) above, and the earlier point A) we discussed, this is fine.';
+    expect(isProseAUQVisible(sample)).toBe(false);
+  });
 });
 
 describe('classifyVisible (runtime path through the runner classifier)', () => {
@@ -552,6 +656,45 @@ describe('runPlanSkillObservation env passthrough surface', () => {
   });
 });
 
+describe('launchClaudePty model pin (static tripwire)', () => {
+  // Why static-grep, not a behavioral assert: the spawn fires immediately
+  // inside launchClaudePty, so asserting the built args array would require
+  // extracting an arg-builder seam — which rewrites the exact region kyoto-v5's
+  // hermetic --strict-mcp-config insertion edits, reintroducing a merge
+  // conflict the placement deliberately avoids. The end-to-end behavioral proof
+  // is the live PTY smoke (skill-e2e-plan-*-plan-mode.test.ts) running under the
+  // pinned model. These grep-level guards stop a refactor from silently
+  // dropping the pin or reordering it past extraArgs.
+  const src = readFileSync(new URL('./claude-pty-runner.ts', import.meta.url), 'utf-8');
+
+  test('ClaudePtyOptions exposes model?: string', () => {
+    const opts: ClaudePtyOptions = { model: 'claude-sonnet-4-6' };
+    expect(opts.model).toBe('claude-sonnet-4-6');
+  });
+
+  test('spawn args push --model from the EVALS_MODEL fallback chain', () => {
+    expect(src).toContain("args.push('--model', model)");
+    // opts.model -> EVALS_MODEL -> 'claude-sonnet-4-6' (mirrors session-runner.ts:144)
+    expect(src).toMatch(
+      /opts\.model\s*\?\?\s*process\.env\.EVALS_MODEL\s*\?\?\s*'claude-sonnet-4-6'/,
+    );
+  });
+
+  test('--model is pushed BEFORE extraArgs so a per-test --model override wins', () => {
+    const modelPush = src.indexOf("args.push('--model', model)");
+    const extraArgsPush = src.indexOf('if (opts.extraArgs) args.push(...opts.extraArgs)');
+    expect(modelPush).toBeGreaterThan(-1);
+    expect(extraArgsPush).toBeGreaterThan(-1);
+    expect(modelPush).toBeLessThan(extraArgsPush);
+  });
+
+  test('all three plan-skill wrappers forward model to launchClaudePty', () => {
+    // Count must match the number of wrappers (observation, counting, floor).
+    const forwards = src.match(/^\s*model: opts\.model,$/gm) ?? [];
+    expect(forwards.length).toBe(3);
+  });
+});
+
 // ────────────────────────────────────────────────────────────────────────────
 // Per-finding count primitives — Section 3 unit tests #1–#5, #7, #12.
 // ────────────────────────────────────────────────────────────────────────────

test/helpers/touchfiles.ts

@@ -523,10 +523,16 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
   'plan-eng-coverage-audit': 'gate',
   'plan-review-report': 'gate',
 
-  // Plan-mode handshake — deterministic safety regression, gate-tier
+  // Plan-mode handshake. plan-ceo/plan-devex ask-first reliably (gate-tier);
+  // plan-eng/plan-design run a long explore/audit before their first
+  // AskUserQuestion, so whether they reach a terminal outcome within the 300s
+  // budget hinges on stochastic ask-first compliance (~50-67%/run measured).
+  // Per the "non-deterministic -> periodic" tiering rule they are periodic:
+  // the hardened ask-first gate + the collapsed-form detector lifted them from
+  // always-failing to mostly-passing, but they are not deterministic gates.
   'plan-ceo-review-plan-mode': 'gate',
-  'plan-eng-review-plan-mode': 'gate',
-  'plan-design-review-plan-mode': 'gate',
+  'plan-eng-review-plan-mode': 'periodic',
+  'plan-design-review-plan-mode': 'periodic',
   'plan-devex-review-plan-mode': 'gate',
   'plan-mode-no-op': 'gate',
   // v1.21+ auto-mode regression tests
@@ -556,9 +562,9 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
   'plan-eng-finding-count':    'periodic',
   'plan-design-finding-count': 'periodic',
   'plan-devex-finding-count':  'periodic',
-  'plan-eng-finding-floor':    'gate',
+  'plan-eng-finding-floor':    'periodic',  // stochastic ask-first (see plan-mode-handshake note); periodic
   'plan-ceo-finding-floor':    'gate',
-  'plan-design-finding-floor': 'gate',
+  'plan-design-finding-floor': 'periodic',  // stochastic ask-first (see plan-mode-handshake note); periodic
   'plan-devex-finding-floor':  'gate',
   'plan-eng-multi-finding-batching': 'periodic',
   'plan-ceo-split-overflow': 'periodic',