From 5526fca45cf2d07d9393f78832da1aa039b1e223 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 21:40:12 +0000 Subject: [PATCH] Q3: office-hours asks open-ended questions in a Q prose form (#2719) In Conductor, and whenever AskUserQuestion fails, prose is the only channel, and the prose fallback defined only the D decision brief. Phase 2A/2B diagnostic questions have no option set, so the model dropped the phase. Office-hours' preamble now defines the Q open-question form (question, why I'm asking, what a strong answer sounds like) and routes free-text replies to it; the diagnostic sections name it. Scoped to office-hours so every other skill's preamble is unchanged. Ported from PR #2729. Co-authored-by: Bharadwaj Mittapelli --- office-hours/SKILL.md.tmpl | 2 +- .../phase-2a-startup-diagnostic.md.tmpl | 2 + .../phase-2b-builder-brainstorm.md.tmpl | 2 + .../preamble/generate-ask-user-format.ts | 15 +++++++- test/office-hours-open-question-form.test.ts | 38 +++++++++++++++++++ 5 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 test/office-hours-open-question-form.test.ts diff --git a/office-hours/SKILL.md.tmpl b/office-hours/SKILL.md.tmpl index f0a4bb9a1..2ef970cb1 100644 --- a/office-hours/SKILL.md.tmpl +++ b/office-hours/SKILL.md.tmpl @@ -335,7 +335,7 @@ Confirm you Read every section the Section index named as applying to this run, ## Important Rules - **Never start implementation.** This skill produces design docs, not code. Not even scaffolding. -- **Questions ONE AT A TIME.** Never batch multiple questions into one AskUserQuestion. +- **Questions ONE AT A TIME.** Never batch multiple questions into one AskUserQuestion. Without AskUserQuestion, use `Q` for open-ended questions and `D` for discrete decisions. - **The assignment is mandatory.** Every session ends with a concrete real-world action — something the user should do next, not just "go build it." - **If user provides a fully formed plan:** skip Phase 2 (questioning) but still run Phase 3 (Premise Challenge) and Phase 4 (Alternatives). Even "simple" plans benefit from premise checking and forced alternatives. - **Completion status:** diff --git a/office-hours/sections/phase-2a-startup-diagnostic.md.tmpl b/office-hours/sections/phase-2a-startup-diagnostic.md.tmpl index bc0d0c074..bc3689c34 100644 --- a/office-hours/sections/phase-2a-startup-diagnostic.md.tmpl +++ b/office-hours/sections/phase-2a-startup-diagnostic.md.tmpl @@ -68,6 +68,8 @@ These examples show the difference between soft exploration and rigorous diagnos Ask these questions **ONE AT A TIME** via AskUserQuestion. Push on each one until the answer is specific, evidence-based, and uncomfortable. Comfort means the founder hasn't gone deep enough. +These questions are open-ended (no fixed option set): when AskUserQuestion is unavailable (Conductor, or a failed call), ask each in the `Q` open-question prose form, never as a `D` decision brief. + When a forcing question's options describe the founder's own evidence, the `Recommendation:` still takes a position: recommend the option the founder's own words already support ("zero users" supports the no-evidence-yet answer) because of what that answer means for the next step, and name the evidence that would change it. Never recommend an option only because it would be the best position to be in. **Smart routing based on product stage — you don't always need all six:** diff --git a/office-hours/sections/phase-2b-builder-brainstorm.md.tmpl b/office-hours/sections/phase-2b-builder-brainstorm.md.tmpl index 0964eb7e5..5b8d5ddaa 100644 --- a/office-hours/sections/phase-2b-builder-brainstorm.md.tmpl +++ b/office-hours/sections/phase-2b-builder-brainstorm.md.tmpl @@ -26,6 +26,8 @@ Both are outcome-framed. Only one has the 'whoa.' Builder mode's job is to surfa Ask these **ONE AT A TIME** via AskUserQuestion. The goal is to brainstorm and sharpen the idea, not interrogate. +These questions are open-ended (no fixed option set): when AskUserQuestion is unavailable (Conductor, or a failed call), ask each in the `Q` open-question prose form, never as a `D` decision brief. + - **What's the coolest version of this?** What would make it genuinely delightful? - **Who would you show this to?** What would make them say "whoa"? - **What's the fastest path to something you can actually use or share?** diff --git a/scripts/resolvers/preamble/generate-ask-user-format.ts b/scripts/resolvers/preamble/generate-ask-user-format.ts index 4451a165c..4ba1a5412 100644 --- a/scripts/resolvers/preamble/generate-ask-user-format.ts +++ b/scripts/resolvers/preamble/generate-ask-user-format.ts @@ -1,6 +1,8 @@ import type { TemplateContext } from '../types'; export function generateAskUserFormat(ctx: TemplateContext): string { + // Q3 (#2719, PR #2729): office-hours asks open-ended diagnostic questions, which have no option set. + const openQuestions = ctx.skillName === 'office-hours'; const planReview = ['plan-ceo-review', 'plan-eng-review', 'plan-design-review', 'plan-devex-review'].includes(ctx.skillName); return `## AskUserQuestion Format @@ -33,7 +35,7 @@ Tell three outcomes apart: 2. **Completeness scores per choice** — explicit on EACH choice, per the Completeness rule in the Format section below; never silently drop the score. 3. **The recommendation and why** — the \`Recommendation: because \` line plus the \`(recommended)\` marker on that choice. -Layout: a \`D\` title; an explicit reply line listing the offered selectors; the issue ELI10; the Recommendation line; ONE paragraph per choice with its \`(recommended)\` marker, \`Completeness: X/10\`, and 2-4 sentences of reasoning (never a bare bullet list); a closing \`Net:\` line. With \`QUESTION_TUNING: true\`, append the checked \`\` to the explicit reply line. Split chains / 5+ options: one prose block per per-option call, in sequence. Before an interactive prose question, finish preparatory tool calls that do not depend on its answer. Then send the complete brief as the final message of the turn and STOP and wait for the user's typed answer. Do not publish an earlier copy during tool work or follow it with tools or a summary-only waiting message. In plan mode this satisfies end-of-turn like a tool call. +Layout: a \`D\` title; an explicit reply line listing the offered selectors; the issue ELI10; the Recommendation line; ONE paragraph per choice with its \`(recommended)\` marker, \`Completeness: X/10\`, and 2-4 sentences of reasoning (never a bare bullet list); a closing \`Net:\` line. With \`QUESTION_TUNING: true\`, append the checked \`\` to the explicit reply line. Split chains / 5+ options: one prose block per per-option call, in sequence. Before an interactive prose question, finish preparatory tool calls that do not depend on its answer. Then send the complete brief as the final message of the turn and STOP and wait for the user's typed answer.${openQuestions ? ' An open-ended question with no options list uses the `Q` form below instead of a `D` brief; a free-text reply answers the most recent unanswered `Q`.' : ''} Do not publish an earlier copy during tool work or follow it with tools or a summary-only waiting message. In plan mode this satisfies end-of-turn like a tool call. **Continuation — mapping a typed reply back to a brief.** Each brief carries a stable label (\`D\`, or \`D.k\` in a split chain). The user references it (e.g. "3.2: B"). A bare letter maps to the single most-recent UNANSWERED brief; if more than one is open (a split chain), do NOT guess — ask which \`D.k\` it answers. Never apply a bare letter ambiguously across a chain. @@ -42,7 +44,18 @@ Layout: a \`D\` title; an explicit reply line listing the offered selectors; ### Format Every AskUserQuestion is a decision brief and must be sent as tool_use, not prose — unless the documented failure fallback above applies (interactive session + the call is unavailable/erroring), in which case the prose fallback is the correct output. +${openQuestions ? ` +**Open-question prose form (\`Q\`)** — for open-ended questions with no fixed option set (the Phase 2A/2B diagnostic questions) when you are in prose: +\`\`\`text +Q — +Why I'm asking: <1-2 sentences: stakes, what a weak answer would mean> +What a strong answer sounds like: +Reply in your own words — I'll wait. +\`\`\` + +Q-numbering starts at \`Q1\` per invocation, independent of D-numbering. Questions with discrete options always use \`D\`. +` : ''} \`\`\` D — Project/branch/task: <1 short grounding sentence using _BRANCH> diff --git a/test/office-hours-open-question-form.test.ts b/test/office-hours-open-question-form.test.ts new file mode 100644 index 000000000..d506e34f7 --- /dev/null +++ b/test/office-hours-open-question-form.test.ts @@ -0,0 +1,38 @@ +/** + * Q3 (#2719, PR #2729): in Conductor (or when AskUserQuestion fails) prose is + * the only channel, and the prose fallback defined only the `D` decision + * brief. Office-hours' Phase 2A/2B diagnostic questions have no option set, so + * the model dropped them. Office-hours now carries a `Q` open-question form + * and its diagnostic phases name it; other skills are unchanged. + */ +import { describe, expect, test } from 'bun:test'; +import { generateAskUserFormat } from '../scripts/resolvers/preamble/generate-ask-user-format'; +import { HOST_PATHS, type TemplateContext } from '../scripts/resolvers/types'; +import * as fs from 'fs'; +import * as path from 'path'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const ctx = (host: string, skillName: string): TemplateContext => ({ host, skillName, tmplPath: '', paths: HOST_PATHS[host] }); + +describe('Q3: open-question prose form', () => { + for (const host of ['claude', 'codex']) { + test(`${host}: office-hours defines the Q form and routes free-text replies to it`, () => { + const text = generateAskUserFormat(ctx(host, 'office-hours')); + expect(text).toContain('**Open-question prose form (`Q`)**'); + expect(text).toContain("Q — \nWhy I'm asking:"); + expect(text).toContain("Reply in your own words — I'll wait."); + expect(text).toContain('a free-text reply answers the most recent unanswered `Q`'); + expect(text).toContain('Questions with discrete options always use `D`'); + }); + + test(`${host}: other skills keep only the D brief`, () => { + expect(generateAskUserFormat(ctx(host, 'review'))).not.toContain('Q'); + }); + } + + test('the Phase 2A and 2B diagnostic sections ask in Q form when prose is the channel', () => { + for (const file of ['office-hours/sections/phase-2a-startup-diagnostic.md.tmpl', 'office-hours/sections/phase-2b-builder-brainstorm.md.tmpl']) { + expect(fs.readFileSync(path.join(ROOT, file), 'utf8')).toContain('ask each in the `Q` open-question prose form, never as a `D` decision brief'); + } + }); +});