From a487a09bf1125e21a34b41e9b3f5b1b27f6495c9 Mon Sep 17 00:00:00 2001 From: garrytan Date: Wed, 30 Sep 2026 22:06:05 +0000 Subject: [PATCH] test(office-hours): a fork whose outer options are the seeded shapes is the Phase 4 question Census trials 1-2 captured complete Phase 4 forks (A) Server-side B) Client-side C) Hybrid, recommendation with because) whose prose used none of the vocabulary words. Accept two seeded shapes as outer options as Phase 4 specificity; the earlier-phase, nested, fenced and single-shape controls still fail. --- test/office-hours-phase4-caller.test.ts | 13 +++++++++++++ test/skill-e2e-office-hours-phase4.test.ts | 21 +++++++++++++++------ 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/test/office-hours-phase4-caller.test.ts b/test/office-hours-phase4-caller.test.ts index 9dd574f66..43c00f767 100644 --- a/test/office-hours-phase4-caller.test.ts +++ b/test/office-hours-phase4-caller.test.ts @@ -96,3 +96,16 @@ test('Phase4 caller fixture retains fork validation under either path convention expect(rejected.thrown).toBeDefined(); expect(rejected.judged).toBe(0); expect(rejected.rows[0].passed).toBe(false); } }); + +test('Phase4 caller accepts a fork whose outer options are the seeded shapes, without Phase 4 vocabulary', async () => { + // Census 36776104571 trials 1-2: a complete Phase 4 fork naming the three seeded shapes as its options. + const seeded = (options: string) => `Where should the cross-skill retrieval smarts live?\nThe choice locks in who owns the ranking logic.\n${options}\nRecommendation: C because we can tune the salience signal before freezing a contract.\n`; + for (const options of ['A) Server-side — new MCP tools in gbrain\nB) Client-side — a gstack helper\nC) Hybrid — V1 client-side, V1.5 in gbrain', + 'A) **Server-side** gbrain tools\nB) **Hybrid** promotion later']) { + const x = await runCaller(seeded(options)); expect(x.thrown).toBeUndefined(); expect(x.judged).toBe(1); expect(x.rows[0].passed).toBe(true); + } + for (const options of ['A) Server-side — new MCP tools\nB) Put retrieval on the client', 'A) Server-side\n 1) Client-side\n 2) Hybrid', + 'A) Use a queue\n```text\nB) Server-side\nC) Client-side\n```', 'A) Ask customers first\nB) Ask later\nServer-side and Client-side are both options.']) { + const x = await runCaller(seeded(options)); expect(x.thrown).toBeDefined(); expect(x.judged).toBe(0); expect(x.rows[0].passed).toBe(false); + } +}); diff --git a/test/skill-e2e-office-hours-phase4.test.ts b/test/skill-e2e-office-hours-phase4.test.ts index 051803e6a..2043f4a55 100644 --- a/test/skill-e2e-office-hours-phase4.test.ts +++ b/test/skill-e2e-office-hours-phase4.test.ts @@ -45,8 +45,8 @@ const evalCollector = createEvalCollector('e2e-office-hours-phase4'); const BECAUSE_RE = /\bbecause\b/i; // At least 2 numbered/lettered options (A/B or 1/2). Office-hours Phase 4 says // "2-3 distinct alternatives," so 2+ is the minimum bar. -function hasTwoAlternatives(text: string): boolean { - const options: Array<{ indent: number; label: string }> = []; +function outerOptions(text: string): Array<{ label: string; text: string }> { + const options: Array<{ indent: number; label: string; text: string }> = []; let fence: { char: string; length: number } | undefined; for (const line of text.split('\n')) { const marker = /^ {0,3}(`{3,}|~{3,})(.*)$/.exec(line); @@ -55,13 +55,16 @@ function hasTwoAlternatives(text: string): boolean { continue; } if (marker) { fence = { char: marker[1]![0]!, length: marker[1]!.length }; continue; } - const option = /^( {0,3})([A-Z]|[1-9]\d*)\)\s+\S/.exec(line); - if (option) options.push({ indent: option[1]!.length, label: option[2]! }); + const option = /^( {0,3})([A-Z]|[1-9]\d*)\)\s+(\S.*)$/.exec(line); + if (option) options.push({ indent: option[1]!.length, label: option[2]!, text: option[3]! }); } // Only the outer option list counts; numbered steps inside one option and // Markdown source/quoted blocks do not supply another alternative. const outerIndent = Math.min(...options.map(option => option.indent)); - const labels = options.filter(option => option.indent === outerIndent).map(option => option.label); + return options.filter(option => option.indent === outerIndent); +} +function hasTwoAlternatives(text: string): boolean { + const labels = outerOptions(text).map(option => option.label); return new Set(labels.filter(label => /^[A-Z]$/.test(label))).size >= 2 || new Set(labels.filter(label => /^\d+$/.test(label))).size >= 2; } @@ -69,6 +72,12 @@ function hasTwoAlternatives(text: string): boolean { // question. Without this, a captured AskUserQuestion from an earlier phase // would false-pass. const PHASE4_VOCAB_RE = /approach|alternative|architectur(?:e|al)|implementation/i; +// A question whose outer options are two of the seeded Phase 4 shapes is the +// Phase 4 fork even when its prose names none of those words. +function offersSeededAlternatives(text: string): boolean { + const shapes = outerOptions(text).map(option => /^\**(Server-side|Client-side|Hybrid)\b/i.exec(option.text)?.[1]?.toLowerCase()); + return new Set(shapes.filter(Boolean)).size >= 2; +} function setupOfficeHoursDir(): string { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-office-hours-phase4-')); @@ -181,7 +190,7 @@ After writing the file with that ONE Phase 4 question, stop. Do not continue to expect(captured).toMatch(BECAUSE_RE); expect(hasTwoAlternatives(captured)).toBe(true); // Phase-4 specificity: prevents a stray earlier-phase AUQ from false-passing. - expect(captured).toMatch(PHASE4_VOCAB_RE); + expect(PHASE4_VOCAB_RE.test(captured) || offersSeededAlternatives(captured), 'captured question is not the Phase 4 fork').toBe(true); // Recommendation-quality judge: same threshold as plan-format tests. await assertRecommendationQuality({