From aab8009b685099743c25a130a9d8b48b45480b65 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 22:15:44 +0000 Subject: [PATCH] I1: grade the AUQ matrix's (recommended) marker on option labels, the field the auto-decide hook reads The decision-brief format puts a Pros / cons block with 'A) ... (recommended)' in the question text and the same marker on the option label. The matrix counted both and failed all six skills in periodic run 37151477069 with 'expected exactly one (recommended) option, found 2'. The question-preference hook reads labels only, so the gate now checks the native question: a Recommendation line in the question text and exactly one label ending in (recommended). Fixtures move to the native question shape and add the real spec capture plus a marker-only-in-text negative control. --- test/auq-native-capture.test.ts | 12 +-- test/fixtures/auq-matrix/briefs.json | 129 +++++++++++++++++++++++++-- test/helpers/auq-sdk-capture.ts | 16 ++-- test/skill-e2e-auq-matrix.test.ts | 9 +- 4 files changed, 142 insertions(+), 24 deletions(-) diff --git a/test/auq-native-capture.test.ts b/test/auq-native-capture.test.ts index 8517f5939..270314ade 100644 --- a/test/auq-native-capture.test.ts +++ b/test/auq-native-capture.test.ts @@ -636,17 +636,17 @@ fs.writeFileSync(path.join(root,'results.json'),JSON.stringify({results,privateR // Stored first-AUQ briefs for the matrix's machine-read format gate. { const briefs = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/auq-matrix/briefs.json'), 'utf8')) as - { known_good: Record; known_bad: Record }; + { known_good: Record; known_bad: Record }; describe('AUQ matrix format gate', () => { - test.each(Object.entries(briefs.known_good))('passes %s', (_name, text) => { - expect(auqMachineFormatProblems(text)).toEqual([]); + test.each(Object.entries(briefs.known_good))('passes %s', (_name, question) => { + expect(auqMachineFormatProblems(question)).toEqual([]); }); - test.each(Object.entries(briefs.known_bad))('fails %s', (_name, text) => { - expect(auqMachineFormatProblems(text).length).toBeGreaterThan(0); + test.each(Object.entries(briefs.known_bad))('fails %s', (_name, question) => { + expect(auqMachineFormatProblems(question).length).toBeGreaterThan(0); }); test('a brief missing only presentation elements no longer fails, but they are still reported', () => { - expect(scoreAuqFormat(briefs.known_good['no-emoji-or-net']!).missing).toEqual(['Pros / cons:', '✅', '❌', 'Net:']); + expect(scoreAuqFormat(serializeNativeAuq(briefs.known_good['no-emoji-or-net']!)).missing).toEqual(['Pros / cons:', '✅', '❌', 'Net:']); }); }); } diff --git a/test/fixtures/auq-matrix/briefs.json b/test/fixtures/auq-matrix/briefs.json index 00437fcfc..71275c2e9 100644 --- a/test/fixtures/auq-matrix/briefs.json +++ b/test/fixtures/auq-matrix/briefs.json @@ -1,14 +1,127 @@ { - "_comment": "First-AUQ texts in serializeNativeAuq shape. known_good passes the matrix's format gate (substance is graded separately). known_bad must keep failing it.", + "_comment": "First-AUQ native questions (header, question text, option labels and descriptions). known_good passes the matrix's machine-read format gate (substance is graded separately); known_bad must keep failing it. pros-cons-block-in-question is the spec capture from periodic run 37151477069: the format's Pros / cons block repeats (recommended) in the question text while exactly one label carries it.", "known_good": { - "full-format": "Scope\n\nD1 — Which test harness should the plan adopt?\nELI10: The plan needs one test runner.\nStakes if we pick wrong: flaky CI for weeks.\nRecommendation: A because the repo already uses bun test.\nPros / cons:\nNet: reuse vs. migration.\n\nA) bun test (recommended)\n✅ already installed\n❌ fewer plugins\n\nB) vitest\n✅ plugins\n❌ new dependency", - "no-emoji-or-net": "Scope\n\nD1 — Which test harness should the plan adopt?\nELI10: The plan needs one test runner.\nRecommendation: A because the repo already uses bun test and adding vitest means a second config.\n\nA) bun test (recommended)\nAlready installed; fewer plugins.\n\nB) vitest\nMore plugins; new dependency.", - "bold-recommendation": "Scope\n\n**Recommendation:** B, because the migration is already half done.\n\nA) Stay\nNo churn.\n\nB) Finish migration (recommended)\nOne config." + "full-format": { + "header": "Scope", + "question": "D1 — Which test harness should the plan adopt?\nELI10: The plan needs one test runner.\nStakes if we pick wrong: flaky CI for weeks.\nRecommendation: A because the repo already uses bun test.\nPros / cons:\nNet: reuse vs. migration.", + "options": [ + { + "label": "A) bun test (recommended)", + "description": "✅ already installed\n❌ fewer plugins" + }, + { + "label": "B) vitest", + "description": "✅ plugins\n❌ new dependency" + } + ] + }, + "no-emoji-or-net": { + "header": "Scope", + "question": "D1 — Which test harness should the plan adopt?\nELI10: The plan needs one test runner.\nRecommendation: A because the repo already uses bun test and adding vitest means a second config.", + "options": [ + { + "label": "A) bun test (recommended)", + "description": "Already installed; fewer plugins." + }, + { + "label": "B) vitest", + "description": "More plugins; new dependency." + } + ] + }, + "bold-recommendation": { + "header": "Scope", + "question": "**Recommendation:** B, because the migration is already half done.", + "options": [ + { + "label": "A) Stay", + "description": "No churn." + }, + { + "label": "B) Finish migration (recommended)", + "description": "One config." + } + ] + }, + "pros-cons-block-in-question": { + "header": "Recipients", + "question": "D1 — Who receives the email when a task is assigned? \nProject/branch/task: main branch, speccing \"email notifications when a task is assigned to someone\".\nELI10: Right now nobody gets told when a task lands on their plate. The obvious fix is to email the new assignee, but assignment also matters to the person who lost the task and to whoever created it. Each extra audience adds templates, preference toggles, and tests, and emailing people who do not care trains them to ignore your mail.\nStakes if we pick wrong: Too narrow and reassigned work silently vanishes from someone's queue. Too broad and your notifications become spam that users filter out.\nRecommendation: A because the assignee is the one person who must act, and the other audiences can be added as a follow-up issue once the pipeline exists.\nCompleteness: A=7/10, B=9/10, C=10/10\nPros / cons:\nA) New assignee only (recommended)\n ✅ One template, one recipient, one preference flag; smallest surface to test and ship (human: ~1 day / CC: ~10 min)\n ✅ Matches what every major tracker does by default, so users will not be surprised\n ❌ On reassignment the previous assignee is never told the task left their queue\nB) New assignee plus previous assignee on reassignment\n ✅ Closes the \"task silently disappeared\" gap that causes duplicated work\n ✅ Still a small audience; two templates instead of one (human: ~1.5 days / CC: ~15 min)\n ❌ Requires reading the old assignee before the write, which constrains where the hook lives\nC) Assignee, previous assignee, and task creator or watchers\n ✅ Full visibility for everyone with a stake in the task\n ✅ Lays groundwork for a general watcher model later (human: ~3 days / CC: ~30 min)\n ❌ Highest spam risk and needs per-audience opt-out from day one\nNet: You are trading a small gap on reassignment against the cost of more templates, more preferences, and more inbox noise.", + "options": [ + { + "label": "New assignee only (recommended)", + "description": "✅ One template, one recipient, smallest surface to test and ship\n✅ Matches what every major tracker does by default\n❌ Previous assignee is never told the task left their queue" + }, + { + "label": "Assignee + previous assignee", + "description": "✅ Closes the \"task silently disappeared\" gap on reassignment\n✅ Still a small audience, two templates instead of one\n❌ Must read the old assignee before the write, constraining hook placement" + }, + { + "label": "Assignee, previous, creator/watchers", + "description": "✅ Full visibility for everyone with a stake in the task\n✅ Groundwork for a general watcher model later\n❌ Highest spam risk, needs per-audience opt-out from day one" + } + ] + } }, "known_bad": { - "no-recommendation-line": "Scope\n\nD1 — Which test harness?\nELI10: pick one.\n✅ ❌ Net: tradeoff. Pros / cons\n\nA) bun test (recommended)\n\nB) vitest", - "two-recommended": "Scope\n\nRecommendation: A because it is installed.\n\nA) bun test (recommended)\n\nB) vitest (recommended)", - "no-recommended-option": "Scope\n\nRecommendation: A because it is installed.\n\nA) bun test\n\nB) vitest", - "empty-recommendation": "Scope\n\nRecommendation:\n\nA) bun test (recommended)\n\nB) vitest" + "no-recommendation-line": { + "header": "Scope", + "question": "D1 — Which test harness?\nELI10: pick one.\n✅ ❌ Net: tradeoff. Pros / cons", + "options": [ + { + "label": "A) bun test (recommended)" + }, + { + "label": "B) vitest" + } + ] + }, + "two-recommended": { + "header": "Scope", + "question": "Recommendation: A because it is installed.", + "options": [ + { + "label": "A) bun test (recommended)" + }, + { + "label": "B) vitest (recommended)" + } + ] + }, + "no-recommended-option": { + "header": "Scope", + "question": "Recommendation: A because it is installed.", + "options": [ + { + "label": "A) bun test" + }, + { + "label": "B) vitest" + } + ] + }, + "recommended-only-in-question-text": { + "header": "Scope", + "question": "Recommendation: A because it is installed.\nPros / cons:\nA) bun test (recommended)\n ✅ already installed\nB) vitest\n ✅ plugins", + "options": [ + { + "label": "A) bun test" + }, + { + "label": "B) vitest" + } + ] + }, + "empty-recommendation": { + "header": "Scope", + "question": "Recommendation:", + "options": [ + { + "label": "A) bun test (recommended)" + }, + { + "label": "B) vitest" + } + ] + } } } diff --git a/test/helpers/auq-sdk-capture.ts b/test/helpers/auq-sdk-capture.ts index 5713c73a1..6fa716d19 100644 --- a/test/helpers/auq-sdk-capture.ts +++ b/test/helpers/auq-sdk-capture.ts @@ -44,14 +44,18 @@ export function scoreAuqFormat(text: string): { present: number; total: number; /** * Format problems that fail a first-question matrix run: only the fields - * software reads (a `Recommendation:` line and exactly one `(recommended)` - * option). ELI10, Pros / cons, ✅/❌ and Net: are reported, not failed. + * software reads. The question text needs a `Recommendation:` line, and + * exactly one option label ends in `(recommended)`, the suffix the AUTO_DECIDE + * hook parses from labels (hosts/claude/hooks/question-preference-hook.ts). + * The format's `Pros / cons:` block in the question text repeats the marker + * by design, so it is not counted. ELI10, Pros / cons, ✅/❌ and Net: are + * reported, not failed. */ -export function auqMachineFormatProblems(text: string): string[] { +export function auqMachineFormatProblems(question: NativePlanQuestion): string[] { const problems: string[] = []; - if (!/^[*_]*[ \t]*recommendation[ \t]*[*_]*[ \t]*:[*_ \t]*\S/im.test(text)) problems.push('missing Recommendation: line'); - const recommended = text.match(/\(recommended\)/gi)?.length ?? 0; - if (recommended !== 1) problems.push(`expected exactly one (recommended) option, found ${recommended}`); + if (!/^[*_]*[ \t]*recommendation[ \t]*[*_]*[ \t]*:[*_ \t]*\S/im.test(question.question)) problems.push('missing Recommendation: line'); + const recommended = question.options.filter(option => /\(recommended\)\s*$/i.test(option.label)).length; + if (recommended !== 1) problems.push(`expected exactly one (recommended) option label, found ${recommended}`); return problems; } diff --git a/test/skill-e2e-auq-matrix.test.ts b/test/skill-e2e-auq-matrix.test.ts index 0eb699a0d..eb2985712 100644 --- a/test/skill-e2e-auq-matrix.test.ts +++ b/test/skill-e2e-auq-matrix.test.ts @@ -32,9 +32,9 @@ import { resolveEvalModel } from '../lib/eval-model'; import { CAPTURE_MS } from './helpers/eval-budgets'; import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'node:fs'; +import { captureNativeFirstAuq, type NativeAuqCapture } from './helpers/auq-native-capture'; import { setupSkillDir, - captureFirstAuq, scoreAuqFormat, auqMachineFormatProblems, skillFromWorktree, @@ -123,9 +123,9 @@ describeE2E('AUQ behavioral matrix (periodic)', () => { fixtures: m.fixtures, tmpPrefix: `auq-matrix-${m.skill}-`, }); - let text = ''; + let capture: NativeAuqCapture; try { - text = await captureFirstAuq({ + capture = await captureNativeFirstAuq({ planDir: dir, skillName: m.skill, scenario: m.scenario, @@ -137,6 +137,7 @@ describeE2E('AUQ behavioral matrix (periodic)', () => { fs.rmSync(dir, { recursive: true, force: true }); } + const text = capture.text; const fmt = scoreAuqFormat(text); let substance = 0; let recPresent = false; @@ -159,7 +160,7 @@ describeE2E('AUQ behavioral matrix (periodic)', () => { } const problems: string[] = []; // Presentation elements (ELI10, Pros / cons, ✅/❌, Net:) are logged above, not failed. - problems.push(...auqMachineFormatProblems(text)); + problems.push(...auqMachineFormatProblems(capture.question)); if (substance < 4) problems.push(`recommendation substance ${substance} < 4 (boilerplate/weak)`); if (problems.length > 0) { throw new Error(