From e9cc336c7e6839c7041f9dbd36900ed1d31d2ab5 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sun, 4 Oct 2026 17:23:57 +0000 Subject: [PATCH] C4: score auq-matrix substance with a 3-sample judge panel Each skill keeps one native capture in the same execution; the recommendation-substance grader now runs as a JUDGE_PANEL_SAMPLES panel on that capture and gates on the mean against the unchanged minimum of 4. A failed sample fails the panel without resampling. Approved at the gate 2026-10-04 for censuses after this change. --- test/auq-substance-panel.test.ts | 49 +++++++++++++++++++++++++++++ test/helpers/auq-substance-panel.ts | 18 +++++++++++ test/skill-e2e-auq-matrix.test.ts | 21 ++++++++----- 3 files changed, 80 insertions(+), 8 deletions(-) create mode 100644 test/auq-substance-panel.test.ts create mode 100644 test/helpers/auq-substance-panel.ts diff --git a/test/auq-substance-panel.test.ts b/test/auq-substance-panel.test.ts new file mode 100644 index 000000000..115b119d4 --- /dev/null +++ b/test/auq-substance-panel.test.ts @@ -0,0 +1,49 @@ +import { describe, test, expect } from 'bun:test'; +import * as fs from 'fs'; +import * as path from 'path'; +import { auqSubstancePanel, AUQ_SUBSTANCE_MIN } from './helpers/auq-substance-panel'; +import { JUDGE_PANEL_SAMPLES } from './helpers/llm-judge'; + +const grader = (scores: number[]) => { + const seen: string[] = []; + let index = 0; + const grade = async (text: string) => { + seen.push(text); + const substance = scores[index++]; + if (substance === undefined) throw new Error('grader called more than the panel size'); + return { substance, present: true, hadLiteralBecause: true, reason: 'r' }; + }; + return { grade, seen }; +}; + +describe('auq-matrix substance panel (C4)', () => { + test('one capture is graded by a full panel and gates on the mean at the unchanged minimum', async () => { + expect(AUQ_SUBSTANCE_MIN).toBe(4); + const { grade, seen } = grader([5, 3, 4]); + const panel = await auqSubstancePanel('Recommendation: A because B', grade); + expect(seen).toEqual(Array(JUDGE_PANEL_SAMPLES).fill('Recommendation: A because B')); + expect(panel).toMatchObject({ mean: 4, passed: true }); + }); + test('a single low sample no longer decides the case, but a low mean still fails', async () => { + expect((await auqSubstancePanel('q', grader([4, 3, 5]).grade)).passed).toBe(true); + expect((await auqSubstancePanel('q', grader([4, 3, 4]).grade))).toMatchObject({ passed: false }); + expect((await auqSubstancePanel('q', grader([3, 3, 3]).grade)).mean).toBe(3); + }); + test('a failed sample fails the panel without resampling', async () => { + let calls = 0; + const grade = async () => { + calls++; + if (calls === 2) throw new Error('judge returned non-JSON'); + return { substance: 5, present: true, hadLiteralBecause: true, reason: 'r' }; + }; + await expect(auqSubstancePanel('q', grade)).rejects.toThrow('non-JSON'); + expect(calls).toBe(JUDGE_PANEL_SAMPLES); + }); + test('the matrix scores its single capture through the panel', () => { + const source = fs.readFileSync(path.join(import.meta.dir, 'skill-e2e-auq-matrix.test.ts'), 'utf8'); + expect(source.match(/captureNativeFirstAuq\(/g)).toHaveLength(1); + expect(source).toContain('auqSubstancePanel(text)'); + expect(source).not.toMatch(/gradeAuqRecommendation\(/); + expect(source).toContain('substance < AUQ_SUBSTANCE_MIN'); + }); +}); diff --git a/test/helpers/auq-substance-panel.ts b/test/helpers/auq-substance-panel.ts new file mode 100644 index 000000000..85d785f25 --- /dev/null +++ b/test/helpers/auq-substance-panel.ts @@ -0,0 +1,18 @@ +/** + * auq-matrix recommendation substance (C4, approved 2026-10-04): one native + * capture per skill, scored by a JUDGE_PANEL_SAMPLES panel of the same grader + * on that capture. The panel mean gates against the unchanged minimum; a sample + * error fails the panel and is never resampled (judgePanel). + */ +import { gradeAuqRecommendation } from './auq-sdk-capture'; +import { judgePanel, judgePanelMean } from './llm-judge'; + +export const AUQ_SUBSTANCE_MIN = 4; + +type Grade = Awaited>; + +export async function auqSubstancePanel(text: string, grade: (text: string) => Promise = gradeAuqRecommendation) { + const samples = await judgePanel(() => grade(text)); + const mean = judgePanelMean(samples, ['substance']).substance; + return { samples, mean, passed: mean >= AUQ_SUBSTANCE_MIN }; +} diff --git a/test/skill-e2e-auq-matrix.test.ts b/test/skill-e2e-auq-matrix.test.ts index eb2985712..1f2205ab9 100644 --- a/test/skill-e2e-auq-matrix.test.ts +++ b/test/skill-e2e-auq-matrix.test.ts @@ -6,7 +6,8 @@ * listed skill SHIPS the format spec in its always-loaded skeleton. This test * proves each skill's model OBEYS it: that the first real AUQ it fires carries * the fields software reads (a Recommendation: line and exactly one - * (recommended) option) with a substantive recommendation (>= 4). The other + * (recommended) option) with a substantive recommendation (3-sample judge panel + * mean >= 4 on the one capture; C4). The other * format elements are logged. One parametrized case per skill so a single weak skill * is an isolated failure, not a blocker for the rest. * @@ -38,8 +39,8 @@ import { scoreAuqFormat, auqMachineFormatProblems, skillFromWorktree, - gradeAuqRecommendation, } from './helpers/auq-sdk-capture'; +import { auqSubstancePanel, AUQ_SUBSTANCE_MIN } from './helpers/auq-substance-panel'; const describeE2E = describeE2ETier('periodic'); const runId = `auq-matrix-${process.env.EVALS_RUN_ID ?? 'local'}`; @@ -140,18 +141,20 @@ describeE2E('AUQ behavioral matrix (periodic)', () => { const text = capture.text; const fmt = scoreAuqFormat(text); let substance = 0; + let samples: number[] = []; let recPresent = false; let hadBecause = false; if (text.trim()) { - const g = await gradeAuqRecommendation(text); - substance = g.substance; - recPresent = g.present; - hadBecause = g.hadLiteralBecause; + const panel = await auqSubstancePanel(text); + substance = panel.mean; + samples = panel.samples.map(g => g.substance); + recPresent = panel.samples[0].present; + hadBecause = panel.samples[0].hadLiteralBecause; } // eslint-disable-next-line no-console console.log( `[AUQ-matrix ${m.skill}] captured=${text.length}B format=${fmt.present}/${fmt.total} ` + - `missing=[${fmt.missing.join(',')}] recPresent=${recPresent} substance=${substance} ` + + `missing=[${fmt.missing.join(',')}] recPresent=${recPresent} substance=${substance.toFixed(2)} [${samples.join(',')}] ` + `literalBecause=${hadBecause}`, ); @@ -161,7 +164,9 @@ describeE2E('AUQ behavioral matrix (periodic)', () => { const problems: string[] = []; // Presentation elements (ELI10, Pros / cons, ✅/❌, Net:) are logged above, not failed. problems.push(...auqMachineFormatProblems(capture.question)); - if (substance < 4) problems.push(`recommendation substance ${substance} < 4 (boilerplate/weak)`); + if (substance < AUQ_SUBSTANCE_MIN) { + problems.push(`recommendation substance mean ${substance.toFixed(2)} < ${AUQ_SUBSTANCE_MIN} over samples [${samples.join(',')}] (boilerplate/weak)`); + } if (problems.length > 0) { throw new Error( `${m.skill} AUQ not at plan-ceo bar:\n - ${problems.join('\n - ')}\n--- captured AUQ ---\n${text}`,