C4: score auq-matrix substance with a 3-sample judge panel

Each skill keeps one native capture in the same execution; the
recommendation-substance grader now runs as a JUDGE_PANEL_SAMPLES panel on
that capture and gates on the mean against the unchanged minimum of 4. A
failed sample fails the panel without resampling. Approved at the gate
2026-10-04 for censuses after this change.

test/auq-substance-panel.test.ts

@@ -0,0 +1,49 @@
+import { describe, test, expect } from 'bun:test';
+import * as fs from 'fs';
+import * as path from 'path';
+import { auqSubstancePanel, AUQ_SUBSTANCE_MIN } from './helpers/auq-substance-panel';
+import { JUDGE_PANEL_SAMPLES } from './helpers/llm-judge';
+
+const grader = (scores: number[]) => {
+  const seen: string[] = [];
+  let index = 0;
+  const grade = async (text: string) => {
+    seen.push(text);
+    const substance = scores[index++];
+    if (substance === undefined) throw new Error('grader called more than the panel size');
+    return { substance, present: true, hadLiteralBecause: true, reason: 'r' };
+  };
+  return { grade, seen };
+};
+
+describe('auq-matrix substance panel (C4)', () => {
+  test('one capture is graded by a full panel and gates on the mean at the unchanged minimum', async () => {
+    expect(AUQ_SUBSTANCE_MIN).toBe(4);
+    const { grade, seen } = grader([5, 3, 4]);
+    const panel = await auqSubstancePanel('Recommendation: A because B', grade);
+    expect(seen).toEqual(Array(JUDGE_PANEL_SAMPLES).fill('Recommendation: A because B'));
+    expect(panel).toMatchObject({ mean: 4, passed: true });
+  });
+  test('a single low sample no longer decides the case, but a low mean still fails', async () => {
+    expect((await auqSubstancePanel('q', grader([4, 3, 5]).grade)).passed).toBe(true);
+    expect((await auqSubstancePanel('q', grader([4, 3, 4]).grade))).toMatchObject({ passed: false });
+    expect((await auqSubstancePanel('q', grader([3, 3, 3]).grade)).mean).toBe(3);
+  });
+  test('a failed sample fails the panel without resampling', async () => {
+    let calls = 0;
+    const grade = async () => {
+      calls++;
+      if (calls === 2) throw new Error('judge returned non-JSON');
+      return { substance: 5, present: true, hadLiteralBecause: true, reason: 'r' };
+    };
+    await expect(auqSubstancePanel('q', grade)).rejects.toThrow('non-JSON');
+    expect(calls).toBe(JUDGE_PANEL_SAMPLES);
+  });
+  test('the matrix scores its single capture through the panel', () => {
+    const source = fs.readFileSync(path.join(import.meta.dir, 'skill-e2e-auq-matrix.test.ts'), 'utf8');
+    expect(source.match(/captureNativeFirstAuq\(/g)).toHaveLength(1);
+    expect(source).toContain('auqSubstancePanel(text)');
+    expect(source).not.toMatch(/gradeAuqRecommendation\(/);
+    expect(source).toContain('substance < AUQ_SUBSTANCE_MIN');
+  });
+});

test/helpers/auq-substance-panel.ts

@@ -0,0 +1,18 @@
+/**
+ * auq-matrix recommendation substance (C4, approved 2026-10-04): one native
+ * capture per skill, scored by a JUDGE_PANEL_SAMPLES panel of the same grader
+ * on that capture. The panel mean gates against the unchanged minimum; a sample
+ * error fails the panel and is never resampled (judgePanel).
+ */
+import { gradeAuqRecommendation } from './auq-sdk-capture';
+import { judgePanel, judgePanelMean } from './llm-judge';
+
+export const AUQ_SUBSTANCE_MIN = 4;
+
+type Grade = Awaited<ReturnType<typeof gradeAuqRecommendation>>;
+
+export async function auqSubstancePanel(text: string, grade: (text: string) => Promise<Grade> = gradeAuqRecommendation) {
+  const samples = await judgePanel(() => grade(text));
+  const mean = judgePanelMean(samples, ['substance']).substance;
+  return { samples, mean, passed: mean >= AUQ_SUBSTANCE_MIN };
+}

test/skill-e2e-auq-matrix.test.ts

@@ -6,7 +6,8 @@
  * listed skill SHIPS the format spec in its always-loaded skeleton. This test
  * proves each skill's model OBEYS it: that the first real AUQ it fires carries
  * the fields software reads (a Recommendation: line and exactly one
- * (recommended) option) with a substantive recommendation (>= 4). The other
+ * (recommended) option) with a substantive recommendation (3-sample judge panel
+ * mean >= 4 on the one capture; C4). The other
  * format elements are logged. One parametrized case per skill so a single weak skill
  * is an isolated failure, not a blocker for the rest.
  *
@@ -38,8 +39,8 @@ import {
   scoreAuqFormat,
   auqMachineFormatProblems,
   skillFromWorktree,
-  gradeAuqRecommendation,
 } from './helpers/auq-sdk-capture';
+import { auqSubstancePanel, AUQ_SUBSTANCE_MIN } from './helpers/auq-substance-panel';
 
 const describeE2E = describeE2ETier('periodic');
 const runId = `auq-matrix-${process.env.EVALS_RUN_ID ?? 'local'}`;
@@ -140,18 +141,20 @@ describeE2E('AUQ behavioral matrix (periodic)', () => {
         const text = capture.text;
         const fmt = scoreAuqFormat(text);
         let substance = 0;
+        let samples: number[] = [];
         let recPresent = false;
         let hadBecause = false;
         if (text.trim()) {
-          const g = await gradeAuqRecommendation(text);
-          substance = g.substance;
-          recPresent = g.present;
-          hadBecause = g.hadLiteralBecause;
+          const panel = await auqSubstancePanel(text);
+          substance = panel.mean;
+          samples = panel.samples.map(g => g.substance);
+          recPresent = panel.samples[0].present;
+          hadBecause = panel.samples[0].hadLiteralBecause;
         }
         // eslint-disable-next-line no-console
         console.log(
           `[AUQ-matrix ${m.skill}] captured=${text.length}B format=${fmt.present}/${fmt.total} ` +
-            `missing=[${fmt.missing.join(',')}] recPresent=${recPresent} substance=${substance} ` +
+            `missing=[${fmt.missing.join(',')}] recPresent=${recPresent} substance=${substance.toFixed(2)} [${samples.join(',')}] ` +
             `literalBecause=${hadBecause}`,
         );
 
@@ -161,7 +164,9 @@ describeE2E('AUQ behavioral matrix (periodic)', () => {
         const problems: string[] = [];
         // Presentation elements (ELI10, Pros / cons, ✅/❌, Net:) are logged above, not failed.
         problems.push(...auqMachineFormatProblems(capture.question));
-        if (substance < 4) problems.push(`recommendation substance ${substance} < 4 (boilerplate/weak)`);
+        if (substance < AUQ_SUBSTANCE_MIN) {
+          problems.push(`recommendation substance mean ${substance.toFixed(2)} < ${AUQ_SUBSTANCE_MIN} over samples [${samples.join(',')}] (boilerplate/weak)`);
+        }
         if (problems.length > 0) {
           throw new Error(
             `${m.skill} AUQ not at plan-ceo bar:\n  - ${problems.join('\n  - ')}\n--- captured AUQ ---\n${text}`,