fix(evals): AUQ A/B vendored pre-cut arm + judge-error inconclusive taxonomy

- The PRE arm read a branch-local SHA (3263fffe) that becomes unreachable on
  fresh clones after the squash-merge; the pre-cut render is now a vendored
  fixture.
- A judge failure on one side no longer coerces substance to 0 (which
  fabricated DEGRADATION on POST-side failures and masked regressions on
  PRE-side failures): null substance = inconclusive, format still gates.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-08-29 05:18:52 +00:00
co-authored by Claude Fable 5
parent 83de97e152
commit 4d5af0efe4
2 changed files with 1153 additions and 12 deletions
File diff suppressed because it is too large Load Diff
+26 -12
View File
@@ -15,41 +15,48 @@
* NOT WORSE than the pre-cut AUQ on format elements and recommendation
* substance. Same harness and bar as skill-e2e-auq-verbose-vs-carved-ab.
*
* - PRE : plan-ceo-review/SKILL.md read from git at 3263fffe (the last
* commit before the cut), with the current sections/ (the cut
* touched only the preamble skeleton, not the sections).
* - PRE : the pre-cut plan-ceo-review/SKILL.md render, vendored at
* test/fixtures/auq-pre-cut-plan-ceo-review-SKILL.md (captured
* from branch commit 3263fffe, the last commit before the cut —
* vendored because that SHA is branch-local and unreachable from
* fresh clones after the squash-merge), with the current
* sections/ (the cut touched only the preamble skeleton).
* - POST : this worktree's render.
*/
import { test } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs';
import * as path from 'node:path';
import {
setupPlanCeoDir,
captureModeSelectionAuq,
scoreAuqFormat,
carvedSkill,
verboseSkill,
} from './helpers/auq-sdk-capture';
import { judgeRecommendation } from './helpers/llm-judge';
const describeE2E = describeE2ETier('periodic');
const runId = `auq-cut-ab-${process.env.EVALS_RUN_ID ?? 'local'}`;
const PRE_CUT_REF = '3263fffe';
const PRE_CUT_FIXTURE = path.join(import.meta.dir, 'fixtures', 'auq-pre-cut-plan-ceo-review-SKILL.md');
async function grade(label: string, dir: string) {
const text = await captureModeSelectionAuq({ planDir: dir, testName: `auq-cut-ab-${label}`, runId });
const fmt = scoreAuqFormat(text);
let substance = 0;
// null = judge unavailable. Never coerced to 0: a transient judge failure
// on one side must read as INCONCLUSIVE, not as a fabricated degradation
// (POST-side failure) or a masked regression (PRE-side failure) — same
// taxonomy as armJudge's judge_error cells.
let substance: number | null = null;
if (text.trim()) {
try {
const r = await judgeRecommendation(text);
substance = r.reason_substance;
} catch { /* judge unavailable */ }
} catch { /* judge unavailable — recorded as null */ }
}
// eslint-disable-next-line no-console
console.log(
`[AUQ-CUT-AB ${label}] captured=${text.length}B format=${fmt.present}/${fmt.total} ` +
`missing=[${fmt.missing.join(',')}] substance=${substance}`,
`missing=[${fmt.missing.join(',')}] substance=${substance ?? 'inconclusive'}`,
);
return { text, fmt, substance };
}
@@ -65,7 +72,7 @@ describeE2E('AUQ no-degradation: repetition cut (periodic)', () => {
tmpPrefix: 'auq-cut-ab-post-',
});
const preDir = setupPlanCeoDir({
skillMd: verboseSkill(PRE_CUT_REF),
skillMd: fs.readFileSync(PRE_CUT_FIXTURE, 'utf-8'),
sectionsFrom: post.sectionsFrom,
tmpPrefix: 'auq-cut-ab-pre-',
});
@@ -80,8 +87,8 @@ describeE2E('AUQ no-degradation: repetition cut (periodic)', () => {
}
const summary = [
`POST: format ${q.fmt.present}/${q.fmt.total}, substance ${q.substance}`,
`PRE : format ${p.fmt.present}/${p.fmt.total}, substance ${p.substance}`,
`POST: format ${q.fmt.present}/${q.fmt.total}, substance ${q.substance ?? 'inconclusive'}`,
`PRE : format ${p.fmt.present}/${p.fmt.total}, substance ${p.substance ?? 'inconclusive'}`,
].join('\n');
if (!q.text.trim() || !p.text.trim()) {
@@ -92,7 +99,14 @@ describeE2E('AUQ no-degradation: repetition cut (periodic)', () => {
}
const formatRegressed = q.fmt.present < p.fmt.present;
const substanceRegressed = q.substance < p.substance - 1; // 1-pt judge tolerance
// Substance compares only when BOTH judge calls succeeded; a null on
// either side logs as inconclusive and the format comparison still gates.
const substanceComparable = q.substance !== null && p.substance !== null;
if (!substanceComparable) {
// eslint-disable-next-line no-console
console.log('[AUQ-CUT-AB] substance inconclusive (judge error on at least one side) — format elements still compared.');
}
const substanceRegressed = substanceComparable && q.substance! < p.substance! - 1; // 1-pt judge tolerance
if (formatRegressed || substanceRegressed) {
throw new Error(
`AUQ DEGRADATION from the repetition cut — the gate outranks the approval; revert the cut:\n${summary}` +