mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-09 14:38:59 +02:00
fix(evals): AUQ A/B vendored pre-cut arm + judge-error inconclusive taxonomy
- The PRE arm read a branch-local SHA (3263fffe) that becomes unreachable on
fresh clones after the squash-merge; the pre-cut render is now a vendored
fixture.
- A judge failure on one side no longer coerces substance to 0 (which
fabricated DEGRADATION on POST-side failures and masked regressions on
PRE-side failures): null substance = inconclusive, format still gates.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
83de97e152
commit
4d5af0efe4
+1127
File diff suppressed because it is too large
Load Diff
@@ -15,41 +15,48 @@
|
||||
* NOT WORSE than the pre-cut AUQ on format elements and recommendation
|
||||
* substance. Same harness and bar as skill-e2e-auq-verbose-vs-carved-ab.
|
||||
*
|
||||
* - PRE : plan-ceo-review/SKILL.md read from git at 3263fffe (the last
|
||||
* commit before the cut), with the current sections/ (the cut
|
||||
* touched only the preamble skeleton, not the sections).
|
||||
* - PRE : the pre-cut plan-ceo-review/SKILL.md render, vendored at
|
||||
* test/fixtures/auq-pre-cut-plan-ceo-review-SKILL.md (captured
|
||||
* from branch commit 3263fffe, the last commit before the cut —
|
||||
* vendored because that SHA is branch-local and unreachable from
|
||||
* fresh clones after the squash-merge), with the current
|
||||
* sections/ (the cut touched only the preamble skeleton).
|
||||
* - POST : this worktree's render.
|
||||
*/
|
||||
import { test } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import {
|
||||
setupPlanCeoDir,
|
||||
captureModeSelectionAuq,
|
||||
scoreAuqFormat,
|
||||
carvedSkill,
|
||||
verboseSkill,
|
||||
} from './helpers/auq-sdk-capture';
|
||||
import { judgeRecommendation } from './helpers/llm-judge';
|
||||
|
||||
const describeE2E = describeE2ETier('periodic');
|
||||
const runId = `auq-cut-ab-${process.env.EVALS_RUN_ID ?? 'local'}`;
|
||||
const PRE_CUT_REF = '3263fffe';
|
||||
const PRE_CUT_FIXTURE = path.join(import.meta.dir, 'fixtures', 'auq-pre-cut-plan-ceo-review-SKILL.md');
|
||||
|
||||
async function grade(label: string, dir: string) {
|
||||
const text = await captureModeSelectionAuq({ planDir: dir, testName: `auq-cut-ab-${label}`, runId });
|
||||
const fmt = scoreAuqFormat(text);
|
||||
let substance = 0;
|
||||
// null = judge unavailable. Never coerced to 0: a transient judge failure
|
||||
// on one side must read as INCONCLUSIVE, not as a fabricated degradation
|
||||
// (POST-side failure) or a masked regression (PRE-side failure) — same
|
||||
// taxonomy as armJudge's judge_error cells.
|
||||
let substance: number | null = null;
|
||||
if (text.trim()) {
|
||||
try {
|
||||
const r = await judgeRecommendation(text);
|
||||
substance = r.reason_substance;
|
||||
} catch { /* judge unavailable */ }
|
||||
} catch { /* judge unavailable — recorded as null */ }
|
||||
}
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(
|
||||
`[AUQ-CUT-AB ${label}] captured=${text.length}B format=${fmt.present}/${fmt.total} ` +
|
||||
`missing=[${fmt.missing.join(',')}] substance=${substance}`,
|
||||
`missing=[${fmt.missing.join(',')}] substance=${substance ?? 'inconclusive'}`,
|
||||
);
|
||||
return { text, fmt, substance };
|
||||
}
|
||||
@@ -65,7 +72,7 @@ describeE2E('AUQ no-degradation: repetition cut (periodic)', () => {
|
||||
tmpPrefix: 'auq-cut-ab-post-',
|
||||
});
|
||||
const preDir = setupPlanCeoDir({
|
||||
skillMd: verboseSkill(PRE_CUT_REF),
|
||||
skillMd: fs.readFileSync(PRE_CUT_FIXTURE, 'utf-8'),
|
||||
sectionsFrom: post.sectionsFrom,
|
||||
tmpPrefix: 'auq-cut-ab-pre-',
|
||||
});
|
||||
@@ -80,8 +87,8 @@ describeE2E('AUQ no-degradation: repetition cut (periodic)', () => {
|
||||
}
|
||||
|
||||
const summary = [
|
||||
`POST: format ${q.fmt.present}/${q.fmt.total}, substance ${q.substance}`,
|
||||
`PRE : format ${p.fmt.present}/${p.fmt.total}, substance ${p.substance}`,
|
||||
`POST: format ${q.fmt.present}/${q.fmt.total}, substance ${q.substance ?? 'inconclusive'}`,
|
||||
`PRE : format ${p.fmt.present}/${p.fmt.total}, substance ${p.substance ?? 'inconclusive'}`,
|
||||
].join('\n');
|
||||
|
||||
if (!q.text.trim() || !p.text.trim()) {
|
||||
@@ -92,7 +99,14 @@ describeE2E('AUQ no-degradation: repetition cut (periodic)', () => {
|
||||
}
|
||||
|
||||
const formatRegressed = q.fmt.present < p.fmt.present;
|
||||
const substanceRegressed = q.substance < p.substance - 1; // 1-pt judge tolerance
|
||||
// Substance compares only when BOTH judge calls succeeded; a null on
|
||||
// either side logs as inconclusive and the format comparison still gates.
|
||||
const substanceComparable = q.substance !== null && p.substance !== null;
|
||||
if (!substanceComparable) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('[AUQ-CUT-AB] substance inconclusive (judge error on at least one side) — format elements still compared.');
|
||||
}
|
||||
const substanceRegressed = substanceComparable && q.substance! < p.substance! - 1; // 1-pt judge tolerance
|
||||
if (formatRegressed || substanceRegressed) {
|
||||
throw new Error(
|
||||
`AUQ DEGRADATION from the repetition cut — the gate outranks the approval; revert the cut:\n${summary}` +
|
||||
|
||||
Reference in New Issue
Block a user