mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-17 18:32:19 +02:00
fix(evals): AUQ A/B vendored pre-cut arm + judge-error inconclusive taxonomy
- The PRE arm read a branch-local SHA (3263fffe) that becomes unreachable on
fresh clones after the squash-merge; the pre-cut render is now a vendored
fixture.
- A judge failure on one side no longer coerces substance to 0 (which
fabricated DEGRADATION on POST-side failures and masked regressions on
PRE-side failures): null substance = inconclusive, format still gates.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
83de97e152
commit
4d5af0efe4
+1127
File diff suppressed because it is too large
Load Diff
@@ -15,41 +15,48 @@
|
|||||||
* NOT WORSE than the pre-cut AUQ on format elements and recommendation
|
* NOT WORSE than the pre-cut AUQ on format elements and recommendation
|
||||||
* substance. Same harness and bar as skill-e2e-auq-verbose-vs-carved-ab.
|
* substance. Same harness and bar as skill-e2e-auq-verbose-vs-carved-ab.
|
||||||
*
|
*
|
||||||
* - PRE : plan-ceo-review/SKILL.md read from git at 3263fffe (the last
|
* - PRE : the pre-cut plan-ceo-review/SKILL.md render, vendored at
|
||||||
* commit before the cut), with the current sections/ (the cut
|
* test/fixtures/auq-pre-cut-plan-ceo-review-SKILL.md (captured
|
||||||
* touched only the preamble skeleton, not the sections).
|
* from branch commit 3263fffe, the last commit before the cut —
|
||||||
|
* vendored because that SHA is branch-local and unreachable from
|
||||||
|
* fresh clones after the squash-merge), with the current
|
||||||
|
* sections/ (the cut touched only the preamble skeleton).
|
||||||
* - POST : this worktree's render.
|
* - POST : this worktree's render.
|
||||||
*/
|
*/
|
||||||
import { test } from 'bun:test';
|
import { test } from 'bun:test';
|
||||||
import { describeE2ETier } from './helpers/e2e-gate';
|
import { describeE2ETier } from './helpers/e2e-gate';
|
||||||
import * as fs from 'node:fs';
|
import * as fs from 'node:fs';
|
||||||
|
import * as path from 'node:path';
|
||||||
import {
|
import {
|
||||||
setupPlanCeoDir,
|
setupPlanCeoDir,
|
||||||
captureModeSelectionAuq,
|
captureModeSelectionAuq,
|
||||||
scoreAuqFormat,
|
scoreAuqFormat,
|
||||||
carvedSkill,
|
carvedSkill,
|
||||||
verboseSkill,
|
|
||||||
} from './helpers/auq-sdk-capture';
|
} from './helpers/auq-sdk-capture';
|
||||||
import { judgeRecommendation } from './helpers/llm-judge';
|
import { judgeRecommendation } from './helpers/llm-judge';
|
||||||
|
|
||||||
const describeE2E = describeE2ETier('periodic');
|
const describeE2E = describeE2ETier('periodic');
|
||||||
const runId = `auq-cut-ab-${process.env.EVALS_RUN_ID ?? 'local'}`;
|
const runId = `auq-cut-ab-${process.env.EVALS_RUN_ID ?? 'local'}`;
|
||||||
const PRE_CUT_REF = '3263fffe';
|
const PRE_CUT_FIXTURE = path.join(import.meta.dir, 'fixtures', 'auq-pre-cut-plan-ceo-review-SKILL.md');
|
||||||
|
|
||||||
async function grade(label: string, dir: string) {
|
async function grade(label: string, dir: string) {
|
||||||
const text = await captureModeSelectionAuq({ planDir: dir, testName: `auq-cut-ab-${label}`, runId });
|
const text = await captureModeSelectionAuq({ planDir: dir, testName: `auq-cut-ab-${label}`, runId });
|
||||||
const fmt = scoreAuqFormat(text);
|
const fmt = scoreAuqFormat(text);
|
||||||
let substance = 0;
|
// null = judge unavailable. Never coerced to 0: a transient judge failure
|
||||||
|
// on one side must read as INCONCLUSIVE, not as a fabricated degradation
|
||||||
|
// (POST-side failure) or a masked regression (PRE-side failure) — same
|
||||||
|
// taxonomy as armJudge's judge_error cells.
|
||||||
|
let substance: number | null = null;
|
||||||
if (text.trim()) {
|
if (text.trim()) {
|
||||||
try {
|
try {
|
||||||
const r = await judgeRecommendation(text);
|
const r = await judgeRecommendation(text);
|
||||||
substance = r.reason_substance;
|
substance = r.reason_substance;
|
||||||
} catch { /* judge unavailable */ }
|
} catch { /* judge unavailable — recorded as null */ }
|
||||||
}
|
}
|
||||||
// eslint-disable-next-line no-console
|
// eslint-disable-next-line no-console
|
||||||
console.log(
|
console.log(
|
||||||
`[AUQ-CUT-AB ${label}] captured=${text.length}B format=${fmt.present}/${fmt.total} ` +
|
`[AUQ-CUT-AB ${label}] captured=${text.length}B format=${fmt.present}/${fmt.total} ` +
|
||||||
`missing=[${fmt.missing.join(',')}] substance=${substance}`,
|
`missing=[${fmt.missing.join(',')}] substance=${substance ?? 'inconclusive'}`,
|
||||||
);
|
);
|
||||||
return { text, fmt, substance };
|
return { text, fmt, substance };
|
||||||
}
|
}
|
||||||
@@ -65,7 +72,7 @@ describeE2E('AUQ no-degradation: repetition cut (periodic)', () => {
|
|||||||
tmpPrefix: 'auq-cut-ab-post-',
|
tmpPrefix: 'auq-cut-ab-post-',
|
||||||
});
|
});
|
||||||
const preDir = setupPlanCeoDir({
|
const preDir = setupPlanCeoDir({
|
||||||
skillMd: verboseSkill(PRE_CUT_REF),
|
skillMd: fs.readFileSync(PRE_CUT_FIXTURE, 'utf-8'),
|
||||||
sectionsFrom: post.sectionsFrom,
|
sectionsFrom: post.sectionsFrom,
|
||||||
tmpPrefix: 'auq-cut-ab-pre-',
|
tmpPrefix: 'auq-cut-ab-pre-',
|
||||||
});
|
});
|
||||||
@@ -80,8 +87,8 @@ describeE2E('AUQ no-degradation: repetition cut (periodic)', () => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const summary = [
|
const summary = [
|
||||||
`POST: format ${q.fmt.present}/${q.fmt.total}, substance ${q.substance}`,
|
`POST: format ${q.fmt.present}/${q.fmt.total}, substance ${q.substance ?? 'inconclusive'}`,
|
||||||
`PRE : format ${p.fmt.present}/${p.fmt.total}, substance ${p.substance}`,
|
`PRE : format ${p.fmt.present}/${p.fmt.total}, substance ${p.substance ?? 'inconclusive'}`,
|
||||||
].join('\n');
|
].join('\n');
|
||||||
|
|
||||||
if (!q.text.trim() || !p.text.trim()) {
|
if (!q.text.trim() || !p.text.trim()) {
|
||||||
@@ -92,7 +99,14 @@ describeE2E('AUQ no-degradation: repetition cut (periodic)', () => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const formatRegressed = q.fmt.present < p.fmt.present;
|
const formatRegressed = q.fmt.present < p.fmt.present;
|
||||||
const substanceRegressed = q.substance < p.substance - 1; // 1-pt judge tolerance
|
// Substance compares only when BOTH judge calls succeeded; a null on
|
||||||
|
// either side logs as inconclusive and the format comparison still gates.
|
||||||
|
const substanceComparable = q.substance !== null && p.substance !== null;
|
||||||
|
if (!substanceComparable) {
|
||||||
|
// eslint-disable-next-line no-console
|
||||||
|
console.log('[AUQ-CUT-AB] substance inconclusive (judge error on at least one side) — format elements still compared.');
|
||||||
|
}
|
||||||
|
const substanceRegressed = substanceComparable && q.substance! < p.substance! - 1; // 1-pt judge tolerance
|
||||||
if (formatRegressed || substanceRegressed) {
|
if (formatRegressed || substanceRegressed) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`AUQ DEGRADATION from the repetition cut — the gate outranks the approval; revert the cut:\n${summary}` +
|
`AUQ DEGRADATION from the repetition cut — the gate outranks the approval; revert the cut:\n${summary}` +
|
||||||
|
|||||||
Reference in New Issue
Block a user