test(judges): sample the recommendation rubric as a panel; never re-ask armJudge

llm-judge-recommendation is a judge case: each fixture now draws a
3-sample judgePanel, gates reason_substance on the panel mean and the
present/commits/has_because checks on a 2-of-3 majority, thresholds
unchanged. armJudge no longer re-asks on a malformed verdict; it is a
failed sample, as the judge policy requires.
This commit is contained in:
garrytan committed 2026-09-29 19:51:35 +00:00
1 parent 2fb55e4c7a
commit f4ab5ee75f
3 files changed
+43 -44

No files matched your search

+12 -15
View File
@@ -13,7 +13,7 @@ import {
} from './helpers/arm-benchmark-harness';
import {
armJudge, buildArmJudgePrompt, parseArmJudgeResponse,
ARM_JUDGE_ATTEMPTS, callJudge,
callJudge,
} from './helpers/llm-judge';
import * as fs from 'fs';
import * as path from 'path';
@@ -182,28 +182,25 @@ describe('arm benchmark selftest (free, no API)', () => {
expect(score.construct).toBe('none');
});
test('armJudge: bounded retry-on-malformed — recovers once, then gives up', async () => {
// Malformed first, valid second: recovers within the 2-attempt bound.
test('armJudge: a malformed verdict is a failed sample, never re-asked', async () => {
let calls = 0;
const flaky = (async () => {
const malformedFirst = (async () => {
calls++;
return calls === 1
? { over_engineering: 9, construct: 'garbage' }
: { over_engineering: 2, construct: 'repository layer in app.js', reasoning: 'ok' };
}) as unknown as typeof callJudge;
const recovered = await armJudge('ticket', 'diff --git a/x b/x\n+1\n', { call: flaky });
expect(recovered.over_engineering).toBe(2);
expect(calls).toBe(ARM_JUDGE_ATTEMPTS);
await expect(armJudge('ticket', 'diff --git a/x b/x\n+1\n', { call: malformedFirst }))
.rejects.toThrow(/malformed verdict \(never resampled\)/);
expect(calls).toBe(1);
// Always malformed: throws after exactly ARM_JUDGE_ATTEMPTS attempts.
let badCalls = 0;
const alwaysBad = (async () => {
badCalls++;
return { nonsense: true };
let goodCalls = 0;
const wellFormed = (async () => {
goodCalls++;
return { over_engineering: 2, construct: 'repository layer in app.js', reasoning: 'ok' };
}) as unknown as typeof callJudge;
await expect(armJudge('ticket', 'diff --git a/x b/x\n+1\n', { call: alwaysBad }))
.rejects.toThrow(/no well-formed verdict after 2 attempts/);
expect(badCalls).toBe(ARM_JUDGE_ATTEMPTS);
expect((await armJudge('ticket', 'diff --git a/x b/x\n+1\n', { call: wellFormed })).over_engineering).toBe(2);
expect(goodCalls).toBe(1);
});
});