mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-03 01:46:55 +02:00
test(judges): sample the recommendation rubric as a panel; never re-ask armJudge
llm-judge-recommendation is a judge case: each fixture now draws a 3-sample judgePanel, gates reason_substance on the panel mean and the present/commits/has_because checks on a 2-of-3 majority, thresholds unchanged. armJudge no longer re-asks on a malformed verdict; it is a failed sample, as the judge policy requires.
This commit is contained in:
1 parent
2fb55e4c7a
commit
f4ab5ee75f
3 files changed
+43
-44
No files matched your search
@@ -13,7 +13,7 @@ import {
|
||||
} from './helpers/arm-benchmark-harness';
|
||||
import {
|
||||
armJudge, buildArmJudgePrompt, parseArmJudgeResponse,
|
||||
ARM_JUDGE_ATTEMPTS, callJudge,
|
||||
callJudge,
|
||||
} from './helpers/llm-judge';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
@@ -182,28 +182,25 @@ describe('arm benchmark selftest (free, no API)', () => {
|
||||
expect(score.construct).toBe('none');
|
||||
});
|
||||
|
||||
test('armJudge: bounded retry-on-malformed — recovers once, then gives up', async () => {
|
||||
// Malformed first, valid second: recovers within the 2-attempt bound.
|
||||
test('armJudge: a malformed verdict is a failed sample, never re-asked', async () => {
|
||||
let calls = 0;
|
||||
const flaky = (async () => {
|
||||
const malformedFirst = (async () => {
|
||||
calls++;
|
||||
return calls === 1
|
||||
? { over_engineering: 9, construct: 'garbage' }
|
||||
: { over_engineering: 2, construct: 'repository layer in app.js', reasoning: 'ok' };
|
||||
}) as unknown as typeof callJudge;
|
||||
const recovered = await armJudge('ticket', 'diff --git a/x b/x\n+1\n', { call: flaky });
|
||||
expect(recovered.over_engineering).toBe(2);
|
||||
expect(calls).toBe(ARM_JUDGE_ATTEMPTS);
|
||||
await expect(armJudge('ticket', 'diff --git a/x b/x\n+1\n', { call: malformedFirst }))
|
||||
.rejects.toThrow(/malformed verdict \(never resampled\)/);
|
||||
expect(calls).toBe(1);
|
||||
|
||||
// Always malformed: throws after exactly ARM_JUDGE_ATTEMPTS attempts.
|
||||
let badCalls = 0;
|
||||
const alwaysBad = (async () => {
|
||||
badCalls++;
|
||||
return { nonsense: true };
|
||||
let goodCalls = 0;
|
||||
const wellFormed = (async () => {
|
||||
goodCalls++;
|
||||
return { over_engineering: 2, construct: 'repository layer in app.js', reasoning: 'ok' };
|
||||
}) as unknown as typeof callJudge;
|
||||
await expect(armJudge('ticket', 'diff --git a/x b/x\n+1\n', { call: alwaysBad }))
|
||||
.rejects.toThrow(/no well-formed verdict after 2 attempts/);
|
||||
expect(badCalls).toBe(ARM_JUDGE_ATTEMPTS);
|
||||
expect((await armJudge('ticket', 'diff --git a/x b/x\n+1\n', { call: wellFormed })).over_engineering).toBe(2);
|
||||
expect(goodCalls).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
Reference in new issue
Block a user