diff --git a/test/browse-judge-floors.test.ts b/test/browse-judge-floors.test.ts new file mode 100644 index 000000000..53a1a160f --- /dev/null +++ b/test/browse-judge-floors.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, test } from 'bun:test'; +import * as fs from 'fs'; +import * as path from 'path'; +import { browseJudgeFloorsMet } from './helpers/workflow-judge-cache'; + +type Scores = { clarity: number; completeness: number; actionability: number }; +const stored = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/browse-judge/panel-means.json'), 'utf8')) as + { known_good: Record; known_bad: Record }; + +describe('browse judge floors', () => { + test.each(Object.entries(stored.known_good))('passes %s', (_name, scores) => expect(browseJudgeFloorsMet(scores)).toBe(true)); + test.each(Object.entries(stored.known_bad))('fails %s', (_name, scores) => expect(browseJudgeFloorsMet(scores)).toBe(false)); +}); diff --git a/test/fixtures/browse-judge/panel-means.json b/test/fixtures/browse-judge/panel-means.json new file mode 100644 index 000000000..ad4864e77 --- /dev/null +++ b/test/fixtures/browse-judge/panel-means.json @@ -0,0 +1,13 @@ +{ + "_comment": "Browse reference judge panel means against baseline {clarity:4, completeness:4, actionability:4}. known_good passes the floors (the old ratchet failed the first two); known_bad fails them.", + "known_good": { + "clarity-dip-to-3.67": { "clarity": 3.67, "completeness": 4, "actionability": 4 }, + "clarity-at-floor": { "clarity": 3, "completeness": 4.33, "actionability": 4 }, + "at-baseline": { "clarity": 4, "completeness": 4, "actionability": 4 } + }, + "known_bad": { + "clarity-below-floor": { "clarity": 2.67, "completeness": 4, "actionability": 4 }, + "completeness-below-floor": { "clarity": 4, "completeness": 3.67, "actionability": 4 }, + "actionability-below-floor": { "clarity": 4, "completeness": 4, "actionability": 3.33 } + } +} diff --git a/test/helpers/workflow-judge-cache.ts b/test/helpers/workflow-judge-cache.ts index f1bd4024e..3a6ad000f 100644 --- a/test/helpers/workflow-judge-cache.ts +++ b/test/helpers/workflow-judge-cache.ts @@ -135,3 +135,15 @@ export function prepareWorkflowJudgeCache(opts: WorkflowCacheOptions): { }, }; } + +/** + * Pass floors for the browse reference judge panel mean. The stored baseline + * is recorded for comparison only: a three-sample mean is too noisy for a + * no-dip ratchet, and gating on it silently raised the clarity floor to 4. + */ +export const BROWSE_JUDGE_FLOORS = { clarity: 3, completeness: 4, actionability: 4 } as const; + +export function browseJudgeFloorsMet(scores: Record): boolean { + return (Object.keys(BROWSE_JUDGE_FLOORS) as Array) + .every(dim => scores[dim] >= BROWSE_JUDGE_FLOORS[dim]); +} diff --git a/test/skill-llm-eval.test.ts b/test/skill-llm-eval.test.ts index cb6465e0d..40a91a410 100644 --- a/test/skill-llm-eval.test.ts +++ b/test/skill-llm-eval.test.ts @@ -18,7 +18,7 @@ import { callJudge, judge, JudgeRefusalError, DEFAULT_JUDGE_MAX_TOKENS, judgePan import { ASK_QUESTIONS_HEADING, ENG_REVIEW_EXCERPT } from './helpers/workflow-excerpt'; import type { JudgeScore } from './helpers/llm-judge'; import { readWorkflowJudgeInput, buildWorkflowJudgePrompt, QA_DISCOVERY_REFERENCES, WORKFLOW_JUDGE_RESPONSE_SCHEMA, type WorkflowJudgeInput } from './helpers/workflow-judge-input'; -import { prepareWorkflowJudgeCache, validWorkflowJudgeScore } from './helpers/workflow-judge-cache'; +import { BROWSE_JUDGE_FLOORS, browseJudgeFloorsMet, prepareWorkflowJudgeCache, validWorkflowJudgeScore } from './helpers/workflow-judge-cache'; import { buildCookieWorkflowJudgeInput, COOKIE_WORKFLOW_JUDGE } from './helpers/cookie-workflow-judge-input'; import { getCookieWorkflowManualReview, type ManualJudgeReview } from './helpers/cookie-workflow-manual-review'; import { resolveEvalModel } from '../lib/eval-model'; @@ -96,8 +96,8 @@ describeIfSelected('LLM-as-judge quality evals', [ const t0 = Date.now(); // Browse carve: snapshot flags + the full command list are the whole // generated section file; one judge grades the union. Scores are also - // pinned against test/fixtures/eval-baselines.json (UPDATE_BASELINES=1 - // rewrites the pin). + // compared with test/fixtures/eval-baselines.json and the difference is + // recorded (UPDATE_BASELINES=1 rewrites it); only the floors gate. const section = sliceBrowseSection('## Snapshot Flags'); const samples = await judgePanel(() => judge('browse skill reference (flags + commands)', section)); @@ -119,17 +119,16 @@ describeIfSelected('LLM-as-judge quality evals', [ name: 'browse/SKILL.md reference', suite: 'LLM-as-judge quality evals', tier: 'llm-judge', - passed: scores.clarity >= 3 && scores.completeness >= 4 && scores.actionability >= 4 && regressions.length === 0, + passed: browseJudgeFloorsMet(scores), duration_ms: Date.now() - t0, cost_usd: 0.02 * samples.length, judge_scores: { clarity: scores.clarity, completeness: scores.completeness, actionability: scores.actionability }, judge_reasoning: regressions.length ? `${judgePanelReasoning(samples)} | ${regressions.join('; ')}` : judgePanelReasoning(samples), }); - expect(scores.clarity).toBeGreaterThanOrEqual(3); - expect(scores.completeness).toBeGreaterThanOrEqual(4); - expect(scores.actionability).toBeGreaterThanOrEqual(4); - expect(regressions).toEqual([]); + expect(scores.clarity).toBeGreaterThanOrEqual(BROWSE_JUDGE_FLOORS.clarity); + expect(scores.completeness).toBeGreaterThanOrEqual(BROWSE_JUDGE_FLOORS.completeness); + expect(scores.actionability).toBeGreaterThanOrEqual(BROWSE_JUDGE_FLOORS.actionability); }, JUDGE_MS); testIfSelected('setup block', async () => {