mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-08 04:11:18 +02:00
test: PTY harness handles clipped reviews and bundled setup tabs; AUQ judge uses structured output; design-consultation carve declines optional outside voices
- ceo mode routing: a Submit review taller than the viewport, a setup tab bundled after the mode tab, and a clip through the mode question each hung or misread the run; the native answer is still verified after Submit. - judgeRecommendation requests a 1-5 enum schema; a malformed Haiku reply had scored substance 0 for a 4/5 brief. Judge failures now propagate. - carve section-loading for design-consultation declines the optional outside voices (a supported path) and treats DESIGN.md as the report; timeout unchanged. The Step 0E handoff defect is not fixed (0/15 samples across four wordings, none shipped) and is filed in TODOS.
This commit is contained in:
1 parent
dfe5e733fb
commit
a27365bffe
16 files changed
+893
-28
No files matched your search
@@ -1,6 +1,7 @@
|
||||
import { afterEach, beforeEach, describe, expect, spyOn, test } from 'bun:test';
|
||||
import Anthropic from '@anthropic-ai/sdk';
|
||||
import { armJudge, callJudge, JudgeRefusalError } from './helpers/llm-judge';
|
||||
import { armJudge, callJudge, JudgeRefusalError, judgeRecommendation, RECOMMENDATION_JUDGE_SCHEMA } from './helpers/llm-judge';
|
||||
import { gradeAuqRecommendation } from './helpers/auq-sdk-capture';
|
||||
|
||||
describe('frontier Claude judge compatibility', () => {
|
||||
let originalKey: string | undefined;
|
||||
@@ -194,6 +195,35 @@ describe('frontier Claude judge compatibility', () => {
|
||||
}
|
||||
});
|
||||
|
||||
// Census 36641820398 slice 7: Haiku's free-form reply left inner quotes unescaped.
|
||||
const CENSUS_UNESCAPED_REPLY = '```json\n{"reason_substance": 4, "reasoning": "The clause is concrete and option-specific ("at 1/10 every dimension has gaps and none are obviously safe to skip") but does not explicitly compare the chosen option (A) against alternatives B or C in the because-clause itself\u2014the comparison lives in the surrounding context, not in the reason block."}\n```';
|
||||
const CENSUS_BRIEF = 'D1 \u2014 Review all 7 design dimensions?\nRecommendation: A because at 1/10 every dimension has gaps and none are obviously safe to skip.\nA) All 7 dimensions (recommended)';
|
||||
|
||||
test('recommendation judge requests structured output, so quoted evidence cannot break its JSON', async () => {
|
||||
create.mockResolvedValue({ stop_reason: 'end_turn', content: [{ type: 'text',
|
||||
text: JSON.stringify({ reason_substance: 4, reasoning: 'Concrete ("at 1/10 every dimension has gaps") but no comparison.' }) }] } as never);
|
||||
const score = await judgeRecommendation(CENSUS_BRIEF);
|
||||
expect(score.reason_substance).toBe(4);
|
||||
expect(score.reason_text).toBe('at 1/10 every dimension has gaps and none are obviously safe to skip.');
|
||||
expect(create.mock.calls[0][0].output_config).toEqual({ format: { type: 'json_schema', schema: RECOMMENDATION_JUDGE_SCHEMA } });
|
||||
expect(create.mock.calls[0][0].model).toBe('claude-haiku-4-5-20251001');
|
||||
});
|
||||
|
||||
test('the captured unescaped reply is unparseable free-form text and fails without a score', async () => {
|
||||
create.mockResolvedValue({ stop_reason: 'end_turn', content: [{ type: 'text', text: CENSUS_UNESCAPED_REPLY }] } as never);
|
||||
await expect(callJudge('score this', 'claude-haiku-4-5-20251001')).rejects.toThrow(SyntaxError);
|
||||
create.mockClear();
|
||||
create.mockResolvedValue({ stop_reason: 'end_turn', content: [{ type: 'text', text: CENSUS_UNESCAPED_REPLY }] } as never);
|
||||
await expect(gradeAuqRecommendation(CENSUS_BRIEF)).rejects.toThrow(SyntaxError);
|
||||
expect(create).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test('a weak structured score still fails the substance bar', async () => {
|
||||
create.mockResolvedValue({ stop_reason: 'end_turn', content: [{ type: 'text',
|
||||
text: JSON.stringify({ reason_substance: 1, reasoning: 'Boilerplate.' }) }] } as never);
|
||||
expect((await gradeAuqRecommendation('Recommendation: A because it is better.')).substance).toBe(1);
|
||||
});
|
||||
|
||||
test('arm judge sends no unsupported temperature to Fable', async () => {
|
||||
create.mockResolvedValue({ content: [
|
||||
{ type: 'thinking', thinking: '', signature: 'fixture' },
|
||||
|
||||
Reference in new issue
Block a user