test: PTY harness handles clipped reviews and bundled setup tabs; AUQ judge uses structured output; design-consultation carve declines optional outside voices

- ceo mode routing: a Submit review taller than the viewport, a setup tab
  bundled after the mode tab, and a clip through the mode question each hung
  or misread the run; the native answer is still verified after Submit.
- judgeRecommendation requests a 1-5 enum schema; a malformed Haiku reply had
  scored substance 0 for a 4/5 brief. Judge failures now propagate.
- carve section-loading for design-consultation declines the optional outside
  voices (a supported path) and treats DESIGN.md as the report; timeout unchanged.
The Step 0E handoff defect is not fixed (0/15 samples across four wordings,
none shipped) and is filed in TODOS.
This commit is contained in:
garrytan committed 2026-09-30 12:52:56 +00:00
1 parent dfe5e733fb
commit a27365bffe
16 files changed
+893 -28

No files matched your search

+31 -1
View File
@@ -1,6 +1,7 @@
import { afterEach, beforeEach, describe, expect, spyOn, test } from 'bun:test';
import Anthropic from '@anthropic-ai/sdk';
import { armJudge, callJudge, JudgeRefusalError } from './helpers/llm-judge';
import { armJudge, callJudge, JudgeRefusalError, judgeRecommendation, RECOMMENDATION_JUDGE_SCHEMA } from './helpers/llm-judge';
import { gradeAuqRecommendation } from './helpers/auq-sdk-capture';
describe('frontier Claude judge compatibility', () => {
let originalKey: string | undefined;
@@ -194,6 +195,35 @@ describe('frontier Claude judge compatibility', () => {
}
});
// Census 36641820398 slice 7: Haiku's free-form reply left inner quotes unescaped.
const CENSUS_UNESCAPED_REPLY = '```json\n{"reason_substance": 4, "reasoning": "The clause is concrete and option-specific ("at 1/10 every dimension has gaps and none are obviously safe to skip") but does not explicitly compare the chosen option (A) against alternatives B or C in the because-clause itself\u2014the comparison lives in the surrounding context, not in the reason block."}\n```';
const CENSUS_BRIEF = 'D1 \u2014 Review all 7 design dimensions?\nRecommendation: A because at 1/10 every dimension has gaps and none are obviously safe to skip.\nA) All 7 dimensions (recommended)';
test('recommendation judge requests structured output, so quoted evidence cannot break its JSON', async () => {
create.mockResolvedValue({ stop_reason: 'end_turn', content: [{ type: 'text',
text: JSON.stringify({ reason_substance: 4, reasoning: 'Concrete ("at 1/10 every dimension has gaps") but no comparison.' }) }] } as never);
const score = await judgeRecommendation(CENSUS_BRIEF);
expect(score.reason_substance).toBe(4);
expect(score.reason_text).toBe('at 1/10 every dimension has gaps and none are obviously safe to skip.');
expect(create.mock.calls[0][0].output_config).toEqual({ format: { type: 'json_schema', schema: RECOMMENDATION_JUDGE_SCHEMA } });
expect(create.mock.calls[0][0].model).toBe('claude-haiku-4-5-20251001');
});
test('the captured unescaped reply is unparseable free-form text and fails without a score', async () => {
create.mockResolvedValue({ stop_reason: 'end_turn', content: [{ type: 'text', text: CENSUS_UNESCAPED_REPLY }] } as never);
await expect(callJudge('score this', 'claude-haiku-4-5-20251001')).rejects.toThrow(SyntaxError);
create.mockClear();
create.mockResolvedValue({ stop_reason: 'end_turn', content: [{ type: 'text', text: CENSUS_UNESCAPED_REPLY }] } as never);
await expect(gradeAuqRecommendation(CENSUS_BRIEF)).rejects.toThrow(SyntaxError);
expect(create).toHaveBeenCalledTimes(1);
});
test('a weak structured score still fails the substance bar', async () => {
create.mockResolvedValue({ stop_reason: 'end_turn', content: [{ type: 'text',
text: JSON.stringify({ reason_substance: 1, reasoning: 'Boilerplate.' }) }] } as never);
expect((await gradeAuqRecommendation('Recommendation: A because it is better.')).substance).toBe(1);
});
test('arm judge sends no unsupported temperature to Fable', async () => {
create.mockResolvedValue({ content: [
{ type: 'thinking', thinking: '', signature: 'fixture' },