test: record llm-judge abort diagnostic as non-behavioral; abort tests wait on events, not sleeps

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed 2026-10-28 13:53:16 +00:00
1 parent 71a3a7a1bd
commit 2d6c44b142
2 files changed
+20 -11

No files matched your search

+1 -1
View File
@@ -29,7 +29,7 @@
"test/helpers/eval-budgets.ts": {"blob":"667d0e09b20fe5d7366f57b77b5fb9b04baa37ef","since":6},
"test/helpers/eval-store.ts": {"blob":"a50ea2c28f508ad9f35ea9353dd2f502ee0dcfc2","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
"test/helpers/hermetic-env.ts": {"blob":"cc072895461dc52ea317d1beff72121342420d30","since":6},
"test/helpers/llm-judge.ts": {"blob":"a3177665b662932ce485ccc976c2b36a1531a0bb","since":6,"nonBehavioral":"callJudge rewrites only the message of a JSON parse failure (names the judge output, stop reason, length and the retained llm-judge-response-parse-error event); it still throws a SyntaxError before any score is parsed, the request is unchanged, and no case's pass or fail changes"},
"test/helpers/llm-judge.ts": {"blob":"6f742f6a0f751ddcc9616d7d590a885c7bb78d62","since":6,"nonBehavioral":"callJudge emits one llm-judge-aborted diagnostic line (timings, request and retry counts, block kind, character counts, message and request ids; no content) when its caller's signal aborts it; the request, retries, parsing, scores and the thrown abort reason are unchanged, so no case's pass or fail changes"},
"test/helpers/paid-test-set.ts": {"blob":"ff9bfe43c43ec2bdd11114f7b8ec7c5736116f75","since":6},
"test/helpers/pty-screen.ts": {"blob":"4f77569f9ce11f9cee9f347216e468cd7c956820","since":6},
"test/helpers/pty/auq.ts": {"blob":"16e5ea2bb5b9f28f44808a1a09024dbd3e6f28c4","since":6},
+19 -10
View File
@@ -114,13 +114,19 @@ test('a judge aborted mid-stream retains what it had received, as counts without
];
let delivered!: () => void;
const received = new Promise<void>(resolve => { delivered = resolve; });
transport.mockImplementation(async (_url: string, init: RequestInit) => new Response(new ReadableStream({
start(stream) {
for (const event of partial) stream.enqueue(encoder.encode(`event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`));
init.signal!.addEventListener('abort', () => stream.error(init.signal!.reason), { once: true });
setTimeout(delivered, 20);
},
}), { headers: { 'content-type': 'text/event-stream', 'request-id': 'req_stalled' } }));
transport.mockImplementation(async (_url: string, init: RequestInit) => {
const chunks = partial.map(event => encoder.encode(`event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`));
return new Response(new ReadableStream({
start(stream) {
init.signal!.addEventListener('abort', () => stream.error(init.signal!.reason), { once: true });
},
pull(stream) {
const chunk = chunks.shift();
if (chunk) stream.enqueue(chunk);
else delivered();
},
}, { highWaterMark: 0 }), { headers: { 'content-type': 'text/event-stream', 'request-id': 'req_stalled' } });
});
const controller = new AbortController();
const rejected = callJudge('score the whole bundle', 'claude-fable-5-1', {
max_tokens: 65_536, stream: true, jsonSchema: WORKFLOW_JUDGE_RESPONSE_SCHEMA, signal: controller.signal,
@@ -137,17 +143,20 @@ test('a judge aborted mid-stream retains what it had received, as counts without
retryWaitsMs: [], block: 'thinking', thinkingChars: 'PRIVATE_THINKING'.length, textChars: 0,
messageId: 'msg_stalled', requestId: 'req_stalled', reason: failure.message });
expect(event.firstEventMs).toBeGreaterThanOrEqual(0);
expect(event.lastEventMs).toBeGreaterThanOrEqual(event.firstEventMs);
expect(event.elapsedMs).toBeGreaterThanOrEqual(event.lastEventMs);
expect(event.lastEventMs).toBeGreaterThanOrEqual(event.firstEventMs); // gstack-timing-ok: ordering of two recorded offsets, not an elapsed-time bound
expect(event.elapsedMs).toBeGreaterThanOrEqual(event.lastEventMs); // gstack-timing-ok: ordering of two recorded offsets, not an elapsed-time bound
});
test('a stream that never sent an event is retained as such, and provider errors without an abort are not', async () => {
let sent!: () => void;
const requested = new Promise<void>(resolve => { sent = resolve; });
transport.mockImplementation((_url: string, init: RequestInit) => new Promise((_resolve, reject) => {
init.signal!.addEventListener('abort', () => reject(init.signal!.reason), { once: true });
sent();
}));
const controller = new AbortController();
const rejected = callJudge('score', 'claude-fable-5-1', { max_tokens: 65_536, stream: true, signal: controller.signal }).catch(error => error);
await new Promise(resolve => setTimeout(resolve, 10));
await requested;
controller.abort(new Error('deadline'));
await rejected;
expect(JSON.parse(diagnostics.mock.calls[0]![0])).toMatchObject({ type: 'llm-judge-aborted', firstEventMs: null,