mirror of
https://github.com/garrytan/gstack.git
synced 2026-05-02 03:35:09 +02:00
test(eval): unit tests for agent-sdk-runner (36 tests, free tier)
Stub `queryProvider` feeds hand-crafted SDKMessage streams. Covers: happy-path shape, all 3 rate-limit shapes + retry, workspace reset on retry, persistent 429 -> `RateLimitExhaustedError`, non-429 propagation, process-level concurrency cap, options propagation, artifact path uniqueness, cost/turn mapping, and every validator rejection case.
This commit is contained in:
Garry Tan
@@ -0,0 +1,725 @@
|
||||
/**
|
||||
* Unit tests for test/helpers/agent-sdk-runner.ts.
|
||||
*
|
||||
* Runs in free `bun test` (no API calls). Uses a stub QueryProvider to
|
||||
* simulate SDK event streams — happy path, rate-limit retries across all
|
||||
* three shapes, persistent failure, non-retryable error, options
|
||||
* propagation, concurrency cap.
|
||||
*
|
||||
* Also covers validateFixtures() rejections.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
import type {
|
||||
SDKMessage,
|
||||
Options,
|
||||
Query,
|
||||
} from '@anthropic-ai/claude-agent-sdk';
|
||||
import {
|
||||
runAgentSdkTest,
|
||||
toSkillTestResult,
|
||||
firstTurnParallelism,
|
||||
isRateLimitThrown,
|
||||
isRateLimitResult,
|
||||
isRateLimitEvent,
|
||||
RateLimitExhaustedError,
|
||||
__resetSemaphoreForTests,
|
||||
type QueryProvider,
|
||||
type AgentSdkResult,
|
||||
} from '../test/helpers/agent-sdk-runner';
|
||||
import {
|
||||
validateFixtures,
|
||||
fanoutPass,
|
||||
type OverlayFixture,
|
||||
} from '../test/fixtures/overlay-nudges';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Stub SDK event builders
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
let uuidCounter = 0;
|
||||
function uuid(): string {
|
||||
return `00000000-0000-0000-0000-${String(++uuidCounter).padStart(12, '0')}`;
|
||||
}
|
||||
|
||||
function systemInit(model = 'claude-opus-4-7', version = '2.1.117'): SDKMessage {
|
||||
return {
|
||||
type: 'system',
|
||||
subtype: 'init',
|
||||
apiKeySource: 'user',
|
||||
claude_code_version: version,
|
||||
cwd: '/tmp/x',
|
||||
tools: ['Read'],
|
||||
mcp_servers: [],
|
||||
model,
|
||||
permissionMode: 'bypassPermissions',
|
||||
slash_commands: [],
|
||||
output_style: 'default',
|
||||
skills: [],
|
||||
plugins: [],
|
||||
uuid: uuid(),
|
||||
session_id: 'test-session',
|
||||
} as unknown as SDKMessage;
|
||||
}
|
||||
|
||||
function assistantTurn(
|
||||
blocks: Array<{ type: 'text'; text: string } | { type: 'tool_use'; name: string; input: unknown }>,
|
||||
): SDKMessage {
|
||||
return {
|
||||
type: 'assistant',
|
||||
parent_tool_use_id: null,
|
||||
uuid: uuid(),
|
||||
session_id: 'test-session',
|
||||
message: {
|
||||
id: 'msg_' + uuid(),
|
||||
type: 'message',
|
||||
role: 'assistant',
|
||||
model: 'claude-opus-4-7',
|
||||
content: blocks.map((b) => ({ ...b })),
|
||||
stop_reason: 'end_turn',
|
||||
stop_sequence: null,
|
||||
usage: {
|
||||
input_tokens: 10,
|
||||
output_tokens: 20,
|
||||
cache_creation_input_tokens: 0,
|
||||
cache_read_input_tokens: 0,
|
||||
service_tier: 'standard',
|
||||
},
|
||||
},
|
||||
} as unknown as SDKMessage;
|
||||
}
|
||||
|
||||
function resultSuccess(cost = 0.01, turns = 1): SDKMessage {
|
||||
return {
|
||||
type: 'result',
|
||||
subtype: 'success',
|
||||
duration_ms: 100,
|
||||
duration_api_ms: 50,
|
||||
is_error: false,
|
||||
num_turns: turns,
|
||||
result: 'done',
|
||||
stop_reason: 'end_turn',
|
||||
total_cost_usd: cost,
|
||||
usage: {
|
||||
input_tokens: 10,
|
||||
output_tokens: 20,
|
||||
cache_creation_input_tokens: 0,
|
||||
cache_read_input_tokens: 0,
|
||||
server_tool_use: {},
|
||||
service_tier: 'standard',
|
||||
},
|
||||
modelUsage: {},
|
||||
permission_denials: [],
|
||||
uuid: uuid(),
|
||||
session_id: 'test-session',
|
||||
} as unknown as SDKMessage;
|
||||
}
|
||||
|
||||
function resultRateLimit(): SDKMessage {
|
||||
return {
|
||||
type: 'result',
|
||||
subtype: 'error_during_execution',
|
||||
duration_ms: 100,
|
||||
duration_api_ms: 50,
|
||||
is_error: true,
|
||||
num_turns: 0,
|
||||
stop_reason: null,
|
||||
total_cost_usd: 0,
|
||||
usage: {
|
||||
input_tokens: 0,
|
||||
output_tokens: 0,
|
||||
cache_creation_input_tokens: 0,
|
||||
cache_read_input_tokens: 0,
|
||||
server_tool_use: {},
|
||||
service_tier: 'standard',
|
||||
},
|
||||
modelUsage: {},
|
||||
permission_denials: [],
|
||||
errors: ['rate limit exceeded (429)'],
|
||||
uuid: uuid(),
|
||||
session_id: 'test-session',
|
||||
} as unknown as SDKMessage;
|
||||
}
|
||||
|
||||
function rateLimitEvent(): SDKMessage {
|
||||
return {
|
||||
type: 'rate_limit_event',
|
||||
rate_limit_info: {
|
||||
status: 'rejected',
|
||||
rateLimitType: 'five_hour',
|
||||
},
|
||||
uuid: uuid(),
|
||||
session_id: 'test-session',
|
||||
} as unknown as SDKMessage;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Stub query provider
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface StubConfig {
|
||||
/** One event stream per call. Exhausted calls throw. */
|
||||
streams: SDKMessage[][];
|
||||
/** Throw this error on the Nth call (0-indexed). */
|
||||
throwAt?: number;
|
||||
throwError?: unknown;
|
||||
/** Track calls for assertions. */
|
||||
calls: Array<{ prompt: string; options: Options | undefined; startedAt: number; endedAt?: number }>;
|
||||
}
|
||||
|
||||
function makeStubProvider(config: StubConfig): QueryProvider {
|
||||
let callIdx = -1;
|
||||
const provider: QueryProvider = (params) => {
|
||||
callIdx++;
|
||||
const idx = callIdx;
|
||||
const startedAt = Date.now();
|
||||
const prompt = typeof params.prompt === 'string' ? params.prompt : '<iterable>';
|
||||
config.calls.push({ prompt, options: params.options, startedAt });
|
||||
|
||||
if (config.throwAt !== undefined && idx === config.throwAt) {
|
||||
const err = config.throwError ?? new Error('stub throw');
|
||||
// Return an async generator that throws on first next().
|
||||
const gen = (async function* (): AsyncGenerator<SDKMessage, void> {
|
||||
throw err;
|
||||
})();
|
||||
return gen as unknown as Query;
|
||||
}
|
||||
|
||||
const stream = config.streams[idx];
|
||||
if (!stream) {
|
||||
const gen = (async function* (): AsyncGenerator<SDKMessage, void> {
|
||||
throw new Error(`stub has no stream for call ${idx}`);
|
||||
})();
|
||||
return gen as unknown as Query;
|
||||
}
|
||||
|
||||
const gen = (async function* (): AsyncGenerator<SDKMessage, void> {
|
||||
try {
|
||||
for (const ev of stream) {
|
||||
yield ev;
|
||||
}
|
||||
} finally {
|
||||
config.calls[idx]!.endedAt = Date.now();
|
||||
}
|
||||
})();
|
||||
return gen as unknown as Query;
|
||||
};
|
||||
return provider;
|
||||
}
|
||||
|
||||
const BASE_OPTS = {
|
||||
systemPrompt: '',
|
||||
userPrompt: 'test prompt',
|
||||
workingDirectory: '/tmp/test-dir',
|
||||
maxRetries: 3,
|
||||
};
|
||||
|
||||
// Reset semaphore before each test that depends on fresh capacity.
|
||||
function freshSem(cap = 10): void {
|
||||
__resetSemaphoreForTests(cap);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Happy path
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('runAgentSdkTest — happy path', () => {
|
||||
test('collects events, assistantTurns, toolCalls, and result fields', async () => {
|
||||
freshSem();
|
||||
const stub: StubConfig = {
|
||||
streams: [
|
||||
[
|
||||
systemInit(),
|
||||
assistantTurn([
|
||||
{ type: 'text', text: 'reading files' },
|
||||
{ type: 'tool_use', name: 'Read', input: { path: 'a.txt' } },
|
||||
{ type: 'tool_use', name: 'Read', input: { path: 'b.txt' } },
|
||||
]),
|
||||
assistantTurn([{ type: 'text', text: 'done' }]),
|
||||
resultSuccess(0.05, 2),
|
||||
],
|
||||
],
|
||||
calls: [],
|
||||
};
|
||||
const result = await runAgentSdkTest({
|
||||
...BASE_OPTS,
|
||||
queryProvider: makeStubProvider(stub),
|
||||
});
|
||||
|
||||
expect(result.events.length).toBe(4);
|
||||
expect(result.assistantTurns.length).toBe(2);
|
||||
expect(result.toolCalls.length).toBe(2);
|
||||
expect(result.toolCalls[0]!.tool).toBe('Read');
|
||||
expect(result.output).toContain('reading files');
|
||||
expect(result.output).toContain('done');
|
||||
expect(result.exitReason).toBe('success');
|
||||
expect(result.turnsUsed).toBe(2);
|
||||
expect(result.costUsd).toBe(0.05);
|
||||
expect(result.sdkClaudeCodeVersion).toBe('2.1.117');
|
||||
expect(result.model).toBe('claude-opus-4-7');
|
||||
expect(result.firstResponseMs).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
|
||||
test('first-turn parallelism: 3 tool_use blocks in first assistant turn', async () => {
|
||||
freshSem();
|
||||
const stub: StubConfig = {
|
||||
streams: [
|
||||
[
|
||||
systemInit(),
|
||||
assistantTurn([
|
||||
{ type: 'tool_use', name: 'Read', input: { path: 'a' } },
|
||||
{ type: 'tool_use', name: 'Read', input: { path: 'b' } },
|
||||
{ type: 'tool_use', name: 'Read', input: { path: 'c' } },
|
||||
]),
|
||||
resultSuccess(),
|
||||
],
|
||||
],
|
||||
calls: [],
|
||||
};
|
||||
const result = await runAgentSdkTest({
|
||||
...BASE_OPTS,
|
||||
queryProvider: makeStubProvider(stub),
|
||||
});
|
||||
expect(firstTurnParallelism(result.assistantTurns[0])).toBe(3);
|
||||
});
|
||||
|
||||
test('first-turn parallelism: 0 when first turn is text-only', async () => {
|
||||
freshSem();
|
||||
const stub: StubConfig = {
|
||||
streams: [
|
||||
[
|
||||
systemInit(),
|
||||
assistantTurn([{ type: 'text', text: 'thinking' }]),
|
||||
resultSuccess(),
|
||||
],
|
||||
],
|
||||
calls: [],
|
||||
};
|
||||
const result = await runAgentSdkTest({
|
||||
...BASE_OPTS,
|
||||
queryProvider: makeStubProvider(stub),
|
||||
});
|
||||
expect(firstTurnParallelism(result.assistantTurns[0])).toBe(0);
|
||||
});
|
||||
|
||||
test('first-turn parallelism: 0 when no first turn', () => {
|
||||
expect(firstTurnParallelism(undefined)).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Options propagation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('runAgentSdkTest — options propagation', () => {
|
||||
test('systemPrompt, model, cwd, allowedTools, disallowedTools, permissionMode, settingSources, env, pathToClaudeCodeExecutable reach query()', async () => {
|
||||
freshSem();
|
||||
const stub: StubConfig = {
|
||||
streams: [[systemInit(), assistantTurn([{ type: 'text', text: 'ok' }]), resultSuccess()]],
|
||||
calls: [],
|
||||
};
|
||||
await runAgentSdkTest({
|
||||
systemPrompt: 'you are a test overlay',
|
||||
userPrompt: 'go',
|
||||
workingDirectory: '/tmp/spec-dir',
|
||||
model: 'claude-opus-4-7',
|
||||
maxTurns: 7,
|
||||
allowedTools: ['Read', 'Glob'],
|
||||
disallowedTools: ['Bash', 'Write'],
|
||||
permissionMode: 'bypassPermissions',
|
||||
settingSources: [],
|
||||
env: { ANTHROPIC_API_KEY: 'fake' },
|
||||
pathToClaudeCodeExecutable: '/fake/path/claude',
|
||||
queryProvider: makeStubProvider(stub),
|
||||
});
|
||||
|
||||
const opts = stub.calls[0]!.options!;
|
||||
expect(opts.systemPrompt).toBe('you are a test overlay');
|
||||
expect(opts.model).toBe('claude-opus-4-7');
|
||||
expect(opts.cwd).toBe('/tmp/spec-dir');
|
||||
expect(opts.maxTurns).toBe(7);
|
||||
expect(opts.tools).toEqual(['Read', 'Glob']);
|
||||
expect(opts.allowedTools).toEqual(['Read', 'Glob']);
|
||||
expect(opts.disallowedTools).toEqual(['Bash', 'Write']);
|
||||
expect(opts.permissionMode).toBe('bypassPermissions');
|
||||
expect(opts.allowDangerouslySkipPermissions).toBe(true);
|
||||
expect(opts.settingSources).toEqual([]);
|
||||
expect(opts.env).toEqual({ ANTHROPIC_API_KEY: 'fake' });
|
||||
expect(opts.pathToClaudeCodeExecutable).toBe('/fake/path/claude');
|
||||
});
|
||||
|
||||
test('empty systemPrompt means no systemPrompt option passed', async () => {
|
||||
freshSem();
|
||||
const stub: StubConfig = {
|
||||
streams: [[systemInit(), assistantTurn([{ type: 'text', text: 'ok' }]), resultSuccess()]],
|
||||
calls: [],
|
||||
};
|
||||
await runAgentSdkTest({
|
||||
...BASE_OPTS,
|
||||
queryProvider: makeStubProvider(stub),
|
||||
});
|
||||
// systemPrompt is undefined when empty string passed (so SDK uses no override)
|
||||
expect(stub.calls[0]!.options!.systemPrompt).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Rate-limit retry (three shapes)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('runAgentSdkTest — rate-limit retry', () => {
|
||||
test('retryable on thrown 429-shaped error, then succeeds on 2nd attempt', async () => {
|
||||
freshSem();
|
||||
const stub: StubConfig = {
|
||||
streams: [
|
||||
// call 0: throws (handled via throwAt below)
|
||||
[],
|
||||
// call 1: success
|
||||
[systemInit(), assistantTurn([{ type: 'text', text: 'ok' }]), resultSuccess()],
|
||||
],
|
||||
throwAt: 0,
|
||||
throwError: Object.assign(new Error('429 too many requests'), { status: 429 }),
|
||||
calls: [],
|
||||
};
|
||||
const result = await runAgentSdkTest({
|
||||
...BASE_OPTS,
|
||||
queryProvider: makeStubProvider(stub),
|
||||
maxRetries: 2,
|
||||
});
|
||||
expect(result.exitReason).toBe('success');
|
||||
expect(stub.calls.length).toBe(2);
|
||||
});
|
||||
|
||||
test('retryable on result-message rate-limit, then succeeds', async () => {
|
||||
freshSem();
|
||||
const stub: StubConfig = {
|
||||
streams: [
|
||||
[systemInit(), resultRateLimit()],
|
||||
[systemInit(), assistantTurn([{ type: 'text', text: 'ok' }]), resultSuccess()],
|
||||
],
|
||||
calls: [],
|
||||
};
|
||||
const result = await runAgentSdkTest({
|
||||
...BASE_OPTS,
|
||||
queryProvider: makeStubProvider(stub),
|
||||
maxRetries: 2,
|
||||
});
|
||||
expect(result.exitReason).toBe('success');
|
||||
expect(stub.calls.length).toBe(2);
|
||||
});
|
||||
|
||||
test('retryable on mid-stream SDKRateLimitEvent, then succeeds', async () => {
|
||||
freshSem();
|
||||
const stub: StubConfig = {
|
||||
streams: [
|
||||
[systemInit(), rateLimitEvent()],
|
||||
[systemInit(), assistantTurn([{ type: 'text', text: 'ok' }]), resultSuccess()],
|
||||
],
|
||||
calls: [],
|
||||
};
|
||||
const result = await runAgentSdkTest({
|
||||
...BASE_OPTS,
|
||||
queryProvider: makeStubProvider(stub),
|
||||
maxRetries: 2,
|
||||
});
|
||||
expect(result.exitReason).toBe('success');
|
||||
expect(stub.calls.length).toBe(2);
|
||||
});
|
||||
|
||||
test('onRetry callback is invoked between attempts', async () => {
|
||||
freshSem();
|
||||
const resets: string[] = [];
|
||||
const stub: StubConfig = {
|
||||
streams: [
|
||||
[],
|
||||
[systemInit(), assistantTurn([{ type: 'text', text: 'ok' }]), resultSuccess()],
|
||||
],
|
||||
throwAt: 0,
|
||||
throwError: Object.assign(new Error('429'), { status: 429 }),
|
||||
calls: [],
|
||||
};
|
||||
await runAgentSdkTest({
|
||||
...BASE_OPTS,
|
||||
queryProvider: makeStubProvider(stub),
|
||||
maxRetries: 2,
|
||||
onRetry: (dir) => resets.push(dir),
|
||||
});
|
||||
expect(resets.length).toBe(1);
|
||||
expect(resets[0]).toBe('/tmp/test-dir');
|
||||
});
|
||||
|
||||
test('persistent 429 throws RateLimitExhaustedError after maxRetries', async () => {
|
||||
freshSem();
|
||||
const stub: StubConfig = {
|
||||
streams: [[], [], [], []], // 4 empty streams; throw on each
|
||||
calls: [],
|
||||
};
|
||||
// Every call throws:
|
||||
let callCount = 0;
|
||||
const alwaysThrowProvider: QueryProvider = (params) => {
|
||||
callCount++;
|
||||
stub.calls.push({
|
||||
prompt: typeof params.prompt === 'string' ? params.prompt : '',
|
||||
options: params.options,
|
||||
startedAt: Date.now(),
|
||||
});
|
||||
const gen = (async function* (): AsyncGenerator<SDKMessage, void> {
|
||||
throw Object.assign(new Error('429 always'), { status: 429 });
|
||||
})();
|
||||
return gen as unknown as Query;
|
||||
};
|
||||
|
||||
let caught: unknown = null;
|
||||
try {
|
||||
await runAgentSdkTest({
|
||||
...BASE_OPTS,
|
||||
queryProvider: alwaysThrowProvider,
|
||||
maxRetries: 2,
|
||||
});
|
||||
} catch (err) {
|
||||
caught = err;
|
||||
}
|
||||
expect(caught).toBeInstanceOf(RateLimitExhaustedError);
|
||||
expect((caught as RateLimitExhaustedError).attempts).toBe(3); // initial + 2 retries
|
||||
expect(callCount).toBe(3);
|
||||
});
|
||||
|
||||
test('non-429 error is NOT retried, propagates immediately', async () => {
|
||||
__resetSemaphoreForTests(10);
|
||||
let callCount = 0;
|
||||
const throwOnce: QueryProvider = () => {
|
||||
callCount++;
|
||||
const gen = (async function* (): AsyncGenerator<SDKMessage, void> {
|
||||
throw new Error('generic auth failure');
|
||||
})();
|
||||
return gen as unknown as Query;
|
||||
};
|
||||
let caught: unknown = null;
|
||||
try {
|
||||
await runAgentSdkTest({
|
||||
...BASE_OPTS,
|
||||
queryProvider: throwOnce,
|
||||
maxRetries: 3,
|
||||
});
|
||||
} catch (err) {
|
||||
caught = err;
|
||||
}
|
||||
expect(caught).toBeInstanceOf(Error);
|
||||
expect((caught as Error).message).toBe('generic auth failure');
|
||||
expect(callCount).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Rate-limit detectors (unit)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('rate-limit detectors', () => {
|
||||
test('isRateLimitThrown matches status 429, message, name', () => {
|
||||
expect(isRateLimitThrown(Object.assign(new Error('boom'), { status: 429 }))).toBe(true);
|
||||
expect(isRateLimitThrown(new Error('429 Too Many Requests'))).toBe(true);
|
||||
expect(isRateLimitThrown(new Error('rate-limit exceeded'))).toBe(true);
|
||||
expect(isRateLimitThrown(Object.assign(new Error('x'), { name: 'RateLimitError' }))).toBe(true);
|
||||
expect(isRateLimitThrown(new Error('auth failed'))).toBe(false);
|
||||
expect(isRateLimitThrown(null)).toBe(false);
|
||||
});
|
||||
|
||||
test('isRateLimitResult matches error_during_execution with 429-shaped errors', () => {
|
||||
expect(isRateLimitResult(resultRateLimit())).toBe(true);
|
||||
expect(isRateLimitResult(resultSuccess())).toBe(false);
|
||||
});
|
||||
|
||||
test('isRateLimitEvent matches rate_limit_event with status=rejected', () => {
|
||||
expect(isRateLimitEvent(rateLimitEvent())).toBe(true);
|
||||
expect(isRateLimitEvent(resultSuccess())).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Semaphore concurrency cap
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('runAgentSdkTest — concurrency', () => {
|
||||
test('process-level semaphore caps concurrent queries', async () => {
|
||||
__resetSemaphoreForTests(2);
|
||||
let inFlight = 0;
|
||||
let peakInFlight = 0;
|
||||
const slowStub: QueryProvider = () => {
|
||||
const gen = (async function* (): AsyncGenerator<SDKMessage, void> {
|
||||
inFlight++;
|
||||
if (inFlight > peakInFlight) peakInFlight = inFlight;
|
||||
yield systemInit();
|
||||
await new Promise((r) => setTimeout(r, 30));
|
||||
yield assistantTurn([{ type: 'text', text: 'ok' }]);
|
||||
yield resultSuccess();
|
||||
inFlight--;
|
||||
})();
|
||||
return gen as unknown as Query;
|
||||
};
|
||||
|
||||
await Promise.all(
|
||||
Array.from({ length: 6 }, (_, i) =>
|
||||
runAgentSdkTest({
|
||||
...BASE_OPTS,
|
||||
userPrompt: `trial-${i}`,
|
||||
queryProvider: slowStub,
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
expect(peakInFlight).toBeLessThanOrEqual(2);
|
||||
expect(peakInFlight).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// toSkillTestResult shape
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('toSkillTestResult', () => {
|
||||
test('produces a SkillTestResult-shaped object', async () => {
|
||||
freshSem();
|
||||
const stub: StubConfig = {
|
||||
streams: [[systemInit(), assistantTurn([{ type: 'text', text: 'hi' }]), resultSuccess(0.02, 1)]],
|
||||
calls: [],
|
||||
};
|
||||
const r = await runAgentSdkTest({
|
||||
...BASE_OPTS,
|
||||
queryProvider: makeStubProvider(stub),
|
||||
});
|
||||
const s = toSkillTestResult(r);
|
||||
expect(s.toolCalls).toBeArray();
|
||||
expect(s.browseErrors).toBeArray();
|
||||
expect(s.exitReason).toBe('success');
|
||||
expect(s.duration).toBeNumber();
|
||||
expect(s.output).toBe('hi');
|
||||
expect(s.costEstimate.estimatedCost).toBe(0.02);
|
||||
expect(s.costEstimate.turnsUsed).toBe(1);
|
||||
expect(s.model).toBe('claude-opus-4-7');
|
||||
expect(s.firstResponseMs).toBeNumber();
|
||||
expect(s.maxInterTurnMs).toBeNumber();
|
||||
expect(s.transcript).toBeArray();
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Fixture validator
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('validateFixtures', () => {
|
||||
function base(overrides: Partial<OverlayFixture> = {}): OverlayFixture {
|
||||
return {
|
||||
id: 'test-fixture',
|
||||
overlayPath: 'model-overlays/opus-4-7.md',
|
||||
model: 'claude-opus-4-7',
|
||||
trials: 10,
|
||||
setupWorkspace: () => {},
|
||||
userPrompt: 'go',
|
||||
metric: () => 0,
|
||||
pass: fanoutPass,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('passes for a valid fixture', () => {
|
||||
expect(() => validateFixtures([base()])).not.toThrow();
|
||||
});
|
||||
|
||||
test('rejects empty id', () => {
|
||||
expect(() => validateFixtures([base({ id: '' })])).toThrow(/id must be/);
|
||||
});
|
||||
|
||||
test('rejects id with uppercase or unsafe chars', () => {
|
||||
expect(() => validateFixtures([base({ id: 'Test_Fixture' })])).toThrow(/id must be/);
|
||||
});
|
||||
|
||||
test('rejects duplicate ids', () => {
|
||||
expect(() => validateFixtures([base(), base()])).toThrow(/duplicate fixture id/);
|
||||
});
|
||||
|
||||
test('rejects non-integer trials', () => {
|
||||
expect(() => validateFixtures([base({ trials: 3.5 })])).toThrow(/trials must be/);
|
||||
});
|
||||
|
||||
test('rejects trials < 3', () => {
|
||||
expect(() => validateFixtures([base({ trials: 2 })])).toThrow(/trials must be/);
|
||||
});
|
||||
|
||||
test('rejects concurrency < 1', () => {
|
||||
expect(() => validateFixtures([base({ concurrency: 0 })])).toThrow(/concurrency must be/);
|
||||
});
|
||||
|
||||
test('rejects non-integer concurrency', () => {
|
||||
expect(() => validateFixtures([base({ concurrency: 2.5 })])).toThrow(/concurrency must be/);
|
||||
});
|
||||
|
||||
test('rejects empty model', () => {
|
||||
expect(() => validateFixtures([base({ model: '' })])).toThrow(/model must be/);
|
||||
});
|
||||
|
||||
test('rejects empty userPrompt', () => {
|
||||
expect(() => validateFixtures([base({ userPrompt: '' })])).toThrow(/userPrompt must be/);
|
||||
});
|
||||
|
||||
test('rejects absolute overlayPath', () => {
|
||||
expect(() => validateFixtures([base({ overlayPath: '/etc/passwd' })])).toThrow(/overlayPath must be/);
|
||||
});
|
||||
|
||||
test("rejects overlayPath containing '..'", () => {
|
||||
expect(() =>
|
||||
validateFixtures([base({ overlayPath: '../outside/file.md' })]),
|
||||
).toThrow(/overlayPath must be/);
|
||||
});
|
||||
|
||||
test('rejects missing overlay file', () => {
|
||||
expect(() =>
|
||||
validateFixtures([base({ overlayPath: 'model-overlays/nonexistent.md' })]),
|
||||
).toThrow(/overlay file not found/);
|
||||
});
|
||||
|
||||
test('rejects non-function setupWorkspace', () => {
|
||||
expect(() =>
|
||||
validateFixtures([base({ setupWorkspace: 'not a function' as unknown as (d: string) => void })]),
|
||||
).toThrow(/setupWorkspace must be a function/);
|
||||
});
|
||||
|
||||
test('rejects non-function metric', () => {
|
||||
expect(() =>
|
||||
validateFixtures([base({ metric: null as unknown as (r: AgentSdkResult) => number })]),
|
||||
).toThrow(/metric must be a function/);
|
||||
});
|
||||
|
||||
test('rejects non-function pass', () => {
|
||||
expect(() =>
|
||||
validateFixtures([base({ pass: undefined as unknown as OverlayFixture['pass'] })]),
|
||||
).toThrow(/pass must be a function/);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// fanoutPass predicate
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('fanoutPass predicate', () => {
|
||||
test('accepts mean lift >= 0.5 AND >=3/10 overlay trials >= 2', () => {
|
||||
const overlay = [2, 2, 2, 2, 2, 2, 2, 2, 2, 2];
|
||||
const off = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
|
||||
expect(fanoutPass({ overlay, off })).toBe(true);
|
||||
});
|
||||
|
||||
test('rejects when mean lift < 0.5', () => {
|
||||
const overlay = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1];
|
||||
const off = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1];
|
||||
expect(fanoutPass({ overlay, off })).toBe(false);
|
||||
});
|
||||
|
||||
test('rejects when mean lift >= 0.5 but <3 overlay trials emit >=2', () => {
|
||||
// Mean overlay = 1.2, off = 0.0, lift 1.2 but only 2 trials at >=2
|
||||
const overlay = [2, 2, 1, 1, 1, 1, 1, 1, 1, 1];
|
||||
const off = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
|
||||
expect(fanoutPass({ overlay, off })).toBe(false);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user