gstack/test/agent-sdk-runner.test.ts

/**
 * Unit tests for test/helpers/agent-sdk-runner.ts.
 *
 * Runs in free `bun test` (no API calls). Uses a stub QueryProvider to
 * simulate SDK event streams — happy path, rate-limit retries across all
 * three shapes, persistent failure, non-retryable error, options
 * propagation, concurrency cap.
 *
 * Also covers validateFixtures() rejections.
 */

import { describe, test, expect } from 'bun:test';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import type {
  SDKMessage,
  Options,
  Query,
} from '@anthropic-ai/claude-agent-sdk';
import {
  runAgentSdkTest,
  toSkillTestResult,
  firstTurnParallelism,
  isRateLimitThrown,
  isRateLimitResult,
  isRateLimitEvent,
  RateLimitExhaustedError,
  __resetSemaphoreForTests,
  type QueryProvider,
  type AgentSdkResult,
} from '../test/helpers/agent-sdk-runner';
import {
  validateFixtures,
  fanoutPass,
  type OverlayFixture,
} from '../test/fixtures/overlay-nudges';
import { firstAssistantMessageToolCount } from './helpers/overlay-measurement';
import { CLAUDE_CAPTURE_EVAL_MODEL } from '../lib/eval-model';

// ---------------------------------------------------------------------------
// Stub SDK event builders
// ---------------------------------------------------------------------------

let uuidCounter = 0;
function uuid(): string {
  return `00000000-0000-0000-0000-${String(++uuidCounter).padStart(12, '0')}`;
}

function systemInit(model = CLAUDE_CAPTURE_EVAL_MODEL, version = '2.1.117'): SDKMessage {
  return {
    type: 'system',
    subtype: 'init',
    apiKeySource: 'user',
    claude_code_version: version,
    cwd: '/tmp/x',
    tools: ['Read'],
    mcp_servers: [],
    model,
    permissionMode: 'bypassPermissions',
    slash_commands: [],
    output_style: 'default',
    skills: [],
    plugins: [],
    uuid: uuid(),
    session_id: 'test-session',
  } as unknown as SDKMessage;
}

function assistantTurn(
  blocks: Array<{ type: 'text'; text: string } | { type: 'tool_use'; name: string; input: unknown }>,
): SDKMessage {
  return {
    type: 'assistant',
    parent_tool_use_id: null,
    uuid: uuid(),
    session_id: 'test-session',
    message: {
      id: 'msg_' + uuid(),
      type: 'message',
      role: 'assistant',
      model: 'claude-sonnet-4-6',
      content: blocks.map((b) => ({ ...b })),
      stop_reason: 'end_turn',
      stop_sequence: null,
      usage: {
        input_tokens: 10,
        output_tokens: 20,
        cache_creation_input_tokens: 0,
        cache_read_input_tokens: 0,
        service_tier: 'standard',
      },
    },
  } as unknown as SDKMessage;
}

function resultSuccess(cost = 0.01, turns = 1): SDKMessage {
  return {
    type: 'result',
    subtype: 'success',
    duration_ms: 100,
    duration_api_ms: 50,
    is_error: false,
    num_turns: turns,
    result: 'done',
    stop_reason: 'end_turn',
    total_cost_usd: cost,
    usage: {
      input_tokens: 10,
      output_tokens: 20,
      cache_creation_input_tokens: 0,
      cache_read_input_tokens: 0,
      server_tool_use: {},
      service_tier: 'standard',
    },
    modelUsage: {},
    permission_denials: [],
    uuid: uuid(),
    session_id: 'test-session',
  } as unknown as SDKMessage;
}

function resultRateLimit(): SDKMessage {
  return {
    type: 'result',
    subtype: 'error_during_execution',
    duration_ms: 100,
    duration_api_ms: 50,
    is_error: true,
    num_turns: 0,
    stop_reason: null,
    total_cost_usd: 0,
    usage: {
      input_tokens: 0,
      output_tokens: 0,
      cache_creation_input_tokens: 0,
      cache_read_input_tokens: 0,
      server_tool_use: {},
      service_tier: 'standard',
    },
    modelUsage: {},
    permission_denials: [],
    errors: ['rate limit exceeded (429)'],
    uuid: uuid(),
    session_id: 'test-session',
  } as unknown as SDKMessage;
}

function rateLimitEvent(): SDKMessage {
  return {
    type: 'rate_limit_event',
    rate_limit_info: {
      status: 'rejected',
      rateLimitType: 'five_hour',
    },
    uuid: uuid(),
    session_id: 'test-session',
  } as unknown as SDKMessage;
}

// ---------------------------------------------------------------------------
// Stub query provider
// ---------------------------------------------------------------------------

interface StubConfig {
  /** One event stream per call. Exhausted calls throw. */
  streams: SDKMessage[][];
  /** Throw this error on the Nth call (0-indexed). */
  throwAt?: number;
  throwError?: unknown;
  /** Track calls for assertions. */
  calls: Array<{ prompt: string; options: Options | undefined; startedAt: number; endedAt?: number }>;
}

function makeStubProvider(config: StubConfig): QueryProvider {
  let callIdx = -1;
  const provider: QueryProvider = (params) => {
    callIdx++;
    const idx = callIdx;
    const startedAt = Date.now();
    const prompt = typeof params.prompt === 'string' ? params.prompt : '<iterable>';
    config.calls.push({ prompt, options: params.options, startedAt });

    if (config.throwAt !== undefined && idx === config.throwAt) {
      const err = config.throwError ?? new Error('stub throw');
      // Return an async generator that throws on first next().
      const gen = (async function* (): AsyncGenerator<SDKMessage, void> {
        throw err;
      })();
      return gen as unknown as Query;
    }

    const stream = config.streams[idx];
    if (!stream) {
      const gen = (async function* (): AsyncGenerator<SDKMessage, void> {
        throw new Error(`stub has no stream for call ${idx}`);
      })();
      return gen as unknown as Query;
    }

    const gen = (async function* (): AsyncGenerator<SDKMessage, void> {
      try {
        for (const ev of stream) {
          yield ev;
        }
      } finally {
        config.calls[idx]!.endedAt = Date.now();
      }
    })();
    return gen as unknown as Query;
  };
  return provider;
}

const BASE_OPTS = {
  systemPrompt: { type: 'sdk-default', reason: 'stub queryProvider: no model reads the prompt' } as const,
  userPrompt: 'test prompt',
  workingDirectory: '/tmp/test-dir',
  maxRetries: 3,
};

// Reset semaphore before each test that depends on fresh capacity.
function freshSem(cap = 10): void {
  __resetSemaphoreForTests(cap);
}

// ---------------------------------------------------------------------------
// Happy path
// ---------------------------------------------------------------------------

describe('runAgentSdkTest — happy path', () => {
  test('collects events, assistantTurns, toolCalls, and result fields', async () => {
    freshSem();
    const stub: StubConfig = {
      streams: [
        [
          systemInit(),
          assistantTurn([
            { type: 'text', text: 'reading files' },
            { type: 'tool_use', name: 'Read', input: { path: 'a.txt' } },
            { type: 'tool_use', name: 'Read', input: { path: 'b.txt' } },
          ]),
          assistantTurn([{ type: 'text', text: 'done' }]),
          resultSuccess(0.05, 2),
        ],
      ],
      calls: [],
    };
    const result = await runAgentSdkTest({
      ...BASE_OPTS,
      queryProvider: makeStubProvider(stub),
    });

    expect(result.events.length).toBe(4);
    expect(result.assistantTurns.length).toBe(2);
    expect(result.toolCalls.length).toBe(2);
    expect(result.toolCalls[0]!.tool).toBe('Read');
    expect(result.output).toContain('reading files');
    expect(result.output).toContain('done');
    expect(result.exitReason).toBe('success');
    expect(result.turnsUsed).toBe(2);
    expect(result.costUsd).toBe(0.05);
    expect(result.sdkClaudeCodeVersion).toBe('2.1.117');
    expect(result.model).toBe(CLAUDE_CAPTURE_EVAL_MODEL);
    expect(result.firstResponseMs).toBeGreaterThanOrEqual(0);
  });

  test('first-turn parallelism: 3 tool_use blocks in first assistant turn', async () => {
    freshSem();
    const stub: StubConfig = {
      streams: [
        [
          systemInit(),
          assistantTurn([
            { type: 'tool_use', name: 'Read', input: { path: 'a' } },
            { type: 'tool_use', name: 'Read', input: { path: 'b' } },
            { type: 'tool_use', name: 'Read', input: { path: 'c' } },
          ]),
          resultSuccess(),
        ],
      ],
      calls: [],
    };
    const result = await runAgentSdkTest({
      ...BASE_OPTS,
      queryProvider: makeStubProvider(stub),
    });
    expect(firstTurnParallelism(result.assistantTurns[0])).toBe(3);
  });

  test('first-turn parallelism: 0 when first turn is text-only', async () => {
    freshSem();
    const stub: StubConfig = {
      streams: [
        [
          systemInit(),
          assistantTurn([{ type: 'text', text: 'thinking' }]),
          resultSuccess(),
        ],
      ],
      calls: [],
    };
    const result = await runAgentSdkTest({
      ...BASE_OPTS,
      queryProvider: makeStubProvider(stub),
    });
    expect(firstTurnParallelism(result.assistantTurns[0])).toBe(0);
  });

  test('first-turn parallelism: 0 when no first turn', () => {
    expect(firstTurnParallelism(undefined)).toBe(0);
  });
});

// ---------------------------------------------------------------------------
// Terminal usage, including SDK errors thrown after a terminal event
// ---------------------------------------------------------------------------

describe('runAgentSdkTest — terminal usage', () => {
  for (const throwsAfterTerminal of [false, true]) {
    test(`preserves max-turns terminal usage through serialization (${throwsAfterTerminal ? 'then throws' : 'EOF'})`, async () => {
      freshSem();
      const terminal = {
        ...resultRateLimit(), subtype: 'error_max_turns', num_turns: 26,
        total_cost_usd: 0.5610850000000001,
        errors: ['Reached maximum number of turns (25)'],
      } as SDKMessage;
      // Assistant event chunks are not the SDK's authoritative turn count.
      const stream = [systemInit(), assistantTurn([{ type: 'text', text: 'working' }]),
        assistantTurn([{ type: 'text', text: 'still working' }]), terminal];
      let calls = 0;
      const queryProvider: QueryProvider = () => {
        calls++;
        return (async function* () {
          yield* stream;
          if (throwsAfterTerminal) throw new Error('Reached maximum number of turns (25)');
        })() as unknown as Query;
      };
      const result = await runAgentSdkTest({ ...BASE_OPTS, queryProvider });
      expect(calls).toBe(1);
      expect(result.exitReason).toBe('error_max_turns');
      expect(result.assistantTurns).toHaveLength(2);
      expect({ turnsUsed: result.turnsUsed, costUsd: result.costUsd })
        .toEqual({ turnsUsed: 26, costUsd: 0.5610850000000001 });
      expect(result.events).toEqual(stream);
      const stored = JSON.parse(JSON.stringify(toSkillTestResult(result)));
      expect(stored.exitReason).toBe('error_max_turns');
      expect(stored.costEstimate.turnsUsed).toBe(26);
      expect(stored.costEstimate.estimatedCost).toBe(0.5610850000000001);
      expect(stored.transcript.at(-1)).toEqual(terminal);
    });
  }

  for (const subtype of ['error_during_execution', 'error_max_budget_usd']) {
    test(`preserves non-rate-limit ${subtype} terminal fields`, async () => {
      freshSem();
      const terminal = { ...resultRateLimit(), subtype, num_turns: 3,
        total_cost_usd: 0.25, errors: ['model execution stopped'] } as SDKMessage;
      const stub: StubConfig = { streams: [[systemInit(), terminal]], calls: [] };
      const result = await runAgentSdkTest({ ...BASE_OPTS, queryProvider: makeStubProvider(stub) });
      expect(stub.calls).toHaveLength(1);
      expect(result.exitReason).toBe(subtype);
      expect(result.turnsUsed).toBe(3);
      expect(result.costUsd).toBe(0.25);
      expect(result.events.at(-1)).toEqual(terminal);
    });
  }

  test('keeps the existing unknown-usage fallback when max turns throws without a terminal', async () => {
    freshSem();
    const queryProvider: QueryProvider = () => (async function* () {
      yield systemInit();
      yield assistantTurn([{ type: 'text', text: 'partial output' }]);
      throw new Error('Reached maximum number of turns (25)');
    })() as unknown as Query;
    const result = await runAgentSdkTest({ ...BASE_OPTS, queryProvider });
    expect(result.exitReason).toBe('error_max_turns');
    expect(result.turnsUsed).toBe(1);
    expect(result.costUsd).toBe(0); // Still unknown, not a zero-cost billing claim.
    expect(result.output).toBe('partial output');
    expect(result.events.some(event => event.type === 'result')).toBe(false);
  });

  test('does not swallow a generic error after a terminal event', async () => {
    freshSem();
    const failure = new Error('stream transport failed after terminal');
    const queryProvider: QueryProvider = () => (async function* () {
      yield resultSuccess(0.1, 2);
      throw failure;
    })() as unknown as Query;
    await expect(runAgentSdkTest({ ...BASE_OPTS, queryProvider })).rejects.toBe(failure);
  });

  test('a max-turns throw remains an error even after a prior success terminal', async () => {
    freshSem();
    const queryProvider: QueryProvider = () => (async function* () {
      yield resultSuccess(0.1, 2);
      throw new Error('Reached maximum number of turns (2)');
    })() as unknown as Query;
    const result = await runAgentSdkTest({ ...BASE_OPTS, queryProvider });
    expect(result.exitReason).toBe('error_max_turns');
    expect(result.turnsUsed).toBe(2);
    expect(result.costUsd).toBe(0.1);
  });
});

// ---------------------------------------------------------------------------
// Options propagation
// ---------------------------------------------------------------------------

describe('runAgentSdkTest — options propagation', () => {
  test('systemPrompt, model, cwd, allowedTools, disallowedTools, permissionMode, settingSources, env, pathToClaudeCodeExecutable reach query()', async () => {
    freshSem();
    const stub: StubConfig = {
      streams: [[systemInit(), assistantTurn([{ type: 'text', text: 'ok' }]), resultSuccess()]],
      calls: [],
    };
    await runAgentSdkTest({
      systemPrompt: { type: 'custom', text: 'you are a test overlay', reason: 'checks that custom text reaches query()' },
      userPrompt: 'go',
      workingDirectory: '/tmp/spec-dir',
      model: 'claude-opus-4-7',
      maxTurns: 7,
      allowedTools: ['Read', 'Glob'],
      disallowedTools: ['Bash', 'Write'],
      permissionMode: 'bypassPermissions',
      settingSources: [],
      env: { ANTHROPIC_API_KEY: 'fake' },
      pathToClaudeCodeExecutable: '/fake/path/claude',
      queryProvider: makeStubProvider(stub),
    });

    const opts = stub.calls[0]!.options!;
    expect(opts.systemPrompt).toBe('you are a test overlay');
    expect(opts.model).toBe('claude-opus-4-7');
    expect(opts.cwd).toBe('/tmp/spec-dir');
    expect(opts.maxTurns).toBe(7);
    expect(opts.tools).toEqual(['Read', 'Glob']);
    expect(opts.allowedTools).toEqual(['Read', 'Glob']);
    expect(opts.disallowedTools).toEqual(['Bash', 'Write']);
    expect(opts.permissionMode).toBe('bypassPermissions');
    expect(opts.allowDangerouslySkipPermissions).toBe(true);
    expect(opts.settingSources).toEqual([]);
    // env is the COMPLETE hermetic env with the per-test override merged
    // last — partial pass-through was the documented SDK auth-breaker
    // (Options.env replaces the child's entire environment).
    expect(opts.env?.ANTHROPIC_API_KEY).toBe('fake');
    expect(opts.env?.PATH).toBeTruthy();
    expect(opts.env?.CLAUDE_CONFIG_DIR).toMatch(/\/\.claude$/);
    expect(opts.env?.GSTACK_HOME).toContain('gstack-home');
    expect(opts.pathToClaudeCodeExecutable).toBe('/fake/path/claude');
  });

  test('an sdk-default systemPrompt passes no systemPrompt option', async () => {
    freshSem();
    const stub: StubConfig = {
      streams: [[systemInit(), assistantTurn([{ type: 'text', text: 'ok' }]), resultSuccess()]],
      calls: [],
    };
    await runAgentSdkTest({
      ...BASE_OPTS,
      queryProvider: makeStubProvider(stub),
    });
    expect(stub.calls[0]!.options!.systemPrompt).toBeUndefined();
  });
});

// ---------------------------------------------------------------------------
// canUseTool extension (D10 CEO / D4 eng)
// ---------------------------------------------------------------------------

describe('runAgentSdkTest — canUseTool extension', () => {
  test('permissionMode flips to "default" when canUseTool is supplied', async () => {
    freshSem();
    const stub: StubConfig = {
      streams: [[systemInit(), assistantTurn([{ type: 'text', text: 'ok' }]), resultSuccess()]],
      calls: [],
    };
    await runAgentSdkTest({
      ...BASE_OPTS,
      queryProvider: makeStubProvider(stub),
      canUseTool: async (_toolName, input) => ({ behavior: 'allow', updatedInput: input }),
    });
    const opts = stub.calls[0]!.options!;
    expect(opts.permissionMode).toBe('default');
    expect(opts.allowDangerouslySkipPermissions).toBe(false);
  });

  test('permissionMode stays "bypassPermissions" when canUseTool is NOT supplied', async () => {
    freshSem();
    const stub: StubConfig = {
      streams: [[systemInit(), assistantTurn([{ type: 'text', text: 'ok' }]), resultSuccess()]],
      calls: [],
    };
    await runAgentSdkTest({
      ...BASE_OPTS,
      queryProvider: makeStubProvider(stub),
    });
    const opts = stub.calls[0]!.options!;
    expect(opts.permissionMode).toBe('bypassPermissions');
    expect(opts.allowDangerouslySkipPermissions).toBe(true);
  });

  test('canUseTool callback reaches the SDK options', async () => {
    freshSem();
    const stub: StubConfig = {
      streams: [[systemInit(), assistantTurn([{ type: 'text', text: 'ok' }]), resultSuccess()]],
      calls: [],
    };
    const cb = async (_toolName: string, input: Record<string, unknown>) => ({
      behavior: 'allow' as const,
      updatedInput: input,
    });
    await runAgentSdkTest({
      ...BASE_OPTS,
      queryProvider: makeStubProvider(stub),
      canUseTool: cb,
    });
    const opts = stub.calls[0]!.options! as Options & { canUseTool?: unknown };
    expect(typeof opts.canUseTool).toBe('function');
  });

  test('AskUserQuestion is auto-added to allowedTools when canUseTool is supplied', async () => {
    freshSem();
    const stub: StubConfig = {
      streams: [[systemInit(), assistantTurn([{ type: 'text', text: 'ok' }]), resultSuccess()]],
      calls: [],
    };
    await runAgentSdkTest({
      ...BASE_OPTS,
      allowedTools: ['Read', 'Grep'], // explicitly omits AskUserQuestion
      queryProvider: makeStubProvider(stub),
      canUseTool: async (_toolName, input) => ({ behavior: 'allow', updatedInput: input }),
    });
    const opts = stub.calls[0]!.options!;
    expect(opts.allowedTools).toContain('AskUserQuestion');
    expect(opts.tools).toContain('AskUserQuestion');
  });

  test('AskUserQuestion is NOT auto-added when canUseTool is absent', async () => {
    freshSem();
    const stub: StubConfig = {
      streams: [[systemInit(), assistantTurn([{ type: 'text', text: 'ok' }]), resultSuccess()]],
      calls: [],
    };
    await runAgentSdkTest({
      ...BASE_OPTS,
      allowedTools: ['Read', 'Grep'],
      queryProvider: makeStubProvider(stub),
    });
    const opts = stub.calls[0]!.options!;
    expect(opts.allowedTools).not.toContain('AskUserQuestion');
  });

  test('passThroughNonAskUserQuestion helper returns allow+updatedInput', async () => {
    const { passThroughNonAskUserQuestion } = await import('../test/helpers/agent-sdk-runner');
    const result = passThroughNonAskUserQuestion('Read', { file_path: '/tmp/x' });
    expect(result.behavior).toBe('allow');
    expect(result.updatedInput).toEqual({ file_path: '/tmp/x' });
  });
});

// ---------------------------------------------------------------------------
// Rate-limit retry (three shapes)
// ---------------------------------------------------------------------------

describe('runAgentSdkTest — rate-limit retry', () => {
  test('retryable on thrown 429-shaped error, then succeeds on 2nd attempt', async () => {
    freshSem();
    const stub: StubConfig = {
      streams: [
        // call 0: throws (handled via throwAt below)
        [],
        // call 1: success
        [systemInit(), assistantTurn([{ type: 'text', text: 'ok' }]), resultSuccess()],
      ],
      throwAt: 0,
      throwError: Object.assign(new Error('429 too many requests'), { status: 429 }),
      calls: [],
    };
    const result = await runAgentSdkTest({
      ...BASE_OPTS,
      queryProvider: makeStubProvider(stub),
      maxRetries: 2,
    });
    expect(result.exitReason).toBe('success');
    expect(stub.calls.length).toBe(2);
  });

  test('retryable on result-message rate-limit, then succeeds', async () => {
    freshSem();
    const stub: StubConfig = {
      streams: [
        [systemInit(), resultRateLimit()],
        [systemInit(), assistantTurn([{ type: 'text', text: 'ok' }]), resultSuccess()],
      ],
      calls: [],
    };
    const result = await runAgentSdkTest({
      ...BASE_OPTS,
      queryProvider: makeStubProvider(stub),
      maxRetries: 2,
    });
    expect(result.exitReason).toBe('success');
    expect(stub.calls.length).toBe(2);
  });

  test('retryable on mid-stream SDKRateLimitEvent, then succeeds', async () => {
    freshSem();
    const stub: StubConfig = {
      streams: [
        [systemInit(), rateLimitEvent()],
        [systemInit(), assistantTurn([{ type: 'text', text: 'ok' }]), resultSuccess()],
      ],
      calls: [],
    };
    const result = await runAgentSdkTest({
      ...BASE_OPTS,
      queryProvider: makeStubProvider(stub),
      maxRetries: 2,
    });
    expect(result.exitReason).toBe('success');
    expect(stub.calls.length).toBe(2);
  });

  test('onRetry callback is invoked between attempts', async () => {
    freshSem();
    const resets: string[] = [];
    const stub: StubConfig = {
      streams: [
        [],
        [systemInit(), assistantTurn([{ type: 'text', text: 'ok' }]), resultSuccess()],
      ],
      throwAt: 0,
      throwError: Object.assign(new Error('429'), { status: 429 }),
      calls: [],
    };
    await runAgentSdkTest({
      ...BASE_OPTS,
      queryProvider: makeStubProvider(stub),
      maxRetries: 2,
      onRetry: (dir) => resets.push(dir),
    });
    expect(resets.length).toBe(1);
    expect(resets[0]).toBe('/tmp/test-dir');
  });

  test('persistent 429 throws RateLimitExhaustedError after maxRetries', async () => {
    freshSem();
    const stub: StubConfig = {
      streams: [[], [], [], []], // 4 empty streams; throw on each
      calls: [],
    };
    // Every call throws:
    let callCount = 0;
    const alwaysThrowProvider: QueryProvider = (params) => {
      callCount++;
      stub.calls.push({
        prompt: typeof params.prompt === 'string' ? params.prompt : '',
        options: params.options,
        startedAt: Date.now(),
      });
      const gen = (async function* (): AsyncGenerator<SDKMessage, void> {
        throw Object.assign(new Error('429 always'), { status: 429 });
      })();
      return gen as unknown as Query;
    };

    let caught: unknown = null;
    try {
      await runAgentSdkTest({
        ...BASE_OPTS,
        queryProvider: alwaysThrowProvider,
        maxRetries: 2,
      });
    } catch (err) {
      caught = err;
    }
    expect(caught).toBeInstanceOf(RateLimitExhaustedError);
    expect((caught as RateLimitExhaustedError).attempts).toBe(3); // initial + 2 retries
    expect(callCount).toBe(3);
  });

  test('non-429 error is NOT retried, propagates immediately', async () => {
    __resetSemaphoreForTests(10);
    let callCount = 0;
    const throwOnce: QueryProvider = () => {
      callCount++;
      const gen = (async function* (): AsyncGenerator<SDKMessage, void> {
        throw new Error('generic auth failure');
      })();
      return gen as unknown as Query;
    };
    let caught: unknown = null;
    try {
      await runAgentSdkTest({
        ...BASE_OPTS,
        queryProvider: throwOnce,
        maxRetries: 3,
      });
    } catch (err) {
      caught = err;
    }
    expect(caught).toBeInstanceOf(Error);
    expect((caught as Error).message).toBe('generic auth failure');
    expect(callCount).toBe(1);
  });
});

// ---------------------------------------------------------------------------
// Rate-limit detectors (unit)
// ---------------------------------------------------------------------------

describe('rate-limit detectors', () => {
  test('isRateLimitThrown matches status 429, message, name', () => {
    expect(isRateLimitThrown(Object.assign(new Error('boom'), { status: 429 }))).toBe(true);
    expect(isRateLimitThrown(new Error('429 Too Many Requests'))).toBe(true);
    expect(isRateLimitThrown(new Error('rate-limit exceeded'))).toBe(true);
    expect(isRateLimitThrown(Object.assign(new Error('x'), { name: 'RateLimitError' }))).toBe(true);
    expect(isRateLimitThrown(new Error('auth failed'))).toBe(false);
    expect(isRateLimitThrown(null)).toBe(false);
  });

  test('isRateLimitResult matches error_during_execution with 429-shaped errors', () => {
    expect(isRateLimitResult(resultRateLimit())).toBe(true);
    expect(isRateLimitResult(resultSuccess())).toBe(false);
  });

  test('isRateLimitEvent matches rate_limit_event with status=rejected', () => {
    expect(isRateLimitEvent(rateLimitEvent())).toBe(true);
    expect(isRateLimitEvent(resultSuccess())).toBe(false);
  });
});

// ---------------------------------------------------------------------------
// Semaphore concurrency cap
// ---------------------------------------------------------------------------

describe('runAgentSdkTest — concurrency', () => {
  test('admission runs only once across a rate-limit retry', async () => {
    __resetSemaphoreForTests(1);
    let admissions = 0;
    const stub: StubConfig = {
      streams: [[systemInit(), resultRateLimit()], [systemInit(), resultSuccess()]], calls: [],
    };
    await runAgentSdkTest({ ...BASE_OPTS, queryProvider: makeStubProvider(stub), maxRetries: 1,
      onAdmission: () => { admissions++; } });
    expect(admissions).toBe(1);
    expect(stub.calls).toHaveLength(2);
  });

  test.each(['throw', 'abort'] as const)('admission %s prevents transport creation and releases its slot', async mode => {
    __resetSemaphoreForTests(1);
    const controller = new AbortController(), failure = new Error(`admission ${mode}`);
    const stub: StubConfig = { streams: [[systemInit(), resultSuccess()]], calls: [] };
    const queryProvider = makeStubProvider(stub);
    const failed = runAgentSdkTest({ ...BASE_OPTS, queryProvider, signal: controller.signal,
      onAdmission: () => { if (mode === 'throw') throw failure; controller.abort(failure); } });
    const sibling = runAgentSdkTest({ ...BASE_OPTS, queryProvider });
    const results = await Promise.allSettled([failed, sibling]);
    expect(results[0]).toEqual({ status: 'rejected', reason: failure });
    expect(results[1].status).toBe('fulfilled');
    expect(stub.calls).toHaveLength(1);
  });

  test('queued cancellation never calls admission or transport and leaves later siblings runnable', async () => {
    __resetSemaphoreForTests(1);
    const controller = new AbortController(), reason = new Error('queued cancellation');
    let unblock!: () => void, started!: () => void;
    const held = new Promise<void>(resolve => { unblock = resolve; });
    const ready = new Promise<void>(resolve => { started = resolve; });
    const provider: QueryProvider = () => (async function* () {
      started(); await held; yield resultSuccess();
    })() as unknown as Query;
    const occupying = runAgentSdkTest({ ...BASE_OPTS, queryProvider: provider });
    await ready;
    let admissions = 0;
    const stub: StubConfig = { streams: [[resultSuccess()]], calls: [] };
    const queryProvider = makeStubProvider(stub);
    const queued = runAgentSdkTest({ ...BASE_OPTS, queryProvider, signal: controller.signal,
      onAdmission: () => { admissions++; } });
    const sibling = runAgentSdkTest({ ...BASE_OPTS, queryProvider });
    controller.abort(reason);
    const cancelled = await Promise.allSettled([queued]);
    expect(cancelled[0]).toEqual({ status: 'rejected', reason });
    expect(admissions).toBe(0); expect(stub.calls).toHaveLength(0);
    unblock();
    await Promise.all([occupying, sibling]);
    expect(stub.calls).toHaveLength(1);
  });

  test('process-level semaphore caps concurrent queries', async () => {
    __resetSemaphoreForTests(2);
    let inFlight = 0;
    let peakInFlight = 0;
    const slowStub: QueryProvider = () => {
      const gen = (async function* (): AsyncGenerator<SDKMessage, void> {
        inFlight++;
        if (inFlight > peakInFlight) peakInFlight = inFlight;
        yield systemInit();
        await new Promise((r) => setTimeout(r, 30));
        yield assistantTurn([{ type: 'text', text: 'ok' }]);
        yield resultSuccess();
        inFlight--;
      })();
      return gen as unknown as Query;
    };

    await Promise.all(
      Array.from({ length: 6 }, (_, i) =>
        runAgentSdkTest({
          ...BASE_OPTS,
          userPrompt: `trial-${i}`,
          queryProvider: slowStub,
        }),
      ),
    );

    expect(peakInFlight).toBeLessThanOrEqual(2);
    expect(peakInFlight).toBeGreaterThan(0);
  });
});

// ---------------------------------------------------------------------------
// toSkillTestResult shape
// ---------------------------------------------------------------------------

describe('toSkillTestResult', () => {
  test('produces a SkillTestResult-shaped object', async () => {
    freshSem();
    const stub: StubConfig = {
      streams: [[systemInit(), assistantTurn([{ type: 'text', text: 'hi' }]), resultSuccess(0.02, 1)]],
      calls: [],
    };
    const r = await runAgentSdkTest({
      ...BASE_OPTS,
      queryProvider: makeStubProvider(stub),
    });
    const s = toSkillTestResult(r);
    expect(s.toolCalls).toBeArray();
    expect(s.browseErrors).toBeArray();
    expect(s.exitReason).toBe('success');
    expect(s.duration).toBeNumber();
    expect(s.output).toBe('hi');
    expect(s.costEstimate.estimatedCost).toBe(0.02);
    expect(s.costEstimate.turnsUsed).toBe(1);
    expect(s.model).toBe(CLAUDE_CAPTURE_EVAL_MODEL);
    expect(s.firstResponseMs).toBeNumber();
    expect(s.maxInterTurnMs).toBeNumber();
    expect(s.transcript).toBeArray();
  });
});

// ---------------------------------------------------------------------------
// Fixture validator
// ---------------------------------------------------------------------------

describe('validateFixtures', () => {
  function base(overrides: Partial<OverlayFixture> = {}): OverlayFixture {
    return {
      id: 'test-fixture',
      overlayPath: 'model-overlays/opus-4-7.md',
      model: 'claude-sonnet-4-6',
      trials: 10,
      setupWorkspace: () => {},
      userPrompt: 'go',
      metric: () => 0,
      pass: fanoutPass,
      ...overrides,
    };
  }

  test('passes for a valid fixture', () => {
    expect(() => validateFixtures([base()])).not.toThrow();
  });

  test('rejects empty id', () => {
    expect(() => validateFixtures([base({ id: '' })])).toThrow(/id must be/);
  });

  test('rejects id with uppercase or unsafe chars', () => {
    expect(() => validateFixtures([base({ id: 'Test_Fixture' })])).toThrow(/id must be/);
  });

  test('rejects duplicate ids', () => {
    expect(() => validateFixtures([base(), base()])).toThrow(/duplicate fixture id/);
  });

  test('rejects non-integer trials', () => {
    expect(() => validateFixtures([base({ trials: 3.5 })])).toThrow(/trials must be/);
  });

  test('rejects trials < 3', () => {
    expect(() => validateFixtures([base({ trials: 2 })])).toThrow(/trials must be/);
  });

  test('rejects concurrency < 1', () => {
    expect(() => validateFixtures([base({ concurrency: 0 })])).toThrow(/concurrency must be/);
  });

  test('rejects non-integer concurrency', () => {
    expect(() => validateFixtures([base({ concurrency: 2.5 })])).toThrow(/concurrency must be/);
  });

  test('rejects empty model', () => {
    expect(() => validateFixtures([base({ model: '' })])).toThrow(/model must be/);
  });

  test('rejects empty userPrompt', () => {
    expect(() => validateFixtures([base({ userPrompt: '' })])).toThrow(/userPrompt must be/);
  });

  test('rejects absolute overlayPath', () => {
    expect(() => validateFixtures([base({ overlayPath: '/etc/passwd' })])).toThrow(/overlayPath must be/);
  });

  test("rejects overlayPath containing '..'", () => {
    expect(() =>
      validateFixtures([base({ overlayPath: '../outside/file.md' })]),
    ).toThrow(/overlayPath must be/);
  });

  test('rejects missing overlay file', () => {
    expect(() =>
      validateFixtures([base({ overlayPath: 'model-overlays/nonexistent.md' })]),
    ).toThrow(/overlay file not found/);
  });

  test('rejects non-function setupWorkspace', () => {
    expect(() =>
      validateFixtures([base({ setupWorkspace: 'not a function' as unknown as (d: string) => void })]),
    ).toThrow(/setupWorkspace must be a function/);
  });

  test('rejects non-function metric', () => {
    expect(() =>
      validateFixtures([base({ metric: null as unknown as (r: AgentSdkResult) => number })]),
    ).toThrow(/metric must be a function/);
  });

  test('rejects non-function pass', () => {
    expect(() =>
      validateFixtures([base({ pass: undefined as unknown as OverlayFixture['pass'] })]),
    ).toThrow(/pass must be a function/);
  });
});

// ---------------------------------------------------------------------------
// fanoutPass predicate
// ---------------------------------------------------------------------------

describe('overlay first logical message metric', () => {
  // Public SDK shape: separate assistant events share one message.id, and
  // tool results may arrive between them. The initial empty public event
  // carries no inspected private content. All IDs here are synthetic.
  function splitResponse(): AgentSdkResult {
    const initial = systemInit();
    const event = (messageId: string, id?: string) => {
      const e = assistantTurn(id ? [{ type: 'tool_use', name: 'Read', input: {} }] : []) as any;
      e.message.id = messageId;
      if (id) e.message.content[0].id = id;
      return e;
    };
    const turns = [event('first'), event('first', 'alpha'), event('first', 'beta'), event('first', 'gamma'), event('later', 'later-tool')];
    const result = { type: 'user', session_id: 'test-session', parent_tool_use_id: null,
      message: { role: 'user', content: [{ type: 'tool_result', tool_use_id: 'alpha', content: 'Alpha' }] } };
    return { events: [initial, turns[0], turns[1], result, ...turns.slice(2)], assistantTurns: turns } as unknown as AgentSdkResult;
  }
  test('counts one split first response across interleaved results independently of paid fixture registration', () => {
    expect(firstAssistantMessageToolCount(splitResponse())).toBe(3);
  });
  test('a combined message and repeated tool ID have the same count', () => {
    for (const combined of [false, true]) {
      const r = splitResponse();
      if (combined) {
        (r.assistantTurns[0]!.message.content as any[]).push(...r.assistantTurns.slice(1, 4).flatMap(e => e.message.content as any[]));
      } else r.assistantTurns.splice(3, 0, structuredClone(r.assistantTurns[1]!));
      expect(firstAssistantMessageToolCount(r)).toBe(3);
    }
  });
  test('child, foreign-session and later-response tools cannot inflate the first response', () => {
    const r = splitResponse();
    const foreign = structuredClone(r.assistantTurns[1]!) as any;
    foreign.session_id = 'other-session'; foreign.message.content[0].id = 'foreign';
    const child = structuredClone(r.assistantTurns[1]!) as any;
    child.parent_tool_use_id = 'agent-tool'; child.message.content[0].id = 'child';
    r.assistantTurns.unshift(child, foreign);
    expect(firstAssistantMessageToolCount(r)).toBe(3);
  });
  test('missing first-response identity cannot borrow a later response', () => {
    for (const field of ['id', 'session_id']) {
      const r = splitResponse();
      if (field === 'id') (r.assistantTurns[0]!.message as any).id = '';
      else (r.events[0] as any).session_id = '';
      expect(() => firstAssistantMessageToolCount(r)).toThrow(field === 'id' ? 'message.id' : 'session_id');
    }
  });
});

describe('fanoutPass predicate', () => {
  test('accepts mean lift >= 0.5 AND >=3/10 overlay trials >= 2', () => {
    const overlay = [2, 2, 2, 2, 2, 2, 2, 2, 2, 2];
    const off = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
    expect(fanoutPass({ overlay, off })).toBe(true);
  });

  test('rejects when mean lift < 0.5', () => {
    const overlay = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1];
    const off = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1];
    expect(fanoutPass({ overlay, off })).toBe(false);
  });

  test('rejects when mean lift >= 0.5 but <3 overlay trials emit >=2', () => {
    // Mean overlay = 1.2, off = 0.0, lift 1.2 but only 2 trials at >=2
    const overlay = [2, 2, 1, 1, 1, 1, 1, 1, 1, 1];
    const off = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
    expect(fanoutPass({ overlay, off })).toBe(false);
  });
});