diff --git a/test/fixtures/overlay-nudges.ts b/test/fixtures/overlay-nudges.ts new file mode 100644 index 00000000..9d9cecc4 --- /dev/null +++ b/test/fixtures/overlay-nudges.ts @@ -0,0 +1,179 @@ +/** + * Overlay-efficacy fixture registry. + * + * Each fixture defines a reproducible A/B test for one behavioral nudge + * embedded in a model-overlays/*.md file. The harness at + * test/skill-e2e-overlay-harness.test.ts iterates this registry and runs + * `fixture.trials` A/B trials per fixture, asserting `fixture.pass(arms)`. + * + * Adding a new overlay eval = one entry in this list. The harness handles + * arm wiring, concurrency, artifact storage, rate-limit retries, and the + * cross-harness diagnostic. + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import { + firstTurnParallelism, + type AgentSdkResult, +} from '../helpers/agent-sdk-runner'; + +const REPO_ROOT = path.resolve(__dirname, '..', '..'); + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export interface OverlayFixture { + /** Unique, lowercase/digits/dash only. Used in artifact paths. */ + id: string; + /** Path to the overlay file, relative to repo root. */ + overlayPath: string; + /** API model ID, not the overlay family name. */ + model: string; + /** Integer >= 3. Trials per arm. */ + trials: number; + /** Max concurrent queries for this fixture's arms. Default 3. */ + concurrency?: number; + /** Populate the workspace dir before each trial. */ + setupWorkspace: (dir: string) => void; + /** The prompt the model receives. Non-empty. */ + userPrompt: string; + /** Compute the per-trial metric from the typed SDK result. */ + metric: (r: AgentSdkResult) => number; + /** Acceptance predicate across all arms' per-trial metrics. */ + pass: (arms: { overlay: number[]; off: number[] }) => boolean; +} + +// --------------------------------------------------------------------------- +// Validation +// --------------------------------------------------------------------------- + +export function validateFixtures(fixtures: OverlayFixture[]): void { + const ids = new Set(); + for (const f of fixtures) { + if (!f.id || !/^[a-z0-9-]+$/.test(f.id)) { + throw new Error( + `fixture id must be non-empty, lowercase/digits/dash only: ${JSON.stringify(f.id)}`, + ); + } + if (ids.has(f.id)) { + throw new Error(`duplicate fixture id: ${f.id}`); + } + ids.add(f.id); + + if (!Number.isInteger(f.trials) || f.trials < 3) { + throw new Error(`${f.id}: trials must be an integer >= 3 (got ${f.trials})`); + } + if ( + f.concurrency !== undefined && + (!Number.isInteger(f.concurrency) || f.concurrency < 1) + ) { + throw new Error( + `${f.id}: concurrency must be an integer >= 1 (got ${f.concurrency})`, + ); + } + + if (!f.model) throw new Error(`${f.id}: model must be non-empty`); + if (!f.userPrompt) throw new Error(`${f.id}: userPrompt must be non-empty`); + + if (path.isAbsolute(f.overlayPath) || f.overlayPath.includes('..')) { + throw new Error( + `${f.id}: overlayPath must be relative and must not contain '..' (got ${f.overlayPath})`, + ); + } + const fullPath = path.resolve(REPO_ROOT, f.overlayPath); + if (!fs.existsSync(fullPath)) { + throw new Error(`${f.id}: overlay file not found at ${f.overlayPath}`); + } + + for (const fn of ['setupWorkspace', 'metric', 'pass'] as const) { + if (typeof f[fn] !== 'function') { + throw new Error(`${f.id}: ${fn} must be a function`); + } + } + } +} + +// --------------------------------------------------------------------------- +// Metric + predicate helpers +// --------------------------------------------------------------------------- + +function mean(xs: number[]): number { + if (xs.length === 0) return 0; + return xs.reduce((a, b) => a + b, 0) / xs.length; +} + +/** + * Standard fanout predicate: overlay mean beats off mean by at least 0.5 + * parallel tool_use blocks in first turn, AND at least 3 of the overlay + * trials emit >= 2 parallel tool_use blocks. + * + * The combined rule catches both "overlay nudges every trial slightly" + * (mean) and "overlay sometimes triggers real fanout" (floor). A single + * 0.5 lift with every trial still emitting 1 call would be suspicious; + * this predicate rejects it. + */ +export function fanoutPass(arms: { overlay: number[]; off: number[] }): boolean { + const lift = mean(arms.overlay) - mean(arms.off); + const floorHits = arms.overlay.filter((n) => n >= 2).length; + return lift >= 0.5 && floorHits >= 3; +} + +// --------------------------------------------------------------------------- +// Fixtures +// --------------------------------------------------------------------------- + +export const OVERLAY_FIXTURES: OverlayFixture[] = [ + { + id: 'opus-4-7-fanout-toy', + overlayPath: 'model-overlays/opus-4-7.md', + model: 'claude-opus-4-7', + trials: 10, + concurrency: 3, + setupWorkspace: (dir) => { + fs.writeFileSync(path.join(dir, 'alpha.txt'), 'Alpha file: used in module A.\n'); + fs.writeFileSync(path.join(dir, 'beta.txt'), 'Beta file: used in module B.\n'); + fs.writeFileSync(path.join(dir, 'gamma.txt'), 'Gamma file: used in module C.\n'); + }, + userPrompt: + 'Read alpha.txt, beta.txt, and gamma.txt and summarize each in one line.', + metric: (r) => firstTurnParallelism(r.assistantTurns[0]), + pass: fanoutPass, + }, + { + id: 'opus-4-7-fanout-realistic', + overlayPath: 'model-overlays/opus-4-7.md', + model: 'claude-opus-4-7', + trials: 10, + concurrency: 3, + setupWorkspace: (dir) => { + fs.writeFileSync( + path.join(dir, 'app.ts'), + "import { config } from './config';\nimport { util } from './src/util';\n\nexport function main() { return config.name + ':' + util(); }\n", + ); + fs.writeFileSync( + path.join(dir, 'config.ts'), + "export const config = { name: 'demo', version: 1 };\n", + ); + fs.writeFileSync( + path.join(dir, 'README.md'), + '# demo project\n\nA small demo. Entry: `app.ts`. Config: `config.ts`.\n', + ); + fs.mkdirSync(path.join(dir, 'src'), { recursive: true }); + fs.writeFileSync( + path.join(dir, 'src', 'util.ts'), + "export function util() { return 'util-result'; }\n", + ); + }, + userPrompt: + 'Audit this project: read app.ts, config.ts, and README.md, and glob for ' + + 'every .ts file under src/. Summarize what you find in 3 bullet points.', + metric: (r) => firstTurnParallelism(r.assistantTurns[0]), + pass: fanoutPass, + }, +]; + +// Validate at module load so a broken fixture fails fast at test startup, +// not mid-run after burning API dollars. +validateFixtures(OVERLAY_FIXTURES); diff --git a/test/helpers/agent-sdk-runner.ts b/test/helpers/agent-sdk-runner.ts new file mode 100644 index 00000000..1504384e --- /dev/null +++ b/test/helpers/agent-sdk-runner.ts @@ -0,0 +1,465 @@ +/** + * Claude Agent SDK wrapper for the overlay-efficacy harness. + * + * This sits alongside session-runner.ts (which drives `claude -p` as a + * subprocess) but runs the model via the published @anthropic-ai/claude-agent-sdk + * instead. The SDK exposes the same harness primitives Claude Code itself uses, + * so overlay-driven behavior change is measured against a closer approximation + * of real Claude Code than the `claude -p` subprocess path provides. + * + * Explicit design rules (from plan review): + * - Use SDK-exported SDKMessage types. No `| unknown` union collapse. + * - Permission surface is explicit: bypassPermissions + settingSources:[] + + * disallowedTools inverse. Without these, the SDK inherits user settings, + * project .claude/, and local hooks, and arms are no longer comparable. + * - Binary pinning via pathToClaudeCodeExecutable. Resolve with `which claude` + * at setup time; the SDK would otherwise use its bundled binary. + * - 3-shape rate-limit detection: thrown error, result-message error subtype, + * mid-stream SDKRateLimitEvent. All three recover on retry. + * - On retry, caller resets workspace via a setupWorkspace callback so + * partial Bash side-effects don't contaminate the next attempt. + * - Process-level semaphore caps concurrent queries across all callers in + * the same bun-test process. Composes with bun's own --concurrent flag. + */ + +import { + query, + type SDKMessage, + type SDKAssistantMessage, + type SDKResultMessage, + type SDKSystemMessage, + type PermissionMode, + type SettingSource, + type Options, +} from '@anthropic-ai/claude-agent-sdk'; +import * as fs from 'fs'; +import * as path from 'path'; +import { execSync } from 'child_process'; +import type { SkillTestResult } from './session-runner'; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export interface AgentSdkResult { + /** Full raw event stream for forensic recovery. */ + events: SDKMessage[]; + /** Assistant-typed subset, in order. */ + assistantTurns: SDKAssistantMessage[]; + /** Flat tool-call list, in order of emission. */ + toolCalls: Array<{ tool: string; input: unknown; output: string }>; + /** Concatenated assistant text, newline-joined. */ + output: string; + /** 'success' | 'error_during_execution' | 'error_max_turns' | ... */ + exitReason: string; + turnsUsed: number; + durationMs: number; + firstResponseMs: number; + maxInterTurnMs: number; + costUsd: number; + model: string; + sdkVersion: string; + /** claude_code_version from the SDK's system/init event (authoritative). */ + sdkClaudeCodeVersion: string; + /** Path to the claude binary we pinned. */ + resolvedBinaryPath: string; + /** browse-error pattern scan for SkillTestResult parity. Always empty here. */ + browseErrors: string[]; +} + +/** Signature matching `query()` from the SDK. DI hook for unit tests. */ +export type QueryProvider = typeof query; + +/** Subset of SDK Options['systemPrompt'] we support. */ +export type SystemPromptOption = + | string + | { type: 'preset'; preset: 'claude_code'; append?: string; excludeDynamicSections?: boolean }; + +export interface RunAgentSdkOptions { + /** + * System prompt surface. + * - bare string "" -> omit entirely (SDK default: no system prompt) + * - bare string "...text..." -> REPLACE default with given text (use sparingly) + * - { type:'preset', preset:'claude_code' } -> use Claude Code default + * - { type:'preset', preset:'claude_code', append: "..." } -> default + append + * + * For overlay-efficacy measurement, the preset+append pattern is the right + * one: it measures "does adding overlay text to the REAL Claude Code system + * prompt change behavior" rather than "does the overlay alone (stripped of + * base scaffolding) change behavior". + */ + systemPrompt: SystemPromptOption; + userPrompt: string; + workingDirectory: string; + model?: string; + maxTurns?: number; + allowedTools?: string[]; + disallowedTools?: string[]; + permissionMode?: PermissionMode; + settingSources?: SettingSource[]; + env?: Record; + pathToClaudeCodeExecutable?: string; + testName?: string; + runId?: string; + fixtureId?: string; + queryProvider?: QueryProvider; + /** Max 429 retries per call. Default 3. */ + maxRetries?: number; + /** + * Caller provides this when retry should reset the workspace. The harness + * invokes it with a fresh dir after a rate-limit failure. When omitted, + * retries reuse the original workingDirectory (fine for read-only tests). + */ + onRetry?: (freshDir: string) => void; +} + +export class RateLimitExhaustedError extends Error { + readonly attempts: number; + constructor(attempts: number, cause?: unknown) { + super(`rate limit exhausted after ${attempts} attempts`); + this.name = 'RateLimitExhaustedError'; + this.attempts = attempts; + if (cause !== undefined) (this as { cause?: unknown }).cause = cause; + } +} + +// --------------------------------------------------------------------------- +// Process-level semaphore for API concurrency +// --------------------------------------------------------------------------- + +/** + * Bounded token bucket. Shared across all runAgentSdkTest calls in this + * process so that bun's --concurrent flag does not compound with in-test + * concurrency to blow past Anthropic's rate limits. + * + * Default capacity 3. Override via GSTACK_SDK_MAX_CONCURRENCY env var. + */ +class Semaphore { + private available: number; + private readonly queue: Array<() => void> = []; + constructor(capacity: number) { + this.available = capacity; + } + async acquire(): Promise { + if (this.available > 0) { + this.available--; + return; + } + await new Promise((resolve) => this.queue.push(resolve)); + } + release(): void { + const next = this.queue.shift(); + if (next) { + next(); + } else { + this.available++; + } + } + /** For tests. Returns tokens currently in-flight. */ + inFlight(): number { + // Not introspectable from outside without tracking; approximate. + return this.queue.length; + } +} + +const DEFAULT_SDK_CONCURRENCY = Number(process.env.GSTACK_SDK_MAX_CONCURRENCY ?? 3); +let _apiSemaphore: Semaphore | null = null; +function getApiSemaphore(): Semaphore { + if (!_apiSemaphore) _apiSemaphore = new Semaphore(DEFAULT_SDK_CONCURRENCY); + return _apiSemaphore; +} + +/** Test-only. Resets the process-level semaphore. */ +export function __resetSemaphoreForTests(capacity: number): void { + _apiSemaphore = new Semaphore(capacity); +} + +// --------------------------------------------------------------------------- +// Rate-limit detection +// --------------------------------------------------------------------------- + +/** True if `err` looks like a rate-limit thrown from the SDK. */ +export function isRateLimitThrown(err: unknown): boolean { + if (!err || typeof err !== 'object') return false; + const msg = (err as { message?: string }).message ?? ''; + const name = (err as { name?: string }).name ?? ''; + const status = (err as { status?: number }).status; + return ( + status === 429 || + /rate.?limit|429|too many requests/i.test(msg) || + /RateLimit/i.test(name) + ); +} + +/** True if a SDKResultMessage is a rate-limit-shaped error. */ +export function isRateLimitResult(msg: SDKMessage): boolean { + if (msg.type !== 'result') return false; + const r = msg as SDKResultMessage; + if (r.subtype === 'success') return false; + // subtype === 'error_during_execution' | 'error_max_turns' | 'error_max_budget_usd' | ... + if (r.subtype !== 'error_during_execution') return false; + const errs = (r as { errors?: string[] }).errors ?? []; + return errs.some((e) => /rate.?limit|429|too many requests/i.test(e)); +} + +/** True if mid-stream SDKRateLimitEvent indicates a blocking rate-limit. */ +export function isRateLimitEvent(msg: SDKMessage): boolean { + if (msg.type !== 'rate_limit_event') return false; + const info = (msg as { rate_limit_info?: { status?: string } }).rate_limit_info; + return info?.status === 'rejected'; +} + +// --------------------------------------------------------------------------- +// Version resolution (cached) +// --------------------------------------------------------------------------- + +let _sdkVersionCache: string | null = null; +function resolveSdkVersion(): string { + if (_sdkVersionCache) return _sdkVersionCache; + try { + const pkgPath = require.resolve('@anthropic-ai/claude-agent-sdk/package.json'); + const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf-8')) as { version?: string }; + _sdkVersionCache = pkg.version ?? 'unknown'; + } catch { + _sdkVersionCache = 'unknown'; + } + return _sdkVersionCache; +} + +export function resolveClaudeBinary(): string | null { + try { + return execSync('which claude', { encoding: 'utf-8' }).trim() || null; + } catch { + return null; + } +} + +// --------------------------------------------------------------------------- +// Main runner +// --------------------------------------------------------------------------- + +/** + * Execute a single SDK query with retries. Returns a typed result. + * + * The retry loop treats 429 as recoverable and any other error as fatal. + * Exponential backoff: 1s, 2s, 4s. After maxRetries failures, throws + * RateLimitExhaustedError so the caller can decide what to do with the run. + */ +export async function runAgentSdkTest( + opts: RunAgentSdkOptions, +): Promise { + const sem = getApiSemaphore(); + const maxRetries = opts.maxRetries ?? 3; + const queryImpl: QueryProvider = opts.queryProvider ?? query; + const model = opts.model ?? 'claude-opus-4-7'; + + let attempt = 0; + let lastErr: unknown = null; + + while (attempt <= maxRetries) { + await sem.acquire(); + const startMs = Date.now(); + try { + const sdkOpts: Options = { + model, + cwd: opts.workingDirectory, + maxTurns: opts.maxTurns ?? 5, + tools: opts.allowedTools ?? ['Read', 'Glob', 'Grep', 'Bash'], + disallowedTools: opts.disallowedTools, + allowedTools: opts.allowedTools ?? ['Read', 'Glob', 'Grep', 'Bash'], + permissionMode: opts.permissionMode ?? 'bypassPermissions', + allowDangerouslySkipPermissions: + (opts.permissionMode ?? 'bypassPermissions') === 'bypassPermissions', + settingSources: opts.settingSources ?? [], + env: opts.env, + pathToClaudeCodeExecutable: opts.pathToClaudeCodeExecutable, + }; + // Empty bare string means "omit entirely" (SDK runs with no override). + // Any object or non-empty string is passed through. + if (typeof opts.systemPrompt === 'object' || opts.systemPrompt !== '') { + sdkOpts.systemPrompt = opts.systemPrompt; + } + + const events: SDKMessage[] = []; + const assistantTurns: SDKAssistantMessage[] = []; + const toolCalls: Array<{ tool: string; input: unknown; output: string }> = []; + const assistantTextParts: string[] = []; + let firstResponseMs = 0; + let lastEventMs = startMs; + let maxInterTurnMs = 0; + let systemInitVersion = 'unknown'; + let rateLimited: unknown = null; + let terminalResult: SDKResultMessage | null = null; + + const q = queryImpl({ + prompt: opts.userPrompt, + options: sdkOpts, + }); + + for await (const ev of q) { + const now = Date.now(); + if (firstResponseMs === 0) firstResponseMs = now - startMs; + const interTurn = now - lastEventMs; + if (interTurn > maxInterTurnMs) maxInterTurnMs = interTurn; + lastEventMs = now; + + events.push(ev); + + if (ev.type === 'system' && (ev as SDKSystemMessage).subtype === 'init') { + systemInitVersion = + (ev as SDKSystemMessage).claude_code_version ?? 'unknown'; + } else if (ev.type === 'assistant') { + const am = ev as SDKAssistantMessage; + assistantTurns.push(am); + const content = am.message?.content; + if (Array.isArray(content)) { + for (const block of content as Array< + | { type: 'text'; text?: string } + | { type: 'tool_use'; name?: string; input?: unknown } + | { type: string } + >) { + if (block.type === 'text') { + const t = (block as { text?: string }).text; + if (t) assistantTextParts.push(t); + } else if (block.type === 'tool_use') { + const tb = block as { name?: string; input?: unknown }; + toolCalls.push({ + tool: tb.name ?? 'unknown', + input: tb.input ?? {}, + output: '', + }); + } + } + } + } else if (isRateLimitEvent(ev)) { + rateLimited = new Error( + `mid-stream rate limit: ${JSON.stringify( + (ev as { rate_limit_info?: unknown }).rate_limit_info, + )}`, + ); + } else if (ev.type === 'result') { + terminalResult = ev as SDKResultMessage; + if (isRateLimitResult(ev)) { + rateLimited = new Error( + `result-message rate limit: ${((ev as { errors?: string[] }).errors ?? []).join('; ')}`, + ); + } + } + } + + if (rateLimited) { + throw rateLimited; + } + if (!terminalResult) { + throw new Error('query stream ended without a result event'); + } + + const durationMs = Date.now() - startMs; + const costUsd = + (terminalResult as { total_cost_usd?: number }).total_cost_usd ?? 0; + const turnsUsed = + (terminalResult as { num_turns?: number }).num_turns ?? + assistantTurns.length; + const exitReason = + (terminalResult as { subtype?: string }).subtype ?? 'unknown'; + + return { + events, + assistantTurns, + toolCalls, + output: assistantTextParts.join('\n'), + exitReason, + turnsUsed, + durationMs, + firstResponseMs, + maxInterTurnMs, + costUsd, + model, + sdkVersion: resolveSdkVersion(), + sdkClaudeCodeVersion: systemInitVersion, + resolvedBinaryPath: opts.pathToClaudeCodeExecutable ?? 'sdk-default', + browseErrors: [], + }; + } catch (err) { + lastErr = err; + const isRetryable = isRateLimitThrown(err); + if (!isRetryable || attempt >= maxRetries) { + if (isRetryable) { + throw new RateLimitExhaustedError(attempt + 1, err); + } + throw err; + } + attempt++; + // backoff: 1s, 2s, 4s + await new Promise((r) => setTimeout(r, 1000 * Math.pow(2, attempt - 1))); + // Let caller reset workspace since prior attempt may have partially + // mutated files via Bash. + if (opts.onRetry) { + opts.onRetry(opts.workingDirectory); + } + } finally { + sem.release(); + } + } + + throw new RateLimitExhaustedError(attempt + 1, lastErr); +} + +// --------------------------------------------------------------------------- +// Legacy shape mapper +// --------------------------------------------------------------------------- + +/** + * Adapt AgentSdkResult to the legacy SkillTestResult shape so helpers that + * expect the old `claude -p` output (extractToolSummary, etc) work unchanged. + */ +export function toSkillTestResult(r: AgentSdkResult): SkillTestResult { + // Cost estimate: use SDK's authoritative cost; back-compute chars. + // session-runner.ts:30 requires inputChars/outputChars/estimatedTokens. + // These are rough; real consumers of CostEstimate use cost + turns. + const outputChars = r.output.length; + const inputChars = 0; // unknown from SDK path; not used for pass/fail + const estimatedTokens = Math.round((inputChars + outputChars) / 4); + + // Build a flat transcript list mimicking the NDJSON shape: + // parseNDJSON emits [{ type: 'assistant', message: {...} }, ...]. + // Use the SDK's assistantTurns directly since their shape matches. + const transcript: unknown[] = r.events.slice(); + + return { + toolCalls: r.toolCalls, + browseErrors: r.browseErrors, + exitReason: r.exitReason, + duration: r.durationMs, + output: r.output, + costEstimate: { + inputChars, + outputChars, + estimatedTokens, + estimatedCost: r.costUsd, + turnsUsed: r.turnsUsed, + }, + transcript, + model: r.model, + firstResponseMs: r.firstResponseMs, + maxInterTurnMs: r.maxInterTurnMs, + }; +} + +// --------------------------------------------------------------------------- +// Metric helpers (re-exported for fixtures) +// --------------------------------------------------------------------------- + +/** + * Count `tool_use` blocks in the first assistant turn of an SDK result. + * Returns 0 if there is no first turn or no content array. + * + * This is the core "fanout" metric. A turn with N tool_use blocks = N + * parallel tool invocations. + */ +export function firstTurnParallelism(firstTurn: SDKAssistantMessage | undefined): number { + if (!firstTurn) return 0; + const content = firstTurn.message?.content; + if (!Array.isArray(content)) return 0; + return (content as Array<{ type: string }>).filter((b) => b.type === 'tool_use').length; +}