mirror of
https://github.com/garrytan/gstack.git
synced 2026-05-07 05:56:41 +02:00
Merge remote-tracking branch 'origin/main' into garrytan/create-gbrain-skill
This commit is contained in:
Garry Tan
@@ -0,0 +1,565 @@
|
||||
/**
|
||||
* Claude Agent SDK wrapper for the overlay-efficacy harness.
|
||||
*
|
||||
* This sits alongside session-runner.ts (which drives `claude -p` as a
|
||||
* subprocess) but runs the model via the published @anthropic-ai/claude-agent-sdk
|
||||
* instead. The SDK exposes the same harness primitives Claude Code itself uses,
|
||||
* so overlay-driven behavior change is measured against a closer approximation
|
||||
* of real Claude Code than the `claude -p` subprocess path provides.
|
||||
*
|
||||
* Explicit design rules (from plan review):
|
||||
* - Use SDK-exported SDKMessage types. No `| unknown` union collapse.
|
||||
* - Permission surface is explicit: bypassPermissions + settingSources:[] +
|
||||
* disallowedTools inverse. Without these, the SDK inherits user settings,
|
||||
* project .claude/, and local hooks, and arms are no longer comparable.
|
||||
* - Binary pinning via pathToClaudeCodeExecutable. Resolve with `which claude`
|
||||
* at setup time; the SDK would otherwise use its bundled binary.
|
||||
* - 3-shape rate-limit detection: thrown error, result-message error subtype,
|
||||
* mid-stream SDKRateLimitEvent. All three recover on retry.
|
||||
* - On retry, caller resets workspace via a setupWorkspace callback so
|
||||
* partial Bash side-effects don't contaminate the next attempt.
|
||||
* - Process-level semaphore caps concurrent queries across all callers in
|
||||
* the same bun-test process. Composes with bun's own --concurrent flag.
|
||||
*/
|
||||
|
||||
import {
|
||||
query,
|
||||
type SDKMessage,
|
||||
type SDKAssistantMessage,
|
||||
type SDKResultMessage,
|
||||
type SDKSystemMessage,
|
||||
type PermissionMode,
|
||||
type SettingSource,
|
||||
type Options,
|
||||
type CanUseTool,
|
||||
} from '@anthropic-ai/claude-agent-sdk';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { execSync } from 'child_process';
|
||||
import type { SkillTestResult } from './session-runner';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface AgentSdkResult {
|
||||
/** Full raw event stream for forensic recovery. */
|
||||
events: SDKMessage[];
|
||||
/** Assistant-typed subset, in order. */
|
||||
assistantTurns: SDKAssistantMessage[];
|
||||
/** Flat tool-call list, in order of emission. */
|
||||
toolCalls: Array<{ tool: string; input: unknown; output: string }>;
|
||||
/** Concatenated assistant text, newline-joined. */
|
||||
output: string;
|
||||
/** 'success' | 'error_during_execution' | 'error_max_turns' | ... */
|
||||
exitReason: string;
|
||||
turnsUsed: number;
|
||||
durationMs: number;
|
||||
firstResponseMs: number;
|
||||
maxInterTurnMs: number;
|
||||
costUsd: number;
|
||||
model: string;
|
||||
sdkVersion: string;
|
||||
/** claude_code_version from the SDK's system/init event (authoritative). */
|
||||
sdkClaudeCodeVersion: string;
|
||||
/** Path to the claude binary we pinned. */
|
||||
resolvedBinaryPath: string;
|
||||
/** browse-error pattern scan for SkillTestResult parity. Always empty here. */
|
||||
browseErrors: string[];
|
||||
}
|
||||
|
||||
/** Signature matching `query()` from the SDK. DI hook for unit tests. */
|
||||
export type QueryProvider = typeof query;
|
||||
|
||||
/** Subset of SDK Options['systemPrompt'] we support. */
|
||||
export type SystemPromptOption =
|
||||
| string
|
||||
| { type: 'preset'; preset: 'claude_code'; append?: string; excludeDynamicSections?: boolean };
|
||||
|
||||
export interface RunAgentSdkOptions {
|
||||
/**
|
||||
* System prompt surface.
|
||||
* - bare string "" -> omit entirely (SDK default: no system prompt)
|
||||
* - bare string "...text..." -> REPLACE default with given text (use sparingly)
|
||||
* - { type:'preset', preset:'claude_code' } -> use Claude Code default
|
||||
* - { type:'preset', preset:'claude_code', append: "..." } -> default + append
|
||||
*
|
||||
* For overlay-efficacy measurement, the preset+append pattern is the right
|
||||
* one: it measures "does adding overlay text to the REAL Claude Code system
|
||||
* prompt change behavior" rather than "does the overlay alone (stripped of
|
||||
* base scaffolding) change behavior".
|
||||
*/
|
||||
systemPrompt: SystemPromptOption;
|
||||
userPrompt: string;
|
||||
workingDirectory: string;
|
||||
model?: string;
|
||||
maxTurns?: number;
|
||||
allowedTools?: string[];
|
||||
disallowedTools?: string[];
|
||||
permissionMode?: PermissionMode;
|
||||
settingSources?: SettingSource[];
|
||||
env?: Record<string, string>;
|
||||
pathToClaudeCodeExecutable?: string;
|
||||
testName?: string;
|
||||
runId?: string;
|
||||
fixtureId?: string;
|
||||
queryProvider?: QueryProvider;
|
||||
/** Max 429 retries per call. Default 3. */
|
||||
maxRetries?: number;
|
||||
/**
|
||||
* Caller provides this when retry should reset the workspace. The harness
|
||||
* invokes it with a fresh dir after a rate-limit failure. When omitted,
|
||||
* retries reuse the original workingDirectory (fine for read-only tests).
|
||||
*/
|
||||
onRetry?: (freshDir: string) => void;
|
||||
/**
|
||||
* Optional canUseTool callback. When supplied, the harness flips
|
||||
* permissionMode from 'bypassPermissions' to 'default' so the SDK actually
|
||||
* routes tool-use approval decisions through the callback. Without this
|
||||
* flip, bypassPermissions short-circuits the callback and tests that want
|
||||
* to assert on AskUserQuestion content silently pass without asserting.
|
||||
*
|
||||
* Callback contract matches the SDK: fires on every tool-use approval
|
||||
* request and on AskUserQuestion invocations. For non-AskUserQuestion
|
||||
* tools that tests don't care about, use `passThroughNonAskUserQuestion`
|
||||
* to auto-allow them.
|
||||
*/
|
||||
canUseTool?: CanUseTool;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pass-through helper: auto-allows any tool_use that isn't AskUserQuestion.
|
||||
* Most plan-mode handshake tests only care about the handshake AskUserQuestion;
|
||||
* every other tool (Read, Grep, Bash, Write, Edit, ExitPlanMode) should just
|
||||
* run. Compose with a test-specific AskUserQuestion handler:
|
||||
*
|
||||
* canUseTool: async (toolName, input, options) => {
|
||||
* if (toolName === 'AskUserQuestion') {
|
||||
* // custom assertions + canned answer
|
||||
* return { behavior: 'allow', updatedInput: { questions: input.questions, answers: {...} } };
|
||||
* }
|
||||
* return passThroughNonAskUserQuestion(toolName, input);
|
||||
* }
|
||||
*/
|
||||
export function passThroughNonAskUserQuestion(
|
||||
toolName: string,
|
||||
input: Record<string, unknown>,
|
||||
): { behavior: 'allow'; updatedInput: Record<string, unknown> } {
|
||||
// SDK requires an allow response to include updatedInput — pass the original
|
||||
// input through unchanged so the tool runs as the model intended.
|
||||
void toolName;
|
||||
return { behavior: 'allow', updatedInput: input };
|
||||
}
|
||||
|
||||
export class RateLimitExhaustedError extends Error {
|
||||
readonly attempts: number;
|
||||
constructor(attempts: number, cause?: unknown) {
|
||||
super(`rate limit exhausted after ${attempts} attempts`);
|
||||
this.name = 'RateLimitExhaustedError';
|
||||
this.attempts = attempts;
|
||||
if (cause !== undefined) (this as { cause?: unknown }).cause = cause;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Process-level semaphore for API concurrency
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Bounded token bucket. Shared across all runAgentSdkTest calls in this
|
||||
* process so that bun's --concurrent flag does not compound with in-test
|
||||
* concurrency to blow past Anthropic's rate limits.
|
||||
*
|
||||
* Default capacity 3. Override via GSTACK_SDK_MAX_CONCURRENCY env var.
|
||||
*/
|
||||
class Semaphore {
|
||||
private available: number;
|
||||
private readonly queue: Array<() => void> = [];
|
||||
constructor(capacity: number) {
|
||||
this.available = capacity;
|
||||
}
|
||||
async acquire(): Promise<void> {
|
||||
if (this.available > 0) {
|
||||
this.available--;
|
||||
return;
|
||||
}
|
||||
await new Promise<void>((resolve) => this.queue.push(resolve));
|
||||
}
|
||||
release(): void {
|
||||
const next = this.queue.shift();
|
||||
if (next) {
|
||||
next();
|
||||
} else {
|
||||
this.available++;
|
||||
}
|
||||
}
|
||||
/** For tests. Returns tokens currently in-flight. */
|
||||
inFlight(): number {
|
||||
// Not introspectable from outside without tracking; approximate.
|
||||
return this.queue.length;
|
||||
}
|
||||
}
|
||||
|
||||
const DEFAULT_SDK_CONCURRENCY = Number(process.env.GSTACK_SDK_MAX_CONCURRENCY ?? 3);
|
||||
let _apiSemaphore: Semaphore | null = null;
|
||||
function getApiSemaphore(): Semaphore {
|
||||
if (!_apiSemaphore) _apiSemaphore = new Semaphore(DEFAULT_SDK_CONCURRENCY);
|
||||
return _apiSemaphore;
|
||||
}
|
||||
|
||||
/** Test-only. Resets the process-level semaphore. */
|
||||
export function __resetSemaphoreForTests(capacity: number): void {
|
||||
_apiSemaphore = new Semaphore(capacity);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Rate-limit detection
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** True if `err` looks like a rate-limit thrown from the SDK. */
|
||||
export function isRateLimitThrown(err: unknown): boolean {
|
||||
if (!err || typeof err !== 'object') return false;
|
||||
const msg = (err as { message?: string }).message ?? '';
|
||||
const name = (err as { name?: string }).name ?? '';
|
||||
const status = (err as { status?: number }).status;
|
||||
return (
|
||||
status === 429 ||
|
||||
/rate.?limit|429|too many requests/i.test(msg) ||
|
||||
/RateLimit/i.test(name)
|
||||
);
|
||||
}
|
||||
|
||||
/** True if a SDKResultMessage is a rate-limit-shaped error. */
|
||||
export function isRateLimitResult(msg: SDKMessage): boolean {
|
||||
if (msg.type !== 'result') return false;
|
||||
const r = msg as SDKResultMessage;
|
||||
if (r.subtype === 'success') return false;
|
||||
// subtype === 'error_during_execution' | 'error_max_turns' | 'error_max_budget_usd' | ...
|
||||
if (r.subtype !== 'error_during_execution') return false;
|
||||
const errs = (r as { errors?: string[] }).errors ?? [];
|
||||
return errs.some((e) => /rate.?limit|429|too many requests/i.test(e));
|
||||
}
|
||||
|
||||
/** True if mid-stream SDKRateLimitEvent indicates a blocking rate-limit. */
|
||||
export function isRateLimitEvent(msg: SDKMessage): boolean {
|
||||
if (msg.type !== 'rate_limit_event') return false;
|
||||
const info = (msg as { rate_limit_info?: { status?: string } }).rate_limit_info;
|
||||
return info?.status === 'rejected';
|
||||
}
|
||||
|
||||
/**
|
||||
* True if `err` is the SDK's "max turns reached" throw. Some SDK versions
|
||||
* raise this as an exception from the generator instead of emitting a
|
||||
* result message with subtype='error_max_turns'. We treat it as terminal-
|
||||
* but-recoverable: record what we collected and continue, rather than
|
||||
* failing the whole run.
|
||||
*/
|
||||
export function isMaxTurnsError(err: unknown): boolean {
|
||||
if (!err || typeof err !== 'object') return false;
|
||||
const msg = (err as { message?: string }).message ?? '';
|
||||
return /reached maximum number of turns|max.?turns/i.test(msg);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Version resolution (cached)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
let _sdkVersionCache: string | null = null;
|
||||
function resolveSdkVersion(): string {
|
||||
if (_sdkVersionCache) return _sdkVersionCache;
|
||||
try {
|
||||
const pkgPath = require.resolve('@anthropic-ai/claude-agent-sdk/package.json');
|
||||
const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf-8')) as { version?: string };
|
||||
_sdkVersionCache = pkg.version ?? 'unknown';
|
||||
} catch {
|
||||
_sdkVersionCache = 'unknown';
|
||||
}
|
||||
return _sdkVersionCache;
|
||||
}
|
||||
|
||||
export function resolveClaudeBinary(): string | null {
|
||||
try {
|
||||
return execSync('which claude', { encoding: 'utf-8' }).trim() || null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main runner
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Execute a single SDK query with retries. Returns a typed result.
|
||||
*
|
||||
* The retry loop treats 429 as recoverable and any other error as fatal.
|
||||
* Exponential backoff: 1s, 2s, 4s. After maxRetries failures, throws
|
||||
* RateLimitExhaustedError so the caller can decide what to do with the run.
|
||||
*/
|
||||
export async function runAgentSdkTest(
|
||||
opts: RunAgentSdkOptions,
|
||||
): Promise<AgentSdkResult> {
|
||||
const sem = getApiSemaphore();
|
||||
const maxRetries = opts.maxRetries ?? 3;
|
||||
const queryImpl: QueryProvider = opts.queryProvider ?? query;
|
||||
const model = opts.model ?? 'claude-opus-4-7';
|
||||
|
||||
let attempt = 0;
|
||||
let lastErr: unknown = null;
|
||||
|
||||
while (attempt <= maxRetries) {
|
||||
await sem.acquire();
|
||||
const startMs = Date.now();
|
||||
|
||||
// Hoisted so the max-turns catch branch can synthesize a result from
|
||||
// whatever we captured before the SDK threw.
|
||||
const events: SDKMessage[] = [];
|
||||
const assistantTurns: SDKAssistantMessage[] = [];
|
||||
const toolCalls: Array<{ tool: string; input: unknown; output: string }> = [];
|
||||
const assistantTextParts: string[] = [];
|
||||
let firstResponseMs = 0;
|
||||
let lastEventMs = startMs;
|
||||
let maxInterTurnMs = 0;
|
||||
let systemInitVersion = 'unknown';
|
||||
let rateLimited: unknown = null;
|
||||
let terminalResult: SDKResultMessage | null = null;
|
||||
|
||||
try {
|
||||
// When canUseTool is supplied, the SDK must route tool-use approval
|
||||
// decisions through the callback. bypassPermissions short-circuits
|
||||
// that. Flip to 'default' mode so canUseTool actually fires. Tests
|
||||
// that want AskUserQuestion interception without this flip would
|
||||
// silently auto-pass — the exact testability gap D14/D4-eng fix.
|
||||
const hasCanUseTool = typeof opts.canUseTool === 'function';
|
||||
const resolvedPermissionMode: PermissionMode =
|
||||
opts.permissionMode ?? (hasCanUseTool ? 'default' : 'bypassPermissions');
|
||||
|
||||
// When canUseTool is supplied, ensure AskUserQuestion is in the allowed
|
||||
// tools list. Without it, Claude can't invoke AskUserQuestion at all
|
||||
// and the callback never has a chance to fire on it.
|
||||
const baseTools = opts.allowedTools ?? ['Read', 'Glob', 'Grep', 'Bash'];
|
||||
const resolvedTools =
|
||||
hasCanUseTool && !baseTools.includes('AskUserQuestion')
|
||||
? [...baseTools, 'AskUserQuestion']
|
||||
: baseTools;
|
||||
|
||||
const sdkOpts: Options = {
|
||||
model,
|
||||
cwd: opts.workingDirectory,
|
||||
maxTurns: opts.maxTurns ?? 5,
|
||||
tools: resolvedTools,
|
||||
disallowedTools: opts.disallowedTools,
|
||||
allowedTools: resolvedTools,
|
||||
permissionMode: resolvedPermissionMode,
|
||||
allowDangerouslySkipPermissions: resolvedPermissionMode === 'bypassPermissions',
|
||||
settingSources: opts.settingSources ?? [],
|
||||
env: opts.env,
|
||||
pathToClaudeCodeExecutable: opts.pathToClaudeCodeExecutable,
|
||||
...(hasCanUseTool ? { canUseTool: opts.canUseTool } : {}),
|
||||
};
|
||||
// Empty bare string means "omit entirely" (SDK runs with no override).
|
||||
// Any object or non-empty string is passed through.
|
||||
if (typeof opts.systemPrompt === 'object' || opts.systemPrompt !== '') {
|
||||
sdkOpts.systemPrompt = opts.systemPrompt;
|
||||
}
|
||||
|
||||
const q = queryImpl({
|
||||
prompt: opts.userPrompt,
|
||||
options: sdkOpts,
|
||||
});
|
||||
|
||||
for await (const ev of q) {
|
||||
const now = Date.now();
|
||||
if (firstResponseMs === 0) firstResponseMs = now - startMs;
|
||||
const interTurn = now - lastEventMs;
|
||||
if (interTurn > maxInterTurnMs) maxInterTurnMs = interTurn;
|
||||
lastEventMs = now;
|
||||
|
||||
events.push(ev);
|
||||
|
||||
if (ev.type === 'system' && (ev as SDKSystemMessage).subtype === 'init') {
|
||||
systemInitVersion =
|
||||
(ev as SDKSystemMessage).claude_code_version ?? 'unknown';
|
||||
} else if (ev.type === 'assistant') {
|
||||
const am = ev as SDKAssistantMessage;
|
||||
assistantTurns.push(am);
|
||||
const content = am.message?.content;
|
||||
if (Array.isArray(content)) {
|
||||
for (const block of content as Array<
|
||||
| { type: 'text'; text?: string }
|
||||
| { type: 'tool_use'; name?: string; input?: unknown }
|
||||
| { type: string }
|
||||
>) {
|
||||
if (block.type === 'text') {
|
||||
const t = (block as { text?: string }).text;
|
||||
if (t) assistantTextParts.push(t);
|
||||
} else if (block.type === 'tool_use') {
|
||||
const tb = block as { name?: string; input?: unknown };
|
||||
toolCalls.push({
|
||||
tool: tb.name ?? 'unknown',
|
||||
input: tb.input ?? {},
|
||||
output: '',
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (isRateLimitEvent(ev)) {
|
||||
rateLimited = new Error(
|
||||
`mid-stream rate limit: ${JSON.stringify(
|
||||
(ev as { rate_limit_info?: unknown }).rate_limit_info,
|
||||
)}`,
|
||||
);
|
||||
} else if (ev.type === 'result') {
|
||||
terminalResult = ev as SDKResultMessage;
|
||||
if (isRateLimitResult(ev)) {
|
||||
rateLimited = new Error(
|
||||
`result-message rate limit: ${((ev as { errors?: string[] }).errors ?? []).join('; ')}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (rateLimited) {
|
||||
throw rateLimited;
|
||||
}
|
||||
if (!terminalResult) {
|
||||
throw new Error('query stream ended without a result event');
|
||||
}
|
||||
|
||||
const durationMs = Date.now() - startMs;
|
||||
const costUsd =
|
||||
(terminalResult as { total_cost_usd?: number }).total_cost_usd ?? 0;
|
||||
const turnsUsed =
|
||||
(terminalResult as { num_turns?: number }).num_turns ??
|
||||
assistantTurns.length;
|
||||
const exitReason =
|
||||
(terminalResult as { subtype?: string }).subtype ?? 'unknown';
|
||||
|
||||
return {
|
||||
events,
|
||||
assistantTurns,
|
||||
toolCalls,
|
||||
output: assistantTextParts.join('\n'),
|
||||
exitReason,
|
||||
turnsUsed,
|
||||
durationMs,
|
||||
firstResponseMs,
|
||||
maxInterTurnMs,
|
||||
costUsd,
|
||||
model,
|
||||
sdkVersion: resolveSdkVersion(),
|
||||
sdkClaudeCodeVersion: systemInitVersion,
|
||||
resolvedBinaryPath: opts.pathToClaudeCodeExecutable ?? 'sdk-default',
|
||||
browseErrors: [],
|
||||
};
|
||||
} catch (err) {
|
||||
lastErr = err;
|
||||
|
||||
// "Max turns reached" is the SDK's way of saying "this session ran
|
||||
// out of turns." It's thrown from the generator instead of emitted
|
||||
// as a result message. Treat as a successful-but-capped trial: the
|
||||
// assistant turns we collected are real and carry a metric. Record
|
||||
// them with exitReason='error_max_turns' rather than failing the
|
||||
// whole run.
|
||||
if (isMaxTurnsError(err)) {
|
||||
const durationMs = Date.now() - startMs;
|
||||
return {
|
||||
events,
|
||||
assistantTurns,
|
||||
toolCalls,
|
||||
output: assistantTextParts.join('\n'),
|
||||
exitReason: 'error_max_turns',
|
||||
turnsUsed: assistantTurns.length,
|
||||
durationMs,
|
||||
firstResponseMs,
|
||||
maxInterTurnMs,
|
||||
costUsd: 0, // unknown from thrown-error path
|
||||
model,
|
||||
sdkVersion: resolveSdkVersion(),
|
||||
sdkClaudeCodeVersion: systemInitVersion,
|
||||
resolvedBinaryPath: opts.pathToClaudeCodeExecutable ?? 'sdk-default',
|
||||
browseErrors: [],
|
||||
};
|
||||
}
|
||||
|
||||
const isRetryable = isRateLimitThrown(err);
|
||||
if (!isRetryable || attempt >= maxRetries) {
|
||||
if (isRetryable) {
|
||||
throw new RateLimitExhaustedError(attempt + 1, err);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
attempt++;
|
||||
// backoff: 1s, 2s, 4s
|
||||
await new Promise((r) => setTimeout(r, 1000 * Math.pow(2, attempt - 1)));
|
||||
// Let caller reset workspace since prior attempt may have partially
|
||||
// mutated files via Bash.
|
||||
if (opts.onRetry) {
|
||||
opts.onRetry(opts.workingDirectory);
|
||||
}
|
||||
} finally {
|
||||
sem.release();
|
||||
}
|
||||
}
|
||||
|
||||
throw new RateLimitExhaustedError(attempt + 1, lastErr);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Legacy shape mapper
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Adapt AgentSdkResult to the legacy SkillTestResult shape so helpers that
|
||||
* expect the old `claude -p` output (extractToolSummary, etc) work unchanged.
|
||||
*/
|
||||
export function toSkillTestResult(r: AgentSdkResult): SkillTestResult {
|
||||
// Cost estimate: use SDK's authoritative cost; back-compute chars.
|
||||
// session-runner.ts:30 requires inputChars/outputChars/estimatedTokens.
|
||||
// These are rough; real consumers of CostEstimate use cost + turns.
|
||||
const outputChars = r.output.length;
|
||||
const inputChars = 0; // unknown from SDK path; not used for pass/fail
|
||||
const estimatedTokens = Math.round((inputChars + outputChars) / 4);
|
||||
|
||||
// Build a flat transcript list mimicking the NDJSON shape:
|
||||
// parseNDJSON emits [{ type: 'assistant', message: {...} }, ...].
|
||||
// Use the SDK's assistantTurns directly since their shape matches.
|
||||
const transcript: unknown[] = r.events.slice();
|
||||
|
||||
return {
|
||||
toolCalls: r.toolCalls,
|
||||
browseErrors: r.browseErrors,
|
||||
exitReason: r.exitReason,
|
||||
duration: r.durationMs,
|
||||
output: r.output,
|
||||
costEstimate: {
|
||||
inputChars,
|
||||
outputChars,
|
||||
estimatedTokens,
|
||||
estimatedCost: r.costUsd,
|
||||
turnsUsed: r.turnsUsed,
|
||||
},
|
||||
transcript,
|
||||
model: r.model,
|
||||
firstResponseMs: r.firstResponseMs,
|
||||
maxInterTurnMs: r.maxInterTurnMs,
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Metric helpers (re-exported for fixtures)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Count `tool_use` blocks in the first assistant turn of an SDK result.
|
||||
* Returns 0 if there is no first turn or no content array.
|
||||
*
|
||||
* This is the core "fanout" metric. A turn with N tool_use blocks = N
|
||||
* parallel tool invocations.
|
||||
*/
|
||||
export function firstTurnParallelism(firstTurn: SDKAssistantMessage | undefined): number {
|
||||
if (!firstTurn) return 0;
|
||||
const content = firstTurn.message?.content;
|
||||
if (!Array.isArray(content)) return 0;
|
||||
return (content as Array<{ type: string }>).filter((b) => b.type === 'tool_use').length;
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
/**
|
||||
* Shared helpers for plan-mode handshake E2E tests.
|
||||
*
|
||||
* Four sibling test files (plan-ceo, plan-eng, plan-design, plan-devex) exercise
|
||||
* the identical handshake contract against different skills. This helper
|
||||
* centralizes the canUseTool interceptor and the assertion shape so the four
|
||||
* test files are thin wiring (~40 LOC each) and can't drift out of sync.
|
||||
*
|
||||
* See scripts/resolvers/preamble/generate-plan-mode-handshake.ts for the
|
||||
* handshake prose that the tests below assert against.
|
||||
*/
|
||||
|
||||
import { expect } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
import { execSync } from 'child_process';
|
||||
import {
|
||||
runAgentSdkTest,
|
||||
passThroughNonAskUserQuestion,
|
||||
resolveClaudeBinary,
|
||||
type AgentSdkResult,
|
||||
} from './agent-sdk-runner';
|
||||
|
||||
/** Distinctive phrase matching what Claude Code's harness actually injects. */
|
||||
export const PLAN_MODE_REMINDER =
|
||||
'Plan mode is active. The user indicated that they do not want you to execute yet';
|
||||
|
||||
export interface HandshakeCaptureResult {
|
||||
sdkResult: AgentSdkResult;
|
||||
/** Each AskUserQuestion that fired, with its input payload. */
|
||||
askUserQuestions: Array<{ input: Record<string, unknown>; orderIndex: number }>;
|
||||
/** Tool-use events in the order they fired (names only). */
|
||||
toolOrder: string[];
|
||||
/** Whether any Write or Edit tool fired BEFORE the first AskUserQuestion. */
|
||||
writeOrEditBeforeAsk: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a skill via the Agent SDK with canUseTool intercepting every tool use.
|
||||
* Inject the plan-mode distinctive phrase into the system prompt and auto-
|
||||
* answer the handshake with the given answerLabel ("Exit" or "Cancel"). Return
|
||||
* the captured events for assertion.
|
||||
*/
|
||||
export async function runPlanModeHandshakeTest(opts: {
|
||||
/** Skill name, e.g. 'plan-ceo-review'. */
|
||||
skillName: string;
|
||||
/** "Exit" to pick option A (exit-and-rerun) or "Cancel" for option C. */
|
||||
answerLabel: 'Exit' | 'Cancel';
|
||||
/** If true, DO NOT inject the reminder — used by the no-op regression test. */
|
||||
omitPlanModeReminder?: boolean;
|
||||
/** Max turns for the SDK call (default 4 — handshake + exit should fit easily). */
|
||||
maxTurns?: number;
|
||||
}): Promise<HandshakeCaptureResult> {
|
||||
const { skillName, answerLabel, omitPlanModeReminder, maxTurns } = opts;
|
||||
|
||||
const askUserQuestions: HandshakeCaptureResult['askUserQuestions'] = [];
|
||||
const toolOrder: string[] = [];
|
||||
let toolIndex = 0;
|
||||
let firstAskIndex = -1;
|
||||
|
||||
const workingDir = fs.mkdtempSync(
|
||||
path.join(os.tmpdir(), `plan-mode-handshake-${skillName}-`),
|
||||
);
|
||||
|
||||
// The SDK requires AskUserQuestion to be in the allowed tools list. The
|
||||
// harness auto-adds it when canUseTool is supplied, but we also want Read
|
||||
// so the skill can load its own file if it tries to.
|
||||
const binary = resolveClaudeBinary();
|
||||
|
||||
try {
|
||||
// Inject the distinctive phrase into the system prompt by appending it to
|
||||
// the default Claude Code preset. Claude Code's real plan mode uses an
|
||||
// injected system-reminder; in SDK tests we use systemPrompt.append which
|
||||
// the model treats as equally authoritative.
|
||||
const reminderAppend = omitPlanModeReminder
|
||||
? ''
|
||||
: `\n\n<system-reminder>\n${PLAN_MODE_REMINDER}. This supercedes any other instructions you have received.\n</system-reminder>\n`;
|
||||
|
||||
const sdkResult = await runAgentSdkTest({
|
||||
systemPrompt: {
|
||||
type: 'preset',
|
||||
preset: 'claude_code',
|
||||
append: reminderAppend,
|
||||
},
|
||||
userPrompt: `Read the skill file at ${path.resolve(
|
||||
import.meta.dir,
|
||||
'..',
|
||||
'..',
|
||||
skillName,
|
||||
'SKILL.md',
|
||||
)} and follow its instructions. There is no real plan to review — just start the skill and respond to any AskUserQuestion that fires.`,
|
||||
workingDirectory: workingDir,
|
||||
maxTurns: maxTurns ?? 4,
|
||||
allowedTools: ['Read', 'Grep', 'Glob', 'Bash'],
|
||||
...(binary ? { pathToClaudeCodeExecutable: binary } : {}),
|
||||
canUseTool: async (toolName, input) => {
|
||||
toolOrder.push(toolName);
|
||||
if (toolName === 'AskUserQuestion') {
|
||||
if (firstAskIndex === -1) firstAskIndex = toolIndex;
|
||||
askUserQuestions.push({ input, orderIndex: toolIndex });
|
||||
toolIndex++;
|
||||
// Auto-answer with the label the test specified.
|
||||
const q = (input.questions as Array<{ question: string; options: Array<{ label: string }> }>)[0];
|
||||
const matched = q.options.find((o) => o.label.includes(answerLabel));
|
||||
const answer = matched ? matched.label : q.options[0]!.label;
|
||||
return {
|
||||
behavior: 'allow',
|
||||
updatedInput: {
|
||||
questions: input.questions,
|
||||
answers: { [q.question]: answer },
|
||||
},
|
||||
};
|
||||
}
|
||||
toolIndex++;
|
||||
return passThroughNonAskUserQuestion(toolName, input);
|
||||
},
|
||||
});
|
||||
|
||||
const writeOrEditBeforeAsk =
|
||||
firstAskIndex > 0 &&
|
||||
toolOrder.slice(0, firstAskIndex).some((t) => t === 'Write' || t === 'Edit');
|
||||
|
||||
return { sdkResult, askUserQuestions, toolOrder, writeOrEditBeforeAsk };
|
||||
} finally {
|
||||
try {
|
||||
fs.rmSync(workingDir, { recursive: true, force: true });
|
||||
} catch { /* ignore cleanup errors */ }
|
||||
}
|
||||
}
|
||||
|
||||
/** Assert the shape of a fired handshake AskUserQuestion. */
|
||||
export function assertHandshakeShape(
|
||||
aq: { input: Record<string, unknown> },
|
||||
): void {
|
||||
const questions = aq.input.questions as Array<{
|
||||
question: string;
|
||||
options: Array<{ label: string }>;
|
||||
}>;
|
||||
expect(questions).toBeDefined();
|
||||
expect(questions.length).toBe(1);
|
||||
const q = questions[0]!;
|
||||
// D8 dropped Option B; handshake has exactly 2 options.
|
||||
expect(q.options.length).toBe(2);
|
||||
const labels = q.options.map((o) => o.label);
|
||||
expect(labels.some((l) => l.includes('Exit'))).toBe(true);
|
||||
expect(labels.some((l) => l.includes('Cancel'))).toBe(true);
|
||||
}
|
||||
|
||||
/** Read the skill-usage.jsonl log and return handshake entries. */
|
||||
export function readHandshakeLog(): Array<Record<string, unknown>> {
|
||||
const logPath = path.join(os.homedir(), '.gstack', 'analytics', 'skill-usage.jsonl');
|
||||
if (!fs.existsSync(logPath)) return [];
|
||||
const lines = fs.readFileSync(logPath, 'utf-8').split('\n').filter(Boolean);
|
||||
return lines
|
||||
.map((line) => {
|
||||
try {
|
||||
return JSON.parse(line);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
})
|
||||
.filter((x): x is Record<string, unknown> => x !== null && x.event === 'plan_mode_handshake');
|
||||
}
|
||||
|
||||
export { execSync };
|
||||
@@ -82,12 +82,40 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'plan-eng-review-artifact': ['plan-eng-review/**'],
|
||||
'plan-review-report': ['plan-eng-review/**', 'scripts/gen-skill-docs.ts'],
|
||||
|
||||
// Plan-mode handshake (v1.10.2.0) — gate-tier safety regression tests.
|
||||
// Each fires when any of: the interactive skill's template, the resolver,
|
||||
// preamble composition, the Agent SDK harness, the question registry, or
|
||||
// the one-way-door classifier changes.
|
||||
'plan-ceo-review-plan-mode': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-plan-mode-handshake.ts', 'scripts/resolvers/preamble.ts', 'scripts/question-registry.ts', 'scripts/one-way-doors.ts', 'test/helpers/agent-sdk-runner.ts'],
|
||||
'plan-eng-review-plan-mode': ['plan-eng-review/**', 'scripts/resolvers/preamble/generate-plan-mode-handshake.ts', 'scripts/resolvers/preamble.ts', 'scripts/question-registry.ts', 'scripts/one-way-doors.ts', 'test/helpers/agent-sdk-runner.ts'],
|
||||
'plan-design-review-plan-mode-handshake': ['plan-design-review/**', 'scripts/resolvers/preamble/generate-plan-mode-handshake.ts', 'scripts/resolvers/preamble.ts', 'scripts/question-registry.ts', 'scripts/one-way-doors.ts', 'test/helpers/agent-sdk-runner.ts'],
|
||||
'plan-devex-review-plan-mode': ['plan-devex-review/**', 'scripts/resolvers/preamble/generate-plan-mode-handshake.ts', 'scripts/resolvers/preamble.ts', 'scripts/question-registry.ts', 'scripts/one-way-doors.ts', 'test/helpers/agent-sdk-runner.ts'],
|
||||
'plan-mode-no-op': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-plan-mode-handshake.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/agent-sdk-runner.ts'],
|
||||
'e2e-harness-audit': ['plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-plan-mode-handshake.ts', 'test/helpers/agent-sdk-runner.ts'],
|
||||
|
||||
// AskUserQuestion format regression (RECOMMENDATION + Completeness: N/10)
|
||||
// Fires when either template OR the two preamble resolvers change.
|
||||
'plan-ceo-review-format-mode': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts'],
|
||||
'plan-ceo-review-format-approach': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts'],
|
||||
'plan-eng-review-format-coverage': ['plan-eng-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts'],
|
||||
'plan-eng-review-format-kind': ['plan-eng-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts'],
|
||||
'plan-ceo-review-format-mode': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
'plan-ceo-review-format-approach': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
'plan-eng-review-format-coverage': ['plan-eng-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
'plan-eng-review-format-kind': ['plan-eng-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
|
||||
// v1.7.0.0 Pros/Cons format cadence + format + negative-escape evals.
|
||||
// Dependencies: same as format-mode + the 4 plan-review templates + overlay.
|
||||
// All periodic-tier (non-deterministic Opus 4.7 behavior).
|
||||
'plan-ceo-review-prosons-cadence': ['plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
'plan-review-prosons-format': ['plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
'plan-review-prosons-hardstop-neg': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
'plan-review-prosons-neutral-neg': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
|
||||
// Expanded coverage (CT3) — 6 non-plan-review skills inherit Pros/Cons via preamble
|
||||
'ship-prosons-format': ['ship/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
'office-hours-prosons-format': ['office-hours/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
'investigate-prosons-format': ['investigate/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
'qa-prosons-format': ['qa/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
'review-prosons-format': ['review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
'design-review-prosons-format': ['design-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
'document-release-prosons-format': ['document-release/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
|
||||
// /plan-tune (v1 observational)
|
||||
'plan-tune-inspect': ['plan-tune/**', 'scripts/question-registry.ts', 'scripts/psychographic-signals.ts', 'scripts/one-way-doors.ts', 'bin/gstack-question-log', 'bin/gstack-question-preference', 'bin/gstack-developer-profile'],
|
||||
@@ -222,6 +250,24 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
['model-overlays/claude.md', 'model-overlays/opus-4-7.md', 'scripts/models.ts', 'scripts/resolvers/model-overlay.ts'],
|
||||
'fanout-arm-overlay-off':
|
||||
['model-overlays/claude.md', 'model-overlays/opus-4-7.md', 'scripts/models.ts', 'scripts/resolvers/model-overlay.ts'],
|
||||
|
||||
// Overlay efficacy harness (SDK) — measures whether overlay nudges change
|
||||
// behavior under @anthropic-ai/claude-agent-sdk (closer to real Claude Code
|
||||
// than `claude -p`). testNames in the file are template literals so the
|
||||
// completeness scanner doesn't require them; these entries exist for
|
||||
// diff-based selection accuracy.
|
||||
'overlay-harness-opus-4-7-fanout-toy': [
|
||||
'model-overlays/**',
|
||||
'test/fixtures/overlay-nudges.ts',
|
||||
'test/helpers/agent-sdk-runner.ts',
|
||||
'scripts/resolvers/model-overlay.ts',
|
||||
],
|
||||
'overlay-harness-opus-4-7-fanout-realistic': [
|
||||
'model-overlays/**',
|
||||
'test/fixtures/overlay-nudges.ts',
|
||||
'test/helpers/agent-sdk-runner.ts',
|
||||
'scripts/resolvers/model-overlay.ts',
|
||||
],
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -282,12 +328,35 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
|
||||
'plan-eng-coverage-audit': 'gate',
|
||||
'plan-review-report': 'gate',
|
||||
|
||||
// Plan-mode handshake — deterministic safety regression, gate-tier
|
||||
'plan-ceo-review-plan-mode': 'gate',
|
||||
'plan-eng-review-plan-mode': 'gate',
|
||||
'plan-design-review-plan-mode-handshake': 'gate',
|
||||
'plan-devex-review-plan-mode': 'gate',
|
||||
'plan-mode-no-op': 'gate',
|
||||
'e2e-harness-audit': 'gate',
|
||||
|
||||
// AskUserQuestion format regression — periodic (Opus 4.7 non-deterministic benchmark)
|
||||
'plan-ceo-review-format-mode': 'periodic',
|
||||
'plan-ceo-review-format-approach': 'periodic',
|
||||
'plan-eng-review-format-coverage': 'periodic',
|
||||
'plan-eng-review-format-kind': 'periodic',
|
||||
|
||||
// v1.7.0.0 Pros/Cons format — cadence + negative-escape evals (all periodic)
|
||||
'plan-ceo-review-prosons-cadence': 'periodic',
|
||||
'plan-review-prosons-format': 'periodic',
|
||||
'plan-review-prosons-hardstop-neg': 'periodic',
|
||||
'plan-review-prosons-neutral-neg': 'periodic',
|
||||
|
||||
// CT3 expanded coverage — non-plan-review skills inheriting Pros/Cons (all periodic)
|
||||
'ship-prosons-format': 'periodic',
|
||||
'office-hours-prosons-format': 'periodic',
|
||||
'investigate-prosons-format': 'periodic',
|
||||
'qa-prosons-format': 'periodic',
|
||||
'review-prosons-format': 'periodic',
|
||||
'design-review-prosons-format': 'periodic',
|
||||
'document-release-prosons-format': 'periodic',
|
||||
|
||||
// /plan-tune — gate (core v1 DX promise: plain-English intent routing)
|
||||
'plan-tune-inspect': 'gate',
|
||||
|
||||
@@ -398,6 +467,10 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
|
||||
// Opus 4.7 overlay evals — periodic (non-deterministic LLM behavior + Opus cost)
|
||||
'fanout-arm-overlay-on': 'periodic',
|
||||
'fanout-arm-overlay-off': 'periodic',
|
||||
|
||||
// Overlay efficacy harness (SDK, paid) — periodic only
|
||||
'overlay-harness-opus-4-7-fanout-toy': 'periodic',
|
||||
'overlay-harness-opus-4-7-fanout-realistic': 'periodic',
|
||||
};
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user