gstack/scripts/preflight-agent-sdk.ts

/**
 * Preflight for the overlay efficacy harness.
 *
 * Confirms, before any paid eval runs:
 *   1. `@anthropic-ai/claude-agent-sdk` loads and `query()` is the expected shape.
 *   2. `claude-opus-4-7` is a live API model ID (not a Claude Code alias).
 *   3. The SDK event stream contains the types we assume (system init, assistant,
 *      result) with the fields we destructure.
 *   4. `scripts/resolvers/model-overlay.ts` resolves `{{INHERIT:claude}}` against
 *      `opus-4-7.md` with no unresolved inheritance directives.
 *   5. A local `claude` binary exists at `which claude` so binary pinning is possible.
 *
 * Run: bun run scripts/preflight-agent-sdk.ts [--live]
 *
 * Checks 1-2 and 4-5 are free. Check 3 (the live SDK query, ~15 tokens of API
 * spend) runs only with --live. Exit 0 on success, 1 on any failed check,
 * 2 on a bad argument.
 */

import '../lib/conductor-env-shim';
import { query, type SDKMessage } from '@anthropic-ai/claude-agent-sdk';
import { readOverlay } from './resolvers/model-overlay';
import { resolveClaudeBinary } from '../lib/claude-bin';
import { sdkSystemPrompt } from '../test/helpers/sdk-system-prompt';

const USAGE = `Usage: bun run scripts/preflight-agent-sdk.ts [--live]

Free checks: overlay resolution and local claude binary pinning.
--live  also runs one end-to-end SDK query against claude-opus-4-7
        (~15 tokens of API spend; needs ANTHROPIC_API_KEY).`;

function parsePreflightArgs(argv: string[]): { help: boolean; live: boolean } {
  const parsed = { help: false, live: false };
  for (const arg of argv) {
    if (arg === '--help' || arg === '-h') parsed.help = true;
    else if (arg === '--live') parsed.live = true;
    else throw new Error(`unknown argument "${arg}"`);
  }
  return parsed;
}

async function main() {
  let args: { help: boolean; live: boolean };
  try {
    args = parsePreflightArgs(process.argv.slice(2));
  } catch (err) {
    console.error(`preflight-agent-sdk: ${(err as Error).message}\n\n${USAGE}`);
    process.exit(2);
  }
  if (args.help) {
    console.log(USAGE);
    return;
  }

  const failures: string[] = [];
  const pass = (msg: string) => console.log(`  ok  ${msg}`);
  const fail = (msg: string) => {
    console.log(`  FAIL  ${msg}`);
    failures.push(msg);
  };

  // 1. Overlay resolver
  console.log('1. Overlay resolver');
  const resolved = readOverlay('opus-4-7');
  if (!resolved) {
    fail("readOverlay('opus-4-7') returned empty");
  } else {
    pass(`resolved overlay length: ${resolved.length} chars`);
    if (resolved.includes('{{INHERIT:')) {
      fail('resolved overlay still contains {{INHERIT:...}} directive');
    } else {
      pass('no unresolved INHERIT directives');
    }
  }

  // 2. Local claude binary exists
  console.log('\n2. Binary pinning');
  let claudePath: string | null = resolveClaudeBinary();
  if (claudePath) {
    pass(`local claude binary: ${claudePath}`);
  } else {
    fail('`Bun.which("claude")` failed — cannot pin binary (set GSTACK_CLAUDE_BIN to override)');
  }

  // 3. SDK query end-to-end
  console.log('\n3. SDK query end-to-end');
  if (!args.live) {
    console.log('  skip  live query not requested — rerun with --live to spend ~15 tokens on it');
  } else if (!process.env.ANTHROPIC_API_KEY) {
    console.log('  skip  ANTHROPIC_API_KEY not set — cannot test live query');
  } else {
    try {
      const events: SDKMessage[] = [];
      const q = query({
        prompt: 'say pong',
        options: {
          model: 'claude-opus-4-7',
          ...sdkSystemPrompt({ type: 'sdk-default', reason: 'connectivity probe with no tools: checks the event stream shape, not skill behavior' }),
          tools: [],
          permissionMode: 'bypassPermissions',
          allowDangerouslySkipPermissions: true,
          settingSources: [],
          maxTurns: 1,
          pathToClaudeCodeExecutable: claudePath ?? undefined,
          env: { ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY },
        },
      });
      for await (const ev of q) events.push(ev);
      pass(`received ${events.length} events`);

      const init = events.find(
        (e) => e.type === 'system' && (e as { subtype?: string }).subtype === 'init',
      ) as { claude_code_version?: string; model?: string } | undefined;
      if (!init) {
        fail('no system/init event received');
      } else {
        pass(`system init: claude_code_version=${init.claude_code_version}, model=${init.model}`);
      }

      const assistantEvents = events.filter((e) => e.type === 'assistant');
      if (assistantEvents.length === 0) {
        fail('no assistant events received — model ID may be rejected');
      } else {
        pass(`received ${assistantEvents.length} assistant event(s)`);
        const first = assistantEvents[0] as { message?: { content?: unknown[] } };
        const content = first.message?.content;
        if (!Array.isArray(content)) {
          fail('first assistant event has no content[] array');
        } else {
          pass(`first assistant content[] has ${content.length} block(s)`);
        }
      }

      const result = events.find((e) => e.type === 'result') as
        | { subtype?: string; total_cost_usd?: number; num_turns?: number }
        | undefined;
      if (!result) {
        fail('no result event received');
      } else {
        pass(
          `result: subtype=${result.subtype}, cost=$${result.total_cost_usd?.toFixed(4)}, turns=${result.num_turns}`,
        );
      }
    } catch (err) {
      fail(`SDK query threw: ${err instanceof Error ? err.message : String(err)}`);
    }
  }

  console.log();
  if (failures.length > 0) {
    console.log(`PREFLIGHT FAILED: ${failures.length} check(s) failed`);
    process.exit(1);
  }
  console.log('PREFLIGHT OK');
}

if (import.meta.main) {
  main().catch((err) => {
    console.error(err);
    process.exit(1);
  });
}