// E2E: /setup-gbrain Path 4 with Step 4.5 "Yes" — local PGLite for code search. // // Drives the skill against a stub HTTP MCP server (200 OK on tools/list). // Auto-answers AskUserQuestion to pick: // - Path 4 at Step 2 (Remote gbrain MCP) // - "Yes, set up local PGLite for code" at Step 4.5 // // Asserts that the model: // 1. ran the verify helper successfully (got past Step 4c) // 2. invoked gstack-gbrain-install (Step 4.5 Yes branch) // 3. invoked `gbrain init --pglite --json` (also Step 4.5 Yes branch) // 4. registered the remote MCP via claude mcp add --transport http // 5. wrote a "Code search ..... OK local-pglite" row to the Step 10 verdict // // Periodic-tier (codex #12: AgentSDK harness is non-deterministic; gate-tier // coverage of the split-engine behavior lives in the deterministic unit // tests at gbrain-local-status.test.ts, gbrain-sync-skip.test.ts, etc). // // Cost: ~$0.50-$1.00 per run. Periodic-tier (EVALS=1 EVALS_TIER=periodic). import { test, expect, afterAll } from 'bun:test'; import { CAPTURE_MS } from './helpers/eval-budgets'; import { describeE2ETier, e2eTierEnabled } from './helpers/e2e-gate'; import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; import * as http from 'http'; import { runAgentSdkTest, passThroughNonAskUserQuestion, resolveClaudeBinary, type AgentSdkResult, } from './helpers/agent-sdk-runner'; import { buildSetupGbrainFixture } from './helpers/setup-gbrain-fixture'; import { EvalCollector } from './helpers/eval-store'; import { redactFindingSpans } from '../lib/redact-engine'; const describeE2E = describeE2ETier('periodic'); const evalCollector = e2eTierEnabled('periodic') ? new EvalCollector('e2e') : null; const FIXTURE_TOKEN = 'gbrain_fake_token_for_test'; /** Public diagnostics only; never serialize SDK thinking or credential values. */ function publicDiagnostics(result: AgentSdkResult) { const transcript = result.events.flatMap((event: any) => { if (event.type === 'system' && event.subtype === 'init') return [{ type: event.type, subtype: event.subtype, session_id: event.session_id, cwd: event.cwd, model: event.model, tools: event.tools, claude_code_version: event.claude_code_version, }]; if (event.type !== 'assistant' && event.type !== 'user') return []; const content = Array.isArray(event.message?.content) ? event.message.content.flatMap((block: any) => { if (block.type === 'text') return [{ type: block.type, text: block.text }]; if (block.type === 'tool_use') return [{ type: block.type, id: block.id, name: block.name, input: block.input }]; if (block.type === 'tool_result') return [{ type: block.type, tool_use_id: block.tool_use_id, is_error: block.is_error, content: typeof block.content === 'string' ? block.content : Array.isArray(block.content) ? block.content.filter((b: any) => b.type === 'text').map((b: any) => ({ type: 'text', text: b.text })) : [] }]; return []; }) : []; return content.length ? [{ type: event.type, session_id: event.session_id, parent_tool_use_id: event.parent_tool_use_id, message: { id: event.message.id, role: event.message.role, content } }] : []; }); const serialized = JSON.stringify({ output: result.output, transcript, browseErrors: result.browseErrors }).replaceAll(FIXTURE_TOKEN, ''); const redacted = redactFindingSpans(serialized, { repoVisibility: 'private' }); if (redacted === null) return { output: '[Public diagnostics omitted: redaction limit]', transcript: [], browseErrors: [] }; try { return JSON.parse(redacted); } catch { // Text redaction can consume JSON delimiters along with a credential URL. // Keep only its redacted text; diagnostics must not mask the test verdict. return { output: `[Redacted public diagnostics; JSON structure changed]\n${redacted}`, transcript: [], browseErrors: [] }; } } afterAll(async () => { if (evalCollector) await evalCollector.finalize(); }); /** * Minimal stub MCP server that returns success on initialize / tools/list. * Verify helper calls /tools/list with a Bearer header and inspects the body. */ function startStubMcp(): Promise<{ url: string; close: () => Promise }> { return new Promise((resolve) => { const server = http.createServer((req, res) => { let body = ''; req.on('data', (c) => (body += c)); req.on('end', () => { res.statusCode = 200; res.setHeader('Content-Type', 'text/event-stream'); // Try to be useful: respond with a fake initialize + tools/list payload. let payload: unknown = { jsonrpc: '2.0', id: 1, result: { tools: [] } }; try { const req = JSON.parse(body); if (req.method === 'initialize') { payload = { jsonrpc: '2.0', id: req.id, result: { protocolVersion: '2024-11-05', capabilities: { tools: {} }, serverInfo: { name: 'gbrain', version: '0.32.3.0' }, }, }; } } catch { // ignore parse failure; default payload } res.end(`event: message\ndata: ${JSON.stringify(payload)}\n\n`); }); }); server.listen(0, '127.0.0.1', () => { const addr = server.address(); if (!addr || typeof addr === 'string') throw new Error('no address'); resolve({ url: `http://127.0.0.1:${addr.port}/mcp`, close: () => new Promise((r) => server.close(() => r())), }); }); }); } /** * Fake gbrain CLI: * - --version → echoes a version * - init --pglite --json → writes a pglite config, exits 0 * - everything else → exits 0 quietly * * Logs every invocation so we can assert init was called. */ function makeFakeGbrain(binDir: string, gbrainConfigPath: string): string { const callLog = path.join(binDir, 'gbrain-calls.log'); const script = `#!/bin/bash echo "gbrain $@" >> "${callLog}" case "$1 $2" in "--version "*) echo "gbrain 0.33.1.0"; exit 0 ;; "init --pglite") cat > "${gbrainConfigPath}" <> "${callLog}" case "$1 $2" in "mcp add") exit 0 ;; "mcp list") echo "gbrain: http://stub/mcp (HTTP) — connected" ; exit 0 ;; "mcp remove") exit 0 ;; "mcp get") echo '{"type":"http","url":"http://stub/mcp"}'; exit 0 ;; esac exit 0 `; fs.writeFileSync(path.join(binDir, 'claude'), script, { mode: 0o755 }); return callLog; } /** * Fake gstack-gbrain-install so we don't actually clone the gbrain repo + * bun-link. The test only cares that the skill INVOKED it on the Yes branch. */ function makeFakeInstall(binDir: string): string { const callLog = path.join(binDir, 'install-calls.log'); const script = `#!/bin/bash echo "install $@" >> "${callLog}" exit 0 `; fs.writeFileSync(path.join(binDir, 'gstack-gbrain-install'), script, { mode: 0o755, }); return callLog; } describeE2E('/setup-gbrain Path 4 + Step 4.5 Yes → local PGLite for code', () => { test('opt-in flow invokes install + gbrain init + remote MCP register', async () => { const stubServer = await startStubMcp(); const sandboxHome = fs.mkdtempSync(path.join(os.tmpdir(), 'path4-pglite-')); const fakeBinDir = fs.mkdtempSync(path.join(os.tmpdir(), 'path4-pglite-bin-')); const gbrainConfigDir = path.join(sandboxHome, '.gbrain'); fs.mkdirSync(gbrainConfigDir, { recursive: true }); const gbrainConfigPath = path.join(gbrainConfigDir, 'config.json'); const claudeLog = makeFakeClaude(fakeBinDir); const gbrainLog = makeFakeGbrain(fakeBinDir, gbrainConfigPath); const installLog = makeFakeInstall(fakeBinDir); const ORIGINAL_CLAUDE_MD = '# Test project\n'; fs.writeFileSync(path.join(sandboxHome, 'CLAUDE.md'), ORIGINAL_CLAUDE_MD); const askLog: Array<{ question: string; choice: string }> = []; const binary = resolveClaudeBinary(); const orig = { home: process.env.HOME, pathEnv: process.env.PATH, mcpToken: process.env.GBRAIN_MCP_TOKEN, }; process.env.HOME = sandboxHome; process.env.PATH = `${fakeBinDir}:${path.join(path.resolve(import.meta.dir, '..'), 'bin')}:${process.env.PATH ?? '/usr/bin:/bin:/opt/homebrew/bin'}`; process.env.GBRAIN_MCP_TOKEN = FIXTURE_TOKEN; try { // Carve-aware fixture (see test/helpers/setup-gbrain-fixture.ts): // skeleton + brain-init (Step 4 Path 4 body incl. the Step 4d local // PGLite offer this test says Yes to) + claude-md-persist (Step 8 sits // on the walked path to Step 10). Non-empty guard inside the builder. const skillPath = path.join(sandboxHome, 'setup-gbrain-SKILL.md'); fs.writeFileSync( skillPath, buildSetupGbrainFixture(['brain-init.md', 'claude-md-persist.md']), ); const result = await runAgentSdkTest({ systemPrompt: { type: 'preset', preset: 'claude_code' }, userPrompt: `Read the skill file at ${skillPath} and follow Path 4 (Remote MCP). ` + `Use this MCP URL: ${stubServer.url}. ` + `The bearer token is already in GBRAIN_MCP_TOKEN. ` + `At Step 4d (the "Want symbol-aware code search?" question), ask me with AskUserQuestion and wait for my answer. ` + `Then continue through Step 5a (MCP registration) → Step 10 (verdict). ` + `I have not yet chosen whether to set up local PGLite; do not skip that decision.`, workingDirectory: sandboxHome, env: { GBRAIN_MCP_TOKEN: FIXTURE_TOKEN }, maxTurns: 25, allowedTools: ['Read', 'Grep', 'Glob', 'Bash', 'Write', 'Edit'], ...(binary ? { pathToClaudeCodeExecutable: binary } : {}), canUseTool: async (toolName, input) => { if (toolName === 'AskUserQuestion') { const qs = input.questions as Array<{ question: string; options: Array<{ label: string }>; }>; const answers: Record = {}; for (const q of qs) { // Heuristics: pick the option that screams "yes/PGLite/code search" for our flow. const yes = q.options.find((o) => /yes.*local|local.*pglite|code search|opt in/i.test(o.label), ) ?? q.options.find((o) => /remote.*mcp|path 4/i.test(o.label)) ?? q.options[0]!; answers[q.question] = yes.label; askLog.push({ question: q.question, choice: yes.label }); } return { behavior: 'allow', updatedInput: { questions: qs, answers }, }; } return passThroughNonAskUserQuestion(toolName, input); }, }); let passed = false; try { // Smoke test contract (codex #12: AgentSDK is non-deterministic, so this // E2E asserts the model followed the SPLIT-ENGINE PATH without depending // on the exact subcommand sequence — deterministic per-step coverage // lives in gbrain-local-status.test.ts, gbrain-sync-skip.test.ts, etc). // Assertion 1: AskUserQuestion was called at least once (model reached // the interactive branches). expect(askLog.length).toBeGreaterThan(0); // Assertion 2: at LEAST ONE of the Path 4 / Step 4.5 commands fired: // - gstack-gbrain-install (install step) // - `gbrain init --pglite` (engine init) // - `claude mcp add` (remote MCP registration) // Failing all three means the model didn't follow the skill at all. const installCalls = fs.existsSync(installLog) ? fs.readFileSync(installLog, 'utf-8') : ''; const gbrainCalls = fs.existsSync(gbrainLog) ? fs.readFileSync(gbrainLog, 'utf-8') : ''; const claudeCalls = fs.existsSync(claudeLog) ? fs.readFileSync(claudeLog, 'utf-8') : ''; const followedPath = installCalls.length > 0 || /gbrain init --pglite/.test(gbrainCalls) || /mcp add/.test(claudeCalls); expect(followedPath).toBe(true); // Assertion 3: token never leaked to CLAUDE.md (security regression). const finalClaudeMd = fs.readFileSync( path.join(sandboxHome, 'CLAUDE.md'), 'utf-8', ); expect(finalClaudeMd).not.toContain(FIXTURE_TOKEN); passed = true; } finally { const diagnostics = publicDiagnostics(result); evalCollector?.addTest({ name: 'setup-gbrain-path4-local-pglite', suite: 'setup-gbrain', tier: 'e2e', passed, duration_ms: result.durationMs, cost_usd: result.costUsd, output: diagnostics.output, transcript: diagnostics.transcript, turns_used: result.turnsUsed, browse_errors: diagnostics.browseErrors, exit_reason: result.exitReason, model: result.model, first_response_ms: result.firstResponseMs, max_inter_turn_ms: result.maxInterTurnMs, }); } } finally { if (orig.home === undefined) delete process.env.HOME; else process.env.HOME = orig.home; if (orig.pathEnv === undefined) delete process.env.PATH; else process.env.PATH = orig.pathEnv; if (orig.mcpToken === undefined) delete process.env.GBRAIN_MCP_TOKEN; else process.env.GBRAIN_MCP_TOKEN = orig.mcpToken; await stubServer.close(); fs.rmSync(sandboxHome, { recursive: true, force: true }); fs.rmSync(fakeBinDir, { recursive: true, force: true }); } }, CAPTURE_MS); });