/** Periodic paid workflow E2Es: real host -> real outside CLI -> seeded defect. */ import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { e2eTierEnabled } from './helpers/e2e-gate'; import { runSkillTest } from './helpers/session-runner'; import { runCodexSkill } from './helpers/codex-session-runner'; import { EvalCollector } from './helpers/eval-store'; import { createOutsideReviewRepo, installOutsideReviewFixture } from './helpers/outside-voice-fixture'; import { claudeOutsideExecutions, codexOutsideExecutions, codexExecutionTranscript, foundInvoiceAuthorizationDefect, outsideExecutionTranscript } from './helpers/outside-voice-evidence'; import { createOutsideReceiptRuntime } from './helpers/outside-voice-receipt'; import { CAPTURE_LONG_MS } from './helpers/eval-budgets'; const ROOT = path.resolve(import.meta.dir, '..'); const periodic = e2eTierEnabled('periodic'); const missing = ['claude', 'codex'].filter(cli => !Bun.which(cli)); const enabled = periodic && missing.length === 0; if (periodic && missing.length) process.stderr.write(`Outside-review E2Es: SKIPPED — missing CLI: ${missing.join(', ')}. No outside review was completed.\n`); // Read the shared diff/tier selection only when a live test is possible. Its // connectivity preflight must not turn a missing-CLI skip into a model request. const selected = enabled ? (await import('./helpers/e2e-helpers')).selectedTests : []; const collector = enabled ? new EvalCollector('e2e-outside-voice') : null; const describeLive = enabled ? describe : describe.skip; let fixtureRoot: string; let rendered: string; function prompt(skill: string): string { return `Run the installed ${skill} skill on this branch. The base branch is main (origin/main exists locally). Complete its adversarial review workflow and report the outside provider's actual findings separately from the native review. Review only; leave source files unchanged. All reviewer calls in this skill are authorized. If a native subagent tool is unavailable, report that and continue the independent outside CLI pass. Do not claim success if the outside CLI cannot complete.`; } function testIfSelected(name: string, fn: () => Promise) { const run = selected === null || selected.includes(name); (run ? test : test.skip)(name, fn, CAPTURE_LONG_MS + 30_000); } describeLive('Installed workflows dispatch outside their host harness', () => { beforeAll(() => { fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-outside-live-')); rendered = path.join(fixtureRoot, 'render'); const result = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'all', '--out-dir', rendered], { cwd: ROOT, stdout: 'pipe', stderr: 'pipe', timeout: 120_000, }); if (result.exitCode !== 0) throw new Error(result.stderr.toString()); }); testIfSelected('outside-voice-codex-to-claude-code', async () => { const repo = createOutsideReviewRepo(fixtureRoot, 'codex'); const receiptRuntime = createOutsideReceiptRuntime(ROOT, fixtureRoot, repo); const skillDir = installOutsideReviewFixture(rendered, 'codex', repo, receiptRuntime.runtimeRoot); const result = await runCodexSkill({ skillDir, skillName: 'gstack-review', cwd: repo, prompt: prompt('gstack-review'), // Reviewer scratch files and review logs need writes; this is an isolated // disposable fixture, and the prompt keeps repository source unchanged. sandbox: 'danger-full-access', timeoutMs: CAPTURE_LONG_MS, }); const executions = codexOutsideExecutions(result.rawLines); const observed = receiptRuntime.read(); const dispatched = foundInvoiceAuthorizationDefect(executions, 'claude-code') || foundInvoiceAuthorizationDefect(observed.executions, 'claude-code'); const passed = result.exitCode === 0 && dispatched; collector?.addTest({ name: 'outside-voice-codex-to-claude-code', suite: 'outside-voice', tier: 'e2e', passed, duration_ms: result.durationMs, cost_usd: 0, output: result.output.slice(0, 2000), transcript: [...outsideExecutionTranscript(executions, 'claude-code'), ...codexExecutionTranscript(executions), ...observed.receipts.map(receipt => ({ type: 'outside_cli_receipt', provider: 'claude-code', receipt }))], turns_used: result.toolCalls.length, exit_reason: result.exitCode === 0 ? 'success' : `exit_${result.exitCode}` }); expect(result.exitCode).toBe(0); expect(dispatched).toBe(true); expect(Bun.spawnSync(['git', 'diff', '--exit-code', 'HEAD', '--', 'invoice.ts'], { cwd: repo, timeout: 10_000 }).exitCode).toBe(0); }); testIfSelected('outside-voice-claude-code-to-codex', async () => { const repo = createOutsideReviewRepo(fixtureRoot, 'claude'); installOutsideReviewFixture(rendered, 'claude', repo, ROOT); const env: Record = {}; // Admit only Codex authentication/config location, never the operator's // session/harness markers or unrelated credentials. for (const key of ['OPENAI_API_KEY', 'CODEX_HOME']) if (process.env[key]) env[key] = process.env[key]!; const result = await runSkillTest({ workingDirectory: repo, prompt: prompt('/review'), timeout: CAPTURE_LONG_MS, maxTurns: 30, allowedTools: ['Bash', 'Read', 'Write', 'Grep', 'Glob', 'Agent', 'Skill'], env, testName: 'outside-voice-claude-code-to-codex', }); const executions = claudeOutsideExecutions(result.transcript); const dispatched = foundInvoiceAuthorizationDefect(executions, 'codex'); const passed = result.exitReason === 'success' && dispatched; collector?.addTest({ name: 'outside-voice-claude-code-to-codex', suite: 'outside-voice', tier: 'e2e', passed, duration_ms: result.duration, cost_usd: result.costEstimate.estimatedCost, output: result.output.slice(0, 2000), transcript: outsideExecutionTranscript(executions, 'codex'), turns_used: result.costEstimate.turnsUsed, exit_reason: result.exitReason }); expect(result.exitReason).toBe('success'); expect(dispatched).toBe(true); expect(Bun.spawnSync(['git', 'diff', '--exit-code', 'HEAD', '--', 'invoice.ts'], { cwd: repo, timeout: 10_000 }).exitCode).toBe(0); }); }); afterAll(async () => { if (collector) await collector.finalize(); if (fixtureRoot) fs.rmSync(fixtureRoot, { recursive: true, force: true }); });