import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets'; import { runSkillTest, SESSION_DRAIN_GRACE_MS } from './helpers/session-runner'; import { ROOT, browseBin, runId, evalsEnabled, selectedTests, describeIfSelected, testConcurrentIfSelected, copyDirSync, setupBrowseShims, logCost, recordE2E, createEvalCollector, finalizeEvalCollector, } from './helpers/e2e-helpers'; import { extractSkillSections, REVIEW_E2E_SECTIONS } from './helpers/skill-fixture'; import { expectContract } from './helpers/eval-store'; import { spawnSync } from 'child_process'; import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; import { carriesDetectorRows, installFakeImpeccable } from './helpers/fake-impeccable'; const evalCollector = createEvalCollector('e2e-review'); // Capture cleanup and recording must finish before Bun starts its retry. const REVIEW_FINALIZE_MS = SESSION_DRAIN_GRACE_MS + 5_000; // --- B5: Review skill E2E --- describeIfSelected('Review skill E2E', ['review-sql-injection'], () => { let reviewDir: string; beforeAll(() => { reviewDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-review-')); // Pre-build a git repo with a vulnerable file on a feature branch (decision 5A) const run = (cmd: string, args: string[]) => spawnSync(cmd, args, { cwd: reviewDir, stdio: 'pipe', timeout: 5000 }); run('git', ['init', '-b', 'main']); run('git', ['config', 'user.email', 'test@test.com']); run('git', ['config', 'user.name', 'Test']); // Commit a clean base on main fs.writeFileSync(path.join(reviewDir, 'app.rb'), '# clean base\nclass App\nend\n'); run('git', ['add', 'app.rb']); run('git', ['commit', '-m', 'initial commit']); // Create feature branch with vulnerable code run('git', ['checkout', '-b', 'feature/add-user-controller']); const vulnContent = fs.readFileSync(path.join(ROOT, 'test', 'fixtures', 'review-eval-vuln.rb'), 'utf-8'); fs.writeFileSync(path.join(reviewDir, 'user_controller.rb'), vulnContent); run('git', ['add', 'user_controller.rb']); run('git', ['commit', '-m', 'add user controller']); // Review skill files — extract only the core review workflow sections // (CLAUDE.md: "E2E test fixtures: extract, don't copy"). fs.writeFileSync( path.join(reviewDir, 'review-SKILL.md'), extractSkillSections(path.join(ROOT, 'review'), REVIEW_E2E_SECTIONS), ); fs.copyFileSync(path.join(ROOT, 'review', 'checklist.md'), path.join(reviewDir, 'review-checklist.md')); fs.copyFileSync(path.join(ROOT, 'review', 'greptile-triage.md'), path.join(reviewDir, 'review-greptile-triage.md')); }); afterAll(() => { try { fs.rmSync(reviewDir, { recursive: true, force: true }); } catch {} }); testConcurrentIfSelected('review-sql-injection', async () => { const result = await runSkillTest({ prompt: `You are in a git repo on a feature branch with changes against main. Read review-SKILL.md for the review workflow instructions. Also read review-checklist.md and apply it. Skip the preamble bash block, lake intro, telemetry, and contributor mode sections — go straight to the review. Run /review on the current diff (git diff main...HEAD). Write your review findings to ${reviewDir}/review-output.md`, workingDirectory: reviewDir, maxTurns: 20, timeout: CAPTURE_MS, testName: 'review-sql-injection', runId, }); logCost('/review', result); let passed = false; try { expect(result.exitReason).toBe('success'); expect(result.browseErrors).toEqual([]); const reviewOutputPath = path.join(reviewDir, 'review-output.md'); expect(fs.existsSync(reviewOutputPath)).toBe(true); const reviewContent = fs.readFileSync(reviewOutputPath, 'utf-8').toLowerCase(); const hasSqlContent = reviewContent.includes('sql') || reviewContent.includes('injection') || reviewContent.includes('sanitiz') || reviewContent.includes('parameteriz') || reviewContent.includes('interpolat') || reviewContent.includes('user_input') || reviewContent.includes('unsanitized'); expect(hasSqlContent).toBe(true); passed = true; } finally { recordE2E(evalCollector, '/review SQL injection', 'Review skill E2E', result, { passed }); } }, CAPTURE_MS + REVIEW_FINALIZE_MS); }); // --- Review: Enum completeness E2E --- describeIfSelected('Review enum completeness E2E', ['review-enum-completeness'], () => { let enumDir: string; let enumCaptureSequence = 0; beforeAll(() => { enumDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-enum-')); const run = (cmd: string, args: string[]) => spawnSync(cmd, args, { cwd: enumDir, stdio: 'pipe', timeout: 5000 }); run('git', ['init', '-b', 'main']); run('git', ['config', 'user.email', 'test@test.com']); run('git', ['config', 'user.name', 'Test']); // Commit baseline on main — order model with 4 statuses const baseContent = fs.readFileSync(path.join(ROOT, 'test', 'fixtures', 'review-eval-enum.rb'), 'utf-8'); fs.writeFileSync(path.join(enumDir, 'order.rb'), baseContent); run('git', ['add', 'order.rb']); run('git', ['commit', '-m', 'initial order model']); // Feature branch adds "returned" status but misses handlers run('git', ['checkout', '-b', 'feature/add-returned-status']); const diffContent = fs.readFileSync(path.join(ROOT, 'test', 'fixtures', 'review-eval-enum-diff.rb'), 'utf-8'); fs.writeFileSync(path.join(enumDir, 'order.rb'), diffContent); run('git', ['add', 'order.rb']); run('git', ['commit', '-m', 'add returned status']); // Review skill files — extracted sections, not the full 1870-line file. fs.writeFileSync( path.join(enumDir, 'review-SKILL.md'), extractSkillSections(path.join(ROOT, 'review'), REVIEW_E2E_SECTIONS), ); fs.copyFileSync(path.join(ROOT, 'review', 'checklist.md'), path.join(enumDir, 'review-checklist.md')); fs.copyFileSync(path.join(ROOT, 'review', 'greptile-triage.md'), path.join(enumDir, 'review-greptile-triage.md')); }); afterAll(() => { try { fs.rmSync(enumDir, { recursive: true, force: true }); } catch {} }); testConcurrentIfSelected('review-enum-completeness', async () => { const attempt = ++enumCaptureSequence; for (const f of fs.readdirSync(enumDir)) if (/^review-output.*\.md$/.test(f)) fs.rmSync(path.join(enumDir, f)); const reviewPath = path.join(enumDir, `review-output-${attempt}.md`); const result = await runSkillTest({ prompt: `You are in a git repo on branch feature/add-returned-status with changes against main. This is a focused, read-only core review: run only the checklist's static Enum & Value Completeness check on this diff. Do not run the full /review lifecycle, QA or exploratory probes (for example Step 4.7), Greptile, hosting/PR/review-log setup, or any command that is not a git read or a file read. This fixture provides only static Ruby source with no configured runnable application, dependencies or runtime/test harness; base main is local and there is no remote or PR. Do not fetch, install, run, or probe an app or dependencies, and any tools that happen to be installed on the host do not expand this scope. Read review-SKILL.md for the review workflow instructions, then read review-checklist.md and apply its Enum & Value Completeness section. Statically inspect the change: read git diff main...HEAD, then grep the sibling status values through the actual authored source and read every match in full, including unchanged consumers, checking whether each handles the new value. Write your review findings once to ${reviewPath} and then stop with a brief final response. Do not re-run the review, reuse a prior report, or invent runtime checks. The diff adds a new "returned" status to the Order model. Your job is to check if all consumers handle it.`, workingDirectory: enumDir, maxTurns: 15, timeout: JUDGE_MS, testName: 'review-enum-completeness', runId: `${process.env.EVALS_RUN_ID ?? runId}-review-enum-${process.pid}-${attempt}`, publicStreamDiagnostics: true, }); logCost('/review enum', result); let passed = false; try { expect(result.exitReason).toBe('success'); // Verify the review caught the missing enum handlers expect(fs.existsSync(reviewPath)).toBe(true); const review = fs.readFileSync(reviewPath, 'utf-8'); const mentionsReturned = review.toLowerCase().includes('returned'); const mentionsEnum = review.toLowerCase().includes('enum') || review.toLowerCase().includes('status'); const mentionsCritical = review.toLowerCase().includes('critical'); expect(mentionsReturned).toBe(true); expect(mentionsEnum || mentionsCritical).toBe(true); passed = result.browseErrors.length === 0; } finally { recordE2E(evalCollector, '/review enum completeness', 'Review enum completeness E2E', result, { passed }); } // The runner can drain stderr for 5s after exit; reserve 1s for assertions/recording. }, JUDGE_MS + REVIEW_FINALIZE_MS); }); // --- Review: Design review lite E2E --- describeIfSelected('Review design lite E2E', ['review-design-lite'], () => { let designDir: string; let fakeEngineDir: string; beforeAll(() => { designDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-design-lite-')); const run = (cmd: string, args: string[]) => spawnSync(cmd, args, { cwd: designDir, stdio: 'pipe', timeout: 5000 }); run('git', ['init', '-b', 'main']); run('git', ['config', 'user.email', 'test@test.com']); run('git', ['config', 'user.name', 'Test']); // Commit clean base on main fs.writeFileSync(path.join(designDir, 'index.html'), '