diff --git a/test/fixtures/coverage-audit-fixture.ts b/test/fixtures/coverage-audit-fixture.ts new file mode 100644 index 00000000..8a7adcc3 --- /dev/null +++ b/test/fixtures/coverage-audit-fixture.ts @@ -0,0 +1,76 @@ +/** + * Shared fixture for test coverage audit E2E tests. + * + * Creates a Node.js project with billing source code that has intentional + * test coverage gaps: processPayment has happy-path-only tests, + * refundPayment has no tests at all. + * + * Used by: ship-coverage-audit E2E, review-coverage-audit E2E + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import { spawnSync } from 'child_process'; + +export function createCoverageAuditFixture(dir: string): void { + // Create a Node.js project WITH test framework but coverage gaps + fs.writeFileSync(path.join(dir, 'package.json'), JSON.stringify({ + name: 'test-coverage-app', + version: '1.0.0', + type: 'module', + scripts: { test: 'echo "no tests yet"' }, + devDependencies: { vitest: '^1.0.0' }, + }, null, 2)); + + // Create vitest config + fs.writeFileSync(path.join(dir, 'vitest.config.ts'), + `import { defineConfig } from 'vitest/config';\nexport default defineConfig({ test: {} });\n`); + + fs.writeFileSync(path.join(dir, 'VERSION'), '0.1.0.0\n'); + fs.writeFileSync(path.join(dir, 'CHANGELOG.md'), '# Changelog\n'); + + // Create source file with multiple code paths + fs.mkdirSync(path.join(dir, 'src'), { recursive: true }); + fs.writeFileSync(path.join(dir, 'src', 'billing.ts'), ` +export function processPayment(amount: number, currency: string) { + if (amount <= 0) throw new Error('Invalid amount'); + if (currency !== 'USD' && currency !== 'EUR') throw new Error('Unsupported currency'); + return { status: 'success', amount, currency }; +} + +export function refundPayment(paymentId: string, reason: string) { + if (!paymentId) throw new Error('Payment ID required'); + if (!reason) throw new Error('Reason required'); + return { status: 'refunded', paymentId, reason }; +} +`); + + // Create a test directory with ONE test (partial coverage) + fs.mkdirSync(path.join(dir, 'test'), { recursive: true }); + fs.writeFileSync(path.join(dir, 'test', 'billing.test.ts'), ` +import { describe, test, expect } from 'vitest'; +import { processPayment } from '../src/billing'; + +describe('processPayment', () => { + test('processes valid payment', () => { + const result = processPayment(100, 'USD'); + expect(result.status).toBe('success'); + }); + // GAP: no test for invalid amount + // GAP: no test for unsupported currency + // GAP: refundPayment not tested at all +}); +`); + + // Init git repo with main branch + const run = (cmd: string, args: string[]) => + spawnSync(cmd, args, { cwd: dir, stdio: 'pipe', timeout: 5000 }); + run('git', ['init', '-b', 'main']); + run('git', ['config', 'user.email', 'test@test.com']); + run('git', ['config', 'user.name', 'Test']); + run('git', ['add', '.']); + run('git', ['commit', '-m', 'initial commit']); + + // Create feature branch + run('git', ['checkout', '-b', 'feature/billing']); +} diff --git a/test/helpers/touchfiles.ts b/test/helpers/touchfiles.ts index 53cc709c..effdbad0 100644 --- a/test/helpers/touchfiles.ts +++ b/test/helpers/touchfiles.ts @@ -83,8 +83,9 @@ export const E2E_TOUCHFILES: Record = { // QA bootstrap 'qa-bootstrap': ['qa/**', 'browse/src/**', 'ship/**'], - // Ship coverage audit - 'ship-coverage-audit': ['ship/**'], + // Coverage audit (shared fixture) + 'ship-coverage-audit': ['ship/**', 'test/fixtures/coverage-audit-fixture.ts'], + 'review-coverage-audit': ['review/**', 'test/fixtures/coverage-audit-fixture.ts'], // Design 'design-consultation-core': ['design-consultation/**'], diff --git a/test/skill-e2e.test.ts b/test/skill-e2e.test.ts index 96019f70..2e576e5d 100644 --- a/test/skill-e2e.test.ts +++ b/test/skill-e2e.test.ts @@ -2727,66 +2727,9 @@ describeIfSelected('Test Coverage Audit E2E', ['ship-coverage-audit'], () => { copyDirSync(path.join(ROOT, 'ship'), path.join(coverageDir, 'ship')); copyDirSync(path.join(ROOT, 'review'), path.join(coverageDir, 'review')); - // Create a Node.js project WITH test framework but coverage gaps - fs.writeFileSync(path.join(coverageDir, 'package.json'), JSON.stringify({ - name: 'test-coverage-app', - version: '1.0.0', - type: 'module', - scripts: { test: 'echo "no tests yet"' }, - devDependencies: { vitest: '^1.0.0' }, - }, null, 2)); - - // Create vitest config - fs.writeFileSync(path.join(coverageDir, 'vitest.config.ts'), - `import { defineConfig } from 'vitest/config';\nexport default defineConfig({ test: {} });\n`); - - fs.writeFileSync(path.join(coverageDir, 'VERSION'), '0.1.0.0\n'); - fs.writeFileSync(path.join(coverageDir, 'CHANGELOG.md'), '# Changelog\n'); - - // Create source file with multiple code paths - fs.mkdirSync(path.join(coverageDir, 'src'), { recursive: true }); - fs.writeFileSync(path.join(coverageDir, 'src', 'billing.ts'), ` -export function processPayment(amount: number, currency: string) { - if (amount <= 0) throw new Error('Invalid amount'); - if (currency !== 'USD' && currency !== 'EUR') throw new Error('Unsupported currency'); - return { status: 'success', amount, currency }; -} - -export function refundPayment(paymentId: string, reason: string) { - if (!paymentId) throw new Error('Payment ID required'); - if (!reason) throw new Error('Reason required'); - return { status: 'refunded', paymentId, reason }; -} -`); - - // Create a test directory with ONE test (partial coverage) - fs.mkdirSync(path.join(coverageDir, 'test'), { recursive: true }); - fs.writeFileSync(path.join(coverageDir, 'test', 'billing.test.ts'), ` -import { describe, test, expect } from 'vitest'; -import { processPayment } from '../src/billing'; - -describe('processPayment', () => { - test('processes valid payment', () => { - const result = processPayment(100, 'USD'); - expect(result.status).toBe('success'); - }); - // GAP: no test for invalid amount - // GAP: no test for unsupported currency - // GAP: refundPayment not tested at all -}); -`); - - // Init git repo with main branch - const run = (cmd: string, args: string[]) => - spawnSync(cmd, args, { cwd: coverageDir, stdio: 'pipe', timeout: 5000 }); - run('git', ['init', '-b', 'main']); - run('git', ['config', 'user.email', 'test@test.com']); - run('git', ['config', 'user.name', 'Test']); - run('git', ['add', '.']); - run('git', ['commit', '-m', 'initial commit']); - - // Create feature branch - run('git', ['checkout', '-b', 'feature/billing']); + // Use shared fixture for billing project with coverage gaps + const { createCoverageAuditFixture } = require('./fixtures/coverage-audit-fixture'); + createCoverageAuditFixture(coverageDir); }); afterAll(() => { @@ -2841,6 +2784,72 @@ Output the diagram directly.`, }, 180_000); }); +// --- Review Coverage Audit E2E --- + +describeIfSelected('Review Coverage Audit E2E', ['review-coverage-audit'], () => { + let reviewCoverageDir: string; + + beforeAll(() => { + reviewCoverageDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-review-coverage-')); + + // Copy review skill files + copyDirSync(path.join(ROOT, 'review'), path.join(reviewCoverageDir, 'review')); + + // Use shared fixture for billing project with coverage gaps + const { createCoverageAuditFixture } = require('./fixtures/coverage-audit-fixture'); + createCoverageAuditFixture(reviewCoverageDir); + }); + + afterAll(() => { + try { fs.rmSync(reviewCoverageDir, { recursive: true, force: true }); } catch {} + }); + + test('/review Step 4.75 produces coverage diagram', async () => { + const result = await runSkillTest({ + prompt: `Read the file review/SKILL.md for the review workflow instructions. + +You are on the feature/billing branch. The base branch is main. +This is a test project — there is no remote, no PR to create. + +ONLY run Step 4.75 (Test Coverage Diagram) from the review workflow. +Skip all other steps (scope drift, checklist, design review, fix-first, etc.). + +The source code is in ${reviewCoverageDir}/src/billing.ts. +Existing tests are in ${reviewCoverageDir}/test/billing.test.ts. + +Produce the ASCII coverage diagram showing which code paths are tested and which have gaps. +Output the diagram directly.`, + workingDirectory: reviewCoverageDir, + maxTurns: 15, + allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'], + timeout: 120_000, + testName: 'review-coverage-audit', + runId, + }); + + logCost('/review coverage audit', result); + recordE2E('/review Step 4.75 coverage audit', 'Review Coverage Audit E2E', result, { + passed: result.exitReason === 'success', + }); + + expect(result.exitReason).toBe('success'); + + // Check output contains coverage diagram elements + const output = result.output || ''; + const hasGap = output.includes('GAP') || output.includes('gap') || output.includes('NO TEST'); + const hasTested = output.includes('TESTED') || output.includes('tested') || output.includes('✓'); + const hasCoverage = output.includes('COVERAGE') || output.includes('coverage') || output.includes('paths tested'); + + console.log(`Output has GAP markers: ${hasGap}`); + console.log(`Output has TESTED markers: ${hasTested}`); + console.log(`Output has coverage summary: ${hasCoverage}`); + + // At minimum, the agent should have read the source and test files + const readCalls = result.toolCalls.filter(tc => tc.tool === 'Read'); + expect(readCalls.length).toBeGreaterThan(0); + }, 180_000); +}); + // --- Codex skill E2E --- describeIfSelected('Codex skill E2E', ['codex-review'], () => {