From 451d7c2fc11a49731bdf359e2ce0a6187f7243bd Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Fri, 20 Mar 2026 07:58:44 -0700 Subject: [PATCH] fix: strengthen E2E assertions for coverage audit tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The coverage audit E2E tests (ship + review) were only asserting exitReason === 'success' and readCalls > 0 — they passed even if the agent produced no coverage diagram. Add assertion that the output contains either GAP or TESTED markers. Found during /review. Co-Authored-By: Claude Opus 4.6 (1M context) --- test/skill-e2e.test.ts | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/test/skill-e2e.test.ts b/test/skill-e2e.test.ts index 2e576e5d..aac7c452 100644 --- a/test/skill-e2e.test.ts +++ b/test/skill-e2e.test.ts @@ -2770,14 +2770,18 @@ Output the diagram directly.`, // Check output contains coverage diagram elements const output = result.output || ''; - const hasGap = output.includes('GAP') || output.includes('gap') || output.includes('NO TEST'); - const hasTested = output.includes('TESTED') || output.includes('tested') || output.includes('✓'); - const hasCoverage = output.includes('COVERAGE') || output.includes('coverage') || output.includes('paths tested'); + const outputLower = output.toLowerCase(); + const hasGap = outputLower.includes('gap') || outputLower.includes('no test'); + const hasTested = outputLower.includes('tested') || output.includes('✓') || output.includes('★'); + const hasCoverage = outputLower.includes('coverage') || outputLower.includes('paths tested'); console.log(`Output has GAP markers: ${hasGap}`); console.log(`Output has TESTED markers: ${hasTested}`); console.log(`Output has coverage summary: ${hasCoverage}`); + // The agent MUST produce a coverage diagram with gap and tested markers + expect(hasGap || hasTested).toBe(true); + // At minimum, the agent should have read the source and test files const readCalls = result.toolCalls.filter(tc => tc.tool === 'Read'); expect(readCalls.length).toBeGreaterThan(0); @@ -2836,14 +2840,18 @@ Output the diagram directly.`, // Check output contains coverage diagram elements const output = result.output || ''; - const hasGap = output.includes('GAP') || output.includes('gap') || output.includes('NO TEST'); - const hasTested = output.includes('TESTED') || output.includes('tested') || output.includes('✓'); - const hasCoverage = output.includes('COVERAGE') || output.includes('coverage') || output.includes('paths tested'); + const outputLower = output.toLowerCase(); + const hasGap = outputLower.includes('gap') || outputLower.includes('no test'); + const hasTested = outputLower.includes('tested') || output.includes('✓') || output.includes('★'); + const hasCoverage = outputLower.includes('coverage') || outputLower.includes('paths tested'); console.log(`Output has GAP markers: ${hasGap}`); console.log(`Output has TESTED markers: ${hasTested}`); console.log(`Output has coverage summary: ${hasCoverage}`); + // The agent MUST produce a coverage diagram with gap and tested markers + expect(hasGap || hasTested).toBe(true); + // At minimum, the agent should have read the source and test files const readCalls = result.toolCalls.filter(tc => tc.tool === 'Read'); expect(readCalls.length).toBeGreaterThan(0);