From 988e985ddde492bfa3ed56e540a4f41fb79c238f Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 00:38:19 +0000 Subject: [PATCH] =?UTF-8?q?test(evals):=20coverage-audit=20detector=20spli?= =?UTF-8?q?ts=20parallel=20columns=20at=20=E2=94=82=20rails=20and=20[+]=20?= =?UTF-8?q?groups;=20coverage=20prompts=20name=20absolute=20skill=20paths;?= =?UTF-8?q?=20TODOs=20from=20the=20gate=20eval=20census?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- TODOS.md | 12 ++++++++++++ test/coverage-audit-evidence.test.ts | 19 +++++++++++++++++++ .../coverage-audit-parallel-column.json | 10 ++++++++++ test/helpers/coverage-audit-evidence.ts | 3 ++- test/skill-e2e-coverage-audit.test.ts | 8 ++++---- 5 files changed, 47 insertions(+), 5 deletions(-) create mode 100644 test/fixtures/coverage-audit-parallel-column.json diff --git a/TODOS.md b/TODOS.md index 79e9225f5..b0aa02201 100644 --- a/TODOS.md +++ b/TODOS.md @@ -87,6 +87,18 @@ by re-running `./setup` from it. Iterate the registry's render column instead. **Effort:** S. **Priority:** P3. +- **Judge actionability sits exactly at the 4.0 gate** (P2, from the PR #3014 eval + census) — all 24 workflow judges score actionability at 4.0 in nearly every run, so one + low sample of three fails the case (qa-only 9/78, qa 6/78, plan-ceo modes 5/78, plan-eng + sections 4/78, review 4/70 over 09-27..10-02). The repair is clearer skill text per case + (as done for qa in v1.91.14.0), not a lower threshold. Effort M. +- **`--case` cannot select loop-registered paid cases** (P3) — coverage audits, TPA and + plan-mode-no-op register their cases in a loop, so `test-paid-shards.ts --case ` runs + nothing; teach `fileCaseRegistration` the loop pattern. Effort S. +- **Mid-response `API Error:` in PTY sessions waits out the whole budget** (P3, policy call) — + a dropped connection after the first model turn returns the CLI to an idle prompt and the + harness waits 300 s. Decide whether the harness should fail fast and classify it. Effort S. + ### P2/P3: impeccable interop deferrals (filed 2026-09-08, from the CEO + eng reviews of docs/designs/IMPECCABLE_INTEROP.md) Each item was weighed during the review and deferred with a reason; none blocks diff --git a/test/coverage-audit-evidence.test.ts b/test/coverage-audit-evidence.test.ts index 142a29bfc..99b48d715 100644 --- a/test/coverage-audit-evidence.test.ts +++ b/test/coverage-audit-evidence.test.ts @@ -14,6 +14,7 @@ import fixture_coverage_checkbox_tail_av from './fixtures/coverage-checkbox-tail import captured_coverage_diagram_legend_as from './fixtures/coverage-diagram-legend-as.json'; import fixture_coverage_shell_display_aq from './fixtures/coverage-shell-display-aq.json'; import billing_coverage_shell_display_aq from './fixtures/coverage-audit-ae.json'; +import captured_parallel_column from './fixtures/coverage-audit-parallel-column.json'; const clone = (v:T):T => structuredClone(v); const diagram = '```text\nsrc/billing.ts\n├── processPayment: happy path [TESTED]\n└── refundPayment [UNTESTED]\n```'; @@ -1000,3 +1001,21 @@ describe('coverage reads with neighboring display commands', () => { }); }); }); + +describe('coverage-audit-parallel-column', () => { + const text = captured_parallel_column.output; + const diagram = (output: string) => { const s = synthetic(); s.result.output = output; return verdict(s).diagram; }; + const covered = '[★★ TESTED] Happy path USD — billing.test.ts:6 │ (no test chains the two functions)'; + + test('a wrapped USER FLOWS rail beside a covered code path stays in its own column', () => { + expect(captured_parallel_column.provenance.recordedPassed).toBe(false); + expect(text).toContain(covered); + expect(diagram(text)).toBe(true); + }); + + test('the parallel column still cannot supply or cancel code-path coverage', () => { + expect(diagram(text.replace(covered, '[GAP] Happy path USD — billing.test.ts:6 │ [★★ TESTED] happy USD flow'))).toBe(false); + expect(diagram(text.replace(covered, '[GAP] Happy path USD — billing.test.ts:6 [+] [★★ TESTED] happy USD flow'))).toBe(false); + expect(diagram(text.replace(/(refundPayment[\s\S]*?)\n\n/, m => m.replaceAll('[GAP] ', '[★★ TESTED] ')))).toBe(false); + }); +}); diff --git a/test/fixtures/coverage-audit-parallel-column.json b/test/fixtures/coverage-audit-parallel-column.json new file mode 100644 index 000000000..542bf9a66 --- /dev/null +++ b/test/fixtures/coverage-audit-parallel-column.json @@ -0,0 +1,10 @@ +{ + "provenance": { + "runId": 37077373640, + "headSha": "147163e0", + "testName": "plan-eng-coverage-audit", + "recordedPassed": false, + "source": "Exact final parent result of the CI capture (paid-slice-12-a1). The diagram follows plan-eng-review's documented CODE PATHS | USER FLOWS layout; a wrapped USER FLOWS row continues on a │ rail beside the covered processPayment row." + }, + "output": "Test review complete for the two billing functions. Framework detected: vitest (config file and devDependency present), though the `test` script in package.json is a placeholder echo, so the one existing test never runs via `npm test`.\n\n```\nCODE PATHS USER FLOWS\n[+] src/billing.ts [+] Payment lifecycle (caller-facing)\n ├── processPayment(amount, currency) ├── [GAP] [→E2E] Pay then refund same paymentId\n │ ├── [★★ TESTED] Happy path USD — billing.test.ts:6 │ (no test chains the two functions)\n │ │ asserts status only; amount/currency └── [GAP] Caller receives rejection error\n │ │ echo not asserted (no test asserts any thrown message)\n │ ├── [GAP] EUR accepted (2nd arm of currency guard, :4)\n │ ├── [GAP] amount <= 0 → throws 'Invalid amount' (:3)\n │ │ boundary 0 and negative both unexercised\n │ └── [GAP] Unsupported currency → throws (:4)\n │ e.g. 'GBP', '', lowercase 'usd'\n └── refundPayment(paymentId, reason) [+] Error states\n ├── [GAP] Happy path → { status: 'refunded' } (:11) └── [GAP] Falsy-ID and falsy-reason both\n ├── [GAP] !paymentId → throws 'Payment ID required' collapse to '' and undefined;\n │ (:9) '' and undefined unexercised neither is exercised\n └── [GAP] !reason → throws 'Reason required' (:10)\n function is never imported by any test\n\nLLM integration: none in scope — no [→EVAL] paths\n\nCOVERAGE: 1/9 paths tested (11%) | Code paths: 1/7 (14%) | User flows: 0/2 (0%)\nQUALITY: ★★★:0 ★★:1 ★:0 | GAPS: 8 (1 E2E, 0 eval)\n```\n\nLegend: ★★★ behavior + edge + error | ★★ happy path | ★ smoke check\n[→E2E] = needs integration test | [→EVAL] = needs LLM eval\n\n**Coverage summary**\n\n- **processPayment** has one happy-path test rated ★★. It only checks the status field, so a regression that mangled the returned amount or currency would pass. Both guard clauses and the EUR arm are untested.\n- **refundPayment** has zero coverage. It is not imported anywhere in the test directory, so all three branches are gaps.\n- **No error message is asserted anywhere.** The four throw sites are the only behavior that protects callers from bad input, and none has a test that would catch the guard being removed or inverted.\n- **The branch diff itself adds no behavior.** It only appends a comment to each file, so the gaps above are pre-existing on main, not introduced by this branch.\n- **The pay-then-refund flow has no test**, and both functions are pure with no integration boundary yet. Unit tests are the right tool for every code-path gap. The lifecycle row is flagged E2E only because the matrix says payment flows should not rely on unit tests alone once a real boundary exists.\n\nPer the requested scope I did not write the Test Plan Artifact, propose tests through the decision gate, or modify the source or tests." +} \ No newline at end of file diff --git a/test/helpers/coverage-audit-evidence.ts b/test/helpers/coverage-audit-evidence.ts index 9f755b5ca..2781d5dd2 100644 --- a/test/helpers/coverage-audit-evidence.ts +++ b/test/helpers/coverage-audit-evidence.ts @@ -295,7 +295,8 @@ function treeRow(line: string): { depth: number; text: string } | undefined { ? { depth: -1, text: line } : undefined; } // A parallel USER FLOWS column cannot supply CODE PATHS coverage markers. - return { depth: match[1]!.length, text: match[2]!.split(/ {3,}(?=[├└+|])/, 1)[0]! }; + // That column starts at a tree glyph, its wrapped-row rail (│) or a [+] group. + return { depth: match[1]!.length, text: match[2]!.split(/ {3,}(?=[├└│|+]|\[\+\])/, 1)[0]! }; } const coverageMapCaption = String.raw`[A-Za-z0-9_.-]+(?:\/[A-Za-z0-9_.-]+)*\.[A-Za-z0-9]+[\t ]+[—–-][\t ]+(?:test[\t ]+)?coverage[\t ]+map`; diff --git a/test/skill-e2e-coverage-audit.test.ts b/test/skill-e2e-coverage-audit.test.ts index b78e0a545..397d9295a 100644 --- a/test/skill-e2e-coverage-audit.test.ts +++ b/test/skill-e2e-coverage-audit.test.ts @@ -22,12 +22,12 @@ import { resolveEvalModel } from '../lib/eval-model'; const evalCollector = createEvalCollector('e2e'); const CASES = [ { id: 'review-coverage-audit', skill: 'review', suite: 'Review Coverage Audit E2E', - instructions: `Read review/SKILL.md and review/sections/review-army.md for the current review workflow. -Apply ONLY the testing specialist checklist in review/specialists/testing.md to the supplied source and tests. + instructions: (dir: string) => `Read ${dir}/review/SKILL.md and ${dir}/review/sections/review-army.md for the current review workflow. +Apply ONLY the testing specialist checklist in ${dir}/review/specialists/testing.md to the supplied source and tests. This is a targeted --testing request, even though this fixture has no branch diff. Run that checklist directly; do not dispatch other specialists or perform fixes.` }, { id: 'plan-eng-coverage-audit', skill: 'plan-eng-review', suite: 'Plan Eng Review Coverage Audit E2E', - instructions: `Read plan-eng-review/SKILL.md and plan-eng-review/sections/review-sections.md. + instructions: (dir: string) => `Read ${dir}/plan-eng-review/SKILL.md and ${dir}/plan-eng-review/sections/review-sections.md. Run ONLY section "3. Test review": codepath tracing and the ASCII coverage diagram. For this targeted audit, treat the two supplied billing functions as the proposed codepaths. Skip architecture, code quality, performance, test generation and all other workflow steps.` }, @@ -57,7 +57,7 @@ for (const entry of CASES) describeIfSelected(entry.suite, [entry.id], () => { return { path: file, content: fs.readFileSync(file, 'utf8') }; }); return runSkillTest({ - prompt: `${entry.instructions} + prompt: `${entry.instructions(cwd!)} You are on the feature/billing branch. The base branch is main. This is a test project — there is no remote, no PR to create.