From 6a4d54fc5559d06a220da4fe81bc547044562606 Mon Sep 17 00:00:00 2001 From: garrytan Date: Wed, 30 Sep 2026 23:17:19 +0000 Subject: [PATCH] test(section-loading): record the CEO capture's report and transcript The 6fcb0981 census failed hasStaleFillRaceFinding (line 98), but the case records nothing beyond junit, so the report the detector judged is gone. Return the SkillTestResult from captureSectionReads and record it, with the full saved report, through the eval collector on pass and fail. --- test/helpers/auq-sdk-capture.ts | 4 +- ...2e-plan-ceo-review-section-loading.test.ts | 50 ++++++++++++------- 2 files changed, 34 insertions(+), 20 deletions(-) diff --git a/test/helpers/auq-sdk-capture.ts b/test/helpers/auq-sdk-capture.ts index 46c4d5c94..b0f5622cb 100644 --- a/test/helpers/auq-sdk-capture.ts +++ b/test/helpers/auq-sdk-capture.ts @@ -253,7 +253,7 @@ export async function captureSectionReads(opts: { nativeReviewOnly?: boolean; }): Promise<{ readSections: Set; reportProduced: boolean; reportWritten: boolean; exitReason: SkillTestResult['exitReason']; toolCalls: SkillTestResult['toolCalls']; - transcript: SkillTestResult['transcript']; output: string }> { + transcript: SkillTestResult['transcript']; output: string; result: SkillTestResult }> { const outFile = path.join(opts.planDir, opts.reportFile ?? 'REPORT.md'); const timeout = opts.timeout ?? 300_000; const fullPlanReview = opts.skillName === 'plan-ceo-review' || opts.skillName === 'plan-eng-review'; @@ -358,7 +358,7 @@ ${fullPlanReview ? `- Save the evolving plan and review outputs to ${outFile} wi // Keep successful terminal-output captures, but a draft left by a failed run // must never satisfy callers that use reportProduced as their completion gate. - return { readSections, reportProduced, reportWritten, exitReason: result.exitReason, toolCalls: result.toolCalls, transcript: result.transcript, output }; + return { readSections, reportProduced, reportWritten, exitReason: result.exitReason, toolCalls: result.toolCalls, transcript: result.transcript, output, result }; } /** A completed CEO review needs its artifact and every summary outcome. */ diff --git a/test/skill-e2e-plan-ceo-review-section-loading.test.ts b/test/skill-e2e-plan-ceo-review-section-loading.test.ts index 7af32cccd..75c8832ab 100644 --- a/test/skill-e2e-plan-ceo-review-section-loading.test.ts +++ b/test/skill-e2e-plan-ceo-review-section-loading.test.ts @@ -25,7 +25,7 @@ * ~$1-2/run. Periodic tier. */ -import { test, expect } from 'bun:test'; +import { afterAll, test, expect } from 'bun:test'; import { CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { describeE2ETier } from './helpers/e2e-gate'; import { @@ -37,9 +37,12 @@ import { LONG_SECTION_CAPTURE_MS, } from './helpers/auq-sdk-capture'; import { CEO_SECTION_CACHE_PLAN, CEO_SECTION_DECISION_POLICY, hasStaleFillRaceFinding } from './helpers/ceo-section-loading-fixture'; +import { createEvalCollector, finalizeEvalCollector, recordE2E } from './helpers/e2e-helpers'; const describeE2E = describeE2ETier('periodic'); const runId = `plan-ceo-section-loading-${process.env.EVALS_RUN_ID ?? 'local'}`; +const collector = createEvalCollector('e2e-plan-ceo-section-loading'); +afterAll(async () => { await finalizeEvalCollector(collector); }); // Sections every plan-ceo-review run must consult after Step 0. const REQUIRED_SECTIONS = ['review-sections.md']; @@ -79,24 +82,35 @@ describeE2E('/plan-ceo-review section-loading E2E (periodic, SDK capture)', () = nativeReviewOnly: true, }); - validateCeoReviewCompletion(capture); - const { readSections, reportProduced, output } = capture; - const missing = REQUIRED_SECTIONS.filter(s => !readSections.has(s)); - expect({ reportProduced, read: [...readSections], missing }).toEqual({ - reportProduced: true, - read: expect.any(Array), - missing: [], - }); - // Guard against an empty pass: the report must have real content. - expect(output.trim().length).toBeGreaterThan(200); - expect(output).toMatch(/^\|\s*Review\s*\|\s*Trigger\s*\|\s*Why\s*\|\s*Runs\s*\|\s*Status\s*\|\s*Findings\s*\|/m); - expect(output).toMatch(/^\|\s*CEO Review\s*\|/m); - // A native capture must not invent an outside dispatch or claim coverage. - expect(hasDisabledOutsideReview(output)).toBe(true); - // Loading a section and producing a table alone must not hide an empty - // review: the complete fixture still contains a real ordering defect. - expect(hasStaleFillRaceFinding(output)).toBe(true); + try { assertSectionLoadingReport(capture); } + catch (error) { + recordE2E(collector, 'plan-ceo-section-loading', 'plan-ceo-section-loading', capture.result, + { passed: false, error: String(error), output: capture.output }); + throw error; + } + recordE2E(collector, 'plan-ceo-section-loading', 'plan-ceo-section-loading', capture.result, + { passed: true, output: capture.output }); }, CAPTURE_LONG_MS, ); }); + +function assertSectionLoadingReport(capture: Awaited>): void { + validateCeoReviewCompletion(capture); + const { readSections, reportProduced, output } = capture; + const missing = REQUIRED_SECTIONS.filter(s => !readSections.has(s)); + expect({ reportProduced, read: [...readSections], missing }).toEqual({ + reportProduced: true, + read: expect.any(Array), + missing: [], + }); + // Guard against an empty pass: the report must have real content. + expect(output.trim().length).toBeGreaterThan(200); + expect(output).toMatch(/^\|\s*Review\s*\|\s*Trigger\s*\|\s*Why\s*\|\s*Runs\s*\|\s*Status\s*\|\s*Findings\s*\|/m); + expect(output).toMatch(/^\|\s*CEO Review\s*\|/m); + // A native capture must not invent an outside dispatch or claim coverage. + expect(hasDisabledOutsideReview(output)).toBe(true); + // Loading a section and producing a table alone must not hide an empty + // review: the complete fixture still contains a real ordering defect. + expect(hasStaleFillRaceFinding(output)).toBe(true); +}