test(section-loading): record the CEO capture's report and transcript

The 6fcb0981 census failed hasStaleFillRaceFinding (line 98), but the case
records nothing beyond junit, so the report the detector judged is gone.
Return the SkillTestResult from captureSectionReads and record it, with the
full saved report, through the eval collector on pass and fail.
This commit is contained in:
garrytan committed 2026-09-30 23:23:18 +00:00
1 parent 7eaefdec59
commit 6a4d54fc55
2 files changed
+34 -20

No files matched your search

+2 -2
View File
@@ -253,7 +253,7 @@ export async function captureSectionReads(opts: {
nativeReviewOnly?: boolean;
}): Promise<{ readSections: Set<string>; reportProduced: boolean; reportWritten: boolean;
exitReason: SkillTestResult['exitReason']; toolCalls: SkillTestResult['toolCalls'];
transcript: SkillTestResult['transcript']; output: string }> {
transcript: SkillTestResult['transcript']; output: string; result: SkillTestResult }> {
const outFile = path.join(opts.planDir, opts.reportFile ?? 'REPORT.md');
const timeout = opts.timeout ?? 300_000;
const fullPlanReview = opts.skillName === 'plan-ceo-review' || opts.skillName === 'plan-eng-review';
@@ -358,7 +358,7 @@ ${fullPlanReview ? `- Save the evolving plan and review outputs to ${outFile} wi
// Keep successful terminal-output captures, but a draft left by a failed run
// must never satisfy callers that use reportProduced as their completion gate.
return { readSections, reportProduced, reportWritten, exitReason: result.exitReason, toolCalls: result.toolCalls, transcript: result.transcript, output };
return { readSections, reportProduced, reportWritten, exitReason: result.exitReason, toolCalls: result.toolCalls, transcript: result.transcript, output, result };
}
/** A completed CEO review needs its artifact and every summary outcome. */
@@ -25,7 +25,7 @@
* ~$1-2/run. Periodic tier.
*/
import { test, expect } from 'bun:test';
import { afterAll, test, expect } from 'bun:test';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import {
@@ -37,9 +37,12 @@ import {
LONG_SECTION_CAPTURE_MS,
} from './helpers/auq-sdk-capture';
import { CEO_SECTION_CACHE_PLAN, CEO_SECTION_DECISION_POLICY, hasStaleFillRaceFinding } from './helpers/ceo-section-loading-fixture';
import { createEvalCollector, finalizeEvalCollector, recordE2E } from './helpers/e2e-helpers';
const describeE2E = describeE2ETier('periodic');
const runId = `plan-ceo-section-loading-${process.env.EVALS_RUN_ID ?? 'local'}`;
const collector = createEvalCollector('e2e-plan-ceo-section-loading');
afterAll(async () => { await finalizeEvalCollector(collector); });
// Sections every plan-ceo-review run must consult after Step 0.
const REQUIRED_SECTIONS = ['review-sections.md'];
@@ -79,24 +82,35 @@ describeE2E('/plan-ceo-review section-loading E2E (periodic, SDK capture)', () =
nativeReviewOnly: true,
});
validateCeoReviewCompletion(capture);
const { readSections, reportProduced, output } = capture;
const missing = REQUIRED_SECTIONS.filter(s => !readSections.has(s));
expect({ reportProduced, read: [...readSections], missing }).toEqual({
reportProduced: true,
read: expect.any(Array),
missing: [],
});
// Guard against an empty pass: the report must have real content.
expect(output.trim().length).toBeGreaterThan(200);
expect(output).toMatch(/^\|\s*Review\s*\|\s*Trigger\s*\|\s*Why\s*\|\s*Runs\s*\|\s*Status\s*\|\s*Findings\s*\|/m);
expect(output).toMatch(/^\|\s*CEO Review\s*\|/m);
// A native capture must not invent an outside dispatch or claim coverage.
expect(hasDisabledOutsideReview(output)).toBe(true);
// Loading a section and producing a table alone must not hide an empty
// review: the complete fixture still contains a real ordering defect.
expect(hasStaleFillRaceFinding(output)).toBe(true);
try { assertSectionLoadingReport(capture); }
catch (error) {
recordE2E(collector, 'plan-ceo-section-loading', 'plan-ceo-section-loading', capture.result,
{ passed: false, error: String(error), output: capture.output });
throw error;
}
recordE2E(collector, 'plan-ceo-section-loading', 'plan-ceo-section-loading', capture.result,
{ passed: true, output: capture.output });
},
CAPTURE_LONG_MS,
);
});
function assertSectionLoadingReport(capture: Awaited<ReturnType<typeof captureSectionReads>>): void {
validateCeoReviewCompletion(capture);
const { readSections, reportProduced, output } = capture;
const missing = REQUIRED_SECTIONS.filter(s => !readSections.has(s));
expect({ reportProduced, read: [...readSections], missing }).toEqual({
reportProduced: true,
read: expect.any(Array),
missing: [],
});
// Guard against an empty pass: the report must have real content.
expect(output.trim().length).toBeGreaterThan(200);
expect(output).toMatch(/^\|\s*Review\s*\|\s*Trigger\s*\|\s*Why\s*\|\s*Runs\s*\|\s*Status\s*\|\s*Findings\s*\|/m);
expect(output).toMatch(/^\|\s*CEO Review\s*\|/m);
// A native capture must not invent an outside dispatch or claim coverage.
expect(hasDisabledOutsideReview(output)).toBe(true);
// Loading a section and producing a table alone must not hide an empty
// review: the complete fixture still contains a real ordering defect.
expect(hasStaleFillRaceFinding(output)).toBe(true);
}