From b6ad9090d7c9cd7123979d23294a5d9da305a1e2 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Fri, 11 Sep 2026 06:02:40 +0000 Subject: [PATCH] fix(cso): require complete evaluation reports Co-Authored-By: OpenAI Codex --- scripts/cso-eval.ts | 19 ++++++++++++------- test/cso-eval.test.ts | 16 ++++++++++++++++ 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/scripts/cso-eval.ts b/scripts/cso-eval.ts index 268e95b84..70050cbde 100644 --- a/scripts/cso-eval.ts +++ b/scripts/cso-eval.ts @@ -493,7 +493,7 @@ function quantile(values: number[], fraction: number): number | null { function measuredRate(cells: EvalCell[], results: Map, field: 'setup' | 'reproduction' | 'repair' | 'recheck', corpus: CorpusManifest): Rate { return rate(cells.filter(cell => { const result = results.get(cell.id); - if (!result || result[field] !== 'passed') return false; + if (!result?.reportPresent || !result.reportComplete || result[field] !== 'passed') return false; if (field === 'setup') return true; if (!result.oracleEvidenceHash || result.oracleVersion !== corpus.version || result.setup !== 'passed') return false; if (field === 'reproduction') return true; @@ -510,7 +510,7 @@ function measuredRate(cells: EvalCell[], results: Map, field function trustedFindingVerification(cell: EvalCell, result: EvalResult, finding: EvalFinding, stage: 'repair' | 'recheck', corpus: CorpusManifest): boolean { const verification = finding.trustedVerification; - if (cell.version !== 'v3' || cell.mode !== 'comprehensive' || cell.variant !== 'vulnerable' || finding.evidence !== 'supported' || + if (!result.reportPresent || !result.reportComplete || cell.version !== 'v3' || cell.mode !== 'comprehensive' || cell.variant !== 'vulnerable' || finding.evidence !== 'supported' || finding.judgment !== 'correct' || finding.matchedCaseId !== cell.caseId || !finding.claimedTested || !verification || result.setup !== 'passed' || result.reproduction !== 'passed' || result.repair !== 'passed' || !result.heldOutAssertionsPassed || verification.repair !== 'passed' || verification.repairEvidenceHash !== result.oracleEvidenceHash || result.oracleVersion !== corpus.version) @@ -532,7 +532,11 @@ export function scoreEval(matrix: EvalMatrix, observations: EvalResult[], qualif validateResult(result, cell, corpus); results.set(result.cellId, result); } const eligibleFinding = (cell: EvalCell, finding: EvalFinding) => finding.evidence === 'supported' || (cell.version === 'v2' && finding.evidence === 'legacy_review'); - const found = (cell: EvalCell) => results.get(cell.id)?.findings.some(finding => eligibleFinding(cell, finding) && finding.judgment === 'correct') ?? false; + const found = (cell: EvalCell) => { + const result = results.get(cell.id); + return result?.reportPresent === true && result.reportComplete === true && + result.findings.some(finding => eligibleFinding(cell, finding) && finding.judgment === 'correct'); + }; const groups: any[] = []; for (const version of ['v2', 'v3'] as const) for (const mode of ['daily', 'comprehensive'] as const) { const selected = matrix.cells.filter(cell => cell.version === version && cell.mode === mode); @@ -541,7 +545,8 @@ export function scoreEval(matrix: EvalMatrix, observations: EvalResult[], qualif const highCritical = expected.filter(cell => ['critical','high'].includes(corpus.cases.find(fixture => fixture.id === cell.caseId)!.severity)); let correct = 0, reported = 0, unadjudicated = 0; for (const cell of selected) { - const findings = results.get(cell.id)?.findings.filter(finding => finding.evidence !== 'hypothesis') ?? []; + const result = results.get(cell.id); + const findings = result?.reportPresent && result.reportComplete ? result.findings.filter(finding => finding.evidence !== 'hypothesis') : []; reported += findings.length; unadjudicated += findings.filter(finding => finding.judgment === 'unadjudicated').length; correct += Number(findings.some(finding => eligibleFinding(cell, finding) && finding.judgment === 'correct')); // Duplicate reports do not increase true positives. @@ -554,7 +559,7 @@ export function scoreEval(matrix: EvalMatrix, observations: EvalResult[], qualif return count + result.findings.filter(finding => finding.claimedTested && !trustedFindingVerification(cell, result, finding, 'repair', corpus)).length; }, 0); groups.push({ version, mode, cells: selected.length, submitted: completed.length, missing: selected.length - completed.length, - reports: rate(completed.filter(result => result.reportPresent).length, selected.length), precision: rate(correct, reported), recall: rate(expected.filter(found).length, expected.length), highCriticalRecall: rate(highCritical.filter(found).length, highCritical.length), + reports: rate(completed.filter(result => result.reportPresent && result.reportComplete).length, selected.length), precision: rate(correct, reported), recall: rate(expected.filter(found).length, expected.length), highCriticalRecall: rate(highCritical.filter(found).length, highCritical.length), unadjudicated, falseTested, setup: mode === 'comprehensive' ? measuredRate(selected, results, 'setup', corpus) : null, reproduction: mode === 'comprehensive' ? measuredRate(expected, results, 'reproduction', corpus) : null, repair: mode === 'comprehensive' ? measuredRate(expected, results, 'repair', corpus) : null, @@ -579,13 +584,13 @@ export function scoreEval(matrix: EvalMatrix, observations: EvalResult[], qualif }).map(cell => cell.caseId)); return [stack, { correctHeldOutRepairs: successful.size, denominator: new Set(eligible.map(cell => cell.caseId)).size }]; })); - const assessedAll = observations.length === matrix.cells.length && groups.every(group => !group.unadjudicated); + const assessedAll = observations.length === matrix.cells.length && observations.every(result => result.reportPresent && result.reportComplete) && groups.every(group => !group.unadjudicated); const gate = (condition: boolean | null, hasData: boolean): 'pass' | 'fail' | 'unmeasured' => !hasData || condition === null ? 'unmeasured' : condition ? 'pass' : 'fail'; const containmentValues = REQUIRED_CONTAINMENT.map(name => qualification.containment?.[name]); if (containmentValues.some(value => value !== undefined && !['passed', 'failed', 'not_run'].includes(value))) throw new Error('INVALID_CONTAINMENT_RESULT'); const gates = { matchedCompleteMatrix: gate(assessedAll, observations.length > 0), - mandatoryReports: gate(groups.every(group => group.reports.numerator === group.reports.denominator), assessedAll), + mandatoryReports: gate(groups.every(group => group.reports.numerator === group.reports.denominator), observations.length === matrix.cells.length), dailyPrecision95: gate(daily.precision.value === null ? null : daily.precision.value >= 0.95, daily.submitted === daily.cells && !daily.unadjudicated), comprehensiveHighCriticalRecall80: gate(comprehensive.highCriticalRecall.value >= 0.8, comprehensive.submitted === comprehensive.cells && !comprehensive.unadjudicated), noHighCriticalRecallRegression: gate(comprehensive.highCriticalRecall.value >= baseline.highCriticalRecall.value, comprehensive.submitted === comprehensive.cells && baseline.submitted === baseline.cells && !comprehensive.unadjudicated && !baseline.unadjudicated), diff --git a/test/cso-eval.test.ts b/test/cso-eval.test.ts index 3102e0d14..33bb71619 100644 --- a/test/cso-eval.test.ts +++ b/test/cso-eval.test.ts @@ -195,6 +195,22 @@ describe('CSO matched evaluation accounting', () => { expect(group(score, 'v3', 'comprehensive').highCriticalRecall).toEqual({ numerator: 96, denominator: 96, value: 1 }); expect(score.perStack.rails).toEqual({ correctHeldOutRepairs: 10, denominator: 10 }); }); + test('incomplete mandatory reports cannot contribute security evidence or qualify a complete matrix', () => { + const results = matrix.cells.map(syntheticResult); + for (const result of results) result.reportComplete = false; + const score = scoreEval(matrix, results, qualification), daily = group(score, 'v3', 'daily'), comprehensive = group(score, 'v3', 'comprehensive'); + expect(score.status).toBe('partial'); + expect(score.gates.matchedCompleteMatrix).toBe('fail'); + expect(score.gates.mandatoryReports).toBe('fail'); + expect(daily.reports).toEqual({ numerator: 0, denominator: 240, value: 0 }); + expect(daily.precision).toEqual({ numerator: 0, denominator: 0, value: null }); + expect(daily.recall.numerator).toBe(0); + expect(comprehensive.setup.numerator).toBe(0); + expect(comprehensive.reproduction.numerator).toBe(0); + expect(comprehensive.repair.numerator).toBe(0); + expect(comprehensive.recheck.numerator).toBe(0); + expect(score.perStack.rails.correctHeldOutRepairs).toBe(0); + }); test('release scoring binds every trusted judgment to a complete matched producer batch', () => { const batch = syntheticBatch(); const results = matrix.cells.map(cell => ({ ...syntheticResult(cell), producerReceiptHash: batch.receipts.find(receipt => receipt.cell.id === cell.id)!.receiptHash }));