From d561177ef46e89c6841e31ef642f13fb6f274bbc Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 17 Oct 2026 16:57:02 +0000 Subject: [PATCH] feat(cso-eval): daily precision counts confirmed incidental issues and leaves out declared fixture instrumentation --- docs/cso-release-qualification.md | 12 ++++++++++++ scripts/cso-eval.ts | 18 ++++++++++++++---- test/cso-eval.test.ts | 21 ++++++++++++++++++++- 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/docs/cso-release-qualification.md b/docs/cso-release-qualification.md index dc7786f86..191130310 100644 --- a/docs/cso-release-qualification.md +++ b/docs/cso-release-qualification.md @@ -294,6 +294,18 @@ requalified catalog or withdraws the catalog's runtimes in the same pull request. Independently, the helper refuses to select a runtime recorded under another isolation policy. +The daily precision gate (`dailyPrecision95` in +[`cso-eval.ts`](../scripts/cso-eval.ts)) divides true positives by reported +findings. Hypotheses count in neither term. A finding the evaluator marks +`instrumentation` also counts in neither term: it targets the evaluator's +declared fixture instrumentation, such as the canary overlay files or the egress +probe route. The seeded issue adds one true positive per cell, however many +times it is restated. A finding the evaluator marks `incidental` is a different +issue that a unanimous judge panel confirmed is real in that source. It adds one +true positive per distinct `incidentalKey` in its cell, never counts toward +recall, and needs no seeded match. Recall still measures only the seeded issue, +and the thresholds are unchanged. + The judge observes held-out attacks in one boot: a `VerificationRequest`'s `security` field is either one assertion or an array of up to eight assertions with distinct names, judged in a single before/after pair. The before phase is diff --git a/scripts/cso-eval.ts b/scripts/cso-eval.ts index eef53751d..87d3280b9 100644 --- a/scripts/cso-eval.ts +++ b/scripts/cso-eval.ts @@ -98,8 +98,15 @@ export interface PortableSkillPayloadIdentity { } export interface EvalFinding { id: string; evidence: 'supported' | 'hypothesis' | 'legacy_review'; claimedTested: boolean; - /** Written only by the independent evaluator after reviewing the reported trace. */ - judgment: 'correct' | 'incorrect' | 'unadjudicated'; matchedCaseId?: string; + /** + * Written only by the independent evaluator after reviewing the reported trace. `correct`: the seeded issue. + * `incidental`: a different issue a unanimous judge panel confirmed is real in this source; a precision true + * positive, never recall. `instrumentation`: it targets the evaluator's declared fixture instrumentation + * (canary overlay files, egress probe), so precision leaves it out of both terms. + */ + judgment: 'correct' | 'incidental' | 'instrumentation' | 'incorrect' | 'unadjudicated'; matchedCaseId?: string; + /** `incidental` only: the evaluator's opaque key for the confirmed issue, so restatements count once per cell. */ + incidentalKey?: string; /** * Written only by the independent evaluator. A producer's tested claim is * valid only when this finding, rather than merely another finding in the @@ -642,9 +649,11 @@ function validateResult(result: EvalResult, cell: EvalCell, corpus: EvalCorpusMa if (result.producerReceiptHash !== undefined && !HEX.test(result.producerReceiptHash)) throw new Error('INVALID_PRODUCER_RECEIPT_PROVENANCE'); const ids = new Set(), repairEvidence = new Set(), recheckEvidence = new Set(); for (const finding of result.findings) { - if (!finding.id || ids.has(finding.id) || !['supported', 'hypothesis', 'legacy_review'].includes(finding.evidence) || !['correct', 'incorrect', 'unadjudicated'].includes(finding.judgment) || typeof finding.claimedTested !== 'boolean') throw new Error('INVALID_FINDING_JUDGMENT'); + if (!finding.id || ids.has(finding.id) || !['supported', 'hypothesis', 'legacy_review'].includes(finding.evidence) || !['correct', 'incidental', 'instrumentation', 'incorrect', 'unadjudicated'].includes(finding.judgment) || typeof finding.claimedTested !== 'boolean') throw new Error('INVALID_FINDING_JUDGMENT'); ids.add(finding.id); if (finding.judgment === 'correct' && (cell.variant !== 'vulnerable' || finding.matchedCaseId !== cell.caseId)) throw new Error('INVALID_ORACLE_MATCH'); + if ((finding.judgment === 'incidental' || finding.judgment === 'instrumentation') && (finding.evidence === 'hypothesis' || finding.matchedCaseId !== undefined)) throw new Error('INVALID_FINDING_JUDGMENT'); + if ((finding.judgment === 'incidental') !== (finding.incidentalKey !== undefined) || (finding.incidentalKey !== undefined && !HEX.test(finding.incidentalKey))) throw new Error('INVALID_FINDING_JUDGMENT'); if (finding.trustedVerification !== undefined) { const verification = finding.trustedVerification as Record, allowed = new Set(['repair', 'repairEvidenceHash', 'recheck', 'recheckEvidenceHash']); if (!verification || typeof verification !== 'object' || Array.isArray(verification) || Object.keys(verification).some(key => !allowed.has(key)) || @@ -745,10 +754,11 @@ export function scoreEval(matrix: EvalMatrix, observations: EvalResult[], qualif let correct = 0, reported = 0, unadjudicated = 0; for (const cell of selected) { const result = results.get(cell.id); - const findings = result?.reportFinished ? result.findings.filter(finding => finding.evidence !== 'hypothesis') : []; + const findings = result?.reportFinished ? result.findings.filter(finding => finding.evidence !== 'hypothesis' && finding.judgment !== 'instrumentation') : []; reported += findings.length; unadjudicated += findings.filter(finding => finding.judgment === 'unadjudicated').length; correct += Number(findings.some(finding => eligibleFinding(cell, finding) && finding.judgment === 'correct')); // Duplicate reports do not increase true positives. + correct += new Set(findings.filter(finding => eligibleFinding(cell, finding) && finding.judgment === 'incidental').map(finding => finding.incidentalKey)).size; } const firstUseful = completed.flatMap(result => result.firstUsefulResultMs === null ? [] : [result.firstUsefulResultMs]); const costs = completed.flatMap(result => result.usage?.costUSD === undefined ? [] : [result.usage.costUSD]); diff --git a/test/cso-eval.test.ts b/test/cso-eval.test.ts index 34b773c84..d1892ba87 100644 --- a/test/cso-eval.test.ts +++ b/test/cso-eval.test.ts @@ -4,7 +4,7 @@ import { chmodSync, cpSync, existsSync, linkSync, lstatSync, mkdirSync, mkdtempS import { createHash } from 'node:crypto'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; -import { collectProducerReceipts, createEvalBaseline, createEvalMatrix, createPortableSkillPayload, loadPortableSkillPayload, prepareEvalJobs, REQUIRED_CONTAINMENT, scoreCollectedEval, scoreEval, validateMatrix, validatePortableSkillPayload, type EvalBaseline, type EvalCell, type EvalMatrix, type EvalResult, type PreparedEvalSchedule } from '../scripts/cso-eval'; +import { collectProducerReceipts, createEvalBaseline, createEvalMatrix, createPortableSkillPayload, loadPortableSkillPayload, prepareEvalJobs, REQUIRED_CONTAINMENT, scoreCollectedEval, scoreEval, validateMatrix, validatePortableSkillPayload, type EvalBaseline, type EvalCell, type EvalFinding, type EvalMatrix, type EvalResult, type PreparedEvalSchedule } from '../scripts/cso-eval'; import { assertProducerArtifactPath, PRODUCER_PROVIDER_POLICY, producerFailureMessage, producerProviderTimeoutMs, producerInstallationIdentity, resolveProducerHelperBinding, runProducerCell, sanitizeProducerRun, validateProductionProducerInstallation } from '../scripts/cso-eval-producer'; import { producerArtifactInventoryHash, producerHostPlatform, producerInstallationIdentityHash, producerProviderIdentityHash, producerReceiptHash, sha256, type ProducerArtifactInventory, type ProducerInstallationIdentity, type ProducerProviderIdentity, type ProducerReceipt } from '../scripts/cso-eval-protocol'; import type { Family, ProviderAdapter, RunOpts, RunResult } from './helpers/providers/types'; @@ -316,6 +316,25 @@ describe('CSO matched evaluation accounting', () => { const two = syntheticResult(negative); two.findings.push({ id: 'false-positive', evidence: 'supported', claimedTested: false, judgment: 'incorrect' }); expect(group(scoreEval(matrix, [one, two]), 'v3', 'daily').precision).toEqual({ numerator: 1, denominator: 3, value: 1 / 3 }); }); + test('confirmed incidental issues count once each for precision, declared instrumentation counts in neither term, and recall still needs the seeded issue', () => { + const positive = matrix.cells.find(item => item.version === 'v3' && item.mode === 'daily' && item.variant === 'vulnerable')!; + const negative = matrix.cells.find(item => item.version === 'v3' && item.mode === 'daily' && item.variant === 'fixed')!; + const key = (seed: string) => createHash('sha256').update(seed).digest('hex'); + const one = syntheticResult(positive); + one.findings.push({ id: 'jwt', evidence: 'supported', claimedTested: false, judgment: 'incidental', incidentalKey: key('jwt') }, { id: 'jwt-again', evidence: 'supported', claimedTested: false, judgment: 'incidental', incidentalKey: key('jwt') }); + one.findings.push({ id: 'canary', evidence: 'supported', claimedTested: false, judgment: 'instrumentation' }, { id: 'guess', evidence: 'hypothesis', claimedTested: false, judgment: 'incidental', incidentalKey: key('guess') }); + const two = syntheticResult(negative); + two.findings.push({ id: 'admin', evidence: 'supported', claimedTested: false, judgment: 'incidental', incidentalKey: key('admin') }, { id: 'probe', evidence: 'supported', claimedTested: false, judgment: 'instrumentation' }, { id: 'noise', evidence: 'supported', claimedTested: false, judgment: 'incorrect' }); + expect(() => scoreEval(matrix, [one])).toThrow('INVALID_FINDING_JUDGMENT'); + one.findings.pop(); + expect(group(scoreEval(matrix, [one, two]), 'v3', 'daily').precision).toEqual({ numerator: 3, denominator: 5, value: 3 / 5 }); + const missed = syntheticResult(positive); missed.findings = [{ id: 'only-incidental', evidence: 'supported', claimedTested: false, judgment: 'incidental', incidentalKey: key('x') }]; + expect(group(scoreEval(matrix, [missed]), 'v3', 'daily').recall.numerator).toBe(0); + for (const bad of [{ judgment: 'incidental' }, { judgment: 'incidental', incidentalKey: 'short' }, { judgment: 'incorrect', incidentalKey: key('y') }, { judgment: 'instrumentation', matchedCaseId: positive.caseId }, { judgment: 'incidental', incidentalKey: key('z'), matchedCaseId: positive.caseId }]) { + const result = syntheticResult(positive); result.findings.push({ id: 'bad', evidence: 'supported', claimedTested: false, ...bad } as EvalFinding); + expect(() => scoreEval(matrix, [result])).toThrow('INVALID_FINDING_JUDGMENT'); + } + }); test('producer tested claims require independent successful held-out repair evidence', () => { const cell = matrix.cells.find(item => item.version === 'v3' && item.mode === 'comprehensive' && item.variant === 'vulnerable')!; const result = syntheticResult(cell); delete result.oracleEvidenceHash;