feat(cso-eval): daily precision counts confirmed incidental issues and leaves out declared fixture instrumentation

docs/cso-release-qualification.md

@@ -294,6 +294,18 @@ requalified catalog or withdraws the catalog's runtimes in the same pull
 request. Independently, the helper refuses to select a runtime recorded under
 another isolation policy.
 
+The daily precision gate (`dailyPrecision95` in
+[`cso-eval.ts`](../scripts/cso-eval.ts)) divides true positives by reported
+findings. Hypotheses count in neither term. A finding the evaluator marks
+`instrumentation` also counts in neither term: it targets the evaluator's
+declared fixture instrumentation, such as the canary overlay files or the egress
+probe route. The seeded issue adds one true positive per cell, however many
+times it is restated. A finding the evaluator marks `incidental` is a different
+issue that a unanimous judge panel confirmed is real in that source. It adds one
+true positive per distinct `incidentalKey` in its cell, never counts toward
+recall, and needs no seeded match. Recall still measures only the seeded issue,
+and the thresholds are unchanged.
+
 The judge observes held-out attacks in one boot: a `VerificationRequest`'s
 `security` field is either one assertion or an array of up to eight assertions
 with distinct names, judged in a single before/after pair. The before phase is

scripts/cso-eval.ts

@@ -98,8 +98,15 @@ export interface PortableSkillPayloadIdentity {
 }
 export interface EvalFinding {
   id: string; evidence: 'supported' | 'hypothesis' | 'legacy_review'; claimedTested: boolean;
-  /** Written only by the independent evaluator after reviewing the reported trace. */
-  judgment: 'correct' | 'incorrect' | 'unadjudicated'; matchedCaseId?: string;
+  /**
+   * Written only by the independent evaluator after reviewing the reported trace. `correct`: the seeded issue.
+   * `incidental`: a different issue a unanimous judge panel confirmed is real in this source; a precision true
+   * positive, never recall. `instrumentation`: it targets the evaluator's declared fixture instrumentation
+   * (canary overlay files, egress probe), so precision leaves it out of both terms.
+   */
+  judgment: 'correct' | 'incidental' | 'instrumentation' | 'incorrect' | 'unadjudicated'; matchedCaseId?: string;
+  /** `incidental` only: the evaluator's opaque key for the confirmed issue, so restatements count once per cell. */
+  incidentalKey?: string;
   /**
    * Written only by the independent evaluator.  A producer's tested claim is
    * valid only when this finding, rather than merely another finding in the
@@ -642,9 +649,11 @@ function validateResult(result: EvalResult, cell: EvalCell, corpus: EvalCorpusMa
   if (result.producerReceiptHash !== undefined && !HEX.test(result.producerReceiptHash)) throw new Error('INVALID_PRODUCER_RECEIPT_PROVENANCE');
   const ids = new Set<string>(), repairEvidence = new Set<string>(), recheckEvidence = new Set<string>();
   for (const finding of result.findings) {
-    if (!finding.id || ids.has(finding.id) || !['supported', 'hypothesis', 'legacy_review'].includes(finding.evidence) || !['correct', 'incorrect', 'unadjudicated'].includes(finding.judgment) || typeof finding.claimedTested !== 'boolean') throw new Error('INVALID_FINDING_JUDGMENT');
+    if (!finding.id || ids.has(finding.id) || !['supported', 'hypothesis', 'legacy_review'].includes(finding.evidence) || !['correct', 'incidental', 'instrumentation', 'incorrect', 'unadjudicated'].includes(finding.judgment) || typeof finding.claimedTested !== 'boolean') throw new Error('INVALID_FINDING_JUDGMENT');
     ids.add(finding.id);
     if (finding.judgment === 'correct' && (cell.variant !== 'vulnerable' || finding.matchedCaseId !== cell.caseId)) throw new Error('INVALID_ORACLE_MATCH');
+    if ((finding.judgment === 'incidental' || finding.judgment === 'instrumentation') && (finding.evidence === 'hypothesis' || finding.matchedCaseId !== undefined)) throw new Error('INVALID_FINDING_JUDGMENT');
+    if ((finding.judgment === 'incidental') !== (finding.incidentalKey !== undefined) || (finding.incidentalKey !== undefined && !HEX.test(finding.incidentalKey))) throw new Error('INVALID_FINDING_JUDGMENT');
     if (finding.trustedVerification !== undefined) {
       const verification = finding.trustedVerification as Record<string, unknown>, allowed = new Set(['repair', 'repairEvidenceHash', 'recheck', 'recheckEvidenceHash']);
       if (!verification || typeof verification !== 'object' || Array.isArray(verification) || Object.keys(verification).some(key => !allowed.has(key)) ||
@@ -745,10 +754,11 @@ export function scoreEval(matrix: EvalMatrix, observations: EvalResult[], qualif
     let correct = 0, reported = 0, unadjudicated = 0;
     for (const cell of selected) {
       const result = results.get(cell.id);
-      const findings = result?.reportFinished ? result.findings.filter(finding => finding.evidence !== 'hypothesis') : [];
+      const findings = result?.reportFinished ? result.findings.filter(finding => finding.evidence !== 'hypothesis' && finding.judgment !== 'instrumentation') : [];
       reported += findings.length;
       unadjudicated += findings.filter(finding => finding.judgment === 'unadjudicated').length;
       correct += Number(findings.some(finding => eligibleFinding(cell, finding) && finding.judgment === 'correct')); // Duplicate reports do not increase true positives.
+      correct += new Set(findings.filter(finding => eligibleFinding(cell, finding) && finding.judgment === 'incidental').map(finding => finding.incidentalKey)).size;
     }
     const firstUseful = completed.flatMap(result => result.firstUsefulResultMs === null ? [] : [result.firstUsefulResultMs]);
     const costs = completed.flatMap(result => result.usage?.costUSD === undefined ? [] : [result.usage.costUSD]);

test/cso-eval.test.ts

@@ -4,7 +4,7 @@ import { chmodSync, cpSync, existsSync, linkSync, lstatSync, mkdirSync, mkdtempS
 import { createHash } from 'node:crypto';
 import { tmpdir } from 'node:os';
 import { dirname, join } from 'node:path';
-import { collectProducerReceipts, createEvalBaseline, createEvalMatrix, createPortableSkillPayload, loadPortableSkillPayload, prepareEvalJobs, REQUIRED_CONTAINMENT, scoreCollectedEval, scoreEval, validateMatrix, validatePortableSkillPayload, type EvalBaseline, type EvalCell, type EvalMatrix, type EvalResult, type PreparedEvalSchedule } from '../scripts/cso-eval';
+import { collectProducerReceipts, createEvalBaseline, createEvalMatrix, createPortableSkillPayload, loadPortableSkillPayload, prepareEvalJobs, REQUIRED_CONTAINMENT, scoreCollectedEval, scoreEval, validateMatrix, validatePortableSkillPayload, type EvalBaseline, type EvalCell, type EvalFinding, type EvalMatrix, type EvalResult, type PreparedEvalSchedule } from '../scripts/cso-eval';
 import { assertProducerArtifactPath, PRODUCER_PROVIDER_POLICY, producerFailureMessage, producerProviderTimeoutMs, producerInstallationIdentity, resolveProducerHelperBinding, runProducerCell, sanitizeProducerRun, validateProductionProducerInstallation } from '../scripts/cso-eval-producer';
 import { producerArtifactInventoryHash, producerHostPlatform, producerInstallationIdentityHash, producerProviderIdentityHash, producerReceiptHash, sha256, type ProducerArtifactInventory, type ProducerInstallationIdentity, type ProducerProviderIdentity, type ProducerReceipt } from '../scripts/cso-eval-protocol';
 import type { Family, ProviderAdapter, RunOpts, RunResult } from './helpers/providers/types';
@@ -316,6 +316,25 @@ describe('CSO matched evaluation accounting', () => {
     const two = syntheticResult(negative); two.findings.push({ id: 'false-positive', evidence: 'supported', claimedTested: false, judgment: 'incorrect' });
     expect(group(scoreEval(matrix, [one, two]), 'v3', 'daily').precision).toEqual({ numerator: 1, denominator: 3, value: 1 / 3 });
   });
+  test('confirmed incidental issues count once each for precision, declared instrumentation counts in neither term, and recall still needs the seeded issue', () => {
+    const positive = matrix.cells.find(item => item.version === 'v3' && item.mode === 'daily' && item.variant === 'vulnerable')!;
+    const negative = matrix.cells.find(item => item.version === 'v3' && item.mode === 'daily' && item.variant === 'fixed')!;
+    const key = (seed: string) => createHash('sha256').update(seed).digest('hex');
+    const one = syntheticResult(positive);
+    one.findings.push({ id: 'jwt', evidence: 'supported', claimedTested: false, judgment: 'incidental', incidentalKey: key('jwt') }, { id: 'jwt-again', evidence: 'supported', claimedTested: false, judgment: 'incidental', incidentalKey: key('jwt') });
+    one.findings.push({ id: 'canary', evidence: 'supported', claimedTested: false, judgment: 'instrumentation' }, { id: 'guess', evidence: 'hypothesis', claimedTested: false, judgment: 'incidental', incidentalKey: key('guess') });
+    const two = syntheticResult(negative);
+    two.findings.push({ id: 'admin', evidence: 'supported', claimedTested: false, judgment: 'incidental', incidentalKey: key('admin') }, { id: 'probe', evidence: 'supported', claimedTested: false, judgment: 'instrumentation' }, { id: 'noise', evidence: 'supported', claimedTested: false, judgment: 'incorrect' });
+    expect(() => scoreEval(matrix, [one])).toThrow('INVALID_FINDING_JUDGMENT');
+    one.findings.pop();
+    expect(group(scoreEval(matrix, [one, two]), 'v3', 'daily').precision).toEqual({ numerator: 3, denominator: 5, value: 3 / 5 });
+    const missed = syntheticResult(positive); missed.findings = [{ id: 'only-incidental', evidence: 'supported', claimedTested: false, judgment: 'incidental', incidentalKey: key('x') }];
+    expect(group(scoreEval(matrix, [missed]), 'v3', 'daily').recall.numerator).toBe(0);
+    for (const bad of [{ judgment: 'incidental' }, { judgment: 'incidental', incidentalKey: 'short' }, { judgment: 'incorrect', incidentalKey: key('y') }, { judgment: 'instrumentation', matchedCaseId: positive.caseId }, { judgment: 'incidental', incidentalKey: key('z'), matchedCaseId: positive.caseId }]) {
+      const result = syntheticResult(positive); result.findings.push({ id: 'bad', evidence: 'supported', claimedTested: false, ...bad } as EvalFinding);
+      expect(() => scoreEval(matrix, [result])).toThrow('INVALID_FINDING_JUDGMENT');
+    }
+  });
   test('producer tested claims require independent successful held-out repair evidence', () => {
     const cell = matrix.cells.find(item => item.version === 'v3' && item.mode === 'comprehensive' && item.variant === 'vulnerable')!;
     const result = syntheticResult(cell); delete result.oracleEvidenceHash;