CalvinBackup/gstack
1
0 Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-11-09 02:08:19 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat(cso-eval): daily precision counts confirmed incidental issues and leaves out declared fixture instrumentation

Browse Source
This commit is contained in:
garrytan committed 2026-10-17 16:57:02 +00:00
1 parent d804fbb2fc
commit d561177ef4
3 files changed
+46 -5
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/cso-eval.ts
+14 -4
View File
@@ -98,8 +98,15 @@ export interface PortableSkillPayloadIdentity {
}
export interface EvalFinding {
id: string; evidence: 'supported' | 'hypothesis' | 'legacy_review'; claimedTested: boolean;
/** Written only by the independent evaluator after reviewing the reported trace. */
judgment: 'correct' | 'incorrect' | 'unadjudicated'; matchedCaseId?: string;
/**
* Written only by the independent evaluator after reviewing the reported trace. `correct`: the seeded issue.
* `incidental`: a different issue a unanimous judge panel confirmed is real in this source; a precision true
* positive, never recall. `instrumentation`: it targets the evaluator's declared fixture instrumentation
* (canary overlay files, egress probe), so precision leaves it out of both terms.
*/
judgment: 'correct' | 'incidental' | 'instrumentation' | 'incorrect' | 'unadjudicated'; matchedCaseId?: string;
/** `incidental` only: the evaluator's opaque key for the confirmed issue, so restatements count once per cell. */
incidentalKey?: string;
/**
* Written only by the independent evaluator. A producer's tested claim is
* valid only when this finding, rather than merely another finding in the
@@ -642,9 +649,11 @@ function validateResult(result: EvalResult, cell: EvalCell, corpus: EvalCorpusMa
if (result.producerReceiptHash !== undefined && !HEX.test(result.producerReceiptHash)) throw new Error('INVALID_PRODUCER_RECEIPT_PROVENANCE');
const ids = new Set<string>(), repairEvidence = new Set<string>(), recheckEvidence = new Set<string>();
for (const finding of result.findings) {
if (!finding.id || ids.has(finding.id) || !['supported', 'hypothesis', 'legacy_review'].includes(finding.evidence) || !['correct', 'incorrect', 'unadjudicated'].includes(finding.judgment) || typeof finding.claimedTested !== 'boolean') throw new Error('INVALID_FINDING_JUDGMENT');
if (!finding.id || ids.has(finding.id) || !['supported', 'hypothesis', 'legacy_review'].includes(finding.evidence) || !['correct', 'incidental', 'instrumentation', 'incorrect', 'unadjudicated'].includes(finding.judgment) || typeof finding.claimedTested !== 'boolean') throw new Error('INVALID_FINDING_JUDGMENT');
ids.add(finding.id);
if (finding.judgment === 'correct' && (cell.variant !== 'vulnerable' || finding.matchedCaseId !== cell.caseId)) throw new Error('INVALID_ORACLE_MATCH');
if ((finding.judgment === 'incidental' || finding.judgment === 'instrumentation') && (finding.evidence === 'hypothesis' || finding.matchedCaseId !== undefined)) throw new Error('INVALID_FINDING_JUDGMENT');
if ((finding.judgment === 'incidental') !== (finding.incidentalKey !== undefined) || (finding.incidentalKey !== undefined && !HEX.test(finding.incidentalKey))) throw new Error('INVALID_FINDING_JUDGMENT');
if (finding.trustedVerification !== undefined) {
const verification = finding.trustedVerification as Record<string, unknown>, allowed = new Set(['repair', 'repairEvidenceHash', 'recheck', 'recheckEvidenceHash']);
if (!verification || typeof verification !== 'object' || Array.isArray(verification) || Object.keys(verification).some(key => !allowed.has(key)) ||
@@ -745,10 +754,11 @@ export function scoreEval(matrix: EvalMatrix, observations: EvalResult[], qualif
let correct = 0, reported = 0, unadjudicated = 0;
for (const cell of selected) {
const result = results.get(cell.id);
const findings = result?.reportFinished ? result.findings.filter(finding => finding.evidence !== 'hypothesis') : [];
const findings = result?.reportFinished ? result.findings.filter(finding => finding.evidence !== 'hypothesis' && finding.judgment !== 'instrumentation') : [];
reported += findings.length;
unadjudicated += findings.filter(finding => finding.judgment === 'unadjudicated').length;
correct += Number(findings.some(finding => eligibleFinding(cell, finding) && finding.judgment === 'correct')); // Duplicate reports do not increase true positives.
correct += new Set(findings.filter(finding => eligibleFinding(cell, finding) && finding.judgment === 'incidental').map(finding => finding.incidentalKey)).size;
}
const firstUseful = completed.flatMap(result => result.firstUsefulResultMs === null ? [] : [result.firstUsefulResultMs]);
const costs = completed.flatMap(result => result.usage?.costUSD === undefined ? [] : [result.usage.costUSD]);