Merge remote-tracking branch 'origin/main' into capy/gate-cost-replay-and

This commit is contained in:
garrytan committed 2026-10-18 17:39:27 +00:00
commit 995406dcdb
8 files changed
+118 -6

No files matched your search

+60
View File
@@ -348,6 +348,66 @@ describe('CSO matched evaluation accounting', () => {
result.findings[1].trustedVerification = structuredClone(result.findings[0].trustedVerification);
expect(() => scoreEval(matrix, results, qualification)).toThrow('DUPLICATE_TRUSTED_REPAIR_BINDING');
});
test('tested claims on declared instrumentation count in neither direction; an incidental claim needs a passing replay of its signed bundle', () => {
const key = (seed: string) => createHash('sha256').update(seed).digest('hex');
const vulnerable = matrix.cells.find(item => item.version === 'v3' && item.mode === 'comprehensive' && item.variant === 'vulnerable')!;
const fixed = matrix.cells.find(item => item.version === 'v3' && item.mode === 'comprehensive' && item.variant === 'fixed')!;
const daily = matrix.cells.find(item => item.version === 'v3' && item.mode === 'daily' && item.variant === 'fixed')!;
const score = (edit: (byId: Map<string, EvalResult>) => void) => {
const results = matrix.cells.map(syntheticResult), byId = new Map(results.map(result => [result.cellId, result]));
edit(byId);
return scoreEval(matrix, results, qualification);
};
const clean = score(() => {});
expect(clean.gates.zeroFalselyTestedRepairs).toBe('pass');
const incidental = (id: string, replay?: EvalFinding['trustedReplay']): EvalFinding => ({ id, evidence: 'supported', claimedTested: true, judgment: 'incidental', incidentalKey: key(id), ...(replay ? { trustedReplay: replay } : {}) });
const instrumented = score(byId => {
byId.get(fixed.id)!.findings.push({ id: 'canary-claim', evidence: 'supported', claimedTested: true, judgment: 'instrumentation' });
byId.get(daily.id)!.findings.push({ id: 'probe-claim', evidence: 'supported', claimedTested: true, judgment: 'instrumentation' });
});
expect(group(instrumented, 'v3', 'comprehensive').falseTested).toBe(0); expect(group(instrumented, 'v3', 'daily').falseTested).toBe(0);
expect(instrumented.gates.zeroFalselyTestedRepairs).toBe('pass');
const instrumentationOnly = score(byId => {
const result = byId.get(vulnerable.id)!, binding = result.findings[0].trustedVerification!;
result.findings = [{ id: 'canary-claim', evidence: 'supported', claimedTested: true, judgment: 'instrumentation', trustedVerification: structuredClone(binding) }];
});
const empty = score(byId => { byId.get(vulnerable.id)!.findings = []; }), baseline = group(clean, 'v3', 'comprehensive');
const sameAsNoFinding = (scored: ReturnType<typeof scoreEval>) => {
const actual = group(scored, 'v3', 'comprehensive'), none = group(empty, 'v3', 'comprehensive');
expect(actual.falseTested).toBe(0);
expect([actual.recall, actual.highCriticalRecall, actual.repair, actual.recheck]).toEqual([none.recall, none.highCriticalRecall, none.repair, none.recheck]);
expect(scored.perStack).toEqual(empty.perStack);
expect(actual.recall.numerator).toBe(baseline.recall.numerator - 1);
expect(actual.repair.numerator).toBe(baseline.repair.numerator - 1);
};
sameAsNoFinding(instrumentationOnly);
for (const [replay, expected] of [[undefined, 1], [{ result: 'missing' }, 1], [{ result: 'failed' }, 1], [{ result: 'passed', replayEvidenceHash: key('replay') }, 0]] as const) {
const scored = score(byId => byId.get(fixed.id)!.findings.push(incidental('jwt-claim', replay as EvalFinding['trustedReplay'])));
expect(group(scored, 'v3', 'comprehensive').falseTested).toBe(expected);
expect(scored.gates.zeroFalselyTestedRepairs).toBe(expected ? 'fail' : 'pass');
}
const replayedOnly = score(byId => { byId.get(vulnerable.id)!.findings = [incidental('admin-claim', { result: 'passed', replayEvidenceHash: key('admin') })]; });
sameAsNoFinding(replayedOnly);
const passed = { result: 'passed', replayEvidenceHash: key('r') } as const;
for (const bad of [
{ id: 'on-correct', evidence: 'supported', claimedTested: true, judgment: 'correct', matchedCaseId: vulnerable.caseId, trustedReplay: passed },
{ id: 'on-instrumentation', evidence: 'supported', claimedTested: true, judgment: 'instrumentation', trustedReplay: passed },
{ id: 'on-incorrect', evidence: 'supported', claimedTested: true, judgment: 'incorrect', trustedReplay: { result: 'failed' } },
{ ...incidental('unclaimed', passed), claimedTested: false },
incidental('no-hash', { result: 'passed' } as EvalFinding['trustedReplay']),
incidental('failed-hash', { result: 'failed', replayEvidenceHash: key('f') } as EvalFinding['trustedReplay']),
incidental('short-hash', { result: 'passed', replayEvidenceHash: 'abc' }),
incidental('extra-key', { ...passed, repair: 'passed' } as EvalFinding['trustedReplay']),
incidental('bad-result', { result: 'skipped' } as unknown as EvalFinding['trustedReplay']),
] as EvalFinding[]) {
expect(() => score(byId => byId.get(vulnerable.id)!.findings.push(bad))).toThrow('INVALID_TRUSTED_REPLAY');
}
expect(() => score(byId => byId.get(fixed.id)!.findings.push(incidental('one', passed), incidental('two', passed)))).toThrow('DUPLICATE_TRUSTED_REPLAY_BINDING');
});
test('per-stack held-out repair requires the matching supported discovery', () => {
const results = matrix.cells.map(syntheticResult);
for (let index = 0; index < matrix.cells.length; index++) {
+3 -1
View File
@@ -247,7 +247,9 @@ unknown, and wall-clock budgets are not represented as model-spend caps.
The scorer checks 95% daily precision, 80% comprehensive high/critical recall,
no aggregate high/critical recall regression against v2, all core cold starts,
zero falsely certified runtime-tested repairs, and a correct held-out repair in
zero falsely certified runtime-tested repairs (claims on declared instrumentation
count in neither direction; an incidental claim needs a passing replay of its
signed bundle), and a correct held-out repair in
every application stack. It also requires complete matched results, mandatory
reports, containment/canary receipts, and enforced oracle separation. Missing
evaluations return