diff --git a/CHANGELOG.md b/CHANGELOG.md index f0d0e97d9..f9839d4ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## [1.91.136.0] - 2026-10-18 + +**/cso release qualification counts a tested claim as false only when trusted evidence could have certified it.** In the second paid runtime qualification, the falsely-tested gate counted 33 claims. 12 of them were on the evaluator's own fixture instrumentation and 2 were on real incidental issues, which only the seeded repair verification could ever certify. Garry chose option 1. + +### Changed +- **Tested claims on declared instrumentation count in neither direction.** A finding the evaluator marks `instrumentation` is never falsely tested and never certified, the same way precision leaves it out. +- **A tested claim on an incidental finding needs a passing replay.** The evaluator replays the helper's signed repair bundle for that finding against the trusted source and records `trustedReplay`. Only `passed` with a 64-hex `replayEvidenceHash` certifies the claim. `failed` or `missing` (no bound bundle) leaves it falsely tested. +- **Recall, repair, held-out repair, the v2 baseline and every threshold are unchanged.** Those still need the seeded issue and the judge's own held-out verification. Validation rejects `trustedReplay` on any judgment other than a claimed `incidental`, a malformed or reused hash, and unknown fields. + +### If you… +- **…write an evaluator for `cso-eval.ts`:** write `trustedReplay` only on claimed `incidental` findings, with `result` `passed`, `failed` or `missing`, and `replayEvidenceHash` on `passed` only. + ## [1.91.135.0] - 2026-10-18 **/cso evaluation cells that run a repair no longer fail on the helper's own file names.** In the second paid runtime qualification, one comprehensive Rails cell failed with REDACTION_FAILED. Once a repair ran, the helper retained dependency closures as `-before-.json`. The producer's artifact inventory checks each path component, and this name didn't match any known helper-ID shape. So the secret redactor read the 64-character hex finding ID as a credential and failed the whole cell. diff --git a/VERSION b/VERSION index ec3b8f365..ebbca0fda 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.135.0 +1.91.136.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index cbb0c7cc4..e5bcd63fa 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.135.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.136.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/docs/cso-release-qualification.md b/docs/cso-release-qualification.md index 191130310..2927b04f7 100644 --- a/docs/cso-release-qualification.md +++ b/docs/cso-release-qualification.md @@ -306,6 +306,18 @@ true positive per distinct `incidentalKey` in its cell, never counts toward recall, and needs no seeded match. Recall still measures only the seeded issue, and the thresholds are unchanged. +The falsely-tested gate (`zeroFalselyTestedRepairs`) counts every finding the +producer claimed was tested that trusted evidence does not certify. A claim on +the seeded issue is certified only by the judge's own held-out repair +verification. A claim on an `incidental` finding is certified only when the +evaluator replayed the helper's signed repair bundle for that finding against +the trusted source and the replay passed: the evaluator records +`trustedReplay: { result: "passed", replayEvidenceHash }`. A `failed` replay or a +`missing` bundle leaves the claim falsely tested. A claim on an +`instrumentation` finding counts in neither direction, the same way precision +leaves it out. Neither a replay nor an instrumentation claim ever counts toward +recall, repair or held-out rates, which need the seeded issue. + The judge observes held-out attacks in one boot: a `VerificationRequest`'s `security` field is either one assertion or an array of up to eight assertions with distinct names, judged in a single before/after pair. The before phase is diff --git a/package.json b/package.json index 36e793f2a..75a61bf43 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.135", + "version": "1.91.136", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/scripts/cso-eval.ts b/scripts/cso-eval.ts index 87d3280b9..a9b0f9290 100644 --- a/scripts/cso-eval.ts +++ b/scripts/cso-eval.ts @@ -118,6 +118,13 @@ export interface EvalFinding { recheck: Outcome; recheckEvidenceHash?: string; }; + /** + * Written only by the independent evaluator, for a tested claim on an `incidental` finding: the result of + * replaying the helper's signed repair bundle for that finding against the trusted source. `passed` (with the + * replay's evidence hash) certifies the claim; `failed` or `missing` (no bound bundle) leaves it falsely tested. + * It never counts toward recall, repair or held-out rates, which need the seeded issue. + */ + trustedReplay?: { result: 'passed' | 'failed' | 'missing'; replayEvidenceHash?: string }; } export interface EvalResult { cellId: string; sourceHash: string; skillHash: string; model: string; host: string; budgetSeconds: number; @@ -647,7 +654,7 @@ function validateResult(result: EvalResult, cell: EvalCell, corpus: EvalCorpusMa if (result.recheckEvidenceHash !== undefined && !HEX.test(result.recheckEvidenceHash)) throw new Error('INVALID_RECHECK_EVIDENCE'); if (result.usage && (result.usage.source !== 'host' || (result.usage.tokens !== undefined && !validNumber(result.usage.tokens)) || (result.usage.costUSD !== undefined && !validNumber(result.usage.costUSD)))) throw new Error('INVALID_EVAL_USAGE'); if (result.producerReceiptHash !== undefined && !HEX.test(result.producerReceiptHash)) throw new Error('INVALID_PRODUCER_RECEIPT_PROVENANCE'); - const ids = new Set(), repairEvidence = new Set(), recheckEvidence = new Set(); + const ids = new Set(), repairEvidence = new Set(), recheckEvidence = new Set(), replayEvidence = new Set(); for (const finding of result.findings) { if (!finding.id || ids.has(finding.id) || !['supported', 'hypothesis', 'legacy_review'].includes(finding.evidence) || !['correct', 'incidental', 'instrumentation', 'incorrect', 'unadjudicated'].includes(finding.judgment) || typeof finding.claimedTested !== 'boolean') throw new Error('INVALID_FINDING_JUDGMENT'); ids.add(finding.id); @@ -676,6 +683,17 @@ function validateResult(result: EvalResult, cell: EvalCell, corpus: EvalCorpusMa recheckEvidence.add(recheckHash); } } + if (finding.trustedReplay !== undefined) { + const replay = finding.trustedReplay as Record, hash = replay?.replayEvidenceHash; + if (!replay || typeof replay !== 'object' || Array.isArray(replay) || Object.keys(replay).some(key => key !== 'result' && key !== 'replayEvidenceHash') || + !['passed', 'failed', 'missing'].includes(String(replay.result)) || finding.judgment !== 'incidental' || !finding.claimedTested || + (replay.result === 'passed') !== (typeof hash === 'string' && HEX.test(hash)) || (replay.result !== 'passed' && hash !== undefined)) + throw new Error('INVALID_TRUSTED_REPLAY'); + if (typeof hash === 'string') { + if (replayEvidence.has(hash)) throw new Error('DUPLICATE_TRUSTED_REPLAY_BINDING'); + replayEvidence.add(hash); + } + } } if (cell.mode === 'daily' && [result.setup, result.reproduction, result.repair, result.recheck].some(value => value !== 'not_attempted')) throw new Error('DAILY_EVAL_EXECUTED_APPLICATION'); } @@ -728,6 +746,14 @@ function validatedResults(matrix: EvalMatrix, observations: EvalResult[], corpus const eligibleFinding = (cell: EvalCell, finding: EvalFinding) => finding.evidence === 'supported' || (cell.version === 'v2' && finding.evidence === 'legacy_review'); const foundSeededIssue = (cell: EvalCell, result: EvalResult | undefined) => result?.reportFinished === true && result.findings.some(finding => eligibleFinding(cell, finding) && finding.judgment === 'correct'); +/** + * A tested claim is false unless trusted evidence certifies it: the seeded repair's held-out verification, or, for an + * `incidental` finding, a passing replay of the helper's signed bundle. Claims on declared instrumentation count in + * neither direction (Garry, 2026-10-18, option 1), the same way precision leaves them out. + */ +const falselyTested = (cell: EvalCell, result: EvalResult, finding: EvalFinding, corpus: EvalCorpusManifest) => + finding.claimedTested && finding.judgment !== 'instrumentation' && !trustedFindingVerification(cell, result, finding, 'repair', corpus) && + !(finding.judgment === 'incidental' && finding.trustedReplay?.result === 'passed'); const isHighCritical = (corpus: EvalCorpusManifest, caseId: string) => ['critical', 'high'].includes(corpus.cases.find(fixture => fixture.id === caseId)!.severity); /** High/critical recall of a pre-registered external baseline, after proving it binds to this release matrix. */ @@ -765,7 +791,7 @@ export function scoreEval(matrix: EvalMatrix, observations: EvalResult[], qualif const tokens = completed.flatMap(result => result.usage?.tokens === undefined ? [] : [result.usage.tokens]); const falseTested = selected.reduce((count, cell) => { const result = results.get(cell.id); if (!result) return count; - return count + result.findings.filter(finding => finding.claimedTested && !trustedFindingVerification(cell, result, finding, 'repair', corpus)).length; + return count + result.findings.filter(finding => falselyTested(cell, result, finding, corpus)).length; }, 0); groups.push({ version, mode, cells: selected.length, submitted: completed.length, missing: selected.length - completed.length, reports: rate(completed.filter(result => result.reportPresent && result.reportComplete).length, selected.length), finishedReports: rate(completed.filter(result => result.reportFinished).length, selected.length), precision: rate(correct, reported), recall: rate(expected.filter(found).length, expected.length), highCriticalRecall: rate(highCritical.filter(found).length, highCritical.length), diff --git a/test/cso-eval.test.ts b/test/cso-eval.test.ts index b398f7300..b64390551 100644 --- a/test/cso-eval.test.ts +++ b/test/cso-eval.test.ts @@ -348,6 +348,66 @@ describe('CSO matched evaluation accounting', () => { result.findings[1].trustedVerification = structuredClone(result.findings[0].trustedVerification); expect(() => scoreEval(matrix, results, qualification)).toThrow('DUPLICATE_TRUSTED_REPAIR_BINDING'); }); + test('tested claims on declared instrumentation count in neither direction; an incidental claim needs a passing replay of its signed bundle', () => { + const key = (seed: string) => createHash('sha256').update(seed).digest('hex'); + const vulnerable = matrix.cells.find(item => item.version === 'v3' && item.mode === 'comprehensive' && item.variant === 'vulnerable')!; + const fixed = matrix.cells.find(item => item.version === 'v3' && item.mode === 'comprehensive' && item.variant === 'fixed')!; + const daily = matrix.cells.find(item => item.version === 'v3' && item.mode === 'daily' && item.variant === 'fixed')!; + const score = (edit: (byId: Map) => void) => { + const results = matrix.cells.map(syntheticResult), byId = new Map(results.map(result => [result.cellId, result])); + edit(byId); + return scoreEval(matrix, results, qualification); + }; + const clean = score(() => {}); + expect(clean.gates.zeroFalselyTestedRepairs).toBe('pass'); + const incidental = (id: string, replay?: EvalFinding['trustedReplay']): EvalFinding => ({ id, evidence: 'supported', claimedTested: true, judgment: 'incidental', incidentalKey: key(id), ...(replay ? { trustedReplay: replay } : {}) }); + + const instrumented = score(byId => { + byId.get(fixed.id)!.findings.push({ id: 'canary-claim', evidence: 'supported', claimedTested: true, judgment: 'instrumentation' }); + byId.get(daily.id)!.findings.push({ id: 'probe-claim', evidence: 'supported', claimedTested: true, judgment: 'instrumentation' }); + }); + expect(group(instrumented, 'v3', 'comprehensive').falseTested).toBe(0); expect(group(instrumented, 'v3', 'daily').falseTested).toBe(0); + expect(instrumented.gates.zeroFalselyTestedRepairs).toBe('pass'); + + const instrumentationOnly = score(byId => { + const result = byId.get(vulnerable.id)!, binding = result.findings[0].trustedVerification!; + result.findings = [{ id: 'canary-claim', evidence: 'supported', claimedTested: true, judgment: 'instrumentation', trustedVerification: structuredClone(binding) }]; + }); + const empty = score(byId => { byId.get(vulnerable.id)!.findings = []; }), baseline = group(clean, 'v3', 'comprehensive'); + const sameAsNoFinding = (scored: ReturnType) => { + const actual = group(scored, 'v3', 'comprehensive'), none = group(empty, 'v3', 'comprehensive'); + expect(actual.falseTested).toBe(0); + expect([actual.recall, actual.highCriticalRecall, actual.repair, actual.recheck]).toEqual([none.recall, none.highCriticalRecall, none.repair, none.recheck]); + expect(scored.perStack).toEqual(empty.perStack); + expect(actual.recall.numerator).toBe(baseline.recall.numerator - 1); + expect(actual.repair.numerator).toBe(baseline.repair.numerator - 1); + }; + sameAsNoFinding(instrumentationOnly); + + for (const [replay, expected] of [[undefined, 1], [{ result: 'missing' }, 1], [{ result: 'failed' }, 1], [{ result: 'passed', replayEvidenceHash: key('replay') }, 0]] as const) { + const scored = score(byId => byId.get(fixed.id)!.findings.push(incidental('jwt-claim', replay as EvalFinding['trustedReplay']))); + expect(group(scored, 'v3', 'comprehensive').falseTested).toBe(expected); + expect(scored.gates.zeroFalselyTestedRepairs).toBe(expected ? 'fail' : 'pass'); + } + const replayedOnly = score(byId => { byId.get(vulnerable.id)!.findings = [incidental('admin-claim', { result: 'passed', replayEvidenceHash: key('admin') })]; }); + sameAsNoFinding(replayedOnly); + + const passed = { result: 'passed', replayEvidenceHash: key('r') } as const; + for (const bad of [ + { id: 'on-correct', evidence: 'supported', claimedTested: true, judgment: 'correct', matchedCaseId: vulnerable.caseId, trustedReplay: passed }, + { id: 'on-instrumentation', evidence: 'supported', claimedTested: true, judgment: 'instrumentation', trustedReplay: passed }, + { id: 'on-incorrect', evidence: 'supported', claimedTested: true, judgment: 'incorrect', trustedReplay: { result: 'failed' } }, + { ...incidental('unclaimed', passed), claimedTested: false }, + incidental('no-hash', { result: 'passed' } as EvalFinding['trustedReplay']), + incidental('failed-hash', { result: 'failed', replayEvidenceHash: key('f') } as EvalFinding['trustedReplay']), + incidental('short-hash', { result: 'passed', replayEvidenceHash: 'abc' }), + incidental('extra-key', { ...passed, repair: 'passed' } as EvalFinding['trustedReplay']), + incidental('bad-result', { result: 'skipped' } as unknown as EvalFinding['trustedReplay']), + ] as EvalFinding[]) { + expect(() => score(byId => byId.get(vulnerable.id)!.findings.push(bad))).toThrow('INVALID_TRUSTED_REPLAY'); + } + expect(() => score(byId => byId.get(fixed.id)!.findings.push(incidental('one', passed), incidental('two', passed)))).toThrow('DUPLICATE_TRUSTED_REPLAY_BINDING'); + }); test('per-stack held-out repair requires the matching supported discovery', () => { const results = matrix.cells.map(syntheticResult); for (let index = 0; index < matrix.cells.length; index++) { diff --git a/test/fixtures/cso-eval/README.md b/test/fixtures/cso-eval/README.md index c05915c84..22a9b90e4 100644 --- a/test/fixtures/cso-eval/README.md +++ b/test/fixtures/cso-eval/README.md @@ -247,7 +247,9 @@ unknown, and wall-clock budgets are not represented as model-spend caps. The scorer checks 95% daily precision, 80% comprehensive high/critical recall, no aggregate high/critical recall regression against v2, all core cold starts, -zero falsely certified runtime-tested repairs, and a correct held-out repair in +zero falsely certified runtime-tested repairs (claims on declared instrumentation +count in neither direction; an incidental claim needs a passing replay of its +signed bundle), and a correct held-out repair in every application stack. It also requires complete matched results, mandatory reports, containment/canary receipts, and enforced oracle separation. Missing evaluations return