diff --git a/CHANGELOG.md b/CHANGELOG.md index 9792fb1da..7df661fe0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## [1.91.93.0] - 2026-10-12 + +**/cso release qualification gates on finished reports. Report completeness is still measured, but it no longer blocks a release.** + +- **Gate edit: `matchedCompleteMatrix` and `mandatoryReports` (scripts/cso-eval.ts).** Both gates used to require every report's helper completeness to be `complete`: no gap, and every domain assessed. The private corpus fixtures carry gaps by design. They have no workflow files for zizmor, their planted credentials are never tested live, `.env` is kept out of the sealed source, and gitleaks has no history to read. So no report could ever be `complete`, and no release could ever qualify. Both gates now require every report to be `finished`, on a complete, fully adjudicated matrix. A running, missing or failed report still fails them. Completeness stays its own measured rate: `reports` per group, and the evaluator's `completenessRates`. Garry approved this rule on 2026-10-11, choosing option (a). No threshold changed. + ## [1.91.92.0] - 2026-10-11 **Outside reviews honor `codex_reviews` at the send, not in skill prose. Codex reasoning effort is a setting, and the README says what leaves your machine.** diff --git a/VERSION b/VERSION index 09b16c3f2..05b7fbcf8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.92.0 +1.91.93.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 6fb3d36ef..d1685ba3e 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.92.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.93.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/package.json b/package.json index c1c0de65b..d4d689e83 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.92", + "version": "1.91.93", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/scripts/cso-eval.ts b/scripts/cso-eval.ts index 8c19ae1a2..ad02a9d2f 100644 --- a/scripts/cso-eval.ts +++ b/scripts/cso-eval.ts @@ -116,7 +116,8 @@ export interface EvalResult { cellId: string; sourceHash: string; skillHash: string; model: string; host: string; budgetSeconds: number; /** * `reportFinished`: the helper report's status is `finished`; its findings and runtime outcomes count whatever its - * completeness. `reportComplete`: it is also `complete` (no gap, every domain assessed), measured as its own rate. + * completeness, and the report gates require it. `reportComplete`: it is also `complete` (no gap, every domain + * assessed), measured as its own ungated rate because the corpus fixtures carry gaps by design. */ reportPresent: boolean; reportFinished: boolean; reportComplete: boolean; findings: EvalFinding[]; setup: Outcome; reproduction: Outcome; repair: Outcome; recheck: Outcome; @@ -781,13 +782,13 @@ export function scoreEval(matrix: EvalMatrix, observations: EvalResult[], qualif }).map(cell => cell.caseId)); return [stack, { correctHeldOutRepairs: successful.size, denominator: new Set(eligible.map(cell => cell.caseId)).size }]; })); - const assessedAll = observations.length === matrix.cells.length && observations.every(result => result.reportPresent && result.reportComplete) && groups.every(group => !group.unadjudicated); + const assessedAll = observations.length === matrix.cells.length && observations.every(result => result.reportFinished) && groups.every(group => !group.unadjudicated); const gate = (condition: boolean | null, hasData: boolean): 'pass' | 'fail' | 'unmeasured' => !hasData || condition === null ? 'unmeasured' : condition ? 'pass' : 'fail'; const containmentValues = REQUIRED_CONTAINMENT.map(name => qualification.containment?.[name]); if (containmentValues.some(value => value !== undefined && !['passed', 'failed', 'not_run'].includes(value))) throw new Error('INVALID_CONTAINMENT_RESULT'); const gates = { matchedCompleteMatrix: gate(assessedAll, observations.length > 0), - mandatoryReports: gate(groups.every(group => group.reports.numerator === group.reports.denominator), observations.length === matrix.cells.length), + mandatoryReports: gate(groups.every(group => group.finishedReports.numerator === group.finishedReports.denominator), observations.length === matrix.cells.length), dailyPrecision95: gate(daily.precision.value === null ? null : daily.precision.value >= 0.95, daily.submitted === daily.cells && !daily.unadjudicated), comprehensiveHighCriticalRecall80: gate(comprehensive.highCriticalRecall.value >= 0.8, comprehensive.submitted === comprehensive.cells && !comprehensive.unadjudicated), noHighCriticalRecallRegression: profile === 'full' diff --git a/test/cso-eval.test.ts b/test/cso-eval.test.ts index 75f0b77bb..6d6559576 100644 --- a/test/cso-eval.test.ts +++ b/test/cso-eval.test.ts @@ -206,7 +206,7 @@ describe('CSO matched evaluation accounting', () => { expect(group(score, 'v3', 'comprehensive').highCriticalRecall).toEqual({ numerator: 96, denominator: 96, value: 1 }); expect(score.perStack.rails).toEqual({ correctHeldOutRepairs: 10, denominator: 10 }); }); - test('finished partial reports count their findings and outcomes; completeness stays its own measured, gated rate', () => { + test('finished partial reports count their findings and outcomes and satisfy the report gates; completeness stays its own measured rate', () => { const results = matrix.cells.map(syntheticResult); for (const result of results) result.reportComplete = false; const score = scoreEval(matrix, results, qualification), daily = group(score, 'v3', 'daily'), comprehensive = group(score, 'v3', 'comprehensive'); @@ -216,7 +216,14 @@ describe('CSO matched evaluation accounting', () => { expect(comprehensive.highCriticalRecall).toEqual({ numerator: 96, denominator: 96, value: 1 }); expect(comprehensive.repair.numerator).toBe(comprehensive.repair.denominator); expect(score.perStack.rails).toEqual({ correctHeldOutRepairs: 10, denominator: 10 }); - expect(score.gates).toMatchObject({ comprehensiveHighCriticalRecall80: 'pass', dailyPrecision95: 'pass', mandatoryReports: 'fail', matchedCompleteMatrix: 'fail' }); + expect(score.gates).toMatchObject({ comprehensiveHighCriticalRecall80: 'pass', dailyPrecision95: 'pass', mandatoryReports: 'pass', matchedCompleteMatrix: 'pass' }); + expect(score.status).toBe('qualified'); + }); + test('one unfinished report fails both report gates even when every other report is finished', () => { + const results = matrix.cells.map(syntheticResult); + Object.assign(results[0], { reportComplete: false, reportFinished: false }); + const score = scoreEval(matrix, results, qualification); + expect(score.gates).toMatchObject({ mandatoryReports: 'fail', matchedCompleteMatrix: 'fail' }); expect(score.status).toBe('partial'); }); test('running or missing reports contribute no security evidence and cannot qualify a complete matrix', () => {