diff --git a/lib/release-gate.ts b/lib/release-gate.ts index 0829d06bf..af586a1df 100644 --- a/lib/release-gate.ts +++ b/lib/release-gate.ts @@ -302,16 +302,20 @@ function finishRed(env: GateEnv, c: Context, rec: ReleaseRecord, lines: string[] return out('held', rec.sha); } -/** Collect a bisect round's prefix records into its results; the cost they report is added to the spend. */ +/** + * Collect a bisect round's prefix records into its results. Each case on each prefix adds its reported cost to the spend, or + * RELEASE_TRIAL_RESERVATION_USD when the cost is null or the prefix record is missing, so the cap holds across rounds. + */ function collectRound(env: GateEnv, b: ReleaseBisect, round: ReleaseBisectRound, run: Run): void { for (const p of round.prefixes) { const text = artifactFile(env, run.id, `${RELEASE_BISECT_ARTIFACT}-bisect-${p.sha.slice(0, 12)}`, 'bisect-prefix.json'); let rec: { kind?: string; nonce?: string; prefix_sha?: string; checked_out_sha?: string; panels?: Array<{ case: string; status: string; red_class: string | null; cost_usd: number | null }> } | null = null; try { rec = text ? JSON.parse(text) : null; } catch { rec = null; } - if (!rec || rec.kind !== 'train-bisect-prefix' || rec.nonce !== round.nonce || rec.prefix_sha !== p.sha || rec.checked_out_sha !== p.sha || !Array.isArray(rec.panels)) continue; - for (const panel of rec.panels) { - if (!b.cases.includes(panel.case)) continue; - b.spent_usd = Math.round((b.spent_usd + (typeof panel.cost_usd === 'number' ? panel.cost_usd : 0)) * 100) / 100; + const valid = rec && rec.kind === 'train-bisect-prefix' && rec.nonce === round.nonce && rec.prefix_sha === p.sha && rec.checked_out_sha === p.sha && Array.isArray(rec.panels); + for (const c of b.cases) { + const panel = valid ? rec!.panels!.find(x => x.case === c) : undefined; + b.spent_usd = Math.round((b.spent_usd + (typeof panel?.cost_usd === 'number' ? panel.cost_usd : RELEASE_TRIAL_RESERVATION_USD)) * 100) / 100; + if (!panel) continue; if (panel.status === 'PASS') p.results[panel.case] = 'green'; else if (panel.status === 'FAIL' && panel.red_class === 'VERDICT') p.results[panel.case] = 'red'; } diff --git a/test/release-gate.test.ts b/test/release-gate.test.ts index 1a39323b1..957adb238 100644 --- a/test/release-gate.test.ts +++ b/test/release-gate.test.ts @@ -306,6 +306,34 @@ describe('release gate: red, bisect and the hold', () => { }); }); +describe('release gate: the bisect cap with unknown costs', () => { + // Value: protects=a bisect whose prefix records report cost_usd null (what scripts/train-gate.ts writes today) books + // RELEASE_TRIAL_RESERVATION_USD per case and prefix, so a later round sees the spend and the bisect stops capped within its cap; fails_when=null + // costs count as $0 and every round reserves the whole cap again (about 4x the cap over MAX_BISECT_ROUNDS); why_new=review finding (targeted + // money, l1 fix-check); seam=fake gh; no_claim=real case prices + test('null prefix costs are charged at the reservation, and the bisect ends capped within its cap', () => { + const f = fixture(); + const order = [f.base, ...f.c, ...[6, 7, 8, 9, 10, 11, 12].map(i => f.commit('c' + i))]; + const fake = fakeGh((inputs) => { + if (!inputs.prefix_shas) return { conclusion: 'failure', artifacts: report('RED', ['case-a rule FAIL VERDICT']) }; + const files: Record> = {}; + for (const sha of inputs.prefix_shas.split(',')) { + const red = order.indexOf(sha) >= 7; + files['gstack-release-record-bisect-' + sha.slice(0, 12)] = { 'bisect-prefix.json': JSON.stringify({ schema: 1, kind: 'train-bisect-prefix', nonce: inputs.nonce, prefix_sha: sha, checked_out_sha: sha, + panels: [{ case: 'case-a', kind: 'rule', status: red ? 'FAIL' : 'PASS', red_class: red ? 'VERDICT' : null, cost_usd: null }] }) }; + } + return { conclusion: 'success', artifacts: files }; + }); + const out = runGate(envFor(f, fake.gh), { gate: { ...GATE, bisect_spend_cap_usd: 4 }, label: 'test', dryRun: false, ignoreCadence: true }); + expect(out.action, out.lines.join('\n')).toBe('held'); + const bisect = records(f)[0]!.bisect!; + expect(bisect).toMatchObject({ state: 'capped', spent_usd: 3 }); + expect(bisect.spent_usd).toBeLessThanOrEqual(4); + expect(fake.dispatches).toHaveLength(2); + expect(fake.dispatches[1]!.inputs.prefix_shas!.split(',')).toHaveLength(2); + }); +}); + describe('release gate: verdict, range and policy units', () => { test('parseReport and classifyRun: verdict reds bisect, other classes keep a red incomplete, success is green, and a contradiction is incomplete', () => { const r = parseReport(report('RED', ['case-a rule FAIL VERDICT', 'judge one judge FAIL', 'case-z rule FAIL INFRA'])['report-verdict-a1']['report.txt']);