diff --git a/scripts/ship-measure-sweep.ts b/scripts/ship-measure-sweep.ts index 564df72d0..d51aa21b8 100644 --- a/scripts/ship-measure-sweep.ts +++ b/scripts/ship-measure-sweep.ts @@ -30,6 +30,7 @@ import { CASE_QUARANTINE } from '../test/helpers/periodic-exclude-data'; import { E2E_KINDS, E2E_TIERS, LLM_JUDGE_TOUCHFILES } from '../test/helpers/touchfiles-data'; import { parseCliFlags } from './lib/shard-engine'; import { RED_RATE_SHRINK, verdictsOf } from './lib/eval-history'; +import { trialPassed } from './lib/measure-bar'; import { repoSlug } from './lib/ci-history'; import { loadPassRateHistory, type TrialRecord as HistoryRecord } from './eval-flake-rank'; import { @@ -292,12 +293,14 @@ export async function runSweep(o: SweepOptions): Promise { continue; } const captures = path.relative(o.outDir, path.join(dir, caseSlug(c.case), 'baseline')); + const decided = m.status === 'measured' && m.decision !== 'void' && m.decision !== 'incomplete'; Object.assign(row, { status: m.status === 'measured' ? 'measured' : 'budget_exhausted', decision: m.status === 'measured' ? m.decision : m.status.replace('_', ' '), - measuredPasses: m.passes, measuredTrials: m.counted || m.trials.length, actualUsd: m.actualUsd, costUnknownTrials: m.costUnknownTrials, captures, + ...(decided ? { measuredPasses: m.passes, measuredTrials: m.counted } : { measuredPasses: m.trials.filter(t => trialPassed(c.kind, t)).length, measuredTrials: m.trials.length }), + actualUsd: m.actualUsd, costUnknownTrials: m.costUnknownTrials, captures, note: m.status === 'measured' ? m.decisionReason : m.reason ?? '', }); - if (m.status === 'measured' && m.decision !== 'void' && m.decision !== 'incomplete') { + if (decided) { measuredRates[c.case] = measuredRedRate(c.kind, m.counted - m.passes, m.counted, plan.pooledRate); report.allGreenAfter = predictedAllGreen(plan, measuredRates); } diff --git a/test/ship-measure-sweep.test.ts b/test/ship-measure-sweep.test.ts index d4f963130..d7b03f36c 100644 --- a/test/ship-measure-sweep.test.ts +++ b/test/ship-measure-sweep.test.ts @@ -134,6 +134,8 @@ describe('running the sweep with a fake runner', () => { expect(report.priorWeekUsd).toBe(136); // B and C tie (one red in three); ties fall to the case id, so C ranks before B. expect(report.rows.map(r => [r.case, r.status])).toEqual([[A, 'measured'], [C, 'budget_exhausted'], [B, 'skipped']]); + // A partly funded case reports the trials that ran, not the planned N. + expect([report.rows[1]!.measuredPasses, report.rows[1]!.measuredTrials]).toEqual([4, 4]); expect(run.calls).toHaveLength(14); expect(report.chargedUsd).toBe(14); expect(report.rows[2]!.note).toContain('weekly cap $150 exhausted');