mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-10 18:50:18 +01:00
fix(ship-measure): a partly funded sweep row shows the trials that ran
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
a22ac43b07
commit
81a79d54a7
2 files changed
+7
-2
No files matched your search
@@ -30,6 +30,7 @@ import { CASE_QUARANTINE } from '../test/helpers/periodic-exclude-data';
|
||||
import { E2E_KINDS, E2E_TIERS, LLM_JUDGE_TOUCHFILES } from '../test/helpers/touchfiles-data';
|
||||
import { parseCliFlags } from './lib/shard-engine';
|
||||
import { RED_RATE_SHRINK, verdictsOf } from './lib/eval-history';
|
||||
import { trialPassed } from './lib/measure-bar';
|
||||
import { repoSlug } from './lib/ci-history';
|
||||
import { loadPassRateHistory, type TrialRecord as HistoryRecord } from './eval-flake-rank';
|
||||
import {
|
||||
@@ -292,12 +293,14 @@ export async function runSweep(o: SweepOptions): Promise<SweepReport> {
|
||||
continue;
|
||||
}
|
||||
const captures = path.relative(o.outDir, path.join(dir, caseSlug(c.case), 'baseline'));
|
||||
const decided = m.status === 'measured' && m.decision !== 'void' && m.decision !== 'incomplete';
|
||||
Object.assign(row, {
|
||||
status: m.status === 'measured' ? 'measured' : 'budget_exhausted', decision: m.status === 'measured' ? m.decision : m.status.replace('_', ' '),
|
||||
measuredPasses: m.passes, measuredTrials: m.counted || m.trials.length, actualUsd: m.actualUsd, costUnknownTrials: m.costUnknownTrials, captures,
|
||||
...(decided ? { measuredPasses: m.passes, measuredTrials: m.counted } : { measuredPasses: m.trials.filter(t => trialPassed(c.kind, t)).length, measuredTrials: m.trials.length }),
|
||||
actualUsd: m.actualUsd, costUnknownTrials: m.costUnknownTrials, captures,
|
||||
note: m.status === 'measured' ? m.decisionReason : m.reason ?? '',
|
||||
});
|
||||
if (m.status === 'measured' && m.decision !== 'void' && m.decision !== 'incomplete') {
|
||||
if (decided) {
|
||||
measuredRates[c.case] = measuredRedRate(c.kind, m.counted - m.passes, m.counted, plan.pooledRate);
|
||||
report.allGreenAfter = predictedAllGreen(plan, measuredRates);
|
||||
}
|
||||
|
||||
@@ -134,6 +134,8 @@ describe('running the sweep with a fake runner', () => {
|
||||
expect(report.priorWeekUsd).toBe(136);
|
||||
// B and C tie (one red in three); ties fall to the case id, so C ranks before B.
|
||||
expect(report.rows.map(r => [r.case, r.status])).toEqual([[A, 'measured'], [C, 'budget_exhausted'], [B, 'skipped']]);
|
||||
// A partly funded case reports the trials that ran, not the planned N.
|
||||
expect([report.rows[1]!.measuredPasses, report.rows[1]!.measuredTrials]).toEqual([4, 4]);
|
||||
expect(run.calls).toHaveLength(14);
|
||||
expect(report.chargedUsd).toBe(14);
|
||||
expect(report.rows[2]!.note).toContain('weekly cap $150 exhausted');
|
||||
|
||||
Reference in new issue
Block a user