diff --git a/scripts/test-health-report.ts b/scripts/test-health-report.ts index cf11e4f3e..bccc11639 100644 --- a/scripts/test-health-report.ts +++ b/scripts/test-health-report.ts @@ -228,6 +228,30 @@ export function collectHealth(inputs: HealthInputs): HealthReport { detail: modes.map(mode => { const list = entries.filter(entry => (modeOf.get(entry.run.id) ?? 'no manifest') === mode); return `${mode}: ${mean(list)} (${list.length} runs)`; }) }; })); + metrics.push(metric('pr-shadow-map', 'Exercised-path map shadow (not active)', 'over evals.yml pull_request runs whose paid-plan manifest carries prCoverage.shadow: the predicted saving (map per-case known cost of dropped minus added cases; a lower bound, since known cost excludes PTY trials with unknown cost) and shadow drops of red cases (more executed failed than passed trials in the run, on a case the shadow would drop). A drop counts toward the D3 activation bar as a miss unless the case passed in a later run of the same branch at the same head SHA, which marks it a flaky drop. Scaled to 7 days; docs/TESTING_INTERNALS.md#exercised-path-map', () => { + const shadowed = need(prManifests).filter(({ manifest }) => manifest.prCoverage?.shadow && !manifest.prCoverage.shadow.error); + if (shadowed.length === 0) throw new Error('no paid-plan manifest with a prCoverage.shadow block in the window'); + const records = prRecords instanceof Error ? [] : prRecords; + const executed = (trials: TrialRecord[], id: string, outcome: string) => trials.filter(record => record.case === id && record.execution === 'executed' && record.outcome === outcome).length; + const saving = shadowed.reduce((sum, { manifest }) => sum + (Number(manifest.prCoverage.shadow.predictedSavingUsd) || 0), 0); + const shares = shadowed.map(({ manifest }) => Number(manifest.prCoverage.shadow.unknownCostShare)).filter(share => Number.isFinite(share) && share > 0); + const misses: string[] = []; + const flaky: string[] = []; + for (const { run, manifest } of shadowed) { + const trials = records.find(entry => entry.run.id === run.id)?.trials ?? []; + for (const id of (manifest.prCoverage.shadow.wouldDrop ?? []) as string[]) { + if (executed(trials, id, 'failed') <= executed(trials, id, 'passed')) continue; + const laterPass = records.find(entry => entry.run.branch === run.branch && entry.run.sha === run.sha + && (entry.run.createdAt > run.createdAt || (entry.run.id === run.id && entry.run.attempt > run.attempt)) && executed(entry.trials, id, 'passed') > 0); + if (laterPass) flaky.push(`${run.id}:${id} (passed in run ${laterPass.run.id})`); else misses.push(`${run.id}:${id}`); + } + } + const week = 7 / sinceDays; + const bound = shares.length ? `a lower bound: known cost excludes the ${Math.round(100 * Math.max(...shares))}% of observed case runs with unknown cost (PTY)` : 'a lower bound: known cost excludes PTY trials with unknown cost'; + return { value: `$${(saving * week).toFixed(2)} predicted saving/week (${bound}); ${(misses.length * week).toFixed(1)} activation miss(es)/week, ${flaky.length} flaky drop(s) not counted (${shadowed.length} runs)`, + detail: [...misses.slice(0, 20).map(miss => `shadow miss: ${miss}`), ...flaky.slice(0, 20).map(drop => `flaky drop: ${drop}`)] }; + })); + metrics.push(metric('pr-reused-trials', 'Reused PR trial records', 'trial-outcomes records with execution = reused across non-cancelled PR runs', () => { const all = need(prRecords).flatMap(entry => entry.trials); const reused = all.filter(record => record.execution === 'reused').length; diff --git a/test/test-health-report.test.ts b/test/test-health-report.test.ts index 38a2e31f4..0d654dba8 100644 --- a/test/test-health-report.test.ts +++ b/test/test-health-report.test.ts @@ -191,7 +191,8 @@ describe('collectHealth over recorded fixture data', () => { [/runs\/61\/artifacts/, () => ({ artifacts: [] })], [/workflows\/windows-free-tests\.yml\/runs/, () => ({ workflow_runs: windowsRuns })], ], { - 11: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'full-fallback', e2e: ['gate-a', 'gate-b'], judges: ['judge-a'] } }) }, + 11: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'full-fallback', e2e: ['gate-a', 'gate-b'], judges: ['judge-a'], + shadow: { e2e: ['gate-a', 'gate-b'], wouldDrop: [], wouldAdd: [], provenance: [], predictedSavingUsd: 0, unpricedCases: 0 } } }) }, 12: { 'trial-outcomes.jsonl': prTrials }, 51: { 'trial-outcomes.jsonl': census }, 61: { 'flake-ledger.jsonl': ledger('test/a.test.ts') }, @@ -207,6 +208,7 @@ describe('collectHealth over recorded fixture data', () => { expect(value('pr-spend')).toMatchObject({ value: '$2.50 recorded', detail: ['per merged PR: $2.50 (1 merged)', 'per validated revision: $2.50 (1 revisions)'] }); expect(value('pr-cases-per-run')).toMatchObject({ value: '2.0 gate cases, 1.0 judges per run (1 runs)', detail: ['full-fallback: 2.0 gate cases, 1.0 judges (1 runs)'] }); expect(value('pr-known-cost-per-run')).toMatchObject({ value: '$2.50 per run (1 runs; 0 executed trial(s) with unknown cost)', detail: ['full-fallback: $2.50 (1 runs)'] }); + expect(value('pr-shadow-map')).toMatchObject({ value: '$0.00 predicted saving/week (a lower bound: known cost excludes PTY trials with unknown cost); 0.0 activation miss(es)/week, 0 flaky drop(s) not counted (1 runs)', detail: [] }); expect(report.headline).toBe('Cost per merged PR: $2.50 (1 merged PRs, $2.50 recorded)'); expect(formatHealth(report).split('\n')[1]).toBe('Cost per merged PR: $2.50 (1 merged PRs, $2.50 recorded)'); expect(value('cancelled-slice-minutes').value).toBe('30 min/week (1 cancelled run(s))'); @@ -247,6 +249,38 @@ describe('collectHealth over recorded fixture data', () => { expect(report.headline).toBe('Cost per merged PR: unavailable (0 merged PRs)'); }); + test('the shadow map metric counts red drops as activation misses unless the case passed later at the same head SHA, and labels dollars a lower bound', () => { + const gate = (caseId: string, outcome: 'passed' | 'failed', extra: Partial = {}) => trialRecord(caseId, outcome, { tier: 'gate', lane: 'gate/pr', ...extra }); + const first = formatTrialOutcomes([gate('gate-a', 'failed', { cost_usd: 3 }), gate('gate-b', 'failed'), gate('gate-c', 'failed'), + gate('gate-d', 'failed', { execution: 'reused', cost_usd: 0 }), gate('gate-e', 'passed')]); + const rerun = formatTrialOutcomes([gate('gate-b', 'passed'), gate('gate-a', 'failed')]); + const shadow = (wouldDrop: string[], predictedSavingUsd: number, unknownCostShare = 0.391) => + ({ e2e: [], wouldDrop, wouldAdd: [], provenance: [], predictedSavingUsd, unpricedCases: 0, unknownCostShare }); + const client = fakeClient([ + [/workflows\/evals\.yml\/runs/, () => ({ workflow_runs: [ + run(1, { event: 'pull_request', head_branch: 'a', head_sha: 'same', created_at: day(3) }), + run(2, { event: 'pull_request', head_branch: 'a', head_sha: 'same', created_at: day(2) }), + run(3, { event: 'pull_request', head_branch: 'c' }), + run(4, { event: 'pull_request', head_branch: 'd' })] })], + [/runs\/1\/artifacts/, () => ({ artifacts: [{ id: 11, name: 'paid-plan', size_in_bytes: 10 }, { id: 12, name: 'trial-outcomes-pr-a1', size_in_bytes: 10 }] })], + [/runs\/2\/artifacts/, () => ({ artifacts: [{ id: 21, name: 'paid-plan', size_in_bytes: 10 }, { id: 22, name: 'trial-outcomes-pr-a1', size_in_bytes: 10 }] })], + [/runs\/3\/artifacts/, () => ({ artifacts: [{ id: 31, name: 'paid-plan', size_in_bytes: 10 }] })], + [/runs\/4\/artifacts/, () => ({ artifacts: [{ id: 41, name: 'paid-plan', size_in_bytes: 10 }] })], + [/pulls\?state=closed/, () => []], + ], { + 11: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'dependents', e2e: [], judges: [], shadow: shadow(['gate-a', 'gate-b', 'gate-d', 'gate-e'], 4.5) } }) }, + 12: { 'trial-outcomes.jsonl': first }, + 21: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'pr', e2e: [], judges: [], shadow: shadow([], -1.5) } }) }, + 22: { 'trial-outcomes.jsonl': rerun }, + 31: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'pr', e2e: [], judges: [], shadow: { ...shadow([], 99), error: 'bad map' } } }) }, + 41: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'pr', e2e: [], judges: [] } }) }, + }); + const report = collectHealth({ client, repo: 'o/r', now: NOW, sinceDays: 7, cacheDir: tmp('health-cache-'), root: freeRoot(3, 0) }); + expect(report.metrics.find(entry => entry.id === 'pr-shadow-map')).toMatchObject({ + value: '$3.00 predicted saving/week (a lower bound: known cost excludes the 39% of observed case runs with unknown cost (PTY)); 1.0 activation miss(es)/week, 1 flaky drop(s) not counted (2 runs)', + detail: ['shadow miss: 1:gate-a', 'flaky drop: 1:gate-b (passed in run 2)'] }); + }); + test('the Windows wall counts only successful pull_request runs and says so when there are none', () => { const wall = (windowsRuns: unknown[]) => collectHealth({ client: fixture(windowsRuns), repo: 'o/r', now: NOW, sinceDays: 7, cacheDir: tmp('health-cache-'), root: freeRoot(3, 0) }) .metrics.find(entry => entry.id === 'free-wall')!;