diff --git a/scripts/test-health-report.ts b/scripts/test-health-report.ts index e4dc5da46..cf11e4f3e 100644 --- a/scripts/test-health-report.ts +++ b/scripts/test-health-report.ts @@ -33,7 +33,7 @@ export const COMPLEXITY_PATHS = ['scripts', 'test/helpers', '.github/workflows'] export interface Metric { id: string; title: string; rule: string; value?: string; detail?: string[]; unavailable?: string } export interface Check { id: string; title: string; status: 'pass' | 'fail' | 'unavailable'; detail: string; fix: string } export type IssueAction = 'upsert' | 'keep-open' | 'close'; -export interface HealthReport { repo: string; sinceDays: number; generatedAt: string; metrics: Metric[]; checks: Check[]; issueAction: IssueAction } +export interface HealthReport { repo: string; sinceDays: number; generatedAt: string; headline?: string; metrics: Metric[]; checks: Check[]; issueAction: IssueAction } export interface HealthInputs { client: GhClient; @@ -188,14 +188,17 @@ export function collectHealth(inputs: HealthInputs): HealthReport { const need = (value: T | Error): T => { if (value instanceof Error) throw value; return value; }; const prRecords = attempt(() => need(prRuns).filter(run => run.conclusion !== 'cancelled') .map(run => ({ run, trials: records(artifactDirs(run, name => name.startsWith('trial-outcomes'))) }))); + const prManifests = attempt(() => need(prRuns).flatMap(run => { + const file = artifactDirs(run, name => name === 'paid-plan').map(dir => path.join(dir, 'manifest.json')).find(candidate => fs.existsSync(candidate)); + return file ? [{ run, manifest: readJson(file) }] : []; + })); const metrics: Metric[] = []; + let costPerMergedPr: string | undefined; - metrics.push(metric('pr-fallback-rate', 'PR full-fallback rate', 'share of evals.yml pull_request runs whose paid-plan manifest has prCoverage.mode = full-fallback', () => { + metrics.push(metric('pr-fallback-rate', 'PR full-fallback rate', 'share of evals.yml pull_request runs whose paid-plan manifest has prCoverage.mode = full-fallback (a label, not spend: dependents runs select about as much)', () => { const modes = new Map(); - for (const run of need(prRuns)) { - const manifest = artifactDirs(run, name => name === 'paid-plan').flatMap(dir => [path.join(dir, 'manifest.json')]).find(file => fs.existsSync(file)); - if (!manifest) continue; - const mode = String(readJson(manifest).prCoverage?.mode ?? 'none'); + for (const { manifest } of need(prManifests)) { + const mode = String(manifest.prCoverage?.mode ?? 'none'); modes.set(mode, (modes.get(mode) ?? 0) + 1); } const total = [...modes.values()].reduce((a, b) => a + b, 0); @@ -204,6 +207,27 @@ export function collectHealth(inputs: HealthInputs): HealthReport { detail: [...modes].map(([mode, count]) => `${mode}: ${count}`) }; })); + metrics.push(metric('pr-cases-per-run', 'Selected gate cases and judges per PR run', 'mean length of prCoverage.e2e (gate cases) and prCoverage.judges in the paid-plan manifests of evals.yml pull_request runs, overall and per mode', () => { + const planned = need(prManifests).filter(({ manifest }) => Array.isArray(manifest.prCoverage?.e2e) && Array.isArray(manifest.prCoverage?.judges)); + if (planned.length === 0) throw new Error('no paid-plan manifest with prCoverage.e2e and prCoverage.judges in the window'); + const mean = (entries: typeof planned) => `${(entries.reduce((sum, { manifest }) => sum + manifest.prCoverage.e2e.length, 0) / entries.length).toFixed(1)} gate cases, ${(entries.reduce((sum, { manifest }) => sum + manifest.prCoverage.judges.length, 0) / entries.length).toFixed(1)} judges`; + const modes = [...new Set(planned.map(({ manifest }) => String(manifest.prCoverage.mode)))].sort(); + return { value: `${mean(planned)} per run (${planned.length} runs)`, + detail: modes.map(mode => { const entries = planned.filter(({ manifest }) => String(manifest.prCoverage.mode) === mode); return `${mode}: ${mean(entries)} (${entries.length} runs)`; }) }; + })); + + metrics.push(metric('pr-known-cost-per-run', 'Known cost per PR run', 'mean over non-cancelled evals.yml pull_request runs of executed trial cost_usd with a known cost (a run with no trial records counts $0), overall and per manifest mode; unknown-cost trials (PTY) are counted, not priced', () => { + const entries = need(prRecords); + if (entries.length === 0) throw new Error('no non-cancelled PR runs in the window'); + const known = (trials: TrialRecord[]) => trials.filter(record => record.execution === 'executed' && record.cost_known !== false).reduce((sum, record) => sum + record.cost_usd, 0); + const unknown = entries.flatMap(entry => entry.trials).filter(record => record.execution === 'executed' && record.cost_known === false).length; + const modeOf = new Map((prManifests instanceof Error ? [] : prManifests).map(({ run, manifest }) => [run.id, String(manifest.prCoverage?.mode ?? 'none')])); + const mean = (list: typeof entries) => `$${(list.reduce((sum, entry) => sum + known(entry.trials), 0) / list.length).toFixed(2)}`; + const modes = [...new Set(entries.map(entry => modeOf.get(entry.run.id) ?? 'no manifest'))].sort(); + return { value: `${mean(entries)} per run (${entries.length} runs; ${unknown} executed trial(s) with unknown cost)`, + detail: modes.map(mode => { const list = entries.filter(entry => (modeOf.get(entry.run.id) ?? 'no manifest') === mode); return `${mode}: ${mean(list)} (${list.length} runs)`; }) }; + })); + metrics.push(metric('pr-reused-trials', 'Reused PR trial records', 'trial-outcomes records with execution = reused across non-cancelled PR runs', () => { const all = need(prRecords).flatMap(entry => entry.trials); const reused = all.filter(record => record.execution === 'reused').length; @@ -216,6 +240,7 @@ export function collectHealth(inputs: HealthInputs): HealthReport { .reduce((sum, record) => sum + record.cost_usd, 0); const revisions = new Set(entries.filter(entry => entry.run.conclusion === 'success' || entry.run.conclusion === 'failure').map(entry => entry.run.sha)).size; const merged = listMergedPulls(client, repo, since); + if (merged) costPerMergedPr = `$${(spend / merged).toFixed(2)} (${merged} merged PRs, $${spend.toFixed(2)} recorded)`; return { value: `$${spend.toFixed(2)} recorded`, detail: [ `per merged PR: ${merged ? `$${(spend / merged).toFixed(2)} (${merged} merged)` : 'n/a (0 merged)'}`, `per validated revision: ${revisions ? `$${(spend / revisions).toFixed(2)} (${revisions} revisions)` : 'n/a'}`] }; @@ -267,7 +292,8 @@ export function collectHealth(inputs: HealthInputs): HealthReport { })); const checks = [flakeCheck(flakes), unseededCheck(root)]; - return { repo, sinceDays, generatedAt: new Date(now).toISOString(), metrics, checks, issueAction: issueActionFor(checks) }; + const headline = `Cost per merged PR: ${costPerMergedPr ?? `unavailable (${metrics.find(entry => entry.id === 'pr-spend')?.unavailable ?? '0 merged PRs'})`}`; + return { repo, sinceDays, generatedAt: new Date(now).toISOString(), headline, metrics, checks, issueAction: issueActionFor(checks) }; } function listMergedPulls(client: GhClient, repo: string, since: string): number { @@ -340,7 +366,7 @@ function censusMetrics(inputs: HealthInputs, artifactDirs: (run: CiRun, match: ( // --- Rendering --- export function formatHealth(report: HealthReport): string { - const lines = [`test:health ${report.repo}, last ${report.sinceDays} day(s), ${report.generatedAt}`]; + const lines = [`test:health ${report.repo}, last ${report.sinceDays} day(s), ${report.generatedAt}`, ...(report.headline ? [report.headline] : [])]; for (const entry of report.metrics) { lines.push(`- ${entry.title}: ${entry.unavailable ? `unavailable: ${entry.unavailable}` : entry.value}`); lines.push(` rule: ${entry.rule}`); diff --git a/test/test-health-report.test.ts b/test/test-health-report.test.ts index 75970cef3..38a2e31f4 100644 --- a/test/test-health-report.test.ts +++ b/test/test-health-report.test.ts @@ -191,7 +191,7 @@ describe('collectHealth over recorded fixture data', () => { [/runs\/61\/artifacts/, () => ({ artifacts: [] })], [/workflows\/windows-free-tests\.yml\/runs/, () => ({ workflow_runs: windowsRuns })], ], { - 11: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'full-fallback' } }) }, + 11: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'full-fallback', e2e: ['gate-a', 'gate-b'], judges: ['judge-a'] } }) }, 12: { 'trial-outcomes.jsonl': prTrials }, 51: { 'trial-outcomes.jsonl': census }, 61: { 'flake-ledger.jsonl': ledger('test/a.test.ts') }, @@ -205,6 +205,10 @@ describe('collectHealth over recorded fixture data', () => { expect(value('net-complexity').unavailable).toContain('git grep HEAD failed'); expect(value('pr-reused-trials').value).toBe('1 of 2 trial records'); expect(value('pr-spend')).toMatchObject({ value: '$2.50 recorded', detail: ['per merged PR: $2.50 (1 merged)', 'per validated revision: $2.50 (1 revisions)'] }); + expect(value('pr-cases-per-run')).toMatchObject({ value: '2.0 gate cases, 1.0 judges per run (1 runs)', detail: ['full-fallback: 2.0 gate cases, 1.0 judges (1 runs)'] }); + expect(value('pr-known-cost-per-run')).toMatchObject({ value: '$2.50 per run (1 runs; 0 executed trial(s) with unknown cost)', detail: ['full-fallback: $2.50 (1 runs)'] }); + expect(report.headline).toBe('Cost per merged PR: $2.50 (1 merged PRs, $2.50 recorded)'); + expect(formatHealth(report).split('\n')[1]).toBe('Cost per merged PR: $2.50 (1 merged PRs, $2.50 recorded)'); expect(value('cancelled-slice-minutes').value).toBe('30 min/week (1 cancelled run(s))'); expect(value('push-to-verdict').value).toBe('p50 10.0 min, p95 10.0 min (1 runs)'); expect(value('census-periodic').value).toBe('1.00 expected failed trials/run (1/3 over 1 run(s)); P(red) 33.3% in force, 33.3% single-trial'); @@ -223,6 +227,26 @@ describe('collectHealth over recorded fixture data', () => { expect(formatHealth(report)).toContain('rule: share of evals.yml pull_request runs'); }); + test('known cost per PR run prices only known-cost executed trials and counts the unknown ones', () => { + const trials = formatTrialOutcomes([trialRecord('gate-a', 'passed', { tier: 'gate', lane: 'gate/pr', cost_usd: 4 }), + trialRecord('gate-pty', 'passed', { tier: 'gate', lane: 'gate/pr', cost_usd: 0, cost_known: false })]); + const client = fakeClient([ + [/workflows\/evals\.yml\/runs/, () => ({ workflow_runs: [run(1, { event: 'pull_request', head_branch: 'a' }), run(2, { event: 'pull_request', head_branch: 'b' })] })], + [/runs\/1\/artifacts/, () => ({ artifacts: [{ id: 11, name: 'paid-plan', size_in_bytes: 10 }, { id: 12, name: 'trial-outcomes-pr-a1', size_in_bytes: 10 }] })], + [/runs\/2\/artifacts/, () => ({ artifacts: [{ id: 21, name: 'paid-plan', size_in_bytes: 10 }] })], + [/pulls\?state=closed/, () => []], + ], { + 11: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'dependents', e2e: ['gate-a', 'gate-pty', 'gate-c'], judges: ['j'] } }) }, + 12: { 'trial-outcomes.jsonl': trials }, + 21: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'pr', e2e: [], judges: [] } }) }, + }); + const report = collectHealth({ client, repo: 'o/r', now: NOW, sinceDays: 7, cacheDir: tmp('health-cache-'), root: freeRoot(3, 0) }); + const value = (id: string) => report.metrics.find(entry => entry.id === id)!; + expect(value('pr-known-cost-per-run')).toMatchObject({ value: '$2.00 per run (2 runs; 1 executed trial(s) with unknown cost)', detail: ['dependents: $4.00 (1 runs)', 'pr: $0.00 (1 runs)'] }); + expect(value('pr-cases-per-run')).toMatchObject({ value: '1.5 gate cases, 0.5 judges per run (2 runs)', detail: ['dependents: 3.0 gate cases, 1.0 judges (1 runs)', 'pr: 0.0 gate cases, 0.0 judges (1 runs)'] }); + expect(report.headline).toBe('Cost per merged PR: unavailable (0 merged PRs)'); + }); + test('the Windows wall counts only successful pull_request runs and says so when there are none', () => { const wall = (windowsRuns: unknown[]) => collectHealth({ client: fixture(windowsRuns), repo: 'o/r', now: NOW, sinceDays: 7, cacheDir: tmp('health-cache-'), root: freeRoot(3, 0) }) .metrics.find(entry => entry.id === 'free-wall')!;