CalvinBackup/gstack
1
0 Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-11-01 14:34:42 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat(test-health): report gate cases and known cost per PR run, lead with cost per merged PR (A1-3)

Browse Source 
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed 2026-10-17 19:20:56 +00:00
1 parent 4a1da453fa
commit 0d18469121
2 files changed
+59 -9
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/test-health-report.ts
+34 -8
View File
@@ -33,7 +33,7 @@ export const COMPLEXITY_PATHS = ['scripts', 'test/helpers', '.github/workflows']
export interface Metric { id: string; title: string; rule: string; value?: string; detail?: string[]; unavailable?: string }
export interface Check { id: string; title: string; status: 'pass' | 'fail' | 'unavailable'; detail: string; fix: string }
export type IssueAction = 'upsert' | 'keep-open' | 'close';
export interface HealthReport { repo: string; sinceDays: number; generatedAt: string; metrics: Metric[]; checks: Check[]; issueAction: IssueAction }
export interface HealthReport { repo: string; sinceDays: number; generatedAt: string; headline?: string; metrics: Metric[]; checks: Check[]; issueAction: IssueAction }
export interface HealthInputs {
client: GhClient;
@@ -188,14 +188,17 @@ export function collectHealth(inputs: HealthInputs): HealthReport {
const need = <T>(value: T | Error): T => { if (value instanceof Error) throw value; return value; };
const prRecords = attempt(() => need(prRuns).filter(run => run.conclusion !== 'cancelled')
.map(run => ({ run, trials: records(artifactDirs(run, name => name.startsWith('trial-outcomes'))) })));
const prManifests = attempt(() => need(prRuns).flatMap(run => {
const file = artifactDirs(run, name => name === 'paid-plan').map(dir => path.join(dir, 'manifest.json')).find(candidate => fs.existsSync(candidate));
return file ? [{ run, manifest: readJson(file) }] : [];
}));
const metrics: Metric[] = [];
let costPerMergedPr: string | undefined;
metrics.push(metric('pr-fallback-rate', 'PR full-fallback rate', 'share of evals.yml pull_request runs whose paid-plan manifest has prCoverage.mode = full-fallback', () => {
metrics.push(metric('pr-fallback-rate', 'PR full-fallback rate', 'share of evals.yml pull_request runs whose paid-plan manifest has prCoverage.mode = full-fallback (a label, not spend: dependents runs select about as much)', () => {
const modes = new Map<string, number>();
for (const run of need(prRuns)) {
const manifest = artifactDirs(run, name => name === 'paid-plan').flatMap(dir => [path.join(dir, 'manifest.json')]).find(file => fs.existsSync(file));
if (!manifest) continue;
const mode = String(readJson(manifest).prCoverage?.mode ?? 'none');
for (const { manifest } of need(prManifests)) {
const mode = String(manifest.prCoverage?.mode ?? 'none');
modes.set(mode, (modes.get(mode) ?? 0) + 1);
}
const total = [...modes.values()].reduce((a, b) => a + b, 0);
@@ -204,6 +207,27 @@ export function collectHealth(inputs: HealthInputs): HealthReport {
detail: [...modes].map(([mode, count]) => `${mode}: ${count}`) };
}));
metrics.push(metric('pr-cases-per-run', 'Selected gate cases and judges per PR run', 'mean length of prCoverage.e2e (gate cases) and prCoverage.judges in the paid-plan manifests of evals.yml pull_request runs, overall and per mode', () => {
const planned = need(prManifests).filter(({ manifest }) => Array.isArray(manifest.prCoverage?.e2e) && Array.isArray(manifest.prCoverage?.judges));
if (planned.length === 0) throw new Error('no paid-plan manifest with prCoverage.e2e and prCoverage.judges in the window');
const mean = (entries: typeof planned) => `${(entries.reduce((sum, { manifest }) => sum + manifest.prCoverage.e2e.length, 0) / entries.length).toFixed(1)} gate cases, ${(entries.reduce((sum, { manifest }) => sum + manifest.prCoverage.judges.length, 0) / entries.length).toFixed(1)} judges`;
const modes = [...new Set(planned.map(({ manifest }) => String(manifest.prCoverage.mode)))].sort();
return { value: `${mean(planned)} per run (${planned.length} runs)`,
detail: modes.map(mode => { const entries = planned.filter(({ manifest }) => String(manifest.prCoverage.mode) === mode); return `${mode}: ${mean(entries)} (${entries.length} runs)`; }) };
}));
metrics.push(metric('pr-known-cost-per-run', 'Known cost per PR run', 'mean over non-cancelled evals.yml pull_request runs of executed trial cost_usd with a known cost (a run with no trial records counts $0), overall and per manifest mode; unknown-cost trials (PTY) are counted, not priced', () => {
const entries = need(prRecords);
if (entries.length === 0) throw new Error('no non-cancelled PR runs in the window');
const known = (trials: TrialRecord[]) => trials.filter(record => record.execution === 'executed' && record.cost_known !== false).reduce((sum, record) => sum + record.cost_usd, 0);
const unknown = entries.flatMap(entry => entry.trials).filter(record => record.execution === 'executed' && record.cost_known === false).length;
const modeOf = new Map((prManifests instanceof Error ? [] : prManifests).map(({ run, manifest }) => [run.id, String(manifest.prCoverage?.mode ?? 'none')]));
const mean = (list: typeof entries) => `$${(list.reduce((sum, entry) => sum + known(entry.trials), 0) / list.length).toFixed(2)}`;
const modes = [...new Set(entries.map(entry => modeOf.get(entry.run.id) ?? 'no manifest'))].sort();
return { value: `${mean(entries)} per run (${entries.length} runs; ${unknown} executed trial(s) with unknown cost)`,
detail: modes.map(mode => { const list = entries.filter(entry => (modeOf.get(entry.run.id) ?? 'no manifest') === mode); return `${mode}: ${mean(list)} (${list.length} runs)`; }) };
}));
metrics.push(metric('pr-reused-trials', 'Reused PR trial records', 'trial-outcomes records with execution = reused across non-cancelled PR runs', () => {
const all = need(prRecords).flatMap(entry => entry.trials);
const reused = all.filter(record => record.execution === 'reused').length;
@@ -216,6 +240,7 @@ export function collectHealth(inputs: HealthInputs): HealthReport {
.reduce((sum, record) => sum + record.cost_usd, 0);
const revisions = new Set(entries.filter(entry => entry.run.conclusion === 'success' || entry.run.conclusion === 'failure').map(entry => entry.run.sha)).size;
const merged = listMergedPulls(client, repo, since);
if (merged) costPerMergedPr = `$${(spend / merged).toFixed(2)} (${merged} merged PRs, $${spend.toFixed(2)} recorded)`;
return { value: `$${spend.toFixed(2)} recorded`, detail: [
`per merged PR: ${merged ? `$${(spend / merged).toFixed(2)} (${merged} merged)` : 'n/a (0 merged)'}`,
`per validated revision: ${revisions ? `$${(spend / revisions).toFixed(2)} (${revisions} revisions)` : 'n/a'}`] };
@@ -267,7 +292,8 @@ export function collectHealth(inputs: HealthInputs): HealthReport {
}));
const checks = [flakeCheck(flakes), unseededCheck(root)];
return { repo, sinceDays, generatedAt: new Date(now).toISOString(), metrics, checks, issueAction: issueActionFor(checks) };
const headline = `Cost per merged PR: ${costPerMergedPr ?? `unavailable (${metrics.find(entry => entry.id === 'pr-spend')?.unavailable ?? '0 merged PRs'})`}`;
return { repo, sinceDays, generatedAt: new Date(now).toISOString(), headline, metrics, checks, issueAction: issueActionFor(checks) };
}
function listMergedPulls(client: GhClient, repo: string, since: string): number {
@@ -340,7 +366,7 @@ function censusMetrics(inputs: HealthInputs, artifactDirs: (run: CiRun, match: (
// --- Rendering ---
export function formatHealth(report: HealthReport): string {
const lines = [`test:health ${report.repo}, last ${report.sinceDays} day(s), ${report.generatedAt}`];
const lines = [`test:health ${report.repo}, last ${report.sinceDays} day(s), ${report.generatedAt}`, ...(report.headline ? [report.headline] : [])];
for (const entry of report.metrics) {
lines.push(`- ${entry.title}: ${entry.unavailable ? `unavailable: ${entry.unavailable}` : entry.value}`);
lines.push(` rule: ${entry.rule}`);
test/test-health-report.test.ts
+25 -1
View File
@@ -191,7 +191,7 @@ describe('collectHealth over recorded fixture data', () => {
[/runs\/61\/artifacts/, () => ({ artifacts: [] })],
[/workflows\/windows-free-tests\.yml\/runs/, () => ({ workflow_runs: windowsRuns })],
], {
11: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'full-fallback' } }) },
11: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'full-fallback', e2e: ['gate-a', 'gate-b'], judges: ['judge-a'] } }) },
12: { 'trial-outcomes.jsonl': prTrials },
51: { 'trial-outcomes.jsonl': census },
61: { 'flake-ledger.jsonl': ledger('test/a.test.ts') },
@@ -205,6 +205,10 @@ describe('collectHealth over recorded fixture data', () => {
expect(value('net-complexity').unavailable).toContain('git grep HEAD failed');
expect(value('pr-reused-trials').value).toBe('1 of 2 trial records');
expect(value('pr-spend')).toMatchObject({ value: '$2.50 recorded', detail: ['per merged PR: $2.50 (1 merged)', 'per validated revision: $2.50 (1 revisions)'] });
expect(value('pr-cases-per-run')).toMatchObject({ value: '2.0 gate cases, 1.0 judges per run (1 runs)', detail: ['full-fallback: 2.0 gate cases, 1.0 judges (1 runs)'] });
expect(value('pr-known-cost-per-run')).toMatchObject({ value: '$2.50 per run (1 runs; 0 executed trial(s) with unknown cost)', detail: ['full-fallback: $2.50 (1 runs)'] });
expect(report.headline).toBe('Cost per merged PR: $2.50 (1 merged PRs, $2.50 recorded)');
expect(formatHealth(report).split('\n')[1]).toBe('Cost per merged PR: $2.50 (1 merged PRs, $2.50 recorded)');
expect(value('cancelled-slice-minutes').value).toBe('30 min/week (1 cancelled run(s))');
expect(value('push-to-verdict').value).toBe('p50 10.0 min, p95 10.0 min (1 runs)');
expect(value('census-periodic').value).toBe('1.00 expected failed trials/run (1/3 over 1 run(s)); P(red) 33.3% in force, 33.3% single-trial');
@@ -223,6 +227,26 @@ describe('collectHealth over recorded fixture data', () => {
expect(formatHealth(report)).toContain('rule: share of evals.yml pull_request runs');
});
test('known cost per PR run prices only known-cost executed trials and counts the unknown ones', () => {
const trials = formatTrialOutcomes([trialRecord('gate-a', 'passed', { tier: 'gate', lane: 'gate/pr', cost_usd: 4 }),
trialRecord('gate-pty', 'passed', { tier: 'gate', lane: 'gate/pr', cost_usd: 0, cost_known: false })]);
const client = fakeClient([
[/workflows\/evals\.yml\/runs/, () => ({ workflow_runs: [run(1, { event: 'pull_request', head_branch: 'a' }), run(2, { event: 'pull_request', head_branch: 'b' })] })],
[/runs\/1\/artifacts/, () => ({ artifacts: [{ id: 11, name: 'paid-plan', size_in_bytes: 10 }, { id: 12, name: 'trial-outcomes-pr-a1', size_in_bytes: 10 }] })],
[/runs\/2\/artifacts/, () => ({ artifacts: [{ id: 21, name: 'paid-plan', size_in_bytes: 10 }] })],
[/pulls\?state=closed/, () => []],
], {
11: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'dependents', e2e: ['gate-a', 'gate-pty', 'gate-c'], judges: ['j'] } }) },
12: { 'trial-outcomes.jsonl': trials },
21: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'pr', e2e: [], judges: [] } }) },
});
const report = collectHealth({ client, repo: 'o/r', now: NOW, sinceDays: 7, cacheDir: tmp('health-cache-'), root: freeRoot(3, 0) });
const value = (id: string) => report.metrics.find(entry => entry.id === id)!;
expect(value('pr-known-cost-per-run')).toMatchObject({ value: '$2.00 per run (2 runs; 1 executed trial(s) with unknown cost)', detail: ['dependents: $4.00 (1 runs)', 'pr: $0.00 (1 runs)'] });
expect(value('pr-cases-per-run')).toMatchObject({ value: '1.5 gate cases, 0.5 judges per run (2 runs)', detail: ['dependents: 3.0 gate cases, 1.0 judges (1 runs)', 'pr: 0.0 gate cases, 0.0 judges (1 runs)'] });
expect(report.headline).toBe('Cost per merged PR: unavailable (0 merged PRs)');
});
test('the Windows wall counts only successful pull_request runs and says so when there are none', () => {
const wall = (windowsRuns: unknown[]) => collectHealth({ client: fixture(windowsRuns), repo: 'o/r', now: NOW, sinceDays: 7, cacheDir: tmp('health-cache-'), root: freeRoot(3, 0) })
.metrics.find(entry => entry.id === 'free-wall')!;