feat(test-health): report shadow savings as a lower bound and split red drops into activation misses and flaky drops (A2-3)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

scripts/test-health-report.ts

@@ -228,6 +228,30 @@ export function collectHealth(inputs: HealthInputs): HealthReport {
       detail: modes.map(mode => { const list = entries.filter(entry => (modeOf.get(entry.run.id) ?? 'no manifest') === mode); return `${mode}: ${mean(list)} (${list.length} runs)`; }) };
   }));
 
+  metrics.push(metric('pr-shadow-map', 'Exercised-path map shadow (not active)', 'over evals.yml pull_request runs whose paid-plan manifest carries prCoverage.shadow: the predicted saving (map per-case known cost of dropped minus added cases; a lower bound, since known cost excludes PTY trials with unknown cost) and shadow drops of red cases (more executed failed than passed trials in the run, on a case the shadow would drop). A drop counts toward the D3 activation bar as a miss unless the case passed in a later run of the same branch at the same head SHA, which marks it a flaky drop. Scaled to 7 days; docs/TESTING_INTERNALS.md#exercised-path-map', () => {
+    const shadowed = need(prManifests).filter(({ manifest }) => manifest.prCoverage?.shadow && !manifest.prCoverage.shadow.error);
+    if (shadowed.length === 0) throw new Error('no paid-plan manifest with a prCoverage.shadow block in the window');
+    const records = prRecords instanceof Error ? [] : prRecords;
+    const executed = (trials: TrialRecord[], id: string, outcome: string) => trials.filter(record => record.case === id && record.execution === 'executed' && record.outcome === outcome).length;
+    const saving = shadowed.reduce((sum, { manifest }) => sum + (Number(manifest.prCoverage.shadow.predictedSavingUsd) || 0), 0);
+    const shares = shadowed.map(({ manifest }) => Number(manifest.prCoverage.shadow.unknownCostShare)).filter(share => Number.isFinite(share) && share > 0);
+    const misses: string[] = [];
+    const flaky: string[] = [];
+    for (const { run, manifest } of shadowed) {
+      const trials = records.find(entry => entry.run.id === run.id)?.trials ?? [];
+      for (const id of (manifest.prCoverage.shadow.wouldDrop ?? []) as string[]) {
+        if (executed(trials, id, 'failed') <= executed(trials, id, 'passed')) continue;
+        const laterPass = records.find(entry => entry.run.branch === run.branch && entry.run.sha === run.sha
+          && (entry.run.createdAt > run.createdAt || (entry.run.id === run.id && entry.run.attempt > run.attempt)) && executed(entry.trials, id, 'passed') > 0);
+        if (laterPass) flaky.push(`${run.id}:${id} (passed in run ${laterPass.run.id})`); else misses.push(`${run.id}:${id}`);
+      }
+    }
+    const week = 7 / sinceDays;
+    const bound = shares.length ? `a lower bound: known cost excludes the ${Math.round(100 * Math.max(...shares))}% of observed case runs with unknown cost (PTY)` : 'a lower bound: known cost excludes PTY trials with unknown cost';
+    return { value: `$${(saving * week).toFixed(2)} predicted saving/week (${bound}); ${(misses.length * week).toFixed(1)} activation miss(es)/week, ${flaky.length} flaky drop(s) not counted (${shadowed.length} runs)`,
+      detail: [...misses.slice(0, 20).map(miss => `shadow miss: ${miss}`), ...flaky.slice(0, 20).map(drop => `flaky drop: ${drop}`)] };
+  }));
+
   metrics.push(metric('pr-reused-trials', 'Reused PR trial records', 'trial-outcomes records with execution = reused across non-cancelled PR runs', () => {
     const all = need(prRecords).flatMap(entry => entry.trials);
     const reused = all.filter(record => record.execution === 'reused').length;

test/test-health-report.test.ts

@@ -191,7 +191,8 @@ describe('collectHealth over recorded fixture data', () => {
       [/runs\/61\/artifacts/, () => ({ artifacts: [] })],
       [/workflows\/windows-free-tests\.yml\/runs/, () => ({ workflow_runs: windowsRuns })],
     ], {
-      11: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'full-fallback', e2e: ['gate-a', 'gate-b'], judges: ['judge-a'] } }) },
+      11: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'full-fallback', e2e: ['gate-a', 'gate-b'], judges: ['judge-a'],
+        shadow: { e2e: ['gate-a', 'gate-b'], wouldDrop: [], wouldAdd: [], provenance: [], predictedSavingUsd: 0, unpricedCases: 0 } } }) },
       12: { 'trial-outcomes.jsonl': prTrials },
       51: { 'trial-outcomes.jsonl': census },
       61: { 'flake-ledger.jsonl': ledger('test/a.test.ts') },
@@ -207,6 +208,7 @@ describe('collectHealth over recorded fixture data', () => {
     expect(value('pr-spend')).toMatchObject({ value: '$2.50 recorded', detail: ['per merged PR: $2.50 (1 merged)', 'per validated revision: $2.50 (1 revisions)'] });
     expect(value('pr-cases-per-run')).toMatchObject({ value: '2.0 gate cases, 1.0 judges per run (1 runs)', detail: ['full-fallback: 2.0 gate cases, 1.0 judges (1 runs)'] });
     expect(value('pr-known-cost-per-run')).toMatchObject({ value: '$2.50 per run (1 runs; 0 executed trial(s) with unknown cost)', detail: ['full-fallback: $2.50 (1 runs)'] });
+    expect(value('pr-shadow-map')).toMatchObject({ value: '$0.00 predicted saving/week (a lower bound: known cost excludes PTY trials with unknown cost); 0.0 activation miss(es)/week, 0 flaky drop(s) not counted (1 runs)', detail: [] });
     expect(report.headline).toBe('Cost per merged PR: $2.50 (1 merged PRs, $2.50 recorded)');
     expect(formatHealth(report).split('\n')[1]).toBe('Cost per merged PR: $2.50 (1 merged PRs, $2.50 recorded)');
     expect(value('cancelled-slice-minutes').value).toBe('30 min/week (1 cancelled run(s))');
@@ -247,6 +249,38 @@ describe('collectHealth over recorded fixture data', () => {
     expect(report.headline).toBe('Cost per merged PR: unavailable (0 merged PRs)');
   });
 
+  test('the shadow map metric counts red drops as activation misses unless the case passed later at the same head SHA, and labels dollars a lower bound', () => {
+    const gate = (caseId: string, outcome: 'passed' | 'failed', extra: Partial<TrialOutcomeRecord> = {}) => trialRecord(caseId, outcome, { tier: 'gate', lane: 'gate/pr', ...extra });
+    const first = formatTrialOutcomes([gate('gate-a', 'failed', { cost_usd: 3 }), gate('gate-b', 'failed'), gate('gate-c', 'failed'),
+      gate('gate-d', 'failed', { execution: 'reused', cost_usd: 0 }), gate('gate-e', 'passed')]);
+    const rerun = formatTrialOutcomes([gate('gate-b', 'passed'), gate('gate-a', 'failed')]);
+    const shadow = (wouldDrop: string[], predictedSavingUsd: number, unknownCostShare = 0.391) =>
+      ({ e2e: [], wouldDrop, wouldAdd: [], provenance: [], predictedSavingUsd, unpricedCases: 0, unknownCostShare });
+    const client = fakeClient([
+      [/workflows\/evals\.yml\/runs/, () => ({ workflow_runs: [
+        run(1, { event: 'pull_request', head_branch: 'a', head_sha: 'same', created_at: day(3) }),
+        run(2, { event: 'pull_request', head_branch: 'a', head_sha: 'same', created_at: day(2) }),
+        run(3, { event: 'pull_request', head_branch: 'c' }),
+        run(4, { event: 'pull_request', head_branch: 'd' })] })],
+      [/runs\/1\/artifacts/, () => ({ artifacts: [{ id: 11, name: 'paid-plan', size_in_bytes: 10 }, { id: 12, name: 'trial-outcomes-pr-a1', size_in_bytes: 10 }] })],
+      [/runs\/2\/artifacts/, () => ({ artifacts: [{ id: 21, name: 'paid-plan', size_in_bytes: 10 }, { id: 22, name: 'trial-outcomes-pr-a1', size_in_bytes: 10 }] })],
+      [/runs\/3\/artifacts/, () => ({ artifacts: [{ id: 31, name: 'paid-plan', size_in_bytes: 10 }] })],
+      [/runs\/4\/artifacts/, () => ({ artifacts: [{ id: 41, name: 'paid-plan', size_in_bytes: 10 }] })],
+      [/pulls\?state=closed/, () => []],
+    ], {
+      11: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'dependents', e2e: [], judges: [], shadow: shadow(['gate-a', 'gate-b', 'gate-d', 'gate-e'], 4.5) } }) },
+      12: { 'trial-outcomes.jsonl': first },
+      21: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'pr', e2e: [], judges: [], shadow: shadow([], -1.5) } }) },
+      22: { 'trial-outcomes.jsonl': rerun },
+      31: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'pr', e2e: [], judges: [], shadow: { ...shadow([], 99), error: 'bad map' } } }) },
+      41: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'pr', e2e: [], judges: [] } }) },
+    });
+    const report = collectHealth({ client, repo: 'o/r', now: NOW, sinceDays: 7, cacheDir: tmp('health-cache-'), root: freeRoot(3, 0) });
+    expect(report.metrics.find(entry => entry.id === 'pr-shadow-map')).toMatchObject({
+      value: '$3.00 predicted saving/week (a lower bound: known cost excludes the 39% of observed case runs with unknown cost (PTY)); 1.0 activation miss(es)/week, 1 flaky drop(s) not counted (2 runs)',
+      detail: ['shadow miss: 1:gate-a', 'flaky drop: 1:gate-b (passed in run 2)'] });
+  });
+
   test('the Windows wall counts only successful pull_request runs and says so when there are none', () => {
     const wall = (windowsRuns: unknown[]) => collectHealth({ client: fixture(windowsRuns), repo: 'o/r', now: NOW, sinceDays: 7, cacheDir: tmp('health-cache-'), root: freeRoot(3, 0) })
       .metrics.find(entry => entry.id === 'free-wall')!;