test:health and factory scorecard: leave reused main pushes out of free-suite flake counts (#3725) (#3822)

A main push that reused its merged PR's verified run (#3709) runs no shards and uploads nothing, so both readers counted it as a zero-flake run and diluted real flake rates under the 5% bar. reuseChecker (scripts/lib/ci-history.ts) marks a run with no artifacts and a green reuse job; readCiFlakeLedgers leaves it out and refills the window, readFreeRetries leaves it out of the free-retry count. One capped job listing per artifact-less run (40 per read); past the cap or on a failed listing the run counts as before, with a note.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

scripts/lib/ci-history.ts

@@ -259,7 +259,38 @@ export function readFlakeLedgerDir(dir: string): FlakeLedgerEntry[] {
 export const FLAKE_POLICY = { maxRate: 0.05, minRuns: 20 } as const;
 
 export interface FlakyFile { file: string; flakyRuns: number; entries: number; rate: number; enforced: boolean; failing: boolean }
-export interface FlakeWindow { runs: number; flakyRuns: number; since: string; files: FlakyFile[]; enforceable: boolean }
+/** `skipped`: reused main pushes left out of the window (reuseChecker); `note`: runs that could not be checked. */
+export interface FlakeWindow { runs: number; flakyRuns: number; since: string; files: FlakyFile[]; enforceable: boolean; skipped?: number; note?: string }
+
+/** The most job listings one reuseChecker spends. */
+export const REUSE_LOOKUPS_MAX = 40;
+
+/**
+ * Finds runs that reused their merged pull request's verified run (#3709): such
+ * a run uploads no artifact and its `reuse` job succeeds, because every suite
+ * job was skipped. It holds no flake evidence, so flake counts leave it out of
+ * their runs and their denominator (#3725). A run with any artifact costs no
+ * call. A run with none costs one job listing, at most REUSE_LOOKUPS_MAX per
+ * checker; past the cap, or when the listing fails, the run counts as it did
+ * before #3725 and note() says so.
+ */
+export function reuseChecker(client: GhClient, repo: string) {
+  let lookups = 0;
+  let unchecked = 0;
+  let reason = '';
+  return {
+    reused(run: Pick<CiRun, 'id' | 'attempt'>, artifactNames: string[]): boolean {
+      if (artifactNames.length) return false;
+      if (lookups >= REUSE_LOOKUPS_MAX) { unchecked++; reason ||= `cap of ${REUSE_LOOKUPS_MAX} job listings reached`; return false; }
+      lookups++;
+      try {
+        return listRunJobs(client, repo, run.id, run.attempt).some(job => job.name === 'reuse' && job.conclusion === 'success');
+      } catch (error) { unchecked++; reason ||= unavailableReason(error); return false; }
+    },
+    note: (): string | undefined => (unchecked
+      ? `${unchecked} run(s) with no artifacts not checked for main-push reuse (${reason}); counted as before #3725` : undefined),
+  };
+}
 
 /**
  * Rank files by the share of runs in which they needed the retry pass. A file
@@ -286,9 +317,10 @@ export function rankFlakyFiles(runEntries: Map<number, FlakeLedgerEntry[]>, runC
 /**
  * Read the flake ledgers of main's completed free-tests runs: the runs created
  * in the last `sinceDays`, widened to the newest `minRuns` runs (within
- * `maxLookbackDays`) so the 5% rule has a window of at least 20 runs. A run
- * with no `flake-ledger-*` artifact had no flaky passes (the upload is skipped
- * when the ledger file does not exist).
+ * `maxLookbackDays`) so the 5% rule has a window of at least 20 runs. Reused
+ * main pushes (reuseChecker) are left out and older runs take their place. A
+ * run with no `flake-ledger-*` artifact had no flaky passes (the upload is
+ * skipped when the ledger file does not exist).
  */
 export function readCiFlakeLedgers(opts: { client: GhClient; repo: string; sinceDays: number; now: number; cacheDir: string;
   minRuns?: number; maxLookbackDays?: number; workflow?: string }): FlakeWindow {
@@ -297,16 +329,25 @@ export function readCiFlakeLedgers(opts: { client: GhClient; repo: string; since
   const all = listWorkflowRuns(opts.client, opts.repo, opts.workflow ?? 'free-tests.yml',
     { since: iso(Math.max(opts.sinceDays, opts.maxLookbackDays ?? 60)), branch: 'main' })
     .filter(run => run.conclusion === 'success' || run.conclusion === 'failure');
-  const recent = all.filter(run => run.createdAt >= iso(opts.sinceDays));
-  const window = recent.length >= minRuns ? recent : all.slice(0, minRuns);
+  const since = iso(opts.sinceDays);
+  const recent = all.filter(run => run.createdAt >= since).length;
   const fetcher = historyFetcher(opts.client);
+  const reuse = reuseChecker(opts.client, opts.repo);
+  const window: CiRun[] = [];
+  let skipped = 0;
+  for (const [index, run] of all.entries()) {
+    if (index >= recent && window.length >= minRuns) break;
+    if (reuse.reused(run, fetcher.listArtifacts(opts.repo, run.id).map(artifact => artifact.name))) skipped++;
+    else window.push(run);
+  }
   const runEntries = new Map<number, FlakeLedgerEntry[]>();
   for (const run of window) {
     const dirs = downloadRunArtifacts({ repo: opts.repo, run, cacheDir: opts.cacheDir, fetcher, match: name => name.startsWith('flake-ledger-') });
     const entries = dirs.flatMap(readFlakeLedgerDir);
     if (entries.length) runEntries.set(run.id, entries);
   }
-  return rankFlakyFiles(runEntries, window.length, window[window.length - 1]?.createdAt ?? iso(opts.sinceDays));
+  const note = reuse.note();
+  return { ...rankFlakyFiles(runEntries, window.length, window[window.length - 1]?.createdAt ?? since), skipped, ...(note ? { note } : {}) };
 }
 
 /** Markdown table of one run's merged flake ledgers, for a job summary. */

scripts/test-health-report.ts

@@ -128,12 +128,18 @@ export function flakeCheck(window: FlakeWindow | Error): Check {
     fix: 'Fix the race at source (docs/TESTING_INTERNALS.md#flake-ledger); never raise the timeout or add a retry. Reproduce with `bun test <file>` under load.' };
   if (window instanceof Error) return { ...base, status: 'unavailable', detail: `unavailable: ${unavailableReason(window)}` };
   const failing = window.files.filter(file => file.failing);
+  const left = flakeWindowNotes(window);
   if (!window.enforceable) {
-    return { ...base, status: 'pass', detail: `report-only: ${window.runs} main run(s) since ${window.since.slice(0, 10)} (< 20 needed to enforce)` };
+    return { ...base, status: 'pass', detail: `report-only: ${window.runs} main run(s) since ${window.since.slice(0, 10)} (< 20 needed to enforce)${left}` };
   }
-  if (failing.length === 0) return { ...base, status: 'pass', detail: `no file above 5% over ${window.runs} main runs` };
+  if (failing.length === 0) return { ...base, status: 'pass', detail: `no file above 5% over ${window.runs} main runs${left}` };
   return { ...base, status: 'fail',
-    detail: failing.map(file => `${file.file}: flaky in ${file.flakyRuns}/${window.runs} main runs (${pct(file.rate)} > 5%)`).join('; ') };
+    detail: failing.map(file => `${file.file}: flaky in ${file.flakyRuns}/${window.runs} main runs (${pct(file.rate)} > 5%)`).join('; ') + left };
+}
+
+/** The reused main pushes a flake window left out and the runs it could not check, or ''. */
+function flakeWindowNotes(window: FlakeWindow): string {
+  return `${window.skipped ? `; ${window.skipped} reused main push(es) left out` : ''}${window.note ? `; ${window.note}` : ''}`;
 }
 
 export function unseededCheck(root: string): Check {
@@ -286,9 +292,9 @@ export function collectHealth(inputs: HealthInputs): HealthReport {
   metrics.push(...censusMetrics(inputs, artifactDirs));
 
   const flakes = attempt(() => readCiFlakeLedgers({ client, repo, sinceDays, now, cacheDir }));
-  metrics.push(metric('free-flaky-pass-rate', 'Free-suite flaky-pass rate', 'share of main free-tests runs (success or failure) with at least one flaky-pass ledger entry; per-file rate = runs with that file / runs', () => {
+  metrics.push(metric('free-flaky-pass-rate', 'Free-suite flaky-pass rate', 'share of main free-tests runs (success or failure) with at least one flaky-pass ledger entry; per-file rate = runs with that file / runs; a main push that reused its pull request\'s run (no artifacts, `reuse` job green) is left out', () => {
     const window = need(flakes);
-    return { value: `${pct(window.runs ? window.flakyRuns / window.runs : 0)} (${window.flakyRuns}/${window.runs} main runs since ${window.since.slice(0, 10)} needed the retry pass; ${window.files.length} file(s))`,
+    return { value: `${pct(window.runs ? window.flakyRuns / window.runs : 0)} (${window.flakyRuns}/${window.runs} main runs since ${window.since.slice(0, 10)} needed the retry pass; ${window.files.length} file(s)${flakeWindowNotes(window)})`,
       detail: window.files.slice(0, 15).map(file => `${file.file}: ${file.flakyRuns}/${window.runs} runs (${pct(file.rate)})${file.failing ? ' FAIL > 5%' : ''}`) };
   }));