Let ci-carry verify a rerun Actions attempt by reading its jobs in pages of 50 (GST-2215) (#4246)

A rerun attempt's 100-job page took GitHub ~11 s for gbrain's 86-job Test runs, past the git proxy's ~10 s cutoff, so every rerun failed verification with api_error. Read 50 per page and, after a 5xx, reread the whole list once at 25. Any failed page still throws, so a partial job list never counts.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

changelog.d/capy-ci-carry-reads-rerun-attempt-jobs-in-pages-gst-2215.md

@@ -0,0 +1,4 @@
+---
+important: blocker
+---
+- **`gstack-ci-carry` and `gstack-pregate --from-run` can verify a GitHub Actions run again after it has been rerun.** Reading a rerun attempt's job list 100 jobs at a time could take GitHub about 11 seconds for a large workflow (86 jobs in gbrain's Test runs), and a proxy that cuts requests off at about 10 seconds answered 502. So any run whose newest attempt was a rerun failed with `api_error`, and /ship fell back to running the whole suite locally, even after a green rerun of a known flake. The job list is now read 50 per page, every page, and a server error (5xx) reads the whole list once more at 25 per page. A page that still fails leaves the run unverified (`api_error`), as before: a partial job list never counts. (GST-2215)

lib/run-evidence.ts

@@ -332,7 +332,11 @@ export function ghRunDeps(repo: string, repoRoot: string): RunDeps {
   return {
     run: id => json<ApiRun>(`repos/${repo}/actions/runs/${id}`),
     runsForHead: (workflow, sha) => all<ApiRun>(`repos/${repo}/actions/workflows/${workflow}/runs?head_sha=${sha}&per_page=100`, 'workflow_runs'),
-    jobs: (id, attempt) => all<ApiJob>(`repos/${repo}/actions/runs/${id}/attempts/${attempt}/jobs?per_page=100`, 'jobs'),
+    // A long rerun attempt's 100-job page can outlast a proxy's ~10 s cutoff (GST-2215): read 50 per page, and after a 5xx read the whole list once more at 25. A failed page throws, so a partial list never counts.
+    jobs: (id, attempt) => {
+      const read = (n: number) => all<ApiJob>(`repos/${repo}/actions/runs/${id}/attempts/${attempt}/jobs?per_page=${n}`, 'jobs');
+      try { return read(50); } catch (e) { if (!/\bHTTP 5\d\d\b/.test((e as Error).message)) throw e; return read(25); }
+    },
     artifact: (id, name) => {
       const listed = all<{ name: string; size_in_bytes: number; expired: boolean }>(`repos/${repo}/actions/runs/${id}/artifacts?name=${encodeURIComponent(name)}&per_page=100`, 'artifacts').filter(a => a.name === name && !a.expired);
       if (listed.length !== 1) throw new Error(`run ${id} has ${listed.length} unexpired artifacts named ${name}`);

test/run-evidence.test.ts

@@ -2,8 +2,11 @@
  * lib/run-evidence.ts: a GitHub Actions run as evidence for one tree (GSTA-156). Every refusal reason has a fixture;
  * the dependencies are fakes, so nothing here reaches GitHub.
  */
-import { describe, expect, test } from 'bun:test';
-import { parseRunReceipt, parseRunRef, runInventory, runProfileErrors, runReceiptFields, uncoveredFiles, verifyFromRun, verifyRun, type ApiJob, type ApiRun, type RunDeps, type RunProfile, type TreeRecord } from '../lib/run-evidence';
+import { afterAll, describe, expect, test } from 'bun:test';
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import { ghRunDeps, parseRunReceipt, parseRunRef, runInventory, runProfileErrors, runReceiptFields, uncoveredFiles, verifyFromRun, verifyRun, type ApiJob, type ApiRun, type RunDeps, type RunProfile, type TreeRecord } from '../lib/run-evidence';
 
 const REPO = 'garrytan/gbrain';
 const HEAD = 'a'.repeat(40);
@@ -248,3 +251,72 @@ describe('verifyFromRun (P3b --from-run)', () => {
     expect(verifyFromRun(src(world({ others: [red] })), '100', ['test.yml/unit'], TREE, OWN)).toMatchObject({ ok: false, reason: 'newer_run_vetoes' });
   });
 });
+
+describe.skipIf(process.platform === 'win32')('ghRunDeps jobs read through a proxy that cuts slow pages (GST-2215)', () => {
+  // A stub gh that pages an attempt-2 jobs list itself, the way `gh api --paginate` does, and answers 502 for any page
+  // larger than maxPage (the git proxy's ~10 s cutoff), for one failing page, or 404 for everything.
+  const dirs: string[] = [];
+  afterAll(() => { for (const d of dirs) fs.rmSync(d, { recursive: true, force: true }); });
+  function stubGh(o: { jobs: number; maxPage: number; failPage?: number; status?: 404 }) {
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gst2215-'));
+    dirs.push(dir);
+    const log = path.join(dir, 'calls.log');
+    fs.writeFileSync(path.join(dir, 'gh'), [
+      '#!/usr/bin/env bash',
+      `printf '%s\\n' "$*" >> '${log}'`,
+      ...(o.status ? [`echo 'gh: Not Found (HTTP ${o.status})' >&2; exit 1`] : []),
+      'url=""; for a in "$@"; do case "$a" in repos/*) url="$a";; esac; done',
+      'case "$url" in */actions/runs/100/attempts/2/jobs\\?per_page=*) ;; *) echo "gh: unexpected $url" >&2; exit 1;; esac',
+      'n="${url##*per_page=}"',
+      `if [ "$n" -gt ${o.maxPage} ]; then echo 'gh: Bad Gateway (HTTP 502)' >&2; exit 1; fi`,
+      `i=1; page=1; while [ $i -le ${o.jobs} ]; do`,
+      `  if [ $page -eq ${o.failPage ?? 0} ]; then echo 'gh: Bad Gateway (HTTP 502)' >&2; exit 1; fi`,
+      `  end=$((i + n - 1)); [ $end -gt ${o.jobs} ] && end=${o.jobs}`,
+      '  for j in $(seq $i $end); do printf \'{"name":"job-%d","conclusion":"success"}\\n\' $j; done',
+      '  i=$((end + 1)); page=$((page + 1))',
+      'done',
+    ].join('\n'), { mode: 0o755 });
+    return { dir, calls: () => fs.readFileSync(log, 'utf8').trim().split('\n') };
+  }
+  function verifyThrough(stub: { dir: string }) {
+    const saved = { PATH: process.env.PATH, GSTACK_STATE_ROOT: process.env.GSTACK_STATE_ROOT };
+    process.env.PATH = `${stub.dir}:${saved.PATH}`;
+    process.env.GSTACK_STATE_ROOT = path.join(stub.dir, 'state');
+    try {
+      const base = world();
+      const w = world({ run: { ...base.run, run_attempt: 2 }, artifact: { ...(base.artifact as TreeRecord), run_attempt: 2 } });
+      return verifyRun({ ...deps(w), jobs: ghRunDeps(REPO, stub.dir).jobs }, { repo: REPO, runRef: '100', profileName: 'test', profile: { ...PROFILE, jobs: ['job-1', 'job-75'] }, now: NOW });
+    } finally {
+      process.env.PATH = saved.PATH;
+      if (saved.GSTACK_STATE_ROOT === undefined) delete process.env.GSTACK_STATE_ROOT; else process.env.GSTACK_STATE_ROOT = saved.GSTACK_STATE_ROOT;
+    }
+  }
+  const page = (n: number) => `api --paginate repos/${REPO}/actions/runs/100/attempts/2/jobs?per_page=${n} --jq .jobs[]`;
+
+  // Value: protects=a rerun attempt with 86 jobs verifies when the proxy 502s a 100-job page; fails_when=the read asks for 100 per page or stops after page one;
+  //   why_new=GST-2215 (gbrain #8940); seam=ghRunDeps.jobs over a PATH stub; no_claim=GitHub's real latency
+  test('an attempt-2 run with more than 50 jobs is read at 50 per page, every page, and verifies', () => {
+    const stub = stubGh({ jobs: 86, maxPage: 50 });
+    const v = verifyThrough(stub);
+    expect(v.ok ? v.evidence.run_attempt : v.detail).toBe(2);
+    expect(stub.calls()).toEqual([page(50)]);
+  });
+
+  // Value: protects=one smaller-page retry after a 5xx; fails_when=a 502 at 50 per page refuses without retrying
+  test('a 5xx at 50 per page retries the whole list once at 25 per page', () => {
+    const stub = stubGh({ jobs: 86, maxPage: 25 });
+    expect(verifyThrough(stub).ok).toBe(true);
+    expect(stub.calls()).toEqual([page(50), page(25)]);
+  });
+
+  // Value: protects=fail closed, a partial job list never counts; fails_when=the jobs printed before a failed page are used, or the retry repeats
+  test('a page failing in the middle of the list stays api_error after one retry, and a non-5xx error does not retry', () => {
+    const mid = stubGh({ jobs: 86, maxPage: 100, failPage: 2 });
+    const v = verifyThrough(mid);
+    expect(v.ok ? 'ok' : [v.reason, v.detail.includes('HTTP 502')]).toEqual(['api_error', true]);
+    expect(mid.calls()).toEqual([page(50), page(25)]);
+    const missing = stubGh({ jobs: 86, maxPage: 100, status: 404 });
+    expect(verifyThrough(missing)).toMatchObject({ ok: false, reason: 'api_error' });
+    expect(missing.calls()).toEqual([page(50)]);
+  });
+});