diff --git a/changelog.d/capy-ci-carry-reads-rerun-attempt-jobs-in-pages-gst-2215.md b/changelog.d/capy-ci-carry-reads-rerun-attempt-jobs-in-pages-gst-2215.md new file mode 100644 index 000000000..784bc891b --- /dev/null +++ b/changelog.d/capy-ci-carry-reads-rerun-attempt-jobs-in-pages-gst-2215.md @@ -0,0 +1,4 @@ +--- +important: blocker +--- +- **`gstack-ci-carry` and `gstack-pregate --from-run` can verify a GitHub Actions run again after it has been rerun.** Reading a rerun attempt's job list 100 jobs at a time could take GitHub about 11 seconds for a large workflow (86 jobs in gbrain's Test runs), and a proxy that cuts requests off at about 10 seconds answered 502. So any run whose newest attempt was a rerun failed with `api_error`, and /ship fell back to running the whole suite locally, even after a green rerun of a known flake. The job list is now read 50 per page, every page, and a server error (5xx) reads the whole list once more at 25 per page. A page that still fails leaves the run unverified (`api_error`), as before: a partial job list never counts. (GST-2215) diff --git a/lib/run-evidence.ts b/lib/run-evidence.ts index 262c1ceda..fae562338 100644 --- a/lib/run-evidence.ts +++ b/lib/run-evidence.ts @@ -332,7 +332,11 @@ export function ghRunDeps(repo: string, repoRoot: string): RunDeps { return { run: id => json(`repos/${repo}/actions/runs/${id}`), runsForHead: (workflow, sha) => all(`repos/${repo}/actions/workflows/${workflow}/runs?head_sha=${sha}&per_page=100`, 'workflow_runs'), - jobs: (id, attempt) => all(`repos/${repo}/actions/runs/${id}/attempts/${attempt}/jobs?per_page=100`, 'jobs'), + // A long rerun attempt's 100-job page can outlast a proxy's ~10 s cutoff (GST-2215): read 50 per page, and after a 5xx read the whole list once more at 25. A failed page throws, so a partial list never counts. + jobs: (id, attempt) => { + const read = (n: number) => all(`repos/${repo}/actions/runs/${id}/attempts/${attempt}/jobs?per_page=${n}`, 'jobs'); + try { return read(50); } catch (e) { if (!/\bHTTP 5\d\d\b/.test((e as Error).message)) throw e; return read(25); } + }, artifact: (id, name) => { const listed = all<{ name: string; size_in_bytes: number; expired: boolean }>(`repos/${repo}/actions/runs/${id}/artifacts?name=${encodeURIComponent(name)}&per_page=100`, 'artifacts').filter(a => a.name === name && !a.expired); if (listed.length !== 1) throw new Error(`run ${id} has ${listed.length} unexpired artifacts named ${name}`); diff --git a/test/run-evidence.test.ts b/test/run-evidence.test.ts index d5cf8e90c..a68bf735c 100644 --- a/test/run-evidence.test.ts +++ b/test/run-evidence.test.ts @@ -2,8 +2,11 @@ * lib/run-evidence.ts: a GitHub Actions run as evidence for one tree (GSTA-156). Every refusal reason has a fixture; * the dependencies are fakes, so nothing here reaches GitHub. */ -import { describe, expect, test } from 'bun:test'; -import { parseRunReceipt, parseRunRef, runInventory, runProfileErrors, runReceiptFields, uncoveredFiles, verifyFromRun, verifyRun, type ApiJob, type ApiRun, type RunDeps, type RunProfile, type TreeRecord } from '../lib/run-evidence'; +import { afterAll, describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { ghRunDeps, parseRunReceipt, parseRunRef, runInventory, runProfileErrors, runReceiptFields, uncoveredFiles, verifyFromRun, verifyRun, type ApiJob, type ApiRun, type RunDeps, type RunProfile, type TreeRecord } from '../lib/run-evidence'; const REPO = 'garrytan/gbrain'; const HEAD = 'a'.repeat(40); @@ -248,3 +251,72 @@ describe('verifyFromRun (P3b --from-run)', () => { expect(verifyFromRun(src(world({ others: [red] })), '100', ['test.yml/unit'], TREE, OWN)).toMatchObject({ ok: false, reason: 'newer_run_vetoes' }); }); }); + +describe.skipIf(process.platform === 'win32')('ghRunDeps jobs read through a proxy that cuts slow pages (GST-2215)', () => { + // A stub gh that pages an attempt-2 jobs list itself, the way `gh api --paginate` does, and answers 502 for any page + // larger than maxPage (the git proxy's ~10 s cutoff), for one failing page, or 404 for everything. + const dirs: string[] = []; + afterAll(() => { for (const d of dirs) fs.rmSync(d, { recursive: true, force: true }); }); + function stubGh(o: { jobs: number; maxPage: number; failPage?: number; status?: 404 }) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gst2215-')); + dirs.push(dir); + const log = path.join(dir, 'calls.log'); + fs.writeFileSync(path.join(dir, 'gh'), [ + '#!/usr/bin/env bash', + `printf '%s\\n' "$*" >> '${log}'`, + ...(o.status ? [`echo 'gh: Not Found (HTTP ${o.status})' >&2; exit 1`] : []), + 'url=""; for a in "$@"; do case "$a" in repos/*) url="$a";; esac; done', + 'case "$url" in */actions/runs/100/attempts/2/jobs\\?per_page=*) ;; *) echo "gh: unexpected $url" >&2; exit 1;; esac', + 'n="${url##*per_page=}"', + `if [ "$n" -gt ${o.maxPage} ]; then echo 'gh: Bad Gateway (HTTP 502)' >&2; exit 1; fi`, + `i=1; page=1; while [ $i -le ${o.jobs} ]; do`, + ` if [ $page -eq ${o.failPage ?? 0} ]; then echo 'gh: Bad Gateway (HTTP 502)' >&2; exit 1; fi`, + ` end=$((i + n - 1)); [ $end -gt ${o.jobs} ] && end=${o.jobs}`, + ' for j in $(seq $i $end); do printf \'{"name":"job-%d","conclusion":"success"}\\n\' $j; done', + ' i=$((end + 1)); page=$((page + 1))', + 'done', + ].join('\n'), { mode: 0o755 }); + return { dir, calls: () => fs.readFileSync(log, 'utf8').trim().split('\n') }; + } + function verifyThrough(stub: { dir: string }) { + const saved = { PATH: process.env.PATH, GSTACK_STATE_ROOT: process.env.GSTACK_STATE_ROOT }; + process.env.PATH = `${stub.dir}:${saved.PATH}`; + process.env.GSTACK_STATE_ROOT = path.join(stub.dir, 'state'); + try { + const base = world(); + const w = world({ run: { ...base.run, run_attempt: 2 }, artifact: { ...(base.artifact as TreeRecord), run_attempt: 2 } }); + return verifyRun({ ...deps(w), jobs: ghRunDeps(REPO, stub.dir).jobs }, { repo: REPO, runRef: '100', profileName: 'test', profile: { ...PROFILE, jobs: ['job-1', 'job-75'] }, now: NOW }); + } finally { + process.env.PATH = saved.PATH; + if (saved.GSTACK_STATE_ROOT === undefined) delete process.env.GSTACK_STATE_ROOT; else process.env.GSTACK_STATE_ROOT = saved.GSTACK_STATE_ROOT; + } + } + const page = (n: number) => `api --paginate repos/${REPO}/actions/runs/100/attempts/2/jobs?per_page=${n} --jq .jobs[]`; + + // Value: protects=a rerun attempt with 86 jobs verifies when the proxy 502s a 100-job page; fails_when=the read asks for 100 per page or stops after page one; + // why_new=GST-2215 (gbrain #8940); seam=ghRunDeps.jobs over a PATH stub; no_claim=GitHub's real latency + test('an attempt-2 run with more than 50 jobs is read at 50 per page, every page, and verifies', () => { + const stub = stubGh({ jobs: 86, maxPage: 50 }); + const v = verifyThrough(stub); + expect(v.ok ? v.evidence.run_attempt : v.detail).toBe(2); + expect(stub.calls()).toEqual([page(50)]); + }); + + // Value: protects=one smaller-page retry after a 5xx; fails_when=a 502 at 50 per page refuses without retrying + test('a 5xx at 50 per page retries the whole list once at 25 per page', () => { + const stub = stubGh({ jobs: 86, maxPage: 25 }); + expect(verifyThrough(stub).ok).toBe(true); + expect(stub.calls()).toEqual([page(50), page(25)]); + }); + + // Value: protects=fail closed, a partial job list never counts; fails_when=the jobs printed before a failed page are used, or the retry repeats + test('a page failing in the middle of the list stays api_error after one retry, and a non-5xx error does not retry', () => { + const mid = stubGh({ jobs: 86, maxPage: 100, failPage: 2 }); + const v = verifyThrough(mid); + expect(v.ok ? 'ok' : [v.reason, v.detail.includes('HTTP 502')]).toEqual(['api_error', true]); + expect(mid.calls()).toEqual([page(50), page(25)]); + const missing = stubGh({ jobs: 86, maxPage: 100, status: 404 }); + expect(verifyThrough(missing)).toMatchObject({ ok: false, reason: 'api_error' }); + expect(missing.calls()).toEqual([page(50)]); + }); +});