From 22a4a50469c35b512bead2e69f2c2a7aec790068 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 31 Oct 2026 18:21:13 +0000 Subject: [PATCH] ci-audit: rate-limited 403s report ci_audit_rate_limit; changelog Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- changelog.d/garrytan-gsta-204-ci-audit.md | 3 ++- lib/ci-audit/github.ts | 4 ++++ test/ci-audit.test.ts | 8 ++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/changelog.d/garrytan-gsta-204-ci-audit.md b/changelog.d/garrytan-gsta-204-ci-audit.md index 1ca345363..f68e983ab 100644 --- a/changelog.d/garrytan-gsta-204-ci-audit.md +++ b/changelog.d/garrytan-gsta-204-ci-audit.md @@ -6,7 +6,8 @@ bump: minor - **`bin/gstack-ci-audit run`** (lib/ci-audit/): push to all checks green over all-green, first-attempt PR heads (cancelled, superseded, rerun, unsettled and window-edge heads are left out and counted), time to first failure, per-job waiting/setup/running with n, and cost from billed minutes scaled by per-workflow run counts (GitHub's runs list stops at 1,000 runs; the report says when the timing window is shorter). Ubicloud is priced by the VM family read from sampled logs, and cache hit rate is claimed only from restore lines in logs (`--logs N`, at most 20). Exit codes 0 complete, 1 usage, 3 unavailable, 4 partial. Errors are one `CI_AUDIT: unavailable ()` line; ten `ci_audit_*` codes in `lib/gate-outcomes.ts` with docs/troubleshooting.md sections. - **`gstack-ci-audit ship-check`**: a few-second, report-only check for /ship. It checks CI-shaping files in the diff against the lessons, and with `--pr` compares the PR's previous settled push with up to 40 comparable all-green PR heads. "Slower" means above their p95 and at least `ci_audit_slower_min_s` (default 120) over their median; under 8 comparable heads it prints `unknown thin baseline`. About 6 GitHub calls, 10 s deadline. `gstack-config set ci_audit_ship off` turns it off. (The /ship step that calls it lands separately.) - **Saturday audit**: `ci-audit/templates/ci-audit-weekly.yml` runs the report every Saturday at 15:00 UTC and `gstack-ci-audit publish` keeps one "Weekly CI audit" issue current. Only a complete audit replaces the issue or closes it; a partial or failed one comments and keeps the last findings. -- **`--record` / `--fixture`** save and replay GitHub responses with allowlisted fields only; `test/fixtures/ci-audit/gstack` is a recorded gstack window. +- **`--record` / `--fixture`** (or `GSTACK_CI_AUDIT_FIXTURE`) save and replay GitHub responses with allowlisted fields only, plus the recording's clock; `test/fixtures/ci-audit/gstack` is a recorded gstack window. +- **Paid case `ci-audit-planted`** (gate, behavior) runs /ci-audit on a synthetic repo with three planted problems and two decoys; `ci-audit-planted-no-gstack` (periodic) gives the same task and data with no skill text. - **docs/ci-audit.md**, catalog rows, and the doctor's `ci-audit` row. ### Changed diff --git a/lib/ci-audit/github.ts b/lib/ci-audit/github.ts index dcd5266ba..87a9d07a3 100644 --- a/lib/ci-audit/github.ts +++ b/lib/ci-audit/github.ts @@ -86,6 +86,10 @@ function parseIncluded(out: string, err: string, code: number, apiPath: string): if (/gh auth login|not logged in|GH_TOKEN/i.test(err)) throw new AuditError('ci_audit_gh_missing', 'gh is not signed in'); throw new AuditError('ci_audit_api_failed', `gh api ${apiPath}: ${err.trim().split('\n')[0] ?? 'no output'}`, 'now'); } + if ((status === 403 || status === 429) && (/^x-ratelimit-remaining:\s*0\s*$/im.test(head) || /rate limit/i.test(body))) { + const reset = Number(/^x-ratelimit-reset:\s*(\d+)/im.exec(head)?.[1]); + throw new AuditError('ci_audit_rate_limit', `GitHub refused ${apiPath} for rate limit (HTTP ${status})`, Number.isFinite(reset) ? `after ${new Date(reset * 1000).toISOString()}` : 'after an hour'); + } let json: unknown = null; try { json = body ? JSON.parse(body) : null; } catch { throw new AuditError('ci_audit_api_failed', `gh api ${apiPath} returned non-JSON (HTTP ${status})`, 'now'); } return { status, json }; diff --git a/test/ci-audit.test.ts b/test/ci-audit.test.ts index 07b826915..0ebd7f016 100644 --- a/test/ci-audit.test.ts +++ b/test/ci-audit.test.ts @@ -333,6 +333,14 @@ describe('command line', () => { expect(r.stdout.toString()).toContain('(ci_audit_deadline)'); }, 35_000); + test('a rate-limited 403 is reported as ci_audit_rate_limit with the reset time', () => { + const bin = tmp(); + fs.symlinkSync(process.execPath, path.join(bin, 'bun')); + fs.writeFileSync(path.join(bin, 'gh'), '#!/bin/sh\nprintf "HTTP/2.0 403 Forbidden\\r\\nX-Ratelimit-Remaining: 0\\r\\nX-Ratelimit-Reset: 1793472066\\r\\n\\r\\n{\\"message\\":\\"API rate limit exceeded\\"}"\nexit 1\n', { mode: 0o755 }); + const r = Bun.spawnSync([path.join(bin, 'bun'), '--no-env-file', '--config=/dev/null', path.join(ROOT, 'bin/gstack-ci-audit'), 'ship-check', '--base', 'main', '--pr', '5', '--repo', 'o/n', '--root', tmp()], { env: { PATH: `${bin}:/bin:/usr/bin`, HOME: tmp(), GSTACK_STATE_ROOT: tmp() }, timeout: 20_000 }); + expect(r.stdout.toString()).toContain('(ci_audit_rate_limit)'); + }); + test('ship-check against a failing gh reports ci_audit_api_failed', () => { const bin = tmp(); fs.symlinkSync(process.execPath, path.join(bin, 'bun'));