From aaff4741b9694bcad84e79876f1c0aa1d19edb4d Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Fri, 6 Nov 2026 21:51:12 -0800 Subject: [PATCH] fix: gstack's GitHub waits poll REST at most once a minute and wait out rate limits (GST-1713) (#4109) * fix: gstack's GitHub waits poll REST at most once a minute and wait out rate limits (GST-1713) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test: register the gh-poll REST fixtures as free fixtures (GST-1713) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: pre-landing review fixes (GST-1713) ci-gate --registration-wait re-polls while time is left in the window; a rate-limited gh pr merge fails at once under the land lock; a bot PR author reads without its [bot] suffix; ci-carry wait documents its one-poll overrun. Review-Finding: p1/specialist-batched#1 reader=testing Review-Finding: p1/specialist-batched#2 reader=testing Review-Finding: p1/specialist-batched#3 reader=testing Review-Finding: p1/red-team#1 reader=red-team Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs: the GST-1713 changelog fragment names the gh pr merge exception Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- bin/gstack-ci-carry | 6 +- bin/gstack-ship-queue | 3 +- bin/gstack-ship-receipt | 3 +- .../capy-gst-1713-github-poll-floor.md | 1 + docs/ship-policy.md | 2 +- docs/standalone-binaries.md | 2 +- land-and-deploy/SKILL.md | 4 +- land-and-deploy/SKILL.md.tmpl | 4 +- land-and-deploy/sections/merge-and-deploy.md | 2 +- .../sections/merge-and-deploy.md.tmpl | 2 +- lib/ci-carry.ts | 16 +- lib/ci-gate.ts | 22 +- lib/gh-poll.ts | 180 +++ lib/greptile-early.ts | 11 +- lib/pr-wait-github.ts | 116 +- lib/pr-wait.ts | 46 +- lib/release-gate.ts | 3 +- lib/run-evidence.ts | 3 +- lib/ship-bin-common.ts | 3 +- lib/ship-queue-dispatch-ci.ts | 3 +- lib/ship-queue-land-direct.ts | 5 +- lib/ship-queue-land.ts | 3 +- lib/ship-queue-post-merge.ts | 3 +- lib/ship-queue-train-watch.ts | 7 +- lib/ship-queue-train.ts | 3 +- lib/ship-release.ts | 3 +- scripts/eval-bg.ts | 5 +- scripts/recover-receipts.ts | 1 + scripts/train-backstop.ts | 2 + test/eval-detach-timeout-floor.test.ts | 2 +- test/fixtures/gh-poll/check-runs.json | 1221 +++++++++++++++++ test/fixtures/gh-poll/pull-merged.json | 20 + test/fixtures/gh-poll/pull-open.json | 20 + test/fixtures/gh-poll/rate-limit.json | 22 + test/fixtures/gh-poll/status.json | 6 + test/gh-poll-tripwire.test.ts | 98 ++ test/gh-poll.test.ts | 202 +++ test/greptile-early.test.ts | 4 +- test/gstack-ci-gate.test.ts | 24 +- test/helpers/free-fixtures-data.ts | 1 + test/helpers/ship-queue-train-fixture.ts | 2 +- test/pr-wait.test.ts | 97 +- 42 files changed, 2044 insertions(+), 139 deletions(-) create mode 100644 changelog.d/capy-gst-1713-github-poll-floor.md create mode 100644 lib/gh-poll.ts create mode 100644 test/fixtures/gh-poll/check-runs.json create mode 100644 test/fixtures/gh-poll/pull-merged.json create mode 100644 test/fixtures/gh-poll/pull-open.json create mode 100644 test/fixtures/gh-poll/rate-limit.json create mode 100644 test/fixtures/gh-poll/status.json create mode 100644 test/gh-poll-tripwire.test.ts create mode 100644 test/gh-poll.test.ts diff --git a/bin/gstack-ci-carry b/bin/gstack-ci-carry index c5292f5eb..1771c0287 100755 --- a/bin/gstack-ci-carry +++ b/bin/gstack-ci-carry @@ -10,7 +10,7 @@ * ship_lanes_from names) at HEAD's clean tree. With --command, the base branch's mapping for that exact lane * command supplies the profiles and the inventory the runs must cover (a pin-file command, so it needs * --allow-repo-commands). Prints CI_CARRY: accepted (exit 0) or CI_CARRY: none () (exit 1). - * wait: polls the newest run of the profile's workflow at HEAD until it completes (default timeout 1800 s), + * wait: polls the newest run of the profile's workflow at HEAD (REST, at least 60 s apart) until it completes (default timeout 1800 s), * prints CI_WAIT: green|red|running|none, and on green runs find. Exit 0 green and carried, 1 red or not * carried, 3 still running or no run. Logic: lib/ci-carry.ts; the run checks are lib/run-evidence.ts. */ @@ -28,7 +28,7 @@ const HELP = `gstack-ci-carry — use a verified CI run at HEAD's exact tree ins --policy-from head --simulate: the same check under the branch's own map and profiles, every line starting SIMULATION:; a dry run that proves a new mapping before it lands (find never writes evidence; nothing clears) wait [--base ] [--profile ] [--timeout ] [--interval ] - poll the newest run at HEAD (default 1800 s, every 20 s): CI_WAIT: green|red|running|none, then find on green; + poll the newest run at HEAD (default 1800 s, every 60 s at least, plus jitter): CI_WAIT: green|red|running|none, then find on green; exit 0 carried, 1 red or not carried, 3 still running or no run Profiles and ship_lanes_from come from origin/'s .gstack/pregate.json only (a simulation reads the branch's). Only a clean, committed tree @@ -75,7 +75,7 @@ function find(): number { if (cmd === 'find') process.exit(find()); const timeout = Number(flag('--timeout') ?? '1800'); -const interval = Number(flag('--interval') ?? '20'); +const interval = Number(flag('--interval') ?? '60'); if (!(timeout >= 0 && timeout <= 21_600) || !(interval >= 1 && interval <= 600)) usage('--timeout is 0-21600 seconds and --interval 1-600'); const profile = check.profiles[profiles[0]!]; if (!profile) usage(`${profiles[0]} is not a run_profiles name on origin/${base}`); diff --git a/bin/gstack-ship-queue b/bin/gstack-ship-queue index 686138351..a8021c08b 100755 --- a/bin/gstack-ship-queue +++ b/bin/gstack-ship-queue @@ -32,6 +32,7 @@ */ import { spawnSync } from 'node:child_process'; import * as path from 'node:path'; +import { ghSpawnSync } from '../lib/gh-poll'; import { EXIT, exitTableHelp, formatResult } from '../lib/headless-artifacts'; import { versionAtRef } from '../lib/restamp'; import { RESULT_CODES } from '../lib/result-codes'; @@ -166,7 +167,7 @@ async function main(argv: string[]): Promise { const probe = spawnSync('git', ['rev-parse', '--show-toplevel'], { encoding: 'utf8', timeout: 15_000 }); if (probe.status !== 0) return usage('not inside a git repository'); const repoRoot = probe.stdout.trim(); - const gh = (a: string[]) => { const r = spawnSync('gh', a, { cwd: repoRoot, encoding: 'utf8', timeout: 60_000 }); return { ok: r.status === 0, out: r.stdout ?? '', err: r.stderr ?? (r.error ? String(r.error.message) : '') }; }; + const gh = (a: string[]) => { const r = ghSpawnSync(a, { cwd: repoRoot, encoding: 'utf8', timeout: 60_000 }); return { ok: r.status === 0, out: r.stdout ?? '', err: r.stderr ?? (r.error ? String(r.error.message) : '') }; }; const ctx: QueueContext = { repoRoot, base: resolveBaseBranch(repoRoot, flag(args, '--base')), gh }; const pr = flag(args, '--pr'); if (cmd === 'gate-ahead') return gateAhead(ctx, args); diff --git a/bin/gstack-ship-receipt b/bin/gstack-ship-receipt index 6893d6a72..e782202e8 100755 --- a/bin/gstack-ship-receipt +++ b/bin/gstack-ship-receipt @@ -34,6 +34,7 @@ import { spawnSync } from 'node:child_process'; import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; +import { ghSpawnSync } from '../lib/gh-poll'; import { EXIT, exitTableHelp, type ShipReceipt } from '../lib/headless-artifacts'; import { resultLine, resultLineWithDocs } from '../lib/result-codes'; import { versionAtRef } from '../lib/restamp'; @@ -167,7 +168,7 @@ function git(args: string[], cwd?: string): string | undefined { return r.status === 0 ? r.stdout.trim() : undefined; } function gh(args: string[]): { ok: boolean; out: string; err: string } { - const r = spawnSync('gh', args, { encoding: 'utf8', timeout: 120_000, maxBuffer: 64 * 1024 * 1024 }); + const r = ghSpawnSync(args, { encoding: 'utf8', timeout: 120_000, maxBuffer: 64 * 1024 * 1024 }); return { ok: r.status === 0, out: r.stdout ?? '', err: (r.stderr ?? '').trim() }; } diff --git a/changelog.d/capy-gst-1713-github-poll-floor.md b/changelog.d/capy-gst-1713-github-poll-floor.md new file mode 100644 index 000000000..cc63960a3 --- /dev/null +++ b/changelog.d/capy-gst-1713-github-poll-floor.md @@ -0,0 +1 @@ +- **gstack stops draining the shared GitHub GraphQL quota.** `gstack-pr-wait` read the pull request through a GraphQL query and `gh pr checks` every 20 seconds (as often as every 5), about 360 GraphQL calls an hour per watcher, and the Capy app installation's 7,700-point hourly budget ran out right after each reset, stalling merges across crews. It now reads the PR state, base head and head-commit checks from REST, and makes its one GraphQL read (comments, reviews and review-thread resolution) only when the PR changed, every 10 minutes, or once green under `--until ready`. Every gstack wait between GitHub reads now goes through `lib/gh-poll.ts`, which puts a 60-second floor plus up to 15 seconds of jitter under it: `gstack-pr-wait` (was 20 s), `gstack-ci-gate --wait` and its registration re-poll (were 30 s and 10 s), `gstack-ci-carry wait` (was 20 s), `gstack-ship-release accept` (was 15 s), `gstack-greptile-early wait` (was 30 s, and its head read moved to REST), the merge-train and land waits (were 30 s, and 3 s for land's merge readback) and eval-bg's run follow (was 10 to 30 s). A gh call that fails with a rate-limit message is no longer retried at once. The gh runners read `gh api rate_limit`, sleep until that budget resets (at least 60 s) and retry at most twice. The one exception is `gh pr merge`, which runs under the land lock and still fails at once. `test/gh-poll-tripwire.test.ts` fails on any new wait in a GitHub-reading module that skips the floor. (GST-1713) diff --git a/docs/ship-policy.md b/docs/ship-policy.md index 51bf5f362..f161f4b80 100644 --- a/docs/ship-policy.md +++ b/docs/ship-policy.md @@ -2378,7 +2378,7 @@ restamping`, GST-1245), so a reset stamp needs no hand cleanup: keeps the reviewed pin apart from the last pushed head, so a rerun leases on what it pushed and never on a head someone else pushed (`QUEUE_TRAIN_MOVED`, never overwritten). -5. **Wait** for the PR's checks on the stamp commit, polling every 30 seconds +5. **Wait** for the PR's checks on the stamp commit, polling every 60 seconds or more (lib/gh-poll.ts) with the same check read the pin passed (`--timeout `, default 60). `--no-wait` stops after the push and prints `gstack-pr-wait --until green && gstack-ship-queue land --pr ` to run instead. diff --git a/docs/standalone-binaries.md b/docs/standalone-binaries.md index d956cc468..5396baa20 100644 --- a/docs/standalone-binaries.md +++ b/docs/standalone-binaries.md @@ -20,7 +20,7 @@ Beyond the slash-command skills, gstack ships standalone CLIs for workflows that | `gstack-review-read` | **Review freshness** — emits review records with computed `review_freshness.status` and `reason`: CURRENT, STALE, or UNVERIFIED for diff reviews. `/ship` and `/land-and-deploy` use the same grade; a matching commit alone never certifies a diff review. [Dashboard rules](skills.md#review-readiness-dashboard). | | `gstack-evidence` | **Verification-evidence ledger** — `run --label -- ` transparently wraps any test command (the child's exit code always passes through) and records what ran against which working-tree fingerprint (tracked and untracked files that are not gitignored, minus an untracked `.bun/` folder at the repo root, which Bun can create during a run); a run whose files changed while it ran is recorded without a fingerprint and with up to five of the changed paths, which `check` names in its STALE reason. `check` grades each label FRESH/STALE/MISSING with `--expect-cmd`, `--max-age`, and `--allow-paths` binding. A label is stored and looked up with every character outside `A-Za-z0-9._-` turned into `_`, so `pnpm lint` and `pnpm_lint` name the same row. `carried` lists the CI-sourced rows at the current tree that grade FRESH, which the ship receipt records as `ci_carried`. /ship and /land-and-deploy cite fresh evidence instead of re-running suites. Per-run logs are 0600, capped at 2MB, pruned after 30 days; the ledger and logs stay machine-local by design. | | `gstack-flaky` | **Flaky-test rerun for any JUnit runner** — `run --junit -- ` runs your test command (Playwright, `bun test`, jest, vitest, pytest) and, when at most `--max-reruns` (3) tests failed and the runner made no retries of its own, reruns the whole command once (`--rerun-scope test` opts in to rerunning only the failed tests). A test that fails and then passes on that rerun is a flaky pass: the run stays green and one redacted line per flaky pass or failed rerun goes on a ledger (`GSTACK_FLAKE_LEDGER_JUNIT`, default `/flake-ledger-junit.jsonl`). A missing or malformed report, a crash or a killed runner never passes. `quarantine add|renew|release|check|report` keeps `.gstack/quarantine.json`: a quarantined test still runs, gets up to 3 attempts of the whole command, passes only on a full green attempt and is never counted as a pass (`run --base ` reads the registry from the base branch). `ids --junit ` prints a report's test ids. Exit 0 pass, 1 a test failed, 2 usage, 3 no usable report. [Guide](flaky.md). | -| `gstack-pr-wait` | **PR event watcher** — `gstack-pr-wait [--repo owner/name] [--until green\|ready\|any-change] [--interval 20] [--timeout 60] [--exclude CHECK,...]` polls one pull request through `gh` and prints one stdout line per real event (a check failed with its link, all checks finished, a comment or review from someone other than the author, a review bot finished on the head, the base moved and its new commits touch this PR's files, BEHIND or DIRTY, merged or closed), with progress on stderr, so a monitor wakes the agent once per event instead of on a timer. `green` gates on every check on the head (an advisory bot's review is never waited for, its checks gate like any other), as `gstack-ci-gate` does; a check the base branch requires (rulesets and classic branch protection) is pending until it appears, never green by omission; two identical polls confirm a check set is complete; `--exclude` leaves a named non-required check out of the verdict as `gstack-ci-gate --exclude` does (a required name is refused and gates). Exit 0 when the condition is met, 1 when it needs action or `--timeout` passes (`PR_WAIT_BLOCKED`), 2 on bad input, a second watcher for the same PR or a local error (`PR_WAIT_REFUSED`). State under `/pr-wait/`, baseline discarded after 6 hours. Review bots are configuration: `gstack-config set pr_wait_gating_bots greptile,macroscope`, `pr_wait_advisory_bots coderabbit` (advisory bots are reported, never waited on). `/land-and-deploy` Step 3 is one call of it. Design ported from tstack's pr-wait (MIT, Theo Browne; NOTICE.md). [Exit codes](troubleshooting.md#gstack-pr-wait). | +| `gstack-pr-wait` | **PR event watcher** — `gstack-pr-wait [--repo owner/name] [--until green\|ready\|any-change] [--interval 60] [--timeout 60] [--exclude CHECK,...]` polls one pull request through `gh` and prints one stdout line per real event (a check failed with its link, all checks finished, a comment or review from someone other than the author, a review bot finished on the head, the base moved and its new commits touch this PR's files, BEHIND or DIRTY, merged or closed), with progress on stderr, so a monitor wakes the agent once per event instead of on a timer. `green` gates on every check on the head (an advisory bot's review is never waited for, its checks gate like any other), as `gstack-ci-gate` does; a check the base branch requires (rulesets and classic branch protection) is pending until it appears, never green by omission; two identical polls confirm a check set is complete; `--exclude` leaves a named non-required check out of the verdict as `gstack-ci-gate --exclude` does (a required name is refused and gates). Exit 0 when the condition is met, 1 when it needs action or `--timeout` passes (`PR_WAIT_BLOCKED`), 2 on bad input, a second watcher for the same PR or a local error (`PR_WAIT_REFUSED`). State under `/pr-wait/`, baseline discarded after 6 hours. Review bots are configuration: `gstack-config set pr_wait_gating_bots greptile,macroscope`, `pr_wait_advisory_bots coderabbit` (advisory bots are reported, never waited on). `/land-and-deploy` Step 3 is one call of it. Design ported from tstack's pr-wait (MIT, Theo Browne; NOTICE.md). [Exit codes](troubleshooting.md#gstack-pr-wait). | | `gstack-review-bots` | **Review-bot comments** — `fetch [--pr ]` reads every root line comment, conversation comment and newest review body from each bot in `pr_wait_gating_bots` and `pr_wait_advisory_bots` (Bot accounts only, all pages, through `gh`) into a private directory with per-bot counts, or prints `REVIEW_BOTS: skip` or `REVIEW_BOTS: error` (never a false zero). `record --dir --id --type fix\|fp\|already-fixed --category ` appends one triage outcome to the project and global `review-bot-history.md`, taking the bot and file from the fetched comment. `/review` Step 2.5 and `/ship` Step 10 run it. [Triage reference](../review/greptile-triage.md). | | `gstack-issue-guard` | **Tracker-text trust envelope** — fetches GitHub issue/PR text (`issue `, `pr-body`, `pr-comments`, or `--stdin`) and wraps it in a labeled envelope so agents treat it as data: injection-shaped lines get labeled even through fullwidth and invisible-character evasion, and forged envelope banners are defused. Every tracker-text ingress in gstack routes through it, enforced by a CI scanner. | | `gstack-ios-qa-daemon` | **iOS QA daemon** — Mac-side broker between an agent and a connected iPhone or iPad over USB CoreDevice. Loopback by default; `--tailnet` opens a Tailscale-facing listener with identity-gated capability tiers. Single-instance via flock on `~/.gstack/ios-qa-daemon.pid`. See [docs/howto-ios-testing-with-gstack.md](howto-ios-testing-with-gstack.md). | diff --git a/land-and-deploy/SKILL.md b/land-and-deploy/SKILL.md index 2243d5140..0ad025054 100644 --- a/land-and-deploy/SKILL.md +++ b/land-and-deploy/SKILL.md @@ -407,7 +407,7 @@ gh run list --repo "$REPO" --branch "$BASE_BRANCH" --limit 10 --json databaseId, Match `DEPLOY_SHA`, workflow and target environment. If no run appears yet, repeat the lookup within the same 20-minute deadline. A name match on another SHA is not evidence. -Poll every 30 seconds: +Poll every 60 seconds: ```bash gh run view --repo "$REPO" --json status,conclusion ``` @@ -594,7 +594,7 @@ Recheck read-only evidence; never redo a merge/deploy because a section was miss - Never force-push, bypass CI, replay a confirmed merge, or hide missing evidence. - Auto-detect facts; ask when unknown or an explicit approval gate applies. -- Step 3 is one `gstack-pr-wait` call; elsewhere poll at the stated interval (30 s for merge readback and deploy) with progress lines. +- Step 3 is one `gstack-pr-wait` call; elsewhere poll at the stated interval (60 s for merge readback and deploy) with progress lines. - After merge failures, offer approved rollback when appropriate; never revert a rollback automatically. - Verify once; `/canary` does extended monitoring; rechecks need the user's choice. - Use `--delete-branch`; reconcile failed cleanup non-destructively with confirmation. diff --git a/land-and-deploy/SKILL.md.tmpl b/land-and-deploy/SKILL.md.tmpl index e38c32dea..a4ec02a53 100644 --- a/land-and-deploy/SKILL.md.tmpl +++ b/land-and-deploy/SKILL.md.tmpl @@ -255,7 +255,7 @@ gh run list --repo "$REPO" --branch "$BASE_BRANCH" --limit 10 --json databaseId, Match `DEPLOY_SHA`, workflow and target environment. If no run appears yet, repeat the lookup within the same 20-minute deadline. A name match on another SHA is not evidence. -Poll every 30 seconds: +Poll every 60 seconds: ```bash gh run view --repo "$REPO" --json status,conclusion ``` @@ -441,7 +441,7 @@ Recheck read-only evidence; never redo a merge/deploy because a section was miss - Never force-push, bypass CI, replay a confirmed merge, or hide missing evidence. - Auto-detect facts; ask when unknown or an explicit approval gate applies. -- Step 3 is one `gstack-pr-wait` call; elsewhere poll at the stated interval (30 s for merge readback and deploy) with progress lines. +- Step 3 is one `gstack-pr-wait` call; elsewhere poll at the stated interval (60 s for merge readback and deploy) with progress lines. - After merge failures, offer approved rollback when appropriate; never revert a rollback automatically. - Verify once; `/canary` does extended monitoring; rechecks need the user's choice. - Use `--delete-branch`; reconcile failed cleanup non-destructively with confirmation. diff --git a/land-and-deploy/sections/merge-and-deploy.md b/land-and-deploy/sections/merge-and-deploy.md index 46bd99a10..e895eed07 100644 --- a/land-and-deploy/sections/merge-and-deploy.md +++ b/land-and-deploy/sections/merge-and-deploy.md @@ -164,7 +164,7 @@ merge queue is in use. Explain which is observed; an auto request alone does not prove a queue. A queue reruns CI against the proposed merge. Record `MERGE_PATH=queue` only when `mergeQueueEntry` was observed, otherwise `auto`. -Set `WAITED=true`. Repeat the readback every 30 seconds, up to 30 minutes; report progress every 2 minutes. +Set `WAITED=true`. Repeat the readback every 60 seconds, up to 30 minutes; report progress every 2 minutes. While OPEN with an active auto request **or** queue entry, keep waiting. Once waiting has begun, never dispatch START or DIRECT: OPEN with confirmed absence of **both** means removal/cancellation, so **STOP** and point to GitHub's checks/queue page. diff --git a/land-and-deploy/sections/merge-and-deploy.md.tmpl b/land-and-deploy/sections/merge-and-deploy.md.tmpl index c3c69f717..a4de77cd3 100644 --- a/land-and-deploy/sections/merge-and-deploy.md.tmpl +++ b/land-and-deploy/sections/merge-and-deploy.md.tmpl @@ -162,7 +162,7 @@ merge queue is in use. Explain which is observed; an auto request alone does not prove a queue. A queue reruns CI against the proposed merge. Record `MERGE_PATH=queue` only when `mergeQueueEntry` was observed, otherwise `auto`. -Set `WAITED=true`. Repeat the readback every 30 seconds, up to 30 minutes; report progress every 2 minutes. +Set `WAITED=true`. Repeat the readback every 60 seconds, up to 30 minutes; report progress every 2 minutes. While OPEN with an active auto request **or** queue entry, keep waiting. Once waiting has begun, never dispatch START or DIRECT: OPEN with confirmed absence of **both** means removal/cancellation, so **STOP** and point to GitHub's checks/queue page. diff --git a/lib/ci-carry.ts b/lib/ci-carry.ts index aed0a9233..68058d17f 100644 --- a/lib/ci-carry.ts +++ b/lib/ci-carry.ts @@ -23,12 +23,13 @@ * suite as before. Security and advisory scans and the sealed guard are never * carried: callers carry test lanes only. * - * waitForRun polls the newest run of one workflow on HEAD until it completes, - * bounded by a timeout, and returns green, red with the failing jobs, still - * running, or none. + * waitForRun polls the newest run of one workflow on HEAD (REST) until it + * completes, at least 60 s apart (lib/gh-poll.ts), bounded by a timeout, and + * returns green, red with the failing jobs, still running, or none. */ import * as fs from 'node:fs'; import * as path from 'node:path'; +import { pollDelayMs } from './gh-poll'; import { normalPath, runChanged, runInventory, runReceiptFields, uncoveredFiles, verifyRun, type ApiRun, type RunDeps, type RunEvidence, type RunProfile, type RunSource } from './run-evidence'; export type CarryRefusal = 'dirty_tree' | 'no_profile' | 'no_run' | 'no_verified_run' | 'tree_mismatch' | 'no_tests' @@ -136,8 +137,11 @@ export function findCarry(src: RunSource, o: { profiles: string[]; head: string; export type WaitVerdict = { state: 'green'; run: ApiRun } | { state: 'red'; run: ApiRun; failing: string[] } | { state: 'running'; run: ApiRun } | { state: 'none'; reason: string }; const PASSING = ['success', 'skipped', 'neutral']; -/** Poll the newest `workflow` run on `head` (among `events`) until it completes or `timeoutMs` passes. */ -export async function waitForRun(deps: RunDeps, o: { workflow: string; events: string[]; head: string; timeoutMs: number; intervalMs: number; sleep: (ms: number) => Promise; clock: () => number }): Promise { +/** + * Poll the newest `workflow` run on `head` (among `events`) until it completes or `timeoutMs` passes; polls are at least 60 s + * apart, so the last poll can land up to one poll interval (75 s with jitter) after `timeoutMs`. + */ +export async function waitForRun(deps: RunDeps, o: { workflow: string; events: string[]; head: string; timeoutMs: number; intervalMs: number; sleep: (ms: number) => Promise; clock: () => number; random?: () => number }): Promise { const end = o.clock() + o.timeoutMs; for (;;) { const newest = deps.runsForHead(o.workflow, o.head).filter(r => o.events.includes(r.event)).sort(newestFirst)[0]; @@ -147,7 +151,7 @@ export async function waitForRun(deps: RunDeps, o: { workflow: string; events: s return { state: 'red', run: newest, failing }; } if (o.clock() >= end) return newest ? { state: 'running', run: newest } : { state: 'none', reason: `no ${o.workflow} run on ${o.head.slice(0, 12)} after ${Math.round(o.timeoutMs / 1000)} s` }; - await o.sleep(Math.max(1, Math.min(o.intervalMs, end - o.clock()))); + await o.sleep(pollDelayMs(o.intervalMs, o.random)); } } diff --git a/lib/ci-gate.ts b/lib/ci-gate.ts index 006a998ca..e2b666133 100644 --- a/lib/ci-gate.ts +++ b/lib/ci-gate.ts @@ -1,4 +1,5 @@ import { spawnSync } from 'node:child_process'; +import { MIN_POLL_MS, pollDelayMs, syncLimitEnv, withRateLimit } from './gh-poll'; export type CiVerdict = 'PASS' | 'FAIL' | 'PENDING' | 'NO_CHECKS' | 'ERROR'; export const CI_GATE_EXIT: Record = { PASS: 0, FAIL: 1, PENDING: 2, NO_CHECKS: 3, ERROR: 4 }; @@ -57,10 +58,12 @@ Options: --override-head SHA head the exclusions were approved for; must equal --expect-head --wait SECONDS keep polling while PENDING, up to SECONDS (default 0) - --interval SECONDS poll interval (default 30) + --interval SECONDS poll interval (default and minimum 60, plus jitter; + a smaller value is raised to 60) --registration-wait S when no check exists yet, re-poll up to S seconds before reporting NO_CHECKS (default 0; use 60 right after a push, when checks may not be registered yet) + A GitHub rate limit waits for its reset, never retries at once (GST-1713). --help show this help`; export function bucketClass(bucket: string): Klass { @@ -201,6 +204,7 @@ export async function runCiGate( gh: GhRunner, sleep: (ms: number) => Promise = ms => new Promise(resolve => setTimeout(resolve, ms)), now: () => number = Date.now, + random: () => number = Math.random, ): Promise { if (opts.excludes.length > 0 && opts.overrideHead !== opts.expectHead) { return { @@ -210,13 +214,12 @@ export async function runCiGate( } const started = now(); const elapsed = () => (now() - started) / 1000; + const step = Math.max(opts.intervalSeconds, MIN_POLL_MS / 1000); let result = evaluateOnce(gh, opts); while (true) { - if (result.verdict === 'NO_CHECKS' && elapsed() + Math.min(opts.intervalSeconds, 10) <= opts.registrationWaitSeconds) { - await sleep(Math.min(opts.intervalSeconds, 10) * 1000); - } else if (result.verdict === 'PENDING' && elapsed() + opts.intervalSeconds <= opts.waitSeconds) { - await sleep(opts.intervalSeconds * 1000); - } else break; + const window = result.verdict === 'NO_CHECKS' ? opts.registrationWaitSeconds : result.verdict === 'PENDING' ? opts.waitSeconds : 0; + if (elapsed() >= window) break; + await sleep(pollDelayMs(step * 1000, random)); result = evaluateOnce(gh, opts); } if (result.verdict === 'NO_CHECKS') { @@ -230,7 +233,7 @@ export async function runCiGate( export function parseCiGateArgs(argv: string[]): CiGateOptions | { help: true } | { error: string; expectHead?: string } { const opts: CiGateOptions = { - repo: '', pr: '', expectHead: '', excludes: [], waitSeconds: 0, intervalSeconds: 30, registrationWaitSeconds: 0, + repo: '', pr: '', expectHead: '', excludes: [], waitSeconds: 0, intervalSeconds: MIN_POLL_MS / 1000, registrationWaitSeconds: 0, }; const seconds = (value: string | undefined, flag: string, positive = false): number | string => { if (value === undefined || !/^\d+(?:\.\d+)?$/.test(value)) return `${flag} needs a number of seconds`; @@ -270,14 +273,15 @@ export function parseCiGateArgs(argv: string[]): CiGateOptions | { help: true } return opts; } -export const spawnGh: GhRunner = args => { +/** gh as a GhRunner, with gh-poll's rate-limit rule: a rate-limited call sleeps until the reset instead of failing or retrying at once. */ +export const spawnGh: GhRunner = withRateLimit(args => { const result = spawnSync('gh', args, { encoding: 'utf8', timeout: 60_000, maxBuffer: 16 * 1024 * 1024 }); if (result.error) { const code = (result.error as NodeJS.ErrnoException).code; return { status: null, stdout: '', stderr: result.stderr ?? '', error: code === 'ENOENT' ? 'gh is not installed or not on PATH' : `gh failed to run (${code ?? result.error.message})` }; } return { status: result.status, stdout: result.stdout, stderr: result.stderr }; -}; +}, syncLimitEnv); export async function ciGateMain(argv: string[], gh: GhRunner = spawnGh, write: (text: string) => void = text => process.stdout.write(text)): Promise { const parsed = parseCiGateArgs(argv); diff --git a/lib/gh-poll.ts b/lib/gh-poll.ts new file mode 100644 index 000000000..4a960b8e8 --- /dev/null +++ b/lib/gh-poll.ts @@ -0,0 +1,180 @@ +/** + * gh-poll: the one policy for how gstack reads GitHub repeatedly (GST-1713). + * + * The Capy GitHub app installation shares one GraphQL budget across every + * crew, and `gh pr view|list|checks` spend it. Three rules live here: + * + * pollDelayMs every wait between two GitHub reads is at least + * MIN_POLL_MS (60 s) plus up to 15 s of jitter, whatever + * interval a caller or flag asks for. + * rate limits a gh call that fails with a rate-limit message (403/429, + * "API rate limit exceeded", "GraphQL: API rate limit", + * a secondary limit) is never retried at once: the helper + * reads `gh api rate_limit` (exempt from the limit), sleeps + * until that resource resets (at least 60 s), and retries at + * most MAX_LIMIT_WAITS times before returning the failure. + * REST reads pull request state and head-commit checks come from REST + * (`repos/{o}/{r}/pulls/{n}`, `commits/{sha}/check-runs`, + * `commits/{sha}/status`), which never touch the GraphQL budget. + * + * test/gh-poll-tripwire.test.ts keeps every sleep in a GitHub-reading module + * on pollDelayMs. GSTACK_GH_POLL_FLOOR_MS is a test seam only (fixtures that + * drive real binaries with millisecond polls); no shipped file may set it. + */ +import { spawnSync, type SpawnSyncOptionsWithStringEncoding, type SpawnSyncReturns } from 'node:child_process'; +import type { GhResult, GhRunner } from './ci-gate'; + +export const MIN_POLL_MS = 60_000; +export const MAX_LIMIT_WAITS = 2; +/** The longest a rate-limit wait may last: one full hourly window plus slack. */ +const MAX_LIMIT_WAIT_MS = 65 * 60_000; +const RESET_SLACK_MS = 5_000; + +type Vars = Record; + +function floorMs(vars: Vars): number { + const seam = vars.GSTACK_GH_POLL_FLOOR_MS; + return seam !== undefined && /^\d+$/.test(seam) ? Number(seam) : MIN_POLL_MS; +} + +/** The wait before the next GitHub read: at least the floor (60 s), plus jitter up to a quarter of the floor. */ +export function pollDelayMs(requestedMs: number, random: () => number = Math.random, vars: Vars = process.env): number { + const floor = floorMs(vars); + return Math.max(Number.isFinite(requestedMs) ? requestedMs : 0, floor) + Math.round(random() * floor / 4); +} + +export type RateLimitKind = 'graphql' | 'core' | 'secondary'; + +const SECONDARY = /secondary rate limit|abuse detection/i; +const GRAPHQL_LIMIT = /GraphQL: API rate limit|"type":\s*"RATE_LIMITED"/i; +const ANY_LIMIT = /API rate limit (?:already )?exceeded|rate limit exceeded|HTTP 429|\(HTTP 403\).*rate limit|rate limit.*\(HTTP 403\)/i; +/** gh subcommands that read through GraphQL; everything else (gh api , gh run, gh workflow, gh release) is REST. */ +const GRAPHQL_COMMANDS = new Set(['pr', 'issue', 'repo', 'search', 'project', 'label']); + +/** Which budget a failed gh call ran out of, from its output, or null when the failure is not a rate limit. */ +export function rateLimitKind(args: readonly string[], output: string): RateLimitKind | null { + if (SECONDARY.test(output)) return 'secondary'; + if (GRAPHQL_LIMIT.test(output)) return 'graphql'; + if (!ANY_LIMIT.test(output)) return null; + return (args[0] === 'api' && args[1] === 'graphql') || GRAPHQL_COMMANDS.has(args[0] ?? '') ? 'graphql' : 'core'; +} + +/** + * How long to sleep after a rate-limit failure: until the resource in `gh api rate_limit` resets (plus slack) when it + * reads remaining 0, else (a secondary limit, an unreadable reply) the poll floor. Never less than pollDelayMs(0). + */ +export function rateLimitWaitMs(kind: RateLimitKind, rateLimitJson: string, nowMs: number, random: () => number = Math.random, vars: Vars = process.env): number { + let untilReset = 0; + if (kind !== 'secondary') { + try { + const resource = (JSON.parse(rateLimitJson) as { resources?: Record }).resources?.[kind]; + if (resource && resource.remaining === 0 && typeof resource.reset === 'number') untilReset = resource.reset * 1000 - nowMs + RESET_SLACK_MS; + } catch { /* unreadable: fall back to the floor */ } + } + return pollDelayMs(Math.min(Math.max(untilReset, 0), MAX_LIMIT_WAIT_MS), random, vars); +} + +export interface LimitEnv { sleep(ms: number): void; now(): number; log(message: string): void; random?: () => number; vars?: Vars } + +/** + * `gh pr merge` runs under the land lock, which another lander may break once the holder is 10 minutes into a merge + * (PUSH_STALE_MS): a rate-limited merge fails at once, as it did before the wait existed, instead of sleeping through it. + */ +const failsFast = (args: readonly string[]): boolean => args[0] === 'pr' && args[1] === 'merge'; + +/** + * Wrap a gh runner: a rate-limited failure sleeps until the reset and reruns the same call, at most MAX_LIMIT_WAITS + * times; any other result (success or a real failure) and a rate-limited `gh pr merge` return at once. The + * `gh api rate_limit` read goes through the inner runner, unwrapped. + */ +export function withRateLimit(gh: GhRunner, env: LimitEnv): GhRunner { + return args => { + for (let waits = 0; ; waits++) { + const result = gh(args); + const kind = result.status === 0 ? null : rateLimitKind(args, `${result.stderr}\n${result.stdout}`); + if (!kind || waits >= MAX_LIMIT_WAITS || failsFast(args)) return result; + const limits = gh(['api', 'rate_limit']); + const ms = rateLimitWaitMs(kind, limits.status === 0 ? limits.stdout : '', env.now(), env.random, env.vars); + env.log(`GitHub ${kind} rate limit on gh ${args.slice(0, 2).join(' ')}; waiting ${Math.round(ms / 1000)} s for the reset before one retry (${waits + 1}/${MAX_LIMIT_WAITS})`); + env.sleep(ms); + } + }; +} + +/** The CLI environment: a blocking sleep (gh runners are synchronous) and the wait note on stderr. */ +export const syncLimitEnv: LimitEnv = { + sleep: ms => { Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); }, + now: () => Date.now(), + log: message => { process.stderr.write(`gstack-gh: ${message}\n`); }, +}; + +/** Drop-in for `spawnSync('gh', args, options)` with the rate-limit rule; the wait note goes to stderr. */ +export function ghSpawnSync(args: string[], options: SpawnSyncOptionsWithStringEncoding): SpawnSyncReturns { + let last: SpawnSyncReturns | undefined; + const run = withRateLimit(a => { + const r = spawnSync('gh', a, options); + if (a === args) last = r; + return { status: r.status, stdout: r.stdout ?? '', stderr: r.stderr ?? '', ...(r.error ? { error: String(r.error.message) } : {}) }; + }, syncLimitEnv); + run(args); + return last!; +} + +type Json = Record; +const obj = (value: unknown): Json => (value && typeof value === 'object' && !Array.isArray(value) ? (value as Json) : {}); +const str = (value: unknown): string => (typeof value === 'string' ? value : ''); + +/** A pull request's state from REST `repos/{o}/{r}/pulls/{n}`, in GraphQL's spellings (MERGED/CLOSED/OPEN, BEHIND/DIRTY/...). */ +export interface PullState { url: string; state: string; headRefOid: string; baseRefName: string; mergeStateStatus: string; author: string; updatedAt: string } + +export function parsePullState(json: string): PullState | null { + let raw: Json; + try { raw = obj(JSON.parse(json)); } catch { return null; } + const head = str(obj(raw.head).sha); + if (!head) return null; + return { + url: str(raw.html_url), + state: raw.merged === true || str(raw.merged_at) ? 'MERGED' : str(raw.state) === 'closed' ? 'CLOSED' : 'OPEN', + headRefOid: head, + baseRefName: str(obj(raw.base).ref), + mergeStateStatus: (str(raw.mergeable_state) || 'unknown').toUpperCase(), + author: str(obj(raw.user).login).replace(/\[bot\]$/, '') || 'ghost', + updatedAt: str(raw.updated_at), + }; +} + +/** One check on a commit in `gh pr checks` bucket spelling: pass, fail, pending, skipping, neutral, cancel. */ +export interface HeadCheck { name: string; bucket: string; link: string; description: string } + +const CONCLUSION_BUCKET: Record = { + success: 'pass', failure: 'fail', timed_out: 'fail', action_required: 'fail', startup_failure: 'fail', + cancelled: 'cancel', skipped: 'skipping', neutral: 'skipping', +}; +const STATUS_BUCKET: Record = { success: 'pass', failure: 'fail', error: 'fail', pending: 'pending' }; + +/** + * Every check on a commit from the REST `check_runs` and combined-status `statuses` entries, in `gh pr checks` + * buckets: the newest run per check suite and name (a re-run attempt supersedes the run it replaced; two workflow runs + * of one job, push and pull_request, stay two rows for the caller to collapse) and every status context. An unknown + * conclusion keeps its own spelling, which every bucket reader counts as failed. + */ +export function parseHeadChecks(runs: readonly unknown[], statuses: readonly unknown[]): HeadCheck[] { + const newest = new Map(); + const rank = (r: Json) => (str(r.status) !== 'completed' ? Infinity : Date.parse(str(r.completed_at)) || 0); + for (const r of runs.map(obj)) { + const key = `${String(obj(r.check_suite).id ?? obj(r.app).id ?? '')}\0${str(r.name)}`; + const have = newest.get(key); + if (!have || rank(r) > rank(have) || (rank(r) === rank(have) && Number(r.id) > Number(have.id))) newest.set(key, r); + } + const checks: HeadCheck[] = [...newest.values()].filter(r => str(r.name)).map(r => ({ + name: str(r.name), + bucket: str(r.status) !== 'completed' ? 'pending' : CONCLUSION_BUCKET[str(r.conclusion)] ?? (str(r.conclusion) || 'pending'), + link: str(r.details_url) || str(r.html_url), + description: str(obj(r.output).title), + })); + for (const s of statuses.map(obj)) { + if (!str(s.context)) continue; + checks.push({ name: str(s.context), bucket: STATUS_BUCKET[str(s.state)] ?? str(s.state), link: str(s.target_url), description: str(s.description) }); + } + return checks; +} diff --git a/lib/greptile-early.ts b/lib/greptile-early.ts index 3707c9971..ad641883e 100644 --- a/lib/greptile-early.ts +++ b/lib/greptile-early.ts @@ -14,8 +14,8 @@ * open --base B --title-file F [--ready] [--confirm T] * open the early PR through gstack-post and post one * `@greptileai` comment when Greptile skips drafts - * wait --since poll (30 s) for a completed Greptile review of the - * PR's head commit; returns within ~90 s per call, + * wait --since poll (60 s, REST only) for a completed Greptile review + * of the PR's head commit; returns within ~90 s per call, * `timeout` once 10 minutes have passed since * * bin/gstack-greptile-early is the CLI; tests drive `runGreptileEarly` with a @@ -26,6 +26,7 @@ import * as path from "path"; import { resolveStateRoot, readConfigKey, type StateRootEnv } from "./state-root"; import { botLists, botOwning } from "./pr-wait"; import { remoteSlug } from "./remote-identity"; +import { MIN_POLL_MS, pollDelayMs } from "./gh-poll"; export interface EarlyEnv { run(cmd: string, args: string[], input?: string): { status: number; stdout: string; stderr: string }; @@ -41,7 +42,7 @@ export interface EarlyEnv { } export const WAIT_CAP_SECONDS = 600; -export const POLL_SECONDS = 30; +export const POLL_SECONDS = MIN_POLL_MS / 1000; const CALL_BUDGET_SECONDS = 90; const COMMENT_WINDOW_DAYS = 90; const TRIGGER = "@greptileai"; @@ -228,7 +229,7 @@ function wait(env: EarlyEnv, args: string[]): number { const pr = args[0]; const since = Number(flag(args, "--since")); if (!pr || !/^[1-9][0-9]*$/.test(pr) || !Number.isInteger(since) || since <= 0) { env.err("usage: gstack-greptile-early wait --since \n"); return 64; } - const head = env.run("gh", ["pr", "view", pr, "--json", "headRefOid", "--jq", ".headRefOid"]); + const head = env.run("gh", ["api", `repos/{owner}/{repo}/pulls/${pr}`, "--jq", ".head.sha"]); const sha = head.stdout.trim(); if (head.status !== 0 || !/^[0-9a-f]{40}$/.test(sha)) { env.out("GREPTILE_REVIEW: unavailable (could not read the PR head commit)\n"); return 0; } const started = env.now(); @@ -240,7 +241,7 @@ function wait(env: EarlyEnv, args: string[]): number { const mmss = `${Math.floor(elapsed / 60)}m${String(elapsed % 60).padStart(2, "0")}s`; env.out(`Greptile: waiting for its review of ${sha.slice(0, 7)} (${mmss} of ${WAIT_CAP_SECONDS / 60}m). Turn early review off: ${config(env)} set ship_greptile_early false\n`); if ((env.now() - started) / 1000 + POLL_SECONDS > CALL_BUDGET_SECONDS) { env.out("GREPTILE_REVIEW: pending\n"); return 0; } - env.sleep(POLL_SECONDS * 1000); + env.sleep(pollDelayMs(POLL_SECONDS * 1000)); } } diff --git a/lib/pr-wait-github.ts b/lib/pr-wait-github.ts index e999e9cf5..9f01054bd 100644 --- a/lib/pr-wait-github.ts +++ b/lib/pr-wait-github.ts @@ -1,10 +1,11 @@ /** * pr-wait-github — the GitHub read side of gstack-pr-wait: a gh runner that - * retries transient failures, the one GraphQL read of a pull request, its - * checks from `gh pr checks`, the required-check names from rulesets and - * classic branch protection, and the base-branch comparisons. The event logic - * is lib/pr-wait.ts; the two are split so each stays under the module-size - * ratchet. + * retries transient failures (never a rate limit, GST-1713), the pull + * request's state, base head and head-commit checks from REST, the one + * GraphQL read of its conversation (comments, reviews, review threads), the + * required-check names from rulesets and classic branch protection, and the + * base-branch comparisons. The event logic is lib/pr-wait.ts; the two are + * split so each stays under the module-size ratchet. * * Design ported from tstack's `pr-wait` (t3dotgg/tstack, * skills/babysit-pr/scripts/pr-wait at 7de017b, MIT, Copyright (c) 2026 Theo @@ -12,6 +13,7 @@ * gstack's gh runner shape (lib/ci-gate.ts). */ import type { GhResult, GhRunner } from './ci-gate'; +import { parseHeadChecks, parsePullState, rateLimitKind, type PullState } from './gh-poll'; export class GhError extends Error { constructor(message: string, readonly transient: boolean) { @@ -32,22 +34,12 @@ export interface Actor { login: string; bot: boolean } export interface Comment { id: string; url: string; body: string; author: Actor } export interface Review { id: string; url: string; state: string; body: string; commitOid: string | null; author: Actor } export interface Thread { isResolved: boolean; path: string; line: number | null; comments: Comment[] } -export interface PullRequest { - url: string; - state: string; - headRefOid: string; - baseRefName: string; - mergeStateStatus: string; - baseOid: string | null; - author: string; - viewer: string; - comments: Comment[]; - reviews: Review[]; - threads: Thread[]; -} +/** What people said on the PR: the one GraphQL read, which REST cannot answer (review-thread resolution). */ +export interface Conversation { viewer: string; comments: Comment[]; reviews: Review[]; threads: Thread[] } +export interface PullRequest extends PullState, Conversation { baseOid: string | null } export interface Check { name: string; bucket: string; link: string; description: string } -const TRANSIENT = /HTTP (5\d\d|429)|rate limit|timed? ?out|connection (reset|refused)|EOF|temporar|could not resolve host|TLS handshake|Something went wrong/i; +const TRANSIENT = /HTTP 5\d\d|timed? ?out|connection (reset|refused)|EOF|temporar|could not resolve host|TLS handshake|Something went wrong/i; const RETRIES = 5; const PASS_BUCKETS = new Set(['pass', 'skipping', 'neutral']); /** Worst bucket first, for collapsing duplicate check names (push and pull_request runs of one job). */ @@ -58,9 +50,6 @@ query($owner: String!, $name: String!, $number: Int!) { viewer { login } repository(owner: $owner, name: $name) { pullRequest(number: $number) { - url state headRefOid baseRefName mergeStateStatus - author { login } - baseRef { target { oid } } comments(last: 100) { nodes { id url body author { login __typename } } } reviews(last: 100) { nodes { id url state body commit { oid } author { login __typename } } @@ -85,7 +74,11 @@ function lastLine(text: string): string { return text.trim().split(/\r?\n/).filter(Boolean).at(-1) ?? ''; } -/** Run gh. A transient failure (HTTP 5xx or 429, network) is retried with doubling backoff. */ +/** + * Run gh. A transient failure (HTTP 5xx, network) is retried with doubling backoff. A rate limit is never retried + * here: it throws a transient GhError at once, and the poll loop waits its full interval (the production runner, + * lib/gh-poll.ts withRateLimit, has already slept until the reset). + */ export function runGh(env: GhEnv, args: string[], allow: number[] = [0]): GhResult { let delay = 2_000; for (let attempt = 1; ; attempt++) { @@ -93,11 +86,13 @@ export function runGh(env: GhEnv, args: string[], allow: number[] = [0]): GhResu // gh missing is fatal; any other spawn failure (its 60 s timeout, a signal) is a slow API, so retry. if (result.error && /not installed/.test(result.error)) throw new GhError(result.error, false); const status = result.status ?? -1; + if (status !== 0 && rateLimitKind(args, `${result.stderr}\n${result.stdout}`)) throw new GhError(lastLine(result.stderr) || 'GitHub rate limit', true); const transient = status !== 0 && (result.error !== undefined || result.status === null || TRANSIENT.test(result.stderr)); if (!transient && allow.includes(status)) return result; const message = result.error ?? (lastLine(result.stderr) || `gh exited ${status}`); if (!transient || attempt === RETRIES) throw new GhError(message, transient); env.log(`gh ${args.slice(0, 2).join(' ')}: ${message}; retry in ${delay / 1000}s`); + // gh-poll-exempt: a 5xx or network retry of one read (2, 4, 8, 16 s), never a rate limit (thrown above) env.sleep(delay); delay *= 2; } @@ -125,31 +120,43 @@ export function currentRepo(env: GhEnv): string { return runGh(env, ['repo', 'view', '--json', 'nameWithOwner', '--jq', '.nameWithOwner']).stdout.trim(); } -/** Confirms the PR exists and is readable; throws GhError otherwise. */ +/** Confirms the PR exists and is readable (REST); throws GhError otherwise. */ export function confirmPullRequest(env: GhEnv, repo: string, pr: number): void { - runGh(env, ['pr', 'view', String(pr), '-R', repo, '--json', 'number']); + runGh(env, ['api', `repos/${repo}/pulls/${pr}`, '--jq', '.number']); } -export function fetchPullRequest(env: GhEnv, repo: string, pr: number): PullRequest { +/** The PR's state, head, base and merge state from REST `pulls/{n}`: no GraphQL. */ +export function fetchPullState(env: GhEnv, repo: string, pr: number): PullState { + const state = parsePullState(runGh(env, ['api', `repos/${repo}/pulls/${pr}`]).stdout); + if (!state) throw new GhError(`${repo}#${pr} is not a pull request GitHub returned`, false); + return state; +} + +/** The base branch's current head commit (REST), or null when unreadable. */ +export function fetchBaseOid(env: GhEnv, repo: string, base: string): string | null { + try { + return runGh(env, ['api', `repos/${repo}/git/ref/heads/${base}`, '--jq', '.object.sha']).stdout.trim() || null; + } catch (error) { + if (error instanceof GhError && error.transient) throw error; + return null; + } +} + +/** The conversation: the one GraphQL read, made only when lib/pr-wait.ts decides it is stale. */ +export function fetchConversation(env: GhEnv, repo: string, pr: number): Conversation { const [owner, name] = repo.split('/'); const result = runGh(env, ['api', 'graphql', '-f', `query=${PR_QUERY}`, '-f', `owner=${owner}`, '-f', `name=${name}`, '-F', `number=${pr}`]); let parsed: Json; try { parsed = obj(JSON.parse(result.stdout)); } catch { throw new GhError('gh api graphql printed output that is not JSON', false); } - const node = obj(obj(obj(parsed.data).repository).pullRequest); - if (!str(node.headRefOid)) { + const pullRequest = obj(obj(parsed.data).repository).pullRequest; + if (!pullRequest || typeof pullRequest !== 'object') { const errors = Array.isArray(parsed.errors) ? parsed.errors.map(e => str(obj(e).message)).filter(Boolean).join('; ') : ''; throw new GhError(errors || `${repo}#${pr} is not a pull request GitHub returned`, false); } + const node = obj(pullRequest); return { - url: str(node.url), - state: str(node.state), - headRefOid: str(node.headRefOid), - baseRefName: str(node.baseRefName), - mergeStateStatus: str(node.mergeStateStatus) || 'UNKNOWN', - baseOid: str(obj(obj(node.baseRef).target).oid) || null, - author: str(obj(node.author).login) || 'ghost', viewer: str(obj(obj(parsed.data).viewer).login), comments: nodes(node.comments).map(comment), reviews: nodes(node.reviews).map(r => ({ ...comment(r), state: str(r.state), commitOid: str(obj(r.commit).oid) || null })), @@ -162,29 +169,28 @@ export function fetchPullRequest(env: GhEnv, repo: string, pr: number): PullRequ }; } +/** Every entry of a paginated REST list, read with `--paginate --jq '.[]'` (one JSON value per line). */ +function restList(env: GhEnv, path: string, key: string): unknown[] { + const lines = runGh(env, ['api', '--paginate', path, '--jq', `.${key}[]`]).stdout.split(/\r?\n/).filter(Boolean); + try { return lines.map(line => JSON.parse(line)); } catch { + throw new GhError(`gh api ${path} printed output that is not JSON`, false); + } +} + /** - * Every check on the PR head by name, duplicates collapsed to the worst - * bucket. `gh pr checks` exits 8 while checks run and 1 when one failed: - * those are states, so the JSON is read whatever the exit status. + * Every check on the head commit by name, from REST check runs and commit + * statuses (the same rows `gh pr checks` reads through GraphQL), duplicates + * collapsed to the worst bucket. */ -export function fetchChecks(env: GhEnv, repo: string, pr: number): Map { - const result = runGh(env, ['pr', 'checks', String(pr), '-R', repo, '--json', 'name,bucket,link,description'], [0, 1, 8]); +export function fetchChecks(env: GhEnv, repo: string, head: string): Map { + const rows = parseHeadChecks( + restList(env, `repos/${repo}/commits/${head}/check-runs?per_page=100`, 'check_runs'), + restList(env, `repos/${repo}/commits/${head}/status?per_page=100`, 'statuses'), + ); const checks = new Map(); - if (!result.stdout.trim()) { - if (/no checks reported/.test(result.stderr)) return checks; - throw new GhError(lastLine(result.stderr) || `gh pr checks exited ${result.status}`, false); - } - let rows: unknown; - try { rows = JSON.parse(result.stdout); } catch { - throw new GhError('gh pr checks printed output that is not JSON', false); - } - if (!Array.isArray(rows)) throw new GhError('gh pr checks JSON is not a list', false); - for (const raw of rows.map(obj)) { - const name = str(raw.name); - if (!name) continue; - const row: Check = { name, bucket: str(raw.bucket) || 'pending', link: str(raw.link), description: str(raw.description) }; - const current = checks.get(name); - if (!current || RANK[bucketClass(row.bucket)] > RANK[bucketClass(current.bucket)]) checks.set(name, row); + for (const row of rows) { + const current = checks.get(row.name); + if (!current || RANK[bucketClass(row.bucket)] > RANK[bucketClass(current.bucket)]) checks.set(row.name, row); } return checks; } diff --git a/lib/pr-wait.ts b/lib/pr-wait.ts index cff55732e..9caebe325 100644 --- a/lib/pr-wait.ts +++ b/lib/pr-wait.ts @@ -16,9 +16,10 @@ import * as path from 'node:path'; import { createHash } from 'node:crypto'; import type { GhRunner } from './ci-gate'; import { - GhError, behindBy, bucketClass, compareFiles, confirmPullRequest, currentRepo, fetchChecks, fetchPullRequest, - pullRequestFiles, requiredChecks, type Check, type GhEnv, type PullRequest, + GhError, behindBy, bucketClass, compareFiles, confirmPullRequest, currentRepo, fetchBaseOid, fetchChecks, fetchConversation, + fetchPullState, pullRequestFiles, requiredChecks, type Check, type Conversation, type GhEnv, type PullRequest, } from './pr-wait-github'; +import { MIN_POLL_MS, pollDelayMs, type PullState } from './gh-poll'; import { readConfigKey, resolveStateRoot, type StateRootEnv } from './state-root'; import { resultCodeSuffix, resultLineWithDocs } from './result-codes'; @@ -31,7 +32,9 @@ const STATUS_BOTS = ['github-actions', 'vercel', 'netlify', 'railway', 'cloudfla /** A state file older than this describes an old session: start a fresh baseline instead of replaying it. */ export const STALE_STATE_MS = 6 * 3600 * 1000; const MAX_FAILED_POLLS = 10; -const MIN_INTERVAL_SECONDS = 5; +const MIN_INTERVAL_SECONDS = MIN_POLL_MS / 1000; +/** The conversation (the one GraphQL read) is reread when the PR's updated_at or head moves, and at least this often. */ +const CONVERSATION_REFRESH_MS = 10 * 60_000; /** Minutes before a watch that met nothing ends as PR_WAIT_BLOCKED, naming what is still pending; 0 waits without limit. */ export const DEFAULT_TIMEOUT_MINUTES = 60; @@ -77,6 +80,11 @@ comes from the URL or the current checkout. merge state is known (default) any-change stop after the first poll that prints an event +--interval seconds between polls: at least 60 (the default), plus up to 15 s + of jitter. Each poll reads REST (PR, checks, base head); the + GraphQL conversation read runs only when the PR changed, every + 10 minutes, or once green under --until ready. A rate limit waits + for its reset, never retries at once --timeout minutes before a watch that met nothing ends as PR_WAIT_BLOCKED, naming what is still pending (default 60; 0 waits without limit) --exclude check names (comma-separated, repeatable) left out of the verdict, @@ -115,7 +123,7 @@ lock, which the next watcher for the PR takes over once the pid is gone. Docs: docs/troubleshooting.md#gstack-pr-wait`; export function parsePrWaitArgs(argv: string[]): PrWaitOptions | { help: true } | { error: string } { - const opts: PrWaitOptions = { until: 'ready', interval: 20, timeout: DEFAULT_TIMEOUT_MINUTES, excludes: [] }; + const opts: PrWaitOptions = { until: 'ready', interval: MIN_INTERVAL_SECONDS, timeout: DEFAULT_TIMEOUT_MINUTES, excludes: [] }; for (let i = 0; i < argv.length; i++) { const flag = argv[i]; if (flag === '--help' || flag === '-h') return { help: true }; @@ -217,6 +225,7 @@ export class Watcher { private prevSnapshot: string | null = null; private prFiles = new Map>(); private pending: string[] = []; + private conversationCache: { key: string; at: number; value: Conversation } | null = null; private readonly gh: GhEnv; constructor(readonly env: PrWaitEnv, readonly repo: string, readonly pr: number, readonly until: Until, readonly interval: number, readonly bots: BotLists, readonly timeoutMinutes = DEFAULT_TIMEOUT_MINUTES, readonly excludes: readonly string[] = []) { @@ -450,12 +459,26 @@ export class Watcher { private dirtyLine(base: string, head: string): string { return `merge state is DIRTY on ${head.slice(0, 7)}: the branch conflicts with ${base}, rebase it`; } private behindLine(base: string, head: string): string { return `merge state is BEHIND on ${head.slice(0, 7)}: update the branch from ${base} before merging`; } + /** + * The conversation, reread through GraphQL only when the PR's updated_at or head moved, the cached read is ten + * minutes old, or `force` (a ready verdict is near, so review-thread resolution must be current). REST cannot read + * thread resolution; every other fact comes from REST each poll. + */ + private conversation(state: PullState, force: boolean): Conversation { + const key = `${state.headRefOid}|${state.updatedAt}`; + const cached = this.conversationCache; + if (cached && !force && cached.key === key && this.env.now() - cached.at < CONVERSATION_REFRESH_MS) return cached.value; + const value = fetchConversation(this.gh, this.repo, this.pr); + this.conversationCache = { key, at: this.env.now(), value }; + return value; + } + /** One poll. Returns an exit code when the watch ends, null to keep polling. Throws GhError. */ poll(): number | null { - const pr = fetchPullRequest(this.gh, this.repo, this.pr); - if (pr.state === 'MERGED') return this.finish(PR_WAIT_EXIT.met, [`merged: ${pr.url}`]); - if (pr.state === 'CLOSED') return this.finish(PR_WAIT_EXIT.blocked, [`closed without merging: ${pr.url}`]); - const base = pr.baseRefName; + const state = fetchPullState(this.gh, this.repo, this.pr); + if (state.state === 'MERGED') return this.finish(PR_WAIT_EXIT.met, [`merged: ${state.url}`]); + if (state.state === 'CLOSED') return this.finish(PR_WAIT_EXIT.blocked, [`closed without merging: ${state.url}`]); + const base = state.baseRefName; if (this.required === null) { const read = requiredChecks(this.gh, this.repo, base); this.required = read.names; @@ -471,12 +494,13 @@ export class Watcher { if (refused.length) this.log(`exclude refused for required check${refused.length === 1 ? '' : 's'} ${refused.join(', ')}: a required check gates`); } } - const checks = fetchChecks(this.gh, this.repo, this.pr); - const head = pr.headRefOid; + const head = state.headRefOid; + const checks = fetchChecks(this.gh, this.repo, head); const short = head.slice(0, 7); this.events = 0; const ci = this.evaluateChecks(checks, head); + const pr: PullRequest = { ...state, baseOid: fetchBaseOid(this.gh, this.repo, base), ...this.conversation(state, this.until === 'ready' && ci.green) }; const unresolved = this.emitConversation(pr, new Set([pr.author, pr.viewer])); const bots = this.emitBots(pr, checks, head); this.emitBaseAndMerge(pr, head); @@ -540,7 +564,7 @@ export class Watcher { this.log(`poll failed (${failures}/${MAX_FAILED_POLLS}): ${error.message}`); if (failures >= MAX_FAILED_POLLS) return this.finish(PR_WAIT_EXIT.blocked, [`gstack-pr-wait stopped: GitHub unreachable for ${failures} polls: ${error.message}`]); } - this.env.sleep(this.interval * 1000); + this.env.sleep(pollDelayMs(this.interval * 1000)); } } } diff --git a/lib/release-gate.ts b/lib/release-gate.ts index 0b3749abc..6069d2552 100644 --- a/lib/release-gate.ts +++ b/lib/release-gate.ts @@ -26,6 +26,7 @@ import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { DISPATCH_INPUTS, makeNonce } from '../scripts/eval-bg'; +import { pollDelayMs } from './gh-poll'; import { ghFailureReason, type GhRunner } from '../scripts/recover-receipts'; import { MAX_BISECT_CASES, MAX_PREFIXES, RELEASE_BISECT_ARTIFACT, RELEASE_BISECT_REF, type Git } from '../scripts/train-gate'; import type { ReleaseGatePolicy } from './release-gate-policy'; @@ -187,7 +188,7 @@ function follow(env: GateEnv, workflow: string, nonce: string, known: number | n if (run) lines.push(`RELEASE: run ${run.id} ${run.html_url ?? ''}`.trim()); } else if (run.status === 'completed') return run; if (env.now() >= env.deadline) break; - env.sleep(env.pollMs); + env.sleep(pollDelayMs(env.pollMs)); if (run) run = ghJson(env, ['api', `repos/${env.repo}/actions/runs/${run.id}`]) ?? run; } lines.push(`RELEASE: ${run ? `run ${run.id} is still ${run.status}` : `no run for nonce ${nonce} yet`} at the deadline; the record stays resumable`); diff --git a/lib/run-evidence.ts b/lib/run-evidence.ts index 76a1ce3ed..ceb295bb9 100644 --- a/lib/run-evidence.ts +++ b/lib/run-evidence.ts @@ -40,6 +40,7 @@ import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { sha256Hex, writeReceipt } from './egress-receipt'; +import { ghSpawnSync } from './gh-poll'; import { runBashSync } from './script-invocation'; /** @@ -307,7 +308,7 @@ const receiptFailed = (err: unknown) => process.stderr.write(`[run-evidence] egr function gh(args: string[], timeoutMs = 60_000): string { try { writeReceipt(receiptFor('api.github.com', 'actions-run-read', args.join(' '))); } catch (err) { receiptFailed(err); } - const r = spawnSync('gh', args, { encoding: 'utf8', timeout: timeoutMs, maxBuffer: 64 * 1024 * 1024 }); + const r = ghSpawnSync(args, { encoding: 'utf8', timeout: timeoutMs, maxBuffer: 64 * 1024 * 1024 }); if (r.status !== 0 || r.error) throw new Error(`gh ${args.slice(0, 2).join(' ')} failed: ${(r.stderr || String(r.error ?? '')).trim().slice(0, 300)}`); return r.stdout; } diff --git a/lib/ship-bin-common.ts b/lib/ship-bin-common.ts index 0a6787a07..8dca315f5 100644 --- a/lib/ship-bin-common.ts +++ b/lib/ship-bin-common.ts @@ -15,6 +15,7 @@ import { isConfigUnavailable, loadConfigFromBase } from './pregate/config'; import { parseUnifiedDiff, versionAtRef } from './restamp'; import { branchSides, stampOnlyLine } from './stamp-only'; import { DEFAULT_POLICY, matchesAny } from './ship-policy'; +import { ghSpawnSync } from './gh-poll'; import { extractVersion } from './version-source'; import { BIN_DIR, runBashSync, runScriptSync, scriptInvocation } from './script-invocation'; @@ -40,7 +41,7 @@ export function hostedOrigin(cwd: string): boolean { } /** gh with argv, never a shell, so a branch name cannot change what runs. */ export function gh(cwd: string, args: string[], timeout = 60_000): Run { - return done(spawnSync('gh', args, { cwd, encoding: 'utf8', timeout, maxBuffer: 64 * 1024 * 1024, env: { ...process.env, GIT_TERMINAL_PROMPT: '0' } })); + return done(ghSpawnSync(args, { cwd, encoding: 'utf8', timeout, maxBuffer: 64 * 1024 * 1024, env: { ...process.env, GIT_TERMINAL_PROMPT: '0' } })); } export function sh(cwd: string, command: string, timeout = 1_800_000): Run { return done(runBashSync(command, { cwd, encoding: 'utf8', timeout, maxBuffer: 64 * 1024 * 1024 })); diff --git a/lib/ship-queue-dispatch-ci.ts b/lib/ship-queue-dispatch-ci.ts index d099b75c7..2efa2a9ba 100644 --- a/lib/ship-queue-dispatch-ci.ts +++ b/lib/ship-queue-dispatch-ci.ts @@ -16,6 +16,7 @@ */ import * as fs from 'node:fs'; import * as path from 'node:path'; +import { pollDelayMs } from './gh-poll'; import type { PregateFile } from './headless-artifacts'; import { PREGATE_OUT_REL, clearObligation, readPregateFile, writePregateFile } from './pregate/index'; import type { QueueContext } from './ship-queue'; @@ -65,7 +66,7 @@ export async function dispatchCi(ctx: QueueContext, o: DispatchCiOptions): Promi run = run && slug ? runView(ctx, slug, run.databaseId) ?? run : runsAt(ctx, wf, branch, head)[0]; if (run?.status === 'completed') break; if (Date.now() >= deadline) return { lines: [...lines, `CI_DISPATCH: ${wf} --ref ${branch} ${run ? `run ${run.databaseId} ${run.url} ${run.status}` : 'no run appeared'}`], failure: { code: 'QUEUE_TRAIN_PENDING', message: `${wf} ${run ? `run ${run.databaseId} is still ${run.status}` : 'has no workflow_dispatch run at this head yet'} at the --dispatch-ci timeout; rerun ready --dispatch-ci to continue waiting` } }; - await sleep(o.pollMs); + await sleep(pollDelayMs(o.pollMs)); } lines.push(`CI_DISPATCH: ${wf} --ref ${branch} run ${run.databaseId} ${run.url} ${run.conclusion ?? 'unknown'}`); concluded.set(wf, run); diff --git a/lib/ship-queue-land-direct.ts b/lib/ship-queue-land-direct.ts index cdd19f9cc..360cf1ef9 100644 --- a/lib/ship-queue-land-direct.ts +++ b/lib/ship-queue-land-direct.ts @@ -13,6 +13,7 @@ * neither the base land checked nor in the pin), a land note on refs/notes/gstack-trains, and the lock's release. A * rerun after GitHub merged goes straight to the tree check. */ +import { MIN_POLL_MS, pollDelayMs } from './gh-poll'; import { runMergedChecks } from './land-merged-checks'; import { classifyRelease, RELEASE_BRANCH } from './release-stamp'; import { versionAtRef } from './restamp'; @@ -235,8 +236,8 @@ export function landDirect(ctx: QueueContext, o: DirectOptions): StepOutput { if (lost) return fail(lines, lost); const m = ctx.gh(['pr', 'merge', o.pr, method, '--match-head-commit', o.pin]); let merge: string | undefined; - for (let i = 0; i < (m.ok ? 4 : 1) && !isSha(merge); i++) { - if (i) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 3_000); + for (let i = 0; i < (m.ok ? 3 : 1) && !isSha(merge); i++) { + if (i) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, pollDelayMs(MIN_POLL_MS)); const after = view(ctx, o.pr); merge = 'code' in after || after.state !== 'MERGED' ? undefined : after.mergeCommit?.oid; } diff --git a/lib/ship-queue-land.ts b/lib/ship-queue-land.ts index 7bd7c09bd..390bef2fa 100644 --- a/lib/ship-queue-land.ts +++ b/lib/ship-queue-land.ts @@ -15,6 +15,7 @@ import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { mergeChangelogs, parseChangelog } from './changelog-check'; +import { MIN_POLL_MS, pollDelayMs } from './gh-poll'; import { fragmentSlug, readFragments } from './changelog-fragments'; import { restamp, versionAtRef } from './restamp'; import { acquireLandLock, lockLabel, readLock, writeLock, type LandLockPhase } from './ship-queue-land-lock'; @@ -212,7 +213,7 @@ function prInfo(ctx: QueueContext, pr: string): PrInfo | QueueFailure { export function waitChecks(ctx: QueueContext, sha: string, minutes: number): void { const deadline = Date.now() + minutes * 60_000; for (let ci = ciAtHead(ctx, sha); !ci.green && !ci.line.startsWith('red') && Date.now() < deadline; ci = ciAtHead(ctx, sha)) { - Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 30_000); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, pollDelayMs(MIN_POLL_MS)); } } diff --git a/lib/ship-queue-post-merge.ts b/lib/ship-queue-post-merge.ts index 61848d7a0..e3fc688b5 100644 --- a/lib/ship-queue-post-merge.ts +++ b/lib/ship-queue-post-merge.ts @@ -12,6 +12,7 @@ * commit pending, a failed one makes it red (named as a commit status when no * check run failed). Rerun it after an interruption and it continues. */ +import { pollDelayMs } from './gh-poll'; import { ciAtHead, type QueueContext } from './ship-queue'; import { outFailed, outOk, outPending, type TrainOut } from './ship-queue-train'; @@ -46,6 +47,6 @@ export async function verifyPostMerge(ctx: QueueContext, o: PostMergeOptions): P : ci.none ? `no check run has attached to ${at} yet` : settled ? `the post-merge checks the train gate names never ran at ${at}: ${absent.join(', ')}` : `main's push workflows at ${at} have not settled at the timeout: ${pending.join(', ') || 'a commit status'} still running`; return outPending({ code: 'QUEUE_TRAIN_PENDING', message }, lines, again, 'none'); } - await sleep(o.pollMs); + await sleep(pollDelayMs(o.pollMs)); } } diff --git a/lib/ship-queue-train-watch.ts b/lib/ship-queue-train-watch.ts index 922c2aab4..afdea767b 100644 --- a/lib/ship-queue-train-watch.ts +++ b/lib/ship-queue-train-watch.ts @@ -13,6 +13,7 @@ * --land lands each landable slot in order through train land, stopping at the first refusal with its code and NEXT. * It never merges without --land and never merges main into a PR. */ +import { pollDelayMs } from './gh-poll'; import type { QueueContext } from './ship-queue'; import { buildTrain, findTrain, nextFor, outOk, outPending, outRefused, trainEnv, trainStatus, type TrainOut } from './ship-queue-train'; import { repoSlug, runView } from './ship-queue-train-github'; @@ -36,7 +37,7 @@ async function awaitGate(ctx: QueueContext, t: TrainRecord, deadline: number, po const view = runView(ctx, slug, a.run.id); if (view?.status === 'completed') return null; if (Date.now() >= deadline) return `run ${a.run.id} is still ${view?.status ?? 'unreadable'} at the watch timeout`; - await sleep(pollMs); + await sleep(pollDelayMs(pollMs)); } } @@ -49,7 +50,7 @@ async function awaitPrefixRuns(ctx: QueueContext, t: TrainRecord, deadline: numb const running = open.filter(a => runView(ctx, slug, a.run!.id)?.status !== 'completed'); if (!running.length) return null; if (Date.now() >= deadline) return `${running.length} prefix run(s) still running at the watch timeout (${running.map(a => a.run!.id).join(', ')})`; - await sleep(pollMs); + await sleep(pollDelayMs(pollMs)); } } @@ -70,7 +71,7 @@ async function watchGeneric(ctx: QueueContext, env: TrainEnv, start: TrainRecord verdict = trainVerdict(ctx, { train: t.nonce }); reload(); if (!t.verdict && Date.now() >= deadline) return { ...verdict, lines: [...lines, ...verdict.lines], next: again }; - if (!t.verdict) await sleep(o.pollMs); + if (!t.verdict) await sleep(pollDelayMs(o.pollMs)); } verdict ??= trainVerdict(ctx, { train: t.nonce }); lines.push(...verdict.lines); diff --git a/lib/ship-queue-train.ts b/lib/ship-queue-train.ts index 2cc453be2..c6e71a3ef 100644 --- a/lib/ship-queue-train.ts +++ b/lib/ship-queue-train.ts @@ -18,6 +18,7 @@ import { randomBytes } from 'node:crypto'; import { DISPATCH_INPUTS, awaitNonceRun, dispatchWorkflow, listDispatchRuns, makeNonce, runForNonce, type WorkflowDispatch } from '../scripts/eval-bg'; import { carryReleasedParent, changelogDamage, mergeChangelogs, parseChangelog, type ChangelogSection } from './changelog-check'; import type { ResultStatus } from './headless-artifacts'; +import { pollDelayMs } from './gh-poll'; import { baseOverlapLine } from './pr-wait'; import { versionAtRef, versionFiles } from './restamp'; import type { LoadedPolicy, ShipPolicy, TrainPolicy } from './ship-policy'; @@ -495,7 +496,7 @@ async function freePhase(env: TrainEnv, slug: string, t: TrainRecord, o: BuildOp const deadline = Date.now() + o.freeTimeoutSec * 1000; let found = newestBranchRun(env.ctx, slug, env.train.free_workflow, t.branch); while ((found === null || found === 'unreadable' || found.status !== 'completed') && Date.now() < deadline) { - await sleep(o.pollMs); + await sleep(pollDelayMs(o.pollMs)); found = newestBranchRun(env.ctx, slug, env.train.free_workflow, t.branch); } if (found && found !== 'unreadable' && found.status !== 'completed') { diff --git a/lib/ship-release.ts b/lib/ship-release.ts index bef678d7b..bb9326237 100644 --- a/lib/ship-release.ts +++ b/lib/ship-release.ts @@ -29,6 +29,7 @@ import { parseReceiptDocs, type ReceiptDocs } from './ship-receipt-docs'; import { resolveStateRoot } from './state-root'; import { bumpVersion, cmpVersion, fmtVersion, parseVersion, versionWidth, type Bump } from './version-source'; import { waitForRun } from './ci-carry'; +import { MIN_POLL_MS } from './gh-poll'; import { lanesOnlyHeadPolicy } from './pregate/head-policy'; import { PREPUSH_CHECK_IDS } from './pregate/index'; import { buildRunCheck } from './pregate/run-check'; @@ -624,7 +625,7 @@ export async function accept(argv: readonly string[], cwd: string): Promise m.profiles))]) { const p = runCheck.profiles[name]; if (!p) return r.stop(`ship_lanes_from names ${name}, which is not a run_profiles entry on origin/${base}`); - const w = await waitForRun(runCheck.deps, { workflow: p.workflow, events: p.events, head, timeoutMs: waitS * 1000, intervalMs: 15_000, sleep: ms => new Promise(res => setTimeout(res, ms)), clock: Date.now }); + const w = await waitForRun(runCheck.deps, { workflow: p.workflow, events: p.events, head, timeoutMs: waitS * 1000, intervalMs: MIN_POLL_MS, sleep: ms => new Promise(res => setTimeout(res, ms)), clock: Date.now }); r.add(`CI_WAIT: ${w.state} ${name} ${'run' in w ? `run=${w.run.id}/${w.run.run_attempt}` : w.reason}`); if (w.state === 'running') return r.stop(`${p.workflow} run ${w.run.id} at ${head.slice(0, 12)} is still running: rerun accept (or gstack-ci-carry wait --base ${base} in the background, then accept)`); if (w.state === 'red') return r.stop(`${p.workflow} run ${w.run.id} failed (${short(w.failing, 4) || 'no failing job named'}): Step 5 triage on its logs; fix, commit, push, then accept again`); diff --git a/scripts/eval-bg.ts b/scripts/eval-bg.ts index bb44b1ae4..ac265acbb 100644 --- a/scripts/eval-bg.ts +++ b/scripts/eval-bg.ts @@ -28,6 +28,7 @@ import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { spawnSync } from 'node:child_process'; +import { pollDelayMs } from '../lib/gh-poll'; import { detectBaseBranch } from '../test/helpers/touchfiles'; import { DEFAULT_JOBS, ROOT, type PaidTier } from './lib/paid-types'; import { buildRunManifest, loadPaidTestDurations, recordedShardMs } from './lib/paid-plan'; @@ -243,7 +244,7 @@ export function dispatchWorkflow(dispatch: WorkflowDispatch, ref: string, run: E export async function awaitNonceRun(workflow: string, nonce: string, run: Exec, repo = CANONICAL_REPO): Promise { const deadline = Date.now() + 10 * pollMs(); while (Date.now() < deadline) { - await sleep(Math.min(pollMs(), 10_000)); + await sleep(pollDelayMs(pollMs())); const found = runForNonce(listDispatchRuns(workflow, run, repo) ?? [], nonce); if (found) return found; } @@ -335,7 +336,7 @@ async function followWorkflow(dispatch: WorkflowDispatch, ref: string, key: stri console.log(`[eval-bg] ${dispatch.workflow}: conclusion ${state.conclusion ?? 'unknown'} -> exit ${code}`); return code; } - await sleep(pollMs()); + await sleep(pollDelayMs(pollMs())); } } diff --git a/scripts/recover-receipts.ts b/scripts/recover-receipts.ts index 00a780a31..0ff5c7093 100644 --- a/scripts/recover-receipts.ts +++ b/scripts/recover-receipts.ts @@ -235,6 +235,7 @@ export function collectRecovery(options: CollectOptions): { checkpoint: Recovery const id = String(listedRun.id); let run = listedRun; while (run.status !== 'completed' && left() > (options.pollMs ?? 5_000)) { + // gh-poll-exempt: runs in GitHub Actions (evals.yml) on GITHUB_TOKEN, not the shared Capy installation sleep(options.pollMs ?? 5_000); const polled = gh(['api', `repos/${options.repo}/actions/runs/${id}`], left()); if (polled.status !== 0) break; diff --git a/scripts/train-backstop.ts b/scripts/train-backstop.ts index 5492ed4c4..ebc8f053d 100644 --- a/scripts/train-backstop.ts +++ b/scripts/train-backstop.ts @@ -123,6 +123,7 @@ function gateMain(o: Required> run = json(o.gh(['api', `repos/${o.repo}/actions/runs/${run.id}`], GH_MS)) ?? run; if (run.status === 'completed') return run; } + // gh-poll-exempt: runs in GitHub Actions (evals-main-backstop.yml) on GITHUB_TOKEN; default 60 s o.sleep(o.pollMs); } lines.push(`the gate ${run ? `run ${run.id}` : 'dispatch'} did not finish before the backstop deadline`); @@ -151,6 +152,7 @@ export function runBackstop(o: BackstopOptions): { action: BackstopAction; lines if (!merging.length) break; if (now() >= settleUntil) { lines.push(`BACKSTOP: train(s) ${merging.map(t => t.nonce).join(', ')} still merging after the settle window; gating anyway`); break; } lines.push(`BACKSTOP: waiting for train(s) ${merging.map(t => t.nonce).join(', ')} to finish merging${fetched.ok ? '' : ` (${fetched.detail})`}`); + // gh-poll-exempt: runs in GitHub Actions (evals-main-backstop.yml) on GITHUB_TOKEN; default 60 s sleep(pollMs); } const digest = stampFreeTree(git, o.sha, o.repoRoot, loaded.policy); diff --git a/test/eval-detach-timeout-floor.test.ts b/test/eval-detach-timeout-floor.test.ts index bfa3ed553..76291a9db 100644 --- a/test/eval-detach-timeout-floor.test.ts +++ b/test/eval-detach-timeout-floor.test.ts @@ -135,7 +135,7 @@ describe('completion contract on both backends (DX-2)', () => { }; const sandbox = () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'eval-bg-')); - const env: NodeJS.ProcessEnv = { ...process.env, HOME: dir, GSTACK_EVAL_BG_POLL_MS: '50' }; + const env: NodeJS.ProcessEnv = { ...process.env, HOME: dir, GSTACK_EVAL_BG_POLL_MS: '50', GSTACK_GH_POLL_FLOOR_MS: '0' }; return { dir, env }; }; diff --git a/test/fixtures/gh-poll/check-runs.json b/test/fixtures/gh-poll/check-runs.json new file mode 100644 index 000000000..d682623f2 --- /dev/null +++ b/test/fixtures/gh-poll/check-runs.json @@ -0,0 +1,1221 @@ +{ + "total_count": 64, + "check_runs": [ + { + "id": 128269175403, + "name": "free-tests", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:17:27Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128269175403", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128269175403", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114685849622 + } + }, + { + "id": 128267888293, + "name": "free-suite (4)", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:17:08Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267888293", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267888293", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114685849622 + } + }, + { + "id": 128267888243, + "name": "free-suite (10)", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:16:26Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267888243", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267888243", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114685849622 + } + }, + { + "id": 128267888204, + "name": "free-suite (9)", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:15:59Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267888204", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267888204", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114685849622 + } + }, + { + "id": 128267888201, + "name": "free-suite (7)", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:16:00Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267888201", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267888201", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114685849622 + } + }, + { + "id": 128267888196, + "name": "free-suite (6)", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:15:59Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267888196", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267888196", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114685849622 + } + }, + { + "id": 128267888195, + "name": "free-suite (1)", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:15:39Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267888195", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267888195", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114685849622 + } + }, + { + "id": 128267888184, + "name": "free-suite (5)", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:16:32Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267888184", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267888184", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114685849622 + } + }, + { + "id": 128267888159, + "name": "free-suite (8)", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:16:42Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267888159", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267888159", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114685849622 + } + }, + { + "id": 128267888150, + "name": "free-suite (11)", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:13:37Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267888150", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267888150", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114685849622 + } + }, + { + "id": 128267888135, + "name": "free-suite (2)", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:16:37Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267888135", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267888135", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114685849622 + } + }, + { + "id": 128267888108, + "name": "free-suite (3)", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:16:26Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267888108", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267888108", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114685849622 + } + }, + { + "id": 128267800662, + "name": "Trunk Dynamic CI (learning only)", + "status": "completed", + "conclusion": "skipped", + "completed_at": "2026-11-07T02:12:09Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267800662", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267800662", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114685849622 + } + }, + { + "id": 128267800601, + "name": "reuse", + "status": "completed", + "conclusion": "skipped", + "completed_at": "2026-11-07T02:12:09Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267800601", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267800601", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114685849622 + } + }, + { + "id": 128267799958, + "name": "cso-windows-launcher", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:13:46Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267799958", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267799958", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114685849622 + } + }, + { + "id": 128267799937, + "name": "cso-docker-integration", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:12:35Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267799937", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267799937", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114685849622 + } + }, + { + "id": 128267799864, + "name": "cso-macos-launcher", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:13:15Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267799864", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267799864", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114685849622 + } + }, + { + "id": 128267799842, + "name": "typecheck", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:12:34Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267799842", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267799842", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114685849622 + } + }, + { + "id": 128267799809, + "name": "macos-named-regressions", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:15:31Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267799809", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267799809", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114685849622 + } + }, + { + "id": 128267799802, + "name": "quick-bun-default", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:13:55Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267799802", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267799802", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114685849622 + } + }, + { + "id": 128267799588, + "name": "free-plan", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:12:29Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267799588", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42328030770/job/128267799588", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114685849622 + } + }, + { + "id": 128267475495, + "name": "free-tests", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:11:21Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128267475495", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128267475495", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428816 + } + }, + { + "id": 128266879587, + "name": "windows-scoped-tests", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:08:58Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470805/job/128266879587", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470805/job/128266879587", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428588 + } + }, + { + "id": 128266204862, + "name": "slices-comment", + "status": "completed", + "conclusion": "skipped", + "completed_at": "2026-11-07T02:06:28Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470800/job/128266204862", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470800/job/128266204862", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428568 + } + }, + { + "id": 128266204292, + "name": "slices-report", + "status": "completed", + "conclusion": "skipped", + "completed_at": "2026-11-07T02:06:28Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470800/job/128266204292", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470800/job/128266204292", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428568 + } + }, + { + "id": 128266203725, + "name": "rerun-plan", + "status": "completed", + "conclusion": "skipped", + "completed_at": "2026-11-07T02:06:28Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470800/job/128266203725", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470800/job/128266203725", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428568 + } + }, + { + "id": 128266203681, + "name": "train-check", + "status": "completed", + "conclusion": "skipped", + "completed_at": "2026-11-07T02:06:28Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470800/job/128266203681", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470800/job/128266203681", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428568 + } + }, + { + "id": 128266203480, + "name": "eval-rerun", + "status": "completed", + "conclusion": "skipped", + "completed_at": "2026-11-07T02:06:28Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470800/job/128266203480", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470800/job/128266203480", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428568 + } + }, + { + "id": 128266203137, + "name": "train-bisect", + "status": "completed", + "conclusion": "skipped", + "completed_at": "2026-11-07T02:06:28Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470800/job/128266203137", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470800/job/128266203137", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428568 + } + }, + { + "id": 128266202924, + "name": "eval-slices", + "status": "completed", + "conclusion": "skipped", + "completed_at": "2026-11-07T02:06:28Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470800/job/128266202924", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470800/job/128266202924", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428568 + } + }, + { + "id": 128266147691, + "name": "free-suite (9)", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:10:55Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266147691", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266147691", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428816 + } + }, + { + "id": 128266147613, + "name": "free-suite (7)", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:09:16Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266147613", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266147613", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428816 + } + }, + { + "id": 128266147557, + "name": "free-suite (8)", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:10:18Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266147557", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266147557", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428816 + } + }, + { + "id": 128266147542, + "name": "free-suite (3)", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:10:39Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266147542", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266147542", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428816 + } + }, + { + "id": 128266147533, + "name": "free-suite (11)", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:07:23Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266147533", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266147533", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428816 + } + }, + { + "id": 128266147525, + "name": "free-suite (1)", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:10:37Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266147525", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266147525", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428816 + } + }, + { + "id": 128266147515, + "name": "free-suite (2)", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:11:00Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266147515", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266147515", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428816 + } + }, + { + "id": 128266147508, + "name": "free-suite (5)", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:09:08Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266147508", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266147508", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428816 + } + }, + { + "id": 128266147489, + "name": "free-suite (4)", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:10:31Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266147489", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266147489", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428816 + } + }, + { + "id": 128266147486, + "name": "free-suite (10)", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:09:31Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266147486", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266147486", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428816 + } + }, + { + "id": 128266147439, + "name": "free-suite (6)", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:10:08Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266147439", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266147439", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428816 + } + }, + { + "id": 128266081531, + "name": "cso-macos-launcher", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:08:10Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266081531", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266081531", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428816 + } + }, + { + "id": 128266081482, + "name": "cso-docker-integration", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:06:27Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266081482", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266081482", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428816 + } + }, + { + "id": 128266081471, + "name": "cso-windows-launcher", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:07:42Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266081471", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266081471", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428816 + } + }, + { + "id": 128266081452, + "name": "quick-bun-default", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:07:44Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266081452", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266081452", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428816 + } + }, + { + "id": 128266081443, + "name": "free-plan", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:06:16Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266081443", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266081443", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428816 + } + }, + { + "id": 128266081424, + "name": "macos-named-regressions", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:08:37Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266081424", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266081424", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428816 + } + }, + { + "id": 128266081397, + "name": "typecheck", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:06:35Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266081397", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128266081397", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428816 + } + }, + { + "id": 128266055434, + "name": "readiness-comment", + "status": "completed", + "conclusion": "skipped", + "completed_at": "2026-11-07T02:05:55Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470800/job/128266055434", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470800/job/128266055434", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428568 + } + }, + { + "id": 128266055391, + "name": "plan-slices", + "status": "completed", + "conclusion": "skipped", + "completed_at": "2026-11-07T02:05:55Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470800/job/128266055391", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470800/job/128266055391", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428568 + } + }, + { + "id": 128266055247, + "name": "recover-receipts", + "status": "completed", + "conclusion": "skipped", + "completed_at": "2026-11-07T02:05:55Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470800/job/128266055247", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470800/job/128266055247", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428568 + } + }, + { + "id": 128266054919, + "name": "readiness", + "status": "completed", + "conclusion": "skipped", + "completed_at": "2026-11-07T02:05:55Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470800/job/128266054919", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470800/job/128266054919", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428568 + } + }, + { + "id": 128266053505, + "name": "build-image", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:06:27Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470800/job/128266053505", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470800/job/128266053505", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428568 + } + }, + { + "id": 128266053423, + "name": "screen-summary", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:06:14Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470800/job/128266053423", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470800/job/128266053423", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428568 + } + }, + { + "id": 128266037116, + "name": "windows-scoped-shard", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:08:53Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470805/job/128266037116", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470805/job/128266037116", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428588 + } + }, + { + "id": 128265980576, + "name": "Check VERSION is not stale vs queue", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:06:05Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327471124/job/128265980576", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327471124/job/128265980576", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684429384 + } + }, + { + "id": 128265980113, + "name": "actionlint", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:05:53Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470975/job/128265980113", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470975/job/128265980113", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428964 + } + }, + { + "id": 128265979803, + "name": "reuse", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:06:01Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128265979803", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128265979803", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428816 + } + }, + { + "id": 128265979664, + "name": "quality", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:06:07Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470949/job/128265979664", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470949/job/128265979664", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428943 + } + }, + { + "id": 128265979483, + "name": "Trunk Dynamic CI (learning only)", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:05:48Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128265979483", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470893/job/128265979483", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428816 + } + }, + { + "id": 128265979295, + "name": "route", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:05:55Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470800/job/128265979295", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470800/job/128265979295", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428568 + } + }, + { + "id": 128265979247, + "name": "check-freshness", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:05:57Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470857/job/128265979247", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470857/job/128265979247", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428734 + } + }, + { + "id": 128265978989, + "name": "windows-scope", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:05:51Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470805/job/128265978989", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470805/job/128265978989", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428588 + } + }, + { + "id": 128265978805, + "name": "windows-smoke", + "status": "completed", + "conclusion": "success", + "completed_at": "2026-11-07T02:09:14Z", + "details_url": "https://github.com/garrytan/gstack/actions/runs/42327470755/job/128265978805", + "html_url": "https://github.com/garrytan/gstack/actions/runs/42327470755/job/128265978805", + "output": { + "title": null + }, + "app": { + "id": 15368, + "slug": "github-actions" + }, + "check_suite": { + "id": 114684428477 + } + } + ] +} diff --git a/test/fixtures/gh-poll/pull-merged.json b/test/fixtures/gh-poll/pull-merged.json new file mode 100644 index 000000000..8651115c4 --- /dev/null +++ b/test/fixtures/gh-poll/pull-merged.json @@ -0,0 +1,20 @@ +{ + "number": 4101, + "html_url": "https://github.com/garrytan/gstack/pull/4101", + "state": "closed", + "merged": true, + "merged_at": "2026-11-07T02:28:30Z", + "updated_at": "2026-11-07T02:28:33Z", + "mergeable_state": "unknown", + "user": { + "login": "garrytan" + }, + "head": { + "sha": "ff050059a9ebeeaa41618287f0f9687612174bbc", + "ref": "capy/release-stamps-and-a" + }, + "base": { + "sha": "e4b1e3455230a958cc53667e920994a9747d440c", + "ref": "main" + } +} diff --git a/test/fixtures/gh-poll/pull-open.json b/test/fixtures/gh-poll/pull-open.json new file mode 100644 index 000000000..037391685 --- /dev/null +++ b/test/fixtures/gh-poll/pull-open.json @@ -0,0 +1,20 @@ +{ + "number": 4108, + "html_url": "https://github.com/garrytan/gstack/pull/4108", + "state": "open", + "merged": false, + "merged_at": null, + "updated_at": "2026-11-07T03:31:02Z", + "mergeable_state": "unknown", + "user": { + "login": "garrytan" + }, + "head": { + "sha": "8cadbdc985a6259842722f58c82f370f985fd39b", + "ref": "capy/gst-1698-cut-plan-eng-review" + }, + "base": { + "sha": "dd97420b5355ede9e149d023e5f4cf40ad6f0f27", + "ref": "main" + } +} diff --git a/test/fixtures/gh-poll/rate-limit.json b/test/fixtures/gh-poll/rate-limit.json new file mode 100644 index 000000000..eb9c63204 --- /dev/null +++ b/test/fixtures/gh-poll/rate-limit.json @@ -0,0 +1,22 @@ +{ + "resources": { + "core": { + "limit": 7700, + "used": 0, + "remaining": 7700, + "reset": 1794026491 + }, + "graphql": { + "limit": 7700, + "used": 502, + "remaining": 7198, + "reset": 1794023433 + } + }, + "rate": { + "limit": 7700, + "used": 0, + "remaining": 7700, + "reset": 1794026491 + } +} diff --git a/test/fixtures/gh-poll/status.json b/test/fixtures/gh-poll/status.json new file mode 100644 index 000000000..3357b009a --- /dev/null +++ b/test/fixtures/gh-poll/status.json @@ -0,0 +1,6 @@ +{ + "state": "pending", + "sha": "ff050059a9ebeeaa41618287f0f9687612174bbc", + "total_count": 0, + "statuses": [] +} diff --git a/test/gh-poll-tripwire.test.ts b/test/gh-poll-tripwire.test.ts new file mode 100644 index 000000000..97e8ca4dc --- /dev/null +++ b/test/gh-poll-tripwire.test.ts @@ -0,0 +1,98 @@ +/** + * GitHub poll tripwire (GST-1713). The Capy GitHub app installation shares one + * GraphQL budget across every crew; gstack waits that polled every 5 to 30 s + * drained it. lib/gh-poll.ts's pollDelayMs puts a 60 s floor (plus jitter) on + * every wait between GitHub reads. + * + * Rule: in bin/, lib/ and scripts/, a file that reads GitHub (spawns gh, or + * calls one of the GitHub readers named in READERS) may only sleep with a + * duration from pollDelayMs on the same line. Sleep primitives being defined + * or typed (`sleep: ms => Bun.sleepSync(ms)`, `sleep(ms: number): void`) are + * plumbing, not waits. Anything else needs a marker with its reason on the + * line or the line above: + * + * // gh-poll-exempt: + * + * Exemptions are counted and ratcheted (EXEMPT_CEILING only goes down), and + * no shipped file may set the GSTACK_GH_POLL_FLOOR_MS test seam. + */ +import { describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const SCAN_ROOTS = ['bin', 'lib', 'scripts']; +/** The policy owner itself sleeps the rate-limit wait it computes. */ +const OWNER = path.join('lib', 'gh-poll.ts'); + +const READERS = /\b(?:ghJson|ghLines|ghApiJson|runView|newestBranchRun|ciAtHead|readAllPages|runsForHead|fetchPullState|fetchChecks|fetchConversation|reviewDone|ghRunDeps|waitForRun|awaitNonceRun)\(/; +const SPAWNS_GH = /\b(?:spawnSync|spawn|execFileSync)\(\s*['"]gh['"]|\bghSpawnSync\(|\bctx\.gh\(|\brunGh\(|\bspawnGh\b|\.run\(\s*['"]gh['"]|['"]gh['"]\s*,\s*\[/; +const SLEEP = /\b(?:Bun\.sleepSync|Bun\.sleep|Atomics\.wait|sleepSync|sleep)\s*\(|\bsetTimeout\(\s*\w+\s*,/; +const PLUMBING = /\bsleep\w*\s*[:=]\s*\S|function\s+sleep|\bsleep\w*\(\s*\w+\s*:\s*number\s*\)\s*:/; +const COMMENT_LINE = /^\s*(?:\/\/|\*|\/\*)/; +const EXEMPT_MARKER = /gh-poll-exempt:/; +/** Shrink-only: lower it when exemptions burn down; never raise it. */ +const EXEMPT_CEILING = 4; + +function walk(dir: string, out: string[] = []): string[] { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { if (entry.name !== 'node_modules' && entry.name !== 'dist') walk(full, out); } + else if (dir.endsWith(`${path.sep}bin`) || /\.(?:[cm]?[jt]s)$/.test(entry.name)) out.push(full); + } + return out; +} + +interface Hit { file: string; line: number; text: string } + +function scanSleeps(files: Array<{ rel: string; source: string }>): { violations: Hit[]; exempt: Hit[] } { + const violations: Hit[] = []; + const exempt: Hit[] = []; + for (const { rel, source } of files) { + if (rel === OWNER || !(READERS.test(source) || SPAWNS_GH.test(source))) continue; + const lines = source.split('\n'); + lines.forEach((text, i) => { + if (!SLEEP.test(text) || COMMENT_LINE.test(text) || PLUMBING.test(text) || /\bpollDelayMs\(/.test(text)) return; + const hit = { file: rel, line: i + 1, text: text.trim().slice(0, 140) }; + if (EXEMPT_MARKER.test(text) || (i > 0 && EXEMPT_MARKER.test(lines[i - 1]!))) exempt.push(hit); + else violations.push(hit); + }); + } + return { violations, exempt }; +} + +function shipped(): Array<{ rel: string; source: string }> { + return SCAN_ROOTS.flatMap(root => walk(path.join(ROOT, root))).flatMap(file => { + const source = fs.readFileSync(file, 'utf8'); + return source.includes('\u0000') ? [] : [{ rel: path.relative(ROOT, file), source }]; + }); +} + +describe('GitHub poll tripwire', () => { + // Value: protects=the shared GraphQL and REST budgets from a new fast poll loop; fails_when=a GitHub-reading module sleeps a literal or caller interval instead of pollDelayMs + test('every wait in a GitHub-reading module goes through pollDelayMs', () => { + const { violations, exempt } = scanSleeps(shipped()); + if (violations.length) { + throw new Error(`${violations.length} wait(s) in GitHub-reading modules do not use pollDelayMs (lib/gh-poll.ts, 60 s floor):\n${violations.map(v => ` ${v.file}:${v.line}: ${v.text}`).join('\n')}\nFix: sleep pollDelayMs(), or mark a non-poll wait with // gh-poll-exempt: `); + } + expect(exempt.length).toBeLessThanOrEqual(EXEMPT_CEILING); + }); + + test('the scanner catches a planted 20 s poll, and passes pollDelayMs, plumbing and exempted waits', () => { + const planted = (body: string) => [{ rel: 'lib/planted.ts', source: `const r = ctx.gh(['pr', 'checks']);\n${body}\n` }]; + expect(scanSleeps(planted(' Bun.sleepSync(20_000);')).violations.map(v => v.line)).toEqual([2]); + expect(scanSleeps(planted(' await sleep(o.pollMs);')).violations).toHaveLength(1); + expect(scanSleeps(planted(' await new Promise(r => setTimeout(r, 30_000));')).violations).toHaveLength(1); + expect(scanSleeps([{ rel: 'lib/delegated.ts', source: 'for (;;) { if (runView(ctx, slug, id)) break; Atomics.wait(i32, 0, 0, 5_000); }\n' }]).violations).toHaveLength(1); + expect(scanSleeps(planted(' await sleep(pollDelayMs(o.pollMs));')).violations).toEqual([]); + expect(scanSleeps(planted(' sleep: ms => Bun.sleepSync(ms),\n sleep(ms: number): void;')).violations).toEqual([]); + expect(scanSleeps(planted(' // gh-poll-exempt: lock retry, not a GitHub read\n Bun.sleepSync(50);')).exempt).toHaveLength(1); + expect(scanSleeps([{ rel: 'lib/no-github.ts', source: 'Bun.sleepSync(10);\n' }]).violations).toEqual([]); + }); + + test('no shipped file sets the GSTACK_GH_POLL_FLOOR_MS test seam', () => { + const setters = shipped().filter(f => f.rel !== OWNER && /GSTACK_GH_POLL_FLOOR_MS/.test(f.source)); + const workflows = fs.readdirSync(path.join(ROOT, '.github', 'workflows')).filter(name => /GSTACK_GH_POLL_FLOOR_MS/.test(fs.readFileSync(path.join(ROOT, '.github', 'workflows', name), 'utf8'))); + expect([...setters.map(f => f.rel), ...workflows]).toEqual([]); + }); +}); diff --git a/test/gh-poll.test.ts b/test/gh-poll.test.ts new file mode 100644 index 000000000..5f389731c --- /dev/null +++ b/test/gh-poll.test.ts @@ -0,0 +1,202 @@ +/** + * lib/gh-poll.ts (GST-1713): the 60 s poll floor with jitter, rate-limit + * classification and the sleep-until-reset rule, and the REST response + * parsers. test/fixtures/gh-poll/*.json are trimmed recordings of real + * GitHub REST replies (garrytan/gstack#4101 merged, #4108 open, the check runs + * and combined status of #4101's head, and `gh api rate_limit`, recorded + * 2026-11-06). The rate-limit messages are the ones gh prints for the Capy app + * installation (FOUN-32 report on GST-1713) and GitHub's secondary-limit text. + */ +import { describe, expect, test } from 'bun:test'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import type { GhResult } from '../lib/ci-gate'; +import { MAX_LIMIT_WAITS, MIN_POLL_MS, parseHeadChecks, parsePullState, pollDelayMs, rateLimitKind, rateLimitWaitMs, withRateLimit } from '../lib/gh-poll'; +import { GhError, fetchChecks, runGh } from '../lib/pr-wait-github'; +import { waitForRun } from '../lib/ci-carry'; + +const fixture = (name: string): string => readFileSync(join(import.meta.dir, 'fixtures', 'gh-poll', name), 'utf8'); +const ok = (stdout: string): GhResult => ({ status: 0, stdout, stderr: '' }); +const err = (stderr: string, status = 1): GhResult => ({ status, stdout: '', stderr: `${stderr}\n` }); +const NO_SEAM = {}; + +const GRAPHQL_LIMIT = 'GraphQL: API rate limit exceeded for installation ID 161978188.'; +const CORE_LIMIT = 'gh: API rate limit exceeded for installation ID 161978188. If you reach out to GitHub Support for help, please include the request ID 1234:5678. (HTTP 403)'; +const SECONDARY_LIMIT = 'gh: You have exceeded a secondary rate limit. Please wait a few minutes before you try again. (HTTP 403)'; +const RATE_LIMIT_JSON = fixture('rate-limit.json'); +const exhausted = (resource: 'graphql' | 'core', reset: number) => { + const parsed = JSON.parse(RATE_LIMIT_JSON); + parsed.resources[resource] = { ...parsed.resources[resource], remaining: 0, reset }; + return JSON.stringify(parsed); +}; + +describe('the poll floor', () => { + // Value: protects=no gstack wait polls GitHub faster than once a minute; fails_when=a caller's 5, 15, 20 or 30 s interval reaches GitHub unchanged + test('every delay is at least 60 s plus up to 15 s of jitter; a longer request is kept', () => { + expect(MIN_POLL_MS).toBe(60_000); + for (const requested of [0, 5_000, 20_000, 30_000, 59_999]) { + expect(pollDelayMs(requested, () => 0, NO_SEAM)).toBe(60_000); + expect(pollDelayMs(requested, () => 1, NO_SEAM)).toBe(75_000); + } + expect(pollDelayMs(Number.NaN, () => 0, NO_SEAM)).toBe(60_000); + expect(pollDelayMs(300_000, () => 0.5, NO_SEAM)).toBe(307_500); + }); + + test('GSTACK_GH_POLL_FLOOR_MS lowers the floor for fixtures only', () => { + expect(pollDelayMs(20, () => 1, { GSTACK_GH_POLL_FLOOR_MS: '0' })).toBe(20); + expect(pollDelayMs(20, () => 0, { GSTACK_GH_POLL_FLOOR_MS: 'soon' })).toBe(60_000); + }); +}); + +describe('rate limits are classified, never mistaken for other failures', () => { + // Value: protects=a rate-limited call is recognized whichever budget ran out; fails_when=a GraphQL limit reads as core, or a permission 403 reads as a rate limit + test('GraphQL, REST core and secondary limits; permission and server errors are not rate limits', () => { + expect(rateLimitKind(['pr', 'view', '42'], GRAPHQL_LIMIT)).toBe('graphql'); + expect(rateLimitKind(['api', 'repos/o/r/pulls/42'], GRAPHQL_LIMIT)).toBe('graphql'); + expect(rateLimitKind(['api', 'graphql'], '{"errors":[{"type":"RATE_LIMITED","message":"API rate limit exceeded"}]}')).toBe('graphql'); + expect(rateLimitKind(['api', 'repos/o/r/commits/abc/check-runs'], CORE_LIMIT)).toBe('core'); + expect(rateLimitKind(['run', 'view', '9'], CORE_LIMIT)).toBe('core'); + expect(rateLimitKind(['pr', 'checks', '42'], 'gh: HTTP 429: too many requests')).toBe('graphql'); + expect(rateLimitKind(['api', 'repos/o/r/pulls/42'], SECONDARY_LIMIT)).toBe('secondary'); + expect(rateLimitKind(['api', 'repos/o/r/branches/main/protection'], 'gh: Resource not accessible by integration (HTTP 403)')).toBeNull(); + expect(rateLimitKind(['api', 'repos/o/r/pulls/42'], 'gh: Something went wrong (HTTP 502)')).toBeNull(); + }); + + test('the wait runs to the exhausted resource\'s reset (plus slack); a secondary limit or an unreadable reply waits the floor', () => { + const now = 1_794_020_000_000; + expect(rateLimitWaitMs('graphql', exhausted('graphql', now / 1000 + 600), now, () => 0, NO_SEAM)).toBe(605_000); + expect(rateLimitWaitMs('core', exhausted('core', now / 1000 + 30), now, () => 0, NO_SEAM)).toBe(60_000); + expect(rateLimitWaitMs('graphql', RATE_LIMIT_JSON, now, () => 0, NO_SEAM)).toBe(60_000); + expect(rateLimitWaitMs('secondary', exhausted('graphql', now / 1000 + 600), now, () => 0, NO_SEAM)).toBe(60_000); + expect(rateLimitWaitMs('graphql', 'not json', now, () => 0, NO_SEAM)).toBe(60_000); + expect(rateLimitWaitMs('graphql', exhausted('graphql', now / 1000 + 86_400), now, () => 0, NO_SEAM)).toBe(65 * 60_000); + }); +}); + +describe('a rate-limited call backs off until the reset, never retries at once', () => { + const harness = (replies: GhResult[], limits = exhausted('graphql', 1_794_020_600)) => { + const calls: string[][] = []; + const sleeps: number[] = []; + let clock = 1_794_020_000_000; + const gh = withRateLimit(args => { + calls.push(args); + if (args[0] === 'api' && args[1] === 'rate_limit') return ok(limits); + return replies.length > 1 ? replies.shift()! : replies[0]!; + }, { sleep: ms => { sleeps.push(ms); clock += ms; }, now: () => clock, log: () => {}, random: () => 0, vars: NO_SEAM }); + return { gh, calls, sleeps }; + }; + + // Value: protects=GraphQL quota recovery; fails_when=a rate-limited gh call is rerun immediately or in a short loop + test('one rate limit: read rate_limit, sleep until the reset, rerun once', () => { + const h = harness([err(GRAPHQL_LIMIT), ok('{"number":42}')]); + expect(h.gh(['pr', 'view', '42']).stdout).toBe('{"number":42}'); + expect(h.calls).toEqual([['pr', 'view', '42'], ['api', 'rate_limit'], ['pr', 'view', '42']]); + expect(h.sleeps).toEqual([605_000]); + }); + + test('a limit that persists gives up after MAX_LIMIT_WAITS waits, each at least the floor, and returns the failure', () => { + const h = harness([err(SECONDARY_LIMIT)]); + const result = h.gh(['api', 'repos/o/r/pulls/42']); + expect(result.status).toBe(1); + expect(result.stderr).toContain('secondary rate limit'); + expect(h.calls.filter(c => c[1] !== 'rate_limit')).toHaveLength(MAX_LIMIT_WAITS + 1); + expect(h.sleeps).toHaveLength(MAX_LIMIT_WAITS); + for (const ms of h.sleeps) expect(ms).toBeGreaterThanOrEqual(MIN_POLL_MS); + }); + + // Value: protects=the land lock; fails_when=a rate-limited gh pr merge sleeps past PUSH_STALE_MS while another lander may break the lock + test('a rate-limited gh pr merge fails at once: no rate_limit read, no sleep, no rerun', () => { + const h = harness([err(GRAPHQL_LIMIT)]); + expect(h.gh(['pr', 'merge', '42', '--squash', '--match-head-commit', 'abc']).status).toBe(1); + expect(h.calls).toEqual([['pr', 'merge', '42', '--squash', '--match-head-commit', 'abc']]); + expect(h.sleeps).toEqual([]); + }); + + test('success and real failures return at once: no rate_limit read, no sleep', () => { + const h = harness([err('gh: Not Found (HTTP 404)')]); + expect(h.gh(['api', 'repos/o/r/pulls/42']).status).toBe(1); + expect(h.calls).toEqual([['api', 'repos/o/r/pulls/42']]); + expect(h.sleeps).toEqual([]); + }); + + test("gstack-pr-wait's runGh throws a rate limit at once (transient, no 2 s retry), while a 502 still retries", () => { + const sleeps: number[] = []; + let calls = 0; + const limited = { gh: () => { calls += 1; return err(GRAPHQL_LIMIT); }, sleep: (ms: number) => sleeps.push(ms), log: () => {} }; + let thrown: unknown; + try { runGh(limited, ['api', 'graphql']); } catch (error) { thrown = error; } + expect(thrown).toBeInstanceOf(GhError); + expect((thrown as GhError).transient).toBe(true); + expect(calls).toBe(1); + expect(sleeps).toEqual([]); + const flaky = [err('gh: Something went wrong (HTTP 502)'), ok('fine')]; + expect(runGh({ gh: () => flaky.shift()!, sleep: (ms: number) => sleeps.push(ms), log: () => {} }, ['api', 'x']).stdout).toBe('fine'); + expect(sleeps).toEqual([2_000]); + }); +}); + +describe('REST replies parse from recorded fixtures', () => { + // Value: protects=the REST reads that replaced gh pr view and gh pr checks read GitHub's real shapes; fails_when=a merged PR reads OPEN, the head or base is lost, or a skipped check counts as failed + test('pulls/{n}: merged and open pull requests in GraphQL spellings', () => { + expect(parsePullState(fixture('pull-merged.json'))).toEqual({ + url: 'https://github.com/garrytan/gstack/pull/4101', state: 'MERGED', headRefOid: 'ff050059a9ebeeaa41618287f0f9687612174bbc', + baseRefName: 'main', mergeStateStatus: 'UNKNOWN', author: 'garrytan', updatedAt: '2026-11-07T02:28:33Z', + }); + const open = parsePullState(fixture('pull-open.json'))!; + expect(open).toMatchObject({ state: 'OPEN', baseRefName: 'main', headRefOid: '8cadbdc985a6259842722f58c82f370f985fd39b' }); + expect(parsePullState(JSON.stringify({ ...JSON.parse(fixture('pull-open.json')), mergeable_state: 'behind' }))!.mergeStateStatus).toBe('BEHIND'); + expect(parsePullState(JSON.stringify({ ...JSON.parse(fixture('pull-open.json')), state: 'closed' }))!.state).toBe('CLOSED'); + expect(parsePullState(JSON.stringify({ ...JSON.parse(fixture('pull-open.json')), user: { login: 'capy-ai[bot]' } }))!.author).toBe('capy-ai'); + expect(parsePullState('{"message":"Not Found"}')).toBeNull(); + expect(parsePullState('not json')).toBeNull(); + }); + + test('check-runs and status: 64 recorded runs become pass and skipping rows; re-runs supersede, contexts join, unknown conclusions fail closed', () => { + const runs = JSON.parse(fixture('check-runs.json')).check_runs as Array>; + const statuses = JSON.parse(fixture('status.json')).statuses as unknown[]; + const rows = parseHeadChecks(runs, statuses); + expect(rows).toHaveLength(64); + expect(rows.filter(r => r.bucket === 'pass')).toHaveLength(51); + expect(rows.filter(r => r.bucket === 'skipping')).toHaveLength(13); + expect(rows.find(r => r.name === 'free-tests')).toMatchObject({ bucket: 'pass', link: expect.stringContaining('https://github.com/garrytan/gstack/actions/runs/') }); + const first = runs.find(r => r.name === 'free-tests')!; + const rerun = { ...first, id: Number(first.id) + 1, status: 'in_progress', conclusion: null, completed_at: null }; + const stale = { ...first, id: Number(first.id) + 2, name: 'old-check', conclusion: 'stale' }; + const synthetic = parseHeadChecks([first, rerun, stale], [{ context: 'ci/legacy', state: 'failure', target_url: 'https://ci.example/1', description: 'red' }, { context: 'ci/wait', state: 'pending' }]); + expect(synthetic.map(r => [r.name, r.bucket])).toEqual([['free-tests', 'pending'], ['old-check', 'stale'], ['ci/legacy', 'fail'], ['ci/wait', 'pending']]); + }); + + test("gstack-pr-wait's fetchChecks reads the paginated lines and collapses one job's two workflow runs to the worst bucket", () => { + const runs = JSON.parse(fixture('check-runs.json')).check_runs as Array>; + const pushRun = { ...runs.find(r => r.name === 'free-tests')!, id: 1, conclusion: 'failure', check_suite: { id: 7 } }; + const calls: string[][] = []; + const gh = (args: string[]): GhResult => { + calls.push(args); + if (args[2]!.includes('/check-runs')) return ok([...runs, pushRun].map(r => JSON.stringify(r)).join('\n')); + return ok(''); + }; + const checks = fetchChecks({ gh, sleep: () => {}, log: () => {} }, 'garrytan/gstack', 'f'.repeat(40)); + expect(checks.size).toBe(new Set(runs.map(r => r.name)).size); + expect(checks.get('free-tests')!.bucket).toBe('fail'); + expect(calls.map(c => c.slice(0, 3))).toEqual([ + ['api', '--paginate', `repos/garrytan/gstack/commits/${'f'.repeat(40)}/check-runs?per_page=100`], + ['api', '--paginate', `repos/garrytan/gstack/commits/${'f'.repeat(40)}/status?per_page=100`], + ]); + expect(calls.some(c => c[0] === 'pr' || c[1] === 'graphql')).toBe(false); + }); +}); + +describe('waits sleep at least the floor', () => { + // Value: protects=gstack-ci-carry wait and ship-release accept poll at most once a minute; fails_when=waitForRun sleeps its 15 or 20 s interval + test('waitForRun sleeps pollDelayMs between reads, whatever intervalMs asks', async () => { + const sleeps: number[] = []; + let t = 0; + let reads = 0; + const deps = { runsForHead: () => { reads += 1; return [{ id: 1, run_attempt: 1, event: 'pull_request', status: 'in_progress', conclusion: null, created_at: '2026-11-06T00:00:00Z' }]; } } as never; + const verdict = await waitForRun(deps, { workflow: 'free-tests.yml', events: ['pull_request'], head: 'a'.repeat(40), timeoutMs: 300_000, intervalMs: 15_000, sleep: async ms => { sleeps.push(ms); t += ms; }, clock: () => t, random: () => 0 }); + expect(verdict.state).toBe('running'); + expect(sleeps.length).toBeGreaterThan(0); + for (const ms of sleeps) expect(ms).toBeGreaterThanOrEqual(MIN_POLL_MS); + expect(reads).toBe(sleeps.length + 1); + }); +}); diff --git a/test/greptile-early.test.ts b/test/greptile-early.test.ts index 793800d8d..7d166ce0b 100644 --- a/test/greptile-early.test.ts +++ b/test/greptile-early.test.ts @@ -64,7 +64,7 @@ function fixture(opts: { if (cmd === "gh" && args[0] === "repo") return ok("{}"); if (cmd === "gh" && joined.includes("/comments?sort=created")) return ok(JSON.stringify(joined.includes("/issues/") ? opts.comments ?? [] : [])); if (cmd === "gh" && joined.startsWith("pr list")) return ok(JSON.stringify(opts.openPrs ?? [])); - if (cmd === "gh" && joined.startsWith("pr view")) return ok(SHA + "\n"); + if (cmd === "gh" && joined.startsWith("api repos/{owner}/{repo}/pulls/12 ")) return ok(SHA + "\n"); if (cmd === "gh" && joined.includes("/check-runs")) return ok(JSON.stringify(opts.checks ?? [])); if (cmd === "gh" && joined.includes("/reviews")) return ok(JSON.stringify(opts.reviews ?? [])); if (cmd === "gh" && joined.includes("/issues/12/comments")) return ok(JSON.stringify(opts.summaries ?? [])); @@ -237,7 +237,7 @@ describe("waiting for the review (CEO-8 completion signal)", () => { const f = fixture(); const polls = () => f.calls.filter(c => c.args.join(" ").includes("/check-runs")).length; f.run("wait", "12", "--since", since); - expect(polls()).toBe(4); // at 0, 30, 60 and 90 seconds + expect(polls()).toBe(2); // at 0 and 60-75 seconds: polls are at least 60 s apart (GST-1713) f.advance(9 * 60_000); const r = f.run("wait", "12", "--since", since); expect(r.out).toContain("GREPTILE_REVIEW: timeout"); diff --git a/test/gstack-ci-gate.test.ts b/test/gstack-ci-gate.test.ts index e1cd8bb38..8288bbf10 100644 --- a/test/gstack-ci-gate.test.ts +++ b/test/gstack-ci-gate.test.ts @@ -227,30 +227,38 @@ describe('polling', () => { test('delayed registration: zero checks re-polled until checks appear', async () => { const c = clock(); const gh = fakeGh({ required: [NONE, NONE, NONE_REQUIRED], all: [NONE, NONE, ok([row('build', 'pending')])] }); - const result = await runCiGate(options({ registrationWaitSeconds: 60 }), gh, c.sleep, c.now); + const result = await runCiGate(options({ registrationWaitSeconds: 120 }), gh, c.sleep, c.now, () => 0); expect(result.verdict).toBe('PENDING'); - expect(c.now()).toBe(20_000); + expect(c.now()).toBe(120_000); }); test('registration wait is bounded at 60s, then NO_CHECKS', async () => { const c = clock(); - const result = await runCiGate(options({ registrationWaitSeconds: 60 }), fakeGh({ required: [NONE], all: [NONE] }), c.sleep, c.now); + const result = await runCiGate(options({ registrationWaitSeconds: 60 }), fakeGh({ required: [NONE], all: [NONE] }), c.sleep, c.now, () => 0); expect(result.verdict).toBe('NO_CHECKS'); expect(c.now()).toBe(60_000); }); + test('registration wait re-polls once even when the first read takes time', async () => { + const c = clock(); + const base = fakeGh({ required: [NONE, NONE_REQUIRED], all: [NONE, ok([row('build', 'pending')])] }); + const gh: GhRunner = args => { c.sleep(500); return base(args); }; + const result = await runCiGate(options({ registrationWaitSeconds: 60 }), gh, c.sleep, c.now, () => 0); + expect(result.verdict).toBe('PENDING'); + }); + test('--wait polls PENDING until a final verdict', async () => { const c = clock(); const all = [ok([row('t', 'pending')]), ok([row('t', 'pending')]), ok([row('t', 'pass')])]; - const result = await runCiGate(options({ waitSeconds: 240 }), fakeGh({ required: [NONE_REQUIRED], all }), c.sleep, c.now); + const result = await runCiGate(options({ waitSeconds: 240 }), fakeGh({ required: [NONE_REQUIRED], all }), c.sleep, c.now, () => 0); expect(result.verdict).toBe('PASS'); - expect(c.now()).toBe(60_000); + expect(c.now()).toBe(120_000); }); test('--wait timeout reports PENDING and names the still-pending checks', async () => { const c = clock(); const all = [ok([row('build', 'pass'), row('deploy-preview', 'pending')])]; - const result = await runCiGate(options({ waitSeconds: 240 }), fakeGh({ required: [NONE_REQUIRED], all }), c.sleep, c.now); + const result = await runCiGate(options({ waitSeconds: 240 }), fakeGh({ required: [NONE_REQUIRED], all }), c.sleep, c.now, () => 0); expect(result.verdict).toBe('PENDING'); expect(checkLines(result.lines)).toEqual(['CHECK\tdeploy-preview\tpending\trequired=n\thttps://ci.example/deploy-preview']); expect(result.lines.some(line => line.startsWith('NOTE still pending after waiting 240s'))).toBe(true); @@ -259,9 +267,9 @@ describe('polling', () => { test('a failure during the wait stops immediately (fail fast)', async () => { const c = clock(); const all = [ok([row('t', 'pending')]), ok([row('t', 'fail'), row('u', 'pending')])]; - const result = await runCiGate(options({ waitSeconds: 240 }), fakeGh({ required: [NONE_REQUIRED], all }), c.sleep, c.now); + const result = await runCiGate(options({ waitSeconds: 240 }), fakeGh({ required: [NONE_REQUIRED], all }), c.sleep, c.now, () => 0); expect(result.verdict).toBe('FAIL'); - expect(c.now()).toBe(30_000); + expect(c.now()).toBe(60_000); }); }); diff --git a/test/helpers/free-fixtures-data.ts b/test/helpers/free-fixtures-data.ts index 61a735005..f454b6cbc 100644 --- a/test/helpers/free-fixtures-data.ts +++ b/test/helpers/free-fixtures-data.ts @@ -151,6 +151,7 @@ export const FREE_FIXTURES: Readonly> = { 'test/fixtures/exit-propagation/**': ['test/exit-propagation.test.ts'], 'test/fixtures/factory-scorecard/**': ['test/factory-scorecard.test.ts'], 'test/fixtures/free-shard-capture.ts': ['test/test-free-shards-capture.test.ts'], + 'test/fixtures/gh-poll/**': ['test/gh-poll.test.ts'], 'test/fixtures/golden/**': ['test/host-config.test.ts', 'test/ship-review-loop.test.ts'], 'test/fixtures/health-detect-legacy.sh': ['test/health-detect-no-loss.test.ts'], 'test/fixtures/impeccable-antipatterns.json': ['test/design-catalog.test.ts', 'test/impeccable-fixtures.test.ts'], diff --git a/test/helpers/ship-queue-train-fixture.ts b/test/helpers/ship-queue-train-fixture.ts index 17e3f822b..1ec394990 100644 --- a/test/helpers/ship-queue-train-fixture.ts +++ b/test/helpers/ship-queue-train-fixture.ts @@ -63,7 +63,7 @@ export function trainFixture(policy: Record = TRAIN_POLICY, pre fs.writeFileSync(path.join(dir, 'shim.js'), shimSource()); fs.writeFileSync(path.join(dir, 'gh'), `#!/bin/sh\nexec node "${path.join(dir, 'shim.js')}" "${dataFile}" "$@"\n`, { mode: 0o755 }); const state = path.join(repo.root, 'state'); - const env = { PATH: `${dir}:${process.env.PATH ?? ''}`, GSTACK_STATE_ROOT: state, GSTACK_SHIP_QUEUE_POLL_MS: '20', GSTACK_EVAL_BG_POLL_MS: '20', GSTACK_SHIP_QUEUE_NOW: '2026-10-18T12:00:00Z' }; + const env = { PATH: `${dir}:${process.env.PATH ?? ''}`, GSTACK_STATE_ROOT: state, GSTACK_SHIP_QUEUE_POLL_MS: '20', GSTACK_GH_POLL_FLOOR_MS: '0', GSTACK_EVAL_BG_POLL_MS: '20', GSTACK_SHIP_QUEUE_NOW: '2026-10-18T12:00:00Z' }; const read = () => JSON.parse(fs.readFileSync(dataFile, 'utf8')) as ShimData; const write = (fn: (d: ShimData) => void) => { const d = read(); fn(d); fs.writeFileSync(dataFile, JSON.stringify(d)); }; return { repo, state, dataFile, env, read, write }; diff --git a/test/pr-wait.test.ts b/test/pr-wait.test.ts index 5d91a8059..c512d1d02 100644 --- a/test/pr-wait.test.ts +++ b/test/pr-wait.test.ts @@ -53,7 +53,31 @@ const prJson = (shape: PrShape = {}) => JSON.stringify({ const checksJson = (...rows: Array<[string, string] | [string, string, string]>) => JSON.stringify(rows.map(([name, bucket, description]) => ({ name, bucket, link: `https://ci.example/${name}`, description: description ?? '' }))); -/** Scripted replies per gh command shape; a list is consumed one poll at a time, the last reply repeating. */ +/** The REST `pulls/{n}` reply for a scripted GraphQL PR reply: same facts, REST spellings; updated_at moves when the reply does. */ +function restPull(graphql: string): string { + const node = JSON.parse(graphql).data.repository.pullRequest; + return JSON.stringify({ + html_url: node.url, state: node.state === 'OPEN' ? 'open' : 'closed', merged: node.state === 'MERGED', head: { sha: node.headRefOid }, + base: { ref: node.baseRefName }, mergeable_state: String(node.mergeStateStatus).toLowerCase(), user: { login: node.author.login }, + updated_at: `u-${graphql.length}-${[...graphql].reduce((h, ch) => (h * 31 + ch.charCodeAt(0)) | 0, 0)}`, + }); +} +const CONCLUSION: Record = { pass: 'success', fail: 'failure', skipping: 'skipped', cancel: 'cancelled' }; +/** `gh pr checks`-shaped rows as the REST check-runs lines `gh api --paginate --jq '.check_runs[]'` prints, one suite per row. */ +function restCheckRuns(result: GhResult): GhResult { + if (result.status !== 0) return result; + const rows = JSON.parse(result.stdout || '[]') as Array<{ name: string; bucket: string; link: string; description: string }>; + return ok(rows.map((row, i) => JSON.stringify({ + id: i + 1, name: row.name, status: row.bucket === 'pending' ? 'in_progress' : 'completed', conclusion: row.bucket === 'pending' ? null : CONCLUSION[row.bucket] ?? row.bucket, + completed_at: row.bucket === 'pending' ? null : '2026-11-06T00:00:00Z', details_url: row.link, output: { title: row.description }, app: { id: 15368 }, check_suite: { id: 100 + i }, + })).join('\n')); +} + +/** + * Scripted replies per gh command shape; a list is consumed one poll at a time, the last reply repeating. `pr` is the + * GraphQL reply; each REST `pulls/{n}` read takes the next one and the GraphQL conversation read returns that same reply. + * `checks` is `gh pr checks`-shaped and served as REST check runs. + */ interface Script { pr?: string | string[]; checks?: GhResult | GhResult[]; @@ -67,15 +91,26 @@ function fakeGh(script: Script) { // Function declarations, not generic arrows: pregate's import scanner reads tests as TSX, where `(x) =>` is a tag. function queue(value: T | T[] | undefined): T[] { return value === undefined ? [] : Array.isArray(value) ? [...value] : [value]; } function next(q: T[]): T | undefined { return q.length > 1 ? q.shift() : q[0]; } - const prs = queue(script.pr).map(ok); + const prs = queue(script.pr); const checks = queue(script.checks); const calls: string[][] = []; + let current = prJson(); const gh = (args: string[]): GhResult => { calls.push(args); - const [a, b] = args; - if (a === 'api' && b === 'graphql') return next(prs) ?? ok(prJson()); - if (a === 'pr' && b === 'checks') return next(checks) ?? ok(checksJson()); - if (a === 'pr' && b === 'view') return ok('{"number":42}'); + const a = args[0]; + const b = args[1] === '--paginate' ? args[2] : args[1]; + if (a === 'api' && b === 'graphql') return ok(current); + if (a === 'api' && b === `repos/${REPO}/pulls/42`) { + if (args.includes('.number')) return ok('42\n'); + current = next(prs) ?? prJson(); + return ok(restPull(current)); + } + if (a === 'api' && b.includes('/check-runs')) return restCheckRuns(next(checks) ?? ok(checksJson())); + if (a === 'api' && b.includes('/status')) return ok(''); + if (a === 'api' && b.includes('/git/ref/heads/')) { + const oid = JSON.parse(current).data.repository.pullRequest.baseRef?.target.oid; + return oid ? ok(`${oid}\n`) : NOT_FOUND; + } if (a === 'pr' && b === 'diff') return script.prFiles ?? ok(''); if (a === 'repo' && b === 'view') return ok(`${REPO}\n`); if (a === 'api' && b.includes('/rules/branches/')) return script.rulesets ?? NOT_FOUND; @@ -360,7 +395,7 @@ describe('arguments and the gh layer', () => { expect(parsePrTarget('abc')).toMatchObject({ error: "'abc' is not a PR number or PR URL" }); expect(parsePrTarget('0')).toMatchObject({ error: "'0' is not a valid PR number" }); expect(parsePrTarget('42', 'nope')).toMatchObject({ error: "'nope' is not owner/name" }); - expect(parsePrWaitArgs(['42', '--until', 'green', '--interval', '2'])).toEqual({ target: '42', until: 'green', interval: 5, timeout: 60, excludes: [] }); + expect(parsePrWaitArgs(['42', '--until', 'green', '--interval', '2'])).toEqual({ target: '42', until: 'green', interval: 60, timeout: 60, excludes: [] }); expect(parsePrWaitArgs(['42', '--exclude', 'quality-gate, canary', '--exclude', 'canary'])).toMatchObject({ excludes: ['quality-gate', 'canary'] }); expect(parsePrWaitArgs(['42', '--exclude', ' , '])).toMatchObject({ error: expect.stringContaining('--exclude needs') }); expect(parsePrWaitArgs(['42', '--timeout', '0'])).toMatchObject({ timeout: 0 }); @@ -377,9 +412,9 @@ describe('arguments and the gh layer', () => { expect(prWaitMain([], env(fakeGh({})))).toBe(2); expect(lines()[0]).toMatch(/missing PR number.*\(PR_WAIT_REFUSED; .*#pr-wait-refused\)$/); stdout = []; - const gone = (args: string[]): GhResult => (args[1] === 'view' ? err('GraphQL: Could not resolve to a PullRequest with the number of 42. (HTTP 404)') : ok('')); + const gone = (args: string[]): GhResult => (args[1] === `repos/${REPO}/pulls/42` ? err('gh: Not Found (HTTP 404)') : ok('')); expect(prWaitMain(['42', '--repo', REPO], env(gone))).toBe(2); - expect(lines()[0]).toContain('cannot read owner/project#42: GraphQL: Could not resolve'); + expect(lines()[0]).toContain('cannot read owner/project#42: gh: Not Found (HTTP 404)'); expect(RESULT_CODES.PR_WAIT_REFUSED.anchor).toBe('pr-wait-refused'); expect(RESULT_CODES.PR_WAIT_BLOCKED.anchor).toBe('pr-wait-blocked'); }); @@ -539,13 +574,13 @@ describe('pass-1 review fixes: gates agree, bounded waits, bot identity, local f }); test('run() ends as blocked after ten transient failures, and at once on a non-transient gh error', () => { - const down = (args: string[]): GhResult => (args[1] === 'graphql' ? err('gh: Something went wrong (HTTP 502)') : ok('')); + const down = (): GhResult => err('gh: Something went wrong (HTTP 502)'); const e = env(down); expect(new Watcher(e, REPO, 42, 'green', 20, botLists(e.vars)).run()).toBe(PR_WAIT_EXIT.blocked); expect(stderr.join('').match(/poll failed \(\d+\/10\)/g)).toHaveLength(10); expect(lines()[0]).toBe('#42 gstack-pr-wait stopped: GitHub unreachable for 10 polls: gh: Something went wrong (HTTP 502)'); stdout = []; - const gone = (args: string[]): GhResult => (args[1] === 'graphql' ? err('gh: Not Found (HTTP 404)') : ok('')); + const gone = (args: string[]): GhResult => (args[1] === `repos/${REPO}/pulls/42` ? err('gh: Not Found (HTTP 404)') : ok('')); const e2 = env(gone); expect(new Watcher(e2, REPO, 42, 'green', 20, botLists(e2.vars)).run()).toBe(PR_WAIT_EXIT.blocked); expect(lines()[0]).toBe('#42 gstack-pr-wait stopped: gh: Not Found (HTTP 404)'); @@ -595,9 +630,9 @@ describe('pass-1 review fixes: gates agree, bounded waits, bot identity, local f mkdirSync(fakeBin); writeFileSync(join(fakeBin, 'gh'), `#!/bin/sh case "$1 $2" in - 'pr view') echo '{"number":42}' ;; + 'api repos/owner/project/pulls/42') echo '${restPull(prJson({}))}' ;; 'api graphql') echo '${prJson({})}' ;; - 'pr checks') echo '${checksJson(['build', 'pending'])}'; exit 8 ;; + 'api --paginate') echo '${restCheckRuns(ok(checksJson(['build', 'pending']))).stdout}' ;; *) echo 'gh: Not Found (HTTP 404)' >&2; exit 1 ;; esac`, { mode: 0o755 }); const stateRoot = join(dir, 'state'); @@ -619,6 +654,42 @@ esac`, { mode: 0o755 }); }, 30_000); }); +describe('GitHub budget (GST-1713): REST every poll, the GraphQL conversation read only when it can have changed', () => { + const graphqlReads = (gh: ReturnType) => gh.calls.filter(c => c[1] === 'graphql').length; + const run = (script: Script, until: PrWaitOptions['until'], polls: number, between: () => void = () => {}) => { + const gh = fakeGh(script); + const e = env(gh); + const w = new Watcher(e, REPO, 42, until, 60, botLists(e.vars)); + for (let i = 0; i < polls; i++) { expect(w.poll()).toBeNull(); w.save(); w.baseline = false; between(); } + return gh; + }; + + // Value: protects=the shared GraphQL quota; fails_when=gstack-pr-wait spends a GraphQL read (or gh pr checks) on every poll again + test('an unchanged PR is read through GraphQL once across polls; checks and state never touch GraphQL', () => { + const gh = run({ checks: ok(checksJson(['build', 'pending'])) }, 'green', 4); + expect(graphqlReads(gh)).toBe(1); + expect(gh.calls.some(c => c[0] === 'pr')).toBe(false); + expect(gh.calls.filter(c => c.includes(`repos/${REPO}/pulls/42`))).toHaveLength(4); + }); + + test('a new comment (updated_at moves) or ten minutes reread the conversation; a ready watch rereads it once green', () => { + const talk = comment('c9', { login: 'reviewer' }, 'please rename'); + const changed = run({ pr: [prJson(), prJson(), prJson({ comments: [talk] })], checks: ok(checksJson(['build', 'pending'])) }, 'green', 4); + expect(graphqlReads(changed)).toBe(2); + expect(lines()).toEqual([`#42 comment from reviewer: please rename https://github.com/${REPO}/pull/42#issuecomment-c9`]); + expect(graphqlReads(run({ checks: ok(checksJson(['build', 'pending'])) }, 'green', 3, () => { clock += 10 * 60_000; }))).toBe(3); + expect(graphqlReads(run({ checks: ok(checksJson(['build', 'pass'])) }, 'ready', 3))).toBe(3); + }); + + test('run() sleeps at least 60 s between polls, even when asked for 5', () => { + const sleeps: number[] = []; + const e = { ...env(fakeGh({ checks: ok(checksJson(['build', 'pending'])) })), sleep: (ms: number) => { sleeps.push(ms); clock += ms; } }; + expect(new Watcher(e, REPO, 42, 'green', 5, botLists(e.vars), 5).run()).toBe(PR_WAIT_EXIT.blocked); + expect(sleeps.length).toBeGreaterThan(0); + for (const ms of sleeps) expect(ms).toBeGreaterThanOrEqual(60_000); + }); +}); + describe('state matrix cells (gstack-state-matrix: --until × bucket, merge state × --until, lock, baseline × event)', () => { const GREPTILE = review('g1', { login: 'greptile-apps', bot: true }, 'COMMENTED', '');