fix: gstack's GitHub waits poll REST at most once a minute and wait out rate limits (GST-1713) (#4109)

* fix: gstack's GitHub waits poll REST at most once a minute and wait out rate limits (GST-1713)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* test: register the gh-poll REST fixtures as free fixtures (GST-1713)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix: pre-landing review fixes (GST-1713)

ci-gate --registration-wait re-polls while time is left in the window; a rate-limited gh pr merge fails at once under the land lock; a bot PR author reads without its [bot] suffix; ci-carry wait documents its one-poll overrun.

Review-Finding: p1/specialist-batched#1 reader=testing
Review-Finding: p1/specialist-batched#2 reader=testing
Review-Finding: p1/specialist-batched#3 reader=testing
Review-Finding: p1/red-team#1 reader=red-team
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* docs: the GST-1713 changelog fragment names the gh pr merge exception

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

---------

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
Garry Tanandcapy-ai[bot] authored and GitHub committed 2026-11-07 05:51:12 +00:00
1 parent 3742ee9527
commit aaff4741b9
42 files changed
+2044 -139

No files matched your search

+3 -3
View File
@@ -10,7 +10,7 @@
* ship_lanes_from names) at HEAD's clean tree. With --command, the base branch's mapping for that exact lane
* command supplies the profiles and the inventory the runs must cover (a pin-file command, so it needs
* --allow-repo-commands). Prints CI_CARRY: accepted <receipt> (exit 0) or CI_CARRY: none (<reason>) (exit 1).
* wait: polls the newest run of the profile's workflow at HEAD until it completes (default timeout 1800 s),
* wait: polls the newest run of the profile's workflow at HEAD (REST, at least 60 s apart) until it completes (default timeout 1800 s),
* prints CI_WAIT: green|red|running|none, and on green runs find. Exit 0 green and carried, 1 red or not
* carried, 3 still running or no run. Logic: lib/ci-carry.ts; the run checks are lib/run-evidence.ts.
*/
@@ -28,7 +28,7 @@ const HELP = `gstack-ci-carry — use a verified CI run at HEAD's exact tree ins
--policy-from head --simulate: the same check under the branch's own map and profiles, every line starting
SIMULATION:; a dry run that proves a new mapping before it lands (find never writes evidence; nothing clears)
wait [--base <branch>] [--profile <name>] [--timeout <s>] [--interval <s>]
poll the newest run at HEAD (default 1800 s, every 20 s): CI_WAIT: green|red|running|none, then find on green;
poll the newest run at HEAD (default 1800 s, every 60 s at least, plus jitter): CI_WAIT: green|red|running|none, then find on green;
exit 0 carried, 1 red or not carried, 3 still running or no run
Profiles and ship_lanes_from come from origin/<base>'s .gstack/pregate.json only (a simulation reads the branch's). Only a clean, committed tree
@@ -75,7 +75,7 @@ function find(): number {
if (cmd === 'find') process.exit(find());
const timeout = Number(flag('--timeout') ?? '1800');
const interval = Number(flag('--interval') ?? '20');
const interval = Number(flag('--interval') ?? '60');
if (!(timeout >= 0 && timeout <= 21_600) || !(interval >= 1 && interval <= 600)) usage('--timeout is 0-21600 seconds and --interval 1-600');
const profile = check.profiles[profiles[0]!];
if (!profile) usage(`${profiles[0]} is not a run_profiles name on origin/${base}`);
+2 -1
View File
@@ -32,6 +32,7 @@
*/
import { spawnSync } from 'node:child_process';
import * as path from 'node:path';
import { ghSpawnSync } from '../lib/gh-poll';
import { EXIT, exitTableHelp, formatResult } from '../lib/headless-artifacts';
import { versionAtRef } from '../lib/restamp';
import { RESULT_CODES } from '../lib/result-codes';
@@ -166,7 +167,7 @@ async function main(argv: string[]): Promise<number> {
const probe = spawnSync('git', ['rev-parse', '--show-toplevel'], { encoding: 'utf8', timeout: 15_000 });
if (probe.status !== 0) return usage('not inside a git repository');
const repoRoot = probe.stdout.trim();
const gh = (a: string[]) => { const r = spawnSync('gh', a, { cwd: repoRoot, encoding: 'utf8', timeout: 60_000 }); return { ok: r.status === 0, out: r.stdout ?? '', err: r.stderr ?? (r.error ? String(r.error.message) : '') }; };
const gh = (a: string[]) => { const r = ghSpawnSync(a, { cwd: repoRoot, encoding: 'utf8', timeout: 60_000 }); return { ok: r.status === 0, out: r.stdout ?? '', err: r.stderr ?? (r.error ? String(r.error.message) : '') }; };
const ctx: QueueContext = { repoRoot, base: resolveBaseBranch(repoRoot, flag(args, '--base')), gh };
const pr = flag(args, '--pr');
if (cmd === 'gate-ahead') return gateAhead(ctx, args);
+2 -1
View File
@@ -34,6 +34,7 @@ import { spawnSync } from 'node:child_process';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { ghSpawnSync } from '../lib/gh-poll';
import { EXIT, exitTableHelp, type ShipReceipt } from '../lib/headless-artifacts';
import { resultLine, resultLineWithDocs } from '../lib/result-codes';
import { versionAtRef } from '../lib/restamp';
@@ -167,7 +168,7 @@ function git(args: string[], cwd?: string): string | undefined {
return r.status === 0 ? r.stdout.trim() : undefined;
}
function gh(args: string[]): { ok: boolean; out: string; err: string } {
const r = spawnSync('gh', args, { encoding: 'utf8', timeout: 120_000, maxBuffer: 64 * 1024 * 1024 });
const r = ghSpawnSync(args, { encoding: 'utf8', timeout: 120_000, maxBuffer: 64 * 1024 * 1024 });
return { ok: r.status === 0, out: r.stdout ?? '', err: (r.stderr ?? '').trim() };
}
@@ -0,0 +1 @@
- **gstack stops draining the shared GitHub GraphQL quota.** `gstack-pr-wait` read the pull request through a GraphQL query and `gh pr checks` every 20 seconds (as often as every 5), about 360 GraphQL calls an hour per watcher, and the Capy app installation's 7,700-point hourly budget ran out right after each reset, stalling merges across crews. It now reads the PR state, base head and head-commit checks from REST, and makes its one GraphQL read (comments, reviews and review-thread resolution) only when the PR changed, every 10 minutes, or once green under `--until ready`. Every gstack wait between GitHub reads now goes through `lib/gh-poll.ts`, which puts a 60-second floor plus up to 15 seconds of jitter under it: `gstack-pr-wait` (was 20 s), `gstack-ci-gate --wait` and its registration re-poll (were 30 s and 10 s), `gstack-ci-carry wait` (was 20 s), `gstack-ship-release accept` (was 15 s), `gstack-greptile-early wait` (was 30 s, and its head read moved to REST), the merge-train and land waits (were 30 s, and 3 s for land's merge readback) and eval-bg's run follow (was 10 to 30 s). A gh call that fails with a rate-limit message is no longer retried at once. The gh runners read `gh api rate_limit`, sleep until that budget resets (at least 60 s) and retry at most twice. The one exception is `gh pr merge`, which runs under the land lock and still fails at once. `test/gh-poll-tripwire.test.ts` fails on any new wait in a GitHub-reading module that skips the floor. (GST-1713)
+1 -1
View File
@@ -2378,7 +2378,7 @@ restamping`, GST-1245), so a reset stamp needs no hand cleanup:
keeps the reviewed pin apart from the last pushed head, so a rerun leases
on what it pushed and never on a head someone else pushed
(`QUEUE_TRAIN_MOVED`, never overwritten).
5. **Wait** for the PR's checks on the stamp commit, polling every 30 seconds
5. **Wait** for the PR's checks on the stamp commit, polling every 60 seconds or more (lib/gh-poll.ts)
with the same check read the pin passed (`--timeout <min>`, default 60). `--no-wait`
stops after the push and prints `gstack-pr-wait <n> --until green &&
gstack-ship-queue land --pr <n>` to run instead.
+1 -1
View File
@@ -20,7 +20,7 @@ Beyond the slash-command skills, gstack ships standalone CLIs for workflows that
| `gstack-review-read` | **Review freshness** — emits review records with computed `review_freshness.status` and `reason`: CURRENT, STALE, or UNVERIFIED for diff reviews. `/ship` and `/land-and-deploy` use the same grade; a matching commit alone never certifies a diff review. [Dashboard rules](skills.md#review-readiness-dashboard). |
| `gstack-evidence` | **Verification-evidence ledger** — `run --label <lane> -- <cmd>` transparently wraps any test command (the child's exit code always passes through) and records what ran against which working-tree fingerprint (tracked and untracked files that are not gitignored, minus an untracked `.bun/` folder at the repo root, which Bun can create during a run); a run whose files changed while it ran is recorded without a fingerprint and with up to five of the changed paths, which `check` names in its STALE reason. `check` grades each label FRESH/STALE/MISSING with `--expect-cmd`, `--max-age`, and `--allow-paths` binding. A label is stored and looked up with every character outside `A-Za-z0-9._-` turned into `_`, so `pnpm lint` and `pnpm_lint` name the same row. `carried` lists the CI-sourced rows at the current tree that grade FRESH, which the ship receipt records as `ci_carried`. /ship and /land-and-deploy cite fresh evidence instead of re-running suites. Per-run logs are 0600, capped at 2MB, pruned after 30 days; the ledger and logs stay machine-local by design. |
| `gstack-flaky` | **Flaky-test rerun for any JUnit runner** — `run --junit <file.xml> -- <command>` runs your test command (Playwright, `bun test`, jest, vitest, pytest) and, when at most `--max-reruns` (3) tests failed and the runner made no retries of its own, reruns the whole command once (`--rerun-scope test` opts in to rerunning only the failed tests). A test that fails and then passes on that rerun is a flaky pass: the run stays green and one redacted line per flaky pass or failed rerun goes on a ledger (`GSTACK_FLAKE_LEDGER_JUNIT`, default `<state root>/flake-ledger-junit.jsonl`). A missing or malformed report, a crash or a killed runner never passes. `quarantine add|renew|release|check|report` keeps `.gstack/quarantine.json`: a quarantined test still runs, gets up to 3 attempts of the whole command, passes only on a full green attempt and is never counted as a pass (`run --base <ref>` reads the registry from the base branch). `ids --junit <file>` prints a report's test ids. Exit 0 pass, 1 a test failed, 2 usage, 3 no usable report. [Guide](flaky.md). |
| `gstack-pr-wait` | **PR event watcher** — `gstack-pr-wait <pr> [--repo owner/name] [--until green\|ready\|any-change] [--interval 20] [--timeout 60] [--exclude CHECK,...]` polls one pull request through `gh` and prints one stdout line per real event (a check failed with its link, all checks finished, a comment or review from someone other than the author, a review bot finished on the head, the base moved and its new commits touch this PR's files, BEHIND or DIRTY, merged or closed), with progress on stderr, so a monitor wakes the agent once per event instead of on a timer. `green` gates on every check on the head (an advisory bot's review is never waited for, its checks gate like any other), as `gstack-ci-gate` does; a check the base branch requires (rulesets and classic branch protection) is pending until it appears, never green by omission; two identical polls confirm a check set is complete; `--exclude` leaves a named non-required check out of the verdict as `gstack-ci-gate --exclude` does (a required name is refused and gates). Exit 0 when the condition is met, 1 when it needs action or `--timeout` passes (`PR_WAIT_BLOCKED`), 2 on bad input, a second watcher for the same PR or a local error (`PR_WAIT_REFUSED`). State under `<state root>/pr-wait/`, baseline discarded after 6 hours. Review bots are configuration: `gstack-config set pr_wait_gating_bots greptile,macroscope`, `pr_wait_advisory_bots coderabbit` (advisory bots are reported, never waited on). `/land-and-deploy` Step 3 is one call of it. Design ported from tstack's pr-wait (MIT, Theo Browne; NOTICE.md). [Exit codes](troubleshooting.md#gstack-pr-wait). |
| `gstack-pr-wait` | **PR event watcher** — `gstack-pr-wait <pr> [--repo owner/name] [--until green\|ready\|any-change] [--interval 60] [--timeout 60] [--exclude CHECK,...]` polls one pull request through `gh` and prints one stdout line per real event (a check failed with its link, all checks finished, a comment or review from someone other than the author, a review bot finished on the head, the base moved and its new commits touch this PR's files, BEHIND or DIRTY, merged or closed), with progress on stderr, so a monitor wakes the agent once per event instead of on a timer. `green` gates on every check on the head (an advisory bot's review is never waited for, its checks gate like any other), as `gstack-ci-gate` does; a check the base branch requires (rulesets and classic branch protection) is pending until it appears, never green by omission; two identical polls confirm a check set is complete; `--exclude` leaves a named non-required check out of the verdict as `gstack-ci-gate --exclude` does (a required name is refused and gates). Exit 0 when the condition is met, 1 when it needs action or `--timeout` passes (`PR_WAIT_BLOCKED`), 2 on bad input, a second watcher for the same PR or a local error (`PR_WAIT_REFUSED`). State under `<state root>/pr-wait/`, baseline discarded after 6 hours. Review bots are configuration: `gstack-config set pr_wait_gating_bots greptile,macroscope`, `pr_wait_advisory_bots coderabbit` (advisory bots are reported, never waited on). `/land-and-deploy` Step 3 is one call of it. Design ported from tstack's pr-wait (MIT, Theo Browne; NOTICE.md). [Exit codes](troubleshooting.md#gstack-pr-wait). |
| `gstack-review-bots` | **Review-bot comments** — `fetch [--pr <n>]` reads every root line comment, conversation comment and newest review body from each bot in `pr_wait_gating_bots` and `pr_wait_advisory_bots` (Bot accounts only, all pages, through `gh`) into a private directory with per-bot counts, or prints `REVIEW_BOTS: skip` or `REVIEW_BOTS: error` (never a false zero). `record --dir <dir> --id <comment id> --type fix\|fp\|already-fixed --category <category>` appends one triage outcome to the project and global `review-bot-history.md`, taking the bot and file from the fetched comment. `/review` Step 2.5 and `/ship` Step 10 run it. [Triage reference](../review/greptile-triage.md). |
| `gstack-issue-guard` | **Tracker-text trust envelope** — fetches GitHub issue/PR text (`issue <n>`, `pr-body`, `pr-comments`, or `--stdin`) and wraps it in a labeled envelope so agents treat it as data: injection-shaped lines get labeled even through fullwidth and invisible-character evasion, and forged envelope banners are defused. Every tracker-text ingress in gstack routes through it, enforced by a CI scanner. |
| `gstack-ios-qa-daemon` | **iOS QA daemon** — Mac-side broker between an agent and a connected iPhone or iPad over USB CoreDevice. Loopback by default; `--tailnet` opens a Tailscale-facing listener with identity-gated capability tiers. Single-instance via flock on `~/.gstack/ios-qa-daemon.pid`. See [docs/howto-ios-testing-with-gstack.md](howto-ios-testing-with-gstack.md). |
+2 -2
View File
@@ -407,7 +407,7 @@ gh run list --repo "$REPO" --branch "$BASE_BRANCH" --limit 10 --json databaseId,
Match `DEPLOY_SHA`, workflow and target environment. If no run appears yet, repeat
the lookup within the same 20-minute deadline. A name match on another SHA is not evidence.
Poll every 30 seconds:
Poll every 60 seconds:
```bash
gh run view <run-id> --repo "$REPO" --json status,conclusion
```
@@ -594,7 +594,7 @@ Recheck read-only evidence; never redo a merge/deploy because a section was miss
- Never force-push, bypass CI, replay a confirmed merge, or hide missing evidence.
- Auto-detect facts; ask when unknown or an explicit approval gate applies.
- Step 3 is one `gstack-pr-wait` call; elsewhere poll at the stated interval (30 s for merge readback and deploy) with progress lines.
- Step 3 is one `gstack-pr-wait` call; elsewhere poll at the stated interval (60 s for merge readback and deploy) with progress lines.
- After merge failures, offer approved rollback when appropriate; never revert a rollback automatically.
- Verify once; `/canary` does extended monitoring; rechecks need the user's choice.
- Use `--delete-branch`; reconcile failed cleanup non-destructively with confirmation.
+2 -2
View File
@@ -255,7 +255,7 @@ gh run list --repo "$REPO" --branch "$BASE_BRANCH" --limit 10 --json databaseId,
Match `DEPLOY_SHA`, workflow and target environment. If no run appears yet, repeat
the lookup within the same 20-minute deadline. A name match on another SHA is not evidence.
Poll every 30 seconds:
Poll every 60 seconds:
```bash
gh run view <run-id> --repo "$REPO" --json status,conclusion
```
@@ -441,7 +441,7 @@ Recheck read-only evidence; never redo a merge/deploy because a section was miss
- Never force-push, bypass CI, replay a confirmed merge, or hide missing evidence.
- Auto-detect facts; ask when unknown or an explicit approval gate applies.
- Step 3 is one `gstack-pr-wait` call; elsewhere poll at the stated interval (30 s for merge readback and deploy) with progress lines.
- Step 3 is one `gstack-pr-wait` call; elsewhere poll at the stated interval (60 s for merge readback and deploy) with progress lines.
- After merge failures, offer approved rollback when appropriate; never revert a rollback automatically.
- Verify once; `/canary` does extended monitoring; rechecks need the user's choice.
- Use `--delete-branch`; reconcile failed cleanup non-destructively with confirmation.
+1 -1
View File
@@ -164,7 +164,7 @@ merge queue is in use. Explain which is observed; an auto request alone does not
prove a queue. A queue reruns CI against the proposed merge. Record `MERGE_PATH=queue`
only when `mergeQueueEntry` was observed, otherwise `auto`.
Set `WAITED=true`. Repeat the readback every 30 seconds, up to 30 minutes; report progress every 2 minutes.
Set `WAITED=true`. Repeat the readback every 60 seconds, up to 30 minutes; report progress every 2 minutes.
While OPEN with an active auto request **or** queue entry, keep waiting. Once waiting
has begun, never dispatch START or DIRECT: OPEN with confirmed absence of **both**
means removal/cancellation, so **STOP** and point to GitHub's checks/queue page.
@@ -162,7 +162,7 @@ merge queue is in use. Explain which is observed; an auto request alone does not
prove a queue. A queue reruns CI against the proposed merge. Record `MERGE_PATH=queue`
only when `mergeQueueEntry` was observed, otherwise `auto`.
Set `WAITED=true`. Repeat the readback every 30 seconds, up to 30 minutes; report progress every 2 minutes.
Set `WAITED=true`. Repeat the readback every 60 seconds, up to 30 minutes; report progress every 2 minutes.
While OPEN with an active auto request **or** queue entry, keep waiting. Once waiting
has begun, never dispatch START or DIRECT: OPEN with confirmed absence of **both**
means removal/cancellation, so **STOP** and point to GitHub's checks/queue page.
+10 -6
View File
@@ -23,12 +23,13 @@
* suite as before. Security and advisory scans and the sealed guard are never
* carried: callers carry test lanes only.
*
* waitForRun polls the newest run of one workflow on HEAD until it completes,
* bounded by a timeout, and returns green, red with the failing jobs, still
* running, or none.
* waitForRun polls the newest run of one workflow on HEAD (REST) until it
* completes, at least 60 s apart (lib/gh-poll.ts), bounded by a timeout, and
* returns green, red with the failing jobs, still running, or none.
*/
import * as fs from 'node:fs';
import * as path from 'node:path';
import { pollDelayMs } from './gh-poll';
import { normalPath, runChanged, runInventory, runReceiptFields, uncoveredFiles, verifyRun, type ApiRun, type RunDeps, type RunEvidence, type RunProfile, type RunSource } from './run-evidence';
export type CarryRefusal = 'dirty_tree' | 'no_profile' | 'no_run' | 'no_verified_run' | 'tree_mismatch' | 'no_tests'
@@ -136,8 +137,11 @@ export function findCarry(src: RunSource, o: { profiles: string[]; head: string;
export type WaitVerdict = { state: 'green'; run: ApiRun } | { state: 'red'; run: ApiRun; failing: string[] } | { state: 'running'; run: ApiRun } | { state: 'none'; reason: string };
const PASSING = ['success', 'skipped', 'neutral'];
/** Poll the newest `workflow` run on `head` (among `events`) until it completes or `timeoutMs` passes. */
export async function waitForRun(deps: RunDeps, o: { workflow: string; events: string[]; head: string; timeoutMs: number; intervalMs: number; sleep: (ms: number) => Promise<void>; clock: () => number }): Promise<WaitVerdict> {
/**
* Poll the newest `workflow` run on `head` (among `events`) until it completes or `timeoutMs` passes; polls are at least 60 s
* apart, so the last poll can land up to one poll interval (75 s with jitter) after `timeoutMs`.
*/
export async function waitForRun(deps: RunDeps, o: { workflow: string; events: string[]; head: string; timeoutMs: number; intervalMs: number; sleep: (ms: number) => Promise<void>; clock: () => number; random?: () => number }): Promise<WaitVerdict> {
const end = o.clock() + o.timeoutMs;
for (;;) {
const newest = deps.runsForHead(o.workflow, o.head).filter(r => o.events.includes(r.event)).sort(newestFirst)[0];
@@ -147,7 +151,7 @@ export async function waitForRun(deps: RunDeps, o: { workflow: string; events: s
return { state: 'red', run: newest, failing };
}
if (o.clock() >= end) return newest ? { state: 'running', run: newest } : { state: 'none', reason: `no ${o.workflow} run on ${o.head.slice(0, 12)} after ${Math.round(o.timeoutMs / 1000)} s` };
await o.sleep(Math.max(1, Math.min(o.intervalMs, end - o.clock())));
await o.sleep(pollDelayMs(o.intervalMs, o.random));
}
}
+13 -9
View File
@@ -1,4 +1,5 @@
import { spawnSync } from 'node:child_process';
import { MIN_POLL_MS, pollDelayMs, syncLimitEnv, withRateLimit } from './gh-poll';
export type CiVerdict = 'PASS' | 'FAIL' | 'PENDING' | 'NO_CHECKS' | 'ERROR';
export const CI_GATE_EXIT: Record<CiVerdict, number> = { PASS: 0, FAIL: 1, PENDING: 2, NO_CHECKS: 3, ERROR: 4 };
@@ -57,10 +58,12 @@ Options:
--override-head SHA head the exclusions were approved for; must equal
--expect-head
--wait SECONDS keep polling while PENDING, up to SECONDS (default 0)
--interval SECONDS poll interval (default 30)
--interval SECONDS poll interval (default and minimum 60, plus jitter;
a smaller value is raised to 60)
--registration-wait S when no check exists yet, re-poll up to S seconds
before reporting NO_CHECKS (default 0; use 60 right
after a push, when checks may not be registered yet)
A GitHub rate limit waits for its reset, never retries at once (GST-1713).
--help show this help`;
export function bucketClass(bucket: string): Klass {
@@ -201,6 +204,7 @@ export async function runCiGate(
gh: GhRunner,
sleep: (ms: number) => Promise<void> = ms => new Promise(resolve => setTimeout(resolve, ms)),
now: () => number = Date.now,
random: () => number = Math.random,
): Promise<CiGateResult> {
if (opts.excludes.length > 0 && opts.overrideHead !== opts.expectHead) {
return {
@@ -210,13 +214,12 @@ export async function runCiGate(
}
const started = now();
const elapsed = () => (now() - started) / 1000;
const step = Math.max(opts.intervalSeconds, MIN_POLL_MS / 1000);
let result = evaluateOnce(gh, opts);
while (true) {
if (result.verdict === 'NO_CHECKS' && elapsed() + Math.min(opts.intervalSeconds, 10) <= opts.registrationWaitSeconds) {
await sleep(Math.min(opts.intervalSeconds, 10) * 1000);
} else if (result.verdict === 'PENDING' && elapsed() + opts.intervalSeconds <= opts.waitSeconds) {
await sleep(opts.intervalSeconds * 1000);
} else break;
const window = result.verdict === 'NO_CHECKS' ? opts.registrationWaitSeconds : result.verdict === 'PENDING' ? opts.waitSeconds : 0;
if (elapsed() >= window) break;
await sleep(pollDelayMs(step * 1000, random));
result = evaluateOnce(gh, opts);
}
if (result.verdict === 'NO_CHECKS') {
@@ -230,7 +233,7 @@ export async function runCiGate(
export function parseCiGateArgs(argv: string[]): CiGateOptions | { help: true } | { error: string; expectHead?: string } {
const opts: CiGateOptions = {
repo: '', pr: '', expectHead: '', excludes: [], waitSeconds: 0, intervalSeconds: 30, registrationWaitSeconds: 0,
repo: '', pr: '', expectHead: '', excludes: [], waitSeconds: 0, intervalSeconds: MIN_POLL_MS / 1000, registrationWaitSeconds: 0,
};
const seconds = (value: string | undefined, flag: string, positive = false): number | string => {
if (value === undefined || !/^\d+(?:\.\d+)?$/.test(value)) return `${flag} needs a number of seconds`;
@@ -270,14 +273,15 @@ export function parseCiGateArgs(argv: string[]): CiGateOptions | { help: true }
return opts;
}
export const spawnGh: GhRunner = args => {
/** gh as a GhRunner, with gh-poll's rate-limit rule: a rate-limited call sleeps until the reset instead of failing or retrying at once. */
export const spawnGh: GhRunner = withRateLimit(args => {
const result = spawnSync('gh', args, { encoding: 'utf8', timeout: 60_000, maxBuffer: 16 * 1024 * 1024 });
if (result.error) {
const code = (result.error as NodeJS.ErrnoException).code;
return { status: null, stdout: '', stderr: result.stderr ?? '', error: code === 'ENOENT' ? 'gh is not installed or not on PATH' : `gh failed to run (${code ?? result.error.message})` };
}
return { status: result.status, stdout: result.stdout, stderr: result.stderr };
};
}, syncLimitEnv);
export async function ciGateMain(argv: string[], gh: GhRunner = spawnGh, write: (text: string) => void = text => process.stdout.write(text)): Promise<number> {
const parsed = parseCiGateArgs(argv);
+180
View File
@@ -0,0 +1,180 @@
/**
* gh-poll: the one policy for how gstack reads GitHub repeatedly (GST-1713).
*
* The Capy GitHub app installation shares one GraphQL budget across every
* crew, and `gh pr view|list|checks` spend it. Three rules live here:
*
* pollDelayMs every wait between two GitHub reads is at least
* MIN_POLL_MS (60 s) plus up to 15 s of jitter, whatever
* interval a caller or flag asks for.
* rate limits a gh call that fails with a rate-limit message (403/429,
* "API rate limit exceeded", "GraphQL: API rate limit",
* a secondary limit) is never retried at once: the helper
* reads `gh api rate_limit` (exempt from the limit), sleeps
* until that resource resets (at least 60 s), and retries at
* most MAX_LIMIT_WAITS times before returning the failure.
* REST reads pull request state and head-commit checks come from REST
* (`repos/{o}/{r}/pulls/{n}`, `commits/{sha}/check-runs`,
* `commits/{sha}/status`), which never touch the GraphQL budget.
*
* test/gh-poll-tripwire.test.ts keeps every sleep in a GitHub-reading module
* on pollDelayMs. GSTACK_GH_POLL_FLOOR_MS is a test seam only (fixtures that
* drive real binaries with millisecond polls); no shipped file may set it.
*/
import { spawnSync, type SpawnSyncOptionsWithStringEncoding, type SpawnSyncReturns } from 'node:child_process';
import type { GhResult, GhRunner } from './ci-gate';
export const MIN_POLL_MS = 60_000;
export const MAX_LIMIT_WAITS = 2;
/** The longest a rate-limit wait may last: one full hourly window plus slack. */
const MAX_LIMIT_WAIT_MS = 65 * 60_000;
const RESET_SLACK_MS = 5_000;
type Vars = Record<string, string | undefined>;
function floorMs(vars: Vars): number {
const seam = vars.GSTACK_GH_POLL_FLOOR_MS;
return seam !== undefined && /^\d+$/.test(seam) ? Number(seam) : MIN_POLL_MS;
}
/** The wait before the next GitHub read: at least the floor (60 s), plus jitter up to a quarter of the floor. */
export function pollDelayMs(requestedMs: number, random: () => number = Math.random, vars: Vars = process.env): number {
const floor = floorMs(vars);
return Math.max(Number.isFinite(requestedMs) ? requestedMs : 0, floor) + Math.round(random() * floor / 4);
}
export type RateLimitKind = 'graphql' | 'core' | 'secondary';
const SECONDARY = /secondary rate limit|abuse detection/i;
const GRAPHQL_LIMIT = /GraphQL: API rate limit|"type":\s*"RATE_LIMITED"/i;
const ANY_LIMIT = /API rate limit (?:already )?exceeded|rate limit exceeded|HTTP 429|\(HTTP 403\).*rate limit|rate limit.*\(HTTP 403\)/i;
/** gh subcommands that read through GraphQL; everything else (gh api <path>, gh run, gh workflow, gh release) is REST. */
const GRAPHQL_COMMANDS = new Set(['pr', 'issue', 'repo', 'search', 'project', 'label']);
/** Which budget a failed gh call ran out of, from its output, or null when the failure is not a rate limit. */
export function rateLimitKind(args: readonly string[], output: string): RateLimitKind | null {
if (SECONDARY.test(output)) return 'secondary';
if (GRAPHQL_LIMIT.test(output)) return 'graphql';
if (!ANY_LIMIT.test(output)) return null;
return (args[0] === 'api' && args[1] === 'graphql') || GRAPHQL_COMMANDS.has(args[0] ?? '') ? 'graphql' : 'core';
}
/**
* How long to sleep after a rate-limit failure: until the resource in `gh api rate_limit` resets (plus slack) when it
* reads remaining 0, else (a secondary limit, an unreadable reply) the poll floor. Never less than pollDelayMs(0).
*/
export function rateLimitWaitMs(kind: RateLimitKind, rateLimitJson: string, nowMs: number, random: () => number = Math.random, vars: Vars = process.env): number {
let untilReset = 0;
if (kind !== 'secondary') {
try {
const resource = (JSON.parse(rateLimitJson) as { resources?: Record<string, { remaining?: unknown; reset?: unknown }> }).resources?.[kind];
if (resource && resource.remaining === 0 && typeof resource.reset === 'number') untilReset = resource.reset * 1000 - nowMs + RESET_SLACK_MS;
} catch { /* unreadable: fall back to the floor */ }
}
return pollDelayMs(Math.min(Math.max(untilReset, 0), MAX_LIMIT_WAIT_MS), random, vars);
}
export interface LimitEnv { sleep(ms: number): void; now(): number; log(message: string): void; random?: () => number; vars?: Vars }
/**
* `gh pr merge` runs under the land lock, which another lander may break once the holder is 10 minutes into a merge
* (PUSH_STALE_MS): a rate-limited merge fails at once, as it did before the wait existed, instead of sleeping through it.
*/
const failsFast = (args: readonly string[]): boolean => args[0] === 'pr' && args[1] === 'merge';
/**
* Wrap a gh runner: a rate-limited failure sleeps until the reset and reruns the same call, at most MAX_LIMIT_WAITS
* times; any other result (success or a real failure) and a rate-limited `gh pr merge` return at once. The
* `gh api rate_limit` read goes through the inner runner, unwrapped.
*/
export function withRateLimit(gh: GhRunner, env: LimitEnv): GhRunner {
return args => {
for (let waits = 0; ; waits++) {
const result = gh(args);
const kind = result.status === 0 ? null : rateLimitKind(args, `${result.stderr}\n${result.stdout}`);
if (!kind || waits >= MAX_LIMIT_WAITS || failsFast(args)) return result;
const limits = gh(['api', 'rate_limit']);
const ms = rateLimitWaitMs(kind, limits.status === 0 ? limits.stdout : '', env.now(), env.random, env.vars);
env.log(`GitHub ${kind} rate limit on gh ${args.slice(0, 2).join(' ')}; waiting ${Math.round(ms / 1000)} s for the reset before one retry (${waits + 1}/${MAX_LIMIT_WAITS})`);
env.sleep(ms);
}
};
}
/** The CLI environment: a blocking sleep (gh runners are synchronous) and the wait note on stderr. */
export const syncLimitEnv: LimitEnv = {
sleep: ms => { Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); },
now: () => Date.now(),
log: message => { process.stderr.write(`gstack-gh: ${message}\n`); },
};
/** Drop-in for `spawnSync('gh', args, options)` with the rate-limit rule; the wait note goes to stderr. */
export function ghSpawnSync(args: string[], options: SpawnSyncOptionsWithStringEncoding): SpawnSyncReturns<string> {
let last: SpawnSyncReturns<string> | undefined;
const run = withRateLimit(a => {
const r = spawnSync('gh', a, options);
if (a === args) last = r;
return { status: r.status, stdout: r.stdout ?? '', stderr: r.stderr ?? '', ...(r.error ? { error: String(r.error.message) } : {}) };
}, syncLimitEnv);
run(args);
return last!;
}
type Json = Record<string, unknown>;
const obj = (value: unknown): Json => (value && typeof value === 'object' && !Array.isArray(value) ? (value as Json) : {});
const str = (value: unknown): string => (typeof value === 'string' ? value : '');
/** A pull request's state from REST `repos/{o}/{r}/pulls/{n}`, in GraphQL's spellings (MERGED/CLOSED/OPEN, BEHIND/DIRTY/...). */
export interface PullState { url: string; state: string; headRefOid: string; baseRefName: string; mergeStateStatus: string; author: string; updatedAt: string }
export function parsePullState(json: string): PullState | null {
let raw: Json;
try { raw = obj(JSON.parse(json)); } catch { return null; }
const head = str(obj(raw.head).sha);
if (!head) return null;
return {
url: str(raw.html_url),
state: raw.merged === true || str(raw.merged_at) ? 'MERGED' : str(raw.state) === 'closed' ? 'CLOSED' : 'OPEN',
headRefOid: head,
baseRefName: str(obj(raw.base).ref),
mergeStateStatus: (str(raw.mergeable_state) || 'unknown').toUpperCase(),
author: str(obj(raw.user).login).replace(/\[bot\]$/, '') || 'ghost',
updatedAt: str(raw.updated_at),
};
}
/** One check on a commit in `gh pr checks` bucket spelling: pass, fail, pending, skipping, neutral, cancel. */
export interface HeadCheck { name: string; bucket: string; link: string; description: string }
const CONCLUSION_BUCKET: Record<string, string> = {
success: 'pass', failure: 'fail', timed_out: 'fail', action_required: 'fail', startup_failure: 'fail',
cancelled: 'cancel', skipped: 'skipping', neutral: 'skipping',
};
const STATUS_BUCKET: Record<string, string> = { success: 'pass', failure: 'fail', error: 'fail', pending: 'pending' };
/**
* Every check on a commit from the REST `check_runs` and combined-status `statuses` entries, in `gh pr checks`
* buckets: the newest run per check suite and name (a re-run attempt supersedes the run it replaced; two workflow runs
* of one job, push and pull_request, stay two rows for the caller to collapse) and every status context. An unknown
* conclusion keeps its own spelling, which every bucket reader counts as failed.
*/
export function parseHeadChecks(runs: readonly unknown[], statuses: readonly unknown[]): HeadCheck[] {
const newest = new Map<string, Json>();
const rank = (r: Json) => (str(r.status) !== 'completed' ? Infinity : Date.parse(str(r.completed_at)) || 0);
for (const r of runs.map(obj)) {
const key = `${String(obj(r.check_suite).id ?? obj(r.app).id ?? '')}\0${str(r.name)}`;
const have = newest.get(key);
if (!have || rank(r) > rank(have) || (rank(r) === rank(have) && Number(r.id) > Number(have.id))) newest.set(key, r);
}
const checks: HeadCheck[] = [...newest.values()].filter(r => str(r.name)).map(r => ({
name: str(r.name),
bucket: str(r.status) !== 'completed' ? 'pending' : CONCLUSION_BUCKET[str(r.conclusion)] ?? (str(r.conclusion) || 'pending'),
link: str(r.details_url) || str(r.html_url),
description: str(obj(r.output).title),
}));
for (const s of statuses.map(obj)) {
if (!str(s.context)) continue;
checks.push({ name: str(s.context), bucket: STATUS_BUCKET[str(s.state)] ?? str(s.state), link: str(s.target_url), description: str(s.description) });
}
return checks;
}
+6 -5
View File
@@ -14,8 +14,8 @@
* open --base B --title-file F [--ready] [--confirm T]
* open the early PR through gstack-post and post one
* `@greptileai` comment when Greptile skips drafts
* wait <pr> --since <epoch> poll (30 s) for a completed Greptile review of the
* PR's head commit; returns within ~90 s per call,
* wait <pr> --since <epoch> poll (60 s, REST only) for a completed Greptile review
* of the PR's head commit; returns within ~90 s per call,
* `timeout` once 10 minutes have passed since <epoch>
*
* bin/gstack-greptile-early is the CLI; tests drive `runGreptileEarly` with a
@@ -26,6 +26,7 @@ import * as path from "path";
import { resolveStateRoot, readConfigKey, type StateRootEnv } from "./state-root";
import { botLists, botOwning } from "./pr-wait";
import { remoteSlug } from "./remote-identity";
import { MIN_POLL_MS, pollDelayMs } from "./gh-poll";
export interface EarlyEnv {
run(cmd: string, args: string[], input?: string): { status: number; stdout: string; stderr: string };
@@ -41,7 +42,7 @@ export interface EarlyEnv {
}
export const WAIT_CAP_SECONDS = 600;
export const POLL_SECONDS = 30;
export const POLL_SECONDS = MIN_POLL_MS / 1000;
const CALL_BUDGET_SECONDS = 90;
const COMMENT_WINDOW_DAYS = 90;
const TRIGGER = "@greptileai";
@@ -228,7 +229,7 @@ function wait(env: EarlyEnv, args: string[]): number {
const pr = args[0];
const since = Number(flag(args, "--since"));
if (!pr || !/^[1-9][0-9]*$/.test(pr) || !Number.isInteger(since) || since <= 0) { env.err("usage: gstack-greptile-early wait <pr-number> --since <epoch-seconds>\n"); return 64; }
const head = env.run("gh", ["pr", "view", pr, "--json", "headRefOid", "--jq", ".headRefOid"]);
const head = env.run("gh", ["api", `repos/{owner}/{repo}/pulls/${pr}`, "--jq", ".head.sha"]);
const sha = head.stdout.trim();
if (head.status !== 0 || !/^[0-9a-f]{40}$/.test(sha)) { env.out("GREPTILE_REVIEW: unavailable (could not read the PR head commit)\n"); return 0; }
const started = env.now();
@@ -240,7 +241,7 @@ function wait(env: EarlyEnv, args: string[]): number {
const mmss = `${Math.floor(elapsed / 60)}m${String(elapsed % 60).padStart(2, "0")}s`;
env.out(`Greptile: waiting for its review of ${sha.slice(0, 7)} (${mmss} of ${WAIT_CAP_SECONDS / 60}m). Turn early review off: ${config(env)} set ship_greptile_early false\n`);
if ((env.now() - started) / 1000 + POLL_SECONDS > CALL_BUDGET_SECONDS) { env.out("GREPTILE_REVIEW: pending\n"); return 0; }
env.sleep(POLL_SECONDS * 1000);
env.sleep(pollDelayMs(POLL_SECONDS * 1000));
}
}
+61 -55
View File
@@ -1,10 +1,11 @@
/**
* pr-wait-github — the GitHub read side of gstack-pr-wait: a gh runner that
* retries transient failures, the one GraphQL read of a pull request, its
* checks from `gh pr checks`, the required-check names from rulesets and
* classic branch protection, and the base-branch comparisons. The event logic
* is lib/pr-wait.ts; the two are split so each stays under the module-size
* ratchet.
* retries transient failures (never a rate limit, GST-1713), the pull
* request's state, base head and head-commit checks from REST, the one
* GraphQL read of its conversation (comments, reviews, review threads), the
* required-check names from rulesets and classic branch protection, and the
* base-branch comparisons. The event logic is lib/pr-wait.ts; the two are
* split so each stays under the module-size ratchet.
*
* Design ported from tstack's `pr-wait` (t3dotgg/tstack,
* skills/babysit-pr/scripts/pr-wait at 7de017b, MIT, Copyright (c) 2026 Theo
@@ -12,6 +13,7 @@
* gstack's gh runner shape (lib/ci-gate.ts).
*/
import type { GhResult, GhRunner } from './ci-gate';
import { parseHeadChecks, parsePullState, rateLimitKind, type PullState } from './gh-poll';
export class GhError extends Error {
constructor(message: string, readonly transient: boolean) {
@@ -32,22 +34,12 @@ export interface Actor { login: string; bot: boolean }
export interface Comment { id: string; url: string; body: string; author: Actor }
export interface Review { id: string; url: string; state: string; body: string; commitOid: string | null; author: Actor }
export interface Thread { isResolved: boolean; path: string; line: number | null; comments: Comment[] }
export interface PullRequest {
url: string;
state: string;
headRefOid: string;
baseRefName: string;
mergeStateStatus: string;
baseOid: string | null;
author: string;
viewer: string;
comments: Comment[];
reviews: Review[];
threads: Thread[];
}
/** What people said on the PR: the one GraphQL read, which REST cannot answer (review-thread resolution). */
export interface Conversation { viewer: string; comments: Comment[]; reviews: Review[]; threads: Thread[] }
export interface PullRequest extends PullState, Conversation { baseOid: string | null }
export interface Check { name: string; bucket: string; link: string; description: string }
const TRANSIENT = /HTTP (5\d\d|429)|rate limit|timed? ?out|connection (reset|refused)|EOF|temporar|could not resolve host|TLS handshake|Something went wrong/i;
const TRANSIENT = /HTTP 5\d\d|timed? ?out|connection (reset|refused)|EOF|temporar|could not resolve host|TLS handshake|Something went wrong/i;
const RETRIES = 5;
const PASS_BUCKETS = new Set(['pass', 'skipping', 'neutral']);
/** Worst bucket first, for collapsing duplicate check names (push and pull_request runs of one job). */
@@ -58,9 +50,6 @@ query($owner: String!, $name: String!, $number: Int!) {
viewer { login }
repository(owner: $owner, name: $name) {
pullRequest(number: $number) {
url state headRefOid baseRefName mergeStateStatus
author { login }
baseRef { target { oid } }
comments(last: 100) { nodes { id url body author { login __typename } } }
reviews(last: 100) {
nodes { id url state body commit { oid } author { login __typename } }
@@ -85,7 +74,11 @@ function lastLine(text: string): string {
return text.trim().split(/\r?\n/).filter(Boolean).at(-1) ?? '';
}
/** Run gh. A transient failure (HTTP 5xx or 429, network) is retried with doubling backoff. */
/**
* Run gh. A transient failure (HTTP 5xx, network) is retried with doubling backoff. A rate limit is never retried
* here: it throws a transient GhError at once, and the poll loop waits its full interval (the production runner,
* lib/gh-poll.ts withRateLimit, has already slept until the reset).
*/
export function runGh(env: GhEnv, args: string[], allow: number[] = [0]): GhResult {
let delay = 2_000;
for (let attempt = 1; ; attempt++) {
@@ -93,11 +86,13 @@ export function runGh(env: GhEnv, args: string[], allow: number[] = [0]): GhResu
// gh missing is fatal; any other spawn failure (its 60 s timeout, a signal) is a slow API, so retry.
if (result.error && /not installed/.test(result.error)) throw new GhError(result.error, false);
const status = result.status ?? -1;
if (status !== 0 && rateLimitKind(args, `${result.stderr}\n${result.stdout}`)) throw new GhError(lastLine(result.stderr) || 'GitHub rate limit', true);
const transient = status !== 0 && (result.error !== undefined || result.status === null || TRANSIENT.test(result.stderr));
if (!transient && allow.includes(status)) return result;
const message = result.error ?? (lastLine(result.stderr) || `gh exited ${status}`);
if (!transient || attempt === RETRIES) throw new GhError(message, transient);
env.log(`gh ${args.slice(0, 2).join(' ')}: ${message}; retry in ${delay / 1000}s`);
// gh-poll-exempt: a 5xx or network retry of one read (2, 4, 8, 16 s), never a rate limit (thrown above)
env.sleep(delay);
delay *= 2;
}
@@ -125,31 +120,43 @@ export function currentRepo(env: GhEnv): string {
return runGh(env, ['repo', 'view', '--json', 'nameWithOwner', '--jq', '.nameWithOwner']).stdout.trim();
}
/** Confirms the PR exists and is readable; throws GhError otherwise. */
/** Confirms the PR exists and is readable (REST); throws GhError otherwise. */
export function confirmPullRequest(env: GhEnv, repo: string, pr: number): void {
runGh(env, ['pr', 'view', String(pr), '-R', repo, '--json', 'number']);
runGh(env, ['api', `repos/${repo}/pulls/${pr}`, '--jq', '.number']);
}
export function fetchPullRequest(env: GhEnv, repo: string, pr: number): PullRequest {
/** The PR's state, head, base and merge state from REST `pulls/{n}`: no GraphQL. */
export function fetchPullState(env: GhEnv, repo: string, pr: number): PullState {
const state = parsePullState(runGh(env, ['api', `repos/${repo}/pulls/${pr}`]).stdout);
if (!state) throw new GhError(`${repo}#${pr} is not a pull request GitHub returned`, false);
return state;
}
/** The base branch's current head commit (REST), or null when unreadable. */
export function fetchBaseOid(env: GhEnv, repo: string, base: string): string | null {
try {
return runGh(env, ['api', `repos/${repo}/git/ref/heads/${base}`, '--jq', '.object.sha']).stdout.trim() || null;
} catch (error) {
if (error instanceof GhError && error.transient) throw error;
return null;
}
}
/** The conversation: the one GraphQL read, made only when lib/pr-wait.ts decides it is stale. */
export function fetchConversation(env: GhEnv, repo: string, pr: number): Conversation {
const [owner, name] = repo.split('/');
const result = runGh(env, ['api', 'graphql', '-f', `query=${PR_QUERY}`, '-f', `owner=${owner}`, '-f', `name=${name}`, '-F', `number=${pr}`]);
let parsed: Json;
try { parsed = obj(JSON.parse(result.stdout)); } catch {
throw new GhError('gh api graphql printed output that is not JSON', false);
}
const node = obj(obj(obj(parsed.data).repository).pullRequest);
if (!str(node.headRefOid)) {
const pullRequest = obj(obj(parsed.data).repository).pullRequest;
if (!pullRequest || typeof pullRequest !== 'object') {
const errors = Array.isArray(parsed.errors) ? parsed.errors.map(e => str(obj(e).message)).filter(Boolean).join('; ') : '';
throw new GhError(errors || `${repo}#${pr} is not a pull request GitHub returned`, false);
}
const node = obj(pullRequest);
return {
url: str(node.url),
state: str(node.state),
headRefOid: str(node.headRefOid),
baseRefName: str(node.baseRefName),
mergeStateStatus: str(node.mergeStateStatus) || 'UNKNOWN',
baseOid: str(obj(obj(node.baseRef).target).oid) || null,
author: str(obj(node.author).login) || 'ghost',
viewer: str(obj(obj(parsed.data).viewer).login),
comments: nodes(node.comments).map(comment),
reviews: nodes(node.reviews).map(r => ({ ...comment(r), state: str(r.state), commitOid: str(obj(r.commit).oid) || null })),
@@ -162,29 +169,28 @@ export function fetchPullRequest(env: GhEnv, repo: string, pr: number): PullRequ
};
}
/** Every entry of a paginated REST list, read with `--paginate --jq '.<key>[]'` (one JSON value per line). */
function restList(env: GhEnv, path: string, key: string): unknown[] {
const lines = runGh(env, ['api', '--paginate', path, '--jq', `.${key}[]`]).stdout.split(/\r?\n/).filter(Boolean);
try { return lines.map(line => JSON.parse(line)); } catch {
throw new GhError(`gh api ${path} printed output that is not JSON`, false);
}
}
/**
* Every check on the PR head by name, duplicates collapsed to the worst
* bucket. `gh pr checks` exits 8 while checks run and 1 when one failed:
* those are states, so the JSON is read whatever the exit status.
* Every check on the head commit by name, from REST check runs and commit
* statuses (the same rows `gh pr checks` reads through GraphQL), duplicates
* collapsed to the worst bucket.
*/
export function fetchChecks(env: GhEnv, repo: string, pr: number): Map<string, Check> {
const result = runGh(env, ['pr', 'checks', String(pr), '-R', repo, '--json', 'name,bucket,link,description'], [0, 1, 8]);
export function fetchChecks(env: GhEnv, repo: string, head: string): Map<string, Check> {
const rows = parseHeadChecks(
restList(env, `repos/${repo}/commits/${head}/check-runs?per_page=100`, 'check_runs'),
restList(env, `repos/${repo}/commits/${head}/status?per_page=100`, 'statuses'),
);
const checks = new Map<string, Check>();
if (!result.stdout.trim()) {
if (/no checks reported/.test(result.stderr)) return checks;
throw new GhError(lastLine(result.stderr) || `gh pr checks exited ${result.status}`, false);
}
let rows: unknown;
try { rows = JSON.parse(result.stdout); } catch {
throw new GhError('gh pr checks printed output that is not JSON', false);
}
if (!Array.isArray(rows)) throw new GhError('gh pr checks JSON is not a list', false);
for (const raw of rows.map(obj)) {
const name = str(raw.name);
if (!name) continue;
const row: Check = { name, bucket: str(raw.bucket) || 'pending', link: str(raw.link), description: str(raw.description) };
const current = checks.get(name);
if (!current || RANK[bucketClass(row.bucket)] > RANK[bucketClass(current.bucket)]) checks.set(name, row);
for (const row of rows) {
const current = checks.get(row.name);
if (!current || RANK[bucketClass(row.bucket)] > RANK[bucketClass(current.bucket)]) checks.set(row.name, row);
}
return checks;
}
+35 -11
View File
@@ -16,9 +16,10 @@ import * as path from 'node:path';
import { createHash } from 'node:crypto';
import type { GhRunner } from './ci-gate';
import {
GhError, behindBy, bucketClass, compareFiles, confirmPullRequest, currentRepo, fetchChecks, fetchPullRequest,
pullRequestFiles, requiredChecks, type Check, type GhEnv, type PullRequest,
GhError, behindBy, bucketClass, compareFiles, confirmPullRequest, currentRepo, fetchBaseOid, fetchChecks, fetchConversation,
fetchPullState, pullRequestFiles, requiredChecks, type Check, type Conversation, type GhEnv, type PullRequest,
} from './pr-wait-github';
import { MIN_POLL_MS, pollDelayMs, type PullState } from './gh-poll';
import { readConfigKey, resolveStateRoot, type StateRootEnv } from './state-root';
import { resultCodeSuffix, resultLineWithDocs } from './result-codes';
@@ -31,7 +32,9 @@ const STATUS_BOTS = ['github-actions', 'vercel', 'netlify', 'railway', 'cloudfla
/** A state file older than this describes an old session: start a fresh baseline instead of replaying it. */
export const STALE_STATE_MS = 6 * 3600 * 1000;
const MAX_FAILED_POLLS = 10;
const MIN_INTERVAL_SECONDS = 5;
const MIN_INTERVAL_SECONDS = MIN_POLL_MS / 1000;
/** The conversation (the one GraphQL read) is reread when the PR's updated_at or head moves, and at least this often. */
const CONVERSATION_REFRESH_MS = 10 * 60_000;
/** Minutes before a watch that met nothing ends as PR_WAIT_BLOCKED, naming what is still pending; 0 waits without limit. */
export const DEFAULT_TIMEOUT_MINUTES = 60;
@@ -77,6 +80,11 @@ comes from the URL or the current checkout.
merge state is known (default)
any-change stop after the first poll that prints an event
--interval seconds between polls: at least 60 (the default), plus up to 15 s
of jitter. Each poll reads REST (PR, checks, base head); the
GraphQL conversation read runs only when the PR changed, every
10 minutes, or once green under --until ready. A rate limit waits
for its reset, never retries at once
--timeout minutes before a watch that met nothing ends as PR_WAIT_BLOCKED,
naming what is still pending (default 60; 0 waits without limit)
--exclude check names (comma-separated, repeatable) left out of the verdict,
@@ -115,7 +123,7 @@ lock, which the next watcher for the PR takes over once the pid is gone.
Docs: docs/troubleshooting.md#gstack-pr-wait`;
export function parsePrWaitArgs(argv: string[]): PrWaitOptions | { help: true } | { error: string } {
const opts: PrWaitOptions = { until: 'ready', interval: 20, timeout: DEFAULT_TIMEOUT_MINUTES, excludes: [] };
const opts: PrWaitOptions = { until: 'ready', interval: MIN_INTERVAL_SECONDS, timeout: DEFAULT_TIMEOUT_MINUTES, excludes: [] };
for (let i = 0; i < argv.length; i++) {
const flag = argv[i];
if (flag === '--help' || flag === '-h') return { help: true };
@@ -217,6 +225,7 @@ export class Watcher {
private prevSnapshot: string | null = null;
private prFiles = new Map<string, Set<string>>();
private pending: string[] = [];
private conversationCache: { key: string; at: number; value: Conversation } | null = null;
private readonly gh: GhEnv;
constructor(readonly env: PrWaitEnv, readonly repo: string, readonly pr: number, readonly until: Until, readonly interval: number, readonly bots: BotLists, readonly timeoutMinutes = DEFAULT_TIMEOUT_MINUTES, readonly excludes: readonly string[] = []) {
@@ -450,12 +459,26 @@ export class Watcher {
private dirtyLine(base: string, head: string): string { return `merge state is DIRTY on ${head.slice(0, 7)}: the branch conflicts with ${base}, rebase it`; }
private behindLine(base: string, head: string): string { return `merge state is BEHIND on ${head.slice(0, 7)}: update the branch from ${base} before merging`; }
/**
* The conversation, reread through GraphQL only when the PR's updated_at or head moved, the cached read is ten
* minutes old, or `force` (a ready verdict is near, so review-thread resolution must be current). REST cannot read
* thread resolution; every other fact comes from REST each poll.
*/
private conversation(state: PullState, force: boolean): Conversation {
const key = `${state.headRefOid}|${state.updatedAt}`;
const cached = this.conversationCache;
if (cached && !force && cached.key === key && this.env.now() - cached.at < CONVERSATION_REFRESH_MS) return cached.value;
const value = fetchConversation(this.gh, this.repo, this.pr);
this.conversationCache = { key, at: this.env.now(), value };
return value;
}
/** One poll. Returns an exit code when the watch ends, null to keep polling. Throws GhError. */
poll(): number | null {
const pr = fetchPullRequest(this.gh, this.repo, this.pr);
if (pr.state === 'MERGED') return this.finish(PR_WAIT_EXIT.met, [`merged: ${pr.url}`]);
if (pr.state === 'CLOSED') return this.finish(PR_WAIT_EXIT.blocked, [`closed without merging: ${pr.url}`]);
const base = pr.baseRefName;
const state = fetchPullState(this.gh, this.repo, this.pr);
if (state.state === 'MERGED') return this.finish(PR_WAIT_EXIT.met, [`merged: ${state.url}`]);
if (state.state === 'CLOSED') return this.finish(PR_WAIT_EXIT.blocked, [`closed without merging: ${state.url}`]);
const base = state.baseRefName;
if (this.required === null) {
const read = requiredChecks(this.gh, this.repo, base);
this.required = read.names;
@@ -471,12 +494,13 @@ export class Watcher {
if (refused.length) this.log(`exclude refused for required check${refused.length === 1 ? '' : 's'} ${refused.join(', ')}: a required check gates`);
}
}
const checks = fetchChecks(this.gh, this.repo, this.pr);
const head = pr.headRefOid;
const head = state.headRefOid;
const checks = fetchChecks(this.gh, this.repo, head);
const short = head.slice(0, 7);
this.events = 0;
const ci = this.evaluateChecks(checks, head);
const pr: PullRequest = { ...state, baseOid: fetchBaseOid(this.gh, this.repo, base), ...this.conversation(state, this.until === 'ready' && ci.green) };
const unresolved = this.emitConversation(pr, new Set([pr.author, pr.viewer]));
const bots = this.emitBots(pr, checks, head);
this.emitBaseAndMerge(pr, head);
@@ -540,7 +564,7 @@ export class Watcher {
this.log(`poll failed (${failures}/${MAX_FAILED_POLLS}): ${error.message}`);
if (failures >= MAX_FAILED_POLLS) return this.finish(PR_WAIT_EXIT.blocked, [`gstack-pr-wait stopped: GitHub unreachable for ${failures} polls: ${error.message}`]);
}
this.env.sleep(this.interval * 1000);
this.env.sleep(pollDelayMs(this.interval * 1000));
}
}
}
+2 -1
View File
@@ -26,6 +26,7 @@ import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { DISPATCH_INPUTS, makeNonce } from '../scripts/eval-bg';
import { pollDelayMs } from './gh-poll';
import { ghFailureReason, type GhRunner } from '../scripts/recover-receipts';
import { MAX_BISECT_CASES, MAX_PREFIXES, RELEASE_BISECT_ARTIFACT, RELEASE_BISECT_REF, type Git } from '../scripts/train-gate';
import type { ReleaseGatePolicy } from './release-gate-policy';
@@ -187,7 +188,7 @@ function follow(env: GateEnv, workflow: string, nonce: string, known: number | n
if (run) lines.push(`RELEASE: run ${run.id} ${run.html_url ?? ''}`.trim());
} else if (run.status === 'completed') return run;
if (env.now() >= env.deadline) break;
env.sleep(env.pollMs);
env.sleep(pollDelayMs(env.pollMs));
if (run) run = ghJson<Run>(env, ['api', `repos/${env.repo}/actions/runs/${run.id}`]) ?? run;
}
lines.push(`RELEASE: ${run ? `run ${run.id} is still ${run.status}` : `no run for nonce ${nonce} yet`} at the deadline; the record stays resumable`);
+2 -1
View File
@@ -40,6 +40,7 @@ import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { sha256Hex, writeReceipt } from './egress-receipt';
import { ghSpawnSync } from './gh-poll';
import { runBashSync } from './script-invocation';
/**
@@ -307,7 +308,7 @@ const receiptFailed = (err: unknown) => process.stderr.write(`[run-evidence] egr
function gh(args: string[], timeoutMs = 60_000): string {
try { writeReceipt(receiptFor('api.github.com', 'actions-run-read', args.join(' '))); } catch (err) { receiptFailed(err); }
const r = spawnSync('gh', args, { encoding: 'utf8', timeout: timeoutMs, maxBuffer: 64 * 1024 * 1024 });
const r = ghSpawnSync(args, { encoding: 'utf8', timeout: timeoutMs, maxBuffer: 64 * 1024 * 1024 });
if (r.status !== 0 || r.error) throw new Error(`gh ${args.slice(0, 2).join(' ')} failed: ${(r.stderr || String(r.error ?? '')).trim().slice(0, 300)}`);
return r.stdout;
}
+2 -1
View File
@@ -15,6 +15,7 @@ import { isConfigUnavailable, loadConfigFromBase } from './pregate/config';
import { parseUnifiedDiff, versionAtRef } from './restamp';
import { branchSides, stampOnlyLine } from './stamp-only';
import { DEFAULT_POLICY, matchesAny } from './ship-policy';
import { ghSpawnSync } from './gh-poll';
import { extractVersion } from './version-source';
import { BIN_DIR, runBashSync, runScriptSync, scriptInvocation } from './script-invocation';
@@ -40,7 +41,7 @@ export function hostedOrigin(cwd: string): boolean {
}
/** gh with argv, never a shell, so a branch name cannot change what runs. */
export function gh(cwd: string, args: string[], timeout = 60_000): Run {
return done(spawnSync('gh', args, { cwd, encoding: 'utf8', timeout, maxBuffer: 64 * 1024 * 1024, env: { ...process.env, GIT_TERMINAL_PROMPT: '0' } }));
return done(ghSpawnSync(args, { cwd, encoding: 'utf8', timeout, maxBuffer: 64 * 1024 * 1024, env: { ...process.env, GIT_TERMINAL_PROMPT: '0' } }));
}
export function sh(cwd: string, command: string, timeout = 1_800_000): Run {
return done(runBashSync(command, { cwd, encoding: 'utf8', timeout, maxBuffer: 64 * 1024 * 1024 }));
+2 -1
View File
@@ -16,6 +16,7 @@
*/
import * as fs from 'node:fs';
import * as path from 'node:path';
import { pollDelayMs } from './gh-poll';
import type { PregateFile } from './headless-artifacts';
import { PREGATE_OUT_REL, clearObligation, readPregateFile, writePregateFile } from './pregate/index';
import type { QueueContext } from './ship-queue';
@@ -65,7 +66,7 @@ export async function dispatchCi(ctx: QueueContext, o: DispatchCiOptions): Promi
run = run && slug ? runView(ctx, slug, run.databaseId) ?? run : runsAt(ctx, wf, branch, head)[0];
if (run?.status === 'completed') break;
if (Date.now() >= deadline) return { lines: [...lines, `CI_DISPATCH: ${wf} --ref ${branch} ${run ? `run ${run.databaseId} ${run.url} ${run.status}` : 'no run appeared'}`], failure: { code: 'QUEUE_TRAIN_PENDING', message: `${wf} ${run ? `run ${run.databaseId} is still ${run.status}` : 'has no workflow_dispatch run at this head yet'} at the --dispatch-ci timeout; rerun ready --dispatch-ci to continue waiting` } };
await sleep(o.pollMs);
await sleep(pollDelayMs(o.pollMs));
}
lines.push(`CI_DISPATCH: ${wf} --ref ${branch} run ${run.databaseId} ${run.url} ${run.conclusion ?? 'unknown'}`);
concluded.set(wf, run);
+3 -2
View File
@@ -13,6 +13,7 @@
* neither the base land checked nor in the pin), a land note on refs/notes/gstack-trains, and the lock's release. A
* rerun after GitHub merged goes straight to the tree check.
*/
import { MIN_POLL_MS, pollDelayMs } from './gh-poll';
import { runMergedChecks } from './land-merged-checks';
import { classifyRelease, RELEASE_BRANCH } from './release-stamp';
import { versionAtRef } from './restamp';
@@ -235,8 +236,8 @@ export function landDirect(ctx: QueueContext, o: DirectOptions): StepOutput {
if (lost) return fail(lines, lost);
const m = ctx.gh(['pr', 'merge', o.pr, method, '--match-head-commit', o.pin]);
let merge: string | undefined;
for (let i = 0; i < (m.ok ? 4 : 1) && !isSha(merge); i++) {
if (i) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 3_000);
for (let i = 0; i < (m.ok ? 3 : 1) && !isSha(merge); i++) {
if (i) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, pollDelayMs(MIN_POLL_MS));
const after = view(ctx, o.pr);
merge = 'code' in after || after.state !== 'MERGED' ? undefined : after.mergeCommit?.oid;
}
+2 -1
View File
@@ -15,6 +15,7 @@ import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { mergeChangelogs, parseChangelog } from './changelog-check';
import { MIN_POLL_MS, pollDelayMs } from './gh-poll';
import { fragmentSlug, readFragments } from './changelog-fragments';
import { restamp, versionAtRef } from './restamp';
import { acquireLandLock, lockLabel, readLock, writeLock, type LandLockPhase } from './ship-queue-land-lock';
@@ -212,7 +213,7 @@ function prInfo(ctx: QueueContext, pr: string): PrInfo | QueueFailure {
export function waitChecks(ctx: QueueContext, sha: string, minutes: number): void {
const deadline = Date.now() + minutes * 60_000;
for (let ci = ciAtHead(ctx, sha); !ci.green && !ci.line.startsWith('red') && Date.now() < deadline; ci = ciAtHead(ctx, sha)) {
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 30_000);
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, pollDelayMs(MIN_POLL_MS));
}
}
+2 -1
View File
@@ -12,6 +12,7 @@
* commit pending, a failed one makes it red (named as a commit status when no
* check run failed). Rerun it after an interruption and it continues.
*/
import { pollDelayMs } from './gh-poll';
import { ciAtHead, type QueueContext } from './ship-queue';
import { outFailed, outOk, outPending, type TrainOut } from './ship-queue-train';
@@ -46,6 +47,6 @@ export async function verifyPostMerge(ctx: QueueContext, o: PostMergeOptions): P
: ci.none ? `no check run has attached to ${at} yet` : settled ? `the post-merge checks the train gate names never ran at ${at}: ${absent.join(', ')}` : `main's push workflows at ${at} have not settled at the timeout: ${pending.join(', ') || 'a commit status'} still running`;
return outPending({ code: 'QUEUE_TRAIN_PENDING', message }, lines, again, 'none');
}
await sleep(o.pollMs);
await sleep(pollDelayMs(o.pollMs));
}
}
+4 -3
View File
@@ -13,6 +13,7 @@
* --land lands each landable slot in order through train land, stopping at the first refusal with its code and NEXT.
* It never merges without --land and never merges main into a PR.
*/
import { pollDelayMs } from './gh-poll';
import type { QueueContext } from './ship-queue';
import { buildTrain, findTrain, nextFor, outOk, outPending, outRefused, trainEnv, trainStatus, type TrainOut } from './ship-queue-train';
import { repoSlug, runView } from './ship-queue-train-github';
@@ -36,7 +37,7 @@ async function awaitGate(ctx: QueueContext, t: TrainRecord, deadline: number, po
const view = runView(ctx, slug, a.run.id);
if (view?.status === 'completed') return null;
if (Date.now() >= deadline) return `run ${a.run.id} is still ${view?.status ?? 'unreadable'} at the watch timeout`;
await sleep(pollMs);
await sleep(pollDelayMs(pollMs));
}
}
@@ -49,7 +50,7 @@ async function awaitPrefixRuns(ctx: QueueContext, t: TrainRecord, deadline: numb
const running = open.filter(a => runView(ctx, slug, a.run!.id)?.status !== 'completed');
if (!running.length) return null;
if (Date.now() >= deadline) return `${running.length} prefix run(s) still running at the watch timeout (${running.map(a => a.run!.id).join(', ')})`;
await sleep(pollMs);
await sleep(pollDelayMs(pollMs));
}
}
@@ -70,7 +71,7 @@ async function watchGeneric(ctx: QueueContext, env: TrainEnv, start: TrainRecord
verdict = trainVerdict(ctx, { train: t.nonce });
reload();
if (!t.verdict && Date.now() >= deadline) return { ...verdict, lines: [...lines, ...verdict.lines], next: again };
if (!t.verdict) await sleep(o.pollMs);
if (!t.verdict) await sleep(pollDelayMs(o.pollMs));
}
verdict ??= trainVerdict(ctx, { train: t.nonce });
lines.push(...verdict.lines);
+2 -1
View File
@@ -18,6 +18,7 @@ import { randomBytes } from 'node:crypto';
import { DISPATCH_INPUTS, awaitNonceRun, dispatchWorkflow, listDispatchRuns, makeNonce, runForNonce, type WorkflowDispatch } from '../scripts/eval-bg';
import { carryReleasedParent, changelogDamage, mergeChangelogs, parseChangelog, type ChangelogSection } from './changelog-check';
import type { ResultStatus } from './headless-artifacts';
import { pollDelayMs } from './gh-poll';
import { baseOverlapLine } from './pr-wait';
import { versionAtRef, versionFiles } from './restamp';
import type { LoadedPolicy, ShipPolicy, TrainPolicy } from './ship-policy';
@@ -495,7 +496,7 @@ async function freePhase(env: TrainEnv, slug: string, t: TrainRecord, o: BuildOp
const deadline = Date.now() + o.freeTimeoutSec * 1000;
let found = newestBranchRun(env.ctx, slug, env.train.free_workflow, t.branch);
while ((found === null || found === 'unreadable' || found.status !== 'completed') && Date.now() < deadline) {
await sleep(o.pollMs);
await sleep(pollDelayMs(o.pollMs));
found = newestBranchRun(env.ctx, slug, env.train.free_workflow, t.branch);
}
if (found && found !== 'unreadable' && found.status !== 'completed') {
+2 -1
View File
@@ -29,6 +29,7 @@ import { parseReceiptDocs, type ReceiptDocs } from './ship-receipt-docs';
import { resolveStateRoot } from './state-root';
import { bumpVersion, cmpVersion, fmtVersion, parseVersion, versionWidth, type Bump } from './version-source';
import { waitForRun } from './ci-carry';
import { MIN_POLL_MS } from './gh-poll';
import { lanesOnlyHeadPolicy } from './pregate/head-policy';
import { PREPUSH_CHECK_IDS } from './pregate/index';
import { buildRunCheck } from './pregate/run-check';
@@ -624,7 +625,7 @@ export async function accept(argv: readonly string[], cwd: string): Promise<Repo
for (const name of [...new Set([...mapped.values()].flatMap(m => m.profiles))]) {
const p = runCheck.profiles[name];
if (!p) return r.stop(`ship_lanes_from names ${name}, which is not a run_profiles entry on origin/${base}`);
const w = await waitForRun(runCheck.deps, { workflow: p.workflow, events: p.events, head, timeoutMs: waitS * 1000, intervalMs: 15_000, sleep: ms => new Promise(res => setTimeout(res, ms)), clock: Date.now });
const w = await waitForRun(runCheck.deps, { workflow: p.workflow, events: p.events, head, timeoutMs: waitS * 1000, intervalMs: MIN_POLL_MS, sleep: ms => new Promise(res => setTimeout(res, ms)), clock: Date.now });
r.add(`CI_WAIT: ${w.state} ${name} ${'run' in w ? `run=${w.run.id}/${w.run.run_attempt}` : w.reason}`);
if (w.state === 'running') return r.stop(`${p.workflow} run ${w.run.id} at ${head.slice(0, 12)} is still running: rerun accept (or gstack-ci-carry wait --base ${base} in the background, then accept)`);
if (w.state === 'red') return r.stop(`${p.workflow} run ${w.run.id} failed (${short(w.failing, 4) || 'no failing job named'}): Step 5 triage on its logs; fix, commit, push, then accept again`);
+3 -2
View File
@@ -28,6 +28,7 @@ import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { spawnSync } from 'node:child_process';
import { pollDelayMs } from '../lib/gh-poll';
import { detectBaseBranch } from '../test/helpers/touchfiles';
import { DEFAULT_JOBS, ROOT, type PaidTier } from './lib/paid-types';
import { buildRunManifest, loadPaidTestDurations, recordedShardMs } from './lib/paid-plan';
@@ -243,7 +244,7 @@ export function dispatchWorkflow(dispatch: WorkflowDispatch, ref: string, run: E
export async function awaitNonceRun(workflow: string, nonce: string, run: Exec, repo = CANONICAL_REPO): Promise<RunSummary | null> {
const deadline = Date.now() + 10 * pollMs();
while (Date.now() < deadline) {
await sleep(Math.min(pollMs(), 10_000));
await sleep(pollDelayMs(pollMs()));
const found = runForNonce(listDispatchRuns(workflow, run, repo) ?? [], nonce);
if (found) return found;
}
@@ -335,7 +336,7 @@ async function followWorkflow(dispatch: WorkflowDispatch, ref: string, key: stri
console.log(`[eval-bg] ${dispatch.workflow}: conclusion ${state.conclusion ?? 'unknown'} -> exit ${code}`);
return code;
}
await sleep(pollMs());
await sleep(pollDelayMs(pollMs()));
}
}
+1
View File
@@ -235,6 +235,7 @@ export function collectRecovery(options: CollectOptions): { checkpoint: Recovery
const id = String(listedRun.id);
let run = listedRun;
while (run.status !== 'completed' && left() > (options.pollMs ?? 5_000)) {
// gh-poll-exempt: runs in GitHub Actions (evals.yml) on GITHUB_TOKEN, not the shared Capy installation
sleep(options.pollMs ?? 5_000);
const polled = gh(['api', `repos/${options.repo}/actions/runs/${id}`], left());
if (polled.status !== 0) break;
+2
View File
@@ -123,6 +123,7 @@ function gateMain(o: Required<Pick<BackstopOptions, 'repo' | 'branch' | 'sha'>>
run = json<Run>(o.gh(['api', `repos/${o.repo}/actions/runs/${run.id}`], GH_MS)) ?? run;
if (run.status === 'completed') return run;
}
// gh-poll-exempt: runs in GitHub Actions (evals-main-backstop.yml) on GITHUB_TOKEN; default 60 s
o.sleep(o.pollMs);
}
lines.push(`the gate ${run ? `run ${run.id}` : 'dispatch'} did not finish before the backstop deadline`);
@@ -151,6 +152,7 @@ export function runBackstop(o: BackstopOptions): { action: BackstopAction; lines
if (!merging.length) break;
if (now() >= settleUntil) { lines.push(`BACKSTOP: train(s) ${merging.map(t => t.nonce).join(', ')} still merging after the settle window; gating anyway`); break; }
lines.push(`BACKSTOP: waiting for train(s) ${merging.map(t => t.nonce).join(', ')} to finish merging${fetched.ok ? '' : ` (${fetched.detail})`}`);
// gh-poll-exempt: runs in GitHub Actions (evals-main-backstop.yml) on GITHUB_TOKEN; default 60 s
sleep(pollMs);
}
const digest = stampFreeTree(git, o.sha, o.repoRoot, loaded.policy);
+1 -1
View File
@@ -135,7 +135,7 @@ describe('completion contract on both backends (DX-2)', () => {
};
const sandbox = () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'eval-bg-'));
const env: NodeJS.ProcessEnv = { ...process.env, HOME: dir, GSTACK_EVAL_BG_POLL_MS: '50' };
const env: NodeJS.ProcessEnv = { ...process.env, HOME: dir, GSTACK_EVAL_BG_POLL_MS: '50', GSTACK_GH_POLL_FLOOR_MS: '0' };
return { dir, env };
};
File diff suppressed because it is too large. Load diff
+20
View File
@@ -0,0 +1,20 @@
{
"number": 4101,
"html_url": "https://github.com/garrytan/gstack/pull/4101",
"state": "closed",
"merged": true,
"merged_at": "2026-11-07T02:28:30Z",
"updated_at": "2026-11-07T02:28:33Z",
"mergeable_state": "unknown",
"user": {
"login": "garrytan"
},
"head": {
"sha": "ff050059a9ebeeaa41618287f0f9687612174bbc",
"ref": "capy/release-stamps-and-a"
},
"base": {
"sha": "e4b1e3455230a958cc53667e920994a9747d440c",
"ref": "main"
}
}
+20
View File
@@ -0,0 +1,20 @@
{
"number": 4108,
"html_url": "https://github.com/garrytan/gstack/pull/4108",
"state": "open",
"merged": false,
"merged_at": null,
"updated_at": "2026-11-07T03:31:02Z",
"mergeable_state": "unknown",
"user": {
"login": "garrytan"
},
"head": {
"sha": "8cadbdc985a6259842722f58c82f370f985fd39b",
"ref": "capy/gst-1698-cut-plan-eng-review"
},
"base": {
"sha": "dd97420b5355ede9e149d023e5f4cf40ad6f0f27",
"ref": "main"
}
}
+22
View File
@@ -0,0 +1,22 @@
{
"resources": {
"core": {
"limit": 7700,
"used": 0,
"remaining": 7700,
"reset": 1794026491
},
"graphql": {
"limit": 7700,
"used": 502,
"remaining": 7198,
"reset": 1794023433
}
},
"rate": {
"limit": 7700,
"used": 0,
"remaining": 7700,
"reset": 1794026491
}
}
+6
View File
@@ -0,0 +1,6 @@
{
"state": "pending",
"sha": "ff050059a9ebeeaa41618287f0f9687612174bbc",
"total_count": 0,
"statuses": []
}
+98
View File
@@ -0,0 +1,98 @@
/**
* GitHub poll tripwire (GST-1713). The Capy GitHub app installation shares one
* GraphQL budget across every crew; gstack waits that polled every 5 to 30 s
* drained it. lib/gh-poll.ts's pollDelayMs puts a 60 s floor (plus jitter) on
* every wait between GitHub reads.
*
* Rule: in bin/, lib/ and scripts/, a file that reads GitHub (spawns gh, or
* calls one of the GitHub readers named in READERS) may only sleep with a
* duration from pollDelayMs on the same line. Sleep primitives being defined
* or typed (`sleep: ms => Bun.sleepSync(ms)`, `sleep(ms: number): void`) are
* plumbing, not waits. Anything else needs a marker with its reason on the
* line or the line above:
*
* // gh-poll-exempt: <why this wait may be shorter than 60 s>
*
* Exemptions are counted and ratcheted (EXEMPT_CEILING only goes down), and
* no shipped file may set the GSTACK_GH_POLL_FLOOR_MS test seam.
*/
import { describe, expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as path from 'node:path';
const ROOT = path.resolve(import.meta.dir, '..');
const SCAN_ROOTS = ['bin', 'lib', 'scripts'];
/** The policy owner itself sleeps the rate-limit wait it computes. */
const OWNER = path.join('lib', 'gh-poll.ts');
const READERS = /\b(?:ghJson|ghLines|ghApiJson|runView|newestBranchRun|ciAtHead|readAllPages|runsForHead|fetchPullState|fetchChecks|fetchConversation|reviewDone|ghRunDeps|waitForRun|awaitNonceRun)\(/;
const SPAWNS_GH = /\b(?:spawnSync|spawn|execFileSync)\(\s*['"]gh['"]|\bghSpawnSync\(|\bctx\.gh\(|\brunGh\(|\bspawnGh\b|\.run\(\s*['"]gh['"]|['"]gh['"]\s*,\s*\[/;
const SLEEP = /\b(?:Bun\.sleepSync|Bun\.sleep|Atomics\.wait|sleepSync|sleep)\s*\(|\bsetTimeout\(\s*\w+\s*,/;
const PLUMBING = /\bsleep\w*\s*[:=]\s*\S|function\s+sleep|\bsleep\w*\(\s*\w+\s*:\s*number\s*\)\s*:/;
const COMMENT_LINE = /^\s*(?:\/\/|\*|\/\*)/;
const EXEMPT_MARKER = /gh-poll-exempt:/;
/** Shrink-only: lower it when exemptions burn down; never raise it. */
const EXEMPT_CEILING = 4;
function walk(dir: string, out: string[] = []): string[] {
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
const full = path.join(dir, entry.name);
if (entry.isDirectory()) { if (entry.name !== 'node_modules' && entry.name !== 'dist') walk(full, out); }
else if (dir.endsWith(`${path.sep}bin`) || /\.(?:[cm]?[jt]s)$/.test(entry.name)) out.push(full);
}
return out;
}
interface Hit { file: string; line: number; text: string }
function scanSleeps(files: Array<{ rel: string; source: string }>): { violations: Hit[]; exempt: Hit[] } {
const violations: Hit[] = [];
const exempt: Hit[] = [];
for (const { rel, source } of files) {
if (rel === OWNER || !(READERS.test(source) || SPAWNS_GH.test(source))) continue;
const lines = source.split('\n');
lines.forEach((text, i) => {
if (!SLEEP.test(text) || COMMENT_LINE.test(text) || PLUMBING.test(text) || /\bpollDelayMs\(/.test(text)) return;
const hit = { file: rel, line: i + 1, text: text.trim().slice(0, 140) };
if (EXEMPT_MARKER.test(text) || (i > 0 && EXEMPT_MARKER.test(lines[i - 1]!))) exempt.push(hit);
else violations.push(hit);
});
}
return { violations, exempt };
}
function shipped(): Array<{ rel: string; source: string }> {
return SCAN_ROOTS.flatMap(root => walk(path.join(ROOT, root))).flatMap(file => {
const source = fs.readFileSync(file, 'utf8');
return source.includes('\u0000') ? [] : [{ rel: path.relative(ROOT, file), source }];
});
}
describe('GitHub poll tripwire', () => {
// Value: protects=the shared GraphQL and REST budgets from a new fast poll loop; fails_when=a GitHub-reading module sleeps a literal or caller interval instead of pollDelayMs
test('every wait in a GitHub-reading module goes through pollDelayMs', () => {
const { violations, exempt } = scanSleeps(shipped());
if (violations.length) {
throw new Error(`${violations.length} wait(s) in GitHub-reading modules do not use pollDelayMs (lib/gh-poll.ts, 60 s floor):\n${violations.map(v => ` ${v.file}:${v.line}: ${v.text}`).join('\n')}\nFix: sleep pollDelayMs(<interval ms>), or mark a non-poll wait with // gh-poll-exempt: <reason>`);
}
expect(exempt.length).toBeLessThanOrEqual(EXEMPT_CEILING);
});
test('the scanner catches a planted 20 s poll, and passes pollDelayMs, plumbing and exempted waits', () => {
const planted = (body: string) => [{ rel: 'lib/planted.ts', source: `const r = ctx.gh(['pr', 'checks']);\n${body}\n` }];
expect(scanSleeps(planted(' Bun.sleepSync(20_000);')).violations.map(v => v.line)).toEqual([2]);
expect(scanSleeps(planted(' await sleep(o.pollMs);')).violations).toHaveLength(1);
expect(scanSleeps(planted(' await new Promise(r => setTimeout(r, 30_000));')).violations).toHaveLength(1);
expect(scanSleeps([{ rel: 'lib/delegated.ts', source: 'for (;;) { if (runView(ctx, slug, id)) break; Atomics.wait(i32, 0, 0, 5_000); }\n' }]).violations).toHaveLength(1);
expect(scanSleeps(planted(' await sleep(pollDelayMs(o.pollMs));')).violations).toEqual([]);
expect(scanSleeps(planted(' sleep: ms => Bun.sleepSync(ms),\n sleep(ms: number): void;')).violations).toEqual([]);
expect(scanSleeps(planted(' // gh-poll-exempt: lock retry, not a GitHub read\n Bun.sleepSync(50);')).exempt).toHaveLength(1);
expect(scanSleeps([{ rel: 'lib/no-github.ts', source: 'Bun.sleepSync(10);\n' }]).violations).toEqual([]);
});
test('no shipped file sets the GSTACK_GH_POLL_FLOOR_MS test seam', () => {
const setters = shipped().filter(f => f.rel !== OWNER && /GSTACK_GH_POLL_FLOOR_MS/.test(f.source));
const workflows = fs.readdirSync(path.join(ROOT, '.github', 'workflows')).filter(name => /GSTACK_GH_POLL_FLOOR_MS/.test(fs.readFileSync(path.join(ROOT, '.github', 'workflows', name), 'utf8')));
expect([...setters.map(f => f.rel), ...workflows]).toEqual([]);
});
});
+202
View File
@@ -0,0 +1,202 @@
/**
* lib/gh-poll.ts (GST-1713): the 60 s poll floor with jitter, rate-limit
* classification and the sleep-until-reset rule, and the REST response
* parsers. test/fixtures/gh-poll/*.json are trimmed recordings of real
* GitHub REST replies (garrytan/gstack#4101 merged, #4108 open, the check runs
* and combined status of #4101's head, and `gh api rate_limit`, recorded
* 2026-11-06). The rate-limit messages are the ones gh prints for the Capy app
* installation (FOUN-32 report on GST-1713) and GitHub's secondary-limit text.
*/
import { describe, expect, test } from 'bun:test';
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
import type { GhResult } from '../lib/ci-gate';
import { MAX_LIMIT_WAITS, MIN_POLL_MS, parseHeadChecks, parsePullState, pollDelayMs, rateLimitKind, rateLimitWaitMs, withRateLimit } from '../lib/gh-poll';
import { GhError, fetchChecks, runGh } from '../lib/pr-wait-github';
import { waitForRun } from '../lib/ci-carry';
const fixture = (name: string): string => readFileSync(join(import.meta.dir, 'fixtures', 'gh-poll', name), 'utf8');
const ok = (stdout: string): GhResult => ({ status: 0, stdout, stderr: '' });
const err = (stderr: string, status = 1): GhResult => ({ status, stdout: '', stderr: `${stderr}\n` });
const NO_SEAM = {};
const GRAPHQL_LIMIT = 'GraphQL: API rate limit exceeded for installation ID 161978188.';
const CORE_LIMIT = 'gh: API rate limit exceeded for installation ID 161978188. If you reach out to GitHub Support for help, please include the request ID 1234:5678. (HTTP 403)';
const SECONDARY_LIMIT = 'gh: You have exceeded a secondary rate limit. Please wait a few minutes before you try again. (HTTP 403)';
const RATE_LIMIT_JSON = fixture('rate-limit.json');
const exhausted = (resource: 'graphql' | 'core', reset: number) => {
const parsed = JSON.parse(RATE_LIMIT_JSON);
parsed.resources[resource] = { ...parsed.resources[resource], remaining: 0, reset };
return JSON.stringify(parsed);
};
describe('the poll floor', () => {
// Value: protects=no gstack wait polls GitHub faster than once a minute; fails_when=a caller's 5, 15, 20 or 30 s interval reaches GitHub unchanged
test('every delay is at least 60 s plus up to 15 s of jitter; a longer request is kept', () => {
expect(MIN_POLL_MS).toBe(60_000);
for (const requested of [0, 5_000, 20_000, 30_000, 59_999]) {
expect(pollDelayMs(requested, () => 0, NO_SEAM)).toBe(60_000);
expect(pollDelayMs(requested, () => 1, NO_SEAM)).toBe(75_000);
}
expect(pollDelayMs(Number.NaN, () => 0, NO_SEAM)).toBe(60_000);
expect(pollDelayMs(300_000, () => 0.5, NO_SEAM)).toBe(307_500);
});
test('GSTACK_GH_POLL_FLOOR_MS lowers the floor for fixtures only', () => {
expect(pollDelayMs(20, () => 1, { GSTACK_GH_POLL_FLOOR_MS: '0' })).toBe(20);
expect(pollDelayMs(20, () => 0, { GSTACK_GH_POLL_FLOOR_MS: 'soon' })).toBe(60_000);
});
});
describe('rate limits are classified, never mistaken for other failures', () => {
// Value: protects=a rate-limited call is recognized whichever budget ran out; fails_when=a GraphQL limit reads as core, or a permission 403 reads as a rate limit
test('GraphQL, REST core and secondary limits; permission and server errors are not rate limits', () => {
expect(rateLimitKind(['pr', 'view', '42'], GRAPHQL_LIMIT)).toBe('graphql');
expect(rateLimitKind(['api', 'repos/o/r/pulls/42'], GRAPHQL_LIMIT)).toBe('graphql');
expect(rateLimitKind(['api', 'graphql'], '{"errors":[{"type":"RATE_LIMITED","message":"API rate limit exceeded"}]}')).toBe('graphql');
expect(rateLimitKind(['api', 'repos/o/r/commits/abc/check-runs'], CORE_LIMIT)).toBe('core');
expect(rateLimitKind(['run', 'view', '9'], CORE_LIMIT)).toBe('core');
expect(rateLimitKind(['pr', 'checks', '42'], 'gh: HTTP 429: too many requests')).toBe('graphql');
expect(rateLimitKind(['api', 'repos/o/r/pulls/42'], SECONDARY_LIMIT)).toBe('secondary');
expect(rateLimitKind(['api', 'repos/o/r/branches/main/protection'], 'gh: Resource not accessible by integration (HTTP 403)')).toBeNull();
expect(rateLimitKind(['api', 'repos/o/r/pulls/42'], 'gh: Something went wrong (HTTP 502)')).toBeNull();
});
test('the wait runs to the exhausted resource\'s reset (plus slack); a secondary limit or an unreadable reply waits the floor', () => {
const now = 1_794_020_000_000;
expect(rateLimitWaitMs('graphql', exhausted('graphql', now / 1000 + 600), now, () => 0, NO_SEAM)).toBe(605_000);
expect(rateLimitWaitMs('core', exhausted('core', now / 1000 + 30), now, () => 0, NO_SEAM)).toBe(60_000);
expect(rateLimitWaitMs('graphql', RATE_LIMIT_JSON, now, () => 0, NO_SEAM)).toBe(60_000);
expect(rateLimitWaitMs('secondary', exhausted('graphql', now / 1000 + 600), now, () => 0, NO_SEAM)).toBe(60_000);
expect(rateLimitWaitMs('graphql', 'not json', now, () => 0, NO_SEAM)).toBe(60_000);
expect(rateLimitWaitMs('graphql', exhausted('graphql', now / 1000 + 86_400), now, () => 0, NO_SEAM)).toBe(65 * 60_000);
});
});
describe('a rate-limited call backs off until the reset, never retries at once', () => {
const harness = (replies: GhResult[], limits = exhausted('graphql', 1_794_020_600)) => {
const calls: string[][] = [];
const sleeps: number[] = [];
let clock = 1_794_020_000_000;
const gh = withRateLimit(args => {
calls.push(args);
if (args[0] === 'api' && args[1] === 'rate_limit') return ok(limits);
return replies.length > 1 ? replies.shift()! : replies[0]!;
}, { sleep: ms => { sleeps.push(ms); clock += ms; }, now: () => clock, log: () => {}, random: () => 0, vars: NO_SEAM });
return { gh, calls, sleeps };
};
// Value: protects=GraphQL quota recovery; fails_when=a rate-limited gh call is rerun immediately or in a short loop
test('one rate limit: read rate_limit, sleep until the reset, rerun once', () => {
const h = harness([err(GRAPHQL_LIMIT), ok('{"number":42}')]);
expect(h.gh(['pr', 'view', '42']).stdout).toBe('{"number":42}');
expect(h.calls).toEqual([['pr', 'view', '42'], ['api', 'rate_limit'], ['pr', 'view', '42']]);
expect(h.sleeps).toEqual([605_000]);
});
test('a limit that persists gives up after MAX_LIMIT_WAITS waits, each at least the floor, and returns the failure', () => {
const h = harness([err(SECONDARY_LIMIT)]);
const result = h.gh(['api', 'repos/o/r/pulls/42']);
expect(result.status).toBe(1);
expect(result.stderr).toContain('secondary rate limit');
expect(h.calls.filter(c => c[1] !== 'rate_limit')).toHaveLength(MAX_LIMIT_WAITS + 1);
expect(h.sleeps).toHaveLength(MAX_LIMIT_WAITS);
for (const ms of h.sleeps) expect(ms).toBeGreaterThanOrEqual(MIN_POLL_MS);
});
// Value: protects=the land lock; fails_when=a rate-limited gh pr merge sleeps past PUSH_STALE_MS while another lander may break the lock
test('a rate-limited gh pr merge fails at once: no rate_limit read, no sleep, no rerun', () => {
const h = harness([err(GRAPHQL_LIMIT)]);
expect(h.gh(['pr', 'merge', '42', '--squash', '--match-head-commit', 'abc']).status).toBe(1);
expect(h.calls).toEqual([['pr', 'merge', '42', '--squash', '--match-head-commit', 'abc']]);
expect(h.sleeps).toEqual([]);
});
test('success and real failures return at once: no rate_limit read, no sleep', () => {
const h = harness([err('gh: Not Found (HTTP 404)')]);
expect(h.gh(['api', 'repos/o/r/pulls/42']).status).toBe(1);
expect(h.calls).toEqual([['api', 'repos/o/r/pulls/42']]);
expect(h.sleeps).toEqual([]);
});
test("gstack-pr-wait's runGh throws a rate limit at once (transient, no 2 s retry), while a 502 still retries", () => {
const sleeps: number[] = [];
let calls = 0;
const limited = { gh: () => { calls += 1; return err(GRAPHQL_LIMIT); }, sleep: (ms: number) => sleeps.push(ms), log: () => {} };
let thrown: unknown;
try { runGh(limited, ['api', 'graphql']); } catch (error) { thrown = error; }
expect(thrown).toBeInstanceOf(GhError);
expect((thrown as GhError).transient).toBe(true);
expect(calls).toBe(1);
expect(sleeps).toEqual([]);
const flaky = [err('gh: Something went wrong (HTTP 502)'), ok('fine')];
expect(runGh({ gh: () => flaky.shift()!, sleep: (ms: number) => sleeps.push(ms), log: () => {} }, ['api', 'x']).stdout).toBe('fine');
expect(sleeps).toEqual([2_000]);
});
});
describe('REST replies parse from recorded fixtures', () => {
// Value: protects=the REST reads that replaced gh pr view and gh pr checks read GitHub's real shapes; fails_when=a merged PR reads OPEN, the head or base is lost, or a skipped check counts as failed
test('pulls/{n}: merged and open pull requests in GraphQL spellings', () => {
expect(parsePullState(fixture('pull-merged.json'))).toEqual({
url: 'https://github.com/garrytan/gstack/pull/4101', state: 'MERGED', headRefOid: 'ff050059a9ebeeaa41618287f0f9687612174bbc',
baseRefName: 'main', mergeStateStatus: 'UNKNOWN', author: 'garrytan', updatedAt: '2026-11-07T02:28:33Z',
});
const open = parsePullState(fixture('pull-open.json'))!;
expect(open).toMatchObject({ state: 'OPEN', baseRefName: 'main', headRefOid: '8cadbdc985a6259842722f58c82f370f985fd39b' });
expect(parsePullState(JSON.stringify({ ...JSON.parse(fixture('pull-open.json')), mergeable_state: 'behind' }))!.mergeStateStatus).toBe('BEHIND');
expect(parsePullState(JSON.stringify({ ...JSON.parse(fixture('pull-open.json')), state: 'closed' }))!.state).toBe('CLOSED');
expect(parsePullState(JSON.stringify({ ...JSON.parse(fixture('pull-open.json')), user: { login: 'capy-ai[bot]' } }))!.author).toBe('capy-ai');
expect(parsePullState('{"message":"Not Found"}')).toBeNull();
expect(parsePullState('not json')).toBeNull();
});
test('check-runs and status: 64 recorded runs become pass and skipping rows; re-runs supersede, contexts join, unknown conclusions fail closed', () => {
const runs = JSON.parse(fixture('check-runs.json')).check_runs as Array<Record<string, unknown>>;
const statuses = JSON.parse(fixture('status.json')).statuses as unknown[];
const rows = parseHeadChecks(runs, statuses);
expect(rows).toHaveLength(64);
expect(rows.filter(r => r.bucket === 'pass')).toHaveLength(51);
expect(rows.filter(r => r.bucket === 'skipping')).toHaveLength(13);
expect(rows.find(r => r.name === 'free-tests')).toMatchObject({ bucket: 'pass', link: expect.stringContaining('https://github.com/garrytan/gstack/actions/runs/') });
const first = runs.find(r => r.name === 'free-tests')!;
const rerun = { ...first, id: Number(first.id) + 1, status: 'in_progress', conclusion: null, completed_at: null };
const stale = { ...first, id: Number(first.id) + 2, name: 'old-check', conclusion: 'stale' };
const synthetic = parseHeadChecks([first, rerun, stale], [{ context: 'ci/legacy', state: 'failure', target_url: 'https://ci.example/1', description: 'red' }, { context: 'ci/wait', state: 'pending' }]);
expect(synthetic.map(r => [r.name, r.bucket])).toEqual([['free-tests', 'pending'], ['old-check', 'stale'], ['ci/legacy', 'fail'], ['ci/wait', 'pending']]);
});
test("gstack-pr-wait's fetchChecks reads the paginated lines and collapses one job's two workflow runs to the worst bucket", () => {
const runs = JSON.parse(fixture('check-runs.json')).check_runs as Array<Record<string, unknown>>;
const pushRun = { ...runs.find(r => r.name === 'free-tests')!, id: 1, conclusion: 'failure', check_suite: { id: 7 } };
const calls: string[][] = [];
const gh = (args: string[]): GhResult => {
calls.push(args);
if (args[2]!.includes('/check-runs')) return ok([...runs, pushRun].map(r => JSON.stringify(r)).join('\n'));
return ok('');
};
const checks = fetchChecks({ gh, sleep: () => {}, log: () => {} }, 'garrytan/gstack', 'f'.repeat(40));
expect(checks.size).toBe(new Set(runs.map(r => r.name)).size);
expect(checks.get('free-tests')!.bucket).toBe('fail');
expect(calls.map(c => c.slice(0, 3))).toEqual([
['api', '--paginate', `repos/garrytan/gstack/commits/${'f'.repeat(40)}/check-runs?per_page=100`],
['api', '--paginate', `repos/garrytan/gstack/commits/${'f'.repeat(40)}/status?per_page=100`],
]);
expect(calls.some(c => c[0] === 'pr' || c[1] === 'graphql')).toBe(false);
});
});
describe('waits sleep at least the floor', () => {
// Value: protects=gstack-ci-carry wait and ship-release accept poll at most once a minute; fails_when=waitForRun sleeps its 15 or 20 s interval
test('waitForRun sleeps pollDelayMs between reads, whatever intervalMs asks', async () => {
const sleeps: number[] = [];
let t = 0;
let reads = 0;
const deps = { runsForHead: () => { reads += 1; return [{ id: 1, run_attempt: 1, event: 'pull_request', status: 'in_progress', conclusion: null, created_at: '2026-11-06T00:00:00Z' }]; } } as never;
const verdict = await waitForRun(deps, { workflow: 'free-tests.yml', events: ['pull_request'], head: 'a'.repeat(40), timeoutMs: 300_000, intervalMs: 15_000, sleep: async ms => { sleeps.push(ms); t += ms; }, clock: () => t, random: () => 0 });
expect(verdict.state).toBe('running');
expect(sleeps.length).toBeGreaterThan(0);
for (const ms of sleeps) expect(ms).toBeGreaterThanOrEqual(MIN_POLL_MS);
expect(reads).toBe(sleeps.length + 1);
});
});
+2 -2
View File
@@ -64,7 +64,7 @@ function fixture(opts: {
if (cmd === "gh" && args[0] === "repo") return ok("{}");
if (cmd === "gh" && joined.includes("/comments?sort=created")) return ok(JSON.stringify(joined.includes("/issues/") ? opts.comments ?? [] : []));
if (cmd === "gh" && joined.startsWith("pr list")) return ok(JSON.stringify(opts.openPrs ?? []));
if (cmd === "gh" && joined.startsWith("pr view")) return ok(SHA + "\n");
if (cmd === "gh" && joined.startsWith("api repos/{owner}/{repo}/pulls/12 ")) return ok(SHA + "\n");
if (cmd === "gh" && joined.includes("/check-runs")) return ok(JSON.stringify(opts.checks ?? []));
if (cmd === "gh" && joined.includes("/reviews")) return ok(JSON.stringify(opts.reviews ?? []));
if (cmd === "gh" && joined.includes("/issues/12/comments")) return ok(JSON.stringify(opts.summaries ?? []));
@@ -237,7 +237,7 @@ describe("waiting for the review (CEO-8 completion signal)", () => {
const f = fixture();
const polls = () => f.calls.filter(c => c.args.join(" ").includes("/check-runs")).length;
f.run("wait", "12", "--since", since);
expect(polls()).toBe(4); // at 0, 30, 60 and 90 seconds
expect(polls()).toBe(2); // at 0 and 60-75 seconds: polls are at least 60 s apart (GST-1713)
f.advance(9 * 60_000);
const r = f.run("wait", "12", "--since", since);
expect(r.out).toContain("GREPTILE_REVIEW: timeout");
+16 -8
View File
@@ -227,30 +227,38 @@ describe('polling', () => {
test('delayed registration: zero checks re-polled until checks appear', async () => {
const c = clock();
const gh = fakeGh({ required: [NONE, NONE, NONE_REQUIRED], all: [NONE, NONE, ok([row('build', 'pending')])] });
const result = await runCiGate(options({ registrationWaitSeconds: 60 }), gh, c.sleep, c.now);
const result = await runCiGate(options({ registrationWaitSeconds: 120 }), gh, c.sleep, c.now, () => 0);
expect(result.verdict).toBe('PENDING');
expect(c.now()).toBe(20_000);
expect(c.now()).toBe(120_000);
});
test('registration wait is bounded at 60s, then NO_CHECKS', async () => {
const c = clock();
const result = await runCiGate(options({ registrationWaitSeconds: 60 }), fakeGh({ required: [NONE], all: [NONE] }), c.sleep, c.now);
const result = await runCiGate(options({ registrationWaitSeconds: 60 }), fakeGh({ required: [NONE], all: [NONE] }), c.sleep, c.now, () => 0);
expect(result.verdict).toBe('NO_CHECKS');
expect(c.now()).toBe(60_000);
});
test('registration wait re-polls once even when the first read takes time', async () => {
const c = clock();
const base = fakeGh({ required: [NONE, NONE_REQUIRED], all: [NONE, ok([row('build', 'pending')])] });
const gh: GhRunner = args => { c.sleep(500); return base(args); };
const result = await runCiGate(options({ registrationWaitSeconds: 60 }), gh, c.sleep, c.now, () => 0);
expect(result.verdict).toBe('PENDING');
});
test('--wait polls PENDING until a final verdict', async () => {
const c = clock();
const all = [ok([row('t', 'pending')]), ok([row('t', 'pending')]), ok([row('t', 'pass')])];
const result = await runCiGate(options({ waitSeconds: 240 }), fakeGh({ required: [NONE_REQUIRED], all }), c.sleep, c.now);
const result = await runCiGate(options({ waitSeconds: 240 }), fakeGh({ required: [NONE_REQUIRED], all }), c.sleep, c.now, () => 0);
expect(result.verdict).toBe('PASS');
expect(c.now()).toBe(60_000);
expect(c.now()).toBe(120_000);
});
test('--wait timeout reports PENDING and names the still-pending checks', async () => {
const c = clock();
const all = [ok([row('build', 'pass'), row('deploy-preview', 'pending')])];
const result = await runCiGate(options({ waitSeconds: 240 }), fakeGh({ required: [NONE_REQUIRED], all }), c.sleep, c.now);
const result = await runCiGate(options({ waitSeconds: 240 }), fakeGh({ required: [NONE_REQUIRED], all }), c.sleep, c.now, () => 0);
expect(result.verdict).toBe('PENDING');
expect(checkLines(result.lines)).toEqual(['CHECK\tdeploy-preview\tpending\trequired=n\thttps://ci.example/deploy-preview']);
expect(result.lines.some(line => line.startsWith('NOTE still pending after waiting 240s'))).toBe(true);
@@ -259,9 +267,9 @@ describe('polling', () => {
test('a failure during the wait stops immediately (fail fast)', async () => {
const c = clock();
const all = [ok([row('t', 'pending')]), ok([row('t', 'fail'), row('u', 'pending')])];
const result = await runCiGate(options({ waitSeconds: 240 }), fakeGh({ required: [NONE_REQUIRED], all }), c.sleep, c.now);
const result = await runCiGate(options({ waitSeconds: 240 }), fakeGh({ required: [NONE_REQUIRED], all }), c.sleep, c.now, () => 0);
expect(result.verdict).toBe('FAIL');
expect(c.now()).toBe(30_000);
expect(c.now()).toBe(60_000);
});
});
+1
View File
@@ -151,6 +151,7 @@ export const FREE_FIXTURES: Readonly<Record<string, readonly string[]>> = {
'test/fixtures/exit-propagation/**': ['test/exit-propagation.test.ts'],
'test/fixtures/factory-scorecard/**': ['test/factory-scorecard.test.ts'],
'test/fixtures/free-shard-capture.ts': ['test/test-free-shards-capture.test.ts'],
'test/fixtures/gh-poll/**': ['test/gh-poll.test.ts'],
'test/fixtures/golden/**': ['test/host-config.test.ts', 'test/ship-review-loop.test.ts'],
'test/fixtures/health-detect-legacy.sh': ['test/health-detect-no-loss.test.ts'],
'test/fixtures/impeccable-antipatterns.json': ['test/design-catalog.test.ts', 'test/impeccable-fixtures.test.ts'],
+1 -1
View File
@@ -63,7 +63,7 @@ export function trainFixture(policy: Record<string, unknown> = TRAIN_POLICY, pre
fs.writeFileSync(path.join(dir, 'shim.js'), shimSource());
fs.writeFileSync(path.join(dir, 'gh'), `#!/bin/sh\nexec node "${path.join(dir, 'shim.js')}" "${dataFile}" "$@"\n`, { mode: 0o755 });
const state = path.join(repo.root, 'state');
const env = { PATH: `${dir}:${process.env.PATH ?? ''}`, GSTACK_STATE_ROOT: state, GSTACK_SHIP_QUEUE_POLL_MS: '20', GSTACK_EVAL_BG_POLL_MS: '20', GSTACK_SHIP_QUEUE_NOW: '2026-10-18T12:00:00Z' };
const env = { PATH: `${dir}:${process.env.PATH ?? ''}`, GSTACK_STATE_ROOT: state, GSTACK_SHIP_QUEUE_POLL_MS: '20', GSTACK_GH_POLL_FLOOR_MS: '0', GSTACK_EVAL_BG_POLL_MS: '20', GSTACK_SHIP_QUEUE_NOW: '2026-10-18T12:00:00Z' };
const read = () => JSON.parse(fs.readFileSync(dataFile, 'utf8')) as ShimData;
const write = (fn: (d: ShimData) => void) => { const d = read(); fn(d); fs.writeFileSync(dataFile, JSON.stringify(d)); };
return { repo, state, dataFile, env, read, write };
+84 -13
View File
@@ -53,7 +53,31 @@ const prJson = (shape: PrShape = {}) => JSON.stringify({
const checksJson = (...rows: Array<[string, string] | [string, string, string]>) =>
JSON.stringify(rows.map(([name, bucket, description]) => ({ name, bucket, link: `https://ci.example/${name}`, description: description ?? '' })));
/** Scripted replies per gh command shape; a list is consumed one poll at a time, the last reply repeating. */
/** The REST `pulls/{n}` reply for a scripted GraphQL PR reply: same facts, REST spellings; updated_at moves when the reply does. */
function restPull(graphql: string): string {
const node = JSON.parse(graphql).data.repository.pullRequest;
return JSON.stringify({
html_url: node.url, state: node.state === 'OPEN' ? 'open' : 'closed', merged: node.state === 'MERGED', head: { sha: node.headRefOid },
base: { ref: node.baseRefName }, mergeable_state: String(node.mergeStateStatus).toLowerCase(), user: { login: node.author.login },
updated_at: `u-${graphql.length}-${[...graphql].reduce((h, ch) => (h * 31 + ch.charCodeAt(0)) | 0, 0)}`,
});
}
const CONCLUSION: Record<string, string> = { pass: 'success', fail: 'failure', skipping: 'skipped', cancel: 'cancelled' };
/** `gh pr checks`-shaped rows as the REST check-runs lines `gh api --paginate --jq '.check_runs[]'` prints, one suite per row. */
function restCheckRuns(result: GhResult): GhResult {
if (result.status !== 0) return result;
const rows = JSON.parse(result.stdout || '[]') as Array<{ name: string; bucket: string; link: string; description: string }>;
return ok(rows.map((row, i) => JSON.stringify({
id: i + 1, name: row.name, status: row.bucket === 'pending' ? 'in_progress' : 'completed', conclusion: row.bucket === 'pending' ? null : CONCLUSION[row.bucket] ?? row.bucket,
completed_at: row.bucket === 'pending' ? null : '2026-11-06T00:00:00Z', details_url: row.link, output: { title: row.description }, app: { id: 15368 }, check_suite: { id: 100 + i },
})).join('\n'));
}
/**
* Scripted replies per gh command shape; a list is consumed one poll at a time, the last reply repeating. `pr` is the
* GraphQL reply; each REST `pulls/{n}` read takes the next one and the GraphQL conversation read returns that same reply.
* `checks` is `gh pr checks`-shaped and served as REST check runs.
*/
interface Script {
pr?: string | string[];
checks?: GhResult | GhResult[];
@@ -67,15 +91,26 @@ function fakeGh(script: Script) {
// Function declarations, not generic arrows: pregate's import scanner reads tests as TSX, where `<T>(x) =>` is a tag.
function queue<T>(value: T | T[] | undefined): T[] { return value === undefined ? [] : Array.isArray(value) ? [...value] : [value]; }
function next<T>(q: T[]): T | undefined { return q.length > 1 ? q.shift() : q[0]; }
const prs = queue(script.pr).map(ok);
const prs = queue(script.pr);
const checks = queue(script.checks);
const calls: string[][] = [];
let current = prJson();
const gh = (args: string[]): GhResult => {
calls.push(args);
const [a, b] = args;
if (a === 'api' && b === 'graphql') return next(prs) ?? ok(prJson());
if (a === 'pr' && b === 'checks') return next(checks) ?? ok(checksJson());
if (a === 'pr' && b === 'view') return ok('{"number":42}');
const a = args[0];
const b = args[1] === '--paginate' ? args[2] : args[1];
if (a === 'api' && b === 'graphql') return ok(current);
if (a === 'api' && b === `repos/${REPO}/pulls/42`) {
if (args.includes('.number')) return ok('42\n');
current = next(prs) ?? prJson();
return ok(restPull(current));
}
if (a === 'api' && b.includes('/check-runs')) return restCheckRuns(next(checks) ?? ok(checksJson()));
if (a === 'api' && b.includes('/status')) return ok('');
if (a === 'api' && b.includes('/git/ref/heads/')) {
const oid = JSON.parse(current).data.repository.pullRequest.baseRef?.target.oid;
return oid ? ok(`${oid}\n`) : NOT_FOUND;
}
if (a === 'pr' && b === 'diff') return script.prFiles ?? ok('');
if (a === 'repo' && b === 'view') return ok(`${REPO}\n`);
if (a === 'api' && b.includes('/rules/branches/')) return script.rulesets ?? NOT_FOUND;
@@ -360,7 +395,7 @@ describe('arguments and the gh layer', () => {
expect(parsePrTarget('abc')).toMatchObject({ error: "'abc' is not a PR number or PR URL" });
expect(parsePrTarget('0')).toMatchObject({ error: "'0' is not a valid PR number" });
expect(parsePrTarget('42', 'nope')).toMatchObject({ error: "'nope' is not owner/name" });
expect(parsePrWaitArgs(['42', '--until', 'green', '--interval', '2'])).toEqual({ target: '42', until: 'green', interval: 5, timeout: 60, excludes: [] });
expect(parsePrWaitArgs(['42', '--until', 'green', '--interval', '2'])).toEqual({ target: '42', until: 'green', interval: 60, timeout: 60, excludes: [] });
expect(parsePrWaitArgs(['42', '--exclude', 'quality-gate, canary', '--exclude', 'canary'])).toMatchObject({ excludes: ['quality-gate', 'canary'] });
expect(parsePrWaitArgs(['42', '--exclude', ' , '])).toMatchObject({ error: expect.stringContaining('--exclude needs') });
expect(parsePrWaitArgs(['42', '--timeout', '0'])).toMatchObject({ timeout: 0 });
@@ -377,9 +412,9 @@ describe('arguments and the gh layer', () => {
expect(prWaitMain([], env(fakeGh({})))).toBe(2);
expect(lines()[0]).toMatch(/missing PR number.*\(PR_WAIT_REFUSED; .*#pr-wait-refused\)$/);
stdout = [];
const gone = (args: string[]): GhResult => (args[1] === 'view' ? err('GraphQL: Could not resolve to a PullRequest with the number of 42. (HTTP 404)') : ok(''));
const gone = (args: string[]): GhResult => (args[1] === `repos/${REPO}/pulls/42` ? err('gh: Not Found (HTTP 404)') : ok(''));
expect(prWaitMain(['42', '--repo', REPO], env(gone))).toBe(2);
expect(lines()[0]).toContain('cannot read owner/project#42: GraphQL: Could not resolve');
expect(lines()[0]).toContain('cannot read owner/project#42: gh: Not Found (HTTP 404)');
expect(RESULT_CODES.PR_WAIT_REFUSED.anchor).toBe('pr-wait-refused');
expect(RESULT_CODES.PR_WAIT_BLOCKED.anchor).toBe('pr-wait-blocked');
});
@@ -539,13 +574,13 @@ describe('pass-1 review fixes: gates agree, bounded waits, bot identity, local f
});
test('run() ends as blocked after ten transient failures, and at once on a non-transient gh error', () => {
const down = (args: string[]): GhResult => (args[1] === 'graphql' ? err('gh: Something went wrong (HTTP 502)') : ok(''));
const down = (): GhResult => err('gh: Something went wrong (HTTP 502)');
const e = env(down);
expect(new Watcher(e, REPO, 42, 'green', 20, botLists(e.vars)).run()).toBe(PR_WAIT_EXIT.blocked);
expect(stderr.join('').match(/poll failed \(\d+\/10\)/g)).toHaveLength(10);
expect(lines()[0]).toBe('#42 gstack-pr-wait stopped: GitHub unreachable for 10 polls: gh: Something went wrong (HTTP 502)');
stdout = [];
const gone = (args: string[]): GhResult => (args[1] === 'graphql' ? err('gh: Not Found (HTTP 404)') : ok(''));
const gone = (args: string[]): GhResult => (args[1] === `repos/${REPO}/pulls/42` ? err('gh: Not Found (HTTP 404)') : ok(''));
const e2 = env(gone);
expect(new Watcher(e2, REPO, 42, 'green', 20, botLists(e2.vars)).run()).toBe(PR_WAIT_EXIT.blocked);
expect(lines()[0]).toBe('#42 gstack-pr-wait stopped: gh: Not Found (HTTP 404)');
@@ -595,9 +630,9 @@ describe('pass-1 review fixes: gates agree, bounded waits, bot identity, local f
mkdirSync(fakeBin);
writeFileSync(join(fakeBin, 'gh'), `#!/bin/sh
case "$1 $2" in
'pr view') echo '{"number":42}' ;;
'api repos/owner/project/pulls/42') echo '${restPull(prJson({}))}' ;;
'api graphql') echo '${prJson({})}' ;;
'pr checks') echo '${checksJson(['build', 'pending'])}'; exit 8 ;;
'api --paginate') echo '${restCheckRuns(ok(checksJson(['build', 'pending']))).stdout}' ;;
*) echo 'gh: Not Found (HTTP 404)' >&2; exit 1 ;;
esac`, { mode: 0o755 });
const stateRoot = join(dir, 'state');
@@ -619,6 +654,42 @@ esac`, { mode: 0o755 });
}, 30_000);
});
describe('GitHub budget (GST-1713): REST every poll, the GraphQL conversation read only when it can have changed', () => {
const graphqlReads = (gh: ReturnType<typeof fakeGh>) => gh.calls.filter(c => c[1] === 'graphql').length;
const run = (script: Script, until: PrWaitOptions['until'], polls: number, between: () => void = () => {}) => {
const gh = fakeGh(script);
const e = env(gh);
const w = new Watcher(e, REPO, 42, until, 60, botLists(e.vars));
for (let i = 0; i < polls; i++) { expect(w.poll()).toBeNull(); w.save(); w.baseline = false; between(); }
return gh;
};
// Value: protects=the shared GraphQL quota; fails_when=gstack-pr-wait spends a GraphQL read (or gh pr checks) on every poll again
test('an unchanged PR is read through GraphQL once across polls; checks and state never touch GraphQL', () => {
const gh = run({ checks: ok(checksJson(['build', 'pending'])) }, 'green', 4);
expect(graphqlReads(gh)).toBe(1);
expect(gh.calls.some(c => c[0] === 'pr')).toBe(false);
expect(gh.calls.filter(c => c.includes(`repos/${REPO}/pulls/42`))).toHaveLength(4);
});
test('a new comment (updated_at moves) or ten minutes reread the conversation; a ready watch rereads it once green', () => {
const talk = comment('c9', { login: 'reviewer' }, 'please rename');
const changed = run({ pr: [prJson(), prJson(), prJson({ comments: [talk] })], checks: ok(checksJson(['build', 'pending'])) }, 'green', 4);
expect(graphqlReads(changed)).toBe(2);
expect(lines()).toEqual([`#42 comment from reviewer: please rename https://github.com/${REPO}/pull/42#issuecomment-c9`]);
expect(graphqlReads(run({ checks: ok(checksJson(['build', 'pending'])) }, 'green', 3, () => { clock += 10 * 60_000; }))).toBe(3);
expect(graphqlReads(run({ checks: ok(checksJson(['build', 'pass'])) }, 'ready', 3))).toBe(3);
});
test('run() sleeps at least 60 s between polls, even when asked for 5', () => {
const sleeps: number[] = [];
const e = { ...env(fakeGh({ checks: ok(checksJson(['build', 'pending'])) })), sleep: (ms: number) => { sleeps.push(ms); clock += ms; } };
expect(new Watcher(e, REPO, 42, 'green', 5, botLists(e.vars), 5).run()).toBe(PR_WAIT_EXIT.blocked);
expect(sleeps.length).toBeGreaterThan(0);
for (const ms of sleeps) expect(ms).toBeGreaterThanOrEqual(60_000);
});
});
describe('state matrix cells (gstack-state-matrix: --until × bucket, merge state × --until, lock, baseline × event)', () => {
const GREPTILE = review('g1', { login: 'greptile-apps', bot: true }, 'COMMENTED', '');