Stop gstack-pr-wait, the land gate and the release cut from calling a head green while a check on it failed (GST-2252) (#4257)

* Make gstack-pr-wait, gstack-ci-gate and the release cut count a failed check hidden behind an empty rerun (GST-2252)

GitHub rollup and default check-runs list hold only each check suite latest attempt, so a rerun attempt with no jobs hid the failed run before it, and a workflow run that failed before any job never appeared. All three readers now add CI at the commit as the merge queue reads it (ciAtHead via lib/ci-head-checks.ts) and keep the worse bucket per check. Tests reuse the gstack-evals#303 recording from #4235.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* Have the release cut read every check attempt only for commits that pass its named checks (GST-2252)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* Say in the changelog that the release cut now counts failed commit statuses (GST-2252)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* Let the release cut set its own jobs aside in the every-attempt check read too (GST-2252)

After #4297 the cut sets aside check runs from its own workflow and the Windows nightly by job id, but the rows from lib/ci-head-checks.ts skipped that filter, so the cut waited on its own run. headCheckRows now takes the set-aside check run and workflow run ids and removes them from every list before ciAtHead reads it.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

---------

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
Garry Tanandcapy-ai[bot] authored and GitHub committed 2026-11-17 10:03:19 +00:00
1 parent 2af3822f94
commit 83cc10acd2
15 files changed
+307 -43

No files matched your search

+12 -3
View File
@@ -1,4 +1,5 @@
import { spawnSync } from 'node:child_process';
import { headCheckRows, type HeadCheckRow } from './ci-head-checks';
import { MIN_POLL_MS, pollDelayMs, syncLimitEnv, withRateLimit } from './gh-poll';
export type CiVerdict = 'PASS' | 'FAIL' | 'PENDING' | 'NO_CHECKS' | 'ERROR';
@@ -38,7 +39,11 @@ export const CI_GATE_USAGE = `gstack-ci-gate --repo OWNER/NAME --pr NUMBER --exp
Decides whether a pull request's CI allows a merge of exactly SHA. Reads the
PR head, gates on every check (required ones and all others), and re-reads the
head afterwards; a moved head is ERROR.
head afterwards; a moved head is ERROR. Besides gh pr checks, which lists only
each check suite's latest attempt, it reads CI at SHA the way gstack-ship-queue
ready does: a failed check whose rerun attempt created no jobs, and a workflow
run that failed before creating any job ("<workflow> (run <id>, no jobs)"),
count as failed.
Output (stdout), always first:
VERDICT <PASS|FAIL|PENDING|NO_CHECKS|ERROR> <sha>
@@ -106,7 +111,8 @@ export function parseChecks(result: GhResult): ChecksRead {
interface Evaluation { verdict: Exclude<CiVerdict, 'ERROR'>; lines: string[] }
type EvalOrError = Evaluation | { verdict: 'ERROR'; cause: string; stderr: string };
export function evaluateChecks(required: GhResult, all: GhResult, excludes: string[]): EvalOrError {
/** `head`: CI at the head from lib/ci-head-checks.ts, observed like the rest, so a check it reads worse than gh pr checks does gates. */
export function evaluateChecks(required: GhResult, all: GhResult, excludes: string[], head: HeadCheckRow[] = []): EvalOrError {
const allRead = parseChecks(all);
if (allRead.kind === 'error') return { verdict: 'ERROR', cause: allRead.cause, stderr: allRead.stderr };
if (allRead.kind === 'none-required') return { verdict: 'ERROR', cause: 'gh pr checks without --required reported no required checks', stderr: '' };
@@ -129,6 +135,7 @@ export function evaluateChecks(required: GhResult, all: GhResult, excludes: stri
};
for (const obs of allRead.kind === 'rows' ? allRead.rows : []) observe(obs);
for (const obs of requiredRows) observe(obs);
for (const obs of head) observe({ ...obs, link: '' });
if (rows.size === 0) return { verdict: 'NO_CHECKS', lines: notes.filter(n => !n.includes('no required checks')) };
const excluded = new Set<string>();
@@ -180,12 +187,14 @@ function evaluateOnce(gh: GhRunner, opts: CiGateOptions): CiGateResult {
const fields = ['--json', 'name,state,bucket,link'];
const required = gh([...checkArgs, '--required', ...fields]);
const all = gh([...checkArgs, ...fields]);
const head = headCheckRows(gh, opts.repo, opts.expectHead);
if ('error' in head) return fail(errorLines(head.error, ''));
const after = readHead(gh, opts);
if ('cause' in after) return fail(errorLines(after.cause, after.stderr));
if (after.head !== before.head) {
return fail([`HEAD ${after.head}`, 'CAUSE the PR head moved while checks were being read', 'FIX rerun /land-and-deploy from Step 1 for the new head']);
}
const evaluation = evaluateChecks(required, all, opts.excludes);
const evaluation = evaluateChecks(required, all, opts.excludes, head.rows);
if (evaluation.verdict === 'ERROR') return fail(errorLines(evaluation.cause, evaluation.stderr));
return { verdict: evaluation.verdict, sha: opts.expectHead, lines: evaluation.lines };
}
+69
View File
@@ -0,0 +1,69 @@
/**
* ci-head-checks: the merge queue's read of CI at one commit (ciAtHead in lib/ship-queue.ts, GST-2167) as `gh pr checks`
* bucket rows, for the other readers that decide a commit is green: gstack-pr-wait, gstack-ci-gate and the release cut
* (crew/scripts/release-cut.ts) (GST-2252).
*
* GitHub's rollup (`gh pr checks`) and its default check-runs list hold only each check suite's latest attempt, so a rerun
* attempt that created no jobs hides the failed run before it, and a workflow run that failed before creating a job is
* never a check run at all: gstack-evals#303 read green with its `checks` job red. ciAtHead counts both. Each caller adds
* these rows to the ones it already reads and keeps the worst bucket per name, so no check that gated before stops gating.
*/
import type { GhRunner } from './ci-gate';
import { ciAtHead } from './ship-queue';
export interface HeadCheckRow { name: string; bucket: string }
const CONCLUSION_BUCKET: Record<string, string> = {
success: 'pass', failure: 'fail', timed_out: 'fail', action_required: 'fail', startup_failure: 'fail',
cancelled: 'cancel', skipped: 'skipping', neutral: 'skipping',
};
/** Check runs (by id) and workflow runs (by id) a caller sets aside, such as the release cut's own jobs (GST-2463). */
export interface AsideRuns { checkRunIds: ReadonlySet<number>; workflowRunIds: ReadonlySet<number> }
const PAGE = 100;
/**
* Every check and commit-status context ciAtHead counts at `sha` in `repo`, or why they could not all be read (unknown is
* never green). A check with an unknown conclusion keeps its spelling and one with none is `fail`, as ciAtHead counts them.
* `aside` removes those check runs and workflow runs from every list before ciAtHead reads it: each list is read whole and
* served again without them, so ciAtHead's total_count check still holds, and a list that cannot be read whole is unknown.
*/
export function headCheckRows(gh: GhRunner, repo: string, sha: string, aside?: AsideRuns): { rows: HeadCheckRow[] } | { error: string } {
const run = (args: string[]) => gh(args.map(a => a.replace('repos/{owner}/{repo}/', `repos/${repo}/`)));
const lists = new Map<string, Array<Record<string, unknown>> | null>();
const readWhole = (prefix: string, key: string, drop: ReadonlySet<number>) => {
const items: Array<Record<string, unknown>> = [];
for (let page = 1; page <= 10; page++) {
const r = run(['api', `${prefix}per_page=${PAGE}&page=${page}`]);
let body: Record<string, unknown> | null = null;
try { body = !r.error && r.status === 0 ? JSON.parse(r.stdout) : null; } catch {}
const list = body?.[key];
if (!Array.isArray(list) || typeof body?.total_count !== 'number') return null;
items.push(...list);
if (items.length >= body.total_count || list.length < PAGE) return items.length === body.total_count ? items.filter(i => !drop.has(Number(i.id))) : null;
}
return null;
};
const inRepo = (args: string[]) => {
const m = aside && args[0] === 'api' ? /^(.*(?:\/check-runs|\/actions\/runs)\?(?:.*&)?)per_page=\d+&page=(\d+)$/.exec(args[1] ?? '') : null;
if (!aside || !m) {
const r = run(args);
return { ok: !r.error && r.status === 0, out: r.stdout, err: r.error ?? r.stderr };
}
const key = m[1]!.includes('/check-runs') ? 'check_runs' : 'workflow_runs';
if (!lists.has(m[1]!)) lists.set(m[1]!, readWhole(m[1]!, key, key === 'check_runs' ? aside.checkRunIds : aside.workflowRunIds));
const all = lists.get(m[1]!);
if (!all) return { ok: false, out: '', err: `${key} unreadable while setting runs aside` };
const page = Number(m[2]);
return { ok: true, out: JSON.stringify({ total_count: all.length, [key]: all.slice((page - 1) * PAGE, page * PAGE) }), err: '' };
};
const ci = ciAtHead({ repoRoot: process.cwd(), base: '', gh: inRepo }, sha);
if (ci.line.startsWith('unknown')) return { error: ci.line.replace(/^unknown at (\S+)/, 'CI at $1 is unknown') };
return {
rows: [
...ci.checks.map(c => ({ name: c.name, bucket: c.status !== 'completed' ? 'pending' : CONCLUSION_BUCKET[c.conclusion ?? ''] ?? (c.conclusion || 'fail') })),
...ci.contexts.map(s => ({ name: s.name, bucket: s.state === 'success' ? 'pass' : s.state === 'pending' ? 'pending' : 'fail' })),
],
};
}
+10 -3
View File
@@ -13,6 +13,7 @@
* gstack's gh runner shape (lib/ci-gate.ts).
*/
import type { GhResult, GhRunner } from './ci-gate';
import { headCheckRows } from './ci-head-checks';
import { parseHeadChecks, parsePullState, rateLimitKind, type PullState } from './gh-poll';
export class GhError extends Error {
@@ -179,14 +180,20 @@ function restList(env: GhEnv, path: string, key: string): unknown[] {
/**
* Every check on the head commit by name, from REST check runs and commit
* statuses (the same rows `gh pr checks` reads through GraphQL), duplicates
* collapsed to the worst bucket.
* statuses (the same rows `gh pr checks` reads through GraphQL) plus CI at the
* head as gstack-ship-queue ready reads it (lib/ci-head-checks.ts: a failed
* check whose rerun attempt created no jobs, a workflow run that failed with
* no jobs), duplicates collapsed to the worst bucket. An unknown read is a
* transient GhError, so the poll loop tries again.
*/
export function fetchChecks(env: GhEnv, repo: string, head: string): Map<string, Check> {
const rows = parseHeadChecks(
const listed = parseHeadChecks(
restList(env, `repos/${repo}/commits/${head}/check-runs?per_page=100`, 'check_runs'),
restList(env, `repos/${repo}/commits/${head}/status?per_page=100`, 'statuses'),
);
const ci = headCheckRows(args => runGh(env, args), repo, head);
if ('error' in ci) throw new GhError(ci.error, true);
const rows = [...listed, ...ci.rows.map(row => ({ ...row, link: '', description: '' }))];
const checks = new Map<string, Check>();
for (const row of rows) {
const current = checks.get(row.name);