Stop gstack-pr-wait, the land gate and the release cut from calling a head green while a check on it failed (GST-2252) (#4257)

* Make gstack-pr-wait, gstack-ci-gate and the release cut count a failed check hidden behind an empty rerun (GST-2252)

GitHub rollup and default check-runs list hold only each check suite latest attempt, so a rerun attempt with no jobs hid the failed run before it, and a workflow run that failed before any job never appeared. All three readers now add CI at the commit as the merge queue reads it (ciAtHead via lib/ci-head-checks.ts) and keep the worse bucket per check. Tests reuse the gstack-evals#303 recording from #4235.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* Have the release cut read every check attempt only for commits that pass its named checks (GST-2252)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* Say in the changelog that the release cut now counts failed commit statuses (GST-2252)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* Let the release cut set its own jobs aside in the every-attempt check read too (GST-2252)

After #4297 the cut sets aside check runs from its own workflow and the Windows nightly by job id, but the rows from lib/ci-head-checks.ts skipped that filter, so the cut waited on its own run. headCheckRows now takes the set-aside check run and workflow run ids and removes them from every list before ciAtHead reads it.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

---------

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
Garry Tanandcapy-ai[bot] authored and GitHub committed 2026-11-17 10:03:19 +00:00
1 parent 2af3822f94
commit 83cc10acd2
15 files changed
+307 -43

No files matched your search

+6 -2
View File
@@ -14,6 +14,7 @@ import type { GhResult } from '../lib/ci-gate';
import { MAX_LIMIT_WAITS, MIN_POLL_MS, parseHeadChecks, parsePullState, pollDelayMs, rateLimitKind, rateLimitWaitMs, withRateLimit } from '../lib/gh-poll';
import { GhError, fetchChecks, runGh } from '../lib/pr-wait-github';
import { waitForRun } from '../lib/ci-carry';
import { ciRest } from './helpers/ci-attempts-fixture';
const fixture = (name: string): string => readFileSync(join(import.meta.dir, 'fixtures', 'gh-poll', name), 'utf8');
const ok = (stdout: string): GhResult => ({ status: 0, stdout, stderr: '' });
@@ -172,8 +173,7 @@ describe('REST replies parse from recorded fixtures', () => {
const calls: string[][] = [];
const gh = (args: string[]): GhResult => {
calls.push(args);
if (args[2]!.includes('/check-runs')) return ok([...runs, pushRun].map(r => JSON.stringify(r)).join('\n'));
return ok('');
return ciRest(args, { check_runs: [...runs, pushRun] }) ?? ok('');
};
const checks = fetchChecks({ gh, sleep: () => {}, log: () => {} }, 'garrytan/gstack', 'f'.repeat(40));
expect(checks.size).toBe(new Set(runs.map(r => r.name)).size);
@@ -181,6 +181,10 @@ describe('REST replies parse from recorded fixtures', () => {
expect(calls.map(c => c.slice(0, 3))).toEqual([
['api', '--paginate', `repos/garrytan/gstack/commits/${'f'.repeat(40)}/check-runs?per_page=100`],
['api', '--paginate', `repos/garrytan/gstack/commits/${'f'.repeat(40)}/status?per_page=100`],
['api', `repos/garrytan/gstack/commits/${'f'.repeat(40)}/check-runs?per_page=100&page=1`],
['api', `repos/garrytan/gstack/commits/${'f'.repeat(40)}/check-runs?filter=all&per_page=100&page=1`],
['api', `repos/garrytan/gstack/commits/${'f'.repeat(40)}/status?per_page=100&page=1`],
['api', `repos/garrytan/gstack/actions/runs?head_sha=${'f'.repeat(40)}&per_page=100&page=1`],
]);
expect(calls.some(c => c[0] === 'pr' || c[1] === 'graphql')).toBe(false);
});
+26 -1
View File
@@ -10,6 +10,7 @@ import { describe, expect, test } from 'bun:test';
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
import { ciGateMain, evaluateChecks, parseCiGateArgs, runCiGate, type CiGateOptions, type GhResult, type GhRunner } from '../lib/ci-gate';
import { PR303, PR303_RERUN_PASSED, ciRest, type CiAtCommit } from './helpers/ci-attempts-fixture';
const HEAD = 'a'.repeat(40);
const OTHER = 'b'.repeat(40);
@@ -21,13 +22,16 @@ const row = (name: string, bucket?: string) => ({ name, state: 'X', link: `https
const NONE = err("no checks reported on the 'feature' branch");
const NONE_REQUIRED = err("no required checks reported on the 'feature' branch");
interface Script { heads?: string[]; required?: GhResult[]; all?: GhResult[]; api?: GhResult }
/** `rest` is REST CI at the head (none by default); null answers none of those reads, so they fall to `api`. */
interface Script { heads?: string[]; required?: GhResult[]; all?: GhResult[]; api?: GhResult; rest?: CiAtCommit | null }
function fakeGh(script: Script): GhRunner & { calls: string[][] } {
const queues = { head: script.heads ?? [HEAD], required: script.required ?? [], all: script.all ?? [] };
const next = <T>(queue: T[]): T => (queue.length > 1 ? queue.shift()! : queue[0]);
const calls: string[][] = [];
const gh = ((args: string[]) => {
calls.push(args);
const rest = script.rest === null ? null : ciRest(args, script.rest ?? { check_runs: [] });
if (rest) return rest;
if (args[0] === 'api') return script.api ?? err('gh: Not Found (HTTP 404)');
if (args[1] === 'view') return { status: 0, stdout: `${next(queues.head)}\n`, stderr: '' };
if (args[1] === 'merge') throw new Error('the gate must never merge');
@@ -181,11 +185,32 @@ describe('head binding', () => {
['pr', 'view', '42', '--repo', 'owner/project', '--json', 'headRefOid', '--jq', '.headRefOid'],
['pr', 'checks', '42', '--repo', 'owner/project', '--required', '--json', 'name,state,bucket,link'],
['pr', 'checks', '42', '--repo', 'owner/project', '--json', 'name,state,bucket,link'],
['api', `repos/owner/project/commits/${HEAD}/check-runs?per_page=100&page=1`],
['api', `repos/owner/project/commits/${HEAD}/check-runs?filter=all&per_page=100&page=1`],
['api', `repos/owner/project/commits/${HEAD}/status?per_page=100&page=1`],
['api', `repos/owner/project/actions/runs?head_sha=${HEAD}&per_page=100&page=1`],
['pr', 'view', '42', '--repo', 'owner/project', '--json', 'headRefOid', '--jq', '.headRefOid'],
]);
});
});
describe('every attempt of each check suite counts (GST-2252)', () => {
// Value: protects=the merge gate counts a failed check whose rerun attempt created no jobs and a workflow run that failed before any job, as
// gstack-ship-queue ready does, while a rerun that passed stays PASS and an unreadable REST read is ERROR; fails_when=the gate trusts gh pr checks
// (GitHub's rollup lists only each suite's latest attempt) and passes gstack-evals#303; seam=lib/ci-head-checks.ts (ciAtHead) in evaluateOnce,
// through the fake gh's filter=latest/all split; no_claim=GitHub's attempt grouping (the fixture is #4235's recording)
test('a failed check hidden behind a rerun attempt with no jobs, and a workflow run that failed with no jobs, are FAIL (gstack-evals#303)', async () => {
const red = await gate({ required: [NONE_REQUIRED], all: [ok([row('sealed', 'pass')])], rest: PR303 });
expect(red.verdict).toBe('FAIL');
expect(checkLines(red.lines)).toEqual(['CHECK\tchecks\tfail\trequired=n\t', 'CHECK\tsealed (run 42671854799, no jobs)\tfail\trequired=n\t']);
const rerun = await gate({ required: [NONE_REQUIRED], all: [ok([row('checks', 'pass'), row('sealed', 'pass')])], rest: PR303_RERUN_PASSED });
expect(rerun.verdict).toBe('PASS');
const unreadable = await gate({ required: [NONE_REQUIRED], all: [ok([row('sealed', 'pass')])], rest: null, api: err('gh: HTTP 403') });
expect(unreadable.verdict).toBe('ERROR');
expect(unreadable.lines[0]).toStartWith('CAUSE CI at aaaaaaaaaaaa is unknown');
});
});
describe('per-head, per-check overrides', () => {
const nonRequiredRed = { required: [ok([row('build', 'pass')])], all: [ok([row('build', 'pass'), row('lint', 'fail'), row('e2e', 'pending')])] };
+63
View File
@@ -0,0 +1,63 @@
/**
* GitHub's REST answers about CI at one commit, for unit fakes of the readers that share ciAtHead (lib/ci-head-checks.ts,
* GST-2252), with the check runs and workflow runs #4235 (GST-2167) recorded from gstack-evals#303 in
* test/gstack-ship-queue.test.ts. A check run marked earlier_attempt belongs to a superseded workflow-run attempt: like
* GitHub's default filter=latest and ghQueueShim (test/helpers/ship-queue-fixture.ts), it is listed only when the call asks
* for filter=all.
*/
export interface CiAtCommit { check_runs: Array<Record<string, unknown>>; statuses?: Array<Record<string, unknown>>; workflow_runs?: Array<Record<string, unknown>> }
const app = { id: 15368 };
const wf = (id: number, name: string, workflow_id: number, suite: number, conclusion: string, created_at: string) => ({ id, name, workflow_id, check_suite_id: suite, status: 'completed', conclusion, created_at });
const checks = { id: 129322614246, name: 'checks', status: 'completed', conclusion: 'failure', completed_at: '2026-11-09T17:49:57Z', app, check_suite: { id: 115553115622 } };
const sealed = { id: 129322588977, name: 'sealed', status: 'completed', conclusion: 'success', completed_at: '2026-11-09T17:22:47Z', app, check_suite: { id: 115553095053 } };
/**
* gstack-evals#303: `checks` failed in its first attempt and the rerun attempt created no jobs, so GitHub's default list shows
* only the passing `sealed`; a second sealed run failed without creating a job. ciAtHead reads it red (passed=1 failed=2).
*/
export const PR303: CiAtCommit = {
check_runs: [{ ...checks, earlier_attempt: true }, sealed],
workflow_runs: [
wf(42671854799, 'sealed', 2, 115574560782, 'failure', '2026-11-09T18:11:40Z'),
wf(42663983836, 'ci', 1, 115553115622, 'failure', '2026-11-09T17:22:30Z'),
wf(42663976278, 'sealed', 2, 115553095053, 'success', '2026-11-09T17:22:28Z'),
],
};
/** The negative control from the same test: the failed `checks` attempt was rerun and passed, so the head is green. */
export const PR303_RERUN_PASSED: CiAtCommit = {
check_runs: [{ ...checks, earlier_attempt: true }, { ...checks, id: 129330000000, conclusion: 'success', completed_at: '2026-11-09T18:20:00Z' }, sealed],
workflow_runs: [wf(42663983836, 'ci', 1, 115553115622, 'success', '2026-11-09T17:22:30Z'), wf(42663976278, 'sealed', 2, 115553095053, 'success', '2026-11-09T17:22:28Z')],
};
/**
* Bash `case "$*"` arms for a gh shim that answers every read of CI at a commit with `checkRuns` (all in their latest attempt):
* gateVerdict's `--paginate --slurp` check-runs pages and ciAtHead's single pages, with no statuses and no workflow runs.
*/
export function ciShimCases(checkRuns: unknown[]): string {
return [
` *--slurp*check-runs*) echo '${JSON.stringify([{ check_runs: checkRuns }])}' ;;`,
` *check-runs*) echo '${JSON.stringify({ total_count: checkRuns.length, check_runs: checkRuns })}' ;;`,
` */status\\?*) echo '{"total_count":0,"statuses":[]}' ;;`,
` *actions/runs*) echo '{"total_count":0,"workflow_runs":[]}' ;;`,
].join('\n') + '\n';
}
/**
* gh's reply to one REST read of CI at a commit, or null when `args` is not one: `commits/<sha>/check-runs`, `commits/<sha>/status`
* or `actions/runs?head_sha=<sha>`, as one page (`page=N`), `--paginate --jq '.<key>[]'` lines or `--paginate --slurp` pages.
*/
export function ciRest(args: string[], data: CiAtCommit): { status: number; stdout: string; stderr: string } | null {
if (args[0] !== 'api') return null;
const path = args.find((a, i) => i > 0 && !a.startsWith('--') && args[i - 1] !== '--jq') ?? '';
const m = /commits\/[0-9a-f]+\/(check-runs|status)\b|actions\/runs\?head_sha=/.exec(path);
if (!m) return null;
const key = m[1] === 'check-runs' ? 'check_runs' : m[1] === 'status' ? 'statuses' : 'workflow_runs';
const list = key === 'check_runs' ? data.check_runs.filter(r => /[?&]filter=all(&|$)/.test(path) || !r.earlier_attempt) : data[key] ?? [];
const out = (stdout: string) => ({ status: 0, stdout, stderr: '' });
if (args.includes('--jq')) return out(list.map(item => JSON.stringify(item)).join('\n'));
if (args.includes('--slurp')) return out(JSON.stringify([{ total_count: list.length, [key]: list }]));
const page = Number(/[?&]page=(\d+)/.exec(path)?.[1] ?? 1);
return out(JSON.stringify({ total_count: list.length, [key]: page === 1 ? list : [] }));
}
+7 -1
View File
@@ -20,7 +20,13 @@ const OTHER = 'b'.repeat(40);
const FAKE_GH = `#!/usr/bin/env bash
D="$FAKE_GH_DIR"
if [ "$1" = api ]; then kind=api
if [ "$1" = api ]; then
case "$2" in
*/check-runs\?*) echo '{"total_count":0,"check_runs":[]}'; exit 0 ;;
*/status\?*) echo '{"total_count":0,"statuses":[]}'; exit 0 ;;
*/actions/runs\?*) echo '{"total_count":0,"workflow_runs":[]}'; exit 0 ;;
esac
kind=api
elif [ "$1 $2" = "pr merge" ]; then printf '%s\\n' "$*" >> "$D/merges.log"; exit 0
elif [ "$1 $2" = "pr view" ]; then kind=head
elif [ "$1 $2" = "pr checks" ]; then case " $* " in *" --required "*) kind=required ;; *) kind=all ;; esac
+28 -8
View File
@@ -15,6 +15,7 @@ import type { GhResult } from '../lib/ci-gate';
import { DEFAULT_ADVISORY_BOTS, DEFAULT_GATING_BOTS, DEFAULT_TIMEOUT_MINUTES, PR_WAIT_EXIT, Watcher, botLists, parsePrTarget, parsePrWaitArgs, prWaitMain, stateDir, type PrWaitEnv, type PrWaitOptions } from '../lib/pr-wait';
import { GhError, runGh } from '../lib/pr-wait-github';
import { RESULT_CODES } from '../lib/result-codes';
import { PR303, PR303_RERUN_PASSED, ciRest, type CiAtCommit } from './helpers/ci-attempts-fixture';
const ROOT = join(import.meta.dir, '..');
const HEAD = 'a'.repeat(40);
@@ -63,14 +64,13 @@ function restPull(graphql: string): string {
});
}
const CONCLUSION: Record<string, string> = { pass: 'success', fail: 'failure', skipping: 'skipped', cancel: 'cancelled' };
/** `gh pr checks`-shaped rows as the REST check-runs lines `gh api --paginate --jq '.check_runs[]'` prints, one suite per row. */
function restCheckRuns(result: GhResult): GhResult {
if (result.status !== 0) return result;
/** `gh pr checks`-shaped rows as REST check runs, one suite per row. */
function checkRunObjects(result: GhResult): Array<Record<string, unknown>> {
const rows = JSON.parse(result.stdout || '[]') as Array<{ name: string; bucket: string; link: string; description: string }>;
return ok(rows.map((row, i) => JSON.stringify({
return rows.map((row, i) => ({
id: i + 1, name: row.name, status: row.bucket === 'pending' ? 'in_progress' : 'completed', conclusion: row.bucket === 'pending' ? null : CONCLUSION[row.bucket] ?? row.bucket,
completed_at: row.bucket === 'pending' ? null : '2026-11-06T00:00:00Z', details_url: row.link, output: { title: row.description }, app: { id: 15368 }, check_suite: { id: 100 + i },
})).join('\n'));
}));
}
/**
@@ -81,6 +81,8 @@ function restCheckRuns(result: GhResult): GhResult {
interface Script {
pr?: string | string[];
checks?: GhResult | GhResult[];
/** Raw REST CI at the head (check runs with attempts, workflow runs), served instead of `checks`. */
rest?: CiAtCommit;
rulesets?: GhResult;
protection?: GhResult;
prFiles?: GhResult;
@@ -95,6 +97,7 @@ function fakeGh(script: Script) {
const checks = queue(script.checks);
const calls: string[][] = [];
let current = prJson();
let shown: GhResult = ok(checksJson());
const gh = (args: string[]): GhResult => {
calls.push(args);
const a = args[0];
@@ -105,8 +108,10 @@ function fakeGh(script: Script) {
current = next(prs) ?? prJson();
return ok(restPull(current));
}
if (a === 'api' && b.includes('/check-runs')) return restCheckRuns(next(checks) ?? ok(checksJson()));
if (a === 'api' && b.includes('/status')) return ok('');
if (script.rest) { const rest = ciRest(args, script.rest); if (rest) return rest; }
// One poll reads the paginated list once (which takes the next scripted reply), then ciAtHead's pages of that same reply.
if (a === 'api' && args[1] === '--paginate' && b.includes('/check-runs')) shown = next(checks) ?? ok(checksJson());
if (a === 'api' && /\/check-runs|\/status|\/actions\/runs/.test(b)) return shown.status === 0 ? ciRest(args, { check_runs: checkRunObjects(shown) })! : shown;
if (a === 'api' && b.includes('/git/ref/heads/')) {
const oid = JSON.parse(current).data.repository.pullRequest.baseRef?.target.oid;
return oid ? ok(`${oid}\n`) : NOT_FOUND;
@@ -458,6 +463,20 @@ describe('pass-1 review fixes: gates agree, bounded waits, bot identity, local f
expect(lines()).toEqual(['#42 check failed: quality-gate https://ci.example/quality-gate', expect.stringContaining('(PR_WAIT_BLOCKED;')]);
});
// Value: protects=green counts a failed check whose rerun attempt created no jobs and a workflow run that failed before any job, as gstack-ship-queue
// ready does, while a rerun that passed stays green; fails_when=pr-wait reads only GitHub's default (latest-attempt) check-runs list and calls
// gstack-evals#303 green; seam=lib/ci-head-checks.ts (ciAtHead) behind fetchChecks, through the fake gh's filter=latest/all split;
// no_claim=GitHub's attempt grouping (the fixture is #4235's recording)
test('a failed check hidden behind a rerun attempt with no jobs, and a workflow run that failed with no jobs, block green (gstack-evals#303, GST-2252)', () => {
const red = watcher({ rest: PR303 }, 'green');
expect(red.poll()).toBe(PR_WAIT_EXIT.blocked);
expect(lines()).toEqual(['#42 check failed: checks', '#42 check failed: sealed (run 42671854799, no jobs)', expect.stringContaining('(PR_WAIT_BLOCKED;')]);
stdout = [];
const rerun = watcher({ rest: PR303_RERUN_PASSED }, 'green');
expect(pollUntilDone(rerun, 3)).toBe(PR_WAIT_EXIT.met);
expect(lines()).toEqual([expect.stringMatching(/^#42 green on aaaaaaa: 2 pass;/)]);
});
// #3495 follow-up (ruled by GStack Minder, 2026-10-26): gstack-ci-gate --exclude has a watcher equivalent, so a non-required
// check stuck in pending (a canary, a deploy preview) no longer holds --until green for the whole timeout.
test('--exclude leaves a named non-required check out of the verdict: pending does not hold green, failed does not block; a required name is refused and gates', () => {
@@ -632,7 +651,8 @@ describe('pass-1 review fixes: gates agree, bounded waits, bot identity, local f
case "$1 $2" in
'api repos/owner/project/pulls/42') echo '${restPull(prJson({}))}' ;;
'api graphql') echo '${prJson({})}' ;;
'api --paginate') echo '${restCheckRuns(ok(checksJson(['build', 'pending']))).stdout}' ;;
'api --paginate') echo '${checkRunObjects(ok(checksJson(['build', 'pending']))).map(r => JSON.stringify(r)).join('\n')}' ;;
'api repos/owner/project/commits/'*|'api repos/owner/project/actions/runs'*) echo '{"total_count":0,"check_runs":[],"statuses":[],"workflow_runs":[]}' ;;
*) echo 'gh: Not Found (HTTP 404)' >&2; exit 1 ;;
esac`, { mode: 0o755 });
const stateRoot = join(dir, 'state');