mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-17 13:44:32 +01:00
Stop gstack-pr-wait, the land gate and the release cut from calling a head green while a check on it failed (GST-2252) (#4257)
* Make gstack-pr-wait, gstack-ci-gate and the release cut count a failed check hidden behind an empty rerun (GST-2252) GitHub rollup and default check-runs list hold only each check suite latest attempt, so a rerun attempt with no jobs hid the failed run before it, and a workflow run that failed before any job never appeared. All three readers now add CI at the commit as the merge queue reads it (ciAtHead via lib/ci-head-checks.ts) and keep the worse bucket per check. Tests reuse the gstack-evals#303 recording from #4235. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Have the release cut read every check attempt only for commits that pass its named checks (GST-2252) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Say in the changelog that the release cut now counts failed commit statuses (GST-2252) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Let the release cut set its own jobs aside in the every-attempt check read too (GST-2252) After #4297 the cut sets aside check runs from its own workflow and the Windows nightly by job id, but the rows from lib/ci-head-checks.ts skipped that filter, so the cut waited on its own run. headCheckRows now takes the set-aside check run and workflow run ids and removes them from every list before ciAtHead reads it. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
2af3822f94
commit
83cc10acd2
15 files changed
+307
-43
No files matched your search
+12
-3
@@ -1,4 +1,5 @@
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { headCheckRows, type HeadCheckRow } from './ci-head-checks';
|
||||
import { MIN_POLL_MS, pollDelayMs, syncLimitEnv, withRateLimit } from './gh-poll';
|
||||
|
||||
export type CiVerdict = 'PASS' | 'FAIL' | 'PENDING' | 'NO_CHECKS' | 'ERROR';
|
||||
@@ -38,7 +39,11 @@ export const CI_GATE_USAGE = `gstack-ci-gate --repo OWNER/NAME --pr NUMBER --exp
|
||||
|
||||
Decides whether a pull request's CI allows a merge of exactly SHA. Reads the
|
||||
PR head, gates on every check (required ones and all others), and re-reads the
|
||||
head afterwards; a moved head is ERROR.
|
||||
head afterwards; a moved head is ERROR. Besides gh pr checks, which lists only
|
||||
each check suite's latest attempt, it reads CI at SHA the way gstack-ship-queue
|
||||
ready does: a failed check whose rerun attempt created no jobs, and a workflow
|
||||
run that failed before creating any job ("<workflow> (run <id>, no jobs)"),
|
||||
count as failed.
|
||||
|
||||
Output (stdout), always first:
|
||||
VERDICT <PASS|FAIL|PENDING|NO_CHECKS|ERROR> <sha>
|
||||
@@ -106,7 +111,8 @@ export function parseChecks(result: GhResult): ChecksRead {
|
||||
interface Evaluation { verdict: Exclude<CiVerdict, 'ERROR'>; lines: string[] }
|
||||
type EvalOrError = Evaluation | { verdict: 'ERROR'; cause: string; stderr: string };
|
||||
|
||||
export function evaluateChecks(required: GhResult, all: GhResult, excludes: string[]): EvalOrError {
|
||||
/** `head`: CI at the head from lib/ci-head-checks.ts, observed like the rest, so a check it reads worse than gh pr checks does gates. */
|
||||
export function evaluateChecks(required: GhResult, all: GhResult, excludes: string[], head: HeadCheckRow[] = []): EvalOrError {
|
||||
const allRead = parseChecks(all);
|
||||
if (allRead.kind === 'error') return { verdict: 'ERROR', cause: allRead.cause, stderr: allRead.stderr };
|
||||
if (allRead.kind === 'none-required') return { verdict: 'ERROR', cause: 'gh pr checks without --required reported no required checks', stderr: '' };
|
||||
@@ -129,6 +135,7 @@ export function evaluateChecks(required: GhResult, all: GhResult, excludes: stri
|
||||
};
|
||||
for (const obs of allRead.kind === 'rows' ? allRead.rows : []) observe(obs);
|
||||
for (const obs of requiredRows) observe(obs);
|
||||
for (const obs of head) observe({ ...obs, link: '' });
|
||||
if (rows.size === 0) return { verdict: 'NO_CHECKS', lines: notes.filter(n => !n.includes('no required checks')) };
|
||||
|
||||
const excluded = new Set<string>();
|
||||
@@ -180,12 +187,14 @@ function evaluateOnce(gh: GhRunner, opts: CiGateOptions): CiGateResult {
|
||||
const fields = ['--json', 'name,state,bucket,link'];
|
||||
const required = gh([...checkArgs, '--required', ...fields]);
|
||||
const all = gh([...checkArgs, ...fields]);
|
||||
const head = headCheckRows(gh, opts.repo, opts.expectHead);
|
||||
if ('error' in head) return fail(errorLines(head.error, ''));
|
||||
const after = readHead(gh, opts);
|
||||
if ('cause' in after) return fail(errorLines(after.cause, after.stderr));
|
||||
if (after.head !== before.head) {
|
||||
return fail([`HEAD ${after.head}`, 'CAUSE the PR head moved while checks were being read', 'FIX rerun /land-and-deploy from Step 1 for the new head']);
|
||||
}
|
||||
const evaluation = evaluateChecks(required, all, opts.excludes);
|
||||
const evaluation = evaluateChecks(required, all, opts.excludes, head.rows);
|
||||
if (evaluation.verdict === 'ERROR') return fail(errorLines(evaluation.cause, evaluation.stderr));
|
||||
return { verdict: evaluation.verdict, sha: opts.expectHead, lines: evaluation.lines };
|
||||
}
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
/**
|
||||
* ci-head-checks: the merge queue's read of CI at one commit (ciAtHead in lib/ship-queue.ts, GST-2167) as `gh pr checks`
|
||||
* bucket rows, for the other readers that decide a commit is green: gstack-pr-wait, gstack-ci-gate and the release cut
|
||||
* (crew/scripts/release-cut.ts) (GST-2252).
|
||||
*
|
||||
* GitHub's rollup (`gh pr checks`) and its default check-runs list hold only each check suite's latest attempt, so a rerun
|
||||
* attempt that created no jobs hides the failed run before it, and a workflow run that failed before creating a job is
|
||||
* never a check run at all: gstack-evals#303 read green with its `checks` job red. ciAtHead counts both. Each caller adds
|
||||
* these rows to the ones it already reads and keeps the worst bucket per name, so no check that gated before stops gating.
|
||||
*/
|
||||
import type { GhRunner } from './ci-gate';
|
||||
import { ciAtHead } from './ship-queue';
|
||||
|
||||
export interface HeadCheckRow { name: string; bucket: string }
|
||||
|
||||
const CONCLUSION_BUCKET: Record<string, string> = {
|
||||
success: 'pass', failure: 'fail', timed_out: 'fail', action_required: 'fail', startup_failure: 'fail',
|
||||
cancelled: 'cancel', skipped: 'skipping', neutral: 'skipping',
|
||||
};
|
||||
|
||||
/** Check runs (by id) and workflow runs (by id) a caller sets aside, such as the release cut's own jobs (GST-2463). */
|
||||
export interface AsideRuns { checkRunIds: ReadonlySet<number>; workflowRunIds: ReadonlySet<number> }
|
||||
|
||||
const PAGE = 100;
|
||||
|
||||
/**
|
||||
* Every check and commit-status context ciAtHead counts at `sha` in `repo`, or why they could not all be read (unknown is
|
||||
* never green). A check with an unknown conclusion keeps its spelling and one with none is `fail`, as ciAtHead counts them.
|
||||
* `aside` removes those check runs and workflow runs from every list before ciAtHead reads it: each list is read whole and
|
||||
* served again without them, so ciAtHead's total_count check still holds, and a list that cannot be read whole is unknown.
|
||||
*/
|
||||
export function headCheckRows(gh: GhRunner, repo: string, sha: string, aside?: AsideRuns): { rows: HeadCheckRow[] } | { error: string } {
|
||||
const run = (args: string[]) => gh(args.map(a => a.replace('repos/{owner}/{repo}/', `repos/${repo}/`)));
|
||||
const lists = new Map<string, Array<Record<string, unknown>> | null>();
|
||||
const readWhole = (prefix: string, key: string, drop: ReadonlySet<number>) => {
|
||||
const items: Array<Record<string, unknown>> = [];
|
||||
for (let page = 1; page <= 10; page++) {
|
||||
const r = run(['api', `${prefix}per_page=${PAGE}&page=${page}`]);
|
||||
let body: Record<string, unknown> | null = null;
|
||||
try { body = !r.error && r.status === 0 ? JSON.parse(r.stdout) : null; } catch {}
|
||||
const list = body?.[key];
|
||||
if (!Array.isArray(list) || typeof body?.total_count !== 'number') return null;
|
||||
items.push(...list);
|
||||
if (items.length >= body.total_count || list.length < PAGE) return items.length === body.total_count ? items.filter(i => !drop.has(Number(i.id))) : null;
|
||||
}
|
||||
return null;
|
||||
};
|
||||
const inRepo = (args: string[]) => {
|
||||
const m = aside && args[0] === 'api' ? /^(.*(?:\/check-runs|\/actions\/runs)\?(?:.*&)?)per_page=\d+&page=(\d+)$/.exec(args[1] ?? '') : null;
|
||||
if (!aside || !m) {
|
||||
const r = run(args);
|
||||
return { ok: !r.error && r.status === 0, out: r.stdout, err: r.error ?? r.stderr };
|
||||
}
|
||||
const key = m[1]!.includes('/check-runs') ? 'check_runs' : 'workflow_runs';
|
||||
if (!lists.has(m[1]!)) lists.set(m[1]!, readWhole(m[1]!, key, key === 'check_runs' ? aside.checkRunIds : aside.workflowRunIds));
|
||||
const all = lists.get(m[1]!);
|
||||
if (!all) return { ok: false, out: '', err: `${key} unreadable while setting runs aside` };
|
||||
const page = Number(m[2]);
|
||||
return { ok: true, out: JSON.stringify({ total_count: all.length, [key]: all.slice((page - 1) * PAGE, page * PAGE) }), err: '' };
|
||||
};
|
||||
const ci = ciAtHead({ repoRoot: process.cwd(), base: '', gh: inRepo }, sha);
|
||||
if (ci.line.startsWith('unknown')) return { error: ci.line.replace(/^unknown at (\S+)/, 'CI at $1 is unknown') };
|
||||
return {
|
||||
rows: [
|
||||
...ci.checks.map(c => ({ name: c.name, bucket: c.status !== 'completed' ? 'pending' : CONCLUSION_BUCKET[c.conclusion ?? ''] ?? (c.conclusion || 'fail') })),
|
||||
...ci.contexts.map(s => ({ name: s.name, bucket: s.state === 'success' ? 'pass' : s.state === 'pending' ? 'pending' : 'fail' })),
|
||||
],
|
||||
};
|
||||
}
|
||||
+10
-3
@@ -13,6 +13,7 @@
|
||||
* gstack's gh runner shape (lib/ci-gate.ts).
|
||||
*/
|
||||
import type { GhResult, GhRunner } from './ci-gate';
|
||||
import { headCheckRows } from './ci-head-checks';
|
||||
import { parseHeadChecks, parsePullState, rateLimitKind, type PullState } from './gh-poll';
|
||||
|
||||
export class GhError extends Error {
|
||||
@@ -179,14 +180,20 @@ function restList(env: GhEnv, path: string, key: string): unknown[] {
|
||||
|
||||
/**
|
||||
* Every check on the head commit by name, from REST check runs and commit
|
||||
* statuses (the same rows `gh pr checks` reads through GraphQL), duplicates
|
||||
* collapsed to the worst bucket.
|
||||
* statuses (the same rows `gh pr checks` reads through GraphQL) plus CI at the
|
||||
* head as gstack-ship-queue ready reads it (lib/ci-head-checks.ts: a failed
|
||||
* check whose rerun attempt created no jobs, a workflow run that failed with
|
||||
* no jobs), duplicates collapsed to the worst bucket. An unknown read is a
|
||||
* transient GhError, so the poll loop tries again.
|
||||
*/
|
||||
export function fetchChecks(env: GhEnv, repo: string, head: string): Map<string, Check> {
|
||||
const rows = parseHeadChecks(
|
||||
const listed = parseHeadChecks(
|
||||
restList(env, `repos/${repo}/commits/${head}/check-runs?per_page=100`, 'check_runs'),
|
||||
restList(env, `repos/${repo}/commits/${head}/status?per_page=100`, 'statuses'),
|
||||
);
|
||||
const ci = headCheckRows(args => runGh(env, args), repo, head);
|
||||
if ('error' in ci) throw new GhError(ci.error, true);
|
||||
const rows = [...listed, ...ci.rows.map(row => ({ ...row, link: '', description: '' }))];
|
||||
const checks = new Map<string, Check>();
|
||||
for (const row of rows) {
|
||||
const current = checks.get(row.name);
|
||||
|
||||
Reference in new issue
Block a user