mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-17 13:44:32 +01:00
Stop gstack-pr-wait, the land gate and the release cut from calling a head green while a check on it failed (GST-2252) (#4257)
* Make gstack-pr-wait, gstack-ci-gate and the release cut count a failed check hidden behind an empty rerun (GST-2252) GitHub rollup and default check-runs list hold only each check suite latest attempt, so a rerun attempt with no jobs hid the failed run before it, and a workflow run that failed before any job never appeared. All three readers now add CI at the commit as the merge queue reads it (ciAtHead via lib/ci-head-checks.ts) and keep the worse bucket per check. Tests reuse the gstack-evals#303 recording from #4235. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Have the release cut read every check attempt only for commits that pass its named checks (GST-2252) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Say in the changelog that the release cut now counts failed commit statuses (GST-2252) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Let the release cut set its own jobs aside in the every-attempt check read too (GST-2252) After #4297 the cut sets aside check runs from its own workflow and the Windows nightly by job id, but the rows from lib/ci-head-checks.ts skipped that filter, so the cut waited on its own run. headCheckRows now takes the set-aside check run and workflow run ids and removes them from every list before ciAtHead reads it. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
2af3822f94
commit
83cc10acd2
15 files changed
+307
-43
No files matched your search
@@ -0,0 +1 @@
|
||||
- **`gstack-pr-wait`, `gstack-ci-gate` and the release cut no longer call a commit green while a check on it failed.** They read CI from GitHub's rollup or its default check-runs list, which hold only each check suite's latest attempt, so when a rerun attempt created no jobs the failed run before it disappeared, and a workflow run that failed before creating any job never appeared at all. That is the false green the merge queue had on gstack-evals#303 (#4235). All three now also read CI at the commit through the merge queue's own helper, which reads every attempt and every workflow run, and keep the worse answer per check. So `gstack-pr-wait --until green` blocks with `check failed: checks` and `check failed: sealed (run <id>, no jobs)`, the land gate says `VERDICT FAIL`, and the release cut skips the commit. The release cut now also counts a failed commit status as red, as the merge queue does. A rerun that passed still reads green, and an unreadable read is an error, never a pass (GST-2252).
|
||||
+23
-10
@@ -69,6 +69,8 @@ import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { FLOOR_HOURS, IMPORTANT_FLOOR_HOURS, SCHEDULED_GRACE_HOURS, decideDaily, importantChanges, type DailyDecision } from './release-cadence';
|
||||
import { bucketClass } from '../../lib/ci-gate';
|
||||
import { headCheckRows, type AsideRuns } from '../../lib/ci-head-checks';
|
||||
|
||||
export { FLOOR_HOURS, SCHEDULED_GRACE_HOURS };
|
||||
|
||||
@@ -120,7 +122,11 @@ interface CheckRun { id?: number; name: string; status: string; conclusion: stri
|
||||
export interface TrainEvidence { bySha: Map<string, string>; byTree: Map<string, string>; release?: Map<string, string> }
|
||||
export interface GateOptions { requiredCheck?: string; backstopCheck?: string; evidence?: TrainEvidence; treeOf?: (sha: string) => string | null }
|
||||
|
||||
/** Why a commit is not releasable, or green and through which evidence. */
|
||||
/**
|
||||
* Why a commit is not releasable, or green and through which evidence. Red and running checks also come from CI at the commit as the
|
||||
* merge queue reads it (lib/ci-head-checks.ts), so a failed check whose rerun attempt created no jobs, or a workflow run that failed
|
||||
* before any job, is red here too (GST-2252).
|
||||
*/
|
||||
export function gateVerdict(gh: Gh, repo: string, sha: string, o: GateOptions = {}): Verdict {
|
||||
const required = o.requiredCheck ?? REQUIRED_CHECK;
|
||||
const backstop = o.backstopCheck ?? BACKSTOP_CHECK;
|
||||
@@ -128,17 +134,21 @@ export function gateVerdict(gh: Gh, repo: string, sha: string, o: GateOptions =
|
||||
if (!paged) return { green: false, why: 'cannot read its check runs', pending: false };
|
||||
const pages = (Array.isArray(paged) ? paged : [paged]) as Array<{ check_runs?: CheckRun[] }>;
|
||||
let runs = pages.flatMap(p => p.check_runs ?? []);
|
||||
if (runs.some(c => c.status !== 'completed' || RED.has(c.conclusion ?? ''))) {
|
||||
const setAside = (): AsideRuns => {
|
||||
const wf = parseJson<unknown>(gh(['api', '--paginate', '--slurp', `repos/${repo}/actions/runs?head_sha=${sha}&per_page=100`]));
|
||||
const wfRuns = (Array.isArray(wf) ? wf : wf ? [wf] : []) as Array<{ workflow_runs?: Array<{ id?: number; path?: string }> }>;
|
||||
const asideWorkflows = new Set([CUT_WORKFLOW, ...NEVER_GATING_WORKFLOWS]);
|
||||
const aside = new Set<number>();
|
||||
const checkRunIds = new Set<number>();
|
||||
const workflowRunIds = new Set<number>();
|
||||
for (const w of wfRuns.flatMap(p => p.workflow_runs ?? []).filter(w => asideWorkflows.has(w.path ?? '') && typeof w.id === 'number')) {
|
||||
workflowRunIds.add(w.id!);
|
||||
const jobs = parseJson<unknown>(gh(['api', '--paginate', '--slurp', `repos/${repo}/actions/runs/${w.id}/jobs?filter=all&per_page=100`]));
|
||||
for (const p of (Array.isArray(jobs) ? jobs : jobs ? [jobs] : []) as Array<{ jobs?: Array<{ id?: number }> }>) for (const j of p.jobs ?? []) if (typeof j.id === 'number') aside.add(j.id);
|
||||
for (const p of (Array.isArray(jobs) ? jobs : jobs ? [jobs] : []) as Array<{ jobs?: Array<{ id?: number }> }>) for (const j of p.jobs ?? []) if (typeof j.id === 'number') checkRunIds.add(j.id);
|
||||
}
|
||||
runs = runs.filter(c => !(typeof c.id === 'number' && aside.has(c.id)));
|
||||
}
|
||||
return { checkRunIds, workflowRunIds };
|
||||
};
|
||||
let aside = runs.some(c => c.status !== 'completed' || RED.has(c.conclusion ?? '')) ? setAside() : undefined;
|
||||
if (aside) runs = runs.filter(c => !(typeof c.id === 'number' && aside!.checkRunIds.has(c.id)));
|
||||
let via = backstop;
|
||||
for (const name of [required, backstop]) {
|
||||
const named = runs.filter(c => c.name === name);
|
||||
@@ -151,10 +161,13 @@ export function gateVerdict(gh: Gh, repo: string, sha: string, o: GateOptions =
|
||||
if (named.length === 0) return { green: false, why: `no ${name} check run`, pending: false };
|
||||
return { green: false, why: `${name} concluded ${named.map(c => c.conclusion).join(',')}`, pending: false };
|
||||
}
|
||||
const red = runs.filter(c => c.status === 'completed' && RED.has(c.conclusion ?? ''));
|
||||
if (red.length > 0) return { green: false, why: `red check runs: ${[...new Set(red.map(c => c.name))].join(', ')}`, pending: false };
|
||||
const running = runs.filter(c => c.status !== 'completed');
|
||||
if (running.length > 0) return { green: false, why: `check runs still going: ${[...new Set(running.map(c => c.name))].join(', ')}`, pending: true };
|
||||
let head = headCheckRows(gh, repo, sha, aside);
|
||||
if (!aside && 'rows' in head && head.rows.some(r => bucketClass(r.bucket) !== 'pass')) head = headCheckRows(gh, repo, sha, aside = setAside());
|
||||
if ('error' in head) return { green: false, why: 'cannot read its check runs', pending: false };
|
||||
const red = [...runs.filter(c => c.status === 'completed' && RED.has(c.conclusion ?? '')).map(c => c.name), ...head.rows.filter(r => bucketClass(r.bucket) === 'fail').map(r => r.name)];
|
||||
if (red.length > 0) return { green: false, why: `red check runs: ${[...new Set(red)].join(', ')}`, pending: false };
|
||||
const running = [...runs.filter(c => c.status !== 'completed').map(c => c.name), ...head.rows.filter(r => r.bucket === 'pending').map(r => r.name)];
|
||||
if (running.length > 0) return { green: false, why: `check runs still going: ${[...new Set(running)].join(', ')}`, pending: true };
|
||||
return { green: true, via };
|
||||
}
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ import * as path from 'node:path';
|
||||
import { bunIsolationArgs } from '../../lib/bun-isolation';
|
||||
import { FLOOR_HOURS, FORCED_AFTER_HOURS, IMPORTANT_FLOOR_HOURS, SCHEDULED_GRACE_HOURS, decideDaily, importantChanges, type DailyInput, type ImportantChange } from '../scripts/release-cadence';
|
||||
import { gitIn, planDaily, type Gh } from '../scripts/release-cut';
|
||||
import { ciRest, ciShimCases } from '../../test/helpers/ci-attempts-fixture';
|
||||
|
||||
const H = 3_600_000;
|
||||
// Tuesday 2026-11-10 14:00 UTC: not a regular day.
|
||||
@@ -138,8 +139,9 @@ const GREEN = [{ name: 'free-tests', status: 'completed', conclusion: 'success'
|
||||
function fakeGh(pending: Set<string> = new Set()): Gh {
|
||||
return (args) => {
|
||||
const api = args[0] === 'api' ? args[args.length - 1]! : '';
|
||||
const m = /commits\/([0-9a-f]+)\/check-runs/.exec(api);
|
||||
if (m) return { status: 0, stdout: JSON.stringify([{ check_runs: pending.has(m[1]!) ? [{ name: 'main-backstop', status: 'in_progress', conclusion: null }] : GREEN }]), stderr: '' };
|
||||
const m = /commits\/([0-9a-f]+)\/|head_sha=([0-9a-f]+)/.exec(api);
|
||||
const rest = m && ciRest(args, { check_runs: pending.has(m[1] ?? m[2]!) ? [{ name: 'main-backstop', status: 'in_progress', conclusion: null }] : GREEN });
|
||||
if (rest) return rest;
|
||||
if (args[0] === 'release' && args[1] === 'view' && args[2] === 'v1.0.0') return { status: 0, stdout: '{}', stderr: '' };
|
||||
return { status: 1, stdout: '', stderr: 'gh: Not Found (HTTP 404)' };
|
||||
};
|
||||
@@ -245,7 +247,7 @@ describe.skipIf(process.platform === 'win32')('release-cut --daily (the workflow
|
||||
const r = repo(30);
|
||||
const shim = path.join(r.root, 'shim');
|
||||
fs.mkdirSync(shim);
|
||||
fs.writeFileSync(path.join(shim, 'gh'), '#!/usr/bin/env bash\ncase "$*" in\n *check-runs*) echo \'' + JSON.stringify([{ check_runs: GREEN }]) + '\' ;;\n "release view v1.0.0 "*) echo \'{}\' ;;\n *) exit 1 ;;\nesac\n', { mode: 0o755 });
|
||||
fs.writeFileSync(path.join(shim, 'gh'), '#!/usr/bin/env bash\ncase "$*" in\n' + ciShimCases(GREEN) + ' "release view v1.0.0 "*) echo \'{}\' ;;\n *) exit 1 ;;\nesac\n', { mode: 0o755 });
|
||||
const env: Record<string, string | undefined> = { ...process.env, PATH: shim + path.delimiter + process.env.PATH, GITHUB_ACTIONS: 'true' };
|
||||
for (const k of ['GITHUB_EVENT_NAME', 'GITHUB_STEP_SUMMARY']) delete env[k];
|
||||
const run = (...args: string[]) => spawnSync(process.execPath, [...bunIsolationArgs(), 'run', path.join(path.resolve(import.meta.dir, '..', '..'), 'crew', 'scripts', 'release-cut.ts'), '--repo', 'o/r', ...args],
|
||||
|
||||
@@ -11,6 +11,7 @@ import { execFileSync, spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { bunIsolationArgs } from '../../lib/bun-isolation';
|
||||
import { PR303, PR303_RERUN_PASSED, ciRest, ciShimCases, type CiAtCommit } from '../../test/helpers/ci-attempts-fixture';
|
||||
import { gitFixtureTemplate, rebindGitDir } from '../../test/helpers/git-fixture-template';
|
||||
import {
|
||||
BACKSTOP_CHECK, CUT_WORKFLOW, FLOOR_HOURS, NEVER_GATING_WORKFLOWS, NOTES_CAP, NOTES_REF, REQUIRED_CHECK, SCHEDULED_GRACE_HOURS, changelogEntries, compareVersions, cutRelease, describePlan, floorStatus,
|
||||
@@ -42,8 +43,12 @@ function fakeGh(state: State): Gh {
|
||||
return (args) => {
|
||||
state.calls.push(args);
|
||||
const api = args[0] === 'api' ? args[args.length - 1]! : '';
|
||||
const m = /commits\/([0-9a-f]+)\/check-runs/.exec(api);
|
||||
if (m) return { status: 0, stdout: JSON.stringify([{ check_runs: state.checks[m[1]!] ?? GREEN }]), stderr: '' };
|
||||
if (api && state.api && api in state.api) return { status: 0, stdout: JSON.stringify(state.api[api]), stderr: '' };
|
||||
const m = /commits\/([0-9a-f]+)\/|head_sha=([0-9a-f]+)/.exec(api);
|
||||
const sha = m?.[1] ?? m?.[2];
|
||||
const listed = sha ? state.api?.['repos/o/r/actions/runs?head_sha=' + sha + '&per_page=100'] as Array<{ workflow_runs?: Array<Record<string, unknown>> }> | undefined : undefined;
|
||||
const rest = sha && ciRest(args, { check_runs: state.checks[sha] ?? GREEN, workflow_runs: listed?.flatMap(p => p.workflow_runs ?? []) });
|
||||
if (rest) return rest;
|
||||
if (api && state.api && api in state.api) return { status: 0, stdout: JSON.stringify(state.api[api]), stderr: '' };
|
||||
if (api) return { status: 1, stdout: '', stderr: 'gh: Not Found (HTTP 404)' };
|
||||
if (args[0] === 'release' && args[1] === 'view') return state.released.has(args[2]!) ? { status: 0, stdout: '{}', stderr: '' } : { status: 1, stdout: '', stderr: 'release not found' };
|
||||
@@ -171,9 +176,8 @@ describe('release-cut: planning', () => {
|
||||
// never a failed required run, a gh error, an unrelated running check or a second page; seam=none; no_claim=GitHub's real check-runs payload shape (a fake gh answers; unit only)
|
||||
test('gateVerdict: a failed free-tests run, an unreadable check-runs API, an unrelated running check and a paginated check list', () => {
|
||||
const checks = (runs: unknown[]): Gh => (args) => {
|
||||
const api = args[args.length - 1]!;
|
||||
if (/check-runs/.test(api)) return { status: 0, stdout: JSON.stringify(runs), stderr: '' };
|
||||
return { status: 1, stdout: '', stderr: 'unexpected gh ' + args.join(' ') };
|
||||
if (args.includes('--slurp')) return { status: 0, stdout: JSON.stringify(runs), stderr: '' };
|
||||
return ciRest(args, { check_runs: (runs as Array<{ check_runs: Array<Record<string, unknown>> }>).flatMap(p => p.check_runs) }) ?? { status: 1, stdout: '', stderr: 'unexpected gh ' + args.join(' ') };
|
||||
};
|
||||
const failed = checks([{ check_runs: [{ name: REQUIRED_CHECK, status: 'completed', conclusion: 'failure' }] }]);
|
||||
expect(gateVerdict(failed, 'o/r', 'a'.repeat(40))).toEqual({ green: false, why: REQUIRED_CHECK + ' concluded failure', pending: false });
|
||||
@@ -193,6 +197,16 @@ describe('release-cut: planning', () => {
|
||||
expect(gateVerdict(skipped, 'o/r', 'a'.repeat(40))).toEqual({ green: true, via: BACKSTOP_CHECK });
|
||||
});
|
||||
|
||||
// Value: protects=the release cut counts a failed check whose rerun attempt created no jobs and a workflow run that failed before any job, as the merge
|
||||
// queue does, while a rerun that passed stays releasable; fails_when=gateVerdict reads only GitHub's default (latest-attempt) check-runs list and calls a
|
||||
// main commit carrying gstack-evals#303's failures releasable; seam=lib/ci-head-checks.ts (ciAtHead) in gateVerdict, through ciRest's filter=latest/all
|
||||
// split; no_claim=GitHub's attempt grouping (the fixture is #4235's recording)
|
||||
test('gateVerdict: a failed check hidden behind a rerun attempt with no jobs, and a workflow run that failed with no jobs, are red (gstack-evals#303, GST-2252)', () => {
|
||||
const at = (data: CiAtCommit): Gh => args => ciRest(args, { ...data, check_runs: [...GREEN, ...data.check_runs] }) ?? { status: 1, stdout: '', stderr: 'unexpected gh ' + args.join(' ') };
|
||||
expect(gateVerdict(at(PR303), 'o/r', 'a'.repeat(40))).toEqual({ green: false, why: 'red check runs: checks, sealed (run 42671854799, no jobs)', pending: false });
|
||||
expect(gateVerdict(at(PR303_RERUN_PASSED), 'o/r', 'a'.repeat(40))).toEqual({ green: true, via: BACKSTOP_CHECK });
|
||||
});
|
||||
|
||||
test('an active freeze in refs/notes/gstack-trains refuses the cut; a cleared one does not', () => {
|
||||
const f = fixture();
|
||||
const freeze = { schema: 1, kind: 'freeze', repo_key: 'k', train: 'abc', reason: 'backstop red on 2 cases', created_at: 'x', cleared_at: null };
|
||||
@@ -346,6 +360,27 @@ describe('release-cut: the cut sets aside its own check runs (GST-2463)', () =>
|
||||
.toEqual({ green: false, why: 'red check runs: Cut the release', pending: false });
|
||||
});
|
||||
|
||||
// Value: protects=the every-attempt read (lib/ci-head-checks.ts) sets the cut's own runs aside by job id and workflow run id, the way the default list
|
||||
// does, so a failed attempt of the cut's own job or a cut run that failed with no jobs never keeps a gated commit out, while another workflow's jobless
|
||||
// failure still does; fails_when=headCheckRows rows skip GST-2463's exclusion and the cut deadlocks on its own "Cut the release" run (GST-2252 after
|
||||
// #4297); seam=gh is faked with check-runs (filter=latest/all), actions/runs and jobs payloads; no_claim=a live workflow run
|
||||
test('gateVerdict: the every-attempt read sets the cut\'s own earlier attempt and its jobless run aside by id, never by name (GST-2252, GST-2463)', () => {
|
||||
const evidence = gatedBy(sha, '43072520821');
|
||||
const wf = (id: number, path: string, conclusion: string) => ({ id, path, check_suite_id: id, workflow_id: id, name: path, status: 'completed', conclusion, created_at: '2026-11-11T00:00:00Z' });
|
||||
const api = (runs: Array<Record<string, unknown>>, jobs: Record<number, number[]>) => Object.assign(
|
||||
{ ['repos/o/r/actions/runs?head_sha=' + sha + '&per_page=100']: [{ total_count: runs.length, workflow_runs: runs }] },
|
||||
...Object.entries(jobs).map(([id, ids]) => ({ ['repos/o/r/actions/runs/' + id + '/jobs?filter=all&per_page=100']: [{ total_count: ids.length, jobs: ids.map(j => ({ id: j })) }] })));
|
||||
const earlier = { ...ownRun('completed', 'failure'), earlier_attempt: true } as CheckRun;
|
||||
expect(gateVerdict(ghFor([...unbacked, earlier], api([wf(9001, CUT_WORKFLOW, 'success')], { 9001: [OWN_JOB] })), 'o/r', sha, { evidence }))
|
||||
.toEqual({ green: true, via: 'release gate run 43072520821' });
|
||||
expect(gateVerdict(ghFor(unbacked, api([wf(9001, CUT_WORKFLOW, 'failure')], { 9001: [] })), 'o/r', sha, { evidence }))
|
||||
.toEqual({ green: true, via: 'release gate run 43072520821' });
|
||||
expect(gateVerdict(ghFor(unbacked, api([wf(9001, CUT_WORKFLOW, 'failure'), wf(555, '.github/workflows/evals.yml', 'failure')], { 9001: [] })), 'o/r', sha, { evidence }))
|
||||
.toEqual({ green: false, why: 'red check runs: .github/workflows/evals.yml (run 555, no jobs)', pending: false });
|
||||
expect(gateVerdict(ghFor([...unbacked, { ...earlier, id: 70555 } as CheckRun], api([wf(9001, CUT_WORKFLOW, 'success')], { 9001: [OWN_JOB] })), 'o/r', sha, { evidence }))
|
||||
.toEqual({ green: false, why: 'red check runs: Cut the release', pending: false });
|
||||
});
|
||||
|
||||
// Value: protects=check runs from the never-gating Windows nightly (timed out, cancelled, failed or still running) never keep a gated commit out of the
|
||||
// cut, while a red required check, a red or running check run from any other workflow, a same-named job outside the nightly and a missing gate still
|
||||
// refuse; fails_when=the nightly's shard 4 timeout holds stable (0326b144 behind a GREEN release gate, 6 nights), or the exclusion is keyed on the check
|
||||
@@ -527,7 +562,7 @@ describe('release-cut: train evidence gates a commit whose backstop never settle
|
||||
if (process.platform === 'win32') return;
|
||||
const shim = path.join(f.base, 'shim');
|
||||
fs.mkdirSync(shim);
|
||||
fs.writeFileSync(path.join(shim, 'gh'), '#!/usr/bin/env bash\ncase "$*" in\n *check-runs*) echo \'' + JSON.stringify([{ check_runs: f.state.checks[head] }]) + '\' ;;\n *) exit 1 ;;\nesac\n', { mode: 0o755 });
|
||||
fs.writeFileSync(path.join(shim, 'gh'), '#!/usr/bin/env bash\ncase "$*" in\n' + ciShimCases(f.state.checks[head]) + ' *) exit 1 ;;\nesac\n', { mode: 0o755 });
|
||||
const run = (...args: string[]) => spawnSync(process.execPath, [...bunIsolationArgs(), 'run', path.join(path.resolve(import.meta.dir, '..', '..'), 'crew', 'scripts', 'release-cut.ts'), ...args],
|
||||
{ cwd: f.work, encoding: 'utf8', timeout: 60_000, env: cliEnv(shim) });
|
||||
const cli = run('--repo', f.repo, '--dry-run', '--ref', head, '--required-check', 'ci', '--backstop-check', 'nightly-backstop', '--notes-ref', 'refs/notes/other-trains');
|
||||
@@ -690,7 +725,7 @@ describe.skipIf(process.platform === 'win32')('release-cut: the workflow entry p
|
||||
const f = fixture();
|
||||
const shim = path.join(f.base, 'shim');
|
||||
fs.mkdirSync(shim);
|
||||
fs.writeFileSync(path.join(shim, 'gh'), '#!/usr/bin/env bash\necho "gh $*" >> "' + path.join(f.base, 'gh.log') + '"\ncase "$*" in\n *check-runs*) echo \'' + JSON.stringify([{ check_runs: GREEN }]) + '\' ;;\n "release view v1.0.3 "*) echo \'{\"tagName\":\"v1.0.3\"}\' ;;\n *) exit 1 ;;\nesac\n', { mode: 0o755 });
|
||||
fs.writeFileSync(path.join(shim, 'gh'), '#!/usr/bin/env bash\necho "gh $*" >> "' + path.join(f.base, 'gh.log') + '"\ncase "$*" in\n' + ciShimCases(GREEN) + ' "release view v1.0.3 "*) echo \'{\"tagName\":\"v1.0.3\"}\' ;;\n *) exit 1 ;;\nesac\n', { mode: 0o755 });
|
||||
const run = (...args: string[]) => spawnSync(process.execPath, [...bunIsolationArgs(), 'run', path.join(path.resolve(import.meta.dir, '..', '..'), 'crew', 'scripts', 'release-cut.ts'), ...args],
|
||||
{ cwd: f.work, encoding: 'utf8', timeout: 60_000, env: cliEnv(shim) });
|
||||
|
||||
@@ -724,7 +759,7 @@ describe.skipIf(process.platform === 'win32')('release-cut: the workflow entry p
|
||||
expect(refused.stdout).toContain('REFUSED: non-fast-forward');
|
||||
|
||||
// Nothing green: the real cut goes red (exit 1), a dry run has answered (exit 0) and names every skip.
|
||||
fs.writeFileSync(path.join(shim, 'gh'), '#!/usr/bin/env bash\ncase "$*" in\n *check-runs*) echo \'' + JSON.stringify([{ check_runs: GREEN.filter(c => c.name !== BACKSTOP_CHECK) }]) + '\' ;;\n *) exit 1 ;;\nesac\n', { mode: 0o755 });
|
||||
fs.writeFileSync(path.join(shim, 'gh'), '#!/usr/bin/env bash\ncase "$*" in\n' + ciShimCases(GREEN.filter(c => c.name !== BACKSTOP_CHECK)) + ' *) exit 1 ;;\nesac\n', { mode: 0o755 });
|
||||
git(f.seed, 'push', '-q', '--delete', 'origin', 'stable', 'refs/tags/v1.0.3');
|
||||
f.refresh(); git(f.work, 'tag', '-d', 'v1.0.3');
|
||||
const noneDry = run('--repo', f.repo, '--dry-run');
|
||||
@@ -1186,7 +1221,7 @@ describe.skipIf(process.platform === 'win32')('release-cut: refusals at the work
|
||||
f.refresh();
|
||||
const shim = path.join(f.base, 'shim');
|
||||
fs.mkdirSync(shim);
|
||||
fs.writeFileSync(path.join(shim, 'gh'), '#!/usr/bin/env bash\ncase "$*" in\n *check-runs*) echo \'' + JSON.stringify([{ check_runs: GREEN }]) + '\' ;;\n "release view v1.0.3 "*) echo \'{"publishedAt":"' + isoZ(at) + '"}\' ;;\n *) echo "release not found" >&2; exit 1 ;;\nesac\n', { mode: 0o755 });
|
||||
fs.writeFileSync(path.join(shim, 'gh'), '#!/usr/bin/env bash\ncase "$*" in\n' + ciShimCases(GREEN) + ' "release view v1.0.3 "*) echo \'{"publishedAt":"' + isoZ(at) + '"}\' ;;\n *) echo "release not found" >&2; exit 1 ;;\nesac\n', { mode: 0o755 });
|
||||
const run = (env: Record<string, string>, ...args: string[]) => spawnSync(process.execPath, [...bunIsolationArgs(), 'run', path.join(path.resolve(import.meta.dir, '..', '..'), 'crew', 'scripts', 'release-cut.ts'), '--repo', f.repo, ...args],
|
||||
{ cwd: f.work, encoding: 'utf8', timeout: 60_000, env: cliEnv(shim, env) });
|
||||
const stable = () => git(f.origin, 'rev-parse', 'refs/heads/stable');
|
||||
|
||||
+4
-1
@@ -44,7 +44,10 @@ regression fix (below).
|
||||
**Green** means, on that exact commit: the `free-tests` check run succeeded
|
||||
(the status main requires), the commit was gated (below), every check run on
|
||||
it has completed and none failed, and `refs/notes/gstack-trains` holds no
|
||||
active freeze. A cancelled check run counts as red. A commit with a check run
|
||||
active freeze. A cancelled check run counts as red. Check runs are read the
|
||||
way the merge queue reads them, so a failed check whose rerun attempt created
|
||||
no jobs, and a workflow run that failed before creating any job, count as red
|
||||
even though GitHub's default list hides them. A commit with a check run
|
||||
still going is skipped and the next older one is considered, back through 7
|
||||
days or 200 first-parent commits. The cut's own job lands a check run on
|
||||
main's head, the very commit it may cut, so check runs from this workflow's
|
||||
|
||||
+3
-1
@@ -277,7 +277,9 @@ no name or no readable `completed_at` counts on its own; commit-status contexts
|
||||
are counted one each. A workflow run that failed without starting any job
|
||||
counts as a failed check `<workflow> (run <id>, no jobs)` unless a later run of
|
||||
the same workflow replaced it; a run cancelled before any job is not counted.
|
||||
An unreadable list is unknown, never green.
|
||||
An unreadable list is unknown, never green. `gstack-pr-wait`, `gstack-ci-gate`
|
||||
and the release cut read CI through the same helper (`lib/ci-head-checks.ts`),
|
||||
on top of what they read before, so none of them calls such a head green.
|
||||
|
||||
When a check counts as passing only because a newer run from another check
|
||||
suite replaced a red one (a branch-only `free-tests.yml` dispatch beside the red
|
||||
|
||||
@@ -7308,7 +7308,12 @@ one stdout line per real event, prefixed `#<pr>`; progress and recoverable
|
||||
errors go to stderr. `/land-and-deploy` Step 3 runs it with `--until green`.
|
||||
Exit 0 means the `--until` condition is met or the PR merged; exit 1 and 2
|
||||
end with one of the two codes below. `green` gates on every check on the
|
||||
head, as `gstack-ci-gate` does, so the two never disagree; a check the base
|
||||
head, as `gstack-ci-gate` does, so the two never disagree. Both read every
|
||||
attempt of each check suite as `gstack-ship-queue ready` does: a failed check
|
||||
whose rerun attempt created no jobs (GitHub's default list then hides it) and a
|
||||
workflow run that failed before creating any job (`check failed: <workflow>
|
||||
(run <id>, no jobs)`) block `green`, and a rerun that passed clears its
|
||||
failure. A check the base
|
||||
branch requires (rulesets and classic branch protection) counts as pending
|
||||
until it appears. `--exclude <check,...>` leaves a named non-required check
|
||||
out of the verdict, as `gstack-ci-gate --exclude` does: it neither holds
|
||||
|
||||
+12
-3
@@ -1,4 +1,5 @@
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { headCheckRows, type HeadCheckRow } from './ci-head-checks';
|
||||
import { MIN_POLL_MS, pollDelayMs, syncLimitEnv, withRateLimit } from './gh-poll';
|
||||
|
||||
export type CiVerdict = 'PASS' | 'FAIL' | 'PENDING' | 'NO_CHECKS' | 'ERROR';
|
||||
@@ -38,7 +39,11 @@ export const CI_GATE_USAGE = `gstack-ci-gate --repo OWNER/NAME --pr NUMBER --exp
|
||||
|
||||
Decides whether a pull request's CI allows a merge of exactly SHA. Reads the
|
||||
PR head, gates on every check (required ones and all others), and re-reads the
|
||||
head afterwards; a moved head is ERROR.
|
||||
head afterwards; a moved head is ERROR. Besides gh pr checks, which lists only
|
||||
each check suite's latest attempt, it reads CI at SHA the way gstack-ship-queue
|
||||
ready does: a failed check whose rerun attempt created no jobs, and a workflow
|
||||
run that failed before creating any job ("<workflow> (run <id>, no jobs)"),
|
||||
count as failed.
|
||||
|
||||
Output (stdout), always first:
|
||||
VERDICT <PASS|FAIL|PENDING|NO_CHECKS|ERROR> <sha>
|
||||
@@ -106,7 +111,8 @@ export function parseChecks(result: GhResult): ChecksRead {
|
||||
interface Evaluation { verdict: Exclude<CiVerdict, 'ERROR'>; lines: string[] }
|
||||
type EvalOrError = Evaluation | { verdict: 'ERROR'; cause: string; stderr: string };
|
||||
|
||||
export function evaluateChecks(required: GhResult, all: GhResult, excludes: string[]): EvalOrError {
|
||||
/** `head`: CI at the head from lib/ci-head-checks.ts, observed like the rest, so a check it reads worse than gh pr checks does gates. */
|
||||
export function evaluateChecks(required: GhResult, all: GhResult, excludes: string[], head: HeadCheckRow[] = []): EvalOrError {
|
||||
const allRead = parseChecks(all);
|
||||
if (allRead.kind === 'error') return { verdict: 'ERROR', cause: allRead.cause, stderr: allRead.stderr };
|
||||
if (allRead.kind === 'none-required') return { verdict: 'ERROR', cause: 'gh pr checks without --required reported no required checks', stderr: '' };
|
||||
@@ -129,6 +135,7 @@ export function evaluateChecks(required: GhResult, all: GhResult, excludes: stri
|
||||
};
|
||||
for (const obs of allRead.kind === 'rows' ? allRead.rows : []) observe(obs);
|
||||
for (const obs of requiredRows) observe(obs);
|
||||
for (const obs of head) observe({ ...obs, link: '' });
|
||||
if (rows.size === 0) return { verdict: 'NO_CHECKS', lines: notes.filter(n => !n.includes('no required checks')) };
|
||||
|
||||
const excluded = new Set<string>();
|
||||
@@ -180,12 +187,14 @@ function evaluateOnce(gh: GhRunner, opts: CiGateOptions): CiGateResult {
|
||||
const fields = ['--json', 'name,state,bucket,link'];
|
||||
const required = gh([...checkArgs, '--required', ...fields]);
|
||||
const all = gh([...checkArgs, ...fields]);
|
||||
const head = headCheckRows(gh, opts.repo, opts.expectHead);
|
||||
if ('error' in head) return fail(errorLines(head.error, ''));
|
||||
const after = readHead(gh, opts);
|
||||
if ('cause' in after) return fail(errorLines(after.cause, after.stderr));
|
||||
if (after.head !== before.head) {
|
||||
return fail([`HEAD ${after.head}`, 'CAUSE the PR head moved while checks were being read', 'FIX rerun /land-and-deploy from Step 1 for the new head']);
|
||||
}
|
||||
const evaluation = evaluateChecks(required, all, opts.excludes);
|
||||
const evaluation = evaluateChecks(required, all, opts.excludes, head.rows);
|
||||
if (evaluation.verdict === 'ERROR') return fail(errorLines(evaluation.cause, evaluation.stderr));
|
||||
return { verdict: evaluation.verdict, sha: opts.expectHead, lines: evaluation.lines };
|
||||
}
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
/**
|
||||
* ci-head-checks: the merge queue's read of CI at one commit (ciAtHead in lib/ship-queue.ts, GST-2167) as `gh pr checks`
|
||||
* bucket rows, for the other readers that decide a commit is green: gstack-pr-wait, gstack-ci-gate and the release cut
|
||||
* (crew/scripts/release-cut.ts) (GST-2252).
|
||||
*
|
||||
* GitHub's rollup (`gh pr checks`) and its default check-runs list hold only each check suite's latest attempt, so a rerun
|
||||
* attempt that created no jobs hides the failed run before it, and a workflow run that failed before creating a job is
|
||||
* never a check run at all: gstack-evals#303 read green with its `checks` job red. ciAtHead counts both. Each caller adds
|
||||
* these rows to the ones it already reads and keeps the worst bucket per name, so no check that gated before stops gating.
|
||||
*/
|
||||
import type { GhRunner } from './ci-gate';
|
||||
import { ciAtHead } from './ship-queue';
|
||||
|
||||
export interface HeadCheckRow { name: string; bucket: string }
|
||||
|
||||
const CONCLUSION_BUCKET: Record<string, string> = {
|
||||
success: 'pass', failure: 'fail', timed_out: 'fail', action_required: 'fail', startup_failure: 'fail',
|
||||
cancelled: 'cancel', skipped: 'skipping', neutral: 'skipping',
|
||||
};
|
||||
|
||||
/** Check runs (by id) and workflow runs (by id) a caller sets aside, such as the release cut's own jobs (GST-2463). */
|
||||
export interface AsideRuns { checkRunIds: ReadonlySet<number>; workflowRunIds: ReadonlySet<number> }
|
||||
|
||||
const PAGE = 100;
|
||||
|
||||
/**
|
||||
* Every check and commit-status context ciAtHead counts at `sha` in `repo`, or why they could not all be read (unknown is
|
||||
* never green). A check with an unknown conclusion keeps its spelling and one with none is `fail`, as ciAtHead counts them.
|
||||
* `aside` removes those check runs and workflow runs from every list before ciAtHead reads it: each list is read whole and
|
||||
* served again without them, so ciAtHead's total_count check still holds, and a list that cannot be read whole is unknown.
|
||||
*/
|
||||
export function headCheckRows(gh: GhRunner, repo: string, sha: string, aside?: AsideRuns): { rows: HeadCheckRow[] } | { error: string } {
|
||||
const run = (args: string[]) => gh(args.map(a => a.replace('repos/{owner}/{repo}/', `repos/${repo}/`)));
|
||||
const lists = new Map<string, Array<Record<string, unknown>> | null>();
|
||||
const readWhole = (prefix: string, key: string, drop: ReadonlySet<number>) => {
|
||||
const items: Array<Record<string, unknown>> = [];
|
||||
for (let page = 1; page <= 10; page++) {
|
||||
const r = run(['api', `${prefix}per_page=${PAGE}&page=${page}`]);
|
||||
let body: Record<string, unknown> | null = null;
|
||||
try { body = !r.error && r.status === 0 ? JSON.parse(r.stdout) : null; } catch {}
|
||||
const list = body?.[key];
|
||||
if (!Array.isArray(list) || typeof body?.total_count !== 'number') return null;
|
||||
items.push(...list);
|
||||
if (items.length >= body.total_count || list.length < PAGE) return items.length === body.total_count ? items.filter(i => !drop.has(Number(i.id))) : null;
|
||||
}
|
||||
return null;
|
||||
};
|
||||
const inRepo = (args: string[]) => {
|
||||
const m = aside && args[0] === 'api' ? /^(.*(?:\/check-runs|\/actions\/runs)\?(?:.*&)?)per_page=\d+&page=(\d+)$/.exec(args[1] ?? '') : null;
|
||||
if (!aside || !m) {
|
||||
const r = run(args);
|
||||
return { ok: !r.error && r.status === 0, out: r.stdout, err: r.error ?? r.stderr };
|
||||
}
|
||||
const key = m[1]!.includes('/check-runs') ? 'check_runs' : 'workflow_runs';
|
||||
if (!lists.has(m[1]!)) lists.set(m[1]!, readWhole(m[1]!, key, key === 'check_runs' ? aside.checkRunIds : aside.workflowRunIds));
|
||||
const all = lists.get(m[1]!);
|
||||
if (!all) return { ok: false, out: '', err: `${key} unreadable while setting runs aside` };
|
||||
const page = Number(m[2]);
|
||||
return { ok: true, out: JSON.stringify({ total_count: all.length, [key]: all.slice((page - 1) * PAGE, page * PAGE) }), err: '' };
|
||||
};
|
||||
const ci = ciAtHead({ repoRoot: process.cwd(), base: '', gh: inRepo }, sha);
|
||||
if (ci.line.startsWith('unknown')) return { error: ci.line.replace(/^unknown at (\S+)/, 'CI at $1 is unknown') };
|
||||
return {
|
||||
rows: [
|
||||
...ci.checks.map(c => ({ name: c.name, bucket: c.status !== 'completed' ? 'pending' : CONCLUSION_BUCKET[c.conclusion ?? ''] ?? (c.conclusion || 'fail') })),
|
||||
...ci.contexts.map(s => ({ name: s.name, bucket: s.state === 'success' ? 'pass' : s.state === 'pending' ? 'pending' : 'fail' })),
|
||||
],
|
||||
};
|
||||
}
|
||||
+10
-3
@@ -13,6 +13,7 @@
|
||||
* gstack's gh runner shape (lib/ci-gate.ts).
|
||||
*/
|
||||
import type { GhResult, GhRunner } from './ci-gate';
|
||||
import { headCheckRows } from './ci-head-checks';
|
||||
import { parseHeadChecks, parsePullState, rateLimitKind, type PullState } from './gh-poll';
|
||||
|
||||
export class GhError extends Error {
|
||||
@@ -179,14 +180,20 @@ function restList(env: GhEnv, path: string, key: string): unknown[] {
|
||||
|
||||
/**
|
||||
* Every check on the head commit by name, from REST check runs and commit
|
||||
* statuses (the same rows `gh pr checks` reads through GraphQL), duplicates
|
||||
* collapsed to the worst bucket.
|
||||
* statuses (the same rows `gh pr checks` reads through GraphQL) plus CI at the
|
||||
* head as gstack-ship-queue ready reads it (lib/ci-head-checks.ts: a failed
|
||||
* check whose rerun attempt created no jobs, a workflow run that failed with
|
||||
* no jobs), duplicates collapsed to the worst bucket. An unknown read is a
|
||||
* transient GhError, so the poll loop tries again.
|
||||
*/
|
||||
export function fetchChecks(env: GhEnv, repo: string, head: string): Map<string, Check> {
|
||||
const rows = parseHeadChecks(
|
||||
const listed = parseHeadChecks(
|
||||
restList(env, `repos/${repo}/commits/${head}/check-runs?per_page=100`, 'check_runs'),
|
||||
restList(env, `repos/${repo}/commits/${head}/status?per_page=100`, 'statuses'),
|
||||
);
|
||||
const ci = headCheckRows(args => runGh(env, args), repo, head);
|
||||
if ('error' in ci) throw new GhError(ci.error, true);
|
||||
const rows = [...listed, ...ci.rows.map(row => ({ ...row, link: '', description: '' }))];
|
||||
const checks = new Map<string, Check>();
|
||||
for (const row of rows) {
|
||||
const current = checks.get(row.name);
|
||||
|
||||
@@ -14,6 +14,7 @@ import type { GhResult } from '../lib/ci-gate';
|
||||
import { MAX_LIMIT_WAITS, MIN_POLL_MS, parseHeadChecks, parsePullState, pollDelayMs, rateLimitKind, rateLimitWaitMs, withRateLimit } from '../lib/gh-poll';
|
||||
import { GhError, fetchChecks, runGh } from '../lib/pr-wait-github';
|
||||
import { waitForRun } from '../lib/ci-carry';
|
||||
import { ciRest } from './helpers/ci-attempts-fixture';
|
||||
|
||||
const fixture = (name: string): string => readFileSync(join(import.meta.dir, 'fixtures', 'gh-poll', name), 'utf8');
|
||||
const ok = (stdout: string): GhResult => ({ status: 0, stdout, stderr: '' });
|
||||
@@ -172,8 +173,7 @@ describe('REST replies parse from recorded fixtures', () => {
|
||||
const calls: string[][] = [];
|
||||
const gh = (args: string[]): GhResult => {
|
||||
calls.push(args);
|
||||
if (args[2]!.includes('/check-runs')) return ok([...runs, pushRun].map(r => JSON.stringify(r)).join('\n'));
|
||||
return ok('');
|
||||
return ciRest(args, { check_runs: [...runs, pushRun] }) ?? ok('');
|
||||
};
|
||||
const checks = fetchChecks({ gh, sleep: () => {}, log: () => {} }, 'garrytan/gstack', 'f'.repeat(40));
|
||||
expect(checks.size).toBe(new Set(runs.map(r => r.name)).size);
|
||||
@@ -181,6 +181,10 @@ describe('REST replies parse from recorded fixtures', () => {
|
||||
expect(calls.map(c => c.slice(0, 3))).toEqual([
|
||||
['api', '--paginate', `repos/garrytan/gstack/commits/${'f'.repeat(40)}/check-runs?per_page=100`],
|
||||
['api', '--paginate', `repos/garrytan/gstack/commits/${'f'.repeat(40)}/status?per_page=100`],
|
||||
['api', `repos/garrytan/gstack/commits/${'f'.repeat(40)}/check-runs?per_page=100&page=1`],
|
||||
['api', `repos/garrytan/gstack/commits/${'f'.repeat(40)}/check-runs?filter=all&per_page=100&page=1`],
|
||||
['api', `repos/garrytan/gstack/commits/${'f'.repeat(40)}/status?per_page=100&page=1`],
|
||||
['api', `repos/garrytan/gstack/actions/runs?head_sha=${'f'.repeat(40)}&per_page=100&page=1`],
|
||||
]);
|
||||
expect(calls.some(c => c[0] === 'pr' || c[1] === 'graphql')).toBe(false);
|
||||
});
|
||||
|
||||
@@ -10,6 +10,7 @@ import { describe, expect, test } from 'bun:test';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { ciGateMain, evaluateChecks, parseCiGateArgs, runCiGate, type CiGateOptions, type GhResult, type GhRunner } from '../lib/ci-gate';
|
||||
import { PR303, PR303_RERUN_PASSED, ciRest, type CiAtCommit } from './helpers/ci-attempts-fixture';
|
||||
|
||||
const HEAD = 'a'.repeat(40);
|
||||
const OTHER = 'b'.repeat(40);
|
||||
@@ -21,13 +22,16 @@ const row = (name: string, bucket?: string) => ({ name, state: 'X', link: `https
|
||||
const NONE = err("no checks reported on the 'feature' branch");
|
||||
const NONE_REQUIRED = err("no required checks reported on the 'feature' branch");
|
||||
|
||||
interface Script { heads?: string[]; required?: GhResult[]; all?: GhResult[]; api?: GhResult }
|
||||
/** `rest` is REST CI at the head (none by default); null answers none of those reads, so they fall to `api`. */
|
||||
interface Script { heads?: string[]; required?: GhResult[]; all?: GhResult[]; api?: GhResult; rest?: CiAtCommit | null }
|
||||
function fakeGh(script: Script): GhRunner & { calls: string[][] } {
|
||||
const queues = { head: script.heads ?? [HEAD], required: script.required ?? [], all: script.all ?? [] };
|
||||
const next = <T>(queue: T[]): T => (queue.length > 1 ? queue.shift()! : queue[0]);
|
||||
const calls: string[][] = [];
|
||||
const gh = ((args: string[]) => {
|
||||
calls.push(args);
|
||||
const rest = script.rest === null ? null : ciRest(args, script.rest ?? { check_runs: [] });
|
||||
if (rest) return rest;
|
||||
if (args[0] === 'api') return script.api ?? err('gh: Not Found (HTTP 404)');
|
||||
if (args[1] === 'view') return { status: 0, stdout: `${next(queues.head)}\n`, stderr: '' };
|
||||
if (args[1] === 'merge') throw new Error('the gate must never merge');
|
||||
@@ -181,11 +185,32 @@ describe('head binding', () => {
|
||||
['pr', 'view', '42', '--repo', 'owner/project', '--json', 'headRefOid', '--jq', '.headRefOid'],
|
||||
['pr', 'checks', '42', '--repo', 'owner/project', '--required', '--json', 'name,state,bucket,link'],
|
||||
['pr', 'checks', '42', '--repo', 'owner/project', '--json', 'name,state,bucket,link'],
|
||||
['api', `repos/owner/project/commits/${HEAD}/check-runs?per_page=100&page=1`],
|
||||
['api', `repos/owner/project/commits/${HEAD}/check-runs?filter=all&per_page=100&page=1`],
|
||||
['api', `repos/owner/project/commits/${HEAD}/status?per_page=100&page=1`],
|
||||
['api', `repos/owner/project/actions/runs?head_sha=${HEAD}&per_page=100&page=1`],
|
||||
['pr', 'view', '42', '--repo', 'owner/project', '--json', 'headRefOid', '--jq', '.headRefOid'],
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('every attempt of each check suite counts (GST-2252)', () => {
|
||||
// Value: protects=the merge gate counts a failed check whose rerun attempt created no jobs and a workflow run that failed before any job, as
|
||||
// gstack-ship-queue ready does, while a rerun that passed stays PASS and an unreadable REST read is ERROR; fails_when=the gate trusts gh pr checks
|
||||
// (GitHub's rollup lists only each suite's latest attempt) and passes gstack-evals#303; seam=lib/ci-head-checks.ts (ciAtHead) in evaluateOnce,
|
||||
// through the fake gh's filter=latest/all split; no_claim=GitHub's attempt grouping (the fixture is #4235's recording)
|
||||
test('a failed check hidden behind a rerun attempt with no jobs, and a workflow run that failed with no jobs, are FAIL (gstack-evals#303)', async () => {
|
||||
const red = await gate({ required: [NONE_REQUIRED], all: [ok([row('sealed', 'pass')])], rest: PR303 });
|
||||
expect(red.verdict).toBe('FAIL');
|
||||
expect(checkLines(red.lines)).toEqual(['CHECK\tchecks\tfail\trequired=n\t', 'CHECK\tsealed (run 42671854799, no jobs)\tfail\trequired=n\t']);
|
||||
const rerun = await gate({ required: [NONE_REQUIRED], all: [ok([row('checks', 'pass'), row('sealed', 'pass')])], rest: PR303_RERUN_PASSED });
|
||||
expect(rerun.verdict).toBe('PASS');
|
||||
const unreadable = await gate({ required: [NONE_REQUIRED], all: [ok([row('sealed', 'pass')])], rest: null, api: err('gh: HTTP 403') });
|
||||
expect(unreadable.verdict).toBe('ERROR');
|
||||
expect(unreadable.lines[0]).toStartWith('CAUSE CI at aaaaaaaaaaaa is unknown');
|
||||
});
|
||||
});
|
||||
|
||||
describe('per-head, per-check overrides', () => {
|
||||
const nonRequiredRed = { required: [ok([row('build', 'pass')])], all: [ok([row('build', 'pass'), row('lint', 'fail'), row('e2e', 'pending')])] };
|
||||
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
/**
|
||||
* GitHub's REST answers about CI at one commit, for unit fakes of the readers that share ciAtHead (lib/ci-head-checks.ts,
|
||||
* GST-2252), with the check runs and workflow runs #4235 (GST-2167) recorded from gstack-evals#303 in
|
||||
* test/gstack-ship-queue.test.ts. A check run marked earlier_attempt belongs to a superseded workflow-run attempt: like
|
||||
* GitHub's default filter=latest and ghQueueShim (test/helpers/ship-queue-fixture.ts), it is listed only when the call asks
|
||||
* for filter=all.
|
||||
*/
|
||||
export interface CiAtCommit { check_runs: Array<Record<string, unknown>>; statuses?: Array<Record<string, unknown>>; workflow_runs?: Array<Record<string, unknown>> }
|
||||
|
||||
const app = { id: 15368 };
|
||||
const wf = (id: number, name: string, workflow_id: number, suite: number, conclusion: string, created_at: string) => ({ id, name, workflow_id, check_suite_id: suite, status: 'completed', conclusion, created_at });
|
||||
const checks = { id: 129322614246, name: 'checks', status: 'completed', conclusion: 'failure', completed_at: '2026-11-09T17:49:57Z', app, check_suite: { id: 115553115622 } };
|
||||
const sealed = { id: 129322588977, name: 'sealed', status: 'completed', conclusion: 'success', completed_at: '2026-11-09T17:22:47Z', app, check_suite: { id: 115553095053 } };
|
||||
|
||||
/**
|
||||
* gstack-evals#303: `checks` failed in its first attempt and the rerun attempt created no jobs, so GitHub's default list shows
|
||||
* only the passing `sealed`; a second sealed run failed without creating a job. ciAtHead reads it red (passed=1 failed=2).
|
||||
*/
|
||||
export const PR303: CiAtCommit = {
|
||||
check_runs: [{ ...checks, earlier_attempt: true }, sealed],
|
||||
workflow_runs: [
|
||||
wf(42671854799, 'sealed', 2, 115574560782, 'failure', '2026-11-09T18:11:40Z'),
|
||||
wf(42663983836, 'ci', 1, 115553115622, 'failure', '2026-11-09T17:22:30Z'),
|
||||
wf(42663976278, 'sealed', 2, 115553095053, 'success', '2026-11-09T17:22:28Z'),
|
||||
],
|
||||
};
|
||||
|
||||
/** The negative control from the same test: the failed `checks` attempt was rerun and passed, so the head is green. */
|
||||
export const PR303_RERUN_PASSED: CiAtCommit = {
|
||||
check_runs: [{ ...checks, earlier_attempt: true }, { ...checks, id: 129330000000, conclusion: 'success', completed_at: '2026-11-09T18:20:00Z' }, sealed],
|
||||
workflow_runs: [wf(42663983836, 'ci', 1, 115553115622, 'success', '2026-11-09T17:22:30Z'), wf(42663976278, 'sealed', 2, 115553095053, 'success', '2026-11-09T17:22:28Z')],
|
||||
};
|
||||
|
||||
/**
|
||||
* Bash `case "$*"` arms for a gh shim that answers every read of CI at a commit with `checkRuns` (all in their latest attempt):
|
||||
* gateVerdict's `--paginate --slurp` check-runs pages and ciAtHead's single pages, with no statuses and no workflow runs.
|
||||
*/
|
||||
export function ciShimCases(checkRuns: unknown[]): string {
|
||||
return [
|
||||
` *--slurp*check-runs*) echo '${JSON.stringify([{ check_runs: checkRuns }])}' ;;`,
|
||||
` *check-runs*) echo '${JSON.stringify({ total_count: checkRuns.length, check_runs: checkRuns })}' ;;`,
|
||||
` */status\\?*) echo '{"total_count":0,"statuses":[]}' ;;`,
|
||||
` *actions/runs*) echo '{"total_count":0,"workflow_runs":[]}' ;;`,
|
||||
].join('\n') + '\n';
|
||||
}
|
||||
|
||||
/**
|
||||
* gh's reply to one REST read of CI at a commit, or null when `args` is not one: `commits/<sha>/check-runs`, `commits/<sha>/status`
|
||||
* or `actions/runs?head_sha=<sha>`, as one page (`page=N`), `--paginate --jq '.<key>[]'` lines or `--paginate --slurp` pages.
|
||||
*/
|
||||
export function ciRest(args: string[], data: CiAtCommit): { status: number; stdout: string; stderr: string } | null {
|
||||
if (args[0] !== 'api') return null;
|
||||
const path = args.find((a, i) => i > 0 && !a.startsWith('--') && args[i - 1] !== '--jq') ?? '';
|
||||
const m = /commits\/[0-9a-f]+\/(check-runs|status)\b|actions\/runs\?head_sha=/.exec(path);
|
||||
if (!m) return null;
|
||||
const key = m[1] === 'check-runs' ? 'check_runs' : m[1] === 'status' ? 'statuses' : 'workflow_runs';
|
||||
const list = key === 'check_runs' ? data.check_runs.filter(r => /[?&]filter=all(&|$)/.test(path) || !r.earlier_attempt) : data[key] ?? [];
|
||||
const out = (stdout: string) => ({ status: 0, stdout, stderr: '' });
|
||||
if (args.includes('--jq')) return out(list.map(item => JSON.stringify(item)).join('\n'));
|
||||
if (args.includes('--slurp')) return out(JSON.stringify([{ total_count: list.length, [key]: list }]));
|
||||
const page = Number(/[?&]page=(\d+)/.exec(path)?.[1] ?? 1);
|
||||
return out(JSON.stringify({ total_count: list.length, [key]: page === 1 ? list : [] }));
|
||||
}
|
||||
@@ -20,7 +20,13 @@ const OTHER = 'b'.repeat(40);
|
||||
|
||||
const FAKE_GH = `#!/usr/bin/env bash
|
||||
D="$FAKE_GH_DIR"
|
||||
if [ "$1" = api ]; then kind=api
|
||||
if [ "$1" = api ]; then
|
||||
case "$2" in
|
||||
*/check-runs\?*) echo '{"total_count":0,"check_runs":[]}'; exit 0 ;;
|
||||
*/status\?*) echo '{"total_count":0,"statuses":[]}'; exit 0 ;;
|
||||
*/actions/runs\?*) echo '{"total_count":0,"workflow_runs":[]}'; exit 0 ;;
|
||||
esac
|
||||
kind=api
|
||||
elif [ "$1 $2" = "pr merge" ]; then printf '%s\\n' "$*" >> "$D/merges.log"; exit 0
|
||||
elif [ "$1 $2" = "pr view" ]; then kind=head
|
||||
elif [ "$1 $2" = "pr checks" ]; then case " $* " in *" --required "*) kind=required ;; *) kind=all ;; esac
|
||||
|
||||
+28
-8
@@ -15,6 +15,7 @@ import type { GhResult } from '../lib/ci-gate';
|
||||
import { DEFAULT_ADVISORY_BOTS, DEFAULT_GATING_BOTS, DEFAULT_TIMEOUT_MINUTES, PR_WAIT_EXIT, Watcher, botLists, parsePrTarget, parsePrWaitArgs, prWaitMain, stateDir, type PrWaitEnv, type PrWaitOptions } from '../lib/pr-wait';
|
||||
import { GhError, runGh } from '../lib/pr-wait-github';
|
||||
import { RESULT_CODES } from '../lib/result-codes';
|
||||
import { PR303, PR303_RERUN_PASSED, ciRest, type CiAtCommit } from './helpers/ci-attempts-fixture';
|
||||
|
||||
const ROOT = join(import.meta.dir, '..');
|
||||
const HEAD = 'a'.repeat(40);
|
||||
@@ -63,14 +64,13 @@ function restPull(graphql: string): string {
|
||||
});
|
||||
}
|
||||
const CONCLUSION: Record<string, string> = { pass: 'success', fail: 'failure', skipping: 'skipped', cancel: 'cancelled' };
|
||||
/** `gh pr checks`-shaped rows as the REST check-runs lines `gh api --paginate --jq '.check_runs[]'` prints, one suite per row. */
|
||||
function restCheckRuns(result: GhResult): GhResult {
|
||||
if (result.status !== 0) return result;
|
||||
/** `gh pr checks`-shaped rows as REST check runs, one suite per row. */
|
||||
function checkRunObjects(result: GhResult): Array<Record<string, unknown>> {
|
||||
const rows = JSON.parse(result.stdout || '[]') as Array<{ name: string; bucket: string; link: string; description: string }>;
|
||||
return ok(rows.map((row, i) => JSON.stringify({
|
||||
return rows.map((row, i) => ({
|
||||
id: i + 1, name: row.name, status: row.bucket === 'pending' ? 'in_progress' : 'completed', conclusion: row.bucket === 'pending' ? null : CONCLUSION[row.bucket] ?? row.bucket,
|
||||
completed_at: row.bucket === 'pending' ? null : '2026-11-06T00:00:00Z', details_url: row.link, output: { title: row.description }, app: { id: 15368 }, check_suite: { id: 100 + i },
|
||||
})).join('\n'));
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -81,6 +81,8 @@ function restCheckRuns(result: GhResult): GhResult {
|
||||
interface Script {
|
||||
pr?: string | string[];
|
||||
checks?: GhResult | GhResult[];
|
||||
/** Raw REST CI at the head (check runs with attempts, workflow runs), served instead of `checks`. */
|
||||
rest?: CiAtCommit;
|
||||
rulesets?: GhResult;
|
||||
protection?: GhResult;
|
||||
prFiles?: GhResult;
|
||||
@@ -95,6 +97,7 @@ function fakeGh(script: Script) {
|
||||
const checks = queue(script.checks);
|
||||
const calls: string[][] = [];
|
||||
let current = prJson();
|
||||
let shown: GhResult = ok(checksJson());
|
||||
const gh = (args: string[]): GhResult => {
|
||||
calls.push(args);
|
||||
const a = args[0];
|
||||
@@ -105,8 +108,10 @@ function fakeGh(script: Script) {
|
||||
current = next(prs) ?? prJson();
|
||||
return ok(restPull(current));
|
||||
}
|
||||
if (a === 'api' && b.includes('/check-runs')) return restCheckRuns(next(checks) ?? ok(checksJson()));
|
||||
if (a === 'api' && b.includes('/status')) return ok('');
|
||||
if (script.rest) { const rest = ciRest(args, script.rest); if (rest) return rest; }
|
||||
// One poll reads the paginated list once (which takes the next scripted reply), then ciAtHead's pages of that same reply.
|
||||
if (a === 'api' && args[1] === '--paginate' && b.includes('/check-runs')) shown = next(checks) ?? ok(checksJson());
|
||||
if (a === 'api' && /\/check-runs|\/status|\/actions\/runs/.test(b)) return shown.status === 0 ? ciRest(args, { check_runs: checkRunObjects(shown) })! : shown;
|
||||
if (a === 'api' && b.includes('/git/ref/heads/')) {
|
||||
const oid = JSON.parse(current).data.repository.pullRequest.baseRef?.target.oid;
|
||||
return oid ? ok(`${oid}\n`) : NOT_FOUND;
|
||||
@@ -458,6 +463,20 @@ describe('pass-1 review fixes: gates agree, bounded waits, bot identity, local f
|
||||
expect(lines()).toEqual(['#42 check failed: quality-gate https://ci.example/quality-gate', expect.stringContaining('(PR_WAIT_BLOCKED;')]);
|
||||
});
|
||||
|
||||
// Value: protects=green counts a failed check whose rerun attempt created no jobs and a workflow run that failed before any job, as gstack-ship-queue
|
||||
// ready does, while a rerun that passed stays green; fails_when=pr-wait reads only GitHub's default (latest-attempt) check-runs list and calls
|
||||
// gstack-evals#303 green; seam=lib/ci-head-checks.ts (ciAtHead) behind fetchChecks, through the fake gh's filter=latest/all split;
|
||||
// no_claim=GitHub's attempt grouping (the fixture is #4235's recording)
|
||||
test('a failed check hidden behind a rerun attempt with no jobs, and a workflow run that failed with no jobs, block green (gstack-evals#303, GST-2252)', () => {
|
||||
const red = watcher({ rest: PR303 }, 'green');
|
||||
expect(red.poll()).toBe(PR_WAIT_EXIT.blocked);
|
||||
expect(lines()).toEqual(['#42 check failed: checks', '#42 check failed: sealed (run 42671854799, no jobs)', expect.stringContaining('(PR_WAIT_BLOCKED;')]);
|
||||
stdout = [];
|
||||
const rerun = watcher({ rest: PR303_RERUN_PASSED }, 'green');
|
||||
expect(pollUntilDone(rerun, 3)).toBe(PR_WAIT_EXIT.met);
|
||||
expect(lines()).toEqual([expect.stringMatching(/^#42 green on aaaaaaa: 2 pass;/)]);
|
||||
});
|
||||
|
||||
// #3495 follow-up (ruled by GStack Minder, 2026-10-26): gstack-ci-gate --exclude has a watcher equivalent, so a non-required
|
||||
// check stuck in pending (a canary, a deploy preview) no longer holds --until green for the whole timeout.
|
||||
test('--exclude leaves a named non-required check out of the verdict: pending does not hold green, failed does not block; a required name is refused and gates', () => {
|
||||
@@ -632,7 +651,8 @@ describe('pass-1 review fixes: gates agree, bounded waits, bot identity, local f
|
||||
case "$1 $2" in
|
||||
'api repos/owner/project/pulls/42') echo '${restPull(prJson({}))}' ;;
|
||||
'api graphql') echo '${prJson({})}' ;;
|
||||
'api --paginate') echo '${restCheckRuns(ok(checksJson(['build', 'pending']))).stdout}' ;;
|
||||
'api --paginate') echo '${checkRunObjects(ok(checksJson(['build', 'pending']))).map(r => JSON.stringify(r)).join('\n')}' ;;
|
||||
'api repos/owner/project/commits/'*|'api repos/owner/project/actions/runs'*) echo '{"total_count":0,"check_runs":[],"statuses":[],"workflow_runs":[]}' ;;
|
||||
*) echo 'gh: Not Found (HTTP 404)' >&2; exit 1 ;;
|
||||
esac`, { mode: 0o755 });
|
||||
const stateRoot = join(dir, 'state');
|
||||
|
||||
Reference in new issue
Block a user