From bc8bec4df289ad8039586287c8fe73eb45659f79 Mon Sep 17 00:00:00 2001 From: garrytan Date: Fri, 2 Oct 2026 14:32:50 +0000 Subject: [PATCH] fix(land-and-deploy): never merge over red, pending or missing CI (#2995) /land-and-deploy gated only on required checks, so a repo with none (every GitHub Free private repo, any repo without branch protection) skipped the CI wait and could merge over red CI. The verdict now comes from bin/gstack-ci-gate, a tested helper bound to the head SHA: all checks by default, cancel/unknown buckets fail, NO_CHECKS needs explicit per-head approval, and both merge commands (including --auto) run only after a PASS for the same SHA they pin with --match-head-commit. Co-authored-by: harshitgavita-07 --- bin/gstack-ci-gate | 4 + land-and-deploy/SKILL.md | 32 +- land-and-deploy/SKILL.md.tmpl | 32 +- .../sections/first-run-validation.md | 2 +- .../sections/first-run-validation.md.tmpl | 2 +- land-and-deploy/sections/merge-and-deploy.md | 13 +- .../sections/merge-and-deploy.md.tmpl | 13 +- land-and-deploy/sections/readiness-gate.md | 8 + .../sections/readiness-gate.md.tmpl | 8 + lib/ci-gate.ts | 295 +++++++++++++++++ scripts/question-registry.ts | 16 + test/fixtures/ci-gate/no-checks.json | 14 + .../ci-gate/protected-nonrequired-red.json | 14 + test/fixtures/ci-gate/protected-pass.json | 14 + .../ci-gate/protected-required-red.json | 14 + test/gstack-ci-gate.test.ts | 306 ++++++++++++++++++ test/land-and-deploy-ci-gate.test.ts | 217 +++++++++++++ 17 files changed, 964 insertions(+), 40 deletions(-) create mode 100755 bin/gstack-ci-gate create mode 100644 lib/ci-gate.ts create mode 100644 test/fixtures/ci-gate/no-checks.json create mode 100644 test/fixtures/ci-gate/protected-nonrequired-red.json create mode 100644 test/fixtures/ci-gate/protected-pass.json create mode 100644 test/fixtures/ci-gate/protected-required-red.json create mode 100644 test/gstack-ci-gate.test.ts create mode 100644 test/land-and-deploy-ci-gate.test.ts diff --git a/bin/gstack-ci-gate b/bin/gstack-ci-gate new file mode 100755 index 000000000..088d0c76e --- /dev/null +++ b/bin/gstack-ci-gate @@ -0,0 +1,4 @@ +#!/usr/bin/env bun +import { ciGateMain } from '../lib/ci-gate'; + +process.exit(await ciGateMain(process.argv.slice(2))); diff --git a/land-and-deploy/SKILL.md b/land-and-deploy/SKILL.md index 888e329fa..9bf1df50c 100644 --- a/land-and-deploy/SKILL.md +++ b/land-and-deploy/SKILL.md @@ -581,7 +581,7 @@ When the user types `/land-and-deploy`, run this skill. Automate read-only detection and polling. First-run setup confirmation (Step 1.5) and pre-merge approval (Step 3.5) are mandatory when applicable. Stop on missing -access, unknown target/state, failing required CI, conflicts, or failing tests. +access, unknown target/state, unapproved red, pending or missing CI, conflicts, or failing tests. After any merge error, read server state before deciding whether to stop. Failures, timeouts, staging choices, rollback, and optional cleanup use the explicit decisions below; no approval overrides a blocker or authorizes a different revision. @@ -695,16 +695,16 @@ Choice A saves the fingerprint and continues to Step 2; B/C stop. Tell the user: "Checking CI status and merge readiness..." ```bash -gh pr checks "$PR_NUMBER" --repo "$REPO" --required --json name,state,bucket,link +~/.claude/skills/gstack/bin/gstack-ci-gate --repo "$REPO" --pr "$PR_NUMBER" --expect-head "$PR_HEAD" --registration-wait 60 ``` -Parse valid JSON using `bucket` (pass/fail/pending/skipping/cancel). Exit 8 means -pending; a nonzero exit with valid failing checks is a CI failure. Auth/network/schema -errors are **STOP**, never "no required checks". An empty successful result or the -CLI's explicit "no required checks reported" response means none are configured. -1. Required checks **FAILING/cancelled**: **STOP**, list failures to fix. -2. Required checks **PENDING**: announce the wait and proceed to Step 3. -3. All pass (or none required): report that exact result. Skip only Step 3's wait; +It gates on all checks on this head, required or not. Act on line 1, +`VERDICT ` (`` must be `PR_HEAD`), never on the exit code. +Auth/network/schema errors are ERROR, never "no required checks". +1. ERROR, or FAIL on a `required=y|?` check: **STOP**, show the output. +2. Only `required=n` failures, or NO_CHECKS ("no CI ran on ``" after a 60 s + re-poll): Step 3.5 must approve it. +3. PENDING: Step 3. PASS: skip Step 3's wait; continue to Step 3.4, then Step 3.5 before merging. Also check for merge conflicts: @@ -712,23 +712,21 @@ Also check for merge conflicts: gh pr view "$PR_NUMBER" --repo "$REPO" --json mergeable -q .mergeable ``` If `CONFLICTING`: **STOP**, resolve conflicts first. Failed/UNKNOWN readback: **STOP**, -readiness is not established. Cancelled required checks are failures, not passes. +readiness is not established. --- ## Step 3: Wait for CI (if pending) -If required checks are still pending, wait for them to complete. Use a timeout of 15 minutes: +Poll in 4-minute rounds (300 s shell timeout), 15 minutes max; record the wait: ```bash -gh pr checks "$PR_NUMBER" --repo "$REPO" --required --watch --fail-fast --interval 30 +~/.claude/skills/gstack/bin/gstack-ci-gate --repo "$REPO" --pr "$PR_NUMBER" --expect-head "$PR_HEAD" --wait 240 ``` -Record the CI wait time for the deploy report. - -Pass: report duration and continue to Step 3.4, then Step 3.5 before merging. -Failure: **STOP**, show failing checks. Timeout (15 minutes): **STOP**, point to -GitHub Actions. Enforce the deadline; do not leave an unbounded watch running. +Other verdicts follow Step 2. PENDING at 15 minutes: list the still-pending checks; +ask A) wait 15 more minutes, B) exclude named `required=n` checks (Step 3.5 +approves each), or C) stop. --- diff --git a/land-and-deploy/SKILL.md.tmpl b/land-and-deploy/SKILL.md.tmpl index 724645497..f276a8c4d 100644 --- a/land-and-deploy/SKILL.md.tmpl +++ b/land-and-deploy/SKILL.md.tmpl @@ -50,7 +50,7 @@ When the user types `/land-and-deploy`, run this skill. Automate read-only detection and polling. First-run setup confirmation (Step 1.5) and pre-merge approval (Step 3.5) are mandatory when applicable. Stop on missing -access, unknown target/state, failing required CI, conflicts, or failing tests. +access, unknown target/state, unapproved red, pending or missing CI, conflicts, or failing tests. After any merge error, read server state before deciding whether to stop. Failures, timeouts, staging choices, rollback, and optional cleanup use the explicit decisions below; no approval overrides a blocker or authorizes a different revision. @@ -154,16 +154,16 @@ Choice A saves the fingerprint and continues to Step 2; B/C stop. Tell the user: "Checking CI status and merge readiness..." ```bash -gh pr checks "$PR_NUMBER" --repo "$REPO" --required --json name,state,bucket,link +~/.claude/skills/gstack/bin/gstack-ci-gate --repo "$REPO" --pr "$PR_NUMBER" --expect-head "$PR_HEAD" --registration-wait 60 ``` -Parse valid JSON using `bucket` (pass/fail/pending/skipping/cancel). Exit 8 means -pending; a nonzero exit with valid failing checks is a CI failure. Auth/network/schema -errors are **STOP**, never "no required checks". An empty successful result or the -CLI's explicit "no required checks reported" response means none are configured. -1. Required checks **FAILING/cancelled**: **STOP**, list failures to fix. -2. Required checks **PENDING**: announce the wait and proceed to Step 3. -3. All pass (or none required): report that exact result. Skip only Step 3's wait; +It gates on all checks on this head, required or not. Act on line 1, +`VERDICT ` (`` must be `PR_HEAD`), never on the exit code. +Auth/network/schema errors are ERROR, never "no required checks". +1. ERROR, or FAIL on a `required=y|?` check: **STOP**, show the output. +2. Only `required=n` failures, or NO_CHECKS ("no CI ran on ``" after a 60 s + re-poll): Step 3.5 must approve it. +3. PENDING: Step 3. PASS: skip Step 3's wait; continue to Step 3.4, then Step 3.5 before merging. Also check for merge conflicts: @@ -171,23 +171,21 @@ Also check for merge conflicts: gh pr view "$PR_NUMBER" --repo "$REPO" --json mergeable -q .mergeable ``` If `CONFLICTING`: **STOP**, resolve conflicts first. Failed/UNKNOWN readback: **STOP**, -readiness is not established. Cancelled required checks are failures, not passes. +readiness is not established. --- ## Step 3: Wait for CI (if pending) -If required checks are still pending, wait for them to complete. Use a timeout of 15 minutes: +Poll in 4-minute rounds (300 s shell timeout), 15 minutes max; record the wait: ```bash -gh pr checks "$PR_NUMBER" --repo "$REPO" --required --watch --fail-fast --interval 30 +~/.claude/skills/gstack/bin/gstack-ci-gate --repo "$REPO" --pr "$PR_NUMBER" --expect-head "$PR_HEAD" --wait 240 ``` -Record the CI wait time for the deploy report. - -Pass: report duration and continue to Step 3.4, then Step 3.5 before merging. -Failure: **STOP**, show failing checks. Timeout (15 minutes): **STOP**, point to -GitHub Actions. Enforce the deadline; do not leave an unbounded watch running. +Other verdicts follow Step 2. PENDING at 15 minutes: list the still-pending checks; +ask A) wait 15 more minutes, B) exclude named `required=n` checks (Step 3.5 +approves each), or C) stop. --- diff --git a/land-and-deploy/sections/first-run-validation.md b/land-and-deploy/sections/first-run-validation.md index 392a52bf9..25a227aca 100644 --- a/land-and-deploy/sections/first-run-validation.md +++ b/land-and-deploy/sections/first-run-validation.md @@ -93,7 +93,7 @@ Run whichever commands are relevant based on the detected platform. Build the re ║ ├─ Staging workflow: {file or "not found"} ║ ║ └─ Preview deploys: {detected or "not detected"} ║ ║ WHAT WILL HAPPEN ║ -║ 1. Wait for required CI if pending ║ +║ 1. Wait for all CI on this head (gstack-ci-gate) ║ ║ 2. Readiness checks and explicit merge approval ║ ║ 3. Merge PR via {merge method} ║ ║ 4. {Wait for deploy workflow / Wait 60s / Skip} ║ diff --git a/land-and-deploy/sections/first-run-validation.md.tmpl b/land-and-deploy/sections/first-run-validation.md.tmpl index cb0d9cd3c..bd4221ef5 100644 --- a/land-and-deploy/sections/first-run-validation.md.tmpl +++ b/land-and-deploy/sections/first-run-validation.md.tmpl @@ -55,7 +55,7 @@ Run whichever commands are relevant based on the detected platform. Build the re ║ ├─ Staging workflow: {file or "not found"} ║ ║ └─ Preview deploys: {detected or "not detected"} ║ ║ WHAT WILL HAPPEN ║ -║ 1. Wait for required CI if pending ║ +║ 1. Wait for all CI on this head (gstack-ci-gate) ║ ║ 2. Readiness checks and explicit merge approval ║ ║ 3. Merge PR via {merge method} ║ ║ 4. {Wait for deploy workflow / Wait 60s / Skip} ║ diff --git a/land-and-deploy/sections/merge-and-deploy.md b/land-and-deploy/sections/merge-and-deploy.md index 199f5478a..ff82ae790 100644 --- a/land-and-deploy/sections/merge-and-deploy.md +++ b/land-and-deploy/sections/merge-and-deploy.md @@ -73,7 +73,14 @@ and current state. **If `state == "CLOSED"`: STOP**, the PR closed without mergi Only START makes the first attempt. Immediately before either merge command, repeat readback and Step 1's local HEAD/branch/cleanliness check. Retargeting or local changes invalidate readiness; `--match-head-commit` protects head, not destination. +Each merge block re-runs the CI gate with Step 3.5's approvals (also for `--auto`, +which waits on required checks only). A gate exit means no merge ran: **STOP** with +its output. Only branch protection or a merge queue makes check-to-merge +atomic. ```bash +CI_GATE=$(~/.claude/skills/gstack/bin/gstack-ci-gate --repo "$REPO" --pr "$PR_NUMBER" --expect-head "$PR_HEAD" "${CI_OVERRIDE[@]}") +echo "$CI_GATE" +case "${CI_GATE%%$'\n'*}" in "VERDICT PASS $PR_HEAD"|"VERDICT NO_CHECKS $NO_CI_APPROVED_HEAD") ;; *) exit 1 ;; esac MERGE_ATTEMPT=auto MERGE_EXIT=0 MERGE_ERROR=$(gh pr merge "$MERGE_FLAG" --auto --delete-branch "$PR_NUMBER" --repo "$REPO" --match-head-commit "$PR_HEAD" 2>&1) || MERGE_EXIT=$? @@ -82,9 +89,11 @@ Return to readback, even on exit 0. Only DIRECT permits **one direct fallback**: readback has confirmed OPEN, no auto request and no queue entry, and the auto attempt returned one of the two documented rejection classes: auto-merge disabled, or PR already clean/unstable with nothing required pending. The latter does not mean -auto-merge is disabled. Recheck required CI as in Step 2 before the fallback; failures -or unknown check results stop, even if GitHub calls the PR mergeable. +auto-merge is disabled. ```bash +CI_GATE=$(~/.claude/skills/gstack/bin/gstack-ci-gate --repo "$REPO" --pr "$PR_NUMBER" --expect-head "$PR_HEAD" "${CI_OVERRIDE[@]}") +echo "$CI_GATE" +case "${CI_GATE%%$'\n'*}" in "VERDICT PASS $PR_HEAD"|"VERDICT NO_CHECKS $NO_CI_APPROVED_HEAD") ;; *) exit 1 ;; esac MERGE_ATTEMPT=direct MERGE_EXIT=0 MERGE_ERROR=$(gh pr merge "$MERGE_FLAG" --delete-branch "$PR_NUMBER" --repo "$REPO" --match-head-commit "$PR_HEAD" 2>&1) || MERGE_EXIT=$? diff --git a/land-and-deploy/sections/merge-and-deploy.md.tmpl b/land-and-deploy/sections/merge-and-deploy.md.tmpl index b9814a0bf..6ef5f0fad7 100644 --- a/land-and-deploy/sections/merge-and-deploy.md.tmpl +++ b/land-and-deploy/sections/merge-and-deploy.md.tmpl @@ -71,7 +71,14 @@ and current state. **If `state == "CLOSED"`: STOP**, the PR closed without mergi Only START makes the first attempt. Immediately before either merge command, repeat readback and Step 1's local HEAD/branch/cleanliness check. Retargeting or local changes invalidate readiness; `--match-head-commit` protects head, not destination. +Each merge block re-runs the CI gate with Step 3.5's approvals (also for `--auto`, +which waits on required checks only). A gate exit means no merge ran: **STOP** with +its output. Only branch protection or a merge queue makes check-to-merge +atomic. ```bash +CI_GATE=$(~/.claude/skills/gstack/bin/gstack-ci-gate --repo "$REPO" --pr "$PR_NUMBER" --expect-head "$PR_HEAD" "${CI_OVERRIDE[@]}") +echo "$CI_GATE" +case "${CI_GATE%%$'\n'*}" in "VERDICT PASS $PR_HEAD"|"VERDICT NO_CHECKS $NO_CI_APPROVED_HEAD") ;; *) exit 1 ;; esac MERGE_ATTEMPT=auto MERGE_EXIT=0 MERGE_ERROR=$(gh pr merge "$MERGE_FLAG" --auto --delete-branch "$PR_NUMBER" --repo "$REPO" --match-head-commit "$PR_HEAD" 2>&1) || MERGE_EXIT=$? @@ -80,9 +87,11 @@ Return to readback, even on exit 0. Only DIRECT permits **one direct fallback**: readback has confirmed OPEN, no auto request and no queue entry, and the auto attempt returned one of the two documented rejection classes: auto-merge disabled, or PR already clean/unstable with nothing required pending. The latter does not mean -auto-merge is disabled. Recheck required CI as in Step 2 before the fallback; failures -or unknown check results stop, even if GitHub calls the PR mergeable. +auto-merge is disabled. ```bash +CI_GATE=$(~/.claude/skills/gstack/bin/gstack-ci-gate --repo "$REPO" --pr "$PR_NUMBER" --expect-head "$PR_HEAD" "${CI_OVERRIDE[@]}") +echo "$CI_GATE" +case "${CI_GATE%%$'\n'*}" in "VERDICT PASS $PR_HEAD"|"VERDICT NO_CHECKS $NO_CI_APPROVED_HEAD") ;; *) exit 1 ;; esac MERGE_ATTEMPT=direct MERGE_EXIT=0 MERGE_ERROR=$(gh pr merge "$MERGE_FLAG" --delete-branch "$PR_NUMBER" --repo "$REPO" --match-head-commit "$PR_HEAD" 2>&1) || MERGE_EXIT=$? diff --git a/land-and-deploy/sections/readiness-gate.md b/land-and-deploy/sections/readiness-gate.md index 9c3d5d476..6438e154d 100644 --- a/land-and-deploy/sections/readiness-gate.md +++ b/land-and-deploy/sections/readiness-gate.md @@ -240,6 +240,7 @@ Build the full readiness report: ╠══════════════════════════════════════════════════════════╣ ║ PR: #NNN — title ║ ║ Branch: feature → main ║ +║ CI (head ): PASS / FAIL / PENDING / NO CI RAN ║ ║ REVIEWS ║ ║ ├─ Eng Review: CURRENT / STALE (N commits) / — ║ ║ ├─ CEO Review: CURRENT / — (optional) ║ @@ -259,6 +260,13 @@ Build the full readiness report: ╚══════════════════════════════════════════════════════════╝ ``` +CI row: ERROR or red/pending `required=y|?` checks are BLOCKERS. Red/pending +`required=n` checks or NO_CHECKS first need one-way question +`land-and-deploy-ci-override` / `land-and-deploy-no-ci-confirm`, naming each check +(or "no CI ran on ``"): A) exclude exactly these / accept no CI, this head → set +`CI_OVERRIDE=(--override-head "$PR_HEAD" --exclude "" ...)` or +`NO_CI_APPROVED_HEAD=$PR_HEAD`, never stored or reused; B) stop (BLOCKER). + If there are BLOCKERS (including failing free tests): show the report and **STOP** with repair instructions. Do not offer A or C with blockers. If there are WARNINGS but no blockers: list each warning and recommend A if diff --git a/land-and-deploy/sections/readiness-gate.md.tmpl b/land-and-deploy/sections/readiness-gate.md.tmpl index b81aa5113..dbd7f6fd8 100644 --- a/land-and-deploy/sections/readiness-gate.md.tmpl +++ b/land-and-deploy/sections/readiness-gate.md.tmpl @@ -238,6 +238,7 @@ Build the full readiness report: ╠══════════════════════════════════════════════════════════╣ ║ PR: #NNN — title ║ ║ Branch: feature → main ║ +║ CI (head ): PASS / FAIL / PENDING / NO CI RAN ║ ║ REVIEWS ║ ║ ├─ Eng Review: CURRENT / STALE (N commits) / — ║ ║ ├─ CEO Review: CURRENT / — (optional) ║ @@ -257,6 +258,13 @@ Build the full readiness report: ╚══════════════════════════════════════════════════════════╝ ``` +CI row: ERROR or red/pending `required=y|?` checks are BLOCKERS. Red/pending +`required=n` checks or NO_CHECKS first need one-way question +`land-and-deploy-ci-override` / `land-and-deploy-no-ci-confirm`, naming each check +(or "no CI ran on ``"): A) exclude exactly these / accept no CI, this head → set +`CI_OVERRIDE=(--override-head "$PR_HEAD" --exclude "" ...)` or +`NO_CI_APPROVED_HEAD=$PR_HEAD`, never stored or reused; B) stop (BLOCKER). + If there are BLOCKERS (including failing free tests): show the report and **STOP** with repair instructions. Do not offer A or C with blockers. If there are WARNINGS but no blockers: list each warning and recommend A if diff --git a/lib/ci-gate.ts b/lib/ci-gate.ts new file mode 100644 index 000000000..006a998ca --- /dev/null +++ b/lib/ci-gate.ts @@ -0,0 +1,295 @@ +import { spawnSync } from 'node:child_process'; + +export type CiVerdict = 'PASS' | 'FAIL' | 'PENDING' | 'NO_CHECKS' | 'ERROR'; +export const CI_GATE_EXIT: Record = { PASS: 0, FAIL: 1, PENDING: 2, NO_CHECKS: 3, ERROR: 4 }; +export const MIN_GH_VERSION = '2.50.0'; + +export interface GhResult { status: number | null; stdout: string; stderr: string; error?: string } +export type GhRunner = (args: string[]) => GhResult; + +export interface CiGateOptions { + repo: string; + pr: string; + expectHead: string; + excludes: string[]; + overrideHead?: string; + waitSeconds: number; + intervalSeconds: number; + registrationWaitSeconds: number; +} + +export interface CiGateResult { verdict: CiVerdict; sha: string; lines: string[] } + +type Required = 'y' | 'n' | '?'; +type Klass = 'pass' | 'pending' | 'fail'; +interface Row { name: string; bucket: string; link: string; required: Required } +type ChecksRead = + | { kind: 'rows'; rows: { name: string; bucket: string; link: string }[] } + | { kind: 'none' } + | { kind: 'none-required' } + | { kind: 'error'; cause: string; stderr: string }; + +const PASS_BUCKETS = new Set(['pass', 'skipping', 'neutral']); +const HEAD_RE = /^[0-9a-f]{40}(?:[0-9a-f]{24})?$/; +const RANK: Record = { pass: 0, pending: 1, fail: 2 }; + +export const CI_GATE_USAGE = `gstack-ci-gate --repo OWNER/NAME --pr NUMBER --expect-head SHA [options] + +Decides whether a pull request's CI allows a merge of exactly SHA. Reads the +PR head, gates on every check (required ones and all others), and re-reads the +head afterwards; a moved head is ERROR. + +Output (stdout), always first: + VERDICT +then one line per non-passing check: + CHECKnamebucketrequired=y|n|?link[excluded|override-refused] +then NOTE / HEAD / CAUSE / STDERR / FIX lines. Act on the VERDICT line, not the exit code. + +Exit codes: 0 PASS, 1 FAIL, 2 PENDING, 3 NO_CHECKS, 4 ERROR. + +Buckets: pass, skipping and neutral pass; pending waits; fail, cancel and any +unknown bucket fail; a missing bucket field is ERROR (needs gh >= ${MIN_GH_VERSION}). + +Options: + --exclude NAME drop one named non-required check from the verdict + (repeatable; needs --override-head; required checks + and checks of unknown required status are never dropped) + --override-head SHA head the exclusions were approved for; must equal + --expect-head + --wait SECONDS keep polling while PENDING, up to SECONDS (default 0) + --interval SECONDS poll interval (default 30) + --registration-wait S when no check exists yet, re-poll up to S seconds + before reporting NO_CHECKS (default 0; use 60 right + after a push, when checks may not be registered yet) + --help show this help`; + +export function bucketClass(bucket: string): Klass { + if (PASS_BUCKETS.has(bucket)) return 'pass'; + return bucket === 'pending' ? 'pending' : 'fail'; +} + +function lastLine(text: string): string { + return text.trim().split(/\r?\n/).filter(Boolean).at(-1) ?? ''; +} + +export function parseChecks(result: GhResult): ChecksRead { + if (result.error) return { kind: 'error', cause: result.error, stderr: lastLine(result.stderr) }; + const text = result.stdout.trim(); + if (text) { + let parsed: unknown; + try { parsed = JSON.parse(text); } catch { + return { kind: 'error', cause: 'gh pr checks printed output that is not JSON', stderr: lastLine(result.stderr) }; + } + if (!Array.isArray(parsed)) return { kind: 'error', cause: 'gh pr checks JSON is not a list', stderr: lastLine(result.stderr) }; + const rows = []; + for (const item of parsed) { + if (!item || typeof item !== 'object' || typeof (item as { name?: unknown }).name !== 'string') { + return { kind: 'error', cause: 'gh pr checks returned a check without a name', stderr: lastLine(result.stderr) }; + } + const { name, bucket, link } = item as { name: string; bucket?: unknown; link?: unknown }; + if (typeof bucket !== 'string') { + return { kind: 'error', cause: `gh pr checks returned no bucket field for "${name}"`, stderr: lastLine(result.stderr) }; + } + rows.push({ name, bucket, link: typeof link === 'string' ? link : '' }); + } + return { kind: 'rows', rows }; + } + if (result.status === 0) return { kind: 'rows', rows: [] }; + if (/no required checks reported/.test(result.stderr)) return { kind: 'none-required' }; + if (/no checks reported/.test(result.stderr)) return { kind: 'none' }; + return { kind: 'error', cause: `gh pr checks exited ${result.status ?? 'abnormally'}`, stderr: lastLine(result.stderr) }; +} + +interface Evaluation { verdict: Exclude; lines: string[] } +type EvalOrError = Evaluation | { verdict: 'ERROR'; cause: string; stderr: string }; + +export function evaluateChecks(required: GhResult, all: GhResult, excludes: string[]): EvalOrError { + const allRead = parseChecks(all); + if (allRead.kind === 'error') return { verdict: 'ERROR', cause: allRead.cause, stderr: allRead.stderr }; + if (allRead.kind === 'none-required') return { verdict: 'ERROR', cause: 'gh pr checks without --required reported no required checks', stderr: '' }; + const reqRead = parseChecks(required); + const notes: string[] = []; + const requiredKnown = reqRead.kind === 'rows' || reqRead.kind === 'none-required'; + if (reqRead.kind === 'error') { + notes.push(`NOTE required-check metadata unavailable (${reqRead.stderr || reqRead.cause}); gating on all checks, overrides refused`); + } else if (reqRead.kind === 'none' && allRead.kind === 'rows' && allRead.rows.length > 0) { + notes.push('NOTE checks registered while they were being read; required status unknown, overrides refused'); + } + const requiredRows = reqRead.kind === 'rows' ? reqRead.rows : []; + const requiredNames = new Set(requiredRows.map(row => row.name)); + if (requiredKnown && requiredNames.size === 0) notes.push('NOTE no required checks configured; gating on all checks'); + const rows = new Map(); + const observe = (obs: { name: string; bucket: string; link: string }) => { + const required: Required = !requiredKnown ? '?' : requiredNames.has(obs.name) ? 'y' : 'n'; + const prior = rows.get(obs.name); + if (!prior || RANK[bucketClass(obs.bucket)] > RANK[bucketClass(prior.bucket)]) rows.set(obs.name, { ...obs, required }); + }; + for (const obs of allRead.kind === 'rows' ? allRead.rows : []) observe(obs); + for (const obs of requiredRows) observe(obs); + if (rows.size === 0) return { verdict: 'NO_CHECKS', lines: notes.filter(n => !n.includes('no required checks')) }; + + const excluded = new Set(); + const lines: string[] = []; + for (const name of excludes) { + const row = rows.get(name); + if (!row) notes.push(`NOTE override for unknown check "${name}" ignored`); + else if (row.required === 'n') excluded.add(name); + } + let worst: Klass = 'pass'; + for (const row of [...rows.values()].sort((a, b) => a.name.localeCompare(b.name))) { + const klass = bucketClass(row.bucket); + const isExcluded = excluded.has(row.name); + if (!isExcluded && RANK[klass] > RANK[worst]) worst = klass; + if (klass === 'pass') continue; + const marker = isExcluded ? '\texcluded' : excludes.includes(row.name) ? '\toverride-refused' : ''; + lines.push(`CHECK\t${row.name}\t${row.bucket}\trequired=${row.required}\t${row.link}${marker}`); + } + const verdict = worst === 'fail' ? 'FAIL' : worst === 'pending' ? 'PENDING' : 'PASS'; + return { verdict, lines: [...lines, ...notes] }; +} + +function readHead(gh: GhRunner, opts: CiGateOptions): { head: string } | { cause: string; stderr: string } { + const result = gh(['pr', 'view', opts.pr, '--repo', opts.repo, '--json', 'headRefOid', '--jq', '.headRefOid']); + if (result.error) return { cause: result.error, stderr: lastLine(result.stderr) }; + const head = result.stdout.trim(); + if (result.status !== 0 || !HEAD_RE.test(head)) { + return { cause: `gh pr view could not read the PR head (exit ${result.status ?? 'abnormal'})`, stderr: lastLine(result.stderr) }; + } + return { head }; +} + +function errorLines(cause: string, stderr: string): string[] { + return [ + `CAUSE ${cause}`, + ...(stderr ? [`STDERR ${stderr}`] : []), + `FIX check \`gh auth status\`; gstack-ci-gate needs gh >= ${MIN_GH_VERSION} (\`gh --version\`)`, + ]; +} + +function evaluateOnce(gh: GhRunner, opts: CiGateOptions): CiGateResult { + const fail = (lines: string[]): CiGateResult => ({ verdict: 'ERROR', sha: opts.expectHead, lines }); + const before = readHead(gh, opts); + if ('cause' in before) return fail(errorLines(before.cause, before.stderr)); + if (before.head !== opts.expectHead) { + return fail([`HEAD ${before.head}`, 'CAUSE the PR head is not the approved head', 'FIX rerun /land-and-deploy from Step 1 for the new head']); + } + const checkArgs = ['pr', 'checks', opts.pr, '--repo', opts.repo]; + const fields = ['--json', 'name,state,bucket,link']; + const required = gh([...checkArgs, '--required', ...fields]); + const all = gh([...checkArgs, ...fields]); + const after = readHead(gh, opts); + if ('cause' in after) return fail(errorLines(after.cause, after.stderr)); + if (after.head !== before.head) { + return fail([`HEAD ${after.head}`, 'CAUSE the PR head moved while checks were being read', 'FIX rerun /land-and-deploy from Step 1 for the new head']); + } + const evaluation = evaluateChecks(required, all, opts.excludes); + if (evaluation.verdict === 'ERROR') return fail(errorLines(evaluation.cause, evaluation.stderr)); + return { verdict: evaluation.verdict, sha: opts.expectHead, lines: evaluation.lines }; +} + +function workflowsNote(gh: GhRunner, opts: CiGateOptions): string { + const result = gh(['api', `repos/${opts.repo}/contents/.github/workflows?ref=${opts.expectHead}`, '--jq', 'length']); + if (!result.error && result.status === 0 && /^\d+$/.test(result.stdout.trim())) { + return Number(result.stdout.trim()) > 0 ? 'NOTE .github/workflows exists at this head' : 'NOTE .github/workflows is empty at this head'; + } + if (/HTTP 404|Not Found/.test(result.stderr)) return 'NOTE .github/workflows does not exist at this head'; + return 'NOTE could not tell whether .github/workflows exists at this head'; +} + +export async function runCiGate( + opts: CiGateOptions, + gh: GhRunner, + sleep: (ms: number) => Promise = ms => new Promise(resolve => setTimeout(resolve, ms)), + now: () => number = Date.now, +): Promise { + if (opts.excludes.length > 0 && opts.overrideHead !== opts.expectHead) { + return { + verdict: 'ERROR', sha: opts.expectHead, + lines: ['CAUSE check overrides were not approved for this head', 'FIX approve overrides again in the readiness report for the current head'], + }; + } + const started = now(); + const elapsed = () => (now() - started) / 1000; + let result = evaluateOnce(gh, opts); + while (true) { + if (result.verdict === 'NO_CHECKS' && elapsed() + Math.min(opts.intervalSeconds, 10) <= opts.registrationWaitSeconds) { + await sleep(Math.min(opts.intervalSeconds, 10) * 1000); + } else if (result.verdict === 'PENDING' && elapsed() + opts.intervalSeconds <= opts.waitSeconds) { + await sleep(opts.intervalSeconds * 1000); + } else break; + result = evaluateOnce(gh, opts); + } + if (result.verdict === 'NO_CHECKS') { + result.lines.push(`NOTE no CI ran on ${opts.expectHead} (looked for ${Math.round(elapsed())}s)`, workflowsNote(gh, opts)); + } + if (result.verdict === 'PENDING' && opts.waitSeconds > 0) { + result.lines.push(`NOTE still pending after waiting ${Math.round(elapsed())}s`); + } + return result; +} + +export function parseCiGateArgs(argv: string[]): CiGateOptions | { help: true } | { error: string; expectHead?: string } { + const opts: CiGateOptions = { + repo: '', pr: '', expectHead: '', excludes: [], waitSeconds: 0, intervalSeconds: 30, registrationWaitSeconds: 0, + }; + const seconds = (value: string | undefined, flag: string, positive = false): number | string => { + if (value === undefined || !/^\d+(?:\.\d+)?$/.test(value)) return `${flag} needs a number of seconds`; + const n = Number(value); + return positive && n <= 0 ? `${flag} must be greater than 0` : n; + }; + for (let i = 0; i < argv.length; i++) { + const flag = argv[i]; + const value = argv[i + 1]; + if (flag === '--help' || flag === '-h') return { help: true }; + const take = (): string | undefined => { i++; return value; }; + switch (flag) { + case '--repo': opts.repo = take() ?? ''; break; + case '--pr': opts.pr = take() ?? ''; break; + case '--expect-head': opts.expectHead = take() ?? ''; break; + case '--override-head': opts.overrideHead = take() ?? ''; break; + case '--exclude': { + const name = take(); + if (!name) return { error: '--exclude needs a check name', expectHead: opts.expectHead }; + opts.excludes.push(name); + break; + } + case '--wait': case '--interval': case '--registration-wait': { + const parsed = seconds(take(), flag, flag === '--interval'); + if (typeof parsed === 'string') return { error: parsed, expectHead: opts.expectHead }; + if (flag === '--wait') opts.waitSeconds = parsed; + else if (flag === '--interval') opts.intervalSeconds = parsed; + else opts.registrationWaitSeconds = parsed; + break; + } + default: return { error: `unknown argument ${flag}`, expectHead: opts.expectHead }; + } + } + if (!/^[^/\s]+\/[^/\s]+$/.test(opts.repo)) return { error: '--repo OWNER/NAME is required', expectHead: opts.expectHead }; + if (!/^[1-9]\d*$/.test(opts.pr)) return { error: '--pr NUMBER is required', expectHead: opts.expectHead }; + if (!HEAD_RE.test(opts.expectHead)) return { error: '--expect-head needs a full commit SHA', expectHead: opts.expectHead }; + return opts; +} + +export const spawnGh: GhRunner = args => { + const result = spawnSync('gh', args, { encoding: 'utf8', timeout: 60_000, maxBuffer: 16 * 1024 * 1024 }); + if (result.error) { + const code = (result.error as NodeJS.ErrnoException).code; + return { status: null, stdout: '', stderr: result.stderr ?? '', error: code === 'ENOENT' ? 'gh is not installed or not on PATH' : `gh failed to run (${code ?? result.error.message})` }; + } + return { status: result.status, stdout: result.stdout, stderr: result.stderr }; +}; + +export async function ciGateMain(argv: string[], gh: GhRunner = spawnGh, write: (text: string) => void = text => process.stdout.write(text)): Promise { + const parsed = parseCiGateArgs(argv); + if ('help' in parsed) { + write(`${CI_GATE_USAGE}\n`); + return 0; + } + if ('error' in parsed) { + write(`VERDICT ERROR ${parsed.expectHead || '-'}\nCAUSE ${parsed.error}\nFIX see gstack-ci-gate --help\n`); + return CI_GATE_EXIT.ERROR; + } + const result = await runCiGate(parsed, gh); + write([`VERDICT ${result.verdict} ${result.sha}`, ...result.lines].join('\n') + '\n'); + return CI_GATE_EXIT[result.verdict]; +} diff --git a/scripts/question-registry.ts b/scripts/question-registry.ts index eb1bf0f98..e3fc228c4 100644 --- a/scripts/question-registry.ts +++ b/scripts/question-registry.ts @@ -467,6 +467,22 @@ export const QUESTIONS = { signal_key: 'decision-autonomy', description: "Canary detected regressions — roll back the deploy?", }, + 'land-and-deploy-ci-override': { + id: 'land-and-deploy-ci-override', + skill: 'land-and-deploy', + category: 'approval', + door_type: 'one-way', + options: ['accept', 'reject'], + description: "Merge this head while these named non-required CI checks are red or pending?", + }, + 'land-and-deploy-no-ci-confirm': { + id: 'land-and-deploy-no-ci-confirm', + skill: 'land-and-deploy', + category: 'approval', + door_type: 'one-way', + options: ['accept', 'reject'], + description: "No CI ran on this head — merge it anyway?", + }, // ----------------------------------------------------------------------- // /cso — security audit diff --git a/test/fixtures/ci-gate/no-checks.json b/test/fixtures/ci-gate/no-checks.json new file mode 100644 index 000000000..2a77436f9 --- /dev/null +++ b/test/fixtures/ci-gate/no-checks.json @@ -0,0 +1,14 @@ +{ + "recorded": "gh 2.101.0, garrytan/gstack#3010, 2026-10-02", + "note": "fork PR whose workflows never ran: both queries report no checks", + "required": { + "status": 1, + "stdout": "", + "stderr": "no checks reported on the 'fix/windows-autoplan' branch\n" + }, + "all": { + "status": 1, + "stdout": "", + "stderr": "no checks reported on the 'fix/windows-autoplan' branch\n" + } +} diff --git a/test/fixtures/ci-gate/protected-nonrequired-red.json b/test/fixtures/ci-gate/protected-nonrequired-red.json new file mode 100644 index 000000000..d49827f4a --- /dev/null +++ b/test/fixtures/ci-gate/protected-nonrequired-red.json @@ -0,0 +1,14 @@ +{ + "recorded": "gh 2.101.0, garrytan/gstack#2956, 2026-10-02", + "note": "required free-tests passes; two non-required checks fail", + "required": { + "status": 0, + "stdout": "[{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018433408/job/107698187589\",\"name\":\"free-tests\",\"state\":\"SUCCESS\"}]\n", + "stderr": "" + }, + "all": { + "status": 0, + "stdout": "[{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018432870/job/107705433100\",\"name\":\"slices-comment\",\"state\":\"SUCCESS\"},{\"bucket\":\"fail\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018432870/job/107705239677\",\"name\":\"slices-report\",\"state\":\"FAILURE\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018433408/job/107698187589\",\"name\":\"free-tests\",\"state\":\"SUCCESS\"},{\"bucket\":\"fail\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018432870/job/107697209743\",\"name\":\"eval-slices (5)\",\"state\":\"FAILURE\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018432870/job/107697209626\",\"name\":\"eval-slices (1)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018432870/job/107697209549\",\"name\":\"eval-slices (3)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018432870/job/107697209578\",\"name\":\"eval-slices (2)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018432870/job/107697209506\",\"name\":\"eval-slices (6)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018432870/job/107697209479\",\"name\":\"eval-slices (4)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018433408/job/107696973299\",\"name\":\"free-suite (11)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018433408/job/107696973102\",\"name\":\"free-suite (1)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018433408/job/107696973000\",\"name\":\"free-suite (19)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018433408/job/107696973034\",\"name\":\"free-suite (7)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018433408/job/107696972897\",\"name\":\"free-suite (5)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018433408/job/107696972820\",\"name\":\"free-suite (2)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018433408/job/107696973141\",\"name\":\"free-suite (15)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018433408/job/107696972954\",\"name\":\"free-suite (18)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018433408/job/107696972870\",\"name\":\"free-suite (16)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018433408/job/107696972851\",\"name\":\"free-suite (3)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018433408/job/107696972840\",\"name\":\"free-suite (6)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018433408/job/107696972871\",\"name\":\"free-suite (9)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018433408/job/107696973072\",\"name\":\"free-suite (12)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018433408/job/107696973017\",\"name\":\"free-suite (8)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018433408/job/107696972927\",\"name\":\"free-suite (14)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018433408/job/107696972873\",\"name\":\"free-suite (10)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018433408/job/107696972963\",\"name\":\"free-suite (20)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018432870/job/107696854897\",\"name\":\"build-image\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018433408/job/107696972780\",\"name\":\"free-suite (4)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018433408/job/107696972843\",\"name\":\"free-suite (13)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018432959/job/107696852747\",\"name\":\"actionlint\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018433408/job/107696972884\",\"name\":\"free-suite (17)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018432870/job/107696854436\",\"name\":\"plan-slices\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018433014/job/107696853228\",\"name\":\"quality\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018433160/job/107696855762\",\"name\":\"check-freshness\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018433408/job/107696855422\",\"name\":\"cso-macos-launcher\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018433408/job/107696855378\",\"name\":\"cso-windows-launcher\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018433408/job/107696855530\",\"name\":\"free-plan\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018433408/job/107696854875\",\"name\":\"cso-docker-integration\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36018432506/job/107696851554\",\"name\":\"windows-free-tests\",\"state\":\"SUCCESS\"}]\n", + "stderr": "" + } +} diff --git a/test/fixtures/ci-gate/protected-pass.json b/test/fixtures/ci-gate/protected-pass.json new file mode 100644 index 000000000..04ae5f6d7 --- /dev/null +++ b/test/fixtures/ci-gate/protected-pass.json @@ -0,0 +1,14 @@ +{ + "recorded": "gh 2.101.0, garrytan/gstack#2988, 2026-10-02", + "note": "required free-tests passes; all checks pass or skip", + "required": { + "status": 0, + "stdout": "[{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633770/job/110583563885\",\"name\":\"free-tests\",\"state\":\"SUCCESS\"}]\n", + "stderr": "" + }, + "all": { + "status": 0, + "stdout": "[{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633770/job/110583563885\",\"name\":\"free-tests\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633770/job/110582247757\",\"name\":\"free-suite (21)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633770/job/110582154255\",\"name\":\"cso-windows-launcher\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633059/job/110582346257\",\"name\":\"validate-native (cso-runtime-inputs-2026-09-10, rails-ruby-3.4.4, rails-ruby-3.4.4-arm64, rails, ...\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633059/job/110582345999\",\"name\":\"validate-native (cso-runtime-inputs-2026-09-10, node-24.4.0, node-24.4.0-arm64, node, linux/arm64...\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633770/job/110582247797\",\"name\":\"free-suite (20)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633770/job/110582247546\",\"name\":\"free-suite (5)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633059/job/110582346175\",\"name\":\"validate-native (cso-runtime-inputs-2026-09-10, postgresql-17.2, postgresql-17.2-arm64, postgresq...\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633059/job/110582346096\",\"name\":\"validate-native (cso-runtime-inputs-2026-09-10, bun-1.3.10, bun-1.3.10-arm64, bun, linux/arm64, a...\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633770/job/110582247541\",\"name\":\"free-suite (11)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633770/job/110582247886\",\"name\":\"free-suite (8)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633059/job/110582346456\",\"name\":\"validate-native (cso-runtime-inputs-2026-09-10, python-3.13.4-uv-0.8.0, python-3.13.4-uv-0.8.0-ar...\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633770/job/110582247616\",\"name\":\"free-suite (7)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633059/job/110582346128\",\"name\":\"validate-native (cso-runtime-inputs-2026-09-10, postgresql-17.2, postgresql-17.2-amd64, postgresq...\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633770/job/110582247642\",\"name\":\"free-suite (19)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633770/job/110582247676\",\"name\":\"free-suite (6)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633059/job/110582346229\",\"name\":\"validate-native (cso-runtime-inputs-2026-09-10, rails-ruby-3.4.4, rails-ruby-3.4.4-amd64, rails, ...\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633770/job/110582247385\",\"name\":\"free-suite (1)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633770/job/110582247456\",\"name\":\"free-suite (12)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633059/job/110582346190\",\"name\":\"validate-native (cso-runtime-inputs-2026-09-10, python-3.13.4-uv-0.8.0, python-3.13.4-uv-0.8.0-am...\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633770/job/110582247400\",\"name\":\"free-suite (14)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633059/job/110582346195\",\"name\":\"validate-native (cso-runtime-inputs-2026-09-10, bun-1.3.10, bun-1.3.10-amd64, bun, linux/amd64, a...\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633770/job/110582247704\",\"name\":\"free-suite (17)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633059/job/110582346149\",\"name\":\"validate-native (cso-runtime-inputs-2026-09-10, node-24.4.0, node-24.4.0-amd64, node, linux/amd64...\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633770/job/110582247429\",\"name\":\"free-suite (3)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633770/job/110582247486\",\"name\":\"free-suite (9)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633770/job/110582247427\",\"name\":\"free-suite (16)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633770/job/110582247415\",\"name\":\"free-suite (13)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633770/job/110582247316\",\"name\":\"free-suite (4)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633770/job/110582247405\",\"name\":\"free-suite (18)\",\"state\":\"SUCCESS\"},{\"bucket\":\"skipping\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633059/job/110582348208\",\"name\":\"qualify-native\",\"state\":\"SKIPPED\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633770/job/110582247447\",\"name\":\"free-suite (15)\",\"state\":\"SUCCESS\"},{\"bucket\":\"skipping\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633059/job/110582348702\",\"name\":\"stage\",\"state\":\"SKIPPED\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633770/job/110582247460\",\"name\":\"free-suite (10)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633294/job/110582150127\",\"name\":\"check-freshness\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633770/job/110582247431\",\"name\":\"free-suite (2)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925632769/job/110582149352\",\"name\":\"actionlint\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633059/job/110582149719\",\"name\":\"reviewed-inputs\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633770/job/110582154562\",\"name\":\"cso-macos-launcher\",\"state\":\"SUCCESS\"},{\"bucket\":\"skipping\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925632721/job/110582189042\",\"name\":\"eval-slices\",\"state\":\"SKIPPED\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633770/job/110582154686\",\"name\":\"free-plan\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633770/job/110582154462\",\"name\":\"cso-docker-integration\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633770/job/110582154692\",\"name\":\"typecheck\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925632799/job/110582149617\",\"name\":\"windows-free-tests\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925633002/job/110582149353\",\"name\":\"quality\",\"state\":\"SUCCESS\"},{\"bucket\":\"skipping\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925632721/job/110582153468\",\"name\":\"slices-comment\",\"state\":\"SKIPPED\"},{\"bucket\":\"skipping\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925632721/job/110582151697\",\"name\":\"build-image\",\"state\":\"SKIPPED\"},{\"bucket\":\"skipping\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925632799/job/110582151047\",\"name\":\"cookie-native-qualification\",\"state\":\"SKIPPED\"},{\"bucket\":\"skipping\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925632721/job/110582152960\",\"name\":\"slices-report\",\"state\":\"SKIPPED\"},{\"bucket\":\"skipping\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925632721/job/110582151379\",\"name\":\"plan-slices\",\"state\":\"SKIPPED\"},{\"bucket\":\"skipping\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36925632799/job/110582151987\",\"name\":\"dia-native-qualification\",\"state\":\"SKIPPED\"}]\n", + "stderr": "" + } +} diff --git a/test/fixtures/ci-gate/protected-required-red.json b/test/fixtures/ci-gate/protected-required-red.json new file mode 100644 index 000000000..41deb2bc8 --- /dev/null +++ b/test/fixtures/ci-gate/protected-required-red.json @@ -0,0 +1,14 @@ +{ + "recorded": "gh 2.101.0, garrytan/gstack#3005, 2026-10-02", + "note": "required free-tests fails", + "required": { + "status": 0, + "stdout": "[{\"bucket\":\"fail\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36731112282/job/109942442346\",\"name\":\"free-tests\",\"state\":\"FAILURE\"}]\n", + "stderr": "" + }, + "all": { + "status": 0, + "stdout": "[{\"bucket\":\"fail\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36731112282/job/109942442346\",\"name\":\"free-tests\",\"state\":\"FAILURE\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36731112282/job/109940769622\",\"name\":\"free-suite (21)\",\"state\":\"SUCCESS\"},{\"bucket\":\"fail\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36731112282/job/109940769405\",\"name\":\"free-suite (1)\",\"state\":\"FAILURE\"},{\"bucket\":\"fail\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36731112282/job/109940769674\",\"name\":\"free-suite (13)\",\"state\":\"FAILURE\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36731112282/job/109940769647\",\"name\":\"free-suite (20)\",\"state\":\"SUCCESS\"},{\"bucket\":\"fail\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36731112282/job/109940769835\",\"name\":\"free-suite (3)\",\"state\":\"FAILURE\"},{\"bucket\":\"fail\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36731112282/job/109940769784\",\"name\":\"free-suite (18)\",\"state\":\"FAILURE\"},{\"bucket\":\"fail\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36731112282/job/109940769605\",\"name\":\"free-suite (7)\",\"state\":\"FAILURE\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36731112282/job/109940769847\",\"name\":\"free-suite (4)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36731112282/job/109940769775\",\"name\":\"free-suite (10)\",\"state\":\"SUCCESS\"},{\"bucket\":\"fail\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36731112282/job/109940769659\",\"name\":\"free-suite (15)\",\"state\":\"FAILURE\"},{\"bucket\":\"fail\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36731112282/job/109940769687\",\"name\":\"free-suite (11)\",\"state\":\"FAILURE\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36731112282/job/109940769467\",\"name\":\"free-suite (2)\",\"state\":\"SUCCESS\"},{\"bucket\":\"fail\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36731112282/job/109940769597\",\"name\":\"free-suite (14)\",\"state\":\"FAILURE\"},{\"bucket\":\"fail\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36731112282/job/109940769569\",\"name\":\"free-suite (9)\",\"state\":\"FAILURE\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36731112282/job/109940769539\",\"name\":\"free-suite (16)\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36731112282/job/109940769507\",\"name\":\"free-suite (8)\",\"state\":\"SUCCESS\"},{\"bucket\":\"fail\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36731112282/job/109940769577\",\"name\":\"free-suite (12)\",\"state\":\"FAILURE\"},{\"bucket\":\"fail\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36731112282/job/109940769390\",\"name\":\"free-suite (17)\",\"state\":\"FAILURE\"},{\"bucket\":\"fail\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36731112282/job/109940769541\",\"name\":\"free-suite (5)\",\"state\":\"FAILURE\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36731112282/job/109940769237\",\"name\":\"free-suite (6)\",\"state\":\"SUCCESS\"},{\"bucket\":\"fail\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36731112282/job/109940769331\",\"name\":\"free-suite (19)\",\"state\":\"FAILURE\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36731112282/job/109940659703\",\"name\":\"cso-macos-launcher\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36731112290/job/109940656471\",\"name\":\"Sync PR title to VERSION\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36731112282/job/109940659738\",\"name\":\"cso-docker-integration\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36731112282/job/109940659524\",\"name\":\"cso-windows-launcher\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36731112282/job/109940659062\",\"name\":\"typecheck\",\"state\":\"SUCCESS\"},{\"bucket\":\"pass\",\"link\":\"https://github.com/garrytan/gstack/actions/runs/36731112282/job/109940659494\",\"name\":\"free-plan\",\"state\":\"SUCCESS\"}]\n", + "stderr": "" + } +} diff --git a/test/gstack-ci-gate.test.ts b/test/gstack-ci-gate.test.ts new file mode 100644 index 000000000..e1cd8bb38 --- /dev/null +++ b/test/gstack-ci-gate.test.ts @@ -0,0 +1,306 @@ +/** + * lib/ci-gate.ts — the /land-and-deploy merge gate (#2995). + * + * Fixtures under test/fixtures/ci-gate/ are real `gh pr checks --json` output + * (gh 2.101.0 against garrytan/gstack PRs); the remaining cases are synthetic + * responses in the same shape. gh exits 0 with --json even when checks are red + * or pending, so the verdict must come from `bucket`, never the exit status. + */ +import { describe, expect, test } from 'bun:test'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { ciGateMain, evaluateChecks, parseCiGateArgs, runCiGate, type CiGateOptions, type GhResult, type GhRunner } from '../lib/ci-gate'; + +const HEAD = 'a'.repeat(40); +const OTHER = 'b'.repeat(40); +const fixture = (name: string): { required: GhResult; all: GhResult } => + JSON.parse(readFileSync(join(import.meta.dir, 'fixtures', 'ci-gate', `${name}.json`), 'utf8')); +const ok = (rows: unknown): GhResult => ({ status: 0, stdout: JSON.stringify(rows), stderr: '' }); +const err = (stderr: string, status = 1): GhResult => ({ status, stdout: '', stderr: `${stderr}\n` }); +const row = (name: string, bucket?: string) => ({ name, state: 'X', link: `https://ci.example/${encodeURIComponent(name)}`, ...(bucket === undefined ? {} : { bucket }) }); +const NONE = err("no checks reported on the 'feature' branch"); +const NONE_REQUIRED = err("no required checks reported on the 'feature' branch"); + +interface Script { heads?: string[]; required?: GhResult[]; all?: GhResult[]; api?: GhResult } +function fakeGh(script: Script): GhRunner & { calls: string[][] } { + const queues = { head: script.heads ?? [HEAD], required: script.required ?? [], all: script.all ?? [] }; + const next = (queue: T[]): T => (queue.length > 1 ? queue.shift()! : queue[0]); + const calls: string[][] = []; + const gh = ((args: string[]) => { + calls.push(args); + if (args[0] === 'api') return script.api ?? err('gh: Not Found (HTTP 404)'); + if (args[1] === 'view') return { status: 0, stdout: `${next(queues.head)}\n`, stderr: '' }; + if (args[1] === 'merge') throw new Error('the gate must never merge'); + return next(args.includes('--required') ? queues.required : queues.all); + }) as GhRunner & { calls: string[][] }; + gh.calls = calls; + return gh; +} +const options = (extra: Partial = {}): CiGateOptions => ({ + repo: 'owner/project', pr: '42', expectHead: HEAD, excludes: [], waitSeconds: 0, intervalSeconds: 30, registrationWaitSeconds: 0, ...extra, +}); +const noSleep = async () => {}; +const gate = (script: Script, extra: Partial = {}) => runCiGate(options(extra), fakeGh(script), noSleep); +const checkLines = (lines: string[]) => lines.filter(line => line.startsWith('CHECK\t')); + +describe('verdicts on recorded gh output', () => { + test('protected repo, required and all checks green or skipped → PASS', async () => { + const { required, all } = fixture('protected-pass'); + expect(JSON.parse(all.stdout).some((c: { bucket: string }) => c.bucket === 'skipping')).toBe(true); + const result = await gate({ required: [required], all: [all] }); + expect(result.verdict).toBe('PASS'); + expect(checkLines(result.lines)).toEqual([]); + }); + + test('required check red → FAIL naming it as required', async () => { + const { required, all } = fixture('protected-required-red'); + const result = await gate({ required: [required], all: [all] }); + expect(result.verdict).toBe('FAIL'); + expect(checkLines(result.lines)).toContainEqual(expect.stringMatching(/^CHECK\tfree-tests\tfail\trequired=y\thttps:/)); + expect(checkLines(result.lines).filter(line => line.includes('required=n')).length).toBeGreaterThan(0); + }); + + test('required green but non-required red → FAIL (strict all-checks default)', async () => { + const { required, all } = fixture('protected-nonrequired-red'); + expect(required.status).toBe(0); + const result = await gate({ required: [required], all: [all] }); + expect(result.verdict).toBe('FAIL'); + expect(checkLines(result.lines).map(line => line.split('\t').slice(1, 4))).toEqual([ + ['eval-slices (5)', 'fail', 'required=n'], + ['slices-report', 'fail', 'required=n'], + ]); + }); + + test('no CI on the head at all → NO_CHECKS with SHA and workflow presence', async () => { + const { required, all } = fixture('no-checks'); + const result = await gate({ required: [required], all: [all], api: ok(2) }); + expect(result.verdict).toBe('NO_CHECKS'); + expect(result.sha).toBe(HEAD); + expect(result.lines).toContain('NOTE .github/workflows exists at this head'); + expect(result.lines.some(line => line.includes(`no CI ran on ${HEAD}`))).toBe(true); + const absent = await gate({ required: [required], all: [all] }); + expect(absent.lines).toContain('NOTE .github/workflows does not exist at this head'); + }); +}); + +describe('verdicts on synthetic gh output', () => { + test('unprotected repo: none required, all green → PASS; one red → FAIL (#2995)', async () => { + expect((await gate({ required: [NONE_REQUIRED], all: [ok([row('build', 'pass')])] })).verdict).toBe('PASS'); + const red = await gate({ required: [NONE_REQUIRED], all: [ok([row('build', 'pass'), row('test', 'fail')])] }); + expect(red.verdict).toBe('FAIL'); + expect(red.lines).toContain('NOTE no required checks configured; gating on all checks'); + }); + + test('unprotected repo with a pending check → PENDING, never a pass', async () => { + const result = await gate({ required: [NONE_REQUIRED], all: [ok([row('build', 'pass'), row('test', 'pending')])] }); + expect(result.verdict).toBe('PENDING'); + expect(checkLines(result.lines)).toEqual([`CHECK\ttest\tpending\trequired=n\thttps://ci.example/test`]); + }); + + test('403 protection metadata → gate on all checks with required=?', async () => { + const forbidden = err('GraphQL: Resource not accessible by integration (HTTP 403)'); + const result = await gate({ required: [forbidden], all: [ok([row('build', 'pass'), row('lint', 'fail')])] }); + expect(result.verdict).toBe('FAIL'); + expect(checkLines(result.lines)).toEqual([`CHECK\tlint\tfail\trequired=?\thttps://ci.example/lint`]); + expect(result.lines.some(line => line.startsWith('NOTE required-check metadata unavailable') && line.includes('HTTP 403'))).toBe(true); + expect((await gate({ required: [forbidden], all: [ok([row('build', 'pass')])] })).verdict).toBe('PASS'); + }); + + test('pass buckets are a whitelist; cancel and unknown buckets fail', async () => { + for (const bucket of ['pass', 'skipping', 'neutral']) { + expect((await gate({ required: [NONE_REQUIRED], all: [ok([row('x', bucket)])] })).verdict).toBe('PASS'); + } + for (const bucket of ['cancel', 'fail', 'stale', 'action_required', '']) { + expect((await gate({ required: [NONE_REQUIRED], all: [ok([row('x', bucket)])] })).verdict).toBe('FAIL'); + } + }); + + test('fail outranks pending', async () => { + expect((await gate({ required: [NONE_REQUIRED], all: [ok([row('a', 'pending'), row('b', 'cancel')])] })).verdict).toBe('FAIL'); + }); + + test('missing bucket (old gh), malformed JSON, auth failure and unknown errors → ERROR', async () => { + const cases: GhResult[] = [ + ok([{ name: 'build', state: 'SUCCESS', link: '' }]), + { status: 0, stdout: 'not json', stderr: '' }, + ok({ name: 'build', bucket: 'pass' }), + ok([{ bucket: 'pass' }]), + err('HTTP 401: Bad credentials (https://api.github.com/graphql)'), + err('unknown flag: --json'), + err('no commit found on the pull request'), + { status: null, stdout: '', stderr: '', error: 'gh is not installed or not on PATH' }, + ]; + for (const all of cases) { + const result = await gate({ required: [NONE_REQUIRED], all: [all] }); + expect(result.verdict).toBe('ERROR'); + expect(result.lines.some(line => line.startsWith('CAUSE '))).toBe(true); + expect(result.lines.some(line => line.includes('gh auth status') && line.includes('2.50.0'))).toBe(true); + } + const auth = await gate({ required: [NONE_REQUIRED], all: [err('HTTP 401: Bad credentials')] }); + expect(auth.lines).toContain('STDERR HTTP 401: Bad credentials'); + }); + + test('a nonzero exit that still carries valid JSON is judged by its buckets', async () => { + const legacy = { status: 8, stdout: JSON.stringify([row('build', 'pending')]), stderr: '' }; + expect((await gate({ required: [NONE_REQUIRED], all: [legacy] })).verdict).toBe('PENDING'); + }); + + test('worst observation wins when the two reads disagree', () => { + const result = evaluateChecks(ok([row('build', 'pending')]), ok([row('build', 'pass')]), []); + expect(result.verdict).toBe('PENDING'); + expect(result.verdict !== 'ERROR' && result.lines[0]).toBe('CHECK\tbuild\tpending\trequired=y\thttps://ci.example/build'); + }); + + test('checks that register between the two reads have unknown required status', () => { + const result = evaluateChecks(NONE, ok([row('build', 'fail')]), ['build']); + expect(result.verdict).toBe('FAIL'); + expect(result.verdict !== 'ERROR' && result.lines[0]).toContain('required=?'); + }); +}); + +describe('head binding', () => { + test('a head other than --expect-head is ERROR before any check is read', async () => { + const gh = fakeGh({ heads: [OTHER], required: [NONE_REQUIRED], all: [ok([row('b', 'pass')])] }); + const result = await runCiGate(options(), gh, noSleep); + expect(result.verdict).toBe('ERROR'); + expect(result.sha).toBe(HEAD); + expect(result.lines).toContain(`HEAD ${OTHER}`); + expect(gh.calls.some(args => args[1] === 'checks')).toBe(false); + }); + + test('a head that moves while checks are read is ERROR even when checks pass', async () => { + const result = await gate({ heads: [HEAD, OTHER], required: [NONE_REQUIRED], all: [ok([row('b', 'pass')])] }); + expect(result.verdict).toBe('ERROR'); + expect(result.lines).toContain('CAUSE the PR head moved while checks were being read'); + }); + + test('checks are read with and without --required, for the pinned PR and repo', async () => { + const gh = fakeGh({ required: [NONE_REQUIRED], all: [ok([row('b', 'pass')])] }); + await runCiGate(options(), gh, noSleep); + expect(gh.calls).toEqual([ + ['pr', 'view', '42', '--repo', 'owner/project', '--json', 'headRefOid', '--jq', '.headRefOid'], + ['pr', 'checks', '42', '--repo', 'owner/project', '--required', '--json', 'name,state,bucket,link'], + ['pr', 'checks', '42', '--repo', 'owner/project', '--json', 'name,state,bucket,link'], + ['pr', 'view', '42', '--repo', 'owner/project', '--json', 'headRefOid', '--jq', '.headRefOid'], + ]); + }); +}); + +describe('per-head, per-check overrides', () => { + const nonRequiredRed = { required: [ok([row('build', 'pass')])], all: [ok([row('build', 'pass'), row('lint', 'fail'), row('e2e', 'pending')])] }; + + test('excluding every named non-required failure for this head → PASS, still listed', async () => { + const result = await gate(nonRequiredRed, { excludes: ['lint', 'e2e'], overrideHead: HEAD }); + expect(result.verdict).toBe('PASS'); + expect(checkLines(result.lines).every(line => line.endsWith('\texcluded'))).toBe(true); + expect((await gate(nonRequiredRed, { excludes: ['lint'], overrideHead: HEAD })).verdict).toBe('PENDING'); + }); + + test('overrides approved for another head, or without a head, are ERROR', async () => { + for (const overrideHead of [OTHER, undefined]) { + const result = await gate(nonRequiredRed, { excludes: ['lint', 'e2e'], overrideHead }); + expect(result.verdict).toBe('ERROR'); + expect(result.lines[0]).toBe('CAUSE check overrides were not approved for this head'); + } + }); + + test('required and unknown-required checks can never be excluded', async () => { + const requiredRed = await gate({ required: [ok([row('build', 'fail')])], all: [ok([row('build', 'fail')])] }, { excludes: ['build'], overrideHead: HEAD }); + expect(requiredRed.verdict).toBe('FAIL'); + expect(checkLines(requiredRed.lines)[0]).toEndWith('\toverride-refused'); + const unknown = await gate({ required: [err('HTTP 403')], all: [ok([row('lint', 'fail')])] }, { excludes: ['lint'], overrideHead: HEAD }); + expect(unknown.verdict).toBe('FAIL'); + }); + + test('an override naming no existing check is reported, not silently applied', async () => { + const result = await gate(nonRequiredRed, { excludes: ['lint', 'e2e', 'ghost'], overrideHead: HEAD }); + expect(result.lines).toContain('NOTE override for unknown check "ghost" ignored'); + }); +}); + +describe('polling', () => { + const clock = () => { + let t = 0; + return { now: () => t, sleep: async (ms: number) => { t += ms; } }; + }; + + test('delayed registration: zero checks re-polled until checks appear', async () => { + const c = clock(); + const gh = fakeGh({ required: [NONE, NONE, NONE_REQUIRED], all: [NONE, NONE, ok([row('build', 'pending')])] }); + const result = await runCiGate(options({ registrationWaitSeconds: 60 }), gh, c.sleep, c.now); + expect(result.verdict).toBe('PENDING'); + expect(c.now()).toBe(20_000); + }); + + test('registration wait is bounded at 60s, then NO_CHECKS', async () => { + const c = clock(); + const result = await runCiGate(options({ registrationWaitSeconds: 60 }), fakeGh({ required: [NONE], all: [NONE] }), c.sleep, c.now); + expect(result.verdict).toBe('NO_CHECKS'); + expect(c.now()).toBe(60_000); + }); + + test('--wait polls PENDING until a final verdict', async () => { + const c = clock(); + const all = [ok([row('t', 'pending')]), ok([row('t', 'pending')]), ok([row('t', 'pass')])]; + const result = await runCiGate(options({ waitSeconds: 240 }), fakeGh({ required: [NONE_REQUIRED], all }), c.sleep, c.now); + expect(result.verdict).toBe('PASS'); + expect(c.now()).toBe(60_000); + }); + + test('--wait timeout reports PENDING and names the still-pending checks', async () => { + const c = clock(); + const all = [ok([row('build', 'pass'), row('deploy-preview', 'pending')])]; + const result = await runCiGate(options({ waitSeconds: 240 }), fakeGh({ required: [NONE_REQUIRED], all }), c.sleep, c.now); + expect(result.verdict).toBe('PENDING'); + expect(checkLines(result.lines)).toEqual(['CHECK\tdeploy-preview\tpending\trequired=n\thttps://ci.example/deploy-preview']); + expect(result.lines.some(line => line.startsWith('NOTE still pending after waiting 240s'))).toBe(true); + }); + + test('a failure during the wait stops immediately (fail fast)', async () => { + const c = clock(); + const all = [ok([row('t', 'pending')]), ok([row('t', 'fail'), row('u', 'pending')])]; + const result = await runCiGate(options({ waitSeconds: 240 }), fakeGh({ required: [NONE_REQUIRED], all }), c.sleep, c.now); + expect(result.verdict).toBe('FAIL'); + expect(c.now()).toBe(30_000); + }); +}); + +describe('CLI contract', () => { + const run = async (argv: string[], gh: GhRunner = fakeGh({ required: [NONE_REQUIRED], all: [ok([row('b', 'pass')])] })) => { + let out = ''; + const code = await ciGateMain(argv, gh, text => { out += text; }); + return { code, out }; + }; + const base = ['--repo', 'owner/project', '--pr', '42', '--expect-head', HEAD, '--registration-wait', '0']; + + test('VERDICT is the first line and exit codes are 0/1/2/3/4', async () => { + const cases: Array<[Script, string, number]> = [ + [{ required: [NONE_REQUIRED], all: [ok([row('b', 'pass')])] }, 'PASS', 0], + [{ required: [NONE_REQUIRED], all: [ok([row('b', 'fail')])] }, 'FAIL', 1], + [{ required: [NONE_REQUIRED], all: [ok([row('b', 'pending')])] }, 'PENDING', 2], + [{ required: [NONE], all: [NONE] }, 'NO_CHECKS', 3], + [{ required: [NONE_REQUIRED], all: [err('HTTP 502')] }, 'ERROR', 4], + ]; + for (const [script, verdict, code] of cases) { + const result = await run(base, fakeGh(script)); + expect(result.out.split('\n')[0]).toBe(`VERDICT ${verdict} ${HEAD}`); + expect(result.code).toBe(code); + } + }); + + test('--help documents the verdicts and exit codes', async () => { + const result = await run(['--help']); + expect(result.code).toBe(0); + expect(result.out).toContain('Exit codes: 0 PASS, 1 FAIL, 2 PENDING, 3 NO_CHECKS, 4 ERROR.'); + expect(result.out).toContain('--expect-head'); + }); + + test('bad arguments are ERROR, never a pass', async () => { + for (const argv of [[], ['--repo', 'x', '--pr', '1', '--expect-head', HEAD], [...base, '--interval', '0'], [...base, '--bogus'], [...base.slice(0, 4), '--expect-head', 'abc']]) { + const result = await run(argv); + expect(result.code).toBe(4); + expect(result.out).toStartWith('VERDICT ERROR '); + } + expect(parseCiGateArgs([...base, '--exclude', 'a b', '--override-head', HEAD])).toMatchObject({ excludes: ['a b'], overrideHead: HEAD }); + }); +}); diff --git a/test/land-and-deploy-ci-gate.test.ts b/test/land-and-deploy-ci-gate.test.ts new file mode 100644 index 000000000..0f6a290f6 --- /dev/null +++ b/test/land-and-deploy-ci-gate.test.ts @@ -0,0 +1,217 @@ +/** + * /land-and-deploy merge gate journeys (#2995): v1.90.2.0 gated only on + * `gh pr checks --required`, so repos without required checks merged over red + * or pending CI. These tests run the template's own merge blocks with the real + * bin/gstack-ci-gate against a fake `gh` that records every `gh pr merge`. + */ +import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; +import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { getQuestion } from '../scripts/question-registry'; + +const ROOT = join(import.meta.dir, '..'); +const GATE_BIN = join(ROOT, 'bin', 'gstack-ci-gate'); +const source = (name: string) => readFileSync(join(ROOT, 'land-and-deploy', name), 'utf8'); +const HEAD = 'a'.repeat(40); +const OTHER = 'b'.repeat(40); + +const FAKE_GH = `#!/usr/bin/env bash +D="$FAKE_GH_DIR" +if [ "$1" = api ]; then kind=api +elif [ "$1 $2" = "pr merge" ]; then printf '%s\\n' "$*" >> "$D/merges.log"; exit 0 +elif [ "$1 $2" = "pr view" ]; then kind=head +elif [ "$1 $2" = "pr checks" ]; then case " $* " in *" --required "*) kind=required ;; *) kind=all ;; esac +else echo "fake gh: unexpected $*" >&2; exit 97 +fi +n=$(( $(cat "$D/count.$kind" 2>/dev/null || echo 0) + 1 )); echo "$n" > "$D/count.$kind" +f="$D/$kind.$n"; [ -e "$f.code" ] || f="$D/$kind" +[ -e "$f.out" ] && cat "$f.out" +[ -e "$f.err" ] && cat "$f.err" >&2 +exit "$(cat "$f.code" 2>/dev/null || echo 1)" +`; + +type Reply = { out?: string; err?: string; code?: number }; +const checks = (...rows: Array<[string, string]>): Reply => ({ out: JSON.stringify(rows.map(([name, bucket]) => ({ name, bucket, state: 'X', link: `https://ci.example/${name}` }))), code: 0 }); +const NONE: Reply = { err: "no checks reported on the 'feature' branch", code: 1 }; +const NONE_REQUIRED: Reply = { err: "no required checks reported on the 'feature' branch", code: 1 }; +const head = (sha: string): Reply => ({ out: `${sha}\n`, code: 0 }); + +/** The two fenced blocks in Step 4 that run `gh pr merge`, keyed by mode. */ +function mergeBlocks(): Record<'auto' | 'direct', string> { + const blocks = [...source('sections/merge-and-deploy.md.tmpl').matchAll(/```bash\n([\s\S]*?)\n```/g)] + .map(match => match[1]) + .filter(block => /\bgh pr merge /.test(block)); + expect(blocks).toHaveLength(2); + const auto = blocks.find(block => block.includes('--auto')); + const direct = blocks.find(block => !block.includes('--auto')); + expect(auto && direct).toBeTruthy(); + return { auto: auto!, direct: direct! }; +} + +let dir: string; +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'ci-gate-journey-')); + mkdirSync(join(dir, 'bin')); + writeFileSync(join(dir, 'bin', 'gh'), FAKE_GH); + chmodSync(join(dir, 'bin', 'gh'), 0o755); +}); +afterEach(() => rmSync(dir, { recursive: true, force: true })); + +interface Journey { + heads?: Reply[]; required: Reply | Reply[]; all: Reply | Reply[]; api?: Reply; + mode: 'auto' | 'direct'; noCiApprovedHead?: string; override?: string; +} +function runJourney(j: Journey) { + const replies: Record = { head: j.heads ?? [head(HEAD)], required: j.required, all: j.all, api: j.api ?? { err: 'HTTP 404', code: 1 } }; + for (const [kind, reply] of Object.entries(replies)) { + const list = Array.isArray(reply) ? reply : [reply!]; + list.forEach((r, i) => { + const stem = join(dir, i === list.length - 1 ? kind : `${kind}.${i + 1}`); + writeFileSync(`${stem}.out`, r.out ?? ''); + writeFileSync(`${stem}.err`, r.err ?? ''); + writeFileSync(`${stem}.code`, String(r.code ?? 0)); + }); + } + const block = mergeBlocks()[j.mode].replaceAll('~/.claude/skills/gstack/bin/gstack-ci-gate', GATE_BIN); + const script = `${j.override ? `CI_OVERRIDE=(${j.override})\n` : ''}${block}`; + const result = spawnSync('bash', ['-c', script], { + encoding: 'utf8', timeout: 30_000, + env: { + PATH: `${join(dir, 'bin')}:${process.env.PATH}`, HOME: dir, FAKE_GH_DIR: dir, + REPO: 'owner/project', PR_NUMBER: '42', PR_HEAD: HEAD, MERGE_FLAG: '--squash', + NO_CI_APPROVED_HEAD: j.noCiApprovedHead ?? '', + }, + }); + const log = join(dir, 'merges.log'); + const merges = existsSync(log) ? readFileSync(log, 'utf8').trim().split('\n').filter(Boolean) : []; + for (const name of readdirSync(dir).filter(name => name.startsWith('count.'))) rmSync(join(dir, name)); + rmSync(log, { force: true }); + return { ...result, merges }; +} + +describe('merge blocks never merge without a PASS for the pinned head', () => { + const blocked: Array<[string, Omit]> = [ + ['pending check, no required checks configured', { required: NONE_REQUIRED, all: checks(['build', 'pass'], ['test', 'pending']) }], + ['failing check, no required checks configured', { required: NONE_REQUIRED, all: checks(['build', 'fail']) }], + ['failing required check', { required: checks(['ci', 'fail']), all: checks(['ci', 'fail']) }], + ['cancelled check', { required: NONE_REQUIRED, all: checks(['build', 'cancel']) }], + ['unknown bucket', { required: NONE_REQUIRED, all: checks(['build', 'stale']) }], + ['no checks and no explicit no-CI approval', { required: NONE, all: NONE }], + ['no-CI approval recorded for another head', { required: NONE, all: NONE, noCiApprovedHead: OTHER }], + ['malformed checks JSON', { required: NONE_REQUIRED, all: { out: '{"oops"', code: 0 } }], + ['check without a bucket field (old gh)', { required: NONE_REQUIRED, all: { out: '[{"name":"build","state":"SUCCESS"}]', code: 0 } }], + ['gh auth failure', { required: { err: 'HTTP 401: Bad credentials', code: 1 }, all: { err: 'HTTP 401: Bad credentials', code: 1 } }], + ['checks registered after the no-CI approval', { required: NONE_REQUIRED, all: checks(['build', 'pending']), noCiApprovedHead: HEAD }], + ['head moved mid-gate', { heads: [head(HEAD), head(OTHER)], required: NONE_REQUIRED, all: checks(['build', 'pass']) }], + ['head already moved', { heads: [head(OTHER)], required: NONE_REQUIRED, all: checks(['build', 'pass']) }], + ['required green, non-required pending', { required: checks(['ci', 'pass']), all: checks(['ci', 'pass'], ['preview', 'pending']) }], + ['required green, non-required red', { required: checks(['ci', 'pass']), all: checks(['ci', 'pass'], ['lint', 'fail']) }], + ['override approved for another head', { required: checks(['ci', 'pass']), all: checks(['ci', 'pass'], ['lint', 'fail']), override: `--override-head ${OTHER} --exclude lint` }], + ['override naming a required check', { required: checks(['ci', 'fail']), all: checks(['ci', 'fail']), override: `--override-head ${HEAD} --exclude ci` }], + ]; + for (const mode of ['auto', 'direct'] as const) { + for (const [name, journey] of blocked) { + test(`${mode}: ${name} → zero merges`, () => { + const result = runJourney({ ...journey, mode }); + expect(result.merges).toEqual([]); + expect(result.status).not.toBe(0); + expect(result.stdout).toMatch(/^VERDICT (FAIL|PENDING|NO_CHECKS|ERROR) /); + }); + } + } +}); + +describe('positive controls: the same blocks do merge when the gate passes', () => { + for (const mode of ['auto', 'direct'] as const) { + test(`${mode}: all green → exactly one merge pinned to the head`, () => { + const result = runJourney({ mode, required: checks(['ci', 'pass']), all: checks(['ci', 'pass'], ['docs', 'skipping']) }); + expect(result.status).toBe(0); + expect(result.merges).toHaveLength(1); + expect(result.merges[0]).toContain(`--match-head-commit ${HEAD}`); + expect(result.merges[0].includes('--auto')).toBe(mode === 'auto'); + }); + + test(`${mode}: explicit per-head override of named non-required checks → one merge`, () => { + const result = runJourney({ + mode, required: checks(['ci', 'pass']), all: checks(['ci', 'pass'], ['lint check', 'fail']), + override: `--override-head "$PR_HEAD" --exclude "lint check"`, + }); + expect(result.stdout).toContain('\texcluded'); + expect(result.merges).toHaveLength(1); + }); + + test(`${mode}: no CI on the head, explicitly confirmed for this head → one merge`, () => { + const result = runJourney({ mode, required: NONE, all: NONE, noCiApprovedHead: HEAD }); + expect(result.stdout).toStartWith(`VERDICT NO_CHECKS ${HEAD}`); + expect(result.merges).toHaveLength(1); + }); + } +}); + +describe('template wiring', () => { + const main = source('SKILL.md.tmpl'); + const step = (title: string, next: string) => main.slice(main.indexOf(title), main.indexOf(next)); + + test('Step 2 and the Step 3 CI wait call the helper, not a --required-only query', () => { + const step2 = step('## Step 2: Pre-merge checks', '## Step 3: Wait for CI'); + const step3 = step('## Step 3: Wait for CI', '## Step 3.4'); + expect(step2).toContain('bin/gstack-ci-gate --repo "$REPO" --pr "$PR_NUMBER" --expect-head "$PR_HEAD"'); + expect(step3).toMatch(/bin\/gstack-ci-gate [^\n]*--expect-head "\$PR_HEAD" --wait \d+/); + expect(`${step2}${step3}`).not.toMatch(/gh pr checks[^\n]*--required/); + expect(step3).not.toContain('--watch'); + expect(step3).toContain('still-pending checks'); + expect(step2).toContain('never on the exit code'); + }); + + test('both merge commands, including the direct fallback, run only after the gate', () => { + for (const block of Object.values(mergeBlocks())) { + const gate = block.indexOf('bin/gstack-ci-gate'); + expect(gate).toBeGreaterThanOrEqual(0); + expect(gate).toBeLessThan(block.indexOf('gh pr merge')); + expect(block).toContain('"VERDICT PASS $PR_HEAD"|"VERDICT NO_CHECKS $NO_CI_APPROVED_HEAD") ;;'); + expect(block).toContain('*) exit 1 ;;'); + } + expect(source('sections/merge-and-deploy.md.tmpl')).toMatch(/Only\s+branch\s+protection\s+or\s+a\s+merge\s+queue\s+makes\s+check-to-merge\s+atomic/); + }); + + test('readiness report carries a CI row and one-way override questions', () => { + const readiness = source('sections/readiness-gate.md.tmpl'); + expect(readiness).toContain('CI (head )'); + expect(readiness).toContain('land-and-deploy-ci-override'); + expect(readiness).toContain('land-and-deploy-no-ci-confirm'); + expect(readiness).toContain('never stored or reused'); + }); + + test('generated skill files carry the helper calls', () => { + expect(source('SKILL.md')).toContain('bin/gstack-ci-gate --repo "$REPO" --pr "$PR_NUMBER" --expect-head "$PR_HEAD"'); + expect(source('sections/merge-and-deploy.md').match(/bin\/gstack-ci-gate /g)).toHaveLength(2); + expect(source('sections/readiness-gate.md')).toContain('CI (head )'); + }); +}); + +describe('CI override questions are one-way doors', () => { + const ids = ['land-and-deploy-ci-override', 'land-and-deploy-no-ci-confirm']; + const PREF = join(ROOT, 'bin', 'gstack-question-preference'); + const pref = (...args: string[]) => spawnSync(PREF, args, { encoding: 'utf8', cwd: ROOT, timeout: 30_000, env: { ...process.env, GSTACK_HOME: dir } }); + + test('registered as one-way', () => { + for (const id of ids) expect(getQuestion(id)).toMatchObject({ skill: 'land-and-deploy', door_type: 'one-way' }); + }); + + test('a never-ask preference cannot be written or auto-decide the merge over red', () => { + for (const id of ids) { + const write = pref('--write', JSON.stringify({ question_id: id, preference: 'never-ask', source: 'plan-tune' })); + expect(write.status).toBe(1); + expect(write.stderr).toContain('(door_type: one-way)'); + pref('--read'); + const project = readdirSync(join(dir, 'projects'))[0]; + const file = join(dir, 'projects', project, 'question-preferences.json'); + writeFileSync(file, JSON.stringify({ ...JSON.parse(readFileSync(file, 'utf8')), [id]: 'never-ask' })); + const check = pref('--check', id); + expect(check.stdout).toContain('ASK_NORMALLY'); + expect(check.stdout).not.toContain('AUTO_DECIDE'); + } + }); +});