From 2dcbe9dbbdfb06f979bca57c4f603872fc2fd72e Mon Sep 17 00:00:00 2001 From: garrytan Date: Tue, 3 Nov 2026 02:51:14 +0000 Subject: [PATCH] feat(release-gate): gate main every 8 hours and feed the stable release (inert until adopted) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- .github/workflows/evals-main-backstop.yml | 8 + .github/workflows/evals.yml | 16 +- .github/workflows/release-gate.yml | 118 +++++++ bin/gstack-release-gate | 62 ++++ capy/skill/gstack/scripts/install.sh | 39 ++- changelog.d/gsta258-l1-release-gate.md | 17 + docs/captain.md | 5 +- docs/releases.md | 13 +- docs/ship-policy.md | 75 ++++ lib/release-gate-policy.ts | 57 +++ lib/release-gate.ts | 400 ++++++++++++++++++++++ lib/ship-policy.ts | 9 +- lib/ship-queue-train-record.ts | 19 +- scripts/release-cut.ts | 33 +- scripts/train-gate.ts | 16 +- test/capy-skill-wrapper.test.ts | 70 +++- test/release-cut.test.ts | 44 +++ test/release-gate-workflow.test.ts | 129 +++++++ test/release-gate.test.ts | 301 ++++++++++++++++ test/train-gate.test.ts | 14 + 20 files changed, 1415 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/release-gate.yml create mode 100755 bin/gstack-release-gate create mode 100644 changelog.d/gsta258-l1-release-gate.md create mode 100644 lib/release-gate-policy.ts create mode 100644 lib/release-gate.ts create mode 100644 test/release-gate-workflow.test.ts create mode 100644 test/release-gate.test.ts diff --git a/.github/workflows/evals-main-backstop.yml b/.github/workflows/evals-main-backstop.yml index 2c3f30303..a864b8edf 100644 --- a/.github/workflows/evals-main-backstop.yml +++ b/.github/workflows/evals-main-backstop.yml @@ -12,6 +12,10 @@ name: Main backstop gate # (success when a green train covers the tree or the backstop gated it green, # neutral when the push was superseded or deferred, failure when it froze the # queue or could not finish); the stable release reads it (docs/releases.md). +# +# Once main's policy declares a `release_gate` block, the release gate +# (release-gate.yml, every 8 hours) gates main instead and this backstop stands +# down: no gate, no freeze and no check run on a push. on: push: branches: [main] @@ -60,9 +64,13 @@ jobs: done fi train=$(jq -c 'if (.train | type) == "object" then .train else null end' <<<"$policy") || { echo "::error::${GITHUB_SHA}:.gstack/ship-policy.json is not valid JSON; fix the policy file" >&2; exit 1; } + release=$(jq -r 'if (.release_gate | type) == "object" then "true" else "false" end' <<<"$policy") if [ -z "$train" ] || [ "$train" = null ]; then echo "train=false" >> "$GITHUB_OUTPUT" echo "- Backstop inert: main's .gstack/ship-policy.json declares no train block; per-PR gates cover main" >> "$GITHUB_STEP_SUMMARY" + elif [ "$release" = true ]; then + echo "train=false" >> "$GITHUB_OUTPUT" + echo "- Backstop stands down: main's .gstack/ship-policy.json declares a release_gate block; the scheduled release gate gates main (release-gate.yml, docs/ship-policy.md#release-gate)" >> "$GITHUB_STEP_SUMMARY" else echo "train=true" >> "$GITHUB_OUTPUT" fi diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml index fd5ed557c..b662c8867 100644 --- a/.github/workflows/evals.yml +++ b/.github/workflows/evals.yml @@ -166,7 +166,8 @@ jobs: pull-requests: read outputs: mode: ${{ steps.route.outputs.mode }} - record_artifact: ${{ steps.policy.outputs.record_artifact }} + # A release bisect (gstack-release-gate) names its own prefix-record artifact; everything else uses the train block's. + record_artifact: ${{ steps.route.outputs.record_artifact || steps.policy.outputs.record_artifact }} full_gate_check: ${{ steps.policy.outputs.full_gate_check }} screen_head: ${{ steps.screen.outputs.head_sha }} screen_base: ${{ steps.screen.outputs.base_sha }} @@ -218,10 +219,23 @@ jobs: echo "::warning::the train block at ${base} names no valid record_artifact and full_gate_check; this run routes as before" fi + # The release gate's bisect (gstack-release-gate) routes only under a + # release_gate block at the same base revision; git and jq only. + - name: Read the release gate policy (base revision) + id: release_policy + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: | + policy=$(git show "${BASE_SHA:-origin/${DEFAULT_BRANCH}}:.gstack/ship-policy.json" 2>/dev/null || echo null) + release=$(jq -r 'if (.release_gate | type) == "object" then "true" else "false" end' <<<"$policy" 2>/dev/null || echo false) + echo "release=${release}" >> "$GITHUB_OUTPUT" + - name: Route this run id: route env: TRAIN_POLICY: ${{ steps.policy.outputs.train }} + RELEASE_POLICY: ${{ steps.release_policy.outputs.release }} NONCE: ${{ inputs.nonce }} EXPECTED_SHA: ${{ inputs.expected_sha }} EVALS_ALL: ${{ inputs.evals_all }} diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml new file mode 100644 index 000000000..583444a6d --- /dev/null +++ b/.github/workflows/release-gate.yml @@ -0,0 +1,118 @@ +name: Release gate +# The release gate (docs/ship-policy.md#release-gate; bin/gstack-release-gate, +# lib/release-gate.ts): the full paid gate on main's head every 8 hours, only +# when main moved since the last gate. It records the verdict as a release +# note in refs/notes/gstack-trains, publishes the main-backstop check run on +# the gated commit (what scripts/release-cut.ts reads as gated), and bisects +# a red gate's cases. It never writes the stable branch: release-cut is its +# only writer, and refuses every cut while the newest gate is red. Inert (no +# paid work, nothing written) unless main's .gstack/ship-policy.json declares +# a release_gate block; with one, the per-push main backstop +# (evals-main-backstop.yml) stands down and this replaces it. +on: + schedule: + # Every 8 hours (RELEASE_GATE_SCHEDULE_HOURS in lib/release-gate-policy.ts; + # test/release-gate-workflow.test.ts pins the two together), off the hour. + - cron: '17 */8 * * *' + workflow_dispatch: + inputs: + dry_run: + description: 'Print what the gate would test and dispatch nothing' + type: boolean + default: false + ignore_cadence: + description: 'Gate now even if the last gate is younger than cadence_hours (main must still have moved)' + type: boolean + default: true + +# One gate at a time; a second run waits for the first. +concurrency: + group: release-gate + cancel-in-progress: false + +permissions: {} + +jobs: + release-gate: + runs-on: ubuntu-24.04 + # A dispatch gates only from the default branch, with its tooling. + if: github.ref_name == github.event.repository.default_branch + # The full gate (270) + a bisect round + margin; past --budget-minutes the + # record stays resumable by the next run. + timeout-minutes: 345 + outputs: + action: ${{ steps.gate.outputs.action }} + sha: ${{ steps.gate.outputs.sha }} + check: ${{ steps.gate.outputs.check }} + permissions: + # Pushes the release record to refs/notes/gstack-trains and a bisect's + # release-gate/ branch (deleted after its round). + contents: write + # Dispatches the gate and reads its runs and artifacts. + actions: write + checks: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + fetch-depth: 0 + + # Read with git and jq only: without a release_gate block nothing below runs. + - name: Read the release gate policy (main) + id: policy + run: | + file=.gstack/ship-policy.json + policy=$(git show "${GITHUB_SHA}:${file}" 2>/dev/null || echo null) + gate=$(jq -c 'if (.release_gate | type) == "object" then .release_gate else null end' <<<"$policy") || { echo "::error::${GITHUB_SHA}:${file} is not valid JSON; fix the policy file" >&2; exit 1; } + if [ -z "$gate" ] || [ "$gate" = null ]; then + echo "release=false" >> "$GITHUB_OUTPUT" + echo "- Release gate inert: main's ${file} declares no release_gate block; the per-push backstop gates main" >> "$GITHUB_STEP_SUMMARY" + else + echo "release=true" >> "$GITHUB_OUTPUT" + fi + + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + if: steps.policy.outputs.release == 'true' + with: + bun-version: 1.4.2 + + - name: Gate main's head when it moved + id: gate + if: steps.policy.outputs.release == 'true' + env: + GH_TOKEN: ${{ github.token }} + GSTACK_STATE_ROOT: ${{ runner.temp }}/gstack-state + DRY_RUN: ${{ inputs.dry_run && '1' || '' }} + IGNORE_CADENCE: ${{ inputs.ignore_cadence && '1' || '' }} + run: | + ./bin/gstack-release-gate run --repo "$GITHUB_REPOSITORY" --sha "$GITHUB_SHA" --budget-minutes 330 \ + --summary "$GITHUB_STEP_SUMMARY" --outputs "$GITHUB_OUTPUT" ${DRY_RUN:+--dry-run} ${IGNORE_CADENCE:+--ignore-cadence} + + # The gated commit's check run (release_gate.check_name, main-backstop), the + # evidence scripts/release-cut.ts reads. Its own job with checks: write and + # no repository code, as for backstop-check: only the recorded action word. + release-check: + runs-on: ubuntu-24.04 + needs: release-gate + if: ${{ !cancelled() && contains(fromJSON('["gated","held","failed"]'), needs.release-gate.outputs.action) && needs.release-gate.outputs.sha != '' }} + timeout-minutes: 5 + permissions: + checks: write + steps: + - name: Publish the release gate's check run on the gated commit + env: + GH_TOKEN: ${{ github.token }} + ACTION: ${{ needs.release-gate.outputs.action }} + SHA: ${{ needs.release-gate.outputs.sha }} + CHECK: ${{ needs.release-gate.outputs.check }} + run: | + case "$ACTION" in + gated) conclusion=success ;; + held|failed) conclusion=failure ;; + *) echo "::error::unknown release gate action '$ACTION'"; exit 1 ;; + esac + [[ "$SHA" =~ ^[0-9a-f]{40}$ ]] || { echo "::error::'$SHA' is not a commit SHA"; exit 1; } + [[ "$CHECK" =~ ^[A-Za-z0-9_./-]{1,64}$ ]] || { echo "::error::'$CHECK' is not a check-run name"; exit 1; } + gh api "repos/${GITHUB_REPOSITORY}/check-runs" -X POST -f name="$CHECK" -f head_sha="$SHA" -f status=completed -f conclusion="$conclusion" \ + -f details_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ + -f "output[title]=Release gate: ${ACTION}" -f "output[summary]=${ACTION}: see the release-gate job summary of run ${GITHUB_RUN_ID}" >/dev/null + echo "- ${CHECK}: ${conclusion} (release gate ${ACTION}) on ${SHA}" >> "$GITHUB_STEP_SUMMARY" diff --git a/bin/gstack-release-gate b/bin/gstack-release-gate new file mode 100755 index 000000000..f523a0fd1 --- /dev/null +++ b/bin/gstack-release-gate @@ -0,0 +1,62 @@ +#!/usr/bin/env -S bun --no-env-file --config=/dev/null +/** + * gstack-release-gate — the full paid gate on main every cadence_hours, only + * when main moved since the last gate (docs/ship-policy.md#release-gate; + * logic: lib/release-gate.ts). The gate never moves the stable branch: + * scripts/release-cut.ts is its only writer and holds every cut while the + * newest gate is red. + * + * gstack-release-gate run [--dry-run] [--sha
] [--ignore-cadence] + * gstack-release-gate status + * gstack-release-gate bisect [--sha ] [--spend-cap ] [--dry-run] + * + * Every subcommand takes --repo (default $GITHUB_REPOSITORY, else + * gh repo view), --policy (a whole ship policy, validated; default the + * reviewed .gstack/ship-policy.json on origin/main), --remote (origin), + * --summary (appends each line as a bullet) and --outputs + * (appends action= and sha= for the workflow). run and bisect + * take --poll-seconds (60) and --budget-minutes (330): past the budget nothing + * waits, and the record stays resumable by the next run or bisect. + * Exit 0: inert, skipped, dry run, gated green, or status with no hold; + * 1: red (held), incomplete or could not finish; 2: usage. + */ +import { randomBytes } from 'node:crypto'; +import * as fs from 'node:fs'; +import { bisectGate, gateStatus, loadReleaseGate, runGate, type GateEnv, type GateOut } from '../lib/release-gate'; +import { runGh } from '../scripts/recover-receipts'; +import { gitIn } from '../scripts/train-gate'; + +const USAGE = 'usage: gstack-release-gate run [--dry-run] [--sha ] [--ignore-cadence] | status | bisect [--sha ] [--spend-cap ] [--dry-run]\n' + + ' common: [--repo ] [--policy ] [--remote ] [--summary ] [--outputs ] [--poll-seconds ] [--budget-minutes ]'; +const [command, ...rest] = process.argv.slice(2); +const flag = (name: string) => { const i = rest.indexOf(`--${name}`); return i >= 0 ? rest[i + 1] : undefined; }; +const has = (name: string) => rest.includes(`--${name}`); +const usage = (why?: string): never => { console.error(why ? `${why}\n${USAGE}` : USAGE); process.exit(2); }; +if (command === '--help' || command === '-h') { console.log(USAGE); process.exit(0); } +if (!['run', 'status', 'bisect'].includes(command ?? '')) usage(); +const sha = flag('sha'); +if (sha !== undefined && !/^[0-9a-f]{40}$/.test(sha)) usage('--sha needs a full 40-hex commit'); +const spendCap = flag('spend-cap') === undefined ? undefined : Number(flag('spend-cap')); +if (spendCap !== undefined && !(spendCap >= 0)) usage('--spend-cap needs dollars'); +const repoRoot = process.cwd(); +const repo = flag('repo') ?? process.env.GITHUB_REPOSITORY ?? runGh(['repo', 'view', '--json', 'nameWithOwner', '-q', '.nameWithOwner'], 60_000).stdout.trim(); +if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repo)) usage(`cannot name the repository (${JSON.stringify(repo)}); pass --repo `); +const loaded = loadReleaseGate(repoRoot, { policyFile: flag('policy') }); +const env: GateEnv = { + repo, repoRoot, remote: flag('remote') ?? 'origin', gh: runGh, git: gitIn(repoRoot), now: Date.now, sleep: ms => Bun.sleepSync(ms), + random: () => randomBytes(3).toString('hex'), pollMs: Number(flag('poll-seconds') ?? 60) * 1000, + deadline: Date.now() + Number(flag('budget-minutes') ?? 330) * 60_000, +}; +let out: GateOut; +if ('error' in loaded) out = { action: 'failed', sha: null, lines: [`RELEASE: policy refused: ${loaded.error}`] }; +else if (command === 'run') out = runGate(env, { ...loaded, sha, dryRun: has('dry-run'), ignoreCadence: has('ignore-cadence') }); +else if (command === 'bisect') out = bisectGate(env, { ...loaded, sha, spendCap, dryRun: has('dry-run') }); +else out = gateStatus(env, loaded); +const summary = flag('summary'); +for (const line of out.lines) { + console.log(line); + if (summary) fs.appendFileSync(summary, `- ${line}\n`); +} +const outputs = flag('outputs'); +if (outputs) fs.appendFileSync(outputs, `action=${out.action}\nsha=${out.sha ?? ''}\ncheck=${'gate' in loaded && loaded.gate ? loaded.gate.check_name : ''}\n`); +process.exit(out.action === 'held' || out.action === 'failed' ? 1 : 0); diff --git a/capy/skill/gstack/scripts/install.sh b/capy/skill/gstack/scripts/install.sh index 2cb8ae4c2..f7f2eaa1b 100755 --- a/capy/skill/gstack/scripts/install.sh +++ b/capy/skill/gstack/scripts/install.sh @@ -17,6 +17,12 @@ # checkout it prints `gstack: ok not-needed for=none (...)` and exits 0; with # one it prints the same trailer with the checkout's version, running nothing # from the checkout. That ok admits no workflow (no verdict is written). +# From revision 4 the clone and the floor fast-forward follow the update +# channel (docs/releases.md#channels): GSTACK_UPDATE_CHANNEL=stable|main, else +# the checkout's bin/gstack-update-channel.sh (auto: stable in team mode), +# else main. On stable, a fresh clone checks out origin's stable branch and +# the floor fast-forward goes to stable, each only when stable exists and +# meets the supported minimum; otherwise main, as before. --revision wins. # Everything else (Bun, setup, the doctor, the verdict, the launch line) is # the checkout's. The last line is the `gstack:` trailer; exit 0 only on # `gstack: ok`. Contract: docs/capy-install.md; tests: @@ -27,9 +33,10 @@ # bash install.sh --check --for none this machine runs no workflow: ok not-needed, nothing installed # bash install.sh [--for | --require ] [--with-browser] [--revision ] [--project ] # Environment: GSTACK_CAPY_ROOT (default ~/.local/share/gstack), GSTACK_CAPY_REPO, -# GSTACK_CAPY_AUTO_UPGRADE=0 (never fast-forward), GSTACK_CAPY_FETCH_TIMEOUT (s). +# GSTACK_CAPY_AUTO_UPGRADE=0 (never fast-forward), GSTACK_CAPY_FETCH_TIMEOUT (s), +# GSTACK_UPDATE_CHANNEL=main (follow every merge even in team mode) or stable. set -u -GSTACK_CAPY_WRAPPER=3 +GSTACK_CAPY_WRAPPER=4 self="$(cd "$(dirname "$0")" && pwd -P)/$(basename "$0")" root="${GSTACK_CAPY_ROOT:-$HOME/.local/share/gstack}" @@ -74,6 +81,24 @@ below() { return 1 } support_get() { printf '%s\n' "$1" | sed -n "s/^$2=\([0-9][0-9.]*\)[[:space:]]*\$/\1/p" | head -1; } +# channel_stable ROOT MIN [FETCH_FLAG] — true, with origin's stable commit in $stable_sha, when +# this install follows stable and origin's stable exists and meets MIN; says why not otherwise. +# FETCH_FLAG is --depth=1 for a fresh clone only (never --depth into an existing checkout). +channel_stable() { + local ch="${GSTACK_UPDATE_CHANNEL:-}" ver + case "$ch" in stable|main) ;; *) ch=$(bash "$1/bin/gstack-update-channel.sh" "$1" 2>/dev/null | head -1) ;; esac + [ "$ch" = stable ] || return 1 + if ! bounded git -C "$1" fetch --quiet ${3:+"$3"} origin stable 2>/dev/null; then + echo "gstack-capy-skill: channel stable, but origin has no stable branch yet (or the fetch failed); following main" + return 1 + fi + stable_sha=$(git -C "$1" rev-parse FETCH_HEAD) ver=$(git -C "$1" show FETCH_HEAD:VERSION 2>/dev/null | head -1) + if [ -n "$2" ] && below "${ver:-0}" "$2"; then + echo "gstack-capy-skill: channel stable is at ${ver:-unknown}, below the supported minimum $2; following main" + return 1 + fi + echo "gstack-capy-skill: channel stable ${ver:-unknown} (GSTACK_UPDATE_CHANNEL=main follows every merge)" +} mkdir -p "$(dirname "$root")" if [ "$check" -eq 0 ]; then @@ -100,6 +125,10 @@ if [ ! -f "$root/setup" ]; then fi remote_min=$(support_get "$(cat "$root/capy/support.env" 2>/dev/null)" GSTACK_MIN_REVISION) remote_version=$(head -1 "$root/VERSION" 2>/dev/null) + if [ -z "$revision" ] && channel_stable "$root" "$remote_min" --depth=1; then + git -C "$root" reset --quiet --hard "$stable_sha" \ + || { echo "gstack: fail installer — could not check out origin/stable in $root; fix: GSTACK_UPDATE_CHANNEL=main $fix_cmd"; exit 1; } + fi export GSTACK_CAPY_CLONED="$repo" else # The remote's rules: fetch the default branch into FETCH_HEAD (never --depth @@ -140,9 +169,11 @@ else echo "gstack: fail revision — installed $installed is below the supported minimum $remote_min ($cause); fix: $fix (REVISION_BELOW_MINIMUM)" exit 1 fi - if ! git -C "$root" merge --ff-only -q FETCH_HEAD 2>/dev/null; then + target=$(git -C "$root" rev-parse FETCH_HEAD) + if channel_stable "$root" "$remote_min" && git -C "$root" merge-base --is-ancestor HEAD "$stable_sha" 2>/dev/null; then target=$stable_sha; fi + if ! git -C "$root" merge --ff-only -q "$target" 2>/dev/null; then git -C "$root" fetch --quiet --unshallow origin "$default" 2>/dev/null - git -C "$root" merge --ff-only -q FETCH_HEAD 2>/dev/null \ + git -C "$root" merge --ff-only -q "$target" 2>/dev/null \ || { echo "gstack: fail revision — installed $installed is below the supported minimum $remote_min (fast-forward failed); fix: git -C $root status, then $fix_cmd (REVISION_BELOW_MINIMUM)"; exit 1; } fi echo "gstack-capy-skill: upgraded $installed -> $(head -1 "$root/VERSION")" diff --git a/changelog.d/gsta258-l1-release-gate.md b/changelog.d/gsta258-l1-release-gate.md new file mode 100644 index 000000000..0fd053ea5 --- /dev/null +++ b/changelog.d/gsta258-l1-release-gate.md @@ -0,0 +1,17 @@ +--- +bump: minor +--- +**A release gate can now gate main every 8 hours instead of on every push, and it feeds the stable release.** It is off until main's `.gstack/ship-policy.json` declares a `release_gate` block, so this release changes nothing on its own. With the block, `.github/workflows/release-gate.yml` runs `gstack-release-gate run` every 8 hours. When main moved since the last gate, it runs the full paid gate on main's head once, records the verdict as a `release` note in `refs/notes/gstack-trains`, and publishes the `main-backstop` check run on that commit. A red gate bisects its red cases within `bisect_spend_cap_usd` and names the first bad commit. The per-push main backstop stands down while the block is set, so main is never gated twice. `scripts/release-cut.ts` is still the only thing that moves `stable`, and its 72-hour floor and exceptions are unchanged. It now reads a green release record as proof the commit was gated, and it refuses every cut while the newest release gate is red with no green gate on a later commit. That hold is the freeze. See docs/ship-policy.md#release-gate. + +### Added +- **`release_gate` policy key** (`lib/release-gate-policy.ts`, key version 1.94.0.0): `cadence_hours` (8 to 168), `gate_workflow`, `branch`, `stable_ref`, `check_name`, `on_red: hold-release` and `bisect_spend_cap_usd`, every field required. +- **`bin/gstack-release-gate`** (`lib/release-gate.ts`): `run [--dry-run] [--sha] [--ignore-cadence]`, `status` and `bisect`. `--policy ` reads the block from a file, for a dry run or a proof before adoption. Exit 0 means gated, skipped, inert or a dry run; 1 means held; 2 means failed. +- **`.github/workflows/release-gate.yml`**: an 8-hour schedule and a manual dispatch. The check run comes from a separate job that holds `checks: write` and runs no repository code. +- **Capy skill installer follows the update channel** (`capy/skill/gstack/scripts/install.sh`, wrapper revision 4). With `GSTACK_UPDATE_CHANNEL=stable` or a stable `update_channel`, a fresh clone or a floor upgrade goes to `stable` when it meets the support floor, and to main otherwise. `--revision` still wins. + +### Changed +- **`evals-main-backstop.yml`** stands down (no gate, no freeze, no check run) when main's policy declares `release_gate`. +- **`evals.yml`** routes a release gate bisect (`release-gate/` refs) only under a `release_gate` block at the base revision, and that bisect publishes its prefix records as `gstack-release-record`. + +### Tests +- `test/release-gate.test.ts` drives run, bisect and status against a fake `gh` and a scratch remote. `test/release-gate-workflow.test.ts` runs the workflows' policy steps on scratch repositories and pins the cron to the cadence floor. `test/release-cut.test.ts` proves a red release record refuses the cut and a green one on a later commit releases it. `test/train-gate.test.ts` covers the bisect route, and `test/capy-skill-wrapper.test.ts` covers the stable channel and its fallback to main. diff --git a/docs/captain.md b/docs/captain.md index b09778c50..0a7edfb8b 100644 --- a/docs/captain.md +++ b/docs/captain.md @@ -298,7 +298,10 @@ through an urgent one-PR train (or `train build --repair` while the queue is fro refuses `--urgent`); all prefixes green earns exactly one unchanged rebuild; anything else goes to a repair lane. An infra-only census earns one re-dispatch. A backstop gate runs on every push to the base branch and freezes the queue when it finds -a tree no green train covered; one repair PR at its exact head clears the freeze. +a tree no green train covered; one repair PR at its exact head clears the freeze. A +policy with a `release_gate` block swaps that per-push backstop for one full gate on main +every 8 hours, whose red verdict holds the stable release instead of freezing the queue +([ship-policy.md#release-gate](ship-policy.md#release-gate)). **Wave PRs when the queue is long.** Every slot pays its own stamp, its own round of required checks and its own version, one after another. When more diff --git a/docs/releases.md b/docs/releases.md index de38b8ff4..18b473bad 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -33,7 +33,7 @@ of its own; it only reads what the trains and the backstop already recorded. With nothing green in the window it fails and lists why each commit was skipped. -**Gated** means one of two pieces of evidence. The first is the +**Gated** means one of three pieces of evidence. The first is the `main-backstop` check run: the Main backstop gate workflow publishes it on each push to main from a job of its own, success when a green train record covers the tree or the backstop gated it green, neutral when main moved past @@ -45,7 +45,12 @@ its tree: those entries are the main commits `gstack-ship-queue verify --train` confirmed, so a commit a green train merged is gated even when trains merged faster than the backstop could settle and its own backstop run ended neutral or never ran. The `CANDIDATE` line says which: `gated by -main-backstop` or `gated by train `. Train evidence relaxes nothing +main-backstop` or `gated by train `. The third is a GREEN release +record (kind `release`) on the commit, written by the release gate +([docs/ship-policy.md#release-gate](ship-policy.md#release-gate)) when the +policy declares `release_gate`; the line then says `gated by release gate run +`. A RED release record with no GREEN one on a later commit that +contains it refuses the cut like a freeze. Train evidence relaxes nothing else: a red backstop run, a failed or missing `free-tests` run, any other red check run or a run still going keeps the commit out, and a record in any other state (`red`, `superseded`, `free-green`, …) is not evidence. A commit with @@ -240,6 +245,10 @@ overrides the key for one command. The resolution lives in one helper, - `bin/gstack-upgrade` and `/gstack-upgrade`: they fast-forward to the channel ref (`UPSTREAM_REF=origin/stable` in the skill's output) and say which channel they followed. +- `capy/skill/gstack/scripts/install.sh`, the Capy skill's installer (wrapper + revision 4): a fresh clone or a floor upgrade follows `stable` when the channel + resolves to it and `stable` meets the support floor, and follows main + otherwise. `--revision` still pins an exact commit. Two edge cases are handled on purpose: diff --git a/docs/ship-policy.md b/docs/ship-policy.md index 7fd896a28..3a31c05ee 100644 --- a/docs/ship-policy.md +++ b/docs/ship-policy.md @@ -1118,6 +1118,81 @@ every lane to the per-PR queue at once. `--urgent` ships one PR alone, `--no-screen` skips the paid screen, `--broad` widens it to the full PR selection, and `train build --prs ` drops a PR at any step. + +## `release_gate`: one full gate on main every 8 hours + +Trains gate each batch of PRs, and the main backstop gates every push. The +release gate replaces the backstop with one full paid gate on main's head at +most every `cadence_hours`, and only when main moved since the last gate. It +feeds the stable release (docs/releases.md) and never writes `stable` itself: +`scripts/release-cut.ts` stays the only writer, with its 72-hour floor and its +security and regression exceptions unchanged. + +The surface is off unless main's `.gstack/ship-policy.json` declares the +block (key version 1.94.0.0, so `min_gstack_version` must be at least that): + +```json +"release_gate": { + "cadence_hours": 8, "gate_workflow": "evals.yml", "branch": "main", + "stable_ref": "stable", "check_name": "main-backstop", + "on_red": "hold-release", "bisect_spend_cap_usd": 40 +} +``` + +Every field is required. `cadence_hours` runs from 8 (the workflow's +schedule) to 168. `check_name` is the check run release-cut reads as gated, +so gstack keeps `main-backstop`. `bisect_spend_cap_usd` caps one red gate's +bisect (0 skips it). + +**What runs.** `.github/workflows/release-gate.yml` fires every 8 hours and +on a manual dispatch from main. Without the block it stops after a git-and-jq +policy read, so no paid work starts and nothing is written. With the block it runs +`gstack-release-gate run`, which: + +1. skips when main has not moved since `stable` or since the last gate, or + when the last gate is younger than `cadence_hours` (`--ignore-cadence` + overrides only the age); +2. writes a `release` record (`gstack-trains:release::`) to + `refs/notes/gstack-trains` before it dispatches, so a lost run resumes; +3. dispatches the full gate on that sha and reads its verdict and flake + records. A case that failed once and passed its one rule-16 rerun is green + and is listed as flaky; +4. records GREEN, RED or INCOMPLETE. An INCOMPLETE gate is re-dispatched once + by the next run, then left for a person; +5. on RED, bisects the red cases over main's first-parent commits since the + last green gate (at most 4 rounds, within the spend cap) and records the + first bad commit; +6. publishes `check_name` on the gated commit from a separate job that holds + `checks: write` and runs no repository code: success for a green gate, + failure for a red or failed one. + +**What release-cut reads.** A GREEN release record on a commit counts as +gated, the same as a successful `main-backstop` check run. A RED record +with no GREEN record on a later commit that contains it holds every cut, like a +queue freeze: the `REFUSED` line names the red sha and cases. Land the revert +or the fix-forward, then gate again with `gstack-release-gate run +--ignore-cadence` (or dispatch the workflow). Nothing else clears a hold. + +**The per-push backstop stands down.** `evals-main-backstop.yml` keeps its +push trigger. While the block is set, its policy step reports `train=false`, +so a push dispatches no gate, freezes nothing and publishes no check run. Main +is never gated twice. Trains keep running until the train lane retires them. + +```bash +gstack-release-gate run [--dry-run] [--sha ] [--ignore-cadence] +gstack-release-gate status # the newest record, the hold if any, pending commits +gstack-release-gate bisect [--sha ] [--spend-cap ] [--dry-run] +``` + +Common flags: `--repo`, `--policy ` (read the block from a file instead +of main, for a dry run or a proof before adoption), `--remote`, +`--summary`, `--outputs`, `--poll-seconds`, `--budget-minutes`. Every line +starts with `RELEASE:`. Exit 0 means gated, skipped, inert or a dry run; 1 +means held (red); 2 means failed or incomplete. + +**Escape hatch.** Remove the block on main. The per-push backstop gates the +next push again, and release-cut reads its check runs as before. + ## What `/ship` reads from it diff --git a/lib/release-gate-policy.ts b/lib/release-gate-policy.ts new file mode 100644 index 000000000..5da04d965 --- /dev/null +++ b/lib/release-gate-policy.ts @@ -0,0 +1,57 @@ +/** + * The `release_gate` block of .gstack/ship-policy.json + * (docs/ship-policy.md#release-gate): the full paid gate on the base branch + * every few hours, run by bin/gstack-release-gate from + * .github/workflows/release-gate.yml, instead of a gate per train or + * per push. The gate never writes the stable branch: scripts/release-cut.ts + * stays its only writer, reads a green gate as "gated" and holds every cut + * while the newest gate is red. Validated here so lib/ship-policy.ts stays + * one schema file. + */ +import type { PolicyError } from './ship-policy'; + +export interface ReleaseGatePolicy { + /** Least hours between two gate dispatches; at least RELEASE_GATE_SCHEDULE_HOURS, the workflow's schedule. */ + cadence_hours: number; + /** Workflow file dispatched for the full paid gate and the bisect; it accepts the eval-bg dispatch inputs. */ + gate_workflow: string; + /** The branch the gate tests (its head at each run). */ + branch: string; + /** The branch scripts/release-cut.ts moves; the gate only reads it to count pending commits. */ + stable_ref: string; + /** The check run published on a gated commit, read by release-cut as gated. */ + check_name: string; + /** What a red gate does. hold-release: the red record keeps release-cut from cutting until a later gate is green. */ + on_red: 'hold-release'; + /** Cap over one red gate's bisect, at RELEASE_TRIAL_RESERVATION_USD per case trial; 0 skips the bisect. */ + bisect_spend_cap_usd: number; +} + +/** The cron step of release-gate.yml's schedule (test/release-gate-workflow.test.ts pins it): no cadence gates more often. */ +export const RELEASE_GATE_SCHEDULE_HOURS = 8; +const REF = /^(?!.*\.\.)(?!\/)(?!.*\/$)[A-Za-z0-9._/-]{1,100}$/; +const FIELDS: Record string | null> = { + cadence_hours: v => (Number.isInteger(v) && (v as number) >= RELEASE_GATE_SCHEDULE_HOURS && (v as number) <= 168 ? null : `must be a whole number of hours from ${RELEASE_GATE_SCHEDULE_HOURS} (the workflow's schedule) to 168`), + gate_workflow: v => (typeof v === 'string' && /^[A-Za-z0-9_.-]{1,64}\.ya?ml$/.test(v) ? null : 'must be a workflow file name such as evals.yml'), + branch: v => (typeof v === 'string' && REF.test(v) ? null : 'must be a branch name'), + stable_ref: v => (typeof v === 'string' && REF.test(v) ? null : 'must be a branch name'), + check_name: v => (typeof v === 'string' && /^[A-Za-z0-9_./-]{1,64}$/.test(v) ? null : 'must be a check-run name ([A-Za-z0-9_./-]{1,64})'), + on_red: v => (v === 'hold-release' ? null : 'must be hold-release'), + bisect_spend_cap_usd: v => (typeof v === 'number' && v >= 0 && v <= 500 ? null : 'must be dollars from 0 to 500'), +}; + +export const RELEASE_GATE_EXPLANATION = '{ cadence_hours, gate_workflow, branch, stable_ref, check_name, on_red, bisect_spend_cap_usd } or null: the release gate (docs/ship-policy.md#release-gate). bin/gstack-release-gate dispatches the full paid gate (gate_workflow) on the head of branch at most once per cadence_hours, only when branch moved since the last gate, records the verdict as a release note in refs/notes/gstack-trains and publishes check_name on the gated commit; a red gate bisects its red cases (bisect_spend_cap_usd, 0 skips) and, with on_red hold-release, keeps scripts/release-cut.ts from moving stable_ref until a later gate is green. It never writes stable_ref. While the block is set, the per-push main backstop stands down. null keeps the per-push backstop.'; + +export function releaseGateErrors(raw: unknown): PolicyError[] { + const g = raw as Record | null; + if (!g || typeof g !== 'object' || Array.isArray(g)) return [{ code: 'POLICY_INVALID', path: 'release_gate', message: `must be { ${Object.keys(FIELDS).join(', ')} } or null` }]; + const out: PolicyError[] = []; + for (const k of Object.keys(g)) if (!(k in FIELDS)) out.push({ code: 'POLICY_INVALID', path: `release_gate.${k}`, message: 'unknown key' }); + for (const [k, check] of Object.entries(FIELDS)) { + if (!(k in g)) { out.push({ code: 'POLICY_INVALID', path: `release_gate.${k}`, message: 'missing (every release_gate field is required)' }); continue; } + const why = check(g[k]); + if (why) out.push({ code: 'POLICY_INVALID', path: `release_gate.${k}`, message: `${JSON.stringify(g[k])} ${why}` }); + } + if (typeof g.branch === 'string' && g.branch === g.stable_ref) out.push({ code: 'POLICY_INVALID', path: 'release_gate.stable_ref', message: 'must differ from branch' }); + return out; +} diff --git a/lib/release-gate.ts b/lib/release-gate.ts new file mode 100644 index 000000000..6255a5a0b --- /dev/null +++ b/lib/release-gate.ts @@ -0,0 +1,400 @@ +/** + * The release gate (docs/ship-policy.md#release-gate; plan ship-90-90 choice A): + * the full paid gate on the gated branch's head every cadence_hours, only when + * the branch moved since the last gate, instead of a gate per train or per push. + * + * run skip when the branch head equals the stable branch, already has a + * green or red record, or the last gate is younger than the cadence; + * otherwise write a PENDING `release` record to refs/notes/gstack-trains, + * dispatch gate_workflow on the head (eval-bg's backstop nonce), follow + * it, and record GREEN, RED or INCOMPLETE. Rule-16 reruns happen inside + * the gate run (scripts/lib/paid-rerun.ts): a case that failed once and + * passed its one rerun is green and listed as flaky here. + * bisect a red record's red verdict cases, one trial each, on up to + * MAX_PREFIXES first-parent commits between the last green gate (or + * the stable branch) and the red head, dispatched on release-gate/ + * and narrowed round by round until first_bad is the commit right after + * a green one; spend-capped and resumable. + * status stable, the last gate, pending commits and the hold. + * + * The gate never writes the stable branch. scripts/release-cut.ts is its only + * writer: it reads a green record (or the check_name check run this gate's + * workflow publishes) as gated, and refuses every cut while a red record is not + * followed by a green gate on a later commit (the hold, on_red hold-release). + */ +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { DISPATCH_INPUTS, makeNonce } from '../scripts/eval-bg'; +import { ghFailureReason, type GhRunner } from '../scripts/recover-receipts'; +import { MAX_BISECT_CASES, MAX_PREFIXES, RELEASE_BISECT_ARTIFACT, RELEASE_BISECT_REF, type Git } from '../scripts/train-gate'; +import type { ReleaseGatePolicy } from './release-gate-policy'; +import { loadPolicyFromBase, mergePolicy, validatePolicy } from './ship-policy'; +import { + NOTES_REF, fetchNotes, nowIso, persist, repoIdentity, + type PrefixResult, type ReleaseBisect, type ReleaseBisectRound, type ReleaseRecord, +} from './ship-queue-train-record'; + +export const RELEASE_TRIAL_RESERVATION_USD = 1.5; +/** The gate plus the one re-dispatch EVAL_POLICY.infraRedispatch allows an incomplete census. */ +export const MAX_RELEASE_ATTEMPTS = 2; +/** Cron start jitter the cadence forgives. */ +export const CADENCE_GRACE_HOURS = 1; +export const MAX_BISECT_ROUNDS = 4; +const SHA = /^[0-9a-f]{40}$/; +const CASE_ID = /^[A-Za-z0-9 _.:/-]{1,128}$/; +const HOUR = 3_600_000; +const GH_MS = 60_000; + +export interface Run { id: number; status: string; conclusion: string | null; head_sha: string; display_title?: string; html_url?: string; run_attempt?: number } +export interface GateEnv { + repo: string; repoRoot: string; remote: string; gh: GhRunner; git: Git; + now: () => number; sleep: (ms: number) => void; random: () => string; pollMs: number; + /** Epoch ms after which nothing waits on a run any more; the record stays resumable. */ + deadline: number; +} +export type GateAction = 'inert' | 'skipped' | 'dry-run' | 'gated' | 'held' | 'failed'; +export interface GateOut { action: GateAction; sha: string | null; lines: string[] } + +const short = (sha: string | null | undefined) => (sha ? sha.slice(0, 12) : 'none'); +const usd = (n: number | null) => (n === null ? 'unknown' : n.toFixed(2)); + +/** The release_gate block from origin/'s reviewed policy, or from --policy (validated the same way). */ +export function loadReleaseGate(repoRoot: string, o: { policyFile?: string; base?: string }): { gate: ReleaseGatePolicy | null; label: string } | { error: string } { + if (o.policyFile) { + let raw: unknown; + try { raw = JSON.parse(fs.readFileSync(o.policyFile, 'utf8')); } catch (e: any) { return { error: `${o.policyFile} is not readable JSON: ${e.message}` }; } + const errors = validatePolicy(raw); + if (errors.length) return { error: `${o.policyFile}: ${errors.map(e => `${e.path} ${e.message}`).join('; ')}` }; + return { gate: mergePolicy(raw as Record, o.policyFile).policy.release_gate, label: o.policyFile }; + } + const loaded = loadPolicyFromBase(repoRoot, o.base ?? 'main'); + if ('unavailable' in loaded) return { error: `no reviewed policy: ${loaded.detail}` }; + if (loaded.errors.length) return { error: `${loaded.source.label}: ${loaded.errors.map(e => `${e.path} ${e.message}`).join('; ')}` }; + return { gate: loaded.policy.release_gate, label: loaded.source.label }; +} + +/** This repository's release records from the local notes ref, newest first. */ +export function releaseRecords(git: Git, repoKey: string): ReleaseRecord[] { + const listed = git(['notes', `--ref=${NOTES_REF}`, 'list']); + if (!listed.ok || !listed.out.trim()) return []; + return listed.out.split('\n').map(line => line.split(' ')[0]!).filter(Boolean).flatMap(blob => { + try { const r = JSON.parse(git(['cat-file', 'blob', blob]).out) as ReleaseRecord; return r?.kind === 'release' && r.repo_key === repoKey && SHA.test(r.sha) ? [r] : []; } catch { return []; } + }).sort((a, b) => b.created_at.localeCompare(a.created_at)); +} + +/** The red record that holds releases: red, and no green record sits on a later commit that contains it. Null when none. */ +export function activeHold(records: readonly ReleaseRecord[], isAncestor: (older: string, newer: string) => boolean): ReleaseRecord | null { + const greens = records.filter(r => r.verdict === 'GREEN'); + return records.filter(r => r.verdict === 'RED').find(red => !greens.some(g => g.sha !== red.sha && isAncestor(red.sha, g.sha))) ?? null; +} + +/** The newest green record on an ancestor of sha: where pending commits and a bisect start. */ +export function greenBase(records: readonly ReleaseRecord[], sha: string, isAncestor: (older: string, newer: string) => boolean): ReleaseRecord | null { + return records.find(r => r.verdict === 'GREEN' && r.sha !== sha && isAncestor(r.sha, sha)) ?? null; +} + +export interface ReportFacts { verdict: 'GREEN' | 'RED' | 'NONE'; red: string[]; other: string[]; cost: number | null } + +/** report-verdict's report.txt: the verdict, each failed case (a verdict red, or another class), and the known cost. */ +export function parseReport(text: string): ReportFacts { + const verdict = (/VERDICT (GREEN|RED)/.exec(text)?.[1] as 'GREEN' | 'RED' | undefined) ?? 'NONE'; + const red: string[] = []; + const other: string[] = []; + for (const m of text.matchAll(/^\s+✗ (\S(?:.*?\S)?) {2,}(\w+) {2,}(\w+)(?: {2,}([A-Z_]+))?/gm)) { + (m[3] === 'FAIL' && (!m[4] || m[4] === 'VERDICT') ? red : other).push(m[1]!); + } + const cost = /· cost \$(\d+(?:\.\d+)?)/.exec(text)?.[1]; + return { verdict, red: [...new Set(red)].sort(), other: [...new Set(other)].sort(), cost: cost ? Number(cost) : null }; +} + +/** A finished gate run as one verdict; green needs the run's own success, which follows its rule-16 report. */ +export function classifyRun(run: Pick, report: ReportFacts | null): { verdict: ReleaseRecord['verdict']; red: string[]; detail: string } { + if (run.status !== 'completed') return { verdict: 'PENDING', red: [], detail: `run is ${run.status}` }; + if (run.conclusion === 'success') { + if (report?.verdict === 'RED') return { verdict: 'INCOMPLETE', red: [], detail: 'the run concluded success but its report says RED' }; + return { verdict: 'GREEN', red: [], detail: report ? 'run and report green' : 'run concluded success (no report artifact)' }; + } + if (report?.verdict === 'RED' && report.red.length) { + return { verdict: 'RED', red: report.red, detail: `${report.red.length} red case(s)${report.other.length ? `; also not green: ${report.other.join(', ')}` : ''}` }; + } + return { verdict: 'INCOMPLETE', red: [], detail: `run concluded ${run.conclusion ?? 'unknown'}; report ${report ? `${report.verdict}${report.other.length ? ` (${report.other.join(', ')})` : ''}` : 'missing'}` }; +} + +/** Up to max commits spread evenly over the interior of a range (oldest first), never the endpoints. */ +export function planPrefixes(interior: readonly string[], max = MAX_PREFIXES): string[] { + if (interior.length <= max) return [...interior]; + const picked = Array.from({ length: max }, (_, i) => interior[Math.round(((i + 1) * (interior.length + 1)) / (max + 1)) - 1]!); + return [...new Set(picked)]; +} + +/** + * One round's results narrow (good, bad]: the earliest prefix where any case is + * red becomes bad, the prefix before it good. Every case must be monotone + * (green, then red) over the tested prefixes and have a result on each. + */ +export function narrow(range: { good: string; bad: string }, tested: ReadonlyArray<{ sha: string; results: Record }>, cases: readonly string[]): { good: string; bad: string } | { unknown: string } { + let firstRed = tested.length; + for (const c of cases) { + const seq = tested.map(p => (p.results[c] === 'red' ? 'R' : p.results[c] === 'green' ? 'G' : '?')).join(''); + if (seq.includes('?')) return { unknown: `no result for ${c} on ${tested.filter(p => !p.results[c]).map(p => short(p.sha)).join(', ')}` }; + const at = seq.indexOf('R'); + if (at >= 0 && seq.slice(at).includes('G')) return { unknown: `${c} is not monotone over the tested commits (${seq})` }; + if (at >= 0) firstRed = Math.min(firstRed, at); + } + return { good: firstRed > 0 ? tested[firstRed - 1]!.sha : range.good, bad: firstRed < tested.length ? tested[firstRed]!.sha : range.bad }; +} + +function ghJson(env: GateEnv, args: string[]): T | null { + const r = env.gh(args, GH_MS); + if (r.status !== 0) return null; + try { return JSON.parse(r.stdout) as T; } catch { return null; } +} + +/** One file of a run's artifact, or null when the artifact or the file is missing. */ +function artifactFile(env: GateEnv, run: number, name: string, file: string): string | null { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'release-gate-')); + try { + if (env.gh(['run', 'download', String(run), '-R', env.repo, '-n', name, '-D', dir], GH_MS).status !== 0) return null; + const find = (d: string): string | null => { + for (const e of fs.readdirSync(d, { withFileTypes: true })) { + const p = path.join(d, e.name); + if (e.isDirectory()) { const hit = find(p); if (hit) return hit; } else if (e.name === file) return p; + } + return null; + }; + const hit = find(dir); + return hit ? fs.readFileSync(hit, 'utf8') : null; + } finally { fs.rmSync(dir, { recursive: true, force: true }); } +} + +function dispatch(env: GateEnv, workflow: string, ref: string, inputs: Record): string | null { + const r = env.gh(['workflow', 'run', workflow, '--repo', env.repo, '--ref', ref, ...Object.entries(inputs).flatMap(([k, v]) => ['-f', `${k}=${v}`])], GH_MS); + return r.status === 0 ? null : ghFailureReason(r.stderr); +} + +/** Find a dispatch's run by its nonce, then follow it until it completes or the deadline passes. */ +function follow(env: GateEnv, workflow: string, nonce: string, known: number | null, lines: string[]): Run | null { + let run: Run | null = known ? ghJson(env, ['api', `repos/${env.repo}/actions/runs/${known}`]) : null; + for (;;) { + if (!run) { + const runs = ghJson<{ workflow_runs?: Run[] }>(env, ['api', '-X', 'GET', `repos/${env.repo}/actions/workflows/${workflow}/runs`, '-f', 'event=workflow_dispatch', '-f', 'per_page=30']); + run = runs?.workflow_runs?.find(r => (r.display_title ?? '').includes(`eval-bg ${nonce}]`)) ?? null; + if (run) lines.push(`RELEASE: run ${run.id} ${run.html_url ?? ''}`.trim()); + } else if (run.status === 'completed') return run; + if (env.now() >= env.deadline) break; + env.sleep(env.pollMs); + if (run) run = ghJson(env, ['api', `repos/${env.repo}/actions/runs/${run.id}`]) ?? run; + } + lines.push(`RELEASE: ${run ? `run ${run.id} is still ${run.status}` : `no run for nonce ${nonce} yet`} at the deadline; the record stays resumable`); + return run; +} + +const runRef = (env: GateEnv, run: Run) => ({ id: run.id, url: run.html_url ?? `https://github.com/${env.repo}/actions/runs/${run.id}` }); + +function save(env: GateEnv, rec: ReleaseRecord, lines: string[]): void { + rec.updated_at = nowIso(); + const p = persist(env.repoRoot, [rec], { remote: env.remote }); + if (!p.pushed) lines.push(`RELEASE: record ${p.detail}`); +} + +interface Context { gate: ReleaseGatePolicy; repoKey: string; stable: string | null; records: ReleaseRecord[]; isAncestor: (a: string, b: string) => boolean } + +function context(env: GateEnv, gate: ReleaseGatePolicy): Context | { error: string } { + const f = fetchNotes(env.repoRoot, env.remote); + if (!f.ok) return { error: f.detail }; + const stable = env.git(['ls-remote', env.remote, `refs/heads/${gate.stable_ref}`]).out.split(/\s/)[0] || null; + const isAncestor = (a: string, b: string) => env.git(['merge-base', '--is-ancestor', a, b]).ok; + const repoKey = repoIdentity(env.repoRoot).key; + return { gate, repoKey, stable: stable && SHA.test(stable) ? stable : null, records: releaseRecords(env.git, repoKey), isAncestor }; +} + +/** Make sure the gated branch's history is here (a shallow checkout gets it), so ancestry and ranges read true. */ +function ensureHistory(env: GateEnv, gate: ReleaseGatePolicy, sha: string): void { + if (env.git(['rev-parse', '--is-shallow-repository']).out === 'true') env.git(['fetch', '--no-tags', '-q', '--unshallow', env.remote, gate.branch, gate.stable_ref]); + if (!env.git(['cat-file', '-e', `${sha}^{commit}`]).ok) env.git(['fetch', '--no-tags', '-q', env.remote, gate.branch]); +} + +const firstParents = (env: GateEnv, from: string, to: string): string[] | null => { + const r = env.git(['rev-list', '--first-parent', '--reverse', `${from}..${to}`]); + return r.ok ? r.out.split('\n').filter(Boolean) : null; +}; + +export interface RunOptions { gate: ReleaseGatePolicy | null; label: string; sha?: string; dryRun: boolean; ignoreCadence: boolean } + +/** `run`: gate the branch head once when it moved; bisect a red. */ +export function runGate(env: GateEnv, o: RunOptions): GateOut { + const lines: string[] = []; + const out = (action: GateAction, sha: string | null): GateOut => ({ action, sha, lines }); + if (!o.gate) { lines.push(`RELEASE: inert (${o.label} declares no release_gate block; the per-push main backstop gates main)`); return out('inert', null); } + const gate = o.gate; + const c = context(env, gate); + if ('error' in c) { lines.push(`RELEASE: cannot read refs/notes/gstack-trains (${c.error}); nothing dispatched`); return out('failed', null); } + const sha = o.sha ?? env.git(['ls-remote', env.remote, `refs/heads/${gate.branch}`]).out.split(/\s/)[0] ?? ''; + if (!SHA.test(sha)) { lines.push(`RELEASE: cannot resolve ${gate.branch} on ${env.remote} (${JSON.stringify(sha)})`); return out('failed', null); } + ensureHistory(env, gate, sha); + const prior = c.records.find(r => r.sha === sha) ?? null; + const skip = (why: string) => { lines.push(`RELEASE: skip sha=${sha} (${why})`); return out('skipped', sha); }; + if (sha === c.stable) return skip(`${gate.branch} has not moved since ${gate.stable_ref}`); + if (prior?.verdict === 'GREEN' || prior?.verdict === 'RED') { + if (prior.verdict === 'RED' && prior.bisect?.state === 'running' && !o.dryRun) return finishRed(env, c, prior, lines, out); + return skip(`${gate.branch} has not moved since the last gate: ${prior.verdict} run ${prior.run?.id ?? '-'}`); + } + if (prior?.verdict === 'INCOMPLETE' && prior.attempts >= MAX_RELEASE_ATTEMPTS) return skip(`the gate on this commit was incomplete ${prior.attempts} times (${prior.detail}); diagnose run ${prior.run?.url ?? prior.run?.id ?? '-'}`); + const newest = c.records[0]; + const age = newest ? (env.now() - Date.parse(newest.created_at)) / HOUR : Infinity; + if (!o.ignoreCadence && prior?.verdict !== 'PENDING' && age < gate.cadence_hours - CADENCE_GRACE_HOURS) { + return skip(`the last gate was dispatched ${age.toFixed(1)} h ago on ${short(newest!.sha)}; cadence ${gate.cadence_hours} h`); + } + const since = greenBase(c.records, sha, c.isAncestor)?.sha ?? c.stable; + const pending = since ? firstParents(env, since, sha)?.length ?? null : null; + lines.push(`RELEASE: gate sha=${sha} since=${short(since)} (${since === c.stable ? gate.stable_ref : 'last green gate'}) pending=${pending ?? 'unknown'} stable=${short(c.stable)}`); + if (o.dryRun) { lines.push(`RELEASE: dry run: would dispatch ${gate.gate_workflow} on ${gate.branch}@${short(sha)} (full gate, about $37); nothing written`); return out('dry-run', sha); } + const now = nowIso(); + const rec: ReleaseRecord = prior?.verdict === 'PENDING' ? prior : { + schema: 1, kind: 'release', repo_key: c.repoKey, sha, tree: env.git(['rev-parse', `${sha}^{tree}`]).out, branch: gate.branch, stable: c.stable, since, + nonce: makeNonce('gate', sha, sha, env.random(), { purpose: 'backstop' }), attempts: (prior?.attempts ?? 0) + 1, run: null, verdict: 'PENDING', + red_cases: [], flaky: [], cost_usd: null, detail: 'dispatching', bisect: null, created_at: now, updated_at: now, + }; + if (rec !== prior) { + save(env, rec, lines); + const failed = dispatch(env, gate.gate_workflow, gate.branch, { [DISPATCH_INPUTS.evalsAll]: 'true', [DISPATCH_INPUTS.expectedSha]: sha, [DISPATCH_INPUTS.nonce]: rec.nonce }); + if (failed) { Object.assign(rec, { verdict: 'INCOMPLETE', detail: `the gate dispatch failed (${failed})` }); save(env, rec, lines); lines.push(`RELEASE: incomplete sha=${sha} run=- (${rec.detail})`); return out('failed', sha); } + lines.push(`RELEASE: dispatched ${gate.gate_workflow} on ${gate.branch}@${short(sha)} (nonce ${rec.nonce})`); + } else lines.push(`RELEASE: resuming the pending gate on ${short(sha)} (nonce ${rec.nonce})`); + const run = follow(env, gate.gate_workflow, rec.nonce, rec.run?.id ?? null, lines); + if (run) rec.run = runRef(env, run); + if (!run || run.status !== 'completed') { rec.detail = run ? `run ${run.id} ${run.status}` : 'no run found for the nonce yet'; save(env, rec, lines); return out('failed', sha); } + const attempt = run.run_attempt ?? 1; + const reportText = artifactFile(env, run.id, `report-verdict-a${attempt}`, 'report.txt'); + const report = reportText === null ? null : parseReport(reportText); + const flakes = artifactFile(env, run.id, `flake-records-a${attempt}`, 'flake-records.jsonl') ?? ''; + const k = classifyRun(run, report); + Object.assign(rec, { + verdict: k.verdict, red_cases: k.red, detail: k.detail, cost_usd: report?.cost ?? null, + flaky: [...new Set(flakes.split('\n').flatMap(l => { try { const f = JSON.parse(l); return typeof f?.case === 'string' ? [f.case] : []; } catch { return []; } }))].sort(), + }); + save(env, rec, lines); + if (rec.flaky.length) lines.push(`RELEASE: flaky (failed once, passed its one rule-16 rerun in run ${run.id}): ${rec.flaky.join(', ')}`); + if (rec.verdict === 'GREEN') { + lines.push(`RELEASE: green sha=${sha} stable=${c.stable ?? 'none'} run=${run.id} cost=${usd(rec.cost_usd)}`, + `RELEASE: ${gate.check_name} success goes on ${short(sha)}; scripts/release-cut.ts moves ${gate.stable_ref} on its own schedule and floor`); + return out('gated', sha); + } + if (rec.verdict === 'RED') return finishRed(env, c, rec, lines, out); + lines.push(`RELEASE: incomplete sha=${sha} run=${run.id} (${rec.detail})${rec.attempts < MAX_RELEASE_ATTEMPTS ? '; the next run re-dispatches it once' : ''}`); + return out('failed', sha); +} + +function finishRed(env: GateEnv, c: Context, rec: ReleaseRecord, lines: string[], out: (a: GateAction, s: string | null) => GateOut): GateOut { + bisectRecord(env, c, rec, { spendCap: c.gate.bisect_spend_cap_usd, dryRun: false }, lines); + lines.push(`RELEASE: red sha=${rec.sha} first_bad=${rec.bisect?.first_bad ?? 'unknown'} run=${rec.run?.id ?? '-'}`, + `RELEASE: hold: scripts/release-cut.ts refuses to move ${c.gate.stable_ref} until a gate on a later commit is green (red: ${rec.red_cases.join(', ')}); land the revert or the fix-forward`); + return out('held', rec.sha); +} + +/** Collect a bisect round's prefix records into its results; the cost they report is added to the spend. */ +function collectRound(env: GateEnv, b: ReleaseBisect, round: ReleaseBisectRound, run: Run): void { + for (const p of round.prefixes) { + const text = artifactFile(env, run.id, `${RELEASE_BISECT_ARTIFACT}-bisect-${p.sha.slice(0, 12)}`, 'bisect-prefix.json'); + let rec: { kind?: string; nonce?: string; prefix_sha?: string; checked_out_sha?: string; panels?: Array<{ case: string; status: string; red_class: string | null; cost_usd: number | null }> } | null = null; + try { rec = text ? JSON.parse(text) : null; } catch { rec = null; } + if (!rec || rec.kind !== 'train-bisect-prefix' || rec.nonce !== round.nonce || rec.prefix_sha !== p.sha || rec.checked_out_sha !== p.sha || !Array.isArray(rec.panels)) continue; + for (const panel of rec.panels) { + if (!b.cases.includes(panel.case)) continue; + b.spent_usd = Math.round((b.spent_usd + (typeof panel.cost_usd === 'number' ? panel.cost_usd : 0)) * 100) / 100; + if (panel.status === 'PASS') p.results[panel.case] = 'green'; + else if (panel.status === 'FAIL' && panel.red_class === 'VERDICT') p.results[panel.case] = 'red'; + } + } +} + +/** + * Bisect a red record (resumable): finish a round still running, then narrow + * (good, bad] with up to MAX_PREFIXES commits a round until they are adjacent, + * the pattern is not monotone, the cap is spent, or MAX_BISECT_ROUNDS pass. + */ +export function bisectRecord(env: GateEnv, c: Context, rec: ReleaseRecord, o: { spendCap: number; dryRun: boolean }, lines: string[]): void { + const b: ReleaseBisect = rec.bisect ?? { cases: rec.red_cases, good: rec.since, bad: rec.sha, first_bad: null, spend_cap: o.spendCap, spent_usd: 0, state: 'running', detail: '', rounds: [] }; + b.spend_cap = Math.max(b.spend_cap, o.spendCap); + const stop = (state: ReleaseBisect['state'], detail: string) => { Object.assign(b, { state, detail }); }; + if (!o.dryRun) rec.bisect = b; + if (b.state === 'running' && o.spendCap <= 0 && !b.rounds.length) stop('unknown', 'bisect_spend_cap_usd is 0: no bisect'); + else if (b.state === 'running' && (!b.cases.length || b.cases.length > MAX_BISECT_CASES || b.cases.some(id => !CASE_ID.test(id)))) stop('unknown', `the red cases cannot be bisected (${b.cases.length} case ids, at most ${MAX_BISECT_CASES})`); + else if (b.state === 'running' && !b.good) stop('unknown', 'no green gate and no stable branch before the red commit to bisect from'); + while (b.state === 'running') { + const open = b.rounds.at(-1); + if (open && open.prefixes.some(p => b.cases.some(cs => !p.results[cs])) && open.run?.id !== -1) { + if (o.dryRun) { lines.push(`RELEASE: dry run: round ${b.rounds.length} (nonce ${open.nonce}) still has prefixes to collect`); return; } + const run = follow(env, c.gate.gate_workflow, open.nonce, open.run?.id ?? null, lines); + if (run) open.run = runRef(env, run); + if (!run || run.status !== 'completed') { b.detail = `round ${b.rounds.length} is still running`; save(env, rec, lines); return; } + collectRound(env, b, open, run); + env.git(['push', '-q', env.remote, '--delete', `refs/heads/${open.ref}`]); + const n = narrow({ good: b.good!, bad: b.bad }, open.prefixes, b.cases); + if ('unknown' in n) { stop('unknown', n.unknown); break; } + Object.assign(b, n); + lines.push(`RELEASE: bisect round ${b.rounds.length}: good=${short(b.good)} bad=${short(b.bad)} spent=$${b.spent_usd.toFixed(2)}`); + save(env, rec, lines); + } + const interior = firstParents(env, b.good!, b.bad); + if (!interior) { stop('unknown', `cannot list ${short(b.good)}..${short(b.bad)}`); break; } + interior.pop(); + if (!interior.length) { b.first_bad = b.bad; stop('found', `${short(b.bad)} is the first red commit after green ${short(b.good)}`); break; } + if (b.rounds.length >= MAX_BISECT_ROUNDS) { stop('unknown', `${MAX_BISECT_ROUNDS} rounds left ${interior.length} commit(s) between ${short(b.good)} and ${short(b.bad)}`); break; } + const perPrefix = b.cases.length * RELEASE_TRIAL_RESERVATION_USD; + const fits = Math.floor((b.spend_cap - b.spent_usd + 1e-9) / perPrefix); + const prefixes = planPrefixes(interior, Math.min(MAX_PREFIXES, fits)); + if (!prefixes.length) { stop('capped', `the next round needs $${perPrefix.toFixed(2)} a prefix; $${(b.spend_cap - b.spent_usd).toFixed(2)} of the $${b.spend_cap.toFixed(2)} cap is left`); break; } + lines.push(`RELEASE: bisect round ${b.rounds.length + 1}: ${b.cases.length} case(s) on ${prefixes.length} of ${interior.length} commit(s) between ${short(b.good)} and ${short(b.bad)}, $${(perPrefix * prefixes.length).toFixed(2)} reserved`); + if (o.dryRun) { lines.push('RELEASE: dry run: nothing dispatched'); return; } + const nonce = makeNonce('gate', b.bad, b.good!, env.random(), { purpose: 'bisect' }); + const ref = `${RELEASE_BISECT_REF}${nonce.slice(-6)}-${b.bad.slice(0, 12)}`; + const round: ReleaseBisectRound = { nonce, ref, run: null, prefixes: prefixes.map(sha => ({ sha, results: {} })) }; + b.rounds.push(round); + save(env, rec, lines); + const pushed = env.git(['push', '-q', env.remote, `${b.bad}:refs/heads/${ref}`]); + const failed = pushed.ok ? dispatch(env, c.gate.gate_workflow, ref, { + [DISPATCH_INPUTS.cases]: b.cases.join(','), [DISPATCH_INPUTS.prefixShas]: prefixes.join(','), [DISPATCH_INPUTS.expectedSha]: b.bad, [DISPATCH_INPUTS.nonce]: nonce, + }) : `git push ${ref} failed: ${pushed.out.slice(0, 120)}`; + if (failed) { round.run = { id: -1, url: '' }; env.git(['push', '-q', env.remote, '--delete', `refs/heads/${ref}`]); stop('unknown', `round ${b.rounds.length} dispatch failed: ${failed}`); } + } + lines.push(`RELEASE: bisect ${b.state}: ${b.detail} (spent $${b.spent_usd.toFixed(2)} of $${b.spend_cap.toFixed(2)})`); + if (!o.dryRun) save(env, rec, lines); +} + +/** `bisect`: resume or start the bisect of the newest red record (or the one at sha). */ +export function bisectGate(env: GateEnv, o: { gate: ReleaseGatePolicy | null; label: string; sha?: string; spendCap?: number; dryRun: boolean }): GateOut { + const lines: string[] = []; + if (!o.gate) { lines.push(`RELEASE: inert (${o.label} declares no release_gate block)`); return { action: 'inert', sha: null, lines }; } + const c = context(env, o.gate); + if ('error' in c) { lines.push(`RELEASE: cannot read refs/notes/gstack-trains (${c.error})`); return { action: 'failed', sha: null, lines }; } + const rec = o.sha ? c.records.find(r => r.sha === o.sha) : c.records.find(r => r.verdict === 'RED'); + if (!rec || rec.verdict !== 'RED') { lines.push(`RELEASE: no red release record${o.sha ? ` at ${o.sha}` : ''} to bisect`); return { action: 'failed', sha: o.sha ?? null, lines }; } + ensureHistory(env, o.gate, rec.sha); + if (rec.bisect?.state === 'capped' && (o.spendCap ?? 0) > rec.bisect.spend_cap) rec.bisect.state = 'running'; + bisectRecord(env, c, rec, { spendCap: o.spendCap ?? o.gate.bisect_spend_cap_usd, dryRun: o.dryRun }, lines); + lines.push(`RELEASE: red sha=${rec.sha} first_bad=${rec.bisect?.first_bad ?? 'unknown'} run=${rec.run?.id ?? '-'}`); + return { action: o.dryRun ? 'dry-run' : 'held', sha: rec.sha, lines }; +} + +/** `status`: never spends and never writes. */ +export function gateStatus(env: GateEnv, o: { gate: ReleaseGatePolicy | null; label: string }): GateOut { + const lines: string[] = []; + if (!o.gate) { lines.push(`RELEASE: inert (${o.label} declares no release_gate block; the per-push main backstop gates main)`); return { action: 'inert', sha: null, lines }; } + const c = context(env, o.gate); + if ('error' in c) { lines.push(`RELEASE: cannot read refs/notes/gstack-trains (${c.error})`); return { action: 'failed', sha: null, lines }; } + const head = env.git(['ls-remote', env.remote, `refs/heads/${o.gate.branch}`]).out.split(/\s/)[0] ?? ''; + if (SHA.test(head)) ensureHistory(env, o.gate, head); + const last = c.records[0]; + const base = SHA.test(head) ? greenBase(c.records, head, c.isAncestor)?.sha ?? c.stable : null; + const pending = base && SHA.test(head) ? (base === head ? 0 : firstParents(env, base, head)?.length ?? null) : null; + const hold = activeHold(c.records, c.isAncestor); + lines.push( + `STABLE: ${o.gate.stable_ref} ${c.stable ?? 'absent'} (scripts/release-cut.ts moves it; the gate never does)`, + `LAST_GATE: ${last ? `${last.verdict} sha=${last.sha} run=${last.run?.id ?? '-'} at=${last.created_at} cost=${usd(last.cost_usd)}${last.flaky.length ? ` flaky=${last.flaky.join(',')}` : ''}` : 'none'}`, + `PENDING: ${pending ?? 'unknown'} commit(s) on ${o.gate.branch} ${short(head)} since ${short(base)}`, + `HOLD: ${hold ? `red on ${hold.sha} since ${hold.created_at} (red: ${hold.red_cases.join(', ')}; first_bad=${hold.bisect?.first_bad ?? 'unknown'}, bisect ${hold.bisect?.state ?? 'not run'}); release-cut refuses until a gate on a later commit is green` : 'none'}`, + ); + return { action: hold ? 'held' : 'skipped', sha: SHA.test(head) ? head : null, lines }; +} diff --git a/lib/ship-policy.ts b/lib/ship-policy.ts index 4268b007e..61d520ced 100644 --- a/lib/ship-policy.ts +++ b/lib/ship-policy.ts @@ -15,6 +15,7 @@ import { spawnSync } from 'node:child_process'; import * as fs from 'node:fs'; import * as path from 'node:path'; +import { RELEASE_GATE_EXPLANATION, releaseGateErrors, type ReleaseGatePolicy } from './release-gate-policy'; import { RESULT_CODES, type ResultCodeName } from './result-codes'; import { cmpVersion, parseVersion } from './version-source'; @@ -77,6 +78,8 @@ export interface ShipPolicy { docs: { caps: Array<{ path: string; max_bytes: number }> } | null; /** How /ship reproduces a Step 5 failure on the base branch before calling it base-red (docs/ship-policy.md#base-red); null never calls one base-red. */ base_red: BaseRedPolicy | null; + /** The release gate on the base branch (docs/ship-policy.md#release-gate); null keeps the per-push main backstop. */ + release_gate: ReleaseGatePolicy | null; } /** `base_red`: `setup` installs dependencies in a scratch checkout (null when none are needed); `select` runs the failing test files, named by its `{files}` slot. */ @@ -165,6 +168,7 @@ export const DEFAULT_POLICY: ShipPolicy = { train: null, docs: null, base_red: null, + release_gate: null, }; export type PolicyKey = Exclude; @@ -195,6 +199,7 @@ export const POLICY_EXPLANATIONS: Record = { train: '{ gate_workflow, free_workflow, full_gate_check, record_artifact, max_size, fill_hours, screen_selection, trial_reservation_usd, land? } (the gstack-evals gate) or { gates, land, max_size, fill_hours } (any repo: gates lists { kind: workflows, workflows, required_jobs, record?, post_merge_checks? } and { kind: command, command, timeout_sec, spend_cap_usd, pass_env? }, each GREEN on every prefix; land is { merge_method: squash|merge|rebase, required_checks? }) or null: turns on merge trains in gstack-ship-queue (train build|status|verdict|land|watch|init|doctor|prune, ready --screen|--no-screen, verify --train). gates and land need min_gstack_version at the release that reads them (validate names it). null keeps today\'s per-PR queue unchanged.', docs: '{ caps: [{ path, max_bytes }] } or null: documentation files with a byte cap. /ship Step 14.5 runs `gstack-ship-receipt docs-caps` before the verification gate and prints one DOCS_CAP: line per cap; a file over its cap (or absent) blocks the ship until the doc is trimmed or the cap raised on the base branch. null declares none.', base_red: '{ setup, select, timeout_sec } or null: how /ship Step 5 proves a failure is the base branch\'s own (docs/ship-policy.md#base-red). gstack-base-red checks out the base head and the PR head in scratch worktrees, runs setup (dependency install, or null), then select with the failing test files in its {files} slot; only the same failures in both, in files the PR does not touch, are base-red, and /ship then pauses before READY instead of failing. Read from the base branch only. null keeps every failure in-branch.', + release_gate: RELEASE_GATE_EXPLANATION, min_gstack_version: 'The oldest gstack that may read this file. Any key newer than 1.91.98.0 set to a non-default value requires it at that key’s introducing version (validate prints each); a gstack older than it refuses the file (POLICY_GSTACK_TOO_OLD).', }; @@ -231,6 +236,7 @@ export const BUMP_NEVER_KEY_VERSION = '1.91.353.0'; export const BASE_RED_KEY_VERSION = '1.91.374.0'; /** `train.gates` and `train.land` are values of the older train key, so compatErrors checks them by value, as for bump: never. */ export const TRAIN_GATES_KEY_VERSION = '1.92.0.0'; +export const RELEASE_GATE_KEY_VERSION = '1.94.0.0'; export const POLICY_KEY_INTRODUCED: Record = { bump: '1.91.73.0', queue_mode: '1.91.73.0', release_tool: '1.91.73.0', release_outputs: '1.91.73.0', stamp_paths: '1.91.73.0', changelog: '1.91.73.0', mirror: '1.91.73.0', history: '1.91.73.0', preregistration_shas: '1.91.73.0', stamp_sensitive_gates: '1.91.73.0', @@ -238,7 +244,7 @@ export const POLICY_KEY_INTRODUCED: Record = { migrations: '1.91.98.0', gate_command: SHIP_QUEUE_KEYS_VERSION, full_e2e: SHIP_QUEUE_KEYS_VERSION, min_gstack_version: SHIP_QUEUE_KEYS_VERSION, train: TRAIN_KEYS_VERSION, review_scope: REVIEW_SCOPE_KEY_VERSION, docs: DOCS_CAPS_KEY_VERSION, changelog_fragments: CHANGELOG_FRAGMENTS_KEY_VERSION, - base_red: BASE_RED_KEY_VERSION, + base_red: BASE_RED_KEY_VERSION, release_gate: RELEASE_GATE_KEY_VERSION, }; const newer = (a: string, b: string): boolean => cmpVersion(parseVersion(a)!, parseVersion(b)!) > 0; @@ -496,6 +502,7 @@ export function validatePolicy(raw: unknown, opts: { gstackVersion?: string } = if ('docs' in obj && obj.docs !== null) docsErrors(obj.docs, errors); if ('changelog_fragments' in obj && obj.changelog_fragments !== null) fragmentsErrors(obj.changelog_fragments, errors); if ('base_red' in obj && obj.base_red !== null) baseRedErrors(obj.base_red, errors); + if ('release_gate' in obj && obj.release_gate !== null) errors.push(...releaseGateErrors(obj.release_gate)); if ('stamp_paths' in obj) listErrors('stamp_paths', obj.stamp_paths, errors, globError); for (const key of ['changelog', 'mirror'] as const) { if (!(key in obj) || obj[key] === null) continue; diff --git a/lib/ship-queue-train-record.ts b/lib/ship-queue-train-record.ts index 5d8f7df97..0f38d14f1 100644 --- a/lib/ship-queue-train-record.ts +++ b/lib/ship-queue-train-record.ts @@ -117,7 +117,20 @@ export interface ReadyReceipt { schema: 1; kind: 'ready'; repo_key: string; pr: string; head: string; base_sha: string; ready_at: string; screen: ScreenChoice; updated_at: string; base_red?: BaseRedCondition; fixes_base_red?: string[]; } -export type AnyRecord = TrainRecord | AttemptRecord | BisectRecord | FreezeRecord | ReadyReceipt | WaveRecord; +/** One release-gate bisect round (lib/release-gate.ts): the prefixes it dispatched on release-gate/ and what each case did there. */ +export interface ReleaseBisectRound { nonce: string; ref: string; run: { id: number; url: string } | null; prefixes: Array<{ sha: string; results: Record }> } +/** A red release gate's bisect over the first-parent commits after the last green gate: first_bad once good and bad are adjacent. */ +export interface ReleaseBisect { + cases: string[]; good: string | null; bad: string; first_bad: string | null; spend_cap: number; spent_usd: number; + state: 'running' | 'found' | 'unknown' | 'capped'; detail: string; rounds: ReleaseBisectRound[]; +} +/** The release gate's verdict on one commit of the gated branch (lib/release-gate.ts), the evidence scripts/release-cut.ts reads. */ +export interface ReleaseRecord { + schema: 1; kind: 'release'; repo_key: string; sha: string; tree: string; branch: string; stable: string | null; since: string | null; + nonce: string; attempts: number; run: { id: number; url: string } | null; verdict: 'PENDING' | 'GREEN' | 'RED' | 'INCOMPLETE'; + red_cases: string[]; flaky: string[]; cost_usd: number | null; detail: string; bisect: ReleaseBisect | null; created_at: string; updated_at: string; +} +export type AnyRecord = TrainRecord | AttemptRecord | BisectRecord | FreezeRecord | ReadyReceipt | WaveRecord | ReleaseRecord; /** Clock seam for fill-time and record tests (`GSTACK_SHIP_QUEUE_NOW`, ISO-8601). */ export function nowIso(): string { @@ -141,6 +154,7 @@ const bisectPath = (nonce: string) => path.join(trainDir(nonce), 'bisect.json'); const freezePath = (repoKey: string) => path.join(queueRoot(), 'freeze', `${repoKey}.json`); export const readyPath = (repoKey: string, pr: string, head: string) => path.join(queueRoot(), 'ready', repoKey, `${pr}-${head}.json`); export const wavePath = (nonce: string) => path.join(queueRoot(), 'waves', `${nonce}.json`); +export const releasePath = (repoKey: string, sha: string) => path.join(queueRoot(), 'release', repoKey, `${sha}.json`); /** Hosts that serve GitHub repositories at // (git.capy.ai is Capy's GitHub proxy, set through url.insteadOf). */ const GITHUB_HOSTS = ['github.com', 'www.github.com', 'ssh.github.com', 'git.capy.ai']; @@ -226,6 +240,7 @@ export function recordPath(r: AnyRecord): string { if (r.kind === 'train-bisect') return bisectPath(r.train); if (r.kind === 'freeze') return freezePath(r.repo_key); if (r.kind === 'wave') return wavePath(r.nonce); + if (r.kind === 'release') return releasePath(r.repo_key, r.sha); return readyPath(r.repo_key, r.pr, r.head); } @@ -235,6 +250,7 @@ export function noteName(r: AnyRecord): string { if (r.kind === 'train-bisect') return `gstack-trains:bisect:${r.train}`; if (r.kind === 'freeze') return `gstack-trains:freeze:${r.repo_key}`; if (r.kind === 'wave') return `gstack-trains:wave:${r.nonce}`; + if (r.kind === 'release') return `gstack-trains:release:${r.repo_key}:${r.sha}`; return `gstack-trains:ready:${r.repo_key}:${r.pr}:${r.head}`; } @@ -299,6 +315,7 @@ function parseNote(text: string): AnyRecord | null { if ((r.kind === 'train-attempt' && ids.test(r.train) && ids.test(r.id)) || (r.kind === 'train-bisect' && ids.test(r.train))) return r; if (r.kind === 'freeze' && ids.test(r.repo_key)) return r; if (r.kind === 'wave' && ids.test(r.nonce) && ids.test(r.repo_key) && Array.isArray(r.constituents)) return r; + if (r.kind === 'release' && ids.test(r.repo_key) && /^[0-9a-f]{40,64}$/.test(r.sha)) return r; if (r.kind === 'ready' && ids.test(r.repo_key) && /^\d+$/.test(r.pr) && /^[0-9a-f]{40,64}$/.test(r.head)) return r; } catch {} return null; diff --git a/scripts/release-cut.ts b/scripts/release-cut.ts index 16efe3e21..9dbe1caa0 100755 --- a/scripts/release-cut.ts +++ b/scripts/release-cut.ts @@ -19,7 +19,7 @@ * the window (7 days, 200 first-parent commits) is the candidate; a commit * with a check run still going is skipped, not waited for. * - * Gated means one of two kinds of evidence. Either the backstop check run + * Gated means one of three kinds of evidence. Either the backstop check run * (--backstop-check, default main-backstop) succeeded: evals-main-backstop.yml * publishes it, success only when a green train record covers the tree or the * backstop gated it green, neutral when main moved past the push before it was @@ -27,8 +27,11 @@ * or merged) lists the commit's sha or its tree in merged[]: those entries are * the main commits `gstack-ship-queue verify --train` confirmed, so a neutral * or absent backstop run on such a commit is not held against it, and the - * CANDIDATE line says "gated by train ". A red backstop run stays red - * either way. A commit with neither is not green. + * CANDIDATE line says "gated by train ". Or a GREEN release record + * (kind release, lib/release-gate.ts) names the commit: "gated by release gate + * run ". A red backstop run stays red either way. A commit with none of + * these is not green. A RED release record with no GREEN one on a later + * commit that contains it holds every cut, like a freeze. * * stable moves only forward: a candidate behind the current stable is refused * (REFUSED non-fast-forward), as is a tag v that already names another @@ -99,8 +102,8 @@ export function compareVersions(a: string, b: string): number { export type Verdict = { green: true; via: string } | { green: false; why: string; pending: boolean }; interface CheckRun { name: string; status: string; conclusion: string | null } -/** Main commits a green, merging or merged train record lists in merged[], by sha and by tree, each naming the train. */ -export interface TrainEvidence { bySha: Map; byTree: Map } +/** Main commits a green, merging or merged train record lists in merged[], by sha and by tree, each naming the train; `release` maps a commit a green release gate passed to its run (lib/release-gate.ts). */ +export interface TrainEvidence { bySha: Map; byTree: Map; release?: Map } export interface GateOptions { requiredCheck?: string; backstopCheck?: string; evidence?: TrainEvidence; treeOf?: (sha: string) => string | null } /** Why a commit is not releasable, or green and through which evidence. */ @@ -118,6 +121,8 @@ export function gateVerdict(gh: Gh, repo: string, sha: string, o: GateOptions = if (named.some(c => c.conclusion === 'success')) continue; const train = name === backstop && name !== required && o.evidence ? o.evidence.bySha.get(sha) ?? o.evidence.byTree.get(o.treeOf?.(sha) ?? '') : undefined; if (train) { via = `train ${train}`; continue; } + const gate = name === backstop && name !== required ? o.evidence?.release?.get(sha) : undefined; + if (gate) { via = `release gate run ${gate}`; continue; } if (named.length === 0) return { green: false, why: `no ${name} check run`, pending: false }; return { green: false, why: `${name} concluded ${named.map(c => c.conclusion).join(',')}`, pending: false }; } @@ -129,14 +134,20 @@ export function gateVerdict(gh: Gh, repo: string, sha: string, o: GateOptions = } export interface TrainNotes { error: string | null; freeze: string | null; evidence: TrainEvidence } -interface NoteRecord { kind?: string; state?: string; nonce?: string; merged?: Array<{ sha?: string; tree?: string }>; cleared_at?: string | null; reason?: string; train?: string } +interface NoteRecord { + kind?: string; state?: string; nonce?: string; merged?: Array<{ sha?: string; tree?: string }>; cleared_at?: string | null; reason?: string; train?: string; + sha?: string; verdict?: string; run?: { id?: number } | null; red_cases?: string[]; +} /** * One read of the trains notes ref (fetched from origin): the active freeze * record (kind freeze, cleared_at null) and the train evidence (lib/ * ship-queue-train-record.ts TrainRecord: kind train, state green, merging or * merged, merged[].sha and .tree). A fetch failure other than a missing ref is - * an error; a missing ref holds nothing. + * an error; a missing ref holds nothing. Release-gate records (kind release, + * lib/release-gate.ts) add evidence when GREEN, and a RED one with no GREEN + * record on a later commit that contains it is the hold: it refuses the cut + * like a freeze (a train freeze, when both exist, is the one named). */ export function readTrainNotes(git: Git, notesRef = NOTES_REF): TrainNotes { const notes: TrainNotes = { error: null, freeze: null, evidence: { bySha: new Map(), byTree: new Map() } }; @@ -147,6 +158,8 @@ export function readTrainNotes(git: Git, notesRef = NOTES_REF): TrainNotes { } const listed = git(['notes', `--ref=${NOTES_LOCAL}`, 'list']); if (!listed.ok || !listed.out.trim()) return notes; + const gates: Array<{ sha: string; verdict: string; red: string[] }> = []; + notes.evidence.release = new Map(); for (const line of listed.out.split('\n')) { const blob = line.split(' ')[0]; if (!blob) continue; @@ -155,6 +168,10 @@ export function readTrainNotes(git: Git, notesRef = NOTES_REF): TrainNotes { if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) continue; const r = parsed as NoteRecord; if (r.kind === 'freeze' && !r.cleared_at) notes.freeze = `queue frozen since train ${r.train ?? '?'}: ${r.reason ?? 'red backstop'}`; + if (r.kind === 'release' && typeof r.sha === 'string' && /^[0-9a-f]{40}$/.test(r.sha)) { + gates.push({ sha: r.sha, verdict: String(r.verdict), red: Array.isArray(r.red_cases) ? r.red_cases.filter(c => typeof c === 'string') : [] }); + if (r.verdict === 'GREEN') notes.evidence.release.set(r.sha, String(r.run?.id ?? '?')); + } if (r.kind !== 'train' || typeof r.nonce !== 'string' || !r.nonce || !['green', 'merging', 'merged'].includes(r.state ?? '')) continue; for (const m of Array.isArray(r.merged) ? r.merged : []) { if (!m || typeof m !== 'object') continue; @@ -162,6 +179,8 @@ export function readTrainNotes(git: Git, notesRef = NOTES_REF): TrainNotes { if (typeof m.tree === 'string' && m.tree) notes.evidence.byTree.set(m.tree, r.nonce); } } + const held = gates.find(red => red.verdict === 'RED' && !gates.some(g => g.verdict === 'GREEN' && g.sha !== red.sha && git(['merge-base', '--is-ancestor', red.sha, g.sha]).ok)); + if (held && !notes.freeze) notes.freeze = `release gate red on ${held.sha.slice(0, 12)} (${held.red.join(', ') || 'red cases not recorded'}) and no gate on a later commit is green; land the revert or fix-forward, then run gstack-release-gate run --ignore-cadence`; return notes; } diff --git a/scripts/train-gate.ts b/scripts/train-gate.ts index 1613dfe2b..5baa4818d 100644 --- a/scripts/train-gate.ts +++ b/scripts/train-gate.ts @@ -61,8 +61,14 @@ export type RunMode = 'screen' | 'train' | 'bisect' | 'dispatch'; export interface RouteInput { nonce: string; refName: string; head: string; expectedSha: string; evalsAll: boolean; trainPolicy: boolean; cases: string; prefixShas: string; screenPr: string; screenMergeSha: string; screenPolicy: string; + /** The default branch's policy declares a release_gate block (lib/release-gate-policy.ts). */ + releasePolicy?: boolean; } -export interface Route { mode: RunMode; cases: string[]; prefixes: string[] } +/** artifact: the prefix of a release bisect's per-prefix records (a train bisect uses the train block's record_artifact). */ +export interface Route { mode: RunMode; cases: string[]; prefixes: string[]; artifact?: string } +/** The release gate's bisect runs on its own ref and names its own prefix records (lib/release-gate.ts RELEASE_REF_PREFIX, RELEASE_BISECT_ARTIFACT). */ +export const RELEASE_BISECT_REF = 'release-gate/'; +export const RELEASE_BISECT_ARTIFACT = 'gstack-release-record'; const list = (text: string) => text.split(',').map(item => item.trim()).filter(Boolean); @@ -88,7 +94,8 @@ export function routeDispatch(i: RouteInput, deps: { isAncestor(commit: string, const cases = list(i.cases); const prefixes = list(i.prefixShas); if (scope?.purpose !== 'bisect') return { error: `a bisect needs a bisect nonce (got ${JSON.stringify(i.nonce)})` }; - if (!i.refName.startsWith('train/')) return { error: `a bisect runs on train/, not ${i.refName}` }; + const release = i.refName.startsWith(RELEASE_BISECT_REF); + if (!release && !i.refName.startsWith('train/')) return { error: `a bisect runs on train/ (or the release gate's ${RELEASE_BISECT_REF}), not ${i.refName}` }; if (i.expectedSha !== i.head) return { error: `expected_sha ${i.expectedSha.slice(0, 12) || '(empty)'} is not the train head ${i.head.slice(0, 12)} checked out from ${i.refName}` }; if (!cases.length || cases.length > MAX_BISECT_CASES || new Set(cases).size !== cases.length) return { error: `cases must name 1 to ${MAX_BISECT_CASES} distinct case ids` }; if (!prefixes.length || prefixes.length > MAX_PREFIXES || new Set(prefixes).size !== prefixes.length) return { error: `prefix_shas must name 1 to ${MAX_PREFIXES} distinct commits` }; @@ -99,6 +106,7 @@ export function routeDispatch(i: RouteInput, deps: { isAncestor(commit: string, const plan = deps.gateCases(); const unknown = cases.filter(id => !CASE_ID.test(id) || !plan.has(id)); if (unknown.length) return { error: `case(s) not in the full gate's plan: ${unknown.slice(0, 5).join(', ')}` }; + if (release) return i.releasePolicy ? { mode: 'bisect', cases, prefixes, artifact: RELEASE_BISECT_ARTIFACT } : { error: "a release bisect needs a release_gate block in the default branch's .gstack/ship-policy.json" }; return trainOnly('bisect') ?? { mode: 'bisect', cases, prefixes }; } if (scope?.purpose === 'train') { @@ -359,12 +367,12 @@ if (import.meta.main) { if (command === 'route') { const env = process.env; const route = routeDispatch({ nonce: env.NONCE ?? '', refName: env.GITHUB_REF_NAME ?? '', head: git(['rev-parse', 'HEAD']).out, expectedSha: env.EXPECTED_SHA ?? '', - evalsAll: env.EVALS_ALL === 'true', trainPolicy: env.TRAIN_POLICY === 'true', cases: env.CASES ?? '', prefixShas: env.PREFIX_SHAS ?? '', + evalsAll: env.EVALS_ALL === 'true', trainPolicy: env.TRAIN_POLICY === 'true', releasePolicy: env.RELEASE_POLICY === 'true', cases: env.CASES ?? '', prefixShas: env.PREFIX_SHAS ?? '', screenPr: env.SCREEN_PR ?? '', screenMergeSha: env.SCREEN_MERGE_SHA ?? '', screenPolicy: env.SCREEN_POLICY ?? '' }, { isAncestor: (commit, head) => git(['merge-base', '--is-ancestor', commit, head]).ok, gateCases: () => fullGateCases(process.cwd()) }); if ('error' in route) fail(`dispatch refused before any paid work: ${route.error}`); const ok = route as Route; - output([`mode=${ok.mode}`, `cases=${ok.cases.join(',')}`, `prefixes=${ok.prefixes.join(',')}`]); + output([`mode=${ok.mode}`, `cases=${ok.cases.join(',')}`, `prefixes=${ok.prefixes.join(',')}`, ...(ok.artifact ? [`record_artifact=${ok.artifact}`] : [])]); console.log(`- Run mode: ${ok.mode}${ok.prefixes.length ? ` (${ok.cases.length} case(s) on ${ok.prefixes.length} prefix(es))` : ''}`); } else if (command === 'bisect-plan') { const [cases, prefixes, nonce, out] = [list(flag('cases') ?? ''), list(flag('prefixes') ?? ''), flag('nonce'), flag('out')]; diff --git a/test/capy-skill-wrapper.test.ts b/test/capy-skill-wrapper.test.ts index 4fbc82da3..fe3f066b4 100644 --- a/test/capy-skill-wrapper.test.ts +++ b/test/capy-skill-wrapper.test.ts @@ -53,14 +53,18 @@ const UPGRADE_DEPS = [ 'bin/gstack-upgrade', 'bin/gstack-bun-version.sh', 'bin/gstack-paths', 'bin/gstack-state-root.sh', 'bin/gstack-egress-lib.sh', 'bin/gstack-egress-receipt', 'lib/egress-receipt.ts', 'lib/state-root.ts', 'bin/gstack-lease', 'bin/gstack-harness-pid.sh', + // Sourced by gstack-upgrade: without gstack-config in the fixture it resolves main unless GSTACK_UPDATE_CHANNEL says stable. + 'bin/gstack-update-channel.sh', ]; /** * cloneAt 'A' (default): the clone sits at A (1.0.0), below B's minimum 1.0.1. * cloneAt 'B': the clone sits at B, AT the minimum, and the remote moves on to * C (1.0.2, minimum unchanged): behind origin but not below the floor. + * next: the remote moves on to C for a clone at A too. stable: origin's stable + * branch (the release, docs/releases.md) at that commit. */ -function fixture(opts: { minWrapper?: number; cloneAt?: 'A' | 'B' } = {}) { +function fixture(opts: { minWrapper?: number; cloneAt?: 'A' | 'B'; next?: boolean; stable?: 'A' | 'B' | 'C' } = {}) { const host = capyHost({ systemBun: '1.4.2' }); hosts.push(host); const git = (args: string[], cwd: string) => { @@ -94,13 +98,14 @@ function fixture(opts: { minWrapper?: number; cloneAt?: 'A' | 'B' } = {}) { const b = git(['rev-parse', 'HEAD'], seed); git(['push', '-q', bare, 'main'], seed); let c = ''; - if (opts.cloneAt === 'B') { - git(['clone', '-q', '--single-branch', '--depth', '1', `file://${bare}`, root], host.base); + if (opts.cloneAt === 'B' || opts.next) { + if (opts.cloneAt === 'B') git(['clone', '-q', '--single-branch', '--depth', '1', `file://${bare}`, root], host.base); write(path.join(seed, 'VERSION'), '1.0.2\n'); git(['commit', '-qam', 'C'], seed); c = git(['rev-parse', 'HEAD'], seed); git(['push', '-q', bare, 'main'], seed); } + if (opts.stable) git(['push', '-q', bare, `${{ A: a, B: b, C: c }[opts.stable]}:refs/heads/stable`], seed); const env = { ...host.env, GSTACK_CAPY_ROOT: root, GSTACK_CAPY_REPO: `file://${bare}`, WRAP_LOG: log, WRAP_FAIL_ONCE: path.join(host.base, 'fail-once') }; const run = (args: string[] = [], extra: Record = {}) => { const r = spawnSync('bash', [BOOT, ...args], { cwd: host.home, encoding: 'utf8', env: { ...env, ...extra }, timeout: 120_000 }); @@ -140,13 +145,13 @@ describe('capy skill bootstrapper (capy/skill/gstack/scripts/install.sh)', () => expect(f.git(['rev-parse', 'HEAD'])).toBe(f.b); expect(f.git(['rev-parse', '--is-shallow-repository'])).toBe('true'); expect(install.last).toBe('gstack: ok 1.0.1 for=autoplan'); - expect(f.calls()).toEqual([`installer --root ${f.root} --for autoplan lock=1 rev=3 min=1.0.1 from=1.0.0`]); + expect(f.calls()).toEqual([`installer --root ${f.root} --for autoplan lock=1 rev=4 min=1.0.1 from=1.0.0`]); const next = f.run(['--check', '--for', 'autoplan']); expect(next.status, next.stdout + next.stderr).toBe(0); expect(next.stdout).not.toContain('upgraded'); expect(next.last).toBe('gstack: ok 1.0.1 for=autoplan'); - expect(f.calls().at(-1)).toBe(`installer --check --root ${f.root} --for autoplan lock= rev=3 min=1.0.1 from=`); + expect(f.calls().at(-1)).toBe(`installer --check --root ${f.root} --for autoplan lock= rev=4 min=1.0.1 from=`); expect(f.git(['rev-parse', 'HEAD'])).toBe(f.b); }); @@ -263,7 +268,7 @@ describe('capy skill: always current when online (the minimum is the floor, not expect(r.stdout).toMatch(/^SELF_UPGRADE: done 1\.0\.1 -> 1\.0\.2$/m); expect(f.git(['rev-parse', 'HEAD'])).toBe(f.c); expect(r.last).toBe('gstack: ok 1.0.2 for=autoplan'); - expect(f.calls()).toEqual([`installer --root ${f.root} --for autoplan lock=1 rev=3 min=1.0.1 from=1.0.1`]); + expect(f.calls()).toEqual([`installer --root ${f.root} --for autoplan lock=1 rev=4 min=1.0.1 from=1.0.1`]); expect(fs.existsSync(path.join(f.root, '.git/gstack-upgrade.pending'))).toBe(false); const next = f.run(['--for', 'autoplan']); @@ -271,7 +276,7 @@ describe('capy skill: always current when online (the minimum is the floor, not expect(next.stdout).toMatch(/^SELF_UPGRADE: none \(current\)$/m); expect(next.stdout).not.toContain('SELF_UPGRADE: done'); expect(next.last).toBe('gstack: ok 1.0.2 for=autoplan'); - expect(f.calls().at(-1)).toBe(`installer --root ${f.root} --for autoplan lock=1 rev=3 min=1.0.1 from=`); + expect(f.calls().at(-1)).toBe(`installer --root ${f.root} --for autoplan lock=1 rev=4 min=1.0.1 from=`); }); test('--check mode never moves a behind checkout: it names the gap and the fast-forwarding command, then certifies the installed revision', () => { @@ -282,7 +287,7 @@ describe('capy skill: always current when online (the minimum is the floor, not expect(r.stdout).not.toContain('SELF_UPGRADE'); expect(f.git(['rev-parse', 'HEAD'])).toBe(f.b); expect(r.last).toBe('gstack: ok 1.0.1 for=autoplan'); - expect(f.calls()).toEqual([`installer --check --root ${f.root} --for autoplan lock= rev=3 min=1.0.1 from=`]); + expect(f.calls()).toEqual([`installer --check --root ${f.root} --for autoplan lock= rev=4 min=1.0.1 from=`]); }); test('a dirty behind checkout is skipped with gstack-upgrade\'s reason and still ends ok: above the floor, the trailer is the doctor\'s', () => { @@ -295,7 +300,7 @@ describe('capy skill: always current when online (the minimum is the floor, not expect(f.git(['rev-parse', 'HEAD'])).toBe(f.b); expect(fs.readFileSync(path.join(f.root, 'README.md'), 'utf8')).toBe('local edit\n'); expect(r.last).toBe('gstack: ok 1.0.1 for=autoplan'); - expect(f.calls()).toEqual([`installer --root ${f.root} --for autoplan lock=1 rev=3 min=1.0.1 from=`]); + expect(f.calls()).toEqual([`installer --root ${f.root} --for autoplan lock=1 rev=4 min=1.0.1 from=`]); }); test('GSTACK_CAPY_SELF_UPGRADE=off and GSTACK_CAPY_AUTO_UPGRADE=0 each disable the fast-forward by name and leave HEAD where it is', () => { @@ -315,6 +320,53 @@ describe('capy skill: always current when online (the minimum is the floor, not // Plan A9: the `latest:` trailer note comes only from origin's VERSION the // bootstrapper read in this run; a FETCH_HEAD left by an earlier run never // reaches the installer. +describe('capy skill bootstrapper: the update channel (wrapper 4, docs/releases.md#channels)', () => { + // Value: protects=an install that follows stable (GSTACK_UPDATE_CHANNEL=stable, or team mode through bin/gstack-update-channel.sh) is cloned at + // origin's stable branch, not main, and stays there through the checkout half's self-upgrade; GSTACK_UPDATE_CHANNEL=main and a remote with no stable + // branch (before the first release) clone main; fails_when=a fresh team install lands on unreleased main, the override cannot pick main, or a missing + // stable branch fails the install; why_new=the clone test covers the default branch only; seam=GSTACK_UPDATE_CHANNEL instead of gstack-config's + // team_mode; no_claim=GitHub's real stable branch + test('a fresh clone follows stable when origin has it; GSTACK_UPDATE_CHANNEL=main and a missing stable branch clone main', () => { + const f = fixture({ cloneAt: 'B', stable: 'B' }); + fs.rmSync(f.root, { recursive: true, force: true }); + const stable = f.run(['--for', 'autoplan'], { GSTACK_UPDATE_CHANNEL: 'stable' }); + expect(stable.status, stable.stdout + stable.stderr).toBe(0); + expect(stable.stdout).toContain('gstack-capy-skill: channel stable 1.0.1 (GSTACK_UPDATE_CHANNEL=main follows every merge)'); + expect(f.git(['rev-parse', 'HEAD'])).toBe(f.b); + expect(stable.last).toBe('gstack: ok 1.0.1 for=autoplan'); + + fs.rmSync(f.root, { recursive: true, force: true }); + const main = f.run(['--for', 'autoplan'], { GSTACK_UPDATE_CHANNEL: 'main' }); + expect(main.status, main.stdout + main.stderr).toBe(0); + expect(main.stdout).not.toContain('channel stable'); + expect(f.git(['rev-parse', 'HEAD'])).toBe(f.c); + + spawnSync('git', ['--git-dir', f.bare, 'update-ref', '-d', 'refs/heads/stable'], { env: f.host.env, timeout: 30_000 }); + fs.rmSync(f.root, { recursive: true, force: true }); + const bootstrap = f.run(['--for', 'autoplan'], { GSTACK_UPDATE_CHANNEL: 'stable' }); + expect(bootstrap.status, bootstrap.stdout + bootstrap.stderr).toBe(0); + expect(bootstrap.stdout).toContain('channel stable, but origin has no stable branch yet (or the fetch failed); following main'); + expect(f.git(['rev-parse', 'HEAD'])).toBe(f.c); + }); + + // Value: protects=the bootstrapper's floor fast-forward of a checkout below the supported minimum goes to stable on the stable channel when stable + // meets the minimum, and to main when stable is itself below it; fails_when=a stable install is pushed past the release to main, or left below the + // floor because stable is too old; why_new=the floor tests cover main only; seam=GSTACK_UPDATE_CHANNEL; no_claim=GitHub's real stable branch + test('below the floor: the fast-forward goes to stable when stable meets the minimum, else to main', () => { + const atRelease = fixture({ next: true, stable: 'B' }); + const r = atRelease.run(['--for', 'autoplan'], { GSTACK_UPDATE_CHANNEL: 'stable' }); + expect(r.status, r.stdout + r.stderr).toBe(0); + expect(r.stdout).toContain('gstack-capy-skill: upgraded 1.0.0 -> 1.0.1'); + expect(atRelease.git(['rev-parse', 'HEAD'])).toBe(atRelease.b); + + const old = fixture({ next: true, stable: 'A' }); + const o = old.run(['--for', 'autoplan'], { GSTACK_UPDATE_CHANNEL: 'stable' }); + expect(o.status, o.stdout + o.stderr).toBe(0); + expect(o.stdout).toContain('gstack-capy-skill: channel stable is at 1.0.0, below the supported minimum 1.0.1; following main'); + expect(old.git(['rev-parse', 'HEAD'])).toBe(old.c); + }); +}); + describe('capy skill bootstrapper: GSTACK_CAPY_REMOTE_VERSION', () => { test('a fresh fetch exports origin\'s VERSION; a failed fetch exports nothing even with a stale FETCH_HEAD', () => { const f = fixture({ cloneAt: 'B' }); diff --git a/test/release-cut.test.ts b/test/release-cut.test.ts index 6986b5630..0de0eba80 100644 --- a/test/release-cut.test.ts +++ b/test/release-cut.test.ts @@ -249,6 +249,50 @@ describe('release-cut: planning', () => { }); }); +describe('release-cut: the release gate holds and gates (lib/release-gate.ts records)', () => { + const release = (sha: string, verdict: string, run: number) => ({ schema: 1, kind: 'release', repo_key: 'k', sha, verdict, run: { id: run, url: 'u' }, red_cases: verdict === 'RED' ? ['case-a'] : [], created_at: 'x', updated_at: 'x' }); + const note = (f: ReturnType, record: unknown, on: string) => { + git(f.seed, 'notes', '--ref=' + NOTES_REF, 'add', '-f', '-m', JSON.stringify(record), on); + git(f.seed, 'push', '-q', '-f', 'origin', NOTES_REF); + }; + + // Value: protects=a red release record is the freeze: every cut is refused while no green gate sits on a later commit that contains the red one, + // even when every check run is green, and an incomplete gate or a green gate on an older commit does not lift it; fails_when=release-cut releases a + // commit while the newest release gate is red, a green record anywhere clears the hold, or an INCOMPLETE gate clears it; why_new=the freeze tests know + // only the train freeze record; seam=none; no_claim=real refs/notes/gstack-trains content (fixture records shaped like ReleaseRecord) + test('a red release record refuses the cut; only a green gate on a later commit lifts it', () => { + const f = fixture(); + note(f, release(f.shas['1.0.2']!, 'RED', 41), f.shas['1.0.2']!); + note(f, release(f.shas['1.0.1']!, 'GREEN', 40), f.shas['1.0.1']!); + const held = planRelease({ repo: f.repo, git: f.git, gh: f.gh }); + expect(held.ok).toBe(false); + if (!held.ok) { expect(held.code).toBe('REFUSED'); expect(held.message).toContain('release gate red on ' + f.shas['1.0.2']!.slice(0, 12)); } + note(f, release(f.shas['1.0.3']!, 'INCOMPLETE', 42), f.shas['1.0.3']!); + expect(planRelease({ repo: f.repo, git: f.git, gh: f.gh }).ok).toBe(false); + note(f, release(f.shas['1.0.3']!, 'GREEN', 43), f.shas['1.0.3']!); + const lifted = planRelease({ repo: f.repo, git: f.git, gh: f.gh }); + expect(lifted.ok, JSON.stringify(lifted)).toBe(true); + if (lifted.ok) expect(lifted.plan.sha).toBe(f.shas['1.0.3']); + }); + + // Value: protects=a green release record gates its commit when the main-backstop check run is absent or neutral (the check publish failed or was + // superseded), and the CANDIDATE line names the gate run; a red check run still keeps the commit out; fails_when=a commit the release gate passed is + // never released without the check, or release evidence overrides a red check run; why_new=train records were the only notes evidence; seam=none; + // no_claim=a live gate run + test('a green release record gates a commit whose main-backstop check never ran; a red check run still wins', () => { + const f = fixture(); + const head = f.shas['1.0.3']!; + f.state.checks[head] = GREEN.filter(c => c.name !== BACKSTOP_CHECK); + note(f, release(head, 'GREEN', 77), head); + const gated = planRelease({ repo: f.repo, git: f.git, gh: f.gh }); + expect(gated.ok, JSON.stringify(gated)).toBe(true); + if (gated.ok) { expect(gated.plan.sha).toBe(head); expect(gated.plan.via).toBe('release gate run 77'); } + f.state.checks[head] = [...GREEN.filter(c => c.name !== BACKSTOP_CHECK), { name: 'quality', status: 'completed', conclusion: 'failure' }]; + const red = planRelease({ repo: f.repo, git: f.git, gh: f.gh }); + if (red.ok) expect(red.plan.sha).not.toBe(head); + }); +}); + describe('release-cut: train evidence gates a commit whose backstop never settled', () => { const train = (nonce: string, state: string, merged: Array<{ sha: string; tree: string }>) => ({ schema: 1, kind: 'train', nonce, state, merged: merged.map(m => ({ pr: '1', at: 'x', ...m })), updated_at: 'x' }); diff --git a/test/release-gate-workflow.test.ts b/test/release-gate-workflow.test.ts new file mode 100644 index 000000000..a2f8a5542 --- /dev/null +++ b/test/release-gate-workflow.test.ts @@ -0,0 +1,129 @@ +/** + * The release gate's workflow wiring (docs/ship-policy.md#release-gate): + * release-gate.yml runs bin/gstack-release-gate on its schedule and publishes + * the gated commit's check run from a code-free job; the per-push backstop + * stands down once main's policy declares a release_gate block; evals.yml routes + * a release bisect only under that block. Policy steps run for real (bash, git, + * jq) against a scratch repository; the rest is read from the YAML. + */ +import { afterAll, describe, expect, test } from 'bun:test'; +import { execFileSync, spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { RELEASE_GATE_SCHEDULE_HOURS } from '../lib/release-gate-policy'; +import { RELEASE_BISECT_ARTIFACT } from '../scripts/train-gate'; + +const ROOT = path.resolve(import.meta.dir, '..'); +type Step = { id?: string; name?: string; uses?: string; if?: string; run?: string; env?: Record }; +type Job = { if?: string; needs?: string; outputs?: Record; permissions?: Record; steps: Step[] }; +type Workflow = { on: Record; concurrency: unknown; permissions?: unknown; jobs: Record }; +const load = (rel: string) => Bun.YAML.parse(fs.readFileSync(path.join(ROOT, rel), 'utf8')) as Workflow; +const release = load('.github/workflows/release-gate.yml'); +const backstop = load('.github/workflows/evals-main-backstop.yml'); +const evals = load('.github/workflows/evals.yml'); +const scratch: string[] = []; +afterAll(() => { for (const d of scratch) fs.rmSync(d, { recursive: true, force: true }); }); +const GIT_ENV = { GIT_AUTHOR_NAME: 't', GIT_AUTHOR_EMAIL: 't@x', GIT_COMMITTER_NAME: 't', GIT_COMMITTER_EMAIL: 't@x' }; + +/** Run a policy step's script in a scratch repo whose one commit holds this policy; returns its GITHUB_OUTPUT lines as an object. */ +function runStep(step: Step, policy: unknown): Record { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'release-gate-wf-')); + scratch.push(dir); + const g = (...args: string[]) => execFileSync('git', args, { cwd: dir, encoding: 'utf8', timeout: 30_000, env: { ...process.env, ...GIT_ENV } }).trim(); + g('init', '-q', '-b', 'main'); + if (policy !== undefined) { + fs.mkdirSync(path.join(dir, '.gstack')); + fs.writeFileSync(path.join(dir, '.gstack/ship-policy.json'), JSON.stringify(policy)); + } + fs.writeFileSync(path.join(dir, 'README'), 'x'); + g('add', '-A'); + g('commit', '-q', '-m', 'base'); + const sha = g('rev-parse', 'HEAD'); + g('update-ref', 'refs/remotes/origin/main', sha); + const out = path.join(dir, 'out'); + const r = spawnSync('bash', ['-e', '-c', step.run!], { + cwd: dir, encoding: 'utf8', timeout: 30_000, + env: { ...process.env, GITHUB_OUTPUT: out, GITHUB_STEP_SUMMARY: path.join(dir, 'summary'), GITHUB_SHA: sha, BASE_SHA: '', DEFAULT_BRANCH: 'main' }, + }); + expect(r.status, r.stderr).toBe(0); + const lines = (fs.existsSync(out) ? fs.readFileSync(out, 'utf8') : '').split('\n').filter(Boolean); + return Object.fromEntries(lines.map(l => [l.slice(0, l.indexOf('=')), l.slice(l.indexOf('=') + 1)])); +} +const GATE = { cadence_hours: 8, gate_workflow: 'evals.yml', branch: 'main', stable_ref: 'stable', check_name: 'main-backstop', on_red: 'hold-release', bisect_spend_cap_usd: 40 }; +const TRAIN = { gate_workflow: 'evals.yml', record_artifact: 'gstack-train-record', full_gate_check: 'gstack/full-gate' }; + +describe('release-gate.yml', () => { + // Value: protects=the schedule fires every RELEASE_GATE_SCHEDULE_HOURS (the cadence floor the policy validates), one gate at a time, only from the + // default branch, with contents and actions write but no checks write, and runs the bin on the run's own sha with its outputs wired; the policy step + // uses git and jq only and is inert without a release_gate block; fails_when=the cron and the cadence drift apart, two gates overlap, the job that + // runs repository code can write check runs, or an inert policy reaches bun; why_new=new workflow; seam=none; no_claim=GitHub's cron jitter + test('schedule, concurrency, permissions and the gate step', () => { + const cron = release.on.schedule[0].cron as string; + expect(cron).toMatch(/^\d+ \*\/\d+ \* \* \*$/); + expect(Number(cron.split(' ')[1]!.slice(2))).toBe(RELEASE_GATE_SCHEDULE_HOURS); + expect(Object.keys(release.on).sort()).toEqual(['schedule', 'workflow_dispatch']); + expect(Object.keys(release.on.workflow_dispatch.inputs).sort()).toEqual(['dry_run', 'ignore_cadence']); + expect(release.concurrency).toEqual({ group: 'release-gate', 'cancel-in-progress': false }); + expect(release.permissions).toEqual({}); + const job = release.jobs['release-gate']!; + expect(job.if).toBe('github.ref_name == github.event.repository.default_branch'); + expect(job.permissions).toEqual({ contents: 'write', actions: 'write', checks: 'read' }); + const policy = job.steps.find(s => s.id === 'policy')!; + expect(policy.run).not.toContain('bun'); + expect(runStep(policy, undefined)).toEqual({ release: 'false' }); + expect(runStep(policy, { train: TRAIN })).toEqual({ release: 'false' }); + expect(runStep(policy, { release_gate: GATE })).toEqual({ release: 'true' }); + const gate = job.steps.find(s => s.id === 'gate')!; + expect(gate.if).toBe("steps.policy.outputs.release == 'true'"); + expect(gate.run).toContain('./bin/gstack-release-gate run --repo "$GITHUB_REPOSITORY" --sha "$GITHUB_SHA"'); + expect(gate.run).toContain('--outputs "$GITHUB_OUTPUT"'); + expect(job.outputs).toEqual({ action: '${{ steps.gate.outputs.action }}', sha: '${{ steps.gate.outputs.sha }}', check: '${{ steps.gate.outputs.check }}' }); + }); + + // Value: protects=the gated commit's check run (what release-cut reads) comes from a job with checks: write and no checkout or action, only for a + // gate that ran (success for gated, failure for held or failed), on a validated sha and check name; fails_when=repository code runs with checks: + // write, a skipped or inert run publishes a check, or a red gate publishes success; why_new=new workflow; seam=none; no_claim=GitHub's check API + test('the check run is published from a code-free job, success only for gated', () => { + const job = release.jobs['release-check']!; + expect(job.needs).toBe('release-gate'); + expect(job.if).toContain("contains(fromJSON('[\"gated\",\"held\",\"failed\"]'), needs.release-gate.outputs.action)"); + expect(job.permissions).toEqual({ checks: 'write' }); + expect(job.steps.filter(s => s.uses)).toEqual([]); + const run = job.steps[0]!.run!; + expect(run).toContain('gated) conclusion=success ;;'); + expect(run).toContain('held|failed) conclusion=failure ;;'); + expect(run).toContain('[[ "$SHA" =~ ^[0-9a-f]{40}$ ]]'); + expect(run).toContain('-f name="$CHECK" -f head_sha="$SHA"'); + expect(run).not.toContain('bun'); + }); +}); + +describe('the backstop stands down and evals.yml routes the release bisect', () => { + // Value: protects=while trains exist the per-push backstop keeps gating, and once main declares release_gate it stands down (train=false: no gate, + // no freeze, no check run), so main is never gated twice; fails_when=both gates run on main, or the backstop stops before the release gate is on; + // why_new=new stand-down rule; seam=none; no_claim=a live push + test('evals-main-backstop.yml: train=true with only a train block, train=false once release_gate is declared', () => { + const policy = backstop.jobs.backstop!.steps.find(s => s.id === 'policy')!; + expect(runStep(policy, { train: TRAIN })).toEqual({ train: 'true' }); + expect(runStep(policy, { train: TRAIN, release_gate: GATE })).toEqual({ train: 'false' }); + }); + + // Value: protects=the route job reads release_gate from the base revision with git and jq only and hands it to the route as RELEASE_POLICY, and the + // bisect jobs name their artifact from the route's record_artifact first (gstack-release-record for a release bisect); fails_when=a release bisect + // publishes under the train artifact name the gate never reads, or repository code decides the release policy; why_new=new route input; + // seam=none; no_claim=a live bisect run + test('evals.yml: the release policy step, the RELEASE_POLICY input and the record_artifact override', () => { + const route = evals.jobs.route!; + const step = route.steps.find(s => s.id === 'release_policy')!; + expect(step.run).not.toContain('bun'); + expect(runStep(step, { release_gate: GATE })).toEqual({ release: 'true' }); + expect(runStep(step, { train: TRAIN })).toEqual({ release: 'false' }); + expect(runStep(step, undefined)).toEqual({ release: 'false' }); + const ids = route.steps.map(s => s.id); + expect(ids.indexOf('release_policy')).toBeLessThan(ids.indexOf('route')); + expect(route.steps.find(s => s.id === 'route')!.env!.RELEASE_POLICY).toBe('${{ steps.release_policy.outputs.release }}'); + expect(route.outputs!.record_artifact).toBe('${{ steps.route.outputs.record_artifact || steps.policy.outputs.record_artifact }}'); + expect(RELEASE_BISECT_ARTIFACT).toBe('gstack-release-record'); + }); +}); diff --git a/test/release-gate.test.ts b/test/release-gate.test.ts new file mode 100644 index 000000000..29fe65cbc --- /dev/null +++ b/test/release-gate.test.ts @@ -0,0 +1,301 @@ +/** + * gstack-release-gate (lib/release-gate.ts, docs/ship-policy.md#release-gate): + * the full paid gate on main's head at most once per cadence and only when main + * moved, a release record in refs/notes/gstack-trains, a bisect of a red gate's + * cases over the commits since the last green gate, and the hold release-cut + * reads. GitHub is a fake gh (dispatch, run list, run view, artifact download); + * git, the notes ref and origin are real (a bare origin on disk). + */ +import { afterAll, afterEach, describe, expect, test } from 'bun:test'; +import { execFileSync, spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { bunIsolationArgs } from '../lib/bun-isolation'; +import { + MAX_RELEASE_ATTEMPTS, activeHold, bisectGate, classifyRun, gateStatus, loadReleaseGate, narrow, parseReport, planPrefixes, releaseRecords, runGate, + type GateEnv, type Run, +} from '../lib/release-gate'; +import { RELEASE_GATE_SCHEDULE_HOURS, releaseGateErrors, type ReleaseGatePolicy } from '../lib/release-gate-policy'; +import { RELEASE_GATE_KEY_VERSION, validatePolicy } from '../lib/ship-policy'; +import { NOTES_REF, repoIdentity, type ReleaseRecord } from '../lib/ship-queue-train-record'; +import { gitIn } from '../scripts/train-gate'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const GATE: ReleaseGatePolicy = { cadence_hours: 8, gate_workflow: 'evals.yml', branch: 'main', stable_ref: 'stable', check_name: 'main-backstop', on_red: 'hold-release', bisect_spend_cap_usd: 40 }; +const roots: string[] = []; +const savedStateRoot = process.env.GSTACK_STATE_ROOT; +afterEach(() => { if (savedStateRoot === undefined) delete process.env.GSTACK_STATE_ROOT; else process.env.GSTACK_STATE_ROOT = savedStateRoot; }); +afterAll(() => { for (const r of roots) fs.rmSync(r, { recursive: true, force: true }); }); + +const ID = { GIT_AUTHOR_NAME: 't', GIT_AUTHOR_EMAIL: 't@x', GIT_COMMITTER_NAME: 't', GIT_COMMITTER_EMAIL: 't@x' }; +const git = (cwd: string, ...args: string[]) => execFileSync('git', args, { cwd, encoding: 'utf8', timeout: 30_000, env: { ...process.env, ...ID } }).trim(); + +/** origin with base (also stable) and c1..c5 on main, and work cloned from it; the state root is the fixture's own. */ +function fixture() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'release-gate-')); + roots.push(root); + const origin = path.join(root, 'origin.git'); + const seed = path.join(root, 'seed'); + const work = path.join(root, 'work'); + git(root, 'init', '-q', '--bare', '-b', 'main', origin); + fs.mkdirSync(seed); + git(seed, 'init', '-q', '-b', 'main'); + git(seed, 'remote', 'add', 'origin', origin); + const commit = (name: string) => { fs.writeFileSync(path.join(seed, 'VERSION'), name + '\n'); git(seed, 'add', '-A'); git(seed, 'commit', '-q', '-m', name); git(seed, 'push', '-q', 'origin', 'main'); return git(seed, 'rev-parse', 'HEAD'); }; + const base = commit('base'); + git(seed, 'push', '-q', 'origin', 'main:stable'); + const c = [1, 2, 3, 4, 5].map(i => commit('c' + i)); + git(root, 'clone', '-q', origin, work); + process.env.GSTACK_STATE_ROOT = path.join(root, 'state'); + return { root, origin, seed, work, base, c, commit }; +} + +interface Outcome { conclusion: string; artifacts: Record> } +/** A fake gh: each dispatch becomes a completed run whose outcome the test decides; run list, run view and artifact download read it back. */ +function fakeGh(onDispatch: (inputs: Record, ref: string) => Outcome) { + const runs: Run[] = []; + const artifacts = new Map>>(); + const dispatches: Array<{ ref: string; inputs: Record }> = []; + const gh = (args: string[]) => { + const ok = (out: unknown) => ({ status: 0, stdout: typeof out === 'string' ? out : JSON.stringify(out), stderr: '' }); + if (args[0] === 'workflow' && args[1] === 'run') { + const ref = args[args.indexOf('--ref') + 1]!; + const inputs = Object.fromEntries(args.flatMap((a, i) => (args[i - 1] === '-f' ? [a.split(/=(.*)/s).slice(0, 2)] : []))); + dispatches.push({ ref, inputs }); + const id = 1000 + runs.length; + const o = onDispatch(inputs, ref); + runs.push({ id, status: 'completed', conclusion: o.conclusion, head_sha: inputs.expected_sha!, display_title: 'E2E Evals [eval-bg ' + inputs.nonce + ']', html_url: 'https://github.com/o/r/actions/runs/' + id, run_attempt: 1 }); + artifacts.set(id, o.artifacts); + return ok(''); + } + if (args[0] === 'api' && args.some(a => /\/actions\/workflows\/[^/]+\/runs$/.test(a))) return ok({ workflow_runs: [...runs].reverse() }); + const view = args[0] === 'api' ? /actions\/runs\/(\d+)$/.exec(args[1] ?? '') : null; + if (view) return ok(runs.find(r => r.id === Number(view[1]))); + if (args[0] === 'run' && args[1] === 'download') { + const files = artifacts.get(Number(args[2]))?.[args[args.indexOf('-n') + 1]!]; + if (!files) return { status: 1, stdout: '', stderr: 'no artifact matches any of the names or patterns provided' }; + const dir = args[args.indexOf('-D') + 1]!; + for (const [name, text] of Object.entries(files)) { fs.mkdirSync(dir, { recursive: true }); fs.writeFileSync(path.join(dir, name), text); } + return ok(''); + } + return { status: 1, stdout: '', stderr: 'unexpected gh ' + args.join(' ') }; + }; + return { gh, runs, dispatches }; +} + +const report = (verdict: 'GREEN' | 'RED', failed: string[] = [], cost = '37.40') => ({ 'report-verdict-a1': { 'report.txt': [ + '[test:paid] report: 17/17 slices, 85 planned shards, tier=gate, attempt 1', + ...(failed.length ? ['FAILED cases:', ...failed.map(l => ' ✗ ' + l)] : []), + '[test:paid] VERDICT ' + verdict + ' — lane gate/full census, attempt 1', + ' wall 41m02s · cost $' + cost + ' known + 4 of 139 trial(s) cost unknown', +].join('\n') } }); +const green = (flaky: string[] = []): Outcome => ({ conclusion: 'success', artifacts: { ...report('GREEN'), ...(flaky.length ? { 'flake-records-a1': { 'flake-records.jsonl': flaky.map(c => JSON.stringify({ schema: 1, kind: 'gate-flake', case: c })).join('\n') } } : {}) } }); + +function envFor(f: ReturnType, gh: (args: string[]) => { status: number; stdout: string; stderr: string }, shiftHours = 0): GateEnv { + return { repo: 'o/r', repoRoot: f.work, remote: 'origin', gh, git: gitIn(f.work), now: () => Date.now() + shiftHours * 3_600_000, sleep: () => {}, random: () => 'abc123', pollMs: 1, deadline: Date.now() + 3_600_000 }; +} +const records = (f: ReturnType) => { + git(f.work, 'fetch', '-q', 'origin', '+' + NOTES_REF + ':' + NOTES_REF); + return releaseRecords(gitIn(f.work), repoIdentity(f.work).key); +}; + +describe('release gate: run', () => { + // Value: protects=a gate only when main moved: the first run dispatches the full gate once on main's head (evals_all, expected_sha, a backstop nonce), + // records GREEN with its run, cost and rule-16 flaky cases in origin's notes and prints the RELEASE green line without moving stable; a second run on + // the same head, and a head equal to stable, skip without dispatching; fails_when=the gate dispatches again on an unchanged main (a $37 repeat), a + // green gate moves stable, or the record never reaches origin; why_new=new module; seam=fake gh; no_claim=a live gate run (the PR's one live proof) + test('green: one dispatch on the moved head, a GREEN release record on origin, stable untouched; an unchanged main skips', () => { + const f = fixture(); + const fake = fakeGh(() => green(['judge-x'])); + const head = f.c[4]!; + const out = runGate(envFor(f, fake.gh), { gate: GATE, label: 'test', dryRun: false, ignoreCadence: false }); + expect(out.action, out.lines.join('\n')).toBe('gated'); + expect(fake.dispatches).toHaveLength(1); + expect(fake.dispatches[0]!.ref).toBe('main'); + expect(fake.dispatches[0]!.inputs).toMatchObject({ evals_all: 'true', expected_sha: head }); + expect(fake.dispatches[0]!.inputs.nonce).toMatch(/^gate\.backstop-/); + expect(out.lines).toContain('RELEASE: green sha=' + head + ' stable=' + f.base + ' run=1000 cost=37.40'); + expect(out.lines.some(l => l.includes('flaky (failed once, passed its one rule-16 rerun in run 1000): judge-x'))).toBe(true); + const [rec] = records(f); + expect(rec).toMatchObject({ kind: 'release', sha: head, verdict: 'GREEN', flaky: ['judge-x'], cost_usd: 37.4, since: f.base, attempts: 1, run: { id: 1000 } }); + expect(git(f.seed, 'ls-remote', 'origin', 'refs/heads/stable').split('\t')[0]).toBe(f.base); + + const again = runGate(envFor(f, fake.gh, 9), { gate: GATE, label: 'test', dryRun: false, ignoreCadence: true }); + expect(again.action).toBe('skipped'); + expect(again.lines[0]).toContain('main has not moved since the last gate: GREEN run 1000'); + git(f.seed, 'push', '-q', '-f', 'origin', 'main:stable'); + const atStable = runGate(envFor(f, fake.gh), { gate: GATE, label: 'test', dryRun: false, ignoreCadence: true }); + expect(atStable.lines[0]).toContain('main has not moved since stable'); + expect(fake.dispatches).toHaveLength(1); + }); + + // Value: protects=without a release_gate block nothing runs; --dry-run names the head, the base and the pending count and writes and dispatches + // nothing; the cadence holds a moved main until cadence_hours minus the grace pass, and --ignore-cadence gates now; fails_when=an inert policy spends, + // a dry run writes a record, or the schedule gates more often than the policy says; why_new=new module; seam=fake gh and a shifted clock; + // no_claim=GitHub's cron timing + test('inert, dry run and cadence: nothing is spent until the policy and the clock allow it', () => { + const f = fixture(); + const fake = fakeGh(() => green()); + expect(runGate(envFor(f, fake.gh), { gate: null, label: 'origin/main@x', dryRun: false, ignoreCadence: false }).action).toBe('inert'); + const dry = runGate(envFor(f, fake.gh), { gate: GATE, label: 'test', dryRun: true, ignoreCadence: false }); + expect(dry.action).toBe('dry-run'); + expect(dry.lines[0]).toBe('RELEASE: gate sha=' + f.c[4] + ' since=' + f.base.slice(0, 12) + ' (stable) pending=5 stable=' + f.base.slice(0, 12)); + expect(fake.dispatches).toHaveLength(0); + expect(spawnSync('git', ['ls-remote', 'origin', NOTES_REF], { cwd: f.work, encoding: 'utf8', timeout: 30_000 }).stdout).toBe(''); + + expect(runGate(envFor(f, fake.gh), { gate: GATE, label: 'test', dryRun: false, ignoreCadence: false }).action).toBe('gated'); + const next = f.commit('c6'); + const early = runGate(envFor(f, fake.gh, 2), { gate: GATE, label: 'test', dryRun: false, ignoreCadence: false }); + expect(early.action).toBe('skipped'); + expect(early.lines[0]).toMatch(/the last gate was dispatched 2\.0 h ago on [0-9a-f]{12}; cadence 8 h/); + const due = runGate(envFor(f, fake.gh, 7.5), { gate: GATE, label: 'test', dryRun: false, ignoreCadence: false }); + expect(due.action, due.lines.join('\n')).toBe('gated'); + expect(fake.dispatches.map(d => d.inputs.expected_sha)).toEqual([f.c[4], next]); + expect(records(f).find(r => r.sha === next)?.since).toBe(f.c[4]); + }); + + // Value: protects=an INCOMPLETE census (only INFRA reds) is recorded as such, re-dispatched once by the next run (EVAL_POLICY.infraRedispatch), then + // skipped with its run named, and never treated as green or red; fails_when=an infra-only failure holds the release like a red, is retried without + // end, or is never retried; why_new=new module; seam=fake gh; no_claim=a real runner loss + test('incomplete: recorded, re-dispatched once, then skipped with the run named', () => { + const f = fixture(); + const fake = fakeGh(() => ({ conclusion: 'failure', artifacts: report('RED', ['case-a rule FAIL INFRA']) })); + const env = envFor(f, fake.gh); + const first = runGate(env, { gate: GATE, label: 'test', dryRun: false, ignoreCadence: true }); + expect(first.action).toBe('failed'); + expect(first.lines.at(-1)).toContain('RELEASE: incomplete sha=' + f.c[4] + ' run=1000'); + expect(first.lines.at(-1)).toContain('the next run re-dispatches it once'); + expect(runGate(env, { gate: GATE, label: 'test', dryRun: false, ignoreCadence: true }).action).toBe('failed'); + const third = runGate(env, { gate: GATE, label: 'test', dryRun: false, ignoreCadence: true }); + expect(third.action).toBe('skipped'); + expect(third.lines[0]).toContain('incomplete ' + MAX_RELEASE_ATTEMPTS + ' times'); + expect(fake.dispatches).toHaveLength(MAX_RELEASE_ATTEMPTS); + expect(activeHold(records(f), () => true)).toBeNull(); + }); +}); + +describe('release gate: red, bisect and the hold', () => { + // Value: protects=a red gate records RED (the hold release-cut reads), bisects only its red verdict cases on release-gate/ with prefixes between + // stable and the red head, narrows round by round to the first red commit after a green one, deletes the bisect branch, and prints first_bad; a later + // green gate on a newer commit lifts the hold; fails_when=a red gate moves or leaves stable releasable, the bisect blames the wrong commit, tests the + // known-red head again, leaks its branch, or a green gate on an older commit lifts the hold; why_new=new module; seam=fake gh; no_claim=a live bisect + test('red: RED record, a bisect that names first_bad, the branch cleaned up, and the hold until a later green', () => { + const f = fixture(); + const order = [f.base, ...f.c]; + const fake = fakeGh((inputs) => { + if (!inputs.prefix_shas) return { conclusion: 'failure', artifacts: report('RED', ['case-a rule FAIL VERDICT', 'case-z rule FAIL INFRA']) }; + const files: Record> = {}; + for (const sha of inputs.prefix_shas.split(',')) { + const red = order.indexOf(sha) >= 3; + files['gstack-release-record-bisect-' + sha.slice(0, 12)] = { 'bisect-prefix.json': JSON.stringify({ schema: 1, kind: 'train-bisect-prefix', nonce: inputs.nonce, prefix_sha: sha, checked_out_sha: sha, + panels: [{ case: 'case-a', kind: 'rule', status: red ? 'FAIL' : 'PASS', red_class: red ? 'VERDICT' : null, cost_usd: 1.25 }] }) }; + } + return { conclusion: 'success', artifacts: files }; + }); + const out = runGate(envFor(f, fake.gh), { gate: GATE, label: 'test', dryRun: false, ignoreCadence: true }); + expect(out.action, out.lines.join('\n')).toBe('held'); + expect(out.lines).toContain('RELEASE: red sha=' + f.c[4] + ' first_bad=' + f.c[2] + ' run=1000'); + expect(fake.dispatches).toHaveLength(2); + const bisect = fake.dispatches[1]!; + expect(bisect.ref).toMatch(/^release-gate\/abc123-[0-9a-f]{12}$/); + expect(bisect.inputs).toMatchObject({ cases: 'case-a', prefix_shas: f.c.slice(0, 4).join(','), expected_sha: f.c[4] }); + expect(bisect.inputs.nonce).toMatch(/^gate\.bisect-/); + expect(git(f.seed, 'ls-remote', 'origin', 'refs/heads/release-gate/*')).toBe(''); + const recs = records(f); + expect(recs[0]).toMatchObject({ verdict: 'RED', red_cases: ['case-a'], bisect: { state: 'found', first_bad: f.c[2], good: f.c[1], bad: f.c[2], spent_usd: 5 } }); + expect(git(f.seed, 'ls-remote', 'origin', 'refs/heads/stable').split('\t')[0]).toBe(f.base); + const isAncestor = (a: string, b: string) => spawnSync('git', ['merge-base', '--is-ancestor', a, b], { cwd: f.work, timeout: 30_000 }).status === 0; + expect(activeHold(recs, isAncestor)?.sha).toBe(f.c[4]); + const status = gateStatus(envFor(f, fake.gh), { gate: GATE, label: 'test' }); + expect(status.action).toBe('held'); + expect(status.lines.find(l => l.startsWith('HOLD:'))).toContain('red on ' + f.c[4]); + const older: ReleaseRecord = { ...recs[0]!, sha: f.c[1]!, verdict: 'GREEN', bisect: null }; + expect(activeHold([...recs, older], isAncestor)?.sha).toBe(f.c[4]); + const later: ReleaseRecord = { ...recs[0]!, sha: f.commit('fix'), verdict: 'GREEN', bisect: null }; + git(f.work, 'fetch', '-q', 'origin'); + expect(activeHold([later, ...recs], isAncestor)).toBeNull(); + }); + + // Value: protects=the bisect spends within its cap: a 0 cap records no bisect and first_bad unknown, a small cap tests only the prefixes it covers + // and stops capped, and bisect --spend-cap resumes a capped one; fails_when=a bisect dispatches past its cap or a capped bisect cannot be resumed; + // why_new=new module; seam=fake gh; no_claim=real case prices + test('bisect spend: a zero cap skips it, a small cap stops capped, and bisect --spend-cap resumes', () => { + const f = fixture(); + const fake = fakeGh((inputs) => (inputs.prefix_shas ? { conclusion: 'success', artifacts: {} } : { conclusion: 'failure', artifacts: report('RED', ['case-a rule FAIL VERDICT', 'case-b judge FAIL']) })); + const none = runGate(envFor(f, fake.gh), { gate: { ...GATE, bisect_spend_cap_usd: 0 }, label: 'test', dryRun: false, ignoreCadence: true }); + expect(none.lines).toContain('RELEASE: red sha=' + f.c[4] + ' first_bad=unknown run=1000'); + expect(none.lines.some(l => l.includes('bisect unknown: bisect_spend_cap_usd is 0'))).toBe(true); + expect(fake.dispatches).toHaveLength(1); + + const g = fixture(); + const fake2 = fakeGh((inputs) => (inputs.prefix_shas ? { conclusion: 'success', artifacts: {} } : { conclusion: 'failure', artifacts: report('RED', ['case-a rule FAIL VERDICT', 'case-b judge FAIL']) })); + const capped = runGate(envFor(g, fake2.gh), { gate: { ...GATE, bisect_spend_cap_usd: 2 }, label: 'test', dryRun: false, ignoreCadence: true }); + expect(capped.lines.some(l => l.includes('bisect capped: the next round needs $3.00 a prefix'))).toBe(true); + expect(fake2.dispatches).toHaveLength(1); + const plan = bisectGate(envFor(g, fake2.gh), { gate: GATE, label: 'test', spendCap: 30, dryRun: true }); + expect(plan.lines.some(l => l.startsWith('RELEASE: bisect round 1: 2 case(s) on 4 of 4 commit(s)'))).toBe(true); + expect(fake2.dispatches).toHaveLength(1); + }); +}); + +describe('release gate: verdict, range and policy units', () => { + test('parseReport and classifyRun: verdict reds bisect, other classes keep a red incomplete, success is green, and a contradiction is incomplete', () => { + const r = parseReport(report('RED', ['case-a rule FAIL VERDICT', 'judge one judge FAIL', 'case-z rule FAIL INFRA'])['report-verdict-a1']['report.txt']); + expect(r).toEqual({ verdict: 'RED', red: ['case-a', 'judge one'], other: ['case-z'], cost: 37.4 }); + expect(classifyRun({ status: 'completed', conclusion: 'failure' }, r)).toMatchObject({ verdict: 'RED', red: ['case-a', 'judge one'] }); + expect(classifyRun({ status: 'completed', conclusion: 'failure' }, { ...r, red: [] }).verdict).toBe('INCOMPLETE'); + expect(classifyRun({ status: 'completed', conclusion: 'cancelled' }, null).verdict).toBe('INCOMPLETE'); + expect(classifyRun({ status: 'completed', conclusion: 'success' }, { ...r, verdict: 'GREEN' }).verdict).toBe('GREEN'); + expect(classifyRun({ status: 'completed', conclusion: 'success' }, r).verdict).toBe('INCOMPLETE'); + expect(classifyRun({ status: 'in_progress', conclusion: null }, null).verdict).toBe('PENDING'); + }); + + test('planPrefixes spreads at most MAX_PREFIXES over the interior; narrow moves good and bad and refuses a non-monotone or missing result', () => { + const shas = Array.from({ length: 20 }, (_, i) => String(i).padStart(40, '0')); + expect(planPrefixes(shas.slice(0, 3))).toEqual(shas.slice(0, 3)); + const picked = planPrefixes(shas, 7); + expect(picked).toHaveLength(7); + expect(new Set(picked).size).toBe(7); + expect(picked[0]).not.toBe(shas[0]); + const t = (rs: string[]) => rs.map((r, i) => ({ sha: shas[i]!, results: r === '?' ? {} : { a: r === 'R' ? 'red' as const : 'green' as const } })); + expect(narrow({ good: 'g', bad: 'b' }, t(['G', 'G', 'R']), ['a'])).toEqual({ good: shas[1], bad: shas[2] }); + expect(narrow({ good: 'g', bad: 'b' }, t(['G', 'G']), ['a'])).toEqual({ good: shas[1], bad: 'b' }); + expect(narrow({ good: 'g', bad: 'b' }, t(['R', 'R']), ['a'])).toEqual({ good: 'g', bad: shas[0] }); + expect(narrow({ good: 'g', bad: 'b' }, t(['G', 'R', 'G']), ['a'])).toEqual({ unknown: 'a is not monotone over the tested commits (GRG)' }); + expect(narrow({ good: 'g', bad: 'b' }, t(['G', '?']), ['a'])).toHaveProperty('unknown'); + }); + + // Value: protects=the release_gate key is read only with min_gstack_version at its introducing version, validates every field (cadence at least the + // workflow schedule, a workflow file, branch names, hold-release), and --policy runs the same validation; fails_when=an older gstack reads the block + // silently, a bad field passes, or the command-line policy skips validation; why_new=new key; seam=none; no_claim=the json adoption (a follow-up PR) + test('policy: the release_gate key needs its min_gstack_version and every field; --policy is validated the same way', () => { + const raw = { release_gate: GATE }; + expect(validatePolicy(raw).map(e => e.path)).toEqual(['release_gate']); + expect(validatePolicy({ ...raw, min_gstack_version: RELEASE_GATE_KEY_VERSION })).toEqual([]); + expect(releaseGateErrors({ ...GATE, cadence_hours: RELEASE_GATE_SCHEDULE_HOURS - 1, on_red: 'freeze', stable_ref: 'main', extra: 1 }).map(e => e.path).sort()) + .toEqual(['release_gate.cadence_hours', 'release_gate.extra', 'release_gate.on_red', 'release_gate.stable_ref']); + expect(releaseGateErrors({ cadence_hours: 8 }).filter(e => e.message.startsWith('missing'))).toHaveLength(6); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'release-gate-policy-')); + roots.push(dir); + const good = path.join(dir, 'good.json'); + fs.writeFileSync(good, JSON.stringify({ ...raw, min_gstack_version: RELEASE_GATE_KEY_VERSION })); + expect(loadReleaseGate(ROOT, { policyFile: good })).toEqual({ gate: GATE, label: good }); + const bad = path.join(dir, 'bad.json'); + fs.writeFileSync(bad, JSON.stringify(raw)); + expect((loadReleaseGate(ROOT, { policyFile: bad }) as { error: string }).error).toContain('release_gate introduced in gstack ' + RELEASE_GATE_KEY_VERSION); + }); + + test('bin/gstack-release-gate: usage exits 2; an inert policy file exits 0 without touching GitHub', () => { + const bin = path.join(ROOT, 'bin/gstack-release-gate'); + const usage = spawnSync('bun', [...bunIsolationArgs(), bin, 'frobnicate'], { cwd: ROOT, encoding: 'utf8', timeout: 60_000 }); + expect(usage.status).toBe(2); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'release-gate-bin-')); + roots.push(dir); + const inert = path.join(dir, 'policy.json'); + fs.writeFileSync(inert, '{}'); + const r = spawnSync('bun', [...bunIsolationArgs(), bin, 'run', '--repo', 'o/r', '--policy', inert], { cwd: ROOT, encoding: 'utf8', timeout: 60_000 }); + expect(r.status, r.stderr).toBe(0); + expect(r.stdout.trim()).toBe('RELEASE: inert (' + inert + ' declares no release_gate block; the per-push main backstop gates main)'); + }); +}); diff --git a/test/train-gate.test.ts b/test/train-gate.test.ts index 17702d1f0..2aaad83db 100644 --- a/test/train-gate.test.ts +++ b/test/train-gate.test.ts @@ -54,6 +54,20 @@ describe('routeDispatch: every run keeps its mode, and train inputs are validate expect((route({ ...screen, trainPolicy: false }) as { error: string }).error).toContain('needs a train block'); }); + // Value: protects=the release gate's bisect (lib/release-gate.ts) routes on its own release-gate/ ref under a release_gate policy, names its own + // prefix artifact, keeps every train bisect check (nonce, head, cases, prefixes), and is refused without the policy or on any other ref; + // fails_when=a release bisect needs a train block (it stops working once trains go), a release-gate/ ref routes without a release_gate policy, or a + // train/ bisect starts accepting the release policy alone; why_new=the bisect test below covers train/ only; seam=none; no_claim=a live run + test('a release bisect runs on release-gate/ under a release_gate policy, with the train checks and its own artifact', () => { + const bisect = { nonce: bisectNonce, refName: 'release-gate/abc123-aaaaaaaaaaaa', expectedSha: A, cases: 'case-a', prefixShas: B, releasePolicy: true }; + expect(route(bisect)).toEqual({ mode: 'bisect', cases: ['case-a'], prefixes: [B], artifact: 'gstack-release-record' }); + expect((route({ ...bisect, releasePolicy: false, trainPolicy: true }) as { error: string }).error).toContain('needs a release_gate block'); + expect((route({ ...bisect, prefixShas: C }) as { error: string }).error).toContain('is not an ancestor'); + expect((route({ ...bisect, expectedSha: B }) as { error: string }).error).toContain('is not the train head'); + expect((route({ ...bisect, refName: 'train/t1', trainPolicy: false }) as { error: string }).error).toContain('needs a train block'); + expect((route({ ...bisect, refName: 'main' }) as { error: string }).error).toContain('runs on train/'); + }); + test('a bisect needs a bisect nonce on train/, the train head, plan case ids and ancestor prefixes', () => { const bisect = { nonce: bisectNonce, refName: 'train/t1', expectedSha: A, cases: 'case-a, review/SKILL.md workflow', prefixShas: `${A},${B}`, trainPolicy: true }; expect(route(bisect)).toEqual({ mode: 'bisect', cases: ['case-a', 'review/SKILL.md workflow'], prefixes: [A, B] });