From 2dcbe9dbbdfb06f979bca57c4f603872fc2fd72e Mon Sep 17 00:00:00 2001 From: garrytan Date: Tue, 3 Nov 2026 02:51:14 +0000 Subject: [PATCH 01/15] feat(release-gate): gate main every 8 hours and feed the stable release (inert until adopted) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- .github/workflows/evals-main-backstop.yml | 8 + .github/workflows/evals.yml | 16 +- .github/workflows/release-gate.yml | 118 +++++++ bin/gstack-release-gate | 62 ++++ capy/skill/gstack/scripts/install.sh | 39 ++- changelog.d/gsta258-l1-release-gate.md | 17 + docs/captain.md | 5 +- docs/releases.md | 13 +- docs/ship-policy.md | 75 ++++ lib/release-gate-policy.ts | 57 +++ lib/release-gate.ts | 400 ++++++++++++++++++++++ lib/ship-policy.ts | 9 +- lib/ship-queue-train-record.ts | 19 +- scripts/release-cut.ts | 33 +- scripts/train-gate.ts | 16 +- test/capy-skill-wrapper.test.ts | 70 +++- test/release-cut.test.ts | 44 +++ test/release-gate-workflow.test.ts | 129 +++++++ test/release-gate.test.ts | 301 ++++++++++++++++ test/train-gate.test.ts | 14 + 20 files changed, 1415 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/release-gate.yml create mode 100755 bin/gstack-release-gate create mode 100644 changelog.d/gsta258-l1-release-gate.md create mode 100644 lib/release-gate-policy.ts create mode 100644 lib/release-gate.ts create mode 100644 test/release-gate-workflow.test.ts create mode 100644 test/release-gate.test.ts diff --git a/.github/workflows/evals-main-backstop.yml b/.github/workflows/evals-main-backstop.yml index 2c3f30303..a864b8edf 100644 --- a/.github/workflows/evals-main-backstop.yml +++ b/.github/workflows/evals-main-backstop.yml @@ -12,6 +12,10 @@ name: Main backstop gate # (success when a green train covers the tree or the backstop gated it green, # neutral when the push was superseded or deferred, failure when it froze the # queue or could not finish); the stable release reads it (docs/releases.md). +# +# Once main's policy declares a `release_gate` block, the release gate +# (release-gate.yml, every 8 hours) gates main instead and this backstop stands +# down: no gate, no freeze and no check run on a push. on: push: branches: [main] @@ -60,9 +64,13 @@ jobs: done fi train=$(jq -c 'if (.train | type) == "object" then .train else null end' <<<"$policy") || { echo "::error::${GITHUB_SHA}:.gstack/ship-policy.json is not valid JSON; fix the policy file" >&2; exit 1; } + release=$(jq -r 'if (.release_gate | type) == "object" then "true" else "false" end' <<<"$policy") if [ -z "$train" ] || [ "$train" = null ]; then echo "train=false" >> "$GITHUB_OUTPUT" echo "- Backstop inert: main's .gstack/ship-policy.json declares no train block; per-PR gates cover main" >> "$GITHUB_STEP_SUMMARY" + elif [ "$release" = true ]; then + echo "train=false" >> "$GITHUB_OUTPUT" + echo "- Backstop stands down: main's .gstack/ship-policy.json declares a release_gate block; the scheduled release gate gates main (release-gate.yml, docs/ship-policy.md#release-gate)" >> "$GITHUB_STEP_SUMMARY" else echo "train=true" >> "$GITHUB_OUTPUT" fi diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml index fd5ed557c..b662c8867 100644 --- a/.github/workflows/evals.yml +++ b/.github/workflows/evals.yml @@ -166,7 +166,8 @@ jobs: pull-requests: read outputs: mode: ${{ steps.route.outputs.mode }} - record_artifact: ${{ steps.policy.outputs.record_artifact }} + # A release bisect (gstack-release-gate) names its own prefix-record artifact; everything else uses the train block's. + record_artifact: ${{ steps.route.outputs.record_artifact || steps.policy.outputs.record_artifact }} full_gate_check: ${{ steps.policy.outputs.full_gate_check }} screen_head: ${{ steps.screen.outputs.head_sha }} screen_base: ${{ steps.screen.outputs.base_sha }} @@ -218,10 +219,23 @@ jobs: echo "::warning::the train block at ${base} names no valid record_artifact and full_gate_check; this run routes as before" fi + # The release gate's bisect (gstack-release-gate) routes only under a + # release_gate block at the same base revision; git and jq only. + - name: Read the release gate policy (base revision) + id: release_policy + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: | + policy=$(git show "${BASE_SHA:-origin/${DEFAULT_BRANCH}}:.gstack/ship-policy.json" 2>/dev/null || echo null) + release=$(jq -r 'if (.release_gate | type) == "object" then "true" else "false" end' <<<"$policy" 2>/dev/null || echo false) + echo "release=${release}" >> "$GITHUB_OUTPUT" + - name: Route this run id: route env: TRAIN_POLICY: ${{ steps.policy.outputs.train }} + RELEASE_POLICY: ${{ steps.release_policy.outputs.release }} NONCE: ${{ inputs.nonce }} EXPECTED_SHA: ${{ inputs.expected_sha }} EVALS_ALL: ${{ inputs.evals_all }} diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml new file mode 100644 index 000000000..583444a6d --- /dev/null +++ b/.github/workflows/release-gate.yml @@ -0,0 +1,118 @@ +name: Release gate +# The release gate (docs/ship-policy.md#release-gate; bin/gstack-release-gate, +# lib/release-gate.ts): the full paid gate on main's head every 8 hours, only +# when main moved since the last gate. It records the verdict as a release +# note in refs/notes/gstack-trains, publishes the main-backstop check run on +# the gated commit (what scripts/release-cut.ts reads as gated), and bisects +# a red gate's cases. It never writes the stable branch: release-cut is its +# only writer, and refuses every cut while the newest gate is red. Inert (no +# paid work, nothing written) unless main's .gstack/ship-policy.json declares +# a release_gate block; with one, the per-push main backstop +# (evals-main-backstop.yml) stands down and this replaces it. +on: + schedule: + # Every 8 hours (RELEASE_GATE_SCHEDULE_HOURS in lib/release-gate-policy.ts; + # test/release-gate-workflow.test.ts pins the two together), off the hour. + - cron: '17 */8 * * *' + workflow_dispatch: + inputs: + dry_run: + description: 'Print what the gate would test and dispatch nothing' + type: boolean + default: false + ignore_cadence: + description: 'Gate now even if the last gate is younger than cadence_hours (main must still have moved)' + type: boolean + default: true + +# One gate at a time; a second run waits for the first. +concurrency: + group: release-gate + cancel-in-progress: false + +permissions: {} + +jobs: + release-gate: + runs-on: ubuntu-24.04 + # A dispatch gates only from the default branch, with its tooling. + if: github.ref_name == github.event.repository.default_branch + # The full gate (270) + a bisect round + margin; past --budget-minutes the + # record stays resumable by the next run. + timeout-minutes: 345 + outputs: + action: ${{ steps.gate.outputs.action }} + sha: ${{ steps.gate.outputs.sha }} + check: ${{ steps.gate.outputs.check }} + permissions: + # Pushes the release record to refs/notes/gstack-trains and a bisect's + # release-gate/ branch (deleted after its round). + contents: write + # Dispatches the gate and reads its runs and artifacts. + actions: write + checks: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + fetch-depth: 0 + + # Read with git and jq only: without a release_gate block nothing below runs. + - name: Read the release gate policy (main) + id: policy + run: | + file=.gstack/ship-policy.json + policy=$(git show "${GITHUB_SHA}:${file}" 2>/dev/null || echo null) + gate=$(jq -c 'if (.release_gate | type) == "object" then .release_gate else null end' <<<"$policy") || { echo "::error::${GITHUB_SHA}:${file} is not valid JSON; fix the policy file" >&2; exit 1; } + if [ -z "$gate" ] || [ "$gate" = null ]; then + echo "release=false" >> "$GITHUB_OUTPUT" + echo "- Release gate inert: main's ${file} declares no release_gate block; the per-push backstop gates main" >> "$GITHUB_STEP_SUMMARY" + else + echo "release=true" >> "$GITHUB_OUTPUT" + fi + + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + if: steps.policy.outputs.release == 'true' + with: + bun-version: 1.4.2 + + - name: Gate main's head when it moved + id: gate + if: steps.policy.outputs.release == 'true' + env: + GH_TOKEN: ${{ github.token }} + GSTACK_STATE_ROOT: ${{ runner.temp }}/gstack-state + DRY_RUN: ${{ inputs.dry_run && '1' || '' }} + IGNORE_CADENCE: ${{ inputs.ignore_cadence && '1' || '' }} + run: | + ./bin/gstack-release-gate run --repo "$GITHUB_REPOSITORY" --sha "$GITHUB_SHA" --budget-minutes 330 \ + --summary "$GITHUB_STEP_SUMMARY" --outputs "$GITHUB_OUTPUT" ${DRY_RUN:+--dry-run} ${IGNORE_CADENCE:+--ignore-cadence} + + # The gated commit's check run (release_gate.check_name, main-backstop), the + # evidence scripts/release-cut.ts reads. Its own job with checks: write and + # no repository code, as for backstop-check: only the recorded action word. + release-check: + runs-on: ubuntu-24.04 + needs: release-gate + if: ${{ !cancelled() && contains(fromJSON('["gated","held","failed"]'), needs.release-gate.outputs.action) && needs.release-gate.outputs.sha != '' }} + timeout-minutes: 5 + permissions: + checks: write + steps: + - name: Publish the release gate's check run on the gated commit + env: + GH_TOKEN: ${{ github.token }} + ACTION: ${{ needs.release-gate.outputs.action }} + SHA: ${{ needs.release-gate.outputs.sha }} + CHECK: ${{ needs.release-gate.outputs.check }} + run: | + case "$ACTION" in + gated) conclusion=success ;; + held|failed) conclusion=failure ;; + *) echo "::error::unknown release gate action '$ACTION'"; exit 1 ;; + esac + [[ "$SHA" =~ ^[0-9a-f]{40}$ ]] || { echo "::error::'$SHA' is not a commit SHA"; exit 1; } + [[ "$CHECK" =~ ^[A-Za-z0-9_./-]{1,64}$ ]] || { echo "::error::'$CHECK' is not a check-run name"; exit 1; } + gh api "repos/${GITHUB_REPOSITORY}/check-runs" -X POST -f name="$CHECK" -f head_sha="$SHA" -f status=completed -f conclusion="$conclusion" \ + -f details_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ + -f "output[title]=Release gate: ${ACTION}" -f "output[summary]=${ACTION}: see the release-gate job summary of run ${GITHUB_RUN_ID}" >/dev/null + echo "- ${CHECK}: ${conclusion} (release gate ${ACTION}) on ${SHA}" >> "$GITHUB_STEP_SUMMARY" diff --git a/bin/gstack-release-gate b/bin/gstack-release-gate new file mode 100755 index 000000000..f523a0fd1 --- /dev/null +++ b/bin/gstack-release-gate @@ -0,0 +1,62 @@ +#!/usr/bin/env -S bun --no-env-file --config=/dev/null +/** + * gstack-release-gate — the full paid gate on main every cadence_hours, only + * when main moved since the last gate (docs/ship-policy.md#release-gate; + * logic: lib/release-gate.ts). The gate never moves the stable branch: + * scripts/release-cut.ts is its only writer and holds every cut while the + * newest gate is red. + * + * gstack-release-gate run [--dry-run] [--sha
] [--ignore-cadence] + * gstack-release-gate status + * gstack-release-gate bisect [--sha ] [--spend-cap ] [--dry-run] + * + * Every subcommand takes --repo (default $GITHUB_REPOSITORY, else + * gh repo view), --policy (a whole ship policy, validated; default the + * reviewed .gstack/ship-policy.json on origin/main), --remote (origin), + * --summary (appends each line as a bullet) and --outputs + * (appends action= and sha= for the workflow). run and bisect + * take --poll-seconds (60) and --budget-minutes (330): past the budget nothing + * waits, and the record stays resumable by the next run or bisect. + * Exit 0: inert, skipped, dry run, gated green, or status with no hold; + * 1: red (held), incomplete or could not finish; 2: usage. + */ +import { randomBytes } from 'node:crypto'; +import * as fs from 'node:fs'; +import { bisectGate, gateStatus, loadReleaseGate, runGate, type GateEnv, type GateOut } from '../lib/release-gate'; +import { runGh } from '../scripts/recover-receipts'; +import { gitIn } from '../scripts/train-gate'; + +const USAGE = 'usage: gstack-release-gate run [--dry-run] [--sha ] [--ignore-cadence] | status | bisect [--sha ] [--spend-cap ] [--dry-run]\n' + + ' common: [--repo ] [--policy ] [--remote ] [--summary ] [--outputs ] [--poll-seconds ] [--budget-minutes ]'; +const [command, ...rest] = process.argv.slice(2); +const flag = (name: string) => { const i = rest.indexOf(`--${name}`); return i >= 0 ? rest[i + 1] : undefined; }; +const has = (name: string) => rest.includes(`--${name}`); +const usage = (why?: string): never => { console.error(why ? `${why}\n${USAGE}` : USAGE); process.exit(2); }; +if (command === '--help' || command === '-h') { console.log(USAGE); process.exit(0); } +if (!['run', 'status', 'bisect'].includes(command ?? '')) usage(); +const sha = flag('sha'); +if (sha !== undefined && !/^[0-9a-f]{40}$/.test(sha)) usage('--sha needs a full 40-hex commit'); +const spendCap = flag('spend-cap') === undefined ? undefined : Number(flag('spend-cap')); +if (spendCap !== undefined && !(spendCap >= 0)) usage('--spend-cap needs dollars'); +const repoRoot = process.cwd(); +const repo = flag('repo') ?? process.env.GITHUB_REPOSITORY ?? runGh(['repo', 'view', '--json', 'nameWithOwner', '-q', '.nameWithOwner'], 60_000).stdout.trim(); +if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repo)) usage(`cannot name the repository (${JSON.stringify(repo)}); pass --repo `); +const loaded = loadReleaseGate(repoRoot, { policyFile: flag('policy') }); +const env: GateEnv = { + repo, repoRoot, remote: flag('remote') ?? 'origin', gh: runGh, git: gitIn(repoRoot), now: Date.now, sleep: ms => Bun.sleepSync(ms), + random: () => randomBytes(3).toString('hex'), pollMs: Number(flag('poll-seconds') ?? 60) * 1000, + deadline: Date.now() + Number(flag('budget-minutes') ?? 330) * 60_000, +}; +let out: GateOut; +if ('error' in loaded) out = { action: 'failed', sha: null, lines: [`RELEASE: policy refused: ${loaded.error}`] }; +else if (command === 'run') out = runGate(env, { ...loaded, sha, dryRun: has('dry-run'), ignoreCadence: has('ignore-cadence') }); +else if (command === 'bisect') out = bisectGate(env, { ...loaded, sha, spendCap, dryRun: has('dry-run') }); +else out = gateStatus(env, loaded); +const summary = flag('summary'); +for (const line of out.lines) { + console.log(line); + if (summary) fs.appendFileSync(summary, `- ${line}\n`); +} +const outputs = flag('outputs'); +if (outputs) fs.appendFileSync(outputs, `action=${out.action}\nsha=${out.sha ?? ''}\ncheck=${'gate' in loaded && loaded.gate ? loaded.gate.check_name : ''}\n`); +process.exit(out.action === 'held' || out.action === 'failed' ? 1 : 0); diff --git a/capy/skill/gstack/scripts/install.sh b/capy/skill/gstack/scripts/install.sh index 2cb8ae4c2..f7f2eaa1b 100755 --- a/capy/skill/gstack/scripts/install.sh +++ b/capy/skill/gstack/scripts/install.sh @@ -17,6 +17,12 @@ # checkout it prints `gstack: ok not-needed for=none (...)` and exits 0; with # one it prints the same trailer with the checkout's version, running nothing # from the checkout. That ok admits no workflow (no verdict is written). +# From revision 4 the clone and the floor fast-forward follow the update +# channel (docs/releases.md#channels): GSTACK_UPDATE_CHANNEL=stable|main, else +# the checkout's bin/gstack-update-channel.sh (auto: stable in team mode), +# else main. On stable, a fresh clone checks out origin's stable branch and +# the floor fast-forward goes to stable, each only when stable exists and +# meets the supported minimum; otherwise main, as before. --revision wins. # Everything else (Bun, setup, the doctor, the verdict, the launch line) is # the checkout's. The last line is the `gstack:` trailer; exit 0 only on # `gstack: ok`. Contract: docs/capy-install.md; tests: @@ -27,9 +33,10 @@ # bash install.sh --check --for none this machine runs no workflow: ok not-needed, nothing installed # bash install.sh [--for | --require ] [--with-browser] [--revision ] [--project ] # Environment: GSTACK_CAPY_ROOT (default ~/.local/share/gstack), GSTACK_CAPY_REPO, -# GSTACK_CAPY_AUTO_UPGRADE=0 (never fast-forward), GSTACK_CAPY_FETCH_TIMEOUT (s). +# GSTACK_CAPY_AUTO_UPGRADE=0 (never fast-forward), GSTACK_CAPY_FETCH_TIMEOUT (s), +# GSTACK_UPDATE_CHANNEL=main (follow every merge even in team mode) or stable. set -u -GSTACK_CAPY_WRAPPER=3 +GSTACK_CAPY_WRAPPER=4 self="$(cd "$(dirname "$0")" && pwd -P)/$(basename "$0")" root="${GSTACK_CAPY_ROOT:-$HOME/.local/share/gstack}" @@ -74,6 +81,24 @@ below() { return 1 } support_get() { printf '%s\n' "$1" | sed -n "s/^$2=\([0-9][0-9.]*\)[[:space:]]*\$/\1/p" | head -1; } +# channel_stable ROOT MIN [FETCH_FLAG] — true, with origin's stable commit in $stable_sha, when +# this install follows stable and origin's stable exists and meets MIN; says why not otherwise. +# FETCH_FLAG is --depth=1 for a fresh clone only (never --depth into an existing checkout). +channel_stable() { + local ch="${GSTACK_UPDATE_CHANNEL:-}" ver + case "$ch" in stable|main) ;; *) ch=$(bash "$1/bin/gstack-update-channel.sh" "$1" 2>/dev/null | head -1) ;; esac + [ "$ch" = stable ] || return 1 + if ! bounded git -C "$1" fetch --quiet ${3:+"$3"} origin stable 2>/dev/null; then + echo "gstack-capy-skill: channel stable, but origin has no stable branch yet (or the fetch failed); following main" + return 1 + fi + stable_sha=$(git -C "$1" rev-parse FETCH_HEAD) ver=$(git -C "$1" show FETCH_HEAD:VERSION 2>/dev/null | head -1) + if [ -n "$2" ] && below "${ver:-0}" "$2"; then + echo "gstack-capy-skill: channel stable is at ${ver:-unknown}, below the supported minimum $2; following main" + return 1 + fi + echo "gstack-capy-skill: channel stable ${ver:-unknown} (GSTACK_UPDATE_CHANNEL=main follows every merge)" +} mkdir -p "$(dirname "$root")" if [ "$check" -eq 0 ]; then @@ -100,6 +125,10 @@ if [ ! -f "$root/setup" ]; then fi remote_min=$(support_get "$(cat "$root/capy/support.env" 2>/dev/null)" GSTACK_MIN_REVISION) remote_version=$(head -1 "$root/VERSION" 2>/dev/null) + if [ -z "$revision" ] && channel_stable "$root" "$remote_min" --depth=1; then + git -C "$root" reset --quiet --hard "$stable_sha" \ + || { echo "gstack: fail installer — could not check out origin/stable in $root; fix: GSTACK_UPDATE_CHANNEL=main $fix_cmd"; exit 1; } + fi export GSTACK_CAPY_CLONED="$repo" else # The remote's rules: fetch the default branch into FETCH_HEAD (never --depth @@ -140,9 +169,11 @@ else echo "gstack: fail revision — installed $installed is below the supported minimum $remote_min ($cause); fix: $fix (REVISION_BELOW_MINIMUM)" exit 1 fi - if ! git -C "$root" merge --ff-only -q FETCH_HEAD 2>/dev/null; then + target=$(git -C "$root" rev-parse FETCH_HEAD) + if channel_stable "$root" "$remote_min" && git -C "$root" merge-base --is-ancestor HEAD "$stable_sha" 2>/dev/null; then target=$stable_sha; fi + if ! git -C "$root" merge --ff-only -q "$target" 2>/dev/null; then git -C "$root" fetch --quiet --unshallow origin "$default" 2>/dev/null - git -C "$root" merge --ff-only -q FETCH_HEAD 2>/dev/null \ + git -C "$root" merge --ff-only -q "$target" 2>/dev/null \ || { echo "gstack: fail revision — installed $installed is below the supported minimum $remote_min (fast-forward failed); fix: git -C $root status, then $fix_cmd (REVISION_BELOW_MINIMUM)"; exit 1; } fi echo "gstack-capy-skill: upgraded $installed -> $(head -1 "$root/VERSION")" diff --git a/changelog.d/gsta258-l1-release-gate.md b/changelog.d/gsta258-l1-release-gate.md new file mode 100644 index 000000000..0fd053ea5 --- /dev/null +++ b/changelog.d/gsta258-l1-release-gate.md @@ -0,0 +1,17 @@ +--- +bump: minor +--- +**A release gate can now gate main every 8 hours instead of on every push, and it feeds the stable release.** It is off until main's `.gstack/ship-policy.json` declares a `release_gate` block, so this release changes nothing on its own. With the block, `.github/workflows/release-gate.yml` runs `gstack-release-gate run` every 8 hours. When main moved since the last gate, it runs the full paid gate on main's head once, records the verdict as a `release` note in `refs/notes/gstack-trains`, and publishes the `main-backstop` check run on that commit. A red gate bisects its red cases within `bisect_spend_cap_usd` and names the first bad commit. The per-push main backstop stands down while the block is set, so main is never gated twice. `scripts/release-cut.ts` is still the only thing that moves `stable`, and its 72-hour floor and exceptions are unchanged. It now reads a green release record as proof the commit was gated, and it refuses every cut while the newest release gate is red with no green gate on a later commit. That hold is the freeze. See docs/ship-policy.md#release-gate. + +### Added +- **`release_gate` policy key** (`lib/release-gate-policy.ts`, key version 1.94.0.0): `cadence_hours` (8 to 168), `gate_workflow`, `branch`, `stable_ref`, `check_name`, `on_red: hold-release` and `bisect_spend_cap_usd`, every field required. +- **`bin/gstack-release-gate`** (`lib/release-gate.ts`): `run [--dry-run] [--sha] [--ignore-cadence]`, `status` and `bisect`. `--policy ` reads the block from a file, for a dry run or a proof before adoption. Exit 0 means gated, skipped, inert or a dry run; 1 means held; 2 means failed. +- **`.github/workflows/release-gate.yml`**: an 8-hour schedule and a manual dispatch. The check run comes from a separate job that holds `checks: write` and runs no repository code. +- **Capy skill installer follows the update channel** (`capy/skill/gstack/scripts/install.sh`, wrapper revision 4). With `GSTACK_UPDATE_CHANNEL=stable` or a stable `update_channel`, a fresh clone or a floor upgrade goes to `stable` when it meets the support floor, and to main otherwise. `--revision` still wins. + +### Changed +- **`evals-main-backstop.yml`** stands down (no gate, no freeze, no check run) when main's policy declares `release_gate`. +- **`evals.yml`** routes a release gate bisect (`release-gate/` refs) only under a `release_gate` block at the base revision, and that bisect publishes its prefix records as `gstack-release-record`. + +### Tests +- `test/release-gate.test.ts` drives run, bisect and status against a fake `gh` and a scratch remote. `test/release-gate-workflow.test.ts` runs the workflows' policy steps on scratch repositories and pins the cron to the cadence floor. `test/release-cut.test.ts` proves a red release record refuses the cut and a green one on a later commit releases it. `test/train-gate.test.ts` covers the bisect route, and `test/capy-skill-wrapper.test.ts` covers the stable channel and its fallback to main. diff --git a/docs/captain.md b/docs/captain.md index b09778c50..0a7edfb8b 100644 --- a/docs/captain.md +++ b/docs/captain.md @@ -298,7 +298,10 @@ through an urgent one-PR train (or `train build --repair` while the queue is fro refuses `--urgent`); all prefixes green earns exactly one unchanged rebuild; anything else goes to a repair lane. An infra-only census earns one re-dispatch. A backstop gate runs on every push to the base branch and freezes the queue when it finds -a tree no green train covered; one repair PR at its exact head clears the freeze. +a tree no green train covered; one repair PR at its exact head clears the freeze. A +policy with a `release_gate` block swaps that per-push backstop for one full gate on main +every 8 hours, whose red verdict holds the stable release instead of freezing the queue +([ship-policy.md#release-gate](ship-policy.md#release-gate)). **Wave PRs when the queue is long.** Every slot pays its own stamp, its own round of required checks and its own version, one after another. When more diff --git a/docs/releases.md b/docs/releases.md index de38b8ff4..18b473bad 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -33,7 +33,7 @@ of its own; it only reads what the trains and the backstop already recorded. With nothing green in the window it fails and lists why each commit was skipped. -**Gated** means one of two pieces of evidence. The first is the +**Gated** means one of three pieces of evidence. The first is the `main-backstop` check run: the Main backstop gate workflow publishes it on each push to main from a job of its own, success when a green train record covers the tree or the backstop gated it green, neutral when main moved past @@ -45,7 +45,12 @@ its tree: those entries are the main commits `gstack-ship-queue verify --train` confirmed, so a commit a green train merged is gated even when trains merged faster than the backstop could settle and its own backstop run ended neutral or never ran. The `CANDIDATE` line says which: `gated by -main-backstop` or `gated by train `. Train evidence relaxes nothing +main-backstop` or `gated by train `. The third is a GREEN release +record (kind `release`) on the commit, written by the release gate +([docs/ship-policy.md#release-gate](ship-policy.md#release-gate)) when the +policy declares `release_gate`; the line then says `gated by release gate run +`. A RED release record with no GREEN one on a later commit that +contains it refuses the cut like a freeze. Train evidence relaxes nothing else: a red backstop run, a failed or missing `free-tests` run, any other red check run or a run still going keeps the commit out, and a record in any other state (`red`, `superseded`, `free-green`, …) is not evidence. A commit with @@ -240,6 +245,10 @@ overrides the key for one command. The resolution lives in one helper, - `bin/gstack-upgrade` and `/gstack-upgrade`: they fast-forward to the channel ref (`UPSTREAM_REF=origin/stable` in the skill's output) and say which channel they followed. +- `capy/skill/gstack/scripts/install.sh`, the Capy skill's installer (wrapper + revision 4): a fresh clone or a floor upgrade follows `stable` when the channel + resolves to it and `stable` meets the support floor, and follows main + otherwise. `--revision` still pins an exact commit. Two edge cases are handled on purpose: diff --git a/docs/ship-policy.md b/docs/ship-policy.md index 7fd896a28..3a31c05ee 100644 --- a/docs/ship-policy.md +++ b/docs/ship-policy.md @@ -1118,6 +1118,81 @@ every lane to the per-PR queue at once. `--urgent` ships one PR alone, `--no-screen` skips the paid screen, `--broad` widens it to the full PR selection, and `train build --prs ` drops a PR at any step. + +## `release_gate`: one full gate on main every 8 hours + +Trains gate each batch of PRs, and the main backstop gates every push. The +release gate replaces the backstop with one full paid gate on main's head at +most every `cadence_hours`, and only when main moved since the last gate. It +feeds the stable release (docs/releases.md) and never writes `stable` itself: +`scripts/release-cut.ts` stays the only writer, with its 72-hour floor and its +security and regression exceptions unchanged. + +The surface is off unless main's `.gstack/ship-policy.json` declares the +block (key version 1.94.0.0, so `min_gstack_version` must be at least that): + +```json +"release_gate": { + "cadence_hours": 8, "gate_workflow": "evals.yml", "branch": "main", + "stable_ref": "stable", "check_name": "main-backstop", + "on_red": "hold-release", "bisect_spend_cap_usd": 40 +} +``` + +Every field is required. `cadence_hours` runs from 8 (the workflow's +schedule) to 168. `check_name` is the check run release-cut reads as gated, +so gstack keeps `main-backstop`. `bisect_spend_cap_usd` caps one red gate's +bisect (0 skips it). + +**What runs.** `.github/workflows/release-gate.yml` fires every 8 hours and +on a manual dispatch from main. Without the block it stops after a git-and-jq +policy read, so no paid work starts and nothing is written. With the block it runs +`gstack-release-gate run`, which: + +1. skips when main has not moved since `stable` or since the last gate, or + when the last gate is younger than `cadence_hours` (`--ignore-cadence` + overrides only the age); +2. writes a `release` record (`gstack-trains:release::`) to + `refs/notes/gstack-trains` before it dispatches, so a lost run resumes; +3. dispatches the full gate on that sha and reads its verdict and flake + records. A case that failed once and passed its one rule-16 rerun is green + and is listed as flaky; +4. records GREEN, RED or INCOMPLETE. An INCOMPLETE gate is re-dispatched once + by the next run, then left for a person; +5. on RED, bisects the red cases over main's first-parent commits since the + last green gate (at most 4 rounds, within the spend cap) and records the + first bad commit; +6. publishes `check_name` on the gated commit from a separate job that holds + `checks: write` and runs no repository code: success for a green gate, + failure for a red or failed one. + +**What release-cut reads.** A GREEN release record on a commit counts as +gated, the same as a successful `main-backstop` check run. A RED record +with no GREEN record on a later commit that contains it holds every cut, like a +queue freeze: the `REFUSED` line names the red sha and cases. Land the revert +or the fix-forward, then gate again with `gstack-release-gate run +--ignore-cadence` (or dispatch the workflow). Nothing else clears a hold. + +**The per-push backstop stands down.** `evals-main-backstop.yml` keeps its +push trigger. While the block is set, its policy step reports `train=false`, +so a push dispatches no gate, freezes nothing and publishes no check run. Main +is never gated twice. Trains keep running until the train lane retires them. + +```bash +gstack-release-gate run [--dry-run] [--sha ] [--ignore-cadence] +gstack-release-gate status # the newest record, the hold if any, pending commits +gstack-release-gate bisect [--sha ] [--spend-cap ] [--dry-run] +``` + +Common flags: `--repo`, `--policy ` (read the block from a file instead +of main, for a dry run or a proof before adoption), `--remote`, +`--summary`, `--outputs`, `--poll-seconds`, `--budget-minutes`. Every line +starts with `RELEASE:`. Exit 0 means gated, skipped, inert or a dry run; 1 +means held (red); 2 means failed or incomplete. + +**Escape hatch.** Remove the block on main. The per-push backstop gates the +next push again, and release-cut reads its check runs as before. + ## What `/ship` reads from it diff --git a/lib/release-gate-policy.ts b/lib/release-gate-policy.ts new file mode 100644 index 000000000..5da04d965 --- /dev/null +++ b/lib/release-gate-policy.ts @@ -0,0 +1,57 @@ +/** + * The `release_gate` block of .gstack/ship-policy.json + * (docs/ship-policy.md#release-gate): the full paid gate on the base branch + * every few hours, run by bin/gstack-release-gate from + * .github/workflows/release-gate.yml, instead of a gate per train or + * per push. The gate never writes the stable branch: scripts/release-cut.ts + * stays its only writer, reads a green gate as "gated" and holds every cut + * while the newest gate is red. Validated here so lib/ship-policy.ts stays + * one schema file. + */ +import type { PolicyError } from './ship-policy'; + +export interface ReleaseGatePolicy { + /** Least hours between two gate dispatches; at least RELEASE_GATE_SCHEDULE_HOURS, the workflow's schedule. */ + cadence_hours: number; + /** Workflow file dispatched for the full paid gate and the bisect; it accepts the eval-bg dispatch inputs. */ + gate_workflow: string; + /** The branch the gate tests (its head at each run). */ + branch: string; + /** The branch scripts/release-cut.ts moves; the gate only reads it to count pending commits. */ + stable_ref: string; + /** The check run published on a gated commit, read by release-cut as gated. */ + check_name: string; + /** What a red gate does. hold-release: the red record keeps release-cut from cutting until a later gate is green. */ + on_red: 'hold-release'; + /** Cap over one red gate's bisect, at RELEASE_TRIAL_RESERVATION_USD per case trial; 0 skips the bisect. */ + bisect_spend_cap_usd: number; +} + +/** The cron step of release-gate.yml's schedule (test/release-gate-workflow.test.ts pins it): no cadence gates more often. */ +export const RELEASE_GATE_SCHEDULE_HOURS = 8; +const REF = /^(?!.*\.\.)(?!\/)(?!.*\/$)[A-Za-z0-9._/-]{1,100}$/; +const FIELDS: Record string | null> = { + cadence_hours: v => (Number.isInteger(v) && (v as number) >= RELEASE_GATE_SCHEDULE_HOURS && (v as number) <= 168 ? null : `must be a whole number of hours from ${RELEASE_GATE_SCHEDULE_HOURS} (the workflow's schedule) to 168`), + gate_workflow: v => (typeof v === 'string' && /^[A-Za-z0-9_.-]{1,64}\.ya?ml$/.test(v) ? null : 'must be a workflow file name such as evals.yml'), + branch: v => (typeof v === 'string' && REF.test(v) ? null : 'must be a branch name'), + stable_ref: v => (typeof v === 'string' && REF.test(v) ? null : 'must be a branch name'), + check_name: v => (typeof v === 'string' && /^[A-Za-z0-9_./-]{1,64}$/.test(v) ? null : 'must be a check-run name ([A-Za-z0-9_./-]{1,64})'), + on_red: v => (v === 'hold-release' ? null : 'must be hold-release'), + bisect_spend_cap_usd: v => (typeof v === 'number' && v >= 0 && v <= 500 ? null : 'must be dollars from 0 to 500'), +}; + +export const RELEASE_GATE_EXPLANATION = '{ cadence_hours, gate_workflow, branch, stable_ref, check_name, on_red, bisect_spend_cap_usd } or null: the release gate (docs/ship-policy.md#release-gate). bin/gstack-release-gate dispatches the full paid gate (gate_workflow) on the head of branch at most once per cadence_hours, only when branch moved since the last gate, records the verdict as a release note in refs/notes/gstack-trains and publishes check_name on the gated commit; a red gate bisects its red cases (bisect_spend_cap_usd, 0 skips) and, with on_red hold-release, keeps scripts/release-cut.ts from moving stable_ref until a later gate is green. It never writes stable_ref. While the block is set, the per-push main backstop stands down. null keeps the per-push backstop.'; + +export function releaseGateErrors(raw: unknown): PolicyError[] { + const g = raw as Record | null; + if (!g || typeof g !== 'object' || Array.isArray(g)) return [{ code: 'POLICY_INVALID', path: 'release_gate', message: `must be { ${Object.keys(FIELDS).join(', ')} } or null` }]; + const out: PolicyError[] = []; + for (const k of Object.keys(g)) if (!(k in FIELDS)) out.push({ code: 'POLICY_INVALID', path: `release_gate.${k}`, message: 'unknown key' }); + for (const [k, check] of Object.entries(FIELDS)) { + if (!(k in g)) { out.push({ code: 'POLICY_INVALID', path: `release_gate.${k}`, message: 'missing (every release_gate field is required)' }); continue; } + const why = check(g[k]); + if (why) out.push({ code: 'POLICY_INVALID', path: `release_gate.${k}`, message: `${JSON.stringify(g[k])} ${why}` }); + } + if (typeof g.branch === 'string' && g.branch === g.stable_ref) out.push({ code: 'POLICY_INVALID', path: 'release_gate.stable_ref', message: 'must differ from branch' }); + return out; +} diff --git a/lib/release-gate.ts b/lib/release-gate.ts new file mode 100644 index 000000000..6255a5a0b --- /dev/null +++ b/lib/release-gate.ts @@ -0,0 +1,400 @@ +/** + * The release gate (docs/ship-policy.md#release-gate; plan ship-90-90 choice A): + * the full paid gate on the gated branch's head every cadence_hours, only when + * the branch moved since the last gate, instead of a gate per train or per push. + * + * run skip when the branch head equals the stable branch, already has a + * green or red record, or the last gate is younger than the cadence; + * otherwise write a PENDING `release` record to refs/notes/gstack-trains, + * dispatch gate_workflow on the head (eval-bg's backstop nonce), follow + * it, and record GREEN, RED or INCOMPLETE. Rule-16 reruns happen inside + * the gate run (scripts/lib/paid-rerun.ts): a case that failed once and + * passed its one rerun is green and listed as flaky here. + * bisect a red record's red verdict cases, one trial each, on up to + * MAX_PREFIXES first-parent commits between the last green gate (or + * the stable branch) and the red head, dispatched on release-gate/ + * and narrowed round by round until first_bad is the commit right after + * a green one; spend-capped and resumable. + * status stable, the last gate, pending commits and the hold. + * + * The gate never writes the stable branch. scripts/release-cut.ts is its only + * writer: it reads a green record (or the check_name check run this gate's + * workflow publishes) as gated, and refuses every cut while a red record is not + * followed by a green gate on a later commit (the hold, on_red hold-release). + */ +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { DISPATCH_INPUTS, makeNonce } from '../scripts/eval-bg'; +import { ghFailureReason, type GhRunner } from '../scripts/recover-receipts'; +import { MAX_BISECT_CASES, MAX_PREFIXES, RELEASE_BISECT_ARTIFACT, RELEASE_BISECT_REF, type Git } from '../scripts/train-gate'; +import type { ReleaseGatePolicy } from './release-gate-policy'; +import { loadPolicyFromBase, mergePolicy, validatePolicy } from './ship-policy'; +import { + NOTES_REF, fetchNotes, nowIso, persist, repoIdentity, + type PrefixResult, type ReleaseBisect, type ReleaseBisectRound, type ReleaseRecord, +} from './ship-queue-train-record'; + +export const RELEASE_TRIAL_RESERVATION_USD = 1.5; +/** The gate plus the one re-dispatch EVAL_POLICY.infraRedispatch allows an incomplete census. */ +export const MAX_RELEASE_ATTEMPTS = 2; +/** Cron start jitter the cadence forgives. */ +export const CADENCE_GRACE_HOURS = 1; +export const MAX_BISECT_ROUNDS = 4; +const SHA = /^[0-9a-f]{40}$/; +const CASE_ID = /^[A-Za-z0-9 _.:/-]{1,128}$/; +const HOUR = 3_600_000; +const GH_MS = 60_000; + +export interface Run { id: number; status: string; conclusion: string | null; head_sha: string; display_title?: string; html_url?: string; run_attempt?: number } +export interface GateEnv { + repo: string; repoRoot: string; remote: string; gh: GhRunner; git: Git; + now: () => number; sleep: (ms: number) => void; random: () => string; pollMs: number; + /** Epoch ms after which nothing waits on a run any more; the record stays resumable. */ + deadline: number; +} +export type GateAction = 'inert' | 'skipped' | 'dry-run' | 'gated' | 'held' | 'failed'; +export interface GateOut { action: GateAction; sha: string | null; lines: string[] } + +const short = (sha: string | null | undefined) => (sha ? sha.slice(0, 12) : 'none'); +const usd = (n: number | null) => (n === null ? 'unknown' : n.toFixed(2)); + +/** The release_gate block from origin/'s reviewed policy, or from --policy (validated the same way). */ +export function loadReleaseGate(repoRoot: string, o: { policyFile?: string; base?: string }): { gate: ReleaseGatePolicy | null; label: string } | { error: string } { + if (o.policyFile) { + let raw: unknown; + try { raw = JSON.parse(fs.readFileSync(o.policyFile, 'utf8')); } catch (e: any) { return { error: `${o.policyFile} is not readable JSON: ${e.message}` }; } + const errors = validatePolicy(raw); + if (errors.length) return { error: `${o.policyFile}: ${errors.map(e => `${e.path} ${e.message}`).join('; ')}` }; + return { gate: mergePolicy(raw as Record, o.policyFile).policy.release_gate, label: o.policyFile }; + } + const loaded = loadPolicyFromBase(repoRoot, o.base ?? 'main'); + if ('unavailable' in loaded) return { error: `no reviewed policy: ${loaded.detail}` }; + if (loaded.errors.length) return { error: `${loaded.source.label}: ${loaded.errors.map(e => `${e.path} ${e.message}`).join('; ')}` }; + return { gate: loaded.policy.release_gate, label: loaded.source.label }; +} + +/** This repository's release records from the local notes ref, newest first. */ +export function releaseRecords(git: Git, repoKey: string): ReleaseRecord[] { + const listed = git(['notes', `--ref=${NOTES_REF}`, 'list']); + if (!listed.ok || !listed.out.trim()) return []; + return listed.out.split('\n').map(line => line.split(' ')[0]!).filter(Boolean).flatMap(blob => { + try { const r = JSON.parse(git(['cat-file', 'blob', blob]).out) as ReleaseRecord; return r?.kind === 'release' && r.repo_key === repoKey && SHA.test(r.sha) ? [r] : []; } catch { return []; } + }).sort((a, b) => b.created_at.localeCompare(a.created_at)); +} + +/** The red record that holds releases: red, and no green record sits on a later commit that contains it. Null when none. */ +export function activeHold(records: readonly ReleaseRecord[], isAncestor: (older: string, newer: string) => boolean): ReleaseRecord | null { + const greens = records.filter(r => r.verdict === 'GREEN'); + return records.filter(r => r.verdict === 'RED').find(red => !greens.some(g => g.sha !== red.sha && isAncestor(red.sha, g.sha))) ?? null; +} + +/** The newest green record on an ancestor of sha: where pending commits and a bisect start. */ +export function greenBase(records: readonly ReleaseRecord[], sha: string, isAncestor: (older: string, newer: string) => boolean): ReleaseRecord | null { + return records.find(r => r.verdict === 'GREEN' && r.sha !== sha && isAncestor(r.sha, sha)) ?? null; +} + +export interface ReportFacts { verdict: 'GREEN' | 'RED' | 'NONE'; red: string[]; other: string[]; cost: number | null } + +/** report-verdict's report.txt: the verdict, each failed case (a verdict red, or another class), and the known cost. */ +export function parseReport(text: string): ReportFacts { + const verdict = (/VERDICT (GREEN|RED)/.exec(text)?.[1] as 'GREEN' | 'RED' | undefined) ?? 'NONE'; + const red: string[] = []; + const other: string[] = []; + for (const m of text.matchAll(/^\s+✗ (\S(?:.*?\S)?) {2,}(\w+) {2,}(\w+)(?: {2,}([A-Z_]+))?/gm)) { + (m[3] === 'FAIL' && (!m[4] || m[4] === 'VERDICT') ? red : other).push(m[1]!); + } + const cost = /· cost \$(\d+(?:\.\d+)?)/.exec(text)?.[1]; + return { verdict, red: [...new Set(red)].sort(), other: [...new Set(other)].sort(), cost: cost ? Number(cost) : null }; +} + +/** A finished gate run as one verdict; green needs the run's own success, which follows its rule-16 report. */ +export function classifyRun(run: Pick, report: ReportFacts | null): { verdict: ReleaseRecord['verdict']; red: string[]; detail: string } { + if (run.status !== 'completed') return { verdict: 'PENDING', red: [], detail: `run is ${run.status}` }; + if (run.conclusion === 'success') { + if (report?.verdict === 'RED') return { verdict: 'INCOMPLETE', red: [], detail: 'the run concluded success but its report says RED' }; + return { verdict: 'GREEN', red: [], detail: report ? 'run and report green' : 'run concluded success (no report artifact)' }; + } + if (report?.verdict === 'RED' && report.red.length) { + return { verdict: 'RED', red: report.red, detail: `${report.red.length} red case(s)${report.other.length ? `; also not green: ${report.other.join(', ')}` : ''}` }; + } + return { verdict: 'INCOMPLETE', red: [], detail: `run concluded ${run.conclusion ?? 'unknown'}; report ${report ? `${report.verdict}${report.other.length ? ` (${report.other.join(', ')})` : ''}` : 'missing'}` }; +} + +/** Up to max commits spread evenly over the interior of a range (oldest first), never the endpoints. */ +export function planPrefixes(interior: readonly string[], max = MAX_PREFIXES): string[] { + if (interior.length <= max) return [...interior]; + const picked = Array.from({ length: max }, (_, i) => interior[Math.round(((i + 1) * (interior.length + 1)) / (max + 1)) - 1]!); + return [...new Set(picked)]; +} + +/** + * One round's results narrow (good, bad]: the earliest prefix where any case is + * red becomes bad, the prefix before it good. Every case must be monotone + * (green, then red) over the tested prefixes and have a result on each. + */ +export function narrow(range: { good: string; bad: string }, tested: ReadonlyArray<{ sha: string; results: Record }>, cases: readonly string[]): { good: string; bad: string } | { unknown: string } { + let firstRed = tested.length; + for (const c of cases) { + const seq = tested.map(p => (p.results[c] === 'red' ? 'R' : p.results[c] === 'green' ? 'G' : '?')).join(''); + if (seq.includes('?')) return { unknown: `no result for ${c} on ${tested.filter(p => !p.results[c]).map(p => short(p.sha)).join(', ')}` }; + const at = seq.indexOf('R'); + if (at >= 0 && seq.slice(at).includes('G')) return { unknown: `${c} is not monotone over the tested commits (${seq})` }; + if (at >= 0) firstRed = Math.min(firstRed, at); + } + return { good: firstRed > 0 ? tested[firstRed - 1]!.sha : range.good, bad: firstRed < tested.length ? tested[firstRed]!.sha : range.bad }; +} + +function ghJson(env: GateEnv, args: string[]): T | null { + const r = env.gh(args, GH_MS); + if (r.status !== 0) return null; + try { return JSON.parse(r.stdout) as T; } catch { return null; } +} + +/** One file of a run's artifact, or null when the artifact or the file is missing. */ +function artifactFile(env: GateEnv, run: number, name: string, file: string): string | null { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'release-gate-')); + try { + if (env.gh(['run', 'download', String(run), '-R', env.repo, '-n', name, '-D', dir], GH_MS).status !== 0) return null; + const find = (d: string): string | null => { + for (const e of fs.readdirSync(d, { withFileTypes: true })) { + const p = path.join(d, e.name); + if (e.isDirectory()) { const hit = find(p); if (hit) return hit; } else if (e.name === file) return p; + } + return null; + }; + const hit = find(dir); + return hit ? fs.readFileSync(hit, 'utf8') : null; + } finally { fs.rmSync(dir, { recursive: true, force: true }); } +} + +function dispatch(env: GateEnv, workflow: string, ref: string, inputs: Record): string | null { + const r = env.gh(['workflow', 'run', workflow, '--repo', env.repo, '--ref', ref, ...Object.entries(inputs).flatMap(([k, v]) => ['-f', `${k}=${v}`])], GH_MS); + return r.status === 0 ? null : ghFailureReason(r.stderr); +} + +/** Find a dispatch's run by its nonce, then follow it until it completes or the deadline passes. */ +function follow(env: GateEnv, workflow: string, nonce: string, known: number | null, lines: string[]): Run | null { + let run: Run | null = known ? ghJson(env, ['api', `repos/${env.repo}/actions/runs/${known}`]) : null; + for (;;) { + if (!run) { + const runs = ghJson<{ workflow_runs?: Run[] }>(env, ['api', '-X', 'GET', `repos/${env.repo}/actions/workflows/${workflow}/runs`, '-f', 'event=workflow_dispatch', '-f', 'per_page=30']); + run = runs?.workflow_runs?.find(r => (r.display_title ?? '').includes(`eval-bg ${nonce}]`)) ?? null; + if (run) lines.push(`RELEASE: run ${run.id} ${run.html_url ?? ''}`.trim()); + } else if (run.status === 'completed') return run; + if (env.now() >= env.deadline) break; + env.sleep(env.pollMs); + if (run) run = ghJson(env, ['api', `repos/${env.repo}/actions/runs/${run.id}`]) ?? run; + } + lines.push(`RELEASE: ${run ? `run ${run.id} is still ${run.status}` : `no run for nonce ${nonce} yet`} at the deadline; the record stays resumable`); + return run; +} + +const runRef = (env: GateEnv, run: Run) => ({ id: run.id, url: run.html_url ?? `https://github.com/${env.repo}/actions/runs/${run.id}` }); + +function save(env: GateEnv, rec: ReleaseRecord, lines: string[]): void { + rec.updated_at = nowIso(); + const p = persist(env.repoRoot, [rec], { remote: env.remote }); + if (!p.pushed) lines.push(`RELEASE: record ${p.detail}`); +} + +interface Context { gate: ReleaseGatePolicy; repoKey: string; stable: string | null; records: ReleaseRecord[]; isAncestor: (a: string, b: string) => boolean } + +function context(env: GateEnv, gate: ReleaseGatePolicy): Context | { error: string } { + const f = fetchNotes(env.repoRoot, env.remote); + if (!f.ok) return { error: f.detail }; + const stable = env.git(['ls-remote', env.remote, `refs/heads/${gate.stable_ref}`]).out.split(/\s/)[0] || null; + const isAncestor = (a: string, b: string) => env.git(['merge-base', '--is-ancestor', a, b]).ok; + const repoKey = repoIdentity(env.repoRoot).key; + return { gate, repoKey, stable: stable && SHA.test(stable) ? stable : null, records: releaseRecords(env.git, repoKey), isAncestor }; +} + +/** Make sure the gated branch's history is here (a shallow checkout gets it), so ancestry and ranges read true. */ +function ensureHistory(env: GateEnv, gate: ReleaseGatePolicy, sha: string): void { + if (env.git(['rev-parse', '--is-shallow-repository']).out === 'true') env.git(['fetch', '--no-tags', '-q', '--unshallow', env.remote, gate.branch, gate.stable_ref]); + if (!env.git(['cat-file', '-e', `${sha}^{commit}`]).ok) env.git(['fetch', '--no-tags', '-q', env.remote, gate.branch]); +} + +const firstParents = (env: GateEnv, from: string, to: string): string[] | null => { + const r = env.git(['rev-list', '--first-parent', '--reverse', `${from}..${to}`]); + return r.ok ? r.out.split('\n').filter(Boolean) : null; +}; + +export interface RunOptions { gate: ReleaseGatePolicy | null; label: string; sha?: string; dryRun: boolean; ignoreCadence: boolean } + +/** `run`: gate the branch head once when it moved; bisect a red. */ +export function runGate(env: GateEnv, o: RunOptions): GateOut { + const lines: string[] = []; + const out = (action: GateAction, sha: string | null): GateOut => ({ action, sha, lines }); + if (!o.gate) { lines.push(`RELEASE: inert (${o.label} declares no release_gate block; the per-push main backstop gates main)`); return out('inert', null); } + const gate = o.gate; + const c = context(env, gate); + if ('error' in c) { lines.push(`RELEASE: cannot read refs/notes/gstack-trains (${c.error}); nothing dispatched`); return out('failed', null); } + const sha = o.sha ?? env.git(['ls-remote', env.remote, `refs/heads/${gate.branch}`]).out.split(/\s/)[0] ?? ''; + if (!SHA.test(sha)) { lines.push(`RELEASE: cannot resolve ${gate.branch} on ${env.remote} (${JSON.stringify(sha)})`); return out('failed', null); } + ensureHistory(env, gate, sha); + const prior = c.records.find(r => r.sha === sha) ?? null; + const skip = (why: string) => { lines.push(`RELEASE: skip sha=${sha} (${why})`); return out('skipped', sha); }; + if (sha === c.stable) return skip(`${gate.branch} has not moved since ${gate.stable_ref}`); + if (prior?.verdict === 'GREEN' || prior?.verdict === 'RED') { + if (prior.verdict === 'RED' && prior.bisect?.state === 'running' && !o.dryRun) return finishRed(env, c, prior, lines, out); + return skip(`${gate.branch} has not moved since the last gate: ${prior.verdict} run ${prior.run?.id ?? '-'}`); + } + if (prior?.verdict === 'INCOMPLETE' && prior.attempts >= MAX_RELEASE_ATTEMPTS) return skip(`the gate on this commit was incomplete ${prior.attempts} times (${prior.detail}); diagnose run ${prior.run?.url ?? prior.run?.id ?? '-'}`); + const newest = c.records[0]; + const age = newest ? (env.now() - Date.parse(newest.created_at)) / HOUR : Infinity; + if (!o.ignoreCadence && prior?.verdict !== 'PENDING' && age < gate.cadence_hours - CADENCE_GRACE_HOURS) { + return skip(`the last gate was dispatched ${age.toFixed(1)} h ago on ${short(newest!.sha)}; cadence ${gate.cadence_hours} h`); + } + const since = greenBase(c.records, sha, c.isAncestor)?.sha ?? c.stable; + const pending = since ? firstParents(env, since, sha)?.length ?? null : null; + lines.push(`RELEASE: gate sha=${sha} since=${short(since)} (${since === c.stable ? gate.stable_ref : 'last green gate'}) pending=${pending ?? 'unknown'} stable=${short(c.stable)}`); + if (o.dryRun) { lines.push(`RELEASE: dry run: would dispatch ${gate.gate_workflow} on ${gate.branch}@${short(sha)} (full gate, about $37); nothing written`); return out('dry-run', sha); } + const now = nowIso(); + const rec: ReleaseRecord = prior?.verdict === 'PENDING' ? prior : { + schema: 1, kind: 'release', repo_key: c.repoKey, sha, tree: env.git(['rev-parse', `${sha}^{tree}`]).out, branch: gate.branch, stable: c.stable, since, + nonce: makeNonce('gate', sha, sha, env.random(), { purpose: 'backstop' }), attempts: (prior?.attempts ?? 0) + 1, run: null, verdict: 'PENDING', + red_cases: [], flaky: [], cost_usd: null, detail: 'dispatching', bisect: null, created_at: now, updated_at: now, + }; + if (rec !== prior) { + save(env, rec, lines); + const failed = dispatch(env, gate.gate_workflow, gate.branch, { [DISPATCH_INPUTS.evalsAll]: 'true', [DISPATCH_INPUTS.expectedSha]: sha, [DISPATCH_INPUTS.nonce]: rec.nonce }); + if (failed) { Object.assign(rec, { verdict: 'INCOMPLETE', detail: `the gate dispatch failed (${failed})` }); save(env, rec, lines); lines.push(`RELEASE: incomplete sha=${sha} run=- (${rec.detail})`); return out('failed', sha); } + lines.push(`RELEASE: dispatched ${gate.gate_workflow} on ${gate.branch}@${short(sha)} (nonce ${rec.nonce})`); + } else lines.push(`RELEASE: resuming the pending gate on ${short(sha)} (nonce ${rec.nonce})`); + const run = follow(env, gate.gate_workflow, rec.nonce, rec.run?.id ?? null, lines); + if (run) rec.run = runRef(env, run); + if (!run || run.status !== 'completed') { rec.detail = run ? `run ${run.id} ${run.status}` : 'no run found for the nonce yet'; save(env, rec, lines); return out('failed', sha); } + const attempt = run.run_attempt ?? 1; + const reportText = artifactFile(env, run.id, `report-verdict-a${attempt}`, 'report.txt'); + const report = reportText === null ? null : parseReport(reportText); + const flakes = artifactFile(env, run.id, `flake-records-a${attempt}`, 'flake-records.jsonl') ?? ''; + const k = classifyRun(run, report); + Object.assign(rec, { + verdict: k.verdict, red_cases: k.red, detail: k.detail, cost_usd: report?.cost ?? null, + flaky: [...new Set(flakes.split('\n').flatMap(l => { try { const f = JSON.parse(l); return typeof f?.case === 'string' ? [f.case] : []; } catch { return []; } }))].sort(), + }); + save(env, rec, lines); + if (rec.flaky.length) lines.push(`RELEASE: flaky (failed once, passed its one rule-16 rerun in run ${run.id}): ${rec.flaky.join(', ')}`); + if (rec.verdict === 'GREEN') { + lines.push(`RELEASE: green sha=${sha} stable=${c.stable ?? 'none'} run=${run.id} cost=${usd(rec.cost_usd)}`, + `RELEASE: ${gate.check_name} success goes on ${short(sha)}; scripts/release-cut.ts moves ${gate.stable_ref} on its own schedule and floor`); + return out('gated', sha); + } + if (rec.verdict === 'RED') return finishRed(env, c, rec, lines, out); + lines.push(`RELEASE: incomplete sha=${sha} run=${run.id} (${rec.detail})${rec.attempts < MAX_RELEASE_ATTEMPTS ? '; the next run re-dispatches it once' : ''}`); + return out('failed', sha); +} + +function finishRed(env: GateEnv, c: Context, rec: ReleaseRecord, lines: string[], out: (a: GateAction, s: string | null) => GateOut): GateOut { + bisectRecord(env, c, rec, { spendCap: c.gate.bisect_spend_cap_usd, dryRun: false }, lines); + lines.push(`RELEASE: red sha=${rec.sha} first_bad=${rec.bisect?.first_bad ?? 'unknown'} run=${rec.run?.id ?? '-'}`, + `RELEASE: hold: scripts/release-cut.ts refuses to move ${c.gate.stable_ref} until a gate on a later commit is green (red: ${rec.red_cases.join(', ')}); land the revert or the fix-forward`); + return out('held', rec.sha); +} + +/** Collect a bisect round's prefix records into its results; the cost they report is added to the spend. */ +function collectRound(env: GateEnv, b: ReleaseBisect, round: ReleaseBisectRound, run: Run): void { + for (const p of round.prefixes) { + const text = artifactFile(env, run.id, `${RELEASE_BISECT_ARTIFACT}-bisect-${p.sha.slice(0, 12)}`, 'bisect-prefix.json'); + let rec: { kind?: string; nonce?: string; prefix_sha?: string; checked_out_sha?: string; panels?: Array<{ case: string; status: string; red_class: string | null; cost_usd: number | null }> } | null = null; + try { rec = text ? JSON.parse(text) : null; } catch { rec = null; } + if (!rec || rec.kind !== 'train-bisect-prefix' || rec.nonce !== round.nonce || rec.prefix_sha !== p.sha || rec.checked_out_sha !== p.sha || !Array.isArray(rec.panels)) continue; + for (const panel of rec.panels) { + if (!b.cases.includes(panel.case)) continue; + b.spent_usd = Math.round((b.spent_usd + (typeof panel.cost_usd === 'number' ? panel.cost_usd : 0)) * 100) / 100; + if (panel.status === 'PASS') p.results[panel.case] = 'green'; + else if (panel.status === 'FAIL' && panel.red_class === 'VERDICT') p.results[panel.case] = 'red'; + } + } +} + +/** + * Bisect a red record (resumable): finish a round still running, then narrow + * (good, bad] with up to MAX_PREFIXES commits a round until they are adjacent, + * the pattern is not monotone, the cap is spent, or MAX_BISECT_ROUNDS pass. + */ +export function bisectRecord(env: GateEnv, c: Context, rec: ReleaseRecord, o: { spendCap: number; dryRun: boolean }, lines: string[]): void { + const b: ReleaseBisect = rec.bisect ?? { cases: rec.red_cases, good: rec.since, bad: rec.sha, first_bad: null, spend_cap: o.spendCap, spent_usd: 0, state: 'running', detail: '', rounds: [] }; + b.spend_cap = Math.max(b.spend_cap, o.spendCap); + const stop = (state: ReleaseBisect['state'], detail: string) => { Object.assign(b, { state, detail }); }; + if (!o.dryRun) rec.bisect = b; + if (b.state === 'running' && o.spendCap <= 0 && !b.rounds.length) stop('unknown', 'bisect_spend_cap_usd is 0: no bisect'); + else if (b.state === 'running' && (!b.cases.length || b.cases.length > MAX_BISECT_CASES || b.cases.some(id => !CASE_ID.test(id)))) stop('unknown', `the red cases cannot be bisected (${b.cases.length} case ids, at most ${MAX_BISECT_CASES})`); + else if (b.state === 'running' && !b.good) stop('unknown', 'no green gate and no stable branch before the red commit to bisect from'); + while (b.state === 'running') { + const open = b.rounds.at(-1); + if (open && open.prefixes.some(p => b.cases.some(cs => !p.results[cs])) && open.run?.id !== -1) { + if (o.dryRun) { lines.push(`RELEASE: dry run: round ${b.rounds.length} (nonce ${open.nonce}) still has prefixes to collect`); return; } + const run = follow(env, c.gate.gate_workflow, open.nonce, open.run?.id ?? null, lines); + if (run) open.run = runRef(env, run); + if (!run || run.status !== 'completed') { b.detail = `round ${b.rounds.length} is still running`; save(env, rec, lines); return; } + collectRound(env, b, open, run); + env.git(['push', '-q', env.remote, '--delete', `refs/heads/${open.ref}`]); + const n = narrow({ good: b.good!, bad: b.bad }, open.prefixes, b.cases); + if ('unknown' in n) { stop('unknown', n.unknown); break; } + Object.assign(b, n); + lines.push(`RELEASE: bisect round ${b.rounds.length}: good=${short(b.good)} bad=${short(b.bad)} spent=$${b.spent_usd.toFixed(2)}`); + save(env, rec, lines); + } + const interior = firstParents(env, b.good!, b.bad); + if (!interior) { stop('unknown', `cannot list ${short(b.good)}..${short(b.bad)}`); break; } + interior.pop(); + if (!interior.length) { b.first_bad = b.bad; stop('found', `${short(b.bad)} is the first red commit after green ${short(b.good)}`); break; } + if (b.rounds.length >= MAX_BISECT_ROUNDS) { stop('unknown', `${MAX_BISECT_ROUNDS} rounds left ${interior.length} commit(s) between ${short(b.good)} and ${short(b.bad)}`); break; } + const perPrefix = b.cases.length * RELEASE_TRIAL_RESERVATION_USD; + const fits = Math.floor((b.spend_cap - b.spent_usd + 1e-9) / perPrefix); + const prefixes = planPrefixes(interior, Math.min(MAX_PREFIXES, fits)); + if (!prefixes.length) { stop('capped', `the next round needs $${perPrefix.toFixed(2)} a prefix; $${(b.spend_cap - b.spent_usd).toFixed(2)} of the $${b.spend_cap.toFixed(2)} cap is left`); break; } + lines.push(`RELEASE: bisect round ${b.rounds.length + 1}: ${b.cases.length} case(s) on ${prefixes.length} of ${interior.length} commit(s) between ${short(b.good)} and ${short(b.bad)}, $${(perPrefix * prefixes.length).toFixed(2)} reserved`); + if (o.dryRun) { lines.push('RELEASE: dry run: nothing dispatched'); return; } + const nonce = makeNonce('gate', b.bad, b.good!, env.random(), { purpose: 'bisect' }); + const ref = `${RELEASE_BISECT_REF}${nonce.slice(-6)}-${b.bad.slice(0, 12)}`; + const round: ReleaseBisectRound = { nonce, ref, run: null, prefixes: prefixes.map(sha => ({ sha, results: {} })) }; + b.rounds.push(round); + save(env, rec, lines); + const pushed = env.git(['push', '-q', env.remote, `${b.bad}:refs/heads/${ref}`]); + const failed = pushed.ok ? dispatch(env, c.gate.gate_workflow, ref, { + [DISPATCH_INPUTS.cases]: b.cases.join(','), [DISPATCH_INPUTS.prefixShas]: prefixes.join(','), [DISPATCH_INPUTS.expectedSha]: b.bad, [DISPATCH_INPUTS.nonce]: nonce, + }) : `git push ${ref} failed: ${pushed.out.slice(0, 120)}`; + if (failed) { round.run = { id: -1, url: '' }; env.git(['push', '-q', env.remote, '--delete', `refs/heads/${ref}`]); stop('unknown', `round ${b.rounds.length} dispatch failed: ${failed}`); } + } + lines.push(`RELEASE: bisect ${b.state}: ${b.detail} (spent $${b.spent_usd.toFixed(2)} of $${b.spend_cap.toFixed(2)})`); + if (!o.dryRun) save(env, rec, lines); +} + +/** `bisect`: resume or start the bisect of the newest red record (or the one at sha). */ +export function bisectGate(env: GateEnv, o: { gate: ReleaseGatePolicy | null; label: string; sha?: string; spendCap?: number; dryRun: boolean }): GateOut { + const lines: string[] = []; + if (!o.gate) { lines.push(`RELEASE: inert (${o.label} declares no release_gate block)`); return { action: 'inert', sha: null, lines }; } + const c = context(env, o.gate); + if ('error' in c) { lines.push(`RELEASE: cannot read refs/notes/gstack-trains (${c.error})`); return { action: 'failed', sha: null, lines }; } + const rec = o.sha ? c.records.find(r => r.sha === o.sha) : c.records.find(r => r.verdict === 'RED'); + if (!rec || rec.verdict !== 'RED') { lines.push(`RELEASE: no red release record${o.sha ? ` at ${o.sha}` : ''} to bisect`); return { action: 'failed', sha: o.sha ?? null, lines }; } + ensureHistory(env, o.gate, rec.sha); + if (rec.bisect?.state === 'capped' && (o.spendCap ?? 0) > rec.bisect.spend_cap) rec.bisect.state = 'running'; + bisectRecord(env, c, rec, { spendCap: o.spendCap ?? o.gate.bisect_spend_cap_usd, dryRun: o.dryRun }, lines); + lines.push(`RELEASE: red sha=${rec.sha} first_bad=${rec.bisect?.first_bad ?? 'unknown'} run=${rec.run?.id ?? '-'}`); + return { action: o.dryRun ? 'dry-run' : 'held', sha: rec.sha, lines }; +} + +/** `status`: never spends and never writes. */ +export function gateStatus(env: GateEnv, o: { gate: ReleaseGatePolicy | null; label: string }): GateOut { + const lines: string[] = []; + if (!o.gate) { lines.push(`RELEASE: inert (${o.label} declares no release_gate block; the per-push main backstop gates main)`); return { action: 'inert', sha: null, lines }; } + const c = context(env, o.gate); + if ('error' in c) { lines.push(`RELEASE: cannot read refs/notes/gstack-trains (${c.error})`); return { action: 'failed', sha: null, lines }; } + const head = env.git(['ls-remote', env.remote, `refs/heads/${o.gate.branch}`]).out.split(/\s/)[0] ?? ''; + if (SHA.test(head)) ensureHistory(env, o.gate, head); + const last = c.records[0]; + const base = SHA.test(head) ? greenBase(c.records, head, c.isAncestor)?.sha ?? c.stable : null; + const pending = base && SHA.test(head) ? (base === head ? 0 : firstParents(env, base, head)?.length ?? null) : null; + const hold = activeHold(c.records, c.isAncestor); + lines.push( + `STABLE: ${o.gate.stable_ref} ${c.stable ?? 'absent'} (scripts/release-cut.ts moves it; the gate never does)`, + `LAST_GATE: ${last ? `${last.verdict} sha=${last.sha} run=${last.run?.id ?? '-'} at=${last.created_at} cost=${usd(last.cost_usd)}${last.flaky.length ? ` flaky=${last.flaky.join(',')}` : ''}` : 'none'}`, + `PENDING: ${pending ?? 'unknown'} commit(s) on ${o.gate.branch} ${short(head)} since ${short(base)}`, + `HOLD: ${hold ? `red on ${hold.sha} since ${hold.created_at} (red: ${hold.red_cases.join(', ')}; first_bad=${hold.bisect?.first_bad ?? 'unknown'}, bisect ${hold.bisect?.state ?? 'not run'}); release-cut refuses until a gate on a later commit is green` : 'none'}`, + ); + return { action: hold ? 'held' : 'skipped', sha: SHA.test(head) ? head : null, lines }; +} diff --git a/lib/ship-policy.ts b/lib/ship-policy.ts index 4268b007e..61d520ced 100644 --- a/lib/ship-policy.ts +++ b/lib/ship-policy.ts @@ -15,6 +15,7 @@ import { spawnSync } from 'node:child_process'; import * as fs from 'node:fs'; import * as path from 'node:path'; +import { RELEASE_GATE_EXPLANATION, releaseGateErrors, type ReleaseGatePolicy } from './release-gate-policy'; import { RESULT_CODES, type ResultCodeName } from './result-codes'; import { cmpVersion, parseVersion } from './version-source'; @@ -77,6 +78,8 @@ export interface ShipPolicy { docs: { caps: Array<{ path: string; max_bytes: number }> } | null; /** How /ship reproduces a Step 5 failure on the base branch before calling it base-red (docs/ship-policy.md#base-red); null never calls one base-red. */ base_red: BaseRedPolicy | null; + /** The release gate on the base branch (docs/ship-policy.md#release-gate); null keeps the per-push main backstop. */ + release_gate: ReleaseGatePolicy | null; } /** `base_red`: `setup` installs dependencies in a scratch checkout (null when none are needed); `select` runs the failing test files, named by its `{files}` slot. */ @@ -165,6 +168,7 @@ export const DEFAULT_POLICY: ShipPolicy = { train: null, docs: null, base_red: null, + release_gate: null, }; export type PolicyKey = Exclude; @@ -195,6 +199,7 @@ export const POLICY_EXPLANATIONS: Record = { train: '{ gate_workflow, free_workflow, full_gate_check, record_artifact, max_size, fill_hours, screen_selection, trial_reservation_usd, land? } (the gstack-evals gate) or { gates, land, max_size, fill_hours } (any repo: gates lists { kind: workflows, workflows, required_jobs, record?, post_merge_checks? } and { kind: command, command, timeout_sec, spend_cap_usd, pass_env? }, each GREEN on every prefix; land is { merge_method: squash|merge|rebase, required_checks? }) or null: turns on merge trains in gstack-ship-queue (train build|status|verdict|land|watch|init|doctor|prune, ready --screen|--no-screen, verify --train). gates and land need min_gstack_version at the release that reads them (validate names it). null keeps today\'s per-PR queue unchanged.', docs: '{ caps: [{ path, max_bytes }] } or null: documentation files with a byte cap. /ship Step 14.5 runs `gstack-ship-receipt docs-caps` before the verification gate and prints one DOCS_CAP: line per cap; a file over its cap (or absent) blocks the ship until the doc is trimmed or the cap raised on the base branch. null declares none.', base_red: '{ setup, select, timeout_sec } or null: how /ship Step 5 proves a failure is the base branch\'s own (docs/ship-policy.md#base-red). gstack-base-red checks out the base head and the PR head in scratch worktrees, runs setup (dependency install, or null), then select with the failing test files in its {files} slot; only the same failures in both, in files the PR does not touch, are base-red, and /ship then pauses before READY instead of failing. Read from the base branch only. null keeps every failure in-branch.', + release_gate: RELEASE_GATE_EXPLANATION, min_gstack_version: 'The oldest gstack that may read this file. Any key newer than 1.91.98.0 set to a non-default value requires it at that key’s introducing version (validate prints each); a gstack older than it refuses the file (POLICY_GSTACK_TOO_OLD).', }; @@ -231,6 +236,7 @@ export const BUMP_NEVER_KEY_VERSION = '1.91.353.0'; export const BASE_RED_KEY_VERSION = '1.91.374.0'; /** `train.gates` and `train.land` are values of the older train key, so compatErrors checks them by value, as for bump: never. */ export const TRAIN_GATES_KEY_VERSION = '1.92.0.0'; +export const RELEASE_GATE_KEY_VERSION = '1.94.0.0'; export const POLICY_KEY_INTRODUCED: Record = { bump: '1.91.73.0', queue_mode: '1.91.73.0', release_tool: '1.91.73.0', release_outputs: '1.91.73.0', stamp_paths: '1.91.73.0', changelog: '1.91.73.0', mirror: '1.91.73.0', history: '1.91.73.0', preregistration_shas: '1.91.73.0', stamp_sensitive_gates: '1.91.73.0', @@ -238,7 +244,7 @@ export const POLICY_KEY_INTRODUCED: Record = { migrations: '1.91.98.0', gate_command: SHIP_QUEUE_KEYS_VERSION, full_e2e: SHIP_QUEUE_KEYS_VERSION, min_gstack_version: SHIP_QUEUE_KEYS_VERSION, train: TRAIN_KEYS_VERSION, review_scope: REVIEW_SCOPE_KEY_VERSION, docs: DOCS_CAPS_KEY_VERSION, changelog_fragments: CHANGELOG_FRAGMENTS_KEY_VERSION, - base_red: BASE_RED_KEY_VERSION, + base_red: BASE_RED_KEY_VERSION, release_gate: RELEASE_GATE_KEY_VERSION, }; const newer = (a: string, b: string): boolean => cmpVersion(parseVersion(a)!, parseVersion(b)!) > 0; @@ -496,6 +502,7 @@ export function validatePolicy(raw: unknown, opts: { gstackVersion?: string } = if ('docs' in obj && obj.docs !== null) docsErrors(obj.docs, errors); if ('changelog_fragments' in obj && obj.changelog_fragments !== null) fragmentsErrors(obj.changelog_fragments, errors); if ('base_red' in obj && obj.base_red !== null) baseRedErrors(obj.base_red, errors); + if ('release_gate' in obj && obj.release_gate !== null) errors.push(...releaseGateErrors(obj.release_gate)); if ('stamp_paths' in obj) listErrors('stamp_paths', obj.stamp_paths, errors, globError); for (const key of ['changelog', 'mirror'] as const) { if (!(key in obj) || obj[key] === null) continue; diff --git a/lib/ship-queue-train-record.ts b/lib/ship-queue-train-record.ts index 5d8f7df97..0f38d14f1 100644 --- a/lib/ship-queue-train-record.ts +++ b/lib/ship-queue-train-record.ts @@ -117,7 +117,20 @@ export interface ReadyReceipt { schema: 1; kind: 'ready'; repo_key: string; pr: string; head: string; base_sha: string; ready_at: string; screen: ScreenChoice; updated_at: string; base_red?: BaseRedCondition; fixes_base_red?: string[]; } -export type AnyRecord = TrainRecord | AttemptRecord | BisectRecord | FreezeRecord | ReadyReceipt | WaveRecord; +/** One release-gate bisect round (lib/release-gate.ts): the prefixes it dispatched on release-gate/ and what each case did there. */ +export interface ReleaseBisectRound { nonce: string; ref: string; run: { id: number; url: string } | null; prefixes: Array<{ sha: string; results: Record }> } +/** A red release gate's bisect over the first-parent commits after the last green gate: first_bad once good and bad are adjacent. */ +export interface ReleaseBisect { + cases: string[]; good: string | null; bad: string; first_bad: string | null; spend_cap: number; spent_usd: number; + state: 'running' | 'found' | 'unknown' | 'capped'; detail: string; rounds: ReleaseBisectRound[]; +} +/** The release gate's verdict on one commit of the gated branch (lib/release-gate.ts), the evidence scripts/release-cut.ts reads. */ +export interface ReleaseRecord { + schema: 1; kind: 'release'; repo_key: string; sha: string; tree: string; branch: string; stable: string | null; since: string | null; + nonce: string; attempts: number; run: { id: number; url: string } | null; verdict: 'PENDING' | 'GREEN' | 'RED' | 'INCOMPLETE'; + red_cases: string[]; flaky: string[]; cost_usd: number | null; detail: string; bisect: ReleaseBisect | null; created_at: string; updated_at: string; +} +export type AnyRecord = TrainRecord | AttemptRecord | BisectRecord | FreezeRecord | ReadyReceipt | WaveRecord | ReleaseRecord; /** Clock seam for fill-time and record tests (`GSTACK_SHIP_QUEUE_NOW`, ISO-8601). */ export function nowIso(): string { @@ -141,6 +154,7 @@ const bisectPath = (nonce: string) => path.join(trainDir(nonce), 'bisect.json'); const freezePath = (repoKey: string) => path.join(queueRoot(), 'freeze', `${repoKey}.json`); export const readyPath = (repoKey: string, pr: string, head: string) => path.join(queueRoot(), 'ready', repoKey, `${pr}-${head}.json`); export const wavePath = (nonce: string) => path.join(queueRoot(), 'waves', `${nonce}.json`); +export const releasePath = (repoKey: string, sha: string) => path.join(queueRoot(), 'release', repoKey, `${sha}.json`); /** Hosts that serve GitHub repositories at // (git.capy.ai is Capy's GitHub proxy, set through url.insteadOf). */ const GITHUB_HOSTS = ['github.com', 'www.github.com', 'ssh.github.com', 'git.capy.ai']; @@ -226,6 +240,7 @@ export function recordPath(r: AnyRecord): string { if (r.kind === 'train-bisect') return bisectPath(r.train); if (r.kind === 'freeze') return freezePath(r.repo_key); if (r.kind === 'wave') return wavePath(r.nonce); + if (r.kind === 'release') return releasePath(r.repo_key, r.sha); return readyPath(r.repo_key, r.pr, r.head); } @@ -235,6 +250,7 @@ export function noteName(r: AnyRecord): string { if (r.kind === 'train-bisect') return `gstack-trains:bisect:${r.train}`; if (r.kind === 'freeze') return `gstack-trains:freeze:${r.repo_key}`; if (r.kind === 'wave') return `gstack-trains:wave:${r.nonce}`; + if (r.kind === 'release') return `gstack-trains:release:${r.repo_key}:${r.sha}`; return `gstack-trains:ready:${r.repo_key}:${r.pr}:${r.head}`; } @@ -299,6 +315,7 @@ function parseNote(text: string): AnyRecord | null { if ((r.kind === 'train-attempt' && ids.test(r.train) && ids.test(r.id)) || (r.kind === 'train-bisect' && ids.test(r.train))) return r; if (r.kind === 'freeze' && ids.test(r.repo_key)) return r; if (r.kind === 'wave' && ids.test(r.nonce) && ids.test(r.repo_key) && Array.isArray(r.constituents)) return r; + if (r.kind === 'release' && ids.test(r.repo_key) && /^[0-9a-f]{40,64}$/.test(r.sha)) return r; if (r.kind === 'ready' && ids.test(r.repo_key) && /^\d+$/.test(r.pr) && /^[0-9a-f]{40,64}$/.test(r.head)) return r; } catch {} return null; diff --git a/scripts/release-cut.ts b/scripts/release-cut.ts index 16efe3e21..9dbe1caa0 100755 --- a/scripts/release-cut.ts +++ b/scripts/release-cut.ts @@ -19,7 +19,7 @@ * the window (7 days, 200 first-parent commits) is the candidate; a commit * with a check run still going is skipped, not waited for. * - * Gated means one of two kinds of evidence. Either the backstop check run + * Gated means one of three kinds of evidence. Either the backstop check run * (--backstop-check, default main-backstop) succeeded: evals-main-backstop.yml * publishes it, success only when a green train record covers the tree or the * backstop gated it green, neutral when main moved past the push before it was @@ -27,8 +27,11 @@ * or merged) lists the commit's sha or its tree in merged[]: those entries are * the main commits `gstack-ship-queue verify --train` confirmed, so a neutral * or absent backstop run on such a commit is not held against it, and the - * CANDIDATE line says "gated by train ". A red backstop run stays red - * either way. A commit with neither is not green. + * CANDIDATE line says "gated by train ". Or a GREEN release record + * (kind release, lib/release-gate.ts) names the commit: "gated by release gate + * run ". A red backstop run stays red either way. A commit with none of + * these is not green. A RED release record with no GREEN one on a later + * commit that contains it holds every cut, like a freeze. * * stable moves only forward: a candidate behind the current stable is refused * (REFUSED non-fast-forward), as is a tag v that already names another @@ -99,8 +102,8 @@ export function compareVersions(a: string, b: string): number { export type Verdict = { green: true; via: string } | { green: false; why: string; pending: boolean }; interface CheckRun { name: string; status: string; conclusion: string | null } -/** Main commits a green, merging or merged train record lists in merged[], by sha and by tree, each naming the train. */ -export interface TrainEvidence { bySha: Map; byTree: Map } +/** Main commits a green, merging or merged train record lists in merged[], by sha and by tree, each naming the train; `release` maps a commit a green release gate passed to its run (lib/release-gate.ts). */ +export interface TrainEvidence { bySha: Map; byTree: Map; release?: Map } export interface GateOptions { requiredCheck?: string; backstopCheck?: string; evidence?: TrainEvidence; treeOf?: (sha: string) => string | null } /** Why a commit is not releasable, or green and through which evidence. */ @@ -118,6 +121,8 @@ export function gateVerdict(gh: Gh, repo: string, sha: string, o: GateOptions = if (named.some(c => c.conclusion === 'success')) continue; const train = name === backstop && name !== required && o.evidence ? o.evidence.bySha.get(sha) ?? o.evidence.byTree.get(o.treeOf?.(sha) ?? '') : undefined; if (train) { via = `train ${train}`; continue; } + const gate = name === backstop && name !== required ? o.evidence?.release?.get(sha) : undefined; + if (gate) { via = `release gate run ${gate}`; continue; } if (named.length === 0) return { green: false, why: `no ${name} check run`, pending: false }; return { green: false, why: `${name} concluded ${named.map(c => c.conclusion).join(',')}`, pending: false }; } @@ -129,14 +134,20 @@ export function gateVerdict(gh: Gh, repo: string, sha: string, o: GateOptions = } export interface TrainNotes { error: string | null; freeze: string | null; evidence: TrainEvidence } -interface NoteRecord { kind?: string; state?: string; nonce?: string; merged?: Array<{ sha?: string; tree?: string }>; cleared_at?: string | null; reason?: string; train?: string } +interface NoteRecord { + kind?: string; state?: string; nonce?: string; merged?: Array<{ sha?: string; tree?: string }>; cleared_at?: string | null; reason?: string; train?: string; + sha?: string; verdict?: string; run?: { id?: number } | null; red_cases?: string[]; +} /** * One read of the trains notes ref (fetched from origin): the active freeze * record (kind freeze, cleared_at null) and the train evidence (lib/ * ship-queue-train-record.ts TrainRecord: kind train, state green, merging or * merged, merged[].sha and .tree). A fetch failure other than a missing ref is - * an error; a missing ref holds nothing. + * an error; a missing ref holds nothing. Release-gate records (kind release, + * lib/release-gate.ts) add evidence when GREEN, and a RED one with no GREEN + * record on a later commit that contains it is the hold: it refuses the cut + * like a freeze (a train freeze, when both exist, is the one named). */ export function readTrainNotes(git: Git, notesRef = NOTES_REF): TrainNotes { const notes: TrainNotes = { error: null, freeze: null, evidence: { bySha: new Map(), byTree: new Map() } }; @@ -147,6 +158,8 @@ export function readTrainNotes(git: Git, notesRef = NOTES_REF): TrainNotes { } const listed = git(['notes', `--ref=${NOTES_LOCAL}`, 'list']); if (!listed.ok || !listed.out.trim()) return notes; + const gates: Array<{ sha: string; verdict: string; red: string[] }> = []; + notes.evidence.release = new Map(); for (const line of listed.out.split('\n')) { const blob = line.split(' ')[0]; if (!blob) continue; @@ -155,6 +168,10 @@ export function readTrainNotes(git: Git, notesRef = NOTES_REF): TrainNotes { if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) continue; const r = parsed as NoteRecord; if (r.kind === 'freeze' && !r.cleared_at) notes.freeze = `queue frozen since train ${r.train ?? '?'}: ${r.reason ?? 'red backstop'}`; + if (r.kind === 'release' && typeof r.sha === 'string' && /^[0-9a-f]{40}$/.test(r.sha)) { + gates.push({ sha: r.sha, verdict: String(r.verdict), red: Array.isArray(r.red_cases) ? r.red_cases.filter(c => typeof c === 'string') : [] }); + if (r.verdict === 'GREEN') notes.evidence.release.set(r.sha, String(r.run?.id ?? '?')); + } if (r.kind !== 'train' || typeof r.nonce !== 'string' || !r.nonce || !['green', 'merging', 'merged'].includes(r.state ?? '')) continue; for (const m of Array.isArray(r.merged) ? r.merged : []) { if (!m || typeof m !== 'object') continue; @@ -162,6 +179,8 @@ export function readTrainNotes(git: Git, notesRef = NOTES_REF): TrainNotes { if (typeof m.tree === 'string' && m.tree) notes.evidence.byTree.set(m.tree, r.nonce); } } + const held = gates.find(red => red.verdict === 'RED' && !gates.some(g => g.verdict === 'GREEN' && g.sha !== red.sha && git(['merge-base', '--is-ancestor', red.sha, g.sha]).ok)); + if (held && !notes.freeze) notes.freeze = `release gate red on ${held.sha.slice(0, 12)} (${held.red.join(', ') || 'red cases not recorded'}) and no gate on a later commit is green; land the revert or fix-forward, then run gstack-release-gate run --ignore-cadence`; return notes; } diff --git a/scripts/train-gate.ts b/scripts/train-gate.ts index 1613dfe2b..5baa4818d 100644 --- a/scripts/train-gate.ts +++ b/scripts/train-gate.ts @@ -61,8 +61,14 @@ export type RunMode = 'screen' | 'train' | 'bisect' | 'dispatch'; export interface RouteInput { nonce: string; refName: string; head: string; expectedSha: string; evalsAll: boolean; trainPolicy: boolean; cases: string; prefixShas: string; screenPr: string; screenMergeSha: string; screenPolicy: string; + /** The default branch's policy declares a release_gate block (lib/release-gate-policy.ts). */ + releasePolicy?: boolean; } -export interface Route { mode: RunMode; cases: string[]; prefixes: string[] } +/** artifact: the prefix of a release bisect's per-prefix records (a train bisect uses the train block's record_artifact). */ +export interface Route { mode: RunMode; cases: string[]; prefixes: string[]; artifact?: string } +/** The release gate's bisect runs on its own ref and names its own prefix records (lib/release-gate.ts RELEASE_REF_PREFIX, RELEASE_BISECT_ARTIFACT). */ +export const RELEASE_BISECT_REF = 'release-gate/'; +export const RELEASE_BISECT_ARTIFACT = 'gstack-release-record'; const list = (text: string) => text.split(',').map(item => item.trim()).filter(Boolean); @@ -88,7 +94,8 @@ export function routeDispatch(i: RouteInput, deps: { isAncestor(commit: string, const cases = list(i.cases); const prefixes = list(i.prefixShas); if (scope?.purpose !== 'bisect') return { error: `a bisect needs a bisect nonce (got ${JSON.stringify(i.nonce)})` }; - if (!i.refName.startsWith('train/')) return { error: `a bisect runs on train/, not ${i.refName}` }; + const release = i.refName.startsWith(RELEASE_BISECT_REF); + if (!release && !i.refName.startsWith('train/')) return { error: `a bisect runs on train/ (or the release gate's ${RELEASE_BISECT_REF}), not ${i.refName}` }; if (i.expectedSha !== i.head) return { error: `expected_sha ${i.expectedSha.slice(0, 12) || '(empty)'} is not the train head ${i.head.slice(0, 12)} checked out from ${i.refName}` }; if (!cases.length || cases.length > MAX_BISECT_CASES || new Set(cases).size !== cases.length) return { error: `cases must name 1 to ${MAX_BISECT_CASES} distinct case ids` }; if (!prefixes.length || prefixes.length > MAX_PREFIXES || new Set(prefixes).size !== prefixes.length) return { error: `prefix_shas must name 1 to ${MAX_PREFIXES} distinct commits` }; @@ -99,6 +106,7 @@ export function routeDispatch(i: RouteInput, deps: { isAncestor(commit: string, const plan = deps.gateCases(); const unknown = cases.filter(id => !CASE_ID.test(id) || !plan.has(id)); if (unknown.length) return { error: `case(s) not in the full gate's plan: ${unknown.slice(0, 5).join(', ')}` }; + if (release) return i.releasePolicy ? { mode: 'bisect', cases, prefixes, artifact: RELEASE_BISECT_ARTIFACT } : { error: "a release bisect needs a release_gate block in the default branch's .gstack/ship-policy.json" }; return trainOnly('bisect') ?? { mode: 'bisect', cases, prefixes }; } if (scope?.purpose === 'train') { @@ -359,12 +367,12 @@ if (import.meta.main) { if (command === 'route') { const env = process.env; const route = routeDispatch({ nonce: env.NONCE ?? '', refName: env.GITHUB_REF_NAME ?? '', head: git(['rev-parse', 'HEAD']).out, expectedSha: env.EXPECTED_SHA ?? '', - evalsAll: env.EVALS_ALL === 'true', trainPolicy: env.TRAIN_POLICY === 'true', cases: env.CASES ?? '', prefixShas: env.PREFIX_SHAS ?? '', + evalsAll: env.EVALS_ALL === 'true', trainPolicy: env.TRAIN_POLICY === 'true', releasePolicy: env.RELEASE_POLICY === 'true', cases: env.CASES ?? '', prefixShas: env.PREFIX_SHAS ?? '', screenPr: env.SCREEN_PR ?? '', screenMergeSha: env.SCREEN_MERGE_SHA ?? '', screenPolicy: env.SCREEN_POLICY ?? '' }, { isAncestor: (commit, head) => git(['merge-base', '--is-ancestor', commit, head]).ok, gateCases: () => fullGateCases(process.cwd()) }); if ('error' in route) fail(`dispatch refused before any paid work: ${route.error}`); const ok = route as Route; - output([`mode=${ok.mode}`, `cases=${ok.cases.join(',')}`, `prefixes=${ok.prefixes.join(',')}`]); + output([`mode=${ok.mode}`, `cases=${ok.cases.join(',')}`, `prefixes=${ok.prefixes.join(',')}`, ...(ok.artifact ? [`record_artifact=${ok.artifact}`] : [])]); console.log(`- Run mode: ${ok.mode}${ok.prefixes.length ? ` (${ok.cases.length} case(s) on ${ok.prefixes.length} prefix(es))` : ''}`); } else if (command === 'bisect-plan') { const [cases, prefixes, nonce, out] = [list(flag('cases') ?? ''), list(flag('prefixes') ?? ''), flag('nonce'), flag('out')]; diff --git a/test/capy-skill-wrapper.test.ts b/test/capy-skill-wrapper.test.ts index 4fbc82da3..fe3f066b4 100644 --- a/test/capy-skill-wrapper.test.ts +++ b/test/capy-skill-wrapper.test.ts @@ -53,14 +53,18 @@ const UPGRADE_DEPS = [ 'bin/gstack-upgrade', 'bin/gstack-bun-version.sh', 'bin/gstack-paths', 'bin/gstack-state-root.sh', 'bin/gstack-egress-lib.sh', 'bin/gstack-egress-receipt', 'lib/egress-receipt.ts', 'lib/state-root.ts', 'bin/gstack-lease', 'bin/gstack-harness-pid.sh', + // Sourced by gstack-upgrade: without gstack-config in the fixture it resolves main unless GSTACK_UPDATE_CHANNEL says stable. + 'bin/gstack-update-channel.sh', ]; /** * cloneAt 'A' (default): the clone sits at A (1.0.0), below B's minimum 1.0.1. * cloneAt 'B': the clone sits at B, AT the minimum, and the remote moves on to * C (1.0.2, minimum unchanged): behind origin but not below the floor. + * next: the remote moves on to C for a clone at A too. stable: origin's stable + * branch (the release, docs/releases.md) at that commit. */ -function fixture(opts: { minWrapper?: number; cloneAt?: 'A' | 'B' } = {}) { +function fixture(opts: { minWrapper?: number; cloneAt?: 'A' | 'B'; next?: boolean; stable?: 'A' | 'B' | 'C' } = {}) { const host = capyHost({ systemBun: '1.4.2' }); hosts.push(host); const git = (args: string[], cwd: string) => { @@ -94,13 +98,14 @@ function fixture(opts: { minWrapper?: number; cloneAt?: 'A' | 'B' } = {}) { const b = git(['rev-parse', 'HEAD'], seed); git(['push', '-q', bare, 'main'], seed); let c = ''; - if (opts.cloneAt === 'B') { - git(['clone', '-q', '--single-branch', '--depth', '1', `file://${bare}`, root], host.base); + if (opts.cloneAt === 'B' || opts.next) { + if (opts.cloneAt === 'B') git(['clone', '-q', '--single-branch', '--depth', '1', `file://${bare}`, root], host.base); write(path.join(seed, 'VERSION'), '1.0.2\n'); git(['commit', '-qam', 'C'], seed); c = git(['rev-parse', 'HEAD'], seed); git(['push', '-q', bare, 'main'], seed); } + if (opts.stable) git(['push', '-q', bare, `${{ A: a, B: b, C: c }[opts.stable]}:refs/heads/stable`], seed); const env = { ...host.env, GSTACK_CAPY_ROOT: root, GSTACK_CAPY_REPO: `file://${bare}`, WRAP_LOG: log, WRAP_FAIL_ONCE: path.join(host.base, 'fail-once') }; const run = (args: string[] = [], extra: Record = {}) => { const r = spawnSync('bash', [BOOT, ...args], { cwd: host.home, encoding: 'utf8', env: { ...env, ...extra }, timeout: 120_000 }); @@ -140,13 +145,13 @@ describe('capy skill bootstrapper (capy/skill/gstack/scripts/install.sh)', () => expect(f.git(['rev-parse', 'HEAD'])).toBe(f.b); expect(f.git(['rev-parse', '--is-shallow-repository'])).toBe('true'); expect(install.last).toBe('gstack: ok 1.0.1 for=autoplan'); - expect(f.calls()).toEqual([`installer --root ${f.root} --for autoplan lock=1 rev=3 min=1.0.1 from=1.0.0`]); + expect(f.calls()).toEqual([`installer --root ${f.root} --for autoplan lock=1 rev=4 min=1.0.1 from=1.0.0`]); const next = f.run(['--check', '--for', 'autoplan']); expect(next.status, next.stdout + next.stderr).toBe(0); expect(next.stdout).not.toContain('upgraded'); expect(next.last).toBe('gstack: ok 1.0.1 for=autoplan'); - expect(f.calls().at(-1)).toBe(`installer --check --root ${f.root} --for autoplan lock= rev=3 min=1.0.1 from=`); + expect(f.calls().at(-1)).toBe(`installer --check --root ${f.root} --for autoplan lock= rev=4 min=1.0.1 from=`); expect(f.git(['rev-parse', 'HEAD'])).toBe(f.b); }); @@ -263,7 +268,7 @@ describe('capy skill: always current when online (the minimum is the floor, not expect(r.stdout).toMatch(/^SELF_UPGRADE: done 1\.0\.1 -> 1\.0\.2$/m); expect(f.git(['rev-parse', 'HEAD'])).toBe(f.c); expect(r.last).toBe('gstack: ok 1.0.2 for=autoplan'); - expect(f.calls()).toEqual([`installer --root ${f.root} --for autoplan lock=1 rev=3 min=1.0.1 from=1.0.1`]); + expect(f.calls()).toEqual([`installer --root ${f.root} --for autoplan lock=1 rev=4 min=1.0.1 from=1.0.1`]); expect(fs.existsSync(path.join(f.root, '.git/gstack-upgrade.pending'))).toBe(false); const next = f.run(['--for', 'autoplan']); @@ -271,7 +276,7 @@ describe('capy skill: always current when online (the minimum is the floor, not expect(next.stdout).toMatch(/^SELF_UPGRADE: none \(current\)$/m); expect(next.stdout).not.toContain('SELF_UPGRADE: done'); expect(next.last).toBe('gstack: ok 1.0.2 for=autoplan'); - expect(f.calls().at(-1)).toBe(`installer --root ${f.root} --for autoplan lock=1 rev=3 min=1.0.1 from=`); + expect(f.calls().at(-1)).toBe(`installer --root ${f.root} --for autoplan lock=1 rev=4 min=1.0.1 from=`); }); test('--check mode never moves a behind checkout: it names the gap and the fast-forwarding command, then certifies the installed revision', () => { @@ -282,7 +287,7 @@ describe('capy skill: always current when online (the minimum is the floor, not expect(r.stdout).not.toContain('SELF_UPGRADE'); expect(f.git(['rev-parse', 'HEAD'])).toBe(f.b); expect(r.last).toBe('gstack: ok 1.0.1 for=autoplan'); - expect(f.calls()).toEqual([`installer --check --root ${f.root} --for autoplan lock= rev=3 min=1.0.1 from=`]); + expect(f.calls()).toEqual([`installer --check --root ${f.root} --for autoplan lock= rev=4 min=1.0.1 from=`]); }); test('a dirty behind checkout is skipped with gstack-upgrade\'s reason and still ends ok: above the floor, the trailer is the doctor\'s', () => { @@ -295,7 +300,7 @@ describe('capy skill: always current when online (the minimum is the floor, not expect(f.git(['rev-parse', 'HEAD'])).toBe(f.b); expect(fs.readFileSync(path.join(f.root, 'README.md'), 'utf8')).toBe('local edit\n'); expect(r.last).toBe('gstack: ok 1.0.1 for=autoplan'); - expect(f.calls()).toEqual([`installer --root ${f.root} --for autoplan lock=1 rev=3 min=1.0.1 from=`]); + expect(f.calls()).toEqual([`installer --root ${f.root} --for autoplan lock=1 rev=4 min=1.0.1 from=`]); }); test('GSTACK_CAPY_SELF_UPGRADE=off and GSTACK_CAPY_AUTO_UPGRADE=0 each disable the fast-forward by name and leave HEAD where it is', () => { @@ -315,6 +320,53 @@ describe('capy skill: always current when online (the minimum is the floor, not // Plan A9: the `latest:` trailer note comes only from origin's VERSION the // bootstrapper read in this run; a FETCH_HEAD left by an earlier run never // reaches the installer. +describe('capy skill bootstrapper: the update channel (wrapper 4, docs/releases.md#channels)', () => { + // Value: protects=an install that follows stable (GSTACK_UPDATE_CHANNEL=stable, or team mode through bin/gstack-update-channel.sh) is cloned at + // origin's stable branch, not main, and stays there through the checkout half's self-upgrade; GSTACK_UPDATE_CHANNEL=main and a remote with no stable + // branch (before the first release) clone main; fails_when=a fresh team install lands on unreleased main, the override cannot pick main, or a missing + // stable branch fails the install; why_new=the clone test covers the default branch only; seam=GSTACK_UPDATE_CHANNEL instead of gstack-config's + // team_mode; no_claim=GitHub's real stable branch + test('a fresh clone follows stable when origin has it; GSTACK_UPDATE_CHANNEL=main and a missing stable branch clone main', () => { + const f = fixture({ cloneAt: 'B', stable: 'B' }); + fs.rmSync(f.root, { recursive: true, force: true }); + const stable = f.run(['--for', 'autoplan'], { GSTACK_UPDATE_CHANNEL: 'stable' }); + expect(stable.status, stable.stdout + stable.stderr).toBe(0); + expect(stable.stdout).toContain('gstack-capy-skill: channel stable 1.0.1 (GSTACK_UPDATE_CHANNEL=main follows every merge)'); + expect(f.git(['rev-parse', 'HEAD'])).toBe(f.b); + expect(stable.last).toBe('gstack: ok 1.0.1 for=autoplan'); + + fs.rmSync(f.root, { recursive: true, force: true }); + const main = f.run(['--for', 'autoplan'], { GSTACK_UPDATE_CHANNEL: 'main' }); + expect(main.status, main.stdout + main.stderr).toBe(0); + expect(main.stdout).not.toContain('channel stable'); + expect(f.git(['rev-parse', 'HEAD'])).toBe(f.c); + + spawnSync('git', ['--git-dir', f.bare, 'update-ref', '-d', 'refs/heads/stable'], { env: f.host.env, timeout: 30_000 }); + fs.rmSync(f.root, { recursive: true, force: true }); + const bootstrap = f.run(['--for', 'autoplan'], { GSTACK_UPDATE_CHANNEL: 'stable' }); + expect(bootstrap.status, bootstrap.stdout + bootstrap.stderr).toBe(0); + expect(bootstrap.stdout).toContain('channel stable, but origin has no stable branch yet (or the fetch failed); following main'); + expect(f.git(['rev-parse', 'HEAD'])).toBe(f.c); + }); + + // Value: protects=the bootstrapper's floor fast-forward of a checkout below the supported minimum goes to stable on the stable channel when stable + // meets the minimum, and to main when stable is itself below it; fails_when=a stable install is pushed past the release to main, or left below the + // floor because stable is too old; why_new=the floor tests cover main only; seam=GSTACK_UPDATE_CHANNEL; no_claim=GitHub's real stable branch + test('below the floor: the fast-forward goes to stable when stable meets the minimum, else to main', () => { + const atRelease = fixture({ next: true, stable: 'B' }); + const r = atRelease.run(['--for', 'autoplan'], { GSTACK_UPDATE_CHANNEL: 'stable' }); + expect(r.status, r.stdout + r.stderr).toBe(0); + expect(r.stdout).toContain('gstack-capy-skill: upgraded 1.0.0 -> 1.0.1'); + expect(atRelease.git(['rev-parse', 'HEAD'])).toBe(atRelease.b); + + const old = fixture({ next: true, stable: 'A' }); + const o = old.run(['--for', 'autoplan'], { GSTACK_UPDATE_CHANNEL: 'stable' }); + expect(o.status, o.stdout + o.stderr).toBe(0); + expect(o.stdout).toContain('gstack-capy-skill: channel stable is at 1.0.0, below the supported minimum 1.0.1; following main'); + expect(old.git(['rev-parse', 'HEAD'])).toBe(old.c); + }); +}); + describe('capy skill bootstrapper: GSTACK_CAPY_REMOTE_VERSION', () => { test('a fresh fetch exports origin\'s VERSION; a failed fetch exports nothing even with a stale FETCH_HEAD', () => { const f = fixture({ cloneAt: 'B' }); diff --git a/test/release-cut.test.ts b/test/release-cut.test.ts index 6986b5630..0de0eba80 100644 --- a/test/release-cut.test.ts +++ b/test/release-cut.test.ts @@ -249,6 +249,50 @@ describe('release-cut: planning', () => { }); }); +describe('release-cut: the release gate holds and gates (lib/release-gate.ts records)', () => { + const release = (sha: string, verdict: string, run: number) => ({ schema: 1, kind: 'release', repo_key: 'k', sha, verdict, run: { id: run, url: 'u' }, red_cases: verdict === 'RED' ? ['case-a'] : [], created_at: 'x', updated_at: 'x' }); + const note = (f: ReturnType, record: unknown, on: string) => { + git(f.seed, 'notes', '--ref=' + NOTES_REF, 'add', '-f', '-m', JSON.stringify(record), on); + git(f.seed, 'push', '-q', '-f', 'origin', NOTES_REF); + }; + + // Value: protects=a red release record is the freeze: every cut is refused while no green gate sits on a later commit that contains the red one, + // even when every check run is green, and an incomplete gate or a green gate on an older commit does not lift it; fails_when=release-cut releases a + // commit while the newest release gate is red, a green record anywhere clears the hold, or an INCOMPLETE gate clears it; why_new=the freeze tests know + // only the train freeze record; seam=none; no_claim=real refs/notes/gstack-trains content (fixture records shaped like ReleaseRecord) + test('a red release record refuses the cut; only a green gate on a later commit lifts it', () => { + const f = fixture(); + note(f, release(f.shas['1.0.2']!, 'RED', 41), f.shas['1.0.2']!); + note(f, release(f.shas['1.0.1']!, 'GREEN', 40), f.shas['1.0.1']!); + const held = planRelease({ repo: f.repo, git: f.git, gh: f.gh }); + expect(held.ok).toBe(false); + if (!held.ok) { expect(held.code).toBe('REFUSED'); expect(held.message).toContain('release gate red on ' + f.shas['1.0.2']!.slice(0, 12)); } + note(f, release(f.shas['1.0.3']!, 'INCOMPLETE', 42), f.shas['1.0.3']!); + expect(planRelease({ repo: f.repo, git: f.git, gh: f.gh }).ok).toBe(false); + note(f, release(f.shas['1.0.3']!, 'GREEN', 43), f.shas['1.0.3']!); + const lifted = planRelease({ repo: f.repo, git: f.git, gh: f.gh }); + expect(lifted.ok, JSON.stringify(lifted)).toBe(true); + if (lifted.ok) expect(lifted.plan.sha).toBe(f.shas['1.0.3']); + }); + + // Value: protects=a green release record gates its commit when the main-backstop check run is absent or neutral (the check publish failed or was + // superseded), and the CANDIDATE line names the gate run; a red check run still keeps the commit out; fails_when=a commit the release gate passed is + // never released without the check, or release evidence overrides a red check run; why_new=train records were the only notes evidence; seam=none; + // no_claim=a live gate run + test('a green release record gates a commit whose main-backstop check never ran; a red check run still wins', () => { + const f = fixture(); + const head = f.shas['1.0.3']!; + f.state.checks[head] = GREEN.filter(c => c.name !== BACKSTOP_CHECK); + note(f, release(head, 'GREEN', 77), head); + const gated = planRelease({ repo: f.repo, git: f.git, gh: f.gh }); + expect(gated.ok, JSON.stringify(gated)).toBe(true); + if (gated.ok) { expect(gated.plan.sha).toBe(head); expect(gated.plan.via).toBe('release gate run 77'); } + f.state.checks[head] = [...GREEN.filter(c => c.name !== BACKSTOP_CHECK), { name: 'quality', status: 'completed', conclusion: 'failure' }]; + const red = planRelease({ repo: f.repo, git: f.git, gh: f.gh }); + if (red.ok) expect(red.plan.sha).not.toBe(head); + }); +}); + describe('release-cut: train evidence gates a commit whose backstop never settled', () => { const train = (nonce: string, state: string, merged: Array<{ sha: string; tree: string }>) => ({ schema: 1, kind: 'train', nonce, state, merged: merged.map(m => ({ pr: '1', at: 'x', ...m })), updated_at: 'x' }); diff --git a/test/release-gate-workflow.test.ts b/test/release-gate-workflow.test.ts new file mode 100644 index 000000000..a2f8a5542 --- /dev/null +++ b/test/release-gate-workflow.test.ts @@ -0,0 +1,129 @@ +/** + * The release gate's workflow wiring (docs/ship-policy.md#release-gate): + * release-gate.yml runs bin/gstack-release-gate on its schedule and publishes + * the gated commit's check run from a code-free job; the per-push backstop + * stands down once main's policy declares a release_gate block; evals.yml routes + * a release bisect only under that block. Policy steps run for real (bash, git, + * jq) against a scratch repository; the rest is read from the YAML. + */ +import { afterAll, describe, expect, test } from 'bun:test'; +import { execFileSync, spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { RELEASE_GATE_SCHEDULE_HOURS } from '../lib/release-gate-policy'; +import { RELEASE_BISECT_ARTIFACT } from '../scripts/train-gate'; + +const ROOT = path.resolve(import.meta.dir, '..'); +type Step = { id?: string; name?: string; uses?: string; if?: string; run?: string; env?: Record }; +type Job = { if?: string; needs?: string; outputs?: Record; permissions?: Record; steps: Step[] }; +type Workflow = { on: Record; concurrency: unknown; permissions?: unknown; jobs: Record }; +const load = (rel: string) => Bun.YAML.parse(fs.readFileSync(path.join(ROOT, rel), 'utf8')) as Workflow; +const release = load('.github/workflows/release-gate.yml'); +const backstop = load('.github/workflows/evals-main-backstop.yml'); +const evals = load('.github/workflows/evals.yml'); +const scratch: string[] = []; +afterAll(() => { for (const d of scratch) fs.rmSync(d, { recursive: true, force: true }); }); +const GIT_ENV = { GIT_AUTHOR_NAME: 't', GIT_AUTHOR_EMAIL: 't@x', GIT_COMMITTER_NAME: 't', GIT_COMMITTER_EMAIL: 't@x' }; + +/** Run a policy step's script in a scratch repo whose one commit holds this policy; returns its GITHUB_OUTPUT lines as an object. */ +function runStep(step: Step, policy: unknown): Record { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'release-gate-wf-')); + scratch.push(dir); + const g = (...args: string[]) => execFileSync('git', args, { cwd: dir, encoding: 'utf8', timeout: 30_000, env: { ...process.env, ...GIT_ENV } }).trim(); + g('init', '-q', '-b', 'main'); + if (policy !== undefined) { + fs.mkdirSync(path.join(dir, '.gstack')); + fs.writeFileSync(path.join(dir, '.gstack/ship-policy.json'), JSON.stringify(policy)); + } + fs.writeFileSync(path.join(dir, 'README'), 'x'); + g('add', '-A'); + g('commit', '-q', '-m', 'base'); + const sha = g('rev-parse', 'HEAD'); + g('update-ref', 'refs/remotes/origin/main', sha); + const out = path.join(dir, 'out'); + const r = spawnSync('bash', ['-e', '-c', step.run!], { + cwd: dir, encoding: 'utf8', timeout: 30_000, + env: { ...process.env, GITHUB_OUTPUT: out, GITHUB_STEP_SUMMARY: path.join(dir, 'summary'), GITHUB_SHA: sha, BASE_SHA: '', DEFAULT_BRANCH: 'main' }, + }); + expect(r.status, r.stderr).toBe(0); + const lines = (fs.existsSync(out) ? fs.readFileSync(out, 'utf8') : '').split('\n').filter(Boolean); + return Object.fromEntries(lines.map(l => [l.slice(0, l.indexOf('=')), l.slice(l.indexOf('=') + 1)])); +} +const GATE = { cadence_hours: 8, gate_workflow: 'evals.yml', branch: 'main', stable_ref: 'stable', check_name: 'main-backstop', on_red: 'hold-release', bisect_spend_cap_usd: 40 }; +const TRAIN = { gate_workflow: 'evals.yml', record_artifact: 'gstack-train-record', full_gate_check: 'gstack/full-gate' }; + +describe('release-gate.yml', () => { + // Value: protects=the schedule fires every RELEASE_GATE_SCHEDULE_HOURS (the cadence floor the policy validates), one gate at a time, only from the + // default branch, with contents and actions write but no checks write, and runs the bin on the run's own sha with its outputs wired; the policy step + // uses git and jq only and is inert without a release_gate block; fails_when=the cron and the cadence drift apart, two gates overlap, the job that + // runs repository code can write check runs, or an inert policy reaches bun; why_new=new workflow; seam=none; no_claim=GitHub's cron jitter + test('schedule, concurrency, permissions and the gate step', () => { + const cron = release.on.schedule[0].cron as string; + expect(cron).toMatch(/^\d+ \*\/\d+ \* \* \*$/); + expect(Number(cron.split(' ')[1]!.slice(2))).toBe(RELEASE_GATE_SCHEDULE_HOURS); + expect(Object.keys(release.on).sort()).toEqual(['schedule', 'workflow_dispatch']); + expect(Object.keys(release.on.workflow_dispatch.inputs).sort()).toEqual(['dry_run', 'ignore_cadence']); + expect(release.concurrency).toEqual({ group: 'release-gate', 'cancel-in-progress': false }); + expect(release.permissions).toEqual({}); + const job = release.jobs['release-gate']!; + expect(job.if).toBe('github.ref_name == github.event.repository.default_branch'); + expect(job.permissions).toEqual({ contents: 'write', actions: 'write', checks: 'read' }); + const policy = job.steps.find(s => s.id === 'policy')!; + expect(policy.run).not.toContain('bun'); + expect(runStep(policy, undefined)).toEqual({ release: 'false' }); + expect(runStep(policy, { train: TRAIN })).toEqual({ release: 'false' }); + expect(runStep(policy, { release_gate: GATE })).toEqual({ release: 'true' }); + const gate = job.steps.find(s => s.id === 'gate')!; + expect(gate.if).toBe("steps.policy.outputs.release == 'true'"); + expect(gate.run).toContain('./bin/gstack-release-gate run --repo "$GITHUB_REPOSITORY" --sha "$GITHUB_SHA"'); + expect(gate.run).toContain('--outputs "$GITHUB_OUTPUT"'); + expect(job.outputs).toEqual({ action: '${{ steps.gate.outputs.action }}', sha: '${{ steps.gate.outputs.sha }}', check: '${{ steps.gate.outputs.check }}' }); + }); + + // Value: protects=the gated commit's check run (what release-cut reads) comes from a job with checks: write and no checkout or action, only for a + // gate that ran (success for gated, failure for held or failed), on a validated sha and check name; fails_when=repository code runs with checks: + // write, a skipped or inert run publishes a check, or a red gate publishes success; why_new=new workflow; seam=none; no_claim=GitHub's check API + test('the check run is published from a code-free job, success only for gated', () => { + const job = release.jobs['release-check']!; + expect(job.needs).toBe('release-gate'); + expect(job.if).toContain("contains(fromJSON('[\"gated\",\"held\",\"failed\"]'), needs.release-gate.outputs.action)"); + expect(job.permissions).toEqual({ checks: 'write' }); + expect(job.steps.filter(s => s.uses)).toEqual([]); + const run = job.steps[0]!.run!; + expect(run).toContain('gated) conclusion=success ;;'); + expect(run).toContain('held|failed) conclusion=failure ;;'); + expect(run).toContain('[[ "$SHA" =~ ^[0-9a-f]{40}$ ]]'); + expect(run).toContain('-f name="$CHECK" -f head_sha="$SHA"'); + expect(run).not.toContain('bun'); + }); +}); + +describe('the backstop stands down and evals.yml routes the release bisect', () => { + // Value: protects=while trains exist the per-push backstop keeps gating, and once main declares release_gate it stands down (train=false: no gate, + // no freeze, no check run), so main is never gated twice; fails_when=both gates run on main, or the backstop stops before the release gate is on; + // why_new=new stand-down rule; seam=none; no_claim=a live push + test('evals-main-backstop.yml: train=true with only a train block, train=false once release_gate is declared', () => { + const policy = backstop.jobs.backstop!.steps.find(s => s.id === 'policy')!; + expect(runStep(policy, { train: TRAIN })).toEqual({ train: 'true' }); + expect(runStep(policy, { train: TRAIN, release_gate: GATE })).toEqual({ train: 'false' }); + }); + + // Value: protects=the route job reads release_gate from the base revision with git and jq only and hands it to the route as RELEASE_POLICY, and the + // bisect jobs name their artifact from the route's record_artifact first (gstack-release-record for a release bisect); fails_when=a release bisect + // publishes under the train artifact name the gate never reads, or repository code decides the release policy; why_new=new route input; + // seam=none; no_claim=a live bisect run + test('evals.yml: the release policy step, the RELEASE_POLICY input and the record_artifact override', () => { + const route = evals.jobs.route!; + const step = route.steps.find(s => s.id === 'release_policy')!; + expect(step.run).not.toContain('bun'); + expect(runStep(step, { release_gate: GATE })).toEqual({ release: 'true' }); + expect(runStep(step, { train: TRAIN })).toEqual({ release: 'false' }); + expect(runStep(step, undefined)).toEqual({ release: 'false' }); + const ids = route.steps.map(s => s.id); + expect(ids.indexOf('release_policy')).toBeLessThan(ids.indexOf('route')); + expect(route.steps.find(s => s.id === 'route')!.env!.RELEASE_POLICY).toBe('${{ steps.release_policy.outputs.release }}'); + expect(route.outputs!.record_artifact).toBe('${{ steps.route.outputs.record_artifact || steps.policy.outputs.record_artifact }}'); + expect(RELEASE_BISECT_ARTIFACT).toBe('gstack-release-record'); + }); +}); diff --git a/test/release-gate.test.ts b/test/release-gate.test.ts new file mode 100644 index 000000000..29fe65cbc --- /dev/null +++ b/test/release-gate.test.ts @@ -0,0 +1,301 @@ +/** + * gstack-release-gate (lib/release-gate.ts, docs/ship-policy.md#release-gate): + * the full paid gate on main's head at most once per cadence and only when main + * moved, a release record in refs/notes/gstack-trains, a bisect of a red gate's + * cases over the commits since the last green gate, and the hold release-cut + * reads. GitHub is a fake gh (dispatch, run list, run view, artifact download); + * git, the notes ref and origin are real (a bare origin on disk). + */ +import { afterAll, afterEach, describe, expect, test } from 'bun:test'; +import { execFileSync, spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { bunIsolationArgs } from '../lib/bun-isolation'; +import { + MAX_RELEASE_ATTEMPTS, activeHold, bisectGate, classifyRun, gateStatus, loadReleaseGate, narrow, parseReport, planPrefixes, releaseRecords, runGate, + type GateEnv, type Run, +} from '../lib/release-gate'; +import { RELEASE_GATE_SCHEDULE_HOURS, releaseGateErrors, type ReleaseGatePolicy } from '../lib/release-gate-policy'; +import { RELEASE_GATE_KEY_VERSION, validatePolicy } from '../lib/ship-policy'; +import { NOTES_REF, repoIdentity, type ReleaseRecord } from '../lib/ship-queue-train-record'; +import { gitIn } from '../scripts/train-gate'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const GATE: ReleaseGatePolicy = { cadence_hours: 8, gate_workflow: 'evals.yml', branch: 'main', stable_ref: 'stable', check_name: 'main-backstop', on_red: 'hold-release', bisect_spend_cap_usd: 40 }; +const roots: string[] = []; +const savedStateRoot = process.env.GSTACK_STATE_ROOT; +afterEach(() => { if (savedStateRoot === undefined) delete process.env.GSTACK_STATE_ROOT; else process.env.GSTACK_STATE_ROOT = savedStateRoot; }); +afterAll(() => { for (const r of roots) fs.rmSync(r, { recursive: true, force: true }); }); + +const ID = { GIT_AUTHOR_NAME: 't', GIT_AUTHOR_EMAIL: 't@x', GIT_COMMITTER_NAME: 't', GIT_COMMITTER_EMAIL: 't@x' }; +const git = (cwd: string, ...args: string[]) => execFileSync('git', args, { cwd, encoding: 'utf8', timeout: 30_000, env: { ...process.env, ...ID } }).trim(); + +/** origin with base (also stable) and c1..c5 on main, and work cloned from it; the state root is the fixture's own. */ +function fixture() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'release-gate-')); + roots.push(root); + const origin = path.join(root, 'origin.git'); + const seed = path.join(root, 'seed'); + const work = path.join(root, 'work'); + git(root, 'init', '-q', '--bare', '-b', 'main', origin); + fs.mkdirSync(seed); + git(seed, 'init', '-q', '-b', 'main'); + git(seed, 'remote', 'add', 'origin', origin); + const commit = (name: string) => { fs.writeFileSync(path.join(seed, 'VERSION'), name + '\n'); git(seed, 'add', '-A'); git(seed, 'commit', '-q', '-m', name); git(seed, 'push', '-q', 'origin', 'main'); return git(seed, 'rev-parse', 'HEAD'); }; + const base = commit('base'); + git(seed, 'push', '-q', 'origin', 'main:stable'); + const c = [1, 2, 3, 4, 5].map(i => commit('c' + i)); + git(root, 'clone', '-q', origin, work); + process.env.GSTACK_STATE_ROOT = path.join(root, 'state'); + return { root, origin, seed, work, base, c, commit }; +} + +interface Outcome { conclusion: string; artifacts: Record> } +/** A fake gh: each dispatch becomes a completed run whose outcome the test decides; run list, run view and artifact download read it back. */ +function fakeGh(onDispatch: (inputs: Record, ref: string) => Outcome) { + const runs: Run[] = []; + const artifacts = new Map>>(); + const dispatches: Array<{ ref: string; inputs: Record }> = []; + const gh = (args: string[]) => { + const ok = (out: unknown) => ({ status: 0, stdout: typeof out === 'string' ? out : JSON.stringify(out), stderr: '' }); + if (args[0] === 'workflow' && args[1] === 'run') { + const ref = args[args.indexOf('--ref') + 1]!; + const inputs = Object.fromEntries(args.flatMap((a, i) => (args[i - 1] === '-f' ? [a.split(/=(.*)/s).slice(0, 2)] : []))); + dispatches.push({ ref, inputs }); + const id = 1000 + runs.length; + const o = onDispatch(inputs, ref); + runs.push({ id, status: 'completed', conclusion: o.conclusion, head_sha: inputs.expected_sha!, display_title: 'E2E Evals [eval-bg ' + inputs.nonce + ']', html_url: 'https://github.com/o/r/actions/runs/' + id, run_attempt: 1 }); + artifacts.set(id, o.artifacts); + return ok(''); + } + if (args[0] === 'api' && args.some(a => /\/actions\/workflows\/[^/]+\/runs$/.test(a))) return ok({ workflow_runs: [...runs].reverse() }); + const view = args[0] === 'api' ? /actions\/runs\/(\d+)$/.exec(args[1] ?? '') : null; + if (view) return ok(runs.find(r => r.id === Number(view[1]))); + if (args[0] === 'run' && args[1] === 'download') { + const files = artifacts.get(Number(args[2]))?.[args[args.indexOf('-n') + 1]!]; + if (!files) return { status: 1, stdout: '', stderr: 'no artifact matches any of the names or patterns provided' }; + const dir = args[args.indexOf('-D') + 1]!; + for (const [name, text] of Object.entries(files)) { fs.mkdirSync(dir, { recursive: true }); fs.writeFileSync(path.join(dir, name), text); } + return ok(''); + } + return { status: 1, stdout: '', stderr: 'unexpected gh ' + args.join(' ') }; + }; + return { gh, runs, dispatches }; +} + +const report = (verdict: 'GREEN' | 'RED', failed: string[] = [], cost = '37.40') => ({ 'report-verdict-a1': { 'report.txt': [ + '[test:paid] report: 17/17 slices, 85 planned shards, tier=gate, attempt 1', + ...(failed.length ? ['FAILED cases:', ...failed.map(l => ' ✗ ' + l)] : []), + '[test:paid] VERDICT ' + verdict + ' — lane gate/full census, attempt 1', + ' wall 41m02s · cost $' + cost + ' known + 4 of 139 trial(s) cost unknown', +].join('\n') } }); +const green = (flaky: string[] = []): Outcome => ({ conclusion: 'success', artifacts: { ...report('GREEN'), ...(flaky.length ? { 'flake-records-a1': { 'flake-records.jsonl': flaky.map(c => JSON.stringify({ schema: 1, kind: 'gate-flake', case: c })).join('\n') } } : {}) } }); + +function envFor(f: ReturnType, gh: (args: string[]) => { status: number; stdout: string; stderr: string }, shiftHours = 0): GateEnv { + return { repo: 'o/r', repoRoot: f.work, remote: 'origin', gh, git: gitIn(f.work), now: () => Date.now() + shiftHours * 3_600_000, sleep: () => {}, random: () => 'abc123', pollMs: 1, deadline: Date.now() + 3_600_000 }; +} +const records = (f: ReturnType) => { + git(f.work, 'fetch', '-q', 'origin', '+' + NOTES_REF + ':' + NOTES_REF); + return releaseRecords(gitIn(f.work), repoIdentity(f.work).key); +}; + +describe('release gate: run', () => { + // Value: protects=a gate only when main moved: the first run dispatches the full gate once on main's head (evals_all, expected_sha, a backstop nonce), + // records GREEN with its run, cost and rule-16 flaky cases in origin's notes and prints the RELEASE green line without moving stable; a second run on + // the same head, and a head equal to stable, skip without dispatching; fails_when=the gate dispatches again on an unchanged main (a $37 repeat), a + // green gate moves stable, or the record never reaches origin; why_new=new module; seam=fake gh; no_claim=a live gate run (the PR's one live proof) + test('green: one dispatch on the moved head, a GREEN release record on origin, stable untouched; an unchanged main skips', () => { + const f = fixture(); + const fake = fakeGh(() => green(['judge-x'])); + const head = f.c[4]!; + const out = runGate(envFor(f, fake.gh), { gate: GATE, label: 'test', dryRun: false, ignoreCadence: false }); + expect(out.action, out.lines.join('\n')).toBe('gated'); + expect(fake.dispatches).toHaveLength(1); + expect(fake.dispatches[0]!.ref).toBe('main'); + expect(fake.dispatches[0]!.inputs).toMatchObject({ evals_all: 'true', expected_sha: head }); + expect(fake.dispatches[0]!.inputs.nonce).toMatch(/^gate\.backstop-/); + expect(out.lines).toContain('RELEASE: green sha=' + head + ' stable=' + f.base + ' run=1000 cost=37.40'); + expect(out.lines.some(l => l.includes('flaky (failed once, passed its one rule-16 rerun in run 1000): judge-x'))).toBe(true); + const [rec] = records(f); + expect(rec).toMatchObject({ kind: 'release', sha: head, verdict: 'GREEN', flaky: ['judge-x'], cost_usd: 37.4, since: f.base, attempts: 1, run: { id: 1000 } }); + expect(git(f.seed, 'ls-remote', 'origin', 'refs/heads/stable').split('\t')[0]).toBe(f.base); + + const again = runGate(envFor(f, fake.gh, 9), { gate: GATE, label: 'test', dryRun: false, ignoreCadence: true }); + expect(again.action).toBe('skipped'); + expect(again.lines[0]).toContain('main has not moved since the last gate: GREEN run 1000'); + git(f.seed, 'push', '-q', '-f', 'origin', 'main:stable'); + const atStable = runGate(envFor(f, fake.gh), { gate: GATE, label: 'test', dryRun: false, ignoreCadence: true }); + expect(atStable.lines[0]).toContain('main has not moved since stable'); + expect(fake.dispatches).toHaveLength(1); + }); + + // Value: protects=without a release_gate block nothing runs; --dry-run names the head, the base and the pending count and writes and dispatches + // nothing; the cadence holds a moved main until cadence_hours minus the grace pass, and --ignore-cadence gates now; fails_when=an inert policy spends, + // a dry run writes a record, or the schedule gates more often than the policy says; why_new=new module; seam=fake gh and a shifted clock; + // no_claim=GitHub's cron timing + test('inert, dry run and cadence: nothing is spent until the policy and the clock allow it', () => { + const f = fixture(); + const fake = fakeGh(() => green()); + expect(runGate(envFor(f, fake.gh), { gate: null, label: 'origin/main@x', dryRun: false, ignoreCadence: false }).action).toBe('inert'); + const dry = runGate(envFor(f, fake.gh), { gate: GATE, label: 'test', dryRun: true, ignoreCadence: false }); + expect(dry.action).toBe('dry-run'); + expect(dry.lines[0]).toBe('RELEASE: gate sha=' + f.c[4] + ' since=' + f.base.slice(0, 12) + ' (stable) pending=5 stable=' + f.base.slice(0, 12)); + expect(fake.dispatches).toHaveLength(0); + expect(spawnSync('git', ['ls-remote', 'origin', NOTES_REF], { cwd: f.work, encoding: 'utf8', timeout: 30_000 }).stdout).toBe(''); + + expect(runGate(envFor(f, fake.gh), { gate: GATE, label: 'test', dryRun: false, ignoreCadence: false }).action).toBe('gated'); + const next = f.commit('c6'); + const early = runGate(envFor(f, fake.gh, 2), { gate: GATE, label: 'test', dryRun: false, ignoreCadence: false }); + expect(early.action).toBe('skipped'); + expect(early.lines[0]).toMatch(/the last gate was dispatched 2\.0 h ago on [0-9a-f]{12}; cadence 8 h/); + const due = runGate(envFor(f, fake.gh, 7.5), { gate: GATE, label: 'test', dryRun: false, ignoreCadence: false }); + expect(due.action, due.lines.join('\n')).toBe('gated'); + expect(fake.dispatches.map(d => d.inputs.expected_sha)).toEqual([f.c[4], next]); + expect(records(f).find(r => r.sha === next)?.since).toBe(f.c[4]); + }); + + // Value: protects=an INCOMPLETE census (only INFRA reds) is recorded as such, re-dispatched once by the next run (EVAL_POLICY.infraRedispatch), then + // skipped with its run named, and never treated as green or red; fails_when=an infra-only failure holds the release like a red, is retried without + // end, or is never retried; why_new=new module; seam=fake gh; no_claim=a real runner loss + test('incomplete: recorded, re-dispatched once, then skipped with the run named', () => { + const f = fixture(); + const fake = fakeGh(() => ({ conclusion: 'failure', artifacts: report('RED', ['case-a rule FAIL INFRA']) })); + const env = envFor(f, fake.gh); + const first = runGate(env, { gate: GATE, label: 'test', dryRun: false, ignoreCadence: true }); + expect(first.action).toBe('failed'); + expect(first.lines.at(-1)).toContain('RELEASE: incomplete sha=' + f.c[4] + ' run=1000'); + expect(first.lines.at(-1)).toContain('the next run re-dispatches it once'); + expect(runGate(env, { gate: GATE, label: 'test', dryRun: false, ignoreCadence: true }).action).toBe('failed'); + const third = runGate(env, { gate: GATE, label: 'test', dryRun: false, ignoreCadence: true }); + expect(third.action).toBe('skipped'); + expect(third.lines[0]).toContain('incomplete ' + MAX_RELEASE_ATTEMPTS + ' times'); + expect(fake.dispatches).toHaveLength(MAX_RELEASE_ATTEMPTS); + expect(activeHold(records(f), () => true)).toBeNull(); + }); +}); + +describe('release gate: red, bisect and the hold', () => { + // Value: protects=a red gate records RED (the hold release-cut reads), bisects only its red verdict cases on release-gate/ with prefixes between + // stable and the red head, narrows round by round to the first red commit after a green one, deletes the bisect branch, and prints first_bad; a later + // green gate on a newer commit lifts the hold; fails_when=a red gate moves or leaves stable releasable, the bisect blames the wrong commit, tests the + // known-red head again, leaks its branch, or a green gate on an older commit lifts the hold; why_new=new module; seam=fake gh; no_claim=a live bisect + test('red: RED record, a bisect that names first_bad, the branch cleaned up, and the hold until a later green', () => { + const f = fixture(); + const order = [f.base, ...f.c]; + const fake = fakeGh((inputs) => { + if (!inputs.prefix_shas) return { conclusion: 'failure', artifacts: report('RED', ['case-a rule FAIL VERDICT', 'case-z rule FAIL INFRA']) }; + const files: Record> = {}; + for (const sha of inputs.prefix_shas.split(',')) { + const red = order.indexOf(sha) >= 3; + files['gstack-release-record-bisect-' + sha.slice(0, 12)] = { 'bisect-prefix.json': JSON.stringify({ schema: 1, kind: 'train-bisect-prefix', nonce: inputs.nonce, prefix_sha: sha, checked_out_sha: sha, + panels: [{ case: 'case-a', kind: 'rule', status: red ? 'FAIL' : 'PASS', red_class: red ? 'VERDICT' : null, cost_usd: 1.25 }] }) }; + } + return { conclusion: 'success', artifacts: files }; + }); + const out = runGate(envFor(f, fake.gh), { gate: GATE, label: 'test', dryRun: false, ignoreCadence: true }); + expect(out.action, out.lines.join('\n')).toBe('held'); + expect(out.lines).toContain('RELEASE: red sha=' + f.c[4] + ' first_bad=' + f.c[2] + ' run=1000'); + expect(fake.dispatches).toHaveLength(2); + const bisect = fake.dispatches[1]!; + expect(bisect.ref).toMatch(/^release-gate\/abc123-[0-9a-f]{12}$/); + expect(bisect.inputs).toMatchObject({ cases: 'case-a', prefix_shas: f.c.slice(0, 4).join(','), expected_sha: f.c[4] }); + expect(bisect.inputs.nonce).toMatch(/^gate\.bisect-/); + expect(git(f.seed, 'ls-remote', 'origin', 'refs/heads/release-gate/*')).toBe(''); + const recs = records(f); + expect(recs[0]).toMatchObject({ verdict: 'RED', red_cases: ['case-a'], bisect: { state: 'found', first_bad: f.c[2], good: f.c[1], bad: f.c[2], spent_usd: 5 } }); + expect(git(f.seed, 'ls-remote', 'origin', 'refs/heads/stable').split('\t')[0]).toBe(f.base); + const isAncestor = (a: string, b: string) => spawnSync('git', ['merge-base', '--is-ancestor', a, b], { cwd: f.work, timeout: 30_000 }).status === 0; + expect(activeHold(recs, isAncestor)?.sha).toBe(f.c[4]); + const status = gateStatus(envFor(f, fake.gh), { gate: GATE, label: 'test' }); + expect(status.action).toBe('held'); + expect(status.lines.find(l => l.startsWith('HOLD:'))).toContain('red on ' + f.c[4]); + const older: ReleaseRecord = { ...recs[0]!, sha: f.c[1]!, verdict: 'GREEN', bisect: null }; + expect(activeHold([...recs, older], isAncestor)?.sha).toBe(f.c[4]); + const later: ReleaseRecord = { ...recs[0]!, sha: f.commit('fix'), verdict: 'GREEN', bisect: null }; + git(f.work, 'fetch', '-q', 'origin'); + expect(activeHold([later, ...recs], isAncestor)).toBeNull(); + }); + + // Value: protects=the bisect spends within its cap: a 0 cap records no bisect and first_bad unknown, a small cap tests only the prefixes it covers + // and stops capped, and bisect --spend-cap resumes a capped one; fails_when=a bisect dispatches past its cap or a capped bisect cannot be resumed; + // why_new=new module; seam=fake gh; no_claim=real case prices + test('bisect spend: a zero cap skips it, a small cap stops capped, and bisect --spend-cap resumes', () => { + const f = fixture(); + const fake = fakeGh((inputs) => (inputs.prefix_shas ? { conclusion: 'success', artifacts: {} } : { conclusion: 'failure', artifacts: report('RED', ['case-a rule FAIL VERDICT', 'case-b judge FAIL']) })); + const none = runGate(envFor(f, fake.gh), { gate: { ...GATE, bisect_spend_cap_usd: 0 }, label: 'test', dryRun: false, ignoreCadence: true }); + expect(none.lines).toContain('RELEASE: red sha=' + f.c[4] + ' first_bad=unknown run=1000'); + expect(none.lines.some(l => l.includes('bisect unknown: bisect_spend_cap_usd is 0'))).toBe(true); + expect(fake.dispatches).toHaveLength(1); + + const g = fixture(); + const fake2 = fakeGh((inputs) => (inputs.prefix_shas ? { conclusion: 'success', artifacts: {} } : { conclusion: 'failure', artifacts: report('RED', ['case-a rule FAIL VERDICT', 'case-b judge FAIL']) })); + const capped = runGate(envFor(g, fake2.gh), { gate: { ...GATE, bisect_spend_cap_usd: 2 }, label: 'test', dryRun: false, ignoreCadence: true }); + expect(capped.lines.some(l => l.includes('bisect capped: the next round needs $3.00 a prefix'))).toBe(true); + expect(fake2.dispatches).toHaveLength(1); + const plan = bisectGate(envFor(g, fake2.gh), { gate: GATE, label: 'test', spendCap: 30, dryRun: true }); + expect(plan.lines.some(l => l.startsWith('RELEASE: bisect round 1: 2 case(s) on 4 of 4 commit(s)'))).toBe(true); + expect(fake2.dispatches).toHaveLength(1); + }); +}); + +describe('release gate: verdict, range and policy units', () => { + test('parseReport and classifyRun: verdict reds bisect, other classes keep a red incomplete, success is green, and a contradiction is incomplete', () => { + const r = parseReport(report('RED', ['case-a rule FAIL VERDICT', 'judge one judge FAIL', 'case-z rule FAIL INFRA'])['report-verdict-a1']['report.txt']); + expect(r).toEqual({ verdict: 'RED', red: ['case-a', 'judge one'], other: ['case-z'], cost: 37.4 }); + expect(classifyRun({ status: 'completed', conclusion: 'failure' }, r)).toMatchObject({ verdict: 'RED', red: ['case-a', 'judge one'] }); + expect(classifyRun({ status: 'completed', conclusion: 'failure' }, { ...r, red: [] }).verdict).toBe('INCOMPLETE'); + expect(classifyRun({ status: 'completed', conclusion: 'cancelled' }, null).verdict).toBe('INCOMPLETE'); + expect(classifyRun({ status: 'completed', conclusion: 'success' }, { ...r, verdict: 'GREEN' }).verdict).toBe('GREEN'); + expect(classifyRun({ status: 'completed', conclusion: 'success' }, r).verdict).toBe('INCOMPLETE'); + expect(classifyRun({ status: 'in_progress', conclusion: null }, null).verdict).toBe('PENDING'); + }); + + test('planPrefixes spreads at most MAX_PREFIXES over the interior; narrow moves good and bad and refuses a non-monotone or missing result', () => { + const shas = Array.from({ length: 20 }, (_, i) => String(i).padStart(40, '0')); + expect(planPrefixes(shas.slice(0, 3))).toEqual(shas.slice(0, 3)); + const picked = planPrefixes(shas, 7); + expect(picked).toHaveLength(7); + expect(new Set(picked).size).toBe(7); + expect(picked[0]).not.toBe(shas[0]); + const t = (rs: string[]) => rs.map((r, i) => ({ sha: shas[i]!, results: r === '?' ? {} : { a: r === 'R' ? 'red' as const : 'green' as const } })); + expect(narrow({ good: 'g', bad: 'b' }, t(['G', 'G', 'R']), ['a'])).toEqual({ good: shas[1], bad: shas[2] }); + expect(narrow({ good: 'g', bad: 'b' }, t(['G', 'G']), ['a'])).toEqual({ good: shas[1], bad: 'b' }); + expect(narrow({ good: 'g', bad: 'b' }, t(['R', 'R']), ['a'])).toEqual({ good: 'g', bad: shas[0] }); + expect(narrow({ good: 'g', bad: 'b' }, t(['G', 'R', 'G']), ['a'])).toEqual({ unknown: 'a is not monotone over the tested commits (GRG)' }); + expect(narrow({ good: 'g', bad: 'b' }, t(['G', '?']), ['a'])).toHaveProperty('unknown'); + }); + + // Value: protects=the release_gate key is read only with min_gstack_version at its introducing version, validates every field (cadence at least the + // workflow schedule, a workflow file, branch names, hold-release), and --policy runs the same validation; fails_when=an older gstack reads the block + // silently, a bad field passes, or the command-line policy skips validation; why_new=new key; seam=none; no_claim=the json adoption (a follow-up PR) + test('policy: the release_gate key needs its min_gstack_version and every field; --policy is validated the same way', () => { + const raw = { release_gate: GATE }; + expect(validatePolicy(raw).map(e => e.path)).toEqual(['release_gate']); + expect(validatePolicy({ ...raw, min_gstack_version: RELEASE_GATE_KEY_VERSION })).toEqual([]); + expect(releaseGateErrors({ ...GATE, cadence_hours: RELEASE_GATE_SCHEDULE_HOURS - 1, on_red: 'freeze', stable_ref: 'main', extra: 1 }).map(e => e.path).sort()) + .toEqual(['release_gate.cadence_hours', 'release_gate.extra', 'release_gate.on_red', 'release_gate.stable_ref']); + expect(releaseGateErrors({ cadence_hours: 8 }).filter(e => e.message.startsWith('missing'))).toHaveLength(6); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'release-gate-policy-')); + roots.push(dir); + const good = path.join(dir, 'good.json'); + fs.writeFileSync(good, JSON.stringify({ ...raw, min_gstack_version: RELEASE_GATE_KEY_VERSION })); + expect(loadReleaseGate(ROOT, { policyFile: good })).toEqual({ gate: GATE, label: good }); + const bad = path.join(dir, 'bad.json'); + fs.writeFileSync(bad, JSON.stringify(raw)); + expect((loadReleaseGate(ROOT, { policyFile: bad }) as { error: string }).error).toContain('release_gate introduced in gstack ' + RELEASE_GATE_KEY_VERSION); + }); + + test('bin/gstack-release-gate: usage exits 2; an inert policy file exits 0 without touching GitHub', () => { + const bin = path.join(ROOT, 'bin/gstack-release-gate'); + const usage = spawnSync('bun', [...bunIsolationArgs(), bin, 'frobnicate'], { cwd: ROOT, encoding: 'utf8', timeout: 60_000 }); + expect(usage.status).toBe(2); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'release-gate-bin-')); + roots.push(dir); + const inert = path.join(dir, 'policy.json'); + fs.writeFileSync(inert, '{}'); + const r = spawnSync('bun', [...bunIsolationArgs(), bin, 'run', '--repo', 'o/r', '--policy', inert], { cwd: ROOT, encoding: 'utf8', timeout: 60_000 }); + expect(r.status, r.stderr).toBe(0); + expect(r.stdout.trim()).toBe('RELEASE: inert (' + inert + ' declares no release_gate block; the per-push main backstop gates main)'); + }); +}); diff --git a/test/train-gate.test.ts b/test/train-gate.test.ts index 17702d1f0..2aaad83db 100644 --- a/test/train-gate.test.ts +++ b/test/train-gate.test.ts @@ -54,6 +54,20 @@ describe('routeDispatch: every run keeps its mode, and train inputs are validate expect((route({ ...screen, trainPolicy: false }) as { error: string }).error).toContain('needs a train block'); }); + // Value: protects=the release gate's bisect (lib/release-gate.ts) routes on its own release-gate/ ref under a release_gate policy, names its own + // prefix artifact, keeps every train bisect check (nonce, head, cases, prefixes), and is refused without the policy or on any other ref; + // fails_when=a release bisect needs a train block (it stops working once trains go), a release-gate/ ref routes without a release_gate policy, or a + // train/ bisect starts accepting the release policy alone; why_new=the bisect test below covers train/ only; seam=none; no_claim=a live run + test('a release bisect runs on release-gate/ under a release_gate policy, with the train checks and its own artifact', () => { + const bisect = { nonce: bisectNonce, refName: 'release-gate/abc123-aaaaaaaaaaaa', expectedSha: A, cases: 'case-a', prefixShas: B, releasePolicy: true }; + expect(route(bisect)).toEqual({ mode: 'bisect', cases: ['case-a'], prefixes: [B], artifact: 'gstack-release-record' }); + expect((route({ ...bisect, releasePolicy: false, trainPolicy: true }) as { error: string }).error).toContain('needs a release_gate block'); + expect((route({ ...bisect, prefixShas: C }) as { error: string }).error).toContain('is not an ancestor'); + expect((route({ ...bisect, expectedSha: B }) as { error: string }).error).toContain('is not the train head'); + expect((route({ ...bisect, refName: 'train/t1', trainPolicy: false }) as { error: string }).error).toContain('needs a train block'); + expect((route({ ...bisect, refName: 'main' }) as { error: string }).error).toContain('runs on train/'); + }); + test('a bisect needs a bisect nonce on train/, the train head, plan case ids and ancestor prefixes', () => { const bisect = { nonce: bisectNonce, refName: 'train/t1', expectedSha: A, cases: 'case-a, review/SKILL.md workflow', prefixShas: `${A},${B}`, trainPolicy: true }; expect(route(bisect)).toEqual({ mode: 'bisect', cases: ['case-a', 'review/SKILL.md workflow'], prefixes: [A, B] }); From 6a5817c52755a361e9419f638a715db92be0a12f Mon Sep 17 00:00:00 2001 From: garrytan Date: Tue, 3 Nov 2026 02:52:51 +0000 Subject: [PATCH 02/15] docs(release-gate): exit codes match the bin Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- changelog.d/gsta258-l1-release-gate.md | 2 +- docs/ship-policy.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/changelog.d/gsta258-l1-release-gate.md b/changelog.d/gsta258-l1-release-gate.md index 0fd053ea5..91b2d6a63 100644 --- a/changelog.d/gsta258-l1-release-gate.md +++ b/changelog.d/gsta258-l1-release-gate.md @@ -5,7 +5,7 @@ bump: minor ### Added - **`release_gate` policy key** (`lib/release-gate-policy.ts`, key version 1.94.0.0): `cadence_hours` (8 to 168), `gate_workflow`, `branch`, `stable_ref`, `check_name`, `on_red: hold-release` and `bisect_spend_cap_usd`, every field required. -- **`bin/gstack-release-gate`** (`lib/release-gate.ts`): `run [--dry-run] [--sha] [--ignore-cadence]`, `status` and `bisect`. `--policy ` reads the block from a file, for a dry run or a proof before adoption. Exit 0 means gated, skipped, inert or a dry run; 1 means held; 2 means failed. +- **`bin/gstack-release-gate`** (`lib/release-gate.ts`): `run [--dry-run] [--sha] [--ignore-cadence]`, `status` and `bisect`. `--policy ` reads the block from a file, for a dry run or a proof before adoption. Exit 0 means gated, skipped, inert or a dry run; 1 means held, failed or a refused policy; 2 is a usage error. - **`.github/workflows/release-gate.yml`**: an 8-hour schedule and a manual dispatch. The check run comes from a separate job that holds `checks: write` and runs no repository code. - **Capy skill installer follows the update channel** (`capy/skill/gstack/scripts/install.sh`, wrapper revision 4). With `GSTACK_UPDATE_CHANNEL=stable` or a stable `update_channel`, a fresh clone or a floor upgrade goes to `stable` when it meets the support floor, and to main otherwise. `--revision` still wins. diff --git a/docs/ship-policy.md b/docs/ship-policy.md index 3a31c05ee..706e885d4 100644 --- a/docs/ship-policy.md +++ b/docs/ship-policy.md @@ -1188,7 +1188,7 @@ Common flags: `--repo`, `--policy ` (read the block from a file instead of main, for a dry run or a proof before adoption), `--remote`, `--summary`, `--outputs`, `--poll-seconds`, `--budget-minutes`. Every line starts with `RELEASE:`. Exit 0 means gated, skipped, inert or a dry run; 1 -means held (red); 2 means failed or incomplete. +means held (red), failed, incomplete or a refused policy; 2 is a usage error. **Escape hatch.** Remove the block on main. The per-push backstop gates the next push again, and release-cut reads its check runs as before. From 080bc1c156e760b94446bac352436dde64b1f551 Mon Sep 17 00:00:00 2001 From: garrytan Date: Tue, 3 Nov 2026 03:01:33 +0000 Subject: [PATCH 03/15] fix(release-gate): blobless checkout and free-only workflow classification Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- .github/workflows/release-gate.yml | 1 + scripts/test-pr-profile.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml index 583444a6d..9ab3cd24a 100644 --- a/.github/workflows/release-gate.yml +++ b/.github/workflows/release-gate.yml @@ -55,6 +55,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 + filter: blob:none # Read with git and jq only: without a release_gate block nothing below runs. - name: Read the release gate policy (main) diff --git a/scripts/test-pr-profile.ts b/scripts/test-pr-profile.ts index dd0ca92bc..401bda3a3 100644 --- a/scripts/test-pr-profile.ts +++ b/scripts/test-pr-profile.ts @@ -288,6 +288,7 @@ export const FREE_ONLY_PR_FILES = [ '.github/workflows/test-health.yml', '.github/workflows/eval-sweep.yml', // The weekly off-ship sweep on main; no PR-gate case reads it, and it never gates a PR. '.github/workflows/evals-main-backstop.yml', // The merge-train backstop on main: it dispatches evals.yml after a push to main; no PR-gate case reads it, and it never gates a PR. + '.github/workflows/release-gate.yml', // The 8-hour release gate on main: it dispatches evals.yml on main's head; no PR-gate case reads it, and it never gates a PR. '.github/workflows/weekly-release.yml', // The Monday release cut (tag, stable, GitHub Release): reads check runs on main; no paid case reads it, and it never gates a PR. 'scripts/release-cut.ts', // The release cut's logic, run only by weekly-release.yml; covered by the free test/release-cut.test.ts. '.github/actionlint.yaml', // Runner labels for actionlint only. From cbe0bc81ad2ba88f64310bc4485ed6083ba1ed15 Mon Sep 17 00:00:00 2001 From: garrytan Date: Tue, 3 Nov 2026 03:48:22 +0000 Subject: [PATCH 04/15] fix: pre-landing review fixes (finish an open gate before a new head; neutral check for an unfinished gate) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- .github/workflows/release-gate.yml | 3 ++- docs/ship-policy.md | 19 ++++++++++------- lib/release-gate.ts | 8 ++++++-- scripts/harness-version.json | 2 +- test/release-gate-workflow.test.ts | 8 +++++--- test/release-gate.test.ts | 33 ++++++++++++++++++++++++++++-- 6 files changed, 57 insertions(+), 16 deletions(-) diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml index 9ab3cd24a..4f92f95bc 100644 --- a/.github/workflows/release-gate.yml +++ b/.github/workflows/release-gate.yml @@ -108,7 +108,8 @@ jobs: run: | case "$ACTION" in gated) conclusion=success ;; - held|failed) conclusion=failure ;; + held) conclusion=failure ;; + failed) conclusion=neutral ;; *) echo "::error::unknown release gate action '$ACTION'"; exit 1 ;; esac [[ "$SHA" =~ ^[0-9a-f]{40}$ ]] || { echo "::error::'$SHA' is not a commit SHA"; exit 1; } diff --git a/docs/ship-policy.md b/docs/ship-policy.md index 706e885d4..3632b1313 100644 --- a/docs/ship-policy.md +++ b/docs/ship-policy.md @@ -1149,22 +1149,27 @@ on a manual dispatch from main. Without the block it stops after a git-and-jq policy read, so no paid work starts and nothing is written. With the block it runs `gstack-release-gate run`, which: -1. skips when main has not moved since `stable` or since the last gate, or +1. first finishes an open gate on an older commit (a run that outlived its + budget, or a red gate's unfinished bisect), so main moving never orphans a + verdict; only then does it look at main's head; +2. skips when main has not moved since `stable` or since the last gate, or when the last gate is younger than `cadence_hours` (`--ignore-cadence` overrides only the age); -2. writes a `release` record (`gstack-trains:release::`) to +3. writes a `release` record (`gstack-trains:release::`) to `refs/notes/gstack-trains` before it dispatches, so a lost run resumes; -3. dispatches the full gate on that sha and reads its verdict and flake +4. dispatches the full gate on that sha and reads its verdict and flake records. A case that failed once and passed its one rule-16 rerun is green and is listed as flaky; -4. records GREEN, RED or INCOMPLETE. An INCOMPLETE gate is re-dispatched once +5. records GREEN, RED or INCOMPLETE. An INCOMPLETE gate is re-dispatched once by the next run, then left for a person; -5. on RED, bisects the red cases over main's first-parent commits since the +6. on RED, bisects the red cases over main's first-parent commits since the last green gate (at most 4 rounds, within the spend cap) and records the first bad commit; -6. publishes `check_name` on the gated commit from a separate job that holds +7. publishes `check_name` on the gated commit from a separate job that holds `checks: write` and runs no repository code: success for a green gate, - failure for a red or failed one. + failure for a red one, and neutral when the gate has no verdict yet (still + running at the budget, or incomplete), so a later green on the same commit + is not undercut by an earlier failure. **What release-cut reads.** A GREEN release record on a commit counts as gated, the same as a successful `main-backstop` check run. A RED record diff --git a/lib/release-gate.ts b/lib/release-gate.ts index 6255a5a0b..bdedc6b05 100644 --- a/lib/release-gate.ts +++ b/lib/release-gate.ts @@ -230,8 +230,12 @@ export function runGate(env: GateEnv, o: RunOptions): GateOut { const gate = o.gate; const c = context(env, gate); if ('error' in c) { lines.push(`RELEASE: cannot read refs/notes/gstack-trains (${c.error}); nothing dispatched`); return out('failed', null); } - const sha = o.sha ?? env.git(['ls-remote', env.remote, `refs/heads/${gate.branch}`]).out.split(/\s/)[0] ?? ''; - if (!SHA.test(sha)) { lines.push(`RELEASE: cannot resolve ${gate.branch} on ${env.remote} (${JSON.stringify(sha)})`); return out('failed', null); } + const head = o.sha ?? env.git(['ls-remote', env.remote, `refs/heads/${gate.branch}`]).out.split(/\s/)[0] ?? ''; + if (!SHA.test(head)) { lines.push(`RELEASE: cannot resolve ${gate.branch} on ${env.remote} (${JSON.stringify(head)})`); return out('failed', null); } + const open = o.sha ? null : c.records.find(r => r.sha !== head && ((r.verdict === 'PENDING' && r.run) || (r.verdict === 'RED' && r.bisect?.state === 'running'))); + if (open && o.dryRun) { lines.push(`RELEASE: dry run: would finish the open gate on ${short(open.sha)} (${open.verdict}, run ${open.run?.id ?? '-'}) before gating ${gate.branch}@${short(head)}; nothing written`); return out('dry-run', open.sha); } + if (open) lines.push(`RELEASE: finishing the open gate on ${short(open.sha)} (${open.verdict}, run ${open.run?.id ?? '-'}) before gating ${gate.branch}@${short(head)}`); + const sha = open?.sha ?? head; ensureHistory(env, gate, sha); const prior = c.records.find(r => r.sha === sha) ?? null; const skip = (why: string) => { lines.push(`RELEASE: skip sha=${sha} (${why})`); return out('skipped', sha); }; diff --git a/scripts/harness-version.json b/scripts/harness-version.json index 012cc08ed..41b77c5ba 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -17,7 +17,7 @@ "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6}, "scripts/lib/shard-engine.ts": {"blob":"dc714d7d897aed185c0ec3d2c92ad22d954039e5","since":6,"nonBehavioral":"createShardSandbox drops the Capy launch markers (GSTACK_SESSION_KIND, GSTACK_CAPY, GSTACK_EPHEMERAL, GSTACK_INSTALL_VERDICT) from shard environments; CI never sets them, so no case's input or verdict changes there, and on a Capy machine fixtures now see the session kind they set themselves instead of the installer's"}, "scripts/test-paid-shards.ts": {"blob":"5439d0a8f6e1569fa7c924af20d67c30c81d5d30","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, - "scripts/test-pr-profile.ts": {"blob":"dd0ca92bcf8493fd5c1e9d399cc3bc74ec70c469","since":6,"nonBehavioral":"FREE_ONLY_PR_FILES gains USING_GBRAIN_WITH_GSTACK.md (a root guide no skill prompt or paid fixture reads); it changes only which PR paths select paid cases, never how a case runs or is judged (GST-1027)"}, + "scripts/test-pr-profile.ts": {"blob":"401bda3a3243bc836f39ac4dd0e5933ddcbaee17","since":6,"nonBehavioral":"scripts/test-pr-profile.ts only classifies the new release-gate.yml as free-only; no case selection or verdict reads that entry"}, "scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":6}, "test/helpers/agent-sdk-runner.ts": {"blob":"d66dac458b8c13250973e550edf8087d4044cd0b","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets. Also: runAgentSdkTest now validates systemPrompt before starting; every live caller already passed the claude_code preset, so the prompt the model sees is unchanged and no verdict can change"}, "test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":6}, diff --git a/test/release-gate-workflow.test.ts b/test/release-gate-workflow.test.ts index a2f8a5542..e881221c9 100644 --- a/test/release-gate-workflow.test.ts +++ b/test/release-gate-workflow.test.ts @@ -82,8 +82,9 @@ describe('release-gate.yml', () => { }); // Value: protects=the gated commit's check run (what release-cut reads) comes from a job with checks: write and no checkout or action, only for a - // gate that ran (success for gated, failure for held or failed), on a validated sha and check name; fails_when=repository code runs with checks: - // write, a skipped or inert run publishes a check, or a red gate publishes success; why_new=new workflow; seam=none; no_claim=GitHub's check API + // gate that ran (success for gated, failure for held, neutral for failed so a later green on the same sha stays releasable), on a validated sha + // and check name; fails_when=repository code runs with checks: write, a skipped or inert run publishes a check, a red gate publishes success, or + // an unfinished gate leaves a failure release-cut reads as red; why_new=new workflow; seam=none; no_claim=GitHub's check API test('the check run is published from a code-free job, success only for gated', () => { const job = release.jobs['release-check']!; expect(job.needs).toBe('release-gate'); @@ -92,7 +93,8 @@ describe('release-gate.yml', () => { expect(job.steps.filter(s => s.uses)).toEqual([]); const run = job.steps[0]!.run!; expect(run).toContain('gated) conclusion=success ;;'); - expect(run).toContain('held|failed) conclusion=failure ;;'); + expect(run).toContain('held) conclusion=failure ;;'); + expect(run).toContain('failed) conclusion=neutral ;;'); expect(run).toContain('[[ "$SHA" =~ ^[0-9a-f]{40}$ ]]'); expect(run).toContain('-f name="$CHECK" -f head_sha="$SHA"'); expect(run).not.toContain('bun'); diff --git a/test/release-gate.test.ts b/test/release-gate.test.ts index 29fe65cbc..cfee00ce7 100644 --- a/test/release-gate.test.ts +++ b/test/release-gate.test.ts @@ -51,7 +51,7 @@ function fixture() { return { root, origin, seed, work, base, c, commit }; } -interface Outcome { conclusion: string; artifacts: Record> } +interface Outcome { conclusion: string; artifacts: Record>; status?: string } /** A fake gh: each dispatch becomes a completed run whose outcome the test decides; run list, run view and artifact download read it back. */ function fakeGh(onDispatch: (inputs: Record, ref: string) => Outcome) { const runs: Run[] = []; @@ -65,7 +65,7 @@ function fakeGh(onDispatch: (inputs: Record, ref: string) => Out dispatches.push({ ref, inputs }); const id = 1000 + runs.length; const o = onDispatch(inputs, ref); - runs.push({ id, status: 'completed', conclusion: o.conclusion, head_sha: inputs.expected_sha!, display_title: 'E2E Evals [eval-bg ' + inputs.nonce + ']', html_url: 'https://github.com/o/r/actions/runs/' + id, run_attempt: 1 }); + runs.push({ id, status: o.status ?? 'completed', conclusion: o.conclusion, head_sha: inputs.expected_sha!, display_title: 'E2E Evals [eval-bg ' + inputs.nonce + ']', html_url: 'https://github.com/o/r/actions/runs/' + id, run_attempt: 1 }); artifacts.set(id, o.artifacts); return ok(''); } @@ -173,6 +173,35 @@ describe('release gate: run', () => { expect(fake.dispatches).toHaveLength(MAX_RELEASE_ATTEMPTS); expect(activeHold(records(f), () => true)).toBeNull(); }); + + // Value: protects=a gate run still going at the budget stays PENDING with its run, and the next run finishes that verdict on the old commit before + // gating main's new head, so a red verdict still records RED and holds release-cut after main moved; the new head is gated on the run after; + // fails_when=main moving orphans a PENDING record and its red verdict never becomes a hold; why_new=review finding (red-team, l1-p1); seam=fake gh; + // no_claim=a real six-hour run + test('an unfinished gate is finished on its own commit after main moves, then the new head is gated', () => { + const f = fixture(); + let dispatched = 0; + const fake = fakeGh(() => (dispatched++ === 0 ? { conclusion: 'failure', status: 'in_progress', artifacts: report('RED', ['case-a rule FAIL VERDICT']) } : green())); + const gate = { ...GATE, bisect_spend_cap_usd: 0 }; + const first = runGate({ ...envFor(f, fake.gh), deadline: 0 }, { gate, label: 'test', dryRun: false, ignoreCadence: true }); + expect(first.action).toBe('failed'); + expect(records(f)[0]).toMatchObject({ sha: f.c[4], verdict: 'PENDING', run: { id: 1000 } }); + const next = f.commit('c6'); + fake.runs[0]!.status = 'completed'; + const dry = runGate(envFor(f, fake.gh), { gate, label: 'test', dryRun: true, ignoreCadence: true }); + expect(dry.lines[0]).toContain('would finish the open gate on ' + f.c[4]!.slice(0, 12)); + const second = runGate(envFor(f, fake.gh), { gate, label: 'test', dryRun: false, ignoreCadence: true }); + expect(second.action, second.lines.join('\n')).toBe('held'); + expect(second.sha).toBe(f.c[4]); + expect(second.lines[0]).toContain('finishing the open gate on ' + f.c[4]!.slice(0, 12)); + expect(fake.dispatches).toHaveLength(1); + expect(activeHold(records(f), () => true)?.sha).toBe(f.c[4]); + const third = runGate(envFor(f, fake.gh), { gate, label: 'test', dryRun: false, ignoreCadence: true }); + expect(third.action, third.lines.join('\n')).toBe('gated'); + expect(third.sha).toBe(next); + expect(fake.dispatches).toHaveLength(2); + expect(fake.dispatches[1]!.inputs.expected_sha).toBe(next); + }); }); describe('release gate: red, bisect and the hold', () => { From 70ba48cb1e64feff014555a3a72808e21a1d4f5a Mon Sep 17 00:00:00 2001 From: garrytan Date: Tue, 3 Nov 2026 03:53:52 +0000 Subject: [PATCH 05/15] fix: pre-landing review fixes, pass 2 (the workflow reads main head itself; an open gate on stable or older than three cadences) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- .github/workflows/release-gate.yml | 2 +- docs/ship-policy.md | 3 ++- lib/release-gate.ts | 7 +++++-- test/release-gate-workflow.test.ts | 6 ++++-- test/release-gate.test.ts | 26 +++++++++++++++++++++++++- 5 files changed, 37 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml index 4f92f95bc..688fb1e72 100644 --- a/.github/workflows/release-gate.yml +++ b/.github/workflows/release-gate.yml @@ -85,7 +85,7 @@ jobs: DRY_RUN: ${{ inputs.dry_run && '1' || '' }} IGNORE_CADENCE: ${{ inputs.ignore_cadence && '1' || '' }} run: | - ./bin/gstack-release-gate run --repo "$GITHUB_REPOSITORY" --sha "$GITHUB_SHA" --budget-minutes 330 \ + ./bin/gstack-release-gate run --repo "$GITHUB_REPOSITORY" --budget-minutes 330 \ --summary "$GITHUB_STEP_SUMMARY" --outputs "$GITHUB_OUTPUT" ${DRY_RUN:+--dry-run} ${IGNORE_CADENCE:+--ignore-cadence} # The gated commit's check run (release_gate.check_name, main-backstop), the diff --git a/docs/ship-policy.md b/docs/ship-policy.md index 3632b1313..3ec2fd63a 100644 --- a/docs/ship-policy.md +++ b/docs/ship-policy.md @@ -1151,7 +1151,8 @@ policy read, so no paid work starts and nothing is written. With the block it ru 1. first finishes an open gate on an older commit (a run that outlived its budget, or a red gate's unfinished bisect), so main moving never orphans a - verdict; only then does it look at main's head; + verdict; only then does it look at main's head. An open gate older than + three cadences stops holding up the head; 2. skips when main has not moved since `stable` or since the last gate, or when the last gate is younger than `cadence_hours` (`--ignore-cadence` overrides only the age); diff --git a/lib/release-gate.ts b/lib/release-gate.ts index bdedc6b05..e424d2c46 100644 --- a/lib/release-gate.ts +++ b/lib/release-gate.ts @@ -41,6 +41,8 @@ export const MAX_RELEASE_ATTEMPTS = 2; /** Cron start jitter the cadence forgives. */ export const CADENCE_GRACE_HOURS = 1; export const MAX_BISECT_ROUNDS = 4; +/** An open gate (a run past its budget, or a red gate's running bisect) on an older commit is finished first for this many cadences; after that it stops holding up main's head. */ +export const OPEN_GATE_CADENCES = 3; const SHA = /^[0-9a-f]{40}$/; const CASE_ID = /^[A-Za-z0-9 _.:/-]{1,128}$/; const HOUR = 3_600_000; @@ -232,14 +234,15 @@ export function runGate(env: GateEnv, o: RunOptions): GateOut { if ('error' in c) { lines.push(`RELEASE: cannot read refs/notes/gstack-trains (${c.error}); nothing dispatched`); return out('failed', null); } const head = o.sha ?? env.git(['ls-remote', env.remote, `refs/heads/${gate.branch}`]).out.split(/\s/)[0] ?? ''; if (!SHA.test(head)) { lines.push(`RELEASE: cannot resolve ${gate.branch} on ${env.remote} (${JSON.stringify(head)})`); return out('failed', null); } - const open = o.sha ? null : c.records.find(r => r.sha !== head && ((r.verdict === 'PENDING' && r.run) || (r.verdict === 'RED' && r.bisect?.state === 'running'))); + const fresh = (r: ReleaseRecord) => env.now() - Date.parse(r.created_at) < OPEN_GATE_CADENCES * gate.cadence_hours * HOUR; + const open = o.sha ? null : c.records.find(r => r.sha !== head && fresh(r) && ((r.verdict === 'PENDING' && r.run) || (r.verdict === 'RED' && r.bisect?.state === 'running'))); if (open && o.dryRun) { lines.push(`RELEASE: dry run: would finish the open gate on ${short(open.sha)} (${open.verdict}, run ${open.run?.id ?? '-'}) before gating ${gate.branch}@${short(head)}; nothing written`); return out('dry-run', open.sha); } if (open) lines.push(`RELEASE: finishing the open gate on ${short(open.sha)} (${open.verdict}, run ${open.run?.id ?? '-'}) before gating ${gate.branch}@${short(head)}`); const sha = open?.sha ?? head; ensureHistory(env, gate, sha); const prior = c.records.find(r => r.sha === sha) ?? null; const skip = (why: string) => { lines.push(`RELEASE: skip sha=${sha} (${why})`); return out('skipped', sha); }; - if (sha === c.stable) return skip(`${gate.branch} has not moved since ${gate.stable_ref}`); + if (!open && sha === c.stable) return skip(`${gate.branch} has not moved since ${gate.stable_ref}`); if (prior?.verdict === 'GREEN' || prior?.verdict === 'RED') { if (prior.verdict === 'RED' && prior.bisect?.state === 'running' && !o.dryRun) return finishRed(env, c, prior, lines, out); return skip(`${gate.branch} has not moved since the last gate: ${prior.verdict} run ${prior.run?.id ?? '-'}`); diff --git a/test/release-gate-workflow.test.ts b/test/release-gate-workflow.test.ts index e881221c9..9f4e5402e 100644 --- a/test/release-gate-workflow.test.ts +++ b/test/release-gate-workflow.test.ts @@ -55,7 +55,8 @@ const TRAIN = { gate_workflow: 'evals.yml', record_artifact: 'gstack-train-recor describe('release-gate.yml', () => { // Value: protects=the schedule fires every RELEASE_GATE_SCHEDULE_HOURS (the cadence floor the policy validates), one gate at a time, only from the - // default branch, with contents and actions write but no checks write, and runs the bin on the run's own sha with its outputs wired; the policy step + // default branch, with contents and actions write but no checks write, and runs the bin without --sha (main's head from the remote, so an open gate on an older commit + // is finished first) with its outputs wired; the policy step // uses git and jq only and is inert without a release_gate block; fails_when=the cron and the cadence drift apart, two gates overlap, the job that // runs repository code can write check runs, or an inert policy reaches bun; why_new=new workflow; seam=none; no_claim=GitHub's cron jitter test('schedule, concurrency, permissions and the gate step', () => { @@ -76,7 +77,8 @@ describe('release-gate.yml', () => { expect(runStep(policy, { release_gate: GATE })).toEqual({ release: 'true' }); const gate = job.steps.find(s => s.id === 'gate')!; expect(gate.if).toBe("steps.policy.outputs.release == 'true'"); - expect(gate.run).toContain('./bin/gstack-release-gate run --repo "$GITHUB_REPOSITORY" --sha "$GITHUB_SHA"'); + expect(gate.run).toContain('./bin/gstack-release-gate run --repo "$GITHUB_REPOSITORY" --budget-minutes'); + expect(gate.run).not.toContain('--sha'); expect(gate.run).toContain('--outputs "$GITHUB_OUTPUT"'); expect(job.outputs).toEqual({ action: '${{ steps.gate.outputs.action }}', sha: '${{ steps.gate.outputs.sha }}', check: '${{ steps.gate.outputs.check }}' }); }); diff --git a/test/release-gate.test.ts b/test/release-gate.test.ts index cfee00ce7..03c3103d1 100644 --- a/test/release-gate.test.ts +++ b/test/release-gate.test.ts @@ -13,7 +13,7 @@ import * as os from 'node:os'; import * as path from 'node:path'; import { bunIsolationArgs } from '../lib/bun-isolation'; import { - MAX_RELEASE_ATTEMPTS, activeHold, bisectGate, classifyRun, gateStatus, loadReleaseGate, narrow, parseReport, planPrefixes, releaseRecords, runGate, + MAX_RELEASE_ATTEMPTS, OPEN_GATE_CADENCES, activeHold, bisectGate, classifyRun, gateStatus, loadReleaseGate, narrow, parseReport, planPrefixes, releaseRecords, runGate, type GateEnv, type Run, } from '../lib/release-gate'; import { RELEASE_GATE_SCHEDULE_HOURS, releaseGateErrors, type ReleaseGatePolicy } from '../lib/release-gate-policy'; @@ -202,6 +202,30 @@ describe('release gate: run', () => { expect(fake.dispatches).toHaveLength(2); expect(fake.dispatches[1]!.inputs.expected_sha).toBe(next); }); + + // Value: protects=an open gate is finished even after release-cut moved stable onto its commit, and an open gate older than OPEN_GATE_CADENCES + // cadences stops holding up main's head; fails_when=an open record on stable's commit skips forever, or an unresolvable run blocks every later gate; + // why_new=review findings (state-machines and red-team, l1-p2); seam=fake gh; no_claim=a real deleted run + test('an open gate on the commit stable moved to is still finished; a stale open gate no longer blocks the head', () => { + const f = fixture(); + let dispatched = 0; + const fake = fakeGh(() => (dispatched++ === 0 ? { conclusion: 'success', status: 'in_progress', artifacts: report('GREEN') } : green())); + expect(runGate({ ...envFor(f, fake.gh), deadline: 0 }, { gate: GATE, label: 'test', dryRun: false, ignoreCadence: true }).action).toBe('failed'); + git(f.seed, 'push', '-q', '-f', 'origin', f.c[4] + ':refs/heads/stable'); + f.commit('c6'); + fake.runs[0]!.status = 'completed'; + const finished = runGate(envFor(f, fake.gh), { gate: GATE, label: 'test', dryRun: false, ignoreCadence: true }); + expect(finished.action, finished.lines.join('\n')).toBe('gated'); + expect(finished.sha).toBe(f.c[4]); + + const g = fixture(); + const fake2 = fakeGh(() => ({ conclusion: 'success', status: 'in_progress', artifacts: report('GREEN') })); + expect(runGate({ ...envFor(g, fake2.gh), deadline: 0 }, { gate: GATE, label: 'test', dryRun: false, ignoreCadence: true }).action).toBe('failed'); + const head = g.commit('c6'); + const late = runGate({ ...envFor(g, fake2.gh, OPEN_GATE_CADENCES * GATE.cadence_hours + 1), deadline: 0 }, { gate: GATE, label: 'test', dryRun: false, ignoreCadence: true }); + expect(late.lines.some(l => l.includes('finishing the open gate'))).toBe(false); + expect(fake2.dispatches.at(-1)!.inputs.expected_sha).toBe(head); + }); }); describe('release gate: red, bisect and the hold', () => { From 98b60a1e1aa8530922a3a9a6ab4cadf9f0895a75 Mon Sep 17 00:00:00 2001 From: garrytan Date: Tue, 3 Nov 2026 03:59:51 +0000 Subject: [PATCH 06/15] fix: Step 11 review fixes (no paid dispatch without the record on origin; held or failed leaves the gate job green) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- .github/workflows/release-gate.yml | 7 ++++++- lib/release-gate.ts | 8 +++++--- test/release-gate-workflow.test.ts | 5 ++++- test/release-gate.test.ts | 14 ++++++++++++++ 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml index 688fb1e72..c696e2793 100644 --- a/.github/workflows/release-gate.yml +++ b/.github/workflows/release-gate.yml @@ -84,9 +84,14 @@ jobs: GSTACK_STATE_ROOT: ${{ runner.temp }}/gstack-state DRY_RUN: ${{ inputs.dry_run && '1' || '' }} IGNORE_CADENCE: ${{ inputs.ignore_cadence && '1' || '' }} + # held and failed are verdicts the release-check job publishes on the gated commit. Failing this job would + # leave a failed check on main's head, which release-cut reads as red even after that head gates green, so + # only a run that recorded no action (a crash) fails it. run: | + rc=0 ./bin/gstack-release-gate run --repo "$GITHUB_REPOSITORY" --budget-minutes 330 \ - --summary "$GITHUB_STEP_SUMMARY" --outputs "$GITHUB_OUTPUT" ${DRY_RUN:+--dry-run} ${IGNORE_CADENCE:+--ignore-cadence} + --summary "$GITHUB_STEP_SUMMARY" --outputs "$GITHUB_OUTPUT" ${DRY_RUN:+--dry-run} ${IGNORE_CADENCE:+--ignore-cadence} || rc=$? + if [ "$rc" != 0 ] && ! grep -qE '^action=(held|failed)$' "$GITHUB_OUTPUT"; then exit "$rc"; fi # The gated commit's check run (release_gate.check_name, main-backstop), the # evidence scripts/release-cut.ts reads. Its own job with checks: write and diff --git a/lib/release-gate.ts b/lib/release-gate.ts index e424d2c46..0829d06bf 100644 --- a/lib/release-gate.ts +++ b/lib/release-gate.ts @@ -194,10 +194,12 @@ function follow(env: GateEnv, workflow: string, nonce: string, known: number | n const runRef = (env: GateEnv, run: Run) => ({ id: run.id, url: run.html_url ?? `https://github.com/${env.repo}/actions/runs/${run.id}` }); -function save(env: GateEnv, rec: ReleaseRecord, lines: string[]): void { +/** Writes the record and pushes it to origin's notes; false when the push failed, so no paid dispatch may follow. */ +function save(env: GateEnv, rec: ReleaseRecord, lines: string[]): boolean { rec.updated_at = nowIso(); const p = persist(env.repoRoot, [rec], { remote: env.remote }); if (!p.pushed) lines.push(`RELEASE: record ${p.detail}`); + return p.pushed; } interface Context { gate: ReleaseGatePolicy; repoKey: string; stable: string | null; records: ReleaseRecord[]; isAncestor: (a: string, b: string) => boolean } @@ -264,7 +266,7 @@ export function runGate(env: GateEnv, o: RunOptions): GateOut { red_cases: [], flaky: [], cost_usd: null, detail: 'dispatching', bisect: null, created_at: now, updated_at: now, }; if (rec !== prior) { - save(env, rec, lines); + if (!save(env, rec, lines)) { lines.push(`RELEASE: incomplete sha=${sha} run=- (the record did not reach ${env.remote}; nothing dispatched, so no run repeats a paid gate)`); return out('failed', sha); } const failed = dispatch(env, gate.gate_workflow, gate.branch, { [DISPATCH_INPUTS.evalsAll]: 'true', [DISPATCH_INPUTS.expectedSha]: sha, [DISPATCH_INPUTS.nonce]: rec.nonce }); if (failed) { Object.assign(rec, { verdict: 'INCOMPLETE', detail: `the gate dispatch failed (${failed})` }); save(env, rec, lines); lines.push(`RELEASE: incomplete sha=${sha} run=- (${rec.detail})`); return out('failed', sha); } lines.push(`RELEASE: dispatched ${gate.gate_workflow} on ${gate.branch}@${short(sha)} (nonce ${rec.nonce})`); @@ -359,7 +361,7 @@ export function bisectRecord(env: GateEnv, c: Context, rec: ReleaseRecord, o: { const ref = `${RELEASE_BISECT_REF}${nonce.slice(-6)}-${b.bad.slice(0, 12)}`; const round: ReleaseBisectRound = { nonce, ref, run: null, prefixes: prefixes.map(sha => ({ sha, results: {} })) }; b.rounds.push(round); - save(env, rec, lines); + if (!save(env, rec, lines)) { b.rounds.pop(); lines.push('RELEASE: bisect round not dispatched: the record did not reach ' + env.remote); return; } const pushed = env.git(['push', '-q', env.remote, `${b.bad}:refs/heads/${ref}`]); const failed = pushed.ok ? dispatch(env, c.gate.gate_workflow, ref, { [DISPATCH_INPUTS.cases]: b.cases.join(','), [DISPATCH_INPUTS.prefixShas]: prefixes.join(','), [DISPATCH_INPUTS.expectedSha]: b.bad, [DISPATCH_INPUTS.nonce]: nonce, diff --git a/test/release-gate-workflow.test.ts b/test/release-gate-workflow.test.ts index 9f4e5402e..cc94e9f62 100644 --- a/test/release-gate-workflow.test.ts +++ b/test/release-gate-workflow.test.ts @@ -56,7 +56,8 @@ const TRAIN = { gate_workflow: 'evals.yml', record_artifact: 'gstack-train-recor describe('release-gate.yml', () => { // Value: protects=the schedule fires every RELEASE_GATE_SCHEDULE_HOURS (the cadence floor the policy validates), one gate at a time, only from the // default branch, with contents and actions write but no checks write, and runs the bin without --sha (main's head from the remote, so an open gate on an older commit - // is finished first) with its outputs wired; the policy step + // is finished first) with its outputs wired, and a held or failed verdict leaves the job green (only a crash fails it, so main's head never carries a + // failed native check release-cut would read as red); the policy step // uses git and jq only and is inert without a release_gate block; fails_when=the cron and the cadence drift apart, two gates overlap, the job that // runs repository code can write check runs, or an inert policy reaches bun; why_new=new workflow; seam=none; no_claim=GitHub's cron jitter test('schedule, concurrency, permissions and the gate step', () => { @@ -79,6 +80,8 @@ describe('release-gate.yml', () => { expect(gate.if).toBe("steps.policy.outputs.release == 'true'"); expect(gate.run).toContain('./bin/gstack-release-gate run --repo "$GITHUB_REPOSITORY" --budget-minutes'); expect(gate.run).not.toContain('--sha'); + expect(gate.run).toContain("|| rc=$?"); + expect(gate.run).toContain(`if [ "$rc" != 0 ] && ! grep -qE '^action=(held|failed)$' "$GITHUB_OUTPUT"; then exit "$rc"; fi`); expect(gate.run).toContain('--outputs "$GITHUB_OUTPUT"'); expect(job.outputs).toEqual({ action: '${{ steps.gate.outputs.action }}', sha: '${{ steps.gate.outputs.sha }}', check: '${{ steps.gate.outputs.check }}' }); }); diff --git a/test/release-gate.test.ts b/test/release-gate.test.ts index 03c3103d1..f51287734 100644 --- a/test/release-gate.test.ts +++ b/test/release-gate.test.ts @@ -203,6 +203,20 @@ describe('release gate: run', () => { expect(fake.dispatches[1]!.inputs.expected_sha).toBe(next); }); + // Value: protects=no paid dispatch follows a record that did not reach origin's notes (a fresh checkout reads only origin, so the next run would + // dispatch the same $37 gate again); fails_when=a notes push failure repeats the full gate every cadence; why_new=review finding (targeted money, + // l1-p2); seam=fake gh and a pre-receive hook on the bare origin; no_claim=a real ruleset + test('a record that cannot reach origin dispatches nothing', () => { + const f = fixture(); + const hook = path.join(f.origin, 'hooks', 'pre-receive'); + fs.writeFileSync(hook, '#!/bin/sh\nwhile read old new ref; do case "$ref" in refs/notes/*) exit 1;; esac; done\nexit 0\n', { mode: 0o755 }); + const fake = fakeGh(() => green()); + const out = runGate(envFor(f, fake.gh), { gate: GATE, label: 'test', dryRun: false, ignoreCadence: true }); + expect(out.action).toBe('failed'); + expect(out.lines.at(-1)).toContain('nothing dispatched'); + expect(fake.dispatches).toHaveLength(0); + }); + // Value: protects=an open gate is finished even after release-cut moved stable onto its commit, and an open gate older than OPEN_GATE_CADENCES // cadences stops holding up main's head; fails_when=an open record on stable's commit skips forever, or an unresolvable run blocks every later gate; // why_new=review findings (state-machines and red-team, l1-p2); seam=fake gh; no_claim=a real deleted run From ac106dda064c1cc78061f17dd79d7874f067cf52 Mon Sep 17 00:00:00 2001 From: garrytan Date: Tue, 3 Nov 2026 04:07:28 +0000 Subject: [PATCH 07/15] fix(release-gate): re-pin the key to 1.95.0.0 above main; move assertInsideRoot to lib/path-containment.ts to keep ship-policy.ts under 800 lines Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- changelog.d/gsta258-l1-release-gate.md | 2 +- docs/ship-policy.md | 3 ++- lib/path-containment.ts | 23 +++++++++++++++++++++++ lib/ship-policy.ts | 22 ++-------------------- 4 files changed, 28 insertions(+), 22 deletions(-) create mode 100644 lib/path-containment.ts diff --git a/changelog.d/gsta258-l1-release-gate.md b/changelog.d/gsta258-l1-release-gate.md index 91b2d6a63..19c22ce53 100644 --- a/changelog.d/gsta258-l1-release-gate.md +++ b/changelog.d/gsta258-l1-release-gate.md @@ -4,7 +4,7 @@ bump: minor **A release gate can now gate main every 8 hours instead of on every push, and it feeds the stable release.** It is off until main's `.gstack/ship-policy.json` declares a `release_gate` block, so this release changes nothing on its own. With the block, `.github/workflows/release-gate.yml` runs `gstack-release-gate run` every 8 hours. When main moved since the last gate, it runs the full paid gate on main's head once, records the verdict as a `release` note in `refs/notes/gstack-trains`, and publishes the `main-backstop` check run on that commit. A red gate bisects its red cases within `bisect_spend_cap_usd` and names the first bad commit. The per-push main backstop stands down while the block is set, so main is never gated twice. `scripts/release-cut.ts` is still the only thing that moves `stable`, and its 72-hour floor and exceptions are unchanged. It now reads a green release record as proof the commit was gated, and it refuses every cut while the newest release gate is red with no green gate on a later commit. That hold is the freeze. See docs/ship-policy.md#release-gate. ### Added -- **`release_gate` policy key** (`lib/release-gate-policy.ts`, key version 1.94.0.0): `cadence_hours` (8 to 168), `gate_workflow`, `branch`, `stable_ref`, `check_name`, `on_red: hold-release` and `bisect_spend_cap_usd`, every field required. +- **`release_gate` policy key** (`lib/release-gate-policy.ts`, which needs `min_gstack_version` at this release): `cadence_hours` (8 to 168), `gate_workflow`, `branch`, `stable_ref`, `check_name`, `on_red: hold-release` and `bisect_spend_cap_usd`, every field required. - **`bin/gstack-release-gate`** (`lib/release-gate.ts`): `run [--dry-run] [--sha] [--ignore-cadence]`, `status` and `bisect`. `--policy ` reads the block from a file, for a dry run or a proof before adoption. Exit 0 means gated, skipped, inert or a dry run; 1 means held, failed or a refused policy; 2 is a usage error. - **`.github/workflows/release-gate.yml`**: an 8-hour schedule and a manual dispatch. The check run comes from a separate job that holds `checks: write` and runs no repository code. - **Capy skill installer follows the update channel** (`capy/skill/gstack/scripts/install.sh`, wrapper revision 4). With `GSTACK_UPDATE_CHANNEL=stable` or a stable `update_channel`, a fresh clone or a floor upgrade goes to `stable` when it meets the support floor, and to main otherwise. `--revision` still wins. diff --git a/docs/ship-policy.md b/docs/ship-policy.md index 6af7b8393..93da97c4f 100644 --- a/docs/ship-policy.md +++ b/docs/ship-policy.md @@ -1130,7 +1130,8 @@ feeds the stable release (docs/releases.md) and never writes `stable` itself: security and regression exceptions unchanged. The surface is off unless main's `.gstack/ship-policy.json` declares the -block (key version 1.94.0.0, so `min_gstack_version` must be at least that): +block, with `min_gstack_version` at least the release that introduced the key +(validation names it): ```json "release_gate": { diff --git a/lib/path-containment.ts b/lib/path-containment.ts new file mode 100644 index 000000000..a45a03bb9 --- /dev/null +++ b/lib/path-containment.ts @@ -0,0 +1,23 @@ +/** Containment of a repository-relative write path (moved from lib/ship-policy.ts, which re-exports it, to keep that module under its size ratchet). */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +/** + * A working-tree path a stamp may write: inside `root` after realpath (a + * symlink out of the repository is refused even when the glob matched). + */ +export function assertInsideRoot(root: string, rel: string): string | null { + const realRoot = fs.realpathSync(root); + const abs = path.resolve(realRoot, rel); + if (abs !== realRoot && !abs.startsWith(realRoot + path.sep)) return `${rel} resolves outside ${root}`; + let probe = abs; + while (!fs.existsSync(probe)) { + const parent = path.dirname(probe); + if (parent === probe) break; + probe = parent; + } + let real: string; + try { real = fs.realpathSync(probe); } catch { return null; } + if (real !== realRoot && !real.startsWith(realRoot + path.sep)) return `${rel} resolves through a symlink to outside ${root}`; + return null; +} diff --git a/lib/ship-policy.ts b/lib/ship-policy.ts index aad929f3b..374a23242 100644 --- a/lib/ship-policy.ts +++ b/lib/ship-policy.ts @@ -242,7 +242,7 @@ export const BASE_RED_KEY_VERSION = '1.91.374.0'; /** `train.gates` and `train.land` are values of the older train key, so compatErrors checks them by value, as for bump: never. */ export const TRAIN_GATES_KEY_VERSION = '1.92.0.0'; export const REVIEW_EXCLUDE_KEY_VERSION = '1.93.33.0'; -export const RELEASE_GATE_KEY_VERSION = '1.94.0.0'; +export const RELEASE_GATE_KEY_VERSION = '1.95.0.0'; export const POLICY_KEY_INTRODUCED: Record = { bump: '1.91.73.0', queue_mode: '1.91.73.0', release_tool: '1.91.73.0', release_outputs: '1.91.73.0', stamp_paths: '1.91.73.0', changelog: '1.91.73.0', mirror: '1.91.73.0', history: '1.91.73.0', preregistration_shas: '1.91.73.0', stamp_sensitive_gates: '1.91.73.0', @@ -786,22 +786,4 @@ export function stampFilesInTree(repoRoot: string, treeish: string, globs: reado return listTreeFiles(repoRoot, treeish).filter(e => e.mode !== '120000' && matchesAny(globs, e.path)).map(e => e.path); } -/** - * A working-tree path a stamp may write: inside `root` after realpath (a - * symlink out of the repository is refused even when the glob matched). - */ -export function assertInsideRoot(root: string, rel: string): string | null { - const realRoot = fs.realpathSync(root); - const abs = path.resolve(realRoot, rel); - if (abs !== realRoot && !abs.startsWith(realRoot + path.sep)) return `${rel} resolves outside ${root}`; - let probe = abs; - while (!fs.existsSync(probe)) { - const parent = path.dirname(probe); - if (parent === probe) break; - probe = parent; - } - let real: string; - try { real = fs.realpathSync(probe); } catch { return null; } - if (real !== realRoot && !real.startsWith(realRoot + path.sep)) return `${rel} resolves through a symlink to outside ${root}`; - return null; -} +export { assertInsideRoot } from './path-containment'; From 37af351ee6f15fc1a1df56197e2b5aebbd146fbd Mon Sep 17 00:00:00 2001 From: garrytan Date: Tue, 3 Nov 2026 04:09:39 +0000 Subject: [PATCH 08/15] test(release-gate): type the narrow fixture rows (test typecheck ratchet) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- test/release-gate.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/release-gate.test.ts b/test/release-gate.test.ts index f51287734..1a39323b1 100644 --- a/test/release-gate.test.ts +++ b/test/release-gate.test.ts @@ -18,7 +18,7 @@ import { } from '../lib/release-gate'; import { RELEASE_GATE_SCHEDULE_HOURS, releaseGateErrors, type ReleaseGatePolicy } from '../lib/release-gate-policy'; import { RELEASE_GATE_KEY_VERSION, validatePolicy } from '../lib/ship-policy'; -import { NOTES_REF, repoIdentity, type ReleaseRecord } from '../lib/ship-queue-train-record'; +import { NOTES_REF, repoIdentity, type PrefixResult, type ReleaseRecord } from '../lib/ship-queue-train-record'; import { gitIn } from '../scripts/train-gate'; const ROOT = path.resolve(import.meta.dir, '..'); @@ -325,7 +325,7 @@ describe('release gate: verdict, range and policy units', () => { expect(picked).toHaveLength(7); expect(new Set(picked).size).toBe(7); expect(picked[0]).not.toBe(shas[0]); - const t = (rs: string[]) => rs.map((r, i) => ({ sha: shas[i]!, results: r === '?' ? {} : { a: r === 'R' ? 'red' as const : 'green' as const } })); + const t = (rs: string[]) => rs.map((r, i): { sha: string; results: Record } => ({ sha: shas[i]!, results: r === '?' ? {} : { a: r === 'R' ? 'red' : 'green' } })); expect(narrow({ good: 'g', bad: 'b' }, t(['G', 'G', 'R']), ['a'])).toEqual({ good: shas[1], bad: shas[2] }); expect(narrow({ good: 'g', bad: 'b' }, t(['G', 'G']), ['a'])).toEqual({ good: shas[1], bad: 'b' }); expect(narrow({ good: 'g', bad: 'b' }, t(['R', 'R']), ['a'])).toEqual({ good: 'g', bad: shas[0] }); From 82f91f853708a0bca8f996b1c88ad0badf2f17e8 Mon Sep 17 00:00:00 2001 From: garrytan Date: Tue, 3 Nov 2026 04:44:19 +0000 Subject: [PATCH 09/15] fix(release-gate): charge the trial reservation for a null or missing bisect prefix cost so the cap holds across rounds Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- lib/release-gate.ts | 14 +++++++++----- test/release-gate.test.ts | 28 ++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/lib/release-gate.ts b/lib/release-gate.ts index 0829d06bf..af586a1df 100644 --- a/lib/release-gate.ts +++ b/lib/release-gate.ts @@ -302,16 +302,20 @@ function finishRed(env: GateEnv, c: Context, rec: ReleaseRecord, lines: string[] return out('held', rec.sha); } -/** Collect a bisect round's prefix records into its results; the cost they report is added to the spend. */ +/** + * Collect a bisect round's prefix records into its results. Each case on each prefix adds its reported cost to the spend, or + * RELEASE_TRIAL_RESERVATION_USD when the cost is null or the prefix record is missing, so the cap holds across rounds. + */ function collectRound(env: GateEnv, b: ReleaseBisect, round: ReleaseBisectRound, run: Run): void { for (const p of round.prefixes) { const text = artifactFile(env, run.id, `${RELEASE_BISECT_ARTIFACT}-bisect-${p.sha.slice(0, 12)}`, 'bisect-prefix.json'); let rec: { kind?: string; nonce?: string; prefix_sha?: string; checked_out_sha?: string; panels?: Array<{ case: string; status: string; red_class: string | null; cost_usd: number | null }> } | null = null; try { rec = text ? JSON.parse(text) : null; } catch { rec = null; } - if (!rec || rec.kind !== 'train-bisect-prefix' || rec.nonce !== round.nonce || rec.prefix_sha !== p.sha || rec.checked_out_sha !== p.sha || !Array.isArray(rec.panels)) continue; - for (const panel of rec.panels) { - if (!b.cases.includes(panel.case)) continue; - b.spent_usd = Math.round((b.spent_usd + (typeof panel.cost_usd === 'number' ? panel.cost_usd : 0)) * 100) / 100; + const valid = rec && rec.kind === 'train-bisect-prefix' && rec.nonce === round.nonce && rec.prefix_sha === p.sha && rec.checked_out_sha === p.sha && Array.isArray(rec.panels); + for (const c of b.cases) { + const panel = valid ? rec!.panels!.find(x => x.case === c) : undefined; + b.spent_usd = Math.round((b.spent_usd + (typeof panel?.cost_usd === 'number' ? panel.cost_usd : RELEASE_TRIAL_RESERVATION_USD)) * 100) / 100; + if (!panel) continue; if (panel.status === 'PASS') p.results[panel.case] = 'green'; else if (panel.status === 'FAIL' && panel.red_class === 'VERDICT') p.results[panel.case] = 'red'; } diff --git a/test/release-gate.test.ts b/test/release-gate.test.ts index 1a39323b1..957adb238 100644 --- a/test/release-gate.test.ts +++ b/test/release-gate.test.ts @@ -306,6 +306,34 @@ describe('release gate: red, bisect and the hold', () => { }); }); +describe('release gate: the bisect cap with unknown costs', () => { + // Value: protects=a bisect whose prefix records report cost_usd null (what scripts/train-gate.ts writes today) books + // RELEASE_TRIAL_RESERVATION_USD per case and prefix, so a later round sees the spend and the bisect stops capped within its cap; fails_when=null + // costs count as $0 and every round reserves the whole cap again (about 4x the cap over MAX_BISECT_ROUNDS); why_new=review finding (targeted + // money, l1 fix-check); seam=fake gh; no_claim=real case prices + test('null prefix costs are charged at the reservation, and the bisect ends capped within its cap', () => { + const f = fixture(); + const order = [f.base, ...f.c, ...[6, 7, 8, 9, 10, 11, 12].map(i => f.commit('c' + i))]; + const fake = fakeGh((inputs) => { + if (!inputs.prefix_shas) return { conclusion: 'failure', artifacts: report('RED', ['case-a rule FAIL VERDICT']) }; + const files: Record> = {}; + for (const sha of inputs.prefix_shas.split(',')) { + const red = order.indexOf(sha) >= 7; + files['gstack-release-record-bisect-' + sha.slice(0, 12)] = { 'bisect-prefix.json': JSON.stringify({ schema: 1, kind: 'train-bisect-prefix', nonce: inputs.nonce, prefix_sha: sha, checked_out_sha: sha, + panels: [{ case: 'case-a', kind: 'rule', status: red ? 'FAIL' : 'PASS', red_class: red ? 'VERDICT' : null, cost_usd: null }] }) }; + } + return { conclusion: 'success', artifacts: files }; + }); + const out = runGate(envFor(f, fake.gh), { gate: { ...GATE, bisect_spend_cap_usd: 4 }, label: 'test', dryRun: false, ignoreCadence: true }); + expect(out.action, out.lines.join('\n')).toBe('held'); + const bisect = records(f)[0]!.bisect!; + expect(bisect).toMatchObject({ state: 'capped', spent_usd: 3 }); + expect(bisect.spent_usd).toBeLessThanOrEqual(4); + expect(fake.dispatches).toHaveLength(2); + expect(fake.dispatches[1]!.inputs.prefix_shas!.split(',')).toHaveLength(2); + }); +}); + describe('release gate: verdict, range and policy units', () => { test('parseReport and classifyRun: verdict reds bisect, other classes keep a red incomplete, success is green, and a contradiction is incomplete', () => { const r = parseReport(report('RED', ['case-a rule FAIL VERDICT', 'judge one judge FAIL', 'case-z rule FAIL INFRA'])['report-verdict-a1']['report.txt']); From 5bf607a0cbf4e7fecc14bce419698224d39759ff Mon Sep 17 00:00:00 2001 From: garrytan Date: Tue, 3 Nov 2026 04:50:21 +0000 Subject: [PATCH 10/15] docs(capy-install): wrapper 4 follows the update channel (Step 14.5 audit) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- docs/capy-install.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/capy-install.md b/docs/capy-install.md index 1aa27b2dc..8c878dd5a 100644 --- a/docs/capy-install.md +++ b/docs/capy-install.md @@ -4,7 +4,7 @@ Three layers, each owned by gstack: | Layer | Where | Job | |---|---|---| -| Bootstrapper | `capy/skill/gstack/scripts/install.sh` (copied once into the owner's Drive skill folder; frozen) | clone when absent; fetch origin's default branch and read the supported minimum from `FETCH_HEAD:capy/support.env`; in install mode fast-forward a clean, unpinned checkout below it; exec the full wrapper | +| Bootstrapper | `capy/skill/gstack/scripts/install.sh` (copied once into the owner's Drive skill folder; frozen) | clone when absent; fetch origin's default branch and read the supported minimum from `FETCH_HEAD:capy/support.env` (from wrapper 4 a fresh clone or the floor fast-forward follows `stable` instead when the update channel resolves to it and `stable` meets that minimum, see [releases.md](releases.md#channels)); in install mode fast-forward a clean, unpinned checkout below it; exec the full wrapper | | Full wrapper | `bin/gstack-capy-session` (moves with the checkout) | map the session start onto the installer: `--check` certifies (and reports a checkout behind origin), otherwise fast-forward a checkout behind origin (`bin/gstack-upgrade --ff-only-no-reinstall`), then install or repair | | Installer | `bin/gstack-capy-install` | Bun, both host profiles, the doctor, the install verdict, the launch line, the trailer | @@ -591,7 +591,10 @@ Wrapper 2 only added optional exports (the trailer's `latest:` note and the `cloned` line). Wrapper 3 answers `--check --for none` itself (a copy at 1 or 2 prints `fail not-installed` for a coordinator machine and still works for every other call), so the minimum is 3: every older Drive copy prints the owner -action until the owner recopies `capy/skill/gstack/`. +action until the owner recopies `capy/skill/gstack/`. Wrapper 4 only adds the +update-channel choice for the clone and the floor fast-forward +(`GSTACK_UPDATE_CHANNEL=stable|main`, else the checkout's channel, else main); +the minimum stays 3, so a copy at 3 keeps working. ## Exit codes From 7432a067657774ad528429a5bdc667cad56a330c Mon Sep 17 00:00:00 2001 From: garrytan Date: Tue, 3 Nov 2026 05:03:21 +0000 Subject: [PATCH 11/15] test(pregate-selection): mark the fixture wrapper bump as a deliberate old literal Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- test/pregate-selection.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pregate-selection.test.ts b/test/pregate-selection.test.ts index 2069493f7..3e4b983b7 100644 --- a/test/pregate-selection.test.ts +++ b/test/pregate-selection.test.ts @@ -253,7 +253,7 @@ describe('literals (pure)', () => { for (const [rel, text] of Object.entries(files)) { fs.mkdirSync(path.dirname(path.join(root, rel)), { recursive: true }); fs.writeFileSync(path.join(root, rel), text); } const all = Object.keys(files); // A one-character value: only the file that names the constant counts; the path fragment alone is not enough for "2". - expect(consumersOf(root, all, { name: 'GSTACK_CAPY_WRAPPER', old: '2', new: '3', file: 'capy/skill/gstack/scripts/install.sh' })).toEqual(['test/wrapper.test.ts:2']); + expect(consumersOf(root, all, { name: 'GSTACK_CAPY_WRAPPER', old: '2', new: '3', file: 'capy/skill/gstack/scripts/install.sh' })).toEqual(['test/wrapper.test.ts:2']); // pregate-literal: GSTACK_CAPY_WRAPPER=3 a fixture's own 2 -> 3 bump, not the shipped wrapper revision // A longer value: the constant's name, the module's full file name, or its stem as a path fragment; the bare word "retry" in prose and "preretry.js" do not count. expect(consumersOf(root, all, { name: 'TIMEOUT_MS', old: '5000', new: '9000', file: 'lib/retry.js' })).toEqual(['test/retry.test.ts:1', 'test/retry-path.test.ts:2', 'test/retry-bare.test.ts:2']); fs.rmSync(root, { recursive: true, force: true }); From 289d7b63b5eae9b17002721d568fdff371f6d161 Mon Sep 17 00:00:00 2001 From: garrytan Date: Tue, 3 Nov 2026 05:17:01 +0000 Subject: [PATCH 12/15] test(pregate-selection): put the literal marker on its own comment line Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- test/pregate-selection.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/pregate-selection.test.ts b/test/pregate-selection.test.ts index 3e4b983b7..bb6918d09 100644 --- a/test/pregate-selection.test.ts +++ b/test/pregate-selection.test.ts @@ -253,7 +253,8 @@ describe('literals (pure)', () => { for (const [rel, text] of Object.entries(files)) { fs.mkdirSync(path.dirname(path.join(root, rel)), { recursive: true }); fs.writeFileSync(path.join(root, rel), text); } const all = Object.keys(files); // A one-character value: only the file that names the constant counts; the path fragment alone is not enough for "2". - expect(consumersOf(root, all, { name: 'GSTACK_CAPY_WRAPPER', old: '2', new: '3', file: 'capy/skill/gstack/scripts/install.sh' })).toEqual(['test/wrapper.test.ts:2']); // pregate-literal: GSTACK_CAPY_WRAPPER=3 a fixture's own 2 -> 3 bump, not the shipped wrapper revision + // pregate-literal: GSTACK_CAPY_WRAPPER=3 a fixture's own 2 -> 3 bump, not the shipped wrapper revision + expect(consumersOf(root, all, { name: 'GSTACK_CAPY_WRAPPER', old: '2', new: '3', file: 'capy/skill/gstack/scripts/install.sh' })).toEqual(['test/wrapper.test.ts:2']); // A longer value: the constant's name, the module's full file name, or its stem as a path fragment; the bare word "retry" in prose and "preretry.js" do not count. expect(consumersOf(root, all, { name: 'TIMEOUT_MS', old: '5000', new: '9000', file: 'lib/retry.js' })).toEqual(['test/retry.test.ts:1', 'test/retry-path.test.ts:2', 'test/retry-bare.test.ts:2']); fs.rmSync(root, { recursive: true, force: true }); From 0fc4c0770e7c2eb9c367883b05003891b6180d43 Mon Sep 17 00:00:00 2001 From: garrytan Date: Tue, 3 Nov 2026 09:10:56 +0000 Subject: [PATCH 13/15] test(release-gate-workflow): run policy steps with Git Bash argument conversion off, as on the Linux runners (windows-scoped-shard) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- test/release-gate-workflow.test.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/release-gate-workflow.test.ts b/test/release-gate-workflow.test.ts index cc94e9f62..cc8ce50e9 100644 --- a/test/release-gate-workflow.test.ts +++ b/test/release-gate-workflow.test.ts @@ -26,7 +26,10 @@ const scratch: string[] = []; afterAll(() => { for (const d of scratch) fs.rmSync(d, { recursive: true, force: true }); }); const GIT_ENV = { GIT_AUTHOR_NAME: 't', GIT_AUTHOR_EMAIL: 't@x', GIT_COMMITTER_NAME: 't', GIT_COMMITTER_EMAIL: 't@x' }; -/** Run a policy step's script in a scratch repo whose one commit holds this policy; returns its GITHUB_OUTPUT lines as an object. */ +/** + * Run a policy step's script in a scratch repo whose one commit holds this policy; returns its GITHUB_OUTPUT lines as an object. + * Git Bash's argument conversion is off, as on the Linux runners these steps run on: it rewrites `origin/main:.gstack/…` into a path list. + */ function runStep(step: Step, policy: unknown): Record { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'release-gate-wf-')); scratch.push(dir); @@ -44,7 +47,7 @@ function runStep(step: Step, policy: unknown): Record { const out = path.join(dir, 'out'); const r = spawnSync('bash', ['-e', '-c', step.run!], { cwd: dir, encoding: 'utf8', timeout: 30_000, - env: { ...process.env, GITHUB_OUTPUT: out, GITHUB_STEP_SUMMARY: path.join(dir, 'summary'), GITHUB_SHA: sha, BASE_SHA: '', DEFAULT_BRANCH: 'main' }, + env: { ...process.env, MSYS_NO_PATHCONV: '1', MSYS2_ARG_CONV_EXCL: '*', GITHUB_OUTPUT: out, GITHUB_STEP_SUMMARY: path.join(dir, 'summary'), GITHUB_SHA: sha, BASE_SHA: '', DEFAULT_BRANCH: 'main' }, }); expect(r.status, r.stderr).toBe(0); const lines = (fs.existsSync(out) ? fs.readFileSync(out, 'utf8') : '').split('\n').filter(Boolean); From d107fc4a1e06cdfce25ca2de992bd587208f818e Mon Sep 17 00:00:00 2001 From: garrytan Date: Tue, 3 Nov 2026 09:26:35 +0000 Subject: [PATCH 14/15] docs: release gate evidence and hold in releases.md, release_gate row in the policy key table (Step 14.5 audit) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- docs/releases.md | 9 ++++++--- docs/ship-policy.md | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/releases.md b/docs/releases.md index 18b473bad..520a22987 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -29,7 +29,8 @@ it has completed and none failed, and `refs/notes/gstack-trains` holds no active freeze. A cancelled check run counts as red. A commit with a check run still going is skipped and the next older one is considered, back through 7 days or 200 first-parent commits. The workflow runs no tests and no paid evals -of its own; it only reads what the trains and the backstop already recorded. +of its own; it only reads what the trains, the backstop and the release gate +already recorded. With nothing green in the window it fails and lists why each commit was skipped. @@ -54,7 +55,8 @@ contains it refuses the cut like a freeze. Train evidence relaxes nothing else: a red backstop run, a failed or missing `free-tests` run, any other red check run or a run still going keeps the commit out, and a record in any other state (`red`, `superseded`, `free-green`, …) is not evidence. A commit with -neither a successful backstop run nor a train record is not released: a +none of the three (no successful backstop run, train record or GREEN release +record) is not released: a dependabot push, a policy without a train block, or a tree that was never gated. @@ -311,11 +313,12 @@ is exercised against real check runs before it merges. | Line | Meaning | |---|---| -| `CANDIDATE: () gated by main-backstop|train ` | the commit that will be (or was) released, and which evidence gated it (`[hotfix: --ref]` before `gated by` on a `--ref` cut) | +| `CANDIDATE: () gated by main-backstop|train |release gate run ` | the commit that will be (or was) released, and which evidence gated it (`[hotfix: --ref]` before `gated by` on a `--ref` cut) | | `NOTHING_NEW: origin/main is already released as v` | nothing merged since the last release; exit 0 | | `NO_GREEN_COMMIT: ...` with `SKIPPED:` lines | nothing in the window passed the gate; exit 1 (a dry run exits 0: the skips are its answer) | | `REFUSED: non-fast-forward: ...` | the candidate is behind stable; stable only moves forward | | `REFUSED: queue frozen since train ...` | the main backstop froze the queue; clear it first (docs/ship-policy.md) | +| `REFUSED: release gate red on () and no gate on a later commit is green; ...` | the release gate's hold: land the revert or fix-forward, then `gstack-release-gate run --ignore-cadence` ([docs/ship-policy.md#release-gate](ship-policy.md#release-gate)) | | `REFUSED: tag v already names ` | a version was stamped twice; needs a look before anything is cut | | `REFUSED_FLOOR: stable moved to v at