feat(release-gate): gate main every 8 hours and feed the stable release (inert until adopted)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed 2026-11-03 02:51:14 +00:00
1 parent 923fc0a6b8
commit 2dcbe9dbbd
20 files changed
+1415 -30

No files matched your search

@@ -12,6 +12,10 @@ name: Main backstop gate
# (success when a green train covers the tree or the backstop gated it green,
# neutral when the push was superseded or deferred, failure when it froze the
# queue or could not finish); the stable release reads it (docs/releases.md).
#
# Once main's policy declares a `release_gate` block, the release gate
# (release-gate.yml, every 8 hours) gates main instead and this backstop stands
# down: no gate, no freeze and no check run on a push.
on:
push:
branches: [main]
@@ -60,9 +64,13 @@ jobs:
done
fi
train=$(jq -c 'if (.train | type) == "object" then .train else null end' <<<"$policy") || { echo "::error::${GITHUB_SHA}:.gstack/ship-policy.json is not valid JSON; fix the policy file" >&2; exit 1; }
release=$(jq -r 'if (.release_gate | type) == "object" then "true" else "false" end' <<<"$policy")
if [ -z "$train" ] || [ "$train" = null ]; then
echo "train=false" >> "$GITHUB_OUTPUT"
echo "- Backstop inert: main's .gstack/ship-policy.json declares no train block; per-PR gates cover main" >> "$GITHUB_STEP_SUMMARY"
elif [ "$release" = true ]; then
echo "train=false" >> "$GITHUB_OUTPUT"
echo "- Backstop stands down: main's .gstack/ship-policy.json declares a release_gate block; the scheduled release gate gates main (release-gate.yml, docs/ship-policy.md#release-gate)" >> "$GITHUB_STEP_SUMMARY"
else
echo "train=true" >> "$GITHUB_OUTPUT"
fi
+15 -1
View File
@@ -166,7 +166,8 @@ jobs:
pull-requests: read
outputs:
mode: ${{ steps.route.outputs.mode }}
record_artifact: ${{ steps.policy.outputs.record_artifact }}
# A release bisect (gstack-release-gate) names its own prefix-record artifact; everything else uses the train block's.
record_artifact: ${{ steps.route.outputs.record_artifact || steps.policy.outputs.record_artifact }}
full_gate_check: ${{ steps.policy.outputs.full_gate_check }}
screen_head: ${{ steps.screen.outputs.head_sha }}
screen_base: ${{ steps.screen.outputs.base_sha }}
@@ -218,10 +219,23 @@ jobs:
echo "::warning::the train block at ${base} names no valid record_artifact and full_gate_check; this run routes as before"
fi
# The release gate's bisect (gstack-release-gate) routes only under a
# release_gate block at the same base revision; git and jq only.
- name: Read the release gate policy (base revision)
id: release_policy
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
run: |
policy=$(git show "${BASE_SHA:-origin/${DEFAULT_BRANCH}}:.gstack/ship-policy.json" 2>/dev/null || echo null)
release=$(jq -r 'if (.release_gate | type) == "object" then "true" else "false" end' <<<"$policy" 2>/dev/null || echo false)
echo "release=${release}" >> "$GITHUB_OUTPUT"
- name: Route this run
id: route
env:
TRAIN_POLICY: ${{ steps.policy.outputs.train }}
RELEASE_POLICY: ${{ steps.release_policy.outputs.release }}
NONCE: ${{ inputs.nonce }}
EXPECTED_SHA: ${{ inputs.expected_sha }}
EVALS_ALL: ${{ inputs.evals_all }}
+118
View File
@@ -0,0 +1,118 @@
name: Release gate
# The release gate (docs/ship-policy.md#release-gate; bin/gstack-release-gate,
# lib/release-gate.ts): the full paid gate on main's head every 8 hours, only
# when main moved since the last gate. It records the verdict as a release
# note in refs/notes/gstack-trains, publishes the main-backstop check run on
# the gated commit (what scripts/release-cut.ts reads as gated), and bisects
# a red gate's cases. It never writes the stable branch: release-cut is its
# only writer, and refuses every cut while the newest gate is red. Inert (no
# paid work, nothing written) unless main's .gstack/ship-policy.json declares
# a release_gate block; with one, the per-push main backstop
# (evals-main-backstop.yml) stands down and this replaces it.
on:
schedule:
# Every 8 hours (RELEASE_GATE_SCHEDULE_HOURS in lib/release-gate-policy.ts;
# test/release-gate-workflow.test.ts pins the two together), off the hour.
- cron: '17 */8 * * *'
workflow_dispatch:
inputs:
dry_run:
description: 'Print what the gate would test and dispatch nothing'
type: boolean
default: false
ignore_cadence:
description: 'Gate now even if the last gate is younger than cadence_hours (main must still have moved)'
type: boolean
default: true
# One gate at a time; a second run waits for the first.
concurrency:
group: release-gate
cancel-in-progress: false
permissions: {}
jobs:
release-gate:
runs-on: ubuntu-24.04
# A dispatch gates only from the default branch, with its tooling.
if: github.ref_name == github.event.repository.default_branch
# The full gate (270) + a bisect round + margin; past --budget-minutes the
# record stays resumable by the next run.
timeout-minutes: 345
outputs:
action: ${{ steps.gate.outputs.action }}
sha: ${{ steps.gate.outputs.sha }}
check: ${{ steps.gate.outputs.check }}
permissions:
# Pushes the release record to refs/notes/gstack-trains and a bisect's
# release-gate/<id> branch (deleted after its round).
contents: write
# Dispatches the gate and reads its runs and artifacts.
actions: write
checks: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
fetch-depth: 0
# Read with git and jq only: without a release_gate block nothing below runs.
- name: Read the release gate policy (main)
id: policy
run: |
file=.gstack/ship-policy.json
policy=$(git show "${GITHUB_SHA}:${file}" 2>/dev/null || echo null)
gate=$(jq -c 'if (.release_gate | type) == "object" then .release_gate else null end' <<<"$policy") || { echo "::error::${GITHUB_SHA}:${file} is not valid JSON; fix the policy file" >&2; exit 1; }
if [ -z "$gate" ] || [ "$gate" = null ]; then
echo "release=false" >> "$GITHUB_OUTPUT"
echo "- Release gate inert: main's ${file} declares no release_gate block; the per-push backstop gates main" >> "$GITHUB_STEP_SUMMARY"
else
echo "release=true" >> "$GITHUB_OUTPUT"
fi
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
if: steps.policy.outputs.release == 'true'
with:
bun-version: 1.4.2
- name: Gate main's head when it moved
id: gate
if: steps.policy.outputs.release == 'true'
env:
GH_TOKEN: ${{ github.token }}
GSTACK_STATE_ROOT: ${{ runner.temp }}/gstack-state
DRY_RUN: ${{ inputs.dry_run && '1' || '' }}
IGNORE_CADENCE: ${{ inputs.ignore_cadence && '1' || '' }}
run: |
./bin/gstack-release-gate run --repo "$GITHUB_REPOSITORY" --sha "$GITHUB_SHA" --budget-minutes 330 \
--summary "$GITHUB_STEP_SUMMARY" --outputs "$GITHUB_OUTPUT" ${DRY_RUN:+--dry-run} ${IGNORE_CADENCE:+--ignore-cadence}
# The gated commit's check run (release_gate.check_name, main-backstop), the
# evidence scripts/release-cut.ts reads. Its own job with checks: write and
# no repository code, as for backstop-check: only the recorded action word.
release-check:
runs-on: ubuntu-24.04
needs: release-gate
if: ${{ !cancelled() && contains(fromJSON('["gated","held","failed"]'), needs.release-gate.outputs.action) && needs.release-gate.outputs.sha != '' }}
timeout-minutes: 5
permissions:
checks: write
steps:
- name: Publish the release gate's check run on the gated commit
env:
GH_TOKEN: ${{ github.token }}
ACTION: ${{ needs.release-gate.outputs.action }}
SHA: ${{ needs.release-gate.outputs.sha }}
CHECK: ${{ needs.release-gate.outputs.check }}
run: |
case "$ACTION" in
gated) conclusion=success ;;
held|failed) conclusion=failure ;;
*) echo "::error::unknown release gate action '$ACTION'"; exit 1 ;;
esac
[[ "$SHA" =~ ^[0-9a-f]{40}$ ]] || { echo "::error::'$SHA' is not a commit SHA"; exit 1; }
[[ "$CHECK" =~ ^[A-Za-z0-9_./-]{1,64}$ ]] || { echo "::error::'$CHECK' is not a check-run name"; exit 1; }
gh api "repos/${GITHUB_REPOSITORY}/check-runs" -X POST -f name="$CHECK" -f head_sha="$SHA" -f status=completed -f conclusion="$conclusion" \
-f details_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
-f "output[title]=Release gate: ${ACTION}" -f "output[summary]=${ACTION}: see the release-gate job summary of run ${GITHUB_RUN_ID}" >/dev/null
echo "- ${CHECK}: ${conclusion} (release gate ${ACTION}) on ${SHA}" >> "$GITHUB_STEP_SUMMARY"
+62
View File
@@ -0,0 +1,62 @@
#!/usr/bin/env -S bun --no-env-file --config=/dev/null
/**
* gstack-release-gate — the full paid gate on main every cadence_hours, only
* when main moved since the last gate (docs/ship-policy.md#release-gate;
* logic: lib/release-gate.ts). The gate never moves the stable branch:
* scripts/release-cut.ts is its only writer and holds every cut while the
* newest gate is red.
*
* gstack-release-gate run [--dry-run] [--sha <main sha>] [--ignore-cadence]
* gstack-release-gate status
* gstack-release-gate bisect [--sha <red sha>] [--spend-cap <usd>] [--dry-run]
*
* Every subcommand takes --repo <owner/name> (default $GITHUB_REPOSITORY, else
* gh repo view), --policy <file> (a whole ship policy, validated; default the
* reviewed .gstack/ship-policy.json on origin/main), --remote <name> (origin),
* --summary <file> (appends each line as a bullet) and --outputs <file>
* (appends action=<action> and sha=<sha> for the workflow). run and bisect
* take --poll-seconds (60) and --budget-minutes (330): past the budget nothing
* waits, and the record stays resumable by the next run or bisect.
* Exit 0: inert, skipped, dry run, gated green, or status with no hold;
* 1: red (held), incomplete or could not finish; 2: usage.
*/
import { randomBytes } from 'node:crypto';
import * as fs from 'node:fs';
import { bisectGate, gateStatus, loadReleaseGate, runGate, type GateEnv, type GateOut } from '../lib/release-gate';
import { runGh } from '../scripts/recover-receipts';
import { gitIn } from '../scripts/train-gate';
const USAGE = 'usage: gstack-release-gate run [--dry-run] [--sha <sha>] [--ignore-cadence] | status | bisect [--sha <sha>] [--spend-cap <usd>] [--dry-run]\n'
+ ' common: [--repo <owner/name>] [--policy <file>] [--remote <name>] [--summary <file>] [--outputs <file>] [--poll-seconds <n>] [--budget-minutes <n>]';
const [command, ...rest] = process.argv.slice(2);
const flag = (name: string) => { const i = rest.indexOf(`--${name}`); return i >= 0 ? rest[i + 1] : undefined; };
const has = (name: string) => rest.includes(`--${name}`);
const usage = (why?: string): never => { console.error(why ? `${why}\n${USAGE}` : USAGE); process.exit(2); };
if (command === '--help' || command === '-h') { console.log(USAGE); process.exit(0); }
if (!['run', 'status', 'bisect'].includes(command ?? '')) usage();
const sha = flag('sha');
if (sha !== undefined && !/^[0-9a-f]{40}$/.test(sha)) usage('--sha needs a full 40-hex commit');
const spendCap = flag('spend-cap') === undefined ? undefined : Number(flag('spend-cap'));
if (spendCap !== undefined && !(spendCap >= 0)) usage('--spend-cap needs dollars');
const repoRoot = process.cwd();
const repo = flag('repo') ?? process.env.GITHUB_REPOSITORY ?? runGh(['repo', 'view', '--json', 'nameWithOwner', '-q', '.nameWithOwner'], 60_000).stdout.trim();
if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repo)) usage(`cannot name the repository (${JSON.stringify(repo)}); pass --repo <owner/name>`);
const loaded = loadReleaseGate(repoRoot, { policyFile: flag('policy') });
const env: GateEnv = {
repo, repoRoot, remote: flag('remote') ?? 'origin', gh: runGh, git: gitIn(repoRoot), now: Date.now, sleep: ms => Bun.sleepSync(ms),
random: () => randomBytes(3).toString('hex'), pollMs: Number(flag('poll-seconds') ?? 60) * 1000,
deadline: Date.now() + Number(flag('budget-minutes') ?? 330) * 60_000,
};
let out: GateOut;
if ('error' in loaded) out = { action: 'failed', sha: null, lines: [`RELEASE: policy refused: ${loaded.error}`] };
else if (command === 'run') out = runGate(env, { ...loaded, sha, dryRun: has('dry-run'), ignoreCadence: has('ignore-cadence') });
else if (command === 'bisect') out = bisectGate(env, { ...loaded, sha, spendCap, dryRun: has('dry-run') });
else out = gateStatus(env, loaded);
const summary = flag('summary');
for (const line of out.lines) {
console.log(line);
if (summary) fs.appendFileSync(summary, `- ${line}\n`);
}
const outputs = flag('outputs');
if (outputs) fs.appendFileSync(outputs, `action=${out.action}\nsha=${out.sha ?? ''}\ncheck=${'gate' in loaded && loaded.gate ? loaded.gate.check_name : ''}\n`);
process.exit(out.action === 'held' || out.action === 'failed' ? 1 : 0);
+35 -4
View File
@@ -17,6 +17,12 @@
# checkout it prints `gstack: ok not-needed for=none (...)` and exits 0; with
# one it prints the same trailer with the checkout's version, running nothing
# from the checkout. That ok admits no workflow (no verdict is written).
# From revision 4 the clone and the floor fast-forward follow the update
# channel (docs/releases.md#channels): GSTACK_UPDATE_CHANNEL=stable|main, else
# the checkout's bin/gstack-update-channel.sh (auto: stable in team mode),
# else main. On stable, a fresh clone checks out origin's stable branch and
# the floor fast-forward goes to stable, each only when stable exists and
# meets the supported minimum; otherwise main, as before. --revision wins.
# Everything else (Bun, setup, the doctor, the verdict, the launch line) is
# the checkout's. The last line is the `gstack:` trailer; exit 0 only on
# `gstack: ok`. Contract: docs/capy-install.md; tests:
@@ -27,9 +33,10 @@
# bash install.sh --check --for none this machine runs no workflow: ok not-needed, nothing installed
# bash install.sh [--for <skill> | --require <id,id>] [--with-browser] [--revision <ref>] [--project <path>]
# Environment: GSTACK_CAPY_ROOT (default ~/.local/share/gstack), GSTACK_CAPY_REPO,
# GSTACK_CAPY_AUTO_UPGRADE=0 (never fast-forward), GSTACK_CAPY_FETCH_TIMEOUT (s).
# GSTACK_CAPY_AUTO_UPGRADE=0 (never fast-forward), GSTACK_CAPY_FETCH_TIMEOUT (s),
# GSTACK_UPDATE_CHANNEL=main (follow every merge even in team mode) or stable.
set -u
GSTACK_CAPY_WRAPPER=3
GSTACK_CAPY_WRAPPER=4
self="$(cd "$(dirname "$0")" && pwd -P)/$(basename "$0")"
root="${GSTACK_CAPY_ROOT:-$HOME/.local/share/gstack}"
@@ -74,6 +81,24 @@ below() {
return 1
}
support_get() { printf '%s\n' "$1" | sed -n "s/^$2=\([0-9][0-9.]*\)[[:space:]]*\$/\1/p" | head -1; }
# channel_stable ROOT MIN [FETCH_FLAG] — true, with origin's stable commit in $stable_sha, when
# this install follows stable and origin's stable exists and meets MIN; says why not otherwise.
# FETCH_FLAG is --depth=1 for a fresh clone only (never --depth into an existing checkout).
channel_stable() {
local ch="${GSTACK_UPDATE_CHANNEL:-}" ver
case "$ch" in stable|main) ;; *) ch=$(bash "$1/bin/gstack-update-channel.sh" "$1" 2>/dev/null | head -1) ;; esac
[ "$ch" = stable ] || return 1
if ! bounded git -C "$1" fetch --quiet ${3:+"$3"} origin stable 2>/dev/null; then
echo "gstack-capy-skill: channel stable, but origin has no stable branch yet (or the fetch failed); following main"
return 1
fi
stable_sha=$(git -C "$1" rev-parse FETCH_HEAD) ver=$(git -C "$1" show FETCH_HEAD:VERSION 2>/dev/null | head -1)
if [ -n "$2" ] && below "${ver:-0}" "$2"; then
echo "gstack-capy-skill: channel stable is at ${ver:-unknown}, below the supported minimum $2; following main"
return 1
fi
echo "gstack-capy-skill: channel stable ${ver:-unknown} (GSTACK_UPDATE_CHANNEL=main follows every merge)"
}
mkdir -p "$(dirname "$root")"
if [ "$check" -eq 0 ]; then
@@ -100,6 +125,10 @@ if [ ! -f "$root/setup" ]; then
fi
remote_min=$(support_get "$(cat "$root/capy/support.env" 2>/dev/null)" GSTACK_MIN_REVISION)
remote_version=$(head -1 "$root/VERSION" 2>/dev/null)
if [ -z "$revision" ] && channel_stable "$root" "$remote_min" --depth=1; then
git -C "$root" reset --quiet --hard "$stable_sha" \
|| { echo "gstack: fail installer — could not check out origin/stable in $root; fix: GSTACK_UPDATE_CHANNEL=main $fix_cmd"; exit 1; }
fi
export GSTACK_CAPY_CLONED="$repo"
else
# The remote's rules: fetch the default branch into FETCH_HEAD (never --depth
@@ -140,9 +169,11 @@ else
echo "gstack: fail revision — installed $installed is below the supported minimum $remote_min ($cause); fix: $fix (REVISION_BELOW_MINIMUM)"
exit 1
fi
if ! git -C "$root" merge --ff-only -q FETCH_HEAD 2>/dev/null; then
target=$(git -C "$root" rev-parse FETCH_HEAD)
if channel_stable "$root" "$remote_min" && git -C "$root" merge-base --is-ancestor HEAD "$stable_sha" 2>/dev/null; then target=$stable_sha; fi
if ! git -C "$root" merge --ff-only -q "$target" 2>/dev/null; then
git -C "$root" fetch --quiet --unshallow origin "$default" 2>/dev/null
git -C "$root" merge --ff-only -q FETCH_HEAD 2>/dev/null \
git -C "$root" merge --ff-only -q "$target" 2>/dev/null \
|| { echo "gstack: fail revision — installed $installed is below the supported minimum $remote_min (fast-forward failed); fix: git -C $root status, then $fix_cmd (REVISION_BELOW_MINIMUM)"; exit 1; }
fi
echo "gstack-capy-skill: upgraded $installed -> $(head -1 "$root/VERSION")"
+17
View File
@@ -0,0 +1,17 @@
---
bump: minor
---
**A release gate can now gate main every 8 hours instead of on every push, and it feeds the stable release.** It is off until main's `.gstack/ship-policy.json` declares a `release_gate` block, so this release changes nothing on its own. With the block, `.github/workflows/release-gate.yml` runs `gstack-release-gate run` every 8 hours. When main moved since the last gate, it runs the full paid gate on main's head once, records the verdict as a `release` note in `refs/notes/gstack-trains`, and publishes the `main-backstop` check run on that commit. A red gate bisects its red cases within `bisect_spend_cap_usd` and names the first bad commit. The per-push main backstop stands down while the block is set, so main is never gated twice. `scripts/release-cut.ts` is still the only thing that moves `stable`, and its 72-hour floor and exceptions are unchanged. It now reads a green release record as proof the commit was gated, and it refuses every cut while the newest release gate is red with no green gate on a later commit. That hold is the freeze. See docs/ship-policy.md#release-gate.
### Added
- **`release_gate` policy key** (`lib/release-gate-policy.ts`, key version 1.94.0.0): `cadence_hours` (8 to 168), `gate_workflow`, `branch`, `stable_ref`, `check_name`, `on_red: hold-release` and `bisect_spend_cap_usd`, every field required.
- **`bin/gstack-release-gate`** (`lib/release-gate.ts`): `run [--dry-run] [--sha] [--ignore-cadence]`, `status` and `bisect`. `--policy <file>` reads the block from a file, for a dry run or a proof before adoption. Exit 0 means gated, skipped, inert or a dry run; 1 means held; 2 means failed.
- **`.github/workflows/release-gate.yml`**: an 8-hour schedule and a manual dispatch. The check run comes from a separate job that holds `checks: write` and runs no repository code.
- **Capy skill installer follows the update channel** (`capy/skill/gstack/scripts/install.sh`, wrapper revision 4). With `GSTACK_UPDATE_CHANNEL=stable` or a stable `update_channel`, a fresh clone or a floor upgrade goes to `stable` when it meets the support floor, and to main otherwise. `--revision` still wins.
### Changed
- **`evals-main-backstop.yml`** stands down (no gate, no freeze, no check run) when main's policy declares `release_gate`.
- **`evals.yml`** routes a release gate bisect (`release-gate/<id>` refs) only under a `release_gate` block at the base revision, and that bisect publishes its prefix records as `gstack-release-record`.
### Tests
- `test/release-gate.test.ts` drives run, bisect and status against a fake `gh` and a scratch remote. `test/release-gate-workflow.test.ts` runs the workflows' policy steps on scratch repositories and pins the cron to the cadence floor. `test/release-cut.test.ts` proves a red release record refuses the cut and a green one on a later commit releases it. `test/train-gate.test.ts` covers the bisect route, and `test/capy-skill-wrapper.test.ts` covers the stable channel and its fallback to main.
+4 -1
View File
@@ -298,7 +298,10 @@ through an urgent one-PR train (or `train build --repair` while the queue is fro
refuses `--urgent`); all prefixes green earns exactly one unchanged rebuild;
anything else goes to a repair lane. An infra-only census earns one re-dispatch. A
backstop gate runs on every push to the base branch and freezes the queue when it finds
a tree no green train covered; one repair PR at its exact head clears the freeze.
a tree no green train covered; one repair PR at its exact head clears the freeze. A
policy with a `release_gate` block swaps that per-push backstop for one full gate on main
every 8 hours, whose red verdict holds the stable release instead of freezing the queue
([ship-policy.md#release-gate](ship-policy.md#release-gate)).
**Wave PRs when the queue is long.** Every slot pays its own
stamp, its own round of required checks and its own version, one after another. When more
+11 -2
View File
@@ -33,7 +33,7 @@ of its own; it only reads what the trains and the backstop already recorded.
With nothing green in the window it fails and lists why each commit was
skipped.
**Gated** means one of two pieces of evidence. The first is the
**Gated** means one of three pieces of evidence. The first is the
`main-backstop` check run: the Main backstop gate workflow publishes it on
each push to main from a job of its own, success when a green train record
covers the tree or the backstop gated it green, neutral when main moved past
@@ -45,7 +45,12 @@ its tree: those entries are the main commits `gstack-ship-queue verify
--train` confirmed, so a commit a green train merged is gated even when trains
merged faster than the backstop could settle and its own backstop run ended
neutral or never ran. The `CANDIDATE` line says which: `gated by
main-backstop` or `gated by train <nonce>`. Train evidence relaxes nothing
main-backstop` or `gated by train <nonce>`. The third is a GREEN release
record (kind `release`) on the commit, written by the release gate
([docs/ship-policy.md#release-gate](ship-policy.md#release-gate)) when the
policy declares `release_gate`; the line then says `gated by release gate run
<id>`. A RED release record with no GREEN one on a later commit that
contains it refuses the cut like a freeze. Train evidence relaxes nothing
else: a red backstop run, a failed or missing `free-tests` run, any other red
check run or a run still going keeps the commit out, and a record in any other
state (`red`, `superseded`, `free-green`, …) is not evidence. A commit with
@@ -240,6 +245,10 @@ overrides the key for one command. The resolution lives in one helper,
- `bin/gstack-upgrade` and `/gstack-upgrade`: they fast-forward to the channel
ref (`UPSTREAM_REF=origin/stable` in the skill's output) and say which channel
they followed.
- `capy/skill/gstack/scripts/install.sh`, the Capy skill's installer (wrapper
revision 4): a fresh clone or a floor upgrade follows `stable` when the channel
resolves to it and `stable` meets the support floor, and follows main
otherwise. `--revision` still pins an exact commit.
Two edge cases are handled on purpose:
+75
View File
@@ -1118,6 +1118,81 @@ every lane to the per-PR queue at once. `--urgent` ships one PR alone,
`--no-screen` skips the paid screen, `--broad` widens it to the full PR selection,
and `train build --prs <list without it>` drops a PR at any step.
<a id="release-gate"></a>
## `release_gate`: one full gate on main every 8 hours
Trains gate each batch of PRs, and the main backstop gates every push. The
release gate replaces the backstop with one full paid gate on main's head at
most every `cadence_hours`, and only when main moved since the last gate. It
feeds the stable release (docs/releases.md) and never writes `stable` itself:
`scripts/release-cut.ts` stays the only writer, with its 72-hour floor and its
security and regression exceptions unchanged.
The surface is off unless main's `.gstack/ship-policy.json` declares the
block (key version 1.94.0.0, so `min_gstack_version` must be at least that):
```json
"release_gate": {
"cadence_hours": 8, "gate_workflow": "evals.yml", "branch": "main",
"stable_ref": "stable", "check_name": "main-backstop",
"on_red": "hold-release", "bisect_spend_cap_usd": 40
}
```
Every field is required. `cadence_hours` runs from 8 (the workflow's
schedule) to 168. `check_name` is the check run release-cut reads as gated,
so gstack keeps `main-backstop`. `bisect_spend_cap_usd` caps one red gate's
bisect (0 skips it).
**What runs.** `.github/workflows/release-gate.yml` fires every 8 hours and
on a manual dispatch from main. Without the block it stops after a git-and-jq
policy read, so no paid work starts and nothing is written. With the block it runs
`gstack-release-gate run`, which:
1. skips when main has not moved since `stable` or since the last gate, or
when the last gate is younger than `cadence_hours` (`--ignore-cadence`
overrides only the age);
2. writes a `release` record (`gstack-trains:release:<repo>:<sha>`) to
`refs/notes/gstack-trains` before it dispatches, so a lost run resumes;
3. dispatches the full gate on that sha and reads its verdict and flake
records. A case that failed once and passed its one rule-16 rerun is green
and is listed as flaky;
4. records GREEN, RED or INCOMPLETE. An INCOMPLETE gate is re-dispatched once
by the next run, then left for a person;
5. on RED, bisects the red cases over main's first-parent commits since the
last green gate (at most 4 rounds, within the spend cap) and records the
first bad commit;
6. publishes `check_name` on the gated commit from a separate job that holds
`checks: write` and runs no repository code: success for a green gate,
failure for a red or failed one.
**What release-cut reads.** A GREEN release record on a commit counts as
gated, the same as a successful `main-backstop` check run. A RED record
with no GREEN record on a later commit that contains it holds every cut, like a
queue freeze: the `REFUSED` line names the red sha and cases. Land the revert
or the fix-forward, then gate again with `gstack-release-gate run
--ignore-cadence` (or dispatch the workflow). Nothing else clears a hold.
**The per-push backstop stands down.** `evals-main-backstop.yml` keeps its
push trigger. While the block is set, its policy step reports `train=false`,
so a push dispatches no gate, freezes nothing and publishes no check run. Main
is never gated twice. Trains keep running until the train lane retires them.
```bash
gstack-release-gate run [--dry-run] [--sha <sha>] [--ignore-cadence]
gstack-release-gate status # the newest record, the hold if any, pending commits
gstack-release-gate bisect [--sha <sha>] [--spend-cap <usd>] [--dry-run]
```
Common flags: `--repo`, `--policy <file>` (read the block from a file instead
of main, for a dry run or a proof before adoption), `--remote`,
`--summary`, `--outputs`, `--poll-seconds`, `--budget-minutes`. Every line
starts with `RELEASE:`. Exit 0 means gated, skipped, inert or a dry run; 1
means held (red); 2 means failed or incomplete.
**Escape hatch.** Remove the block on main. The per-push backstop gates the
next push again, and release-cut reads its check runs as before.
<a id="ship"></a>
## What `/ship` reads from it
+57
View File
@@ -0,0 +1,57 @@
/**
* The `release_gate` block of .gstack/ship-policy.json
* (docs/ship-policy.md#release-gate): the full paid gate on the base branch
* every few hours, run by bin/gstack-release-gate from
* .github/workflows/release-gate.yml, instead of a gate per train or
* per push. The gate never writes the stable branch: scripts/release-cut.ts
* stays its only writer, reads a green gate as "gated" and holds every cut
* while the newest gate is red. Validated here so lib/ship-policy.ts stays
* one schema file.
*/
import type { PolicyError } from './ship-policy';
export interface ReleaseGatePolicy {
/** Least hours between two gate dispatches; at least RELEASE_GATE_SCHEDULE_HOURS, the workflow's schedule. */
cadence_hours: number;
/** Workflow file dispatched for the full paid gate and the bisect; it accepts the eval-bg dispatch inputs. */
gate_workflow: string;
/** The branch the gate tests (its head at each run). */
branch: string;
/** The branch scripts/release-cut.ts moves; the gate only reads it to count pending commits. */
stable_ref: string;
/** The check run published on a gated commit, read by release-cut as gated. */
check_name: string;
/** What a red gate does. hold-release: the red record keeps release-cut from cutting until a later gate is green. */
on_red: 'hold-release';
/** Cap over one red gate's bisect, at RELEASE_TRIAL_RESERVATION_USD per case trial; 0 skips the bisect. */
bisect_spend_cap_usd: number;
}
/** The cron step of release-gate.yml's schedule (test/release-gate-workflow.test.ts pins it): no cadence gates more often. */
export const RELEASE_GATE_SCHEDULE_HOURS = 8;
const REF = /^(?!.*\.\.)(?!\/)(?!.*\/$)[A-Za-z0-9._/-]{1,100}$/;
const FIELDS: Record<keyof ReleaseGatePolicy, (v: unknown) => string | null> = {
cadence_hours: v => (Number.isInteger(v) && (v as number) >= RELEASE_GATE_SCHEDULE_HOURS && (v as number) <= 168 ? null : `must be a whole number of hours from ${RELEASE_GATE_SCHEDULE_HOURS} (the workflow's schedule) to 168`),
gate_workflow: v => (typeof v === 'string' && /^[A-Za-z0-9_.-]{1,64}\.ya?ml$/.test(v) ? null : 'must be a workflow file name such as evals.yml'),
branch: v => (typeof v === 'string' && REF.test(v) ? null : 'must be a branch name'),
stable_ref: v => (typeof v === 'string' && REF.test(v) ? null : 'must be a branch name'),
check_name: v => (typeof v === 'string' && /^[A-Za-z0-9_./-]{1,64}$/.test(v) ? null : 'must be a check-run name ([A-Za-z0-9_./-]{1,64})'),
on_red: v => (v === 'hold-release' ? null : 'must be hold-release'),
bisect_spend_cap_usd: v => (typeof v === 'number' && v >= 0 && v <= 500 ? null : 'must be dollars from 0 to 500'),
};
export const RELEASE_GATE_EXPLANATION = '{ cadence_hours, gate_workflow, branch, stable_ref, check_name, on_red, bisect_spend_cap_usd } or null: the release gate (docs/ship-policy.md#release-gate). bin/gstack-release-gate dispatches the full paid gate (gate_workflow) on the head of branch at most once per cadence_hours, only when branch moved since the last gate, records the verdict as a release note in refs/notes/gstack-trains and publishes check_name on the gated commit; a red gate bisects its red cases (bisect_spend_cap_usd, 0 skips) and, with on_red hold-release, keeps scripts/release-cut.ts from moving stable_ref until a later gate is green. It never writes stable_ref. While the block is set, the per-push main backstop stands down. null keeps the per-push backstop.';
export function releaseGateErrors(raw: unknown): PolicyError[] {
const g = raw as Record<string, unknown> | null;
if (!g || typeof g !== 'object' || Array.isArray(g)) return [{ code: 'POLICY_INVALID', path: 'release_gate', message: `must be { ${Object.keys(FIELDS).join(', ')} } or null` }];
const out: PolicyError[] = [];
for (const k of Object.keys(g)) if (!(k in FIELDS)) out.push({ code: 'POLICY_INVALID', path: `release_gate.${k}`, message: 'unknown key' });
for (const [k, check] of Object.entries(FIELDS)) {
if (!(k in g)) { out.push({ code: 'POLICY_INVALID', path: `release_gate.${k}`, message: 'missing (every release_gate field is required)' }); continue; }
const why = check(g[k]);
if (why) out.push({ code: 'POLICY_INVALID', path: `release_gate.${k}`, message: `${JSON.stringify(g[k])} ${why}` });
}
if (typeof g.branch === 'string' && g.branch === g.stable_ref) out.push({ code: 'POLICY_INVALID', path: 'release_gate.stable_ref', message: 'must differ from branch' });
return out;
}
+400
View File
@@ -0,0 +1,400 @@
/**
* The release gate (docs/ship-policy.md#release-gate; plan ship-90-90 choice A):
* the full paid gate on the gated branch's head every cadence_hours, only when
* the branch moved since the last gate, instead of a gate per train or per push.
*
* run skip when the branch head equals the stable branch, already has a
* green or red record, or the last gate is younger than the cadence;
* otherwise write a PENDING `release` record to refs/notes/gstack-trains,
* dispatch gate_workflow on the head (eval-bg's backstop nonce), follow
* it, and record GREEN, RED or INCOMPLETE. Rule-16 reruns happen inside
* the gate run (scripts/lib/paid-rerun.ts): a case that failed once and
* passed its one rerun is green and listed as flaky here.
* bisect a red record's red verdict cases, one trial each, on up to
* MAX_PREFIXES first-parent commits between the last green gate (or
* the stable branch) and the red head, dispatched on release-gate/<id>
* and narrowed round by round until first_bad is the commit right after
* a green one; spend-capped and resumable.
* status stable, the last gate, pending commits and the hold.
*
* The gate never writes the stable branch. scripts/release-cut.ts is its only
* writer: it reads a green record (or the check_name check run this gate's
* workflow publishes) as gated, and refuses every cut while a red record is not
* followed by a green gate on a later commit (the hold, on_red hold-release).
*/
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { DISPATCH_INPUTS, makeNonce } from '../scripts/eval-bg';
import { ghFailureReason, type GhRunner } from '../scripts/recover-receipts';
import { MAX_BISECT_CASES, MAX_PREFIXES, RELEASE_BISECT_ARTIFACT, RELEASE_BISECT_REF, type Git } from '../scripts/train-gate';
import type { ReleaseGatePolicy } from './release-gate-policy';
import { loadPolicyFromBase, mergePolicy, validatePolicy } from './ship-policy';
import {
NOTES_REF, fetchNotes, nowIso, persist, repoIdentity,
type PrefixResult, type ReleaseBisect, type ReleaseBisectRound, type ReleaseRecord,
} from './ship-queue-train-record';
export const RELEASE_TRIAL_RESERVATION_USD = 1.5;
/** The gate plus the one re-dispatch EVAL_POLICY.infraRedispatch allows an incomplete census. */
export const MAX_RELEASE_ATTEMPTS = 2;
/** Cron start jitter the cadence forgives. */
export const CADENCE_GRACE_HOURS = 1;
export const MAX_BISECT_ROUNDS = 4;
const SHA = /^[0-9a-f]{40}$/;
const CASE_ID = /^[A-Za-z0-9 _.:/-]{1,128}$/;
const HOUR = 3_600_000;
const GH_MS = 60_000;
export interface Run { id: number; status: string; conclusion: string | null; head_sha: string; display_title?: string; html_url?: string; run_attempt?: number }
export interface GateEnv {
repo: string; repoRoot: string; remote: string; gh: GhRunner; git: Git;
now: () => number; sleep: (ms: number) => void; random: () => string; pollMs: number;
/** Epoch ms after which nothing waits on a run any more; the record stays resumable. */
deadline: number;
}
export type GateAction = 'inert' | 'skipped' | 'dry-run' | 'gated' | 'held' | 'failed';
export interface GateOut { action: GateAction; sha: string | null; lines: string[] }
const short = (sha: string | null | undefined) => (sha ? sha.slice(0, 12) : 'none');
const usd = (n: number | null) => (n === null ? 'unknown' : n.toFixed(2));
/** The release_gate block from origin/<base>'s reviewed policy, or from --policy <file> (validated the same way). */
export function loadReleaseGate(repoRoot: string, o: { policyFile?: string; base?: string }): { gate: ReleaseGatePolicy | null; label: string } | { error: string } {
if (o.policyFile) {
let raw: unknown;
try { raw = JSON.parse(fs.readFileSync(o.policyFile, 'utf8')); } catch (e: any) { return { error: `${o.policyFile} is not readable JSON: ${e.message}` }; }
const errors = validatePolicy(raw);
if (errors.length) return { error: `${o.policyFile}: ${errors.map(e => `${e.path} ${e.message}`).join('; ')}` };
return { gate: mergePolicy(raw as Record<string, unknown>, o.policyFile).policy.release_gate, label: o.policyFile };
}
const loaded = loadPolicyFromBase(repoRoot, o.base ?? 'main');
if ('unavailable' in loaded) return { error: `no reviewed policy: ${loaded.detail}` };
if (loaded.errors.length) return { error: `${loaded.source.label}: ${loaded.errors.map(e => `${e.path} ${e.message}`).join('; ')}` };
return { gate: loaded.policy.release_gate, label: loaded.source.label };
}
/** This repository's release records from the local notes ref, newest first. */
export function releaseRecords(git: Git, repoKey: string): ReleaseRecord[] {
const listed = git(['notes', `--ref=${NOTES_REF}`, 'list']);
if (!listed.ok || !listed.out.trim()) return [];
return listed.out.split('\n').map(line => line.split(' ')[0]!).filter(Boolean).flatMap(blob => {
try { const r = JSON.parse(git(['cat-file', 'blob', blob]).out) as ReleaseRecord; return r?.kind === 'release' && r.repo_key === repoKey && SHA.test(r.sha) ? [r] : []; } catch { return []; }
}).sort((a, b) => b.created_at.localeCompare(a.created_at));
}
/** The red record that holds releases: red, and no green record sits on a later commit that contains it. Null when none. */
export function activeHold(records: readonly ReleaseRecord[], isAncestor: (older: string, newer: string) => boolean): ReleaseRecord | null {
const greens = records.filter(r => r.verdict === 'GREEN');
return records.filter(r => r.verdict === 'RED').find(red => !greens.some(g => g.sha !== red.sha && isAncestor(red.sha, g.sha))) ?? null;
}
/** The newest green record on an ancestor of sha: where pending commits and a bisect start. */
export function greenBase(records: readonly ReleaseRecord[], sha: string, isAncestor: (older: string, newer: string) => boolean): ReleaseRecord | null {
return records.find(r => r.verdict === 'GREEN' && r.sha !== sha && isAncestor(r.sha, sha)) ?? null;
}
export interface ReportFacts { verdict: 'GREEN' | 'RED' | 'NONE'; red: string[]; other: string[]; cost: number | null }
/** report-verdict's report.txt: the verdict, each failed case (a verdict red, or another class), and the known cost. */
export function parseReport(text: string): ReportFacts {
const verdict = (/VERDICT (GREEN|RED)/.exec(text)?.[1] as 'GREEN' | 'RED' | undefined) ?? 'NONE';
const red: string[] = [];
const other: string[] = [];
for (const m of text.matchAll(/^\s+✗ (\S(?:.*?\S)?) {2,}(\w+) {2,}(\w+)(?: {2,}([A-Z_]+))?/gm)) {
(m[3] === 'FAIL' && (!m[4] || m[4] === 'VERDICT') ? red : other).push(m[1]!);
}
const cost = /· cost \$(\d+(?:\.\d+)?)/.exec(text)?.[1];
return { verdict, red: [...new Set(red)].sort(), other: [...new Set(other)].sort(), cost: cost ? Number(cost) : null };
}
/** A finished gate run as one verdict; green needs the run's own success, which follows its rule-16 report. */
export function classifyRun(run: Pick<Run, 'status' | 'conclusion'>, report: ReportFacts | null): { verdict: ReleaseRecord['verdict']; red: string[]; detail: string } {
if (run.status !== 'completed') return { verdict: 'PENDING', red: [], detail: `run is ${run.status}` };
if (run.conclusion === 'success') {
if (report?.verdict === 'RED') return { verdict: 'INCOMPLETE', red: [], detail: 'the run concluded success but its report says RED' };
return { verdict: 'GREEN', red: [], detail: report ? 'run and report green' : 'run concluded success (no report artifact)' };
}
if (report?.verdict === 'RED' && report.red.length) {
return { verdict: 'RED', red: report.red, detail: `${report.red.length} red case(s)${report.other.length ? `; also not green: ${report.other.join(', ')}` : ''}` };
}
return { verdict: 'INCOMPLETE', red: [], detail: `run concluded ${run.conclusion ?? 'unknown'}; report ${report ? `${report.verdict}${report.other.length ? ` (${report.other.join(', ')})` : ''}` : 'missing'}` };
}
/** Up to max commits spread evenly over the interior of a range (oldest first), never the endpoints. */
export function planPrefixes(interior: readonly string[], max = MAX_PREFIXES): string[] {
if (interior.length <= max) return [...interior];
const picked = Array.from({ length: max }, (_, i) => interior[Math.round(((i + 1) * (interior.length + 1)) / (max + 1)) - 1]!);
return [...new Set(picked)];
}
/**
* One round's results narrow (good, bad]: the earliest prefix where any case is
* red becomes bad, the prefix before it good. Every case must be monotone
* (green, then red) over the tested prefixes and have a result on each.
*/
export function narrow(range: { good: string; bad: string }, tested: ReadonlyArray<{ sha: string; results: Record<string, PrefixResult> }>, cases: readonly string[]): { good: string; bad: string } | { unknown: string } {
let firstRed = tested.length;
for (const c of cases) {
const seq = tested.map(p => (p.results[c] === 'red' ? 'R' : p.results[c] === 'green' ? 'G' : '?')).join('');
if (seq.includes('?')) return { unknown: `no result for ${c} on ${tested.filter(p => !p.results[c]).map(p => short(p.sha)).join(', ')}` };
const at = seq.indexOf('R');
if (at >= 0 && seq.slice(at).includes('G')) return { unknown: `${c} is not monotone over the tested commits (${seq})` };
if (at >= 0) firstRed = Math.min(firstRed, at);
}
return { good: firstRed > 0 ? tested[firstRed - 1]!.sha : range.good, bad: firstRed < tested.length ? tested[firstRed]!.sha : range.bad };
}
function ghJson<T>(env: GateEnv, args: string[]): T | null {
const r = env.gh(args, GH_MS);
if (r.status !== 0) return null;
try { return JSON.parse(r.stdout) as T; } catch { return null; }
}
/** One file of a run's artifact, or null when the artifact or the file is missing. */
function artifactFile(env: GateEnv, run: number, name: string, file: string): string | null {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'release-gate-'));
try {
if (env.gh(['run', 'download', String(run), '-R', env.repo, '-n', name, '-D', dir], GH_MS).status !== 0) return null;
const find = (d: string): string | null => {
for (const e of fs.readdirSync(d, { withFileTypes: true })) {
const p = path.join(d, e.name);
if (e.isDirectory()) { const hit = find(p); if (hit) return hit; } else if (e.name === file) return p;
}
return null;
};
const hit = find(dir);
return hit ? fs.readFileSync(hit, 'utf8') : null;
} finally { fs.rmSync(dir, { recursive: true, force: true }); }
}
function dispatch(env: GateEnv, workflow: string, ref: string, inputs: Record<string, string>): string | null {
const r = env.gh(['workflow', 'run', workflow, '--repo', env.repo, '--ref', ref, ...Object.entries(inputs).flatMap(([k, v]) => ['-f', `${k}=${v}`])], GH_MS);
return r.status === 0 ? null : ghFailureReason(r.stderr);
}
/** Find a dispatch's run by its nonce, then follow it until it completes or the deadline passes. */
function follow(env: GateEnv, workflow: string, nonce: string, known: number | null, lines: string[]): Run | null {
let run: Run | null = known ? ghJson<Run>(env, ['api', `repos/${env.repo}/actions/runs/${known}`]) : null;
for (;;) {
if (!run) {
const runs = ghJson<{ workflow_runs?: Run[] }>(env, ['api', '-X', 'GET', `repos/${env.repo}/actions/workflows/${workflow}/runs`, '-f', 'event=workflow_dispatch', '-f', 'per_page=30']);
run = runs?.workflow_runs?.find(r => (r.display_title ?? '').includes(`eval-bg ${nonce}]`)) ?? null;
if (run) lines.push(`RELEASE: run ${run.id} ${run.html_url ?? ''}`.trim());
} else if (run.status === 'completed') return run;
if (env.now() >= env.deadline) break;
env.sleep(env.pollMs);
if (run) run = ghJson<Run>(env, ['api', `repos/${env.repo}/actions/runs/${run.id}`]) ?? run;
}
lines.push(`RELEASE: ${run ? `run ${run.id} is still ${run.status}` : `no run for nonce ${nonce} yet`} at the deadline; the record stays resumable`);
return run;
}
const runRef = (env: GateEnv, run: Run) => ({ id: run.id, url: run.html_url ?? `https://github.com/${env.repo}/actions/runs/${run.id}` });
function save(env: GateEnv, rec: ReleaseRecord, lines: string[]): void {
rec.updated_at = nowIso();
const p = persist(env.repoRoot, [rec], { remote: env.remote });
if (!p.pushed) lines.push(`RELEASE: record ${p.detail}`);
}
interface Context { gate: ReleaseGatePolicy; repoKey: string; stable: string | null; records: ReleaseRecord[]; isAncestor: (a: string, b: string) => boolean }
function context(env: GateEnv, gate: ReleaseGatePolicy): Context | { error: string } {
const f = fetchNotes(env.repoRoot, env.remote);
if (!f.ok) return { error: f.detail };
const stable = env.git(['ls-remote', env.remote, `refs/heads/${gate.stable_ref}`]).out.split(/\s/)[0] || null;
const isAncestor = (a: string, b: string) => env.git(['merge-base', '--is-ancestor', a, b]).ok;
const repoKey = repoIdentity(env.repoRoot).key;
return { gate, repoKey, stable: stable && SHA.test(stable) ? stable : null, records: releaseRecords(env.git, repoKey), isAncestor };
}
/** Make sure the gated branch's history is here (a shallow checkout gets it), so ancestry and ranges read true. */
function ensureHistory(env: GateEnv, gate: ReleaseGatePolicy, sha: string): void {
if (env.git(['rev-parse', '--is-shallow-repository']).out === 'true') env.git(['fetch', '--no-tags', '-q', '--unshallow', env.remote, gate.branch, gate.stable_ref]);
if (!env.git(['cat-file', '-e', `${sha}^{commit}`]).ok) env.git(['fetch', '--no-tags', '-q', env.remote, gate.branch]);
}
const firstParents = (env: GateEnv, from: string, to: string): string[] | null => {
const r = env.git(['rev-list', '--first-parent', '--reverse', `${from}..${to}`]);
return r.ok ? r.out.split('\n').filter(Boolean) : null;
};
export interface RunOptions { gate: ReleaseGatePolicy | null; label: string; sha?: string; dryRun: boolean; ignoreCadence: boolean }
/** `run`: gate the branch head once when it moved; bisect a red. */
export function runGate(env: GateEnv, o: RunOptions): GateOut {
const lines: string[] = [];
const out = (action: GateAction, sha: string | null): GateOut => ({ action, sha, lines });
if (!o.gate) { lines.push(`RELEASE: inert (${o.label} declares no release_gate block; the per-push main backstop gates main)`); return out('inert', null); }
const gate = o.gate;
const c = context(env, gate);
if ('error' in c) { lines.push(`RELEASE: cannot read refs/notes/gstack-trains (${c.error}); nothing dispatched`); return out('failed', null); }
const sha = o.sha ?? env.git(['ls-remote', env.remote, `refs/heads/${gate.branch}`]).out.split(/\s/)[0] ?? '';
if (!SHA.test(sha)) { lines.push(`RELEASE: cannot resolve ${gate.branch} on ${env.remote} (${JSON.stringify(sha)})`); return out('failed', null); }
ensureHistory(env, gate, sha);
const prior = c.records.find(r => r.sha === sha) ?? null;
const skip = (why: string) => { lines.push(`RELEASE: skip sha=${sha} (${why})`); return out('skipped', sha); };
if (sha === c.stable) return skip(`${gate.branch} has not moved since ${gate.stable_ref}`);
if (prior?.verdict === 'GREEN' || prior?.verdict === 'RED') {
if (prior.verdict === 'RED' && prior.bisect?.state === 'running' && !o.dryRun) return finishRed(env, c, prior, lines, out);
return skip(`${gate.branch} has not moved since the last gate: ${prior.verdict} run ${prior.run?.id ?? '-'}`);
}
if (prior?.verdict === 'INCOMPLETE' && prior.attempts >= MAX_RELEASE_ATTEMPTS) return skip(`the gate on this commit was incomplete ${prior.attempts} times (${prior.detail}); diagnose run ${prior.run?.url ?? prior.run?.id ?? '-'}`);
const newest = c.records[0];
const age = newest ? (env.now() - Date.parse(newest.created_at)) / HOUR : Infinity;
if (!o.ignoreCadence && prior?.verdict !== 'PENDING' && age < gate.cadence_hours - CADENCE_GRACE_HOURS) {
return skip(`the last gate was dispatched ${age.toFixed(1)} h ago on ${short(newest!.sha)}; cadence ${gate.cadence_hours} h`);
}
const since = greenBase(c.records, sha, c.isAncestor)?.sha ?? c.stable;
const pending = since ? firstParents(env, since, sha)?.length ?? null : null;
lines.push(`RELEASE: gate sha=${sha} since=${short(since)} (${since === c.stable ? gate.stable_ref : 'last green gate'}) pending=${pending ?? 'unknown'} stable=${short(c.stable)}`);
if (o.dryRun) { lines.push(`RELEASE: dry run: would dispatch ${gate.gate_workflow} on ${gate.branch}@${short(sha)} (full gate, about $37); nothing written`); return out('dry-run', sha); }
const now = nowIso();
const rec: ReleaseRecord = prior?.verdict === 'PENDING' ? prior : {
schema: 1, kind: 'release', repo_key: c.repoKey, sha, tree: env.git(['rev-parse', `${sha}^{tree}`]).out, branch: gate.branch, stable: c.stable, since,
nonce: makeNonce('gate', sha, sha, env.random(), { purpose: 'backstop' }), attempts: (prior?.attempts ?? 0) + 1, run: null, verdict: 'PENDING',
red_cases: [], flaky: [], cost_usd: null, detail: 'dispatching', bisect: null, created_at: now, updated_at: now,
};
if (rec !== prior) {
save(env, rec, lines);
const failed = dispatch(env, gate.gate_workflow, gate.branch, { [DISPATCH_INPUTS.evalsAll]: 'true', [DISPATCH_INPUTS.expectedSha]: sha, [DISPATCH_INPUTS.nonce]: rec.nonce });
if (failed) { Object.assign(rec, { verdict: 'INCOMPLETE', detail: `the gate dispatch failed (${failed})` }); save(env, rec, lines); lines.push(`RELEASE: incomplete sha=${sha} run=- (${rec.detail})`); return out('failed', sha); }
lines.push(`RELEASE: dispatched ${gate.gate_workflow} on ${gate.branch}@${short(sha)} (nonce ${rec.nonce})`);
} else lines.push(`RELEASE: resuming the pending gate on ${short(sha)} (nonce ${rec.nonce})`);
const run = follow(env, gate.gate_workflow, rec.nonce, rec.run?.id ?? null, lines);
if (run) rec.run = runRef(env, run);
if (!run || run.status !== 'completed') { rec.detail = run ? `run ${run.id} ${run.status}` : 'no run found for the nonce yet'; save(env, rec, lines); return out('failed', sha); }
const attempt = run.run_attempt ?? 1;
const reportText = artifactFile(env, run.id, `report-verdict-a${attempt}`, 'report.txt');
const report = reportText === null ? null : parseReport(reportText);
const flakes = artifactFile(env, run.id, `flake-records-a${attempt}`, 'flake-records.jsonl') ?? '';
const k = classifyRun(run, report);
Object.assign(rec, {
verdict: k.verdict, red_cases: k.red, detail: k.detail, cost_usd: report?.cost ?? null,
flaky: [...new Set(flakes.split('\n').flatMap(l => { try { const f = JSON.parse(l); return typeof f?.case === 'string' ? [f.case] : []; } catch { return []; } }))].sort(),
});
save(env, rec, lines);
if (rec.flaky.length) lines.push(`RELEASE: flaky (failed once, passed its one rule-16 rerun in run ${run.id}): ${rec.flaky.join(', ')}`);
if (rec.verdict === 'GREEN') {
lines.push(`RELEASE: green sha=${sha} stable=${c.stable ?? 'none'} run=${run.id} cost=${usd(rec.cost_usd)}`,
`RELEASE: ${gate.check_name} success goes on ${short(sha)}; scripts/release-cut.ts moves ${gate.stable_ref} on its own schedule and floor`);
return out('gated', sha);
}
if (rec.verdict === 'RED') return finishRed(env, c, rec, lines, out);
lines.push(`RELEASE: incomplete sha=${sha} run=${run.id} (${rec.detail})${rec.attempts < MAX_RELEASE_ATTEMPTS ? '; the next run re-dispatches it once' : ''}`);
return out('failed', sha);
}
function finishRed(env: GateEnv, c: Context, rec: ReleaseRecord, lines: string[], out: (a: GateAction, s: string | null) => GateOut): GateOut {
bisectRecord(env, c, rec, { spendCap: c.gate.bisect_spend_cap_usd, dryRun: false }, lines);
lines.push(`RELEASE: red sha=${rec.sha} first_bad=${rec.bisect?.first_bad ?? 'unknown'} run=${rec.run?.id ?? '-'}`,
`RELEASE: hold: scripts/release-cut.ts refuses to move ${c.gate.stable_ref} until a gate on a later commit is green (red: ${rec.red_cases.join(', ')}); land the revert or the fix-forward`);
return out('held', rec.sha);
}
/** Collect a bisect round's prefix records into its results; the cost they report is added to the spend. */
function collectRound(env: GateEnv, b: ReleaseBisect, round: ReleaseBisectRound, run: Run): void {
for (const p of round.prefixes) {
const text = artifactFile(env, run.id, `${RELEASE_BISECT_ARTIFACT}-bisect-${p.sha.slice(0, 12)}`, 'bisect-prefix.json');
let rec: { kind?: string; nonce?: string; prefix_sha?: string; checked_out_sha?: string; panels?: Array<{ case: string; status: string; red_class: string | null; cost_usd: number | null }> } | null = null;
try { rec = text ? JSON.parse(text) : null; } catch { rec = null; }
if (!rec || rec.kind !== 'train-bisect-prefix' || rec.nonce !== round.nonce || rec.prefix_sha !== p.sha || rec.checked_out_sha !== p.sha || !Array.isArray(rec.panels)) continue;
for (const panel of rec.panels) {
if (!b.cases.includes(panel.case)) continue;
b.spent_usd = Math.round((b.spent_usd + (typeof panel.cost_usd === 'number' ? panel.cost_usd : 0)) * 100) / 100;
if (panel.status === 'PASS') p.results[panel.case] = 'green';
else if (panel.status === 'FAIL' && panel.red_class === 'VERDICT') p.results[panel.case] = 'red';
}
}
}
/**
* Bisect a red record (resumable): finish a round still running, then narrow
* (good, bad] with up to MAX_PREFIXES commits a round until they are adjacent,
* the pattern is not monotone, the cap is spent, or MAX_BISECT_ROUNDS pass.
*/
export function bisectRecord(env: GateEnv, c: Context, rec: ReleaseRecord, o: { spendCap: number; dryRun: boolean }, lines: string[]): void {
const b: ReleaseBisect = rec.bisect ?? { cases: rec.red_cases, good: rec.since, bad: rec.sha, first_bad: null, spend_cap: o.spendCap, spent_usd: 0, state: 'running', detail: '', rounds: [] };
b.spend_cap = Math.max(b.spend_cap, o.spendCap);
const stop = (state: ReleaseBisect['state'], detail: string) => { Object.assign(b, { state, detail }); };
if (!o.dryRun) rec.bisect = b;
if (b.state === 'running' && o.spendCap <= 0 && !b.rounds.length) stop('unknown', 'bisect_spend_cap_usd is 0: no bisect');
else if (b.state === 'running' && (!b.cases.length || b.cases.length > MAX_BISECT_CASES || b.cases.some(id => !CASE_ID.test(id)))) stop('unknown', `the red cases cannot be bisected (${b.cases.length} case ids, at most ${MAX_BISECT_CASES})`);
else if (b.state === 'running' && !b.good) stop('unknown', 'no green gate and no stable branch before the red commit to bisect from');
while (b.state === 'running') {
const open = b.rounds.at(-1);
if (open && open.prefixes.some(p => b.cases.some(cs => !p.results[cs])) && open.run?.id !== -1) {
if (o.dryRun) { lines.push(`RELEASE: dry run: round ${b.rounds.length} (nonce ${open.nonce}) still has prefixes to collect`); return; }
const run = follow(env, c.gate.gate_workflow, open.nonce, open.run?.id ?? null, lines);
if (run) open.run = runRef(env, run);
if (!run || run.status !== 'completed') { b.detail = `round ${b.rounds.length} is still running`; save(env, rec, lines); return; }
collectRound(env, b, open, run);
env.git(['push', '-q', env.remote, '--delete', `refs/heads/${open.ref}`]);
const n = narrow({ good: b.good!, bad: b.bad }, open.prefixes, b.cases);
if ('unknown' in n) { stop('unknown', n.unknown); break; }
Object.assign(b, n);
lines.push(`RELEASE: bisect round ${b.rounds.length}: good=${short(b.good)} bad=${short(b.bad)} spent=$${b.spent_usd.toFixed(2)}`);
save(env, rec, lines);
}
const interior = firstParents(env, b.good!, b.bad);
if (!interior) { stop('unknown', `cannot list ${short(b.good)}..${short(b.bad)}`); break; }
interior.pop();
if (!interior.length) { b.first_bad = b.bad; stop('found', `${short(b.bad)} is the first red commit after green ${short(b.good)}`); break; }
if (b.rounds.length >= MAX_BISECT_ROUNDS) { stop('unknown', `${MAX_BISECT_ROUNDS} rounds left ${interior.length} commit(s) between ${short(b.good)} and ${short(b.bad)}`); break; }
const perPrefix = b.cases.length * RELEASE_TRIAL_RESERVATION_USD;
const fits = Math.floor((b.spend_cap - b.spent_usd + 1e-9) / perPrefix);
const prefixes = planPrefixes(interior, Math.min(MAX_PREFIXES, fits));
if (!prefixes.length) { stop('capped', `the next round needs $${perPrefix.toFixed(2)} a prefix; $${(b.spend_cap - b.spent_usd).toFixed(2)} of the $${b.spend_cap.toFixed(2)} cap is left`); break; }
lines.push(`RELEASE: bisect round ${b.rounds.length + 1}: ${b.cases.length} case(s) on ${prefixes.length} of ${interior.length} commit(s) between ${short(b.good)} and ${short(b.bad)}, $${(perPrefix * prefixes.length).toFixed(2)} reserved`);
if (o.dryRun) { lines.push('RELEASE: dry run: nothing dispatched'); return; }
const nonce = makeNonce('gate', b.bad, b.good!, env.random(), { purpose: 'bisect' });
const ref = `${RELEASE_BISECT_REF}${nonce.slice(-6)}-${b.bad.slice(0, 12)}`;
const round: ReleaseBisectRound = { nonce, ref, run: null, prefixes: prefixes.map(sha => ({ sha, results: {} })) };
b.rounds.push(round);
save(env, rec, lines);
const pushed = env.git(['push', '-q', env.remote, `${b.bad}:refs/heads/${ref}`]);
const failed = pushed.ok ? dispatch(env, c.gate.gate_workflow, ref, {
[DISPATCH_INPUTS.cases]: b.cases.join(','), [DISPATCH_INPUTS.prefixShas]: prefixes.join(','), [DISPATCH_INPUTS.expectedSha]: b.bad, [DISPATCH_INPUTS.nonce]: nonce,
}) : `git push ${ref} failed: ${pushed.out.slice(0, 120)}`;
if (failed) { round.run = { id: -1, url: '' }; env.git(['push', '-q', env.remote, '--delete', `refs/heads/${ref}`]); stop('unknown', `round ${b.rounds.length} dispatch failed: ${failed}`); }
}
lines.push(`RELEASE: bisect ${b.state}: ${b.detail} (spent $${b.spent_usd.toFixed(2)} of $${b.spend_cap.toFixed(2)})`);
if (!o.dryRun) save(env, rec, lines);
}
/** `bisect`: resume or start the bisect of the newest red record (or the one at sha). */
export function bisectGate(env: GateEnv, o: { gate: ReleaseGatePolicy | null; label: string; sha?: string; spendCap?: number; dryRun: boolean }): GateOut {
const lines: string[] = [];
if (!o.gate) { lines.push(`RELEASE: inert (${o.label} declares no release_gate block)`); return { action: 'inert', sha: null, lines }; }
const c = context(env, o.gate);
if ('error' in c) { lines.push(`RELEASE: cannot read refs/notes/gstack-trains (${c.error})`); return { action: 'failed', sha: null, lines }; }
const rec = o.sha ? c.records.find(r => r.sha === o.sha) : c.records.find(r => r.verdict === 'RED');
if (!rec || rec.verdict !== 'RED') { lines.push(`RELEASE: no red release record${o.sha ? ` at ${o.sha}` : ''} to bisect`); return { action: 'failed', sha: o.sha ?? null, lines }; }
ensureHistory(env, o.gate, rec.sha);
if (rec.bisect?.state === 'capped' && (o.spendCap ?? 0) > rec.bisect.spend_cap) rec.bisect.state = 'running';
bisectRecord(env, c, rec, { spendCap: o.spendCap ?? o.gate.bisect_spend_cap_usd, dryRun: o.dryRun }, lines);
lines.push(`RELEASE: red sha=${rec.sha} first_bad=${rec.bisect?.first_bad ?? 'unknown'} run=${rec.run?.id ?? '-'}`);
return { action: o.dryRun ? 'dry-run' : 'held', sha: rec.sha, lines };
}
/** `status`: never spends and never writes. */
export function gateStatus(env: GateEnv, o: { gate: ReleaseGatePolicy | null; label: string }): GateOut {
const lines: string[] = [];
if (!o.gate) { lines.push(`RELEASE: inert (${o.label} declares no release_gate block; the per-push main backstop gates main)`); return { action: 'inert', sha: null, lines }; }
const c = context(env, o.gate);
if ('error' in c) { lines.push(`RELEASE: cannot read refs/notes/gstack-trains (${c.error})`); return { action: 'failed', sha: null, lines }; }
const head = env.git(['ls-remote', env.remote, `refs/heads/${o.gate.branch}`]).out.split(/\s/)[0] ?? '';
if (SHA.test(head)) ensureHistory(env, o.gate, head);
const last = c.records[0];
const base = SHA.test(head) ? greenBase(c.records, head, c.isAncestor)?.sha ?? c.stable : null;
const pending = base && SHA.test(head) ? (base === head ? 0 : firstParents(env, base, head)?.length ?? null) : null;
const hold = activeHold(c.records, c.isAncestor);
lines.push(
`STABLE: ${o.gate.stable_ref} ${c.stable ?? 'absent'} (scripts/release-cut.ts moves it; the gate never does)`,
`LAST_GATE: ${last ? `${last.verdict} sha=${last.sha} run=${last.run?.id ?? '-'} at=${last.created_at} cost=${usd(last.cost_usd)}${last.flaky.length ? ` flaky=${last.flaky.join(',')}` : ''}` : 'none'}`,
`PENDING: ${pending ?? 'unknown'} commit(s) on ${o.gate.branch} ${short(head)} since ${short(base)}`,
`HOLD: ${hold ? `red on ${hold.sha} since ${hold.created_at} (red: ${hold.red_cases.join(', ')}; first_bad=${hold.bisect?.first_bad ?? 'unknown'}, bisect ${hold.bisect?.state ?? 'not run'}); release-cut refuses until a gate on a later commit is green` : 'none'}`,
);
return { action: hold ? 'held' : 'skipped', sha: SHA.test(head) ? head : null, lines };
}
+8 -1
View File
@@ -15,6 +15,7 @@
import { spawnSync } from 'node:child_process';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { RELEASE_GATE_EXPLANATION, releaseGateErrors, type ReleaseGatePolicy } from './release-gate-policy';
import { RESULT_CODES, type ResultCodeName } from './result-codes';
import { cmpVersion, parseVersion } from './version-source';
@@ -77,6 +78,8 @@ export interface ShipPolicy {
docs: { caps: Array<{ path: string; max_bytes: number }> } | null;
/** How /ship reproduces a Step 5 failure on the base branch before calling it base-red (docs/ship-policy.md#base-red); null never calls one base-red. */
base_red: BaseRedPolicy | null;
/** The release gate on the base branch (docs/ship-policy.md#release-gate); null keeps the per-push main backstop. */
release_gate: ReleaseGatePolicy | null;
}
/** `base_red`: `setup` installs dependencies in a scratch checkout (null when none are needed); `select` runs the failing test files, named by its `{files}` slot. */
@@ -165,6 +168,7 @@ export const DEFAULT_POLICY: ShipPolicy = {
train: null,
docs: null,
base_red: null,
release_gate: null,
};
export type PolicyKey = Exclude<keyof ShipPolicy, '$schema'>;
@@ -195,6 +199,7 @@ export const POLICY_EXPLANATIONS: Record<PolicyKey, string> = {
train: '{ gate_workflow, free_workflow, full_gate_check, record_artifact, max_size, fill_hours, screen_selection, trial_reservation_usd, land? } (the gstack-evals gate) or { gates, land, max_size, fill_hours } (any repo: gates lists { kind: workflows, workflows, required_jobs, record?, post_merge_checks? } and { kind: command, command, timeout_sec, spend_cap_usd, pass_env? }, each GREEN on every prefix; land is { merge_method: squash|merge|rebase, required_checks? }) or null: turns on merge trains in gstack-ship-queue (train build|status|verdict|land|watch|init|doctor|prune, ready --screen|--no-screen, verify --train). gates and land need min_gstack_version at the release that reads them (validate names it). null keeps today\'s per-PR queue unchanged.',
docs: '{ caps: [{ path, max_bytes }] } or null: documentation files with a byte cap. /ship Step 14.5 runs `gstack-ship-receipt docs-caps` before the verification gate and prints one DOCS_CAP: line per cap; a file over its cap (or absent) blocks the ship until the doc is trimmed or the cap raised on the base branch. null declares none.',
base_red: '{ setup, select, timeout_sec } or null: how /ship Step 5 proves a failure is the base branch\'s own (docs/ship-policy.md#base-red). gstack-base-red checks out the base head and the PR head in scratch worktrees, runs setup (dependency install, or null), then select with the failing test files in its {files} slot; only the same failures in both, in files the PR does not touch, are base-red, and /ship then pauses before READY instead of failing. Read from the base branch only. null keeps every failure in-branch.',
release_gate: RELEASE_GATE_EXPLANATION,
min_gstack_version: 'The oldest gstack that may read this file. Any key newer than 1.91.98.0 set to a non-default value requires it at that key’s introducing version (validate prints each); a gstack older than it refuses the file (POLICY_GSTACK_TOO_OLD).',
};
@@ -231,6 +236,7 @@ export const BUMP_NEVER_KEY_VERSION = '1.91.353.0';
export const BASE_RED_KEY_VERSION = '1.91.374.0';
/** `train.gates` and `train.land` are values of the older train key, so compatErrors checks them by value, as for bump: never. */
export const TRAIN_GATES_KEY_VERSION = '1.92.0.0';
export const RELEASE_GATE_KEY_VERSION = '1.94.0.0';
export const POLICY_KEY_INTRODUCED: Record<PolicyKey, string> = {
bump: '1.91.73.0', queue_mode: '1.91.73.0', release_tool: '1.91.73.0', release_outputs: '1.91.73.0', stamp_paths: '1.91.73.0',
changelog: '1.91.73.0', mirror: '1.91.73.0', history: '1.91.73.0', preregistration_shas: '1.91.73.0', stamp_sensitive_gates: '1.91.73.0',
@@ -238,7 +244,7 @@ export const POLICY_KEY_INTRODUCED: Record<PolicyKey, string> = {
migrations: '1.91.98.0', gate_command: SHIP_QUEUE_KEYS_VERSION, full_e2e: SHIP_QUEUE_KEYS_VERSION, min_gstack_version: SHIP_QUEUE_KEYS_VERSION,
train: TRAIN_KEYS_VERSION, review_scope: REVIEW_SCOPE_KEY_VERSION, docs: DOCS_CAPS_KEY_VERSION,
changelog_fragments: CHANGELOG_FRAGMENTS_KEY_VERSION,
base_red: BASE_RED_KEY_VERSION,
base_red: BASE_RED_KEY_VERSION, release_gate: RELEASE_GATE_KEY_VERSION,
};
const newer = (a: string, b: string): boolean => cmpVersion(parseVersion(a)!, parseVersion(b)!) > 0;
@@ -496,6 +502,7 @@ export function validatePolicy(raw: unknown, opts: { gstackVersion?: string } =
if ('docs' in obj && obj.docs !== null) docsErrors(obj.docs, errors);
if ('changelog_fragments' in obj && obj.changelog_fragments !== null) fragmentsErrors(obj.changelog_fragments, errors);
if ('base_red' in obj && obj.base_red !== null) baseRedErrors(obj.base_red, errors);
if ('release_gate' in obj && obj.release_gate !== null) errors.push(...releaseGateErrors(obj.release_gate));
if ('stamp_paths' in obj) listErrors('stamp_paths', obj.stamp_paths, errors, globError);
for (const key of ['changelog', 'mirror'] as const) {
if (!(key in obj) || obj[key] === null) continue;
+18 -1
View File
@@ -117,7 +117,20 @@ export interface ReadyReceipt {
schema: 1; kind: 'ready'; repo_key: string; pr: string; head: string; base_sha: string; ready_at: string; screen: ScreenChoice; updated_at: string;
base_red?: BaseRedCondition; fixes_base_red?: string[];
}
export type AnyRecord = TrainRecord | AttemptRecord | BisectRecord | FreezeRecord | ReadyReceipt | WaveRecord;
/** One release-gate bisect round (lib/release-gate.ts): the prefixes it dispatched on release-gate/<id> and what each case did there. */
export interface ReleaseBisectRound { nonce: string; ref: string; run: { id: number; url: string } | null; prefixes: Array<{ sha: string; results: Record<string, PrefixResult> }> }
/** A red release gate's bisect over the first-parent commits after the last green gate: first_bad once good and bad are adjacent. */
export interface ReleaseBisect {
cases: string[]; good: string | null; bad: string; first_bad: string | null; spend_cap: number; spent_usd: number;
state: 'running' | 'found' | 'unknown' | 'capped'; detail: string; rounds: ReleaseBisectRound[];
}
/** The release gate's verdict on one commit of the gated branch (lib/release-gate.ts), the evidence scripts/release-cut.ts reads. */
export interface ReleaseRecord {
schema: 1; kind: 'release'; repo_key: string; sha: string; tree: string; branch: string; stable: string | null; since: string | null;
nonce: string; attempts: number; run: { id: number; url: string } | null; verdict: 'PENDING' | 'GREEN' | 'RED' | 'INCOMPLETE';
red_cases: string[]; flaky: string[]; cost_usd: number | null; detail: string; bisect: ReleaseBisect | null; created_at: string; updated_at: string;
}
export type AnyRecord = TrainRecord | AttemptRecord | BisectRecord | FreezeRecord | ReadyReceipt | WaveRecord | ReleaseRecord;
/** Clock seam for fill-time and record tests (`GSTACK_SHIP_QUEUE_NOW`, ISO-8601). */
export function nowIso(): string {
@@ -141,6 +154,7 @@ const bisectPath = (nonce: string) => path.join(trainDir(nonce), 'bisect.json');
const freezePath = (repoKey: string) => path.join(queueRoot(), 'freeze', `${repoKey}.json`);
export const readyPath = (repoKey: string, pr: string, head: string) => path.join(queueRoot(), 'ready', repoKey, `${pr}-${head}.json`);
export const wavePath = (nonce: string) => path.join(queueRoot(), 'waves', `${nonce}.json`);
export const releasePath = (repoKey: string, sha: string) => path.join(queueRoot(), 'release', repoKey, `${sha}.json`);
/** Hosts that serve GitHub repositories at /<owner>/<repo> (git.capy.ai is Capy's GitHub proxy, set through url.insteadOf). */
const GITHUB_HOSTS = ['github.com', 'www.github.com', 'ssh.github.com', 'git.capy.ai'];
@@ -226,6 +240,7 @@ export function recordPath(r: AnyRecord): string {
if (r.kind === 'train-bisect') return bisectPath(r.train);
if (r.kind === 'freeze') return freezePath(r.repo_key);
if (r.kind === 'wave') return wavePath(r.nonce);
if (r.kind === 'release') return releasePath(r.repo_key, r.sha);
return readyPath(r.repo_key, r.pr, r.head);
}
@@ -235,6 +250,7 @@ export function noteName(r: AnyRecord): string {
if (r.kind === 'train-bisect') return `gstack-trains:bisect:${r.train}`;
if (r.kind === 'freeze') return `gstack-trains:freeze:${r.repo_key}`;
if (r.kind === 'wave') return `gstack-trains:wave:${r.nonce}`;
if (r.kind === 'release') return `gstack-trains:release:${r.repo_key}:${r.sha}`;
return `gstack-trains:ready:${r.repo_key}:${r.pr}:${r.head}`;
}
@@ -299,6 +315,7 @@ function parseNote(text: string): AnyRecord | null {
if ((r.kind === 'train-attempt' && ids.test(r.train) && ids.test(r.id)) || (r.kind === 'train-bisect' && ids.test(r.train))) return r;
if (r.kind === 'freeze' && ids.test(r.repo_key)) return r;
if (r.kind === 'wave' && ids.test(r.nonce) && ids.test(r.repo_key) && Array.isArray(r.constituents)) return r;
if (r.kind === 'release' && ids.test(r.repo_key) && /^[0-9a-f]{40,64}$/.test(r.sha)) return r;
if (r.kind === 'ready' && ids.test(r.repo_key) && /^\d+$/.test(r.pr) && /^[0-9a-f]{40,64}$/.test(r.head)) return r;
} catch {}
return null;
+26 -7
View File
@@ -19,7 +19,7 @@
* the window (7 days, 200 first-parent commits) is the candidate; a commit
* with a check run still going is skipped, not waited for.
*
* Gated means one of two kinds of evidence. Either the backstop check run
* Gated means one of three kinds of evidence. Either the backstop check run
* (--backstop-check, default main-backstop) succeeded: evals-main-backstop.yml
* publishes it, success only when a green train record covers the tree or the
* backstop gated it green, neutral when main moved past the push before it was
@@ -27,8 +27,11 @@
* or merged) lists the commit's sha or its tree in merged[]: those entries are
* the main commits `gstack-ship-queue verify --train` confirmed, so a neutral
* or absent backstop run on such a commit is not held against it, and the
* CANDIDATE line says "gated by train <nonce>". A red backstop run stays red
* either way. A commit with neither is not green.
* CANDIDATE line says "gated by train <nonce>". Or a GREEN release record
* (kind release, lib/release-gate.ts) names the commit: "gated by release gate
* run <id>". A red backstop run stays red either way. A commit with none of
* these is not green. A RED release record with no GREEN one on a later
* commit that contains it holds every cut, like a freeze.
*
* stable moves only forward: a candidate behind the current stable is refused
* (REFUSED non-fast-forward), as is a tag v<VERSION> that already names another
@@ -99,8 +102,8 @@ export function compareVersions(a: string, b: string): number {
export type Verdict = { green: true; via: string } | { green: false; why: string; pending: boolean };
interface CheckRun { name: string; status: string; conclusion: string | null }
/** Main commits a green, merging or merged train record lists in merged[], by sha and by tree, each naming the train. */
export interface TrainEvidence { bySha: Map<string, string>; byTree: Map<string, string> }
/** Main commits a green, merging or merged train record lists in merged[], by sha and by tree, each naming the train; `release` maps a commit a green release gate passed to its run (lib/release-gate.ts). */
export interface TrainEvidence { bySha: Map<string, string>; byTree: Map<string, string>; release?: Map<string, string> }
export interface GateOptions { requiredCheck?: string; backstopCheck?: string; evidence?: TrainEvidence; treeOf?: (sha: string) => string | null }
/** Why a commit is not releasable, or green and through which evidence. */
@@ -118,6 +121,8 @@ export function gateVerdict(gh: Gh, repo: string, sha: string, o: GateOptions =
if (named.some(c => c.conclusion === 'success')) continue;
const train = name === backstop && name !== required && o.evidence ? o.evidence.bySha.get(sha) ?? o.evidence.byTree.get(o.treeOf?.(sha) ?? '') : undefined;
if (train) { via = `train ${train}`; continue; }
const gate = name === backstop && name !== required ? o.evidence?.release?.get(sha) : undefined;
if (gate) { via = `release gate run ${gate}`; continue; }
if (named.length === 0) return { green: false, why: `no ${name} check run`, pending: false };
return { green: false, why: `${name} concluded ${named.map(c => c.conclusion).join(',')}`, pending: false };
}
@@ -129,14 +134,20 @@ export function gateVerdict(gh: Gh, repo: string, sha: string, o: GateOptions =
}
export interface TrainNotes { error: string | null; freeze: string | null; evidence: TrainEvidence }
interface NoteRecord { kind?: string; state?: string; nonce?: string; merged?: Array<{ sha?: string; tree?: string }>; cleared_at?: string | null; reason?: string; train?: string }
interface NoteRecord {
kind?: string; state?: string; nonce?: string; merged?: Array<{ sha?: string; tree?: string }>; cleared_at?: string | null; reason?: string; train?: string;
sha?: string; verdict?: string; run?: { id?: number } | null; red_cases?: string[];
}
/**
* One read of the trains notes ref (fetched from origin): the active freeze
* record (kind freeze, cleared_at null) and the train evidence (lib/
* ship-queue-train-record.ts TrainRecord: kind train, state green, merging or
* merged, merged[].sha and .tree). A fetch failure other than a missing ref is
* an error; a missing ref holds nothing.
* an error; a missing ref holds nothing. Release-gate records (kind release,
* lib/release-gate.ts) add evidence when GREEN, and a RED one with no GREEN
* record on a later commit that contains it is the hold: it refuses the cut
* like a freeze (a train freeze, when both exist, is the one named).
*/
export function readTrainNotes(git: Git, notesRef = NOTES_REF): TrainNotes {
const notes: TrainNotes = { error: null, freeze: null, evidence: { bySha: new Map(), byTree: new Map() } };
@@ -147,6 +158,8 @@ export function readTrainNotes(git: Git, notesRef = NOTES_REF): TrainNotes {
}
const listed = git(['notes', `--ref=${NOTES_LOCAL}`, 'list']);
if (!listed.ok || !listed.out.trim()) return notes;
const gates: Array<{ sha: string; verdict: string; red: string[] }> = [];
notes.evidence.release = new Map();
for (const line of listed.out.split('\n')) {
const blob = line.split(' ')[0];
if (!blob) continue;
@@ -155,6 +168,10 @@ export function readTrainNotes(git: Git, notesRef = NOTES_REF): TrainNotes {
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) continue;
const r = parsed as NoteRecord;
if (r.kind === 'freeze' && !r.cleared_at) notes.freeze = `queue frozen since train ${r.train ?? '?'}: ${r.reason ?? 'red backstop'}`;
if (r.kind === 'release' && typeof r.sha === 'string' && /^[0-9a-f]{40}$/.test(r.sha)) {
gates.push({ sha: r.sha, verdict: String(r.verdict), red: Array.isArray(r.red_cases) ? r.red_cases.filter(c => typeof c === 'string') : [] });
if (r.verdict === 'GREEN') notes.evidence.release.set(r.sha, String(r.run?.id ?? '?'));
}
if (r.kind !== 'train' || typeof r.nonce !== 'string' || !r.nonce || !['green', 'merging', 'merged'].includes(r.state ?? '')) continue;
for (const m of Array.isArray(r.merged) ? r.merged : []) {
if (!m || typeof m !== 'object') continue;
@@ -162,6 +179,8 @@ export function readTrainNotes(git: Git, notesRef = NOTES_REF): TrainNotes {
if (typeof m.tree === 'string' && m.tree) notes.evidence.byTree.set(m.tree, r.nonce);
}
}
const held = gates.find(red => red.verdict === 'RED' && !gates.some(g => g.verdict === 'GREEN' && g.sha !== red.sha && git(['merge-base', '--is-ancestor', red.sha, g.sha]).ok));
if (held && !notes.freeze) notes.freeze = `release gate red on ${held.sha.slice(0, 12)} (${held.red.join(', ') || 'red cases not recorded'}) and no gate on a later commit is green; land the revert or fix-forward, then run gstack-release-gate run --ignore-cadence`;
return notes;
}
+12 -4
View File
@@ -61,8 +61,14 @@ export type RunMode = 'screen' | 'train' | 'bisect' | 'dispatch';
export interface RouteInput {
nonce: string; refName: string; head: string; expectedSha: string; evalsAll: boolean; trainPolicy: boolean;
cases: string; prefixShas: string; screenPr: string; screenMergeSha: string; screenPolicy: string;
/** The default branch's policy declares a release_gate block (lib/release-gate-policy.ts). */
releasePolicy?: boolean;
}
export interface Route { mode: RunMode; cases: string[]; prefixes: string[] }
/** artifact: the prefix of a release bisect's per-prefix records (a train bisect uses the train block's record_artifact). */
export interface Route { mode: RunMode; cases: string[]; prefixes: string[]; artifact?: string }
/** The release gate's bisect runs on its own ref and names its own prefix records (lib/release-gate.ts RELEASE_REF_PREFIX, RELEASE_BISECT_ARTIFACT). */
export const RELEASE_BISECT_REF = 'release-gate/';
export const RELEASE_BISECT_ARTIFACT = 'gstack-release-record';
const list = (text: string) => text.split(',').map(item => item.trim()).filter(Boolean);
@@ -88,7 +94,8 @@ export function routeDispatch(i: RouteInput, deps: { isAncestor(commit: string,
const cases = list(i.cases);
const prefixes = list(i.prefixShas);
if (scope?.purpose !== 'bisect') return { error: `a bisect needs a bisect nonce (got ${JSON.stringify(i.nonce)})` };
if (!i.refName.startsWith('train/')) return { error: `a bisect runs on train/<nonce>, not ${i.refName}` };
const release = i.refName.startsWith(RELEASE_BISECT_REF);
if (!release && !i.refName.startsWith('train/')) return { error: `a bisect runs on train/<nonce> (or the release gate's ${RELEASE_BISECT_REF}<id>), not ${i.refName}` };
if (i.expectedSha !== i.head) return { error: `expected_sha ${i.expectedSha.slice(0, 12) || '(empty)'} is not the train head ${i.head.slice(0, 12)} checked out from ${i.refName}` };
if (!cases.length || cases.length > MAX_BISECT_CASES || new Set(cases).size !== cases.length) return { error: `cases must name 1 to ${MAX_BISECT_CASES} distinct case ids` };
if (!prefixes.length || prefixes.length > MAX_PREFIXES || new Set(prefixes).size !== prefixes.length) return { error: `prefix_shas must name 1 to ${MAX_PREFIXES} distinct commits` };
@@ -99,6 +106,7 @@ export function routeDispatch(i: RouteInput, deps: { isAncestor(commit: string,
const plan = deps.gateCases();
const unknown = cases.filter(id => !CASE_ID.test(id) || !plan.has(id));
if (unknown.length) return { error: `case(s) not in the full gate's plan: ${unknown.slice(0, 5).join(', ')}` };
if (release) return i.releasePolicy ? { mode: 'bisect', cases, prefixes, artifact: RELEASE_BISECT_ARTIFACT } : { error: "a release bisect needs a release_gate block in the default branch's .gstack/ship-policy.json" };
return trainOnly('bisect') ?? { mode: 'bisect', cases, prefixes };
}
if (scope?.purpose === 'train') {
@@ -359,12 +367,12 @@ if (import.meta.main) {
if (command === 'route') {
const env = process.env;
const route = routeDispatch({ nonce: env.NONCE ?? '', refName: env.GITHUB_REF_NAME ?? '', head: git(['rev-parse', 'HEAD']).out, expectedSha: env.EXPECTED_SHA ?? '',
evalsAll: env.EVALS_ALL === 'true', trainPolicy: env.TRAIN_POLICY === 'true', cases: env.CASES ?? '', prefixShas: env.PREFIX_SHAS ?? '',
evalsAll: env.EVALS_ALL === 'true', trainPolicy: env.TRAIN_POLICY === 'true', releasePolicy: env.RELEASE_POLICY === 'true', cases: env.CASES ?? '', prefixShas: env.PREFIX_SHAS ?? '',
screenPr: env.SCREEN_PR ?? '', screenMergeSha: env.SCREEN_MERGE_SHA ?? '', screenPolicy: env.SCREEN_POLICY ?? '' },
{ isAncestor: (commit, head) => git(['merge-base', '--is-ancestor', commit, head]).ok, gateCases: () => fullGateCases(process.cwd()) });
if ('error' in route) fail(`dispatch refused before any paid work: ${route.error}`);
const ok = route as Route;
output([`mode=${ok.mode}`, `cases=${ok.cases.join(',')}`, `prefixes=${ok.prefixes.join(',')}`]);
output([`mode=${ok.mode}`, `cases=${ok.cases.join(',')}`, `prefixes=${ok.prefixes.join(',')}`, ...(ok.artifact ? [`record_artifact=${ok.artifact}`] : [])]);
console.log(`- Run mode: ${ok.mode}${ok.prefixes.length ? ` (${ok.cases.length} case(s) on ${ok.prefixes.length} prefix(es))` : ''}`);
} else if (command === 'bisect-plan') {
const [cases, prefixes, nonce, out] = [list(flag('cases') ?? ''), list(flag('prefixes') ?? ''), flag('nonce'), flag('out')];
+61 -9
View File
@@ -53,14 +53,18 @@ const UPGRADE_DEPS = [
'bin/gstack-upgrade', 'bin/gstack-bun-version.sh', 'bin/gstack-paths', 'bin/gstack-state-root.sh',
'bin/gstack-egress-lib.sh', 'bin/gstack-egress-receipt', 'lib/egress-receipt.ts', 'lib/state-root.ts',
'bin/gstack-lease', 'bin/gstack-harness-pid.sh',
// Sourced by gstack-upgrade: without gstack-config in the fixture it resolves main unless GSTACK_UPDATE_CHANNEL says stable.
'bin/gstack-update-channel.sh',
];
/**
* cloneAt 'A' (default): the clone sits at A (1.0.0), below B's minimum 1.0.1.
* cloneAt 'B': the clone sits at B, AT the minimum, and the remote moves on to
* C (1.0.2, minimum unchanged): behind origin but not below the floor.
* next: the remote moves on to C for a clone at A too. stable: origin's stable
* branch (the release, docs/releases.md) at that commit.
*/
function fixture(opts: { minWrapper?: number; cloneAt?: 'A' | 'B' } = {}) {
function fixture(opts: { minWrapper?: number; cloneAt?: 'A' | 'B'; next?: boolean; stable?: 'A' | 'B' | 'C' } = {}) {
const host = capyHost({ systemBun: '1.4.2' });
hosts.push(host);
const git = (args: string[], cwd: string) => {
@@ -94,13 +98,14 @@ function fixture(opts: { minWrapper?: number; cloneAt?: 'A' | 'B' } = {}) {
const b = git(['rev-parse', 'HEAD'], seed);
git(['push', '-q', bare, 'main'], seed);
let c = '';
if (opts.cloneAt === 'B') {
git(['clone', '-q', '--single-branch', '--depth', '1', `file://${bare}`, root], host.base);
if (opts.cloneAt === 'B' || opts.next) {
if (opts.cloneAt === 'B') git(['clone', '-q', '--single-branch', '--depth', '1', `file://${bare}`, root], host.base);
write(path.join(seed, 'VERSION'), '1.0.2\n');
git(['commit', '-qam', 'C'], seed);
c = git(['rev-parse', 'HEAD'], seed);
git(['push', '-q', bare, 'main'], seed);
}
if (opts.stable) git(['push', '-q', bare, `${{ A: a, B: b, C: c }[opts.stable]}:refs/heads/stable`], seed);
const env = { ...host.env, GSTACK_CAPY_ROOT: root, GSTACK_CAPY_REPO: `file://${bare}`, WRAP_LOG: log, WRAP_FAIL_ONCE: path.join(host.base, 'fail-once') };
const run = (args: string[] = [], extra: Record<string, string> = {}) => {
const r = spawnSync('bash', [BOOT, ...args], { cwd: host.home, encoding: 'utf8', env: { ...env, ...extra }, timeout: 120_000 });
@@ -140,13 +145,13 @@ describe('capy skill bootstrapper (capy/skill/gstack/scripts/install.sh)', () =>
expect(f.git(['rev-parse', 'HEAD'])).toBe(f.b);
expect(f.git(['rev-parse', '--is-shallow-repository'])).toBe('true');
expect(install.last).toBe('gstack: ok 1.0.1 for=autoplan');
expect(f.calls()).toEqual([`installer --root ${f.root} --for autoplan lock=1 rev=3 min=1.0.1 from=1.0.0`]);
expect(f.calls()).toEqual([`installer --root ${f.root} --for autoplan lock=1 rev=4 min=1.0.1 from=1.0.0`]);
const next = f.run(['--check', '--for', 'autoplan']);
expect(next.status, next.stdout + next.stderr).toBe(0);
expect(next.stdout).not.toContain('upgraded');
expect(next.last).toBe('gstack: ok 1.0.1 for=autoplan');
expect(f.calls().at(-1)).toBe(`installer --check --root ${f.root} --for autoplan lock= rev=3 min=1.0.1 from=`);
expect(f.calls().at(-1)).toBe(`installer --check --root ${f.root} --for autoplan lock= rev=4 min=1.0.1 from=`);
expect(f.git(['rev-parse', 'HEAD'])).toBe(f.b);
});
@@ -263,7 +268,7 @@ describe('capy skill: always current when online (the minimum is the floor, not
expect(r.stdout).toMatch(/^SELF_UPGRADE: done 1\.0\.1 -> 1\.0\.2$/m);
expect(f.git(['rev-parse', 'HEAD'])).toBe(f.c);
expect(r.last).toBe('gstack: ok 1.0.2 for=autoplan');
expect(f.calls()).toEqual([`installer --root ${f.root} --for autoplan lock=1 rev=3 min=1.0.1 from=1.0.1`]);
expect(f.calls()).toEqual([`installer --root ${f.root} --for autoplan lock=1 rev=4 min=1.0.1 from=1.0.1`]);
expect(fs.existsSync(path.join(f.root, '.git/gstack-upgrade.pending'))).toBe(false);
const next = f.run(['--for', 'autoplan']);
@@ -271,7 +276,7 @@ describe('capy skill: always current when online (the minimum is the floor, not
expect(next.stdout).toMatch(/^SELF_UPGRADE: none \(current\)$/m);
expect(next.stdout).not.toContain('SELF_UPGRADE: done');
expect(next.last).toBe('gstack: ok 1.0.2 for=autoplan');
expect(f.calls().at(-1)).toBe(`installer --root ${f.root} --for autoplan lock=1 rev=3 min=1.0.1 from=`);
expect(f.calls().at(-1)).toBe(`installer --root ${f.root} --for autoplan lock=1 rev=4 min=1.0.1 from=`);
});
test('--check mode never moves a behind checkout: it names the gap and the fast-forwarding command, then certifies the installed revision', () => {
@@ -282,7 +287,7 @@ describe('capy skill: always current when online (the minimum is the floor, not
expect(r.stdout).not.toContain('SELF_UPGRADE');
expect(f.git(['rev-parse', 'HEAD'])).toBe(f.b);
expect(r.last).toBe('gstack: ok 1.0.1 for=autoplan');
expect(f.calls()).toEqual([`installer --check --root ${f.root} --for autoplan lock= rev=3 min=1.0.1 from=`]);
expect(f.calls()).toEqual([`installer --check --root ${f.root} --for autoplan lock= rev=4 min=1.0.1 from=`]);
});
test('a dirty behind checkout is skipped with gstack-upgrade\'s reason and still ends ok: above the floor, the trailer is the doctor\'s', () => {
@@ -295,7 +300,7 @@ describe('capy skill: always current when online (the minimum is the floor, not
expect(f.git(['rev-parse', 'HEAD'])).toBe(f.b);
expect(fs.readFileSync(path.join(f.root, 'README.md'), 'utf8')).toBe('local edit\n');
expect(r.last).toBe('gstack: ok 1.0.1 for=autoplan');
expect(f.calls()).toEqual([`installer --root ${f.root} --for autoplan lock=1 rev=3 min=1.0.1 from=`]);
expect(f.calls()).toEqual([`installer --root ${f.root} --for autoplan lock=1 rev=4 min=1.0.1 from=`]);
});
test('GSTACK_CAPY_SELF_UPGRADE=off and GSTACK_CAPY_AUTO_UPGRADE=0 each disable the fast-forward by name and leave HEAD where it is', () => {
@@ -315,6 +320,53 @@ describe('capy skill: always current when online (the minimum is the floor, not
// Plan A9: the `latest:` trailer note comes only from origin's VERSION the
// bootstrapper read in this run; a FETCH_HEAD left by an earlier run never
// reaches the installer.
describe('capy skill bootstrapper: the update channel (wrapper 4, docs/releases.md#channels)', () => {
// Value: protects=an install that follows stable (GSTACK_UPDATE_CHANNEL=stable, or team mode through bin/gstack-update-channel.sh) is cloned at
// origin's stable branch, not main, and stays there through the checkout half's self-upgrade; GSTACK_UPDATE_CHANNEL=main and a remote with no stable
// branch (before the first release) clone main; fails_when=a fresh team install lands on unreleased main, the override cannot pick main, or a missing
// stable branch fails the install; why_new=the clone test covers the default branch only; seam=GSTACK_UPDATE_CHANNEL instead of gstack-config's
// team_mode; no_claim=GitHub's real stable branch
test('a fresh clone follows stable when origin has it; GSTACK_UPDATE_CHANNEL=main and a missing stable branch clone main', () => {
const f = fixture({ cloneAt: 'B', stable: 'B' });
fs.rmSync(f.root, { recursive: true, force: true });
const stable = f.run(['--for', 'autoplan'], { GSTACK_UPDATE_CHANNEL: 'stable' });
expect(stable.status, stable.stdout + stable.stderr).toBe(0);
expect(stable.stdout).toContain('gstack-capy-skill: channel stable 1.0.1 (GSTACK_UPDATE_CHANNEL=main follows every merge)');
expect(f.git(['rev-parse', 'HEAD'])).toBe(f.b);
expect(stable.last).toBe('gstack: ok 1.0.1 for=autoplan');
fs.rmSync(f.root, { recursive: true, force: true });
const main = f.run(['--for', 'autoplan'], { GSTACK_UPDATE_CHANNEL: 'main' });
expect(main.status, main.stdout + main.stderr).toBe(0);
expect(main.stdout).not.toContain('channel stable');
expect(f.git(['rev-parse', 'HEAD'])).toBe(f.c);
spawnSync('git', ['--git-dir', f.bare, 'update-ref', '-d', 'refs/heads/stable'], { env: f.host.env, timeout: 30_000 });
fs.rmSync(f.root, { recursive: true, force: true });
const bootstrap = f.run(['--for', 'autoplan'], { GSTACK_UPDATE_CHANNEL: 'stable' });
expect(bootstrap.status, bootstrap.stdout + bootstrap.stderr).toBe(0);
expect(bootstrap.stdout).toContain('channel stable, but origin has no stable branch yet (or the fetch failed); following main');
expect(f.git(['rev-parse', 'HEAD'])).toBe(f.c);
});
// Value: protects=the bootstrapper's floor fast-forward of a checkout below the supported minimum goes to stable on the stable channel when stable
// meets the minimum, and to main when stable is itself below it; fails_when=a stable install is pushed past the release to main, or left below the
// floor because stable is too old; why_new=the floor tests cover main only; seam=GSTACK_UPDATE_CHANNEL; no_claim=GitHub's real stable branch
test('below the floor: the fast-forward goes to stable when stable meets the minimum, else to main', () => {
const atRelease = fixture({ next: true, stable: 'B' });
const r = atRelease.run(['--for', 'autoplan'], { GSTACK_UPDATE_CHANNEL: 'stable' });
expect(r.status, r.stdout + r.stderr).toBe(0);
expect(r.stdout).toContain('gstack-capy-skill: upgraded 1.0.0 -> 1.0.1');
expect(atRelease.git(['rev-parse', 'HEAD'])).toBe(atRelease.b);
const old = fixture({ next: true, stable: 'A' });
const o = old.run(['--for', 'autoplan'], { GSTACK_UPDATE_CHANNEL: 'stable' });
expect(o.status, o.stdout + o.stderr).toBe(0);
expect(o.stdout).toContain('gstack-capy-skill: channel stable is at 1.0.0, below the supported minimum 1.0.1; following main');
expect(old.git(['rev-parse', 'HEAD'])).toBe(old.c);
});
});
describe('capy skill bootstrapper: GSTACK_CAPY_REMOTE_VERSION', () => {
test('a fresh fetch exports origin\'s VERSION; a failed fetch exports nothing even with a stale FETCH_HEAD', () => {
const f = fixture({ cloneAt: 'B' });
+44
View File
@@ -249,6 +249,50 @@ describe('release-cut: planning', () => {
});
});
describe('release-cut: the release gate holds and gates (lib/release-gate.ts records)', () => {
const release = (sha: string, verdict: string, run: number) => ({ schema: 1, kind: 'release', repo_key: 'k', sha, verdict, run: { id: run, url: 'u' }, red_cases: verdict === 'RED' ? ['case-a'] : [], created_at: 'x', updated_at: 'x' });
const note = (f: ReturnType<typeof fixture>, record: unknown, on: string) => {
git(f.seed, 'notes', '--ref=' + NOTES_REF, 'add', '-f', '-m', JSON.stringify(record), on);
git(f.seed, 'push', '-q', '-f', 'origin', NOTES_REF);
};
// Value: protects=a red release record is the freeze: every cut is refused while no green gate sits on a later commit that contains the red one,
// even when every check run is green, and an incomplete gate or a green gate on an older commit does not lift it; fails_when=release-cut releases a
// commit while the newest release gate is red, a green record anywhere clears the hold, or an INCOMPLETE gate clears it; why_new=the freeze tests know
// only the train freeze record; seam=none; no_claim=real refs/notes/gstack-trains content (fixture records shaped like ReleaseRecord)
test('a red release record refuses the cut; only a green gate on a later commit lifts it', () => {
const f = fixture();
note(f, release(f.shas['1.0.2']!, 'RED', 41), f.shas['1.0.2']!);
note(f, release(f.shas['1.0.1']!, 'GREEN', 40), f.shas['1.0.1']!);
const held = planRelease({ repo: f.repo, git: f.git, gh: f.gh });
expect(held.ok).toBe(false);
if (!held.ok) { expect(held.code).toBe('REFUSED'); expect(held.message).toContain('release gate red on ' + f.shas['1.0.2']!.slice(0, 12)); }
note(f, release(f.shas['1.0.3']!, 'INCOMPLETE', 42), f.shas['1.0.3']!);
expect(planRelease({ repo: f.repo, git: f.git, gh: f.gh }).ok).toBe(false);
note(f, release(f.shas['1.0.3']!, 'GREEN', 43), f.shas['1.0.3']!);
const lifted = planRelease({ repo: f.repo, git: f.git, gh: f.gh });
expect(lifted.ok, JSON.stringify(lifted)).toBe(true);
if (lifted.ok) expect(lifted.plan.sha).toBe(f.shas['1.0.3']);
});
// Value: protects=a green release record gates its commit when the main-backstop check run is absent or neutral (the check publish failed or was
// superseded), and the CANDIDATE line names the gate run; a red check run still keeps the commit out; fails_when=a commit the release gate passed is
// never released without the check, or release evidence overrides a red check run; why_new=train records were the only notes evidence; seam=none;
// no_claim=a live gate run
test('a green release record gates a commit whose main-backstop check never ran; a red check run still wins', () => {
const f = fixture();
const head = f.shas['1.0.3']!;
f.state.checks[head] = GREEN.filter(c => c.name !== BACKSTOP_CHECK);
note(f, release(head, 'GREEN', 77), head);
const gated = planRelease({ repo: f.repo, git: f.git, gh: f.gh });
expect(gated.ok, JSON.stringify(gated)).toBe(true);
if (gated.ok) { expect(gated.plan.sha).toBe(head); expect(gated.plan.via).toBe('release gate run 77'); }
f.state.checks[head] = [...GREEN.filter(c => c.name !== BACKSTOP_CHECK), { name: 'quality', status: 'completed', conclusion: 'failure' }];
const red = planRelease({ repo: f.repo, git: f.git, gh: f.gh });
if (red.ok) expect(red.plan.sha).not.toBe(head);
});
});
describe('release-cut: train evidence gates a commit whose backstop never settled', () => {
const train = (nonce: string, state: string, merged: Array<{ sha: string; tree: string }>) =>
({ schema: 1, kind: 'train', nonce, state, merged: merged.map(m => ({ pr: '1', at: 'x', ...m })), updated_at: 'x' });
+129
View File
@@ -0,0 +1,129 @@
/**
* The release gate's workflow wiring (docs/ship-policy.md#release-gate):
* release-gate.yml runs bin/gstack-release-gate on its schedule and publishes
* the gated commit's check run from a code-free job; the per-push backstop
* stands down once main's policy declares a release_gate block; evals.yml routes
* a release bisect only under that block. Policy steps run for real (bash, git,
* jq) against a scratch repository; the rest is read from the YAML.
*/
import { afterAll, describe, expect, test } from 'bun:test';
import { execFileSync, spawnSync } from 'node:child_process';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { RELEASE_GATE_SCHEDULE_HOURS } from '../lib/release-gate-policy';
import { RELEASE_BISECT_ARTIFACT } from '../scripts/train-gate';
const ROOT = path.resolve(import.meta.dir, '..');
type Step = { id?: string; name?: string; uses?: string; if?: string; run?: string; env?: Record<string, string> };
type Job = { if?: string; needs?: string; outputs?: Record<string, string>; permissions?: Record<string, string>; steps: Step[] };
type Workflow = { on: Record<string, any>; concurrency: unknown; permissions?: unknown; jobs: Record<string, Job> };
const load = (rel: string) => Bun.YAML.parse(fs.readFileSync(path.join(ROOT, rel), 'utf8')) as Workflow;
const release = load('.github/workflows/release-gate.yml');
const backstop = load('.github/workflows/evals-main-backstop.yml');
const evals = load('.github/workflows/evals.yml');
const scratch: string[] = [];
afterAll(() => { for (const d of scratch) fs.rmSync(d, { recursive: true, force: true }); });
const GIT_ENV = { GIT_AUTHOR_NAME: 't', GIT_AUTHOR_EMAIL: 't@x', GIT_COMMITTER_NAME: 't', GIT_COMMITTER_EMAIL: 't@x' };
/** Run a policy step's script in a scratch repo whose one commit holds this policy; returns its GITHUB_OUTPUT lines as an object. */
function runStep(step: Step, policy: unknown): Record<string, string> {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'release-gate-wf-'));
scratch.push(dir);
const g = (...args: string[]) => execFileSync('git', args, { cwd: dir, encoding: 'utf8', timeout: 30_000, env: { ...process.env, ...GIT_ENV } }).trim();
g('init', '-q', '-b', 'main');
if (policy !== undefined) {
fs.mkdirSync(path.join(dir, '.gstack'));
fs.writeFileSync(path.join(dir, '.gstack/ship-policy.json'), JSON.stringify(policy));
}
fs.writeFileSync(path.join(dir, 'README'), 'x');
g('add', '-A');
g('commit', '-q', '-m', 'base');
const sha = g('rev-parse', 'HEAD');
g('update-ref', 'refs/remotes/origin/main', sha);
const out = path.join(dir, 'out');
const r = spawnSync('bash', ['-e', '-c', step.run!], {
cwd: dir, encoding: 'utf8', timeout: 30_000,
env: { ...process.env, GITHUB_OUTPUT: out, GITHUB_STEP_SUMMARY: path.join(dir, 'summary'), GITHUB_SHA: sha, BASE_SHA: '', DEFAULT_BRANCH: 'main' },
});
expect(r.status, r.stderr).toBe(0);
const lines = (fs.existsSync(out) ? fs.readFileSync(out, 'utf8') : '').split('\n').filter(Boolean);
return Object.fromEntries(lines.map(l => [l.slice(0, l.indexOf('=')), l.slice(l.indexOf('=') + 1)]));
}
const GATE = { cadence_hours: 8, gate_workflow: 'evals.yml', branch: 'main', stable_ref: 'stable', check_name: 'main-backstop', on_red: 'hold-release', bisect_spend_cap_usd: 40 };
const TRAIN = { gate_workflow: 'evals.yml', record_artifact: 'gstack-train-record', full_gate_check: 'gstack/full-gate' };
describe('release-gate.yml', () => {
// Value: protects=the schedule fires every RELEASE_GATE_SCHEDULE_HOURS (the cadence floor the policy validates), one gate at a time, only from the
// default branch, with contents and actions write but no checks write, and runs the bin on the run's own sha with its outputs wired; the policy step
// uses git and jq only and is inert without a release_gate block; fails_when=the cron and the cadence drift apart, two gates overlap, the job that
// runs repository code can write check runs, or an inert policy reaches bun; why_new=new workflow; seam=none; no_claim=GitHub's cron jitter
test('schedule, concurrency, permissions and the gate step', () => {
const cron = release.on.schedule[0].cron as string;
expect(cron).toMatch(/^\d+ \*\/\d+ \* \* \*$/);
expect(Number(cron.split(' ')[1]!.slice(2))).toBe(RELEASE_GATE_SCHEDULE_HOURS);
expect(Object.keys(release.on).sort()).toEqual(['schedule', 'workflow_dispatch']);
expect(Object.keys(release.on.workflow_dispatch.inputs).sort()).toEqual(['dry_run', 'ignore_cadence']);
expect(release.concurrency).toEqual({ group: 'release-gate', 'cancel-in-progress': false });
expect(release.permissions).toEqual({});
const job = release.jobs['release-gate']!;
expect(job.if).toBe('github.ref_name == github.event.repository.default_branch');
expect(job.permissions).toEqual({ contents: 'write', actions: 'write', checks: 'read' });
const policy = job.steps.find(s => s.id === 'policy')!;
expect(policy.run).not.toContain('bun');
expect(runStep(policy, undefined)).toEqual({ release: 'false' });
expect(runStep(policy, { train: TRAIN })).toEqual({ release: 'false' });
expect(runStep(policy, { release_gate: GATE })).toEqual({ release: 'true' });
const gate = job.steps.find(s => s.id === 'gate')!;
expect(gate.if).toBe("steps.policy.outputs.release == 'true'");
expect(gate.run).toContain('./bin/gstack-release-gate run --repo "$GITHUB_REPOSITORY" --sha "$GITHUB_SHA"');
expect(gate.run).toContain('--outputs "$GITHUB_OUTPUT"');
expect(job.outputs).toEqual({ action: '${{ steps.gate.outputs.action }}', sha: '${{ steps.gate.outputs.sha }}', check: '${{ steps.gate.outputs.check }}' });
});
// Value: protects=the gated commit's check run (what release-cut reads) comes from a job with checks: write and no checkout or action, only for a
// gate that ran (success for gated, failure for held or failed), on a validated sha and check name; fails_when=repository code runs with checks:
// write, a skipped or inert run publishes a check, or a red gate publishes success; why_new=new workflow; seam=none; no_claim=GitHub's check API
test('the check run is published from a code-free job, success only for gated', () => {
const job = release.jobs['release-check']!;
expect(job.needs).toBe('release-gate');
expect(job.if).toContain("contains(fromJSON('[\"gated\",\"held\",\"failed\"]'), needs.release-gate.outputs.action)");
expect(job.permissions).toEqual({ checks: 'write' });
expect(job.steps.filter(s => s.uses)).toEqual([]);
const run = job.steps[0]!.run!;
expect(run).toContain('gated) conclusion=success ;;');
expect(run).toContain('held|failed) conclusion=failure ;;');
expect(run).toContain('[[ "$SHA" =~ ^[0-9a-f]{40}$ ]]');
expect(run).toContain('-f name="$CHECK" -f head_sha="$SHA"');
expect(run).not.toContain('bun');
});
});
describe('the backstop stands down and evals.yml routes the release bisect', () => {
// Value: protects=while trains exist the per-push backstop keeps gating, and once main declares release_gate it stands down (train=false: no gate,
// no freeze, no check run), so main is never gated twice; fails_when=both gates run on main, or the backstop stops before the release gate is on;
// why_new=new stand-down rule; seam=none; no_claim=a live push
test('evals-main-backstop.yml: train=true with only a train block, train=false once release_gate is declared', () => {
const policy = backstop.jobs.backstop!.steps.find(s => s.id === 'policy')!;
expect(runStep(policy, { train: TRAIN })).toEqual({ train: 'true' });
expect(runStep(policy, { train: TRAIN, release_gate: GATE })).toEqual({ train: 'false' });
});
// Value: protects=the route job reads release_gate from the base revision with git and jq only and hands it to the route as RELEASE_POLICY, and the
// bisect jobs name their artifact from the route's record_artifact first (gstack-release-record for a release bisect); fails_when=a release bisect
// publishes under the train artifact name the gate never reads, or repository code decides the release policy; why_new=new route input;
// seam=none; no_claim=a live bisect run
test('evals.yml: the release policy step, the RELEASE_POLICY input and the record_artifact override', () => {
const route = evals.jobs.route!;
const step = route.steps.find(s => s.id === 'release_policy')!;
expect(step.run).not.toContain('bun');
expect(runStep(step, { release_gate: GATE })).toEqual({ release: 'true' });
expect(runStep(step, { train: TRAIN })).toEqual({ release: 'false' });
expect(runStep(step, undefined)).toEqual({ release: 'false' });
const ids = route.steps.map(s => s.id);
expect(ids.indexOf('release_policy')).toBeLessThan(ids.indexOf('route'));
expect(route.steps.find(s => s.id === 'route')!.env!.RELEASE_POLICY).toBe('${{ steps.release_policy.outputs.release }}');
expect(route.outputs!.record_artifact).toBe('${{ steps.route.outputs.record_artifact || steps.policy.outputs.record_artifact }}');
expect(RELEASE_BISECT_ARTIFACT).toBe('gstack-release-record');
});
});
+301
View File
@@ -0,0 +1,301 @@
/**
* gstack-release-gate (lib/release-gate.ts, docs/ship-policy.md#release-gate):
* the full paid gate on main's head at most once per cadence and only when main
* moved, a release record in refs/notes/gstack-trains, a bisect of a red gate's
* cases over the commits since the last green gate, and the hold release-cut
* reads. GitHub is a fake gh (dispatch, run list, run view, artifact download);
* git, the notes ref and origin are real (a bare origin on disk).
*/
import { afterAll, afterEach, describe, expect, test } from 'bun:test';
import { execFileSync, spawnSync } from 'node:child_process';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { bunIsolationArgs } from '../lib/bun-isolation';
import {
MAX_RELEASE_ATTEMPTS, activeHold, bisectGate, classifyRun, gateStatus, loadReleaseGate, narrow, parseReport, planPrefixes, releaseRecords, runGate,
type GateEnv, type Run,
} from '../lib/release-gate';
import { RELEASE_GATE_SCHEDULE_HOURS, releaseGateErrors, type ReleaseGatePolicy } from '../lib/release-gate-policy';
import { RELEASE_GATE_KEY_VERSION, validatePolicy } from '../lib/ship-policy';
import { NOTES_REF, repoIdentity, type ReleaseRecord } from '../lib/ship-queue-train-record';
import { gitIn } from '../scripts/train-gate';
const ROOT = path.resolve(import.meta.dir, '..');
const GATE: ReleaseGatePolicy = { cadence_hours: 8, gate_workflow: 'evals.yml', branch: 'main', stable_ref: 'stable', check_name: 'main-backstop', on_red: 'hold-release', bisect_spend_cap_usd: 40 };
const roots: string[] = [];
const savedStateRoot = process.env.GSTACK_STATE_ROOT;
afterEach(() => { if (savedStateRoot === undefined) delete process.env.GSTACK_STATE_ROOT; else process.env.GSTACK_STATE_ROOT = savedStateRoot; });
afterAll(() => { for (const r of roots) fs.rmSync(r, { recursive: true, force: true }); });
const ID = { GIT_AUTHOR_NAME: 't', GIT_AUTHOR_EMAIL: 't@x', GIT_COMMITTER_NAME: 't', GIT_COMMITTER_EMAIL: 't@x' };
const git = (cwd: string, ...args: string[]) => execFileSync('git', args, { cwd, encoding: 'utf8', timeout: 30_000, env: { ...process.env, ...ID } }).trim();
/** origin with base (also stable) and c1..c5 on main, and work cloned from it; the state root is the fixture's own. */
function fixture() {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'release-gate-'));
roots.push(root);
const origin = path.join(root, 'origin.git');
const seed = path.join(root, 'seed');
const work = path.join(root, 'work');
git(root, 'init', '-q', '--bare', '-b', 'main', origin);
fs.mkdirSync(seed);
git(seed, 'init', '-q', '-b', 'main');
git(seed, 'remote', 'add', 'origin', origin);
const commit = (name: string) => { fs.writeFileSync(path.join(seed, 'VERSION'), name + '\n'); git(seed, 'add', '-A'); git(seed, 'commit', '-q', '-m', name); git(seed, 'push', '-q', 'origin', 'main'); return git(seed, 'rev-parse', 'HEAD'); };
const base = commit('base');
git(seed, 'push', '-q', 'origin', 'main:stable');
const c = [1, 2, 3, 4, 5].map(i => commit('c' + i));
git(root, 'clone', '-q', origin, work);
process.env.GSTACK_STATE_ROOT = path.join(root, 'state');
return { root, origin, seed, work, base, c, commit };
}
interface Outcome { conclusion: string; artifacts: Record<string, Record<string, string>> }
/** A fake gh: each dispatch becomes a completed run whose outcome the test decides; run list, run view and artifact download read it back. */
function fakeGh(onDispatch: (inputs: Record<string, string>, ref: string) => Outcome) {
const runs: Run[] = [];
const artifacts = new Map<number, Record<string, Record<string, string>>>();
const dispatches: Array<{ ref: string; inputs: Record<string, string> }> = [];
const gh = (args: string[]) => {
const ok = (out: unknown) => ({ status: 0, stdout: typeof out === 'string' ? out : JSON.stringify(out), stderr: '' });
if (args[0] === 'workflow' && args[1] === 'run') {
const ref = args[args.indexOf('--ref') + 1]!;
const inputs = Object.fromEntries(args.flatMap((a, i) => (args[i - 1] === '-f' ? [a.split(/=(.*)/s).slice(0, 2)] : [])));
dispatches.push({ ref, inputs });
const id = 1000 + runs.length;
const o = onDispatch(inputs, ref);
runs.push({ id, status: 'completed', conclusion: o.conclusion, head_sha: inputs.expected_sha!, display_title: 'E2E Evals [eval-bg ' + inputs.nonce + ']', html_url: 'https://github.com/o/r/actions/runs/' + id, run_attempt: 1 });
artifacts.set(id, o.artifacts);
return ok('');
}
if (args[0] === 'api' && args.some(a => /\/actions\/workflows\/[^/]+\/runs$/.test(a))) return ok({ workflow_runs: [...runs].reverse() });
const view = args[0] === 'api' ? /actions\/runs\/(\d+)$/.exec(args[1] ?? '') : null;
if (view) return ok(runs.find(r => r.id === Number(view[1])));
if (args[0] === 'run' && args[1] === 'download') {
const files = artifacts.get(Number(args[2]))?.[args[args.indexOf('-n') + 1]!];
if (!files) return { status: 1, stdout: '', stderr: 'no artifact matches any of the names or patterns provided' };
const dir = args[args.indexOf('-D') + 1]!;
for (const [name, text] of Object.entries(files)) { fs.mkdirSync(dir, { recursive: true }); fs.writeFileSync(path.join(dir, name), text); }
return ok('');
}
return { status: 1, stdout: '', stderr: 'unexpected gh ' + args.join(' ') };
};
return { gh, runs, dispatches };
}
const report = (verdict: 'GREEN' | 'RED', failed: string[] = [], cost = '37.40') => ({ 'report-verdict-a1': { 'report.txt': [
'[test:paid] report: 17/17 slices, 85 planned shards, tier=gate, attempt 1',
...(failed.length ? ['FAILED cases:', ...failed.map(l => ' ✗ ' + l)] : []),
'[test:paid] VERDICT ' + verdict + ' — lane gate/full census, attempt 1',
' wall 41m02s · cost $' + cost + ' known + 4 of 139 trial(s) cost unknown',
].join('\n') } });
const green = (flaky: string[] = []): Outcome => ({ conclusion: 'success', artifacts: { ...report('GREEN'), ...(flaky.length ? { 'flake-records-a1': { 'flake-records.jsonl': flaky.map(c => JSON.stringify({ schema: 1, kind: 'gate-flake', case: c })).join('\n') } } : {}) } });
function envFor(f: ReturnType<typeof fixture>, gh: (args: string[]) => { status: number; stdout: string; stderr: string }, shiftHours = 0): GateEnv {
return { repo: 'o/r', repoRoot: f.work, remote: 'origin', gh, git: gitIn(f.work), now: () => Date.now() + shiftHours * 3_600_000, sleep: () => {}, random: () => 'abc123', pollMs: 1, deadline: Date.now() + 3_600_000 };
}
const records = (f: ReturnType<typeof fixture>) => {
git(f.work, 'fetch', '-q', 'origin', '+' + NOTES_REF + ':' + NOTES_REF);
return releaseRecords(gitIn(f.work), repoIdentity(f.work).key);
};
describe('release gate: run', () => {
// Value: protects=a gate only when main moved: the first run dispatches the full gate once on main's head (evals_all, expected_sha, a backstop nonce),
// records GREEN with its run, cost and rule-16 flaky cases in origin's notes and prints the RELEASE green line without moving stable; a second run on
// the same head, and a head equal to stable, skip without dispatching; fails_when=the gate dispatches again on an unchanged main (a $37 repeat), a
// green gate moves stable, or the record never reaches origin; why_new=new module; seam=fake gh; no_claim=a live gate run (the PR's one live proof)
test('green: one dispatch on the moved head, a GREEN release record on origin, stable untouched; an unchanged main skips', () => {
const f = fixture();
const fake = fakeGh(() => green(['judge-x']));
const head = f.c[4]!;
const out = runGate(envFor(f, fake.gh), { gate: GATE, label: 'test', dryRun: false, ignoreCadence: false });
expect(out.action, out.lines.join('\n')).toBe('gated');
expect(fake.dispatches).toHaveLength(1);
expect(fake.dispatches[0]!.ref).toBe('main');
expect(fake.dispatches[0]!.inputs).toMatchObject({ evals_all: 'true', expected_sha: head });
expect(fake.dispatches[0]!.inputs.nonce).toMatch(/^gate\.backstop-/);
expect(out.lines).toContain('RELEASE: green sha=' + head + ' stable=' + f.base + ' run=1000 cost=37.40');
expect(out.lines.some(l => l.includes('flaky (failed once, passed its one rule-16 rerun in run 1000): judge-x'))).toBe(true);
const [rec] = records(f);
expect(rec).toMatchObject({ kind: 'release', sha: head, verdict: 'GREEN', flaky: ['judge-x'], cost_usd: 37.4, since: f.base, attempts: 1, run: { id: 1000 } });
expect(git(f.seed, 'ls-remote', 'origin', 'refs/heads/stable').split('\t')[0]).toBe(f.base);
const again = runGate(envFor(f, fake.gh, 9), { gate: GATE, label: 'test', dryRun: false, ignoreCadence: true });
expect(again.action).toBe('skipped');
expect(again.lines[0]).toContain('main has not moved since the last gate: GREEN run 1000');
git(f.seed, 'push', '-q', '-f', 'origin', 'main:stable');
const atStable = runGate(envFor(f, fake.gh), { gate: GATE, label: 'test', dryRun: false, ignoreCadence: true });
expect(atStable.lines[0]).toContain('main has not moved since stable');
expect(fake.dispatches).toHaveLength(1);
});
// Value: protects=without a release_gate block nothing runs; --dry-run names the head, the base and the pending count and writes and dispatches
// nothing; the cadence holds a moved main until cadence_hours minus the grace pass, and --ignore-cadence gates now; fails_when=an inert policy spends,
// a dry run writes a record, or the schedule gates more often than the policy says; why_new=new module; seam=fake gh and a shifted clock;
// no_claim=GitHub's cron timing
test('inert, dry run and cadence: nothing is spent until the policy and the clock allow it', () => {
const f = fixture();
const fake = fakeGh(() => green());
expect(runGate(envFor(f, fake.gh), { gate: null, label: 'origin/main@x', dryRun: false, ignoreCadence: false }).action).toBe('inert');
const dry = runGate(envFor(f, fake.gh), { gate: GATE, label: 'test', dryRun: true, ignoreCadence: false });
expect(dry.action).toBe('dry-run');
expect(dry.lines[0]).toBe('RELEASE: gate sha=' + f.c[4] + ' since=' + f.base.slice(0, 12) + ' (stable) pending=5 stable=' + f.base.slice(0, 12));
expect(fake.dispatches).toHaveLength(0);
expect(spawnSync('git', ['ls-remote', 'origin', NOTES_REF], { cwd: f.work, encoding: 'utf8', timeout: 30_000 }).stdout).toBe('');
expect(runGate(envFor(f, fake.gh), { gate: GATE, label: 'test', dryRun: false, ignoreCadence: false }).action).toBe('gated');
const next = f.commit('c6');
const early = runGate(envFor(f, fake.gh, 2), { gate: GATE, label: 'test', dryRun: false, ignoreCadence: false });
expect(early.action).toBe('skipped');
expect(early.lines[0]).toMatch(/the last gate was dispatched 2\.0 h ago on [0-9a-f]{12}; cadence 8 h/);
const due = runGate(envFor(f, fake.gh, 7.5), { gate: GATE, label: 'test', dryRun: false, ignoreCadence: false });
expect(due.action, due.lines.join('\n')).toBe('gated');
expect(fake.dispatches.map(d => d.inputs.expected_sha)).toEqual([f.c[4], next]);
expect(records(f).find(r => r.sha === next)?.since).toBe(f.c[4]);
});
// Value: protects=an INCOMPLETE census (only INFRA reds) is recorded as such, re-dispatched once by the next run (EVAL_POLICY.infraRedispatch), then
// skipped with its run named, and never treated as green or red; fails_when=an infra-only failure holds the release like a red, is retried without
// end, or is never retried; why_new=new module; seam=fake gh; no_claim=a real runner loss
test('incomplete: recorded, re-dispatched once, then skipped with the run named', () => {
const f = fixture();
const fake = fakeGh(() => ({ conclusion: 'failure', artifacts: report('RED', ['case-a rule FAIL INFRA']) }));
const env = envFor(f, fake.gh);
const first = runGate(env, { gate: GATE, label: 'test', dryRun: false, ignoreCadence: true });
expect(first.action).toBe('failed');
expect(first.lines.at(-1)).toContain('RELEASE: incomplete sha=' + f.c[4] + ' run=1000');
expect(first.lines.at(-1)).toContain('the next run re-dispatches it once');
expect(runGate(env, { gate: GATE, label: 'test', dryRun: false, ignoreCadence: true }).action).toBe('failed');
const third = runGate(env, { gate: GATE, label: 'test', dryRun: false, ignoreCadence: true });
expect(third.action).toBe('skipped');
expect(third.lines[0]).toContain('incomplete ' + MAX_RELEASE_ATTEMPTS + ' times');
expect(fake.dispatches).toHaveLength(MAX_RELEASE_ATTEMPTS);
expect(activeHold(records(f), () => true)).toBeNull();
});
});
describe('release gate: red, bisect and the hold', () => {
// Value: protects=a red gate records RED (the hold release-cut reads), bisects only its red verdict cases on release-gate/<id> with prefixes between
// stable and the red head, narrows round by round to the first red commit after a green one, deletes the bisect branch, and prints first_bad; a later
// green gate on a newer commit lifts the hold; fails_when=a red gate moves or leaves stable releasable, the bisect blames the wrong commit, tests the
// known-red head again, leaks its branch, or a green gate on an older commit lifts the hold; why_new=new module; seam=fake gh; no_claim=a live bisect
test('red: RED record, a bisect that names first_bad, the branch cleaned up, and the hold until a later green', () => {
const f = fixture();
const order = [f.base, ...f.c];
const fake = fakeGh((inputs) => {
if (!inputs.prefix_shas) return { conclusion: 'failure', artifacts: report('RED', ['case-a rule FAIL VERDICT', 'case-z rule FAIL INFRA']) };
const files: Record<string, Record<string, string>> = {};
for (const sha of inputs.prefix_shas.split(',')) {
const red = order.indexOf(sha) >= 3;
files['gstack-release-record-bisect-' + sha.slice(0, 12)] = { 'bisect-prefix.json': JSON.stringify({ schema: 1, kind: 'train-bisect-prefix', nonce: inputs.nonce, prefix_sha: sha, checked_out_sha: sha,
panels: [{ case: 'case-a', kind: 'rule', status: red ? 'FAIL' : 'PASS', red_class: red ? 'VERDICT' : null, cost_usd: 1.25 }] }) };
}
return { conclusion: 'success', artifacts: files };
});
const out = runGate(envFor(f, fake.gh), { gate: GATE, label: 'test', dryRun: false, ignoreCadence: true });
expect(out.action, out.lines.join('\n')).toBe('held');
expect(out.lines).toContain('RELEASE: red sha=' + f.c[4] + ' first_bad=' + f.c[2] + ' run=1000');
expect(fake.dispatches).toHaveLength(2);
const bisect = fake.dispatches[1]!;
expect(bisect.ref).toMatch(/^release-gate\/abc123-[0-9a-f]{12}$/);
expect(bisect.inputs).toMatchObject({ cases: 'case-a', prefix_shas: f.c.slice(0, 4).join(','), expected_sha: f.c[4] });
expect(bisect.inputs.nonce).toMatch(/^gate\.bisect-/);
expect(git(f.seed, 'ls-remote', 'origin', 'refs/heads/release-gate/*')).toBe('');
const recs = records(f);
expect(recs[0]).toMatchObject({ verdict: 'RED', red_cases: ['case-a'], bisect: { state: 'found', first_bad: f.c[2], good: f.c[1], bad: f.c[2], spent_usd: 5 } });
expect(git(f.seed, 'ls-remote', 'origin', 'refs/heads/stable').split('\t')[0]).toBe(f.base);
const isAncestor = (a: string, b: string) => spawnSync('git', ['merge-base', '--is-ancestor', a, b], { cwd: f.work, timeout: 30_000 }).status === 0;
expect(activeHold(recs, isAncestor)?.sha).toBe(f.c[4]);
const status = gateStatus(envFor(f, fake.gh), { gate: GATE, label: 'test' });
expect(status.action).toBe('held');
expect(status.lines.find(l => l.startsWith('HOLD:'))).toContain('red on ' + f.c[4]);
const older: ReleaseRecord = { ...recs[0]!, sha: f.c[1]!, verdict: 'GREEN', bisect: null };
expect(activeHold([...recs, older], isAncestor)?.sha).toBe(f.c[4]);
const later: ReleaseRecord = { ...recs[0]!, sha: f.commit('fix'), verdict: 'GREEN', bisect: null };
git(f.work, 'fetch', '-q', 'origin');
expect(activeHold([later, ...recs], isAncestor)).toBeNull();
});
// Value: protects=the bisect spends within its cap: a 0 cap records no bisect and first_bad unknown, a small cap tests only the prefixes it covers
// and stops capped, and bisect --spend-cap resumes a capped one; fails_when=a bisect dispatches past its cap or a capped bisect cannot be resumed;
// why_new=new module; seam=fake gh; no_claim=real case prices
test('bisect spend: a zero cap skips it, a small cap stops capped, and bisect --spend-cap resumes', () => {
const f = fixture();
const fake = fakeGh((inputs) => (inputs.prefix_shas ? { conclusion: 'success', artifacts: {} } : { conclusion: 'failure', artifacts: report('RED', ['case-a rule FAIL VERDICT', 'case-b judge FAIL']) }));
const none = runGate(envFor(f, fake.gh), { gate: { ...GATE, bisect_spend_cap_usd: 0 }, label: 'test', dryRun: false, ignoreCadence: true });
expect(none.lines).toContain('RELEASE: red sha=' + f.c[4] + ' first_bad=unknown run=1000');
expect(none.lines.some(l => l.includes('bisect unknown: bisect_spend_cap_usd is 0'))).toBe(true);
expect(fake.dispatches).toHaveLength(1);
const g = fixture();
const fake2 = fakeGh((inputs) => (inputs.prefix_shas ? { conclusion: 'success', artifacts: {} } : { conclusion: 'failure', artifacts: report('RED', ['case-a rule FAIL VERDICT', 'case-b judge FAIL']) }));
const capped = runGate(envFor(g, fake2.gh), { gate: { ...GATE, bisect_spend_cap_usd: 2 }, label: 'test', dryRun: false, ignoreCadence: true });
expect(capped.lines.some(l => l.includes('bisect capped: the next round needs $3.00 a prefix'))).toBe(true);
expect(fake2.dispatches).toHaveLength(1);
const plan = bisectGate(envFor(g, fake2.gh), { gate: GATE, label: 'test', spendCap: 30, dryRun: true });
expect(plan.lines.some(l => l.startsWith('RELEASE: bisect round 1: 2 case(s) on 4 of 4 commit(s)'))).toBe(true);
expect(fake2.dispatches).toHaveLength(1);
});
});
describe('release gate: verdict, range and policy units', () => {
test('parseReport and classifyRun: verdict reds bisect, other classes keep a red incomplete, success is green, and a contradiction is incomplete', () => {
const r = parseReport(report('RED', ['case-a rule FAIL VERDICT', 'judge one judge FAIL', 'case-z rule FAIL INFRA'])['report-verdict-a1']['report.txt']);
expect(r).toEqual({ verdict: 'RED', red: ['case-a', 'judge one'], other: ['case-z'], cost: 37.4 });
expect(classifyRun({ status: 'completed', conclusion: 'failure' }, r)).toMatchObject({ verdict: 'RED', red: ['case-a', 'judge one'] });
expect(classifyRun({ status: 'completed', conclusion: 'failure' }, { ...r, red: [] }).verdict).toBe('INCOMPLETE');
expect(classifyRun({ status: 'completed', conclusion: 'cancelled' }, null).verdict).toBe('INCOMPLETE');
expect(classifyRun({ status: 'completed', conclusion: 'success' }, { ...r, verdict: 'GREEN' }).verdict).toBe('GREEN');
expect(classifyRun({ status: 'completed', conclusion: 'success' }, r).verdict).toBe('INCOMPLETE');
expect(classifyRun({ status: 'in_progress', conclusion: null }, null).verdict).toBe('PENDING');
});
test('planPrefixes spreads at most MAX_PREFIXES over the interior; narrow moves good and bad and refuses a non-monotone or missing result', () => {
const shas = Array.from({ length: 20 }, (_, i) => String(i).padStart(40, '0'));
expect(planPrefixes(shas.slice(0, 3))).toEqual(shas.slice(0, 3));
const picked = planPrefixes(shas, 7);
expect(picked).toHaveLength(7);
expect(new Set(picked).size).toBe(7);
expect(picked[0]).not.toBe(shas[0]);
const t = (rs: string[]) => rs.map((r, i) => ({ sha: shas[i]!, results: r === '?' ? {} : { a: r === 'R' ? 'red' as const : 'green' as const } }));
expect(narrow({ good: 'g', bad: 'b' }, t(['G', 'G', 'R']), ['a'])).toEqual({ good: shas[1], bad: shas[2] });
expect(narrow({ good: 'g', bad: 'b' }, t(['G', 'G']), ['a'])).toEqual({ good: shas[1], bad: 'b' });
expect(narrow({ good: 'g', bad: 'b' }, t(['R', 'R']), ['a'])).toEqual({ good: 'g', bad: shas[0] });
expect(narrow({ good: 'g', bad: 'b' }, t(['G', 'R', 'G']), ['a'])).toEqual({ unknown: 'a is not monotone over the tested commits (GRG)' });
expect(narrow({ good: 'g', bad: 'b' }, t(['G', '?']), ['a'])).toHaveProperty('unknown');
});
// Value: protects=the release_gate key is read only with min_gstack_version at its introducing version, validates every field (cadence at least the
// workflow schedule, a workflow file, branch names, hold-release), and --policy runs the same validation; fails_when=an older gstack reads the block
// silently, a bad field passes, or the command-line policy skips validation; why_new=new key; seam=none; no_claim=the json adoption (a follow-up PR)
test('policy: the release_gate key needs its min_gstack_version and every field; --policy is validated the same way', () => {
const raw = { release_gate: GATE };
expect(validatePolicy(raw).map(e => e.path)).toEqual(['release_gate']);
expect(validatePolicy({ ...raw, min_gstack_version: RELEASE_GATE_KEY_VERSION })).toEqual([]);
expect(releaseGateErrors({ ...GATE, cadence_hours: RELEASE_GATE_SCHEDULE_HOURS - 1, on_red: 'freeze', stable_ref: 'main', extra: 1 }).map(e => e.path).sort())
.toEqual(['release_gate.cadence_hours', 'release_gate.extra', 'release_gate.on_red', 'release_gate.stable_ref']);
expect(releaseGateErrors({ cadence_hours: 8 }).filter(e => e.message.startsWith('missing'))).toHaveLength(6);
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'release-gate-policy-'));
roots.push(dir);
const good = path.join(dir, 'good.json');
fs.writeFileSync(good, JSON.stringify({ ...raw, min_gstack_version: RELEASE_GATE_KEY_VERSION }));
expect(loadReleaseGate(ROOT, { policyFile: good })).toEqual({ gate: GATE, label: good });
const bad = path.join(dir, 'bad.json');
fs.writeFileSync(bad, JSON.stringify(raw));
expect((loadReleaseGate(ROOT, { policyFile: bad }) as { error: string }).error).toContain('release_gate introduced in gstack ' + RELEASE_GATE_KEY_VERSION);
});
test('bin/gstack-release-gate: usage exits 2; an inert policy file exits 0 without touching GitHub', () => {
const bin = path.join(ROOT, 'bin/gstack-release-gate');
const usage = spawnSync('bun', [...bunIsolationArgs(), bin, 'frobnicate'], { cwd: ROOT, encoding: 'utf8', timeout: 60_000 });
expect(usage.status).toBe(2);
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'release-gate-bin-'));
roots.push(dir);
const inert = path.join(dir, 'policy.json');
fs.writeFileSync(inert, '{}');
const r = spawnSync('bun', [...bunIsolationArgs(), bin, 'run', '--repo', 'o/r', '--policy', inert], { cwd: ROOT, encoding: 'utf8', timeout: 60_000 });
expect(r.status, r.stderr).toBe(0);
expect(r.stdout.trim()).toBe('RELEASE: inert (' + inert + ' declares no release_gate block; the per-push main backstop gates main)');
});
});
+14
View File
@@ -54,6 +54,20 @@ describe('routeDispatch: every run keeps its mode, and train inputs are validate
expect((route({ ...screen, trainPolicy: false }) as { error: string }).error).toContain('needs a train block');
});
// Value: protects=the release gate's bisect (lib/release-gate.ts) routes on its own release-gate/<id> ref under a release_gate policy, names its own
// prefix artifact, keeps every train bisect check (nonce, head, cases, prefixes), and is refused without the policy or on any other ref;
// fails_when=a release bisect needs a train block (it stops working once trains go), a release-gate/ ref routes without a release_gate policy, or a
// train/ bisect starts accepting the release policy alone; why_new=the bisect test below covers train/<nonce> only; seam=none; no_claim=a live run
test('a release bisect runs on release-gate/<id> under a release_gate policy, with the train checks and its own artifact', () => {
const bisect = { nonce: bisectNonce, refName: 'release-gate/abc123-aaaaaaaaaaaa', expectedSha: A, cases: 'case-a', prefixShas: B, releasePolicy: true };
expect(route(bisect)).toEqual({ mode: 'bisect', cases: ['case-a'], prefixes: [B], artifact: 'gstack-release-record' });
expect((route({ ...bisect, releasePolicy: false, trainPolicy: true }) as { error: string }).error).toContain('needs a release_gate block');
expect((route({ ...bisect, prefixShas: C }) as { error: string }).error).toContain('is not an ancestor');
expect((route({ ...bisect, expectedSha: B }) as { error: string }).error).toContain('is not the train head');
expect((route({ ...bisect, refName: 'train/t1', trainPolicy: false }) as { error: string }).error).toContain('needs a train block');
expect((route({ ...bisect, refName: 'main' }) as { error: string }).error).toContain('runs on train/<nonce>');
});
test('a bisect needs a bisect nonce on train/<nonce>, the train head, plan case ids and ancestor prefixes', () => {
const bisect = { nonce: bisectNonce, refName: 'train/t1', expectedSha: A, cases: 'case-a, review/SKILL.md workflow', prefixShas: `${A},${B}`, trainPolicy: true };
expect(route(bisect)).toEqual({ mode: 'bisect', cases: ['case-a', 'review/SKILL.md workflow'], prefixes: [A, B] });