mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-07 09:23:50 +01:00
feat(release-gate): gate main every 8 hours and feed the stable release (inert until adopted)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
923fc0a6b8
commit
2dcbe9dbbd
20 files changed
+1415
-30
No files matched your search
@@ -53,14 +53,18 @@ const UPGRADE_DEPS = [
|
||||
'bin/gstack-upgrade', 'bin/gstack-bun-version.sh', 'bin/gstack-paths', 'bin/gstack-state-root.sh',
|
||||
'bin/gstack-egress-lib.sh', 'bin/gstack-egress-receipt', 'lib/egress-receipt.ts', 'lib/state-root.ts',
|
||||
'bin/gstack-lease', 'bin/gstack-harness-pid.sh',
|
||||
// Sourced by gstack-upgrade: without gstack-config in the fixture it resolves main unless GSTACK_UPDATE_CHANNEL says stable.
|
||||
'bin/gstack-update-channel.sh',
|
||||
];
|
||||
|
||||
/**
|
||||
* cloneAt 'A' (default): the clone sits at A (1.0.0), below B's minimum 1.0.1.
|
||||
* cloneAt 'B': the clone sits at B, AT the minimum, and the remote moves on to
|
||||
* C (1.0.2, minimum unchanged): behind origin but not below the floor.
|
||||
* next: the remote moves on to C for a clone at A too. stable: origin's stable
|
||||
* branch (the release, docs/releases.md) at that commit.
|
||||
*/
|
||||
function fixture(opts: { minWrapper?: number; cloneAt?: 'A' | 'B' } = {}) {
|
||||
function fixture(opts: { minWrapper?: number; cloneAt?: 'A' | 'B'; next?: boolean; stable?: 'A' | 'B' | 'C' } = {}) {
|
||||
const host = capyHost({ systemBun: '1.4.2' });
|
||||
hosts.push(host);
|
||||
const git = (args: string[], cwd: string) => {
|
||||
@@ -94,13 +98,14 @@ function fixture(opts: { minWrapper?: number; cloneAt?: 'A' | 'B' } = {}) {
|
||||
const b = git(['rev-parse', 'HEAD'], seed);
|
||||
git(['push', '-q', bare, 'main'], seed);
|
||||
let c = '';
|
||||
if (opts.cloneAt === 'B') {
|
||||
git(['clone', '-q', '--single-branch', '--depth', '1', `file://${bare}`, root], host.base);
|
||||
if (opts.cloneAt === 'B' || opts.next) {
|
||||
if (opts.cloneAt === 'B') git(['clone', '-q', '--single-branch', '--depth', '1', `file://${bare}`, root], host.base);
|
||||
write(path.join(seed, 'VERSION'), '1.0.2\n');
|
||||
git(['commit', '-qam', 'C'], seed);
|
||||
c = git(['rev-parse', 'HEAD'], seed);
|
||||
git(['push', '-q', bare, 'main'], seed);
|
||||
}
|
||||
if (opts.stable) git(['push', '-q', bare, `${{ A: a, B: b, C: c }[opts.stable]}:refs/heads/stable`], seed);
|
||||
const env = { ...host.env, GSTACK_CAPY_ROOT: root, GSTACK_CAPY_REPO: `file://${bare}`, WRAP_LOG: log, WRAP_FAIL_ONCE: path.join(host.base, 'fail-once') };
|
||||
const run = (args: string[] = [], extra: Record<string, string> = {}) => {
|
||||
const r = spawnSync('bash', [BOOT, ...args], { cwd: host.home, encoding: 'utf8', env: { ...env, ...extra }, timeout: 120_000 });
|
||||
@@ -140,13 +145,13 @@ describe('capy skill bootstrapper (capy/skill/gstack/scripts/install.sh)', () =>
|
||||
expect(f.git(['rev-parse', 'HEAD'])).toBe(f.b);
|
||||
expect(f.git(['rev-parse', '--is-shallow-repository'])).toBe('true');
|
||||
expect(install.last).toBe('gstack: ok 1.0.1 for=autoplan');
|
||||
expect(f.calls()).toEqual([`installer --root ${f.root} --for autoplan lock=1 rev=3 min=1.0.1 from=1.0.0`]);
|
||||
expect(f.calls()).toEqual([`installer --root ${f.root} --for autoplan lock=1 rev=4 min=1.0.1 from=1.0.0`]);
|
||||
|
||||
const next = f.run(['--check', '--for', 'autoplan']);
|
||||
expect(next.status, next.stdout + next.stderr).toBe(0);
|
||||
expect(next.stdout).not.toContain('upgraded');
|
||||
expect(next.last).toBe('gstack: ok 1.0.1 for=autoplan');
|
||||
expect(f.calls().at(-1)).toBe(`installer --check --root ${f.root} --for autoplan lock= rev=3 min=1.0.1 from=`);
|
||||
expect(f.calls().at(-1)).toBe(`installer --check --root ${f.root} --for autoplan lock= rev=4 min=1.0.1 from=`);
|
||||
expect(f.git(['rev-parse', 'HEAD'])).toBe(f.b);
|
||||
});
|
||||
|
||||
@@ -263,7 +268,7 @@ describe('capy skill: always current when online (the minimum is the floor, not
|
||||
expect(r.stdout).toMatch(/^SELF_UPGRADE: done 1\.0\.1 -> 1\.0\.2$/m);
|
||||
expect(f.git(['rev-parse', 'HEAD'])).toBe(f.c);
|
||||
expect(r.last).toBe('gstack: ok 1.0.2 for=autoplan');
|
||||
expect(f.calls()).toEqual([`installer --root ${f.root} --for autoplan lock=1 rev=3 min=1.0.1 from=1.0.1`]);
|
||||
expect(f.calls()).toEqual([`installer --root ${f.root} --for autoplan lock=1 rev=4 min=1.0.1 from=1.0.1`]);
|
||||
expect(fs.existsSync(path.join(f.root, '.git/gstack-upgrade.pending'))).toBe(false);
|
||||
|
||||
const next = f.run(['--for', 'autoplan']);
|
||||
@@ -271,7 +276,7 @@ describe('capy skill: always current when online (the minimum is the floor, not
|
||||
expect(next.stdout).toMatch(/^SELF_UPGRADE: none \(current\)$/m);
|
||||
expect(next.stdout).not.toContain('SELF_UPGRADE: done');
|
||||
expect(next.last).toBe('gstack: ok 1.0.2 for=autoplan');
|
||||
expect(f.calls().at(-1)).toBe(`installer --root ${f.root} --for autoplan lock=1 rev=3 min=1.0.1 from=`);
|
||||
expect(f.calls().at(-1)).toBe(`installer --root ${f.root} --for autoplan lock=1 rev=4 min=1.0.1 from=`);
|
||||
});
|
||||
|
||||
test('--check mode never moves a behind checkout: it names the gap and the fast-forwarding command, then certifies the installed revision', () => {
|
||||
@@ -282,7 +287,7 @@ describe('capy skill: always current when online (the minimum is the floor, not
|
||||
expect(r.stdout).not.toContain('SELF_UPGRADE');
|
||||
expect(f.git(['rev-parse', 'HEAD'])).toBe(f.b);
|
||||
expect(r.last).toBe('gstack: ok 1.0.1 for=autoplan');
|
||||
expect(f.calls()).toEqual([`installer --check --root ${f.root} --for autoplan lock= rev=3 min=1.0.1 from=`]);
|
||||
expect(f.calls()).toEqual([`installer --check --root ${f.root} --for autoplan lock= rev=4 min=1.0.1 from=`]);
|
||||
});
|
||||
|
||||
test('a dirty behind checkout is skipped with gstack-upgrade\'s reason and still ends ok: above the floor, the trailer is the doctor\'s', () => {
|
||||
@@ -295,7 +300,7 @@ describe('capy skill: always current when online (the minimum is the floor, not
|
||||
expect(f.git(['rev-parse', 'HEAD'])).toBe(f.b);
|
||||
expect(fs.readFileSync(path.join(f.root, 'README.md'), 'utf8')).toBe('local edit\n');
|
||||
expect(r.last).toBe('gstack: ok 1.0.1 for=autoplan');
|
||||
expect(f.calls()).toEqual([`installer --root ${f.root} --for autoplan lock=1 rev=3 min=1.0.1 from=`]);
|
||||
expect(f.calls()).toEqual([`installer --root ${f.root} --for autoplan lock=1 rev=4 min=1.0.1 from=`]);
|
||||
});
|
||||
|
||||
test('GSTACK_CAPY_SELF_UPGRADE=off and GSTACK_CAPY_AUTO_UPGRADE=0 each disable the fast-forward by name and leave HEAD where it is', () => {
|
||||
@@ -315,6 +320,53 @@ describe('capy skill: always current when online (the minimum is the floor, not
|
||||
// Plan A9: the `latest:` trailer note comes only from origin's VERSION the
|
||||
// bootstrapper read in this run; a FETCH_HEAD left by an earlier run never
|
||||
// reaches the installer.
|
||||
describe('capy skill bootstrapper: the update channel (wrapper 4, docs/releases.md#channels)', () => {
|
||||
// Value: protects=an install that follows stable (GSTACK_UPDATE_CHANNEL=stable, or team mode through bin/gstack-update-channel.sh) is cloned at
|
||||
// origin's stable branch, not main, and stays there through the checkout half's self-upgrade; GSTACK_UPDATE_CHANNEL=main and a remote with no stable
|
||||
// branch (before the first release) clone main; fails_when=a fresh team install lands on unreleased main, the override cannot pick main, or a missing
|
||||
// stable branch fails the install; why_new=the clone test covers the default branch only; seam=GSTACK_UPDATE_CHANNEL instead of gstack-config's
|
||||
// team_mode; no_claim=GitHub's real stable branch
|
||||
test('a fresh clone follows stable when origin has it; GSTACK_UPDATE_CHANNEL=main and a missing stable branch clone main', () => {
|
||||
const f = fixture({ cloneAt: 'B', stable: 'B' });
|
||||
fs.rmSync(f.root, { recursive: true, force: true });
|
||||
const stable = f.run(['--for', 'autoplan'], { GSTACK_UPDATE_CHANNEL: 'stable' });
|
||||
expect(stable.status, stable.stdout + stable.stderr).toBe(0);
|
||||
expect(stable.stdout).toContain('gstack-capy-skill: channel stable 1.0.1 (GSTACK_UPDATE_CHANNEL=main follows every merge)');
|
||||
expect(f.git(['rev-parse', 'HEAD'])).toBe(f.b);
|
||||
expect(stable.last).toBe('gstack: ok 1.0.1 for=autoplan');
|
||||
|
||||
fs.rmSync(f.root, { recursive: true, force: true });
|
||||
const main = f.run(['--for', 'autoplan'], { GSTACK_UPDATE_CHANNEL: 'main' });
|
||||
expect(main.status, main.stdout + main.stderr).toBe(0);
|
||||
expect(main.stdout).not.toContain('channel stable');
|
||||
expect(f.git(['rev-parse', 'HEAD'])).toBe(f.c);
|
||||
|
||||
spawnSync('git', ['--git-dir', f.bare, 'update-ref', '-d', 'refs/heads/stable'], { env: f.host.env, timeout: 30_000 });
|
||||
fs.rmSync(f.root, { recursive: true, force: true });
|
||||
const bootstrap = f.run(['--for', 'autoplan'], { GSTACK_UPDATE_CHANNEL: 'stable' });
|
||||
expect(bootstrap.status, bootstrap.stdout + bootstrap.stderr).toBe(0);
|
||||
expect(bootstrap.stdout).toContain('channel stable, but origin has no stable branch yet (or the fetch failed); following main');
|
||||
expect(f.git(['rev-parse', 'HEAD'])).toBe(f.c);
|
||||
});
|
||||
|
||||
// Value: protects=the bootstrapper's floor fast-forward of a checkout below the supported minimum goes to stable on the stable channel when stable
|
||||
// meets the minimum, and to main when stable is itself below it; fails_when=a stable install is pushed past the release to main, or left below the
|
||||
// floor because stable is too old; why_new=the floor tests cover main only; seam=GSTACK_UPDATE_CHANNEL; no_claim=GitHub's real stable branch
|
||||
test('below the floor: the fast-forward goes to stable when stable meets the minimum, else to main', () => {
|
||||
const atRelease = fixture({ next: true, stable: 'B' });
|
||||
const r = atRelease.run(['--for', 'autoplan'], { GSTACK_UPDATE_CHANNEL: 'stable' });
|
||||
expect(r.status, r.stdout + r.stderr).toBe(0);
|
||||
expect(r.stdout).toContain('gstack-capy-skill: upgraded 1.0.0 -> 1.0.1');
|
||||
expect(atRelease.git(['rev-parse', 'HEAD'])).toBe(atRelease.b);
|
||||
|
||||
const old = fixture({ next: true, stable: 'A' });
|
||||
const o = old.run(['--for', 'autoplan'], { GSTACK_UPDATE_CHANNEL: 'stable' });
|
||||
expect(o.status, o.stdout + o.stderr).toBe(0);
|
||||
expect(o.stdout).toContain('gstack-capy-skill: channel stable is at 1.0.0, below the supported minimum 1.0.1; following main');
|
||||
expect(old.git(['rev-parse', 'HEAD'])).toBe(old.c);
|
||||
});
|
||||
});
|
||||
|
||||
describe('capy skill bootstrapper: GSTACK_CAPY_REMOTE_VERSION', () => {
|
||||
test('a fresh fetch exports origin\'s VERSION; a failed fetch exports nothing even with a stale FETCH_HEAD', () => {
|
||||
const f = fixture({ cloneAt: 'B' });
|
||||
|
||||
@@ -249,6 +249,50 @@ describe('release-cut: planning', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('release-cut: the release gate holds and gates (lib/release-gate.ts records)', () => {
|
||||
const release = (sha: string, verdict: string, run: number) => ({ schema: 1, kind: 'release', repo_key: 'k', sha, verdict, run: { id: run, url: 'u' }, red_cases: verdict === 'RED' ? ['case-a'] : [], created_at: 'x', updated_at: 'x' });
|
||||
const note = (f: ReturnType<typeof fixture>, record: unknown, on: string) => {
|
||||
git(f.seed, 'notes', '--ref=' + NOTES_REF, 'add', '-f', '-m', JSON.stringify(record), on);
|
||||
git(f.seed, 'push', '-q', '-f', 'origin', NOTES_REF);
|
||||
};
|
||||
|
||||
// Value: protects=a red release record is the freeze: every cut is refused while no green gate sits on a later commit that contains the red one,
|
||||
// even when every check run is green, and an incomplete gate or a green gate on an older commit does not lift it; fails_when=release-cut releases a
|
||||
// commit while the newest release gate is red, a green record anywhere clears the hold, or an INCOMPLETE gate clears it; why_new=the freeze tests know
|
||||
// only the train freeze record; seam=none; no_claim=real refs/notes/gstack-trains content (fixture records shaped like ReleaseRecord)
|
||||
test('a red release record refuses the cut; only a green gate on a later commit lifts it', () => {
|
||||
const f = fixture();
|
||||
note(f, release(f.shas['1.0.2']!, 'RED', 41), f.shas['1.0.2']!);
|
||||
note(f, release(f.shas['1.0.1']!, 'GREEN', 40), f.shas['1.0.1']!);
|
||||
const held = planRelease({ repo: f.repo, git: f.git, gh: f.gh });
|
||||
expect(held.ok).toBe(false);
|
||||
if (!held.ok) { expect(held.code).toBe('REFUSED'); expect(held.message).toContain('release gate red on ' + f.shas['1.0.2']!.slice(0, 12)); }
|
||||
note(f, release(f.shas['1.0.3']!, 'INCOMPLETE', 42), f.shas['1.0.3']!);
|
||||
expect(planRelease({ repo: f.repo, git: f.git, gh: f.gh }).ok).toBe(false);
|
||||
note(f, release(f.shas['1.0.3']!, 'GREEN', 43), f.shas['1.0.3']!);
|
||||
const lifted = planRelease({ repo: f.repo, git: f.git, gh: f.gh });
|
||||
expect(lifted.ok, JSON.stringify(lifted)).toBe(true);
|
||||
if (lifted.ok) expect(lifted.plan.sha).toBe(f.shas['1.0.3']);
|
||||
});
|
||||
|
||||
// Value: protects=a green release record gates its commit when the main-backstop check run is absent or neutral (the check publish failed or was
|
||||
// superseded), and the CANDIDATE line names the gate run; a red check run still keeps the commit out; fails_when=a commit the release gate passed is
|
||||
// never released without the check, or release evidence overrides a red check run; why_new=train records were the only notes evidence; seam=none;
|
||||
// no_claim=a live gate run
|
||||
test('a green release record gates a commit whose main-backstop check never ran; a red check run still wins', () => {
|
||||
const f = fixture();
|
||||
const head = f.shas['1.0.3']!;
|
||||
f.state.checks[head] = GREEN.filter(c => c.name !== BACKSTOP_CHECK);
|
||||
note(f, release(head, 'GREEN', 77), head);
|
||||
const gated = planRelease({ repo: f.repo, git: f.git, gh: f.gh });
|
||||
expect(gated.ok, JSON.stringify(gated)).toBe(true);
|
||||
if (gated.ok) { expect(gated.plan.sha).toBe(head); expect(gated.plan.via).toBe('release gate run 77'); }
|
||||
f.state.checks[head] = [...GREEN.filter(c => c.name !== BACKSTOP_CHECK), { name: 'quality', status: 'completed', conclusion: 'failure' }];
|
||||
const red = planRelease({ repo: f.repo, git: f.git, gh: f.gh });
|
||||
if (red.ok) expect(red.plan.sha).not.toBe(head);
|
||||
});
|
||||
});
|
||||
|
||||
describe('release-cut: train evidence gates a commit whose backstop never settled', () => {
|
||||
const train = (nonce: string, state: string, merged: Array<{ sha: string; tree: string }>) =>
|
||||
({ schema: 1, kind: 'train', nonce, state, merged: merged.map(m => ({ pr: '1', at: 'x', ...m })), updated_at: 'x' });
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
/**
|
||||
* The release gate's workflow wiring (docs/ship-policy.md#release-gate):
|
||||
* release-gate.yml runs bin/gstack-release-gate on its schedule and publishes
|
||||
* the gated commit's check run from a code-free job; the per-push backstop
|
||||
* stands down once main's policy declares a release_gate block; evals.yml routes
|
||||
* a release bisect only under that block. Policy steps run for real (bash, git,
|
||||
* jq) against a scratch repository; the rest is read from the YAML.
|
||||
*/
|
||||
import { afterAll, describe, expect, test } from 'bun:test';
|
||||
import { execFileSync, spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { RELEASE_GATE_SCHEDULE_HOURS } from '../lib/release-gate-policy';
|
||||
import { RELEASE_BISECT_ARTIFACT } from '../scripts/train-gate';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
type Step = { id?: string; name?: string; uses?: string; if?: string; run?: string; env?: Record<string, string> };
|
||||
type Job = { if?: string; needs?: string; outputs?: Record<string, string>; permissions?: Record<string, string>; steps: Step[] };
|
||||
type Workflow = { on: Record<string, any>; concurrency: unknown; permissions?: unknown; jobs: Record<string, Job> };
|
||||
const load = (rel: string) => Bun.YAML.parse(fs.readFileSync(path.join(ROOT, rel), 'utf8')) as Workflow;
|
||||
const release = load('.github/workflows/release-gate.yml');
|
||||
const backstop = load('.github/workflows/evals-main-backstop.yml');
|
||||
const evals = load('.github/workflows/evals.yml');
|
||||
const scratch: string[] = [];
|
||||
afterAll(() => { for (const d of scratch) fs.rmSync(d, { recursive: true, force: true }); });
|
||||
const GIT_ENV = { GIT_AUTHOR_NAME: 't', GIT_AUTHOR_EMAIL: 't@x', GIT_COMMITTER_NAME: 't', GIT_COMMITTER_EMAIL: 't@x' };
|
||||
|
||||
/** Run a policy step's script in a scratch repo whose one commit holds this policy; returns its GITHUB_OUTPUT lines as an object. */
|
||||
function runStep(step: Step, policy: unknown): Record<string, string> {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'release-gate-wf-'));
|
||||
scratch.push(dir);
|
||||
const g = (...args: string[]) => execFileSync('git', args, { cwd: dir, encoding: 'utf8', timeout: 30_000, env: { ...process.env, ...GIT_ENV } }).trim();
|
||||
g('init', '-q', '-b', 'main');
|
||||
if (policy !== undefined) {
|
||||
fs.mkdirSync(path.join(dir, '.gstack'));
|
||||
fs.writeFileSync(path.join(dir, '.gstack/ship-policy.json'), JSON.stringify(policy));
|
||||
}
|
||||
fs.writeFileSync(path.join(dir, 'README'), 'x');
|
||||
g('add', '-A');
|
||||
g('commit', '-q', '-m', 'base');
|
||||
const sha = g('rev-parse', 'HEAD');
|
||||
g('update-ref', 'refs/remotes/origin/main', sha);
|
||||
const out = path.join(dir, 'out');
|
||||
const r = spawnSync('bash', ['-e', '-c', step.run!], {
|
||||
cwd: dir, encoding: 'utf8', timeout: 30_000,
|
||||
env: { ...process.env, GITHUB_OUTPUT: out, GITHUB_STEP_SUMMARY: path.join(dir, 'summary'), GITHUB_SHA: sha, BASE_SHA: '', DEFAULT_BRANCH: 'main' },
|
||||
});
|
||||
expect(r.status, r.stderr).toBe(0);
|
||||
const lines = (fs.existsSync(out) ? fs.readFileSync(out, 'utf8') : '').split('\n').filter(Boolean);
|
||||
return Object.fromEntries(lines.map(l => [l.slice(0, l.indexOf('=')), l.slice(l.indexOf('=') + 1)]));
|
||||
}
|
||||
const GATE = { cadence_hours: 8, gate_workflow: 'evals.yml', branch: 'main', stable_ref: 'stable', check_name: 'main-backstop', on_red: 'hold-release', bisect_spend_cap_usd: 40 };
|
||||
const TRAIN = { gate_workflow: 'evals.yml', record_artifact: 'gstack-train-record', full_gate_check: 'gstack/full-gate' };
|
||||
|
||||
describe('release-gate.yml', () => {
|
||||
// Value: protects=the schedule fires every RELEASE_GATE_SCHEDULE_HOURS (the cadence floor the policy validates), one gate at a time, only from the
|
||||
// default branch, with contents and actions write but no checks write, and runs the bin on the run's own sha with its outputs wired; the policy step
|
||||
// uses git and jq only and is inert without a release_gate block; fails_when=the cron and the cadence drift apart, two gates overlap, the job that
|
||||
// runs repository code can write check runs, or an inert policy reaches bun; why_new=new workflow; seam=none; no_claim=GitHub's cron jitter
|
||||
test('schedule, concurrency, permissions and the gate step', () => {
|
||||
const cron = release.on.schedule[0].cron as string;
|
||||
expect(cron).toMatch(/^\d+ \*\/\d+ \* \* \*$/);
|
||||
expect(Number(cron.split(' ')[1]!.slice(2))).toBe(RELEASE_GATE_SCHEDULE_HOURS);
|
||||
expect(Object.keys(release.on).sort()).toEqual(['schedule', 'workflow_dispatch']);
|
||||
expect(Object.keys(release.on.workflow_dispatch.inputs).sort()).toEqual(['dry_run', 'ignore_cadence']);
|
||||
expect(release.concurrency).toEqual({ group: 'release-gate', 'cancel-in-progress': false });
|
||||
expect(release.permissions).toEqual({});
|
||||
const job = release.jobs['release-gate']!;
|
||||
expect(job.if).toBe('github.ref_name == github.event.repository.default_branch');
|
||||
expect(job.permissions).toEqual({ contents: 'write', actions: 'write', checks: 'read' });
|
||||
const policy = job.steps.find(s => s.id === 'policy')!;
|
||||
expect(policy.run).not.toContain('bun');
|
||||
expect(runStep(policy, undefined)).toEqual({ release: 'false' });
|
||||
expect(runStep(policy, { train: TRAIN })).toEqual({ release: 'false' });
|
||||
expect(runStep(policy, { release_gate: GATE })).toEqual({ release: 'true' });
|
||||
const gate = job.steps.find(s => s.id === 'gate')!;
|
||||
expect(gate.if).toBe("steps.policy.outputs.release == 'true'");
|
||||
expect(gate.run).toContain('./bin/gstack-release-gate run --repo "$GITHUB_REPOSITORY" --sha "$GITHUB_SHA"');
|
||||
expect(gate.run).toContain('--outputs "$GITHUB_OUTPUT"');
|
||||
expect(job.outputs).toEqual({ action: '${{ steps.gate.outputs.action }}', sha: '${{ steps.gate.outputs.sha }}', check: '${{ steps.gate.outputs.check }}' });
|
||||
});
|
||||
|
||||
// Value: protects=the gated commit's check run (what release-cut reads) comes from a job with checks: write and no checkout or action, only for a
|
||||
// gate that ran (success for gated, failure for held or failed), on a validated sha and check name; fails_when=repository code runs with checks:
|
||||
// write, a skipped or inert run publishes a check, or a red gate publishes success; why_new=new workflow; seam=none; no_claim=GitHub's check API
|
||||
test('the check run is published from a code-free job, success only for gated', () => {
|
||||
const job = release.jobs['release-check']!;
|
||||
expect(job.needs).toBe('release-gate');
|
||||
expect(job.if).toContain("contains(fromJSON('[\"gated\",\"held\",\"failed\"]'), needs.release-gate.outputs.action)");
|
||||
expect(job.permissions).toEqual({ checks: 'write' });
|
||||
expect(job.steps.filter(s => s.uses)).toEqual([]);
|
||||
const run = job.steps[0]!.run!;
|
||||
expect(run).toContain('gated) conclusion=success ;;');
|
||||
expect(run).toContain('held|failed) conclusion=failure ;;');
|
||||
expect(run).toContain('[[ "$SHA" =~ ^[0-9a-f]{40}$ ]]');
|
||||
expect(run).toContain('-f name="$CHECK" -f head_sha="$SHA"');
|
||||
expect(run).not.toContain('bun');
|
||||
});
|
||||
});
|
||||
|
||||
describe('the backstop stands down and evals.yml routes the release bisect', () => {
|
||||
// Value: protects=while trains exist the per-push backstop keeps gating, and once main declares release_gate it stands down (train=false: no gate,
|
||||
// no freeze, no check run), so main is never gated twice; fails_when=both gates run on main, or the backstop stops before the release gate is on;
|
||||
// why_new=new stand-down rule; seam=none; no_claim=a live push
|
||||
test('evals-main-backstop.yml: train=true with only a train block, train=false once release_gate is declared', () => {
|
||||
const policy = backstop.jobs.backstop!.steps.find(s => s.id === 'policy')!;
|
||||
expect(runStep(policy, { train: TRAIN })).toEqual({ train: 'true' });
|
||||
expect(runStep(policy, { train: TRAIN, release_gate: GATE })).toEqual({ train: 'false' });
|
||||
});
|
||||
|
||||
// Value: protects=the route job reads release_gate from the base revision with git and jq only and hands it to the route as RELEASE_POLICY, and the
|
||||
// bisect jobs name their artifact from the route's record_artifact first (gstack-release-record for a release bisect); fails_when=a release bisect
|
||||
// publishes under the train artifact name the gate never reads, or repository code decides the release policy; why_new=new route input;
|
||||
// seam=none; no_claim=a live bisect run
|
||||
test('evals.yml: the release policy step, the RELEASE_POLICY input and the record_artifact override', () => {
|
||||
const route = evals.jobs.route!;
|
||||
const step = route.steps.find(s => s.id === 'release_policy')!;
|
||||
expect(step.run).not.toContain('bun');
|
||||
expect(runStep(step, { release_gate: GATE })).toEqual({ release: 'true' });
|
||||
expect(runStep(step, { train: TRAIN })).toEqual({ release: 'false' });
|
||||
expect(runStep(step, undefined)).toEqual({ release: 'false' });
|
||||
const ids = route.steps.map(s => s.id);
|
||||
expect(ids.indexOf('release_policy')).toBeLessThan(ids.indexOf('route'));
|
||||
expect(route.steps.find(s => s.id === 'route')!.env!.RELEASE_POLICY).toBe('${{ steps.release_policy.outputs.release }}');
|
||||
expect(route.outputs!.record_artifact).toBe('${{ steps.route.outputs.record_artifact || steps.policy.outputs.record_artifact }}');
|
||||
expect(RELEASE_BISECT_ARTIFACT).toBe('gstack-release-record');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,301 @@
|
||||
/**
|
||||
* gstack-release-gate (lib/release-gate.ts, docs/ship-policy.md#release-gate):
|
||||
* the full paid gate on main's head at most once per cadence and only when main
|
||||
* moved, a release record in refs/notes/gstack-trains, a bisect of a red gate's
|
||||
* cases over the commits since the last green gate, and the hold release-cut
|
||||
* reads. GitHub is a fake gh (dispatch, run list, run view, artifact download);
|
||||
* git, the notes ref and origin are real (a bare origin on disk).
|
||||
*/
|
||||
import { afterAll, afterEach, describe, expect, test } from 'bun:test';
|
||||
import { execFileSync, spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { bunIsolationArgs } from '../lib/bun-isolation';
|
||||
import {
|
||||
MAX_RELEASE_ATTEMPTS, activeHold, bisectGate, classifyRun, gateStatus, loadReleaseGate, narrow, parseReport, planPrefixes, releaseRecords, runGate,
|
||||
type GateEnv, type Run,
|
||||
} from '../lib/release-gate';
|
||||
import { RELEASE_GATE_SCHEDULE_HOURS, releaseGateErrors, type ReleaseGatePolicy } from '../lib/release-gate-policy';
|
||||
import { RELEASE_GATE_KEY_VERSION, validatePolicy } from '../lib/ship-policy';
|
||||
import { NOTES_REF, repoIdentity, type ReleaseRecord } from '../lib/ship-queue-train-record';
|
||||
import { gitIn } from '../scripts/train-gate';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const GATE: ReleaseGatePolicy = { cadence_hours: 8, gate_workflow: 'evals.yml', branch: 'main', stable_ref: 'stable', check_name: 'main-backstop', on_red: 'hold-release', bisect_spend_cap_usd: 40 };
|
||||
const roots: string[] = [];
|
||||
const savedStateRoot = process.env.GSTACK_STATE_ROOT;
|
||||
afterEach(() => { if (savedStateRoot === undefined) delete process.env.GSTACK_STATE_ROOT; else process.env.GSTACK_STATE_ROOT = savedStateRoot; });
|
||||
afterAll(() => { for (const r of roots) fs.rmSync(r, { recursive: true, force: true }); });
|
||||
|
||||
const ID = { GIT_AUTHOR_NAME: 't', GIT_AUTHOR_EMAIL: 't@x', GIT_COMMITTER_NAME: 't', GIT_COMMITTER_EMAIL: 't@x' };
|
||||
const git = (cwd: string, ...args: string[]) => execFileSync('git', args, { cwd, encoding: 'utf8', timeout: 30_000, env: { ...process.env, ...ID } }).trim();
|
||||
|
||||
/** origin with base (also stable) and c1..c5 on main, and work cloned from it; the state root is the fixture's own. */
|
||||
function fixture() {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'release-gate-'));
|
||||
roots.push(root);
|
||||
const origin = path.join(root, 'origin.git');
|
||||
const seed = path.join(root, 'seed');
|
||||
const work = path.join(root, 'work');
|
||||
git(root, 'init', '-q', '--bare', '-b', 'main', origin);
|
||||
fs.mkdirSync(seed);
|
||||
git(seed, 'init', '-q', '-b', 'main');
|
||||
git(seed, 'remote', 'add', 'origin', origin);
|
||||
const commit = (name: string) => { fs.writeFileSync(path.join(seed, 'VERSION'), name + '\n'); git(seed, 'add', '-A'); git(seed, 'commit', '-q', '-m', name); git(seed, 'push', '-q', 'origin', 'main'); return git(seed, 'rev-parse', 'HEAD'); };
|
||||
const base = commit('base');
|
||||
git(seed, 'push', '-q', 'origin', 'main:stable');
|
||||
const c = [1, 2, 3, 4, 5].map(i => commit('c' + i));
|
||||
git(root, 'clone', '-q', origin, work);
|
||||
process.env.GSTACK_STATE_ROOT = path.join(root, 'state');
|
||||
return { root, origin, seed, work, base, c, commit };
|
||||
}
|
||||
|
||||
interface Outcome { conclusion: string; artifacts: Record<string, Record<string, string>> }
|
||||
/** A fake gh: each dispatch becomes a completed run whose outcome the test decides; run list, run view and artifact download read it back. */
|
||||
function fakeGh(onDispatch: (inputs: Record<string, string>, ref: string) => Outcome) {
|
||||
const runs: Run[] = [];
|
||||
const artifacts = new Map<number, Record<string, Record<string, string>>>();
|
||||
const dispatches: Array<{ ref: string; inputs: Record<string, string> }> = [];
|
||||
const gh = (args: string[]) => {
|
||||
const ok = (out: unknown) => ({ status: 0, stdout: typeof out === 'string' ? out : JSON.stringify(out), stderr: '' });
|
||||
if (args[0] === 'workflow' && args[1] === 'run') {
|
||||
const ref = args[args.indexOf('--ref') + 1]!;
|
||||
const inputs = Object.fromEntries(args.flatMap((a, i) => (args[i - 1] === '-f' ? [a.split(/=(.*)/s).slice(0, 2)] : [])));
|
||||
dispatches.push({ ref, inputs });
|
||||
const id = 1000 + runs.length;
|
||||
const o = onDispatch(inputs, ref);
|
||||
runs.push({ id, status: 'completed', conclusion: o.conclusion, head_sha: inputs.expected_sha!, display_title: 'E2E Evals [eval-bg ' + inputs.nonce + ']', html_url: 'https://github.com/o/r/actions/runs/' + id, run_attempt: 1 });
|
||||
artifacts.set(id, o.artifacts);
|
||||
return ok('');
|
||||
}
|
||||
if (args[0] === 'api' && args.some(a => /\/actions\/workflows\/[^/]+\/runs$/.test(a))) return ok({ workflow_runs: [...runs].reverse() });
|
||||
const view = args[0] === 'api' ? /actions\/runs\/(\d+)$/.exec(args[1] ?? '') : null;
|
||||
if (view) return ok(runs.find(r => r.id === Number(view[1])));
|
||||
if (args[0] === 'run' && args[1] === 'download') {
|
||||
const files = artifacts.get(Number(args[2]))?.[args[args.indexOf('-n') + 1]!];
|
||||
if (!files) return { status: 1, stdout: '', stderr: 'no artifact matches any of the names or patterns provided' };
|
||||
const dir = args[args.indexOf('-D') + 1]!;
|
||||
for (const [name, text] of Object.entries(files)) { fs.mkdirSync(dir, { recursive: true }); fs.writeFileSync(path.join(dir, name), text); }
|
||||
return ok('');
|
||||
}
|
||||
return { status: 1, stdout: '', stderr: 'unexpected gh ' + args.join(' ') };
|
||||
};
|
||||
return { gh, runs, dispatches };
|
||||
}
|
||||
|
||||
const report = (verdict: 'GREEN' | 'RED', failed: string[] = [], cost = '37.40') => ({ 'report-verdict-a1': { 'report.txt': [
|
||||
'[test:paid] report: 17/17 slices, 85 planned shards, tier=gate, attempt 1',
|
||||
...(failed.length ? ['FAILED cases:', ...failed.map(l => ' ✗ ' + l)] : []),
|
||||
'[test:paid] VERDICT ' + verdict + ' — lane gate/full census, attempt 1',
|
||||
' wall 41m02s · cost $' + cost + ' known + 4 of 139 trial(s) cost unknown',
|
||||
].join('\n') } });
|
||||
const green = (flaky: string[] = []): Outcome => ({ conclusion: 'success', artifacts: { ...report('GREEN'), ...(flaky.length ? { 'flake-records-a1': { 'flake-records.jsonl': flaky.map(c => JSON.stringify({ schema: 1, kind: 'gate-flake', case: c })).join('\n') } } : {}) } });
|
||||
|
||||
function envFor(f: ReturnType<typeof fixture>, gh: (args: string[]) => { status: number; stdout: string; stderr: string }, shiftHours = 0): GateEnv {
|
||||
return { repo: 'o/r', repoRoot: f.work, remote: 'origin', gh, git: gitIn(f.work), now: () => Date.now() + shiftHours * 3_600_000, sleep: () => {}, random: () => 'abc123', pollMs: 1, deadline: Date.now() + 3_600_000 };
|
||||
}
|
||||
const records = (f: ReturnType<typeof fixture>) => {
|
||||
git(f.work, 'fetch', '-q', 'origin', '+' + NOTES_REF + ':' + NOTES_REF);
|
||||
return releaseRecords(gitIn(f.work), repoIdentity(f.work).key);
|
||||
};
|
||||
|
||||
describe('release gate: run', () => {
|
||||
// Value: protects=a gate only when main moved: the first run dispatches the full gate once on main's head (evals_all, expected_sha, a backstop nonce),
|
||||
// records GREEN with its run, cost and rule-16 flaky cases in origin's notes and prints the RELEASE green line without moving stable; a second run on
|
||||
// the same head, and a head equal to stable, skip without dispatching; fails_when=the gate dispatches again on an unchanged main (a $37 repeat), a
|
||||
// green gate moves stable, or the record never reaches origin; why_new=new module; seam=fake gh; no_claim=a live gate run (the PR's one live proof)
|
||||
test('green: one dispatch on the moved head, a GREEN release record on origin, stable untouched; an unchanged main skips', () => {
|
||||
const f = fixture();
|
||||
const fake = fakeGh(() => green(['judge-x']));
|
||||
const head = f.c[4]!;
|
||||
const out = runGate(envFor(f, fake.gh), { gate: GATE, label: 'test', dryRun: false, ignoreCadence: false });
|
||||
expect(out.action, out.lines.join('\n')).toBe('gated');
|
||||
expect(fake.dispatches).toHaveLength(1);
|
||||
expect(fake.dispatches[0]!.ref).toBe('main');
|
||||
expect(fake.dispatches[0]!.inputs).toMatchObject({ evals_all: 'true', expected_sha: head });
|
||||
expect(fake.dispatches[0]!.inputs.nonce).toMatch(/^gate\.backstop-/);
|
||||
expect(out.lines).toContain('RELEASE: green sha=' + head + ' stable=' + f.base + ' run=1000 cost=37.40');
|
||||
expect(out.lines.some(l => l.includes('flaky (failed once, passed its one rule-16 rerun in run 1000): judge-x'))).toBe(true);
|
||||
const [rec] = records(f);
|
||||
expect(rec).toMatchObject({ kind: 'release', sha: head, verdict: 'GREEN', flaky: ['judge-x'], cost_usd: 37.4, since: f.base, attempts: 1, run: { id: 1000 } });
|
||||
expect(git(f.seed, 'ls-remote', 'origin', 'refs/heads/stable').split('\t')[0]).toBe(f.base);
|
||||
|
||||
const again = runGate(envFor(f, fake.gh, 9), { gate: GATE, label: 'test', dryRun: false, ignoreCadence: true });
|
||||
expect(again.action).toBe('skipped');
|
||||
expect(again.lines[0]).toContain('main has not moved since the last gate: GREEN run 1000');
|
||||
git(f.seed, 'push', '-q', '-f', 'origin', 'main:stable');
|
||||
const atStable = runGate(envFor(f, fake.gh), { gate: GATE, label: 'test', dryRun: false, ignoreCadence: true });
|
||||
expect(atStable.lines[0]).toContain('main has not moved since stable');
|
||||
expect(fake.dispatches).toHaveLength(1);
|
||||
});
|
||||
|
||||
// Value: protects=without a release_gate block nothing runs; --dry-run names the head, the base and the pending count and writes and dispatches
|
||||
// nothing; the cadence holds a moved main until cadence_hours minus the grace pass, and --ignore-cadence gates now; fails_when=an inert policy spends,
|
||||
// a dry run writes a record, or the schedule gates more often than the policy says; why_new=new module; seam=fake gh and a shifted clock;
|
||||
// no_claim=GitHub's cron timing
|
||||
test('inert, dry run and cadence: nothing is spent until the policy and the clock allow it', () => {
|
||||
const f = fixture();
|
||||
const fake = fakeGh(() => green());
|
||||
expect(runGate(envFor(f, fake.gh), { gate: null, label: 'origin/main@x', dryRun: false, ignoreCadence: false }).action).toBe('inert');
|
||||
const dry = runGate(envFor(f, fake.gh), { gate: GATE, label: 'test', dryRun: true, ignoreCadence: false });
|
||||
expect(dry.action).toBe('dry-run');
|
||||
expect(dry.lines[0]).toBe('RELEASE: gate sha=' + f.c[4] + ' since=' + f.base.slice(0, 12) + ' (stable) pending=5 stable=' + f.base.slice(0, 12));
|
||||
expect(fake.dispatches).toHaveLength(0);
|
||||
expect(spawnSync('git', ['ls-remote', 'origin', NOTES_REF], { cwd: f.work, encoding: 'utf8', timeout: 30_000 }).stdout).toBe('');
|
||||
|
||||
expect(runGate(envFor(f, fake.gh), { gate: GATE, label: 'test', dryRun: false, ignoreCadence: false }).action).toBe('gated');
|
||||
const next = f.commit('c6');
|
||||
const early = runGate(envFor(f, fake.gh, 2), { gate: GATE, label: 'test', dryRun: false, ignoreCadence: false });
|
||||
expect(early.action).toBe('skipped');
|
||||
expect(early.lines[0]).toMatch(/the last gate was dispatched 2\.0 h ago on [0-9a-f]{12}; cadence 8 h/);
|
||||
const due = runGate(envFor(f, fake.gh, 7.5), { gate: GATE, label: 'test', dryRun: false, ignoreCadence: false });
|
||||
expect(due.action, due.lines.join('\n')).toBe('gated');
|
||||
expect(fake.dispatches.map(d => d.inputs.expected_sha)).toEqual([f.c[4], next]);
|
||||
expect(records(f).find(r => r.sha === next)?.since).toBe(f.c[4]);
|
||||
});
|
||||
|
||||
// Value: protects=an INCOMPLETE census (only INFRA reds) is recorded as such, re-dispatched once by the next run (EVAL_POLICY.infraRedispatch), then
|
||||
// skipped with its run named, and never treated as green or red; fails_when=an infra-only failure holds the release like a red, is retried without
|
||||
// end, or is never retried; why_new=new module; seam=fake gh; no_claim=a real runner loss
|
||||
test('incomplete: recorded, re-dispatched once, then skipped with the run named', () => {
|
||||
const f = fixture();
|
||||
const fake = fakeGh(() => ({ conclusion: 'failure', artifacts: report('RED', ['case-a rule FAIL INFRA']) }));
|
||||
const env = envFor(f, fake.gh);
|
||||
const first = runGate(env, { gate: GATE, label: 'test', dryRun: false, ignoreCadence: true });
|
||||
expect(first.action).toBe('failed');
|
||||
expect(first.lines.at(-1)).toContain('RELEASE: incomplete sha=' + f.c[4] + ' run=1000');
|
||||
expect(first.lines.at(-1)).toContain('the next run re-dispatches it once');
|
||||
expect(runGate(env, { gate: GATE, label: 'test', dryRun: false, ignoreCadence: true }).action).toBe('failed');
|
||||
const third = runGate(env, { gate: GATE, label: 'test', dryRun: false, ignoreCadence: true });
|
||||
expect(third.action).toBe('skipped');
|
||||
expect(third.lines[0]).toContain('incomplete ' + MAX_RELEASE_ATTEMPTS + ' times');
|
||||
expect(fake.dispatches).toHaveLength(MAX_RELEASE_ATTEMPTS);
|
||||
expect(activeHold(records(f), () => true)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('release gate: red, bisect and the hold', () => {
|
||||
// Value: protects=a red gate records RED (the hold release-cut reads), bisects only its red verdict cases on release-gate/<id> with prefixes between
|
||||
// stable and the red head, narrows round by round to the first red commit after a green one, deletes the bisect branch, and prints first_bad; a later
|
||||
// green gate on a newer commit lifts the hold; fails_when=a red gate moves or leaves stable releasable, the bisect blames the wrong commit, tests the
|
||||
// known-red head again, leaks its branch, or a green gate on an older commit lifts the hold; why_new=new module; seam=fake gh; no_claim=a live bisect
|
||||
test('red: RED record, a bisect that names first_bad, the branch cleaned up, and the hold until a later green', () => {
|
||||
const f = fixture();
|
||||
const order = [f.base, ...f.c];
|
||||
const fake = fakeGh((inputs) => {
|
||||
if (!inputs.prefix_shas) return { conclusion: 'failure', artifacts: report('RED', ['case-a rule FAIL VERDICT', 'case-z rule FAIL INFRA']) };
|
||||
const files: Record<string, Record<string, string>> = {};
|
||||
for (const sha of inputs.prefix_shas.split(',')) {
|
||||
const red = order.indexOf(sha) >= 3;
|
||||
files['gstack-release-record-bisect-' + sha.slice(0, 12)] = { 'bisect-prefix.json': JSON.stringify({ schema: 1, kind: 'train-bisect-prefix', nonce: inputs.nonce, prefix_sha: sha, checked_out_sha: sha,
|
||||
panels: [{ case: 'case-a', kind: 'rule', status: red ? 'FAIL' : 'PASS', red_class: red ? 'VERDICT' : null, cost_usd: 1.25 }] }) };
|
||||
}
|
||||
return { conclusion: 'success', artifacts: files };
|
||||
});
|
||||
const out = runGate(envFor(f, fake.gh), { gate: GATE, label: 'test', dryRun: false, ignoreCadence: true });
|
||||
expect(out.action, out.lines.join('\n')).toBe('held');
|
||||
expect(out.lines).toContain('RELEASE: red sha=' + f.c[4] + ' first_bad=' + f.c[2] + ' run=1000');
|
||||
expect(fake.dispatches).toHaveLength(2);
|
||||
const bisect = fake.dispatches[1]!;
|
||||
expect(bisect.ref).toMatch(/^release-gate\/abc123-[0-9a-f]{12}$/);
|
||||
expect(bisect.inputs).toMatchObject({ cases: 'case-a', prefix_shas: f.c.slice(0, 4).join(','), expected_sha: f.c[4] });
|
||||
expect(bisect.inputs.nonce).toMatch(/^gate\.bisect-/);
|
||||
expect(git(f.seed, 'ls-remote', 'origin', 'refs/heads/release-gate/*')).toBe('');
|
||||
const recs = records(f);
|
||||
expect(recs[0]).toMatchObject({ verdict: 'RED', red_cases: ['case-a'], bisect: { state: 'found', first_bad: f.c[2], good: f.c[1], bad: f.c[2], spent_usd: 5 } });
|
||||
expect(git(f.seed, 'ls-remote', 'origin', 'refs/heads/stable').split('\t')[0]).toBe(f.base);
|
||||
const isAncestor = (a: string, b: string) => spawnSync('git', ['merge-base', '--is-ancestor', a, b], { cwd: f.work, timeout: 30_000 }).status === 0;
|
||||
expect(activeHold(recs, isAncestor)?.sha).toBe(f.c[4]);
|
||||
const status = gateStatus(envFor(f, fake.gh), { gate: GATE, label: 'test' });
|
||||
expect(status.action).toBe('held');
|
||||
expect(status.lines.find(l => l.startsWith('HOLD:'))).toContain('red on ' + f.c[4]);
|
||||
const older: ReleaseRecord = { ...recs[0]!, sha: f.c[1]!, verdict: 'GREEN', bisect: null };
|
||||
expect(activeHold([...recs, older], isAncestor)?.sha).toBe(f.c[4]);
|
||||
const later: ReleaseRecord = { ...recs[0]!, sha: f.commit('fix'), verdict: 'GREEN', bisect: null };
|
||||
git(f.work, 'fetch', '-q', 'origin');
|
||||
expect(activeHold([later, ...recs], isAncestor)).toBeNull();
|
||||
});
|
||||
|
||||
// Value: protects=the bisect spends within its cap: a 0 cap records no bisect and first_bad unknown, a small cap tests only the prefixes it covers
|
||||
// and stops capped, and bisect --spend-cap resumes a capped one; fails_when=a bisect dispatches past its cap or a capped bisect cannot be resumed;
|
||||
// why_new=new module; seam=fake gh; no_claim=real case prices
|
||||
test('bisect spend: a zero cap skips it, a small cap stops capped, and bisect --spend-cap resumes', () => {
|
||||
const f = fixture();
|
||||
const fake = fakeGh((inputs) => (inputs.prefix_shas ? { conclusion: 'success', artifacts: {} } : { conclusion: 'failure', artifacts: report('RED', ['case-a rule FAIL VERDICT', 'case-b judge FAIL']) }));
|
||||
const none = runGate(envFor(f, fake.gh), { gate: { ...GATE, bisect_spend_cap_usd: 0 }, label: 'test', dryRun: false, ignoreCadence: true });
|
||||
expect(none.lines).toContain('RELEASE: red sha=' + f.c[4] + ' first_bad=unknown run=1000');
|
||||
expect(none.lines.some(l => l.includes('bisect unknown: bisect_spend_cap_usd is 0'))).toBe(true);
|
||||
expect(fake.dispatches).toHaveLength(1);
|
||||
|
||||
const g = fixture();
|
||||
const fake2 = fakeGh((inputs) => (inputs.prefix_shas ? { conclusion: 'success', artifacts: {} } : { conclusion: 'failure', artifacts: report('RED', ['case-a rule FAIL VERDICT', 'case-b judge FAIL']) }));
|
||||
const capped = runGate(envFor(g, fake2.gh), { gate: { ...GATE, bisect_spend_cap_usd: 2 }, label: 'test', dryRun: false, ignoreCadence: true });
|
||||
expect(capped.lines.some(l => l.includes('bisect capped: the next round needs $3.00 a prefix'))).toBe(true);
|
||||
expect(fake2.dispatches).toHaveLength(1);
|
||||
const plan = bisectGate(envFor(g, fake2.gh), { gate: GATE, label: 'test', spendCap: 30, dryRun: true });
|
||||
expect(plan.lines.some(l => l.startsWith('RELEASE: bisect round 1: 2 case(s) on 4 of 4 commit(s)'))).toBe(true);
|
||||
expect(fake2.dispatches).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('release gate: verdict, range and policy units', () => {
|
||||
test('parseReport and classifyRun: verdict reds bisect, other classes keep a red incomplete, success is green, and a contradiction is incomplete', () => {
|
||||
const r = parseReport(report('RED', ['case-a rule FAIL VERDICT', 'judge one judge FAIL', 'case-z rule FAIL INFRA'])['report-verdict-a1']['report.txt']);
|
||||
expect(r).toEqual({ verdict: 'RED', red: ['case-a', 'judge one'], other: ['case-z'], cost: 37.4 });
|
||||
expect(classifyRun({ status: 'completed', conclusion: 'failure' }, r)).toMatchObject({ verdict: 'RED', red: ['case-a', 'judge one'] });
|
||||
expect(classifyRun({ status: 'completed', conclusion: 'failure' }, { ...r, red: [] }).verdict).toBe('INCOMPLETE');
|
||||
expect(classifyRun({ status: 'completed', conclusion: 'cancelled' }, null).verdict).toBe('INCOMPLETE');
|
||||
expect(classifyRun({ status: 'completed', conclusion: 'success' }, { ...r, verdict: 'GREEN' }).verdict).toBe('GREEN');
|
||||
expect(classifyRun({ status: 'completed', conclusion: 'success' }, r).verdict).toBe('INCOMPLETE');
|
||||
expect(classifyRun({ status: 'in_progress', conclusion: null }, null).verdict).toBe('PENDING');
|
||||
});
|
||||
|
||||
test('planPrefixes spreads at most MAX_PREFIXES over the interior; narrow moves good and bad and refuses a non-monotone or missing result', () => {
|
||||
const shas = Array.from({ length: 20 }, (_, i) => String(i).padStart(40, '0'));
|
||||
expect(planPrefixes(shas.slice(0, 3))).toEqual(shas.slice(0, 3));
|
||||
const picked = planPrefixes(shas, 7);
|
||||
expect(picked).toHaveLength(7);
|
||||
expect(new Set(picked).size).toBe(7);
|
||||
expect(picked[0]).not.toBe(shas[0]);
|
||||
const t = (rs: string[]) => rs.map((r, i) => ({ sha: shas[i]!, results: r === '?' ? {} : { a: r === 'R' ? 'red' as const : 'green' as const } }));
|
||||
expect(narrow({ good: 'g', bad: 'b' }, t(['G', 'G', 'R']), ['a'])).toEqual({ good: shas[1], bad: shas[2] });
|
||||
expect(narrow({ good: 'g', bad: 'b' }, t(['G', 'G']), ['a'])).toEqual({ good: shas[1], bad: 'b' });
|
||||
expect(narrow({ good: 'g', bad: 'b' }, t(['R', 'R']), ['a'])).toEqual({ good: 'g', bad: shas[0] });
|
||||
expect(narrow({ good: 'g', bad: 'b' }, t(['G', 'R', 'G']), ['a'])).toEqual({ unknown: 'a is not monotone over the tested commits (GRG)' });
|
||||
expect(narrow({ good: 'g', bad: 'b' }, t(['G', '?']), ['a'])).toHaveProperty('unknown');
|
||||
});
|
||||
|
||||
// Value: protects=the release_gate key is read only with min_gstack_version at its introducing version, validates every field (cadence at least the
|
||||
// workflow schedule, a workflow file, branch names, hold-release), and --policy runs the same validation; fails_when=an older gstack reads the block
|
||||
// silently, a bad field passes, or the command-line policy skips validation; why_new=new key; seam=none; no_claim=the json adoption (a follow-up PR)
|
||||
test('policy: the release_gate key needs its min_gstack_version and every field; --policy is validated the same way', () => {
|
||||
const raw = { release_gate: GATE };
|
||||
expect(validatePolicy(raw).map(e => e.path)).toEqual(['release_gate']);
|
||||
expect(validatePolicy({ ...raw, min_gstack_version: RELEASE_GATE_KEY_VERSION })).toEqual([]);
|
||||
expect(releaseGateErrors({ ...GATE, cadence_hours: RELEASE_GATE_SCHEDULE_HOURS - 1, on_red: 'freeze', stable_ref: 'main', extra: 1 }).map(e => e.path).sort())
|
||||
.toEqual(['release_gate.cadence_hours', 'release_gate.extra', 'release_gate.on_red', 'release_gate.stable_ref']);
|
||||
expect(releaseGateErrors({ cadence_hours: 8 }).filter(e => e.message.startsWith('missing'))).toHaveLength(6);
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'release-gate-policy-'));
|
||||
roots.push(dir);
|
||||
const good = path.join(dir, 'good.json');
|
||||
fs.writeFileSync(good, JSON.stringify({ ...raw, min_gstack_version: RELEASE_GATE_KEY_VERSION }));
|
||||
expect(loadReleaseGate(ROOT, { policyFile: good })).toEqual({ gate: GATE, label: good });
|
||||
const bad = path.join(dir, 'bad.json');
|
||||
fs.writeFileSync(bad, JSON.stringify(raw));
|
||||
expect((loadReleaseGate(ROOT, { policyFile: bad }) as { error: string }).error).toContain('release_gate introduced in gstack ' + RELEASE_GATE_KEY_VERSION);
|
||||
});
|
||||
|
||||
test('bin/gstack-release-gate: usage exits 2; an inert policy file exits 0 without touching GitHub', () => {
|
||||
const bin = path.join(ROOT, 'bin/gstack-release-gate');
|
||||
const usage = spawnSync('bun', [...bunIsolationArgs(), bin, 'frobnicate'], { cwd: ROOT, encoding: 'utf8', timeout: 60_000 });
|
||||
expect(usage.status).toBe(2);
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'release-gate-bin-'));
|
||||
roots.push(dir);
|
||||
const inert = path.join(dir, 'policy.json');
|
||||
fs.writeFileSync(inert, '{}');
|
||||
const r = spawnSync('bun', [...bunIsolationArgs(), bin, 'run', '--repo', 'o/r', '--policy', inert], { cwd: ROOT, encoding: 'utf8', timeout: 60_000 });
|
||||
expect(r.status, r.stderr).toBe(0);
|
||||
expect(r.stdout.trim()).toBe('RELEASE: inert (' + inert + ' declares no release_gate block; the per-push main backstop gates main)');
|
||||
});
|
||||
});
|
||||
@@ -54,6 +54,20 @@ describe('routeDispatch: every run keeps its mode, and train inputs are validate
|
||||
expect((route({ ...screen, trainPolicy: false }) as { error: string }).error).toContain('needs a train block');
|
||||
});
|
||||
|
||||
// Value: protects=the release gate's bisect (lib/release-gate.ts) routes on its own release-gate/<id> ref under a release_gate policy, names its own
|
||||
// prefix artifact, keeps every train bisect check (nonce, head, cases, prefixes), and is refused without the policy or on any other ref;
|
||||
// fails_when=a release bisect needs a train block (it stops working once trains go), a release-gate/ ref routes without a release_gate policy, or a
|
||||
// train/ bisect starts accepting the release policy alone; why_new=the bisect test below covers train/<nonce> only; seam=none; no_claim=a live run
|
||||
test('a release bisect runs on release-gate/<id> under a release_gate policy, with the train checks and its own artifact', () => {
|
||||
const bisect = { nonce: bisectNonce, refName: 'release-gate/abc123-aaaaaaaaaaaa', expectedSha: A, cases: 'case-a', prefixShas: B, releasePolicy: true };
|
||||
expect(route(bisect)).toEqual({ mode: 'bisect', cases: ['case-a'], prefixes: [B], artifact: 'gstack-release-record' });
|
||||
expect((route({ ...bisect, releasePolicy: false, trainPolicy: true }) as { error: string }).error).toContain('needs a release_gate block');
|
||||
expect((route({ ...bisect, prefixShas: C }) as { error: string }).error).toContain('is not an ancestor');
|
||||
expect((route({ ...bisect, expectedSha: B }) as { error: string }).error).toContain('is not the train head');
|
||||
expect((route({ ...bisect, refName: 'train/t1', trainPolicy: false }) as { error: string }).error).toContain('needs a train block');
|
||||
expect((route({ ...bisect, refName: 'main' }) as { error: string }).error).toContain('runs on train/<nonce>');
|
||||
});
|
||||
|
||||
test('a bisect needs a bisect nonce on train/<nonce>, the train head, plan case ids and ancestor prefixes', () => {
|
||||
const bisect = { nonce: bisectNonce, refName: 'train/t1', expectedSha: A, cases: 'case-a, review/SKILL.md workflow', prefixShas: `${A},${B}`, trainPolicy: true };
|
||||
expect(route(bisect)).toEqual({ mode: 'bisect', cases: ['case-a', 'review/SKILL.md workflow'], prefixes: [A, B] });
|
||||
|
||||
Reference in new issue
Block a user