From 1652e37fcfe250942cfab11d5a3311c86e40c3ff Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 31 Oct 2026 13:27:02 +0000 Subject: [PATCH] fix(pregate): clear --run re-reads every run before recording, ties veto, inventory runs under pipefail, freshness reads job completion, and a pregate.json rewritten mid-clear refuses Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- bin/gstack-pregate | 13 +++++----- changelog.d/capy-gsta156-run-evidence.md | 2 +- docs/pregate.md | 14 ++++++++--- lib/pregate/index.ts | 8 +++++- lib/run-evidence.ts | 32 ++++++++++++++++++------ test/pregate-clear-runs.test.ts | 11 ++++++++ test/run-evidence.test.ts | 24 +++++++++++++++++- 7 files changed, 84 insertions(+), 20 deletions(-) diff --git a/bin/gstack-pregate b/bin/gstack-pregate index 1eeb9bb09..d1bb4e609 100755 --- a/bin/gstack-pregate +++ b/bin/gstack-pregate @@ -31,8 +31,8 @@ import { PREGATE_CONFIG_PATH, explainConfig, isConfigUnavailable, loadConfigFrom import { describeDraft, draftConfig } from '../lib/pregate/init-draft'; import { buildContext, defaultUserConfigPath, describeRefusal, loadedDotenvFiles } from '../lib/pregate/context'; import { CHECKS, CHECK_IDS, PREGATE_OUT_REL, PREGATE_TIER2, clearDecision, clearObligation, readPregateFile, renderTable, runPregate, verifyPregate, writePregateFile, type RunCheck } from '../lib/pregate/index'; -import { ghRunDeps } from '../lib/run-evidence'; -import { runBashSync, wtreeFingerprint } from '../lib/script-invocation'; +import { ghRunDeps, runInventory } from '../lib/run-evidence'; +import { wtreeFingerprint } from '../lib/script-invocation'; import { describeLane, discoverLanes } from '../lib/pregate/workflows'; import { REMOTE_GATE_LANE } from '../lib/pregate/checks/lanes'; import { describeSubstitution } from '../lib/pregate/remote-receipt'; @@ -237,7 +237,9 @@ function cmdClear(root: string, rest: string[]): number { process.stderr.write(`gstack-pregate clear: receipt refused — ${decision.message}; fix: ${decision.fix ?? RESULT_CODES.PREGATE_RECEIPT_INVALID.fix} ${resultCodeSuffix('PREGATE_RECEIPT_INVALID')}\n`); return EXIT.fail; } - const file = clearObligation(read.file, lane, decision.receipt, `gstack-pregate clear ${new Date().toISOString()}`, decision.substitution); + const current = readPregateFile(out).file; + if (!current || current.tree !== read.file.tree || current.generated_at !== read.file.generated_at) { process.stderr.write(resultLine('gstack-pregate clear', 'PREGATE_STALE', `${out} changed while ${lane} was being verified; rerun the clear against the new file`) + '\n'); return EXIT.fail; } + const file = clearObligation(current, lane, decision.receipt, `gstack-pregate clear ${new Date().toISOString()}`, decision.substitution); writePregateFile(root, out, file); if (/ runs=\S+ profiles=/.test(decision.receipt)) process.stdout.write(`REUSE: accepted ${lane} ${decision.receipt}\n`); if (decision.substitution) process.stdout.write(`PREGATE_RECEIPT_SUBSTITUTED: ${describeSubstitution(decision.substitution)}\n`); @@ -261,10 +263,9 @@ function runCheck(root: string, lane: string, allowRepoCommands: boolean): RunCh if (!allowRepoCommands) return { error: 'the inventory command is a pin-file command; pass --allow-repo-commands' }; const at = () => { const fp = wtreeFingerprint({ cwd: root }); return 'tree' in fp ? fp.tree : null; }; if (at() !== tree) return { error: `the working tree is not pregate.json's tree ${tree.slice(0, 12)}; the inventory must list that tree's gate files` }; - const r = runBashSync(command, { cwd: root, encoding: 'utf8', timeout: 300_000, maxBuffer: 64 * 1024 * 1024 }); - if (r.status !== 0 || r.error) return { error: `${command} exited ${r.status ?? r.error}: ${(r.stderr ?? '').trim().slice(0, 200)}` }; + const listed = runInventory(command, root); if (at() !== tree) return { error: `the working tree changed while ${command} ran` }; - return [...new Set(r.stdout.split('\n').map(l => l.trim()).filter(Boolean))]; + return listed; }, }; } diff --git a/changelog.d/capy-gsta156-run-evidence.md b/changelog.d/capy-gsta156-run-evidence.md index 188341d69..d7f1bf051 100644 --- a/changelog.d/capy-gsta156-run-evidence.md +++ b/changelog.d/capy-gsta156-run-evidence.md @@ -2,7 +2,7 @@ ### Added - **`gstack-pregate clear --run `** (repeatable; `lib/run-evidence.ts`, docs/pregate.md#run-evidence). A run is accepted only when it belongs to this repository, ran a `push` or `workflow_dispatch` event of the profile's workflow, and its newest attempt succeeded with every named job passing exactly once. Its `tree.json` must name the run, attempt, event and head and record the head's tree, and it must be fresh (`max_age_hours`, default 24) and on the profile's OS. It is refused if any run of that workflow on the head that started after it failed, was cancelled or is still running, or if the run changed attempts while being checked. All runs must be on one tree: pregate.json's, or a stamp-only predecessor under the existing tree-receipt rule. Their `files` must cover the obligation's selected tests plus every file the declared `inventory` command lists for that tree. Refusals print `REUSE: refused reason=`; an accepted clear prints `REUSE: accepted`, and the receipt records `runs=/ profiles= heads= verified=`. -- **`run_profiles` and `remote_gate_runs`** in `.gstack/pregate.json`, read only from `origin/`. A user-owned policy that sets them is refused, because nothing would read them there. A lane named by a profile clears only through verified runs, so its `--run-url` is verified the same way, and `--bundle-lane` is refused. When the base config cannot be read, any lane's `--run-url` or `--bundle-lane` clear refuses, since the base may profile it. When the GitHub origin cannot be read, a profiled lane refuses rather than fall back to an unverified record. +- **`run_profiles` and `remote_gate_runs`** in `.gstack/pregate.json`, read only from `origin/`. A user-owned policy that sets them is refused, because nothing would read them there. `gstack-pregate clear` clears a lane named by a profile only through verified runs, so its `--run-url` is verified the same way, and `--bundle-lane` is refused. `gstack-ship-queue ready --dispatch-ci` still records a plain run URL for such a lane, and `verify` does not yet re-check it (docs/pregate.md#run-evidence). When the base config cannot be read, any lane's `--run-url` or `--bundle-lane` clear refuses, since the base may profile it. When the GitHub origin cannot be read, a profiled lane refuses rather than fall back to an unverified record. ### Changed - **`gstack-pregate verify`** stays offline. It also refuses a run receipt recorded for another tree unless its stamp-only substitution is recorded. diff --git a/docs/pregate.md b/docs/pregate.md index 579b8d77d..b0ba908fc 100644 --- a/docs/pregate.md +++ b/docs/pregate.md @@ -666,8 +666,13 @@ once: whose newest attempt started after this run was created failed, was cancelled or is still running (`run_started_at`, so a red re-run of an older run counts, and re-running this run does not hide a failure in between). -- The run finished within `max_age_hours`, and on the profile's OS when it - names one. +- The run finished within `max_age_hours` (its jobs' latest `completed_at`, + else the run's `updated_at`), and on the profile's OS when it names one. + Re-running only the failed jobs refuses (`artifact_mismatch`) unless the job + that uploads `tree.json` re-runs too, because that file must name the + newest attempt. The 8 MiB cap is read from the artifact's zipped size before + download, so a workflow on the branch itself can still unpack a larger file + into the temporary directory. Then, for the lane: @@ -686,7 +691,10 @@ Then, for the lane: the remote gate, every file the `inventory` command prints. The inventory is a pin-file command, so it runs only with `--allow-repo-commands`, and only while the working tree is pregate.json's tree (checked before and after it - runs); an empty or failing inventory refuses. + runs); an empty or failing inventory refuses, and it runs under `pipefail`, + so a pipeline whose first command fails refuses too. +- Every run is read once more after the inventory: a run that moved to a new + attempt or stopped being a success meanwhile refuses (`run_changed`). Any API or git error refuses. Each refusal prints `REUSE: refused reason= — …` as `PREGATE_RECEIPT_INVALID`, and a run that fails diff --git a/lib/pregate/index.ts b/lib/pregate/index.ts index 71e936abc..81233836c 100644 --- a/lib/pregate/index.ts +++ b/lib/pregate/index.ts @@ -27,7 +27,7 @@ import { NON_DOWNGRADABLE, PREGATE_CONFIG_PATH, userPolicyAbsent } from './confi import { sha256, withFingerprint } from './context'; import { baseBranchOf, baseMismatchFix, describeSubstitution, substituteStampOnlyTree } from './remote-receipt'; import { resolveBaseBranch } from '../ship-policy'; -import { parseRunReceipt, parseRunRef, runReceiptFields, uncoveredFiles, verifyRun, type RemoteGateRuns, type RunDeps, type RunEvidence, type RunProfile } from '../run-evidence'; +import { parseRunReceipt, parseRunRef, runChanged, runReceiptFields, uncoveredFiles, verifyRun, type RemoteGateRuns, type RunDeps, type RunEvidence, type RunProfile } from '../run-evidence'; import type { CheckDef, CheckResult, PregateContext, Stage } from './types'; export const CHECKS: readonly CheckDef[] = [regenCheck, secretsCheck, straysCheck, literalsCheck, timingCheck, guardsCheck, lanesCheck]; @@ -328,6 +328,12 @@ function clearByRuns(file: PregateFile, lane: string, refs: string[], check: Run } const missing = uncoveredFiles([...required], evidence); if (missing.length) return { refused: 'PREGATE_RECEIPT_INVALID', message: `REUSE: refused reason=${isGate ? 'inventory_uncovered' : 'tests_uncovered'} — ${missing.length} of the ${required.size} test file(s) ${lane} must cover appear in no run's tree.json (${missing.slice(0, 5).join(', ')}${missing.length > 5 ? ', …' : ''}); ${lane} stays uncleared` }; + for (const e of evidence) { + let changed: string | null; + try { changed = runChanged(check.deps, e.run_id, e.run_attempt); } + catch (err: any) { changed = `reading run ${e.run_id} again failed: ${String(err?.message ?? err).slice(0, 200)}`; } + if (changed) return { refused: 'PREGATE_RECEIPT_INVALID', message: `REUSE: refused reason=run_changed — ${changed}; ${lane} stays uncleared` }; + } const tests = evidence.reduce((n, e) => n + (e.tests ?? e.files.length), 0); const accepted = sub.substitution ? ` accepted-for=${file.tree.slice(0, 12)} stamp-only=${sub.substitution.files.length}` : ''; return { receipt: `${isGate ? 'remote-gate' : 'run'} tree=${trees[0]!.slice(0, 12)} tests=${tests}${accepted} ${runReceiptFields(evidence, check.now)}`, attested: false, substitution: sub.substitution }; diff --git a/lib/run-evidence.ts b/lib/run-evidence.ts index cb46282fc..bb1dc3de7 100644 --- a/lib/run-evidence.ts +++ b/lib/run-evidence.ts @@ -37,6 +37,7 @@ import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { sha256Hex, writeReceipt } from './egress-receipt'; +import { runBashSync } from './script-invocation'; export interface RunProfile { workflow: string; events: string[]; jobs: string[]; artifact: string; max_age_hours?: number; os?: string } export interface RemoteGateRuns { profiles: string[]; inventory: string } @@ -45,7 +46,7 @@ export interface ApiRun { path?: string; created_at?: string; run_started_at?: string; updated_at?: string; repository?: { full_name: string } | null; head_repository?: { full_name: string } | null; } -export interface ApiJob { name: string; conclusion: string | null } +export interface ApiJob { name: string; conclusion: string | null; completed_at?: string | null } export interface TreeRecord { run_id?: unknown; run_attempt?: unknown; event?: unknown; sha?: unknown; head_sha?: unknown; tree?: unknown; reused?: unknown; reuse?: unknown; os?: unknown; files?: unknown; tests?: unknown } /** Everything verifyRun reads, injectable so every refusal has a fixture. */ @@ -140,11 +141,13 @@ export function verifyRun(deps: RunDeps, o: { repo: string; runRef: string; prof if (run.status !== 'completed') return fail('run_in_progress', `${at} is ${run.status}`); if (run.conclusion !== 'success') return fail('run_not_success', `${at} concluded ${run.conclusion}`); if (!SHA.test(run.head_sha)) return fail('api_error', `${at} has no head commit`); - const finished = Date.parse(run.updated_at ?? ''); + const jobs = deps.jobs(id, run.run_attempt); + const completed = jobs.map(j => j.completed_at).filter((t): t is string => !!t).sort().pop(); + const finishedAt = completed ?? run.updated_at; + const finished = Date.parse(finishedAt ?? ''); const maxAge = p.max_age_hours ?? DEFAULT_MAX_AGE_HOURS; if (!Number.isFinite(finished)) return fail('api_error', `${at} has no completion time`); - if (o.now.getTime() - finished > maxAge * 3600_000) return fail('run_stale', `${at} finished ${run.updated_at}, more than ${maxAge} h before now`); - const jobs = deps.jobs(id, run.run_attempt); + if (o.now.getTime() - finished > maxAge * 3600_000) return fail('run_stale', `${at} finished ${finishedAt}, more than ${maxAge} h before now`); for (const name of p.jobs) { const hits = jobs.filter(j => j.name === name); if (hits.length === 0) return fail('job_missing', `${at} has no job named ${name}`); @@ -153,7 +156,7 @@ export function verifyRun(deps: RunDeps, o: { repo: string; runRef: string; prof } const heads = [...new Set([run.head_sha, ...(o.vetoHeads ?? []).filter(h => SHA.test(h))])]; for (const head of heads) { - const newer = deps.runsForHead(p.workflow, head).filter(r => !(r.id === id && r.run_attempt === run.run_attempt) && started(r) > (run.created_at ?? '') && !(r.status === 'completed' && r.conclusion === 'success')); + const newer = deps.runsForHead(p.workflow, head).filter(r => !(r.id === id && r.run_attempt === run.run_attempt) && started(r) >= (run.created_at ?? '') && !(r.status === 'completed' && r.conclusion === 'success')); if (newer.length) return fail('newer_run_vetoes', `newer run ${newer[0]!.id} of ${p.workflow} on ${head.slice(0, 12)} is ${newer[0]!.status === 'completed' ? newer[0]!.conclusion : newer[0]!.status}`); } let rec: TreeRecord; @@ -172,14 +175,27 @@ export function verifyRun(deps: RunDeps, o: { repo: string; runRef: string; prof if (rec.tests !== undefined && (!Number.isInteger(rec.tests) || (rec.tests as number) < 0)) return fail('artifact_mismatch', `${at}'s ${p.artifact} tests is not a non-negative integer`); const files = ((rec.files ?? []) as string[]).map(normalPath); const tests = rec.tests === undefined ? null : (rec.tests as number); - const again = deps.run(id); - if (again.run_attempt !== run.run_attempt || again.status !== 'completed' || again.conclusion !== 'success') return fail('run_changed', `run ${id} moved to attempt ${again.run_attempt} (${again.status === 'completed' ? again.conclusion : again.status}) while it was being verified`); - return { ok: true, evidence: { run_id: id, run_attempt: run.run_attempt, head_sha: run.head_sha, tree: headTree, profile: o.profileName, event: run.event, finished_at: run.updated_at!, files, tests } }; + const changed = runChanged(deps, id, run.run_attempt); + if (changed) return fail('run_changed', changed); + return { ok: true, evidence: { run_id: id, run_attempt: run.run_attempt, head_sha: run.head_sha, tree: headTree, profile: o.profileName, event: run.event, finished_at: finishedAt!, files, tests } }; } catch (e: any) { return fail('api_error', `reading run ${id} failed: ${String(e?.message ?? e).slice(0, 300)}`); } } +/** Why run `id` no longer stands as attempt `attempt`, a success; null while it does. Read again after the evidence, and for every run once a whole clear is assembled. */ +export function runChanged(deps: RunDeps, id: number, attempt: number): string | null { + const again = deps.run(id); + return again.run_attempt === attempt && again.status === 'completed' && again.conclusion === 'success' ? null : `run ${id} moved to attempt ${again.run_attempt} (${again.status === 'completed' ? again.conclusion : again.status}) while it was being verified`; +} + +/** The remote gate's inventory: `command`'s non-empty output lines under pipefail, so a pipeline whose producer fails refuses instead of listing part of the gate. */ +export function runInventory(command: string, cwd: string): string[] | { error: string } { + const r = runBashSync(`set -o pipefail\n${command}`, { cwd, encoding: 'utf8', timeout: 300_000, maxBuffer: 64 * 1024 * 1024 }); + if (r.status !== 0 || r.error) return { error: `${command} exited ${r.status ?? r.error}: ${(r.stderr ?? '').trim().slice(0, 200)}` }; + return [...new Set(r.stdout.split('\n').map(l => l.trim()).filter(Boolean))]; +} + /** The files in `inventory` that no verified run's tree.json lists. */ export function uncoveredFiles(inventory: string[], evidence: RunEvidence[]): string[] { const ran = new Set(evidence.flatMap(e => e.files.map(normalPath))); diff --git a/test/pregate-clear-runs.test.ts b/test/pregate-clear-runs.test.ts index 8b0937857..b91de7911 100644 --- a/test/pregate-clear-runs.test.ts +++ b/test/pregate-clear-runs.test.ts @@ -209,6 +209,17 @@ describe.skipIf(process.platform === 'win32')('clear --run: Actions runs as remo expect(verifyPregate(repo.work, forged, { tree: file.tree } as any)).toMatchObject({ ok: false, code: 'PREGATE_REMOTE_UNCLEARED' }); }, 120_000); + test('every run is read again once the clear is assembled: a run that turns red while another profile or the inventory is read refuses', async () => { + const repo = fixture(); + const head = repo.git('rev-parse', 'HEAD'); + const file = await pregateAt(repo); + const base = check(repo, [{ id: 200, profile: 'test', head, files: ['test/a.test.ts', 'test/util.test.js'] }, { id: 201, profile: 'e2e', head, files: ['test/b.test.ts'] }]); + let flipped = false; + const evidence: RunCheck = { ...base, deps: { ...base.deps, run: id => flipped && id === 200 ? { ...base.deps.run(id), run_attempt: 2, conclusion: 'failure' } : base.deps.run(id) }, inventory: () => { flipped = true; return ['test/a.test.ts', 'test/b.test.ts']; } }; + const d = clearDecision(file, LANE, { runs: ['200', '201'], evidence }, repo.work); + expect('refused' in d && d.message).toContain('REUSE: refused reason=run_changed — run 200 moved to attempt 2 (failure)'); + }, 120_000); + test('origin is read before url.insteadOf, so a Capy git.capy.ai proxy still names the GitHub repository', async () => { const repo = fixture(); const file = await pregateAt(repo); diff --git a/test/run-evidence.test.ts b/test/run-evidence.test.ts index feaada998..19e1cf361 100644 --- a/test/run-evidence.test.ts +++ b/test/run-evidence.test.ts @@ -3,7 +3,7 @@ * the dependencies are fakes, so nothing here reaches GitHub. */ import { describe, expect, test } from 'bun:test'; -import { parseRunReceipt, parseRunRef, runProfileErrors, runReceiptFields, uncoveredFiles, verifyRun, type ApiJob, type ApiRun, type RunDeps, type RunProfile, type TreeRecord } from '../lib/run-evidence'; +import { parseRunReceipt, parseRunRef, runInventory, runProfileErrors, runReceiptFields, uncoveredFiles, verifyRun, type ApiJob, type ApiRun, type RunDeps, type RunProfile, type TreeRecord } from '../lib/run-evidence'; const REPO = 'garrytan/gbrain'; const HEAD = 'a'.repeat(40); @@ -127,6 +127,20 @@ describe('verifyRun', () => { expect(reason(world({ artifact: { ...(w.artifact as TreeRecord), os: 'Linux' } }), { profile: { ...PROFILE, os: 'Linux' } })).toBe('ok'); }); + test('a failed run created in the same second as the candidate vetoes it: ties count as newer', () => { + const w = world(); + expect(reason(world({ others: [{ ...w.run, id: 101, status: 'completed', conclusion: 'failure' }] }))).toBe('newer_run_vetoes'); + }); + + test('freshness reads the jobs\' latest completed_at when the API gives it, not a later updated_at', () => { + const w = world(); + const jobs: ApiJob[] = [{ name: 'full-suite', conclusion: 'success', completed_at: '2026-10-28T10:00:00Z' }, { name: 'lint', conclusion: 'success', completed_at: '2026-10-28T09:00:00Z' }]; + expect(reason(world({ jobs }))).toBe('run_stale'); + const v = check(world({ jobs: jobs.map(j => ({ ...j, completed_at: '2026-10-29T10:10:00Z' })) })); + expect(v.ok && v.evidence.finished_at).toBe('2026-10-29T10:10:00Z'); + expect(w.run.updated_at).toBe('2026-10-29T10:20:00Z'); + }); + test('max_age_hours widens the freshness window', () => { const stale = world({ run: { ...world().run, updated_at: '2026-10-28T10:00:00Z' } }); expect(reason(stale)).toBe('run_stale'); @@ -134,6 +148,14 @@ describe('verifyRun', () => { }); }); +describe('runInventory', () => { + test('lists unique non-empty lines, and a pipeline whose producer fails refuses instead of listing part of the gate', () => { + const cwd = process.cwd(); + expect(runInventory("printf 'test/b.test.ts\\n\\ntest/a.test.ts\\ntest/b.test.ts\\n'", cwd)).toEqual(['test/b.test.ts', 'test/a.test.ts']); + expect(runInventory("{ printf 'test/a.test.ts\\n'; exit 17; } | sort", cwd)).toMatchObject({ error: expect.stringContaining('exited 17') }); + }); +}); + describe('profiles, coverage and receipts', () => { test('runProfileErrors accepts a well-formed pair and names every malformed field', () => { expect(runProfileErrors({ test: PROFILE, e2e: { ...PROFILE, workflow: 'e2e.yml', jobs: ['e2e-full-suite'], artifact: 'e2e-full-suite-tree', os: 'Linux', max_age_hours: 12 } }, { profiles: ['test', 'e2e'], inventory: 'bash scripts/pregate-remote-gate.sh --list' })).toEqual([]);