fix(pregate): clear --run re-reads every run before recording, ties veto, inventory runs under pipefail, freshness reads job completion, and a pregate.json rewritten mid-clear refuses

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed 2026-10-31 13:27:02 +00:00
1 parent 97d387c04d
commit 1652e37fcf
7 files changed
+84 -20

No files matched your search

+7 -6
View File
@@ -31,8 +31,8 @@ import { PREGATE_CONFIG_PATH, explainConfig, isConfigUnavailable, loadConfigFrom
import { describeDraft, draftConfig } from '../lib/pregate/init-draft';
import { buildContext, defaultUserConfigPath, describeRefusal, loadedDotenvFiles } from '../lib/pregate/context';
import { CHECKS, CHECK_IDS, PREGATE_OUT_REL, PREGATE_TIER2, clearDecision, clearObligation, readPregateFile, renderTable, runPregate, verifyPregate, writePregateFile, type RunCheck } from '../lib/pregate/index';
import { ghRunDeps } from '../lib/run-evidence';
import { runBashSync, wtreeFingerprint } from '../lib/script-invocation';
import { ghRunDeps, runInventory } from '../lib/run-evidence';
import { wtreeFingerprint } from '../lib/script-invocation';
import { describeLane, discoverLanes } from '../lib/pregate/workflows';
import { REMOTE_GATE_LANE } from '../lib/pregate/checks/lanes';
import { describeSubstitution } from '../lib/pregate/remote-receipt';
@@ -237,7 +237,9 @@ function cmdClear(root: string, rest: string[]): number {
process.stderr.write(`gstack-pregate clear: receipt refused — ${decision.message}; fix: ${decision.fix ?? RESULT_CODES.PREGATE_RECEIPT_INVALID.fix} ${resultCodeSuffix('PREGATE_RECEIPT_INVALID')}\n`);
return EXIT.fail;
}
const file = clearObligation(read.file, lane, decision.receipt, `gstack-pregate clear ${new Date().toISOString()}`, decision.substitution);
const current = readPregateFile(out).file;
if (!current || current.tree !== read.file.tree || current.generated_at !== read.file.generated_at) { process.stderr.write(resultLine('gstack-pregate clear', 'PREGATE_STALE', `${out} changed while ${lane} was being verified; rerun the clear against the new file`) + '\n'); return EXIT.fail; }
const file = clearObligation(current, lane, decision.receipt, `gstack-pregate clear ${new Date().toISOString()}`, decision.substitution);
writePregateFile(root, out, file);
if (/ runs=\S+ profiles=/.test(decision.receipt)) process.stdout.write(`REUSE: accepted ${lane} ${decision.receipt}\n`);
if (decision.substitution) process.stdout.write(`PREGATE_RECEIPT_SUBSTITUTED: ${describeSubstitution(decision.substitution)}\n`);
@@ -261,10 +263,9 @@ function runCheck(root: string, lane: string, allowRepoCommands: boolean): RunCh
if (!allowRepoCommands) return { error: 'the inventory command is a pin-file command; pass --allow-repo-commands' };
const at = () => { const fp = wtreeFingerprint({ cwd: root }); return 'tree' in fp ? fp.tree : null; };
if (at() !== tree) return { error: `the working tree is not pregate.json's tree ${tree.slice(0, 12)}; the inventory must list that tree's gate files` };
const r = runBashSync(command, { cwd: root, encoding: 'utf8', timeout: 300_000, maxBuffer: 64 * 1024 * 1024 });
if (r.status !== 0 || r.error) return { error: `${command} exited ${r.status ?? r.error}: ${(r.stderr ?? '').trim().slice(0, 200)}` };
const listed = runInventory(command, root);
if (at() !== tree) return { error: `the working tree changed while ${command} ran` };
return [...new Set(r.stdout.split('\n').map(l => l.trim()).filter(Boolean))];
return listed;
},
};
}
+1 -1
View File
@@ -2,7 +2,7 @@
### Added
- **`gstack-pregate clear --run <id|url>`** (repeatable; `lib/run-evidence.ts`, docs/pregate.md#run-evidence). A run is accepted only when it belongs to this repository, ran a `push` or `workflow_dispatch` event of the profile's workflow, and its newest attempt succeeded with every named job passing exactly once. Its `tree.json` must name the run, attempt, event and head and record the head's tree, and it must be fresh (`max_age_hours`, default 24) and on the profile's OS. It is refused if any run of that workflow on the head that started after it failed, was cancelled or is still running, or if the run changed attempts while being checked. All runs must be on one tree: pregate.json's, or a stamp-only predecessor under the existing tree-receipt rule. Their `files` must cover the obligation's selected tests plus every file the declared `inventory` command lists for that tree. Refusals print `REUSE: refused reason=<code>`; an accepted clear prints `REUSE: accepted`, and the receipt records `runs=<id>/<attempt> profiles= heads= verified=`.
- **`run_profiles` and `remote_gate_runs`** in `.gstack/pregate.json`, read only from `origin/<base>`. A user-owned policy that sets them is refused, because nothing would read them there. A lane named by a profile clears only through verified runs, so its `--run-url` is verified the same way, and `--bundle-lane` is refused. When the base config cannot be read, any lane's `--run-url` or `--bundle-lane` clear refuses, since the base may profile it. When the GitHub origin cannot be read, a profiled lane refuses rather than fall back to an unverified record.
- **`run_profiles` and `remote_gate_runs`** in `.gstack/pregate.json`, read only from `origin/<base>`. A user-owned policy that sets them is refused, because nothing would read them there. `gstack-pregate clear` clears a lane named by a profile only through verified runs, so its `--run-url` is verified the same way, and `--bundle-lane` is refused. `gstack-ship-queue ready --dispatch-ci` still records a plain run URL for such a lane, and `verify` does not yet re-check it (docs/pregate.md#run-evidence). When the base config cannot be read, any lane's `--run-url` or `--bundle-lane` clear refuses, since the base may profile it. When the GitHub origin cannot be read, a profiled lane refuses rather than fall back to an unverified record.
### Changed
- **`gstack-pregate verify`** stays offline. It also refuses a run receipt recorded for another tree unless its stamp-only substitution is recorded.
+11 -3
View File
@@ -666,8 +666,13 @@ once:
whose newest attempt started after this run was created failed, was
cancelled or is still running (`run_started_at`, so a red re-run of an older
run counts, and re-running this run does not hide a failure in between).
- The run finished within `max_age_hours`, and on the profile's OS when it
names one.
- The run finished within `max_age_hours` (its jobs' latest `completed_at`,
else the run's `updated_at`), and on the profile's OS when it names one.
Re-running only the failed jobs refuses (`artifact_mismatch`) unless the job
that uploads `tree.json` re-runs too, because that file must name the
newest attempt. The 8 MiB cap is read from the artifact's zipped size before
download, so a workflow on the branch itself can still unpack a larger file
into the temporary directory.
Then, for the lane:
@@ -686,7 +691,10 @@ Then, for the lane:
the remote gate, every file the `inventory` command prints. The inventory is
a pin-file command, so it runs only with `--allow-repo-commands`, and only
while the working tree is pregate.json's tree (checked before and after it
runs); an empty or failing inventory refuses.
runs); an empty or failing inventory refuses, and it runs under `pipefail`,
so a pipeline whose first command fails refuses too.
- Every run is read once more after the inventory: a run that moved to a new
attempt or stopped being a success meanwhile refuses (`run_changed`).
Any API or git error refuses. Each refusal prints `REUSE: refused
reason=<code> — …` as `PREGATE_RECEIPT_INVALID`, and a run that fails
+7 -1
View File
@@ -27,7 +27,7 @@ import { NON_DOWNGRADABLE, PREGATE_CONFIG_PATH, userPolicyAbsent } from './confi
import { sha256, withFingerprint } from './context';
import { baseBranchOf, baseMismatchFix, describeSubstitution, substituteStampOnlyTree } from './remote-receipt';
import { resolveBaseBranch } from '../ship-policy';
import { parseRunReceipt, parseRunRef, runReceiptFields, uncoveredFiles, verifyRun, type RemoteGateRuns, type RunDeps, type RunEvidence, type RunProfile } from '../run-evidence';
import { parseRunReceipt, parseRunRef, runChanged, runReceiptFields, uncoveredFiles, verifyRun, type RemoteGateRuns, type RunDeps, type RunEvidence, type RunProfile } from '../run-evidence';
import type { CheckDef, CheckResult, PregateContext, Stage } from './types';
export const CHECKS: readonly CheckDef[] = [regenCheck, secretsCheck, straysCheck, literalsCheck, timingCheck, guardsCheck, lanesCheck];
@@ -328,6 +328,12 @@ function clearByRuns(file: PregateFile, lane: string, refs: string[], check: Run
}
const missing = uncoveredFiles([...required], evidence);
if (missing.length) return { refused: 'PREGATE_RECEIPT_INVALID', message: `REUSE: refused reason=${isGate ? 'inventory_uncovered' : 'tests_uncovered'} — ${missing.length} of the ${required.size} test file(s) ${lane} must cover appear in no run's tree.json (${missing.slice(0, 5).join(', ')}${missing.length > 5 ? ', …' : ''}); ${lane} stays uncleared` };
for (const e of evidence) {
let changed: string | null;
try { changed = runChanged(check.deps, e.run_id, e.run_attempt); }
catch (err: any) { changed = `reading run ${e.run_id} again failed: ${String(err?.message ?? err).slice(0, 200)}`; }
if (changed) return { refused: 'PREGATE_RECEIPT_INVALID', message: `REUSE: refused reason=run_changed — ${changed}; ${lane} stays uncleared` };
}
const tests = evidence.reduce((n, e) => n + (e.tests ?? e.files.length), 0);
const accepted = sub.substitution ? ` accepted-for=${file.tree.slice(0, 12)} stamp-only=${sub.substitution.files.length}` : '';
return { receipt: `${isGate ? 'remote-gate' : 'run'} tree=${trees[0]!.slice(0, 12)} tests=${tests}${accepted} ${runReceiptFields(evidence, check.now)}`, attested: false, substitution: sub.substitution };
+24 -8
View File
@@ -37,6 +37,7 @@ import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { sha256Hex, writeReceipt } from './egress-receipt';
import { runBashSync } from './script-invocation';
export interface RunProfile { workflow: string; events: string[]; jobs: string[]; artifact: string; max_age_hours?: number; os?: string }
export interface RemoteGateRuns { profiles: string[]; inventory: string }
@@ -45,7 +46,7 @@ export interface ApiRun {
path?: string; created_at?: string; run_started_at?: string; updated_at?: string;
repository?: { full_name: string } | null; head_repository?: { full_name: string } | null;
}
export interface ApiJob { name: string; conclusion: string | null }
export interface ApiJob { name: string; conclusion: string | null; completed_at?: string | null }
export interface TreeRecord { run_id?: unknown; run_attempt?: unknown; event?: unknown; sha?: unknown; head_sha?: unknown; tree?: unknown; reused?: unknown; reuse?: unknown; os?: unknown; files?: unknown; tests?: unknown }
/** Everything verifyRun reads, injectable so every refusal has a fixture. */
@@ -140,11 +141,13 @@ export function verifyRun(deps: RunDeps, o: { repo: string; runRef: string; prof
if (run.status !== 'completed') return fail('run_in_progress', `${at} is ${run.status}`);
if (run.conclusion !== 'success') return fail('run_not_success', `${at} concluded ${run.conclusion}`);
if (!SHA.test(run.head_sha)) return fail('api_error', `${at} has no head commit`);
const finished = Date.parse(run.updated_at ?? '');
const jobs = deps.jobs(id, run.run_attempt);
const completed = jobs.map(j => j.completed_at).filter((t): t is string => !!t).sort().pop();
const finishedAt = completed ?? run.updated_at;
const finished = Date.parse(finishedAt ?? '');
const maxAge = p.max_age_hours ?? DEFAULT_MAX_AGE_HOURS;
if (!Number.isFinite(finished)) return fail('api_error', `${at} has no completion time`);
if (o.now.getTime() - finished > maxAge * 3600_000) return fail('run_stale', `${at} finished ${run.updated_at}, more than ${maxAge} h before now`);
const jobs = deps.jobs(id, run.run_attempt);
if (o.now.getTime() - finished > maxAge * 3600_000) return fail('run_stale', `${at} finished ${finishedAt}, more than ${maxAge} h before now`);
for (const name of p.jobs) {
const hits = jobs.filter(j => j.name === name);
if (hits.length === 0) return fail('job_missing', `${at} has no job named ${name}`);
@@ -153,7 +156,7 @@ export function verifyRun(deps: RunDeps, o: { repo: string; runRef: string; prof
}
const heads = [...new Set([run.head_sha, ...(o.vetoHeads ?? []).filter(h => SHA.test(h))])];
for (const head of heads) {
const newer = deps.runsForHead(p.workflow, head).filter(r => !(r.id === id && r.run_attempt === run.run_attempt) && started(r) > (run.created_at ?? '') && !(r.status === 'completed' && r.conclusion === 'success'));
const newer = deps.runsForHead(p.workflow, head).filter(r => !(r.id === id && r.run_attempt === run.run_attempt) && started(r) >= (run.created_at ?? '') && !(r.status === 'completed' && r.conclusion === 'success'));
if (newer.length) return fail('newer_run_vetoes', `newer run ${newer[0]!.id} of ${p.workflow} on ${head.slice(0, 12)} is ${newer[0]!.status === 'completed' ? newer[0]!.conclusion : newer[0]!.status}`);
}
let rec: TreeRecord;
@@ -172,14 +175,27 @@ export function verifyRun(deps: RunDeps, o: { repo: string; runRef: string; prof
if (rec.tests !== undefined && (!Number.isInteger(rec.tests) || (rec.tests as number) < 0)) return fail('artifact_mismatch', `${at}'s ${p.artifact} tests is not a non-negative integer`);
const files = ((rec.files ?? []) as string[]).map(normalPath);
const tests = rec.tests === undefined ? null : (rec.tests as number);
const again = deps.run(id);
if (again.run_attempt !== run.run_attempt || again.status !== 'completed' || again.conclusion !== 'success') return fail('run_changed', `run ${id} moved to attempt ${again.run_attempt} (${again.status === 'completed' ? again.conclusion : again.status}) while it was being verified`);
return { ok: true, evidence: { run_id: id, run_attempt: run.run_attempt, head_sha: run.head_sha, tree: headTree, profile: o.profileName, event: run.event, finished_at: run.updated_at!, files, tests } };
const changed = runChanged(deps, id, run.run_attempt);
if (changed) return fail('run_changed', changed);
return { ok: true, evidence: { run_id: id, run_attempt: run.run_attempt, head_sha: run.head_sha, tree: headTree, profile: o.profileName, event: run.event, finished_at: finishedAt!, files, tests } };
} catch (e: any) {
return fail('api_error', `reading run ${id} failed: ${String(e?.message ?? e).slice(0, 300)}`);
}
}
/** Why run `id` no longer stands as attempt `attempt`, a success; null while it does. Read again after the evidence, and for every run once a whole clear is assembled. */
export function runChanged(deps: RunDeps, id: number, attempt: number): string | null {
const again = deps.run(id);
return again.run_attempt === attempt && again.status === 'completed' && again.conclusion === 'success' ? null : `run ${id} moved to attempt ${again.run_attempt} (${again.status === 'completed' ? again.conclusion : again.status}) while it was being verified`;
}
/** The remote gate's inventory: `command`'s non-empty output lines under pipefail, so a pipeline whose producer fails refuses instead of listing part of the gate. */
export function runInventory(command: string, cwd: string): string[] | { error: string } {
const r = runBashSync(`set -o pipefail\n${command}`, { cwd, encoding: 'utf8', timeout: 300_000, maxBuffer: 64 * 1024 * 1024 });
if (r.status !== 0 || r.error) return { error: `${command} exited ${r.status ?? r.error}: ${(r.stderr ?? '').trim().slice(0, 200)}` };
return [...new Set(r.stdout.split('\n').map(l => l.trim()).filter(Boolean))];
}
/** The files in `inventory` that no verified run's tree.json lists. */
export function uncoveredFiles(inventory: string[], evidence: RunEvidence[]): string[] {
const ran = new Set(evidence.flatMap(e => e.files.map(normalPath)));
+11
View File
@@ -209,6 +209,17 @@ describe.skipIf(process.platform === 'win32')('clear --run: Actions runs as remo
expect(verifyPregate(repo.work, forged, { tree: file.tree } as any)).toMatchObject({ ok: false, code: 'PREGATE_REMOTE_UNCLEARED' });
}, 120_000);
test('every run is read again once the clear is assembled: a run that turns red while another profile or the inventory is read refuses', async () => {
const repo = fixture();
const head = repo.git('rev-parse', 'HEAD');
const file = await pregateAt(repo);
const base = check(repo, [{ id: 200, profile: 'test', head, files: ['test/a.test.ts', 'test/util.test.js'] }, { id: 201, profile: 'e2e', head, files: ['test/b.test.ts'] }]);
let flipped = false;
const evidence: RunCheck = { ...base, deps: { ...base.deps, run: id => flipped && id === 200 ? { ...base.deps.run(id), run_attempt: 2, conclusion: 'failure' } : base.deps.run(id) }, inventory: () => { flipped = true; return ['test/a.test.ts', 'test/b.test.ts']; } };
const d = clearDecision(file, LANE, { runs: ['200', '201'], evidence }, repo.work);
expect('refused' in d && d.message).toContain('REUSE: refused reason=run_changed — run 200 moved to attempt 2 (failure)');
}, 120_000);
test('origin is read before url.insteadOf, so a Capy git.capy.ai proxy still names the GitHub repository', async () => {
const repo = fixture();
const file = await pregateAt(repo);
+23 -1
View File
@@ -3,7 +3,7 @@
* the dependencies are fakes, so nothing here reaches GitHub.
*/
import { describe, expect, test } from 'bun:test';
import { parseRunReceipt, parseRunRef, runProfileErrors, runReceiptFields, uncoveredFiles, verifyRun, type ApiJob, type ApiRun, type RunDeps, type RunProfile, type TreeRecord } from '../lib/run-evidence';
import { parseRunReceipt, parseRunRef, runInventory, runProfileErrors, runReceiptFields, uncoveredFiles, verifyRun, type ApiJob, type ApiRun, type RunDeps, type RunProfile, type TreeRecord } from '../lib/run-evidence';
const REPO = 'garrytan/gbrain';
const HEAD = 'a'.repeat(40);
@@ -127,6 +127,20 @@ describe('verifyRun', () => {
expect(reason(world({ artifact: { ...(w.artifact as TreeRecord), os: 'Linux' } }), { profile: { ...PROFILE, os: 'Linux' } })).toBe('ok');
});
test('a failed run created in the same second as the candidate vetoes it: ties count as newer', () => {
const w = world();
expect(reason(world({ others: [{ ...w.run, id: 101, status: 'completed', conclusion: 'failure' }] }))).toBe('newer_run_vetoes');
});
test('freshness reads the jobs\' latest completed_at when the API gives it, not a later updated_at', () => {
const w = world();
const jobs: ApiJob[] = [{ name: 'full-suite', conclusion: 'success', completed_at: '2026-10-28T10:00:00Z' }, { name: 'lint', conclusion: 'success', completed_at: '2026-10-28T09:00:00Z' }];
expect(reason(world({ jobs }))).toBe('run_stale');
const v = check(world({ jobs: jobs.map(j => ({ ...j, completed_at: '2026-10-29T10:10:00Z' })) }));
expect(v.ok && v.evidence.finished_at).toBe('2026-10-29T10:10:00Z');
expect(w.run.updated_at).toBe('2026-10-29T10:20:00Z');
});
test('max_age_hours widens the freshness window', () => {
const stale = world({ run: { ...world().run, updated_at: '2026-10-28T10:00:00Z' } });
expect(reason(stale)).toBe('run_stale');
@@ -134,6 +148,14 @@ describe('verifyRun', () => {
});
});
describe('runInventory', () => {
test('lists unique non-empty lines, and a pipeline whose producer fails refuses instead of listing part of the gate', () => {
const cwd = process.cwd();
expect(runInventory("printf 'test/b.test.ts\\n\\ntest/a.test.ts\\ntest/b.test.ts\\n'", cwd)).toEqual(['test/b.test.ts', 'test/a.test.ts']);
expect(runInventory("{ printf 'test/a.test.ts\\n'; exit 17; } | sort", cwd)).toMatchObject({ error: expect.stringContaining('exited 17') });
});
});
describe('profiles, coverage and receipts', () => {
test('runProfileErrors accepts a well-formed pair and names every malformed field', () => {
expect(runProfileErrors({ test: PROFILE, e2e: { ...PROFILE, workflow: 'e2e.yml', jobs: ['e2e-full-suite'], artifact: 'e2e-full-suite-tree', os: 'Linux', max_age_hours: 12 } }, { profiles: ['test', 'e2e'], inventory: 'bash scripts/pregate-remote-gate.sh --list' })).toEqual([]);