From 560c7c98105e3f063d69c8ae4d1c71c50d0e959c Mon Sep 17 00:00:00 2001 From: garrytan Date: Tue, 3 Nov 2026 07:41:03 +0000 Subject: [PATCH 1/4] feat(ci): pull requests reuse a green Free Tests run they can't control (GSTA-258 L4, plan E) A pull_request run of free-tests.yml reuses a green push or dispatch run on main, or a dispatch run on its own branch, at the merge commit's exact tree, only when the merge commit leaves the workflow, scripts/ci-tree-reuse.ts and lib/run-evidence.ts byte-identical to main. A pull_request run is never a source. Opt-in per workflow with find --pull-request; windows-free-tests.yml is unchanged. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- .github/workflows/free-tests.yml | 39 +++++++------ changelog.d/gsta258-l4-pr-ci-reuse.md | 1 + docs/pregate.md | 33 +++++++++++ lib/run-evidence.ts | 6 +- scripts/ci-tree-reuse.ts | 82 ++++++++++++++++++++++++++- test/ci-reuse-workflow-wiring.test.ts | 25 +++++--- test/ci-tree-reuse.test.ts | 74 +++++++++++++++++++++++- 7 files changed, 228 insertions(+), 32 deletions(-) create mode 100644 changelog.d/gsta258-l4-pr-ci-reuse.md diff --git a/.github/workflows/free-tests.yml b/.github/workflows/free-tests.yml index b7b8f291b..0851743df 100644 --- a/.github/workflows/free-tests.yml +++ b/.github/workflows/free-tests.yml @@ -20,9 +20,14 @@ name: Free Tests # passes. A push to main first runs `reuse`: when the merged pull request's # head has the pushed tree and a green pull_request or workflow_dispatch run of # this workflow verifies (lib/run-evidence.ts), every suite job is skipped and -# the aggregate links that run instead. Any other answer, a reuse failure -# included, runs every job as before. Pull request and dispatch runs never -# consume; reused runs never upload a record, so they are never reused. +# the aggregate links that run instead. A pull request run reuses too, but +# only a run the pull request cannot control: a green push or dispatch run on +# main, or a dispatch run on its own branch, at the merge commit's exact tree, +# while the merge commit leaves this file, scripts/ci-tree-reuse.ts and +# lib/run-evidence.ts byte-identical to main. A pull_request run is never a +# source. Any other answer, a reuse failure included, runs every job as before. +# Dispatch runs never consume; reused runs never upload a record, so they are +# never reused. on: pull_request: @@ -77,14 +82,14 @@ permissions: contents: read jobs: - # Main pushes only, and the one job with actions: read (it reads this - # repository's runs and their tested-tree artifacts). Every suite job below - # runs unless this job succeeded and reused a run; a skipped or failed reuse - # job runs them all. Their conditions start with !cancelled() because a - # skipped needed job would otherwise skip its dependents on pull_request and - # dispatch runs, where reuse never runs. + # Main pushes and pull requests only, and the one job with actions: read (it + # reads this repository's runs and their tested-tree artifacts). Every suite + # job below runs unless this job succeeded and reused a run; a skipped or + # failed reuse job runs them all. Their conditions start with !cancelled() + # because a skipped needed job would otherwise skip its dependents on dispatch + # runs, where reuse never runs. reuse: - if: github.event_name == 'push' && github.ref == 'refs/heads/main' + if: (github.event_name == 'push' && github.ref == 'refs/heads/main') || github.event_name == 'pull_request' runs-on: ubuntu-24.04 timeout-minutes: 5 permissions: @@ -103,10 +108,10 @@ jobs: with: bun-version: 1.4.2 - id: find - name: Find a green pull request or dispatch run of this exact tree + name: Find a green run of this exact tree that this run's pull request cannot control env: GH_TOKEN: ${{ github.token }} - run: bun run scripts/ci-tree-reuse.ts find --workflow free-tests.yml --profile ci-reuse/free-tests + run: bun run scripts/ci-tree-reuse.ts find --workflow free-tests.yml --profile ci-reuse/free-tests --pull-request free-plan: needs: reuse @@ -669,7 +674,7 @@ jobs: # stable name as an always-running aggregate so every platform-specific CSO # gate is merge-blocking without a separate branch-protection migration. # Its first step picks one of two exclusive paths before any download: a - # verified reuse (main pushes only) links the reused run and passes; anything + # verified reuse (main pushes and pull requests) links the reused run and passes; anything # else takes the strict path below unchanged, which alone uploads tested-tree. free-tests: if: always() @@ -677,7 +682,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 5 steps: - - name: Take the reuse path only for a verified main-push reuse + - name: Take the reuse path only for a verified main-push or pull request reuse id: route env: REUSE_RESULT: ${{ needs.reuse.result }} @@ -691,8 +696,10 @@ jobs: echo 'reused=false' >> "$GITHUB_OUTPUT" exit 0 fi - test "$GITHUB_EVENT_NAME" = push - test "$GITHUB_REF" = refs/heads/main + case "$GITHUB_EVENT_NAME:$GITHUB_REF" in + push:refs/heads/main|pull_request:refs/pull/*/merge) ;; + *) echo "::error::reuse claimed on a $GITHUB_EVENT_NAME run of $GITHUB_REF"; exit 1 ;; + esac case "$REUSE_RUN_ID" in ''|*[!0-9]*) echo "::error::the reuse job named no run id"; exit 1 ;; esac case "$REUSE_TREE" in ''|*[!0-9a-f]*) echo "::error::the reuse job named no tree"; exit 1 ;; esac test "$REUSE_RUN_URL" = "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$REUSE_RUN_ID" diff --git a/changelog.d/gsta258-l4-pr-ci-reuse.md b/changelog.d/gsta258-l4-pr-ci-reuse.md new file mode 100644 index 000000000..5d5178852 --- /dev/null +++ b/changelog.d/gsta258-l4-pr-ci-reuse.md @@ -0,0 +1 @@ +- **Pull requests reuse a green Free Tests run they can't control** (`scripts/ci-tree-reuse.ts find --pull-request`, docs/pregate.md#ci-reuse-pr). A pull request run skips the suite when a push or dispatch run on main, or a dispatch run on its own branch, already passed on the merge commit's exact tree, and the pull request leaves the workflow, the reuse script and `lib/run-evidence.ts` unchanged. A pull request run is never a source, so a pull request never certifies itself. diff --git a/docs/pregate.md b/docs/pregate.md index 2a5ca0943..0984609f5 100644 --- a/docs/pregate.md +++ b/docs/pregate.md @@ -1059,6 +1059,39 @@ and without it every push falls back to the full run. Windows Free Tests now runs nightly and by hand only (docs/support-tiers.md), never on a push, so its `reuse` job is always skipped and every nightly run takes the strict path. + +#### Pull requests reuse only a run they can't control + +Free Tests' `reuse` job also runs on `pull_request`, as +`find … --pull-request` (`findPrReuse`). Main pushes stay the only consumers +of anything a pull request could certify for itself, so a `pull_request` run +is never a source. A pull request run reuses a green run only when all of these +hold: + +1. the pull request's head is on this repository (`fork_head`) and it targets + main (`not_main_pr`); +2. GitHub's merge commit has exactly two parents, main and the pull request's + head (`merge_mismatch`); +3. the merge commit leaves `.github/workflows/free-tests.yml` + (`workflow_changed`), `scripts/ci-tree-reuse.ts` and `lib/run-evidence.ts` + (`script_changed`) byte-identical to its main parent, so the rules that + decide the reuse are main's; +4. the source is a `push` or `workflow_dispatch` run on main, or a + `workflow_dispatch` run on the pull request's own branch, newest first and + at most 10 (`no_candidate`), whose head commit has the merge commit's tree + (`tree_mismatch`); and +5. `verifyRun` accepts it under `ci-reuse/free-tests` with `push` and + `workflow_dispatch` as the accepted events, its `tested-tree` record names + the merge commit's tree, and no newer red run on its head vetoes it (the + pull request's own run in progress is not a veto). + +The aggregate's route step accepts a reuse on a `pull_request` run of +`refs/pull//merge` as it does on a push to main. A reused pull request run +uploads no `tested-tree`, so the main push after the merge finds the source run +on the head (a branch dispatch) or runs every job. The usual case is a branch +that already holds main's tip. Dispatch Free Tests on the branch, and the +pull request run that follows reuses it instead of running the suite again. + ## Guards and the lane-done rule diff --git a/lib/run-evidence.ts b/lib/run-evidence.ts index 9a66a99d9..19801123c 100644 --- a/lib/run-evidence.ts +++ b/lib/run-evidence.ts @@ -45,7 +45,7 @@ import { runBashSync } from './script-invocation'; export interface RunProfile { workflow: string; events: string[]; jobs: string[]; artifact: string; max_age_hours?: number; os?: string } export interface RemoteGateRuns { profiles: string[]; inventory: string; skipped?: string } export interface ApiRun { - id: number; run_attempt: number; status: string; conclusion: string | null; event: string; head_sha: string; + id: number; run_attempt: number; status: string; conclusion: string | null; event: string; head_sha: string; head_branch?: string | null; path?: string; created_at?: string; run_started_at?: string; updated_at?: string; repository?: { full_name: string } | null; head_repository?: { full_name: string } | null; } @@ -132,7 +132,7 @@ const normalPath = (f: string): string => f.replace(/^(\.\/)+/, ''); * The verdict for run `runRef` under `profile`. `vetoHeads` are further heads * whose newer runs of the same workflow can veto (the consumer's own head). */ -export function verifyRun(deps: RunDeps, o: { repo: string; runRef: string; profileName: string; profile: RunProfile; vetoHeads?: string[]; now: Date }): RunVerdict { +export function verifyRun(deps: RunDeps, o: { repo: string; runRef: string; profileName: string; profile: RunProfile; vetoHeads?: string[]; ignoreRun?: number; now: Date }): RunVerdict { const id = parseRunRef(o.runRef, o.repo); if (id === null) return fail('run_ref_invalid', `${JSON.stringify(o.runRef)} is not a run id or a run URL on ${o.repo}`); const p = o.profile; @@ -161,7 +161,7 @@ export function verifyRun(deps: RunDeps, o: { repo: string; runRef: string; prof } const heads = [...new Set([run.head_sha, ...(o.vetoHeads ?? []).filter(h => SHA.test(h))])]; for (const head of heads) { - const newer = deps.runsForHead(p.workflow, head).filter(r => !(r.id === id && r.run_attempt === run.run_attempt) && started(r) >= (run.created_at ?? '') && !(r.status === 'completed' && r.conclusion === 'success')); + const newer = deps.runsForHead(p.workflow, head).filter(r => !(r.id === id && r.run_attempt === run.run_attempt) && r.id !== o.ignoreRun && started(r) >= (run.created_at ?? '') && !(r.status === 'completed' && r.conclusion === 'success')); if (newer.length) return fail('newer_run_vetoes', `newer run ${newer[0]!.id} of ${p.workflow} on ${head.slice(0, 12)} is ${newer[0]!.status === 'completed' ? newer[0]!.conclusion : newer[0]!.status}`); } let rec: TreeRecord; diff --git a/scripts/ci-tree-reuse.ts b/scripts/ci-tree-reuse.ts index 9107a0a0a..d0558edbb 100755 --- a/scripts/ci-tree-reuse.ts +++ b/scripts/ci-tree-reuse.ts @@ -17,7 +17,14 @@ * .gstack/pregate.json. A pull_request run counts only when its merge commit's tree is its head's tree * (verifyRun). It writes reused=true, run_id, run_url and tree to GITHUB_OUTPUT, or reused=false and a * reason, with one REUSE: line in the step summary. Every lookup failure is a refusal, never a pass: the - * workflow then runs every job as before. Pull request and dispatch runs never consume. + * workflow then runs every job as before. Dispatch runs never consume. + * + * bun run scripts/ci-tree-reuse.ts find --workflow --profile --pull-request + * Also lets a pull_request run consume, but only a run the pull request cannot control (findPrReuse): a green + * push or dispatch run on main, or a dispatch run on the pull request's own same-repository branch, whose tested + * tree is the merge commit's tree, while the merge commit leaves the workflow file, this script and + * lib/run-evidence.ts byte-identical to its base parent. A pull_request run is never a source, so a pull request + * never certifies itself. Without --pull-request a pull_request run refuses (not_main_push) as before. */ import { spawnSync } from 'node:child_process'; import * as fs from 'node:fs'; @@ -31,6 +38,14 @@ const OSES = ['Linux', 'macOS', 'Windows']; export interface ApiPull { number: number; merged_at: string | null; merge_commit_sha: string | null; base: { ref: string; repo?: { full_name: string } | null }; head: { sha: string; repo: { full_name: string } | null } } export interface ReuseDeps extends RunDeps { pullsForCommit(sha: string): ApiPull[] } +export interface PrReuseDeps extends RunDeps { + runsForBranch(workflow: string, branch: string): ApiRun[]; + commit(sha: string): { tree: string; parents: string[] } | null; + blob(sha: string, file: string): string | null; +} +export interface PrContext { number: number; mergeSha: string; headSha: string; headRef: string; headRepo: string; baseRef: string; runId: number } +/** The files whose bytes decide what a reuse means: a pull request that edits one runs every job. */ +export const reuseGuardedFiles = (workflow: string) => [`.github/workflows/${workflow}`, 'scripts/ci-tree-reuse.ts', 'lib/run-evidence.ts']; export type ReuseVerdict = | { reused: true; run_id: number; run_attempt: number; event: string; pr: number; head_sha: string; tree: string } | { reused: false; reason: string; detail: string }; @@ -71,6 +86,50 @@ export function findReuse(deps: ReuseDeps, o: { repo: string; base: string; push } } +/** + * The run a pull_request run of `o.pr.mergeSha` may reuse, or why none. A source is a green push or dispatch run on + * `o.base`, or a dispatch run on the pull request's own branch, never a pull_request run; its tested tree must be the + * merge commit's, and the merge commit must leave every reuseGuardedFiles path byte-identical to its base parent. + */ +export function findPrReuse(deps: PrReuseDeps, o: { repo: string; base: string; pr: PrContext; workflow: string; profileName: string; profile: RunProfile; now: Date }): ReuseVerdict { + const { pr } = o; + if (!SHA.test(pr.mergeSha) || !SHA.test(pr.headSha)) return refuse('usage', `merge ${JSON.stringify(pr.mergeSha)} or head ${JSON.stringify(pr.headSha)} is not a commit sha`); + if (!Number.isSafeInteger(pr.number) || pr.number < 1 || !pr.headRef) return refuse('usage', 'the pull_request event names no pull request number or head branch'); + if (pr.baseRef !== o.base) return refuse('not_main_pr', `#${pr.number} targets ${pr.baseRef}, not ${o.base}`); + if (o.profile.workflow !== o.workflow) return refuse('profile_mismatch', `profile ${o.profileName} names ${o.profile.workflow}, not ${o.workflow}`); + if (pr.headRepo.toLowerCase() !== o.repo.toLowerCase()) return refuse('fork_head', `#${pr.number}'s head is on ${pr.headRepo || 'a deleted repository'}, not ${o.repo}`); + try { + const merge = deps.commit(pr.mergeSha); + if (!merge) return refuse('api_error', `cannot read merge commit ${pr.mergeSha}`); + if (merge.parents.length !== 2 || merge.parents[1] !== pr.headSha || !SHA.test(merge.parents[0] ?? '')) return refuse('merge_mismatch', `merge commit ${pr.mergeSha.slice(0, 12)} has parents ${merge.parents.map(p => p.slice(0, 12)).join(', ') || 'none'}, not base plus #${pr.number}'s head ${pr.headSha.slice(0, 12)}`); + const baseSha = merge.parents[0]!; + for (const file of reuseGuardedFiles(o.workflow)) { + const mine = deps.blob(pr.mergeSha, file); + const theirs = deps.blob(baseSha, file); + if (!mine || !theirs) return refuse('api_error', `cannot read ${file} at ${(mine ? baseSha : pr.mergeSha).slice(0, 12)}`); + if (mine !== theirs) return refuse(file.startsWith('.github/') ? 'workflow_changed' : 'script_changed', `#${pr.number} changes ${file}; a pull request that edits the reuse rules runs every job`); + } + const sources = [ + ...deps.runsForBranch(o.workflow, o.base).filter(r => (r.event === 'push' || r.event === 'workflow_dispatch') && r.head_branch === o.base), + ...(pr.headRef === o.base ? [] : deps.runsForBranch(o.workflow, pr.headRef).filter(r => r.event === 'workflow_dispatch' && r.head_branch === pr.headRef)), + ].sort((a, b) => startedAt(b).localeCompare(startedAt(a)) || b.id - a.id).slice(0, MAX_CANDIDATES); + if (!sources.length) return refuse('no_candidate', `no push or dispatch run of ${o.workflow} on ${o.base}, and no dispatch run on ${pr.headRef}`); + const candidates = sources.filter(r => SHA.test(r.head_sha) && deps.commitTree(r.head_sha) === merge.tree); + if (!candidates.length) return refuse('tree_mismatch', `none of ${sources.length} push or dispatch runs tested merge commit ${pr.mergeSha.slice(0, 12)}'s tree ${merge.tree.slice(0, 12)}`); + const profile = { ...o.profile, events: ['push', 'workflow_dispatch'] }; + const refusals: string[] = []; + for (const run of candidates) { + const v = verifyRun(deps, { repo: o.repo, runRef: String(run.id), profileName: o.profileName, profile, ignoreRun: pr.runId, now: o.now }); + if (!v.ok) { refusals.push(`run ${run.id} (${run.event}) reason=${v.reason} ${v.detail}`); continue; } + if (v.evidence.tree !== merge.tree) { refusals.push(`run ${run.id} (${run.event}) reason=tree_mismatch tested tree ${v.evidence.tree.slice(0, 12)}`); continue; } + return { reused: true, run_id: v.evidence.run_id, run_attempt: v.evidence.run_attempt, event: v.evidence.event, pr: pr.number, head_sha: v.evidence.head_sha, tree: merge.tree }; + } + return refuse('no_verified_run', refusals.join(' | ')); + } catch (e: any) { + return refuse('api_error', String(e?.message ?? e).slice(0, 300)); + } +} + /** tree.json for the `tested-tree` artifact; throws on any missing or malformed input, so a broken record fails the aggregate instead of uploading. */ export function treeRecord(env: NodeJS.ProcessEnv, o: { plan: { shards: Array<{ files: string[] }> }; results: Array<{ outcome?: { summary?: { testsRan?: number } } }>; os: string[]; tree: string }) { const int = (name: string) => { const v = Number(env[name]); if (!Number.isSafeInteger(v) || v < 1) throw new Error(`${name} is not a positive integer`); return v; }; @@ -125,7 +184,9 @@ function find(args: string[]): number { if (env.GITHUB_OUTPUT) fs.appendFileSync(env.GITHUB_OUTPUT, output); return 0; }; - if (env.GITHUB_EVENT_NAME !== 'push' || env.GITHUB_REF !== 'refs/heads/main') return settle(refuse('not_main_push', `${env.GITHUB_EVENT_NAME ?? 'no'} event on ${env.GITHUB_REF ?? 'no ref'}; only a push to main consumes`)); + const mainPush = env.GITHUB_EVENT_NAME === 'push' && env.GITHUB_REF === 'refs/heads/main'; + const pullRequest = args.includes('--pull-request') && env.GITHUB_EVENT_NAME === 'pull_request'; + if (!mainPush && !pullRequest) return settle(refuse('not_main_push', `${env.GITHUB_EVENT_NAME ?? 'no'} event on ${env.GITHUB_REF ?? 'no ref'}; only a push to main consumes${args.includes('--pull-request') ? ', or a pull request' : ''}`)); if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repo)) return settle(refuse('usage', `GITHUB_REPOSITORY ${JSON.stringify(repo)} is not owner/name`)); let profiles: Record; try { @@ -141,7 +202,22 @@ function find(args: string[]): number { commitTree: sha => (SHA.test(sha) ? ghApiJson<{ tree?: { sha?: string } }>(`repos/${repo}/git/commits/${sha}`).tree?.sha ?? null : null), pullsForCommit: sha => ghApiJson(`repos/${repo}/commits/${sha}/pulls?per_page=${MAX_PULLS}`), }; - return settle(findReuse(deps, { repo, base: 'main', pushSha: env.GITHUB_SHA ?? '', workflow, profileName, profile, now: new Date() })); + if (mainPush) return settle(findReuse(deps, { repo, base: 'main', pushSha: env.GITHUB_SHA ?? '', workflow, profileName, profile, now: new Date() })); + let event: { pull_request?: { number?: number; head?: { sha?: string; ref?: string; repo?: { full_name?: string } | null }; base?: { ref?: string } } }; + try { event = JSON.parse(fs.readFileSync(env.GITHUB_EVENT_PATH ?? '', 'utf8')); } catch (e: any) { return settle(refuse('usage', `GITHUB_EVENT_PATH: ${String(e?.message ?? e).slice(0, 200)}`)); } + const p = event.pull_request ?? {}; + const pr: PrContext = { number: Number(p.number), mergeSha: env.GITHUB_SHA ?? '', headSha: p.head?.sha ?? '', headRef: p.head?.ref ?? '', headRepo: p.head?.repo?.full_name ?? '', baseRef: p.base?.ref ?? '', runId: Number(env.GITHUB_RUN_ID) }; + const prDeps: PrReuseDeps = { + ...deps, + runsForBranch: (wf, branch) => ghApiJson<{ workflow_runs?: ApiRun[] }>(`repos/${repo}/actions/workflows/${wf}/runs?branch=${encodeURIComponent(branch)}&per_page=${MAX_CANDIDATES}`).workflow_runs ?? [], + commit: sha => { + if (!SHA.test(sha)) return null; + const c = ghApiJson<{ tree?: { sha?: string }; parents?: Array<{ sha?: string }> }>(`repos/${repo}/git/commits/${sha}`); + return c.tree?.sha ? { tree: c.tree.sha, parents: (c.parents ?? []).map(x => x.sha ?? '') } : null; + }, + blob: (sha, file) => (SHA.test(sha) ? ghApiJson<{ sha?: string; type?: string }>(`repos/${repo}/contents/${file}?ref=${sha}`) : {}).sha ?? null, + }; + return settle(findPrReuse(prDeps, { repo, base: 'main', pr, workflow, profileName, profile, now: new Date() })); } if (import.meta.main) { diff --git a/test/ci-reuse-workflow-wiring.test.ts b/test/ci-reuse-workflow-wiring.test.ts index 6d9b3b8d5..10923d075 100644 --- a/test/ci-reuse-workflow-wiring.test.ts +++ b/test/ci-reuse-workflow-wiring.test.ts @@ -1,6 +1,8 @@ /** - * Main-push CI reuse wiring (GSTA-156 P3b) for free-tests.yml and windows-free-tests.yml. Pins the properties that - * make reuse safe: only a push to main consumes, the reuse job alone reads Actions with a job-level actions: read, + * CI reuse wiring (GSTA-156 P3b) for free-tests.yml and windows-free-tests.yml. Pins the properties that make reuse + * safe: nothing a pull request controls certifies it (a push to main consumes; free-tests.yml's pull_request runs consume + * only through find --pull-request, whose sources and guarded files test/ci-tree-reuse.test.ts pins; dispatch runs never + * consume), the reuse job alone reads Actions with a job-level actions: read, * a skipped or failed reuse job runs every job exactly as before (GitHub skips a job whose needed job was skipped * unless its condition uses a status function, so every dependent condition starts with !cancelled()), the * aggregate's reuse branch runs before any download and the strict path is otherwise unchanged, tested-tree is @@ -15,8 +17,11 @@ import { parseWorkflow } from '../lib/pregate/workflows'; const ROOT = path.resolve(import.meta.dir, '..'); const read = (rel: string) => fs.readFileSync(path.join(ROOT, rel), 'utf8'); -const FILES = { 'free-tests.yml': { aggregate: 'free-tests', profile: 'ci-reuse/free-tests', plan: 'free-plan', exempt: [] as string[], manual: false }, 'windows-free-tests.yml': { aggregate: 'windows-free-tests', profile: 'ci-reuse/windows-free-tests', plan: 'windows-plan', exempt: ['windows-record-durations'], manual: true } }; -const REUSE_IF = "github.event_name == 'push' && github.ref == 'refs/heads/main'"; +const MAIN_PUSH = "github.event_name == 'push' && github.ref == 'refs/heads/main'"; +const FILES = { + 'free-tests.yml': { aggregate: 'free-tests', profile: 'ci-reuse/free-tests', plan: 'free-plan', exempt: [] as string[], manual: false, reuseIf: `(${MAIN_PUSH}) || github.event_name == 'pull_request'`, pullRequest: true }, + 'windows-free-tests.yml': { aggregate: 'windows-free-tests', profile: 'ci-reuse/windows-free-tests', plan: 'windows-plan', exempt: ['windows-record-durations'], manual: true, reuseIf: MAIN_PUSH, pullRequest: false }, +}; const SKIP = "steps.route.outputs.reused != 'true'"; const PROFILES = JSON.parse(read('.gstack/pregate.json')).run_profiles as Record; @@ -38,8 +43,8 @@ for (const [file, spec] of Object.entries(FILES)) { const needs = (name: string): string[] => [jobs[name].needs ?? []].flat(); const suite = Object.keys(jobs).filter(n => n !== 'reuse' && n !== spec.aggregate && !spec.exempt.includes(n)); - test('only a push to main consumes, with job-level actions: read on the reuse job alone and no secrets', () => { - expect(jobs.reuse.if).toBe(REUSE_IF); + test(spec.pullRequest ? 'a push to main consumes, and a pull request only through find --pull-request (never a run it controls), with job-level actions: read on the reuse job alone and no secrets' : 'only a push to main consumes, with job-level actions: read on the reuse job alone and no secrets', () => { + expect(jobs.reuse.if).toBe(spec.reuseIf); expect(jobs.reuse.permissions).toEqual({ contents: 'read', actions: 'read' }); expect(wf.permissions).toEqual({ contents: 'read' }); for (const name of Object.keys(jobs).filter(n => n !== 'reuse')) expect([name, jobs[name].permissions]).toEqual([name, undefined]); @@ -47,7 +52,7 @@ for (const [file, spec] of Object.entries(FILES)) { expect(source).not.toContain('pull_request_target'); const find = jobs.reuse.steps.find((s: any) => s.id === 'find'); expect(find.env).toEqual({ GH_TOKEN: '${{ github.token }}' }); - expect(find.run).toBe(`bun run scripts/ci-tree-reuse.ts find --workflow ${file} --profile ${spec.profile}`); + expect(find.run).toBe(`bun run scripts/ci-tree-reuse.ts find --workflow ${file} --profile ${spec.profile}${spec.pullRequest ? ' --pull-request' : ''}`); expect(jobs.reuse.steps[0].with['persist-credentials']).toBe(false); expect(Object.keys(jobs.reuse.outputs).sort()).toEqual(['reused', 'run_id', 'run_url', 'tree']); }); @@ -99,7 +104,7 @@ for (const [file, spec] of Object.entries(FILES)) { expect(steps[1].run).toMatch(/set -eu\n(test "\$[A-Z_]+_RESULT" = success\n)+$/); }); - test.skipIf(!Bun.which('bash'))('the route step takes the reuse branch only for a well-formed reuse on a main push, and fails rather than passes on anything else claiming reuse', () => { + test.skipIf(!Bun.which('bash'))(`the route step takes the reuse branch only for a well-formed reuse on a main push${spec.pullRequest ? ' or a pull request' : ''}, and fails rather than passes on anything else claiming reuse`, () => { const route = jobs[spec.aggregate].steps[0]; const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'reuse route ')); const shell = (p: string) => p.split(path.sep).join('/'); @@ -117,7 +122,9 @@ for (const [file, spec] of Object.entries(FILES)) { const reused = go(good); expect([reused.status, reused.out]).toEqual([0, 'reused=true\n']); expect(reused.summary).toContain('Reused https://github.com/garrytan/gstack/actions/runs/123'); - for (const bad of []>[{ REUSE_RUN_ID: '' }, { REUSE_RUN_ID: '12x' }, { REUSE_RUN_URL: 'https://github.com/other/repo/actions/runs/123' }, { REUSE_TREE: '' }, { GITHUB_EVENT_NAME: 'pull_request' }, { GITHUB_REF: 'refs/heads/feature' }]) { + const prRun = { GITHUB_EVENT_NAME: 'pull_request', GITHUB_REF: 'refs/pull/7/merge' }; + if (spec.pullRequest) expect(go({ ...good, ...prRun }).out).toBe('reused=true\n'); + for (const bad of []>[{ REUSE_RUN_ID: '' }, { REUSE_RUN_ID: '12x' }, { REUSE_RUN_URL: 'https://github.com/other/repo/actions/runs/123' }, { REUSE_TREE: '' }, { GITHUB_EVENT_NAME: 'pull_request' }, { GITHUB_REF: 'refs/heads/feature' }, { GITHUB_EVENT_NAME: 'workflow_dispatch' }, ...(spec.pullRequest ? [] : [prRun])]) { const r = go({ ...good, ...bad }); expect([JSON.stringify(bad), r.status === 0, r.out]).toEqual([JSON.stringify(bad), false, '']); } diff --git a/test/ci-tree-reuse.test.ts b/test/ci-tree-reuse.test.ts index 1fc8cc913..4987e0534 100644 --- a/test/ci-tree-reuse.test.ts +++ b/test/ci-tree-reuse.test.ts @@ -9,7 +9,7 @@ import * as os from 'node:os'; import * as path from 'node:path'; import { bunIsolationArgs } from '../lib/bun-isolation'; import type { ApiJob, ApiRun, RunProfile, TreeRecord } from '../lib/run-evidence'; -import { MAX_CANDIDATES, findReuse, reuseOutputs, treeRecord, type ApiPull, type ReuseDeps } from '../scripts/ci-tree-reuse'; +import { MAX_CANDIDATES, findPrReuse, findReuse, reuseOutputs, treeRecord, type ApiPull, type PrReuseDeps, type ReuseDeps } from '../scripts/ci-tree-reuse'; const REPO = 'garrytan/gstack'; const PUSH = '1'.repeat(40); @@ -120,6 +120,78 @@ describe('findReuse: candidate discovery for a main push', () => { }); }); +describe('findPrReuse: a pull request reuses only a run it cannot control', () => { + const BASE = '7'.repeat(40); + const MAIN_HEAD = '8'.repeat(40); + const SELF = 999; + const GUARDED = ['.github/workflows/free-tests.yml', 'scripts/ci-tree-reuse.ts', 'lib/run-evidence.ts']; + interface PrFake { id: number; event: string; branch: string; head: string; conclusion?: string | null; status?: string; started?: string; record?: Partial } + interface PrWorld { runs: PrFake[]; trees: Record; blobs: Record; parents: string[]; headRepo: string } + const blobs = (over: Record = {}) => ({ ...Object.fromEntries(GUARDED.flatMap(p => [[`${MERGE}:${p}`, `blob-${p}`], [`${BASE}:${p}`, `blob-${p}`]])), ...over }); + const prWorld = (over: Partial = {}): PrWorld => ({ + runs: [{ id: 500, event: 'push', branch: 'main', head: MAIN_HEAD }, { id: SELF, event: 'pull_request', branch: 'feat', head: HEAD, status: 'in_progress', conclusion: null, started: '2026-10-30T11:30:00Z' }], + trees: { [MERGE]: TREE, [MAIN_HEAD]: TREE, [HEAD]: TREE, [BASE]: '9'.repeat(40) }, blobs: blobs(), parents: [BASE, HEAD], headRepo: REPO, ...over, + }); + function prDeps(w: PrWorld): PrReuseDeps { + const api = (f: PrFake): ApiRun => ({ id: f.id, run_attempt: 1, status: f.status ?? 'completed', conclusion: f.conclusion === undefined ? 'success' : f.conclusion, event: f.event, head_sha: f.head, head_branch: f.branch, + path: '.github/workflows/free-tests.yml', created_at: f.started ?? '2026-10-30T10:00:00Z', run_started_at: f.started ?? '2026-10-30T10:00:00Z', updated_at: '2026-10-30T10:30:00Z', repository: { full_name: REPO }, head_repository: { full_name: REPO } }); + const byId = (id: number) => { const f = w.runs.find(r => r.id === id); if (!f) throw new Error('HTTP 404'); return f; }; + return { + run: id => api(byId(id)), + runsForHead: (wf, sha) => (wf === 'free-tests.yml' ? w.runs.filter(r => r.head === sha).map(api) : []), + runsForBranch: (wf, branch) => (wf === 'free-tests.yml' ? w.runs.filter(r => r.branch === branch).map(api) : []), + jobs: () => [{ name: 'free-plan', conclusion: 'success' }, { name: 'free-tests', conclusion: 'success' }], + artifact: id => { const f = byId(id); return { run_id: id, run_attempt: 1, event: f.event, sha: f.head, head_sha: f.head, tree: w.trees[f.head], os: ['Linux', 'macOS', 'Windows'], files: ['test/a.test.ts'], tests: 9, ...f.record }; }, + commitTree: sha => w.trees[sha] ?? null, + commit: sha => (sha === MERGE ? { tree: TREE, parents: w.parents } : null), + blob: (sha, p) => w.blobs[`${sha}:${p}`] ?? null, + }; + } + const pr = (w: PrWorld) => findPrReuse(prDeps(w), { repo: REPO, base: 'main', workflow: 'free-tests.yml', profileName: 'ci-reuse/free-tests', profile: PROFILE, now: NOW, + pr: { number: 42, mergeSha: MERGE, headSha: HEAD, headRef: 'feat', headRepo: w.headRepo, baseRef: 'main', runId: SELF } }); + const prWhy = (w: PrWorld) => { const v = pr(w); return v.reused ? 'reused' : v.reason; }; + + test('a green push run on main at the merge commit\'s exact tree is reused', () => { + expect(pr(prWorld())).toEqual({ reused: true, run_id: 500, run_attempt: 1, event: 'push', pr: 42, head_sha: MAIN_HEAD, tree: TREE }); + }); + + test('a green dispatch run on the same branch at that tree is reused, and the pull request\'s own run in progress does not veto it', () => { + const v = pr(prWorld({ runs: [{ id: 501, event: 'workflow_dispatch', branch: 'feat', head: HEAD }, { id: SELF, event: 'pull_request', branch: 'feat', head: HEAD, status: 'in_progress', conclusion: null, started: '2026-10-30T11:30:00Z' }] })); + expect(v).toEqual({ reused: true, run_id: 501, run_attempt: 1, event: 'workflow_dispatch', pr: 42, head_sha: HEAD, tree: TREE }); + }); + + test('refused when the pull request changes the workflow file', () => { + expect(prWhy(prWorld({ blobs: blobs({ [`${MERGE}:.github/workflows/free-tests.yml`]: 'edited' }) }))).toBe('workflow_changed'); + }); + + test('refused when the pull request changes the reuse script or the run verifier it imports', () => { + expect(prWhy(prWorld({ blobs: blobs({ [`${MERGE}:scripts/ci-tree-reuse.ts`]: 'edited' }) }))).toBe('script_changed'); + expect(prWhy(prWorld({ blobs: blobs({ [`${MERGE}:lib/run-evidence.ts`]: 'edited' }) }))).toBe('script_changed'); + expect(prWhy(prWorld({ blobs: blobs({ [`${BASE}:scripts/ci-tree-reuse.ts`]: '' }) }))).toBe('api_error'); + }); + + test('refused on a tree mismatch: the source run tested another tree than this merge commit', () => { + expect(prWhy(prWorld({ trees: { [MERGE]: TREE, [MAIN_HEAD]: '5'.repeat(40), [HEAD]: TREE } }))).toBe('tree_mismatch'); + expect(pr(prWorld({ runs: [{ id: 500, event: 'push', branch: 'main', head: MAIN_HEAD, record: { tree: '5'.repeat(40) } }] }))).toMatchObject({ reused: false, reason: 'no_verified_run', detail: expect.stringContaining('reason=tree_mismatch') }); + }); + + test('refused when the only green source is a pull_request run, which the pull request itself controls', () => { + expect(prWhy(prWorld({ runs: [{ id: 502, event: 'pull_request', branch: 'feat', head: HEAD }] }))).toBe('no_candidate'); + expect(prWhy(prWorld({ runs: [{ id: 503, event: 'pull_request', branch: 'main', head: MAIN_HEAD }] }))).toBe('no_candidate'); + }); + + test('a push to another branch, a dispatch on another branch and a fork head are never sources', () => { + expect(prWhy(prWorld({ runs: [{ id: 504, event: 'push', branch: 'feat', head: HEAD }] }))).toBe('no_candidate'); + expect(prWhy(prWorld({ runs: [{ id: 505, event: 'workflow_dispatch', branch: 'other', head: HEAD }] }))).toBe('no_candidate'); + expect(prWhy(prWorld({ headRepo: 'someone/gstack' }))).toBe('fork_head'); + }); + + test('a merge commit that is not base plus this head, or an unreadable one, is refused', () => { + expect(prWhy(prWorld({ parents: [BASE, '6'.repeat(40)] }))).toBe('merge_mismatch'); + expect(prWhy(prWorld({ parents: [BASE] }))).toBe('merge_mismatch'); + }); +}); + describe('treeRecord: the tested-tree producer', () => { const plan = { shards: [{ files: ['test/b.test.ts', 'test/a.test.ts'] }, { files: ['test/c.test.ts'] }] }; const results = [{ outcome: { summary: { testsRan: 7 } } }, { outcome: { summary: { testsRan: 5 } } }]; From 284bd9123df0c8fa70534ad55a82c111b883c1db Mon Sep 17 00:00:00 2001 From: garrytan Date: Tue, 3 Nov 2026 07:50:52 +0000 Subject: [PATCH 2/4] fix: pre-landing review fixes (PR reuse runs main's checkout; sizing experiments are never evidence; a re-run never reuses) Review-Finding: p1/specialist-batched#1 reader=security Review-Finding: p1/red-team#1 reader=red-team Review-Finding: p1/red-team#2 reader=red-team Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- .github/workflows/free-tests.yml | 7 ++++++- docs/pregate.md | 15 +++++++++++---- lib/run-evidence.ts | 5 +++-- scripts/ci-tree-reuse.ts | 11 +++++++---- test/ci-reuse-workflow-wiring.test.ts | 9 +++++---- test/ci-tree-reuse.test.ts | 15 +++++++++++++-- 6 files changed, 45 insertions(+), 17 deletions(-) diff --git a/.github/workflows/free-tests.yml b/.github/workflows/free-tests.yml index 0851743df..e357d2f74 100644 --- a/.github/workflows/free-tests.yml +++ b/.github/workflows/free-tests.yml @@ -101,8 +101,12 @@ jobs: run_url: ${{ steps.find.outputs.run_url }} tree: ${{ steps.find.outputs.tree }} steps: + # On a pull request this checks out main (the base commit), never the + # merge commit, so the reuse script, its imports and the run profile are + # main's code; bun ignores any .env or bunfig.toml in the checkout. - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: + ref: ${{ github.event.pull_request.base.sha || github.sha }} persist-credentials: false - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: @@ -111,7 +115,7 @@ jobs: name: Find a green run of this exact tree that this run's pull request cannot control env: GH_TOKEN: ${{ github.token }} - run: bun run scripts/ci-tree-reuse.ts find --workflow free-tests.yml --profile ci-reuse/free-tests --pull-request + run: bun --no-env-file --config=/dev/null run scripts/ci-tree-reuse.ts find --workflow free-tests.yml --profile ci-reuse/free-tests --pull-request free-plan: needs: reuse @@ -763,6 +767,7 @@ jobs: if: steps.route.outputs.reused != 'true' env: PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + SIZING_OVERRIDE: ${{ inputs.free_runner }}${{ inputs.free_shards }}${{ inputs.free_jobs }} run: bun run scripts/ci-tree-reuse.ts record --plan "$RUNNER_TEMP/free-plan.json" --results "$RUNNER_TEMP/free-results" --os Linux,macOS,Windows --out "$RUNNER_TEMP/tested-tree/tree.json" - name: Upload the tested tree if: steps.route.outputs.reused != 'true' diff --git a/docs/pregate.md b/docs/pregate.md index 0984609f5..8f6e6a7da 100644 --- a/docs/pregate.md +++ b/docs/pregate.md @@ -1063,13 +1063,17 @@ runs nightly and by hand only (docs/support-tiers.md), never on a push, so its #### Pull requests reuse only a run they can't control Free Tests' `reuse` job also runs on `pull_request`, as -`find … --pull-request` (`findPrReuse`). Main pushes stay the only consumers +`find … --pull-request` (`findPrReuse`). On a pull request it checks out the +base commit, never the merge commit, and runs bun with `--no-env-file +--config=/dev/null`, so the reuse script, everything it imports and the run +profile are main's code. Main pushes stay the only consumers of anything a pull request could certify for itself, so a `pull_request` run is never a source. A pull request run reuses a green run only when all of these hold: -1. the pull request's head is on this repository (`fork_head`) and it targets - main (`not_main_pr`); +1. the pull request's head is on this repository (`fork_head`), it targets + main (`not_main_pr`), and this is the run's first attempt (`rerun`: a re-run + never reuses, so a red earlier attempt stays visible); 2. GitHub's merge commit has exactly two parents, main and the pull request's head (`merge_mismatch`); 3. the merge commit leaves `.github/workflows/free-tests.yml` @@ -1083,7 +1087,10 @@ hold: 5. `verifyRun` accepts it under `ci-reuse/free-tests` with `push` and `workflow_dispatch` as the accepted events, its `tested-tree` record names the merge commit's tree, and no newer red run on its head vetoes it (the - pull request's own run in progress is not a veto). + pull request's own run in progress is not a veto). A dispatch that + overrode the runner, shards or sub-shards records `sizing_experiment` in + its `tested-tree`, and `verifyRun` never accepts it (`sizing_experiment`), + for any consumer. The aggregate's route step accepts a reuse on a `pull_request` run of `refs/pull//merge` as it does on a push to main. A reused pull request run diff --git a/lib/run-evidence.ts b/lib/run-evidence.ts index 19801123c..7266eeefa 100644 --- a/lib/run-evidence.ts +++ b/lib/run-evidence.ts @@ -50,7 +50,7 @@ export interface ApiRun { repository?: { full_name: string } | null; head_repository?: { full_name: string } | null; } export interface ApiJob { name: string; conclusion: string | null; completed_at?: string | null } -export interface TreeRecord { run_id?: unknown; run_attempt?: unknown; event?: unknown; sha?: unknown; head_sha?: unknown; tree?: unknown; reused?: unknown; reuse?: unknown; os?: unknown; files?: unknown; tests?: unknown } +export interface TreeRecord { run_id?: unknown; run_attempt?: unknown; event?: unknown; sha?: unknown; head_sha?: unknown; tree?: unknown; reused?: unknown; reuse?: unknown; sizing_experiment?: unknown; os?: unknown; files?: unknown; tests?: unknown } /** Everything verifyRun reads, injectable so every refusal has a fixture. */ export interface RunDeps { @@ -63,7 +63,7 @@ export interface RunDeps { export type RunRefusal = 'run_ref_invalid' | 'other_repository' | 'fork_run' | 'workflow_mismatch' | 'event_mismatch' | 'run_in_progress' | 'run_not_success' | 'job_missing' | 'job_ambiguous' | 'job_not_success' | 'artifact_missing' | 'artifact_mismatch' - | 'artifact_reused' | 'tree_mismatch' | 'newer_run_vetoes' | 'run_changed' | 'run_stale' | 'os_mismatch' | 'api_error'; + | 'artifact_reused' | 'sizing_experiment' | 'tree_mismatch' | 'newer_run_vetoes' | 'run_changed' | 'run_stale' | 'os_mismatch' | 'api_error'; export interface RunEvidence { run_id: number; run_attempt: number; head_sha: string; tree: string; profile: string; event: string; finished_at: string; files: string[]; tests: number | null } export type RunVerdict = { ok: true; evidence: RunEvidence } | { ok: false; reason: RunRefusal; detail: string }; @@ -169,6 +169,7 @@ export function verifyRun(deps: RunDeps, o: { repo: string; runRef: string; prof catch (e: any) { return fail('artifact_missing', `${at} has no readable ${p.artifact} tree.json: ${String(e?.message ?? e).slice(0, 200)}`); } if (!rec || typeof rec !== 'object') return fail('artifact_mismatch', `${at}'s ${p.artifact} tree.json is not a JSON object`); if (rec.reused === true || rec.reuse === true) return fail('artifact_reused', `${at} reused another run; a reuse is never evidence`); + if (rec.sizing_experiment !== undefined) return fail('sizing_experiment', `${at} was a runner-sizing experiment (a dispatch that overrode the runner, shards or sub-shards); it is never a Free Tests result`); const pr = run.event === 'pull_request'; const recHeads = pr ? [['head_sha', rec.head_sha, run.head_sha] as const] : rec.head_sha === undefined && rec.sha === undefined ? [['head', undefined, run.head_sha] as const] : ([['sha', rec.sha, run.head_sha], ['head_sha', rec.head_sha, run.head_sha]] as const).filter(([, got]) => got !== undefined); const wrong = ([['run_id', rec.run_id, id], ['run_attempt', rec.run_attempt, run.run_attempt], ['event', rec.event, run.event], ...recHeads] as const).filter(([, got, want]) => got !== want); diff --git a/scripts/ci-tree-reuse.ts b/scripts/ci-tree-reuse.ts index d0558edbb..49256933f 100755 --- a/scripts/ci-tree-reuse.ts +++ b/scripts/ci-tree-reuse.ts @@ -6,7 +6,9 @@ * The producer. A Free Tests or Windows Free Tests aggregate runs it only after its strict verification * passed, and uploads the file as the `tested-tree` artifact: run_id, run_attempt, event, sha (GITHUB_SHA), * head_sha (the pull request head for pull_request, else GITHUB_SHA), tree (git's tree of GITHUB_SHA), - * workflow, os (the OSes the run's test jobs ran on), files (every test file in the plan) and tests. + * workflow, os (the OSes the run's test jobs ran on), files (every test file in the plan) and tests, plus + * sizing_experiment: true when a dispatch overrode the runner, shards or sub-shards (SIZING_OVERRIDE), which + * verifyRun never accepts as evidence. * * bun run scripts/ci-tree-reuse.ts find --workflow --profile * The main-push consumer (the `reuse` job). A push to main whose tree a pull request already tested green @@ -43,7 +45,7 @@ export interface PrReuseDeps extends RunDeps { commit(sha: string): { tree: string; parents: string[] } | null; blob(sha: string, file: string): string | null; } -export interface PrContext { number: number; mergeSha: string; headSha: string; headRef: string; headRepo: string; baseRef: string; runId: number } +export interface PrContext { number: number; mergeSha: string; headSha: string; headRef: string; headRepo: string; baseRef: string; runId: number; runAttempt: number } /** The files whose bytes decide what a reuse means: a pull request that edits one runs every job. */ export const reuseGuardedFiles = (workflow: string) => [`.github/workflows/${workflow}`, 'scripts/ci-tree-reuse.ts', 'lib/run-evidence.ts']; export type ReuseVerdict = @@ -95,6 +97,7 @@ export function findPrReuse(deps: PrReuseDeps, o: { repo: string; base: string; const { pr } = o; if (!SHA.test(pr.mergeSha) || !SHA.test(pr.headSha)) return refuse('usage', `merge ${JSON.stringify(pr.mergeSha)} or head ${JSON.stringify(pr.headSha)} is not a commit sha`); if (!Number.isSafeInteger(pr.number) || pr.number < 1 || !pr.headRef) return refuse('usage', 'the pull_request event names no pull request number or head branch'); + if (pr.runAttempt !== 1) return refuse('rerun', `attempt ${pr.runAttempt} of run ${pr.runId}: a re-run never reuses, so an earlier red attempt can't be hidden`); if (pr.baseRef !== o.base) return refuse('not_main_pr', `#${pr.number} targets ${pr.baseRef}, not ${o.base}`); if (o.profile.workflow !== o.workflow) return refuse('profile_mismatch', `profile ${o.profileName} names ${o.profile.workflow}, not ${o.workflow}`); if (pr.headRepo.toLowerCase() !== o.repo.toLowerCase()) return refuse('fork_head', `#${pr.number}'s head is on ${pr.headRepo || 'a deleted repository'}, not ${o.repo}`); @@ -146,7 +149,7 @@ export function treeRecord(env: NodeJS.ProcessEnv, o: { plan: { shards: Array<{ if (!files.length) throw new Error('the plan lists no test files'); const counts = o.results.map(r => r.outcome?.summary?.testsRan); const tests = o.results.length && counts.every(n => Number.isSafeInteger(n) && n! >= 0) ? counts.reduce((a, b) => a! + b!, 0)! : files.length; - return { run_id: int('GITHUB_RUN_ID'), run_attempt: int('GITHUB_RUN_ATTEMPT'), event, sha, head_sha: head, tree: o.tree, workflow, os: o.os, files, tests }; + return { run_id: int('GITHUB_RUN_ID'), run_attempt: int('GITHUB_RUN_ATTEMPT'), event, sha, head_sha: head, tree: o.tree, workflow, os: o.os, files, tests, ...(env.SIZING_OVERRIDE ? { sizing_experiment: true } : {}) }; } function flag(args: string[], name: string): string | undefined { const i = args.indexOf(name); return i >= 0 ? args[i + 1] : undefined; } @@ -206,7 +209,7 @@ function find(args: string[]): number { let event: { pull_request?: { number?: number; head?: { sha?: string; ref?: string; repo?: { full_name?: string } | null }; base?: { ref?: string } } }; try { event = JSON.parse(fs.readFileSync(env.GITHUB_EVENT_PATH ?? '', 'utf8')); } catch (e: any) { return settle(refuse('usage', `GITHUB_EVENT_PATH: ${String(e?.message ?? e).slice(0, 200)}`)); } const p = event.pull_request ?? {}; - const pr: PrContext = { number: Number(p.number), mergeSha: env.GITHUB_SHA ?? '', headSha: p.head?.sha ?? '', headRef: p.head?.ref ?? '', headRepo: p.head?.repo?.full_name ?? '', baseRef: p.base?.ref ?? '', runId: Number(env.GITHUB_RUN_ID) }; + const pr: PrContext = { number: Number(p.number), mergeSha: env.GITHUB_SHA ?? '', headSha: p.head?.sha ?? '', headRef: p.head?.ref ?? '', headRepo: p.head?.repo?.full_name ?? '', baseRef: p.base?.ref ?? '', runId: Number(env.GITHUB_RUN_ID), runAttempt: Number(env.GITHUB_RUN_ATTEMPT) }; const prDeps: PrReuseDeps = { ...deps, runsForBranch: (wf, branch) => ghApiJson<{ workflow_runs?: ApiRun[] }>(`repos/${repo}/actions/workflows/${wf}/runs?branch=${encodeURIComponent(branch)}&per_page=${MAX_CANDIDATES}`).workflow_runs ?? [], diff --git a/test/ci-reuse-workflow-wiring.test.ts b/test/ci-reuse-workflow-wiring.test.ts index 10923d075..6cf72f2a1 100644 --- a/test/ci-reuse-workflow-wiring.test.ts +++ b/test/ci-reuse-workflow-wiring.test.ts @@ -1,8 +1,8 @@ /** * CI reuse wiring (GSTA-156 P3b) for free-tests.yml and windows-free-tests.yml. Pins the properties that make reuse * safe: nothing a pull request controls certifies it (a push to main consumes; free-tests.yml's pull_request runs consume - * only through find --pull-request, whose sources and guarded files test/ci-tree-reuse.test.ts pins; dispatch runs never - * consume), the reuse job alone reads Actions with a job-level actions: read, + * only through find --pull-request run from main's checkout, whose sources and guarded files test/ci-tree-reuse.test.ts + * pins; a sizing-experiment dispatch records sizing_experiment and is never evidence; dispatch runs never consume), the reuse job alone reads Actions with a job-level actions: read, * a skipped or failed reuse job runs every job exactly as before (GitHub skips a job whose needed job was skipped * unless its condition uses a status function, so every dependent condition starts with !cancelled()), the * aggregate's reuse branch runs before any download and the strict path is otherwise unchanged, tested-tree is @@ -52,8 +52,9 @@ for (const [file, spec] of Object.entries(FILES)) { expect(source).not.toContain('pull_request_target'); const find = jobs.reuse.steps.find((s: any) => s.id === 'find'); expect(find.env).toEqual({ GH_TOKEN: '${{ github.token }}' }); - expect(find.run).toBe(`bun run scripts/ci-tree-reuse.ts find --workflow ${file} --profile ${spec.profile}${spec.pullRequest ? ' --pull-request' : ''}`); + expect(find.run).toBe(spec.pullRequest ? `bun --no-env-file --config=/dev/null run scripts/ci-tree-reuse.ts find --workflow ${file} --profile ${spec.profile} --pull-request` : `bun run scripts/ci-tree-reuse.ts find --workflow ${file} --profile ${spec.profile}`); expect(jobs.reuse.steps[0].with['persist-credentials']).toBe(false); + if (spec.pullRequest) expect(jobs.reuse.steps[0].with.ref).toBe('${{ github.event.pull_request.base.sha || github.sha }}'); expect(Object.keys(jobs.reuse.outputs).sort()).toEqual(['reused', 'run_id', 'run_url', 'tree']); }); @@ -100,7 +101,7 @@ for (const [file, spec] of Object.entries(FILES)) { expect(steps.length).toBe(upload + 1); expect(steps[upload].uses).toStartWith('actions/upload-artifact@'); expect(steps[upload].with['if-no-files-found']).toBe('error'); - expect(steps[record].env).toEqual({ PR_HEAD_SHA: '${{ github.event.pull_request.head.sha }}' }); + expect(steps[record].env).toEqual({ PR_HEAD_SHA: '${{ github.event.pull_request.head.sha }}', ...(spec.pullRequest ? { SIZING_OVERRIDE: '${{ inputs.free_runner }}${{ inputs.free_shards }}${{ inputs.free_jobs }}' } : {}) }); expect(steps[1].run).toMatch(/set -eu\n(test "\$[A-Z_]+_RESULT" = success\n)+$/); }); diff --git a/test/ci-tree-reuse.test.ts b/test/ci-tree-reuse.test.ts index 4987e0534..0c697dd3a 100644 --- a/test/ci-tree-reuse.test.ts +++ b/test/ci-tree-reuse.test.ts @@ -126,7 +126,7 @@ describe('findPrReuse: a pull request reuses only a run it cannot control', () = const SELF = 999; const GUARDED = ['.github/workflows/free-tests.yml', 'scripts/ci-tree-reuse.ts', 'lib/run-evidence.ts']; interface PrFake { id: number; event: string; branch: string; head: string; conclusion?: string | null; status?: string; started?: string; record?: Partial } - interface PrWorld { runs: PrFake[]; trees: Record; blobs: Record; parents: string[]; headRepo: string } + interface PrWorld { runs: PrFake[]; trees: Record; blobs: Record; parents: string[]; headRepo: string; attempt?: number } const blobs = (over: Record = {}) => ({ ...Object.fromEntries(GUARDED.flatMap(p => [[`${MERGE}:${p}`, `blob-${p}`], [`${BASE}:${p}`, `blob-${p}`]])), ...over }); const prWorld = (over: Partial = {}): PrWorld => ({ runs: [{ id: 500, event: 'push', branch: 'main', head: MAIN_HEAD }, { id: SELF, event: 'pull_request', branch: 'feat', head: HEAD, status: 'in_progress', conclusion: null, started: '2026-10-30T11:30:00Z' }], @@ -148,7 +148,7 @@ describe('findPrReuse: a pull request reuses only a run it cannot control', () = }; } const pr = (w: PrWorld) => findPrReuse(prDeps(w), { repo: REPO, base: 'main', workflow: 'free-tests.yml', profileName: 'ci-reuse/free-tests', profile: PROFILE, now: NOW, - pr: { number: 42, mergeSha: MERGE, headSha: HEAD, headRef: 'feat', headRepo: w.headRepo, baseRef: 'main', runId: SELF } }); + pr: { number: 42, mergeSha: MERGE, headSha: HEAD, headRef: 'feat', headRepo: w.headRepo, baseRef: 'main', runId: SELF, runAttempt: w.attempt ?? 1 } }); const prWhy = (w: PrWorld) => { const v = pr(w); return v.reused ? 'reused' : v.reason; }; test('a green push run on main at the merge commit\'s exact tree is reused', () => { @@ -186,6 +186,15 @@ describe('findPrReuse: a pull request reuses only a run it cannot control', () = expect(prWhy(prWorld({ headRepo: 'someone/gstack' }))).toBe('fork_head'); }); + test('a runner-sizing experiment dispatch is never a source (red team p1 #1)', () => { + const v = pr(prWorld({ runs: [{ id: 506, event: 'workflow_dispatch', branch: 'feat', head: HEAD, record: { sizing_experiment: true } }] })); + expect(v).toMatchObject({ reused: false, reason: 'no_verified_run', detail: expect.stringContaining('reason=sizing_experiment') }); + }); + + test('a re-run attempt of the pull request run never reuses, so its own red earlier attempt stays visible (red team p1 #2)', () => { + expect(prWhy(prWorld({ attempt: 2, runs: [{ id: 501, event: 'workflow_dispatch', branch: 'feat', head: HEAD }] }))).toBe('rerun'); + }); + test('a merge commit that is not base plus this head, or an unreadable one, is refused', () => { expect(prWhy(prWorld({ parents: [BASE, '6'.repeat(40)] }))).toBe('merge_mismatch'); expect(prWhy(prWorld({ parents: [BASE] }))).toBe('merge_mismatch'); @@ -200,6 +209,8 @@ describe('treeRecord: the tested-tree producer', () => { expect(treeRecord(env(), { plan, results, os: ['Linux', 'macOS', 'Windows'], tree: TREE })).toEqual({ run_id: 300, run_attempt: 2, event: 'workflow_dispatch', sha: HEAD, head_sha: HEAD, tree: TREE, workflow: '.github/workflows/free-tests.yml', os: ['Linux', 'macOS', 'Windows'], files: ['test/a.test.ts', 'test/b.test.ts', 'test/c.test.ts'], tests: 12 }); expect(treeRecord(env({ GITHUB_EVENT_NAME: 'pull_request', GITHUB_SHA: MERGE, PR_HEAD_SHA: HEAD }), { plan, results, os: ['Windows'], tree: TREE })).toMatchObject({ event: 'pull_request', sha: MERGE, head_sha: HEAD }); expect(treeRecord(env(), { plan, results: [{ outcome: {} }], os: ['Linux'], tree: TREE }).tests).toBe(3); + expect(treeRecord(env({ SIZING_OVERRIDE: '1' }), { plan, results, os: ['Linux'], tree: TREE })).toMatchObject({ sizing_experiment: true }); + expect(treeRecord(env({ SIZING_OVERRIDE: '' }), { plan, results, os: ['Linux'], tree: TREE })).not.toHaveProperty('sizing_experiment'); }); test('missing or malformed inputs throw, so the aggregate fails instead of uploading a record', () => { expect(() => treeRecord(env({ GITHUB_EVENT_NAME: 'pull_request', GITHUB_SHA: MERGE }), { plan, results, os: ['Linux'], tree: TREE })).toThrow(/PR_HEAD_SHA/); From 81d54d39275b317c3be79d2041d4c12c0032c45c Mon Sep 17 00:00:00 2001 From: garrytan Date: Tue, 3 Nov 2026 08:04:12 +0000 Subject: [PATCH 3/4] docs: pregate CI reuse sections name pull request reuse (docs audit) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- docs/pregate.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/pregate.md b/docs/pregate.md index 8f6e6a7da..46a889ec5 100644 --- a/docs/pregate.md +++ b/docs/pregate.md @@ -88,7 +88,7 @@ the byte-pinned `/ship` renders in `test/fixtures/golden/`). | `stall_timeout_s` | seconds a test command may print nothing while using under a tenth of one CPU before its process group is killed and the check is `incomplete` (`PREGATE_STALLED`); default 300, `0` disables; `--stall-timeout` overrides; the `--remote` gate is exempt (see [supervision](#supervision)) | | `timeout_s` | seconds each runner process may take (the unit-file group, each serial file, each guard without its own `timeout_s`, and the remote gate's deadline) when `--timeout` is not passed; 1 to 3600, default 600. `gstack-ship-preflight` and the `finish` fallback run pregate without `--timeout`, so this is how a repo with long lanes sets theirs | | `remote_gate` | the repo's remote test gate command; under 8 cores the local test tier becomes a `requires-remote` obligation naming it, and `--remote` runs it (see [remote gate](#remote-gate)) | -| `run_profiles` | name → `{ workflow, events, jobs, artifact, max_age_hours?, os? }`: the GitHub Actions runs `clear --run` verifies as evidence on the identical tree; a profile named after a lane (`free-tests.yml/macos-named-regressions`) clears that lane (see [run evidence](#run-evidence)). `--from-run` and the main-push reuse job verify runs against them too ([from-run](#from-run), [CI reuse](#ci-reuse)) | +| `run_profiles` | name → `{ workflow, events, jobs, artifact, max_age_hours?, os? }`: the GitHub Actions runs `clear --run` verifies as evidence on the identical tree; a profile named after a lane (`free-tests.yml/macos-named-regressions`) clears that lane (see [run evidence](#run-evidence)). `--from-run` and the reuse job verify runs against them too ([from-run](#from-run), [CI reuse](#ci-reuse)) | | `remote_gate_runs` | `{ profiles, inventory, skipped? }`: the run profiles that together clear `pregate.json/remote_gate`, the command that prints the remote gate's test files, one per line, which their runs must cover, and an optional command that prints which selected files the gate itself never runs (see [run evidence](#run-evidence)) | | `ship_lanes_from` | maps each exact /ship lane command to `{ profiles, inventory }`: the run profiles whose verified runs at HEAD's clean tree carry that lane instead of running it, and the command listing its test files, which their runs must cover (see [CI carry](#ci-carry)); read from the base branch only | | `allowlist` | `[{ finding, paths?, pattern?, reason }]`: benign findings the `secrets` check accepts and names as allowlisted, by gstack-redact finding id or non-secret category, path globs and a pattern anchored at the finding; never `env.value` (see [allowlist](#allowlist)) | @@ -881,8 +881,9 @@ merge path. **What is never reusable.** Security and advisory scans (osv, audit, gitleaks) depend on the day, not on the tree, so they are not run profiles: they run on every head as before, and `--from-run` never skips them. -Pull-request required checks and dispatched runs always run fresh; only a -push to main reuses ([below](#ci-reuse)). +Dispatched runs always run fresh. A push to main reuses +([below](#ci-reuse)), and a Free Tests pull request run reuses only a run it +can't control ([below](#ci-reuse-pr)). ### The tests stage from a Free Tests run (`--from-run`) @@ -1027,8 +1028,8 @@ prints `CI_WAIT: green|red|running|none`, then `find`'s verdict on green. A squash merge whose pull request head already held main produces a push with the identical tree, and Free Tests and Windows Free Tests used to run on it a -second time. Now each workflow starts with a `reuse` job that runs only on a -push to `main`, with job-level `contents: read` and `actions: read` (every other +second time. Now each workflow starts with a `reuse` job that runs on a push +to `main` (and, for Free Tests, on pull requests: [below](#ci-reuse-pr)), with job-level `contents: read` and `actions: read` (every other job keeps the workflow's `contents: read`). It runs `scripts/ci-tree-reuse.ts find`, which: @@ -1046,8 +1047,8 @@ It writes `reused=true` with the run's id and link, or `reused=false` and the reason, and prints one `REUSE:` line in the step summary. Every suite job needs `reuse` and runs unless it succeeded with `reused=true`; their conditions start with `!cancelled()` because GitHub otherwise skips a job whose -needed job was skipped, and `reuse` is skipped on every pull request and -dispatch run. The required aggregates (`free-tests`, `windows-free-tests`) +needed job was skipped, and `reuse` is skipped on every dispatch run (and on +every Windows Free Tests run). The required aggregates (`free-tests`, `windows-free-tests`) choose a path in their first step, before any download. A verified reuse writes the reused run's link to the summary and passes; anything else, a failed or skipped `reuse` job included, takes the strict path unchanged. Only the strict From 567cc96732e724e566f10677d4a19c58c17a24f0 Mon Sep 17 00:00:00 2001 From: garrytan Date: Tue, 3 Nov 2026 08:06:35 +0000 Subject: [PATCH 4/4] docs: pregate run_profiles row reads as a sentence (prose lint) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- docs/pregate.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/pregate.md b/docs/pregate.md index 46a889ec5..711d6c2d2 100644 --- a/docs/pregate.md +++ b/docs/pregate.md @@ -88,7 +88,7 @@ the byte-pinned `/ship` renders in `test/fixtures/golden/`). | `stall_timeout_s` | seconds a test command may print nothing while using under a tenth of one CPU before its process group is killed and the check is `incomplete` (`PREGATE_STALLED`); default 300, `0` disables; `--stall-timeout` overrides; the `--remote` gate is exempt (see [supervision](#supervision)) | | `timeout_s` | seconds each runner process may take (the unit-file group, each serial file, each guard without its own `timeout_s`, and the remote gate's deadline) when `--timeout` is not passed; 1 to 3600, default 600. `gstack-ship-preflight` and the `finish` fallback run pregate without `--timeout`, so this is how a repo with long lanes sets theirs | | `remote_gate` | the repo's remote test gate command; under 8 cores the local test tier becomes a `requires-remote` obligation naming it, and `--remote` runs it (see [remote gate](#remote-gate)) | -| `run_profiles` | name → `{ workflow, events, jobs, artifact, max_age_hours?, os? }`: the GitHub Actions runs `clear --run` verifies as evidence on the identical tree; a profile named after a lane (`free-tests.yml/macos-named-regressions`) clears that lane (see [run evidence](#run-evidence)). `--from-run` and the reuse job verify runs against them too ([from-run](#from-run), [CI reuse](#ci-reuse)) | +| `run_profiles` | a map from each profile name to `{ workflow, events, jobs, artifact, max_age_hours?, os? }`: the GitHub Actions runs `clear --run` verifies as evidence on the identical tree; a profile named after a lane (`free-tests.yml/macos-named-regressions`) clears that lane (see [run evidence](#run-evidence)). `--from-run` and the reuse job verify runs against them too ([from-run](#from-run), [CI reuse](#ci-reuse)) | | `remote_gate_runs` | `{ profiles, inventory, skipped? }`: the run profiles that together clear `pregate.json/remote_gate`, the command that prints the remote gate's test files, one per line, which their runs must cover, and an optional command that prints which selected files the gate itself never runs (see [run evidence](#run-evidence)) | | `ship_lanes_from` | maps each exact /ship lane command to `{ profiles, inventory }`: the run profiles whose verified runs at HEAD's clean tree carry that lane instead of running it, and the command listing its test files, which their runs must cover (see [CI carry](#ci-carry)); read from the base branch only | | `allowlist` | `[{ finding, paths?, pattern?, reason }]`: benign findings the `secrets` check accepts and names as allowlisted, by gstack-redact finding id or non-secret category, path globs and a pattern anchored at the finding; never `env.value` (see [allowlist](#allowlist)) |