From f15a34dc352f02d22751acd99e39fa4a57ee67fc Mon Sep 17 00:00:00 2001 From: garrytan Date: Sun, 18 Oct 2026 03:49:10 +0000 Subject: [PATCH] feat(evals): readiness-gated paid dispatch (R1), restamp-only gate reuse (R2a), and a receipt store that restores Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- .github/workflows/evals.yml | 181 ++++++++++++---- CHANGELOG.md | 15 ++ docs/TESTING_INTERNALS.md | 47 ++++- scripts/eval-readiness.ts | 138 +++++++++++++ scripts/gate-reuse.ts | 299 +++++++++++++++++++++++++++ scripts/test-health-report.ts | 2 +- test/ci-eval-cache.test.ts | 8 +- test/ci-paid-coordination.test.ts | 20 +- test/eval-readiness.test.ts | 129 ++++++++++++ test/evals-pr-lane-workflow.test.ts | 74 +++++-- test/gate-reuse.test.ts | 309 ++++++++++++++++++++++++++++ 11 files changed, 1150 insertions(+), 72 deletions(-) create mode 100644 scripts/eval-readiness.ts create mode 100644 scripts/gate-reuse.ts create mode 100644 test/eval-readiness.test.ts create mode 100644 test/gate-reuse.test.ts diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml index 716b6a22e..1dac113f5 100644 --- a/.github/workflows/evals.yml +++ b/.github/workflows/evals.yml @@ -120,47 +120,46 @@ jobs: docker manifest inspect "$EVAL_IMAGE" > /tmp/eval-runtime-manifest.json echo "id=$(sha256sum /tmp/eval-runtime-manifest.json | cut -d ' ' -f1)" >> "$GITHUB_OUTPUT" - # ── Push-burst debounce (W1d: CEO-15, ENG-11, DX-7) ────────────────────── - # Only on `synchronize` when another evals.yml run for this PR started in - # the last 15 minutes (a push burst): wait 90 s before planning, so the next - # push's cancel-in-progress lands before any paid slice starts. A first push - # pays nothing. cancel-in-progress does the superseding; the head-SHA check - # only reports "superseded by " when this run outlives the wait. The PR - # label evals-no-debounce skips the wait. - debounce: + # ── Readiness-gated dispatch (R1, GSTA-48) ─────────────────────────────── + # Paid slices for a PR head start only after the PR has been quiet for the + # window (vars.EVALS_QUIET_MINUTES, default 10; the evals-no-debounce label + # sets it to 0) and Free Tests has passed on the same head SHA. A newer push + # still cancels this run (cancel-in-progress), so the window is a sleep; the + # same heads get the same selection and verdict rules, only the start moves. + # Free Tests red on this head fails the run without paid work (re-run this + # workflow after a green Free Tests rerun). Dispatches never wait: they are + # explicit (eval:bg, census). Version Gate is not an input, so a stamp push + # waits for its own Free Tests only and always gets its gate. + readiness: runs-on: ubuntu-24.04 - if: github.event_name == 'pull_request' && github.event.action == 'synchronize' && github.actor != 'dependabot[bot]' && github.event.pull_request.head.repo.full_name == github.repository && !contains(github.event.pull_request.labels.*.name, 'evals-no-debounce') - timeout-minutes: 5 + if: github.event_name == 'pull_request' && github.actor != 'dependabot[bot]' && github.event.pull_request.head.repo.full_name == github.repository + timeout-minutes: 90 permissions: + contents: read actions: read pull-requests: read outputs: superseded: ${{ steps.wait.outputs.superseded }} steps: - - name: Wait out a push burst + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.4.2 + + - name: Wait for a quiet PR and green Free Tests on this head id: wait env: GH_TOKEN: ${{ github.token }} PR_NUMBER: ${{ github.event.pull_request.number }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} - HEAD_BRANCH: ${{ github.event.pull_request.head.ref }} + QUIET_MINUTES: ${{ contains(github.event.pull_request.labels.*.name, 'evals-no-debounce') && '0' || vars.EVALS_QUIET_MINUTES || '10' }} run: | - since=$(date -u -d '15 minutes ago' +%s) - recent=$(gh api -X GET "repos/$GITHUB_REPOSITORY/actions/workflows/evals.yml/runs" -f event=pull_request \ - -f branch="$HEAD_BRANCH" -f per_page=20 \ - --jq "[.workflow_runs[] | select(.id < $GITHUB_RUN_ID and any(.pull_requests[]; .number == $PR_NUMBER) and ((.created_at | fromdateiso8601) >= $since))] | length" \ - 2>/dev/null || echo 0) - if [ "${recent:-0}" -eq 0 ]; then - echo "- Debounce: no other run for this PR in the last 15 min; no wait" >> "$GITHUB_STEP_SUMMARY" - exit 0 - fi - echo "- Debounce: ${recent} run(s) for this PR started in the last 15 min; waiting 90 s (label evals-no-debounce skips this)" >> "$GITHUB_STEP_SUMMARY" - sleep 90 - head=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER" --jq .head.sha 2>/dev/null || echo "$HEAD_SHA") - if [ "$head" != "$HEAD_SHA" ]; then - echo "superseded=true" >> "$GITHUB_OUTPUT" - echo "- Debounce: superseded by ${head}; this run plans no slices" >> "$GITHUB_STEP_SUMMARY" - fi + bun --no-install run scripts/eval-readiness.ts wait --repo "$GITHUB_REPOSITORY" --pr "$PR_NUMBER" --head-sha "$HEAD_SHA" \ + --run-id "$GITHUB_RUN_ID" --quiet-minutes "$QUIET_MINUTES" --deadline-minutes 80 \ + --github-output "$GITHUB_OUTPUT" --summary "$GITHUB_STEP_SUMMARY" # ── Receipt recovery (W1c: CEO-14/34, ENG-4/5/7) ───────────────────────── # A run cancelled by the next push never reaches slices-report, so its @@ -175,10 +174,10 @@ jobs: # plan-slices turns reuse off, so cases execute unreduced. recover-receipts: runs-on: ubuntu-24.04 - needs: [debounce] - # Runs after the debounce settles, so a superseded run's cancellation has usually completed. + needs: [readiness] + # Runs after readiness settles, so a superseded run's cancellation has usually completed; a head whose Free Tests failed recovers nothing. # A dispatch naming pr_receipts reads that PR's receipts from its runs' artifacts (cache entries are ref-scoped). - if: ${{ !cancelled() && needs.debounce.outputs.superseded != 'true' && github.actor != 'dependabot[bot]' && ((github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && !contains(github.event.pull_request.labels.*.name, 'evals-fresh')) || (github.event_name == 'workflow_dispatch' && inputs.pr_receipts != '')) }} + if: ${{ !cancelled() && (needs.readiness.result == 'success' || needs.readiness.result == 'skipped') && needs.readiness.outputs.superseded != 'true' && github.actor != 'dependabot[bot]' && ((github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && !contains(github.event.pull_request.labels.*.name, 'evals-fresh')) || (github.event_name == 'workflow_dispatch' && inputs.pr_receipts != '')) }} timeout-minutes: 5 permissions: contents: read @@ -221,6 +220,41 @@ jobs: --store /tmp/gstack-eval-input-cache --store-key "$STORE_KEY" --mode "$RECOVERY_MODE" --budget-seconds 60 >> "$GITHUB_STEP_SUMMARY" \ || echo "- Receipt recovery crashed; reuse stays off this run" >> "$GITHUB_STEP_SUMMARY" + # R2a (GSTA-48), a narrow gate edit: base-ref code proposes reusing the + # newest executed GREEN gate of this PR when this head differs from it + # only by stamp files (TREE: same-modulo-stamps from the tree receipt + # under the base's ship policy) and main has not moved (same base SHA). + # plan-slices confirms the selection is identical. Anything else (a red + # or partial gate, a moved base, a non-stamp byte, no receipt, a gh or + # git error, evals-fresh) leaves no candidate: the full gate runs. + - name: Find a restamp-only gate to reuse (R2a, base-ref code) + if: github.event_name == 'pull_request' + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} + HEAD_BRANCH: ${{ github.event.pull_request.head.ref }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + if [ ! -f scripts/gate-reuse.ts ]; then + echo "- Restamp-only gate reuse unavailable: the base ref has no scripts/gate-reuse.ts; the full gate runs" >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + bun --no-install run scripts/gate-reuse.ts find --repo "$GITHUB_REPOSITORY" --pr "$PR_NUMBER" --head-repo "$HEAD_REPO" \ + --branch "$HEAD_BRANCH" --run-id "$GITHUB_RUN_ID" --head-sha "$HEAD_SHA" --base-sha "$BASE_SHA" \ + --out /tmp/gate-reuse --budget-seconds 90 >> "$GITHUB_STEP_SUMMARY" \ + || echo "- Restamp-only gate reuse crashed; the full gate runs" >> "$GITHUB_STEP_SUMMARY" + + - name: Hand a restamp-only reuse candidate to plan-slices + if: github.event_name == 'pull_request' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: gate-reuse-candidate + path: /tmp/gate-reuse + if-no-files-found: ignore + retention-days: 7 + - name: Hand the receipt store to plan-slices uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: @@ -248,15 +282,16 @@ jobs: # branch (see CLAUDE.md's garrytan-agents workflow). plan-slices: runs-on: ubicloud-standard-8 - needs: [debounce, recover-receipts] - # !cancelled(): plans even when the debounce or recovery was skipped (first push, dispatch, evals-fresh) or recovery failed (reuse is then off); a superseded run plans nothing. - if: ${{ !cancelled() && needs.debounce.outputs.superseded != 'true' && github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} + needs: [readiness, recover-receipts] + # !cancelled(): plans even when readiness or recovery was skipped (dispatch, evals-fresh) or recovery failed (reuse is then off); a superseded run, or a head whose readiness failed (Free Tests red), plans nothing. + if: ${{ !cancelled() && (needs.readiness.result == 'success' || needs.readiness.result == 'skipped') && needs.readiness.outputs.superseded != 'true' && github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} timeout-minutes: 10 permissions: contents: read outputs: slices: ${{ steps.matrix.outputs.slices }} slice_timeouts: ${{ steps.matrix.outputs.slice_timeouts }} + gate_reuse: ${{ steps.gate_reuse.outputs.gate_reuse }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: @@ -311,6 +346,24 @@ jobs: EVALS_BASE: ${{ inputs.base_sha || (inputs.base_ref != '' && format('origin/{0}', inputs.base_ref)) || '' }} run: EVALS_TIER=gate bun --no-install run scripts/test-paid-shards.ts --tier gate --emit-plan /tmp/paid-plan/manifest.json --slice-budget 420 --jobs 2 --max-parallel 20 + # R2a: reuse holds only when this head's own plan selects exactly the + # cases and trial plans the reused gate ran (selection fingerprint). + - name: Download a restamp-only reuse candidate + if: github.event_name == 'pull_request' + continue-on-error: true + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + name: gate-reuse-candidate + path: /tmp/gate-reuse + + - name: Confirm restamp-only gate reuse (R2a) + id: gate_reuse + if: github.event_name == 'pull_request' + run: | + mkdir -p /tmp/gate-reuse + bun --no-install run scripts/gate-reuse.ts confirm /tmp/gate-reuse /tmp/paid-plan/manifest.json --out /tmp/paid-plan \ + --github-output "$GITHUB_OUTPUT" >> "$GITHUB_STEP_SUMMARY" + # CEO-13: the mode, case counts and, on fallback, every file that # restored the full gate with the edit that narrows it next time. - name: Summarize PR coverage @@ -336,6 +389,7 @@ jobs: /tmp/paid-plan/manifest.json /tmp/paid-plan/receipts /tmp/paid-plan/store + /tmp/paid-plan/gate-reuse.json retention-days: 30 eval-slices: @@ -346,7 +400,8 @@ jobs: # !cancelled(), not always(): still runs when build-image was skipped # (image already published), but a newer push's cancel-in-progress stops # it instead of letting a superseded run finish its paid slices first. - if: ${{ !cancelled() && needs.build-image.result == 'success' && needs.plan-slices.result == 'success' }} + # A confirmed restamp-only reuse (R2a) runs no paid slice. + if: ${{ !cancelled() && needs.build-image.result == 'success' && needs.plan-slices.result == 'success' && needs.plan-slices.outputs.gate_reuse != 'true' }} # The planner packs ~7 minutes of recorded work per slice (EVALS_JOBS=2 x # EVALS_CONCURRENCY=2 per runner, never the old 40-way per-row fan-out # that queued claude session STARTUP behind 39 siblings). Each slice's job @@ -500,7 +555,11 @@ jobs: retention-days: 30 slices-report: - runs-on: ubicloud-standard-2 + # GitHub-hosted, like recover-receipts: Ubicloud runners send actions/cache + # to Ubicloud's own transparent cache, which a GitHub-hosted restore never + # sees, so a store saved on Ubicloud was never restored (GSTA-48: every + # restore missed, e.g. #3253's restamp gate 3 minutes after a saved store). + runs-on: ubuntu-24.04 needs: [plan-slices, eval-slices] # !cancelled(): the report must run (and FAIL) when an executor died — a # missing slice artifact reading as green is the class this lane kills — @@ -518,7 +577,7 @@ jobs: permissions: contents: read outputs: - reconcile-exit: ${{ steps.reconcile.outputs.exit }} + reconcile-exit: ${{ steps.reconcile.outputs.exit || steps.reused.outputs.exit }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: @@ -543,6 +602,7 @@ jobs: - name: Reconcile slices against the manifest (fail-closed) id: reconcile + if: needs.plan-slices.outputs.gate_reuse != 'true' run: | set +e EVALS_TIER=gate bun --no-install run scripts/test-paid-shards.ts --tier gate --report /tmp/paid-report | tee /tmp/report.txt @@ -552,6 +612,19 @@ jobs: # (caught by the ship review army; the wiring test now pins this). echo "exit=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT" + # R2a: the planner confirmed a restamp-only reuse, so no slice ran; the + # verdict is the reused gate's, printed with its receipt. A missing or + # malformed reuse record exits 1 (fail-closed). + - name: Report the reused restamp-only verdict (R2a) + id: reused + if: needs.plan-slices.outputs.gate_reuse == 'true' + run: | + set +e + bun --no-install run scripts/gate-reuse.ts report /tmp/paid-report/gate-reuse.json --repo "$GITHUB_REPOSITORY" > /tmp/report.txt + code=$? + cat /tmp/report.txt + echo "exit=$code" >> "$GITHUB_OUTPUT" + - name: Summarize PR coverage and reuse if: always() run: bun --no-install run scripts/test-pr-profile.ts summary /tmp/paid-report/manifest.json /tmp/paid-report/collector-outcomes.json >> "$GITHUB_STEP_SUMMARY" @@ -594,6 +667,29 @@ jobs: if-no-files-found: ignore retention-days: 30 + # R2a: what this executed run proved, read by a later restamp-only push. + # Reused runs bind nothing, so reuse never chains off a reuse. + - name: Bind this run's verdict for restamp-only reuse (R2a) + if: always() && github.event_name == 'pull_request' && needs.plan-slices.outputs.gate_reuse != 'true' + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + RECONCILE_EXIT: ${{ steps.reconcile.outputs.exit }} + run: | + bun --no-install run scripts/gate-reuse.ts bind /tmp/paid-report --out /tmp/gate-binding --repo "$GITHUB_REPOSITORY" \ + --pr "$PR_NUMBER" --run-id "$GITHUB_RUN_ID" --run-attempt "$GITHUB_RUN_ATTEMPT" --head-sha "$HEAD_SHA" \ + --base-sha "$BASE_SHA" --profile "$EVALS_PROFILE" --reconcile-exit "$RECONCILE_EXIT" >> "$GITHUB_STEP_SUMMARY" + + - name: Upload the gate binding + if: always() && github.event_name == 'pull_request' && needs.plan-slices.outputs.gate_reuse != 'true' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: gate-binding-a${{ github.run_attempt }} + path: /tmp/gate-binding/gate-binding.json + if-no-files-found: ignore + retention-days: 30 + - name: Upload trial outcomes for pass-rate history if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 @@ -604,7 +700,7 @@ jobs: retention-days: 90 - name: Fail the workflow when reconciliation failed - if: steps.reconcile.outputs.exit != '0' + if: (steps.reconcile.outputs.exit || steps.reused.outputs.exit) != '0' run: exit 1 # PR comment in its OWN job with the write token and ZERO repo code: no @@ -644,6 +740,13 @@ jobs: RECONCILE_EXIT: ${{ needs.slices-report.outputs.reconcile-exit }} run: | # shellcheck disable=SC2086,SC2059 + # R2a: a confirmed restamp-only reuse ran no case; say so, with the + # reused run, the receipt and the identical selection. + REUSE=/tmp/paid-report/gate-reuse.json + if [ "${RECONCILE_EXIT:-1}" = "0" ] && [ -f "$REUSE" ] \ + && jq -e '.schema == 1 and .tree == "same-modulo-stamps" and (.stamp_files | length > 0)' "$REUSE" >/dev/null 2>&1; then + BODY=$(jq -r --arg repo "$GITHUB_REPOSITORY" '"## E2E Evals: ✅ PASS (reused: restamp-only, R2a)\n\nNo paid case ran on `\(.current_head)`. It differs from `\(.gated_head)`, which passed the full gate in [run \(.reused_run_id)](https://github.com/\($repo)/actions/runs/\(.reused_run_id)), only by stamp files (\(.stamp_files | join(", "))), and main has not moved since (base `\(.base_sha)`).\n\n```\nTREE: same-modulo-stamps receipt=\(.receipt_sha256)\nselection: \(.selection_sha256) (identical to the reused gate)\nversion: \(.old_version // "-") -> \(.new_version // "-")\n```\n\n---\n*Restamp-only reuse (R2a): any non-stamp byte, a moved base or a missing receipt runs the full gate; the evals-fresh label opts out.*"' "$REUSE") + else TOTAL=0; PASSED=0; FAILED=0; MANUAL=0; EXECUTED=0; REUSED=0; COST="0" SUITE_LINES="" VERIFIED=/tmp/verdict/paid-report/collector-outcomes.json @@ -739,6 +842,8 @@ jobs: \`\`\`" fi + fi + COMMENT_ID=$(gh api repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments \ --jq '.[] | select(.body | startswith("## E2E Evals")) | .id' | tail -1) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15917b501..6941b7e59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog +## [Unreleased] + +**The paid PR gate now starts only after a PR head's free tests pass and the PR has been quiet for 10 minutes, and a version-only restamp reuses the green gate it already has.** Over Oct 3 to 17, paid work on heads whose free tests failed, plus runs cancelled by a push within the window, cost about $2,990, about $1,390 a week (GSTA-47 estimate). The same heads still get the same selection and verdict rules; only the start time moves. A restamp whose tree equals the last green head modulo stamps, on an unchanged base, reports that verdict instead of paying again. The PR receipt store, which never restored once, now does. + +### Changed +- **Readiness-gated dispatch (R1).** The `readiness` job in `evals.yml` replaces the 90-second push-burst debounce. It sleeps out the quiet window (repository variable `EVALS_QUIET_MINUTES`, default 10; `evals-no-debounce` sets it to 0), then waits for Free Tests to pass on the same head SHA, for up to 80 minutes. Free Tests red on the head fails the run with no paid work. A newer push still cancels the run. Dispatches never wait, and Version Gate is not an input, so a stamp push always gets its gate. +- **The receipt store restores.** `slices-report` now runs on a GitHub-hosted runner like `recover-receipts`. On Ubicloud its `actions/cache` save went to Ubicloud's transparent cache, which the GitHub-hosted restore never sees: GitHub's cache held no `eval-input` entry at all, and #3253's restamp gate restored nothing 3 minutes after its green run saved 56 receipts. + +### Added +- **Restamp-only gate reuse (R2a), a narrow gate edit.** A PR head reuses the newest executed gate's GREEN verdict only when that gate ran on the same base SHA, the tree receipt prints `TREE: same-modulo-stamps`, every differing file is VERSION, package.json, CHANGELOG.md or the agents digest, and this head's own plan has the same selection fingerprint. Base-ref code proposes the reuse (`scripts/gate-reuse.ts find`), the planner confirms the selection, no slice runs, and the PR comment says `PASS (reused: restamp-only, R2a)` with the reused run and receipt digest. A non-stamp byte (CHANGELOG entry text included), a moved base, a missing binding or receipt, a red or partial gate, or `evals-fresh` runs the full gate. Reuse across a main move (R2) is not enabled. + +### If you… +- **…want a restamp to reuse its gate:** put the CHANGELOG entry in the gated push (under `## [Unreleased]` or the tentative version), so the stamp only re-heads it. +- **…see "Free Tests run N concluded failure" on E2E Evals:** fix and push, or re-run Free Tests and then re-run E2E Evals. + ## [1.91.133.0] - 2026-10-18 **A lane on a small machine no longer reports done while its pregate test tier still waits on a remote run.** The coordinator contract every lane inherits (autoplan reviewer prompts, `gstack-autoplan contract`, /spec's gate-and-file section) now says a lane is done only when `gstack-pregate run --allow-repo-commands` passes, and that a `requires-remote` obligation needs `--stage tests --remote --spend-cap `. diff --git a/docs/TESTING_INTERNALS.md b/docs/TESTING_INTERNALS.md index 6e8f0408a..96d13752d 100644 --- a/docs/TESTING_INTERNALS.md +++ b/docs/TESTING_INTERNALS.md @@ -1126,7 +1126,7 @@ and `test:health` both say so. through today's selector and its shadow, and exits 1 on any MISS: a failed gate case the shadow would not run. `test/exercised-map.test.ts` pins zero. -### PR receipts, recovery and debounce +### PR receipts, recovery, readiness and restamp-only reuse Passes and failures of single-trial shards are written as receipts at the execution boundary, so a cancelled run's finished work survives. The @@ -1139,13 +1139,46 @@ unresolved run. `plan-slices` turns reuse off when recovery degraded, the checkpoint is missing or a cancelled run is still unresolved; recovery itself never fails the run, and the job summary names the reason (rate limit with its reset time, `gh auth status`, the 60-second budget). The whole store is saved by -`slices-report`. +`slices-report`. Both the restore and the save run on GitHub-hosted runners: an +Ubicloud runner sends `actions/cache` to Ubicloud's transparent cache, which a +GitHub-hosted restore never sees (until GSTA-48 every restore missed). -On a `synchronize` push, when another run for the same PR started in the last 15 -minutes, the debounce job waits 90 seconds before planning; a run that outlives -the wait reports "superseded by ". PR labels: `evals-no-debounce` skips the -wait and `evals-fresh` turns receipt reuse off. Labels apply from the next push, -because adding a label does not start `evals.yml`. +**Readiness (R1).** Paid slices for a PR head start only when the PR has been +quiet for the window (repository variable `EVALS_QUIET_MINUTES`, default 10, +measured from the run's creation) and Free Tests has concluded success on the +same head SHA. The `readiness` job (`scripts/eval-readiness.ts`) sleeps out the +window with no API calls, since a newer push cancels the run anyway, then polls +Free Tests every minute for up to 80 minutes. Free Tests red on the head fails +the run without paid work; re-run Free Tests, then re-run this workflow (a +re-run keeps the creation time, so it does not wait out the window again). A +head that is no longer the PR's head reports "superseded by " and plans +nothing. Selection, trials and verdict rules do not change. Version Gate is not +an input, so a stamp push waits only for its own Free Tests and always gets a +gate. Dispatches (`eval:bg`, the census) never wait. + +**Restamp-only reuse (R2a, a gate edit).** A PR head reuses the verdict of the +newest executed gate of the same PR instead of running one only when all of +these hold: that gate is GREEN with reconcile exit 0 and a successful run; it +ran the PR profile on the same base SHA as this run (main has not moved; reuse +across a main move is R2 and is not enabled); the heads differ; the tree +receipt (`lib/tree-receipt.ts` under the base's ship policy) prints +`TREE: same-modulo-stamps` and every differing file is the version file, +`package.json`, the CHANGELOG or a release output (the agents digest); and this +head's own plan has the same selection fingerprint (cases, statuses, trial +plans, profile). `recover-receipts` proposes the candidate from base-ref code, +`plan-slices` confirms the selection, `eval-slices` is skipped, and the report and +PR comment say `PASS (reused: restamp-only, R2a)` with the reused run, the +receipt digest and the stamp files. Executed runs upload a `gate-binding-a` +artifact; reused runs bind nothing, so reuse never chains off a reuse. Any +other case (a red, partial or cancelled gate, a moved base, a non-stamp byte +including CHANGELOG entry text, a missing binding or receipt, a gh or git +error) runs the full gate. For a restamp to qualify, write the CHANGELOG entry +before the gated push (under `## [Unreleased]` or the tentative version) so the +stamp only re-heads it. + +PR labels: `evals-no-debounce` sets the quiet window to 0 (Free Tests is still +required) and `evals-fresh` turns receipt reuse and restamp-only reuse off. Labels +apply from the next push, because adding a label does not start `evals.yml`. ## Free suite runner, judge reuse and engine skips diff --git a/scripts/eval-readiness.ts b/scripts/eval-readiness.ts new file mode 100644 index 000000000..ae685f5a8 --- /dev/null +++ b/scripts/eval-readiness.ts @@ -0,0 +1,138 @@ +/** + * Readiness-gated dispatch for the PR paid lane (R1, GSTA-48). + * + * The `readiness` job in evals.yml runs this before any paid work: a PR + * head's paid slices start only after (1) the PR has been quiet for the + * window (no newer push since this run was created; `vars.EVALS_QUIET_MINUTES`, + * default 10; the `evals-no-debounce` label sets it to 0) and (2) Free Tests + * (free-tests.yml) has concluded success on the same head SHA. Selection, + * trials and verdict rules are unchanged: only the start time moves. + * + * A newer push cancels the run through the workflow's cancel-in-progress + * group, so the quiet window is a plain sleep with no API calls. After it, + * Free Tests is polled every `--poll-seconds`. Free Tests concluding anything + * but success on this head fails the run without paid work: that head cannot + * merge, and re-running this workflow after a green Free Tests rerun starts + * the gate (a re-run keeps the run's original creation time, so it does not + * wait out the window again). A head that is no longer the PR's head reports + * superseded and plans nothing. + */ +import * as fs from 'node:fs'; +import { runGh, type GhRunner } from './recover-receipts'; + +export interface FreeTestsRun { id: number; status: string; conclusion: string | null; head_sha: string; event: string } +export type FreeTestsState = + | { state: 'missing' } + | { state: 'running'; runId: number } + | { state: 'success'; runId: number } + | { state: 'failed'; runId: number; conclusion: string }; + +export type ReadinessDecision = + | { kind: 'wait'; reason: string } + | { kind: 'ready'; reason: string } + | { kind: 'superseded'; by: string } + | { kind: 'blocked'; reason: string }; + +/** The newest Free Tests pull_request run on this exact head decides; a rerun updates that run in place. */ +export function freeTestsState(runs: FreeTestsRun[], headSha: string): FreeTestsState { + const newest = runs.filter(run => run.head_sha === headSha && run.event === 'pull_request').sort((a, b) => b.id - a.id)[0]; + if (!newest) return { state: 'missing' }; + if (newest.status !== 'completed') return { state: 'running', runId: newest.id }; + if (newest.conclusion === 'success') return { state: 'success', runId: newest.id }; + return { state: 'failed', runId: newest.id, conclusion: newest.conclusion ?? 'unknown' }; +} + +export function readinessDecision(input: { headSha: string; prHead: string | null; free: FreeTestsState; quietLeftMs: number }): ReadinessDecision { + if (input.prHead && input.prHead !== input.headSha) return { kind: 'superseded', by: input.prHead }; + const free = input.free; + if (free.state === 'failed') { + return { kind: 'blocked', reason: `Free Tests run ${free.runId} concluded ${free.conclusion} on ${input.headSha.slice(0, 12)}; the paid gate does not start for a head that cannot merge. Fix and push, or re-run Free Tests and then re-run this workflow` }; + } + if (input.quietLeftMs > 0) return { kind: 'wait', reason: `quiet window: ${Math.ceil(input.quietLeftMs / 1000)} s left` }; + if (free.state === 'missing') return { kind: 'wait', reason: 'Free Tests has not started on this head yet' }; + if (free.state === 'running') return { kind: 'wait', reason: `Free Tests run ${free.runId} is still running` }; + return { kind: 'ready', reason: `Free Tests run ${free.runId} passed on ${input.headSha.slice(0, 12)} and the PR has been quiet for the window` }; +} + +export interface WaitOptions { + repo: string; + pr: number; + headSha: string; + runId: number; + quietMs: number; + deadlineMs: number; + pollMs: number; + gh?: GhRunner; + now?: () => number; + sleep?: (ms: number) => void; +} + +function ghJson(gh: GhRunner, args: string[]): { ok: true; value: T } | { ok: false; error: string } { + const out = gh(args, 30_000); + if (out.status !== 0) return { ok: false, error: out.stderr.trim().split('\n')[0]?.slice(0, 200) || `gh exited ${out.status}` }; + try { return { ok: true, value: JSON.parse(out.stdout) as T }; } catch { return { ok: false, error: 'gh returned unreadable JSON' }; } +} + +/** Waits until the head is ready, superseded or blocked; never longer than the deadline. */ +export function waitForReadiness(o: WaitOptions): { decision: ReadinessDecision; lines: string[] } { + const gh = o.gh ?? runGh; + const now = o.now ?? Date.now; + const sleep = o.sleep ?? ((ms: number) => Bun.sleepSync(ms)); + const start = now(); + const lines: string[] = []; + const run = ghJson<{ created_at?: string }>(gh, ['api', `repos/${o.repo}/actions/runs/${o.runId}`]); + const created = run.ok ? Date.parse(run.value.created_at ?? '') : NaN; + const quietUntil = (Number.isFinite(created) ? created : start) + o.quietMs; + lines.push(`Readiness (R1): paid slices start after Free Tests passes on ${o.headSha.slice(0, 12)} and the PR is quiet for ${Math.round(o.quietMs / 60_000)} min (from ${new Date(quietUntil - o.quietMs).toISOString()})`); + if (quietUntil > now()) sleep(quietUntil - now()); + let lastError = ''; + for (;;) { + const listed = ghJson<{ workflow_runs?: FreeTestsRun[] }>(gh, ['api', '-X', 'GET', `repos/${o.repo}/actions/workflows/free-tests.yml/runs`, + '-f', `head_sha=${o.headSha}`, '-f', 'event=pull_request', '-f', 'per_page=20']); + const pr = ghJson<{ head?: { sha?: string } }>(gh, ['api', `repos/${o.repo}/pulls/${o.pr}`]); + lastError = !listed.ok ? listed.error : !pr.ok ? pr.error : ''; + if (!lastError && listed.ok && pr.ok) { + const decision = readinessDecision({ headSha: o.headSha, prHead: pr.value.head?.sha ?? null, + free: freeTestsState(listed.value.workflow_runs ?? [], o.headSha), quietLeftMs: quietUntil - now() }); + if (decision.kind !== 'wait') { + lines.push(decision.kind === 'superseded' ? `superseded by ${decision.by}; this run plans no slices` : decision.reason); + lines.push(`Readiness wait: ${Math.round((now() - start) / 1000)} s`); + return { decision, lines }; + } + lastError = decision.reason; + } + if (now() + o.pollMs > start + o.deadlineMs) { + const reason = `not ready after ${Math.round(o.deadlineMs / 60_000)} min (${lastError}); re-run this workflow once Free Tests has passed on this head`; + lines.push(reason); + return { decision: { kind: 'blocked', reason }, lines }; + } + sleep(o.pollMs); + } +} + +const USAGE = `usage: bun run scripts/eval-readiness.ts wait --repo --pr --head-sha --run-id + [--quiet-minutes 10] [--deadline-minutes 80] [--poll-seconds 60] [--github-output ] [--summary ] +Exit 0 when ready or superseded (output superseded=true), 1 when blocked, 2 on usage.`; + +if (import.meta.main) { + const [command, ...rest] = process.argv.slice(2); + const flag = (name: string) => { const i = rest.indexOf(`--${name}`); return i >= 0 ? rest[i + 1] : undefined; }; + const num = (name: string, fallback: number) => { const value = flag(name); return value === undefined || value === '' ? fallback : Number(value); }; + const [repo, headSha] = [flag('repo'), flag('head-sha')]; + const [pr, runId, quiet, deadline, poll] = [Number(flag('pr')), Number(flag('run-id')), num('quiet-minutes', 10), num('deadline-minutes', 80), num('poll-seconds', 60)]; + if (command !== 'wait' || !repo || !headSha || !/^[0-9a-f]{40}$/.test(headSha) || !Number.isSafeInteger(pr) || !Number.isSafeInteger(runId) + || ![quiet, deadline, poll].every(value => Number.isFinite(value) && value >= 0)) { + console.error(USAGE); + process.exit(2); + } + const { decision, lines } = waitForReadiness({ repo, pr, headSha, runId, quietMs: quiet * 60_000, deadlineMs: deadline * 60_000, pollMs: Math.max(5, poll) * 1000 }); + const summary = flag('summary'); + for (const line of lines) { + console.log(line); + if (summary) fs.appendFileSync(summary, `- ${line}\n`); + } + const output = flag('github-output'); + if (output && decision.kind === 'superseded') fs.appendFileSync(output, 'superseded=true\n'); + if (decision.kind === 'blocked') console.error(`::error::${decision.reason}`); + process.exit(decision.kind === 'blocked' ? 1 : 0); +} diff --git a/scripts/gate-reuse.ts b/scripts/gate-reuse.ts new file mode 100644 index 000000000..9fd00e486 --- /dev/null +++ b/scripts/gate-reuse.ts @@ -0,0 +1,299 @@ +/** + * Restamp-only gate reuse for the PR paid lane (R2a, GSTA-48). A gate edit: + * a head may report the verdict of an earlier full paid gate instead of + * running one, only when every condition below holds; anything else runs the + * full gate. + * + * bind (slices-report, every executed pull_request run) records what the + * run proved: PR, head, the base SHA the merge ref was built on, + * the reconcile exit, the report verdict and the selection + * fingerprint of its manifest. Reused runs bind nothing. + * find (recover-receipts, BASE-REF code, actions: read) takes the newest + * earlier completed run of this PR that bound a verdict. It must be + * GREEN with reconcile exit 0 and a successful conclusion, on the + * same base SHA as this run (main did not move; reuse across a + * main move is R2, not this), on a different head, and the tree + * receipt (lib/tree-receipt.ts, the base's ship policy) must say + * `TREE: same-modulo-stamps` with every differing file one of + * the policy's stamp files: the version file, package.json, the + * CHANGELOG and the release outputs. It writes candidate.json. + * confirm (plan-slices, after this run's own plan) requires this run's + * selection fingerprint to equal the bound one, so the reused + * verdict covers exactly the cases and trial plans this head would + * run. It writes gate-reuse.json and `reuse=true`. + * report (slices-report) prints the reused verdict and its evidence. + * + * Every failure (no binding, a red or partial verdict, a moved base, a + * non-stamp byte, a missing receipt, gh or git errors, a selection change) + * leaves no candidate, and the run executes the full gate. + */ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { receiptDigest } from '../lib/ship-queue'; +import { isUnavailable, loadPolicyFromBase } from '../lib/ship-policy'; +import { buildReceipt } from '../lib/tree-receipt'; +import { acceptedRun, ghFailureReason, runGh, type GhRunner, type WorkflowRun } from './recover-receipts'; +import type { PaidRunManifest } from './lib/paid-plan'; + +export const BINDING_FILE = 'gate-binding.json'; +export const CANDIDATE_FILE = 'candidate.json'; +export const REUSE_FILE = 'gate-reuse.json'; +const SHA = /^[0-9a-f]{40}$/; + +export interface GateBinding { + schema: 1; + repository: string; + pr: number; + run_id: number; + run_attempt: number; + head_sha: string; + base_sha: string; + profile: string; + reconcile_exit: number; + verdict: string; + selection_sha256: string; +} + +export interface GateReuse { + schema: 1; + pr: number; + reused_run_id: number; + reused_run_attempt: number; + gated_head: string; + current_head: string; + base_sha: string; + tree: 'same-modulo-stamps'; + receipt_sha256: string; + stamp_files: string[]; + old_version: string | null; + new_version: string | null; + selection_sha256: string; +} + +/** The cases, trial plans and verdict-relevant plan fields a run executes; slice packing and receipts are excluded. */ +export function selectionFingerprint(manifest: PaidRunManifest): string { + const entries = manifest.entries + .map(entry => ({ file: entry.file, status: entry.status, trial: entry.trial ?? null, excludeCases: entry.excludeCases ?? null })) + .sort((a, b) => (a.file < b.file ? -1 : a.file > b.file ? 1 : 0)); + const body = { tier: manifest.tier, evalsAll: manifest.evalsAll, profile: manifest.profile ?? null, selection: manifest.selection ?? null, + mode: manifest.prCoverage?.mode ?? null, entries }; + return createHash('sha256').update(JSON.stringify(body)).digest('hex'); +} + +export function readJson(file: string): T | null { + try { return JSON.parse(fs.readFileSync(file, 'utf8')) as T; } catch { return null; } +} + +export function parseBinding(value: unknown): GateBinding | null { + const b = value as GateBinding | null; + if (!b || b.schema !== 1 || typeof b.repository !== 'string' || !Number.isSafeInteger(b.pr) || !Number.isSafeInteger(b.run_id) + || !Number.isSafeInteger(b.run_attempt) || !SHA.test(b.head_sha ?? '') || !SHA.test(b.base_sha ?? '') || typeof b.profile !== 'string' + || !Number.isSafeInteger(b.reconcile_exit) || typeof b.verdict !== 'string' || !/^[0-9a-f]{64}$/.test(b.selection_sha256 ?? '')) return null; + return b; +} + +export function parseReuse(value: unknown): GateReuse | null { + const r = value as GateReuse | null; + if (!r || r.schema !== 1 || r.tree !== 'same-modulo-stamps' || !Number.isSafeInteger(r.pr) || !Number.isSafeInteger(r.reused_run_id) + || !SHA.test(r.gated_head ?? '') || !SHA.test(r.current_head ?? '') || !SHA.test(r.base_sha ?? '') || r.gated_head === r.current_head + || !/^[0-9a-f]{64}$/.test(r.receipt_sha256 ?? '') || !/^[0-9a-f]{64}$/.test(r.selection_sha256 ?? '') + || !Array.isArray(r.stamp_files) || r.stamp_files.length === 0) return null; + return r; +} + +/** slices-report: what this executed run proved, or why it binds nothing. */ +export function buildBinding(o: { repository: string; pr: number; runId: number; runAttempt: number; headSha: string; baseSha: string; + profile: string; reconcileExit: number; reportDir: string }): { binding: GateBinding | null; reason: string } { + if (fs.existsSync(path.join(o.reportDir, REUSE_FILE))) return { binding: null, reason: 'this run reused a verdict; only executed gates bind' }; + const manifest = readJson(path.join(o.reportDir, 'manifest.json')); + if (!manifest?.entries) return { binding: null, reason: 'no readable manifest' }; + const outcomes = readJson<{ verdict?: { verdict?: string } }>(path.join(o.reportDir, 'collector-outcomes.json')); + const binding: GateBinding = { schema: 1, repository: o.repository, pr: o.pr, run_id: o.runId, run_attempt: o.runAttempt, + head_sha: o.headSha, base_sha: o.baseSha, profile: o.profile, reconcile_exit: o.reconcileExit, + verdict: outcomes?.verdict?.verdict ?? 'MISSING', selection_sha256: selectionFingerprint(manifest) }; + return parseBinding(binding) ? { binding, reason: `bound ${binding.verdict} (reconcile exit ${binding.reconcile_exit})` } : { binding: null, reason: 'binding fields are incomplete' }; +} + +export interface FindOptions { + repo: string; + pr: number; + headRepo: string; + branch: string; + selfRunId: number; + headSha: string; + baseSha: string; + /** The base-ref checkout the receipt runs in (its HEAD must be baseSha). */ + repoRoot: string; + out: string; + budgetMs?: number; + maxRuns?: number; + gh?: GhRunner; + git?: (args: string[]) => { ok: boolean; out: string }; + now?: () => number; +} + +const gitIn = (cwd: string) => (args: string[]) => { + const r = spawnSync('git', args, { cwd, encoding: 'utf8', timeout: 120_000, maxBuffer: 64 * 1024 * 1024 }); + return { ok: r.status === 0, out: (r.stdout ?? '').trim() || (r.stderr ?? '').trim() }; +}; + +/** Never throws for an expected refusal: no candidate means the full gate runs, with the reason named. */ +export function findReusableGate(o: FindOptions): { candidate: GateReuse | null; lines: string[] } { + const gh = o.gh ?? runGh; + const git = o.git ?? gitIn(o.repoRoot); + const now = o.now ?? Date.now; + const deadline = now() + (o.budgetMs ?? 90_000); + const left = () => Math.max(1_000, deadline - now()); + const lines: string[] = []; + const refuse = (reason: string) => { lines.push(`no restamp-only reuse: ${reason}; the full gate runs`); return { candidate: null, lines }; }; + if (!SHA.test(o.headSha) || !SHA.test(o.baseSha)) return refuse('this run has no exact head and base SHA'); + + const listed = gh(['api', '-X', 'GET', `repos/${o.repo}/actions/workflows/evals.yml/runs`, '-f', 'event=pull_request', '-f', `branch=${o.branch}`, '-f', 'per_page=50'], left()); + if (listed.status !== 0) return refuse(ghFailureReason(listed.stderr)); + let runs: WorkflowRun[]; + try { runs = (JSON.parse(listed.stdout).workflow_runs ?? []) as WorkflowRun[]; } catch { return refuse('gh returned an unreadable run list'); } + const scope = { pr: o.pr, headRepo: o.headRepo, selfRunId: o.selfRunId }; + const older = runs.filter(run => acceptedRun(run, scope) && run.id < o.selfRunId && run.status === 'completed') + .sort((a, b) => b.id - a.id).slice(0, o.maxRuns ?? 10); + + const work = fs.mkdtempSync(path.join(os.tmpdir(), 'gate-reuse-')); + try { + let found: { run: WorkflowRun; binding: GateBinding | null } | null = null; + for (const run of older) { + const names = gh(['api', '-X', 'GET', `repos/${o.repo}/actions/runs/${run.id}/artifacts`, '-f', 'per_page=100'], left()); + if (names.status !== 0) return refuse(`cannot list run ${run.id}'s artifacts (${ghFailureReason(names.stderr)})`); + let artifact: string | undefined; + try { + artifact = (JSON.parse(names.stdout).artifacts ?? []) + .filter((a: { name: string; expired?: boolean }) => !a.expired && /^gate-binding-a\d+$/.test(a.name)) + .map((a: { name: string }) => a.name).sort((a: string, b: string) => Number(b.slice(14)) - Number(a.slice(14)))[0]; + } catch { return refuse(`run ${run.id}'s artifact list is unreadable`); } + if (!artifact) continue; + const dir = path.join(work, String(run.id)); + const got = gh(['run', 'download', String(run.id), '-R', o.repo, '-n', artifact, '-D', dir], left()); + if (got.status !== 0) return refuse(`cannot download ${artifact} of run ${run.id} (${ghFailureReason(got.stderr)})`); + found = { run, binding: parseBinding(readJson(path.join(dir, BINDING_FILE))) }; + break; + } + if (!found) return refuse('no earlier executed gate on this PR bound a verdict'); + const { run, binding } = found; + if (!binding) return refuse(`run ${run.id}'s gate binding is missing or malformed`); + if (binding.run_id !== run.id || binding.pr !== o.pr || binding.repository !== o.repo || binding.head_sha !== run.head_sha) { + return refuse(`run ${run.id}'s gate binding does not match the run it came from`); + } + if (run.conclusion !== 'success' || binding.verdict !== 'GREEN' || binding.reconcile_exit !== 0) { + return refuse(`the newest executed gate (run ${run.id}, ${binding.head_sha.slice(0, 12)}) is ${binding.verdict}, reconcile exit ${binding.reconcile_exit}, conclusion ${run.conclusion}`); + } + if (binding.profile !== 'pr') return refuse(`run ${run.id} ran the ${binding.profile} profile, not the PR gate`); + if (binding.base_sha !== o.baseSha) { + return refuse(`main moved: run ${run.id} gated on base ${binding.base_sha.slice(0, 12)}, this run's base is ${o.baseSha.slice(0, 12)} (reuse across a main move is R2, which is not enabled)`); + } + if (binding.head_sha === o.headSha) return refuse(`run ${run.id} gated this same head; a re-run executes fresh`); + + const head = git(['rev-parse', 'HEAD']); + if (!head.ok || head.out !== o.baseSha) return refuse(`the receipt checkout is ${head.out.slice(0, 12) || 'unreadable'}, not the base ${o.baseSha.slice(0, 12)}`); + const fetched = git(['fetch', '--no-tags', '--depth=1', 'origin', binding.head_sha, o.headSha]); + if (!fetched.ok) return refuse(`cannot fetch ${binding.head_sha.slice(0, 12)} and ${o.headSha.slice(0, 12)} (${fetched.out.split('\n')[0]})`); + const pinned = git(['update-ref', 'refs/remotes/origin/gate-reuse-base', o.baseSha]); + if (!pinned.ok) return refuse('cannot pin the base ref for the policy'); + const policy = loadPolicyFromBase(o.repoRoot, 'gate-reuse-base'); + if (isUnavailable(policy)) return refuse(`the base's ship policy is unavailable (${policy.detail})`); + if (policy.errors.length) return refuse(`the base's ship policy is invalid (${policy.errors[0]!.message})`); + const receipt = buildReceipt({ repoRoot: o.repoRoot, gated: binding.head_sha, current: o.headSha, policy: policy.policy, policyLabel: policy.source.label }); + if ('error' in receipt) return refuse(`no tree receipt: ${receipt.error}`); + lines.push(`TREE: ${receipt.tree === 'changed' ? `changed (${receipt.residual.join(', ')})` : 'same-modulo-stamps'} (${binding.head_sha.slice(0, 12)} -> ${o.headSha.slice(0, 12)}, policy ${receipt.policy})`); + if (receipt.tree !== 'same-modulo-stamps') return refuse(`${o.headSha.slice(0, 12)} differs from the gated head beyond stamps: ${receipt.residual.join(', ')}`); + const stampFiles = receipt.files.map(file => file.path); + const allowed = new Set([receipt.files.find(f => f.rule === 'version-file')?.path ?? 'VERSION', 'package.json', policy.policy.changelog ?? '']); + const outside = stampFiles.filter(file => !allowed.has(file) && !policy.policy.release_outputs.includes(file)); + if (outside.length) return refuse(`stamp-classified files outside the version file, package.json, CHANGELOG and release outputs: ${outside.join(', ')}`); + if (!stampFiles.length) return refuse('the heads differ by no stamp file'); + const candidate: GateReuse = { schema: 1, pr: o.pr, reused_run_id: run.id, reused_run_attempt: binding.run_attempt, + gated_head: binding.head_sha, current_head: o.headSha, base_sha: o.baseSha, tree: 'same-modulo-stamps', + receipt_sha256: receiptDigest(receipt), stamp_files: stampFiles, old_version: receipt.old_version, new_version: receipt.new_version, + selection_sha256: binding.selection_sha256 }; + fs.mkdirSync(o.out, { recursive: true }); + fs.writeFileSync(path.join(o.out, CANDIDATE_FILE), `${JSON.stringify(candidate, null, 2)}\n`); + lines.push(`restamp-only candidate: run ${run.id} passed the full gate at ${binding.head_sha.slice(0, 12)} on base ${o.baseSha.slice(0, 12)}; stamp files ${stampFiles.join(', ')}; receipt ${candidate.receipt_sha256}; plan-slices confirms the selection`); + return { candidate, lines }; + } finally { + fs.rmSync(work, { recursive: true, force: true }); + } +} + +/** plan-slices: the reuse holds only when this head's own plan selects exactly what the reused gate ran. */ +export function confirmReuse(candidateDir: string, manifest: PaidRunManifest | null): { reuse: GateReuse | null; reason: string } { + const candidate = parseReuse(readJson(path.join(candidateDir, CANDIDATE_FILE))); + if (!candidate) return { reuse: null, reason: 'no restamp-only candidate; the full gate runs' }; + if (!manifest?.entries) return { reuse: null, reason: 'this run has no readable plan; the full gate runs' }; + const fingerprint = selectionFingerprint(manifest); + if (fingerprint !== candidate.selection_sha256) { + return { reuse: null, reason: `this head's selection ${fingerprint.slice(0, 12)} differs from the reused gate's ${candidate.selection_sha256.slice(0, 12)}; the full gate runs` }; + } + return { reuse: candidate, reason: `reusing run ${candidate.reused_run_id}'s GREEN verdict at ${candidate.gated_head.slice(0, 12)}: TREE same-modulo-stamps (${candidate.stamp_files.join(', ')}), same base ${candidate.base_sha.slice(0, 12)}, same selection ${fingerprint.slice(0, 12)}` }; +} + +export function reuseReport(reuse: GateReuse, repo: string): string { + return [ + `GATE REUSED (R2a, restamp-only): no paid case ran on ${reuse.current_head}`, + `reused verdict: GREEN from run ${reuse.reused_run_id} attempt ${reuse.reused_run_attempt} at ${reuse.gated_head} (https://github.com/${repo}/actions/runs/${reuse.reused_run_id})`, + `base: ${reuse.base_sha} (unchanged since that gate)`, + `TREE: same-modulo-stamps receipt=${reuse.receipt_sha256}`, + `stamp files: ${reuse.stamp_files.join(', ')} (${reuse.old_version ?? '-'} -> ${reuse.new_version ?? '-'})`, + `selection: ${reuse.selection_sha256} (identical to the reused gate)`, + 'cost: $0 (no trials executed)', + ].join('\n'); +} + +const USAGE = `usage: + bun run scripts/gate-reuse.ts bind --out --repo --pr --run-id --run-attempt + --head-sha --base-sha --profile --reconcile-exit + bun run scripts/gate-reuse.ts find --repo --pr --head-repo --branch --run-id + --head-sha --base-sha --out [--budget-seconds 90] + bun run scripts/gate-reuse.ts confirm --out [--github-output ] + bun run scripts/gate-reuse.ts report --repo +bind and find never fail the run; no candidate means the full gate runs.`; + +if (import.meta.main) { + const [command, ...rest] = process.argv.slice(2); + const flag = (name: string) => { const i = rest.indexOf(`--${name}`); return i >= 0 ? rest[i + 1] : undefined; }; + const int = (name: string) => Number(flag(name)); + const usage = () => { console.error(USAGE); process.exit(2); }; + if (command === 'bind') { + const [reportDir, out, repository, headSha, baseSha, profile] = [rest[0], flag('out'), flag('repo'), flag('head-sha'), flag('base-sha'), flag('profile')]; + if (!reportDir || !out || !repository || !headSha || !baseSha || !profile) usage(); + const { binding, reason } = buildBinding({ repository: repository!, pr: int('pr'), runId: int('run-id'), runAttempt: int('run-attempt'), + headSha: headSha!, baseSha: baseSha!, profile: profile!, reconcileExit: /^\d+$/.test(flag('reconcile-exit') ?? '') ? int('reconcile-exit') : -1, reportDir: reportDir! }); + if (binding) { + fs.mkdirSync(out!, { recursive: true }); + fs.writeFileSync(path.join(out!, BINDING_FILE), `${JSON.stringify(binding, null, 2)}\n`); + } + console.log(`- Gate binding: ${reason}`); + } else if (command === 'find') { + const [repo, headRepo, branch, headSha, baseSha, out] = [flag('repo'), flag('head-repo'), flag('branch'), flag('head-sha'), flag('base-sha'), flag('out')]; + if (!repo || !headRepo || !branch || !headSha || !baseSha || !out || !Number.isSafeInteger(int('pr')) || !Number.isSafeInteger(int('run-id'))) usage(); + const { lines } = findReusableGate({ repo: repo!, pr: int('pr'), headRepo: headRepo!, branch: branch!, selfRunId: int('run-id'), headSha: headSha!, + baseSha: baseSha!, repoRoot: process.cwd(), out: out!, budgetMs: Number(flag('budget-seconds') ?? '90') * 1000 }); + console.log('### Restamp-only gate reuse (R2a)'); + for (const line of lines) console.log(`- ${line}`); + } else if (command === 'confirm') { + const [dir, manifestPath, out] = [rest[0], rest[1], flag('out')]; + if (!dir || !manifestPath || !out) usage(); + const { reuse, reason } = confirmReuse(dir!, readJson(manifestPath!)); + if (reuse) { + fs.mkdirSync(out!, { recursive: true }); + fs.writeFileSync(path.join(out!, REUSE_FILE), `${JSON.stringify(reuse, null, 2)}\n`); + } + const output = flag('github-output'); + if (output) fs.appendFileSync(output, `gate_reuse=${reuse ? 'true' : 'false'}\n`); + console.log(`- Restamp-only gate reuse (R2a): ${reason}`); + } else if (command === 'report') { + const reuse = parseReuse(readJson(rest[0] ?? '')); + const repo = flag('repo'); + if (!reuse || !repo) { console.error('gate-reuse report: the reuse record is missing or malformed; the gate is not green'); process.exit(1); } + console.log(reuseReport(reuse!, repo!)); + } else usage(); +} diff --git a/scripts/test-health-report.ts b/scripts/test-health-report.ts index bccc11639..f6784ea8a 100644 --- a/scripts/test-health-report.ts +++ b/scripts/test-health-report.ts @@ -277,7 +277,7 @@ export function collectHealth(inputs: HealthInputs): HealthReport { return { value: `${(total / 60_000 * 7 / sinceDays).toFixed(0)} min/week (${cancelled.length} cancelled run(s))` }; })); - metrics.push(metric('push-to-verdict', 'PR lane push-to-verdict', 'created_at to updated_at of evals.yml pull_request runs that reached success or failure (includes queue and debounce)', () => { + metrics.push(metric('push-to-verdict', 'PR lane push-to-verdict', 'created_at to updated_at of evals.yml pull_request runs that reached success or failure (includes queue and the readiness wait)', () => { const walls = need(prRuns).filter(run => run.conclusion === 'success' || run.conclusion === 'failure').map(run => span(run.createdAt, run.updatedAt)!); if (walls.length === 0) throw new Error('no PR run reached a verdict in the window'); return { value: `p50 ${minutes(percentile(walls, 0.5))}, p95 ${minutes(percentile(walls, 0.95))} (${walls.length} runs)` }; diff --git a/test/ci-eval-cache.test.ts b/test/ci-eval-cache.test.ts index 6980e5d43..7f947a4bd 100644 --- a/test/ci-eval-cache.test.ts +++ b/test/ci-eval-cache.test.ts @@ -39,7 +39,7 @@ test('receipt transport: only recover-receipts restores this repository and PR\' const emit = planner.find((s: any) => s.run?.includes('--emit-plan /tmp/paid-plan/manifest.json')); expect(emit.env.EVALS_CACHE_DIR).toBe("${{ steps.reuse.outputs.reuse == 'on' && '/tmp/gstack-eval-input-cache' || '' }}"); const upload = planner.find((s: any) => s.with?.name === 'paid-plan'); - expect(upload.with.path.trim().split('\n')).toEqual(['/tmp/paid-plan/manifest.json', '/tmp/paid-plan/receipts', '/tmp/paid-plan/store']); + expect(upload.with.path.trim().split('\n')).toEqual(['/tmp/paid-plan/manifest.json', '/tmp/paid-plan/receipts', '/tmp/paid-plan/store', '/tmp/paid-plan/gate-reuse.json']); // Executors never restore or save a cache of their own: every slice sees the plan's one receipt set. const executor = paid.jobs['eval-slices'].steps; expect(executor.filter((s: any) => s.uses?.startsWith('actions/cache/'))).toHaveLength(0); @@ -54,6 +54,12 @@ test('receipt transport: only recover-receipts restores this repository and PR\' expect(save[0].with.key).toBe('eval-input-v1-${{ github.repository_id }}-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }}-merged'); // Only PR runs save; a pr_receipts dispatch reads the PR store read-only. expect(save[0].if).toBe("always() && github.event_name == 'pull_request'"); + // Restore and save share one cache backend (GSTA-48): an Ubicloud runner saves into Ubicloud's transparent + // cache, which a GitHub-hosted restore never sees, so every restore missed while the report ran on Ubicloud. + for (const job of Object.values(paid.jobs) as any[]) { + if (!job.steps.some((s: any) => s.uses?.startsWith('actions/cache/') && s.with?.path === '/tmp/gstack-eval-input-cache')) continue; + expect(job['runs-on']).toBe('ubuntu-24.04'); + } expect(report.indexOf(save[0])).toBeGreaterThan(report.indexOf(merge)); expect(paid.jobs['eval-slices'].permissions).toEqual({ contents: 'read', packages: 'read' }); expect(paid.jobs['slices-report'].permissions).toEqual({ contents: 'read' }); diff --git a/test/ci-paid-coordination.test.ts b/test/ci-paid-coordination.test.ts index 31c6ca253..0d4675c65 100644 --- a/test/ci-paid-coordination.test.ts +++ b/test/ci-paid-coordination.test.ts @@ -75,9 +75,9 @@ describe('paid CI coordination stays off the eval image', () => { test(`${name}: planning is independent of image startup and has no dependency install`, () => { const planner = jobs['plan-slices']; if (name === 'evals.yml') { - // The PR lane plans after its two tiny coordination jobs (push-burst debounce, base-ref receipt recovery), never after the image. - expect(planner.needs).toEqual(['debounce', 'recover-receipts']); - for (const id of ['debounce', 'recover-receipts']) { + // The PR lane plans after its two tiny coordination jobs (readiness, base-ref receipt recovery), never after the image. + expect(planner.needs).toEqual(['readiness', 'recover-receipts']); + for (const id of ['readiness', 'recover-receipts']) { const job = jobs[id] as Job & { 'runs-on'?: string }; expect(job.container, id).toBeUndefined(); expect(job['runs-on'], id).toBe('ubuntu-24.04'); @@ -110,7 +110,8 @@ describe('paid CI coordination stays off the eval image', () => { expect(executor.needs).toEqual(['build-image', 'plan-slices']); expect(JSON.stringify(executor.container)).toContain('needs.build-image.outputs.image-tag'); if (name === 'evals.yml') { - expect(executor.if).toBe("${{ !cancelled() && needs.build-image.result == 'success' && needs.plan-slices.result == 'success' }}"); + // A confirmed restamp-only reuse (R2a) is the one PR run that executes no slice. + expect(executor.if).toBe("${{ !cancelled() && needs.build-image.result == 'success' && needs.plan-slices.result == 'success' && needs.plan-slices.outputs.gate_reuse != 'true' }}"); } else { expect(executor.if).toBeUndefined(); } @@ -165,12 +166,13 @@ describe('paid CI coordination stays off the eval image', () => { test('PR planning preserves the fork and Dependabot trust boundaries across the coordination jobs', () => { const jobs = workflows[0].jobs; const trust = "github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)"; - // !cancelled() lets planning proceed when debounce/recovery were skipped or failed (reuse is then off); the trust clause still gates it. - expect(jobs['plan-slices'].if).toBe(`\${{ !cancelled() && needs.debounce.outputs.superseded != 'true' && ${trust} }}`); + // !cancelled() lets planning proceed when readiness was skipped (dispatch) or recovery was skipped or failed (reuse is then off); + // a readiness failure (Free Tests red on this head) or a superseded head plans nothing; the trust clause still gates it. + expect(jobs['plan-slices'].if).toBe(`\${{ !cancelled() && (needs.readiness.result == 'success' || needs.readiness.result == 'skipped') && needs.readiness.outputs.superseded != 'true' && ${trust} }}`); // The coordination jobs never run for Dependabot or for a fork PR either (recovery also serves an explicit pr_receipts dispatch). - expect(jobs.debounce.if).toContain("github.actor != 'dependabot[bot]'"); - expect(jobs.debounce.if).toContain("github.event_name == 'pull_request'"); - expect(jobs.debounce.if).toContain('github.event.pull_request.head.repo.full_name == github.repository'); + expect(jobs.readiness.if).toContain("github.actor != 'dependabot[bot]'"); + expect(jobs.readiness.if).toContain("github.event_name == 'pull_request'"); + expect(jobs.readiness.if).toContain('github.event.pull_request.head.repo.full_name == github.repository'); expect(jobs['recover-receipts'].if).toContain("github.actor != 'dependabot[bot]'"); expect(jobs['recover-receipts'].if).toContain("(github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository"); expect(jobs['recover-receipts'].if).toContain("(github.event_name == 'workflow_dispatch' && inputs.pr_receipts != '')"); diff --git a/test/eval-readiness.test.ts b/test/eval-readiness.test.ts new file mode 100644 index 000000000..2d37c7241 --- /dev/null +++ b/test/eval-readiness.test.ts @@ -0,0 +1,129 @@ +/** + * Readiness-gated dispatch (R1, GSTA-48), driven from scripted gh responses + * and a fake clock: no network, no sleeping. + */ +import { describe, expect, test } from 'bun:test'; +import { freeTestsState, readinessDecision, waitForReadiness, type FreeTestsRun } from '../scripts/eval-readiness'; +import type { GhRunner } from '../scripts/recover-receipts'; + +const HEAD = 'a'.repeat(40); +const NEWER = 'b'.repeat(40); +const CREATED = Date.parse('2026-10-18T00:00:00Z'); +const free = (id: number, status: string, conclusion: string | null, over: Partial = {}): FreeTestsRun => + ({ id, status, conclusion, head_sha: HEAD, event: 'pull_request', ...over }); + +/** A gh that answers the run, Free Tests and PR reads from a per-poll script, recording each call. */ +function fakeGh(polls: Array<{ free: FreeTestsRun[]; prHead?: string } | 'error'>, createdAt: string | null = new Date(CREATED).toISOString()) { + const calls: string[] = []; + let poll = -1; + const gh: GhRunner = (args) => { + const target = args.find(a => a.startsWith('repos/')) ?? ''; + calls.push(target); + const ok = (body: unknown) => ({ status: 0, stdout: JSON.stringify(body), stderr: '' }); + if (/\/actions\/runs\/\d+$/.test(target)) return createdAt ? ok({ created_at: createdAt }) : { status: 1, stdout: '', stderr: 'HTTP 502' }; + if (target.endsWith('/actions/workflows/free-tests.yml/runs')) { + poll++; + const step = polls[Math.min(poll, polls.length - 1)]!; + return step === 'error' ? { status: 1, stdout: '', stderr: 'HTTP 502' } : ok({ workflow_runs: step.free }); + } + if (/\/pulls\/\d+$/.test(target)) { + const step = polls[Math.min(poll, polls.length - 1)]!; + return ok({ head: { sha: step === 'error' ? HEAD : step.prHead ?? HEAD } }); + } + return { status: 1, stdout: '', stderr: 'unexpected' }; + }; + return { gh, calls }; +} + +function clock(start: number) { + let t = start; + const sleeps: number[] = []; + return { now: () => t, sleep: (ms: number) => { sleeps.push(ms); t += ms; }, sleeps, at: () => t }; +} + +const base = { repo: 'garrytan/gstack', pr: 42, headSha: HEAD, runId: 900, quietMs: 10 * 60_000, deadlineMs: 80 * 60_000, pollMs: 60_000 }; + +describe('readiness decision', () => { + test('the newest Free Tests run on this exact head decides; other heads and events are ignored', () => { + expect(freeTestsState([], HEAD)).toEqual({ state: 'missing' }); + expect(freeTestsState([free(1, 'completed', 'success', { head_sha: NEWER })], HEAD)).toEqual({ state: 'missing' }); + expect(freeTestsState([free(1, 'completed', 'success', { event: 'push' })], HEAD)).toEqual({ state: 'missing' }); + expect(freeTestsState([free(1, 'completed', 'failure'), free(2, 'in_progress', null)], HEAD)).toEqual({ state: 'running', runId: 2 }); + expect(freeTestsState([free(2, 'completed', 'success'), free(1, 'completed', 'failure')], HEAD)).toEqual({ state: 'success', runId: 2 }); + expect(freeTestsState([free(3, 'completed', 'cancelled')], HEAD)).toEqual({ state: 'failed', runId: 3, conclusion: 'cancelled' }); + }); + + test('ready needs both a green Free Tests on the head and an elapsed window; a moved head is superseded', () => { + const green = { state: 'success', runId: 7 } as const; + expect(readinessDecision({ headSha: HEAD, prHead: HEAD, free: green, quietLeftMs: 0 }).kind).toBe('ready'); + expect(readinessDecision({ headSha: HEAD, prHead: HEAD, free: green, quietLeftMs: 1 }).kind).toBe('wait'); + expect(readinessDecision({ headSha: HEAD, prHead: HEAD, free: { state: 'running', runId: 7 }, quietLeftMs: 0 }).kind).toBe('wait'); + expect(readinessDecision({ headSha: HEAD, prHead: HEAD, free: { state: 'missing' }, quietLeftMs: 0 }).kind).toBe('wait'); + expect(readinessDecision({ headSha: HEAD, prHead: NEWER, free: green, quietLeftMs: 0 })).toEqual({ kind: 'superseded', by: NEWER }); + const blocked = readinessDecision({ headSha: HEAD, prHead: HEAD, free: { state: 'failed', runId: 7, conclusion: 'failure' }, quietLeftMs: 0 }); + expect(blocked.kind).toBe('blocked'); + expect(blocked.kind === 'blocked' && blocked.reason).toContain('re-run Free Tests and then re-run this workflow'); + }); +}); + +describe('waitForReadiness', () => { + test('sleeps out the window with no API calls, then starts once Free Tests passes', () => { + const { gh, calls } = fakeGh([{ free: [free(5, 'in_progress', null)] }, { free: [free(5, 'completed', 'success')] }]); + const c = clock(CREATED + 5_000); + const { decision, lines } = waitForReadiness({ ...base, gh, now: c.now, sleep: c.sleep }); + expect(decision.kind).toBe('ready'); + // One run read, then nothing until the window closed at created + 10 min. + expect(calls[0]).toBe('repos/garrytan/gstack/actions/runs/900'); + expect(c.sleeps[0]).toBe(10 * 60_000 - 5_000); + expect(calls.filter(call => call.endsWith('free-tests.yml/runs'))).toHaveLength(2); + expect(c.at()).toBe(CREATED + 11 * 60_000); + expect(lines.join('\n')).toContain('Free Tests run 5 passed'); + }); + + test('a re-run keeps the original creation time, so an elapsed window is not waited again', () => { + const { gh } = fakeGh([{ free: [free(5, 'completed', 'success')] }]); + const c = clock(CREATED + 3 * 3600_000); + expect(waitForReadiness({ ...base, gh, now: c.now, sleep: c.sleep }).decision.kind).toBe('ready'); + expect(c.sleeps).toEqual([]); + }); + + test('a quiet window of 0 (the evals-no-debounce label) still waits for Free Tests', () => { + const { gh } = fakeGh([{ free: [] }, { free: [free(5, 'queued', null)] }, { free: [free(5, 'completed', 'success')] }]); + const c = clock(CREATED); + expect(waitForReadiness({ ...base, quietMs: 0, gh, now: c.now, sleep: c.sleep }).decision.kind).toBe('ready'); + expect(c.sleeps).toEqual([60_000, 60_000]); + }); + + test('Free Tests red on this head blocks the paid gate', () => { + const { gh } = fakeGh([{ free: [free(5, 'completed', 'failure')] }]); + const c = clock(CREATED); + const { decision } = waitForReadiness({ ...base, gh, now: c.now, sleep: c.sleep }); + expect(decision.kind).toBe('blocked'); + }); + + test('a newer head is reported superseded instead of starting', () => { + const { gh } = fakeGh([{ free: [free(5, 'completed', 'success')], prHead: NEWER }]); + const c = clock(CREATED); + const { decision, lines } = waitForReadiness({ ...base, gh, now: c.now, sleep: c.sleep }); + expect(decision).toEqual({ kind: 'superseded', by: NEWER }); + expect(lines.join('\n')).toContain('this run plans no slices'); + }); + + test('a Free Tests run that never finishes, or a gh outage, fails at the deadline with the last reason', () => { + for (const polls of [[{ free: [free(5, 'in_progress', null)] }], ['error' as const]]) { + const { gh } = fakeGh(polls); + const c = clock(CREATED); + const { decision } = waitForReadiness({ ...base, gh, now: c.now, sleep: c.sleep }); + expect(decision.kind).toBe('blocked'); + expect(decision.kind === 'blocked' && decision.reason).toContain('not ready after 80 min'); + expect(c.at() - CREATED).toBeLessThanOrEqual(80 * 60_000); + } + }); + + test('an unreadable run creation time measures the window from now', () => { + const { gh } = fakeGh([{ free: [free(5, 'completed', 'success')] }], null); + const c = clock(CREATED + 60_000); + expect(waitForReadiness({ ...base, gh, now: c.now, sleep: c.sleep }).decision.kind).toBe('ready'); + expect(c.sleeps[0]).toBe(10 * 60_000); + }); +}); diff --git a/test/evals-pr-lane-workflow.test.ts b/test/evals-pr-lane-workflow.test.ts index 4874df3fa..aee186627 100644 --- a/test/evals-pr-lane-workflow.test.ts +++ b/test/evals-pr-lane-workflow.test.ts @@ -1,7 +1,8 @@ /** * PR paid-lane workflow pins (W1, CEO-13/14/34, ENG-7/11/12, DX-7, W8f): - * coverage summaries, cancellation recovery off PR code, the push-burst - * debounce, dispatch inputs, and the absence of the old inline subset program. + * coverage summaries, cancellation recovery off PR code, readiness-gated + * dispatch (R1), restamp-only reuse wiring (R2a), dispatch inputs, and the + * absence of the old inline subset program. */ import { describe, expect, test } from 'bun:test'; import * as fs from 'node:fs'; @@ -77,24 +78,30 @@ describe('evals.yml receipt recovery (CEO-14/34, ENG-7)', () => { }); -describe('evals.yml push-burst debounce (CEO-15, ENG-11, DX-7)', () => { - const debounce = workflow.jobs.debounce!; +describe('evals.yml readiness-gated dispatch (R1, GSTA-48)', () => { + const readiness = workflow.jobs.readiness!; - test('a tiny job debounces only synchronize pushes, skippable by label, before recovery and planning', () => { - expect(debounce['runs-on']).toBe('ubuntu-24.04'); - expect(debounce.if).toContain("github.event.action == 'synchronize'"); - expect(debounce.if).toContain("!contains(github.event.pull_request.labels.*.name, 'evals-no-debounce')"); - expect(debounce.permissions).toEqual({ actions: 'read', 'pull-requests': 'read' }); - const wait = step('debounce', 'Wait out a push burst')!.run!; - expect(wait).toContain("date -u -d '15 minutes ago'"); - expect(wait).toContain('sleep 90'); - expect(wait).toContain('superseded=true'); - expect(wait).toContain('superseded by ${head}'); + test('a tiny PR-only job waits for a quiet PR and green Free Tests before recovery and planning', () => { + expect(readiness['runs-on']).toBe('ubuntu-24.04'); + expect(readiness.if).toBe("github.event_name == 'pull_request' && github.actor != 'dependabot[bot]' && github.event.pull_request.head.repo.full_name == github.repository"); + expect(readiness.permissions).toEqual({ contents: 'read', actions: 'read', 'pull-requests': 'read' }); + expect(Number(readiness['timeout-minutes'])).toBeGreaterThan(80); + const wait = step('readiness', 'Wait for a quiet PR and green Free Tests on this head')!; + expect(wait.id).toBe('wait'); + expect(wait.run).toContain('scripts/eval-readiness.ts wait'); + expect(wait.run).toContain('--deadline-minutes 80'); + expect(wait.run).toContain('--github-output "$GITHUB_OUTPUT"'); + expect(wait.env?.HEAD_SHA).toBe('${{ github.event.pull_request.head.sha }}'); + // The window is configurable and the old debounce label still skips it; nothing skips the Free Tests requirement. + expect(wait.env?.QUIET_MINUTES).toBe("${{ contains(github.event.pull_request.labels.*.name, 'evals-no-debounce') && '0' || vars.EVALS_QUIET_MINUTES || '10' }}"); + expect(readiness.outputs?.superseded).toBe('${{ steps.wait.outputs.superseded }}'); for (const job of ['recover-receipts', 'plan-slices']) { - expect(workflow.jobs[job]!.needs).toContain('debounce'); - expect(workflow.jobs[job]!.if).toContain("needs.debounce.outputs.superseded != 'true'"); + expect(workflow.jobs[job]!.needs).toContain('readiness'); + expect(workflow.jobs[job]!.if).toContain("(needs.readiness.result == 'success' || needs.readiness.result == 'skipped')"); + expect(workflow.jobs[job]!.if).toContain("needs.readiness.outputs.superseded != 'true'"); expect(workflow.jobs[job]!.if).toContain('!cancelled()'); } + expect(workflow.jobs.debounce).toBeUndefined(); }); test('cancel-in-progress stays keyed on the PR, and evals-fresh turns receipt reuse off', () => { @@ -104,6 +111,41 @@ describe('evals.yml push-burst debounce (CEO-15, ENG-11, DX-7)', () => { }); }); +describe('evals.yml restamp-only gate reuse wiring (R2a, GSTA-48)', () => { + test('base-ref recovery proposes a candidate; it never fails the run', () => { + const find = step('recover-receipts', 'Find a restamp-only gate to reuse (R2a, base-ref code)')!; + expect(find.if).toBe("github.event_name == 'pull_request'"); + expect(find.env).toMatchObject({ GH_TOKEN: '${{ github.token }}', HEAD_SHA: '${{ github.event.pull_request.head.sha }}', BASE_SHA: '${{ github.event.pull_request.base.sha }}' }); + expect(find.run).toContain('scripts/gate-reuse.ts find'); + expect(find.run).toContain('exit 0'); + expect(find.run).toContain('|| echo'); + expect(step('recover-receipts', 'Hand a restamp-only reuse candidate to plan-slices')?.with).toMatchObject({ name: 'gate-reuse-candidate', path: '/tmp/gate-reuse' }); + }); + + test('the planner confirms the selection after its own plan, and a confirmed reuse runs no slice', () => { + const planner = workflow.jobs['plan-slices']!; + const names = planner.steps.map(s => s.name); + expect(names.indexOf('Confirm restamp-only gate reuse (R2a)')).toBeGreaterThan(names.indexOf('Emit run manifest')); + expect(step('plan-slices', 'Confirm restamp-only gate reuse (R2a)')!.run).toContain('gate-reuse.ts confirm /tmp/gate-reuse /tmp/paid-plan/manifest.json --out /tmp/paid-plan'); + expect(planner.outputs?.gate_reuse).toBe('${{ steps.gate_reuse.outputs.gate_reuse }}'); + expect(workflow.jobs['eval-slices']!.if).toContain("needs.plan-slices.outputs.gate_reuse != 'true'"); + }); + + test('the report prints a reused verdict fail-closed and only executed runs bind a verdict', () => { + const report = workflow.jobs['slices-report']!; + expect(step('slices-report', 'Reconcile slices against the manifest (fail-closed)')!.if).toBe("needs.plan-slices.outputs.gate_reuse != 'true'"); + const reused = step('slices-report', 'Report the reused restamp-only verdict (R2a)')!; + expect(reused.if).toBe("needs.plan-slices.outputs.gate_reuse == 'true'"); + expect(reused.run).toContain('echo "exit=$code"'); + expect(report.outputs?.['reconcile-exit']).toBe('${{ steps.reconcile.outputs.exit || steps.reused.outputs.exit }}'); + expect(step('slices-report', 'Fail the workflow when reconciliation failed')!.if).toBe("(steps.reconcile.outputs.exit || steps.reused.outputs.exit) != '0'"); + const bind = step('slices-report', "Bind this run's verdict for restamp-only reuse (R2a)")!; + expect(bind.if).toBe("always() && github.event_name == 'pull_request' && needs.plan-slices.outputs.gate_reuse != 'true'"); + expect(bind.env?.RECONCILE_EXIT).toBe('${{ steps.reconcile.outputs.exit }}'); + expect(step('slices-report', 'Upload the gate binding')?.with).toMatchObject({ name: 'gate-binding-a${{ github.run_attempt }}', path: '/tmp/gate-binding/gate-binding.json' }); + }); +}); + describe('evals.yml carries no inline planner program (W8f)', () => { test('the finished validation_phase dispatch and its inline subset planner are gone', () => { expect(source).not.toContain('validation_phase'); diff --git a/test/gate-reuse.test.ts b/test/gate-reuse.test.ts new file mode 100644 index 000000000..683b9517f --- /dev/null +++ b/test/gate-reuse.test.ts @@ -0,0 +1,309 @@ +/** + * Restamp-only gate reuse (R2a, GSTA-48): a gate edit, so every refusal is a + * negative control. Real git fixtures (a bare origin, the base-ref receipt + * checkout) and scripted gh responses: no network, no API spend. + */ +import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; +import { spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { + BINDING_FILE, CANDIDATE_FILE, REUSE_FILE, buildBinding, confirmReuse, findReusableGate, parseReuse, reuseReport, selectionFingerprint, + type GateBinding, type GateReuse, +} from '../scripts/gate-reuse'; +import type { GhRunner, WorkflowRun } from '../scripts/recover-receipts'; +import type { PaidRunManifest } from '../scripts/lib/paid-plan'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const REPO = 'garrytan/gstack'; +const scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'gate-reuse-')); +afterAll(() => fs.rmSync(scratch, { recursive: true, force: true })); + +function git(cwd: string, ...args: string[]): string { + const r = spawnSync('git', ['-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.com', '-c', 'commit.gpgsign=false', ...args], + { cwd, encoding: 'utf8', timeout: 30_000 }); + if (r.status !== 0) throw new Error('git ' + args.join(' ') + ': ' + r.stderr); + return r.stdout.trim(); +} + +const changelog = (sections: string[]) => ['# Changelog', '', ...sections, '## [1.0.0.0] - 2026-10-01', '', '- base release', ''].join('\n'); +const files = (version: string, over: Record = {}): Record => ({ + VERSION: version + '\n', + 'package.json': JSON.stringify({ name: 'fixture', version, scripts: { test: 'bun test' } }, null, 2) + '\n', + 'agents-digest/gstack-AGENTS.md': '# gstack digest v' + version + ' — regenerate after upgrading\n\nbody\n', + 'src/feature.ts': 'export const feature = 1;\n', + ...over, +}); + +let origin: string; +let receipt: string; +const sha: Record = {}; + +function commit(work: string, branch: string, from: string, content: Record, message: string): string { + if (from) git(work, 'checkout', '-q', '-B', branch, from); + else git(work, 'checkout', '-q', '--orphan', branch); + for (const [rel, text] of Object.entries(content)) { + fs.mkdirSync(path.dirname(path.join(work, rel)), { recursive: true }); + fs.writeFileSync(path.join(work, rel), text); + } + git(work, 'add', '-A'); + git(work, 'commit', '-q', '-m', message); + git(work, 'push', '-q', 'origin', branch); + return git(work, 'rev-parse', 'HEAD'); +} + +beforeAll(() => { + origin = path.join(scratch, 'origin.git'); + git(scratch, 'init', '-q', '--bare', origin); + git(origin, 'config', 'uploadpack.allowAnySHA1InWant', 'true'); + const work = path.join(scratch, 'work'); + git(scratch, 'clone', '-q', origin, work); + const policy = fs.readFileSync(path.join(ROOT, '.gstack/ship-policy.json'), 'utf8'); + sha.base = commit(work, 'main', '', { ...files('1.0.0.0', { 'src/feature.ts': 'export const feature = 0;\n' }), 'CHANGELOG.md': changelog([]), + '.gstack/ship-policy.json': policy }, 'base'); + const entry = ['## [1.0.1.0] - 2026-10-02', '', '- the feature', '']; + sha.gated = commit(work, 'feature', sha.base, { ...files('1.0.1.0'), 'CHANGELOG.md': changelog(entry) }, 'feature, tentative stamp'); + sha.restamp = commit(work, 'restamp', sha.gated, { ...files('1.0.2.0'), 'CHANGELOG.md': changelog(['## [1.0.2.0] - 2026-10-03', '', '- the feature', '']) }, 'restamp'); + sha.code = commit(work, 'code', sha.restamp, { 'src/feature.ts': 'export const feature = 2;\n' }, 'restamp plus a code byte'); + sha.entry = commit(work, 'entry', sha.gated, { ...files('1.0.2.0'), 'CHANGELOG.md': changelog(['## [1.0.2.0] - 2026-10-03', '', '- the feature, reworded', '']) }, 'restamp plus entry text'); + sha.lock = commit(work, 'lock', sha.gated, { 'bun.lock': '{\n "lockfileVersion": 1\n}\n' }, 'a non-stamp file beside the stamp'); + const lock = (version: string) => '{\n "lockfileVersion": 1,\n "workspaces": {\n "": {\n "name": "fixture",\n "version": "' + version + '",\n },\n },\n}\n'; + sha.gatedLock = commit(work, 'gated-lock', sha.gated, { 'bun.lock': lock('1.0.1.0') }, 'a lockfile carrying the root version'); + sha.lockStamp = commit(work, 'lock-stamp', sha.gatedLock, { ...files('1.0.2.0'), 'CHANGELOG.md': changelog(['## [1.0.2.0] - 2026-10-03', '', '- the feature', '']), + 'bun.lock': lock('1.0.2.0') }, 'restamp that also moves the lockfile root version'); + sha.moved = commit(work, 'main', sha.base, { 'README.md': 'main moved\n' }, 'main moves'); + receipt = path.join(scratch, 'receipt'); + git(scratch, 'clone', '-q', '--depth=1', 'file://' + origin, receipt); + git(receipt, 'fetch', '-q', '--depth=1', 'origin', sha.base); + git(receipt, 'checkout', '-q', '--detach', sha.base); +}); + +const binding = (over: Partial = {}): GateBinding => ({ schema: 1, repository: REPO, pr: 42, run_id: 300, run_attempt: 1, + head_sha: sha.gated!, base_sha: sha.base!, profile: 'pr', reconcile_exit: 0, verdict: 'GREEN', selection_sha256: 'f'.repeat(64), ...over }); +const run = (id: number, head: string, conclusion: string | null = 'success', over: Partial = {}): WorkflowRun => ({ + id, status: 'completed', conclusion, event: 'pull_request', head_sha: head, run_attempt: 1, head_repository: { full_name: REPO }, + pull_requests: [{ number: 42 }], ...over }); + +/** gh: the run list, each run's artifact list, and downloads that write the scripted binding (or raw text). */ +function fakeGh(runs: WorkflowRun[], bindings: Record, failList = ''): GhRunner { + return (args) => { + const ok = (body: unknown) => ({ status: 0, stdout: JSON.stringify(body), stderr: '' }); + if (args.some(a => a.endsWith('/workflows/evals.yml/runs'))) return failList ? { status: 1, stdout: '', stderr: failList } : ok({ workflow_runs: runs }); + const listed = args.find(a => /\/runs\/\d+\/artifacts$/.test(a)); + if (listed) { + const id = Number(/runs\/(\d+)\//.exec(listed)![1]); + return ok({ artifacts: id in bindings ? [{ name: 'gate-binding-a1', expired: false }, { name: 'paid-plan', expired: false }] : [{ name: 'paid-plan', expired: false }] }); + } + if (args[0] === 'run' && args[1] === 'download') { + const dir = args[args.indexOf('-D') + 1]!; + fs.mkdirSync(dir, { recursive: true }); + const b = bindings[Number(args[2])]!; + fs.writeFileSync(path.join(dir, BINDING_FILE), typeof b === 'string' ? b : JSON.stringify(b)); + return { status: 0, stdout: '', stderr: '' }; + } + return { status: 1, stdout: '', stderr: 'unexpected gh ' + args.join(' ') }; + }; +} + +let outCount = 0; +function find(head: string, runs: WorkflowRun[], bindings: Record, over: { baseSha?: string; failList?: string } = {}) { + const out = path.join(scratch, 'out-' + ++outCount); + const result = findReusableGate({ repo: REPO, pr: 42, headRepo: REPO, branch: 'feature', selfRunId: 500, headSha: head, + baseSha: over.baseSha ?? sha.base!, repoRoot: receipt, out, gh: fakeGh(runs, bindings, over.failList) }); + return { ...result, out, text: result.lines.join('\n') }; +} + +describe('find: the positive case', () => { + test('a restamp of a GREEN head on the same base yields a candidate with the receipt', () => { + const { candidate, out, text } = find(sha.restamp!, [run(300, sha.gated!)], { 300: binding() }); + expect(text).toContain('TREE: same-modulo-stamps'); + expect(candidate).not.toBeNull(); + expect(candidate!.stamp_files.sort()).toEqual(['CHANGELOG.md', 'VERSION', 'agents-digest/gstack-AGENTS.md', 'package.json']); + expect(candidate!).toMatchObject({ reused_run_id: 300, gated_head: sha.gated, current_head: sha.restamp, base_sha: sha.base, + old_version: '1.0.1.0', new_version: '1.0.2.0', selection_sha256: 'f'.repeat(64) }); + expect(candidate!.receipt_sha256).toMatch(/^[0-9a-f]{64}$/); + expect(parseReuse(JSON.parse(fs.readFileSync(path.join(out, CANDIDATE_FILE), 'utf8')))).toEqual(candidate); + }); + + test('cancelled or skipped runs that bound nothing are passed over to the newest executed gate', () => { + const { candidate } = find(sha.restamp!, [run(320, sha.gated!, 'cancelled'), run(310, sha.gated!, 'success'), run(300, sha.gated!)], { 300: binding() }); + expect(candidate?.reused_run_id).toBe(300); + }); +}); + +describe('find: negative controls (each forces the full gate)', () => { + const refused = (r: { candidate: GateReuse | null; out: string; text: string }, reason: string | RegExp) => { + expect(r.candidate).toBeNull(); + expect(fs.existsSync(path.join(r.out, CANDIDATE_FILE))).toBe(false); + if (typeof reason === 'string') expect(r.text).toContain(reason); + else expect(r.text).toMatch(reason); + expect(r.text).toContain('the full gate runs'); + }; + + test('a non-stamp byte change', () => { + refused(find(sha.code!, [run(300, sha.gated!)], { 300: binding() }), 'differs from the gated head beyond stamps: src/feature.ts'); + }); + + test('CHANGELOG entry text, not only its heading', () => { + refused(find(sha.entry!, [run(300, sha.gated!)], { 300: binding() }), 'beyond stamps: CHANGELOG.md'); + }); + + test('a file outside the stamp set', () => { + refused(find(sha.lock!, [run(300, sha.gated!)], { 300: binding() }), 'bun.lock'); + // The receipt calls a lockfile root-version bump a stamp; R2a's stamp set is narrower and still refuses it. + refused(find(sha.lockStamp!, [run(300, sha.gatedLock!)], { 300: binding({ head_sha: sha.gatedLock }) }), + 'stamp-classified files outside the version file, package.json, CHANGELOG and release outputs: bun.lock'); + }); + + test('a different base: main moved between the gate and this run', () => { + refused(find(sha.restamp!, [run(300, sha.gated!)], { 300: binding() }, { baseSha: sha.moved }), 'main moved'); + }); + + test('a missing receipt: no executed gate bound a verdict', () => { + refused(find(sha.restamp!, [run(300, sha.gated!)], {}), 'no earlier executed gate on this PR bound a verdict'); + refused(find(sha.restamp!, [], {}), 'no earlier executed gate'); + }); + + test('a missing receipt: a malformed or mismatched binding', () => { + refused(find(sha.restamp!, [run(300, sha.gated!)], { 300: '{"schema":1' }), 'missing or malformed'); + refused(find(sha.restamp!, [run(300, sha.gated!)], { 300: binding({ run_id: 299 }) }), 'does not match the run it came from'); + refused(find(sha.restamp!, [run(300, sha.restamp!)], { 300: binding() }), 'does not match the run it came from'); + }); + + test('a missing receipt: the gated head cannot be fetched, so no tree receipt exists', () => { + const ghost = 'e'.repeat(40); + refused(find(sha.restamp!, [run(300, ghost)], { 300: binding({ head_sha: ghost }) }), /cannot fetch|no tree receipt/); + }); + + test('a red, partial or failed gate, and a newer red that blocks an older green', () => { + refused(find(sha.restamp!, [run(300, sha.gated!)], { 300: binding({ verdict: 'RED' }) }), 'is RED'); + refused(find(sha.restamp!, [run(300, sha.gated!)], { 300: binding({ reconcile_exit: 1 }) }), 'reconcile exit 1'); + refused(find(sha.restamp!, [run(300, sha.gated!, 'failure')], { 300: binding() }), 'conclusion failure'); + refused(find(sha.restamp!, [run(301, sha.gated!), run(300, sha.gated!)], { 301: binding({ run_id: 301, verdict: 'RED' }), 300: binding() }), 'run 301'); + }); + + test('a full-profile binding, the same head, another PR, and a gh outage', () => { + refused(find(sha.restamp!, [run(300, sha.gated!)], { 300: binding({ profile: 'full' }) }), 'not the PR gate'); + refused(find(sha.gated!, [run(300, sha.gated!)], { 300: binding() }), 'gated this same head'); + refused(find(sha.restamp!, [run(300, sha.gated!, 'success', { pull_requests: [{ number: 7 }] })], { 300: binding() }), 'no earlier executed gate'); + refused(find(sha.restamp!, [run(300, sha.gated!)], { 300: binding() }, { failList: 'HTTP 403: Resource not accessible' }), 'not authorized'); + }); +}); + +const manifest = (over: Partial = {}): PaidRunManifest => ({ version: 1, tier: 'gate', evalsAll: false, sliceCount: 2, + selectionReason: 'diff', profile: 'pr', selection: { e2e: ['case-a'], judges: [] }, prCoverage: { mode: 'pr' } as PaidRunManifest['prCoverage'], + entries: [ + { file: 'test/a.test.ts', slice: 1, status: 'planned', estimatedMs: 1000 }, + { file: 'test/b.test.ts#case-b~t1', slice: 2, status: 'planned', trial: { kind: 'behavior', panel: { n: 3, k: 2 }, quarantined: false } }, + { file: 'test/c.test.ts', slice: 0, status: 'skipped-by-diff', reason: 'no input changed' }, + ], ...over } as PaidRunManifest); + +describe('selection fingerprint and confirm', () => { + test('slice packing and estimates do not change the fingerprint; cases, statuses and trial plans do', () => { + const fp = selectionFingerprint(manifest()); + const repacked = manifest(); + repacked.entries = [...repacked.entries].reverse().map(e => ({ ...e, slice: e.slice ? 3 - e.slice : 0, estimatedMs: 5, reason: 'other' })); + expect(selectionFingerprint(repacked)).toBe(fp); + const extra = manifest(); + extra.entries = [...extra.entries, { file: 'test/d.test.ts', slice: 1, status: 'planned' }]; + const quarantined = manifest(); + quarantined.entries = quarantined.entries.map(e => e.trial ? { ...e, trial: { ...e.trial, quarantined: true } } : e); + const skipped = manifest(); + skipped.entries = skipped.entries.map(e => e.file === 'test/a.test.ts' ? { ...e, status: 'skipped-by-diff' as const } : e); + for (const changed of [extra, quarantined, skipped, manifest({ selection: { e2e: ['case-a', 'case-z'], judges: [] } }), manifest({ evalsAll: true })]) { + expect(selectionFingerprint(changed)).not.toBe(fp); + } + }); + + test('confirm reuses only when the candidate exists and the selection is identical', () => { + const { out } = find(sha.restamp!, [run(300, sha.gated!)], { 300: binding({ selection_sha256: selectionFingerprint(manifest()) }) }); + const same = confirmReuse(out, manifest()); + expect(same.reuse?.reused_run_id).toBe(300); + expect(same.reason).toContain('same selection'); + const extra = manifest(); + extra.entries = [...extra.entries, { file: 'test/d.test.ts', slice: 1, status: 'planned' }]; + expect(confirmReuse(out, extra).reuse).toBeNull(); + expect(confirmReuse(out, extra).reason).toContain('differs from the reused gate'); + expect(confirmReuse(out, null).reuse).toBeNull(); + expect(confirmReuse(path.join(scratch, 'no-candidate'), manifest()).reuse).toBeNull(); + const forged = path.join(scratch, 'forged'); + fs.mkdirSync(forged, { recursive: true }); + fs.writeFileSync(path.join(forged, CANDIDATE_FILE), JSON.stringify({ ...JSON.parse(fs.readFileSync(path.join(out, CANDIDATE_FILE), 'utf8')), stamp_files: [] })); + expect(confirmReuse(forged, manifest()).reuse).toBeNull(); + }); +}); + +describe('bind and report', () => { + const reportDir = (name: string, withReuse = false) => { + const dir = path.join(scratch, name); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, 'manifest.json'), JSON.stringify(manifest())); + fs.writeFileSync(path.join(dir, 'collector-outcomes.json'), JSON.stringify({ verdict: { verdict: 'GREEN' } })); + if (withReuse) fs.writeFileSync(path.join(dir, REUSE_FILE), '{}'); + return dir; + }; + const bindArgs = { repository: REPO, pr: 42, runId: 300, runAttempt: 1, headSha: 'a'.repeat(40), baseSha: 'b'.repeat(40), profile: 'pr', reconcileExit: 0 }; + + test('an executed run binds its verdict, reconcile exit and selection; a reused run binds nothing', () => { + const { binding: bound } = buildBinding({ ...bindArgs, reportDir: reportDir('bind-ok') }); + expect(bound).toMatchObject({ verdict: 'GREEN', reconcile_exit: 0, selection_sha256: selectionFingerprint(manifest()) }); + expect(buildBinding({ ...bindArgs, reportDir: reportDir('bind-reused', true) }).binding).toBeNull(); + const empty = path.join(scratch, 'bind-empty'); + fs.mkdirSync(empty, { recursive: true }); + expect(buildBinding({ ...bindArgs, reportDir: empty }).binding).toBeNull(); + }); + + test('the CLI binds a missing reconcile exit as -1, never as a pass', () => { + const out = path.join(scratch, 'bind-cli'); + const r = spawnSync(process.execPath, ['run', path.join(ROOT, 'scripts/gate-reuse.ts'), 'bind', reportDir('bind-cli-report'), '--out', out, + '--repo', REPO, '--pr', '42', '--run-id', '300', '--run-attempt', '1', '--head-sha', 'a'.repeat(40), '--base-sha', 'b'.repeat(40), + '--profile', 'pr', '--reconcile-exit', ''], { encoding: 'utf8', timeout: 30_000 }); + expect(r.status, r.stderr).toBe(0); + expect(JSON.parse(fs.readFileSync(path.join(out, BINDING_FILE), 'utf8')).reconcile_exit).toBe(-1); + }); + + test('report prints the reused verdict with its receipt, and fails closed without a valid record', () => { + const { out } = find(sha.restamp!, [run(300, sha.gated!)], { 300: binding() }); + const reuse = parseReuse(JSON.parse(fs.readFileSync(path.join(out, CANDIDATE_FILE), 'utf8')))!; + const text = reuseReport(reuse, REPO); + expect(text).toContain('GATE REUSED (R2a, restamp-only)'); + expect(text).toContain('TREE: same-modulo-stamps receipt=' + reuse.receipt_sha256); + expect(text).toContain('https://github.com/garrytan/gstack/actions/runs/300'); + const bad = spawnSync(process.execPath, ['run', path.join(ROOT, 'scripts/gate-reuse.ts'), 'report', path.join(scratch, 'missing.json'), '--repo', REPO], + { encoding: 'utf8', timeout: 30_000 }); + expect(bad.status).toBe(1); + }); +}); + +describe.skipIf(!Bun.which('jq') || !Bun.which('bash'))('the PR comment for a reused verdict', () => { + test('runs no repository code and names the reused run and receipt; without reconcile exit 0 it never says PASS (reused)', () => { + const workflow = Bun.YAML.parse(fs.readFileSync(path.join(ROOT, '.github/workflows/evals.yml'), 'utf8')) as any; + const script = (workflow.jobs['slices-comment'].steps.find((s: any) => s.name === 'Post PR comment').run as string); + const root = path.join(scratch, 'comment'); + const paidDir = path.join(root, 'paid-report'); + const binDir = path.join(root, 'bin'); + fs.mkdirSync(paidDir, { recursive: true }); + fs.mkdirSync(binDir, { recursive: true }); + fs.writeFileSync(path.join(binDir, 'gh'), '#!/bin/sh\ncase "$*" in *--jq*) exit 0;; esac\nfor arg do case "$arg" in body=*) printf "%s\\n" "${arg#body=}";; esac; done\n'); + fs.chmodSync(path.join(binDir, 'gh'), 0o755); + fs.writeFileSync(path.join(binDir, 'bc'), '#!/bin/sh\nread -r expression\nprintf "0\\n"\n'); + fs.chmodSync(path.join(binDir, 'bc'), 0o755); + const { out } = find(sha.restamp!, [run(300, sha.gated!)], { 300: binding() }); + fs.copyFileSync(path.join(out, CANDIDATE_FILE), path.join(paidDir, REUSE_FILE)); + fs.writeFileSync(path.join(paidDir, 'manifest.json'), JSON.stringify({ profile: 'pr', selection: { e2e: [], judges: [] } })); + const body = script.replaceAll('/tmp/paid-report', paidDir).replaceAll('/tmp/verdict', path.join(root, 'verdict')) + .replaceAll('$' + '{{ github.repository }}', REPO).replaceAll('$' + '{{ github.event.pull_request.number }}', '42'); + const exec = (reconcile: string) => spawnSync('bash', ['-e', '-c', body], { cwd: root, encoding: 'utf8', timeout: 10_000, + env: { ...process.env, PATH: binDir + ':' + process.env.PATH, RECONCILE_EXIT: reconcile, GITHUB_REPOSITORY: REPO } }); + const reused = exec('0'); + expect(reused.status, reused.stderr).toBe(0); + expect(reused.stdout).toContain('## E2E Evals: ✅ PASS (reused: restamp-only, R2a)'); + expect(reused.stdout).toContain('TREE: same-modulo-stamps receipt='); + expect(reused.stdout).toContain('actions/runs/300'); + const failed = exec('1'); + expect(failed.status, failed.stderr).toBe(0); + expect(failed.stdout).not.toContain('PASS (reused'); + expect(failed.stdout).toContain('❌ FAIL'); + }); +});