v1.91.134.0 feat(evals): readiness-gated paid dispatch (R1), verdict reuse (R2a/R2), receipt store that restores (#3262)

* feat(evals): readiness-gated paid dispatch (R1), restamp-only gate reuse (R2a), and a receipt store that restores

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* feat(evals): R2 verdict reuse across main moves; review fixes (newer-run classification, receipt parser, manual acceptance)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(evals): R2 own-diff signature covers release-output content and CHANGELOG deletions and preamble

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(evals): R2 normalises release outputs by stamp pattern only; compares stamp-file modes; unreadable blobs refuse

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* chore: stamp v1.91.134.0

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(evals): autoplan-guard-pty actor continues through the phase boundary; name PHASE_BOUNDARY_STOP

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

---------

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
Garry Tanandcapy-ai[bot] authored and GitHub committed 2026-10-18 06:03:18 +00:00
1 parent fd13cf25e8
commit a2a6cabfa6
21 files changed
+1758 -84

No files matched your search

+144 -38
View File
@@ -120,47 +120,46 @@ jobs:
docker manifest inspect "$EVAL_IMAGE" > /tmp/eval-runtime-manifest.json
echo "id=$(sha256sum /tmp/eval-runtime-manifest.json | cut -d ' ' -f1)" >> "$GITHUB_OUTPUT"
# ── Push-burst debounce (W1d: CEO-15, ENG-11, DX-7) ──────────────────────
# Only on `synchronize` when another evals.yml run for this PR started in
# the last 15 minutes (a push burst): wait 90 s before planning, so the next
# push's cancel-in-progress lands before any paid slice starts. A first push
# pays nothing. cancel-in-progress does the superseding; the head-SHA check
# only reports "superseded by <sha>" when this run outlives the wait. The PR
# label evals-no-debounce skips the wait.
debounce:
# ── Readiness-gated dispatch (R1, GSTA-48) ───────────────────────────────
# Paid slices for a PR head start only after the PR has been quiet for the
# window (vars.EVALS_QUIET_MINUTES, default 10; the evals-no-debounce label
# sets it to 0) and Free Tests has passed on the same head SHA. A newer push
# still cancels this run (cancel-in-progress), so the window is a sleep; the
# same heads get the same selection and verdict rules, only the start moves.
# Free Tests red on this head fails the run without paid work (re-run this
# workflow after a green Free Tests rerun). Dispatches never wait: they are
# explicit (eval:bg, census). Version Gate is not an input, so a stamp push
# waits for its own Free Tests only and always gets its gate.
readiness:
runs-on: ubuntu-24.04
if: github.event_name == 'pull_request' && github.event.action == 'synchronize' && github.actor != 'dependabot[bot]' && github.event.pull_request.head.repo.full_name == github.repository && !contains(github.event.pull_request.labels.*.name, 'evals-no-debounce')
timeout-minutes: 5
if: github.event_name == 'pull_request' && github.actor != 'dependabot[bot]' && github.event.pull_request.head.repo.full_name == github.repository
timeout-minutes: 90
permissions:
contents: read
actions: read
pull-requests: read
outputs:
superseded: ${{ steps.wait.outputs.superseded }}
steps:
- name: Wait out a push burst
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: 1.4.2
- name: Wait for a quiet PR and green Free Tests on this head
id: wait
env:
GH_TOKEN: ${{ github.token }}
PR_NUMBER: ${{ github.event.pull_request.number }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
HEAD_BRANCH: ${{ github.event.pull_request.head.ref }}
QUIET_MINUTES: ${{ contains(github.event.pull_request.labels.*.name, 'evals-no-debounce') && '0' || vars.EVALS_QUIET_MINUTES || '10' }}
run: |
since=$(date -u -d '15 minutes ago' +%s)
recent=$(gh api -X GET "repos/$GITHUB_REPOSITORY/actions/workflows/evals.yml/runs" -f event=pull_request \
-f branch="$HEAD_BRANCH" -f per_page=20 \
--jq "[.workflow_runs[] | select(.id < $GITHUB_RUN_ID and any(.pull_requests[]; .number == $PR_NUMBER) and ((.created_at | fromdateiso8601) >= $since))] | length" \
2>/dev/null || echo 0)
if [ "${recent:-0}" -eq 0 ]; then
echo "- Debounce: no other run for this PR in the last 15 min; no wait" >> "$GITHUB_STEP_SUMMARY"
exit 0
fi
echo "- Debounce: ${recent} run(s) for this PR started in the last 15 min; waiting 90 s (label evals-no-debounce skips this)" >> "$GITHUB_STEP_SUMMARY"
sleep 90
head=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER" --jq .head.sha 2>/dev/null || echo "$HEAD_SHA")
if [ "$head" != "$HEAD_SHA" ]; then
echo "superseded=true" >> "$GITHUB_OUTPUT"
echo "- Debounce: superseded by ${head}; this run plans no slices" >> "$GITHUB_STEP_SUMMARY"
fi
bun --no-install run scripts/eval-readiness.ts wait --repo "$GITHUB_REPOSITORY" --pr "$PR_NUMBER" --head-sha "$HEAD_SHA" \
--run-id "$GITHUB_RUN_ID" --quiet-minutes "$QUIET_MINUTES" --deadline-minutes 80 \
--github-output "$GITHUB_OUTPUT" --summary "$GITHUB_STEP_SUMMARY"
# ── Receipt recovery (W1c: CEO-14/34, ENG-4/5/7) ─────────────────────────
# A run cancelled by the next push never reaches slices-report, so its
@@ -175,10 +174,10 @@ jobs:
# plan-slices turns reuse off, so cases execute unreduced.
recover-receipts:
runs-on: ubuntu-24.04
needs: [debounce]
# Runs after the debounce settles, so a superseded run's cancellation has usually completed.
needs: [readiness]
# Runs after readiness settles, so a superseded run's cancellation has usually completed; a head whose Free Tests failed recovers nothing.
# A dispatch naming pr_receipts reads that PR's receipts from its runs' artifacts (cache entries are ref-scoped).
if: ${{ !cancelled() && needs.debounce.outputs.superseded != 'true' && github.actor != 'dependabot[bot]' && ((github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && !contains(github.event.pull_request.labels.*.name, 'evals-fresh')) || (github.event_name == 'workflow_dispatch' && inputs.pr_receipts != '')) }}
if: ${{ !cancelled() && (needs.readiness.result == 'success' || needs.readiness.result == 'skipped') && needs.readiness.outputs.superseded != 'true' && github.actor != 'dependabot[bot]' && ((github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && !contains(github.event.pull_request.labels.*.name, 'evals-fresh')) || (github.event_name == 'workflow_dispatch' && inputs.pr_receipts != '')) }}
timeout-minutes: 5
permissions:
contents: read
@@ -221,6 +220,42 @@ jobs:
--store /tmp/gstack-eval-input-cache --store-key "$STORE_KEY" --mode "$RECOVERY_MODE" --budget-seconds 60 >> "$GITHUB_STEP_SUMMARY" \
|| echo "- Receipt recovery crashed; reuse stays off this run" >> "$GITHUB_STEP_SUMMARY"
# Verdict reuse (GSTA-48), a gate edit: base-ref code proposes reusing
# the newest executed GREEN gate of this PR when this head differs from
# it only by stamp files on the same base (R2a: TREE same-modulo-stamps
# from the tree receipt under the base's ship policy), or when main
# moved and the PR's own diff from its merge-base, stamp files excluded,
# is byte-identical (R2). plan-slices confirms the selection is
# identical. Anything else (a red, partial or cancelled gate, a changed
# own diff, a non-stamp byte, no binding or receipt, a gh or git error,
# evals-fresh) leaves no candidate: the full gate runs.
- name: Find a gate verdict to reuse (R2a/R2, base-ref code)
if: github.event_name == 'pull_request'
env:
GH_TOKEN: ${{ github.token }}
PR_NUMBER: ${{ github.event.pull_request.number }}
HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
HEAD_BRANCH: ${{ github.event.pull_request.head.ref }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
if [ ! -f scripts/gate-reuse.ts ]; then
echo "- Gate verdict reuse unavailable: the base ref has no scripts/gate-reuse.ts; the full gate runs" >> "$GITHUB_STEP_SUMMARY"
exit 0
fi
bun --no-install run scripts/gate-reuse.ts find --repo "$GITHUB_REPOSITORY" --pr "$PR_NUMBER" --head-repo "$HEAD_REPO" \
--branch "$HEAD_BRANCH" --run-id "$GITHUB_RUN_ID" --head-sha "$HEAD_SHA" --tested-sha "$GITHUB_SHA" \
--out /tmp/gate-reuse --budget-seconds 90 >> "$GITHUB_STEP_SUMMARY" \
|| echo "- Gate verdict reuse crashed; the full gate runs" >> "$GITHUB_STEP_SUMMARY"
- name: Hand a gate reuse candidate to plan-slices
if: github.event_name == 'pull_request'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: gate-reuse-candidate
path: /tmp/gate-reuse
if-no-files-found: ignore
retention-days: 7
- name: Hand the receipt store to plan-slices
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
@@ -248,15 +283,16 @@ jobs:
# branch (see CLAUDE.md's garrytan-agents workflow).
plan-slices:
runs-on: ubicloud-standard-8
needs: [debounce, recover-receipts]
# !cancelled(): plans even when the debounce or recovery was skipped (first push, dispatch, evals-fresh) or recovery failed (reuse is then off); a superseded run plans nothing.
if: ${{ !cancelled() && needs.debounce.outputs.superseded != 'true' && github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }}
needs: [readiness, recover-receipts]
# !cancelled(): plans even when readiness or recovery was skipped (dispatch, evals-fresh) or recovery failed (reuse is then off); a superseded run, or a head whose readiness failed (Free Tests red), plans nothing.
if: ${{ !cancelled() && (needs.readiness.result == 'success' || needs.readiness.result == 'skipped') && needs.readiness.outputs.superseded != 'true' && github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }}
timeout-minutes: 10
permissions:
contents: read
outputs:
slices: ${{ steps.matrix.outputs.slices }}
slice_timeouts: ${{ steps.matrix.outputs.slice_timeouts }}
gate_reuse: ${{ steps.gate_reuse.outputs.gate_reuse }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
@@ -311,6 +347,24 @@ jobs:
EVALS_BASE: ${{ inputs.base_sha || (inputs.base_ref != '' && format('origin/{0}', inputs.base_ref)) || '' }}
run: EVALS_TIER=gate bun --no-install run scripts/test-paid-shards.ts --tier gate --emit-plan /tmp/paid-plan/manifest.json --slice-budget 420 --jobs 2 --max-parallel 20
# R2a/R2: reuse holds only when this head's own plan selects exactly the
# cases and trial plans the reused gate ran (selection fingerprint).
- name: Download a gate reuse candidate
if: github.event_name == 'pull_request'
continue-on-error: true
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
with:
name: gate-reuse-candidate
path: /tmp/gate-reuse
- name: Confirm gate verdict reuse (R2a/R2)
id: gate_reuse
if: github.event_name == 'pull_request'
run: |
mkdir -p /tmp/gate-reuse
bun --no-install run scripts/gate-reuse.ts confirm /tmp/gate-reuse /tmp/paid-plan/manifest.json --out /tmp/paid-plan \
--github-output "$GITHUB_OUTPUT" >> "$GITHUB_STEP_SUMMARY"
# CEO-13: the mode, case counts and, on fallback, every file that
# restored the full gate with the edit that narrows it next time.
- name: Summarize PR coverage
@@ -336,6 +390,7 @@ jobs:
/tmp/paid-plan/manifest.json
/tmp/paid-plan/receipts
/tmp/paid-plan/store
/tmp/paid-plan/gate-reuse.json
retention-days: 30
eval-slices:
@@ -346,7 +401,8 @@ jobs:
# !cancelled(), not always(): still runs when build-image was skipped
# (image already published), but a newer push's cancel-in-progress stops
# it instead of letting a superseded run finish its paid slices first.
if: ${{ !cancelled() && needs.build-image.result == 'success' && needs.plan-slices.result == 'success' }}
# A confirmed verdict reuse (R2a/R2) runs no paid slice.
if: ${{ !cancelled() && needs.build-image.result == 'success' && needs.plan-slices.result == 'success' && needs.plan-slices.outputs.gate_reuse != 'true' }}
# The planner packs ~7 minutes of recorded work per slice (EVALS_JOBS=2 x
# EVALS_CONCURRENCY=2 per runner, never the old 40-way per-row fan-out
# that queued claude session STARTUP behind 39 siblings). Each slice's job
@@ -500,7 +556,11 @@ jobs:
retention-days: 30
slices-report:
runs-on: ubicloud-standard-2
# GitHub-hosted, like recover-receipts: Ubicloud runners send actions/cache
# to Ubicloud's own transparent cache, which a GitHub-hosted restore never
# sees, so a store saved on Ubicloud was never restored (GSTA-48: every
# restore missed, e.g. #3253's restamp gate 3 minutes after a saved store).
runs-on: ubuntu-24.04
needs: [plan-slices, eval-slices]
# !cancelled(): the report must run (and FAIL) when an executor died — a
# missing slice artifact reading as green is the class this lane kills —
@@ -518,7 +578,7 @@ jobs:
permissions:
contents: read
outputs:
reconcile-exit: ${{ steps.reconcile.outputs.exit }}
reconcile-exit: ${{ steps.reconcile.outputs.exit || steps.reused.outputs.exit }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
@@ -543,6 +603,7 @@ jobs:
- name: Reconcile slices against the manifest (fail-closed)
id: reconcile
if: needs.plan-slices.outputs.gate_reuse != 'true'
run: |
set +e
EVALS_TIER=gate bun --no-install run scripts/test-paid-shards.ts --tier gate --report /tmp/paid-report | tee /tmp/report.txt
@@ -552,6 +613,19 @@ jobs:
# (caught by the ship review army; the wiring test now pins this).
echo "exit=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT"
# R2a/R2: the planner confirmed a reuse, so no slice ran; the verdict is
# the reused gate's, printed with which run, which head and why. A
# missing or malformed reuse record exits 1 (fail-closed).
- name: Report the reused gate verdict (R2a/R2)
id: reused
if: needs.plan-slices.outputs.gate_reuse == 'true'
run: |
set +e
bun --no-install run scripts/gate-reuse.ts report /tmp/paid-report/gate-reuse.json --repo "$GITHUB_REPOSITORY" > /tmp/report.txt
code=$?
cat /tmp/report.txt
echo "exit=$code" >> "$GITHUB_OUTPUT"
- name: Summarize PR coverage and reuse
if: always()
run: bun --no-install run scripts/test-pr-profile.ts summary /tmp/paid-report/manifest.json /tmp/paid-report/collector-outcomes.json >> "$GITHUB_STEP_SUMMARY"
@@ -594,6 +668,29 @@ jobs:
if-no-files-found: ignore
retention-days: 30
# R2a/R2: what this executed run proved, read by a later push. The base
# is the tested merge commit's first parent. Reused runs bind nothing,
# so reuse never chains off a reuse.
- name: Bind this run's verdict for reuse (R2a/R2)
if: always() && github.event_name == 'pull_request' && needs.plan-slices.outputs.gate_reuse != 'true'
env:
PR_NUMBER: ${{ github.event.pull_request.number }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
RECONCILE_EXIT: ${{ steps.reconcile.outputs.exit }}
run: |
bun --no-install run scripts/gate-reuse.ts bind /tmp/paid-report --out /tmp/gate-binding --repo "$GITHUB_REPOSITORY" \
--pr "$PR_NUMBER" --run-id "$GITHUB_RUN_ID" --run-attempt "$GITHUB_RUN_ATTEMPT" --head-sha "$HEAD_SHA" \
--tested-sha "$GITHUB_SHA" --profile "$EVALS_PROFILE" --reconcile-exit "$RECONCILE_EXIT" >> "$GITHUB_STEP_SUMMARY"
- name: Upload the gate binding
if: always() && github.event_name == 'pull_request' && needs.plan-slices.outputs.gate_reuse != 'true'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: gate-binding-a${{ github.run_attempt }}
path: /tmp/gate-binding/gate-binding.json
if-no-files-found: ignore
retention-days: 30
- name: Upload trial outcomes for pass-rate history
if: always()
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
@@ -604,7 +701,7 @@ jobs:
retention-days: 90
- name: Fail the workflow when reconciliation failed
if: steps.reconcile.outputs.exit != '0'
if: (steps.reconcile.outputs.exit || steps.reused.outputs.exit) != '0'
run: exit 1
# PR comment in its OWN job with the write token and ZERO repo code: no
@@ -644,6 +741,13 @@ jobs:
RECONCILE_EXIT: ${{ needs.slices-report.outputs.reconcile-exit }}
run: |
# shellcheck disable=SC2086,SC2059
# R2a/R2: a confirmed verdict reuse ran no case; say which run,
# which head and why, with the receipt or own-diff signature.
REUSE=/tmp/paid-report/gate-reuse.json
if [ "${RECONCILE_EXIT:-1}" = "0" ] && [ -f "$REUSE" ] \
&& jq -e '.schema == 1 and (.kind == "restamp-only" or .kind == "main-moved")' "$REUSE" >/dev/null 2>&1; then
BODY=$(jq -r --arg repo "$GITHUB_REPOSITORY" '"## E2E Evals: ✅ PASS (reused: \(if .kind == "restamp-only" then "restamp-only, R2a" else "main moved, R2" end))\n\nNo paid case ran on `\(.current_head)`. It carries the GREEN verdict of [run \(.reused_run_id)](https://github.com/\($repo)/actions/runs/\(.reused_run_id)) at `\(.gated_head)`, because \(.why).\n\n```\n\(if .kind == "restamp-only" then "TREE: same-modulo-stamps receipt=\(.receipt_sha256)" else "OWN DIFF: identical sha256=\(.own_diff_sha256) (merge-base \(.gated_merge_base) -> \(.merge_base))" end)\nbase: \(.gated_base) -> \(.base_sha)\nselection: \(.selection_sha256) (identical to the reused gate)\n```\n\n---\n*Verdict reuse (R2a restamp-only, R2 main moved): a changed own diff, a non-stamp byte, a changed selection, a newer red or unbound gate, or a missing binding or receipt runs the full gate; the evals-fresh label opts out. Free Tests and every other check still ran on this head.*"' "$REUSE")
else
TOTAL=0; PASSED=0; FAILED=0; MANUAL=0; EXECUTED=0; REUSED=0; COST="0"
SUITE_LINES=""
VERIFIED=/tmp/verdict/paid-report/collector-outcomes.json
@@ -739,6 +843,8 @@ jobs:
\`\`\`"
fi
fi
COMMENT_ID=$(gh api repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments \
--jq '.[] | select(.body | startswith("## E2E Evals")) | .id' | tail -1)
+20
View File
@@ -1,5 +1,25 @@
# Changelog
## [1.91.134.0] - 2026-10-18
**The paid PR gate now starts only after a PR head's free tests pass and the PR has been quiet for 10 minutes, and a head whose own diff already passed the gate reuses that verdict instead of paying again.** Re-measured over Oct 3 to 17 (505 runs, $21,449) with GSTA-47's scripts: the readiness wait alone would have saved $2,986 (about $1,380 a week), and verdict reuse up to $601 more (restamp-only $97, main moved $504). Together that is up to $3,587, about $1,660 a week; how much of the main-moved share survives the selection check is between $146 and $390 of the $601 by approximation. The same heads still get the same selection and verdict rules; only the start time moves. The PR receipt store, which never restored once, now does.
### Changed
- **Readiness-gated dispatch (R1).** The `readiness` job in `evals.yml` replaces the 90-second push-burst debounce. It sleeps out the quiet window (repository variable `EVALS_QUIET_MINUTES`, default 10; `evals-no-debounce` sets it to 0), then waits for Free Tests to pass on the same head SHA, for up to 80 minutes. Free Tests red on the head fails the run with no paid work. A newer push still cancels the run. Dispatches never wait, and Version Gate is not an input, so a stamp push always gets its gate.
- **The receipt store restores.** `slices-report` now runs on a GitHub-hosted runner like `recover-receipts`. On Ubicloud its `actions/cache` save went to Ubicloud's transparent cache, which the GitHub-hosted restore never sees: GitHub's cache held no `eval-input` entry at all, and #3253's restamp gate restored nothing 3 minutes after its green run saved 56 receipts.
### Added
- **Paid-gate verdict reuse (R2a restamp-only, R2 main moved), a gate edit.** A PR head reuses the newest executed gate's GREEN verdict only when every newer run of the PR provably ran no paid slice, the gate had reconcile exit 0 and no manually accepted case, this head's own plan has the same selection fingerprint, and either (R2a) the base is unchanged and the tree receipt prints `TREE: same-modulo-stamps` with only VERSION, package.json, CHANGELOG.md or the agents digest differing, or (R2) main moved and the PR's own diff from its merge-base, stamp files excluded, is byte-identical. Base-ref code proposes the reuse (`scripts/gate-reuse.ts find`), the planner confirms the selection, no slice runs, and the PR comment says `PASS (reused: restamp-only, R2a)` or `PASS (reused: main moved, R2)` with the reused run, its head, why, and the receipt digest or own-diff signature. Free Tests and every other check still run on the new head. Anything else (a changed own diff, a non-stamp byte, a newer red or partial run, a missing binding, receipt or merge-base, `evals-fresh`) runs the full gate.
### Fixed
- **The tree receipt no longer drops diff lines that start with `--` or `++`.** Its unified-diff parser treated `---`/`+++` inside a hunk as file headers, so a release-output line beginning `++` or `--` could pass as a stamp. It now reads headers only before a file's first hunk, and a file mode change is never a stamp.
- **`autoplan-guard-pty` no longer stalls at the phase boundary.** Its scripted actor (haiku) published Phase 1, printed "Phase 1 complete." and ended its turn, so the case idled to its 480 s deadline with 2 of 4 guarded allows. This is the 489 s red seen on #3190, #3258, #3260 and this PR's stamped gate (run 39099195754). The script now runs steps 1 to 11 in one continuous run, and says the Phase 1 line is a boundary, not the end. The publication and the Phase 2 entry stay in separate messages. The case also stops waiting as soon as the actor ends its turn there, and names the cause `PHASE_BOUNDARY_STOP`.
### If you…
- **…want a restamp to reuse its gate:** put the CHANGELOG entry in the gated push (under `## [Unreleased]` or the tentative version), so the stamp only re-heads it.
- **…want a main merge to reuse its gate:** merge or rebase without touching the PR's own files; if main changed a file your PR also changes, the full gate runs.
- **…see "Free Tests run N concluded failure" on E2E Evals:** fix and push, or re-run Free Tests and then re-run E2E Evals.
## [1.91.133.0] - 2026-10-18
**A lane on a small machine no longer reports done while its pregate test tier still waits on a remote run.** The coordinator contract every lane inherits (autoplan reviewer prompts, `gstack-autoplan contract`, /spec's gate-and-file section) now says a lane is done only when `gstack-pregate run --allow-repo-commands` passes, and that a `requires-remote` obligation needs `--stage tests --remote --spend-cap <cap>`.
+1 -1
View File
@@ -1 +1 @@
1.91.133.0
1.91.134.0
+1 -1
View File
@@ -1,4 +1,4 @@
# gstack digest v1.91.133.0 — regenerate/re-copy after upgrading gstack
# gstack digest v1.91.134.0 — regenerate/re-copy after upgrading gstack
Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed
for agent hosts without a full skill install. The full skills add workflows,
+59 -7
View File
@@ -1126,7 +1126,7 @@ and `test:health` both say so.
through today's selector and its shadow, and exits 1 on any MISS: a failed
gate case the shadow would not run. `test/exercised-map.test.ts` pins zero.
### PR receipts, recovery and debounce
### PR receipts, recovery, readiness and verdict reuse
Passes and failures of single-trial shards are written as receipts at the
execution boundary, so a cancelled run's finished work survives. The
@@ -1139,13 +1139,65 @@ unresolved run. `plan-slices` turns reuse off when recovery degraded, the
checkpoint is missing or a cancelled run is still unresolved; recovery itself
never fails the run, and the job summary names the reason (rate limit with its
reset time, `gh auth status`, the 60-second budget). The whole store is saved by
`slices-report`.
`slices-report`. Both the restore and the save run on GitHub-hosted runners: an
Ubicloud runner sends `actions/cache` to Ubicloud's transparent cache, which a
GitHub-hosted restore never sees (until GSTA-48 every restore missed).
On a `synchronize` push, when another run for the same PR started in the last 15
minutes, the debounce job waits 90 seconds before planning; a run that outlives
the wait reports "superseded by <sha>". PR labels: `evals-no-debounce` skips the
wait and `evals-fresh` turns receipt reuse off. Labels apply from the next push,
because adding a label does not start `evals.yml`.
**Readiness (R1).** Paid slices for a PR head start only when the PR has been
quiet for the window (repository variable `EVALS_QUIET_MINUTES`, default 10,
measured from the run's creation) and Free Tests has concluded success on the
same head SHA. The `readiness` job (`scripts/eval-readiness.ts`) sleeps out the
window with no API calls, since a newer push cancels the run anyway, then polls
Free Tests every minute for up to 80 minutes. Free Tests red on the head fails
the run without paid work; re-run Free Tests, then re-run this workflow (a
re-run keeps the creation time, so it does not wait out the window again). A
head that is no longer the PR's head reports "superseded by <sha>" and plans
nothing. Selection, trials and verdict rules do not change. Version Gate is not
an input, so a stamp push waits only for its own Free Tests and always gets a
gate. Dispatches (`eval:bg`, the census) never wait.
**Verdict reuse (R2a restamp-only, R2 main moved; a gate edit).** A PR head
reuses the verdict of the newest executed gate of the same PR instead of
running one only when all of these hold. Every newer run of the PR either bound
a verdict or provably ran no paid slice (superseded, readiness-blocked, a reuse,
or cancelled before a slice finished); a newer run that is still running,
finished a slice, or holds a `paid-slice-*` artifact without a bound verdict
blocks reuse, so a newer red or partial result is never skipped. The gate is
GREEN with reconcile exit 0, a successful run and no manually accepted
(unscored) case; it ran the PR profile on a different head. Then either:
- **R2a, restamp-only:** it ran on the same base (the tested merge commit's
first parent), the tree receipt (`lib/tree-receipt.ts` under the base's ship
policy) prints `TREE: same-modulo-stamps`, and every differing file is the
version file, `package.json`, the CHANGELOG or a release output (the agents
digest); or
- **R2, main moved:** the base moved, and the PR's own diff from its
merge-base, excluding stamp files, is byte-identical to the gated head's
(`git diff --binary --full-index`, so pre-image blob ids are compared and a
main change to a file the PR also changes is a difference; `package.json`
compared without its version; release outputs with each revision's version
replaced; the CHANGELOG by its preamble and the sections the PR added and
removed, dropping only version or `[Unreleased]` headings).
Finally this head's own plan must have the same selection fingerprint (cases,
statuses, trial plans, profile), so a main move that changes what the PR gate
selects runs the full gate. `recover-receipts` proposes the candidate from
base-ref code (`scripts/gate-reuse.ts find`), `plan-slices` confirms the
selection, `eval-slices` is skipped, and the report and PR comment say
`PASS (reused: restamp-only, R2a)` or `PASS (reused: main moved, R2)` with
the reused run, its head, why, and the receipt digest or own-diff signature.
Free Tests and every other check still run on the new head. Executed runs
upload a `gate-binding-a<attempt>` artifact (base, verdict, reconcile exit,
manual count, selection); reused runs bind nothing, so reuse never chains off a
reuse. Any other case (a changed own diff, a non-stamp byte including CHANGELOG
entry text or a file mode, a missing binding, receipt or merge-base, a gh or git
error, `evals-fresh`) runs the full gate. For a restamp to qualify, write the
CHANGELOG entry before the gated push (under `## [Unreleased]` or the
tentative version) so the stamp only re-heads it.
PR labels: `evals-no-debounce` sets the quiet window to 0 (Free Tests is still
required) and `evals-fresh` turns receipt reuse and verdict reuse off. Labels
apply from the next push, because adding a label does not start `evals.yml`.
## Free suite runner, judge reuse and engine skips
+9 -4
View File
@@ -209,9 +209,13 @@ function allowedOutput(plan: Plan, root: string, rel: string): boolean {
}
interface Hunk { removed: string[]; added: string[]; at: number }
export interface FileDiff { path: string; binary: boolean; hunks: Hunk[]; status: 'M' | 'A' | 'D' | 'R' }
export interface FileDiff { path: string; binary: boolean; hunks: Hunk[]; status: 'M' | 'A' | 'D' | 'R'; modeChanged?: boolean }
/** Parse `git diff --unified=0` into per-file hunks of removed/added lines (shared with lib/tree-receipt.ts). */
/**
* Parse `git diff --unified=0` into per-file hunks of removed/added lines (shared with lib/tree-receipt.ts).
* File headers come only before a file's first hunk: inside a hunk every `-`/`+` line is content, so a
* body line that begins `--` or `++` (`---`/`+++` in the diff) is never dropped as a header.
*/
export function parseUnifiedDiff(text: string): FileDiff[] {
const files: FileDiff[] = [];
let cur: FileDiff | null = null;
@@ -225,11 +229,12 @@ export function parseUnifiedDiff(text: string): FileDiff[] {
} else if (!cur) continue;
else if (line.startsWith('new file mode')) cur.status = 'A';
else if (line.startsWith('deleted file mode')) cur.status = 'D';
else if (!hunk && (line.startsWith('old mode ') || line.startsWith('new mode '))) cur.modeChanged = true;
else if (line.startsWith('rename to ')) cur.status = 'R';
else if (line.startsWith('Binary files')) cur.binary = true;
else if (line.startsWith('@@')) { hunk = { removed: [], added: [], at: Number(/^@@ -\d+(?:,\d+)? \+(\d+)/.exec(line)?.[1] ?? 0) }; cur.hunks.push(hunk); }
else if (hunk && line.startsWith('-') && !line.startsWith('---')) hunk.removed.push(line.slice(1));
else if (hunk && line.startsWith('+') && !line.startsWith('+++')) hunk.added.push(line.slice(1));
else if (hunk && line.startsWith('-')) hunk.removed.push(line.slice(1));
else if (hunk && line.startsWith('+')) hunk.added.push(line.slice(1));
}
return files;
}
+1
View File
@@ -72,6 +72,7 @@ function classifyFile(repoRoot: string, f: FileDiff, gated: string, current: str
const changed = (rule: string, residual: string, line = f.hunks[0]?.at): FileVerdict => ({ path: f.path, rule, verdict: 'changed', residual, line });
const stamp = (rule: string): FileVerdict => ({ path: f.path, rule, verdict: 'stamp' });
if (f.binary) return changed('binary', 'binary content differs');
if (f.modeChanged) return changed('mode', 'file mode changed');
if (f.status !== 'M') return changed('path', `${f.status === 'A' ? 'added' : f.status === 'D' ? 'deleted' : 'renamed'}`);
if (policy.stamp_files.length && !matchesAny(policy.stamp_files, f.path)) return changed('stamp_files', 'not in the repo\'s stamp_files allowlist');
const before = blob(repoRoot, gated, f.path);
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "gstack",
"version": "1.91.133",
"version": "1.91.134",
"description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.",
"license": "MIT",
"type": "module",
+138
View File
@@ -0,0 +1,138 @@
/**
* Readiness-gated dispatch for the PR paid lane (R1, GSTA-48).
*
* The `readiness` job in evals.yml runs this before any paid work: a PR
* head's paid slices start only after (1) the PR has been quiet for the
* window (no newer push since this run was created; `vars.EVALS_QUIET_MINUTES`,
* default 10; the `evals-no-debounce` label sets it to 0) and (2) Free Tests
* (free-tests.yml) has concluded success on the same head SHA. Selection,
* trials and verdict rules are unchanged: only the start time moves.
*
* A newer push cancels the run through the workflow's cancel-in-progress
* group, so the quiet window is a plain sleep with no API calls. After it,
* Free Tests is polled every `--poll-seconds`. Free Tests concluding anything
* but success on this head fails the run without paid work: that head cannot
* merge, and re-running this workflow after a green Free Tests rerun starts
* the gate (a re-run keeps the run's original creation time, so it does not
* wait out the window again). A head that is no longer the PR's head reports
* superseded and plans nothing.
*/
import * as fs from 'node:fs';
import { runGh, type GhRunner } from './recover-receipts';
export interface FreeTestsRun { id: number; status: string; conclusion: string | null; head_sha: string; event: string }
export type FreeTestsState =
| { state: 'missing' }
| { state: 'running'; runId: number }
| { state: 'success'; runId: number }
| { state: 'failed'; runId: number; conclusion: string };
export type ReadinessDecision =
| { kind: 'wait'; reason: string }
| { kind: 'ready'; reason: string }
| { kind: 'superseded'; by: string }
| { kind: 'blocked'; reason: string };
/** The newest Free Tests pull_request run on this exact head decides; a rerun updates that run in place. */
export function freeTestsState(runs: FreeTestsRun[], headSha: string): FreeTestsState {
const newest = runs.filter(run => run.head_sha === headSha && run.event === 'pull_request').sort((a, b) => b.id - a.id)[0];
if (!newest) return { state: 'missing' };
if (newest.status !== 'completed') return { state: 'running', runId: newest.id };
if (newest.conclusion === 'success') return { state: 'success', runId: newest.id };
return { state: 'failed', runId: newest.id, conclusion: newest.conclusion ?? 'unknown' };
}
export function readinessDecision(input: { headSha: string; prHead: string | null; free: FreeTestsState; quietLeftMs: number }): ReadinessDecision {
if (input.prHead && input.prHead !== input.headSha) return { kind: 'superseded', by: input.prHead };
const free = input.free;
if (free.state === 'failed') {
return { kind: 'blocked', reason: `Free Tests run ${free.runId} concluded ${free.conclusion} on ${input.headSha.slice(0, 12)}; the paid gate does not start for a head that cannot merge. Fix and push, or re-run Free Tests and then re-run this workflow` };
}
if (input.quietLeftMs > 0) return { kind: 'wait', reason: `quiet window: ${Math.ceil(input.quietLeftMs / 1000)} s left` };
if (free.state === 'missing') return { kind: 'wait', reason: 'Free Tests has not started on this head yet' };
if (free.state === 'running') return { kind: 'wait', reason: `Free Tests run ${free.runId} is still running` };
return { kind: 'ready', reason: `Free Tests run ${free.runId} passed on ${input.headSha.slice(0, 12)} and the PR has been quiet for the window` };
}
export interface WaitOptions {
repo: string;
pr: number;
headSha: string;
runId: number;
quietMs: number;
deadlineMs: number;
pollMs: number;
gh?: GhRunner;
now?: () => number;
sleep?: (ms: number) => void;
}
function ghJson<T>(gh: GhRunner, args: string[]): { ok: true; value: T } | { ok: false; error: string } {
const out = gh(args, 30_000);
if (out.status !== 0) return { ok: false, error: out.stderr.trim().split('\n')[0]?.slice(0, 200) || `gh exited ${out.status}` };
try { return { ok: true, value: JSON.parse(out.stdout) as T }; } catch { return { ok: false, error: 'gh returned unreadable JSON' }; }
}
/** Waits until the head is ready, superseded or blocked; never longer than the deadline. */
export function waitForReadiness(o: WaitOptions): { decision: ReadinessDecision; lines: string[] } {
const gh = o.gh ?? runGh;
const now = o.now ?? Date.now;
const sleep = o.sleep ?? ((ms: number) => Bun.sleepSync(ms));
const start = now();
const lines: string[] = [];
const run = ghJson<{ created_at?: string }>(gh, ['api', `repos/${o.repo}/actions/runs/${o.runId}`]);
const created = run.ok ? Date.parse(run.value.created_at ?? '') : NaN;
const quietUntil = (Number.isFinite(created) ? created : start) + o.quietMs;
lines.push(`Readiness (R1): paid slices start after Free Tests passes on ${o.headSha.slice(0, 12)} and the PR is quiet for ${Math.round(o.quietMs / 60_000)} min (from ${new Date(quietUntil - o.quietMs).toISOString()})`);
if (quietUntil > now()) sleep(quietUntil - now());
let lastError = '';
for (;;) {
const listed = ghJson<{ workflow_runs?: FreeTestsRun[] }>(gh, ['api', '-X', 'GET', `repos/${o.repo}/actions/workflows/free-tests.yml/runs`,
'-f', `head_sha=${o.headSha}`, '-f', 'event=pull_request', '-f', 'per_page=20']);
const pr = ghJson<{ head?: { sha?: string } }>(gh, ['api', `repos/${o.repo}/pulls/${o.pr}`]);
lastError = !listed.ok ? listed.error : !pr.ok ? pr.error : '';
if (!lastError && listed.ok && pr.ok) {
const decision = readinessDecision({ headSha: o.headSha, prHead: pr.value.head?.sha ?? null,
free: freeTestsState(listed.value.workflow_runs ?? [], o.headSha), quietLeftMs: quietUntil - now() });
if (decision.kind !== 'wait') {
lines.push(decision.kind === 'superseded' ? `superseded by ${decision.by}; this run plans no slices` : decision.reason);
lines.push(`Readiness wait: ${Math.round((now() - start) / 1000)} s`);
return { decision, lines };
}
lastError = decision.reason;
}
if (now() + o.pollMs > start + o.deadlineMs) {
const reason = `not ready after ${Math.round(o.deadlineMs / 60_000)} min (${lastError}); re-run this workflow once Free Tests has passed on this head`;
lines.push(reason);
return { decision: { kind: 'blocked', reason }, lines };
}
sleep(o.pollMs);
}
}
const USAGE = `usage: bun run scripts/eval-readiness.ts wait --repo <owner/name> --pr <n> --head-sha <sha> --run-id <id>
[--quiet-minutes 10] [--deadline-minutes 80] [--poll-seconds 60] [--github-output <file>] [--summary <file>]
Exit 0 when ready or superseded (output superseded=true), 1 when blocked, 2 on usage.`;
if (import.meta.main) {
const [command, ...rest] = process.argv.slice(2);
const flag = (name: string) => { const i = rest.indexOf(`--${name}`); return i >= 0 ? rest[i + 1] : undefined; };
const num = (name: string, fallback: number) => { const value = flag(name); return value === undefined || value === '' ? fallback : Number(value); };
const [repo, headSha] = [flag('repo'), flag('head-sha')];
const [pr, runId, quiet, deadline, poll] = [Number(flag('pr')), Number(flag('run-id')), num('quiet-minutes', 10), num('deadline-minutes', 80), num('poll-seconds', 60)];
if (command !== 'wait' || !repo || !headSha || !/^[0-9a-f]{40}$/.test(headSha) || !Number.isSafeInteger(pr) || !Number.isSafeInteger(runId)
|| ![quiet, deadline, poll].every(value => Number.isFinite(value) && value >= 0)) {
console.error(USAGE);
process.exit(2);
}
const { decision, lines } = waitForReadiness({ repo, pr, headSha, runId, quietMs: quiet * 60_000, deadlineMs: deadline * 60_000, pollMs: Math.max(5, poll) * 1000 });
const summary = flag('summary');
for (const line of lines) {
console.log(line);
if (summary) fs.appendFileSync(summary, `- ${line}\n`);
}
const output = flag('github-output');
if (output && decision.kind === 'superseded') fs.appendFileSync(output, 'superseded=true\n');
if (decision.kind === 'blocked') console.error(`::error::${decision.reason}`);
process.exit(decision.kind === 'blocked' ? 1 : 0);
}
+503
View File
@@ -0,0 +1,503 @@
/**
* Paid-gate verdict reuse for the PR lane (GSTA-48). A gate edit: a head may
* report the verdict of an earlier full paid gate instead of running one only
* when every condition below holds; anything else runs the full gate.
*
* bind (slices-report, every executed pull_request run) records what the
* run proved: PR, head, the base its merge commit was built on
* (the tested commit's first parent), the reconcile exit, the
* report verdict and its plan's selection fingerprint. Reused runs
* bind nothing, so reuse never chains off a reuse.
* find (recover-receipts, BASE-REF code, actions: read) takes the newest
* earlier completed run of this PR that bound a verdict. It must be
* GREEN with reconcile exit 0 and a successful conclusion, ran the
* PR profile, and gated a different head. Then one of:
* restamp-only (R2a): same base; the tree receipt
* (lib/tree-receipt.ts, the base's ship policy) says
* `TREE: same-modulo-stamps` and every differing file is a stamp
* file (the version file, package.json, the CHANGELOG, the
* release outputs).
* main-moved (R2): the base moved; the PR's own diff from its
* merge-base, excluding stamp files, is byte-identical to the
* gated head's (package.json compared without its version,
* the CHANGELOG by the PR's own section bodies).
* It writes candidate.json.
* confirm (plan-slices, after this run's own plan) requires this head's
* selection fingerprint to equal the bound one, so the reused
* verdict covers exactly the cases and trial plans this head would
* run. It writes gate-reuse.json and `gate_reuse=true`.
* report (slices-report) prints the reused verdict: which run, which head,
* and why.
*
* Every failure (no binding, a red, partial or cancelled gate, a non-stamp
* byte, a missing receipt or merge-base, gh or git errors, a selection change)
* leaves no candidate, and the run executes the full gate. The free suite and
* every other check run on the new head regardless.
*/
import { createHash } from 'node:crypto';
import { spawnSync } from 'node:child_process';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { receiptDigest } from '../lib/ship-queue';
import { isUnavailable, loadPolicyFromBase, matchesAny, type ShipPolicy } from '../lib/ship-policy';
import { rewriteStamps } from '../lib/restamp';
import { buildReceipt } from '../lib/tree-receipt';
import { acceptedRun, ghFailureReason, runGh, type GhRunner, type WorkflowRun } from './recover-receipts';
import type { PaidRunManifest } from './lib/paid-plan';
export const BINDING_FILE = 'gate-binding.json';
export const CANDIDATE_FILE = 'candidate.json';
export const REUSE_FILE = 'gate-reuse.json';
const SHA = /^[0-9a-f]{40}$/;
const SHA256 = /^[0-9a-f]{64}$/;
export interface GateBinding {
schema: 1;
repository: string;
pr: number;
run_id: number;
run_attempt: number;
head_sha: string;
/** The tested merge commit's first parent: the main the gate actually ran against. */
base_sha: string;
profile: string;
reconcile_exit: number;
verdict: string;
/** Cases accepted manually (unscored); a gate with any is never reused as a clean pass. */
manual_accepted: number;
selection_sha256: string;
}
export interface GateReuse {
schema: 1;
kind: 'restamp-only' | 'main-moved';
pr: number;
reused_run_id: number;
reused_run_attempt: number;
gated_head: string;
current_head: string;
gated_base: string;
base_sha: string;
selection_sha256: string;
/** One sentence: why this head may carry that verdict. */
why: string;
/** restamp-only: the tree receipt. */
receipt_sha256?: string;
stamp_files?: string[];
old_version?: string | null;
new_version?: string | null;
/** main-moved: the merge-bases and the identical own-diff signature. */
gated_merge_base?: string;
merge_base?: string;
own_diff_sha256?: string;
}
/** The cases, trial plans and verdict-relevant plan fields a run executes; slice packing and receipts are excluded. */
export function selectionFingerprint(manifest: PaidRunManifest): string {
const entries = manifest.entries
.map(entry => ({ file: entry.file, status: entry.status, trial: entry.trial ?? null, excludeCases: entry.excludeCases ?? null }))
.sort((a, b) => (a.file < b.file ? -1 : a.file > b.file ? 1 : 0));
const body = { tier: manifest.tier, evalsAll: manifest.evalsAll, profile: manifest.profile ?? null, selection: manifest.selection ?? null,
mode: manifest.prCoverage?.mode ?? null, entries };
return createHash('sha256').update(JSON.stringify(body)).digest('hex');
}
export function readJson<T>(file: string): T | null {
try { return JSON.parse(fs.readFileSync(file, 'utf8')) as T; } catch { return null; }
}
export function parseBinding(value: unknown): GateBinding | null {
const b = value as GateBinding | null;
if (!b || b.schema !== 1 || typeof b.repository !== 'string' || !Number.isSafeInteger(b.pr) || !Number.isSafeInteger(b.run_id)
|| !Number.isSafeInteger(b.run_attempt) || !SHA.test(b.head_sha ?? '') || !SHA.test(b.base_sha ?? '') || typeof b.profile !== 'string'
|| !Number.isSafeInteger(b.reconcile_exit) || typeof b.verdict !== 'string' || !Number.isSafeInteger(b.manual_accepted) || b.manual_accepted < 0 || !SHA256.test(b.selection_sha256 ?? '')) return null;
return b;
}
export function parseReuse(value: unknown): GateReuse | null {
const r = value as GateReuse | null;
if (!r || r.schema !== 1 || !Number.isSafeInteger(r.pr) || !Number.isSafeInteger(r.reused_run_id) || !Number.isSafeInteger(r.reused_run_attempt)
|| !SHA.test(r.gated_head ?? '') || !SHA.test(r.current_head ?? '') || r.gated_head === r.current_head
|| !SHA.test(r.gated_base ?? '') || !SHA.test(r.base_sha ?? '') || !SHA256.test(r.selection_sha256 ?? '') || typeof r.why !== 'string') return null;
if (r.kind === 'restamp-only') {
return r.gated_base === r.base_sha && SHA256.test(r.receipt_sha256 ?? '') && Array.isArray(r.stamp_files) && r.stamp_files.length > 0 ? r : null;
}
if (r.kind === 'main-moved') {
return r.gated_base !== r.base_sha && SHA.test(r.gated_merge_base ?? '') && SHA.test(r.merge_base ?? '') && SHA256.test(r.own_diff_sha256 ?? '') ? r : null;
}
return null;
}
type Git = (args: string[]) => { ok: boolean; out: string };
const gitIn = (cwd: string): Git => (args) => {
const r = spawnSync('git', args, { cwd, encoding: 'utf8', timeout: 120_000, maxBuffer: 256 * 1024 * 1024 });
return r.status === 0 ? { ok: true, out: r.stdout ?? '' } : { ok: false, out: (r.stderr || String(r.error ?? '')).trim() };
};
/** A merge commit's parents from its own object, so a depth-1 checkout can read them. */
export function commitParents(git: Git, commit: string): string[] | null {
const shown = git(['cat-file', '-p', commit]);
if (!shown.ok) return null;
return shown.out.split('\n').filter(line => line.startsWith('parent ')).map(line => line.slice(7).trim());
}
/** The tested commit of a pull_request run is the merge of the head into the base; returns that base. */
export function testedBase(git: Git, tested: string, head: string): { base: string } | { error: string } {
const parents = commitParents(git, tested);
if (!parents) return { error: `the tested commit ${tested.slice(0, 12)} is not readable` };
if (parents.length !== 2 || parents[1] !== head || !SHA.test(parents[0]!)) {
return { error: `the tested commit ${tested.slice(0, 12)} is not the merge of ${head.slice(0, 12)} into a base` };
}
return { base: parents[0]! };
}
/** The files whose differences count as stamps under the base's policy. */
function stampSet(policy: ShipPolicy): { version: string; changelog: string | null; releaseOutputs: string[]; patterns: string[] } {
return { version: 'VERSION', changelog: policy.changelog ?? null, releaseOutputs: policy.release_outputs, patterns: policy.stamp_patterns };
}
/** A version or [Unreleased] heading, which a restamp rewrites; any other heading is content. */
const STAMP_HEADING = /^## \[(?:Unreleased|\d+(?:\.\d+)+)\](?: - \d{4}-\d{2}-\d{2})?\s*$/;
/** The CHANGELOG as its preamble and the multiset of its sections, each without a stamp heading. */
function changelogParts(text: string | null): { preamble: string; sections: string[] } {
if (text === null) return { preamble: '\u0000missing', sections: [] };
const chunks: string[][] = [[]];
for (const line of text.split('\n')) {
if (line.startsWith('## ')) chunks.push([]);
chunks[chunks.length - 1]!.push(line);
}
const [preamble, ...sections] = chunks;
return { preamble: preamble!.join('\n'), sections: sections.map(lines => (STAMP_HEADING.test(lines[0]!) ? lines.slice(1) : lines).join('\n').replace(/\n+$/, '')) };
}
/**
* The PR's own diff from its merge-base with stamps taken out. Every changed
* path that is not a stamp file contributes its full binary patch (pre-image
* blob ids included, so a file main also changed under the PR is a
* difference). Stamp files contribute everything except the stamp: the root
* package.json without its version, each release output with its revision's
* version replaced (its pre-image too, when anything else changed), and the
* CHANGELOG's preamble plus the sections the PR added and removed, with only
* version or [Unreleased] headings dropped. The version file is the stamp.
*/
export function ownDiffSignature(git: Git, mergeBase: string, head: string, stamps: ReturnType<typeof stampSet>): { sha256: string } | { error: string } {
const names = git(['diff', '--name-only', '-z', '--no-renames', mergeBase, head]);
if (!names.ok) return { error: `git diff ${mergeBase.slice(0, 12)} ${head.slice(0, 12)} failed (${names.out.split('\n')[0]})` };
const changed = names.out.split('\u0000').filter(Boolean);
const isRelease = (file: string) => matchesAny(stamps.releaseOutputs, file);
const others = changed.filter(file => file !== stamps.version && file !== 'package.json' && file !== stamps.changelog && !isRelease(file));
let patch = '';
if (others.length) {
const r = git(['diff', '--no-ext-diff', '--no-renames', '--no-color', '--binary', '--full-index', mergeBase, head, '--', ...others.map(file => `:(literal)${file}`)]);
if (!r.ok) return { error: `git diff ${mergeBase.slice(0, 12)} ${head.slice(0, 12)} failed (${r.out.split('\n')[0]})` };
patch = r.out;
}
const stampPaths = changed.filter(file => !others.includes(file));
let modes: string[] = [];
if (stampPaths.length) {
const raw = git(['diff', '--raw', '-z', '--no-renames', mergeBase, head, '--', ...stampPaths.map(file => `:(literal)${file}`)]);
if (!raw.ok) return { error: `git diff --raw ${mergeBase.slice(0, 12)} ${head.slice(0, 12)} failed (${raw.out.split('\n')[0]})` };
const fields = raw.out.split('\u0000');
for (let i = 0; i + 1 < fields.length; i += 2) {
const [oldMode, newMode] = fields[i]!.replace(/^:/, '').split(' ');
if (oldMode !== newMode) modes.push(`${fields[i + 1]} ${oldMode} ${newMode}`);
}
modes = modes.sort();
}
const blob = (rev: string, file: string): string | null => {
const r = git(['show', `${rev}:${file}`]);
if (r.ok) return r.out;
const listed = git(['ls-tree', '--name-only', rev, '--', `:(literal)${file}`]);
if (!listed.ok || listed.out.trim() !== '') throw new Error(`cannot read ${file} at ${rev.slice(0, 12)}`);
return null;
};
const version = (rev: string) => blob(rev, stamps.version)?.trim() || null;
const unstamped = (rev: string, file: string) => {
const text = blob(rev, file);
const v = version(rev);
return text === null ? null : v ? rewriteStamps(text, v, '\u0000VERSION', stamps.patterns).text : text;
};
try {
const manifest = (text: string | null) => {
if (text === null) return null;
try { const parsed = JSON.parse(text); delete parsed.version; return JSON.stringify(parsed); } catch { return text; }
};
const [pkgBase, pkgHead] = [manifest(blob(mergeBase, 'package.json')), manifest(blob(head, 'package.json'))];
const releases = changed.filter(isRelease).sort().flatMap(file => {
const [before, after] = [unstamped(mergeBase, file), unstamped(head, file)];
return before === after ? [] : [[file, before, after]];
});
let changelog = null;
if (stamps.changelog && changed.includes(stamps.changelog)) {
const [before, after] = [changelogParts(blob(mergeBase, stamps.changelog)), changelogParts(blob(head, stamps.changelog))];
const removed = [...before.sections];
const added: string[] = [];
for (const section of after.sections) {
const i = removed.indexOf(section);
if (i >= 0) removed.splice(i, 1);
else added.push(section);
}
changelog = { preamble: before.preamble === after.preamble ? null : [before.preamble, after.preamble], added: added.sort(), removed: removed.sort() };
}
const body = { patch, modes, package: pkgBase === pkgHead ? null : [pkgBase, pkgHead], releases, changelog };
return { sha256: createHash('sha256').update(JSON.stringify(body)).digest('hex') };
} catch (err) {
return { error: (err as Error).message };
}
}
/** slices-report: what this executed run proved, or why it binds nothing. */
export function buildBinding(o: { repository: string; pr: number; runId: number; runAttempt: number; headSha: string; baseSha: string | null;
profile: string; reconcileExit: number; reportDir: string }): { binding: GateBinding | null; reason: string } {
if (fs.existsSync(path.join(o.reportDir, REUSE_FILE))) return { binding: null, reason: 'this run reused a verdict; only executed gates bind' };
if (!o.baseSha) return { binding: null, reason: 'the tested base is unknown' };
const manifest = readJson<PaidRunManifest>(path.join(o.reportDir, 'manifest.json'));
if (!manifest?.entries) return { binding: null, reason: 'no readable manifest' };
const outcomes = readJson<{ verdict?: { verdict?: string }; totals?: { manual_accepted?: number } }>(path.join(o.reportDir, 'collector-outcomes.json'));
const binding: GateBinding = { schema: 1, repository: o.repository, pr: o.pr, run_id: o.runId, run_attempt: o.runAttempt,
head_sha: o.headSha, base_sha: o.baseSha, profile: o.profile, reconcile_exit: o.reconcileExit,
verdict: outcomes?.verdict?.verdict ?? 'MISSING', manual_accepted: outcomes?.totals?.manual_accepted ?? -1, selection_sha256: selectionFingerprint(manifest) };
return parseBinding(binding)
? { binding, reason: `bound ${binding.verdict} at ${binding.head_sha.slice(0, 12)} on base ${binding.base_sha.slice(0, 12)} (reconcile exit ${binding.reconcile_exit})` }
: { binding: null, reason: 'binding fields are incomplete' };
}
/**
* A newer completed run with no binding is passed over only when no paid
* slice of it finished: superseded, readiness-blocked, a reuse, or cancelled
* before or during its slices. A slice that finished (success, failure,
* timed out) without a bound verdict blocks reuse of an older GREEN, so a
* newer red or partial result is never skipped.
*/
export function unboundRunVerdict(gh: GhRunner, repo: string, run: WorkflowRun, timeoutMs: number): string | null {
const listed = gh(['api', '-X', 'GET', `repos/${repo}/actions/runs/${run.id}/jobs`, '-f', 'filter=latest', '-f', 'per_page=100'], timeoutMs);
if (listed.status !== 0) return `cannot list run ${run.id}'s jobs (${ghFailureReason(listed.stderr)})`;
let slices: Array<{ name: string; conclusion: string | null }>;
try {
slices = ((JSON.parse(listed.stdout).jobs ?? []) as Array<{ name: string; conclusion: string | null }>)
.filter(job => job.name === 'eval-slices' || job.name.startsWith('eval-slices '));
} catch { return `run ${run.id}'s job list is unreadable`; }
const finished = slices.find(job => job.conclusion !== null && job.conclusion !== 'skipped' && job.conclusion !== 'cancelled');
return finished
? `newer run ${run.id} (${run.head_sha.slice(0, 12)}) finished a paid slice (${finished.conclusion}) but bound no verdict (conclusion ${run.conclusion})`
: null;
}
export interface FindOptions {
repo: string;
pr: number;
headRepo: string;
branch: string;
selfRunId: number;
headSha: string;
/** This run's tested merge commit (github.sha); its first parent is the base. */
testedSha: string;
/** A base-ref checkout to fetch into and run the receipt in. */
repoRoot: string;
out: string;
budgetMs?: number;
maxRuns?: number;
gh?: GhRunner;
git?: Git;
now?: () => number;
}
/** Never throws for an expected refusal: no candidate means the full gate runs, with the reason named. */
export function findReusableGate(o: FindOptions): { candidate: GateReuse | null; lines: string[] } {
const gh = o.gh ?? runGh;
const git = o.git ?? gitIn(o.repoRoot);
const now = o.now ?? Date.now;
const deadline = now() + (o.budgetMs ?? 90_000);
const left = () => Math.max(1_000, deadline - now());
const lines: string[] = [];
const refuse = (reason: string) => { lines.push(`no gate reuse: ${reason}; the full gate runs`); return { candidate: null, lines }; };
if (!SHA.test(o.headSha) || !SHA.test(o.testedSha)) return refuse('this run has no exact head and tested commit');
const listed = gh(['api', '-X', 'GET', `repos/${o.repo}/actions/workflows/evals.yml/runs`, '-f', 'event=pull_request', '-f', `branch=${o.branch}`, '-f', 'per_page=50'], left());
if (listed.status !== 0) return refuse(ghFailureReason(listed.stderr));
let runs: WorkflowRun[];
try { runs = (JSON.parse(listed.stdout).workflow_runs ?? []) as WorkflowRun[]; } catch { return refuse('gh returned an unreadable run list'); }
const scope = { pr: o.pr, headRepo: o.headRepo, selfRunId: o.selfRunId };
const older = runs.filter(run => acceptedRun(run, scope) && run.id < o.selfRunId)
.sort((a, b) => b.id - a.id).slice(0, o.maxRuns ?? 10);
const work = fs.mkdtempSync(path.join(os.tmpdir(), 'gate-reuse-'));
try {
let found: { run: WorkflowRun; binding: GateBinding | null } | null = null;
for (const run of older) {
const names = gh(['api', '-X', 'GET', `repos/${o.repo}/actions/runs/${run.id}/artifacts`, '-f', 'per_page=100'], left());
if (names.status !== 0) return refuse(`cannot list run ${run.id}'s artifacts (${ghFailureReason(names.stderr)})`);
let artifact: string | undefined;
let sliceEvidence: string | undefined;
try {
const listedNames = ((JSON.parse(names.stdout).artifacts ?? []) as Array<{ name: string; expired?: boolean }>).map(a => a.name);
artifact = listedNames.filter(name => /^gate-binding-a\d+$/.test(name)).sort((a, b) => Number(b.slice(14)) - Number(a.slice(14)))[0];
sliceEvidence = listedNames.find(name => name.startsWith('paid-slice-'));
} catch { return refuse(`run ${run.id}'s artifact list is unreadable`); }
if (run.status !== 'completed') return refuse(`newer run ${run.id} (${run.head_sha.slice(0, 12)}) is still ${run.status}`);
if (!artifact) {
if (sliceEvidence) return refuse(`newer run ${run.id} (${run.head_sha.slice(0, 12)}) holds paid slice results (${sliceEvidence}) but bound no verdict`);
const unbound = unboundRunVerdict(gh, o.repo, run, left());
if (unbound) return refuse(unbound);
continue;
}
const dir = path.join(work, String(run.id));
const got = gh(['run', 'download', String(run.id), '-R', o.repo, '-n', artifact, '-D', dir], left());
if (got.status !== 0) return refuse(`cannot download ${artifact} of run ${run.id} (${ghFailureReason(got.stderr)})`);
found = { run, binding: parseBinding(readJson(path.join(dir, BINDING_FILE))) };
break;
}
if (!found) return refuse('no earlier executed gate on this PR bound a verdict');
const { run, binding } = found;
if (!binding) return refuse(`run ${run.id}'s gate binding is missing or malformed`);
if (binding.run_id !== run.id || binding.pr !== o.pr || binding.repository !== o.repo || binding.head_sha !== run.head_sha) {
return refuse(`run ${run.id}'s gate binding does not match the run it came from`);
}
if (run.conclusion !== 'success' || binding.verdict !== 'GREEN' || binding.reconcile_exit !== 0) {
return refuse(`the newest executed gate (run ${run.id}, ${binding.head_sha.slice(0, 12)}) is ${binding.verdict}, reconcile exit ${binding.reconcile_exit}, conclusion ${run.conclusion}`);
}
if (binding.manual_accepted !== 0) {
return refuse(`run ${run.id}'s GREEN includes ${binding.manual_accepted} manually accepted (unscored) case(s), which reuse would present as a clean pass`);
}
if (binding.profile !== 'pr') return refuse(`run ${run.id} ran the ${binding.profile} profile, not the PR gate`);
if (binding.head_sha === o.headSha) return refuse(`run ${run.id} gated this same head; a re-run executes fresh`);
const fetched = git(['fetch', '--no-tags', '--depth=1', 'origin', binding.head_sha, o.headSha, o.testedSha]);
if (!fetched.ok) return refuse(`cannot fetch the gated and current commits (${fetched.out.split('\n')[0]})`);
const tested = testedBase(git, o.testedSha, o.headSha);
if ('error' in tested) return refuse(tested.error);
const baseSha = tested.base;
const fetchedBase = git(['fetch', '--no-tags', '--depth=1', 'origin', baseSha]);
const pinned = fetchedBase.ok && git(['update-ref', 'refs/remotes/origin/gate-reuse-base', baseSha]).ok;
if (!pinned) return refuse(`cannot pin this run's base ${baseSha.slice(0, 12)} for the policy`);
const policy = loadPolicyFromBase(o.repoRoot, 'gate-reuse-base');
if (isUnavailable(policy)) return refuse(`the base's ship policy is unavailable (${policy.detail})`);
if (policy.errors.length) return refuse(`the base's ship policy is invalid (${policy.errors[0]!.message})`);
const common = { schema: 1 as const, pr: o.pr, reused_run_id: run.id, reused_run_attempt: binding.run_attempt, gated_head: binding.head_sha,
current_head: o.headSha, gated_base: binding.base_sha, base_sha: baseSha, selection_sha256: binding.selection_sha256 };
let candidate: GateReuse;
if (binding.base_sha === baseSha) {
const receipt = buildReceipt({ repoRoot: o.repoRoot, gated: binding.head_sha, current: o.headSha, policy: policy.policy, policyLabel: policy.source.label });
if ('error' in receipt) return refuse(`no tree receipt: ${receipt.error}`);
lines.push(`TREE: ${receipt.tree === 'changed' ? `changed (${receipt.residual.join(', ')})` : 'same-modulo-stamps'} (${binding.head_sha.slice(0, 12)} -> ${o.headSha.slice(0, 12)}, policy ${receipt.policy})`);
if (receipt.tree !== 'same-modulo-stamps') return refuse(`${o.headSha.slice(0, 12)} differs from the gated head beyond stamps: ${receipt.residual.join(', ')}`);
const stampFiles = receipt.files.map(file => file.path);
const stamps = stampSet(policy.policy);
const allowed = new Set([stamps.version, 'package.json', ...(stamps.changelog ? [stamps.changelog] : []), ...stamps.releaseOutputs]);
const outside = stampFiles.filter(file => !allowed.has(file));
if (outside.length) return refuse(`stamp-classified files outside the version file, package.json, CHANGELOG and release outputs: ${outside.join(', ')}`);
if (!stampFiles.length) return refuse('the heads differ by no stamp file');
candidate = { ...common, kind: 'restamp-only', receipt_sha256: receiptDigest(receipt), stamp_files: stampFiles,
old_version: receipt.old_version, new_version: receipt.new_version,
why: `main has not moved (base ${baseSha.slice(0, 12)}) and ${o.headSha.slice(0, 12)} differs from ${binding.head_sha.slice(0, 12)} only by stamp files (${stampFiles.join(', ')}): TREE same-modulo-stamps` };
} else {
const mergeBase = (base: string, head: string) => {
const r = gh(['api', `repos/${o.repo}/compare/${base}...${head}`, '--jq', '.merge_base_commit.sha'], left());
const sha = r.stdout.trim();
return r.status === 0 && SHA.test(sha) ? sha : null;
};
const [gatedMb, currentMb] = [mergeBase(binding.base_sha, binding.head_sha), mergeBase(baseSha, o.headSha)];
if (!gatedMb || !currentMb) return refuse('a merge-base is unavailable, so the own diffs cannot be compared');
const fetchedMb = git(['fetch', '--no-tags', '--depth=1', 'origin', gatedMb, currentMb]);
if (!fetchedMb.ok) return refuse(`cannot fetch the merge-bases (${fetchedMb.out.split('\n')[0]})`);
const stamps = stampSet(policy.policy);
const [gatedDiff, currentDiff] = [ownDiffSignature(git, gatedMb, binding.head_sha, stamps), ownDiffSignature(git, currentMb, o.headSha, stamps)];
if ('error' in gatedDiff) return refuse(gatedDiff.error);
if ('error' in currentDiff) return refuse(currentDiff.error);
lines.push(`OWN DIFF: ${gatedDiff.sha256 === currentDiff.sha256 ? 'identical' : 'changed'} (${binding.head_sha.slice(0, 12)} from ${gatedMb.slice(0, 12)}: ${gatedDiff.sha256.slice(0, 12)}; ${o.headSha.slice(0, 12)} from ${currentMb.slice(0, 12)}: ${currentDiff.sha256.slice(0, 12)})`);
if (gatedDiff.sha256 !== currentDiff.sha256) return refuse(`the PR's own diff (excluding stamp files) differs from the gated head's`);
candidate = { ...common, kind: 'main-moved', gated_merge_base: gatedMb, merge_base: currentMb, own_diff_sha256: currentDiff.sha256,
why: `main moved (${binding.base_sha.slice(0, 12)} -> ${baseSha.slice(0, 12)}) and the PR's own diff from its merge-base, excluding stamp files, is byte-identical to ${binding.head_sha.slice(0, 12)}'s (sha256 ${currentDiff.sha256.slice(0, 12)})` };
}
fs.mkdirSync(o.out, { recursive: true });
fs.writeFileSync(path.join(o.out, CANDIDATE_FILE), `${JSON.stringify(candidate, null, 2)}\n`);
lines.push(`${candidate.kind} candidate: run ${run.id} passed the full gate at ${binding.head_sha.slice(0, 12)}; ${candidate.why}; plan-slices confirms the selection`);
return { candidate, lines };
} finally {
fs.rmSync(work, { recursive: true, force: true });
}
}
/** plan-slices: the reuse holds only when this head's own plan selects exactly what the reused gate ran. */
export function confirmReuse(candidateDir: string, manifest: PaidRunManifest | null): { reuse: GateReuse | null; reason: string } {
const candidate = parseReuse(readJson(path.join(candidateDir, CANDIDATE_FILE)));
if (!candidate) return { reuse: null, reason: 'no reuse candidate; the full gate runs' };
if (!manifest?.entries) return { reuse: null, reason: 'this run has no readable plan; the full gate runs' };
const fingerprint = selectionFingerprint(manifest);
if (fingerprint !== candidate.selection_sha256) {
return { reuse: null, reason: `this head's selection ${fingerprint.slice(0, 12)} differs from the reused gate's ${candidate.selection_sha256.slice(0, 12)}; the full gate runs` };
}
return { reuse: candidate, reason: `reusing run ${candidate.reused_run_id}'s GREEN verdict at ${candidate.gated_head.slice(0, 12)} (${candidate.kind}): ${candidate.why}; same selection ${fingerprint.slice(0, 12)}` };
}
export function reuseReport(reuse: GateReuse, repo: string): string {
const evidence = reuse.kind === 'restamp-only'
? [`TREE: same-modulo-stamps receipt=${reuse.receipt_sha256}`, `stamp files: ${reuse.stamp_files!.join(', ')} (${reuse.old_version ?? '-'} -> ${reuse.new_version ?? '-'})`]
: [`OWN DIFF: identical sha256=${reuse.own_diff_sha256} (merge-base ${reuse.gated_merge_base} -> ${reuse.merge_base}; stamp files excluded)`];
return [
`GATE REUSED (${reuse.kind === 'restamp-only' ? 'R2a, restamp-only' : 'R2, main moved'}): no paid case ran on ${reuse.current_head}`,
`reused verdict: GREEN from run ${reuse.reused_run_id} attempt ${reuse.reused_run_attempt} at ${reuse.gated_head} (https://github.com/${repo}/actions/runs/${reuse.reused_run_id})`,
`why: ${reuse.why}`,
`base: ${reuse.gated_base} -> ${reuse.base_sha}`,
...evidence,
`selection: ${reuse.selection_sha256} (identical to the reused gate)`,
'cost: $0 (no trials executed)',
].join('\n');
}
const USAGE = `usage:
bun run scripts/gate-reuse.ts bind <report dir> --out <dir> --repo <owner/name> --pr <n> --run-id <id> --run-attempt <n>
--head-sha <sha> --tested-sha <merge sha> --profile <pr|full> --reconcile-exit <n>
bun run scripts/gate-reuse.ts find --repo <owner/name> --pr <n> --head-repo <owner/name> --branch <head branch> --run-id <id>
--head-sha <sha> --tested-sha <merge sha> --out <dir> [--budget-seconds 90]
bun run scripts/gate-reuse.ts confirm <candidate dir> <manifest.json> --out <paid-plan dir> [--github-output <file>]
bun run scripts/gate-reuse.ts report <gate-reuse.json> --repo <owner/name>
bind and find never fail the run; no candidate means the full gate runs.`;
if (import.meta.main) {
const [command, ...rest] = process.argv.slice(2);
const flag = (name: string) => { const i = rest.indexOf(`--${name}`); return i >= 0 ? rest[i + 1] : undefined; };
const int = (name: string) => Number(flag(name));
const usage = () => { console.error(USAGE); process.exit(2); };
if (command === 'bind') {
const [reportDir, out, repository, headSha, testedSha, profile] = [rest[0], flag('out'), flag('repo'), flag('head-sha'), flag('tested-sha'), flag('profile')];
if (!reportDir || !out || !repository || !headSha || !testedSha || !profile) usage();
const git = gitIn(process.cwd());
if (!commitParents(git, testedSha!)) git(['fetch', '--no-tags', '--depth=1', 'origin', testedSha!]);
const tested = testedBase(git, testedSha!, headSha!);
const { binding, reason } = buildBinding({ repository: repository!, pr: int('pr'), runId: int('run-id'), runAttempt: int('run-attempt'),
headSha: headSha!, baseSha: 'base' in tested ? tested.base : null, profile: profile!,
reconcileExit: /^\d+$/.test(flag('reconcile-exit') ?? '') ? int('reconcile-exit') : -1, reportDir: reportDir! });
if (binding) {
fs.mkdirSync(out!, { recursive: true });
fs.writeFileSync(path.join(out!, BINDING_FILE), `${JSON.stringify(binding, null, 2)}\n`);
}
console.log(`- Gate binding: ${reason}${'error' in tested ? ` (${tested.error})` : ''}`);
} else if (command === 'find') {
const [repo, headRepo, branch, headSha, testedSha, out] = [flag('repo'), flag('head-repo'), flag('branch'), flag('head-sha'), flag('tested-sha'), flag('out')];
if (!repo || !headRepo || !branch || !headSha || !testedSha || !out || !Number.isSafeInteger(int('pr')) || !Number.isSafeInteger(int('run-id'))) usage();
const { lines } = findReusableGate({ repo: repo!, pr: int('pr'), headRepo: headRepo!, branch: branch!, selfRunId: int('run-id'), headSha: headSha!,
testedSha: testedSha!, repoRoot: process.cwd(), out: out!, budgetMs: Number(flag('budget-seconds') ?? '90') * 1000 });
console.log('### Paid-gate verdict reuse (R2a restamp-only, R2 main moved)');
for (const line of lines) console.log(`- ${line}`);
} else if (command === 'confirm') {
const [dir, manifestPath, out] = [rest[0], rest[1], flag('out')];
if (!dir || !manifestPath || !out) usage();
const { reuse, reason } = confirmReuse(dir!, readJson<PaidRunManifest>(manifestPath!));
if (reuse) {
fs.mkdirSync(out!, { recursive: true });
fs.writeFileSync(path.join(out!, REUSE_FILE), `${JSON.stringify(reuse, null, 2)}\n`);
}
const output = flag('github-output');
if (output) fs.appendFileSync(output, `gate_reuse=${reuse ? 'true' : 'false'}\n`);
console.log(`- Paid-gate verdict reuse: ${reason}`);
} else if (command === 'report') {
const reuse = parseReuse(readJson(rest[0] ?? ''));
const repo = flag('repo');
if (!reuse || !repo) { console.error('gate-reuse report: the reuse record is missing or malformed; the gate is not green'); process.exit(1); }
console.log(reuseReport(reuse!, repo!));
} else usage();
}
+1 -1
View File
@@ -277,7 +277,7 @@ export function collectHealth(inputs: HealthInputs): HealthReport {
return { value: `${(total / 60_000 * 7 / sinceDays).toFixed(0)} min/week (${cancelled.length} cancelled run(s))` };
}));
metrics.push(metric('push-to-verdict', 'PR lane push-to-verdict', 'created_at to updated_at of evals.yml pull_request runs that reached success or failure (includes queue and debounce)', () => {
metrics.push(metric('push-to-verdict', 'PR lane push-to-verdict', 'created_at to updated_at of evals.yml pull_request runs that reached success or failure (includes queue and the readiness wait)', () => {
const walls = need(prRuns).filter(run => run.conclusion === 'success' || run.conclusion === 'failure').map(run => span(run.createdAt, run.updatedAt)!);
if (walls.length === 0) throw new Error('no PR run reached a verdict in the window');
return { value: `p50 ${minutes(percentile(walls, 0.5))}, p95 ${minutes(percentile(walls, 0.95))} (${walls.length} runs)` };
+32 -1
View File
@@ -1,7 +1,7 @@
import { expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { closePacketReadAfterNotice, prepareGuardSession } from './helpers/autoplan-guard-pty';
import { closePacketReadAfterNotice, prepareGuardSession, stoppedAtPhaseBoundary } from './helpers/autoplan-guard-pty';
const base = '/home/runner/.cache/gstack-paid-shard-x/tmp/ap-guard-pty-y';
@@ -33,3 +33,34 @@ test('the scripted actor ends its turn after a background CEO dispatch and reads
expect(skill.indexOf('\n6. ')).toBeGreaterThan(skill.indexOf('\n5. '));
} finally { s.cleanup(); }
});
// PR gate 39099195754 (ce1da4e, fail): the actor published Phase 1, printed "Phase 1 complete." and ended its
// turn, so the run idled to its 480 s deadline with 2 of 4 guarded allows (the same 489 s shape as 38976253774).
const STOP_SCREEN = fs.readFileSync(path.join(import.meta.dir, 'fixtures', 'autoplan-guard-pty-phase-boundary-stop-screen.txt'), 'utf8');
test('the captured phase-boundary stop is named; a run still working or already in Phase 2 is not', () => {
expect(STOP_SCREEN).toContain('Phase 1 complete.');
expect(stoppedAtPhaseBoundary(STOP_SCREEN)).toBe(true);
const working = STOP_SCREEN.slice(0, STOP_SCREEN.indexOf('✻ Brewed'));
expect(stoppedAtPhaseBoundary(working)).toBe(false);
const continued = working + '●Step8:ReadingtheDesignphasefile\n\nRead 1 file\n\n●Step9:ReadingtheDesigndispatchfile\n\n✻ Brewed for 80s · done 5:17 AM\n❯\n';
expect(stoppedAtPhaseBoundary(continued)).toBe(false);
expect(stoppedAtPhaseBoundary('✻ Brewed for 10s · done 5:00 AM\n❯\n')).toBe(false);
});
test('the scripted actor keeps going through the phase boundary: nothing between step 7 and step 11 lets it end its turn there', () => {
const s = prepareGuardSession(undefined);
try {
const skill = fs.readFileSync(path.join(s.project, '.claude', 'skills', fs.readdirSync(path.join(s.project, '.claude', 'skills'))[0]!, 'SKILL.md'), 'utf8');
const step = (n: number) => skill.slice(skill.indexOf(`\n${n}. `) + 1, skill.indexOf(`\n${n + 1}. `));
const intro = skill.slice(skill.indexOf('\n---\n', 4) + 5, skill.indexOf('\n1. '));
expect(intro).toContain('from step 1 through step 11 in one continuous run');
expect(intro).toContain('End your turn before step 11 only where step 5 or step 10 says to wait for a background agent.');
expect(intro).toContain('**Phase 1 complete.** is a phase boundary, not the end of the task.');
expect(step(7)).toContain('Do not end your turn after it: when its result arrives, go straight on to step 8.');
expect(step(8).startsWith("8. As your next assistant message, in the same turn as step 7's result (no user input is coming), Read the file")).toBe(true);
// The publication and the Phase 2 entry stay in separate messages: the guard's boundary is unchanged.
expect(step(7)).toContain('make exactly one tool call in that same message: the Bash command `true autoplan-published ceo`');
for (const n of [6, 7, 8, 9]) expect(step(n)).not.toMatch(/end your turn(?! after it)|wait for (its|a) completion/i);
} finally { s.cleanup(); }
});
+7 -1
View File
@@ -39,7 +39,7 @@ test('receipt transport: only recover-receipts restores this repository and PR\'
const emit = planner.find((s: any) => s.run?.includes('--emit-plan /tmp/paid-plan/manifest.json'));
expect(emit.env.EVALS_CACHE_DIR).toBe("${{ steps.reuse.outputs.reuse == 'on' && '/tmp/gstack-eval-input-cache' || '' }}");
const upload = planner.find((s: any) => s.with?.name === 'paid-plan');
expect(upload.with.path.trim().split('\n')).toEqual(['/tmp/paid-plan/manifest.json', '/tmp/paid-plan/receipts', '/tmp/paid-plan/store']);
expect(upload.with.path.trim().split('\n')).toEqual(['/tmp/paid-plan/manifest.json', '/tmp/paid-plan/receipts', '/tmp/paid-plan/store', '/tmp/paid-plan/gate-reuse.json']);
// Executors never restore or save a cache of their own: every slice sees the plan's one receipt set.
const executor = paid.jobs['eval-slices'].steps;
expect(executor.filter((s: any) => s.uses?.startsWith('actions/cache/'))).toHaveLength(0);
@@ -54,6 +54,12 @@ test('receipt transport: only recover-receipts restores this repository and PR\'
expect(save[0].with.key).toBe('eval-input-v1-${{ github.repository_id }}-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }}-merged');
// Only PR runs save; a pr_receipts dispatch reads the PR store read-only.
expect(save[0].if).toBe("always() && github.event_name == 'pull_request'");
// Restore and save share one cache backend (GSTA-48): an Ubicloud runner saves into Ubicloud's transparent
// cache, which a GitHub-hosted restore never sees, so every restore missed while the report ran on Ubicloud.
for (const job of Object.values(paid.jobs) as any[]) {
if (!job.steps.some((s: any) => s.uses?.startsWith('actions/cache/') && s.with?.path === '/tmp/gstack-eval-input-cache')) continue;
expect(job['runs-on']).toBe('ubuntu-24.04');
}
expect(report.indexOf(save[0])).toBeGreaterThan(report.indexOf(merge));
expect(paid.jobs['eval-slices'].permissions).toEqual({ contents: 'read', packages: 'read' });
expect(paid.jobs['slices-report'].permissions).toEqual({ contents: 'read' });
+11 -9
View File
@@ -75,9 +75,9 @@ describe('paid CI coordination stays off the eval image', () => {
test(`${name}: planning is independent of image startup and has no dependency install`, () => {
const planner = jobs['plan-slices'];
if (name === 'evals.yml') {
// The PR lane plans after its two tiny coordination jobs (push-burst debounce, base-ref receipt recovery), never after the image.
expect(planner.needs).toEqual(['debounce', 'recover-receipts']);
for (const id of ['debounce', 'recover-receipts']) {
// The PR lane plans after its two tiny coordination jobs (readiness, base-ref receipt recovery), never after the image.
expect(planner.needs).toEqual(['readiness', 'recover-receipts']);
for (const id of ['readiness', 'recover-receipts']) {
const job = jobs[id] as Job & { 'runs-on'?: string };
expect(job.container, id).toBeUndefined();
expect(job['runs-on'], id).toBe('ubuntu-24.04');
@@ -110,7 +110,8 @@ describe('paid CI coordination stays off the eval image', () => {
expect(executor.needs).toEqual(['build-image', 'plan-slices']);
expect(JSON.stringify(executor.container)).toContain('needs.build-image.outputs.image-tag');
if (name === 'evals.yml') {
expect(executor.if).toBe("${{ !cancelled() && needs.build-image.result == 'success' && needs.plan-slices.result == 'success' }}");
// A confirmed verdict reuse (R2a/R2) is the one PR run that executes no slice.
expect(executor.if).toBe("${{ !cancelled() && needs.build-image.result == 'success' && needs.plan-slices.result == 'success' && needs.plan-slices.outputs.gate_reuse != 'true' }}");
} else {
expect(executor.if).toBeUndefined();
}
@@ -165,12 +166,13 @@ describe('paid CI coordination stays off the eval image', () => {
test('PR planning preserves the fork and Dependabot trust boundaries across the coordination jobs', () => {
const jobs = workflows[0].jobs;
const trust = "github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)";
// !cancelled() lets planning proceed when debounce/recovery were skipped or failed (reuse is then off); the trust clause still gates it.
expect(jobs['plan-slices'].if).toBe(`\${{ !cancelled() && needs.debounce.outputs.superseded != 'true' && ${trust} }}`);
// !cancelled() lets planning proceed when readiness was skipped (dispatch) or recovery was skipped or failed (reuse is then off);
// a readiness failure (Free Tests red on this head) or a superseded head plans nothing; the trust clause still gates it.
expect(jobs['plan-slices'].if).toBe(`\${{ !cancelled() && (needs.readiness.result == 'success' || needs.readiness.result == 'skipped') && needs.readiness.outputs.superseded != 'true' && ${trust} }}`);
// The coordination jobs never run for Dependabot or for a fork PR either (recovery also serves an explicit pr_receipts dispatch).
expect(jobs.debounce.if).toContain("github.actor != 'dependabot[bot]'");
expect(jobs.debounce.if).toContain("github.event_name == 'pull_request'");
expect(jobs.debounce.if).toContain('github.event.pull_request.head.repo.full_name == github.repository');
expect(jobs.readiness.if).toContain("github.actor != 'dependabot[bot]'");
expect(jobs.readiness.if).toContain("github.event_name == 'pull_request'");
expect(jobs.readiness.if).toContain('github.event.pull_request.head.repo.full_name == github.repository');
expect(jobs['recover-receipts'].if).toContain("github.actor != 'dependabot[bot]'");
expect(jobs['recover-receipts'].if).toContain("(github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository");
expect(jobs['recover-receipts'].if).toContain("(github.event_name == 'workflow_dispatch' && inputs.pr_receipts != '')");
+129
View File
@@ -0,0 +1,129 @@
/**
* Readiness-gated dispatch (R1, GSTA-48), driven from scripted gh responses
* and a fake clock: no network, no sleeping.
*/
import { describe, expect, test } from 'bun:test';
import { freeTestsState, readinessDecision, waitForReadiness, type FreeTestsRun } from '../scripts/eval-readiness';
import type { GhRunner } from '../scripts/recover-receipts';
const HEAD = 'a'.repeat(40);
const NEWER = 'b'.repeat(40);
const CREATED = Date.parse('2026-10-18T00:00:00Z');
const free = (id: number, status: string, conclusion: string | null, over: Partial<FreeTestsRun> = {}): FreeTestsRun =>
({ id, status, conclusion, head_sha: HEAD, event: 'pull_request', ...over });
/** A gh that answers the run, Free Tests and PR reads from a per-poll script, recording each call. */
function fakeGh(polls: Array<{ free: FreeTestsRun[]; prHead?: string } | 'error'>, createdAt: string | null = new Date(CREATED).toISOString()) {
const calls: string[] = [];
let poll = -1;
const gh: GhRunner = (args) => {
const target = args.find(a => a.startsWith('repos/')) ?? '';
calls.push(target);
const ok = (body: unknown) => ({ status: 0, stdout: JSON.stringify(body), stderr: '' });
if (/\/actions\/runs\/\d+$/.test(target)) return createdAt ? ok({ created_at: createdAt }) : { status: 1, stdout: '', stderr: 'HTTP 502' };
if (target.endsWith('/actions/workflows/free-tests.yml/runs')) {
poll++;
const step = polls[Math.min(poll, polls.length - 1)]!;
return step === 'error' ? { status: 1, stdout: '', stderr: 'HTTP 502' } : ok({ workflow_runs: step.free });
}
if (/\/pulls\/\d+$/.test(target)) {
const step = polls[Math.min(poll, polls.length - 1)]!;
return ok({ head: { sha: step === 'error' ? HEAD : step.prHead ?? HEAD } });
}
return { status: 1, stdout: '', stderr: 'unexpected' };
};
return { gh, calls };
}
function clock(start: number) {
let t = start;
const sleeps: number[] = [];
return { now: () => t, sleep: (ms: number) => { sleeps.push(ms); t += ms; }, sleeps, at: () => t };
}
const base = { repo: 'garrytan/gstack', pr: 42, headSha: HEAD, runId: 900, quietMs: 10 * 60_000, deadlineMs: 80 * 60_000, pollMs: 60_000 };
describe('readiness decision', () => {
test('the newest Free Tests run on this exact head decides; other heads and events are ignored', () => {
expect(freeTestsState([], HEAD)).toEqual({ state: 'missing' });
expect(freeTestsState([free(1, 'completed', 'success', { head_sha: NEWER })], HEAD)).toEqual({ state: 'missing' });
expect(freeTestsState([free(1, 'completed', 'success', { event: 'push' })], HEAD)).toEqual({ state: 'missing' });
expect(freeTestsState([free(1, 'completed', 'failure'), free(2, 'in_progress', null)], HEAD)).toEqual({ state: 'running', runId: 2 });
expect(freeTestsState([free(2, 'completed', 'success'), free(1, 'completed', 'failure')], HEAD)).toEqual({ state: 'success', runId: 2 });
expect(freeTestsState([free(3, 'completed', 'cancelled')], HEAD)).toEqual({ state: 'failed', runId: 3, conclusion: 'cancelled' });
});
test('ready needs both a green Free Tests on the head and an elapsed window; a moved head is superseded', () => {
const green = { state: 'success', runId: 7 } as const;
expect(readinessDecision({ headSha: HEAD, prHead: HEAD, free: green, quietLeftMs: 0 }).kind).toBe('ready');
expect(readinessDecision({ headSha: HEAD, prHead: HEAD, free: green, quietLeftMs: 1 }).kind).toBe('wait');
expect(readinessDecision({ headSha: HEAD, prHead: HEAD, free: { state: 'running', runId: 7 }, quietLeftMs: 0 }).kind).toBe('wait');
expect(readinessDecision({ headSha: HEAD, prHead: HEAD, free: { state: 'missing' }, quietLeftMs: 0 }).kind).toBe('wait');
expect(readinessDecision({ headSha: HEAD, prHead: NEWER, free: green, quietLeftMs: 0 })).toEqual({ kind: 'superseded', by: NEWER });
const blocked = readinessDecision({ headSha: HEAD, prHead: HEAD, free: { state: 'failed', runId: 7, conclusion: 'failure' }, quietLeftMs: 0 });
expect(blocked.kind).toBe('blocked');
expect(blocked.kind === 'blocked' && blocked.reason).toContain('re-run Free Tests and then re-run this workflow');
});
});
describe('waitForReadiness', () => {
test('sleeps out the window with no API calls, then starts once Free Tests passes', () => {
const { gh, calls } = fakeGh([{ free: [free(5, 'in_progress', null)] }, { free: [free(5, 'completed', 'success')] }]);
const c = clock(CREATED + 5_000);
const { decision, lines } = waitForReadiness({ ...base, gh, now: c.now, sleep: c.sleep });
expect(decision.kind).toBe('ready');
// One run read, then nothing until the window closed at created + 10 min.
expect(calls[0]).toBe('repos/garrytan/gstack/actions/runs/900');
expect(c.sleeps[0]).toBe(10 * 60_000 - 5_000);
expect(calls.filter(call => call.endsWith('free-tests.yml/runs'))).toHaveLength(2);
expect(c.at()).toBe(CREATED + 11 * 60_000);
expect(lines.join('\n')).toContain('Free Tests run 5 passed');
});
test('a re-run keeps the original creation time, so an elapsed window is not waited again', () => {
const { gh } = fakeGh([{ free: [free(5, 'completed', 'success')] }]);
const c = clock(CREATED + 3 * 3600_000);
expect(waitForReadiness({ ...base, gh, now: c.now, sleep: c.sleep }).decision.kind).toBe('ready');
expect(c.sleeps).toEqual([]);
});
test('a quiet window of 0 (the evals-no-debounce label) still waits for Free Tests', () => {
const { gh } = fakeGh([{ free: [] }, { free: [free(5, 'queued', null)] }, { free: [free(5, 'completed', 'success')] }]);
const c = clock(CREATED);
expect(waitForReadiness({ ...base, quietMs: 0, gh, now: c.now, sleep: c.sleep }).decision.kind).toBe('ready');
expect(c.sleeps).toEqual([60_000, 60_000]);
});
test('Free Tests red on this head blocks the paid gate', () => {
const { gh } = fakeGh([{ free: [free(5, 'completed', 'failure')] }]);
const c = clock(CREATED);
const { decision } = waitForReadiness({ ...base, gh, now: c.now, sleep: c.sleep });
expect(decision.kind).toBe('blocked');
});
test('a newer head is reported superseded instead of starting', () => {
const { gh } = fakeGh([{ free: [free(5, 'completed', 'success')], prHead: NEWER }]);
const c = clock(CREATED);
const { decision, lines } = waitForReadiness({ ...base, gh, now: c.now, sleep: c.sleep });
expect(decision).toEqual({ kind: 'superseded', by: NEWER });
expect(lines.join('\n')).toContain('this run plans no slices');
});
test('a Free Tests run that never finishes, or a gh outage, fails at the deadline with the last reason', () => {
for (const polls of [[{ free: [free(5, 'in_progress', null)] }], ['error' as const]]) {
const { gh } = fakeGh(polls);
const c = clock(CREATED);
const { decision } = waitForReadiness({ ...base, gh, now: c.now, sleep: c.sleep });
expect(decision.kind).toBe('blocked');
expect(decision.kind === 'blocked' && decision.reason).toContain('not ready after 80 min');
expect(c.at() - CREATED).toBeLessThanOrEqual(80 * 60_000);
}
});
test('an unreadable run creation time measures the window from now', () => {
const { gh } = fakeGh([{ free: [free(5, 'completed', 'success')] }], null);
const c = clock(CREATED + 60_000);
expect(waitForReadiness({ ...base, gh, now: c.now, sleep: c.sleep }).decision.kind).toBe('ready');
expect(c.sleeps[0]).toBe(10 * 60_000);
});
});
+64 -16
View File
@@ -1,7 +1,8 @@
/**
* PR paid-lane workflow pins (W1, CEO-13/14/34, ENG-7/11/12, DX-7, W8f):
* coverage summaries, cancellation recovery off PR code, the push-burst
* debounce, dispatch inputs, and the absence of the old inline subset program.
* coverage summaries, cancellation recovery off PR code, readiness-gated
* dispatch (R1), verdict reuse wiring (R2a/R2), dispatch inputs, and the
* absence of the old inline subset program.
*/
import { describe, expect, test } from 'bun:test';
import * as fs from 'node:fs';
@@ -77,24 +78,30 @@ describe('evals.yml receipt recovery (CEO-14/34, ENG-7)', () => {
});
describe('evals.yml push-burst debounce (CEO-15, ENG-11, DX-7)', () => {
const debounce = workflow.jobs.debounce!;
describe('evals.yml readiness-gated dispatch (R1, GSTA-48)', () => {
const readiness = workflow.jobs.readiness!;
test('a tiny job debounces only synchronize pushes, skippable by label, before recovery and planning', () => {
expect(debounce['runs-on']).toBe('ubuntu-24.04');
expect(debounce.if).toContain("github.event.action == 'synchronize'");
expect(debounce.if).toContain("!contains(github.event.pull_request.labels.*.name, 'evals-no-debounce')");
expect(debounce.permissions).toEqual({ actions: 'read', 'pull-requests': 'read' });
const wait = step('debounce', 'Wait out a push burst')!.run!;
expect(wait).toContain("date -u -d '15 minutes ago'");
expect(wait).toContain('sleep 90');
expect(wait).toContain('superseded=true');
expect(wait).toContain('superseded by ${head}');
test('a tiny PR-only job waits for a quiet PR and green Free Tests before recovery and planning', () => {
expect(readiness['runs-on']).toBe('ubuntu-24.04');
expect(readiness.if).toBe("github.event_name == 'pull_request' && github.actor != 'dependabot[bot]' && github.event.pull_request.head.repo.full_name == github.repository");
expect(readiness.permissions).toEqual({ contents: 'read', actions: 'read', 'pull-requests': 'read' });
expect(Number(readiness['timeout-minutes'])).toBeGreaterThan(80);
const wait = step('readiness', 'Wait for a quiet PR and green Free Tests on this head')!;
expect(wait.id).toBe('wait');
expect(wait.run).toContain('scripts/eval-readiness.ts wait');
expect(wait.run).toContain('--deadline-minutes 80');
expect(wait.run).toContain('--github-output "$GITHUB_OUTPUT"');
expect(wait.env?.HEAD_SHA).toBe('${{ github.event.pull_request.head.sha }}');
// The window is configurable and the old debounce label still skips it; nothing skips the Free Tests requirement.
expect(wait.env?.QUIET_MINUTES).toBe("${{ contains(github.event.pull_request.labels.*.name, 'evals-no-debounce') && '0' || vars.EVALS_QUIET_MINUTES || '10' }}");
expect(readiness.outputs?.superseded).toBe('${{ steps.wait.outputs.superseded }}');
for (const job of ['recover-receipts', 'plan-slices']) {
expect(workflow.jobs[job]!.needs).toContain('debounce');
expect(workflow.jobs[job]!.if).toContain("needs.debounce.outputs.superseded != 'true'");
expect(workflow.jobs[job]!.needs).toContain('readiness');
expect(workflow.jobs[job]!.if).toContain("(needs.readiness.result == 'success' || needs.readiness.result == 'skipped')");
expect(workflow.jobs[job]!.if).toContain("needs.readiness.outputs.superseded != 'true'");
expect(workflow.jobs[job]!.if).toContain('!cancelled()');
}
expect(workflow.jobs.debounce).toBeUndefined();
});
test('cancel-in-progress stays keyed on the PR, and evals-fresh turns receipt reuse off', () => {
@@ -104,6 +111,47 @@ describe('evals.yml push-burst debounce (CEO-15, ENG-11, DX-7)', () => {
});
});
describe('evals.yml gate verdict reuse wiring (R2a restamp-only, R2 main moved; GSTA-48)', () => {
test('base-ref recovery proposes a candidate; it never fails the run', () => {
const find = step('recover-receipts', 'Find a gate verdict to reuse (R2a/R2, base-ref code)')!;
expect(find.if).toBe("github.event_name == 'pull_request'");
expect(find.env).toMatchObject({ GH_TOKEN: '${{ github.token }}', HEAD_SHA: '${{ github.event.pull_request.head.sha }}' });
expect(find.env?.BASE_SHA).toBeUndefined();
expect(find.run).toContain('scripts/gate-reuse.ts find');
// The base is the tested merge commit's first parent, never the event's base.sha.
expect(find.run).toContain('--tested-sha "$GITHUB_SHA"');
expect(find.run).not.toContain('--base-sha');
expect(find.run).toContain('exit 0');
expect(find.run).toContain('|| echo');
expect(step('recover-receipts', 'Hand a gate reuse candidate to plan-slices')?.with).toMatchObject({ name: 'gate-reuse-candidate', path: '/tmp/gate-reuse' });
});
test('the planner confirms the selection after its own plan, and a confirmed reuse runs no slice', () => {
const planner = workflow.jobs['plan-slices']!;
const names = planner.steps.map(s => s.name);
expect(names.indexOf('Confirm gate verdict reuse (R2a/R2)')).toBeGreaterThan(names.indexOf('Emit run manifest'));
expect(step('plan-slices', 'Confirm gate verdict reuse (R2a/R2)')!.run).toContain('gate-reuse.ts confirm /tmp/gate-reuse /tmp/paid-plan/manifest.json --out /tmp/paid-plan');
expect(planner.outputs?.gate_reuse).toBe('${{ steps.gate_reuse.outputs.gate_reuse }}');
expect(workflow.jobs['eval-slices']!.if).toContain("needs.plan-slices.outputs.gate_reuse != 'true'");
});
test('the report prints a reused verdict fail-closed and only executed runs bind a verdict', () => {
const report = workflow.jobs['slices-report']!;
expect(step('slices-report', 'Reconcile slices against the manifest (fail-closed)')!.if).toBe("needs.plan-slices.outputs.gate_reuse != 'true'");
const reused = step('slices-report', 'Report the reused gate verdict (R2a/R2)')!;
expect(reused.if).toBe("needs.plan-slices.outputs.gate_reuse == 'true'");
expect(reused.run).toContain('echo "exit=$code"');
expect(report.outputs?.['reconcile-exit']).toBe('${{ steps.reconcile.outputs.exit || steps.reused.outputs.exit }}');
expect(step('slices-report', 'Fail the workflow when reconciliation failed')!.if).toBe("(steps.reconcile.outputs.exit || steps.reused.outputs.exit) != '0'");
const bind = step('slices-report', "Bind this run's verdict for reuse (R2a/R2)")!;
expect(bind.if).toBe("always() && github.event_name == 'pull_request' && needs.plan-slices.outputs.gate_reuse != 'true'");
expect(bind.env?.RECONCILE_EXIT).toBe('${{ steps.reconcile.outputs.exit }}');
expect(bind.run).toContain('--tested-sha "$GITHUB_SHA"');
expect(bind.env?.BASE_SHA).toBeUndefined();
expect(step('slices-report', 'Upload the gate binding')?.with).toMatchObject({ name: 'gate-binding-a${{ github.run_attempt }}', path: '/tmp/gate-binding/gate-binding.json' });
});
});
describe('evals.yml carries no inline planner program (W8f)', () => {
test('the finished validation_phase dispatch and its inline subset planner are gone', () => {
expect(source).not.toContain('validation_phase');
@@ -0,0 +1,122 @@
pletion
Running1shellcommand…
·400
25
50
●
✢75
●Step2:ReadingtheCEOphasefile
Read 1 file
●Step3:Runningthereviewgenerationcommand
Ran 1 shell command
●Step4:ReadingtheCEOdispatchfile
Read 1 file
●Step5:CallingtheCEOreviewagentwiththedispatchinstructions
●Agent(CEOreview)
⎿  Backgrounded agent (↓ to manage · ctrl+o to expand)
●TheCEOreviewagentisrunninginthebackground.Waitingforitscompletionnotification.
✻Waiting for 1 background agent to finish
●Agent"CEOreview"finished · 27s
●Step6:Readingtheclosepacketfilecompletely
Read 1 file
●Step7:Publishingphase1completion
●Running1shellcommand…
⎿ $ true autoplan-published ceo
Wandering…
Wandering…512
*Wandering…837
✶62
✻87
Wandering…625
Wandering…
thinking)
✽43thinking
5thinking
60thinking
✻1thinking
●
4
9
Read 1 file
●Step3:Runningthereviewgenerationcommand
Ran 1 shell command
●Step4:ReadingtheCEOdispatchfile
Read 1 file
●Step5:CallingtheCEOreviewagentwiththedispatchinstructions
●Agent(CEOreview)
⎿  Backgrounded agent (↓ to manage · ctrl+o to expand)
●TheCEOreviewagentisrunninginthebackground.Waitingforitscompletionnotification.
✻Waiting for 1 background agent to finish
●Agent"CEOreview"finished · 27s
●Step6:Readingtheclosepacketfilecompletely
Read 1 file
●Step7:Publishingphase1completion
Ran 1 shell command
● Phase 1 complete.
●Step2:ReadingtheCEOphasefile
Read 1 file
●Step3:Runningthereviewgenerationcommand
Ran 1 shell command
●Step4:ReadingtheCEOdispatchfile
Read 1 file
●Step5:CallingtheCEOreviewagentwiththedispatchinstructions
● Agent(CEO review)
⎿  Backgroundedagent (↓ to manage · ctrl+o to expand)
●TheCEOreviewagentisrunninginthebackground.Waitingforitscompletionnotification.
✻Waiting for 1 background agent to finish
●Agent"CEOreview"finished · 27s
●Step6:Readingtheclosepacketfilecompletely
Read 1 file
●Step7:Publishingphase1completion
Ran 1 shell command
●Phase 1 complete.
✻ Brewed for 50s · done 5:16 AM
❯
← for agents
? for shrtcut · ← foragents
+490
View File
@@ -0,0 +1,490 @@
/**
* Paid-gate verdict reuse (R2a restamp-only, R2 main moved; GSTA-48): a gate
* edit, so every refusal is a negative control. Real git fixtures (a bare origin, the base-ref receipt
* checkout) and scripted gh responses: no network, no API spend.
*/
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
import { spawnSync } from 'node:child_process';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import {
BINDING_FILE, CANDIDATE_FILE, REUSE_FILE, buildBinding, confirmReuse, findReusableGate, parseReuse, reuseReport, selectionFingerprint,
type GateBinding, type GateReuse,
} from '../scripts/gate-reuse';
import type { GhRunner, WorkflowRun } from '../scripts/recover-receipts';
import type { PaidRunManifest } from '../scripts/lib/paid-plan';
const ROOT = path.resolve(import.meta.dir, '..');
const REPO = 'garrytan/gstack';
const scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'gate-reuse-'));
afterAll(() => fs.rmSync(scratch, { recursive: true, force: true }));
function git(cwd: string, ...args: string[]): string {
const r = spawnSync('git', ['-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.com', '-c', 'commit.gpgsign=false', ...args],
{ cwd, encoding: 'utf8', timeout: 30_000 });
if (r.status !== 0) throw new Error('git ' + args.join(' ') + ': ' + r.stderr);
return r.stdout.trim();
}
const changelog = (sections: string[]) => ['# Changelog', '', ...sections, '## [1.0.0.0] - 2026-10-01', '', '- base release', ''].join('\n');
const files = (version: string, over: Record<string, string> = {}): Record<string, string> => ({
VERSION: version + '\n',
'package.json': JSON.stringify({ name: 'fixture', version, scripts: { test: 'bun test' } }, null, 2) + '\n',
'agents-digest/gstack-AGENTS.md': '# gstack digest v' + version + ' — regenerate after upgrading\n\nbody\n',
'src/feature.ts': 'export const feature = 1;\n',
...over,
});
let origin: string;
let receipt: string;
const sha: Record<string, string> = {};
function commit(work: string, branch: string, from: string, content: Record<string, string>, message: string): string {
if (from) git(work, 'checkout', '-q', '-B', branch, from);
else git(work, 'checkout', '-q', '--orphan', branch);
for (const [rel, text] of Object.entries(content)) {
fs.mkdirSync(path.dirname(path.join(work, rel)), { recursive: true });
fs.writeFileSync(path.join(work, rel), text);
}
git(work, 'add', '-A');
git(work, 'commit', '-q', '-m', message);
git(work, 'push', '-q', 'origin', branch);
return git(work, 'rev-parse', 'HEAD');
}
beforeAll(() => {
origin = path.join(scratch, 'origin.git');
git(scratch, 'init', '-q', '--bare', origin);
git(origin, 'config', 'uploadpack.allowAnySHA1InWant', 'true');
const work = path.join(scratch, 'work');
git(scratch, 'clone', '-q', origin, work);
const policy = fs.readFileSync(path.join(ROOT, '.gstack/ship-policy.json'), 'utf8');
sha.base = commit(work, 'main', '', { ...files('1.0.0.0', { 'src/feature.ts': 'export const feature = 0;\n' }), 'CHANGELOG.md': changelog([]),
'.gstack/ship-policy.json': policy }, 'base');
const entry = ['## [1.0.1.0] - 2026-10-02', '', '- the feature', ''];
sha.gated = commit(work, 'feature', sha.base, { ...files('1.0.1.0'), 'CHANGELOG.md': changelog(entry) }, 'feature, tentative stamp');
sha.restamp = commit(work, 'restamp', sha.gated, { ...files('1.0.2.0'), 'CHANGELOG.md': changelog(['## [1.0.2.0] - 2026-10-03', '', '- the feature', '']) }, 'restamp');
sha.code = commit(work, 'code', sha.restamp, { 'src/feature.ts': 'export const feature = 2;\n' }, 'restamp plus a code byte');
sha.entry = commit(work, 'entry', sha.gated, { ...files('1.0.2.0'), 'CHANGELOG.md': changelog(['## [1.0.2.0] - 2026-10-03', '', '- the feature, reworded', '']) }, 'restamp plus entry text');
sha.lock = commit(work, 'lock', sha.gated, { 'bun.lock': '{\n "lockfileVersion": 1\n}\n' }, 'a non-stamp file beside the stamp');
const lock = (version: string) => '{\n "lockfileVersion": 1,\n "workspaces": {\n "": {\n "name": "fixture",\n "version": "' + version + '",\n },\n },\n}\n';
sha.gatedLock = commit(work, 'gated-lock', sha.gated, { 'bun.lock': lock('1.0.1.0') }, 'a lockfile carrying the root version');
sha.lockStamp = commit(work, 'lock-stamp', sha.gatedLock, { ...files('1.0.2.0'), 'CHANGELOG.md': changelog(['## [1.0.2.0] - 2026-10-03', '', '- the feature', '']),
'bun.lock': lock('1.0.2.0') }, 'restamp that also moves the lockfile root version');
sha.moved = commit(work, 'main', sha.base, { 'README.md': 'main moved\n' }, 'main moves');
git(work, 'checkout', '-q', '-B', 'merged', sha.gated);
git(work, 'merge', '-q', '--no-edit', sha.moved);
git(work, 'push', '-q', 'origin', 'merged');
sha.merged = git(work, 'rev-parse', 'HEAD');
sha.rebased = commit(work, 'rebased', sha.moved, { ...files('1.0.1.0'), 'CHANGELOG.md': changelog(entry) }, 'the feature, rebased onto moved main');
sha.rebasedCode = commit(work, 'rebased-code', sha.rebased, { 'src/feature.ts': 'export const feature = 3;\n' }, 'rebased plus a code byte');
sha.rebasedEntry = commit(work, 'rebased-entry', sha.rebased, { 'CHANGELOG.md': changelog(['## [1.0.1.0] - 2026-10-02', '', '- the feature, reworded', '']) }, 'rebased plus entry text');
sha.rebasedPkg = commit(work, 'rebased-pkg', sha.rebased, { 'package.json': JSON.stringify({ name: 'fixture', version: '1.0.1.0', scripts: { test: 'bun test --bail' } }, null, 2) + '\n' }, 'rebased plus a package script');
sha.rebasedDigest = commit(work, 'rebased-digest', sha.rebased, { 'agents-digest/gstack-AGENTS.md': '# gstack digest v1.0.1.0 — regenerate after upgrading\n\nbody\nhand-written instruction\n' }, 'rebased plus release-output content');
sha.rebasedDrop = commit(work, 'rebased-drop', sha.rebased, { 'CHANGELOG.md': ['# Changelog', '', ...entry].join('\n') }, 'rebased plus a deleted CHANGELOG section');
sha.rebasedPreamble = commit(work, 'rebased-preamble', sha.rebased, { 'CHANGELOG.md': changelog(entry).replace('# Changelog', '# Changelog (edited)') }, 'rebased plus a CHANGELOG preamble edit');
const protocol = (version: string, protocolVersion: string) => ({ 'agents-digest/gstack-AGENTS.md': '# gstack digest v' + version + ' — regenerate after upgrading\n\nbody\ncompatible with protocol ' + protocolVersion + '\n' });
sha.gatedProtocol = commit(work, 'gated-protocol', sha.base, { ...files('1.0.1.0', protocol('1.0.1.0', '1.0.1.0')), 'CHANGELOG.md': changelog(entry) }, 'a digest line that repeats the version as content');
sha.restampProtocol = commit(work, 'restamp-protocol', sha.moved, { ...files('1.0.2.0', protocol('1.0.2.0', '1.0.2.0')), 'CHANGELOG.md': changelog(['## [1.0.2.0] - 2026-10-03', '', '- the feature', '']) },
'rebased restamp that also changes the content line');
sha.restampProtocolKept = commit(work, 'restamp-protocol-kept', sha.moved, { ...files('1.0.2.0', protocol('1.0.2.0', '1.0.1.0')), 'CHANGELOG.md': changelog(['## [1.0.2.0] - 2026-10-03', '', '- the feature', '']) },
'rebased restamp that keeps the content line');
git(work, 'checkout', '-q', '-B', 'rebased-mode', sha.rebased);
git(work, 'update-index', '--chmod=+x', 'package.json');
git(work, 'commit', '-q', '-m', 'rebased plus a package.json mode change');
git(work, 'push', '-q', 'origin', 'rebased-mode');
git(work, 'reset', '-q', '--hard');
sha.rebasedMode = git(work, 'rev-parse', 'HEAD');
sha.mainTouch = commit(work, 'main-touch', sha.base, { 'src/feature.ts': 'export const feature = 0;\n// main note\n' }, 'main edits the file the PR edits');
sha.touchRebased = commit(work, 'touch-rebased', sha.mainTouch, { ...files('1.0.1.0', { 'src/feature.ts': 'export const feature = 1;\n// main note\n' }), 'CHANGELOG.md': changelog(entry) },
'the feature, rebased onto a main that edited its file');
const digest = (version: string, body: string) => ({ 'agents-digest/gstack-AGENTS.md': '# gstack digest v' + version + ' — regenerate after upgrading\n\nbody\n' + body });
const restampLog = { 'CHANGELOG.md': changelog(['## [1.0.2.0] - 2026-10-03', '', '- the feature', '']) };
sha.gatedDash = commit(work, 'gated-dash', sha.base, { ...files('1.0.1.0', digest('1.0.1.0', '-- old banner\n')), 'CHANGELOG.md': changelog(entry) }, 'a digest line that looks like a diff header');
sha.dashReplace = commit(work, 'dash-replace', sha.gatedDash, { ...files('1.0.2.0', digest('1.0.2.0', '++ new banner\n')), ...restampLog }, 'restamp that replaces a -- line with a ++ line');
sha.dashRemove = commit(work, 'dash-remove', sha.gatedDash, { ...files('1.0.2.0', digest('1.0.2.0', '')), ...restampLog }, 'restamp that removes a -- line');
sha.dashAdd = commit(work, 'dash-add', sha.gated, { ...files('1.0.2.0', digest('1.0.2.0', '++ ordinary non-version body text\n')), ...restampLog }, 'restamp that adds a ++ line');
git(work, 'checkout', '-q', '-B', 'mode', sha.restamp);
git(work, 'update-index', '--chmod=+x', 'agents-digest/gstack-AGENTS.md');
git(work, 'commit', '-q', '-m', 'restamp plus a digest mode change');
git(work, 'push', '-q', 'origin', 'mode');
sha.mode = git(work, 'rev-parse', 'HEAD');
receipt = path.join(scratch, 'receipt');
git(scratch, 'clone', '-q', '--depth=1', 'file://' + origin, receipt);
git(receipt, 'fetch', '-q', '--depth=1', 'origin', sha.base);
git(receipt, 'checkout', '-q', '--detach', sha.base);
});
const binding = (over: Partial<GateBinding> = {}): GateBinding => ({ schema: 1, repository: REPO, pr: 42, run_id: 300, run_attempt: 1,
head_sha: sha.gated!, base_sha: sha.base!, profile: 'pr', reconcile_exit: 0, verdict: 'GREEN', manual_accepted: 0, selection_sha256: 'f'.repeat(64), ...over });
const run = (id: number, head: string, conclusion: string | null = 'success', over: Partial<WorkflowRun> = {}): WorkflowRun => ({
id, status: 'completed', conclusion, event: 'pull_request', head_sha: head, run_attempt: 1, head_repository: { full_name: REPO },
pull_requests: [{ number: 42 }], ...over });
type Jobs = Record<number, Array<{ name: string; conclusion: string | null }>>;
/**
* gh: the run list, each run's artifact and job lists (a bound run ran its
* slices; an unbound one ran none unless scripted), downloads that write the
* scripted binding (or raw text), and merge-bases from the fixture repo.
*/
function fakeGh(runs: WorkflowRun[], bindings: Record<number, GateBinding | string>, failList = '', jobs: Jobs = {}, failCompare = false): GhRunner {
return (args) => {
const ok = (body: unknown) => ({ status: 0, stdout: JSON.stringify(body), stderr: '' });
if (args.some(a => a.endsWith('/workflows/evals.yml/runs'))) return failList ? { status: 1, stdout: '', stderr: failList } : ok({ workflow_runs: runs });
const jobList = args.find(a => /\/runs\/\d+\/jobs$/.test(a));
if (jobList) {
const id = Number(/runs\/(\d+)\//.exec(jobList)![1]);
return ok({ jobs: [{ name: 'plan-slices', conclusion: 'success' }, ...(jobs[id] ?? (id in bindings ? [{ name: 'eval-slices (1)', conclusion: 'success' }] : []))] });
}
const compare = args.find(a => /\/compare\/[0-9a-f]{40}\.\.\.[0-9a-f]{40}$/.test(a));
if (compare) {
if (failCompare) return { status: 1, stdout: '', stderr: 'HTTP 404: Not Found' };
const [, a, b] = /compare\/([0-9a-f]{40})\.\.\.([0-9a-f]{40})$/.exec(compare)!;
return { status: 0, stdout: git(path.join(scratch, 'work'), 'merge-base', a!, b!) + '\n', stderr: '' };
}
const listed = args.find(a => /\/runs\/\d+\/artifacts$/.test(a));
if (listed) {
const id = Number(/runs\/(\d+)\//.exec(listed)![1]);
return ok({ artifacts: id in bindings ? [{ name: 'gate-binding-a1', expired: false }, { name: 'paid-plan', expired: false }] : [{ name: 'paid-plan', expired: false }] });
}
if (args[0] === 'run' && args[1] === 'download') {
const dir = args[args.indexOf('-D') + 1]!;
fs.mkdirSync(dir, { recursive: true });
const b = bindings[Number(args[2])]!;
fs.writeFileSync(path.join(dir, BINDING_FILE), typeof b === 'string' ? b : JSON.stringify(b));
return { status: 0, stdout: '', stderr: '' };
}
return { status: 1, stdout: '', stderr: 'unexpected gh ' + args.join(' ') };
};
}
/** The tested commit of a pull_request run: the merge of the head into the base (first parent), pushed like refs/pull/N/merge. */
const merges = new Map<string, string>();
function tested(base: string, head: string): string {
const key = base + head;
if (!merges.has(key)) {
const work = path.join(scratch, 'work');
const merge = git(work, 'commit-tree', head + '^{tree}', '-p', base, '-p', head, '-m', 'merge');
git(work, 'push', '-q', 'origin', merge + ':refs/heads/pull-merge-' + merges.size);
merges.set(key, merge);
}
return merges.get(key)!;
}
let outCount = 0;
function find(head: string, runs: WorkflowRun[], bindings: Record<number, GateBinding | string>,
over: { baseSha?: string; failList?: string; jobs?: Jobs; failCompare?: boolean; testedSha?: string } = {}) {
const out = path.join(scratch, 'out-' + ++outCount);
const result = findReusableGate({ repo: REPO, pr: 42, headRepo: REPO, branch: 'feature', selfRunId: 500, headSha: head,
testedSha: over.testedSha ?? tested(over.baseSha ?? sha.base!, head), repoRoot: receipt, out,
gh: fakeGh(runs, bindings, over.failList, over.jobs, over.failCompare) });
return { ...result, out, text: result.lines.join('\n') };
}
describe('find: the positive case', () => {
test('a restamp of a GREEN head on the same base yields a candidate with the receipt', () => {
const { candidate, out, text } = find(sha.restamp!, [run(300, sha.gated!)], { 300: binding() });
expect(text).toContain('TREE: same-modulo-stamps');
expect(candidate).not.toBeNull();
expect(candidate!.kind).toBe('restamp-only');
expect(candidate!.stamp_files!.sort()).toEqual(['CHANGELOG.md', 'VERSION', 'agents-digest/gstack-AGENTS.md', 'package.json']);
expect(candidate!).toMatchObject({ reused_run_id: 300, gated_head: sha.gated, current_head: sha.restamp, base_sha: sha.base,
old_version: '1.0.1.0', new_version: '1.0.2.0', selection_sha256: 'f'.repeat(64) });
expect(candidate!.receipt_sha256).toMatch(/^[0-9a-f]{64}$/);
expect(parseReuse(JSON.parse(fs.readFileSync(path.join(out, CANDIDATE_FILE), 'utf8')))).toEqual(candidate);
});
test('newer runs whose paid slices never finished are passed over to the newest executed gate', () => {
const { candidate } = find(sha.restamp!, [run(330, sha.gated!, 'cancelled'), run(320, sha.gated!, 'cancelled'), run(310, sha.gated!, 'success'),
run(305, sha.gated!, 'failure'), run(300, sha.gated!)], { 300: binding() },
{ jobs: { 330: [{ name: 'eval-slices (1)', conclusion: 'cancelled' }, { name: 'eval-slices (2)', conclusion: 'skipped' }],
310: [{ name: 'eval-slices', conclusion: 'skipped' }], 305: [{ name: 'readiness', conclusion: 'failure' }] } });
expect(candidate?.reused_run_id).toBe(300);
});
});
describe('find: main moved (R2)', () => {
test('a head merged with the moved main, or rebased onto it, reuses the gate when its own diff is byte-identical', () => {
for (const head of [sha.merged!, sha.rebased!]) {
const { candidate, text } = find(head, [run(300, sha.gated!)], { 300: binding() }, { baseSha: sha.moved });
expect(text).toContain('OWN DIFF: identical');
expect(candidate).toMatchObject({ kind: 'main-moved', reused_run_id: 300, gated_head: sha.gated, current_head: head,
gated_base: sha.base, base_sha: sha.moved, gated_merge_base: sha.base });
expect(candidate!.why).toContain('main moved');
expect(candidate!.own_diff_sha256).toMatch(/^[0-9a-f]{64}$/);
expect(parseReuse(candidate)).toEqual(candidate);
}
});
test('a version-equal content line in a release output: only the stamp is normalised, so keeping the line reuses and changing it does not', () => {
const gated = binding({ head_sha: sha.gatedProtocol });
expect(find(sha.restampProtocolKept!, [run(300, sha.gatedProtocol!)], { 300: gated }, { baseSha: sha.moved }).candidate?.kind).toBe('main-moved');
const changed = find(sha.restampProtocol!, [run(300, sha.gatedProtocol!)], { 300: gated }, { baseSha: sha.moved });
expect(changed.candidate).toBeNull();
expect(changed.text).toContain("the PR's own diff (excluding stamp files) differs");
});
test('a restamp on a moved main: stamp files are outside the own-diff compare', () => {
const { candidate } = find(sha.restamp!, [run(300, sha.gated!)], { 300: binding() }, { baseSha: sha.moved });
expect(candidate?.kind).toBe('main-moved');
});
});
describe('find: negative controls (each forces the full gate)', () => {
const refused = (r: { candidate: GateReuse | null; out: string; text: string }, reason: string | RegExp) => {
expect(r.candidate).toBeNull();
expect(fs.existsSync(path.join(r.out, CANDIDATE_FILE))).toBe(false);
if (typeof reason === 'string') expect(r.text).toContain(reason);
else expect(r.text).toMatch(reason);
expect(r.text).toContain('the full gate runs');
};
test('a non-stamp byte change', () => {
refused(find(sha.code!, [run(300, sha.gated!)], { 300: binding() }), 'differs from the gated head beyond stamps: src/feature.ts');
});
test('CHANGELOG entry text, not only its heading', () => {
refused(find(sha.entry!, [run(300, sha.gated!)], { 300: binding() }), 'beyond stamps: CHANGELOG.md');
});
test('a file outside the stamp set', () => {
refused(find(sha.lock!, [run(300, sha.gated!)], { 300: binding() }), 'bun.lock');
// The receipt calls a lockfile root-version bump a stamp; R2a's stamp set is narrower and still refuses it.
refused(find(sha.lockStamp!, [run(300, sha.gatedLock!)], { 300: binding({ head_sha: sha.gatedLock }) }),
'stamp-classified files outside the version file, package.json, CHANGELOG and release outputs: bun.lock');
});
test('main moved and the own diff changed: a code byte, CHANGELOG entry text, a package.json field', () => {
for (const head of [sha.rebasedCode!, sha.rebasedEntry!, sha.rebasedPkg!]) {
refused(find(head, [run(300, sha.gated!)], { 300: binding() }, { baseSha: sha.moved }), "the PR's own diff (excluding stamp files) differs");
}
});
test('main moved and a stamp file changed beyond its stamp: release-output content, a deleted CHANGELOG section, a CHANGELOG preamble edit, a package.json mode', () => {
for (const head of [sha.rebasedDigest!, sha.rebasedDrop!, sha.rebasedPreamble!, sha.rebasedMode!]) {
refused(find(head, [run(300, sha.gated!)], { 300: binding() }, { baseSha: sha.moved }), "the PR's own diff (excluding stamp files) differs");
}
});
test('main moved and changed a file the PR also changes: the pre-image differs, so the own diff is not byte-identical', () => {
refused(find(sha.touchRebased!, [run(300, sha.gated!)], { 300: binding() }, { baseSha: sha.mainTouch }), 'OWN DIFF: changed');
});
test('main moved and a merge-base or the tested merge commit is unavailable', () => {
refused(find(sha.rebased!, [run(300, sha.gated!)], { 300: binding() }, { baseSha: sha.moved, failCompare: true }), 'a merge-base is unavailable');
refused(find(sha.rebased!, [run(300, sha.gated!)], { 300: binding() }, { testedSha: sha.rebased }), 'is not the merge of');
refused(find(sha.rebased!, [run(300, sha.gated!)], { 300: binding() }, { testedSha: 'd'.repeat(40) }), /cannot fetch|not readable/);
});
test('main moved and the newest executed gate is red, partial or a different profile', () => {
refused(find(sha.rebased!, [run(300, sha.gated!)], { 300: binding({ verdict: 'RED' }) }, { baseSha: sha.moved }), 'is RED');
refused(find(sha.rebased!, [run(300, sha.gated!, 'cancelled')], { 300: binding() }, { baseSha: sha.moved }), 'conclusion cancelled');
refused(find(sha.rebased!, [run(300, sha.gated!)], { 300: binding({ profile: 'full' }) }, { baseSha: sha.moved }), 'not the PR gate');
});
test('a newer run that finished a paid slice but bound no verdict blocks an older GREEN (never skipped)', () => {
for (const conclusion of ['failure', 'timed_out', 'success']) {
for (const runConclusion of ['failure', 'cancelled']) {
refused(find(sha.rebased!, [run(310, sha.merged!, runConclusion), run(300, sha.gated!)], { 300: binding() },
{ baseSha: sha.moved, jobs: { 310: [{ name: 'eval-slices (1)', conclusion }, { name: 'eval-slices (2)', conclusion: 'cancelled' }] } }),
'newer run 310');
}
}
refused(find(sha.restamp!, [run(310, sha.gated!, 'success'), run(300, sha.gated!)], { 300: binding() },
{ jobs: { 310: [{ name: 'eval-slices (1)', conclusion: 'success' }] } }), 'finished a paid slice (success) but bound no verdict');
});
test('release-output lines that look like diff headers are content, not headers (receipt parser)', () => {
for (const head of [sha.dashReplace!, sha.dashRemove!]) {
refused(find(head, [run(300, sha.gatedDash!)], { 300: binding({ head_sha: sha.gatedDash }) }), 'beyond stamps: agents-digest/gstack-AGENTS.md');
}
refused(find(sha.dashAdd!, [run(300, sha.gated!)], { 300: binding() }), 'beyond stamps: agents-digest/gstack-AGENTS.md');
});
test('a file mode change beside a restamp', () => {
refused(find(sha.mode!, [run(300, sha.gated!)], { 300: binding() }), 'beyond stamps: agents-digest/gstack-AGENTS.md');
});
test('a GREEN that includes manually accepted (unscored) cases', () => {
refused(find(sha.restamp!, [run(300, sha.gated!)], { 300: binding({ manual_accepted: 1 }) }), 'manually accepted (unscored)');
refused(find(sha.restamp!, [run(300, sha.gated!)], { 300: binding({ manual_accepted: -1 }) }), 'missing or malformed');
});
test('a newer run that holds slice results, or is still running, blocks an older GREEN', () => {
const gh = fakeGh([run(310, sha.gated!, 'cancelled'), run(300, sha.gated!)], { 300: binding() });
const withSlice: GhRunner = (args, t) => args.some(a => a.endsWith('/runs/310/artifacts'))
? { status: 0, stdout: JSON.stringify({ artifacts: [{ name: 'paid-plan' }, { name: 'paid-slice-1-a1' }] }), stderr: '' } : gh(args, t);
const out = path.join(scratch, 'out-slice');
const r = findReusableGate({ repo: REPO, pr: 42, headRepo: REPO, branch: 'feature', selfRunId: 500, headSha: sha.restamp!,
testedSha: tested(sha.base!, sha.restamp!), repoRoot: receipt, out, gh: withSlice });
refused({ ...r, out, text: r.lines.join('\n') }, 'holds paid slice results (paid-slice-1-a1) but bound no verdict');
refused(find(sha.restamp!, [run(310, sha.gated!, null, { status: 'in_progress' }), run(300, sha.gated!)], { 300: binding() }), 'is still in_progress');
});
test('a missing receipt: no executed gate bound a verdict', () => {
refused(find(sha.restamp!, [run(300, sha.gated!)], {}), 'no earlier executed gate on this PR bound a verdict');
refused(find(sha.restamp!, [], {}), 'no earlier executed gate');
});
test('a missing receipt: a malformed or mismatched binding', () => {
refused(find(sha.restamp!, [run(300, sha.gated!)], { 300: '{"schema":1' }), 'missing or malformed');
refused(find(sha.restamp!, [run(300, sha.gated!)], { 300: binding({ run_id: 299 }) }), 'does not match the run it came from');
refused(find(sha.restamp!, [run(300, sha.restamp!)], { 300: binding() }), 'does not match the run it came from');
});
test('a missing receipt: the gated head cannot be fetched, so no tree receipt exists', () => {
const ghost = 'e'.repeat(40);
refused(find(sha.restamp!, [run(300, ghost)], { 300: binding({ head_sha: ghost }) }), /cannot fetch|no tree receipt/);
});
test('a red, partial or failed gate, and a newer red that blocks an older green', () => {
refused(find(sha.restamp!, [run(300, sha.gated!)], { 300: binding({ verdict: 'RED' }) }), 'is RED');
refused(find(sha.restamp!, [run(300, sha.gated!)], { 300: binding({ reconcile_exit: 1 }) }), 'reconcile exit 1');
refused(find(sha.restamp!, [run(300, sha.gated!, 'failure')], { 300: binding() }), 'conclusion failure');
refused(find(sha.restamp!, [run(301, sha.gated!), run(300, sha.gated!)], { 301: binding({ run_id: 301, verdict: 'RED' }), 300: binding() }), 'run 301');
});
test('a full-profile binding, the same head, another PR, and a gh outage', () => {
refused(find(sha.restamp!, [run(300, sha.gated!)], { 300: binding({ profile: 'full' }) }), 'not the PR gate');
refused(find(sha.gated!, [run(300, sha.gated!)], { 300: binding() }), 'gated this same head');
refused(find(sha.restamp!, [run(300, sha.gated!, 'success', { pull_requests: [{ number: 7 }] })], { 300: binding() }), 'no earlier executed gate');
refused(find(sha.restamp!, [run(300, sha.gated!)], { 300: binding() }, { failList: 'HTTP 403: Resource not accessible' }), 'not authorized');
});
});
const manifest = (over: Partial<PaidRunManifest> = {}): PaidRunManifest => ({ version: 1, tier: 'gate', evalsAll: false, sliceCount: 2,
selectionReason: 'diff', profile: 'pr', selection: { e2e: ['case-a'], judges: [] }, prCoverage: { mode: 'pr' } as PaidRunManifest['prCoverage'],
entries: [
{ file: 'test/a.test.ts', slice: 1, status: 'planned', estimatedMs: 1000 },
{ file: 'test/b.test.ts#case-b~t1', slice: 2, status: 'planned', trial: { kind: 'behavior', panel: { n: 3, k: 2 }, quarantined: false } },
{ file: 'test/c.test.ts', slice: 0, status: 'skipped-by-diff', reason: 'no input changed' },
], ...over } as PaidRunManifest);
describe('selection fingerprint and confirm', () => {
test('slice packing and estimates do not change the fingerprint; cases, statuses and trial plans do', () => {
const fp = selectionFingerprint(manifest());
const repacked = manifest();
repacked.entries = [...repacked.entries].reverse().map(e => ({ ...e, slice: e.slice ? 3 - e.slice : 0, estimatedMs: 5, reason: 'other' }));
expect(selectionFingerprint(repacked)).toBe(fp);
const extra = manifest();
extra.entries = [...extra.entries, { file: 'test/d.test.ts', slice: 1, status: 'planned' }];
const quarantined = manifest();
quarantined.entries = quarantined.entries.map(e => e.trial ? { ...e, trial: { ...e.trial, quarantined: true } } : e);
const skipped = manifest();
skipped.entries = skipped.entries.map(e => e.file === 'test/a.test.ts' ? { ...e, status: 'skipped-by-diff' as const } : e);
for (const changed of [extra, quarantined, skipped, manifest({ selection: { e2e: ['case-a', 'case-z'], judges: [] } }), manifest({ evalsAll: true })]) {
expect(selectionFingerprint(changed)).not.toBe(fp);
}
});
test('confirm reuses only when the candidate exists and the selection is identical', () => {
const { out } = find(sha.restamp!, [run(300, sha.gated!)], { 300: binding({ selection_sha256: selectionFingerprint(manifest()) }) });
const same = confirmReuse(out, manifest());
expect(same.reuse?.reused_run_id).toBe(300);
expect(same.reason).toContain('same selection');
const extra = manifest();
extra.entries = [...extra.entries, { file: 'test/d.test.ts', slice: 1, status: 'planned' }];
expect(confirmReuse(out, extra).reuse).toBeNull();
expect(confirmReuse(out, extra).reason).toContain('differs from the reused gate');
expect(confirmReuse(out, null).reuse).toBeNull();
expect(confirmReuse(path.join(scratch, 'no-candidate'), manifest()).reuse).toBeNull();
const forged = path.join(scratch, 'forged');
fs.mkdirSync(forged, { recursive: true });
fs.writeFileSync(path.join(forged, CANDIDATE_FILE), JSON.stringify({ ...JSON.parse(fs.readFileSync(path.join(out, CANDIDATE_FILE), 'utf8')), stamp_files: [] }));
expect(confirmReuse(forged, manifest()).reuse).toBeNull();
});
});
describe('bind and report', () => {
const reportDir = (name: string, withReuse = false) => {
const dir = path.join(scratch, name);
fs.mkdirSync(dir, { recursive: true });
fs.writeFileSync(path.join(dir, 'manifest.json'), JSON.stringify(manifest()));
fs.writeFileSync(path.join(dir, 'collector-outcomes.json'), JSON.stringify({ verdict: { verdict: 'GREEN' }, totals: { manual_accepted: 0 } }));
if (withReuse) fs.writeFileSync(path.join(dir, REUSE_FILE), '{}');
return dir;
};
const bindArgs = { repository: REPO, pr: 42, runId: 300, runAttempt: 1, headSha: 'a'.repeat(40), baseSha: 'b'.repeat(40), profile: 'pr', reconcileExit: 0 };
test('an executed run binds its verdict, reconcile exit and selection; a reused run binds nothing', () => {
const { binding: bound } = buildBinding({ ...bindArgs, reportDir: reportDir('bind-ok') });
expect(bound).toMatchObject({ verdict: 'GREEN', reconcile_exit: 0, selection_sha256: selectionFingerprint(manifest()) });
expect(buildBinding({ ...bindArgs, reportDir: reportDir('bind-reused', true) }).binding).toBeNull();
const empty = path.join(scratch, 'bind-empty');
fs.mkdirSync(empty, { recursive: true });
expect(buildBinding({ ...bindArgs, reportDir: empty }).binding).toBeNull();
});
test('the CLI binds a missing reconcile exit as -1, never as a pass', () => {
const out = path.join(scratch, 'bind-cli');
const bind = (head: string, testedSha: string, dir: string) => spawnSync(process.execPath, ['run', path.join(ROOT, 'scripts/gate-reuse.ts'), 'bind',
reportDir('bind-cli-report'), '--out', dir, '--repo', REPO, '--pr', '42', '--run-id', '300', '--run-attempt', '1', '--head-sha', head,
'--tested-sha', testedSha, '--profile', 'pr', '--reconcile-exit', ''], { cwd: receipt, encoding: 'utf8', timeout: 30_000 });
const r = bind(sha.gated!, tested(sha.moved!, sha.gated!), out);
expect(r.status, r.stderr).toBe(0);
expect(JSON.parse(fs.readFileSync(path.join(out, BINDING_FILE), 'utf8'))).toMatchObject({ reconcile_exit: -1, base_sha: sha.moved, head_sha: sha.gated });
// A tested commit that is not the merge of this head binds nothing.
const none = path.join(scratch, 'bind-cli-none');
expect(bind(sha.restamp!, tested(sha.moved!, sha.gated!), none).status).toBe(0);
expect(fs.existsSync(path.join(none, BINDING_FILE))).toBe(false);
});
test('report prints the reused verdict with its receipt, and fails closed without a valid record', () => {
const { out } = find(sha.restamp!, [run(300, sha.gated!)], { 300: binding() });
const reuse = parseReuse(JSON.parse(fs.readFileSync(path.join(out, CANDIDATE_FILE), 'utf8')))!;
const text = reuseReport(reuse, REPO);
expect(text).toContain('GATE REUSED (R2a, restamp-only)');
expect(text).toContain('TREE: same-modulo-stamps receipt=' + reuse.receipt_sha256);
expect(text).toContain('https://github.com/garrytan/gstack/actions/runs/300');
const moved = find(sha.rebased!, [run(300, sha.gated!)], { 300: binding() }, { baseSha: sha.moved }).candidate!;
const movedText = reuseReport(moved, REPO);
expect(movedText).toContain('GATE REUSED (R2, main moved)');
expect(movedText).toContain('OWN DIFF: identical sha256=' + moved.own_diff_sha256);
expect(movedText).toContain('why: main moved');
expect(movedText).toContain('at ' + sha.gated);
const bad = spawnSync(process.execPath, ['run', path.join(ROOT, 'scripts/gate-reuse.ts'), 'report', path.join(scratch, 'missing.json'), '--repo', REPO],
{ encoding: 'utf8', timeout: 30_000 });
expect(bad.status).toBe(1);
});
});
describe.skipIf(!Bun.which('jq') || !Bun.which('bash'))('the PR comment for a reused verdict', () => {
test('runs no repository code and names the reused run and receipt; without reconcile exit 0 it never says PASS (reused)', () => {
const workflow = Bun.YAML.parse(fs.readFileSync(path.join(ROOT, '.github/workflows/evals.yml'), 'utf8')) as any;
const script = (workflow.jobs['slices-comment'].steps.find((s: any) => s.name === 'Post PR comment').run as string);
const root = path.join(scratch, 'comment');
const paidDir = path.join(root, 'paid-report');
const binDir = path.join(root, 'bin');
fs.mkdirSync(paidDir, { recursive: true });
fs.mkdirSync(binDir, { recursive: true });
fs.writeFileSync(path.join(binDir, 'gh'), '#!/bin/sh\ncase "$*" in *--jq*) exit 0;; esac\nfor arg do case "$arg" in body=*) printf "%s\\n" "${arg#body=}";; esac; done\n');
fs.chmodSync(path.join(binDir, 'gh'), 0o755);
fs.writeFileSync(path.join(binDir, 'bc'), '#!/bin/sh\nread -r expression\nprintf "0\\n"\n');
fs.chmodSync(path.join(binDir, 'bc'), 0o755);
const { out } = find(sha.restamp!, [run(300, sha.gated!)], { 300: binding() });
fs.copyFileSync(path.join(out, CANDIDATE_FILE), path.join(paidDir, REUSE_FILE));
fs.writeFileSync(path.join(paidDir, 'manifest.json'), JSON.stringify({ profile: 'pr', selection: { e2e: [], judges: [] } }));
const body = script.replaceAll('/tmp/paid-report', paidDir).replaceAll('/tmp/verdict', path.join(root, 'verdict'))
.replaceAll('$' + '{{ github.repository }}', REPO).replaceAll('$' + '{{ github.event.pull_request.number }}', '42');
const exec = (reconcile: string) => spawnSync('bash', ['-e', '-c', body], { cwd: root, encoding: 'utf8', timeout: 10_000,
env: { ...process.env, PATH: binDir + ':' + process.env.PATH, RECONCILE_EXIT: reconcile, GITHUB_REPOSITORY: REPO } });
const reused = exec('0');
expect(reused.status, reused.stderr).toBe(0);
expect(reused.stdout).toContain('## E2E Evals: ✅ PASS (reused: restamp-only, R2a)');
expect(reused.stdout).toContain('TREE: same-modulo-stamps receipt=');
expect(reused.stdout).toContain('actions/runs/300');
const moved = find(sha.rebased!, [run(300, sha.gated!)], { 300: binding() }, { baseSha: sha.moved }).candidate!;
fs.writeFileSync(path.join(paidDir, REUSE_FILE), JSON.stringify(moved));
const movedComment = exec('0');
expect(movedComment.status, movedComment.stderr).toBe(0);
expect(movedComment.stdout).toContain('## E2E Evals: ✅ PASS (reused: main moved, R2)');
expect(movedComment.stdout).toContain('OWN DIFF: identical sha256=' + moved.own_diff_sha256);
expect(movedComment.stdout).toContain('because main moved');
const failed = exec('1');
expect(failed.status, failed.stderr).toBe(0);
expect(failed.stdout).not.toContain('PASS (reused');
expect(failed.stdout).toContain('❌ FAIL');
});
});
+20 -3
View File
@@ -37,6 +37,20 @@ export interface GuardSession {
cleanup(): void;
}
/**
* The actor ended its turn at the Phase 1 publication instead of going on to
* Phase 2: the screen after the last "Phase 1 complete." line shows a finished
* turn and no Phase 2 step (PR gate 39099195754, the 489 s "hang" first seen
* in 38976253774). Names the failure; it never passes a run.
*/
export function stoppedAtPhaseBoundary(screen: string): boolean {
const marks = [...screen.matchAll(/Phase\s*1\s*complete\./g)];
if (!marks.length) return false;
const last = marks[marks.length - 1]!;
const after = screen.slice(last.index! + last[0].length);
return /✻[^\n]*\bdone\b/.test(after) && !/Step\s*8|design-phase|Design\s*review/i.test(after);
}
/** The close packet Read (step 6) came after the CEO reviewer's completion notice, so it ran in the notice's new turn. */
export const closePacketReadAfterNotice = (afterNotice: string[]): boolean => afterNotice.some(c => /^Read .*close-packet\.md$/.test(c));
@@ -74,7 +88,9 @@ console.log(close);
const tool = path.join(ROOT, 'bin', 'gstack-autoplan-snapshot.ts');
const sections = path.join(config, 'skills', 'gstack', 'autoplan', 'sections');
const body = [
'Run this scripted phase boundary exactly. Each numbered step is its own assistant message; do nothing else.',
'Run this scripted phase boundary exactly, from step 1 through step 11 in one continuous run. Each numbered step is its own ' +
'assistant message; do nothing else. End your turn before step 11 only where step 5 or step 10 says to wait for a ' +
'background agent. **Phase 1 complete.** is a phase boundary, not the end of the task.',
`1. Run this Bash command alone: bun "${tool}" init "${source}" "${active}" "${restore}"`,
`2. Read the file ${path.join(sections, 'ceo-phase.md')} with offset 1 and limit 20.`,
`3. Run this Bash command alone: bun "${review}"`,
@@ -86,8 +102,9 @@ console.log(close);
'6. Only after the CEO review agent has finished (in the turn its completion notification starts, when it ran in the ' +
'background), Read the file whose path step 3 printed, completely (no offset or limit).',
'7. In one message, write exactly this line as your own text: **Phase 1 complete.** and make exactly one tool call in that ' +
'same message: the Bash command `true autoplan-published ceo`.',
`8. In your next assistant message, without waiting for any user input, Read the file ${path.join(sections, 'design-phase.md')} with offset 1 and limit 20.`,
'same message: the Bash command `true autoplan-published ceo`. Do not end your turn after it: when its result arrives, go ' +
'straight on to step 8.',
`8. As your next assistant message, in the same turn as step 7's result (no user input is coming), Read the file ${path.join(sections, 'design-phase.md')} with offset 1 and limit 20.`,
`9. Run this Bash command alone: cat "${path.join(artifacts, 'dispatch-design.txt')}"`,
'10. Call the Agent tool once with these inputs: prompt = the exact, complete output of step 9 (every line, unchanged), ' +
'description "Design review", subagent_type "general-purpose". If the agent runs in the background, end your turn and wait ' +
+1
View File
@@ -16,6 +16,7 @@
export const FREE_FIXTURES: Readonly<Record<string, readonly string[]>> = {
'test/fixtures/auq-matrix/**': ['test/auq-native-capture.test.ts'],
'test/fixtures/autoplan-edit-digests-al.json': ['test/autoplan-edit-digests-al.test.ts'],
'test/fixtures/autoplan-guard-pty-phase-boundary-stop-screen.txt': ['test/autoplan-guard-pty-fixture.test.ts'],
'test/fixtures/autoplan-home-phase-entry-fb10.json': ['test/autoplan-method-read-audit.test.ts'],
'test/fixtures/autoplan-owned-state-edit.json': ['test/autoplan-owned-state.test.ts'],
'test/fixtures/autoplan-pairing/run-20261014-pairs.json': ['test/autoplan-pairing.test.ts'],
+4 -1
View File
@@ -20,7 +20,7 @@
import { expect } from 'bun:test';
import { describeIfSelected, testIfSelected } from './helpers/e2e-helpers';
import { launchClaudePty, selectPtyNumberedOption } from './helpers/claude-pty-runner';
import { closePacketReadAfterNotice, prepareGuardSession } from './helpers/autoplan-guard-pty';
import { closePacketReadAfterNotice, prepareGuardSession, stoppedAtPhaseBoundary } from './helpers/autoplan-guard-pty';
const MODEL = 'claude-haiku-4-5-20251001';
@@ -48,6 +48,8 @@ describeIfSelected('autoplan publication guard live PTY', ['autoplan-guard-pty']
await Bun.sleep(1_500);
}
if (log.some(e => e.decision === 'deny') || (log.length >= 4 && s.noticeEvidence().notices >= 2)) break;
// The actor ended its turn at the Phase 1 publication; no input is coming, so it cannot reach Phase 2.
if (log.length < 4 && stoppedAtPhaseBoundary(screen)) break;
if (session.exited()) break;
}
await Bun.sleep(3_000);
@@ -55,6 +57,7 @@ describeIfSelected('autoplan publication guard live PTY', ['autoplan-guard-pty']
const summary = { prompts: prompts.length, phase1Ms, totalMs: Math.round(performance.now() - started), evidence,
decisions: log.map(e => ({ decision: e.decision, code: e.code, path: e.path, version: e.claude_code_version })) };
console.log(`[autoplan-guard-pty] ${JSON.stringify(summary)}`);
if (log.length < 4 && stoppedAtPhaseBoundary(session.visibleText())) console.log('[autoplan-guard-pty] PHASE_BOUNDARY_STOP: the actor ended its turn after "Phase 1 complete." instead of going on to step 8');
if (log.length !== 4 || log.some(e => e.decision === 'deny') || prompts.length)
console.log(`[autoplan-guard-pty] journal:\n${s.journalSummary().join('\n')}\n[autoplan-guard-pty] screen:\n${session.visibleText().slice(-2_000)}\n[autoplan-guard-pty] last non-blank screen:\n${session.lastScreen?.() ?? ''}`);
for (const prompt of prompts) console.log(`[autoplan-guard-pty] permission card approved:\n${prompt}`);