v1.91.162.0 feat(evals): merge-train workflow side, inert behind the train policy block (GSTA-49 T1b) (#3279)

* feat(ship-queue): merge trains with pattern bisect and train-record verify (GSTA-49 T1)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* test(windows): curate ship-queue-train.test.ts out of the Windows subset (gh shim not resolved on windows-latest)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* feat(evals): merge-train workflow side, inert behind the train policy block (GSTA-49 T1b)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* chore(evals): T1b CHANGELOG under [Unreleased]; classify evals-main-backstop.yml as free-lane only

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix(evals): plan-design-with-ui focus detector accepts Opus 5.5's 'Review focus' title with the step's rating and an all-7 option; replay run 39298269902

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* v1.91.162.0 feat(evals): merge-train workflow side, inert behind the train policy block (GSTA-49 T1b)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

---------

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
Garry Tanandcapy-ai[bot] authored and GitHub committed 2026-10-20 02:13:39 +00:00
1 parent 5ebda5999f
commit 85b0f531f5
27 files changed
+2547 -96

No files matched your search

+65
View File
@@ -0,0 +1,65 @@
name: Main backstop gate
# GSTA-49 T1b (docs/ship-policy.md#run-a-merge-train). Every push to main is
# covered by a full gate. Under merge trains a train's merges are covered by
# the gate the train already ran on the same trees, so this workflow gates
# main only when no green-train record lists main's tree modulo stamps (a
# merge outside the queue), after any train has finished merging. A red
# backstop writes the freeze record gstack-ship-queue reads. Inert (no paid
# work, nothing written) unless main's .gstack/ship-policy.json declares a
# `train` block; without one the per-PR gates cover main as before.
on:
push:
branches: [main]
# One backstop at a time; a newer push waits and the script hands a superseded
# push to it (only the newest pending run is kept).
concurrency:
group: evals-main-backstop
cancel-in-progress: false
permissions: {}
jobs:
backstop:
runs-on: ubuntu-24.04
if: github.actor != 'dependabot[bot]'
# Settle window (60) + the full gate (270) + margin.
timeout-minutes: 345
permissions:
# Pushes the freeze record to refs/notes/gstack-trains.
contents: write
# Dispatches the gate and reads train runs and their artifacts.
actions: write
checks: read
# Opens the issue a red backstop names.
issues: write
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
fetch-depth: 0
# Read with git and jq only: without a train block nothing below runs.
- name: Read the train policy (main)
id: policy
run: |
train=$(git show "${GITHUB_SHA}:.gstack/ship-policy.json" 2>/dev/null | jq -c 'if (.train | type) == "object" then .train else null end' 2>/dev/null || true)
if [ -z "$train" ] || [ "$train" = null ]; then
echo "train=false" >> "$GITHUB_OUTPUT"
echo "- Backstop inert: main's .gstack/ship-policy.json declares no train block; per-PR gates cover main" >> "$GITHUB_STEP_SUMMARY"
else
echo "train=true" >> "$GITHUB_OUTPUT"
fi
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
if: steps.policy.outputs.train == 'true'
with:
bun-version: 1.4.2
- name: Gate main unless a green train covers its tree
if: steps.policy.outputs.train == 'true'
env:
GH_TOKEN: ${{ github.token }}
GSTACK_STATE_ROOT: ${{ runner.temp }}/gstack-state
run: |
bun --no-env-file --config=/dev/null --no-install run scripts/train-backstop.ts run --repo "$GITHUB_REPOSITORY" --branch "$GITHUB_REF_NAME" --sha "$GITHUB_SHA" \
--summary "$GITHUB_STEP_SUMMARY"
+480 -27
View File
@@ -1,6 +1,8 @@
name: E2E Evals
# eval-bg (scripts/eval-bg.ts) finds its dispatch by the nonce in the run name (ENG-12).
run-name: ${{ inputs.nonce != '' && format('E2E Evals [eval-bg {0}]', inputs.nonce) || '' }}
# A dispatched screen's name also ends ` screen #<pr>`, which recover-receipts and
# gate-reuse filter on to list a PR's screens beside its pull_request runs.
run-name: "${{ inputs.nonce != '' && format('E2E Evals [eval-bg {0}]{1}', inputs.nonce, inputs.screen_pr != '' && format(' screen #{0}', inputs.screen_pr) || '') || '' }}"
on:
pull_request:
branches: [main]
@@ -30,9 +32,33 @@ on:
description: 'Same-repo PR number whose receipt store this run reads read-only (never saves); empty = fresh'
type: string
default: ''
# Merge trains (docs/ship-policy.md#run-a-merge-train): only under a train
# block in the default branch's .gstack/ship-policy.json. The route job
# validates them before any paid work (scripts/train-gate.ts route).
cases:
description: 'Train bisect: comma-separated case ids from the full gate plan (gstack-ship-queue train bisect sets it)'
type: string
default: ''
prefix_shas:
description: 'Train bisect: comma-separated prefix commits, each an ancestor of the train head on train/<nonce>'
type: string
default: ''
screen_pr:
description: 'Dispatched screen: the same-repo PR number (gstack-ship-queue ready --screen sets it)'
type: string
default: ''
screen_merge_sha:
description: "Dispatched screen: the PR's refs/pull/<n>/merge commit to check out and test"
type: string
default: ''
screen_policy:
description: 'Dispatched screen: narrow or broad (a broad screen never adopts a narrow run)'
type: string
default: ''
# A dispatched screen is keyed on its PR, so a newer head's screen cancels the older one.
concurrency:
group: evals-${{ github.event.pull_request.number || github.run_id }}
group: evals-${{ github.event.pull_request.number || (inputs.screen_pr != '' && format('screen-{0}', inputs.screen_pr)) || github.run_id }}
cancel-in-progress: true
env:
@@ -120,6 +146,129 @@ jobs:
docker manifest inspect "$EVAL_IMAGE" > /tmp/eval-runtime-manifest.json
echo "id=$(sha256sum /tmp/eval-runtime-manifest.json | cut -d ' ' -f1)" >> "$GITHUB_OUTPUT"
# ── Route (merge trains, GSTA-49 T1b; docs/ship-policy.md#run-a-merge-train)
# Names the run's mode before any paid work, from the train policy of the
# base revision (the PR's base commit, or the default branch for a
# dispatch). Without a `train` block every run routes as before: a
# pull_request run is `pr`, a dispatch is `dispatch`, and no other job
# changes. Under a train block a PR push is `pr-summary` (screen-summary
# prints the screening selection and its estimated cost; no paid slice
# starts), and gstack-ship-queue's dispatches are `screen`, `train` or
# `bisect` once scripts/train-gate.ts has validated their inputs. A refused
# dispatch fails here, before any paid work.
route:
runs-on: ubuntu-24.04
if: github.actor != 'dependabot[bot]'
timeout-minutes: 30
permissions:
contents: read
actions: read
pull-requests: read
outputs:
mode: ${{ steps.route.outputs.mode }}
record_artifact: ${{ steps.policy.outputs.record_artifact }}
full_gate_check: ${{ steps.policy.outputs.full_gate_check }}
screen_head: ${{ steps.screen.outputs.head_sha }}
screen_base: ${{ steps.screen.outputs.base_sha }}
screen_branch: ${{ steps.screen.outputs.head_branch }}
superseded: ${{ steps.screen.outputs.superseded }}
bisect_slices: ${{ steps.bisect.outputs.slices }}
bisect_timeouts: ${{ steps.bisect.outputs.slice_timeouts }}
bisect_refs: ${{ steps.bisect.outputs.prefix_refs }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
fetch-depth: 0
persist-credentials: false
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: 1.4.2
# Read with git and jq only: no repository code decides whether PR pushes start paid work.
- name: Read the train policy (base revision)
id: policy
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
run: |
base="${BASE_SHA:-origin/${DEFAULT_BRANCH}}"
train=$(git show "${base}:.gstack/ship-policy.json" 2>/dev/null | jq -c 'if (.train | type) == "object" then .train else null end' 2>/dev/null || true)
artifact=$(jq -r '.record_artifact // ""' <<<"${train:-null}" 2>/dev/null || true)
check=$(jq -r '.full_gate_check // ""' <<<"${train:-null}" 2>/dev/null || true)
if [ -z "$train" ] || [ "$train" = null ]; then
echo "train=false" >> "$GITHUB_OUTPUT"
echo "- Train policy: none at ${base}; this run routes as before" >> "$GITHUB_STEP_SUMMARY"
elif [[ "$artifact" =~ ^[A-Za-z0-9_.-]{1,64}$ ]] && [[ "$check" =~ ^[A-Za-z0-9_./-]{1,64}$ ]]; then
{ echo "train=true"; echo "record_artifact=${artifact}"; echo "full_gate_check=${check}"; } >> "$GITHUB_OUTPUT"
echo "- Train policy: declared at ${base} (record artifact ${artifact}, check ${check})" >> "$GITHUB_STEP_SUMMARY"
else
echo "train=false" >> "$GITHUB_OUTPUT"
echo "::warning::the train block at ${base} names no valid record_artifact and full_gate_check; this run routes as before"
fi
- name: Route this run
id: route
env:
TRAIN_POLICY: ${{ steps.policy.outputs.train }}
NONCE: ${{ inputs.nonce }}
EXPECTED_SHA: ${{ inputs.expected_sha }}
EVALS_ALL: ${{ inputs.evals_all }}
CASES: ${{ inputs.cases }}
PREFIX_SHAS: ${{ inputs.prefix_shas }}
SCREEN_PR: ${{ inputs.screen_pr }}
SCREEN_MERGE_SHA: ${{ inputs.screen_merge_sha }}
SCREEN_POLICY: ${{ inputs.screen_policy }}
run: |
if [ "$GITHUB_EVENT_NAME" = pull_request ]; then
mode='pr'
if [ "$TRAIN_POLICY" = true ]; then mode=pr-summary; fi
echo "mode=${mode}" >> "$GITHUB_OUTPUT"
echo "- Run mode: ${mode}" >> "$GITHUB_STEP_SUMMARY"
elif [ -f scripts/train-gate.ts ]; then
bun --no-env-file --config=/dev/null --no-install run scripts/train-gate.ts route --github-output "$GITHUB_OUTPUT" >> "$GITHUB_STEP_SUMMARY"
elif [ -n "${CASES}${PREFIX_SHAS}${SCREEN_PR}${SCREEN_MERGE_SHA}${SCREEN_POLICY}" ]; then
echo "::error::${GITHUB_REF_NAME} has no scripts/train-gate.ts to validate the train or screen inputs; dispatch from a ref that has it"
exit 1
else
echo "mode=dispatch" >> "$GITHUB_OUTPUT"
fi
# The screen's merge commit must merge the PR's current head into its
# base, and Free Tests must be green on that head (base-branch code: a
# screen dispatches on the default branch). No quiet window: a screen is
# an explicit request. A moved PR head reports superseded and plans nothing.
- name: Check the screen's PR head (Free Tests)
id: screen
if: steps.route.outputs.mode == 'screen'
env:
GH_TOKEN: ${{ github.token }}
SCREEN_PR: ${{ inputs.screen_pr }}
SCREEN_MERGE_SHA: ${{ inputs.screen_merge_sha }}
run: |
bun --no-install run scripts/eval-readiness.ts wait --repo "$GITHUB_REPOSITORY" --pr "$SCREEN_PR" --merge-sha "$SCREEN_MERGE_SHA" \
--run-id "$GITHUB_RUN_ID" --quiet-minutes 0 --deadline-minutes 20 \
--github-output "$GITHUB_OUTPUT" --summary "$GITHUB_STEP_SUMMARY"
- name: Plan the bisect (one executor per prefix)
id: bisect
if: steps.route.outputs.mode == 'bisect'
env:
CASES: ${{ steps.route.outputs.cases }}
PREFIXES: ${{ steps.route.outputs.prefixes }}
NONCE: ${{ inputs.nonce }}
run: |
bun --no-env-file --config=/dev/null --no-install run scripts/train-gate.ts bisect-plan --cases "$CASES" --prefixes "$PREFIXES" --nonce "$NONCE" \
--out /tmp/train-bisect/bisect.json --github-output "$GITHUB_OUTPUT" >> "$GITHUB_STEP_SUMMARY"
- name: Hand the bisect plan to its executors
if: steps.route.outputs.mode == 'bisect'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: train-bisect-plan
path: /tmp/train-bisect/bisect.json
retention-days: 30
# ── Readiness-gated dispatch (R1, GSTA-48) ───────────────────────────────
# Paid slices for a PR head start only after the PR has been quiet for the
# window (vars.EVALS_QUIET_MINUTES, default 10; the evals-no-debounce label
@@ -161,6 +310,35 @@ jobs:
--run-id "$GITHUB_RUN_ID" --quiet-minutes "$QUIET_MINUTES" --deadline-minutes 80 \
--github-output "$GITHUB_OUTPUT" --summary "$GITHUB_STEP_SUMMARY"
# GSTA-48 P3-1: a head the readiness job blocked runs no gate, so the PR's
# last "## E2E Evals" comment would still show an older head's verdict.
# Replace it (never post a new one) with what happened to this head. Runs no
# repository code.
readiness-comment:
runs-on: ubuntu-24.04
needs: [readiness]
if: ${{ !cancelled() && needs.readiness.result == 'failure' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }}
timeout-minutes: 5
permissions:
pull-requests: write
issues: write
steps:
- name: Replace a stale verdict comment
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
PR_NUMBER: ${{ github.event.pull_request.number }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
COMMENT_ID=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate \
--jq '.[] | select(.body | startswith("## E2E Evals")) | .id' | tail -1)
if [ -z "$COMMENT_ID" ]; then exit 0; fi
BODY="## E2E Evals: ⛔ NOT RUN on \`${HEAD_SHA}\`
The readiness job blocked this head (Free Tests did not pass on it, or did not finish within the readiness deadline), so no paid gate ran and no verdict covers it. An earlier verdict in this comment belonged to an older head.
Details: [run ${GITHUB_RUN_ID}](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}). Fix and push, or re-run Free Tests and then re-run this workflow."
gh api "repos/${GITHUB_REPOSITORY}/issues/comments/${COMMENT_ID}" -X PATCH -f body="$BODY"
# ── Receipt recovery (W1c: CEO-14/34, ENG-4/5/7) ─────────────────────────
# A run cancelled by the next push never reaches slices-report, so its
# finished receipts (passes AND the negatives executors write at the
@@ -174,10 +352,11 @@ jobs:
# plan-slices turns reuse off, so cases execute unreduced.
recover-receipts:
runs-on: ubuntu-24.04
needs: [readiness]
needs: [readiness, route]
# Runs after readiness settles, so a superseded run's cancellation has usually completed; a head whose Free Tests failed recovers nothing.
# A dispatch naming pr_receipts reads that PR's receipts from its runs' artifacts (cache entries are ref-scoped).
if: ${{ !cancelled() && (needs.readiness.result == 'success' || needs.readiness.result == 'skipped') && needs.readiness.outputs.superseded != 'true' && github.actor != 'dependabot[bot]' && ((github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && !contains(github.event.pull_request.labels.*.name, 'evals-fresh')) || (github.event_name == 'workflow_dispatch' && inputs.pr_receipts != '')) }}
# A dispatched screen only looks for a gate verdict to reuse (R2a/R2); a PR push under a train policy (pr-summary) skips this job.
if: ${{ !cancelled() && (needs.readiness.result == 'success' || needs.readiness.result == 'skipped') && needs.readiness.outputs.superseded != 'true' && needs.route.result == 'success' && needs.route.outputs.superseded != 'true' && github.actor != 'dependabot[bot]' && ((github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && needs.route.outputs.mode == 'pr' && !contains(github.event.pull_request.labels.*.name, 'evals-fresh')) || (github.event_name == 'workflow_dispatch' && inputs.pr_receipts != '') || needs.route.outputs.mode == 'screen') }}
timeout-minutes: 5
permissions:
contents: read
@@ -185,7 +364,7 @@ jobs:
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ github.event.pull_request.base.sha || inputs.base_sha || github.event.repository.default_branch }}
ref: ${{ github.event.pull_request.base.sha || needs.route.outputs.screen_base || inputs.base_sha || github.event.repository.default_branch }}
persist-credentials: false
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
@@ -202,6 +381,7 @@ jobs:
restore-keys: eval-input-v1-${{ github.repository_id }}-pr-${{ github.event.pull_request.number }}-
- name: Recover receipts of cancelled runs (base-ref code)
if: needs.route.outputs.mode != 'screen'
env:
GH_TOKEN: ${{ github.token }}
STORE_KEY: ${{ steps.store.outputs.cache-matched-key }}
@@ -229,26 +409,32 @@ jobs:
# identical. Anything else (a red, partial or cancelled gate, a changed
# own diff, a non-stamp byte, no binding or receipt, a gh or git error,
# evals-fresh) leaves no candidate: the full gate runs.
# A dispatched screen (GSTA-49 T1b) finds the same way, authenticated by
# PR number, head, repository and tested base (its merge commit's first
# parent), listing the PR's pull_request runs and earlier screens; a
# broad screen never adopts a narrow run.
- name: Find a gate verdict to reuse (R2a/R2, base-ref code)
if: github.event_name == 'pull_request'
if: github.event_name == 'pull_request' || needs.route.outputs.mode == 'screen'
env:
GH_TOKEN: ${{ github.token }}
PR_NUMBER: ${{ github.event.pull_request.number }}
HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
HEAD_BRANCH: ${{ github.event.pull_request.head.ref }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
PR_NUMBER: ${{ github.event.pull_request.number || inputs.screen_pr }}
HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
HEAD_BRANCH: ${{ github.event.pull_request.head.ref || needs.route.outputs.screen_branch }}
HEAD_SHA: ${{ github.event.pull_request.head.sha || needs.route.outputs.screen_head }}
TESTED_SHA: ${{ inputs.screen_merge_sha || github.sha }}
SCREEN_POLICY: ${{ inputs.screen_policy }}
run: |
if [ ! -f scripts/gate-reuse.ts ]; then
echo "- Gate verdict reuse unavailable: the base ref has no scripts/gate-reuse.ts; the full gate runs" >> "$GITHUB_STEP_SUMMARY"
exit 0
fi
bun --no-install run scripts/gate-reuse.ts find --repo "$GITHUB_REPOSITORY" --pr "$PR_NUMBER" --head-repo "$HEAD_REPO" \
--branch "$HEAD_BRANCH" --run-id "$GITHUB_RUN_ID" --head-sha "$HEAD_SHA" --tested-sha "$GITHUB_SHA" \
--out /tmp/gate-reuse --budget-seconds 90 >> "$GITHUB_STEP_SUMMARY" \
--branch "$HEAD_BRANCH" --run-id "$GITHUB_RUN_ID" --head-sha "$HEAD_SHA" --tested-sha "$TESTED_SHA" \
${SCREEN_POLICY:+--screen-policy "$SCREEN_POLICY"} --out /tmp/gate-reuse --budget-seconds 90 >> "$GITHUB_STEP_SUMMARY" \
|| echo "- Gate verdict reuse crashed; the full gate runs" >> "$GITHUB_STEP_SUMMARY"
- name: Hand a gate reuse candidate to plan-slices
if: github.event_name == 'pull_request'
if: github.event_name == 'pull_request' || needs.route.outputs.mode == 'screen'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: gate-reuse-candidate
@@ -264,6 +450,36 @@ jobs:
if-no-files-found: ignore
retention-days: 7
# ── PR push under a train policy (GSTA-49 T1b, a gate edit) ─────────────
# Only when the base revision declares a `train` block: the push starts no
# paid slice. This job prints how to request the paid screen, the
# screening selection and its estimated cost, for free; the train's full
# gate covers every merge. Without a train block it never runs.
screen-summary:
runs-on: ubuntu-24.04
needs: [route]
if: needs.route.outputs.mode == 'pr-summary' && github.event.pull_request.head.repo.full_name == github.repository
timeout-minutes: 10
permissions:
contents: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
fetch-depth: 0
persist-credentials: false
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: 1.4.2
- name: Print the screening selection and its estimated cost (no paid case runs)
env:
PR_NUMBER: ${{ github.event.pull_request.number }}
run: |
EVALS_TIER=gate bun --no-install run scripts/test-paid-shards.ts --tier gate --emit-plan /tmp/paid-plan/manifest.json --slices 1
bun --no-env-file --config=/dev/null --no-install run scripts/train-gate.ts screen-summary /tmp/paid-plan/manifest.json --pr "$PR_NUMBER" >> "$GITHUB_STEP_SUMMARY"
bun --no-install run scripts/test-pr-profile.ts summary /tmp/paid-plan/manifest.json >> "$GITHUB_STEP_SUMMARY"
# ── Sliced lane (the ONLY paid lane; legacy 17-row matrix deleted) ──────────
# One PLANNER computes diff selection + the slice plan ONCE (killing
# per-slice selector divergence); K executors consume the manifest; the
@@ -283,9 +499,10 @@ jobs:
# branch (see CLAUDE.md's garrytan-agents workflow).
plan-slices:
runs-on: ubicloud-standard-8
needs: [readiness, recover-receipts]
needs: [readiness, recover-receipts, route]
# !cancelled(): plans even when readiness or recovery was skipped (dispatch, evals-fresh) or recovery failed (reuse is then off); a superseded run, or a head whose readiness failed (Free Tests red), plans nothing.
if: ${{ !cancelled() && (needs.readiness.result == 'success' || needs.readiness.result == 'skipped') && needs.readiness.outputs.superseded != 'true' && github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }}
# The route job must have named a planned mode: pr-summary (a PR push under a train policy) and bisect plan nothing here, and neither does a refused dispatch.
if: ${{ !cancelled() && (needs.readiness.result == 'success' || needs.readiness.result == 'skipped') && needs.readiness.outputs.superseded != 'true' && needs.route.result == 'success' && needs.route.outputs.superseded != 'true' && contains(fromJSON('["pr","screen","train","dispatch"]'), needs.route.outputs.mode) && github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }}
timeout-minutes: 10
permissions:
contents: read
@@ -293,9 +510,16 @@ jobs:
slices: ${{ steps.matrix.outputs.slices }}
slice_timeouts: ${{ steps.matrix.outputs.slice_timeouts }}
gate_reuse: ${{ steps.gate_reuse.outputs.gate_reuse }}
# The route, forwarded: the executors and the report read it from here.
mode: ${{ needs.route.outputs.mode }}
record_artifact: ${{ needs.route.outputs.record_artifact }}
full_gate_check: ${{ needs.route.outputs.full_gate_check }}
screen_head: ${{ needs.route.outputs.screen_head }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
# A dispatched screen tests the PR's pinned refs/pull/<n>/merge commit; empty for every other run (its own ref).
ref: ${{ inputs.screen_merge_sha }}
# Preserve full history for merge-base diff selection. Moving the
# planner off the eval image must not change its selection inputs.
fetch-depth: 0
@@ -343,14 +567,14 @@ jobs:
env:
EVALS_ALL: ${{ (github.event_name == 'workflow_dispatch' && inputs.evals_all) && '1' || '' }}
EVALS_CACHE_DIR: ${{ steps.reuse.outputs.reuse == 'on' && '/tmp/gstack-eval-input-cache' || '' }}
# Dispatch diff base (CEO-17): the exact merge-base, else the named base branch, else detection.
EVALS_BASE: ${{ inputs.base_sha || (inputs.base_ref != '' && format('origin/{0}', inputs.base_ref)) || '' }}
# Dispatch diff base (CEO-17): a screen's merge-commit first parent, else the exact merge-base, else the named base branch, else detection.
EVALS_BASE: ${{ needs.route.outputs.screen_base || inputs.base_sha || (inputs.base_ref != '' && format('origin/{0}', inputs.base_ref)) || '' }}
run: EVALS_TIER=gate bun --no-install run scripts/test-paid-shards.ts --tier gate --emit-plan /tmp/paid-plan/manifest.json --slice-budget 420 --jobs 2 --max-parallel 20
# R2a/R2: reuse holds only when this head's own plan selects exactly the
# cases and trial plans the reused gate ran (selection fingerprint).
- name: Download a gate reuse candidate
if: github.event_name == 'pull_request'
if: github.event_name == 'pull_request' || needs.route.outputs.mode == 'screen'
continue-on-error: true
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
with:
@@ -359,7 +583,7 @@ jobs:
- name: Confirm gate verdict reuse (R2a/R2)
id: gate_reuse
if: github.event_name == 'pull_request'
if: github.event_name == 'pull_request' || needs.route.outputs.mode == 'screen'
run: |
mkdir -p /tmp/gate-reuse
bun --no-install run scripts/gate-reuse.ts confirm /tmp/gate-reuse /tmp/paid-plan/manifest.json --out /tmp/paid-plan \
@@ -430,6 +654,8 @@ jobs:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
# A dispatched screen's pinned merge commit (the planner verified it); empty otherwise.
ref: ${{ inputs.screen_merge_sha }}
# Full history: files with SELF-derived selection (the LLM-judge
# map, routing) walk git at module load, and selection is
# fail-closed on git errors — a shallow checkout crashed those
@@ -555,6 +781,151 @@ jobs:
if-no-files-found: ignore
retention-days: 30
# ── Train bisect (GSTA-49 T1b) ─────────────────────────────────────────
# One executor per prefix of the train (route's validated prefix_shas, each
# an ancestor of the train head): every red case once, one trial, on that
# prefix's own checkout. Each writes the per-prefix record that
# `gstack-ship-queue train bisect` reads, as <record_artifact>-bisect-<sha12>.
# A bisect never reuses a result and never binds a verdict.
train-bisect:
runs-on: ubicloud-standard-8
needs: [build-image, route]
if: ${{ !cancelled() && needs.build-image.result == 'success' && needs.route.result == 'success' && needs.route.outputs.mode == 'bisect' }}
env:
EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-train-bisect-${{ matrix.slice }}
timeout-minutes: ${{ fromJSON(needs.route.outputs.bisect_timeouts)[matrix.slice] }}
permissions:
contents: read
packages: read
container:
image: ${{ needs.build-image.outputs.image-tag }}
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
options: --user runner
strategy:
fail-fast: false
max-parallel: 7
matrix:
slice: ${{ fromJSON(needs.route.outputs.bisect_slices) }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ fromJSON(needs.route.outputs.bisect_refs)[matrix.slice] }}
fetch-depth: 0
persist-credentials: false
- name: Verify the prefix checkout
id: prefix
env:
PREFIX_SHA: ${{ fromJSON(needs.route.outputs.bisect_refs)[matrix.slice] }}
run: |
actual=$(git rev-parse HEAD)
if [ "$actual" != "$PREFIX_SHA" ]; then
echo "::error::train-bisect checked out ${actual}, not its prefix ${PREFIX_SHA}"
exit 1
fi
echo "short=${PREFIX_SHA:0:12}" >> "$GITHUB_OUTPUT"
- name: Fix bun temp
uses: ./.github/actions/fix-bun-temp
- name: Restore deps
uses: ./.github/actions/restore-deps
- run: bun run build
# Any slice can host a PTY smoke, so the seed/registration steps run
# UNCONDITIONALLY (both are idempotent) — the old matrix keyed them on
# matrix.suite.name, which a sliced lane cannot do. The register
# composite carries the fail-fast dangling-symlink/frontmatter
# verification loop, so a moved skill target fails HERE in seconds,
# not as a wedged PTY session at the shard wall.
- name: Seed claude interactive config
uses: ./.github/actions/seed-claude-config
with:
anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }}
- name: Register gstack skills for PTY smokes
uses: ./.github/actions/register-gstack-skills
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
with:
name: train-bisect-plan
path: /tmp/train-bisect
- name: Authenticate the Codex CLI
shell: bash
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
run: |
set -euo pipefail
codex="${GSTACK_CI_CODEX_BIN_DIR:?}/codex"
export CODEX_HOME="$HOME/.gstack-ci-codex"
mkdir -p "$CODEX_HOME"
echo "GSTACK_CI_CODEX_HOME=$CODEX_HOME" >> "$GITHUB_ENV"
if [ -z "${OPENAI_API_KEY:-}" ]; then
echo "::warning::OPENAI_API_KEY is unavailable; Codex eval cases cannot authenticate and will fail."
exit 0
fi
printenv OPENAI_API_KEY | "$codex" login --with-api-key
"$codex" login status >/dev/null 2>&1
- name: Run the red cases on prefix ${{ matrix.slice }}
env:
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
PLAYWRIGHT_BROWSERS_PATH: /opt/playwright-browsers
EVALS_JOBS: "1"
EVALS_CONCURRENCY: "1"
SLICE: ${{ matrix.slice }}
run: EVALS_TIER=gate bun --no-env-file --config=/dev/null run scripts/train-gate.ts bisect-prefix --plan /tmp/train-bisect/bisect.json --index "$SLICE" --out /tmp/train-bisect/out
- name: Upload the prefix record
if: always() && steps.prefix.outputs.short != ''
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: ${{ needs.route.outputs.record_artifact }}-bisect-${{ steps.prefix.outputs.short }}
path: /tmp/train-bisect/out/bisect-prefix.json
if-no-files-found: ignore
overwrite: true
retention-days: 90
- name: Upload native capture evidence
if: always()
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: native-captures-${{ env.EVALS_RUN_ID }}
include-hidden-files: true
path: |
~/.gstack/projects/*/e2e-runs
~/.gstack/projects/*/evals/qa-callers
~/.gstack-dev/e2e-runs
~/.gstack-dev/evals/qa-callers
if-no-files-found: ignore
retention-days: 90
# The spooled per-shard full logs — a red weekly/PR lane three weeks
# later needs more than a summary line.
# always(), not failure(): a failed behavior trial is a verdict and no
# longer reds its runner, but its full log is the diagnosis evidence.
- name: Upload shard logs
if: always()
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: train-bisect-logs-${{ matrix.slice }}-a${{ github.run_attempt }}
include-hidden-files: true
# The Fix-bun-temp step points TMPDIR at /home/runner/.cache, so the
# runner's spool lands THERE, not /tmp — the original /tmp glob
# uploaded nothing and a red slice's diagnostics were unreachable.
path: |
/home/runner/.cache/gstack-paid-shard-*.log
/tmp/gstack-paid-shard-*.log
if-no-files-found: ignore
retention-days: 30
slices-report:
# GitHub-hosted, like recover-receipts: Ubicloud runners send actions/cache
# to Ubicloud's own transparent cache, which a GitHub-hosted restore never
@@ -582,6 +953,10 @@ jobs:
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
# A screen's pinned merge commit (empty otherwise); a train run needs the
# history of its prefix commits T1..Tk for the train record.
ref: ${{ inputs.screen_merge_sha }}
fetch-depth: ${{ needs.plan-slices.outputs.mode == 'train' && '0' || '1' }}
persist-credentials: false
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
@@ -671,19 +1046,24 @@ jobs:
# R2a/R2: what this executed run proved, read by a later push. The base
# is the tested merge commit's first parent. Reused runs bind nothing,
# so reuse never chains off a reuse.
# A dispatched screen binds too (purpose screen, its tested merge commit
# and policy). Train, bisect and other dispatches never bind for reuse.
- name: Bind this run's verdict for reuse (R2a/R2)
if: always() && github.event_name == 'pull_request' && needs.plan-slices.outputs.gate_reuse != 'true'
if: always() && (github.event_name == 'pull_request' || needs.plan-slices.outputs.mode == 'screen') && needs.plan-slices.outputs.gate_reuse != 'true'
env:
PR_NUMBER: ${{ github.event.pull_request.number }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
PR_NUMBER: ${{ github.event.pull_request.number || inputs.screen_pr }}
HEAD_SHA: ${{ github.event.pull_request.head.sha || needs.plan-slices.outputs.screen_head }}
TESTED_SHA: ${{ inputs.screen_merge_sha || github.sha }}
SCREEN_POLICY: ${{ inputs.screen_policy }}
RECONCILE_EXIT: ${{ steps.reconcile.outputs.exit }}
run: |
bun --no-install run scripts/gate-reuse.ts bind /tmp/paid-report --out /tmp/gate-binding --repo "$GITHUB_REPOSITORY" \
--pr "$PR_NUMBER" --run-id "$GITHUB_RUN_ID" --run-attempt "$GITHUB_RUN_ATTEMPT" --head-sha "$HEAD_SHA" \
--tested-sha "$GITHUB_SHA" --profile "$EVALS_PROFILE" --reconcile-exit "$RECONCILE_EXIT" >> "$GITHUB_STEP_SUMMARY"
--tested-sha "$TESTED_SHA" --profile "$EVALS_PROFILE" --reconcile-exit "$RECONCILE_EXIT" \
${SCREEN_POLICY:+--purpose screen --screen-policy "$SCREEN_POLICY"} >> "$GITHUB_STEP_SUMMARY"
- name: Upload the gate binding
if: always() && github.event_name == 'pull_request' && needs.plan-slices.outputs.gate_reuse != 'true'
if: always() && (github.event_name == 'pull_request' || needs.plan-slices.outputs.mode == 'screen') && needs.plan-slices.outputs.gate_reuse != 'true'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: gate-binding-a${{ github.run_attempt }}
@@ -700,17 +1080,81 @@ jobs:
if-no-files-found: ignore
retention-days: 90
# GSTA-49 T1b: a train's own gate run publishes its record, read by
# gstack-ship-queue's verdict and verify and by the main backstop: the
# census from this run's plan manifest, one panel per planned case, and
# the prefix trees T1..Tk modulo stamps. GREEN only for a complete census.
- name: Write the train record
if: always() && needs.plan-slices.outputs.mode == 'train'
env:
NONCE: ${{ inputs.nonce }}
HEAD_SHA: ${{ inputs.expected_sha }}
RECORD: ${{ needs.plan-slices.outputs.record_artifact }}
RECONCILE_EXIT: ${{ steps.reconcile.outputs.exit }}
run: |
bun --no-env-file --config=/dev/null --no-install run scripts/train-gate.ts train-record /tmp/paid-report --nonce "$NONCE" --head "$HEAD_SHA" \
--reconcile-exit "$RECONCILE_EXIT" --run-id "$GITHUB_RUN_ID" --run-attempt "$GITHUB_RUN_ATTEMPT" --repo "$GITHUB_REPOSITORY" \
--out "/tmp/train-record/${RECORD}.json" >> "$GITHUB_STEP_SUMMARY"
- name: Upload the train record
if: always() && needs.plan-slices.outputs.mode == 'train'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: ${{ needs.plan-slices.outputs.record_artifact }}
path: /tmp/train-record
if-no-files-found: ignore
overwrite: true
retention-days: 90
- name: Fail the workflow when reconciliation failed
if: (steps.reconcile.outputs.exit || steps.reused.outputs.exit) != '0'
run: exit 1
# GSTA-49 T1b: the check run gstack-ship-queue requires on the train head,
# published only for a complete GREEN census. Its own job with checks:
# write and no repository code (the same split as slices-comment).
train-check:
runs-on: ubuntu-24.04
needs: [plan-slices, slices-report]
if: ${{ !cancelled() && needs.plan-slices.outputs.mode == 'train' && needs.slices-report.result == 'success' }}
timeout-minutes: 5
permissions:
checks: write
steps:
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
with:
name: ${{ needs.plan-slices.outputs.record_artifact }}
path: /tmp/train-record
- name: Publish the full-gate check on the train head
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
CHECK: ${{ needs.plan-slices.outputs.full_gate_check }}
RECORD: ${{ needs.plan-slices.outputs.record_artifact }}
NONCE: ${{ inputs.nonce }}
HEAD_SHA: ${{ inputs.expected_sha }}
run: |
record="/tmp/train-record/${RECORD}.json"
if ! jq -e --arg nonce "$NONCE" --arg head "$HEAD_SHA" \
'.schema == 1 and .kind == "train-gate" and .verdict == "GREEN" and .census.complete == true and .nonce == $nonce and .head_sha == $head' \
"$record" >/dev/null 2>&1; then
echo "::error::the train record is not a complete GREEN census of ${HEAD_SHA} (nonce ${NONCE}); no ${CHECK} check run"
exit 1
fi
summary=$(jq -r '"rule \(.census.executed.rule)/\(.census.planned.rule), judge \(.census.executed.judge)/\(.census.planned.judge); prefixes T1..T\(.prefixes | length)"' "$record")
gh api "repos/${GITHUB_REPOSITORY}/check-runs" -X POST -f name="$CHECK" -f head_sha="$HEAD_SHA" -f status=completed -f conclusion=success \
-f details_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
-f "output[title]=Train ${NONCE}: complete GREEN census" -f "output[summary]=${summary}" >/dev/null
echo "- ${CHECK}: success on ${HEAD_SHA} (${summary})" >> "$GITHUB_STEP_SUMMARY"
# PR comment in its OWN job with the write token and ZERO repo code: no
# checkout, no bun install — only downloaded artifacts, jq, and gh. See the
# trust-boundary note on slices-report.
slices-comment:
runs-on: ubicloud-standard-2
needs: slices-report
if: ${{ !cancelled() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && needs.slices-report.result != 'skipped' }}
# A dispatched screen comments on its PR too (screen_pr was validated by the route job, or nothing ran).
if: ${{ !cancelled() && ((github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || inputs.screen_pr != '') && needs.slices-report.result != 'skipped' }}
timeout-minutes: 5
permissions:
pull-requests: write
@@ -739,6 +1183,9 @@ jobs:
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
RECONCILE_EXIT: ${{ needs.slices-report.outputs.reconcile-exit }}
PR_NUMBER: ${{ github.event.pull_request.number || inputs.screen_pr }}
SCREEN_POLICY: ${{ inputs.screen_policy }}
SCREEN_MERGE_SHA: ${{ inputs.screen_merge_sha }}
run: |
# shellcheck disable=SC2086,SC2059
# R2a/R2: a confirmed verdict reuse ran no case; say which run,
@@ -845,7 +1292,13 @@ jobs:
fi
COMMENT_ID=$(gh api repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments \
if [ -n "$SCREEN_POLICY" ]; then
BODY="${BODY}
*Dispatched ${SCREEN_POLICY} screen (gstack-ship-queue ready --screen) of merge commit \`${SCREEN_MERGE_SHA}\`.*"
fi
COMMENT_ID=$(gh api "repos/${{ github.repository }}/issues/${PR_NUMBER}/comments" \
--jq '.[] | select(.body | startswith("## E2E Evals")) | .id' | tail -1)
if [ -n "$COMMENT_ID" ]; then
@@ -855,6 +1308,6 @@ jobs:
# REST, not gh's pr-comment subcommand: this job runs with NO
# checkout (the token/exec split), and that subcommand resolves
# the repo FROM git — it dies with "not a git repository" here.
gh api "repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments" \
gh api "repos/${{ github.repository }}/issues/${PR_NUMBER}/comments" \
-X POST -f body="$BODY"
fi
+15
View File
@@ -1,5 +1,20 @@
# Changelog
## [1.91.162.0] - 2026-10-19
**The evals workflow carries merge trains: a train's own gate publishes its record and the `gstack/full-gate` check, bisects run one executor per prefix, `ready --screen` dispatches a real screen, and a main backstop gates any tree no green train covered.** All of it stays off unless the base revision's `.gstack/ship-policy.json` declares a `train` block. Without one, a PR push and every eval-bg dispatch run today's jobs and steps with today's values. The one addition is a free `route` job that reads the policy with git and jq. gstack's own policy still has no `train` block; turning it on is a separate one-line policy PR. Plan GSTA-49, lane T1b.
### Added
- **Train gate runs** (`train/<nonce>` with a train nonce): the full census runs without verdict reuse or reuse binding. The report job writes the train record (`scripts/train-gate.ts train-record`) from this run's plan manifest, one panel per planned case, plus the prefix trees modulo stamps. It uploads the record as the policy's `record_artifact`. A separate `train-check` job, with `checks: write` and no repository code, publishes `full_gate_check` as success only for a complete GREEN census whose nonce and head match the dispatch. A train nonce dispatched on main is refused before any paid work.
- **Bisect inputs `cases` and `prefix_shas`.** Case ids must be in the run's plan, and each prefix must be an ancestor of the train head on `train/<nonce>`, or the run refuses. A `train-bisect` matrix runs one executor per prefix, checks out and verifies that prefix, and uploads `<record_artifact>-bisect-<sha12>`, the name `train bisect` reads.
- **Dispatched screens** (`screen_pr`, `screen_merge_sha`, `screen_policy`): the run checks out and verifies the pinned merge commit. It waits on the PR head's Free Tests, cancels an older screen of the same PR, and comments on that PR. Verdict reuse (R2a/R2) authenticates a screen by PR number, head SHA, repository and tested base (the merge commit's first parent). It keeps every selection and newer-run veto, and lists pull_request and screen runs filtered by PR. A broad screen never adopts a narrow run.
- **`.github/workflows/evals-main-backstop.yml`** runs on every push to main, one at a time. It does nothing without a `train` block. With one, it waits until no train is merging. It runs the full gate on main only when no green train record lists main's tree modulo stamps, and writes the freeze record `gstack-ship-queue` reads. A five-PR train fires zero backstop gates.
- **The free screening summary:** under a `train` block, a PR push runs readiness and prints the screen selection and an estimated cost instead of starting paid slices.
### Changed
- **Gate integrity (train block only):** under a `train` block, PR pushes no longer start paid slices; trains, screens and the backstop gate instead. Train and bisect runs never reuse a verdict and never bind one for reuse. No threshold, trial count, case kind or selection changed.
- **Gate integrity (always, GSTA-48 P3-1):** when the readiness job blocks a PR head, a new `readiness-comment` job replaces an existing `## E2E Evals` comment with "NOT RUN on <head>", so an earlier PASS no longer stands beside a head that never ran. It posts nothing when no such comment exists. A slice killed before its upload step counts as incomplete.
- **Gate integrity, widening (`plan-design-with-ui-scope` focus detector, GStack Minder-approved):** the Step 0D focus question also counts when its title names the focus (the title ends at the first line break or before `Project/branch/task:`), its body carries the step's own rating sentence ("I('ve) rated this plan N/10 on design completeness"), and it offers at least two options, one of which starts "All 7" or "All seven" (design or review) dimensions or passes. Opus 5.5 titled the question only "Review focus" in both attempts of run 39298269902, so the detector missed it. In the second attempt the finding detector also counted the focus question as the review finding, because its one-line title names hierarchy and panels. A recognized focus question can no longer count as a finding. Free replays of both captured attempts (`test/fixtures/design-ui-focus-capture-39298269902-a1.json` and `-a2.json`, registered in the case's touchfiles) pass. Every existing negative control still fails, as do these new ones: an issue-labeled finding that carries the rating, a single-option focus question, a focus question without a leading all-7 option, and a 4/7 rating. No threshold, trial count, case kind or ceiling changed.
## [1.91.161.0] - 2026-10-19
**`gstack-pregate` works on Django, Dart and Flutter repos, and on branches that only touch assets or tool config.** Before, a Django migration, an `admin.py`, a template, a font, a vendored licence or `.pre-commit-config.yaml` stopped `/ship` with `PREGATE_NO_LANE` in a repo that had never opted in, and every touched `_test.dart` file did the same. `gstack-pregate init` now drafts a policy from the repo instead of writing a fixed example. Contributed by @arlophoenix (#3285) and @szsunyuan (#3290); reported by @dgrant (#3266, #3267, #3268, #3269).
+1 -1
View File
@@ -1 +1 @@
1.91.161.0
1.91.162.0
+1 -1
View File
@@ -1,4 +1,4 @@
# gstack digest v1.91.161.0 — regenerate/re-copy after upgrading gstack
# gstack digest v1.91.162.0 — regenerate/re-copy after upgrading gstack
Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed
for agent hosts without a full skill install. The full skills add workflows,
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "gstack",
"version": "1.91.161",
"version": "1.91.162",
"description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.",
"license": "MIT",
"type": "module",
+41 -6
View File
@@ -16,6 +16,13 @@
* the gate (a re-run keeps the run's original creation time, so it does not
* wait out the window again). A head that is no longer the PR's head reports
* superseded and plans nothing.
*
* Dispatched screens (plan GSTA-49 T1b): `--merge-sha` names the
* refs/pull/<n>/merge commit `gstack-ship-queue ready --screen` pinned. The
* screen is accepted only for an open same-repository PR whose current head
* is that merge commit's second parent; its first parent is the tested base.
* The same Free Tests rule then applies to the PR head, with no quiet window
* (a screen is an explicit request).
*/
import * as fs from 'node:fs';
import { runGh, type GhRunner } from './recover-receipts';
@@ -73,6 +80,23 @@ function ghJson<T>(gh: GhRunner, args: string[]): { ok: true; value: T } | { ok:
try { return { ok: true, value: JSON.parse(out.stdout) as T }; } catch { return { ok: false, error: 'gh returned unreadable JSON' }; }
}
export interface ScreenTarget { head: string; base: string; branch: string }
/** A screen's merge commit, checked against the PR: same repository, open, and merging the PR's current head into its base. */
export function screenTarget(gh: GhRunner, repo: string, pr: number, mergeSha: string): ScreenTarget | { error: string } {
if (!/^[0-9a-f]{40}$/.test(mergeSha)) return { error: `screen_merge_sha ${JSON.stringify(mergeSha)} is not a full commit SHA` };
const pull = ghJson<{ state?: string; head?: { sha?: string; ref?: string; repo?: { full_name?: string } | null }; base?: { repo?: { full_name?: string } } }>(gh, ['api', `repos/${repo}/pulls/${pr}`]);
if (!pull.ok) return { error: `cannot read PR #${pr} (${pull.error})` };
const p = pull.value;
if (p.base?.repo?.full_name !== repo || p.head?.repo?.full_name !== repo) return { error: `PR #${pr} is not a same-repository PR of ${repo}; fork work is screened on a trusted branch` };
if (p.state !== 'open') return { error: `PR #${pr} is ${p.state ?? 'unreadable'}, not open` };
const commit = ghJson<{ parents?: Array<{ sha?: string }> }>(gh, ['api', `repos/${repo}/commits/${mergeSha}`]);
if (!commit.ok) return { error: `cannot read the merge commit ${mergeSha.slice(0, 12)} (${commit.error})` };
const parents = (commit.value.parents ?? []).map(parent => parent.sha ?? '');
if (parents.length !== 2 || !parents.every(sha => /^[0-9a-f]{40}$/.test(sha))) return { error: `${mergeSha.slice(0, 12)} is not a two-parent merge commit` };
return { head: parents[1]!, base: parents[0]!, branch: p.head?.ref ?? '' };
}
/** Waits until the head is ready, superseded or blocked; never longer than the deadline. */
export function waitForReadiness(o: WaitOptions): { decision: ReadinessDecision; lines: string[] } {
const gh = o.gh ?? runGh;
@@ -110,28 +134,39 @@ export function waitForReadiness(o: WaitOptions): { decision: ReadinessDecision;
}
}
const USAGE = `usage: bun run scripts/eval-readiness.ts wait --repo <owner/name> --pr <n> --head-sha <sha> --run-id <id>
const USAGE = `usage: bun run scripts/eval-readiness.ts wait --repo <owner/name> --pr <n> (--head-sha <sha> | --merge-sha <sha>) --run-id <id>
[--quiet-minutes 10] [--deadline-minutes 80] [--poll-seconds 60] [--github-output <file>] [--summary <file>]
Exit 0 when ready or superseded (output superseded=true), 1 when blocked, 2 on usage.`;
--merge-sha (a dispatched screen) waits on the PR head that merge commit merges and writes head_sha, base_sha and
head_branch outputs. Exit 0 when ready or superseded (output superseded=true), 1 when blocked, 2 on usage.`;
if (import.meta.main) {
const [command, ...rest] = process.argv.slice(2);
const flag = (name: string) => { const i = rest.indexOf(`--${name}`); return i >= 0 ? rest[i + 1] : undefined; };
const num = (name: string, fallback: number) => { const value = flag(name); return value === undefined || value === '' ? fallback : Number(value); };
const [repo, headSha] = [flag('repo'), flag('head-sha')];
const [repo, mergeSha] = [flag('repo'), flag('merge-sha')];
const [pr, runId, quiet, deadline, poll] = [Number(flag('pr')), Number(flag('run-id')), num('quiet-minutes', 10), num('deadline-minutes', 80), num('poll-seconds', 60)];
if (command !== 'wait' || !repo || !headSha || !/^[0-9a-f]{40}$/.test(headSha) || !Number.isSafeInteger(pr) || !Number.isSafeInteger(runId)
if (command !== 'wait' || !repo || !(flag('head-sha') ?? mergeSha) || !Number.isSafeInteger(pr) || !Number.isSafeInteger(runId)
|| ![quiet, deadline, poll].every(value => Number.isFinite(value) && value >= 0)) {
console.error(USAGE);
process.exit(2);
}
const { decision, lines } = waitForReadiness({ repo, pr, headSha, runId, quietMs: quiet * 60_000, deadlineMs: deadline * 60_000, pollMs: Math.max(5, poll) * 1000 });
const output = flag('github-output');
const summary = flag('summary');
const target = mergeSha ? screenTarget(runGh, repo, pr, mergeSha) : null;
if (target && 'error' in target) {
console.error(`::error::screen refused: ${target.error}`);
if (summary) fs.appendFileSync(summary, `- Screen refused: ${target.error}\n`);
process.exit(1);
}
const headSha = target ? target.head : flag('head-sha')!;
if (!/^[0-9a-f]{40}$/.test(headSha)) { console.error(USAGE); process.exit(2); }
if (target && output) fs.appendFileSync(output, `head_sha=${target.head}\nbase_sha=${target.base}\nhead_branch=${target.branch}\n`);
const { decision, lines } = waitForReadiness({ repo, pr, headSha, runId, quietMs: quiet * 60_000, deadlineMs: deadline * 60_000, pollMs: Math.max(5, poll) * 1000 });
if (target) lines.unshift(`Screen of PR #${pr}: merge ${mergeSha} merges head ${target.head.slice(0, 12)} into base ${target.base.slice(0, 12)}`);
for (const line of lines) {
console.log(line);
if (summary) fs.appendFileSync(summary, `- ${line}\n`);
}
const output = flag('github-output');
if (output && decision.kind === 'superseded') fs.appendFileSync(output, 'superseded=true\n');
if (decision.kind === 'blocked') console.error(`::error::${decision.reason}`);
process.exit(decision.kind === 'blocked' ? 1 : 0);
+80 -29
View File
@@ -33,6 +33,17 @@
* byte, a missing receipt or merge-base, gh or git errors, a selection change)
* leaves no candidate, and the run executes the full gate. The free suite and
* every other check run on the new head regardless.
*
* Dispatched screens (plan GSTA-49 T1b): a workflow_dispatch run with
* screen_pr tests the refs/pull/<n>/merge commit gstack-ship-queue pinned.
* Its run name ends ` screen #<n>`, and its binding adds the tested merge
* commit and the screen policy. A screen's find lists both events filtered by
* the PR, so every newer-run veto spans pull_request runs and screens. A
* dispatched binding is accepted only when it names this PR and repository,
* its head is one of the PR's commits, its tested commit merges that head
* into the bound base, and its policy equals this screen's (a pull_request
* run counts as broad: today's full PR selection). Train and bisect runs never
* bind and are never listed, so they neither reuse nor are reused.
*/
import { createHash } from 'node:crypto';
import { spawnSync } from 'node:child_process';
@@ -43,7 +54,7 @@ import { receiptDigest } from '../lib/ship-queue';
import { isUnavailable, loadPolicyFromBase, matchesAny, type ShipPolicy } from '../lib/ship-policy';
import { rewriteStamps } from '../lib/restamp';
import { buildReceipt } from '../lib/tree-receipt';
import { acceptedRun, ghFailureReason, runGh, type GhRunner, type WorkflowRun } from './recover-receipts';
import { ghFailureReason, listGateRuns, runGh, type GhRunner, type WorkflowRun } from './recover-receipts';
import type { PaidRunManifest } from './lib/paid-plan';
export const BINDING_FILE = 'gate-binding.json';
@@ -67,8 +78,16 @@ export interface GateBinding {
/** Cases accepted manually (unscored); a gate with any is never reused as a clean pass. */
manual_accepted: number;
selection_sha256: string;
/** Dispatched screens only: the purpose, the refs/pull/<n>/merge commit it tested, and the screen policy. */
purpose?: 'screen';
tested_sha?: string;
screen_policy?: ScreenPolicy;
}
export type ScreenPolicy = 'narrow' | 'broad';
/** What a run is for: train and bisect runs never bind a verdict and are never reused. */
export type RunPurpose = 'pr' | 'screen' | 'train' | 'bisect' | 'dispatch';
export interface GateReuse {
schema: 1;
kind: 'restamp-only' | 'main-moved';
@@ -91,6 +110,8 @@ export interface GateReuse {
gated_merge_base?: string;
merge_base?: string;
own_diff_sha256?: string;
/** The reused run was a dispatched screen with this policy. */
reused_screen_policy?: ScreenPolicy;
}
/** The cases, trial plans and verdict-relevant plan fields a run executes; slice packing and receipts are excluded. */
@@ -112,6 +133,7 @@ export function parseBinding(value: unknown): GateBinding | null {
if (!b || b.schema !== 1 || typeof b.repository !== 'string' || !Number.isSafeInteger(b.pr) || !Number.isSafeInteger(b.run_id)
|| !Number.isSafeInteger(b.run_attempt) || !SHA.test(b.head_sha ?? '') || !SHA.test(b.base_sha ?? '') || typeof b.profile !== 'string'
|| !Number.isSafeInteger(b.reconcile_exit) || typeof b.verdict !== 'string' || !Number.isSafeInteger(b.manual_accepted) || b.manual_accepted < 0 || !SHA256.test(b.selection_sha256 ?? '')) return null;
if (b.purpose !== undefined && (b.purpose !== 'screen' || !SHA.test(b.tested_sha ?? '') || (b.screen_policy !== 'narrow' && b.screen_policy !== 'broad'))) return null;
return b;
}
@@ -250,7 +272,8 @@ export function ownDiffSignature(git: Git, mergeBase: string, head: string, stam
/** slices-report: what this executed run proved, or why it binds nothing. */
export function buildBinding(o: { repository: string; pr: number; runId: number; runAttempt: number; headSha: string; baseSha: string | null;
profile: string; reconcileExit: number; reportDir: string }): { binding: GateBinding | null; reason: string } {
profile: string; reconcileExit: number; reportDir: string; purpose?: RunPurpose; testedSha?: string; screenPolicy?: string }): { binding: GateBinding | null; reason: string } {
if (o.purpose && o.purpose !== 'pr' && o.purpose !== 'screen') return { binding: null, reason: `a ${o.purpose} run never binds a verdict for reuse` };
if (fs.existsSync(path.join(o.reportDir, REUSE_FILE))) return { binding: null, reason: 'this run reused a verdict; only executed gates bind' };
if (!o.baseSha) return { binding: null, reason: 'the tested base is unknown' };
const manifest = readJson<PaidRunManifest>(path.join(o.reportDir, 'manifest.json'));
@@ -258,7 +281,8 @@ export function buildBinding(o: { repository: string; pr: number; runId: number;
const outcomes = readJson<{ verdict?: { verdict?: string }; totals?: { manual_accepted?: number } }>(path.join(o.reportDir, 'collector-outcomes.json'));
const binding: GateBinding = { schema: 1, repository: o.repository, pr: o.pr, run_id: o.runId, run_attempt: o.runAttempt,
head_sha: o.headSha, base_sha: o.baseSha, profile: o.profile, reconcile_exit: o.reconcileExit,
verdict: outcomes?.verdict?.verdict ?? 'MISSING', manual_accepted: outcomes?.totals?.manual_accepted ?? -1, selection_sha256: selectionFingerprint(manifest) };
verdict: outcomes?.verdict?.verdict ?? 'MISSING', manual_accepted: outcomes?.totals?.manual_accepted ?? -1, selection_sha256: selectionFingerprint(manifest),
...(o.purpose === 'screen' ? { purpose: 'screen' as const, tested_sha: o.testedSha ?? '', screen_policy: o.screenPolicy as ScreenPolicy } : {}) };
return parseBinding(binding)
? { binding, reason: `bound ${binding.verdict} at ${binding.head_sha.slice(0, 12)} on base ${binding.base_sha.slice(0, 12)} (reconcile exit ${binding.reconcile_exit})` }
: { binding: null, reason: 'binding fields are incomplete' };
@@ -266,22 +290,27 @@ export function buildBinding(o: { repository: string; pr: number; runId: number;
/**
* A newer completed run with no binding is passed over only when no paid
* slice of it finished: superseded, readiness-blocked, a reuse, or cancelled
* before or during its slices. A slice that finished (success, failure,
* timed out) without a bound verdict blocks reuse of an older GREEN, so a
* newer red or partial result is never skipped.
* slice of it ran: superseded, readiness-blocked, a reuse, or cancelled
* before its slice step started. A slice that finished (success, failure,
* timed out) without a bound verdict blocks reuse of an older GREEN, and so
* does a slice cancelled after its slice step started (killed before its
* upload step: its paid results are unknown, so it counts as incomplete), so a
* newer red, partial or unknown result is never skipped.
*/
export function unboundRunVerdict(gh: GhRunner, repo: string, run: WorkflowRun, timeoutMs: number): string | null {
const listed = gh(['api', '-X', 'GET', `repos/${repo}/actions/runs/${run.id}/jobs`, '-f', 'filter=latest', '-f', 'per_page=100'], timeoutMs);
if (listed.status !== 0) return `cannot list run ${run.id}'s jobs (${ghFailureReason(listed.stderr)})`;
let slices: Array<{ name: string; conclusion: string | null }>;
type Job = { name: string; conclusion: string | null; steps?: Array<{ name: string; status?: string; conclusion?: string | null }> };
let slices: Job[];
try {
slices = ((JSON.parse(listed.stdout).jobs ?? []) as Array<{ name: string; conclusion: string | null }>)
.filter(job => job.name === 'eval-slices' || job.name.startsWith('eval-slices '));
slices = ((JSON.parse(listed.stdout).jobs ?? []) as Job[]).filter(job => job.name === 'eval-slices' || job.name.startsWith('eval-slices '));
} catch { return `run ${run.id}'s job list is unreadable`; }
const finished = slices.find(job => job.conclusion !== null && job.conclusion !== 'skipped' && job.conclusion !== 'cancelled');
return finished
? `newer run ${run.id} (${run.head_sha.slice(0, 12)}) finished a paid slice (${finished.conclusion}) but bound no verdict (conclusion ${run.conclusion})`
if (finished) return `newer run ${run.id} (${run.head_sha.slice(0, 12)}) finished a paid slice (${finished.conclusion}) but bound no verdict (conclusion ${run.conclusion})`;
const killed = slices.find(job => job.conclusion === 'cancelled' && (job.steps ?? []).some(step => step.name.startsWith('Run slice')
&& step.status !== 'queued' && step.status !== 'pending' && step.conclusion !== 'skipped'));
return killed
? `newer run ${run.id} (${run.head_sha.slice(0, 12)}) had a paid slice (${killed.name}) killed after its slice step started and before its upload step; its results are unknown, so it counts as incomplete`
: null;
}
@@ -292,8 +321,10 @@ export interface FindOptions {
branch: string;
selfRunId: number;
headSha: string;
/** This run's tested merge commit (github.sha); its first parent is the base. */
/** This run's tested merge commit (github.sha, or a screen's refs/pull/<n>/merge commit); its first parent is the base. */
testedSha: string;
/** A dispatched screen: its policy; pull_request runs and earlier screens of this PR are both candidates. */
screen?: { policy: ScreenPolicy };
/** A base-ref checkout to fetch into and run the receipt in. */
repoRoot: string;
out: string;
@@ -315,13 +346,9 @@ export function findReusableGate(o: FindOptions): { candidate: GateReuse | null;
const refuse = (reason: string) => { lines.push(`no gate reuse: ${reason}; the full gate runs`); return { candidate: null, lines }; };
if (!SHA.test(o.headSha) || !SHA.test(o.testedSha)) return refuse('this run has no exact head and tested commit');
const listed = gh(['api', '-X', 'GET', `repos/${o.repo}/actions/workflows/evals.yml/runs`, '-f', 'event=pull_request', '-f', `branch=${o.branch}`, '-f', 'per_page=50'], left());
if (listed.status !== 0) return refuse(ghFailureReason(listed.stderr));
let runs: WorkflowRun[];
try { runs = (JSON.parse(listed.stdout).workflow_runs ?? []) as WorkflowRun[]; } catch { return refuse('gh returned an unreadable run list'); }
const scope = { pr: o.pr, headRepo: o.headRepo, selfRunId: o.selfRunId };
const older = runs.filter(run => acceptedRun(run, scope) && run.id < o.selfRunId)
.sort((a, b) => b.id - a.id).slice(0, o.maxRuns ?? 10);
const listed = listGateRuns(gh, { repo: o.repo, pr: o.pr, headRepo: o.headRepo, branch: o.branch, selfRunId: o.selfRunId, screens: !!o.screen }, left);
if ('error' in listed) return refuse(listed.error);
const older = listed.runs.filter(run => run.id < o.selfRunId).sort((a, b) => b.id - a.id).slice(0, o.maxRuns ?? 10);
const work = fs.mkdtempSync(path.join(os.tmpdir(), 'gate-reuse-'));
try {
@@ -352,9 +379,15 @@ export function findReusableGate(o: FindOptions): { candidate: GateReuse | null;
if (!found) return refuse('no earlier executed gate on this PR bound a verdict');
const { run, binding } = found;
if (!binding) return refuse(`run ${run.id}'s gate binding is missing or malformed`);
if (binding.run_id !== run.id || binding.pr !== o.pr || binding.repository !== o.repo || binding.head_sha !== run.head_sha) {
const screenRun = run.event === 'workflow_dispatch';
if (binding.run_id !== run.id || binding.pr !== o.pr || binding.repository !== o.repo
|| (screenRun ? binding.purpose !== 'screen' : binding.head_sha !== run.head_sha || binding.purpose !== undefined)) {
return refuse(`run ${run.id}'s gate binding does not match the run it came from`);
}
const boundPolicy = binding.screen_policy ?? 'broad';
if (o.screen && boundPolicy !== o.screen.policy) {
return refuse(`run ${run.id} gated the ${boundPolicy} selection; a ${o.screen.policy} screen never adopts it`);
}
if (run.conclusion !== 'success' || binding.verdict !== 'GREEN' || binding.reconcile_exit !== 0) {
return refuse(`the newest executed gate (run ${run.id}, ${binding.head_sha.slice(0, 12)}) is ${binding.verdict}, reconcile exit ${binding.reconcile_exit}, conclusion ${run.conclusion}`);
}
@@ -364,8 +397,17 @@ export function findReusableGate(o: FindOptions): { candidate: GateReuse | null;
if (binding.profile !== 'pr') return refuse(`run ${run.id} ran the ${binding.profile} profile, not the PR gate`);
if (binding.head_sha === o.headSha) return refuse(`run ${run.id} gated this same head; a re-run executes fresh`);
const fetched = git(['fetch', '--no-tags', '--depth=1', 'origin', binding.head_sha, o.headSha, o.testedSha]);
if (screenRun) {
const commits = gh(['api', `repos/${o.repo}/pulls/${o.pr}/commits`, '--paginate', '--jq', '.[].sha'], left());
if (commits.status !== 0) return refuse(`cannot list PR #${o.pr}'s commits (${ghFailureReason(commits.stderr)})`);
if (!commits.stdout.split('\n').includes(binding.head_sha)) return refuse(`screen run ${run.id}'s head ${binding.head_sha.slice(0, 12)} is not a commit of PR #${o.pr}`);
}
const fetched = git(['fetch', '--no-tags', '--depth=1', 'origin', binding.head_sha, o.headSha, o.testedSha, ...(binding.tested_sha ? [binding.tested_sha] : [])]);
if (!fetched.ok) return refuse(`cannot fetch the gated and current commits (${fetched.out.split('\n')[0]})`);
if (screenRun) {
const gatedBase = testedBase(git, binding.tested_sha!, binding.head_sha);
if ('error' in gatedBase || gatedBase.base !== binding.base_sha) return refuse(`screen run ${run.id}'s tested commit does not merge its head into its bound base (${'error' in gatedBase ? gatedBase.error : gatedBase.base.slice(0, 12)})`);
}
const tested = testedBase(git, o.testedSha, o.headSha);
if ('error' in tested) return refuse(tested.error);
const baseSha = tested.base;
@@ -376,7 +418,8 @@ export function findReusableGate(o: FindOptions): { candidate: GateReuse | null;
if (isUnavailable(policy)) return refuse(`the base's ship policy is unavailable (${policy.detail})`);
if (policy.errors.length) return refuse(`the base's ship policy is invalid (${policy.errors[0]!.message})`);
const common = { schema: 1 as const, pr: o.pr, reused_run_id: run.id, reused_run_attempt: binding.run_attempt, gated_head: binding.head_sha,
current_head: o.headSha, gated_base: binding.base_sha, base_sha: baseSha, selection_sha256: binding.selection_sha256 };
current_head: o.headSha, gated_base: binding.base_sha, base_sha: baseSha, selection_sha256: binding.selection_sha256,
...(binding.screen_policy ? { reused_screen_policy: binding.screen_policy } : {}) };
let candidate: GateReuse;
if (binding.base_sha === baseSha) {
@@ -439,7 +482,7 @@ export function reuseReport(reuse: GateReuse, repo: string): string {
: [`OWN DIFF: identical sha256=${reuse.own_diff_sha256} (merge-base ${reuse.gated_merge_base} -> ${reuse.merge_base}; stamp files excluded)`];
return [
`GATE REUSED (${reuse.kind === 'restamp-only' ? 'R2a, restamp-only' : 'R2, main moved'}): no paid case ran on ${reuse.current_head}`,
`reused verdict: GREEN from run ${reuse.reused_run_id} attempt ${reuse.reused_run_attempt} at ${reuse.gated_head} (https://github.com/${repo}/actions/runs/${reuse.reused_run_id})`,
`reused verdict: GREEN from run ${reuse.reused_run_id} attempt ${reuse.reused_run_attempt}${reuse.reused_screen_policy ? ` (a ${reuse.reused_screen_policy} screen)` : ''} at ${reuse.gated_head} (https://github.com/${repo}/actions/runs/${reuse.reused_run_id})`,
`why: ${reuse.why}`,
`base: ${reuse.gated_base} -> ${reuse.base_sha}`,
...evidence,
@@ -450,9 +493,11 @@ export function reuseReport(reuse: GateReuse, repo: string): string {
const USAGE = `usage:
bun run scripts/gate-reuse.ts bind <report dir> --out <dir> --repo <owner/name> --pr <n> --run-id <id> --run-attempt <n>
--head-sha <sha> --tested-sha <merge sha> --profile <pr|full> --reconcile-exit <n>
--head-sha <sha> --tested-sha <merge sha> --profile <pr|full> --reconcile-exit <n> [--purpose pr|screen|train|bisect|dispatch]
[--screen-policy narrow|broad]
bun run scripts/gate-reuse.ts find --repo <owner/name> --pr <n> --head-repo <owner/name> --branch <head branch> --run-id <id>
--head-sha <sha> --tested-sha <merge sha> --out <dir> [--budget-seconds 90]
--head-sha <sha> --tested-sha <merge sha> --out <dir> [--screen-policy narrow|broad] [--budget-seconds 90]
--screen-policy marks a dispatched screen (its tested sha is the pinned refs/pull/<n>/merge commit).
bun run scripts/gate-reuse.ts confirm <candidate dir> <manifest.json> --out <paid-plan dir> [--github-output <file>]
bun run scripts/gate-reuse.ts report <gate-reuse.json> --repo <owner/name>
bind and find never fail the run; no candidate means the full gate runs.`;
@@ -468,9 +513,12 @@ if (import.meta.main) {
const git = gitIn(process.cwd());
if (!commitParents(git, testedSha!)) git(['fetch', '--no-tags', '--depth=1', 'origin', testedSha!]);
const tested = testedBase(git, testedSha!, headSha!);
const purpose = (flag('purpose') || 'pr') as RunPurpose;
if (!['pr', 'screen', 'train', 'bisect', 'dispatch'].includes(purpose) || (purpose === 'screen' && !['narrow', 'broad'].includes(flag('screen-policy') ?? ''))) usage();
const { binding, reason } = buildBinding({ repository: repository!, pr: int('pr'), runId: int('run-id'), runAttempt: int('run-attempt'),
headSha: headSha!, baseSha: 'base' in tested ? tested.base : null, profile: profile!,
reconcileExit: /^\d+$/.test(flag('reconcile-exit') ?? '') ? int('reconcile-exit') : -1, reportDir: reportDir! });
reconcileExit: /^\d+$/.test(flag('reconcile-exit') ?? '') ? int('reconcile-exit') : -1, reportDir: reportDir!,
purpose, testedSha: testedSha!, screenPolicy: flag('screen-policy') });
if (binding) {
fs.mkdirSync(out!, { recursive: true });
fs.writeFileSync(path.join(out!, BINDING_FILE), `${JSON.stringify(binding, null, 2)}\n`);
@@ -478,9 +526,12 @@ if (import.meta.main) {
console.log(`- Gate binding: ${reason}${'error' in tested ? ` (${tested.error})` : ''}`);
} else if (command === 'find') {
const [repo, headRepo, branch, headSha, testedSha, out] = [flag('repo'), flag('head-repo'), flag('branch'), flag('head-sha'), flag('tested-sha'), flag('out')];
if (!repo || !headRepo || !branch || !headSha || !testedSha || !out || !Number.isSafeInteger(int('pr')) || !Number.isSafeInteger(int('run-id'))) usage();
const policy = flag('screen-policy');
if (!repo || !headRepo || !branch || !headSha || !testedSha || !out || !Number.isSafeInteger(int('pr')) || !Number.isSafeInteger(int('run-id'))
|| (policy !== undefined && policy !== 'narrow' && policy !== 'broad')) usage();
const { lines } = findReusableGate({ repo: repo!, pr: int('pr'), headRepo: headRepo!, branch: branch!, selfRunId: int('run-id'), headSha: headSha!,
testedSha: testedSha!, repoRoot: process.cwd(), out: out!, budgetMs: Number(flag('budget-seconds') ?? '90') * 1000 });
testedSha: testedSha!, repoRoot: process.cwd(), out: out!, budgetMs: Number(flag('budget-seconds') ?? '90') * 1000,
...(policy ? { screen: { policy: policy as ScreenPolicy } } : {}) });
console.log('### Paid-gate verdict reuse (R2a restamp-only, R2 main moved)');
for (const line of lines) console.log(`- ${line}`);
} else if (command === 'confirm') {
+1 -1
View File
@@ -17,7 +17,7 @@
"scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6},
"scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":6},
"scripts/test-paid-shards.ts": {"blob":"7a362c457cd0536dad60aecc675d67599759c8a3","since":6,"nonBehavioral":"GSTA-56: only the shebang changed, to the isolated form (--no-env-file --config=/dev/null); the paid runner is launched via bun run, which ignores the shebang, so no verdict can change"},
"scripts/test-pr-profile.ts": {"blob":"579ca1bbd590f709c1ea3d6f28fa6c629688c79f","since":6,"nonBehavioral":"Classifies scripts/free-test-seed-allowlist.json as free-only; only the free seed ratchet reads it, so no verdict can change"},
"scripts/test-pr-profile.ts": {"blob":"d334195b0d0393622e2c380dc3e2feebe099b6e6","since":6,"nonBehavioral":"classify the new evals-main-backstop.yml workflow as free-lane only: it dispatches evals.yml on main and runs inside no case, so no case's selection on a PR, input or verdict changes"},
"scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":6},
"test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":6},
"test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":6},
+33
View File
@@ -57,6 +57,7 @@ export interface WorkflowRun {
updated_at?: string;
head_repository?: { full_name?: string } | null;
pull_requests?: Array<{ number: number }>;
display_title?: string;
}
/** One `gh` invocation; injectable so free tests drive recovery from recorded responses. */
@@ -104,6 +105,38 @@ export function acceptedRun(run: WorkflowRun, scope: { pr: number; headRepo: str
&& (run.pull_requests ?? []).some(pr => pr.number === scope.pr);
}
/** The PR a dispatched screen names in its run name (evals.yml: `E2E Evals [eval-bg <nonce>] screen #<n>`); null for any other run. */
export function screenRunPr(run: Pick<WorkflowRun, 'event' | 'display_title'>): number | null {
const match = run.event === 'workflow_dispatch' ? /^E2E Evals \[eval-bg [A-Za-z0-9._-]+\] screen #(\d+)$/.exec(run.display_title ?? '') : null;
return match ? Number(match[1]) : null;
}
/** A dispatched screen of this PR from the base repository (same-repository PRs only), never this run. */
export function acceptedScreenRun(run: WorkflowRun, scope: { pr: number; repo: string; selfRunId: number }): boolean {
return run.id !== scope.selfRunId && screenRunPr(run) === scope.pr && run.head_repository?.full_name === scope.repo;
}
function listRuns(gh: GhRunner, repo: string, filter: string[], timeoutMs: number): WorkflowRun[] | { error: string } {
const listed = gh(['api', '-X', 'GET', `repos/${repo}/actions/workflows/evals.yml/runs`, ...filter.flatMap(f => ['-f', f]), '-f', 'per_page=50'], timeoutMs);
if (listed.status !== 0) return { error: ghFailureReason(listed.stderr) };
try { return (JSON.parse(listed.stdout).workflow_runs ?? []) as WorkflowRun[]; } catch { return { error: 'gh returned an unreadable run list' }; }
}
/**
* The PR's evals.yml runs whose results may carry over: its pull_request runs
* and, with `screens`, its dispatched screens (both events, filtered by the PR).
*/
export function listGateRuns(gh: GhRunner, o: { repo: string; pr: number; headRepo: string; branch: string; selfRunId: number; screens: boolean },
left: () => number): { runs: WorkflowRun[] } | { error: string } {
const prRuns = listRuns(gh, o.repo, ['event=pull_request', `branch=${o.branch}`], left());
if ('error' in prRuns) return prRuns;
const runs = prRuns.filter(run => acceptedRun(run, { pr: o.pr, headRepo: o.headRepo, selfRunId: o.selfRunId }));
if (!o.screens) return { runs };
const screens = listRuns(gh, o.repo, ['event=workflow_dispatch'], left());
if ('error' in screens) return screens;
return { runs: [...runs, ...screens.filter(run => acceptedScreenRun(run, { pr: o.pr, repo: o.repo, selfRunId: o.selfRunId }))] };
}
/**
* Negative receipts a cancelled run's completed work proves (ENG-5): every
* completed FAIL panel whose trials share one input identity, and every
+1
View File
@@ -274,6 +274,7 @@ export const FREE_ONLY_PR_FILES = [
'.github/workflows/skill-docs.yml',
'.github/workflows/test-health.yml',
'.github/workflows/eval-sweep.yml', // The weekly off-ship sweep on main; no PR-gate case reads it, and it never gates a PR.
'.github/workflows/evals-main-backstop.yml', // The merge-train backstop on main: it dispatches evals.yml after a push to main; no PR-gate case reads it, and it never gates a PR.
'.github/actionlint.yaml', // Runner labels for actionlint only.
'scripts/ship-measure-sweep.ts', // The weekly sweep CLI; it launches paid cases, never runs inside one.
'.github/workflows/version-gate.yml',
+212
View File
@@ -0,0 +1,212 @@
/**
* The main backstop (evals-main-backstop.yml; plan GSTA-49 lane T1b): every
* push to main is covered by a full gate, and a train's merges are covered by
* the gate the train already ran, so a train costs no second gate.
*
* 1. Inert unless main's .gstack/ship-policy.json declares a `train` block.
* 2. Debounce: wait until no train is merging (refs/notes/gstack-trains
* holds no train in state `merging`), up to the settle window; a newer
* push to main supersedes this run.
* 3. Compute main's tree modulo stamps (scripts/train-gate.ts stampFreeTree)
* and look for a green-train record that lists it among its prefix trees
* T1..Tk: a successful train dispatch of the gate workflow whose record
* artifact is GREEN with a complete census and whose `full_gate_check`
* check run on the train head succeeded.
* 4. Otherwise dispatch the full gate on main (eval-bg's backstop nonce),
* wait for it, and on anything but success write the freeze record
* gstack-ship-queue reads (red cases, their declared inputs, the run, an
* issue) to refs/notes/gstack-trains, unless a freeze is already active.
*/
import { randomBytes } from 'node:crypto';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { loadPolicyFromHead, type TrainPolicy } from '../lib/ship-policy';
import { NOTES_REF, fetchNotes, nowIso, persist, repoIdentity, type FreezeRecord } from '../lib/ship-queue-train-record';
import { E2E_TOUCHFILES, LLM_JUDGE_TOUCHFILES } from '../test/helpers/touchfiles-data';
import { DISPATCH_INPUTS, makeNonce, nonceScope } from './eval-bg';
import { ghFailureReason, runGh, type GhRunner } from './recover-receipts';
import { gitIn, stampFreeTree, type Git, type TrainRecord } from './train-gate';
const SHA = /^[0-9a-f]{40}$/;
const GH_MS = 60_000;
export type BackstopAction = 'inert' | 'superseded' | 'covered' | 'deferred' | 'gated' | 'frozen' | 'failed';
export interface BackstopOptions {
repo: string; branch: string; sha: string; repoRoot: string;
gh?: GhRunner; git?: Git; now?: () => number; sleep?: (ms: number) => void;
settleMs?: number; pollMs?: number; gateDeadlineMs?: number; random?: string;
}
interface Run { id: number; status: string; conclusion: string | null; head_sha: string; display_title?: string; html_url?: string }
function json<T>(r: { status: number | null; stdout: string }): T | null {
if (r.status !== 0) return null;
try { return JSON.parse(r.stdout) as T; } catch { return null; }
}
/** Notes records of one kind for this repository (the local notes ref, after a fetch). */
function noteRecords<T extends { kind: string; repo_key?: string }>(git: Git, kind: string, repoKey: string): T[] {
const listed = git(['notes', `--ref=${NOTES_REF}`, 'list']);
if (!listed.ok || !listed.out.trim()) return [];
return listed.out.split('\n').map(line => line.split(' ')[0]!).filter(Boolean).flatMap(blob => {
try { const r = JSON.parse(git(['cat-file', 'blob', blob]).out) as T; return r?.kind === kind && r.repo_key === repoKey ? [r] : []; } catch { return []; }
});
}
/** The newest green-train record whose prefix trees include this digest, with its run; null when none. */
export function coveringRecord(gh: GhRunner, repo: string, train: TrainPolicy, digest: string, lines: string[]): { run: Run; record: TrainRecord; index: number } | null {
const runs = json<{ workflow_runs?: Run[] }>(gh(['api', '-X', 'GET', `repos/${repo}/actions/workflows/${train.gate_workflow}/runs`, '-f', 'event=workflow_dispatch', '-f', 'status=success', '-f', 'per_page=50'], GH_MS));
if (!runs) { lines.push('cannot list the train gate runs; the backstop gates main'); return null; }
const trains = (runs.workflow_runs ?? []).map(run => ({ run, nonce: /\[eval-bg ([A-Za-z0-9._-]+)\]/.exec(run.display_title ?? '')?.[1] ?? '' }))
.filter(t => t.run.conclusion === 'success' && nonceScope(t.nonce)?.purpose === 'train').sort((a, b) => b.run.id - a.run.id).slice(0, 20);
const scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'train-backstop-'));
try {
for (const { run, nonce } of trains) {
const dir = path.join(scratch, String(run.id));
if (gh(['run', 'download', String(run.id), '-R', repo, '-n', train.record_artifact, '-D', dir], GH_MS).status !== 0) continue;
const file = fs.existsSync(dir) ? fs.readdirSync(dir).find(name => name.endsWith('.json')) : undefined;
let record: TrainRecord | null = null;
try { record = file ? JSON.parse(fs.readFileSync(path.join(dir, file), 'utf8')) as TrainRecord : null; } catch { record = null; }
if (!record || record.kind !== 'train-gate' || record.verdict !== 'GREEN' || record.census?.complete !== true || record.nonce !== nonce
|| record.head_sha !== run.head_sha || !Array.isArray(record.prefixes)) continue;
const hit = record.prefixes.find(p => p.stamp_free_tree === digest);
if (!hit) continue;
const checks = json<{ check_runs?: Array<{ name?: string; conclusion?: string | null }> }>(
gh(['api', `repos/${repo}/commits/${run.head_sha}/check-runs?check_name=${encodeURIComponent(train.full_gate_check)}`], GH_MS));
if (!checks?.check_runs?.some(c => c.name === train.full_gate_check && c.conclusion === 'success')) {
lines.push(`train ${nonce} (run ${run.id}) lists this tree but its ${train.full_gate_check} check run is not a success`);
continue;
}
return { run, record, index: hit.index };
}
return null;
} finally {
fs.rmSync(scratch, { recursive: true, force: true });
}
}
/** Red cases of a finished gate run, from its trial outcomes (first attempt), with their declared inputs. */
function redCases(gh: GhRunner, repo: string, run: Run): Array<{ id: string; inputs: string[] }> {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'train-backstop-red-'));
try {
if (gh(['run', 'download', String(run.id), '-R', repo, '-n', 'trial-outcomes-pr-a1', '-D', dir], GH_MS).status !== 0) return [];
const text = fs.existsSync(path.join(dir, 'trial-outcomes.jsonl')) ? fs.readFileSync(path.join(dir, 'trial-outcomes.jsonl'), 'utf8') : '';
const ids = new Set<string>();
for (const line of text.split('\n').filter(Boolean)) {
try { const r = JSON.parse(line) as { case?: string; outcome?: string; attempt?: number }; if (r.outcome === 'failed' && (r.attempt ?? 1) === 1 && r.case) ids.add(r.case); } catch { /* skip */ }
}
return [...ids].sort().map(id => ({ id, inputs: E2E_TOUCHFILES[id] ?? LLM_JUDGE_TOUCHFILES[id] ?? [] }));
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
}
/** Dispatch the full gate on main, find its run by nonce, and follow it to its conclusion. */
function gateMain(o: Required<Pick<BackstopOptions, 'repo' | 'branch' | 'sha'>> & { gh: GhRunner; train: TrainPolicy; now: () => number; sleep: (ms: number) => void; pollMs: number; deadline: number; nonce: string },
lines: string[]): Run | null {
const inputs = { [DISPATCH_INPUTS.evalsAll]: 'true', [DISPATCH_INPUTS.expectedSha]: o.sha, [DISPATCH_INPUTS.nonce]: o.nonce };
const sent = o.gh(['workflow', 'run', o.train.gate_workflow, '--repo', o.repo, '--ref', o.branch, ...Object.entries(inputs).flatMap(([k, v]) => ['-f', `${k}=${v}`])], GH_MS);
if (sent.status !== 0) { lines.push(`the gate dispatch failed (${ghFailureReason(sent.stderr)})`); return null; }
lines.push(`BACKSTOP: dispatched ${o.train.gate_workflow} on ${o.branch}@${o.sha.slice(0, 12)} (nonce ${o.nonce})`);
let run: Run | null = null;
while (o.now() < o.deadline) {
if (!run) {
const runs = json<{ workflow_runs?: Run[] }>(o.gh(['api', '-X', 'GET', `repos/${o.repo}/actions/workflows/${o.train.gate_workflow}/runs`, '-f', 'event=workflow_dispatch', '-f', 'per_page=30'], GH_MS));
run = runs?.workflow_runs?.find(r => (r.display_title ?? '').includes(`eval-bg ${o.nonce}]`)) ?? null;
if (run) lines.push(`BACKSTOP: gate run ${run.id} ${run.html_url ?? ''}`.trim());
} else {
run = json<Run>(o.gh(['api', `repos/${o.repo}/actions/runs/${run.id}`], GH_MS)) ?? run;
if (run.status === 'completed') return run;
}
o.sleep(o.pollMs);
}
lines.push(`the gate ${run ? `run ${run.id}` : 'dispatch'} did not finish before the backstop deadline`);
return run;
}
export function runBackstop(o: BackstopOptions): { action: BackstopAction; lines: string[] } {
const gh = o.gh ?? runGh;
const git = o.git ?? gitIn(o.repoRoot);
const now = o.now ?? Date.now;
const sleep = o.sleep ?? ((ms: number) => Bun.sleepSync(ms));
const pollMs = o.pollMs ?? 60_000;
const lines: string[] = [];
const done = (action: BackstopAction) => ({ action, lines });
const loaded = loadPolicyFromHead(o.repoRoot);
const train = loaded.policy.train;
if (!train) { lines.push(`BACKSTOP: inert (${loaded.source.label} declares no train block; per-PR gates cover main)`); return done('inert'); }
if (!SHA.test(o.sha)) { lines.push(`BACKSTOP: ${o.sha} is not a commit SHA`); return done('failed'); }
const repoKey = repoIdentity(o.repoRoot).key;
const settleUntil = now() + (o.settleMs ?? 60 * 60_000);
for (;;) {
const fetched = fetchNotes(o.repoRoot);
const merging = noteRecords<{ kind: string; repo_key: string; nonce: string; state: string }>(git, 'train', repoKey).filter(t => t.state === 'merging');
const tip = git(['ls-remote', 'origin', `refs/heads/${o.branch}`]).out.split(/\s/)[0];
if (tip && tip !== o.sha) { lines.push(`BACKSTOP: superseded (${o.branch} moved to ${tip.slice(0, 12)}; that push's run gates it)`); return done('superseded'); }
if (!merging.length) break;
if (now() >= settleUntil) { lines.push(`BACKSTOP: train(s) ${merging.map(t => t.nonce).join(', ')} still merging after the settle window; gating anyway`); break; }
lines.push(`BACKSTOP: waiting for train(s) ${merging.map(t => t.nonce).join(', ')} to finish merging${fetched.ok ? '' : ` (${fetched.detail})`}`);
sleep(pollMs);
}
const digest = stampFreeTree(git, o.sha, o.repoRoot, loaded.policy);
if (typeof digest !== 'string') { lines.push(`BACKSTOP: no tree modulo stamps (${digest.error}); gating main`); }
const covered = typeof digest === 'string' ? coveringRecord(gh, o.repo, train, digest, lines) : null;
if (covered) {
lines.push(`BACKSTOP: covered: main ${o.sha.slice(0, 12)} is T${covered.index} of green train ${covered.record.nonce} (run ${covered.run.id}, tree modulo stamps ${String(digest).slice(0, 12)}); no gate`);
return done('covered');
}
const owned = noteRecords<FreezeRecord>(git, 'freeze', repoKey).find(f => !f.cleared_at && f.main.sha === o.sha && f.backstop);
if (owned?.backstop) {
lines.push(`BACKSTOP: deferred: train ${owned.train} froze the queue on this commit and dispatched backstop run ${owned.backstop.run ?? 'unknown'} (nonce ${owned.backstop.nonce}); no second gate`);
return done('deferred');
}
lines.push(`BACKSTOP: no green-train record lists main's tree modulo stamps${typeof digest === 'string' ? ` ${digest.slice(0, 12)}` : ''}; gating main`);
const nonce = makeNonce('gate', o.sha, o.sha, o.random ?? randomBytes(3).toString('hex'), { purpose: 'backstop' });
const run = gateMain({ repo: o.repo, branch: o.branch, sha: o.sha, gh, train, now, sleep, pollMs, deadline: now() + (o.gateDeadlineMs ?? 270 * 60_000), nonce }, lines);
if (run?.status === 'completed' && run.conclusion === 'success') { lines.push(`BACKSTOP: gated: run ${run.id} is green on main ${o.sha.slice(0, 12)}`); return done('gated'); }
return freeze(o, { gh, git, repoKey, nonce, run }, lines);
}
/** Write the freeze record (local state, then refs/notes/gstack-trains) unless a freeze is already active. */
function freeze(o: BackstopOptions, c: { gh: GhRunner; git: Git; repoKey: string; nonce: string; run: Run | null }, lines: string[]): { action: BackstopAction; lines: string[] } {
const active = noteRecords<FreezeRecord>(c.git, 'freeze', c.repoKey).find(f => !f.cleared_at);
if (active) { lines.push(`BACKSTOP: red, and the queue is already frozen (${active.reason}); the existing freeze stands`); return { action: 'frozen', lines }; }
const red = c.run?.status === 'completed' ? redCases(c.gh, o.repo, c.run) : [];
const runText = c.run ? `run ${c.run.id} (${c.run.status}/${c.run.conclusion ?? '-'})` : 'no run';
const cases = red.length ? red.map(r => `${r.id} (inputs ${r.inputs.slice(0, 4).join(', ') || 'none declared'})`).join('; ') : 'no red case recorded';
const body = [`The main backstop gate is not green on ${o.sha}: ${runText}.`, '', `Red cases: ${cases}.`, '', 'gstack-ship-queue refuses ordinary trains until a repair train merges (docs/ship-policy.md#run-a-merge-train). Land the revert or fix-forward.'].join('\n');
const issue = c.gh(['issue', 'create', '-R', o.repo, '--title', `Main backstop red on ${o.sha.slice(0, 12)}`, '--body', body], GH_MS);
const issueUrl = issue.status === 0 ? issue.stdout.trim().split('\n').at(-1) ?? null : null;
const at = nowIso();
const record: FreezeRecord & { red_cases: typeof red; run_url: string | null; issue: string | null } = {
schema: 1, kind: 'freeze', repo_key: c.repoKey, train: 'backstop', merged: [], unmerged: [],
main: { sha: o.sha, tree: c.git(['rev-parse', `${o.sha}^{tree}`]).out },
reason: `backstop red on main ${o.sha.slice(0, 12)}: ${cases}; ${runText}; fix: land the revert or fix-forward${issueUrl ? ` (${issueUrl})` : ''}`,
backstop: { nonce: c.nonce, run: c.run?.id ?? null, state: c.run ? `${c.run.status}/${c.run.conclusion ?? '-'}` : 'unconfirmed' },
repair: null, created_at: at, cleared_at: null, cleared_by: null, updated_at: at,
red_cases: red, run_url: c.run?.html_url ?? null, issue: issueUrl,
};
const pushed = persist(o.repoRoot, [record]);
lines.push(`BACKSTOP: frozen: ${record.reason}`, `BACKSTOP: freeze record ${pushed.detail}`);
return { action: pushed.pushed ? 'frozen' : 'failed', lines };
}
const USAGE = `usage: bun run scripts/train-backstop.ts run --repo <owner/name> --branch <main> --sha <sha>
[--settle-minutes 60] [--poll-seconds 60] [--gate-deadline-minutes 270] [--summary <file>]
Exit 0 when inert, superseded, covered, deferred to a freeze's own backstop, or gated green; 1 when it froze the queue or could not finish.`;
if (import.meta.main) {
const [command, ...rest] = process.argv.slice(2);
const flag = (name: string) => { const i = rest.indexOf(`--${name}`); return i >= 0 ? rest[i + 1] : undefined; };
const minutes = (name: string, fallback: number) => Number(flag(name) ?? fallback) * 60_000;
const [repo, branch, sha] = [flag('repo'), flag('branch'), flag('sha')];
if (command !== 'run' || !repo || !branch || !sha) { console.error(USAGE); process.exit(2); }
const { action, lines } = runBackstop({ repo: repo!, branch: branch!, sha: sha!, repoRoot: process.cwd(), settleMs: minutes('settle-minutes', 60),
pollMs: Number(flag('poll-seconds') ?? 60) * 1000, gateDeadlineMs: minutes('gate-deadline-minutes', 270) });
const summary = flag('summary');
for (const line of lines) {
console.log(line);
if (summary) fs.appendFileSync(summary, `- ${line}\n`);
}
process.exit(action === 'frozen' || action === 'failed' ? 1 : 0);
}
+365
View File
@@ -0,0 +1,365 @@
/**
* The merge-train side of evals.yml (plan GSTA-49 lane T1b;
* docs/ship-policy.md#run-a-merge-train). gstack-ship-queue (lib/ship-queue-train*.ts)
* dispatches the runs and reads what they publish; this file is the workflow's half.
*
* route (route job, dispatches) names the run's mode and validates
* its inputs: a screen needs a PR, its pinned merge SHA and a
* policy; a bisect needs case ids from the full gate's own plan
* and prefix SHAs that are ancestors of the train head on
* train/<nonce>. Anything else is refused before any spend.
* bisect-plan (plan-slices) one executor per prefix, each expecting its SHA.
* bisect-prefix (eval-slices) one prefix: every red case once, one trial,
* written as the per-prefix record `train bisect` reads.
* train-record (slices-report, train runs) the green-train record: the census
* from the run's own plan manifest, one panel per planned case,
* and the prefix trees T1..Tk modulo stamps. GREEN only for a
* complete census: every planned case executed and passed, no
* unbound result, partial or cancelled shard, or slice killed
* before its upload step.
* screen-summary (plan-slices, PR runs under a train policy) the screening
* selection and its estimated cost; no paid case runs.
*/
import { spawnSync } from 'node:child_process';
import { createHash } from 'node:crypto';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { normalizeChangelog } from '../lib/changelog-check';
import { normalizeLockfile } from '../lib/evidence-bundle';
import { rewriteStamps, versionAtRef } from '../lib/restamp';
import { loadPolicyFromHead, matchesAny, type ShipPolicy } from '../lib/ship-policy';
import type { BisectPrefixRecord, TrainGateRecord } from '../lib/ship-queue-train-verdict';
import type { CasePanel } from '../lib/ship-queue-train-record';
import { E2E_TIERS, LLM_JUDGE_TOUCHFILES } from '../test/helpers/touchfiles-data';
import { panelVerdict } from '../test/helpers/eval-store';
import { UNRECORDED_SHARD_MS, nonceScope, plannedSerialMs } from './eval-bg';
import { caseTrialPlan, fileCaseRegistration, shardCaseId, shardFile } from './lib/paid-cases';
import { buildRunManifest, loadPaidTestDurations, parseRunManifest, recordedShardMs, type PaidRunManifest } from './lib/paid-plan';
import { loadSliceArtifacts } from './lib/paid-report';
import { paidSelectionEnv } from './lib/paid-select';
import { ROOT } from './lib/paid-types';
const SHA = /^[0-9a-f]{40}$/;
const CASE_ID = /^[A-Za-z0-9 _.:/-]{1,128}$/;
export const JUDGE_FILE = 'test/skill-llm-eval.test.ts';
/** T1 caps a train at 6 PRs, so a bisect has at most 7 prefixes (T0..T6). */
export const MAX_PREFIXES = 7;
export const MAX_BISECT_CASES = 40;
/** A full gate's recorded cost (MEASUREMENT.md: $90.93 to $94.70; lib/gate-cost.ts uses the same scale). */
export const FULL_GATE_USD = 93.14;
/** T1's prefix commits: `train T<i>: merge #<pr> at <sha12>` (lib/ship-queue-train.ts buildChain). */
const PREFIX_SUBJECT = /^train T(\d+): merge #(\d+) at [0-9a-f]{12}$/;
export type Git = (args: string[]) => { ok: boolean; out: string };
export const gitIn = (cwd: string): Git => (args) => {
const r = spawnSync('git', args, { cwd, encoding: 'utf8', timeout: 60_000, maxBuffer: 256 * 1024 * 1024 });
return r.status === 0 ? { ok: true, out: (r.stdout ?? '').replace(/\n$/, '') } : { ok: false, out: (r.stderr || String(r.error ?? '')).trim() };
};
export type RunMode = 'screen' | 'train' | 'bisect' | 'dispatch';
export interface RouteInput {
nonce: string; refName: string; head: string; expectedSha: string; evalsAll: boolean; trainPolicy: boolean;
cases: string; prefixShas: string; screenPr: string; screenMergeSha: string; screenPolicy: string;
}
export interface Route { mode: RunMode; cases: string[]; prefixes: string[] }
const list = (text: string) => text.split(',').map(item => item.trim()).filter(Boolean);
/**
* The run's mode from a dispatch's inputs, or why it is refused. Screens,
* trains and bisects exist only under a train policy; a bisect's case ids must
* be in the full gate's plan and each prefix an ancestor of the train head.
*/
export function routeDispatch(i: RouteInput, deps: { isAncestor(commit: string, head: string): boolean; gateCases(): Map<string, string> }): Route | { error: string } {
const scope = i.nonce ? nonceScope(i.nonce) : null;
const trainOnly = (mode: string) => (i.trainPolicy ? null : { error: `a ${mode} dispatch needs a train block in the default branch's .gstack/ship-policy.json` });
if (i.screenPr || i.screenMergeSha || i.screenPolicy) {
if (!/^[1-9]\d{0,8}$/.test(i.screenPr)) return { error: `screen_pr ${JSON.stringify(i.screenPr)} is not a PR number` };
if (!SHA.test(i.screenMergeSha)) return { error: `screen_merge_sha ${JSON.stringify(i.screenMergeSha)} is not a full commit SHA` };
if (i.screenPolicy !== 'narrow' && i.screenPolicy !== 'broad') return { error: `screen_policy ${JSON.stringify(i.screenPolicy)} is not narrow or broad` };
if (i.cases || i.prefixShas) return { error: 'a screen takes no cases or prefix_shas' };
if (i.evalsAll) return { error: 'a screen runs the PR selection: dispatch it with evals_all=false' };
if (i.expectedSha && i.expectedSha !== i.screenMergeSha) return { error: `expected_sha ${i.expectedSha.slice(0, 12)} is not the screen's merge SHA ${i.screenMergeSha.slice(0, 12)}` };
if (scope && (scope.purpose !== 'screen' || scope.policy !== i.screenPolicy)) return { error: `nonce ${i.nonce} is not a ${i.screenPolicy} screen nonce` };
return trainOnly('screen') ?? { mode: 'screen', cases: [], prefixes: [] };
}
if (i.cases || i.prefixShas || scope?.purpose === 'bisect') {
const cases = list(i.cases);
const prefixes = list(i.prefixShas);
if (scope?.purpose !== 'bisect') return { error: `a bisect needs a bisect nonce (got ${JSON.stringify(i.nonce)})` };
if (!i.refName.startsWith('train/')) return { error: `a bisect runs on train/<nonce>, not ${i.refName}` };
if (i.expectedSha !== i.head) return { error: `expected_sha ${i.expectedSha.slice(0, 12) || '(empty)'} is not the train head ${i.head.slice(0, 12)} checked out from ${i.refName}` };
if (!cases.length || cases.length > MAX_BISECT_CASES || new Set(cases).size !== cases.length) return { error: `cases must name 1 to ${MAX_BISECT_CASES} distinct case ids` };
if (!prefixes.length || prefixes.length > MAX_PREFIXES || new Set(prefixes).size !== prefixes.length) return { error: `prefix_shas must name 1 to ${MAX_PREFIXES} distinct commits` };
const bad = prefixes.find(sha => !SHA.test(sha));
if (bad) return { error: `prefix ${JSON.stringify(bad)} is not a full commit SHA` };
const outside = prefixes.find(sha => !deps.isAncestor(sha, i.head));
if (outside) return { error: `prefix ${outside.slice(0, 12)} is not an ancestor of the train head ${i.head.slice(0, 12)} on ${i.refName}` };
const plan = deps.gateCases();
const unknown = cases.filter(id => !CASE_ID.test(id) || !plan.has(id));
if (unknown.length) return { error: `case(s) not in the full gate's plan: ${unknown.slice(0, 5).join(', ')}` };
return trainOnly('bisect') ?? { mode: 'bisect', cases, prefixes };
}
if (scope?.purpose === 'train') {
if (!i.refName.startsWith('train/')) return { error: `a train gate runs on train/<nonce>, not ${i.refName}` };
if (!i.evalsAll) return { error: 'a train gate is the full gate: dispatch it with evals_all=true' };
if (i.expectedSha !== i.head) return { error: `expected_sha ${i.expectedSha.slice(0, 12) || '(empty)'} is not the train head ${i.head.slice(0, 12)}` };
return trainOnly('train') ?? { mode: 'train', cases: [], prefixes: [] };
}
return { mode: 'dispatch', cases: [], prefixes: [] };
}
/** Every case a plan manifest plans, with its kind: the census a complete gate must execute. */
export function plannedCases(manifest: PaidRunManifest, rootDir = ROOT): Map<string, string> {
const cases = new Map<string, string>();
const e2e = manifest.selection?.e2e ?? null;
for (const entry of manifest.entries.filter(item => item.status === 'planned')) {
const file = shardFile(entry.file);
if (entry.trial) { cases.set(shardCaseId(entry.file)!, entry.trial.kind); continue; }
if (file === JUDGE_FILE) { for (const id of manifest.selection?.judges ?? Object.keys(LLM_JUDGE_TOUCHFILES)) cases.set(id, 'judge'); continue; }
const only = shardCaseId(entry.file);
const source = fs.readFileSync(path.join(rootDir, file), 'utf8');
for (const id of fileCaseRegistration(file, source).registered) {
if (E2E_TIERS[id] === manifest.tier && (only === null || id === only) && (e2e === null || e2e.includes(id)) && !(entry.excludeCases ?? []).includes(id)) cases.set(id, 'rule');
}
}
return cases;
}
/** The full gate's planned cases on this checkout (what a bisect may name). */
export function fullGateCases(rootDir = ROOT): Map<string, string> {
return plannedCases(buildRunManifest({ tier: 'gate', profile: 'full', sliceCount: 1, evalsAll: true, env: { EVALS_ALL: '1' }, rootDir }), rootDir);
}
/**
* A commit's tree modulo stamps: every path with its mode and blob, except the
* files a restamp rewrites, which contribute their content without the stamp
* (the version file nothing, package.json without its version, lockfiles
* without their root version, the CHANGELOG with its headings normalized,
* release outputs and stamp paths with their own version replaced). Two
* commits that differ only by a restamp have the same digest.
*/
export function stampFreeTree(git: Git, commit: string, repoRoot: string, policy: ShipPolicy): string | { error: string } {
const listed = git(['ls-tree', '-r', '-z', '--full-tree', commit]);
if (!listed.ok) return { error: `cannot list ${commit.slice(0, 12)} (${listed.out.split('\n')[0]})` };
const at = versionAtRef(repoRoot, commit);
const versionRel = 'rel' in at ? at.rel : 'VERSION';
const version = 'version' in at ? at.version : null;
const lines: string[] = [];
for (const item of listed.out.split('\0').filter(Boolean)) {
const [meta, rel] = [item.slice(0, item.indexOf('\t')), item.slice(item.indexOf('\t') + 1)];
const [mode, , blob] = meta.split(' ') as [string, string, string];
const base = path.basename(rel);
const stamp = rel === versionRel ? 'version' : base === 'package.json' ? 'manifest' : ['bun.lock', 'package-lock.json', 'npm-shrinkwrap.json'].includes(base) ? 'lockfile'
: rel === policy.changelog ? 'changelog' : matchesAny([...policy.stamp_paths, ...policy.release_outputs, ...policy.stamp_files], rel) ? 'release' : null;
if (!stamp) { lines.push(`${rel}\0${mode}\0${blob}`); continue; }
if (stamp === 'version') { lines.push(`${rel}\0${mode}\0(version)`); continue; }
const shown = git(['cat-file', 'blob', blob]);
if (!shown.ok) return { error: `cannot read ${rel} at ${commit.slice(0, 12)}` };
let text = shown.out;
if (stamp === 'manifest') { try { const parsed = JSON.parse(text); delete parsed.version; text = JSON.stringify(parsed); } catch { /* not JSON: kept verbatim */ } }
else if (stamp === 'lockfile') text = normalizeLockfile(base, text);
else if (stamp === 'changelog') text = normalizeChangelog(text);
else if (version) text = rewriteStamps(text, version, '\0VERSION', policy.stamp_patterns).text;
lines.push(`${rel}\0${mode}\0${createHash('sha256').update(text).digest('hex')}`);
}
return createHash('sha256').update(lines.sort().join('\n')).digest('hex');
}
export interface TrainPrefixTree { index: number; commit: string; tree: string; stamp_free_tree: string }
/** T1..Tk from the train head, walking first parents over T1's prefix commits. */
export function trainPrefixes(git: Git, head: string, repoRoot: string, policy: ShipPolicy): TrainPrefixTree[] | { error: string } {
const log = git(['log', '--first-parent', `--max-count=${MAX_PREFIXES}`, '--format=%H%x00%P%x00%s', head]);
if (!log.ok) return { error: `cannot walk ${head.slice(0, 12)}'s first parents (${log.out.split('\n')[0]})` };
const found: TrainPrefixTree[] = [];
for (const line of log.out.split('\n')) {
const [commit, parents, subject] = line.split('\0') as [string, string, string];
const m = PREFIX_SUBJECT.exec(subject ?? '');
if (!m || parents.split(' ').length !== 2) break;
const index = Number(m[1]);
if (found.length && index !== found.at(-1)!.index - 1) return { error: `prefix T${index} follows T${found.at(-1)!.index} at ${commit.slice(0, 12)}` };
const digest = stampFreeTree(git, commit, repoRoot, policy);
if (typeof digest !== 'string') return digest;
found.push({ index, commit, tree: git(['rev-parse', `${commit}^{tree}`]).out, stamp_free_tree: digest });
}
if (!found.length || found.at(-1)!.index !== 1) return { error: `${head.slice(0, 12)} is not a train head (no T1..Tk prefix chain)` };
return found.reverse();
}
interface OutcomeLine { case?: string; kind?: string; attempt?: number; outcome?: string; failure_class?: string; cost_usd?: number; cost_known?: boolean; execution?: string }
interface CollectorPanel { case: string; status: string; redClass: string | null }
function readJson<T>(file: string): T | null {
try { return JSON.parse(fs.readFileSync(file, 'utf8')) as T; } catch { return null; }
}
/** One panel per planned case from the run's trial outcomes (trial panels from the collector's panelVerdict). */
function casePanels(planned: Map<string, string>, outcomes: OutcomeLine[], collector: CollectorPanel[]): CasePanel[] {
return [...planned].sort(([a], [b]) => (a < b ? -1 : 1)).map(([id, kind]) => {
const mine = outcomes.filter(o => o.case === id && o.execution !== 'reused');
const cost = mine.length && mine.every(o => o.cost_known !== false && typeof o.cost_usd === 'number')
? Math.round(mine.reduce((sum, o) => sum + o.cost_usd!, 0) * 10_000) / 10_000 : null;
const panel = collector.find(p => p.case === id);
if (panel) return { case: id, kind, status: panel.status, red_class: panel.redClass, cost_usd: cost };
const failed = mine.filter(o => o.outcome === 'failed');
if (failed.length) return { case: id, kind, status: 'FAIL', red_class: failed.every(o => o.failure_class === 'infra') ? 'INFRA' : 'VERDICT', cost_usd: cost };
if (mine.some(o => o.outcome === 'passed')) return { case: id, kind, status: 'PASS', red_class: null, cost_usd: cost };
return { case: id, kind, status: mine.length ? 'SKIPPED' : 'MISSING', red_class: 'INCOMPLETE', cost_usd: cost };
});
}
export type TrainRecord = TrainGateRecord & { run: { id: number; attempt: number; url: string }; prefixes: TrainPrefixTree[] };
/** The green-train record a train's own run publishes (T1's TrainGateRecord plus the run and the prefix trees). */
export function buildTrainRecord(o: { reportDir: string; nonce: string; head: string; tree: string; reconcileExit: number; rootDir?: string;
prefixes: TrainPrefixTree[]; run: TrainRecord['run'] }): TrainRecord {
const manifest = parseRunManifest(fs.readFileSync(path.join(o.reportDir, 'manifest.json'), 'utf8'));
const planned = plannedCases(manifest, o.rootDir);
let lines: OutcomeLine[] = [];
try { lines = fs.readFileSync(path.join(o.reportDir, 'trial-outcomes.jsonl'), 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line) as OutcomeLine); } catch { lines = []; }
const primary = Math.min(...lines.map(l => l.attempt ?? 1), Infinity);
const collector = readJson<{ verdict?: { verdict?: string }; census?: { unattributed?: unknown[] }; panels?: CollectorPanel[] }>(path.join(o.reportDir, 'collector-outcomes.json'));
const panels = casePanels(planned, lines.filter(l => (l.attempt ?? 1) === primary), collector?.panels ?? []);
const artifacts = loadSliceArtifacts(o.reportDir);
const first = Math.min(...artifacts.map(a => a.result.attempt ?? 1), Infinity);
const results = artifacts.filter(a => (a.result.attempt ?? 1) === first).map(a => a.result);
const reportedSlices = new Set(results.map(r => r.sliceIndex));
const reportedFiles = new Set(results.flatMap(r => r.outcomes.flatMap(outcome => outcome.files)));
const count = (judge: boolean, from: Iterable<[string, string]>) => [...from].filter(([, kind]) => (kind === 'judge') === judge).length;
const executed = panels.filter(p => p.status === 'PASS' || p.status === 'FAIL').map(p => [p.case, p.kind] as [string, string]);
const census = {
complete: false,
planned: { rule: count(false, planned), judge: count(true, planned) },
executed: { rule: count(false, executed), judge: count(true, executed) },
unbound: collector?.census?.unattributed?.length ?? 0,
partial_shards: results.filter(r => r.checkpoint).length,
cancelled_shards: manifest.entries.filter(e => e.status === 'planned' && reportedSlices.has(e.slice) && !reportedFiles.has(e.file)).length,
killed_before_upload: Array.from({ length: manifest.sliceCount }, (_, i) => i + 1).filter(index => !reportedSlices.has(index)).length,
};
census.complete = o.reconcileExit === 0 && !!collector && planned.size > 0 && census.executed.rule === census.planned.rule
&& census.executed.judge === census.planned.judge && !census.unbound && !census.partial_shards && !census.cancelled_shards
&& !census.killed_before_upload && panels.every(p => p.status === 'PASS');
const verdict = census.complete && collector?.verdict?.verdict === 'GREEN' ? 'GREEN' : 'RED';
return { schema: 1, kind: 'train-gate', nonce: o.nonce, head_sha: o.head, tree: o.tree, verdict, census, panels, run: o.run, prefixes: o.prefixes };
}
export interface BisectPlan { schema: 1; nonce: string; cases: Array<{ id: string; kind: string }>; prefixes: string[] }
/** One executor per prefix; each case runs one trial, so a slice's ceiling grows with the case count. */
export function bisectMatrix(plan: BisectPlan): { slices: number[]; timeouts: number[]; refs: string[] } {
const minutes = Math.min(350, 20 + 30 * plan.cases.length);
return { slices: plan.prefixes.map((_, i) => i + 1), timeouts: [0, ...plan.prefixes.map(() => minutes)], refs: ['', ...plan.prefixes] };
}
export type CaseRunner = (id: string, kind: string, evalDir: string) => Promise<{ status: string; redClass: string | null }>;
/** Run every red case once on the checked-out prefix; a case that leaves no verdict is INCOMPLETE (T1 relaunches it). */
export async function runBisectPrefix(plan: BisectPlan, index: number, o: { rootDir: string; evalDir: string; checkedOut: string; runCase?: CaseRunner }): Promise<BisectPrefixRecord> {
const sha = plan.prefixes[index - 1];
if (!sha) throw new Error(`bisect slice ${index} names no prefix (${plan.prefixes.length} planned)`);
if (o.checkedOut !== sha) throw new Error(`bisect slice ${index} checked out ${o.checkedOut.slice(0, 12)}, not its prefix ${sha.slice(0, 12)}`);
const runCase = o.runCase ?? ((id, kind, evalDir) => runOneTrial(id, kind, evalDir, o.rootDir));
const panels: CasePanel[] = [];
for (const c of plan.cases) {
let verdict: { status: string; redClass: string | null };
try { verdict = await runCase(c.id, c.kind, path.join(o.evalDir, c.id.replace(/[^A-Za-z0-9._-]/g, '_'))); }
catch (error) { console.error(`[train-gate] ${c.id}: ${error instanceof Error ? error.message : String(error)}`); verdict = { status: 'INCOMPLETE', redClass: 'INCOMPLETE' }; }
panels.push({ case: c.id, kind: c.kind, status: verdict.status, red_class: verdict.redClass, cost_usd: null });
console.log(`[train-gate] prefix ${sha.slice(0, 12)} (slice ${index}): ${c.id} ${verdict.status}${verdict.redClass ? ` (${verdict.redClass})` : ''}`);
}
return { schema: 1, kind: 'train-bisect-prefix', nonce: plan.nonce, prefix_sha: sha, checked_out_sha: o.checkedOut, panels };
}
/** One trial of one case exactly as CI runs it: E2E cases through the panel runner, a judge through its file with only it selected. */
async function runOneTrial(id: string, kind: string, evalDir: string, rootDir: string): Promise<{ status: string; redClass: string | null }> {
const { runPaidShards } = await import('./test-paid-shards');
if (kind !== 'judge') {
const { runCaseDiagnosis } = await import('./lib/paid-report');
const v = await runCaseDiagnosis(id, { trials: 1, jobs: 1, rootDir, evalDirBase: evalDir, runShards: runPaidShards });
return { status: v.status, redClass: v.redClass };
}
const env = { ...process.env, EVALS: '1', EVALS_TIER: 'gate', EVALS_PREFLIGHT_OK: '1', EVALS_ALL: '1', ...paidSelectionEnv('full', { e2e: [], judges: [id] }, 'train bisect') };
const [outcome] = (await runPaidShards([[JUDGE_FILE]], { jobs: 1, rootDir, env, evalDirBase: evalDir })).outcomes;
const trials = outcome && outcome.runnerError === undefined && ['passed', 'failed', 'timed-out'].includes(outcome.status)
? [{ trial: 1, outcome: outcome.status === 'passed' ? 'passed' as const : 'failed' as const,
...(outcome.status === 'passed' ? {} : { failure_class: outcome.status === 'timed-out' ? 'timeout' as const : 'assertion' as const }) }] : [];
const v = panelVerdict({ case: id, kind: 'judge', panel: { n: 1, k: 1 }, trials, quarantined: caseTrialPlan(id).quarantined });
return { status: v.status, redClass: v.redClass };
}
/** The free summary of a PR push under a train policy: how to ask for the screen, then its selection and estimated cost. */
export function screenSummary(manifest: PaidRunManifest, pr: string, rootDir = ROOT): string[] {
const planned = plannedCases(manifest, rootDir);
const recorded = loadPaidTestDurations(rootDir, manifest.tier);
const serial = manifest.entries.filter(e => e.status === 'planned').reduce((sum, e) => sum + (recordedShardMs(recorded, e.file) ?? UNRECORDED_SHARD_MS), 0);
const full = plannedSerialMs('gate', rootDir);
const usd = full > 0 ? Math.min(FULL_GATE_USD, FULL_GATE_USD * serial / full) : null;
const judges = [...planned.values()].filter(kind => kind === 'judge').length;
return [
`To run the paid screen of this head: \`gstack-ship-queue ready --pr ${pr} --screen\` (add \`--broad\` for today's full PR selection). PR pushes start no paid slices under this repository's train policy; the train's full gate covers every merge.`,
`- Screening selection: ${planned.size - judges} case(s) + ${judges} judge(s), PR coverage mode \`${manifest.prCoverage?.mode ?? 'broad'}\``,
`- Estimated screen cost: ${usd === null ? 'unknown' : `~$${usd.toFixed(2)}`} (recorded shard durations scaled to the $${FULL_GATE_USD} full gate; shards without a recorded duration count ${UNRECORDED_SHARD_MS / 60_000} min). No paid case ran on this push.`,
];
}
const USAGE = `usage:
bun run scripts/train-gate.ts route --github-output <file> (inputs from the route job's env)
bun run scripts/train-gate.ts bisect-plan --cases <ids> --prefixes <shas> --nonce <nonce> --out <bisect.json> [--github-output <file>]
bun run scripts/train-gate.ts bisect-prefix --plan <bisect.json> --index <n> --out <dir>
bun run scripts/train-gate.ts train-record <report dir> --nonce <nonce> --head <sha> --reconcile-exit <n> --run-id <id> --run-attempt <n> --repo <owner/name> --out <file>
bun run scripts/train-gate.ts screen-summary <manifest.json> --pr <n>`;
if (import.meta.main) {
const [command, ...rest] = process.argv.slice(2);
const flag = (name: string) => { const i = rest.indexOf(`--${name}`); return i >= 0 ? rest[i + 1] : undefined; };
const fail = (message: string): never => { console.error(`::error::${message}`); process.exit(1); };
const usage = (): never => { console.error(USAGE); process.exit(2); };
const output = (lines: string[]) => { const file = flag('github-output'); if (file) fs.appendFileSync(file, `${lines.join('\n')}\n`); };
const git = gitIn(process.cwd());
if (command === 'route') {
const env = process.env;
const route = routeDispatch({ nonce: env.NONCE ?? '', refName: env.GITHUB_REF_NAME ?? '', head: git(['rev-parse', 'HEAD']).out, expectedSha: env.EXPECTED_SHA ?? '',
evalsAll: env.EVALS_ALL === 'true', trainPolicy: env.TRAIN_POLICY === 'true', cases: env.CASES ?? '', prefixShas: env.PREFIX_SHAS ?? '',
screenPr: env.SCREEN_PR ?? '', screenMergeSha: env.SCREEN_MERGE_SHA ?? '', screenPolicy: env.SCREEN_POLICY ?? '' },
{ isAncestor: (commit, head) => git(['merge-base', '--is-ancestor', commit, head]).ok, gateCases: () => fullGateCases(process.cwd()) });
if ('error' in route) fail(`dispatch refused before any paid work: ${route.error}`);
const ok = route as Route;
output([`mode=${ok.mode}`, `cases=${ok.cases.join(',')}`, `prefixes=${ok.prefixes.join(',')}`]);
console.log(`- Run mode: ${ok.mode}${ok.prefixes.length ? ` (${ok.cases.length} case(s) on ${ok.prefixes.length} prefix(es))` : ''}`);
} else if (command === 'bisect-plan') {
const [cases, prefixes, nonce, out] = [list(flag('cases') ?? ''), list(flag('prefixes') ?? ''), flag('nonce'), flag('out')];
if (!cases.length || !prefixes.length || !nonce || !out) usage();
const plan = fullGateCases(process.cwd());
const bisect: BisectPlan = { schema: 1, nonce: nonce!, cases: cases.map(id => ({ id, kind: plan.get(id) ?? fail(`case ${id} is not in the full gate's plan`) })), prefixes };
fs.mkdirSync(path.dirname(out!), { recursive: true });
fs.writeFileSync(out!, `${JSON.stringify(bisect, null, 2)}\n`);
const m = bisectMatrix(bisect);
output([`slices=${JSON.stringify(m.slices)}`, `slice_timeouts=${JSON.stringify(m.timeouts)}`, `prefix_refs=${JSON.stringify(m.refs)}`]);
console.log(`- Bisect: ${cases.length} case(s) x ${prefixes.length} prefix(es), one trial each`);
} else if (command === 'bisect-prefix') {
const [planPath, index, out] = [flag('plan'), Number(flag('index')), flag('out')];
if (!planPath || !Number.isSafeInteger(index) || index < 1 || !out) usage();
const plan = readJson<BisectPlan>(planPath!) ?? fail(`unreadable bisect plan ${planPath}`);
const record = await runBisectPrefix(plan, index, { rootDir: process.cwd(), evalDir: path.join(out!, 'eval'), checkedOut: git(['rev-parse', 'HEAD']).out });
fs.mkdirSync(out!, { recursive: true });
fs.writeFileSync(path.join(out!, 'bisect-prefix.json'), `${JSON.stringify(record, null, 2)}\n`);
} else if (command === 'train-record') {
const [reportDir, nonce, head, repo, out] = [rest[0], flag('nonce'), flag('head'), flag('repo'), flag('out')];
if (!reportDir || !nonce || !head || !SHA.test(head) || !repo || !out) usage();
const prefixes = trainPrefixes(git, head!, process.cwd(), loadPolicyFromHead(process.cwd()).policy);
if ('error' in prefixes) fail(`no train record: ${prefixes.error}`);
const runId = Number(flag('run-id'));
const record = buildTrainRecord({ reportDir: reportDir!, nonce: nonce!, head: head!, tree: git(['rev-parse', `${head}^{tree}`]).out,
reconcileExit: /^\d+$/.test(flag('reconcile-exit') ?? '') ? Number(flag('reconcile-exit')) : -1, prefixes: prefixes as TrainPrefixTree[],
run: { id: runId, attempt: Number(flag('run-attempt')), url: `https://github.com/${repo}/actions/runs/${runId}` } });
fs.mkdirSync(path.dirname(out!), { recursive: true });
fs.writeFileSync(out!, `${JSON.stringify(record, null, 2)}\n`);
const c = record.census;
console.log(`- Train record: ${record.verdict} (rule ${c.executed.rule}/${c.planned.rule}, judge ${c.executed.judge}/${c.planned.judge}, unbound ${c.unbound}, partial ${c.partial_shards}, cancelled ${c.cancelled_shards}, killed before upload ${c.killed_before_upload}); prefixes T1..T${record.prefixes.length}`);
} else if (command === 'screen-summary') {
const [manifestPath, pr] = [rest[0], flag('pr')];
if (!manifestPath || !pr) usage();
for (const line of screenSummary(parseRunManifest(fs.readFileSync(manifestPath!, 'utf8')), pr!)) console.log(line);
} else usage();
}
+13 -6
View File
@@ -75,9 +75,9 @@ describe('paid CI coordination stays off the eval image', () => {
test(`${name}: planning is independent of image startup and has no dependency install`, () => {
const planner = jobs['plan-slices'];
if (name === 'evals.yml') {
// The PR lane plans after its two tiny coordination jobs (readiness, base-ref receipt recovery), never after the image.
expect(planner.needs).toEqual(['readiness', 'recover-receipts']);
for (const id of ['readiness', 'recover-receipts']) {
// The PR lane plans after its three tiny coordination jobs (route, readiness, base-ref receipt recovery), never after the image.
expect(planner.needs).toEqual(['readiness', 'recover-receipts', 'route']);
for (const id of ['readiness', 'recover-receipts', 'route']) {
const job = jobs[id] as Job & { 'runs-on'?: string };
expect(job.container, id).toBeUndefined();
expect(job['runs-on'], id).toBe('ubuntu-24.04');
@@ -145,7 +145,8 @@ describe('paid CI coordination stays off the eval image', () => {
const captures = hiddenUploads.filter(step => step.with?.name === 'native-captures-${{ env.EVALS_RUN_ID }}');
// evals-periodic's host-run Codex job mounts $RUNNER_TEMP/eval-home as the container's HOME.
const hostHome = (pattern: string) => pattern.replace('${{ runner.temp }}/eval-home/', '~/');
expect(captures).toHaveLength(name === 'evals.yml' ? 1 : name === 'evals-periodic.yml' ? 3 : 2);
// evals.yml: eval-slices and train-bisect (GSTA-49 T1b), each under its own EVALS_RUN_ID.
expect(captures).toHaveLength(name === 'evals.yml' ? 2 : name === 'evals-periodic.yml' ? 3 : 2);
for (const capture of captures) {
expect(capture.if).toBe('always()');
expect(String(capture.with?.path).trim().split('\n').map(hostHome)).toEqual([
@@ -160,7 +161,11 @@ describe('paid CI coordination stays off the eval image', () => {
// The container sets TMPDIR under HOME, so shard logs land in the mounted HOME's cache.
expect(String(codexLogs.with?.path).trim().split('\n').map(hostHome)).toEqual(['~/.cache/gstack-paid-shard-*.log']);
}
expect(hiddenUploads.filter(step => !captures.includes(step))).toEqual(codexLogs ? [logs!, codexLogs] : [logs!]);
// GSTA-49 T1b: train-bisect keeps the same shard-log upload under its own name.
const bisectLogs = name === 'evals.yml'
? jobs['train-bisect'].steps.find(step => step.with?.name === 'train-bisect-logs-${{ matrix.slice }}-a${{ github.run_attempt }}') : undefined;
if (name === 'evals.yml') expect(String(bisectLogs?.with?.path)).toBe(String(logs?.with?.path));
expect(hiddenUploads.filter(step => !captures.includes(step))).toEqual([logs!, ...(codexLogs ? [codexLogs] : []), ...(bisectLogs ? [bisectLogs] : [])]);
});
}
@@ -169,7 +174,9 @@ describe('paid CI coordination stays off the eval image', () => {
const trust = "github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)";
// !cancelled() lets planning proceed when readiness was skipped (dispatch) or recovery was skipped or failed (reuse is then off);
// a readiness failure (Free Tests red on this head) or a superseded head plans nothing; the trust clause still gates it.
expect(jobs['plan-slices'].if).toBe(`\${{ !cancelled() && (needs.readiness.result == 'success' || needs.readiness.result == 'skipped') && needs.readiness.outputs.superseded != 'true' && ${trust} }}`);
// GSTA-49 T1b: the route job must have succeeded and named a planned mode (pr, screen, train, dispatch); a superseded screen plans nothing.
const route = "needs.route.result == 'success' && needs.route.outputs.superseded != 'true' && contains(fromJSON('[\"pr\",\"screen\",\"train\",\"dispatch\"]'), needs.route.outputs.mode)";
expect(jobs['plan-slices'].if).toBe(`\${{ !cancelled() && (needs.readiness.result == 'success' || needs.readiness.result == 'skipped') && needs.readiness.outputs.superseded != 'true' && ${route} && ${trust} }}`);
// The coordination jobs never run for Dependabot or for a fork PR either (recovery also serves an explicit pr_receipts dispatch).
expect(jobs.readiness.if).toContain("github.actor != 'dependabot[bot]'");
expect(jobs.readiness.if).toContain("github.event_name == 'pull_request'");
+39 -1
View File
@@ -3,7 +3,7 @@
* and a fake clock: no network, no sleeping.
*/
import { describe, expect, test } from 'bun:test';
import { freeTestsState, readinessDecision, waitForReadiness, type FreeTestsRun } from '../scripts/eval-readiness';
import { freeTestsState, readinessDecision, screenTarget, waitForReadiness, type FreeTestsRun } from '../scripts/eval-readiness';
import type { GhRunner } from '../scripts/recover-receipts';
const HEAD = 'a'.repeat(40);
@@ -127,3 +127,41 @@ describe('waitForReadiness', () => {
expect(c.sleeps[0]).toBe(10 * 60_000);
});
});
describe('dispatched screens (GSTA-49 T1b): the merge commit is checked against the PR before Free Tests is read on its head', () => {
const MERGE = 'c'.repeat(40);
const BASE = 'd'.repeat(40);
const pull = (over: Record<string, unknown> = {}) => ({ state: 'open', head: { sha: HEAD, ref: 'feature', repo: { full_name: 'garrytan/gstack' } }, base: { repo: { full_name: 'garrytan/gstack' } }, ...over });
const gh = (pr: unknown, parents: string[] | null): GhRunner => (args) => {
const target = args.find(a => a.startsWith('repos/')) ?? '';
if (target.endsWith('/pulls/42')) return pr ? { status: 0, stdout: JSON.stringify(pr), stderr: '' } : { status: 1, stdout: '', stderr: 'HTTP 404: Not Found' };
if (target.endsWith(`/commits/${MERGE}`)) return parents ? { status: 0, stdout: JSON.stringify({ parents: parents.map(sha => ({ sha })) }), stderr: '' } : { status: 1, stdout: '', stderr: 'HTTP 422' };
return { status: 1, stdout: '', stderr: 'unexpected' };
};
test("an open same-repository PR's merge commit names its head (second parent) and tested base (first parent)", () => {
expect(screenTarget(gh(pull(), [BASE, HEAD]), 'garrytan/gstack', 42, MERGE)).toEqual({ head: HEAD, base: BASE, branch: 'feature' });
});
test('a fork PR, a closed PR, a non-merge commit, a short SHA or an unreadable PR is refused', () => {
const refused: Array<[GhRunner, string, string]> = [
[gh(pull({ head: { sha: HEAD, ref: 'feature', repo: { full_name: 'fork/gstack' } } }), [BASE, HEAD]), MERGE, 'not a same-repository PR'],
[gh(pull({ head: { sha: HEAD, ref: 'feature', repo: null } }), [BASE, HEAD]), MERGE, 'not a same-repository PR'],
[gh(pull({ state: 'closed' }), [BASE, HEAD]), MERGE, 'is closed, not open'],
[gh(pull(), [BASE]), MERGE, 'is not a two-parent merge commit'],
[gh(pull(), null), MERGE, 'cannot read the merge commit'],
[gh(null, [BASE, HEAD]), MERGE, 'cannot read PR #42'],
[gh(pull(), [BASE, HEAD]), MERGE.slice(0, 12), 'is not a full commit SHA'],
];
for (const [runner, merge, why] of refused) expect((screenTarget(runner, 'garrytan/gstack', 42, merge) as { error?: string }).error, why).toContain(why);
});
test('the screen then waits on Free Tests for the PR head with no quiet window, and a moved head is superseded', () => {
const { gh: ready } = fakeGh([{ free: [free(5, 'completed', 'success')] }]);
const c = clock(CREATED);
expect(waitForReadiness({ ...base, quietMs: 0, gh: ready, now: c.now, sleep: c.sleep }).decision.kind).toBe('ready');
expect(c.sleeps).toEqual([]);
const { gh: moved } = fakeGh([{ free: [free(5, 'completed', 'success')], prHead: NEWER }]);
expect(waitForReadiness({ ...base, quietMs: 0, gh: moved, now: c.now, sleep: c.sleep }).decision).toEqual({ kind: 'superseded', by: NEWER });
});
});
+20 -12
View File
@@ -44,7 +44,7 @@ describe('evals.yml receipt recovery (CEO-14/34, ENG-7)', () => {
expect(recover.permissions).toEqual({ contents: 'read', actions: 'read' });
expect(planner.permissions).toEqual({ contents: 'read' });
const checkout = recover.steps.find(s => s.uses?.startsWith('actions/checkout@'))!;
expect(checkout.with).toMatchObject({ ref: '${{ github.event.pull_request.base.sha || inputs.base_sha || github.event.repository.default_branch }}', 'persist-credentials': false });
expect(checkout.with).toMatchObject({ ref: '${{ github.event.pull_request.base.sha || needs.route.outputs.screen_base || inputs.base_sha || github.event.repository.default_branch }}', 'persist-credentials': false });
const collect = step('recover-receipts', 'Recover receipts of cancelled runs (base-ref code)')!;
expect(collect.env?.GH_TOKEN).toBe('${{ github.token }}');
expect(collect.run).toContain('scripts/recover-receipts.ts collect');
@@ -105,7 +105,8 @@ describe('evals.yml readiness-gated dispatch (R1, GSTA-48)', () => {
});
test('cancel-in-progress stays keyed on the PR, and evals-fresh turns receipt reuse off', () => {
expect(source).toContain('group: evals-${{ github.event.pull_request.number || github.run_id }}');
// A dispatched screen (GSTA-49 T1b) is keyed on its PR too: a newer head's screen cancels the older one.
expect(source).toContain("group: evals-${{ github.event.pull_request.number || (inputs.screen_pr != '' && format('screen-{0}', inputs.screen_pr)) || github.run_id }}");
expect(source).toContain('cancel-in-progress: true');
expect(workflow.env.EVALS_FRESH).toContain("contains(github.event.pull_request.labels.*.name, 'evals-fresh')");
});
@@ -114,12 +115,14 @@ describe('evals.yml readiness-gated dispatch (R1, GSTA-48)', () => {
describe('evals.yml gate verdict reuse wiring (R2a restamp-only, R2 main moved; GSTA-48)', () => {
test('base-ref recovery proposes a candidate; it never fails the run', () => {
const find = step('recover-receipts', 'Find a gate verdict to reuse (R2a/R2, base-ref code)')!;
expect(find.if).toBe("github.event_name == 'pull_request'");
expect(find.env).toMatchObject({ GH_TOKEN: '${{ github.token }}', HEAD_SHA: '${{ github.event.pull_request.head.sha }}' });
// GSTA-49 T1b: a dispatched screen finds too, with its route-checked head and its pinned merge commit as the tested sha.
expect(find.if).toBe("github.event_name == 'pull_request' || needs.route.outputs.mode == 'screen'");
expect(find.env).toMatchObject({ GH_TOKEN: '${{ github.token }}', HEAD_SHA: '${{ github.event.pull_request.head.sha || needs.route.outputs.screen_head }}',
TESTED_SHA: '${{ inputs.screen_merge_sha || github.sha }}' });
expect(find.env?.BASE_SHA).toBeUndefined();
expect(find.run).toContain('scripts/gate-reuse.ts find');
// The base is the tested merge commit's first parent, never the event's base.sha.
expect(find.run).toContain('--tested-sha "$GITHUB_SHA"');
expect(find.run).toContain('--tested-sha "$TESTED_SHA"');
expect(find.run).not.toContain('--base-sha');
expect(find.run).toContain('exit 0');
expect(find.run).toContain('|| echo');
@@ -144,9 +147,11 @@ describe('evals.yml gate verdict reuse wiring (R2a restamp-only, R2 main moved;
expect(report.outputs?.['reconcile-exit']).toBe('${{ steps.reconcile.outputs.exit || steps.reused.outputs.exit }}');
expect(step('slices-report', 'Fail the workflow when reconciliation failed')!.if).toBe("(steps.reconcile.outputs.exit || steps.reused.outputs.exit) != '0'");
const bind = step('slices-report', "Bind this run's verdict for reuse (R2a/R2)")!;
expect(bind.if).toBe("always() && github.event_name == 'pull_request' && needs.plan-slices.outputs.gate_reuse != 'true'");
// GSTA-49 T1b: a dispatched screen binds too; train, bisect and plain dispatches never bind.
expect(bind.if).toBe("always() && (github.event_name == 'pull_request' || needs.plan-slices.outputs.mode == 'screen') && needs.plan-slices.outputs.gate_reuse != 'true'");
expect(bind.env?.TESTED_SHA).toBe('${{ inputs.screen_merge_sha || github.sha }}');
expect(bind.env?.RECONCILE_EXIT).toBe('${{ steps.reconcile.outputs.exit }}');
expect(bind.run).toContain('--tested-sha "$GITHUB_SHA"');
expect(bind.run).toContain('--tested-sha "$TESTED_SHA"');
expect(bind.env?.BASE_SHA).toBeUndefined();
expect(step('slices-report', 'Upload the gate binding')?.with).toMatchObject({ name: 'gate-binding-a${{ github.run_attempt }}', path: '/tmp/gate-binding/gate-binding.json' });
});
@@ -163,19 +168,22 @@ describe('evals.yml carries no inline planner program (W8f)', () => {
describe('evals.yml dispatch contract for eval:bg (CEO-17/29, DX-1, ENG-8/12)', () => {
const inputs = workflow.on.workflow_dispatch.inputs as Record<string, { type: string; default: unknown }>;
test('dispatch inputs: evals_all, base_ref, base_sha, expected_sha, nonce, pr_receipts', () => {
expect(Object.keys(inputs).sort()).toEqual(['base_ref', 'base_sha', 'evals_all', 'expected_sha', 'nonce', 'pr_receipts']);
test('dispatch inputs: evals_all, base_ref, base_sha, expected_sha, nonce, pr_receipts, and the train inputs', () => {
// cases, prefix_shas, screen_pr, screen_merge_sha and screen_policy (GSTA-49 T1b) are validated by the route job.
expect(Object.keys(inputs).sort()).toEqual(['base_ref', 'base_sha', 'cases', 'evals_all', 'expected_sha', 'nonce', 'pr_receipts', 'prefix_shas', 'screen_merge_sha', 'screen_policy', 'screen_pr']);
expect(inputs.evals_all).toMatchObject({ type: 'boolean', default: true });
for (const name of ['base_ref', 'base_sha', 'expected_sha', 'nonce', 'pr_receipts']) expect(inputs[name]).toMatchObject({ type: 'string', default: '' });
for (const name of ['base_ref', 'base_sha', 'expected_sha', 'nonce', 'pr_receipts', 'cases', 'prefix_shas', 'screen_pr', 'screen_merge_sha', 'screen_policy']) expect(inputs[name]).toMatchObject({ type: 'string', default: '' });
});
test('the nonce is reflected in the run name so the dispatcher resolves its own run', () => {
expect(workflow['run-name']).toBe("${{ inputs.nonce != '' && format('E2E Evals [eval-bg {0}]', inputs.nonce) || '' }}");
// A dispatched screen's name also ends ' screen #<pr>' (GSTA-49 T1b); eval-bg's runForNonce still matches 'eval-bg <nonce>]'.
expect(workflow['run-name']).toBe("${{ inputs.nonce != '' && format('E2E Evals [eval-bg {0}]{1}', inputs.nonce, inputs.screen_pr != '' && format(' screen #{0}', inputs.screen_pr) || '') || '' }}");
});
test('evals_all=false dispatches run the PR profile on the requested base; evals_all runs the full census', () => {
expect(workflow.env.EVALS_PROFILE).toBe("${{ (github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && !inputs.evals_all)) && 'pr' || 'full' }}");
expect(step('plan-slices', 'Emit run manifest')?.env?.EVALS_BASE).toBe("${{ inputs.base_sha || (inputs.base_ref != '' && format('origin/{0}', inputs.base_ref)) || '' }}");
// A dispatched screen diffs against its merge commit's first parent (route's screen_base; empty for every other run).
expect(step('plan-slices', 'Emit run manifest')?.env?.EVALS_BASE).toBe("${{ needs.route.outputs.screen_base || inputs.base_sha || (inputs.base_ref != '' && format('origin/{0}', inputs.base_ref)) || '' }}");
});
test('a dispatch fails fast with a named fix when the checkout is not the expected revision', () => {
+353
View File
@@ -0,0 +1,353 @@
/**
* evals.yml and evals-main-backstop.yml under merge trains (plan GSTA-49 T1b).
*
* The job graph is simulated with a small evaluator of the GitHub expression
* subset these workflows use (literals, contexts, ==, !=, &&, ||, !, contains,
* fromJSON, format, status functions), so each run mode is checked by which
* jobs and steps actually run and what their expressions evaluate to. The
* inert case: without a train block a PR push and a plain dispatch run today's
* jobs and steps with today's values, plus the free route job.
*/
import { describe, expect, test } from 'bun:test';
import { spawnSync } from 'node:child_process';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { bisectArtifact } from '../lib/ship-queue-train-verdict';
const ROOT = path.resolve(import.meta.dir, '..');
type Step = { id?: string; name?: string; uses?: string; if?: string; run?: string; env?: Record<string, unknown>; with?: Record<string, unknown> };
type Job = { if?: string; needs?: string | string[]; outputs?: Record<string, string>; permissions?: Record<string, string>; steps: Step[]; strategy?: { matrix: Record<string, string> } };
type Workflow = { 'run-name'?: string; on: Record<string, any>; concurrency: { group: string; 'cancel-in-progress': boolean }; permissions?: unknown; jobs: Record<string, Job> };
const load = (rel: string) => Bun.YAML.parse(fs.readFileSync(path.join(ROOT, rel), 'utf8')) as Workflow;
const evals = load('.github/workflows/evals.yml');
const backstopWf = load('.github/workflows/evals-main-backstop.yml');
// ── expression subset ────────────────────────────────────────────────────
const STAR = Symbol('star');
type Star = { [STAR]: unknown[] };
const isStar = (v: unknown): v is Star => typeof v === 'object' && v !== null && STAR in v;
const truthy = (v: unknown) => !(v === false || v === 0 || v === '' || v === null || v === undefined || Number.isNaN(v));
function looseEq(a: unknown, b: unknown): boolean {
if (a === undefined) a = null;
if (b === undefined) b = null;
if (typeof a === 'string' && typeof b === 'string') return a.toLowerCase() === b.toLowerCase();
if ((typeof a === 'object' && a !== null) || (typeof b === 'object' && b !== null)) return a === b;
if (typeof a === typeof b) return a === b;
const num = (v: unknown) => (v === null ? 0 : typeof v === 'boolean' ? Number(v) : typeof v === 'string' ? (v.trim() === '' ? 0 : Number(v)) : Number(v));
return num(a) === num(b);
}
const str = (v: unknown) => (v === null || v === undefined ? '' : typeof v === 'object' ? JSON.stringify(v) : String(v));
function evaluate(expr: string, ctx: Record<string, unknown>, status: { success: boolean; failure: boolean }): unknown {
const src = expr.trim().replace(/^\$\{\{([\s\S]*)\}\}$/, '$1');
const re = /\s*(?:('(?:[^']|'')*')|(\d+(?:\.\d+)?)|(==|!=|&&|\|\||[!()[\],.*])|([A-Za-z_][A-Za-z0-9_-]*))/y;
const tokens: string[] = [];
while (re.lastIndex < src.length) {
const at = re.lastIndex;
const m = re.exec(src);
if (!m) { if (/^\s*$/.test(src.slice(at))) break; throw new Error(`cannot tokenize ${src.slice(at)}`); }
tokens.push(m[1] ?? m[2] ?? m[3] ?? m[4]!);
}
let i = 0;
const take = (want?: string) => { const t = tokens[i++]; if (want !== undefined && t !== want) throw new Error(`expected ${want}, got ${t} in ${src}`); return t!; };
const fns: Record<string, (...a: unknown[]) => unknown> = {
always: () => true, cancelled: () => false, success: () => status.success, failure: () => status.failure,
contains: (hay, needle) => (Array.isArray(hay) || isStar(hay) ? (isStar(hay) ? hay[STAR] : hay as unknown[]).some(v => looseEq(v, needle)) : str(hay).toLowerCase().includes(str(needle).toLowerCase())),
fromJSON: (s) => JSON.parse(str(s)),
format: (f, ...args) => str(f).replace(/\{(\d+)\}/g, (_, n) => str(args[Number(n)])),
startsWith: (a, b) => str(a).toLowerCase().startsWith(str(b).toLowerCase()),
hashFiles: () => 'hash',
};
const member = (v: unknown, key: string): unknown => {
if (isStar(v)) return { [STAR]: v[STAR].map(item => member(item, key)) };
return v !== null && typeof v === 'object' ? (v as Record<string, unknown>)[key] ?? null : null;
};
const primary = (): unknown => {
const t = take();
if (t === '(') { const v = or(); take(')'); return v; }
if (t.startsWith("'")) return t.slice(1, -1).replace(/''/g, "'");
if (/^\d/.test(t)) return Number(t);
if (t === 'true' || t === 'false') return t === 'true';
if (t === 'null') return null;
if (tokens[i] === '(') {
take('(');
const args: unknown[] = [];
while (tokens[i] !== ')') { args.push(or()); if (tokens[i] === ',') take(','); }
take(')');
if (!fns[t]) throw new Error(`unknown function ${t}`);
return fns[t]!(...args);
}
return ctx[t] ?? null;
};
const postfix = (): unknown => {
let v = primary();
for (;;) {
if (tokens[i] === '.') { take('.'); const key = take(); v = key === '*' ? { [STAR]: Array.isArray(v) ? v : v && typeof v === 'object' ? Object.values(v) : [] } : member(v, key); }
else if (tokens[i] === '[') { take('['); const k = or(); take(']'); v = member(v, str(k)); }
else return v;
}
};
const unary = (): unknown => (tokens[i] === '!' ? (take('!'), !truthy(unary())) : postfix());
const eq = (): unknown => {
let l = unary();
while (tokens[i] === '==' || tokens[i] === '!=') { const op = take(); const r = unary(); l = op === '==' ? looseEq(l, r) : !looseEq(l, r); }
return l;
};
const and = (): unknown => { let l = eq(); while (tokens[i] === '&&') { take(); const r = eq(); l = truthy(l) ? r : l; } return l; };
const or = (): unknown => { let l = and(); while (tokens[i] === '||') { take(); const r = and(); l = truthy(l) ? l : r; } return l; };
const value = or();
if (i !== tokens.length) throw new Error(`trailing tokens in ${src}`);
return value;
}
const interpolate = (text: unknown, ctx: Record<string, unknown>) =>
typeof text === 'string' ? text.replace(/\$\{\{([\s\S]*?)\}\}/g, (_, e) => str(evaluate(e, ctx, { success: true, failure: false }))) : text;
// ── job graph ─────────────────────────────────────────────────────────────
const SHA = { main: 'm'.repeat(40), head: 'h'.repeat(40), base: 'b'.repeat(40), merge: 'c'.repeat(40), screenBase: 'e'.repeat(40), screenHead: 'f'.repeat(40) };
interface Scenario { github: Record<string, unknown>; inputs: Record<string, unknown>; outputs?: Record<string, Record<string, string>>; results?: Record<string, string> }
const ROUTE = { mode: '', record_artifact: '', full_gate_check: '', screen_head: '', screen_base: '', screen_branch: '', superseded: '', bisect_slices: '', bisect_timeouts: '', bisect_refs: '' };
function simulate(wf: Workflow, s: Scenario) {
const result: Record<string, string> = {};
const outputs: Record<string, Record<string, unknown>> = {};
const ran: Record<string, { ctx: Record<string, unknown>; steps: Step[] }> = {};
const pending = Object.keys(wf.jobs);
while (pending.length) {
const id = pending.find(j => [wf.jobs[j]!.needs ?? []].flat().every(n => n in result));
if (!id) throw new Error('needs cycle');
pending.splice(pending.indexOf(id), 1);
const job = wf.jobs[id]!;
const needs = [job.needs ?? []].flat();
const ctx = { github: s.github, inputs: s.inputs, vars: {}, secrets: {}, env: {}, steps: {}, matrix: {}, runner: {},
needs: Object.fromEntries(needs.map(n => [n, { result: result[n], outputs: outputs[n] ?? {} }])) };
const allOk = needs.every(n => result[n] === 'success');
const cond = job.if === undefined ? 'success()' : String(job.if);
const value = truthy(evaluate(cond, ctx, { success: allOk, failure: needs.some(n => result[n] === 'failure') }));
const runs = /\b(success|failure|always|cancelled)\(\)/.test(cond) ? value : allOk && value;
result[id] = runs ? s.results?.[id] ?? 'success' : 'skipped';
outputs[id] = runs ? Object.fromEntries(Object.entries(job.outputs ?? {}).map(([k, v]) => [k, s.outputs?.[id]?.[k] ?? interpolate(v, ctx)])) : {};
if (runs) ran[id] = { ctx, steps: job.steps.filter(step => step.if === undefined || truthy(evaluate(String(step.if), ctx, { success: true, failure: false }))) };
}
const step = (job: string, name: string) => {
const found = ran[job]?.steps.find(st => st.name === name);
return found ? { env: Object.fromEntries(Object.entries(found.env ?? {}).map(([k, v]) => [k, interpolate(v, ran[job]!.ctx)])),
with: Object.fromEntries(Object.entries(found.with ?? {}).map(([k, v]) => [k, interpolate(v, ran[job]!.ctx)])), run: found.run } : null;
};
const checkout = (job: string) => {
const found = ran[job]?.steps.find(st => st.uses?.startsWith('actions/checkout@'));
return found ? Object.fromEntries(Object.entries(found.with ?? {}).map(([k, v]) => [k, interpolate(v, ran[job]!.ctx)])) : null;
};
const top = { github: s.github, inputs: s.inputs };
return { jobs: Object.keys(ran).sort(), steps: (job: string) => ran[job]?.steps.map(st => st.name ?? st.uses) ?? [], step, checkout, result,
group: interpolate(wf.concurrency.group, top), runName: interpolate(wf['run-name'] ?? '', top) };
}
const pr = (labels: string[] = []) => ({ event_name: 'pull_request', actor: 'garrytan', repository: 'garrytan/gstack', repository_id: 1, sha: SHA.main, run_id: 7, run_attempt: 1, ref_name: '42/merge',
event: { pull_request: { number: 42, head: { sha: SHA.head, ref: 'feature', repo: { full_name: 'garrytan/gstack' } }, base: { sha: SHA.base }, labels: labels.map(name => ({ name })) },
repository: { default_branch: 'main' } } });
const dispatch = (ref = 'main') => ({ event_name: 'workflow_dispatch', actor: 'garrytan', repository: 'garrytan/gstack', repository_id: 1, sha: SHA.main, run_id: 7, run_attempt: 1, ref_name: ref,
event: { repository: { default_branch: 'main' } } });
const INPUTS = { evals_all: true, base_ref: '', base_sha: '', expected_sha: '', nonce: '', pr_receipts: '', cases: '', prefix_shas: '', screen_pr: '', screen_merge_sha: '', screen_policy: '' };
const TODAY_PR_JOBS = ['build-image', 'eval-slices', 'plan-slices', 'readiness', 'recover-receipts', 'slices-comment', 'slices-report'];
const TODAY_DISPATCH_JOBS = ['build-image', 'eval-slices', 'plan-slices', 'slices-report'];
const TRAIN_STEPS = ['Write the train record', 'Upload the train record', "Check the screen's PR head (Free Tests)", 'Plan the bisect (one executor per prefix)', 'Hand the bisect plan to its executors'];
const train = { record_artifact: 'gstack-train-record', full_gate_check: 'gstack/full-gate' };
describe('without a train block nothing changes (the inert proof)', () => {
test("a PR push runs today's jobs and steps with today's values, plus the free route job", () => {
const sim = simulate(evals, { github: pr(), inputs: {}, outputs: { route: { ...ROUTE, mode: 'pr' } } });
expect(sim.jobs).toEqual([...TODAY_PR_JOBS, 'route'].sort());
for (const job of sim.jobs) for (const name of TRAIN_STEPS) expect(sim.steps(job), job).not.toContain(name);
expect(sim.group).toBe('evals-42');
expect(sim.runName).toBe('');
expect(sim.steps('recover-receipts')).toEqual(expect.arrayContaining(["Restore this PR's verified judge and E2E results", 'Recover receipts of cancelled runs (base-ref code)',
'Find a gate verdict to reuse (R2a/R2, base-ref code)', 'Hand a gate reuse candidate to plan-slices', 'Hand the receipt store to plan-slices']));
expect(sim.checkout('recover-receipts')?.ref).toBe(SHA.base);
expect(sim.step('recover-receipts', 'Find a gate verdict to reuse (R2a/R2, base-ref code)')!.env).toMatchObject({
PR_NUMBER: '42', HEAD_REPO: 'garrytan/gstack', HEAD_BRANCH: 'feature', HEAD_SHA: SHA.head, TESTED_SHA: SHA.main, SCREEN_POLICY: '' });
for (const job of ['plan-slices', 'eval-slices', 'slices-report']) expect(sim.checkout(job)?.ref ?? '', job).toBe('');
expect(sim.checkout('slices-report')?.['fetch-depth']).toBe('1');
expect(sim.step('plan-slices', 'Emit run manifest')!.env.EVALS_BASE).toBe('');
expect(sim.steps('plan-slices')).toEqual(expect.arrayContaining(['Download a gate reuse candidate', 'Confirm gate verdict reuse (R2a/R2)', 'Carry the receipt store to the report']));
expect(sim.step('slices-report', "Bind this run's verdict for reuse (R2a/R2)")!.env).toMatchObject({ PR_NUMBER: '42', HEAD_SHA: SHA.head, TESTED_SHA: SHA.main, SCREEN_POLICY: '' });
expect(sim.step('slices-comment', 'Post PR comment')!.env).toMatchObject({ PR_NUMBER: '42', SCREEN_POLICY: '' });
});
test("a plain dispatch (eval:bg, backstop) runs today's jobs, plus route", () => {
const sim = simulate(evals, { github: dispatch('feature'), inputs: { ...INPUTS, nonce: 'pr-x', evals_all: false, expected_sha: SHA.main }, outputs: { route: { ...ROUTE, mode: 'dispatch' } } });
expect(sim.jobs).toEqual([...TODAY_DISPATCH_JOBS, 'route'].sort());
expect(sim.group).toBe('evals-7');
expect(sim.runName).toBe('E2E Evals [eval-bg pr-x]');
for (const job of ['plan-slices', 'eval-slices', 'slices-report']) expect(sim.checkout(job)?.ref ?? '', job).toBe('');
expect(sim.steps('slices-report')).not.toContain("Bind this run's verdict for reuse (R2a/R2)");
const receipts = simulate(evals, { github: dispatch('feature'), inputs: { ...INPUTS, nonce: 'pr-x', evals_all: false, pr_receipts: '42' }, outputs: { route: { ...ROUTE, mode: 'dispatch' } } });
expect(receipts.jobs).toContain('recover-receipts');
expect(receipts.steps('recover-receipts')).toContain('Recover receipts of cancelled runs (base-ref code)');
});
test('a fork PR and a Dependabot PR still start nothing paid', () => {
const fork = pr();
(fork.event.pull_request.head.repo as { full_name: string }).full_name = 'fork/gstack';
expect(simulate(evals, { github: fork, inputs: {}, outputs: { route: { ...ROUTE, mode: 'pr' } } }).jobs).toEqual(['build-image', 'route']);
expect(simulate(evals, { github: { ...pr(), actor: 'dependabot[bot]' }, inputs: {} }).jobs).toEqual([]);
});
});
describe('under a train block (the gate edits, each only on its mode)', () => {
test('a PR push starts no paid slice: route, readiness and the free screening summary only', () => {
const sim = simulate(evals, { github: pr(), inputs: {}, outputs: { route: { ...ROUTE, ...train, mode: 'pr-summary' } } });
expect(sim.jobs).toEqual(['build-image', 'readiness', 'route', 'screen-summary']);
const summary = evals.jobs['screen-summary']!;
expect(summary.permissions).toEqual({ contents: 'read' });
expect(JSON.stringify(summary)).not.toMatch(/secrets\.|restore-deps|bun install/);
expect(summary.steps.at(-1)!.run).toContain('--emit-plan /tmp/paid-plan/manifest.json --slices 1');
expect(summary.steps.at(-1)!.run).toContain('scripts/train-gate.ts screen-summary /tmp/paid-plan/manifest.json --pr "$PR_NUMBER"');
});
test('a dispatched screen tests its pinned merge commit, reuses and binds as a screen, comments on its PR, and is keyed on the PR', () => {
const inputs = { ...INPUTS, evals_all: false, nonce: 'pr.screen.narrow-x', expected_sha: SHA.merge, base_ref: 'main', base_sha: SHA.base, screen_pr: '42', screen_merge_sha: SHA.merge, screen_policy: 'narrow' };
const route = { ...ROUTE, ...train, mode: 'screen', screen_head: SHA.screenHead, screen_base: SHA.screenBase, screen_branch: 'feature' };
const sim = simulate(evals, { github: dispatch(), inputs, outputs: { route } });
expect(sim.jobs).toEqual(['build-image', 'eval-slices', 'plan-slices', 'recover-receipts', 'route', 'slices-comment', 'slices-report']);
expect(sim.group).toBe('evals-screen-42');
expect(sim.runName).toBe('E2E Evals [eval-bg pr.screen.narrow-x] screen #42');
expect(sim.steps('recover-receipts')).not.toContain('Recover receipts of cancelled runs (base-ref code)');
expect(sim.checkout('recover-receipts')?.ref).toBe(SHA.screenBase);
expect(sim.step('recover-receipts', 'Find a gate verdict to reuse (R2a/R2, base-ref code)')!.env).toMatchObject({
PR_NUMBER: '42', HEAD_REPO: 'garrytan/gstack', HEAD_BRANCH: 'feature', HEAD_SHA: SHA.screenHead, TESTED_SHA: SHA.merge, SCREEN_POLICY: 'narrow' });
for (const job of ['plan-slices', 'eval-slices', 'slices-report']) expect(sim.checkout(job)?.ref, job).toBe(SHA.merge);
expect(sim.step('plan-slices', 'Emit run manifest')!.env.EVALS_BASE).toBe(SHA.screenBase);
expect(sim.steps('plan-slices')).toContain('Confirm gate verdict reuse (R2a/R2)');
const bind = sim.step('slices-report', "Bind this run's verdict for reuse (R2a/R2)")!;
expect(bind.env).toMatchObject({ PR_NUMBER: '42', HEAD_SHA: SHA.screenHead, TESTED_SHA: SHA.merge, SCREEN_POLICY: 'narrow' });
expect(bind.run).toContain('${SCREEN_POLICY:+--purpose screen --screen-policy "$SCREEN_POLICY"}');
expect(sim.steps('slices-report')).not.toContain('Write the train record');
expect(sim.step('slices-comment', 'Post PR comment')!.env).toMatchObject({ PR_NUMBER: '42', SCREEN_POLICY: 'narrow', SCREEN_MERGE_SHA: SHA.merge });
const superseded = simulate(evals, { github: dispatch(), inputs, outputs: { route: { ...route, superseded: 'true' } } });
expect(superseded.jobs).toEqual(['build-image', 'route']);
});
test('a train gate runs the full gate, writes its record from the report and publishes the check; it never reuses or binds', () => {
const inputs = { ...INPUTS, nonce: 'gate.train-x', expected_sha: SHA.head };
const sim = simulate(evals, { github: dispatch('train/t1'), inputs, outputs: { route: { ...ROUTE, ...train, mode: 'train' } } });
expect(sim.jobs).toEqual(['build-image', 'eval-slices', 'plan-slices', 'route', 'slices-report', 'train-check']);
expect(sim.steps('plan-slices')).not.toContain('Confirm gate verdict reuse (R2a/R2)');
expect(sim.steps('slices-report')).not.toContain("Bind this run's verdict for reuse (R2a/R2)");
expect(sim.steps('slices-report')).toEqual(expect.arrayContaining(['Write the train record', 'Upload the train record']));
expect(sim.checkout('slices-report')?.['fetch-depth']).toBe('0');
expect(sim.step('slices-report', 'Write the train record')!.run).toContain('scripts/train-gate.ts train-record /tmp/paid-report --nonce "$NONCE" --head "$HEAD_SHA"');
expect(sim.step('slices-report', 'Upload the train record')!.with).toMatchObject({ name: 'gstack-train-record', path: '/tmp/train-record', overwrite: true });
expect(simulate(evals, { github: dispatch('train/t1'), inputs, outputs: { route: { ...ROUTE, ...train, mode: 'train' } }, results: { 'slices-report': 'failure' } }).jobs).not.toContain('train-check');
});
test('the check run is published only for a complete GREEN census of the train head, by a job with checks: write and no repository code', () => {
const check = evals.jobs['train-check']!;
expect(check.permissions).toEqual({ checks: 'write' });
expect(check.steps.some(st => st.uses?.startsWith('actions/checkout@') || st.uses?.startsWith('oven-sh/setup-bun@'))).toBe(false);
const publish = check.steps.find(st => st.name === 'Publish the full-gate check on the train head')!;
expect(publish.run).toContain('.verdict == "GREEN" and .census.complete == true and .nonce == $nonce and .head_sha == $head');
expect(publish.run).toContain('-f status=completed -f conclusion=success');
expect(publish.env).toMatchObject({ CHECK: '${{ needs.plan-slices.outputs.full_gate_check }}', HEAD_SHA: '${{ inputs.expected_sha }}' });
});
test("a bisect runs one executor per prefix only, uploading each prefix's record under the name T1 reads", () => {
const inputs = { ...INPUTS, nonce: 'gate.bisect-x', expected_sha: SHA.head, cases: 'case-a', prefix_shas: SHA.base };
const sim = simulate(evals, { github: dispatch('train/t1'), inputs, outputs: { route: { ...ROUTE, ...train, mode: 'bisect' } } });
expect(sim.jobs).toEqual(['build-image', 'route', 'train-bisect']);
const job = evals.jobs['train-bisect']!;
expect(job.strategy!.matrix.slice).toBe('${{ fromJSON(needs.route.outputs.bisect_slices) }}');
const upload = job.steps.find(st => st.name === 'Upload the prefix record')!;
const name = String(upload.with!.name).replace('${{ needs.route.outputs.record_artifact }}', 'gstack-train-record').replace('${{ steps.prefix.outputs.short }}', SHA.base.slice(0, 12));
expect(name).toBe(bisectArtifact('gstack-train-record', SHA.base));
expect(job.steps.find(st => st.name === 'Verify the prefix checkout')!.run).toContain('echo "short=${PREFIX_SHA:0:12}"');
});
test('a refused dispatch starts nothing paid', () => {
const sim = simulate(evals, { github: dispatch('train/t1'), inputs: { ...INPUTS, nonce: 'gate.bisect-x' }, outputs: { route: { ...ROUTE, mode: '' } }, results: { route: 'failure' } });
expect(sim.jobs).toEqual(['build-image', 'route']);
});
});
describe('GSTA-48 open items', () => {
test('a head the readiness job blocked replaces the stale verdict comment and plans nothing (P3-1)', () => {
const sim = simulate(evals, { github: pr(), inputs: {}, outputs: { route: { ...ROUTE, mode: 'pr' } }, results: { readiness: 'failure' } });
expect(sim.jobs).toEqual(['build-image', 'readiness', 'readiness-comment', 'route']);
const comment = evals.jobs['readiness-comment']!;
expect(comment.steps.some(st => st.uses)).toBe(false);
expect(comment.steps[0]!.run).toContain('if [ -z "$COMMENT_ID" ]; then exit 0; fi');
expect(comment.steps[0]!.run).toContain('## E2E Evals: ⛔ NOT RUN on');
expect(comment.steps[0]!.run).not.toContain('PASS');
expect(simulate(evals, { github: pr(), inputs: {}, outputs: { route: { ...ROUTE, mode: 'pr' } } }).jobs).not.toContain('readiness-comment');
});
});
describe.skipIf(!Bun.which('jq') || !Bun.which('bash'))('the route job reads the train policy from the base revision with git and jq only', () => {
const route = evals.jobs.route!;
const policyStep = route.steps.find(st => st.name === 'Read the train policy (base revision)')!;
const routeStep = route.steps.find(st => st.name === 'Route this run')!;
function fixture(policy: unknown): { dir: string; base: string } {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'evals-route-'));
const git = (...args: string[]) => spawnSync('git', ['-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.com', '-c', 'commit.gpgsign=false', ...args], { cwd: dir, encoding: 'utf8', timeout: 30_000 });
git('init', '-q', '-b', 'main');
if (policy !== undefined) { fs.mkdirSync(path.join(dir, '.gstack')); fs.writeFileSync(path.join(dir, '.gstack/ship-policy.json'), typeof policy === 'string' ? policy : JSON.stringify(policy)); }
fs.writeFileSync(path.join(dir, 'README.md'), 'x\n');
git('add', '-A');
git('commit', '-q', '-m', 'base');
return { dir, base: git('rev-parse', 'HEAD').stdout.trim() };
}
function run(script: string, dir: string, env: Record<string, string>): Record<string, string> {
const out = path.join(dir, '.github-output');
fs.writeFileSync(out, '');
const r = spawnSync('bash', ['-e', '-c', script], { cwd: dir, encoding: 'utf8', timeout: 30_000,
env: { PATH: process.env.PATH ?? '', HOME: dir, GITHUB_OUTPUT: out, GITHUB_STEP_SUMMARY: path.join(dir, '.summary'), ...env } });
if (r.status !== 0) throw new Error(r.stderr);
return Object.fromEntries(fs.readFileSync(out, 'utf8').split('\n').filter(Boolean).map(line => [line.slice(0, line.indexOf('=')), line.slice(line.indexOf('=') + 1)]));
}
const block = { gate_workflow: 'evals.yml', free_workflow: 'free-tests.yml', full_gate_check: 'gstack/full-gate', record_artifact: 'gstack-train-record', max_size: 5, fill_hours: 3, screen_selection: 'none', trial_reservation_usd: 1.5 };
test('no policy, no train block, or an invalid one: train=false, and a PR push routes pr', () => {
for (const policy of [undefined, { bump: 'patch' }, { bump: 'patch', train: null }, { train: { ...block, record_artifact: 'bad name!' } }, '{not json']) {
const fx = fixture(policy);
const out = run(policyStep.run!, fx.dir, { BASE_SHA: fx.base, DEFAULT_BRANCH: 'main' });
expect(out, JSON.stringify(policy)).toEqual({ train: 'false' });
expect(run(routeStep.run!, fx.dir, { GITHUB_EVENT_NAME: 'pull_request', TRAIN_POLICY: out.train! })).toEqual({ mode: 'pr' });
fs.rmSync(fx.dir, { recursive: true, force: true });
}
});
test('a train block names the record artifact and check, and a PR push routes pr-summary', () => {
const fx = fixture({ bump: 'patch', train: block });
const out = run(policyStep.run!, fx.dir, { BASE_SHA: fx.base, DEFAULT_BRANCH: 'main' });
expect(out).toEqual({ train: 'true', record_artifact: 'gstack-train-record', full_gate_check: 'gstack/full-gate' });
expect(run(routeStep.run!, fx.dir, { GITHUB_EVENT_NAME: 'pull_request', TRAIN_POLICY: 'true' })).toEqual({ mode: 'pr-summary' });
fs.rmSync(fx.dir, { recursive: true, force: true });
});
test('a ref without scripts/train-gate.ts routes a plain dispatch as before and refuses train or screen inputs', () => {
const fx = fixture(undefined);
expect(run(routeStep.run!, fx.dir, { GITHUB_EVENT_NAME: 'workflow_dispatch', GITHUB_REF_NAME: 'old-branch' })).toEqual({ mode: 'dispatch' });
expect(() => run(routeStep.run!, fx.dir, { GITHUB_EVENT_NAME: 'workflow_dispatch', GITHUB_REF_NAME: 'old-branch', SCREEN_PR: '42' })).toThrow();
fs.rmSync(fx.dir, { recursive: true, force: true });
});
test('the policy is read before any repository code runs', () => {
const steps = route.steps.map(st => st.name ?? st.uses ?? '');
expect(steps.indexOf('Read the train policy (base revision)')).toBeLessThan(steps.indexOf('Route this run'));
expect(policyStep.run).not.toContain('bun');
expect(route.permissions).toEqual({ contents: 'read', actions: 'read', 'pull-requests': 'read' });
});
});
describe('evals-main-backstop.yml', () => {
test('push to main only, one at a time without cancelling, inert unless main declares a train block', () => {
expect(Object.keys(backstopWf.on)).toEqual(['push']);
expect(backstopWf.on.push).toEqual({ branches: ['main'] });
expect(backstopWf.concurrency).toEqual({ group: 'evals-main-backstop', 'cancel-in-progress': false });
expect(backstopWf.permissions).toEqual({});
const job = backstopWf.jobs.backstop!;
expect(job.permissions).toEqual({ contents: 'write', actions: 'write', checks: 'read', issues: 'write' });
const policy = job.steps.find(st => st.name === 'Read the train policy (main)')!;
expect(policy.run).not.toContain('bun');
const gate = job.steps.find(st => st.name === 'Gate main unless a green train covers its tree')!;
expect(gate.if).toBe("steps.policy.outputs.train == 'true'");
expect(gate.run).toContain('scripts/train-backstop.ts run --repo "$GITHUB_REPOSITORY" --branch "$GITHUB_REF_NAME" --sha "$GITHUB_SHA"');
});
});
+4 -1
View File
@@ -422,7 +422,10 @@ describe('panel verdict surfaces (eval reliability policy)', () => {
for (const source of [evalsYml, periodicYml, marathonYml]) {
const jobs = jobsOf(source);
const uploads = Object.values(jobs).flatMap(job => job.steps).filter(step => step.uses?.startsWith('actions/upload-artifact@'))
.map(step => step.with?.name ?? '').filter(name => /slice|census-\$/.test(name));
.map(step => step.with?.name ?? '').filter(name => /slice|census-\$/.test(name))
// GSTA-49 T1b: the train record keeps the fixed name gstack-ship-queue reads; it is built from the
// first attempt's slice results only (scripts/train-gate.ts buildTrainRecord), so a re-run cannot change it.
.filter(name => name !== '${{ needs.plan-slices.outputs.record_artifact }}');
expect(uploads.length).toBeGreaterThan(0);
for (const name of uploads) expect(name, name).toContain('-a${{ github.run_attempt }}');
const downloads = Object.values(jobs).flatMap(job => job.steps).filter(step => step.uses?.startsWith('actions/download-artifact@') && step.with?.pattern);
@@ -0,0 +1,181 @@
{
"provenance": {
"kind": "public native AskUserQuestion calls and answers only (observation.json fingerprints[].nativeCall, byte-exact)",
"runId": 39298269902,
"artifact": "paid-slice-4-a1",
"headSha": "072d4b95e5f3b9cf329fbd91fa97e339388f5fe6",
"recordedOutcome": "ceiling_reached",
"recordedReviewCount": 1,
"recordedStep0Count": 4,
"recordedPassed": false,
"note": "The Step 0D question was titled 'Review focus for the User Dashboard plan' (header Focus) and carried 'I rated this plan 4/10 on design completeness' in its body, with options 'All 7 dimensions (recommended)' and 'Specific areas only'. The runner reached the review ceiling on the issue-labeled D5 finding, but the focus predicate required 'all 7' in the title, so the gate failed."
},
"calls": [
{
"preReview": true,
"observedAtMs": 41013,
"call": {
"sessionId": "8baee8e7-0997-47ec-9eb3-5e4487a80424",
"toolUseId": "toolu_01JVwSk6LQqTFjxJegSV8r5v",
"questions": [
{
"question": "D1 — Add gstack skill routing rules to this project's CLAUDE.md?\nProject/branch/task: main branch, reviewing the User Dashboard Page plan.\nELI10: gstack can add a short \"Skill routing\" section to CLAUDE.md so future sessions automatically pick the right skill (e.g. design review, QA). Without it, you invoke skills by name yourself.\nStakes if we pick wrong: Either an unwanted CLAUDE.md edit + commit, or future sessions don't auto-route to skills.\nRecommendation: A because it is one-time and makes skill selection automatic.\nNote: options differ in kind, not coverage — no completeness score.\nNote: plan mode is active, so if you pick A the CLAUDE.md edit + commit is deferred until after plan mode ends; nothing is written now.\nPros / cons:\nA) Add routing rules (recommended)\n ✅ Future sessions route requests to matching skills without you naming them\n ✅ One-time setup, a small appended section at the end of CLAUDE.md\n ❌ Edits and commits CLAUDE.md, which in this repo is a fixture file\nB) No thanks\n ✅ CLAUDE.md stays exactly as it is, no commit is created\n ✅ You keep full manual control over which skill runs\n ❌ You must invoke skills by name every time going forward\nNet: automatic routing versus leaving CLAUDE.md untouched.",
"header": "Routing",
"multiSelect": false,
"options": [
{
"label": "Add routing rules (recommended)",
"description": "Append the Skill routing section to CLAUDE.md and commit (deferred until plan mode ends)."
},
{
"label": "No thanks",
"description": "Set routing_declined; invoke skills manually."
}
]
}
],
"answered": true,
"failed": false,
"answers": {
"D1 — Add gstack skill routing rules to this project's CLAUDE.md?\nProject/branch/task: main branch, reviewing the User Dashboard Page plan.\nELI10: gstack can add a short \"Skill routing\" section to CLAUDE.md so future sessions automatically pick the right skill (e.g. design review, QA). Without it, you invoke skills by name yourself.\nStakes if we pick wrong: Either an unwanted CLAUDE.md edit + commit, or future sessions don't auto-route to skills.\nRecommendation: A because it is one-time and makes skill selection automatic.\nNote: options differ in kind, not coverage — no completeness score.\nNote: plan mode is active, so if you pick A the CLAUDE.md edit + commit is deferred until after plan mode ends; nothing is written now.\nPros / cons:\nA) Add routing rules (recommended)\n ✅ Future sessions route requests to matching skills without you naming them\n ✅ One-time setup, a small appended section at the end of CLAUDE.md\n ❌ Edits and commits CLAUDE.md, which in this repo is a fixture file\nB) No thanks\n ✅ CLAUDE.md stays exactly as it is, no commit is created\n ✅ You keep full manual control over which skill runs\n ❌ You must invoke skills by name every time going forward\nNet: automatic routing versus leaving CLAUDE.md untouched.": "Add routing rules (recommended)"
},
"unansweredQuestionIndices": [],
"answeredAt": "2026-10-19T18:09:26.151Z"
}
},
{
"preReview": true,
"observedAtMs": 88160,
"call": {
"sessionId": "8baee8e7-0997-47ec-9eb3-5e4487a80424",
"toolUseId": "toolu_01WMTWHwmpvAHVSW4BS2UKZb",
"questions": [
{
"question": "D2 — Review focus for the User Dashboard plan\nProject/branch/task: main, design review of the User Dashboard Page plan (review-input.md).\nELI10: I rated this plan 4/10 on design completeness. The biggest gaps are page hierarchy per breakpoint, what each activity/notification row actually shows, and the real copy/behavior for empty, error, modal and toast states. Next I generate visual mockups, then walk all 7 design dimensions. You can narrow that.\nStakes if we pick wrong: Narrowing skips dimensions, so an unreviewed gap (e.g. accessibility or responsive) ships as whatever the implementer guesses.\nRecommendation: A because the plan has gaps in nearly every dimension, so a narrow pass would leave known holes.\nCompleteness: A=10/10, B=6/10\nPros / cons:\nA) All 7 dimensions (recommended)\n ✅ Every dimension gets rated and every gap gets its own decision\n ✅ Hierarchy, states, a11y and responsive all land in the plan before code\n ❌ More questions to answer, one per gap (human: ~1 hr / CC: ~15 min)\nB) Focus on specific areas only\n ✅ Fewer questions, faster to a revised plan for the areas you care about\n ✅ Lets you skip dimensions you already have opinions on\n ❌ Unreviewed dimensions stay at their current low scores in the report\nNet: full coverage versus fewer questions.",
"header": "Focus",
"multiSelect": false,
"options": [
{
"label": "All 7 dimensions (recommended)",
"description": "Mockups first, then full 7-pass review with per-gap decisions."
},
{
"label": "Specific areas only",
"description": "Tell me which dimensions (e.g. hierarchy, states, a11y) to focus on."
}
]
}
],
"answered": true,
"failed": false,
"answers": {
"D2 — Review focus for the User Dashboard plan\nProject/branch/task: main, design review of the User Dashboard Page plan (review-input.md).\nELI10: I rated this plan 4/10 on design completeness. The biggest gaps are page hierarchy per breakpoint, what each activity/notification row actually shows, and the real copy/behavior for empty, error, modal and toast states. Next I generate visual mockups, then walk all 7 design dimensions. You can narrow that.\nStakes if we pick wrong: Narrowing skips dimensions, so an unreviewed gap (e.g. accessibility or responsive) ships as whatever the implementer guesses.\nRecommendation: A because the plan has gaps in nearly every dimension, so a narrow pass would leave known holes.\nCompleteness: A=10/10, B=6/10\nPros / cons:\nA) All 7 dimensions (recommended)\n ✅ Every dimension gets rated and every gap gets its own decision\n ✅ Hierarchy, states, a11y and responsive all land in the plan before code\n ❌ More questions to answer, one per gap (human: ~1 hr / CC: ~15 min)\nB) Focus on specific areas only\n ✅ Fewer questions, faster to a revised plan for the areas you care about\n ✅ Lets you skip dimensions you already have opinions on\n ❌ Unreviewed dimensions stay at their current low scores in the report\nNet: full coverage versus fewer questions.": "All 7 dimensions (recommended)"
},
"unansweredQuestionIndices": [],
"answeredAt": "2026-10-19T18:10:17.876Z"
}
},
{
"preReview": true,
"observedAtMs": 123730,
"call": {
"sessionId": "8baee8e7-0997-47ec-9eb3-5e4487a80424",
"toolUseId": "toolu_01PDpzEdw9gtfax268HSxc9a",
"questions": [
{
"question": "Want outside design voices before the detailed review?\nD3 — Outside design voices\nProject/branch/task: main, User Dashboard Page plan design review.\nELI10: Before the 7 passes, I can get second opinions: Codex scores the plan against hard design rules and litmus checks, and a separate Claude agent does an independent completeness read. It costs a few minutes and gives a cross-check.\nStakes if we pick wrong: Skipping means one reviewer's judgment; running costs time and Codex may be unavailable.\nRecommendation: B because the plan's gaps are already concrete and a single native pass covers them; this is a taste call.\nNote: options differ in kind, not coverage — no completeness score.\nNet: a cross-model check versus a faster path to the fixed plan.",
"header": "Voices",
"multiSelect": false,
"options": [
{
"label": "Yes, run outside design voices",
"description": "✅ Codex hard-rule + litmus scorecard plus an independent Claude completeness read. ✅ Disagreements surface with both perspectives in the relevant pass. ❌ Adds a few minutes; Codex may be missing here, in which case only the Claude voice runs and it's tagged single-model."
},
{
"label": "No, proceed without (recommended)",
"description": "✅ Straight into the 7 passes with the approved mockup as anchor. ✅ Fewest steps to a fixed plan. ❌ Findings rest on one reviewer; no cross-model check on slop risk or hierarchy."
}
]
}
],
"answered": true,
"failed": false,
"answers": {
"Want outside design voices before the detailed review?\nD3 — Outside design voices\nProject/branch/task: main, User Dashboard Page plan design review.\nELI10: Before the 7 passes, I can get second opinions: Codex scores the plan against hard design rules and litmus checks, and a separate Claude agent does an independent completeness read. It costs a few minutes and gives a cross-check.\nStakes if we pick wrong: Skipping means one reviewer's judgment; running costs time and Codex may be unavailable.\nRecommendation: B because the plan's gaps are already concrete and a single native pass covers them; this is a taste call.\nNote: options differ in kind, not coverage — no completeness score.\nNet: a cross-model check versus a faster path to the fixed plan.": "No, proceed without (recommended)"
},
"unansweredQuestionIndices": [],
"answeredAt": "2026-10-19T18:10:53.393Z"
}
},
{
"preReview": true,
"observedAtMs": 140130,
"call": {
"sessionId": "8baee8e7-0997-47ec-9eb3-5e4487a80424",
"toolUseId": "toolu_0116C69RgYwN7PJ2H7FpKdqq",
"questions": [
{
"question": "D4 — Search learnings across your other projects?\nProject/branch/task: main, User Dashboard plan design review.\nELI10: gstack keeps local notes (\"learnings\") from past sessions. It can search notes from your other projects on this machine too, which helps when patterns repeat. Nothing leaves your machine.\nStakes if we pick wrong: Cross-project on can mix in patterns from unrelated client codebases; off misses reusable lessons.\nRecommendation: A because for a solo developer reusable lessons outweigh cross-contamination risk.\nNote: options differ in kind, not coverage — no completeness score.\nPros / cons:\nA) Enable cross-project learnings (recommended)\n ✅ Past lessons from other repos on this machine can inform this review\n ✅ Stays fully local, one-time setting stored in gstack config\n ❌ Patterns from unrelated client projects could leak into recommendations\nB) Project-scoped only\n ✅ Each project's learnings stay isolated, no cross-client contamination\n ✅ Simpler mental model of where advice comes from\n ❌ Misses lessons already learned elsewhere on this machine\nNet: broader recall versus strict per-project isolation.",
"header": "Learnings",
"multiSelect": false,
"options": [
{
"label": "Enable cross-project (recommended)",
"description": "Set cross_project_learnings=true and search across local projects."
},
{
"label": "Project-scoped only",
"description": "Set cross_project_learnings=false."
}
]
}
],
"answered": true,
"failed": false,
"answers": {
"D4 — Search learnings across your other projects?\nProject/branch/task: main, User Dashboard plan design review.\nELI10: gstack keeps local notes (\"learnings\") from past sessions. It can search notes from your other projects on this machine too, which helps when patterns repeat. Nothing leaves your machine.\nStakes if we pick wrong: Cross-project on can mix in patterns from unrelated client codebases; off misses reusable lessons.\nRecommendation: A because for a solo developer reusable lessons outweigh cross-contamination risk.\nNote: options differ in kind, not coverage — no completeness score.\nPros / cons:\nA) Enable cross-project learnings (recommended)\n ✅ Past lessons from other repos on this machine can inform this review\n ✅ Stays fully local, one-time setting stored in gstack config\n ❌ Patterns from unrelated client projects could leak into recommendations\nB) Project-scoped only\n ✅ Each project's learnings stay isolated, no cross-client contamination\n ✅ Simpler mental model of where advice comes from\n ❌ Misses lessons already learned elsewhere on this machine\nNet: broader recall versus strict per-project isolation.": "Enable cross-project (recommended)"
},
"unansweredQuestionIndices": [],
"answeredAt": "2026-10-19T18:11:10.623Z"
}
},
{
"preReview": false,
"observedAtMs": 168630,
"call": {
"sessionId": "8baee8e7-0997-47ec-9eb3-5e4487a80424",
"toolUseId": "toolu_01HhdTUnvoyntgagQCG8UjBH",
"questions": [
{
"question": "D5 — Issue 1: Which content leads the dashboard, and how do panels lay out?\nProject/branch/task: main, User Dashboard plan, Pass 1 Information Architecture.\nELI10: The plan names three panels but never says which one the user sees first. Without that, the engineer picks an order (usually the order in the spec) and every panel gets equal weight, so nothing stands out. Unread notifications are the only thing here that asks the user to do something, so they should win.\nStakes if we pick wrong: Users landing after login scan an undifferentiated grid and miss unread items, or the page is rebuilt after launch.\nRecommendation: 1A because notifications are the only actionable content and \"hierarchy as service\" says the actionable thing goes first.\nCompleteness: 1A=9/10, 1B=8/10, 1C=5/10\nPros / cons:\n1A) Notifications primary, Activity secondary (recommended)\n ✅ Unread items are the first thing seen on every viewport, matching the user's real job\n ✅ lg: two columns (Notifications 7/12 left, Activity 5/12 right); base/sm/md: single column Notifications then Activity\n ❌ Activity gets less width on desktop, so long activity descriptions truncate sooner\n1B) Single column at all widths, Notifications then Activity\n ✅ Simplest layout, one reading path, easiest to make accessible and responsive\n ✅ Same priority order on every device, nothing to re-learn between phone and desktop\n ❌ Wastes horizontal space on lg screens; Activity sits below the fold on desktop\n1C) Leave the order to the implementer\n ✅ No spec time now, implementer has freedom (human: 0 / CC: 0)\n ✅ Can be tuned later once real data exists\n ❌ Most likely result is three equal-weight panels with no clear first read\nNet: a ranked, actionable-first layout versus simplicity or deferral.",
"header": "Hierarchy",
"multiSelect": false,
"options": [
{
"label": "1A Notifications first (recommended)",
"description": "lg: 2 columns, Notifications 7/12 + Activity 5/12; below lg: single column Notifications → Activity.",
"preview": "lg (≥1024px)\n+----------------------------------------------+\n| App header: logo | Dashboard* Workspace | ava|\n+----------------------------------------------+\n| Dashboard |\n+---------------------------+------------------+\n| Notifications 3 unread | Recent activity |\n| [Mark all as read] | row |\n| ● row | row |\n| ● row | row |\n| row | View all → |\n+---------------------------+------------------+\n\nbase/sm/md\n+------------------+\n| Dashboard |\n| Notifications |\n| rows... |\n| Recent activity |\n| rows... |\n+------------------+"
},
{
"label": "1B Single column always",
"description": "Notifications then Activity, one column at every width (max-w ~720px centered).",
"preview": "all widths\n+--------------------------+\n| Dashboard |\n| Notifications 3 unread |\n| rows... |\n| Recent activity |\n| rows... |\n+--------------------------+"
},
{
"label": "1C Leave to implementer",
"description": "No hierarchy spec; implementer decides order and layout."
}
]
}
],
"answered": true,
"failed": false,
"answers": {
"D5 — Issue 1: Which content leads the dashboard, and how do panels lay out?\nProject/branch/task: main, User Dashboard plan, Pass 1 Information Architecture.\nELI10: The plan names three panels but never says which one the user sees first. Without that, the engineer picks an order (usually the order in the spec) and every panel gets equal weight, so nothing stands out. Unread notifications are the only thing here that asks the user to do something, so they should win.\nStakes if we pick wrong: Users landing after login scan an undifferentiated grid and miss unread items, or the page is rebuilt after launch.\nRecommendation: 1A because notifications are the only actionable content and \"hierarchy as service\" says the actionable thing goes first.\nCompleteness: 1A=9/10, 1B=8/10, 1C=5/10\nPros / cons:\n1A) Notifications primary, Activity secondary (recommended)\n ✅ Unread items are the first thing seen on every viewport, matching the user's real job\n ✅ lg: two columns (Notifications 7/12 left, Activity 5/12 right); base/sm/md: single column Notifications then Activity\n ❌ Activity gets less width on desktop, so long activity descriptions truncate sooner\n1B) Single column at all widths, Notifications then Activity\n ✅ Simplest layout, one reading path, easiest to make accessible and responsive\n ✅ Same priority order on every device, nothing to re-learn between phone and desktop\n ❌ Wastes horizontal space on lg screens; Activity sits below the fold on desktop\n1C) Leave the order to the implementer\n ✅ No spec time now, implementer has freedom (human: 0 / CC: 0)\n ✅ Can be tuned later once real data exists\n ❌ Most likely result is three equal-weight panels with no clear first read\nNet: a ranked, actionable-first layout versus simplicity or deferral.": "1A Notifications first (recommended)"
},
"unansweredQuestionIndices": [],
"answeredAt": "2026-10-19T18:11:38.592Z"
}
}
]
}
@@ -0,0 +1,47 @@
{
"provenance": {
"kind": "public native AskUserQuestion calls and answers only (observation.json fingerprints[].nativeCall, byte-exact)",
"runId": 39298269902,
"artifact": "paid-slice-4-a2",
"headSha": "072d4b95e5f3b9cf329fbd91fa97e339388f5fe6",
"recordedOutcome": "ceiling_reached",
"recordedReviewCount": 1,
"recordedStep0Count": 0,
"recordedPassed": false,
"note": "The Step 0D question arrived on one line, 'Review focus. Project/branch/task: ...', with 'I rated this plan 4/10 on design completeness' in the body and options 'All 7 passes (recommended)' and 'States + hierarchy only'. The focus predicate missed it, and the finding predicate counted it as the review question (its one-line title names hierarchy and panels), so the run stopped at the ceiling on the focus question itself."
},
"calls": [
{
"preReview": false,
"observedAtMs": 63729,
"call": {
"sessionId": "84af3460-cb15-42a6-a835-364da70c0be4",
"toolUseId": "toolu_01JaP4QULFteQDRNuaxtMvmz",
"questions": [
{
"question": "D1 — Review focus. Project/branch/task: User Dashboard plan on main. ELI10: I rated this plan 4/10 on design completeness. Biggest gaps: no visual hierarchy (which panel wins on first view), panel states named but never designed (empty copy, skeleton shape, error/retry layout), and no responsive layout per breakpoint. Mockups come next, then all 7 review dimensions unless you want a narrower focus. Stakes if we pick wrong: a narrow focus leaves accessibility or state gaps for the implementer to guess. Recommendation: A because the plan is early and every dimension has open gaps. Completeness: A=10/10, B=6/10. Pros / cons: A) All 7 passes (recommended) ✅ Covers hierarchy, states, journey, slop risk, specificity, responsive and accessibility in one sitting ✅ Each gap still gets its own decision, so you can skip any fix ❌ More questions to answer, roughly one per real gap found. B) Focus on states + hierarchy only ✅ Faster; targets the two gaps most likely to ship looking broken ✅ Fewer decisions to make today ❌ Responsive, modal/toast accessibility and copy stay unspecified for implementer. Net: thoroughness now versus fewer questions today.",
"header": "Focus",
"multiSelect": false,
"options": [
{
"label": "All 7 passes (recommended)",
"description": "✅ Full coverage of every design dimension, one decision per gap. ✅ Nothing left for the implementer to guess. ❌ More questions overall."
},
{
"label": "States + hierarchy only",
"description": "✅ Faster, targets the riskiest gaps. ✅ Fewer decisions. ❌ Responsive, a11y and copy stay unspecified."
}
]
}
],
"answered": true,
"failed": false,
"answers": {
"D1 — Review focus. Project/branch/task: User Dashboard plan on main. ELI10: I rated this plan 4/10 on design completeness. Biggest gaps: no visual hierarchy (which panel wins on first view), panel states named but never designed (empty copy, skeleton shape, error/retry layout), and no responsive layout per breakpoint. Mockups come next, then all 7 review dimensions unless you want a narrower focus. Stakes if we pick wrong: a narrow focus leaves accessibility or state gaps for the implementer to guess. Recommendation: A because the plan is early and every dimension has open gaps. Completeness: A=10/10, B=6/10. Pros / cons: A) All 7 passes (recommended) ✅ Covers hierarchy, states, journey, slop risk, specificity, responsive and accessibility in one sitting ✅ Each gap still gets its own decision, so you can skip any fix ❌ More questions to answer, roughly one per real gap found. B) Focus on states + hierarchy only ✅ Faster; targets the two gaps most likely to ship looking broken ✅ Fewer decisions to make today ❌ Responsive, modal/toast accessibility and copy stay unspecified for implementer. Net: thoroughness now versus fewer questions today.": "All 7 passes (recommended)"
},
"unansweredQuestionIndices": [],
"answeredAt": "2026-10-19T18:23:49.901Z"
}
}
]
}
+74 -4
View File
@@ -129,9 +129,10 @@ type Jobs = Record<number, Array<{ name: string; conclusion: string | null }>>;
* slices; an unbound one ran none unless scripted), downloads that write the
* scripted binding (or raw text), and merge-bases from the fixture repo.
*/
function fakeGh(runs: WorkflowRun[], bindings: Record<number, GateBinding | string>, failList = '', jobs: Jobs = {}, failCompare = false): GhRunner {
function fakeGh(runs: WorkflowRun[], bindings: Record<number, GateBinding | string>, failList = '', jobs: Jobs = {}, failCompare = false, prCommits: string[] = []): GhRunner {
return (args) => {
const ok = (body: unknown) => ({ status: 0, stdout: JSON.stringify(body), stderr: '' });
if (args.some(a => /\/pulls\/42\/commits$/.test(a))) return { status: 0, stdout: prCommits.join('\n') + '\n', stderr: '' };
if (args.some(a => a.endsWith('/workflows/evals.yml/runs'))) return failList ? { status: 1, stdout: '', stderr: failList } : ok({ workflow_runs: runs });
const jobList = args.find(a => /\/runs\/\d+\/jobs$/.test(a));
if (jobList) {
@@ -175,11 +176,11 @@ function tested(base: string, head: string): string {
let outCount = 0;
function find(head: string, runs: WorkflowRun[], bindings: Record<number, GateBinding | string>,
over: { baseSha?: string; failList?: string; jobs?: Jobs; failCompare?: boolean; testedSha?: string } = {}) {
over: { baseSha?: string; failList?: string; jobs?: Jobs; failCompare?: boolean; testedSha?: string; screen?: { policy: 'narrow' | 'broad' }; prCommits?: string[] } = {}) {
const out = path.join(scratch, 'out-' + ++outCount);
const result = findReusableGate({ repo: REPO, pr: 42, headRepo: REPO, branch: 'feature', selfRunId: 500, headSha: head,
testedSha: over.testedSha ?? tested(over.baseSha ?? sha.base!, head), repoRoot: receipt, out,
gh: fakeGh(runs, bindings, over.failList, over.jobs, over.failCompare) });
testedSha: over.testedSha ?? tested(over.baseSha ?? sha.base!, head), repoRoot: receipt, out, ...(over.screen ? { screen: over.screen } : {}),
gh: fakeGh(runs, bindings, over.failList, over.jobs, over.failCompare, over.prCommits) });
return { ...result, out, text: result.lines.join('\n') };
}
@@ -488,3 +489,72 @@ describe.skipIf(!Bun.which('jq') || !Bun.which('bash'))('the PR comment for a re
expect(failed.stdout).toContain('❌ FAIL');
});
});
describe('dispatched screens (GSTA-49 T1b): authenticated by PR, head, repository and tested base', () => {
const title = (policy: string, pr = 42) => `E2E Evals [eval-bg pr.screen.${policy}-${'c'.repeat(12)}-${'d'.repeat(12)}-abc123] screen #${pr}`;
// A screen dispatches on main: the run's head_sha is main's tip, never the PR head; the binding carries the PR head and the merge commit.
const screenRun = (id: number, policy = 'narrow', over: Partial<WorkflowRun> = {}) => run(id, sha.base!, 'success', { event: 'workflow_dispatch', pull_requests: [], display_title: title(policy), ...over });
const screenBinding = (head: string, policy: 'narrow' | 'broad' = 'narrow', over: Partial<GateBinding> = {}) =>
binding({ head_sha: head, purpose: 'screen', tested_sha: tested(sha.base!, head), screen_policy: policy, ...over });
const commits = () => [sha.gated!, sha.restamp!];
const screen = (policy: 'narrow' | 'broad', runs: WorkflowRun[], bindings: Record<number, GateBinding | string>, over: { prCommits?: string[]; jobs?: Jobs } = {}) =>
find(sha.restamp!, runs, bindings, { screen: { policy }, prCommits: over.prCommits ?? commits(), jobs: over.jobs });
test('a screen, a restamp-only push, then a re-screen of the same policy reuses through R2a', () => {
const { candidate, text } = screen('narrow', [screenRun(300)], { 300: screenBinding(sha.gated!) });
expect(text).toContain('TREE: same-modulo-stamps');
expect(candidate).toMatchObject({ kind: 'restamp-only', reused_run_id: 300, gated_head: sha.gated, current_head: sha.restamp, base_sha: sha.base, reused_screen_policy: 'narrow' });
expect(reuseReport(candidate!, REPO)).toContain('(a narrow screen)');
});
test("a PR push under a train policy (no paid slice, no binding) is passed over; the PR's earlier screen still reuses", () => {
const summary = run(320, sha.restamp!, 'success');
const { candidate } = screen('narrow', [summary, screenRun(300)], { 300: screenBinding(sha.gated!) }, { jobs: { 320: [{ name: 'eval-slices', conclusion: 'skipped' }] } });
expect(candidate?.reused_run_id).toBe(300);
});
test('a broad screen never adopts a narrow run; a narrow screen never adopts a pull_request gate (broad); a broad screen may', () => {
const narrowRun = screen('broad', [screenRun(300)], { 300: screenBinding(sha.gated!) });
expect(narrowRun.candidate).toBeNull();
expect(narrowRun.text).toContain('gated the narrow selection; a broad screen never adopts it');
const prRun = screen('narrow', [run(300, sha.gated!)], { 300: binding() });
expect(prRun.candidate).toBeNull();
expect(prRun.text).toContain('gated the broad selection; a narrow screen never adopts it');
expect(screen('broad', [run(300, sha.gated!)], { 300: binding() }).candidate?.reused_run_id).toBe(300);
});
test('a forged screen is refused: another PR, a head outside the PR, a mismatched binding, a tested commit off its bound base, another repository', () => {
const refused: Array<[WorkflowRun[], Record<number, GateBinding | string>, string, string[]?]> = [
[[screenRun(300, 'narrow', { display_title: title('narrow', 43) })], { 300: screenBinding(sha.gated!) }, 'no earlier executed gate on this PR bound a verdict'],
[[screenRun(300)], { 300: screenBinding(sha.gated!) }, 'is not a commit of PR #42', [sha.restamp!]],
[[screenRun(300)], { 300: screenBinding(sha.gated!, 'narrow', { pr: 43 }) }, 'does not match the run it came from'],
[[screenRun(300)], { 300: binding() }, 'does not match the run it came from'],
[[screenRun(300)], { 300: screenBinding(sha.gated!, 'narrow', { tested_sha: tested(sha.moved!, sha.gated!) }) }, 'does not merge its head into its bound base'],
[[screenRun(300, 'narrow', { head_repository: { full_name: 'fork/gstack' } })], { 300: screenBinding(sha.gated!) }, 'no earlier executed gate on this PR bound a verdict'],
[[run(300, sha.gated!)], { 300: screenBinding(sha.gated!) }, 'does not match the run it came from'],
];
for (const [runs, bindings, why, prCommits] of refused) {
const { candidate, text } = screen('narrow', runs, bindings, { prCommits });
expect(candidate, why).toBeNull();
expect(text, why).toContain(why);
}
});
test('train, bisect and plain dispatch runs never bind a verdict, so they never reuse or get reused', () => {
const report = path.join(scratch, 'bind-purpose');
fs.mkdirSync(report, { recursive: true });
fs.writeFileSync(path.join(report, 'manifest.json'), JSON.stringify(manifest()));
fs.writeFileSync(path.join(report, 'collector-outcomes.json'), JSON.stringify({ verdict: { verdict: 'GREEN' }, totals: { manual_accepted: 0 } }));
const args = { repository: REPO, pr: 42, runId: 300, runAttempt: 1, headSha: 'a'.repeat(40), baseSha: 'b'.repeat(40), profile: 'full', reconcileExit: 0, reportDir: report };
for (const purpose of ['train', 'bisect', 'dispatch'] as const) {
const bound = buildBinding({ ...args, purpose });
expect(bound.binding, purpose).toBeNull();
expect(bound.reason).toBe(`a ${purpose} run never binds a verdict for reuse`);
}
expect(buildBinding({ ...args, profile: 'pr', purpose: 'screen', testedSha: 'c'.repeat(40), screenPolicy: 'broad' }).binding)
.toMatchObject({ purpose: 'screen', tested_sha: 'c'.repeat(40), screen_policy: 'broad' });
// A train dispatch in the history is neither a screen nor a pull_request run of this PR: never listed, never a candidate.
const trainRun = run(310, sha.base!, 'success', { event: 'workflow_dispatch', pull_requests: [], display_title: 'E2E Evals [eval-bg gate.train-x]' });
expect(screen('narrow', [trainRun, screenRun(300)], { 300: screenBinding(sha.gated!), 310: screenBinding(sha.gated!) }).candidate?.reused_run_id).toBe(300);
});
});
+1 -1
View File
@@ -379,7 +379,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
"test/fixtures/plan-scope-recovery-av.json",
"test/fixtures/design-scope-checkpoint-at.json",'plan-design-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty/**', 'test/fixtures/design-ui-boxed-question.json', 'test/helpers/hermetic-skill-runtime.ts', 'test/helpers/pty-trust-dialog.ts', 'test/skill-e2e-plan-design-with-ui.test.ts', 'test/fixtures/eng-d2-truncated-border-0bcd.json', 'test/fixtures/eng-d1-clipped-elision-1579.json', 'test/fixtures/eng-d2-planning-prelude-4d.json', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt',
"test/fixtures/design-scope-checkpoint-at.json",'plan-design-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/fixtures/design-ui-focus-capture-39298269902-a1.json', 'test/fixtures/design-ui-focus-capture-39298269902-a2.json', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty/**', 'test/fixtures/design-ui-boxed-question.json', 'test/helpers/hermetic-skill-runtime.ts', 'test/helpers/pty-trust-dialog.ts', 'test/skill-e2e-plan-design-with-ui.test.ts', 'test/fixtures/eng-d2-truncated-border-0bcd.json', 'test/fixtures/eng-d1-clipped-elision-1579.json', 'test/fixtures/eng-d2-planning-prelude-4d.json', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt',
"scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts",
+25 -2
View File
@@ -83,6 +83,10 @@ const captured = JSON.parse(fs.readFileSync(${JSON.stringify(path.join(ROOT, 'te
// Run 39199692993 slice 4 (521fd755): the Step 0D title omitted 'design'; its body carried the rating.
const capturedRated = JSON.parse(fs.readFileSync(${JSON.stringify(path.join(ROOT, 'test/fixtures/design-ui-focus-capture-39199692993.json'))}, 'utf8'))
.calls.map(({ preReview, observedAtMs, call }) => nativePlanCallFingerprint(call, observedAtMs, preReview));
// Run 39298269902 slice 4 (072d4b9), attempts 1 and 2: Opus 5.5 titled Step 0D only
// "Review focus", with the rating in the body and an all-7 option (a2 on one line).
const captured39298269902 = [1, 2].map(attempt => JSON.parse(fs.readFileSync(${JSON.stringify(ROOT)} + '/test/fixtures/design-ui-focus-capture-39298269902-a' + attempt + '.json', 'utf8'))
.calls.map(({ preReview, observedAtMs, call }) => nativePlanCallFingerprint(call, observedAtMs, preReview)));
const unnumberedFinding = fp('unnumbered-finding', "Which region is the dashboard's primary anchor, and how is the page composed?", false);
unnumberedFinding.nativeCall.questions[0].options = nativeFinding.nativeCall.questions[0].options;
const finding = fp('finding', 'Which loading feedback should Save show?', false);
@@ -198,6 +202,22 @@ mock.module(${JSON.stringify(path.join(ROOT, 'test/helpers/claude-pty-runner.ts'
['D14 — Record this follow-up?', ['1A) Record a TODO']],
['D15 — Want outside design voices before the detailed review?', ['1A) Yes, run outside voices', '1B) No, proceed without']],
]) expect(opts.isReviewAUQ(labeled('not-a-finding', question, labels))).toBe(false);
const [focusA1, focusA2] = [captured39298269902[0][1], captured39298269902[1][0]];
for (const focusCall of [focusA1, focusA2]) {
expect(opts.isLastStep0AUQ(focusCall)).toBe(true);
expect(opts.isReviewAUQ(focusCall)).toBe(false);
}
expect(opts.isReviewAUQ(captured39298269902[0][4])).toBe(true);
const ratedBody = 'Project/branch/task: main. ELI10: I rated this plan 4/10 on design completeness.';
const ratedFinding = labeled('rated-finding', 'D5 — Issue 1: Which panel leads the review focus? ' + ratedBody, ['1A All 7 dimensions first', '1B Activity first']);
expect(opts.isLastStep0AUQ(ratedFinding)).toBe(false);
expect(opts.isReviewAUQ(ratedFinding)).toBe(true);
for (const [label, labels] of [['single-option-focus', ['All 7 dimensions (recommended)']], ['no-all-seven-option', ['Every dimension (recommended)', 'States + hierarchy only']],
['all-seven-not-leading', ['Review all 7 dimensions (recommended)', 'States + hierarchy only']]]) {
expect(opts.isLastStep0AUQ(labeled(label, 'D1 — Review focus. ' + ratedBody, labels))).toBe(false);
}
expect(opts.isLastStep0AUQ(labeled('unrated-focus', 'D1 — Review focus. Project/branch/task: main. ELI10: I rated this plan 4/7 on design completeness.',
['All 7 passes (recommended)', 'States + hierarchy only']))).toBe(false);
const chosenFocus = mode.startsWith('native-') ? nativeFocus : mode === 'paraphrase' ? paraphrase : fp('focus', focus);
const pendingCall = {...chosenFocus.nativeCall, answered: false, unansweredQuestionIndices: [0]};
const pending = nativePlanCallFingerprint(pendingCall, 1000, true);
@@ -240,6 +260,9 @@ mock.module(${JSON.stringify(path.join(ROOT, 'test/helpers/claude-pty-runner.ts'
: mode === 'captured-rated' ? capturedRated
: mode === 'captured-rated-no-focus' ? capturedRated.filter((_, index) => index !== 1)
: mode === 'captured-no-focus' ? captured.filter((_, index) => index !== 1)
: mode === 'captured-39298269902' ? captured39298269902[0]
: mode === 'captured-39298269902-no-focus' ? captured39298269902[0].filter((_, index) => index !== 1)
: mode === 'captured-39298269902-focus-only' ? captured39298269902[1]
: mode === 'captured-focus-after-finding' ? [...captured.filter((_, index) => index !== 1), captured[1]]
: mode.startsWith('target-menu') ? [target, fp('other', 'Which artifact should I inspect?', false)]
: mode === 'early-exit' ? [] : mode === 'focus-only' ? [chosenFocus]
@@ -284,12 +307,12 @@ await import(${JSON.stringify(path.join(ROOT, 'test/skill-e2e-plan-design-with-u
} finally { fs.rmSync(directory, { recursive: true, force: true }); }
}
test.each(['source', 'paraphrase', 'native-sequence', 'native-unnumbered', 'native-passes', 'captured', 'captured-rated'])('UI gate seeds the exact target and accepts Design progress with an unselected no-UI alternative (%s)', mode => {
test.each(['source', 'paraphrase', 'native-sequence', 'native-unnumbered', 'native-passes', 'captured', 'captured-rated', 'captured-39298269902'])('UI gate seeds the exact target and accepts Design progress with an unselected no-UI alternative (%s)', mode => {
const result = exercise(mode);
expect(result.code, result.output).toBe(0);
}, 20_000);
test.each(['target-menu', 'target-menu-design-system', 'focus-only', 'captured-no-focus', 'captured-focus-after-finding', 'captured-rated-no-focus', 'native-setup-only', 'native-voices-only', 'early-exit', 'timeout', 'exited', 'unanswered-finding', 'failed-finding', 'no-native-finding'])('UI gate rejects %s and removes its fixture', mode => {
test.each(['target-menu', 'target-menu-design-system', 'focus-only', 'captured-no-focus', 'captured-focus-after-finding', 'captured-rated-no-focus', 'captured-39298269902-no-focus', 'captured-39298269902-focus-only', 'native-setup-only', 'native-voices-only', 'early-exit', 'timeout', 'exited', 'unanswered-finding', 'failed-finding', 'no-native-finding'])('UI gate rejects %s and removes its fixture', mode => {
const result = exercise(mode);
expect(result.code, result.output).toBe(1);
expect(result.output).toContain('plan-design-review with UI scope FAILED');
+28 -1
View File
@@ -7,7 +7,7 @@ import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import {
collectRecovery, ghFailureReason, readCheckpoint, recoveryDecision, storeRunId,
acceptedScreenRun, collectRecovery, ghFailureReason, listGateRuns, readCheckpoint, recoveryDecision, screenRunPr, storeRunId,
type GhRunner, type WorkflowRun,
} from '../scripts/recover-receipts';
import { selectPlanReceipts, writeNegativeReceipt, writePanelReceipt } from '../scripts/e2e-shard-reuse';
@@ -196,3 +196,30 @@ describe('dispatch reads PR receipts read-only (ENG-8)', () => {
expect(gh.calls.filter(c => c[0] === 'run').map(c => c[2])).toEqual(['300']);
});
});
describe('dispatched screens (GSTA-49 T1b): listed beside the PR\'s pull_request runs, filtered by PR and repository', () => {
const screen = (id: number, pr = 42, over: Partial<WorkflowRun> = {}) => run(id, 'completed', 'success', { event: 'workflow_dispatch', pull_requests: [],
display_title: `E2E Evals [eval-bg pr.screen.narrow-${'c'.repeat(12)}-${'d'.repeat(12)}-abc123] screen #${pr}`, ...over });
test('the run name names the PR only for a dispatched screen', () => {
expect(screenRunPr(screen(1))).toBe(42);
expect(screenRunPr({ event: 'pull_request', display_title: screen(1).display_title })).toBeNull();
expect(screenRunPr({ event: 'workflow_dispatch', display_title: 'E2E Evals [eval-bg gate.train-x]' })).toBeNull();
expect(screenRunPr({ event: 'workflow_dispatch', display_title: `${screen(1).display_title} extra` })).toBeNull();
expect(acceptedScreenRun(screen(1, 42, { head_repository: { full_name: 'fork/gstack' } }), { pr: 42, repo: REPO, selfRunId: 9 })).toBe(false);
expect(acceptedScreenRun(screen(9), { pr: 42, repo: REPO, selfRunId: 9 })).toBe(false);
});
test('a screen lists both events; a pull_request run lists only its own event, exactly as before', () => {
const calls: string[][] = [];
const runs = [run(10, 'completed', 'success'), run(11, 'completed', 'success', { pull_requests: [{ number: 43 }] }), screen(12), screen(13, 43), run(14, 'completed', 'success', { event: 'workflow_dispatch' })];
const gh: GhRunner = (args) => { calls.push(args); return { status: 0, stdout: JSON.stringify({ workflow_runs: runs }), stderr: '' }; };
const o = { repo: REPO, pr: 42, headRepo: REPO, branch: 'feature', selfRunId: 99 };
expect((listGateRuns(gh, { ...o, screens: false }, () => 1000) as { runs: WorkflowRun[] }).runs.map(r => r.id)).toEqual([10]);
expect(calls).toEqual([['api', '-X', 'GET', `repos/${REPO}/actions/workflows/evals.yml/runs`, '-f', 'event=pull_request', '-f', 'branch=feature', '-f', 'per_page=50']]);
expect((listGateRuns(gh, { ...o, screens: true }, () => 1000) as { runs: WorkflowRun[] }).runs.map(r => r.id)).toEqual([10, 12]);
expect(calls.at(-1)).toEqual(['api', '-X', 'GET', `repos/${REPO}/actions/workflows/evals.yml/runs`, '-f', 'event=workflow_dispatch', '-f', 'per_page=50']);
const down: GhRunner = (args, timeoutMs) => args.includes('event=workflow_dispatch') ? { status: 1, stdout: '', stderr: 'HTTP 502' } : gh(args, timeoutMs);
expect(listGateRuns(down, { ...o, screens: true }, () => 1000)).toHaveProperty('error');
});
});
+8 -2
View File
@@ -24,7 +24,7 @@ const ROOT = path.resolve(import.meta.dir, '..');
const FIXTURE = path.join(ROOT, 'test', 'fixtures', 'plans', 'ui-heavy-feature.md');
const designFocusBoundary = (fp: AskUserQuestionFingerprint): boolean =>
fp.nativeCall?.answered === true && !fp.nativeCall.failed && fp.nativeCall.questions.some(({ question }) => {
fp.nativeCall?.answered === true && !fp.nativeCall.failed && fp.nativeCall.questions.some(({ question, options }) => {
const text = question.trim().replace(/^D\d+(?:\.\d+)?\s*[—–:-]\s*/i, '');
// A target menu can mention a design system without reviewing this plan.
// The source Step 0D question, or a native title (first line) that offers all
@@ -33,9 +33,15 @@ const designFocusBoundary = (fp: AskUserQuestionFingerprint): boolean =>
// body carries the step's own design-completeness rating (run 39199692993).
const title = text.split(/\r?\n/, 1)[0]!;
const rated = /\bI(?:['’]ve| have)? rated this plan (?:10(?:\.0+)?|[0-9](?:\.\d+)?)\/10 on design completeness\b/i;
// Runs 39298269902 a1/a2: a title that only names the focus ("Review focus.",
// on one line before Project/branch/task:), with the step's own rating in the
// body and an option that offers all seven dimensions against a narrower one.
const lead = text.split(/\r?\n|\s+(?=Project\/branch\/task:)/, 1)[0]!;
return /^I(?:['’]ve| have) rated this plan (?:10(?:\.0+)?|[0-9](?:\.\d+)?)\/10 on design completeness\.[\s\S]*\bWant me to focus on specific areas instead(?: of all (?:7|seven))?\?/i.test(text)
|| (/\ball (?:7|seven) (?:design )?(?:dimensions|passes)\b/i.test(title) && /\bfocus\b/i.test(title)
&& (/\bdesign\b/i.test(title) || rated.test(text.slice(title.length))) && /\?\s*$/.test(title));
&& (/\bdesign\b/i.test(title) || rated.test(text.slice(title.length))) && /\?\s*$/.test(title))
|| (/\bfocus\b/i.test(lead) && rated.test(text) && options.length >= 2
&& options.some(({ label }) => /^\s*all (?:7|seven) (?:design |review )?(?:dimensions|passes)\b/i.test(label)));
});
// Require a choice about the supplied UI, not a workflow offer after focus.
+208
View File
@@ -0,0 +1,208 @@
/**
* The main backstop (scripts/train-backstop.ts, evals-main-backstop.yml; plan
* GSTA-49 T1b): inert without a train block; a train's merges are covered by
* the train's own green record (a five-PR train fires zero backstop gates); a
* merge outside the queue is gated, and a red gate writes the freeze record
* gstack-ship-queue reads. Fixture repos with a bare origin under os.tmpdir()
* and a scripted gh; no network, no API spend.
*/
import { afterAll, afterEach, beforeEach, describe, expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { loadPolicyFromHead } from '../lib/ship-policy';
import { NOTES_REF, persist, repoIdentity } from '../lib/ship-queue-train-record';
import type { GhRunner } from '../scripts/recover-receipts';
import { runBackstop } from '../scripts/train-backstop';
import { gitIn, trainPrefixes, type TrainRecord } from '../scripts/train-gate';
import { E2E_TOUCHFILES } from './helpers/touchfiles-data';
import { BASE_POLICY, cleanup, makeFixtureRepo, type FixtureRepo } from './helpers/restamp-fixture';
import { TRAIN_BLOCK, TRAIN_POLICY } from './helpers/ship-queue-train-fixture';
const REPO = 'fixture/repo';
const TRAIN_NONCE = 'gate.train-' + 'a'.repeat(12) + '-' + 'b'.repeat(12) + '-abc123';
const RED_CASE = Object.keys(E2E_TOUCHFILES)[0]!;
const repos: FixtureRepo[] = [];
afterAll(() => { for (const repo of repos) cleanup(repo); });
const savedStateRoot = process.env.GSTACK_STATE_ROOT;
afterEach(() => { if (savedStateRoot === undefined) delete process.env.GSTACK_STATE_ROOT; else process.env.GSTACK_STATE_ROOT = savedStateRoot; });
interface Fixture { repo: FixtureRepo; main0: string; prs: string[]; train: string[]; record: TrainRecord }
/** Five PR branches from main and the train T1..T5 gstack-ship-queue would build from them (lib/ship-queue-train.ts buildChain). */
function trainFixture(policy: Record<string, unknown> = TRAIN_POLICY): Fixture {
const repo = makeFixtureRepo({ policy, prefix: 'train-backstop-fx-' });
repos.push(repo);
process.env.GSTACK_STATE_ROOT = path.join(repo.root, 'state');
const main0 = repo.git('rev-parse', 'HEAD');
const prs = [1, 2, 3, 4, 5].map(i => {
repo.git('checkout', '-q', '-B', `pr${i}`, main0);
fs.writeFileSync(path.join(repo.work, `pr${i}.txt`), `pr ${i}\n`);
repo.git('add', '-A');
repo.git('commit', '-q', '-m', `pr ${i}`);
return repo.git('rev-parse', 'HEAD');
});
const train: string[] = [];
let parent = main0;
prs.forEach((head, i) => {
const tree = repo.git('merge-tree', '--write-tree', parent, head);
parent = repo.git('commit-tree', tree, '-p', parent, '-p', head, '-m', `train T${i + 1}: merge #${101 + i} at ${head.slice(0, 12)}`);
train.push(parent);
});
repo.git('checkout', '-q', 'main');
const prefixes = trainPrefixes(gitIn(repo.work), parent, repo.work, loadPolicyFromHead(repo.work).policy);
if (!Array.isArray(prefixes)) throw new Error(prefixes.error);
const record: TrainRecord = { schema: 1, kind: 'train-gate', nonce: TRAIN_NONCE, head_sha: parent, tree: repo.git('rev-parse', `${parent}^{tree}`), verdict: 'GREEN',
census: { complete: true, planned: { rule: 108, judge: 24 }, executed: { rule: 108, judge: 24 }, unbound: 0, partial_shards: 0, cancelled_shards: 0, killed_before_upload: 0 },
panels: [], run: { id: 900, attempt: 1, url: 'https://example.invalid/runs/900' }, prefixes };
return { repo, main0, prs, train, record };
}
/** The coordinator's squash merge of PR i onto main, stamped at merge (VERSION, package.json, the release output). */
function squashMerge(repo: FixtureRepo, i: number, extra: Record<string, string> = {}): string {
repo.git('checkout', '-q', 'main');
const version = `1.2.${3 + i}.0`;
const files: Record<string, string> = { [`pr${i}.txt`]: `pr ${i}\n`, VERSION: version + '\n', 'generated/digest.md': `# digest v${version}\nbody\n`, ...extra };
for (const [rel, text] of Object.entries(files)) fs.writeFileSync(path.join(repo.work, rel), text);
const pkg = JSON.parse(fs.readFileSync(path.join(repo.work, 'package.json'), 'utf8'));
fs.writeFileSync(path.join(repo.work, 'package.json'), JSON.stringify({ ...pkg, version: version.split('.').slice(0, 3).join('.') }, null, 2) + '\n');
repo.git('add', '-A');
repo.git('commit', '-q', '-m', `#${100 + i} (squash, stamped ${version})`);
repo.git('push', '-q', 'origin', 'main');
return repo.git('rev-parse', 'HEAD');
}
/** gh: the train's green run, its record artifact and check run, a gate dispatch whose run concludes as scripted, and an issue. */
function fakeGh(fx: Fixture, o: { record?: TrainRecord | null; check?: string; gate?: string } = {}) {
const calls: string[][] = [];
let dispatched = '';
const ok = (body: unknown) => ({ status: 0, stdout: typeof body === 'string' ? body : JSON.stringify(body), stderr: '' });
const gh: GhRunner = (args) => {
calls.push(args);
const joined = args.join(' ');
if (args[0] === 'api' && joined.includes('/actions/workflows/evals.yml/runs') && joined.includes('status=success')) {
return ok({ workflow_runs: o.record === null ? [] : [{ id: 900, status: 'completed', conclusion: 'success', head_sha: fx.record.head_sha, display_title: `E2E Evals [eval-bg ${TRAIN_NONCE}]` }] });
}
if (args[0] === 'api' && joined.includes('/actions/workflows/evals.yml/runs')) {
return ok({ workflow_runs: dispatched ? [{ id: 901, status: 'queued', conclusion: null, head_sha: '', display_title: `E2E Evals [eval-bg ${dispatched}]`, html_url: 'https://example.invalid/runs/901' }] : [] });
}
if (args[0] === 'run' && args[1] === 'download') {
const dir = args[args.indexOf('-D') + 1]!;
const name = args[args.indexOf('-n') + 1];
fs.mkdirSync(dir, { recursive: true });
if (name === TRAIN_BLOCK.record_artifact) fs.writeFileSync(path.join(dir, 'record.json'), JSON.stringify(o.record ?? fx.record));
if (name === 'trial-outcomes-pr-a1') fs.writeFileSync(path.join(dir, 'trial-outcomes.jsonl'), JSON.stringify({ case: RED_CASE, attempt: 1, outcome: 'failed' }) + '\n');
return ok('');
}
if (args[0] === 'api' && /\/commits\/[0-9a-f]{40}\/check-runs/.test(joined)) return ok({ check_runs: [{ name: TRAIN_BLOCK.full_gate_check, conclusion: o.check ?? 'success' }] });
if (args[0] === 'workflow' && args[1] === 'run') {
dispatched = args.find(a => a.startsWith('nonce='))!.slice('nonce='.length);
return ok('');
}
if (args[0] === 'api' && /actions\/runs\/901$/.test(joined)) return ok({ id: 901, status: 'completed', conclusion: o.gate ?? 'failure', head_sha: '', html_url: 'https://example.invalid/runs/901' });
if (args[0] === 'issue' && args[1] === 'create') return ok('https://github.com/fixture/repo/issues/7\n');
return { status: 1, stdout: '', stderr: `unexpected gh ${joined}` };
};
return { gh, calls, dispatches: () => calls.filter(c => c[0] === 'workflow' && c[1] === 'run') };
}
const backstop = (fx: Fixture, sha: string, gh: GhRunner, over: Partial<Parameters<typeof runBackstop>[0]> = {}) =>
runBackstop({ repo: REPO, branch: 'main', sha, repoRoot: fx.repo.work, gh, sleep: () => {}, pollMs: 1, random: 'abc123', ...over });
function originNotes(fx: Fixture): Array<Record<string, unknown>> {
const list = fx.repo.git('--git-dir', fx.repo.origin, 'notes', `--ref=${NOTES_REF}`, 'list');
return list.split('\n').filter(Boolean).map(line => JSON.parse(fx.repo.git('--git-dir', fx.repo.origin, 'cat-file', 'blob', line.split(' ')[0]!)));
}
describe('backstop: inert without a train block', () => {
test('a push to main under a policy with no train block runs nothing and calls no gh', () => {
const fx = trainFixture(BASE_POLICY);
const { gh, calls } = fakeGh(fx);
const out = backstop(fx, squashMerge(fx.repo, 1), gh);
expect(out.action).toBe('inert');
expect(out.lines[0]).toContain('declares no train block');
expect(calls).toEqual([]);
});
});
describe("backstop: a train's merges are covered by the train's own green record", () => {
test('a five-PR train fires zero backstop gates', () => {
const fx = trainFixture();
const { gh, dispatches } = fakeGh(fx);
for (let i = 1; i <= 5; i++) {
const out = backstop(fx, squashMerge(fx.repo, i), gh);
expect(out.action, out.lines.join('\n')).toBe('covered');
expect(out.lines.at(-1)).toContain(`is T${i} of green train ${TRAIN_NONCE} (run 900`);
}
expect(dispatches()).toEqual([]);
});
test('a superseded push hands main to the newer push without a gh call', () => {
const fx = trainFixture();
const first = squashMerge(fx.repo, 1);
squashMerge(fx.repo, 2);
const { gh, calls } = fakeGh(fx);
expect(backstop(fx, first, gh).action).toBe('superseded');
expect(calls).toEqual([]);
});
test('a record without a successful check run, or not GREEN, covers nothing: the backstop gates main', () => {
const fx = trainFixture();
const sha = squashMerge(fx.repo, 1);
for (const o of [{ check: 'failure' }, { record: { ...fx.record, verdict: 'RED' as const } }, { record: { ...fx.record, census: { ...fx.record.census, complete: false } } }]) {
const { gh, dispatches } = fakeGh(fx, { ...o, gate: 'success' });
const out = backstop(fx, sha, gh);
expect(out.action, JSON.stringify(o)).toBe('gated');
expect(dispatches()).toHaveLength(1);
expect(dispatches()[0]).toEqual(expect.arrayContaining(['evals.yml', '--ref', 'main', 'evals_all=true', `expected_sha=${sha}`]));
}
});
});
describe('backstop: a merge outside the queue', () => {
test('is gated; a red gate writes the freeze record with its red cases, inputs, run and issue; the next run on that commit defers to it', () => {
const fx = trainFixture();
squashMerge(fx.repo, 1);
const outside = squashMerge(fx.repo, 6, { 'outside.txt': 'merged outside the queue\n' });
const { gh, dispatches } = fakeGh(fx);
const out = backstop(fx, outside, gh);
expect(out.action, out.lines.join('\n')).toBe('frozen');
expect(dispatches()).toHaveLength(1);
const nonce = dispatches()[0]!.find(a => a.startsWith('nonce='))!.slice('nonce='.length);
expect(nonce).toStartWith(`gate.backstop-${outside.slice(0, 12)}-${outside.slice(0, 12)}-`);
const freeze = originNotes(fx).find(r => r.kind === 'freeze')!;
expect(freeze).toMatchObject({ schema: 1, kind: 'freeze', repo_key: repoIdentity(fx.repo.work).key, train: 'backstop', cleared_at: null,
main: { sha: outside }, backstop: { nonce, run: 901, state: 'completed/failure' }, issue: 'https://github.com/fixture/repo/issues/7',
red_cases: [{ id: RED_CASE, inputs: E2E_TOUCHFILES[RED_CASE] }] });
expect(String(freeze.reason)).toContain(`${RED_CASE} (inputs `);
expect(String(freeze.reason)).toContain('fix: land the revert or fix-forward');
const again = fakeGh(fx);
expect(backstop(fx, outside, again.gh).action).toBe('deferred');
expect(again.dispatches()).toEqual([]);
});
test('a green gate on main writes nothing', () => {
const fx = trainFixture();
const outside = squashMerge(fx.repo, 6, { 'outside.txt': 'merged outside the queue\n' });
const { gh, dispatches } = fakeGh(fx, { record: null, gate: 'success' });
expect(backstop(fx, outside, gh).action).toBe('gated');
expect(dispatches()).toHaveLength(1);
expect(originNotes(fx)).toEqual([]);
});
});
describe('backstop: debounce while a train merges', () => {
let fx: Fixture;
beforeEach(() => { fx = trainFixture(); });
test('waits while a train is merging, then decides once the settle window ends', () => {
const at = new Date().toISOString();
persist(fx.repo.work, [{ schema: 1, kind: 'train', nonce: TRAIN_NONCE, repo_key: repoIdentity(fx.repo.work).key, state: 'merging', updated_at: at } as never]);
const sha = squashMerge(fx.repo, 1);
let clock = 0;
const { gh, dispatches } = fakeGh(fx);
const out = backstop(fx, sha, gh, { now: () => (clock += 20 * 60_000), settleMs: 60 * 60_000 });
expect(out.lines.some(line => line.includes(`waiting for train(s) ${TRAIN_NONCE} to finish merging`))).toBe(true);
expect(out.lines.some(line => line.includes('still merging after the settle window'))).toBe(true);
expect(out.action).toBe('covered');
expect(dispatches()).toEqual([]);
});
});
+250
View File
@@ -0,0 +1,250 @@
/**
* The merge-train side of evals.yml (scripts/train-gate.ts; plan GSTA-49 T1b):
* dispatch routing and its refusals, the green-train record (census from the
* run's own plan manifest, first attempt only), prefix trees modulo stamps,
* the per-prefix bisect record and the free screening summary. Fixture repos
* under os.tmpdir() with their own git identity; no network, no API spend.
*/
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { classifyGate } from '../lib/ship-queue-train-verdict';
import { buildRunManifest, type PaidRunManifest } from '../scripts/lib/paid-plan';
import { buildReceipt } from '../lib/tree-receipt';
import {
FULL_GATE_USD, JUDGE_FILE, MAX_BISECT_CASES, MAX_PREFIXES, bisectMatrix, buildTrainRecord, fullGateCases, gitIn, routeDispatch, runBisectPrefix,
screenSummary, stampFreeTree, trainPrefixes, type BisectPlan, type RouteInput,
} from '../scripts/train-gate';
import { loadPolicyFromHead } from '../lib/ship-policy';
import { LLM_JUDGE_TOUCHFILES } from './helpers/touchfiles-data';
import { cleanup, makeFixtureRepo, type FixtureRepo } from './helpers/restamp-fixture';
import { TRAIN_POLICY } from './helpers/ship-queue-train-fixture';
const scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'train-gate-'));
afterAll(() => fs.rmSync(scratch, { recursive: true, force: true }));
const A = 'a'.repeat(40);
const B = 'b'.repeat(40);
const C = 'c'.repeat(40);
describe('routeDispatch: every run keeps its mode, and train inputs are validated before any paid work', () => {
const base: RouteInput = { nonce: '', refName: 'main', head: A, expectedSha: '', evalsAll: true, trainPolicy: false, cases: '', prefixShas: '',
screenPr: '', screenMergeSha: '', screenPolicy: '' };
const deps = { isAncestor: (commit: string) => commit !== C, gateCases: () => new Map([['case-a', 'rule'], ['review/SKILL.md workflow', 'judge']]) };
const route = (over: Partial<RouteInput>) => routeDispatch({ ...base, ...over }, deps);
const screenNonce = (policy: string) => `pr.screen.${policy}-${A.slice(0, 12)}-${B.slice(0, 12)}-abc123`;
const bisectNonce = `gate.bisect-${A.slice(0, 12)}-${B.slice(0, 12)}-abc123`;
const trainNonce = `gate.train-${A.slice(0, 12)}-${B.slice(0, 12)}-abc123`;
test('a plain eval-bg or backstop dispatch routes as today, with or without a train policy', () => {
expect(route({})).toEqual({ mode: 'dispatch', cases: [], prefixes: [] });
expect(route({ nonce: `pr-${A.slice(0, 12)}-${B.slice(0, 12)}-abc123`, evalsAll: false })).toMatchObject({ mode: 'dispatch' });
expect(route({ nonce: `gate.backstop-${A.slice(0, 12)}-${A.slice(0, 12)}-abc123`, trainPolicy: true })).toMatchObject({ mode: 'dispatch' });
});
test('a screen needs a PR number, a full merge SHA, a policy, evals_all=false, a matching nonce and a train policy', () => {
const screen = { screenPr: '42', screenMergeSha: C, screenPolicy: 'narrow', evalsAll: false, expectedSha: C, nonce: screenNonce('narrow'), trainPolicy: true };
expect(route(screen)).toEqual({ mode: 'screen', cases: [], prefixes: [] });
expect(route({ ...screen, expectedSha: '' })).toMatchObject({ mode: 'screen' });
const refused = [
{ screenPr: '0' }, { screenPr: '42; rm -rf' }, { screenMergeSha: C.slice(0, 12) }, { screenPolicy: 'wide' }, { evalsAll: true }, { expectedSha: A },
{ nonce: screenNonce('broad') }, { nonce: bisectNonce }, { cases: 'case-a' }, { trainPolicy: false },
];
for (const over of refused) expect(route({ ...screen, ...over }), JSON.stringify(over)).toHaveProperty('error');
expect((route({ ...screen, trainPolicy: false }) as { error: string }).error).toContain('needs a train block');
});
test('a bisect needs a bisect nonce on train/<nonce>, the train head, plan case ids and ancestor prefixes', () => {
const bisect = { nonce: bisectNonce, refName: 'train/t1', expectedSha: A, cases: 'case-a, review/SKILL.md workflow', prefixShas: `${A},${B}`, trainPolicy: true };
expect(route(bisect)).toEqual({ mode: 'bisect', cases: ['case-a', 'review/SKILL.md workflow'], prefixes: [A, B] });
const many = (n: number, f: (i: number) => string) => Array.from({ length: n }, (_, i) => f(i)).join(',');
const refused: Array<[Partial<RouteInput>, string]> = [
[{ nonce: trainNonce }, 'needs a bisect nonce'],
[{ refName: 'main' }, 'runs on train/<nonce>'],
[{ expectedSha: B }, 'is not the train head'],
[{ cases: '' }, 'cases must name'],
[{ cases: 'case-a,case-a' }, 'cases must name'],
[{ cases: many(MAX_BISECT_CASES + 1, i => `case-${i}`) }, 'cases must name'],
[{ prefixShas: many(MAX_PREFIXES + 1, i => i.toString(16).padStart(40, '0')) }, 'prefix_shas must name'],
[{ prefixShas: A.slice(0, 12) }, 'is not a full commit SHA'],
[{ prefixShas: `${A},${C}` }, 'is not an ancestor of the train head'],
[{ cases: 'case-a,not-a-plan-case' }, "not in the full gate's plan"],
[{ trainPolicy: false }, 'needs a train block'],
];
for (const [over, why] of refused) expect((route({ ...bisect, ...over }) as { error?: string }).error, why).toContain(why);
expect((route({ cases: 'case-a', prefixShas: A }) as { error?: string }).error).toContain('needs a bisect nonce');
});
test('a train gate is the full gate of the train head on train/<nonce>, under a train policy', () => {
const train = { nonce: trainNonce, refName: 'train/t1', expectedSha: A, trainPolicy: true };
expect(route(train)).toEqual({ mode: 'train', cases: [], prefixes: [] });
for (const over of [{ refName: 'main' }, { evalsAll: false }, { expectedSha: '' }, { trainPolicy: false }]) expect(route({ ...train, ...over })).toHaveProperty('error');
});
test("the bisect's case universe is the full gate's own plan on this checkout", () => {
const cases = fullGateCases();
expect([...cases.values()].filter(kind => kind === 'judge').length).toBe(Object.keys(LLM_JUDGE_TOUCHFILES).length);
expect([...cases.values()].filter(kind => kind !== 'judge').length).toBeGreaterThan(50);
});
});
describe('train record: the census comes from the run\'s own manifest, first attempt only', () => {
const [j1, j2] = Object.keys(LLM_JUDGE_TOUCHFILES) as [string, string];
const behavior = 'shared-libs-opportunity-judgment';
const judgeEntry = buildRunManifest({ tier: 'gate', profile: 'full', sliceCount: 1, evalsAll: true, env: { EVALS_ALL: '1' } }).entries.find(e => e.file === JUDGE_FILE)!;
const trialFiles = [1, 2, 3].map(t => `test/skill-e2e-shared-libs-periodic.test.ts#${behavior}~t${t}`);
const manifest = { version: 1, tier: 'gate', evalsAll: true, sliceCount: 2, selectionReason: 'evals_all', profile: 'full', selection: { e2e: null, judges: [j1, j2] },
entries: [
{ ...judgeEntry, slice: 1 },
...trialFiles.map(file => ({ file, slice: 2, status: 'planned', trial: { kind: 'behavior', panel: { n: 3, k: 2 }, quarantined: false } })),
] } as unknown as PaidRunManifest;
type Slice = { index: number; attempt?: number; files?: string[]; checkpoint?: true };
let n = 0;
function report(o: { slices?: Slice[]; outcomes?: Array<Record<string, unknown>>; collector?: Record<string, unknown> | null } = {}): string {
const dir = path.join(scratch, `report-${++n}`);
fs.mkdirSync(dir, { recursive: true });
fs.writeFileSync(path.join(dir, 'manifest.json'), JSON.stringify(manifest));
const outcomes = o.outcomes ?? [j1, j2].map(id => ({ case: id, kind: 'judge', attempt: 1, outcome: 'passed', cost_usd: 0.25 }))
.concat([1, 2, 3].map(trial => ({ case: behavior, kind: 'behavior', trial, attempt: 1, outcome: 'passed', cost_usd: 1 })));
fs.writeFileSync(path.join(dir, 'trial-outcomes.jsonl'), outcomes.map(line => JSON.stringify(line)).join('\n') + '\n');
if (o.collector !== null) fs.writeFileSync(path.join(dir, 'collector-outcomes.json'), JSON.stringify(o.collector ?? { verdict: { verdict: 'GREEN' }, census: { unattributed: [] }, panels: [] }));
for (const s of o.slices ?? [{ index: 1 }, { index: 2 }]) {
const at = path.join(dir, `paid-slice-${s.index}-a${s.attempt ?? 1}`);
fs.mkdirSync(at, { recursive: true });
fs.writeFileSync(path.join(at, `slice-${s.index}.json`), JSON.stringify({ version: 1, tier: 'gate', sliceIndex: s.index, sliceCount: 2, attempt: s.attempt ?? 1,
...(s.checkpoint ? { checkpoint: true } : {}), outcomes: (s.files ?? (s.index === 1 ? [JUDGE_FILE] : trialFiles)).map(file => ({ files: [file], status: 'passed' })) }));
}
return dir;
}
const build = (dir: string, reconcileExit = 0) => buildTrainRecord({ reportDir: dir, nonce: 'gate.train-x', head: A, tree: B, reconcileExit,
prefixes: [{ index: 1, commit: C, tree: B, stamp_free_tree: 'd'.repeat(64) }], run: { id: 7, attempt: 1, url: 'https://example.invalid/runs/7' } });
const classify = (record: unknown) => classifyGate(record, { nonce: 'gate.train-x', head: A, tree: B }, { status: 'completed', conclusion: 'success' },
{ status: 'completed', conclusion: 'success' } as never).outcome;
test('a complete census of passing cases is GREEN, and T1 classifies it GREEN', () => {
const record = build(report());
expect(record).toMatchObject({ schema: 1, kind: 'train-gate', nonce: 'gate.train-x', head_sha: A, tree: B, verdict: 'GREEN', run: { id: 7 } });
expect(record.census).toEqual({ complete: true, planned: { rule: 1, judge: 2 }, executed: { rule: 1, judge: 2 }, unbound: 0, partial_shards: 0, cancelled_shards: 0, killed_before_upload: 0 });
expect(record.panels.map(p => [p.case, p.status, p.cost_usd])).toEqual([[j2, 'PASS', 0.25], [j1, 'PASS', 0.25], [behavior, 'PASS', 3]].sort((a, b) => (a[0]! < b[0]! ? -1 : 1)));
expect(classify(record)).toBe('GREEN');
});
test('a slice killed before its upload step is incomplete, never passed over, and a re-run attempt cannot fill it', () => {
const killed = build(report({ slices: [{ index: 1 }], outcomes: [j1, j2].map(id => ({ case: id, kind: 'judge', attempt: 1, outcome: 'passed' })) }));
expect(killed.verdict).toBe('RED');
expect(killed.census).toMatchObject({ complete: false, killed_before_upload: 1, executed: { rule: 0, judge: 2 } });
expect(classify(killed)).toBe('INCOMPLETE');
const rerun = build(report({ slices: [{ index: 1 }, { index: 2, attempt: 2 }], outcomes: [
...[j1, j2].map(id => ({ case: id, kind: 'judge', attempt: 1, outcome: 'passed' })), ...[1, 2, 3].map(trial => ({ case: behavior, kind: 'behavior', trial, attempt: 2, outcome: 'passed' }))] }));
expect(rerun.census).toMatchObject({ complete: false, killed_before_upload: 1, executed: { rule: 0, judge: 2 } });
expect(rerun.verdict).toBe('RED');
});
test('a partial or cancelled shard, a missing collector, a red reconcile or a red collector is never GREEN', () => {
expect(build(report({ slices: [{ index: 1 }, { index: 2, checkpoint: true }] })).census.partial_shards).toBe(1);
expect(build(report({ slices: [{ index: 1 }, { index: 2, files: [] }] })).census.cancelled_shards).toBe(trialFiles.length);
for (const record of [build(report({ slices: [{ index: 1 }, { index: 2, checkpoint: true }] })), build(report({ slices: [{ index: 1 }, { index: 2, files: [] }] })),
build(report({ collector: null })), build(report(), 1), build(report({ collector: { verdict: { verdict: 'RED' }, census: { unattributed: [] }, panels: [] } })),
build(report({ collector: { verdict: { verdict: 'GREEN' }, census: { unattributed: [{ file: 'x' }] }, panels: [] } }))]) {
expect(record.verdict).toBe('RED');
}
});
test('a failed case is a red VERDICT panel T1 bisects on', () => {
const record = build(report({ outcomes: [{ case: j1, kind: 'judge', attempt: 1, outcome: 'failed', failure_class: 'assertion' },
{ case: j2, kind: 'judge', attempt: 1, outcome: 'passed' }, ...[1, 2, 3].map(trial => ({ case: behavior, kind: 'behavior', trial, attempt: 1, outcome: 'passed' }))],
collector: { verdict: { verdict: 'RED' }, census: { unattributed: [] }, panels: [] } }));
expect(record.verdict).toBe('RED');
expect(record.panels.find(p => p.case === j1)).toMatchObject({ status: 'FAIL', red_class: 'VERDICT' });
expect(classify(record)).toBe('RED');
});
});
describe('prefix trees modulo stamps', () => {
let repo: FixtureRepo;
const sha: Record<string, string> = {};
beforeAll(() => {
repo = makeFixtureRepo({ policy: TRAIN_POLICY, prefix: 'train-gate-fx-' });
sha.main = repo.git('rev-parse', 'HEAD');
const branch = (name: string, file: string) => {
repo.git('checkout', '-q', '-B', name, sha.main!);
fs.writeFileSync(path.join(repo.work, file), `${name}\n`);
repo.git('add', '-A');
repo.git('commit', '-q', '-m', name);
return repo.git('rev-parse', 'HEAD');
};
sha.pr1 = branch('pr1', 'one.txt');
sha.pr2 = branch('pr2', 'two.txt');
const merge = (parent: string, head: string, index: number, pr: number) => {
const tree = repo.git('merge-tree', '--write-tree', parent, head);
return repo.git('commit-tree', tree, '-p', parent, '-p', head, '-m', `train T${index}: merge #${pr} at ${head.slice(0, 12)}`);
};
sha.t1 = merge(sha.main!, sha.pr1!, 1, 11);
sha.t2 = merge(sha.t1!, sha.pr2!, 2, 12);
repo.git('checkout', '-q', '-B', 'squash', sha.main!);
fs.writeFileSync(path.join(repo.work, 'one.txt'), 'pr1\n');
fs.writeFileSync(path.join(repo.work, 'VERSION'), '1.2.4.0\n');
fs.writeFileSync(path.join(repo.work, 'generated', 'digest.md'), '# digest v1.2.4.0\nbody\n');
const pkg = JSON.parse(fs.readFileSync(path.join(repo.work, 'package.json'), 'utf8'));
fs.writeFileSync(path.join(repo.work, 'package.json'), JSON.stringify({ ...pkg, version: '1.2.4' }, null, 2) + '\n');
repo.git('add', '-A');
repo.git('commit', '-q', '-m', 'squash #11 (stamped)');
sha.squash1 = repo.git('rev-parse', 'HEAD');
fs.writeFileSync(path.join(repo.work, 'one.txt'), 'pr1, edited\n');
repo.git('commit', '-qam', 'a non-stamp edit');
sha.edited = repo.git('rev-parse', 'HEAD');
});
afterAll(() => cleanup(repo));
test('a squash merge that differs from its prefix only by stamps has the same tree modulo stamps; a code byte does not', () => {
const git = gitIn(repo.work);
const policy = loadPolicyFromHead(repo.work).policy;
const t1 = stampFreeTree(git, sha.t1!, repo.work, policy);
expect(t1).toMatch(/^[0-9a-f]{64}$/);
expect(repo.git('rev-parse', `${sha.t1}^{tree}`)).not.toBe(repo.git('rev-parse', `${sha.squash1}^{tree}`));
expect(stampFreeTree(git, sha.squash1!, repo.work, policy)).toBe(t1);
expect(stampFreeTree(git, sha.edited!, repo.work, policy)).not.toBe(t1);
// The same verdicts as the tree receipt verify and gate reuse use (lib/tree-receipt.ts).
const receipt = (current: string) => (buildReceipt({ repoRoot: repo.work, gated: sha.t1!, current, policy, policyLabel: 'fixture' }) as { tree: string }).tree;
expect(receipt(sha.squash1!)).toBe('same-modulo-stamps');
expect(receipt(sha.edited!)).toBe('changed');
expect(stampFreeTree(git, 'f'.repeat(40), repo.work, policy)).toHaveProperty('error');
});
test('T1..Tk come from the train head by first parents; a head that is not a train is refused', () => {
const git = gitIn(repo.work);
const policy = loadPolicyFromHead(repo.work).policy;
const prefixes = trainPrefixes(git, sha.t2!, repo.work, policy);
expect(Array.isArray(prefixes)).toBe(true);
expect((prefixes as Array<{ index: number; commit: string }>).map(p => [p.index, p.commit])).toEqual([[1, sha.t1], [2, sha.t2]]);
expect(trainPrefixes(git, sha.main!, repo.work, policy)).toHaveProperty('error');
});
});
describe('bisect: one executor per prefix, one trial per case', () => {
const plan: BisectPlan = { schema: 1, nonce: 'gate.bisect-x', cases: [{ id: 'case-a', kind: 'rule' }, { id: 'case-b', kind: 'judge' }], prefixes: [A, B] };
test('the matrix indexes prefixes from 1 and scales its ceiling with the case count', () => {
expect(bisectMatrix(plan)).toEqual({ slices: [1, 2], timeouts: [0, 80, 80], refs: ['', A, B] });
});
test('each prefix writes the record T1 reads; a case that leaves no verdict is INCOMPLETE; a wrong checkout refuses', async () => {
const record = await runBisectPrefix(plan, 2, { rootDir: scratch, evalDir: path.join(scratch, 'bisect'), checkedOut: B,
runCase: async (id) => { if (id === 'case-b') throw new Error('runner crashed'); return { status: 'FAIL', redClass: 'VERDICT' }; } });
expect(record).toEqual({ schema: 1, kind: 'train-bisect-prefix', nonce: 'gate.bisect-x', prefix_sha: B, checked_out_sha: B, panels: [
{ case: 'case-a', kind: 'rule', status: 'FAIL', red_class: 'VERDICT', cost_usd: null },
{ case: 'case-b', kind: 'judge', status: 'INCOMPLETE', red_class: 'INCOMPLETE', cost_usd: null }] });
await expect(runBisectPrefix(plan, 1, { rootDir: scratch, evalDir: scratch, checkedOut: B, runCase: async () => ({ status: 'PASS', redClass: null }) })).rejects.toThrow('not its prefix');
await expect(runBisectPrefix(plan, 3, { rootDir: scratch, evalDir: scratch, checkedOut: B })).rejects.toThrow('names no prefix');
});
});
test('the free screening summary says first how to request the screen, then the selection and its estimated cost', () => {
const [j1] = Object.keys(LLM_JUDGE_TOUCHFILES) as [string];
const manifest = { version: 1, tier: 'gate', evalsAll: false, sliceCount: 1, profile: 'pr', selection: { e2e: [], judges: [j1] }, prCoverage: { mode: 'narrow' },
entries: [{ file: JUDGE_FILE, slice: 1, status: 'planned' }] } as unknown as PaidRunManifest;
const lines = screenSummary(manifest, '42');
expect(lines[0]).toStartWith('To run the paid screen of this head: `gstack-ship-queue ready --pr 42 --screen`');
expect(lines[1]).toBe('- Screening selection: 0 case(s) + 1 judge(s), PR coverage mode `narrow`');
expect(lines[2]).toMatch(new RegExp(`^- Estimated screen cost: ~\\$\\d+\\.\\d{2} .*\\$${FULL_GATE_USD} full gate.* No paid case ran on this push\\.$`));
});