mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-05 08:23:59 +01:00
feat(evals): merge-train workflow side, inert behind the train policy block (GSTA-49 T1b)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
a844821583
commit
8128af38ec
16 files changed
+2265
-87
No files matched your search
@@ -0,0 +1,65 @@
|
||||
name: Main backstop gate
|
||||
# GSTA-49 T1b (docs/ship-policy.md#run-a-merge-train). Every push to main is
|
||||
# covered by a full gate. Under merge trains a train's merges are covered by
|
||||
# the gate the train already ran on the same trees, so this workflow gates
|
||||
# main only when no green-train record lists main's tree modulo stamps (a
|
||||
# merge outside the queue), after any train has finished merging. A red
|
||||
# backstop writes the freeze record gstack-ship-queue reads. Inert (no paid
|
||||
# work, nothing written) unless main's .gstack/ship-policy.json declares a
|
||||
# `train` block; without one the per-PR gates cover main as before.
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
# One backstop at a time; a newer push waits and the script hands a superseded
|
||||
# push to it (only the newest pending run is kept).
|
||||
concurrency:
|
||||
group: evals-main-backstop
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
backstop:
|
||||
runs-on: ubuntu-24.04
|
||||
if: github.actor != 'dependabot[bot]'
|
||||
# Settle window (60) + the full gate (270) + margin.
|
||||
timeout-minutes: 345
|
||||
permissions:
|
||||
# Pushes the freeze record to refs/notes/gstack-trains.
|
||||
contents: write
|
||||
# Dispatches the gate and reads train runs and their artifacts.
|
||||
actions: write
|
||||
checks: read
|
||||
# Opens the issue a red backstop names.
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# Read with git and jq only: without a train block nothing below runs.
|
||||
- name: Read the train policy (main)
|
||||
id: policy
|
||||
run: |
|
||||
train=$(git show "${GITHUB_SHA}:.gstack/ship-policy.json" 2>/dev/null | jq -c 'if (.train | type) == "object" then .train else null end' 2>/dev/null || true)
|
||||
if [ -z "$train" ] || [ "$train" = null ]; then
|
||||
echo "train=false" >> "$GITHUB_OUTPUT"
|
||||
echo "- Backstop inert: main's .gstack/ship-policy.json declares no train block; per-PR gates cover main" >> "$GITHUB_STEP_SUMMARY"
|
||||
else
|
||||
echo "train=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
if: steps.policy.outputs.train == 'true'
|
||||
with:
|
||||
bun-version: 1.4.2
|
||||
|
||||
- name: Gate main unless a green train covers its tree
|
||||
if: steps.policy.outputs.train == 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
GSTACK_STATE_ROOT: ${{ runner.temp }}/gstack-state
|
||||
run: |
|
||||
bun --no-install run scripts/train-backstop.ts run --repo "$GITHUB_REPOSITORY" --branch "$GITHUB_REF_NAME" --sha "$GITHUB_SHA" \
|
||||
--summary "$GITHUB_STEP_SUMMARY"
|
||||
+480
-27
@@ -1,6 +1,8 @@
|
||||
name: E2E Evals
|
||||
# eval-bg (scripts/eval-bg.ts) finds its dispatch by the nonce in the run name (ENG-12).
|
||||
run-name: ${{ inputs.nonce != '' && format('E2E Evals [eval-bg {0}]', inputs.nonce) || '' }}
|
||||
# A dispatched screen's name also ends ` screen #<pr>`, which recover-receipts and
|
||||
# gate-reuse filter on to list a PR's screens beside its pull_request runs.
|
||||
run-name: "${{ inputs.nonce != '' && format('E2E Evals [eval-bg {0}]{1}', inputs.nonce, inputs.screen_pr != '' && format(' screen #{0}', inputs.screen_pr) || '') || '' }}"
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
@@ -30,9 +32,33 @@ on:
|
||||
description: 'Same-repo PR number whose receipt store this run reads read-only (never saves); empty = fresh'
|
||||
type: string
|
||||
default: ''
|
||||
# Merge trains (docs/ship-policy.md#run-a-merge-train): only under a train
|
||||
# block in the default branch's .gstack/ship-policy.json. The route job
|
||||
# validates them before any paid work (scripts/train-gate.ts route).
|
||||
cases:
|
||||
description: 'Train bisect: comma-separated case ids from the full gate plan (gstack-ship-queue train bisect sets it)'
|
||||
type: string
|
||||
default: ''
|
||||
prefix_shas:
|
||||
description: 'Train bisect: comma-separated prefix commits, each an ancestor of the train head on train/<nonce>'
|
||||
type: string
|
||||
default: ''
|
||||
screen_pr:
|
||||
description: 'Dispatched screen: the same-repo PR number (gstack-ship-queue ready --screen sets it)'
|
||||
type: string
|
||||
default: ''
|
||||
screen_merge_sha:
|
||||
description: "Dispatched screen: the PR's refs/pull/<n>/merge commit to check out and test"
|
||||
type: string
|
||||
default: ''
|
||||
screen_policy:
|
||||
description: 'Dispatched screen: narrow or broad (a broad screen never adopts a narrow run)'
|
||||
type: string
|
||||
default: ''
|
||||
|
||||
# A dispatched screen is keyed on its PR, so a newer head's screen cancels the older one.
|
||||
concurrency:
|
||||
group: evals-${{ github.event.pull_request.number || github.run_id }}
|
||||
group: evals-${{ github.event.pull_request.number || (inputs.screen_pr != '' && format('screen-{0}', inputs.screen_pr)) || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
@@ -120,6 +146,129 @@ jobs:
|
||||
docker manifest inspect "$EVAL_IMAGE" > /tmp/eval-runtime-manifest.json
|
||||
echo "id=$(sha256sum /tmp/eval-runtime-manifest.json | cut -d ' ' -f1)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# ── Route (merge trains, GSTA-49 T1b; docs/ship-policy.md#run-a-merge-train)
|
||||
# Names the run's mode before any paid work, from the train policy of the
|
||||
# base revision (the PR's base commit, or the default branch for a
|
||||
# dispatch). Without a `train` block every run routes as before: a
|
||||
# pull_request run is `pr`, a dispatch is `dispatch`, and no other job
|
||||
# changes. Under a train block a PR push is `pr-summary` (screen-summary
|
||||
# prints the screening selection and its estimated cost; no paid slice
|
||||
# starts), and gstack-ship-queue's dispatches are `screen`, `train` or
|
||||
# `bisect` once scripts/train-gate.ts has validated their inputs. A refused
|
||||
# dispatch fails here, before any paid work.
|
||||
route:
|
||||
runs-on: ubuntu-24.04
|
||||
if: github.actor != 'dependabot[bot]'
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
pull-requests: read
|
||||
outputs:
|
||||
mode: ${{ steps.route.outputs.mode }}
|
||||
record_artifact: ${{ steps.policy.outputs.record_artifact }}
|
||||
full_gate_check: ${{ steps.policy.outputs.full_gate_check }}
|
||||
screen_head: ${{ steps.screen.outputs.head_sha }}
|
||||
screen_base: ${{ steps.screen.outputs.base_sha }}
|
||||
screen_branch: ${{ steps.screen.outputs.head_branch }}
|
||||
superseded: ${{ steps.screen.outputs.superseded }}
|
||||
bisect_slices: ${{ steps.bisect.outputs.slices }}
|
||||
bisect_timeouts: ${{ steps.bisect.outputs.slice_timeouts }}
|
||||
bisect_refs: ${{ steps.bisect.outputs.prefix_refs }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.4.2
|
||||
|
||||
# Read with git and jq only: no repository code decides whether PR pushes start paid work.
|
||||
- name: Read the train policy (base revision)
|
||||
id: policy
|
||||
env:
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
run: |
|
||||
base="${BASE_SHA:-origin/${DEFAULT_BRANCH}}"
|
||||
train=$(git show "${base}:.gstack/ship-policy.json" 2>/dev/null | jq -c 'if (.train | type) == "object" then .train else null end' 2>/dev/null || true)
|
||||
artifact=$(jq -r '.record_artifact // ""' <<<"${train:-null}" 2>/dev/null || true)
|
||||
check=$(jq -r '.full_gate_check // ""' <<<"${train:-null}" 2>/dev/null || true)
|
||||
if [ -z "$train" ] || [ "$train" = null ]; then
|
||||
echo "train=false" >> "$GITHUB_OUTPUT"
|
||||
echo "- Train policy: none at ${base}; this run routes as before" >> "$GITHUB_STEP_SUMMARY"
|
||||
elif [[ "$artifact" =~ ^[A-Za-z0-9_.-]{1,64}$ ]] && [[ "$check" =~ ^[A-Za-z0-9_./-]{1,64}$ ]]; then
|
||||
{ echo "train=true"; echo "record_artifact=${artifact}"; echo "full_gate_check=${check}"; } >> "$GITHUB_OUTPUT"
|
||||
echo "- Train policy: declared at ${base} (record artifact ${artifact}, check ${check})" >> "$GITHUB_STEP_SUMMARY"
|
||||
else
|
||||
echo "train=false" >> "$GITHUB_OUTPUT"
|
||||
echo "::warning::the train block at ${base} names no valid record_artifact and full_gate_check; this run routes as before"
|
||||
fi
|
||||
|
||||
- name: Route this run
|
||||
id: route
|
||||
env:
|
||||
TRAIN_POLICY: ${{ steps.policy.outputs.train }}
|
||||
NONCE: ${{ inputs.nonce }}
|
||||
EXPECTED_SHA: ${{ inputs.expected_sha }}
|
||||
EVALS_ALL: ${{ inputs.evals_all }}
|
||||
CASES: ${{ inputs.cases }}
|
||||
PREFIX_SHAS: ${{ inputs.prefix_shas }}
|
||||
SCREEN_PR: ${{ inputs.screen_pr }}
|
||||
SCREEN_MERGE_SHA: ${{ inputs.screen_merge_sha }}
|
||||
SCREEN_POLICY: ${{ inputs.screen_policy }}
|
||||
run: |
|
||||
if [ "$GITHUB_EVENT_NAME" = pull_request ]; then
|
||||
mode='pr'
|
||||
if [ "$TRAIN_POLICY" = true ]; then mode=pr-summary; fi
|
||||
echo "mode=${mode}" >> "$GITHUB_OUTPUT"
|
||||
echo "- Run mode: ${mode}" >> "$GITHUB_STEP_SUMMARY"
|
||||
elif [ -f scripts/train-gate.ts ]; then
|
||||
bun --no-install run scripts/train-gate.ts route --github-output "$GITHUB_OUTPUT" >> "$GITHUB_STEP_SUMMARY"
|
||||
elif [ -n "${CASES}${PREFIX_SHAS}${SCREEN_PR}${SCREEN_MERGE_SHA}${SCREEN_POLICY}" ]; then
|
||||
echo "::error::${GITHUB_REF_NAME} has no scripts/train-gate.ts to validate the train or screen inputs; dispatch from a ref that has it"
|
||||
exit 1
|
||||
else
|
||||
echo "mode=dispatch" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# The screen's merge commit must merge the PR's current head into its
|
||||
# base, and Free Tests must be green on that head (base-branch code: a
|
||||
# screen dispatches on the default branch). No quiet window: a screen is
|
||||
# an explicit request. A moved PR head reports superseded and plans nothing.
|
||||
- name: Check the screen's PR head (Free Tests)
|
||||
id: screen
|
||||
if: steps.route.outputs.mode == 'screen'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
SCREEN_PR: ${{ inputs.screen_pr }}
|
||||
SCREEN_MERGE_SHA: ${{ inputs.screen_merge_sha }}
|
||||
run: |
|
||||
bun --no-install run scripts/eval-readiness.ts wait --repo "$GITHUB_REPOSITORY" --pr "$SCREEN_PR" --merge-sha "$SCREEN_MERGE_SHA" \
|
||||
--run-id "$GITHUB_RUN_ID" --quiet-minutes 0 --deadline-minutes 20 \
|
||||
--github-output "$GITHUB_OUTPUT" --summary "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Plan the bisect (one executor per prefix)
|
||||
id: bisect
|
||||
if: steps.route.outputs.mode == 'bisect'
|
||||
env:
|
||||
CASES: ${{ steps.route.outputs.cases }}
|
||||
PREFIXES: ${{ steps.route.outputs.prefixes }}
|
||||
NONCE: ${{ inputs.nonce }}
|
||||
run: |
|
||||
bun --no-install run scripts/train-gate.ts bisect-plan --cases "$CASES" --prefixes "$PREFIXES" --nonce "$NONCE" \
|
||||
--out /tmp/train-bisect/bisect.json --github-output "$GITHUB_OUTPUT" >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Hand the bisect plan to its executors
|
||||
if: steps.route.outputs.mode == 'bisect'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: train-bisect-plan
|
||||
path: /tmp/train-bisect/bisect.json
|
||||
retention-days: 30
|
||||
|
||||
# ── Readiness-gated dispatch (R1, GSTA-48) ───────────────────────────────
|
||||
# Paid slices for a PR head start only after the PR has been quiet for the
|
||||
# window (vars.EVALS_QUIET_MINUTES, default 10; the evals-no-debounce label
|
||||
@@ -161,6 +310,35 @@ jobs:
|
||||
--run-id "$GITHUB_RUN_ID" --quiet-minutes "$QUIET_MINUTES" --deadline-minutes 80 \
|
||||
--github-output "$GITHUB_OUTPUT" --summary "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# GSTA-48 P3-1: a head the readiness job blocked runs no gate, so the PR's
|
||||
# last "## E2E Evals" comment would still show an older head's verdict.
|
||||
# Replace it (never post a new one) with what happened to this head. Runs no
|
||||
# repository code.
|
||||
readiness-comment:
|
||||
runs-on: ubuntu-24.04
|
||||
needs: [readiness]
|
||||
if: ${{ !cancelled() && needs.readiness.result == 'failure' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }}
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
pull-requests: write
|
||||
issues: write
|
||||
steps:
|
||||
- name: Replace a stale verdict comment
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
COMMENT_ID=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate \
|
||||
--jq '.[] | select(.body | startswith("## E2E Evals")) | .id' | tail -1)
|
||||
if [ -z "$COMMENT_ID" ]; then exit 0; fi
|
||||
BODY="## E2E Evals: ⛔ NOT RUN on \`${HEAD_SHA}\`
|
||||
|
||||
The readiness job blocked this head (Free Tests did not pass on it, or did not finish within the readiness deadline), so no paid gate ran and no verdict covers it. An earlier verdict in this comment belonged to an older head.
|
||||
|
||||
Details: [run ${GITHUB_RUN_ID}](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}). Fix and push, or re-run Free Tests and then re-run this workflow."
|
||||
gh api "repos/${GITHUB_REPOSITORY}/issues/comments/${COMMENT_ID}" -X PATCH -f body="$BODY"
|
||||
|
||||
# ── Receipt recovery (W1c: CEO-14/34, ENG-4/5/7) ─────────────────────────
|
||||
# A run cancelled by the next push never reaches slices-report, so its
|
||||
# finished receipts (passes AND the negatives executors write at the
|
||||
@@ -174,10 +352,11 @@ jobs:
|
||||
# plan-slices turns reuse off, so cases execute unreduced.
|
||||
recover-receipts:
|
||||
runs-on: ubuntu-24.04
|
||||
needs: [readiness]
|
||||
needs: [readiness, route]
|
||||
# Runs after readiness settles, so a superseded run's cancellation has usually completed; a head whose Free Tests failed recovers nothing.
|
||||
# A dispatch naming pr_receipts reads that PR's receipts from its runs' artifacts (cache entries are ref-scoped).
|
||||
if: ${{ !cancelled() && (needs.readiness.result == 'success' || needs.readiness.result == 'skipped') && needs.readiness.outputs.superseded != 'true' && github.actor != 'dependabot[bot]' && ((github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && !contains(github.event.pull_request.labels.*.name, 'evals-fresh')) || (github.event_name == 'workflow_dispatch' && inputs.pr_receipts != '')) }}
|
||||
# A dispatched screen only looks for a gate verdict to reuse (R2a/R2); a PR push under a train policy (pr-summary) skips this job.
|
||||
if: ${{ !cancelled() && (needs.readiness.result == 'success' || needs.readiness.result == 'skipped') && needs.readiness.outputs.superseded != 'true' && needs.route.result == 'success' && needs.route.outputs.superseded != 'true' && github.actor != 'dependabot[bot]' && ((github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && needs.route.outputs.mode == 'pr' && !contains(github.event.pull_request.labels.*.name, 'evals-fresh')) || (github.event_name == 'workflow_dispatch' && inputs.pr_receipts != '') || needs.route.outputs.mode == 'screen') }}
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -185,7 +364,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.base.sha || inputs.base_sha || github.event.repository.default_branch }}
|
||||
ref: ${{ github.event.pull_request.base.sha || needs.route.outputs.screen_base || inputs.base_sha || github.event.repository.default_branch }}
|
||||
persist-credentials: false
|
||||
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
@@ -202,6 +381,7 @@ jobs:
|
||||
restore-keys: eval-input-v1-${{ github.repository_id }}-pr-${{ github.event.pull_request.number }}-
|
||||
|
||||
- name: Recover receipts of cancelled runs (base-ref code)
|
||||
if: needs.route.outputs.mode != 'screen'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
STORE_KEY: ${{ steps.store.outputs.cache-matched-key }}
|
||||
@@ -229,26 +409,32 @@ jobs:
|
||||
# identical. Anything else (a red, partial or cancelled gate, a changed
|
||||
# own diff, a non-stamp byte, no binding or receipt, a gh or git error,
|
||||
# evals-fresh) leaves no candidate: the full gate runs.
|
||||
# A dispatched screen (GSTA-49 T1b) finds the same way, authenticated by
|
||||
# PR number, head, repository and tested base (its merge commit's first
|
||||
# parent), listing the PR's pull_request runs and earlier screens; a
|
||||
# broad screen never adopts a narrow run.
|
||||
- name: Find a gate verdict to reuse (R2a/R2, base-ref code)
|
||||
if: github.event_name == 'pull_request'
|
||||
if: github.event_name == 'pull_request' || needs.route.outputs.mode == 'screen'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
|
||||
HEAD_BRANCH: ${{ github.event.pull_request.head.ref }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number || inputs.screen_pr }}
|
||||
HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
|
||||
HEAD_BRANCH: ${{ github.event.pull_request.head.ref || needs.route.outputs.screen_branch }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha || needs.route.outputs.screen_head }}
|
||||
TESTED_SHA: ${{ inputs.screen_merge_sha || github.sha }}
|
||||
SCREEN_POLICY: ${{ inputs.screen_policy }}
|
||||
run: |
|
||||
if [ ! -f scripts/gate-reuse.ts ]; then
|
||||
echo "- Gate verdict reuse unavailable: the base ref has no scripts/gate-reuse.ts; the full gate runs" >> "$GITHUB_STEP_SUMMARY"
|
||||
exit 0
|
||||
fi
|
||||
bun --no-install run scripts/gate-reuse.ts find --repo "$GITHUB_REPOSITORY" --pr "$PR_NUMBER" --head-repo "$HEAD_REPO" \
|
||||
--branch "$HEAD_BRANCH" --run-id "$GITHUB_RUN_ID" --head-sha "$HEAD_SHA" --tested-sha "$GITHUB_SHA" \
|
||||
--out /tmp/gate-reuse --budget-seconds 90 >> "$GITHUB_STEP_SUMMARY" \
|
||||
--branch "$HEAD_BRANCH" --run-id "$GITHUB_RUN_ID" --head-sha "$HEAD_SHA" --tested-sha "$TESTED_SHA" \
|
||||
${SCREEN_POLICY:+--screen-policy "$SCREEN_POLICY"} --out /tmp/gate-reuse --budget-seconds 90 >> "$GITHUB_STEP_SUMMARY" \
|
||||
|| echo "- Gate verdict reuse crashed; the full gate runs" >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Hand a gate reuse candidate to plan-slices
|
||||
if: github.event_name == 'pull_request'
|
||||
if: github.event_name == 'pull_request' || needs.route.outputs.mode == 'screen'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: gate-reuse-candidate
|
||||
@@ -264,6 +450,36 @@ jobs:
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
# ── PR push under a train policy (GSTA-49 T1b, a gate edit) ─────────────
|
||||
# Only when the base revision declares a `train` block: the push starts no
|
||||
# paid slice. This job prints how to request the paid screen, the
|
||||
# screening selection and its estimated cost, for free; the train's full
|
||||
# gate covers every merge. Without a train block it never runs.
|
||||
screen-summary:
|
||||
runs-on: ubuntu-24.04
|
||||
needs: [route]
|
||||
if: needs.route.outputs.mode == 'pr-summary' && github.event.pull_request.head.repo.full_name == github.repository
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.4.2
|
||||
|
||||
- name: Print the screening selection and its estimated cost (no paid case runs)
|
||||
env:
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
run: |
|
||||
EVALS_TIER=gate bun --no-install run scripts/test-paid-shards.ts --tier gate --emit-plan /tmp/paid-plan/manifest.json --slices 1
|
||||
bun --no-install run scripts/train-gate.ts screen-summary /tmp/paid-plan/manifest.json --pr "$PR_NUMBER" >> "$GITHUB_STEP_SUMMARY"
|
||||
bun --no-install run scripts/test-pr-profile.ts summary /tmp/paid-plan/manifest.json >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# ── Sliced lane (the ONLY paid lane; legacy 17-row matrix deleted) ──────────
|
||||
# One PLANNER computes diff selection + the slice plan ONCE (killing
|
||||
# per-slice selector divergence); K executors consume the manifest; the
|
||||
@@ -283,9 +499,10 @@ jobs:
|
||||
# branch (see CLAUDE.md's garrytan-agents workflow).
|
||||
plan-slices:
|
||||
runs-on: ubicloud-standard-8
|
||||
needs: [readiness, recover-receipts]
|
||||
needs: [readiness, recover-receipts, route]
|
||||
# !cancelled(): plans even when readiness or recovery was skipped (dispatch, evals-fresh) or recovery failed (reuse is then off); a superseded run, or a head whose readiness failed (Free Tests red), plans nothing.
|
||||
if: ${{ !cancelled() && (needs.readiness.result == 'success' || needs.readiness.result == 'skipped') && needs.readiness.outputs.superseded != 'true' && github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }}
|
||||
# The route job must have named a planned mode: pr-summary (a PR push under a train policy) and bisect plan nothing here, and neither does a refused dispatch.
|
||||
if: ${{ !cancelled() && (needs.readiness.result == 'success' || needs.readiness.result == 'skipped') && needs.readiness.outputs.superseded != 'true' && needs.route.result == 'success' && needs.route.outputs.superseded != 'true' && contains(fromJSON('["pr","screen","train","dispatch"]'), needs.route.outputs.mode) && github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }}
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -293,9 +510,16 @@ jobs:
|
||||
slices: ${{ steps.matrix.outputs.slices }}
|
||||
slice_timeouts: ${{ steps.matrix.outputs.slice_timeouts }}
|
||||
gate_reuse: ${{ steps.gate_reuse.outputs.gate_reuse }}
|
||||
# The route, forwarded: the executors and the report read it from here.
|
||||
mode: ${{ needs.route.outputs.mode }}
|
||||
record_artifact: ${{ needs.route.outputs.record_artifact }}
|
||||
full_gate_check: ${{ needs.route.outputs.full_gate_check }}
|
||||
screen_head: ${{ needs.route.outputs.screen_head }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
# A dispatched screen tests the PR's pinned refs/pull/<n>/merge commit; empty for every other run (its own ref).
|
||||
ref: ${{ inputs.screen_merge_sha }}
|
||||
# Preserve full history for merge-base diff selection. Moving the
|
||||
# planner off the eval image must not change its selection inputs.
|
||||
fetch-depth: 0
|
||||
@@ -343,14 +567,14 @@ jobs:
|
||||
env:
|
||||
EVALS_ALL: ${{ (github.event_name == 'workflow_dispatch' && inputs.evals_all) && '1' || '' }}
|
||||
EVALS_CACHE_DIR: ${{ steps.reuse.outputs.reuse == 'on' && '/tmp/gstack-eval-input-cache' || '' }}
|
||||
# Dispatch diff base (CEO-17): the exact merge-base, else the named base branch, else detection.
|
||||
EVALS_BASE: ${{ inputs.base_sha || (inputs.base_ref != '' && format('origin/{0}', inputs.base_ref)) || '' }}
|
||||
# Dispatch diff base (CEO-17): a screen's merge-commit first parent, else the exact merge-base, else the named base branch, else detection.
|
||||
EVALS_BASE: ${{ needs.route.outputs.screen_base || inputs.base_sha || (inputs.base_ref != '' && format('origin/{0}', inputs.base_ref)) || '' }}
|
||||
run: EVALS_TIER=gate bun --no-install run scripts/test-paid-shards.ts --tier gate --emit-plan /tmp/paid-plan/manifest.json --slice-budget 420 --jobs 2 --max-parallel 20
|
||||
|
||||
# R2a/R2: reuse holds only when this head's own plan selects exactly the
|
||||
# cases and trial plans the reused gate ran (selection fingerprint).
|
||||
- name: Download a gate reuse candidate
|
||||
if: github.event_name == 'pull_request'
|
||||
if: github.event_name == 'pull_request' || needs.route.outputs.mode == 'screen'
|
||||
continue-on-error: true
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
||||
with:
|
||||
@@ -359,7 +583,7 @@ jobs:
|
||||
|
||||
- name: Confirm gate verdict reuse (R2a/R2)
|
||||
id: gate_reuse
|
||||
if: github.event_name == 'pull_request'
|
||||
if: github.event_name == 'pull_request' || needs.route.outputs.mode == 'screen'
|
||||
run: |
|
||||
mkdir -p /tmp/gate-reuse
|
||||
bun --no-install run scripts/gate-reuse.ts confirm /tmp/gate-reuse /tmp/paid-plan/manifest.json --out /tmp/paid-plan \
|
||||
@@ -430,6 +654,8 @@ jobs:
|
||||
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
# A dispatched screen's pinned merge commit (the planner verified it); empty otherwise.
|
||||
ref: ${{ inputs.screen_merge_sha }}
|
||||
# Full history: files with SELF-derived selection (the LLM-judge
|
||||
# map, routing) walk git at module load, and selection is
|
||||
# fail-closed on git errors — a shallow checkout crashed those
|
||||
@@ -555,6 +781,151 @@ jobs:
|
||||
if-no-files-found: ignore
|
||||
retention-days: 30
|
||||
|
||||
# ── Train bisect (GSTA-49 T1b) ─────────────────────────────────────────
|
||||
# One executor per prefix of the train (route's validated prefix_shas, each
|
||||
# an ancestor of the train head): every red case once, one trial, on that
|
||||
# prefix's own checkout. Each writes the per-prefix record that
|
||||
# `gstack-ship-queue train bisect` reads, as <record_artifact>-bisect-<sha12>.
|
||||
# A bisect never reuses a result and never binds a verdict.
|
||||
train-bisect:
|
||||
runs-on: ubicloud-standard-8
|
||||
needs: [build-image, route]
|
||||
if: ${{ !cancelled() && needs.build-image.result == 'success' && needs.route.result == 'success' && needs.route.outputs.mode == 'bisect' }}
|
||||
env:
|
||||
EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-train-bisect-${{ matrix.slice }}
|
||||
timeout-minutes: ${{ fromJSON(needs.route.outputs.bisect_timeouts)[matrix.slice] }}
|
||||
permissions:
|
||||
contents: read
|
||||
packages: read
|
||||
container:
|
||||
image: ${{ needs.build-image.outputs.image-tag }}
|
||||
credentials:
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
options: --user runner
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 7
|
||||
matrix:
|
||||
slice: ${{ fromJSON(needs.route.outputs.bisect_slices) }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
ref: ${{ fromJSON(needs.route.outputs.bisect_refs)[matrix.slice] }}
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Verify the prefix checkout
|
||||
id: prefix
|
||||
env:
|
||||
PREFIX_SHA: ${{ fromJSON(needs.route.outputs.bisect_refs)[matrix.slice] }}
|
||||
run: |
|
||||
actual=$(git rev-parse HEAD)
|
||||
if [ "$actual" != "$PREFIX_SHA" ]; then
|
||||
echo "::error::train-bisect checked out ${actual}, not its prefix ${PREFIX_SHA}"
|
||||
exit 1
|
||||
fi
|
||||
echo "short=${PREFIX_SHA:0:12}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Fix bun temp
|
||||
uses: ./.github/actions/fix-bun-temp
|
||||
|
||||
- name: Restore deps
|
||||
uses: ./.github/actions/restore-deps
|
||||
|
||||
- run: bun run build
|
||||
|
||||
# Any slice can host a PTY smoke, so the seed/registration steps run
|
||||
# UNCONDITIONALLY (both are idempotent) — the old matrix keyed them on
|
||||
# matrix.suite.name, which a sliced lane cannot do. The register
|
||||
# composite carries the fail-fast dangling-symlink/frontmatter
|
||||
# verification loop, so a moved skill target fails HERE in seconds,
|
||||
# not as a wedged PTY session at the shard wall.
|
||||
- name: Seed claude interactive config
|
||||
uses: ./.github/actions/seed-claude-config
|
||||
with:
|
||||
anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
|
||||
- name: Register gstack skills for PTY smokes
|
||||
uses: ./.github/actions/register-gstack-skills
|
||||
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
||||
with:
|
||||
name: train-bisect-plan
|
||||
path: /tmp/train-bisect
|
||||
|
||||
- name: Authenticate the Codex CLI
|
||||
shell: bash
|
||||
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
|
||||
env:
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
codex="${GSTACK_CI_CODEX_BIN_DIR:?}/codex"
|
||||
export CODEX_HOME="$HOME/.gstack-ci-codex"
|
||||
mkdir -p "$CODEX_HOME"
|
||||
echo "GSTACK_CI_CODEX_HOME=$CODEX_HOME" >> "$GITHUB_ENV"
|
||||
if [ -z "${OPENAI_API_KEY:-}" ]; then
|
||||
echo "::warning::OPENAI_API_KEY is unavailable; Codex eval cases cannot authenticate and will fail."
|
||||
exit 0
|
||||
fi
|
||||
printenv OPENAI_API_KEY | "$codex" login --with-api-key
|
||||
"$codex" login status >/dev/null 2>&1
|
||||
|
||||
- name: Run the red cases on prefix ${{ matrix.slice }}
|
||||
env:
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
|
||||
PLAYWRIGHT_BROWSERS_PATH: /opt/playwright-browsers
|
||||
EVALS_JOBS: "1"
|
||||
EVALS_CONCURRENCY: "1"
|
||||
SLICE: ${{ matrix.slice }}
|
||||
run: EVALS_TIER=gate bun run scripts/train-gate.ts bisect-prefix --plan /tmp/train-bisect/bisect.json --index "$SLICE" --out /tmp/train-bisect/out
|
||||
|
||||
- name: Upload the prefix record
|
||||
if: always() && steps.prefix.outputs.short != ''
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: ${{ needs.route.outputs.record_artifact }}-bisect-${{ steps.prefix.outputs.short }}
|
||||
path: /tmp/train-bisect/out/bisect-prefix.json
|
||||
if-no-files-found: ignore
|
||||
overwrite: true
|
||||
retention-days: 90
|
||||
|
||||
- name: Upload native capture evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: native-captures-${{ env.EVALS_RUN_ID }}
|
||||
include-hidden-files: true
|
||||
path: |
|
||||
~/.gstack/projects/*/e2e-runs
|
||||
~/.gstack/projects/*/evals/qa-callers
|
||||
~/.gstack-dev/e2e-runs
|
||||
~/.gstack-dev/evals/qa-callers
|
||||
if-no-files-found: ignore
|
||||
retention-days: 90
|
||||
|
||||
# The spooled per-shard full logs — a red weekly/PR lane three weeks
|
||||
# later needs more than a summary line.
|
||||
# always(), not failure(): a failed behavior trial is a verdict and no
|
||||
# longer reds its runner, but its full log is the diagnosis evidence.
|
||||
- name: Upload shard logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: train-bisect-logs-${{ matrix.slice }}-a${{ github.run_attempt }}
|
||||
include-hidden-files: true
|
||||
# The Fix-bun-temp step points TMPDIR at /home/runner/.cache, so the
|
||||
# runner's spool lands THERE, not /tmp — the original /tmp glob
|
||||
# uploaded nothing and a red slice's diagnostics were unreachable.
|
||||
path: |
|
||||
/home/runner/.cache/gstack-paid-shard-*.log
|
||||
/tmp/gstack-paid-shard-*.log
|
||||
if-no-files-found: ignore
|
||||
retention-days: 30
|
||||
|
||||
slices-report:
|
||||
# GitHub-hosted, like recover-receipts: Ubicloud runners send actions/cache
|
||||
# to Ubicloud's own transparent cache, which a GitHub-hosted restore never
|
||||
@@ -582,6 +953,10 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
# A screen's pinned merge commit (empty otherwise); a train run needs the
|
||||
# history of its prefix commits T1..Tk for the train record.
|
||||
ref: ${{ inputs.screen_merge_sha }}
|
||||
fetch-depth: ${{ needs.plan-slices.outputs.mode == 'train' && '0' || '1' }}
|
||||
persist-credentials: false
|
||||
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
@@ -671,19 +1046,24 @@ jobs:
|
||||
# R2a/R2: what this executed run proved, read by a later push. The base
|
||||
# is the tested merge commit's first parent. Reused runs bind nothing,
|
||||
# so reuse never chains off a reuse.
|
||||
# A dispatched screen binds too (purpose screen, its tested merge commit
|
||||
# and policy). Train, bisect and other dispatches never bind for reuse.
|
||||
- name: Bind this run's verdict for reuse (R2a/R2)
|
||||
if: always() && github.event_name == 'pull_request' && needs.plan-slices.outputs.gate_reuse != 'true'
|
||||
if: always() && (github.event_name == 'pull_request' || needs.plan-slices.outputs.mode == 'screen') && needs.plan-slices.outputs.gate_reuse != 'true'
|
||||
env:
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number || inputs.screen_pr }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha || needs.plan-slices.outputs.screen_head }}
|
||||
TESTED_SHA: ${{ inputs.screen_merge_sha || github.sha }}
|
||||
SCREEN_POLICY: ${{ inputs.screen_policy }}
|
||||
RECONCILE_EXIT: ${{ steps.reconcile.outputs.exit }}
|
||||
run: |
|
||||
bun --no-install run scripts/gate-reuse.ts bind /tmp/paid-report --out /tmp/gate-binding --repo "$GITHUB_REPOSITORY" \
|
||||
--pr "$PR_NUMBER" --run-id "$GITHUB_RUN_ID" --run-attempt "$GITHUB_RUN_ATTEMPT" --head-sha "$HEAD_SHA" \
|
||||
--tested-sha "$GITHUB_SHA" --profile "$EVALS_PROFILE" --reconcile-exit "$RECONCILE_EXIT" >> "$GITHUB_STEP_SUMMARY"
|
||||
--tested-sha "$TESTED_SHA" --profile "$EVALS_PROFILE" --reconcile-exit "$RECONCILE_EXIT" \
|
||||
${SCREEN_POLICY:+--purpose screen --screen-policy "$SCREEN_POLICY"} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Upload the gate binding
|
||||
if: always() && github.event_name == 'pull_request' && needs.plan-slices.outputs.gate_reuse != 'true'
|
||||
if: always() && (github.event_name == 'pull_request' || needs.plan-slices.outputs.mode == 'screen') && needs.plan-slices.outputs.gate_reuse != 'true'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: gate-binding-a${{ github.run_attempt }}
|
||||
@@ -700,17 +1080,81 @@ jobs:
|
||||
if-no-files-found: ignore
|
||||
retention-days: 90
|
||||
|
||||
# GSTA-49 T1b: a train's own gate run publishes its record, read by
|
||||
# gstack-ship-queue's verdict and verify and by the main backstop: the
|
||||
# census from this run's plan manifest, one panel per planned case, and
|
||||
# the prefix trees T1..Tk modulo stamps. GREEN only for a complete census.
|
||||
- name: Write the train record
|
||||
if: always() && needs.plan-slices.outputs.mode == 'train'
|
||||
env:
|
||||
NONCE: ${{ inputs.nonce }}
|
||||
HEAD_SHA: ${{ inputs.expected_sha }}
|
||||
RECORD: ${{ needs.plan-slices.outputs.record_artifact }}
|
||||
RECONCILE_EXIT: ${{ steps.reconcile.outputs.exit }}
|
||||
run: |
|
||||
bun --no-install run scripts/train-gate.ts train-record /tmp/paid-report --nonce "$NONCE" --head "$HEAD_SHA" \
|
||||
--reconcile-exit "$RECONCILE_EXIT" --run-id "$GITHUB_RUN_ID" --run-attempt "$GITHUB_RUN_ATTEMPT" --repo "$GITHUB_REPOSITORY" \
|
||||
--out "/tmp/train-record/${RECORD}.json" >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Upload the train record
|
||||
if: always() && needs.plan-slices.outputs.mode == 'train'
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: ${{ needs.plan-slices.outputs.record_artifact }}
|
||||
path: /tmp/train-record
|
||||
if-no-files-found: ignore
|
||||
overwrite: true
|
||||
retention-days: 90
|
||||
|
||||
- name: Fail the workflow when reconciliation failed
|
||||
if: (steps.reconcile.outputs.exit || steps.reused.outputs.exit) != '0'
|
||||
run: exit 1
|
||||
|
||||
# GSTA-49 T1b: the check run gstack-ship-queue requires on the train head,
|
||||
# published only for a complete GREEN census. Its own job with checks:
|
||||
# write and no repository code (the same split as slices-comment).
|
||||
train-check:
|
||||
runs-on: ubuntu-24.04
|
||||
needs: [plan-slices, slices-report]
|
||||
if: ${{ !cancelled() && needs.plan-slices.outputs.mode == 'train' && needs.slices-report.result == 'success' }}
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
checks: write
|
||||
steps:
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
||||
with:
|
||||
name: ${{ needs.plan-slices.outputs.record_artifact }}
|
||||
path: /tmp/train-record
|
||||
|
||||
- name: Publish the full-gate check on the train head
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
CHECK: ${{ needs.plan-slices.outputs.full_gate_check }}
|
||||
RECORD: ${{ needs.plan-slices.outputs.record_artifact }}
|
||||
NONCE: ${{ inputs.nonce }}
|
||||
HEAD_SHA: ${{ inputs.expected_sha }}
|
||||
run: |
|
||||
record="/tmp/train-record/${RECORD}.json"
|
||||
if ! jq -e --arg nonce "$NONCE" --arg head "$HEAD_SHA" \
|
||||
'.schema == 1 and .kind == "train-gate" and .verdict == "GREEN" and .census.complete == true and .nonce == $nonce and .head_sha == $head' \
|
||||
"$record" >/dev/null 2>&1; then
|
||||
echo "::error::the train record is not a complete GREEN census of ${HEAD_SHA} (nonce ${NONCE}); no ${CHECK} check run"
|
||||
exit 1
|
||||
fi
|
||||
summary=$(jq -r '"rule \(.census.executed.rule)/\(.census.planned.rule), judge \(.census.executed.judge)/\(.census.planned.judge); prefixes T1..T\(.prefixes | length)"' "$record")
|
||||
gh api "repos/${GITHUB_REPOSITORY}/check-runs" -X POST -f name="$CHECK" -f head_sha="$HEAD_SHA" -f status=completed -f conclusion=success \
|
||||
-f details_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
|
||||
-f "output[title]=Train ${NONCE}: complete GREEN census" -f "output[summary]=${summary}" >/dev/null
|
||||
echo "- ${CHECK}: success on ${HEAD_SHA} (${summary})" >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# PR comment in its OWN job with the write token and ZERO repo code: no
|
||||
# checkout, no bun install — only downloaded artifacts, jq, and gh. See the
|
||||
# trust-boundary note on slices-report.
|
||||
slices-comment:
|
||||
runs-on: ubicloud-standard-2
|
||||
needs: slices-report
|
||||
if: ${{ !cancelled() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && needs.slices-report.result != 'skipped' }}
|
||||
# A dispatched screen comments on its PR too (screen_pr was validated by the route job, or nothing ran).
|
||||
if: ${{ !cancelled() && ((github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || inputs.screen_pr != '') && needs.slices-report.result != 'skipped' }}
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
pull-requests: write
|
||||
@@ -739,6 +1183,9 @@ jobs:
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
RECONCILE_EXIT: ${{ needs.slices-report.outputs.reconcile-exit }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number || inputs.screen_pr }}
|
||||
SCREEN_POLICY: ${{ inputs.screen_policy }}
|
||||
SCREEN_MERGE_SHA: ${{ inputs.screen_merge_sha }}
|
||||
run: |
|
||||
# shellcheck disable=SC2086,SC2059
|
||||
# R2a/R2: a confirmed verdict reuse ran no case; say which run,
|
||||
@@ -845,7 +1292,13 @@ jobs:
|
||||
|
||||
fi
|
||||
|
||||
COMMENT_ID=$(gh api repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments \
|
||||
if [ -n "$SCREEN_POLICY" ]; then
|
||||
BODY="${BODY}
|
||||
|
||||
*Dispatched ${SCREEN_POLICY} screen (gstack-ship-queue ready --screen) of merge commit \`${SCREEN_MERGE_SHA}\`.*"
|
||||
fi
|
||||
|
||||
COMMENT_ID=$(gh api "repos/${{ github.repository }}/issues/${PR_NUMBER}/comments" \
|
||||
--jq '.[] | select(.body | startswith("## E2E Evals")) | .id' | tail -1)
|
||||
|
||||
if [ -n "$COMMENT_ID" ]; then
|
||||
@@ -855,6 +1308,6 @@ jobs:
|
||||
# REST, not gh's pr-comment subcommand: this job runs with NO
|
||||
# checkout (the token/exec split), and that subcommand resolves
|
||||
# the repo FROM git — it dies with "not a git repository" here.
|
||||
gh api "repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments" \
|
||||
gh api "repos/${{ github.repository }}/issues/${PR_NUMBER}/comments" \
|
||||
-X POST -f body="$BODY"
|
||||
fi
|
||||
@@ -16,6 +16,13 @@
|
||||
* the gate (a re-run keeps the run's original creation time, so it does not
|
||||
* wait out the window again). A head that is no longer the PR's head reports
|
||||
* superseded and plans nothing.
|
||||
*
|
||||
* Dispatched screens (plan GSTA-49 T1b): `--merge-sha` names the
|
||||
* refs/pull/<n>/merge commit `gstack-ship-queue ready --screen` pinned. The
|
||||
* screen is accepted only for an open same-repository PR whose current head
|
||||
* is that merge commit's second parent; its first parent is the tested base.
|
||||
* The same Free Tests rule then applies to the PR head, with no quiet window
|
||||
* (a screen is an explicit request).
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import { runGh, type GhRunner } from './recover-receipts';
|
||||
@@ -73,6 +80,23 @@ function ghJson<T>(gh: GhRunner, args: string[]): { ok: true; value: T } | { ok:
|
||||
try { return { ok: true, value: JSON.parse(out.stdout) as T }; } catch { return { ok: false, error: 'gh returned unreadable JSON' }; }
|
||||
}
|
||||
|
||||
export interface ScreenTarget { head: string; base: string; branch: string }
|
||||
|
||||
/** A screen's merge commit, checked against the PR: same repository, open, and merging the PR's current head into its base. */
|
||||
export function screenTarget(gh: GhRunner, repo: string, pr: number, mergeSha: string): ScreenTarget | { error: string } {
|
||||
if (!/^[0-9a-f]{40}$/.test(mergeSha)) return { error: `screen_merge_sha ${JSON.stringify(mergeSha)} is not a full commit SHA` };
|
||||
const pull = ghJson<{ state?: string; head?: { sha?: string; ref?: string; repo?: { full_name?: string } | null }; base?: { repo?: { full_name?: string } } }>(gh, ['api', `repos/${repo}/pulls/${pr}`]);
|
||||
if (!pull.ok) return { error: `cannot read PR #${pr} (${pull.error})` };
|
||||
const p = pull.value;
|
||||
if (p.base?.repo?.full_name !== repo || p.head?.repo?.full_name !== repo) return { error: `PR #${pr} is not a same-repository PR of ${repo}; fork work is screened on a trusted branch` };
|
||||
if (p.state !== 'open') return { error: `PR #${pr} is ${p.state ?? 'unreadable'}, not open` };
|
||||
const commit = ghJson<{ parents?: Array<{ sha?: string }> }>(gh, ['api', `repos/${repo}/commits/${mergeSha}`]);
|
||||
if (!commit.ok) return { error: `cannot read the merge commit ${mergeSha.slice(0, 12)} (${commit.error})` };
|
||||
const parents = (commit.value.parents ?? []).map(parent => parent.sha ?? '');
|
||||
if (parents.length !== 2 || !parents.every(sha => /^[0-9a-f]{40}$/.test(sha))) return { error: `${mergeSha.slice(0, 12)} is not a two-parent merge commit` };
|
||||
return { head: parents[1]!, base: parents[0]!, branch: p.head?.ref ?? '' };
|
||||
}
|
||||
|
||||
/** Waits until the head is ready, superseded or blocked; never longer than the deadline. */
|
||||
export function waitForReadiness(o: WaitOptions): { decision: ReadinessDecision; lines: string[] } {
|
||||
const gh = o.gh ?? runGh;
|
||||
@@ -110,28 +134,39 @@ export function waitForReadiness(o: WaitOptions): { decision: ReadinessDecision;
|
||||
}
|
||||
}
|
||||
|
||||
const USAGE = `usage: bun run scripts/eval-readiness.ts wait --repo <owner/name> --pr <n> --head-sha <sha> --run-id <id>
|
||||
const USAGE = `usage: bun run scripts/eval-readiness.ts wait --repo <owner/name> --pr <n> (--head-sha <sha> | --merge-sha <sha>) --run-id <id>
|
||||
[--quiet-minutes 10] [--deadline-minutes 80] [--poll-seconds 60] [--github-output <file>] [--summary <file>]
|
||||
Exit 0 when ready or superseded (output superseded=true), 1 when blocked, 2 on usage.`;
|
||||
--merge-sha (a dispatched screen) waits on the PR head that merge commit merges and writes head_sha, base_sha and
|
||||
head_branch outputs. Exit 0 when ready or superseded (output superseded=true), 1 when blocked, 2 on usage.`;
|
||||
|
||||
if (import.meta.main) {
|
||||
const [command, ...rest] = process.argv.slice(2);
|
||||
const flag = (name: string) => { const i = rest.indexOf(`--${name}`); return i >= 0 ? rest[i + 1] : undefined; };
|
||||
const num = (name: string, fallback: number) => { const value = flag(name); return value === undefined || value === '' ? fallback : Number(value); };
|
||||
const [repo, headSha] = [flag('repo'), flag('head-sha')];
|
||||
const [repo, mergeSha] = [flag('repo'), flag('merge-sha')];
|
||||
const [pr, runId, quiet, deadline, poll] = [Number(flag('pr')), Number(flag('run-id')), num('quiet-minutes', 10), num('deadline-minutes', 80), num('poll-seconds', 60)];
|
||||
if (command !== 'wait' || !repo || !headSha || !/^[0-9a-f]{40}$/.test(headSha) || !Number.isSafeInteger(pr) || !Number.isSafeInteger(runId)
|
||||
if (command !== 'wait' || !repo || !(flag('head-sha') ?? mergeSha) || !Number.isSafeInteger(pr) || !Number.isSafeInteger(runId)
|
||||
|| ![quiet, deadline, poll].every(value => Number.isFinite(value) && value >= 0)) {
|
||||
console.error(USAGE);
|
||||
process.exit(2);
|
||||
}
|
||||
const { decision, lines } = waitForReadiness({ repo, pr, headSha, runId, quietMs: quiet * 60_000, deadlineMs: deadline * 60_000, pollMs: Math.max(5, poll) * 1000 });
|
||||
const output = flag('github-output');
|
||||
const summary = flag('summary');
|
||||
const target = mergeSha ? screenTarget(runGh, repo, pr, mergeSha) : null;
|
||||
if (target && 'error' in target) {
|
||||
console.error(`::error::screen refused: ${target.error}`);
|
||||
if (summary) fs.appendFileSync(summary, `- Screen refused: ${target.error}\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
const headSha = target ? target.head : flag('head-sha')!;
|
||||
if (!/^[0-9a-f]{40}$/.test(headSha)) { console.error(USAGE); process.exit(2); }
|
||||
if (target && output) fs.appendFileSync(output, `head_sha=${target.head}\nbase_sha=${target.base}\nhead_branch=${target.branch}\n`);
|
||||
const { decision, lines } = waitForReadiness({ repo, pr, headSha, runId, quietMs: quiet * 60_000, deadlineMs: deadline * 60_000, pollMs: Math.max(5, poll) * 1000 });
|
||||
if (target) lines.unshift(`Screen of PR #${pr}: merge ${mergeSha} merges head ${target.head.slice(0, 12)} into base ${target.base.slice(0, 12)}`);
|
||||
for (const line of lines) {
|
||||
console.log(line);
|
||||
if (summary) fs.appendFileSync(summary, `- ${line}\n`);
|
||||
}
|
||||
const output = flag('github-output');
|
||||
if (output && decision.kind === 'superseded') fs.appendFileSync(output, 'superseded=true\n');
|
||||
if (decision.kind === 'blocked') console.error(`::error::${decision.reason}`);
|
||||
process.exit(decision.kind === 'blocked' ? 1 : 0);
|
||||
|
||||
+80
-29
@@ -33,6 +33,17 @@
|
||||
* byte, a missing receipt or merge-base, gh or git errors, a selection change)
|
||||
* leaves no candidate, and the run executes the full gate. The free suite and
|
||||
* every other check run on the new head regardless.
|
||||
*
|
||||
* Dispatched screens (plan GSTA-49 T1b): a workflow_dispatch run with
|
||||
* screen_pr tests the refs/pull/<n>/merge commit gstack-ship-queue pinned.
|
||||
* Its run name ends ` screen #<n>`, and its binding adds the tested merge
|
||||
* commit and the screen policy. A screen's find lists both events filtered by
|
||||
* the PR, so every newer-run veto spans pull_request runs and screens. A
|
||||
* dispatched binding is accepted only when it names this PR and repository,
|
||||
* its head is one of the PR's commits, its tested commit merges that head
|
||||
* into the bound base, and its policy equals this screen's (a pull_request
|
||||
* run counts as broad: today's full PR selection). Train and bisect runs never
|
||||
* bind and are never listed, so they neither reuse nor are reused.
|
||||
*/
|
||||
import { createHash } from 'node:crypto';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
@@ -43,7 +54,7 @@ import { receiptDigest } from '../lib/ship-queue';
|
||||
import { isUnavailable, loadPolicyFromBase, matchesAny, type ShipPolicy } from '../lib/ship-policy';
|
||||
import { rewriteStamps } from '../lib/restamp';
|
||||
import { buildReceipt } from '../lib/tree-receipt';
|
||||
import { acceptedRun, ghFailureReason, runGh, type GhRunner, type WorkflowRun } from './recover-receipts';
|
||||
import { ghFailureReason, listGateRuns, runGh, type GhRunner, type WorkflowRun } from './recover-receipts';
|
||||
import type { PaidRunManifest } from './lib/paid-plan';
|
||||
|
||||
export const BINDING_FILE = 'gate-binding.json';
|
||||
@@ -67,8 +78,16 @@ export interface GateBinding {
|
||||
/** Cases accepted manually (unscored); a gate with any is never reused as a clean pass. */
|
||||
manual_accepted: number;
|
||||
selection_sha256: string;
|
||||
/** Dispatched screens only: the purpose, the refs/pull/<n>/merge commit it tested, and the screen policy. */
|
||||
purpose?: 'screen';
|
||||
tested_sha?: string;
|
||||
screen_policy?: ScreenPolicy;
|
||||
}
|
||||
|
||||
export type ScreenPolicy = 'narrow' | 'broad';
|
||||
/** What a run is for: train and bisect runs never bind a verdict and are never reused. */
|
||||
export type RunPurpose = 'pr' | 'screen' | 'train' | 'bisect' | 'dispatch';
|
||||
|
||||
export interface GateReuse {
|
||||
schema: 1;
|
||||
kind: 'restamp-only' | 'main-moved';
|
||||
@@ -91,6 +110,8 @@ export interface GateReuse {
|
||||
gated_merge_base?: string;
|
||||
merge_base?: string;
|
||||
own_diff_sha256?: string;
|
||||
/** The reused run was a dispatched screen with this policy. */
|
||||
reused_screen_policy?: ScreenPolicy;
|
||||
}
|
||||
|
||||
/** The cases, trial plans and verdict-relevant plan fields a run executes; slice packing and receipts are excluded. */
|
||||
@@ -112,6 +133,7 @@ export function parseBinding(value: unknown): GateBinding | null {
|
||||
if (!b || b.schema !== 1 || typeof b.repository !== 'string' || !Number.isSafeInteger(b.pr) || !Number.isSafeInteger(b.run_id)
|
||||
|| !Number.isSafeInteger(b.run_attempt) || !SHA.test(b.head_sha ?? '') || !SHA.test(b.base_sha ?? '') || typeof b.profile !== 'string'
|
||||
|| !Number.isSafeInteger(b.reconcile_exit) || typeof b.verdict !== 'string' || !Number.isSafeInteger(b.manual_accepted) || b.manual_accepted < 0 || !SHA256.test(b.selection_sha256 ?? '')) return null;
|
||||
if (b.purpose !== undefined && (b.purpose !== 'screen' || !SHA.test(b.tested_sha ?? '') || (b.screen_policy !== 'narrow' && b.screen_policy !== 'broad'))) return null;
|
||||
return b;
|
||||
}
|
||||
|
||||
@@ -250,7 +272,8 @@ export function ownDiffSignature(git: Git, mergeBase: string, head: string, stam
|
||||
|
||||
/** slices-report: what this executed run proved, or why it binds nothing. */
|
||||
export function buildBinding(o: { repository: string; pr: number; runId: number; runAttempt: number; headSha: string; baseSha: string | null;
|
||||
profile: string; reconcileExit: number; reportDir: string }): { binding: GateBinding | null; reason: string } {
|
||||
profile: string; reconcileExit: number; reportDir: string; purpose?: RunPurpose; testedSha?: string; screenPolicy?: string }): { binding: GateBinding | null; reason: string } {
|
||||
if (o.purpose && o.purpose !== 'pr' && o.purpose !== 'screen') return { binding: null, reason: `a ${o.purpose} run never binds a verdict for reuse` };
|
||||
if (fs.existsSync(path.join(o.reportDir, REUSE_FILE))) return { binding: null, reason: 'this run reused a verdict; only executed gates bind' };
|
||||
if (!o.baseSha) return { binding: null, reason: 'the tested base is unknown' };
|
||||
const manifest = readJson<PaidRunManifest>(path.join(o.reportDir, 'manifest.json'));
|
||||
@@ -258,7 +281,8 @@ export function buildBinding(o: { repository: string; pr: number; runId: number;
|
||||
const outcomes = readJson<{ verdict?: { verdict?: string }; totals?: { manual_accepted?: number } }>(path.join(o.reportDir, 'collector-outcomes.json'));
|
||||
const binding: GateBinding = { schema: 1, repository: o.repository, pr: o.pr, run_id: o.runId, run_attempt: o.runAttempt,
|
||||
head_sha: o.headSha, base_sha: o.baseSha, profile: o.profile, reconcile_exit: o.reconcileExit,
|
||||
verdict: outcomes?.verdict?.verdict ?? 'MISSING', manual_accepted: outcomes?.totals?.manual_accepted ?? -1, selection_sha256: selectionFingerprint(manifest) };
|
||||
verdict: outcomes?.verdict?.verdict ?? 'MISSING', manual_accepted: outcomes?.totals?.manual_accepted ?? -1, selection_sha256: selectionFingerprint(manifest),
|
||||
...(o.purpose === 'screen' ? { purpose: 'screen' as const, tested_sha: o.testedSha ?? '', screen_policy: o.screenPolicy as ScreenPolicy } : {}) };
|
||||
return parseBinding(binding)
|
||||
? { binding, reason: `bound ${binding.verdict} at ${binding.head_sha.slice(0, 12)} on base ${binding.base_sha.slice(0, 12)} (reconcile exit ${binding.reconcile_exit})` }
|
||||
: { binding: null, reason: 'binding fields are incomplete' };
|
||||
@@ -266,22 +290,27 @@ export function buildBinding(o: { repository: string; pr: number; runId: number;
|
||||
|
||||
/**
|
||||
* A newer completed run with no binding is passed over only when no paid
|
||||
* slice of it finished: superseded, readiness-blocked, a reuse, or cancelled
|
||||
* before or during its slices. A slice that finished (success, failure,
|
||||
* timed out) without a bound verdict blocks reuse of an older GREEN, so a
|
||||
* newer red or partial result is never skipped.
|
||||
* slice of it ran: superseded, readiness-blocked, a reuse, or cancelled
|
||||
* before its slice step started. A slice that finished (success, failure,
|
||||
* timed out) without a bound verdict blocks reuse of an older GREEN, and so
|
||||
* does a slice cancelled after its slice step started (killed before its
|
||||
* upload step: its paid results are unknown, so it counts as incomplete), so a
|
||||
* newer red, partial or unknown result is never skipped.
|
||||
*/
|
||||
export function unboundRunVerdict(gh: GhRunner, repo: string, run: WorkflowRun, timeoutMs: number): string | null {
|
||||
const listed = gh(['api', '-X', 'GET', `repos/${repo}/actions/runs/${run.id}/jobs`, '-f', 'filter=latest', '-f', 'per_page=100'], timeoutMs);
|
||||
if (listed.status !== 0) return `cannot list run ${run.id}'s jobs (${ghFailureReason(listed.stderr)})`;
|
||||
let slices: Array<{ name: string; conclusion: string | null }>;
|
||||
type Job = { name: string; conclusion: string | null; steps?: Array<{ name: string; status?: string; conclusion?: string | null }> };
|
||||
let slices: Job[];
|
||||
try {
|
||||
slices = ((JSON.parse(listed.stdout).jobs ?? []) as Array<{ name: string; conclusion: string | null }>)
|
||||
.filter(job => job.name === 'eval-slices' || job.name.startsWith('eval-slices '));
|
||||
slices = ((JSON.parse(listed.stdout).jobs ?? []) as Job[]).filter(job => job.name === 'eval-slices' || job.name.startsWith('eval-slices '));
|
||||
} catch { return `run ${run.id}'s job list is unreadable`; }
|
||||
const finished = slices.find(job => job.conclusion !== null && job.conclusion !== 'skipped' && job.conclusion !== 'cancelled');
|
||||
return finished
|
||||
? `newer run ${run.id} (${run.head_sha.slice(0, 12)}) finished a paid slice (${finished.conclusion}) but bound no verdict (conclusion ${run.conclusion})`
|
||||
if (finished) return `newer run ${run.id} (${run.head_sha.slice(0, 12)}) finished a paid slice (${finished.conclusion}) but bound no verdict (conclusion ${run.conclusion})`;
|
||||
const killed = slices.find(job => job.conclusion === 'cancelled' && (job.steps ?? []).some(step => step.name.startsWith('Run slice')
|
||||
&& step.status !== 'queued' && step.status !== 'pending' && step.conclusion !== 'skipped'));
|
||||
return killed
|
||||
? `newer run ${run.id} (${run.head_sha.slice(0, 12)}) had a paid slice (${killed.name}) killed after its slice step started and before its upload step; its results are unknown, so it counts as incomplete`
|
||||
: null;
|
||||
}
|
||||
|
||||
@@ -292,8 +321,10 @@ export interface FindOptions {
|
||||
branch: string;
|
||||
selfRunId: number;
|
||||
headSha: string;
|
||||
/** This run's tested merge commit (github.sha); its first parent is the base. */
|
||||
/** This run's tested merge commit (github.sha, or a screen's refs/pull/<n>/merge commit); its first parent is the base. */
|
||||
testedSha: string;
|
||||
/** A dispatched screen: its policy; pull_request runs and earlier screens of this PR are both candidates. */
|
||||
screen?: { policy: ScreenPolicy };
|
||||
/** A base-ref checkout to fetch into and run the receipt in. */
|
||||
repoRoot: string;
|
||||
out: string;
|
||||
@@ -315,13 +346,9 @@ export function findReusableGate(o: FindOptions): { candidate: GateReuse | null;
|
||||
const refuse = (reason: string) => { lines.push(`no gate reuse: ${reason}; the full gate runs`); return { candidate: null, lines }; };
|
||||
if (!SHA.test(o.headSha) || !SHA.test(o.testedSha)) return refuse('this run has no exact head and tested commit');
|
||||
|
||||
const listed = gh(['api', '-X', 'GET', `repos/${o.repo}/actions/workflows/evals.yml/runs`, '-f', 'event=pull_request', '-f', `branch=${o.branch}`, '-f', 'per_page=50'], left());
|
||||
if (listed.status !== 0) return refuse(ghFailureReason(listed.stderr));
|
||||
let runs: WorkflowRun[];
|
||||
try { runs = (JSON.parse(listed.stdout).workflow_runs ?? []) as WorkflowRun[]; } catch { return refuse('gh returned an unreadable run list'); }
|
||||
const scope = { pr: o.pr, headRepo: o.headRepo, selfRunId: o.selfRunId };
|
||||
const older = runs.filter(run => acceptedRun(run, scope) && run.id < o.selfRunId)
|
||||
.sort((a, b) => b.id - a.id).slice(0, o.maxRuns ?? 10);
|
||||
const listed = listGateRuns(gh, { repo: o.repo, pr: o.pr, headRepo: o.headRepo, branch: o.branch, selfRunId: o.selfRunId, screens: !!o.screen }, left);
|
||||
if ('error' in listed) return refuse(listed.error);
|
||||
const older = listed.runs.filter(run => run.id < o.selfRunId).sort((a, b) => b.id - a.id).slice(0, o.maxRuns ?? 10);
|
||||
|
||||
const work = fs.mkdtempSync(path.join(os.tmpdir(), 'gate-reuse-'));
|
||||
try {
|
||||
@@ -352,9 +379,15 @@ export function findReusableGate(o: FindOptions): { candidate: GateReuse | null;
|
||||
if (!found) return refuse('no earlier executed gate on this PR bound a verdict');
|
||||
const { run, binding } = found;
|
||||
if (!binding) return refuse(`run ${run.id}'s gate binding is missing or malformed`);
|
||||
if (binding.run_id !== run.id || binding.pr !== o.pr || binding.repository !== o.repo || binding.head_sha !== run.head_sha) {
|
||||
const screenRun = run.event === 'workflow_dispatch';
|
||||
if (binding.run_id !== run.id || binding.pr !== o.pr || binding.repository !== o.repo
|
||||
|| (screenRun ? binding.purpose !== 'screen' : binding.head_sha !== run.head_sha || binding.purpose !== undefined)) {
|
||||
return refuse(`run ${run.id}'s gate binding does not match the run it came from`);
|
||||
}
|
||||
const boundPolicy = binding.screen_policy ?? 'broad';
|
||||
if (o.screen && boundPolicy !== o.screen.policy) {
|
||||
return refuse(`run ${run.id} gated the ${boundPolicy} selection; a ${o.screen.policy} screen never adopts it`);
|
||||
}
|
||||
if (run.conclusion !== 'success' || binding.verdict !== 'GREEN' || binding.reconcile_exit !== 0) {
|
||||
return refuse(`the newest executed gate (run ${run.id}, ${binding.head_sha.slice(0, 12)}) is ${binding.verdict}, reconcile exit ${binding.reconcile_exit}, conclusion ${run.conclusion}`);
|
||||
}
|
||||
@@ -364,8 +397,17 @@ export function findReusableGate(o: FindOptions): { candidate: GateReuse | null;
|
||||
if (binding.profile !== 'pr') return refuse(`run ${run.id} ran the ${binding.profile} profile, not the PR gate`);
|
||||
if (binding.head_sha === o.headSha) return refuse(`run ${run.id} gated this same head; a re-run executes fresh`);
|
||||
|
||||
const fetched = git(['fetch', '--no-tags', '--depth=1', 'origin', binding.head_sha, o.headSha, o.testedSha]);
|
||||
if (screenRun) {
|
||||
const commits = gh(['api', `repos/${o.repo}/pulls/${o.pr}/commits`, '--paginate', '--jq', '.[].sha'], left());
|
||||
if (commits.status !== 0) return refuse(`cannot list PR #${o.pr}'s commits (${ghFailureReason(commits.stderr)})`);
|
||||
if (!commits.stdout.split('\n').includes(binding.head_sha)) return refuse(`screen run ${run.id}'s head ${binding.head_sha.slice(0, 12)} is not a commit of PR #${o.pr}`);
|
||||
}
|
||||
const fetched = git(['fetch', '--no-tags', '--depth=1', 'origin', binding.head_sha, o.headSha, o.testedSha, ...(binding.tested_sha ? [binding.tested_sha] : [])]);
|
||||
if (!fetched.ok) return refuse(`cannot fetch the gated and current commits (${fetched.out.split('\n')[0]})`);
|
||||
if (screenRun) {
|
||||
const gatedBase = testedBase(git, binding.tested_sha!, binding.head_sha);
|
||||
if ('error' in gatedBase || gatedBase.base !== binding.base_sha) return refuse(`screen run ${run.id}'s tested commit does not merge its head into its bound base (${'error' in gatedBase ? gatedBase.error : gatedBase.base.slice(0, 12)})`);
|
||||
}
|
||||
const tested = testedBase(git, o.testedSha, o.headSha);
|
||||
if ('error' in tested) return refuse(tested.error);
|
||||
const baseSha = tested.base;
|
||||
@@ -376,7 +418,8 @@ export function findReusableGate(o: FindOptions): { candidate: GateReuse | null;
|
||||
if (isUnavailable(policy)) return refuse(`the base's ship policy is unavailable (${policy.detail})`);
|
||||
if (policy.errors.length) return refuse(`the base's ship policy is invalid (${policy.errors[0]!.message})`);
|
||||
const common = { schema: 1 as const, pr: o.pr, reused_run_id: run.id, reused_run_attempt: binding.run_attempt, gated_head: binding.head_sha,
|
||||
current_head: o.headSha, gated_base: binding.base_sha, base_sha: baseSha, selection_sha256: binding.selection_sha256 };
|
||||
current_head: o.headSha, gated_base: binding.base_sha, base_sha: baseSha, selection_sha256: binding.selection_sha256,
|
||||
...(binding.screen_policy ? { reused_screen_policy: binding.screen_policy } : {}) };
|
||||
|
||||
let candidate: GateReuse;
|
||||
if (binding.base_sha === baseSha) {
|
||||
@@ -439,7 +482,7 @@ export function reuseReport(reuse: GateReuse, repo: string): string {
|
||||
: [`OWN DIFF: identical sha256=${reuse.own_diff_sha256} (merge-base ${reuse.gated_merge_base} -> ${reuse.merge_base}; stamp files excluded)`];
|
||||
return [
|
||||
`GATE REUSED (${reuse.kind === 'restamp-only' ? 'R2a, restamp-only' : 'R2, main moved'}): no paid case ran on ${reuse.current_head}`,
|
||||
`reused verdict: GREEN from run ${reuse.reused_run_id} attempt ${reuse.reused_run_attempt} at ${reuse.gated_head} (https://github.com/${repo}/actions/runs/${reuse.reused_run_id})`,
|
||||
`reused verdict: GREEN from run ${reuse.reused_run_id} attempt ${reuse.reused_run_attempt}${reuse.reused_screen_policy ? ` (a ${reuse.reused_screen_policy} screen)` : ''} at ${reuse.gated_head} (https://github.com/${repo}/actions/runs/${reuse.reused_run_id})`,
|
||||
`why: ${reuse.why}`,
|
||||
`base: ${reuse.gated_base} -> ${reuse.base_sha}`,
|
||||
...evidence,
|
||||
@@ -450,9 +493,11 @@ export function reuseReport(reuse: GateReuse, repo: string): string {
|
||||
|
||||
const USAGE = `usage:
|
||||
bun run scripts/gate-reuse.ts bind <report dir> --out <dir> --repo <owner/name> --pr <n> --run-id <id> --run-attempt <n>
|
||||
--head-sha <sha> --tested-sha <merge sha> --profile <pr|full> --reconcile-exit <n>
|
||||
--head-sha <sha> --tested-sha <merge sha> --profile <pr|full> --reconcile-exit <n> [--purpose pr|screen|train|bisect|dispatch]
|
||||
[--screen-policy narrow|broad]
|
||||
bun run scripts/gate-reuse.ts find --repo <owner/name> --pr <n> --head-repo <owner/name> --branch <head branch> --run-id <id>
|
||||
--head-sha <sha> --tested-sha <merge sha> --out <dir> [--budget-seconds 90]
|
||||
--head-sha <sha> --tested-sha <merge sha> --out <dir> [--screen-policy narrow|broad] [--budget-seconds 90]
|
||||
--screen-policy marks a dispatched screen (its tested sha is the pinned refs/pull/<n>/merge commit).
|
||||
bun run scripts/gate-reuse.ts confirm <candidate dir> <manifest.json> --out <paid-plan dir> [--github-output <file>]
|
||||
bun run scripts/gate-reuse.ts report <gate-reuse.json> --repo <owner/name>
|
||||
bind and find never fail the run; no candidate means the full gate runs.`;
|
||||
@@ -468,9 +513,12 @@ if (import.meta.main) {
|
||||
const git = gitIn(process.cwd());
|
||||
if (!commitParents(git, testedSha!)) git(['fetch', '--no-tags', '--depth=1', 'origin', testedSha!]);
|
||||
const tested = testedBase(git, testedSha!, headSha!);
|
||||
const purpose = (flag('purpose') || 'pr') as RunPurpose;
|
||||
if (!['pr', 'screen', 'train', 'bisect', 'dispatch'].includes(purpose) || (purpose === 'screen' && !['narrow', 'broad'].includes(flag('screen-policy') ?? ''))) usage();
|
||||
const { binding, reason } = buildBinding({ repository: repository!, pr: int('pr'), runId: int('run-id'), runAttempt: int('run-attempt'),
|
||||
headSha: headSha!, baseSha: 'base' in tested ? tested.base : null, profile: profile!,
|
||||
reconcileExit: /^\d+$/.test(flag('reconcile-exit') ?? '') ? int('reconcile-exit') : -1, reportDir: reportDir! });
|
||||
reconcileExit: /^\d+$/.test(flag('reconcile-exit') ?? '') ? int('reconcile-exit') : -1, reportDir: reportDir!,
|
||||
purpose, testedSha: testedSha!, screenPolicy: flag('screen-policy') });
|
||||
if (binding) {
|
||||
fs.mkdirSync(out!, { recursive: true });
|
||||
fs.writeFileSync(path.join(out!, BINDING_FILE), `${JSON.stringify(binding, null, 2)}\n`);
|
||||
@@ -478,9 +526,12 @@ if (import.meta.main) {
|
||||
console.log(`- Gate binding: ${reason}${'error' in tested ? ` (${tested.error})` : ''}`);
|
||||
} else if (command === 'find') {
|
||||
const [repo, headRepo, branch, headSha, testedSha, out] = [flag('repo'), flag('head-repo'), flag('branch'), flag('head-sha'), flag('tested-sha'), flag('out')];
|
||||
if (!repo || !headRepo || !branch || !headSha || !testedSha || !out || !Number.isSafeInteger(int('pr')) || !Number.isSafeInteger(int('run-id'))) usage();
|
||||
const policy = flag('screen-policy');
|
||||
if (!repo || !headRepo || !branch || !headSha || !testedSha || !out || !Number.isSafeInteger(int('pr')) || !Number.isSafeInteger(int('run-id'))
|
||||
|| (policy !== undefined && policy !== 'narrow' && policy !== 'broad')) usage();
|
||||
const { lines } = findReusableGate({ repo: repo!, pr: int('pr'), headRepo: headRepo!, branch: branch!, selfRunId: int('run-id'), headSha: headSha!,
|
||||
testedSha: testedSha!, repoRoot: process.cwd(), out: out!, budgetMs: Number(flag('budget-seconds') ?? '90') * 1000 });
|
||||
testedSha: testedSha!, repoRoot: process.cwd(), out: out!, budgetMs: Number(flag('budget-seconds') ?? '90') * 1000,
|
||||
...(policy ? { screen: { policy: policy as ScreenPolicy } } : {}) });
|
||||
console.log('### Paid-gate verdict reuse (R2a restamp-only, R2 main moved)');
|
||||
for (const line of lines) console.log(`- ${line}`);
|
||||
} else if (command === 'confirm') {
|
||||
|
||||
@@ -57,6 +57,7 @@ export interface WorkflowRun {
|
||||
updated_at?: string;
|
||||
head_repository?: { full_name?: string } | null;
|
||||
pull_requests?: Array<{ number: number }>;
|
||||
display_title?: string;
|
||||
}
|
||||
|
||||
/** One `gh` invocation; injectable so free tests drive recovery from recorded responses. */
|
||||
@@ -104,6 +105,38 @@ export function acceptedRun(run: WorkflowRun, scope: { pr: number; headRepo: str
|
||||
&& (run.pull_requests ?? []).some(pr => pr.number === scope.pr);
|
||||
}
|
||||
|
||||
/** The PR a dispatched screen names in its run name (evals.yml: `E2E Evals [eval-bg <nonce>] screen #<n>`); null for any other run. */
|
||||
export function screenRunPr(run: Pick<WorkflowRun, 'event' | 'display_title'>): number | null {
|
||||
const match = run.event === 'workflow_dispatch' ? /^E2E Evals \[eval-bg [A-Za-z0-9._-]+\] screen #(\d+)$/.exec(run.display_title ?? '') : null;
|
||||
return match ? Number(match[1]) : null;
|
||||
}
|
||||
|
||||
/** A dispatched screen of this PR from the base repository (same-repository PRs only), never this run. */
|
||||
export function acceptedScreenRun(run: WorkflowRun, scope: { pr: number; repo: string; selfRunId: number }): boolean {
|
||||
return run.id !== scope.selfRunId && screenRunPr(run) === scope.pr && run.head_repository?.full_name === scope.repo;
|
||||
}
|
||||
|
||||
function listRuns(gh: GhRunner, repo: string, filter: string[], timeoutMs: number): WorkflowRun[] | { error: string } {
|
||||
const listed = gh(['api', '-X', 'GET', `repos/${repo}/actions/workflows/evals.yml/runs`, ...filter.flatMap(f => ['-f', f]), '-f', 'per_page=50'], timeoutMs);
|
||||
if (listed.status !== 0) return { error: ghFailureReason(listed.stderr) };
|
||||
try { return (JSON.parse(listed.stdout).workflow_runs ?? []) as WorkflowRun[]; } catch { return { error: 'gh returned an unreadable run list' }; }
|
||||
}
|
||||
|
||||
/**
|
||||
* The PR's evals.yml runs whose results may carry over: its pull_request runs
|
||||
* and, with `screens`, its dispatched screens (both events, filtered by the PR).
|
||||
*/
|
||||
export function listGateRuns(gh: GhRunner, o: { repo: string; pr: number; headRepo: string; branch: string; selfRunId: number; screens: boolean },
|
||||
left: () => number): { runs: WorkflowRun[] } | { error: string } {
|
||||
const prRuns = listRuns(gh, o.repo, ['event=pull_request', `branch=${o.branch}`], left());
|
||||
if ('error' in prRuns) return prRuns;
|
||||
const runs = prRuns.filter(run => acceptedRun(run, { pr: o.pr, headRepo: o.headRepo, selfRunId: o.selfRunId }));
|
||||
if (!o.screens) return { runs };
|
||||
const screens = listRuns(gh, o.repo, ['event=workflow_dispatch'], left());
|
||||
if ('error' in screens) return screens;
|
||||
return { runs: [...runs, ...screens.filter(run => acceptedScreenRun(run, { pr: o.pr, repo: o.repo, selfRunId: o.selfRunId }))] };
|
||||
}
|
||||
|
||||
/**
|
||||
* Negative receipts a cancelled run's completed work proves (ENG-5): every
|
||||
* completed FAIL panel whose trials share one input identity, and every
|
||||
|
||||
@@ -0,0 +1,212 @@
|
||||
/**
|
||||
* The main backstop (evals-main-backstop.yml; plan GSTA-49 lane T1b): every
|
||||
* push to main is covered by a full gate, and a train's merges are covered by
|
||||
* the gate the train already ran, so a train costs no second gate.
|
||||
*
|
||||
* 1. Inert unless main's .gstack/ship-policy.json declares a `train` block.
|
||||
* 2. Debounce: wait until no train is merging (refs/notes/gstack-trains
|
||||
* holds no train in state `merging`), up to the settle window; a newer
|
||||
* push to main supersedes this run.
|
||||
* 3. Compute main's tree modulo stamps (scripts/train-gate.ts stampFreeTree)
|
||||
* and look for a green-train record that lists it among its prefix trees
|
||||
* T1..Tk: a successful train dispatch of the gate workflow whose record
|
||||
* artifact is GREEN with a complete census and whose `full_gate_check`
|
||||
* check run on the train head succeeded.
|
||||
* 4. Otherwise dispatch the full gate on main (eval-bg's backstop nonce),
|
||||
* wait for it, and on anything but success write the freeze record
|
||||
* gstack-ship-queue reads (red cases, their declared inputs, the run, an
|
||||
* issue) to refs/notes/gstack-trains, unless a freeze is already active.
|
||||
*/
|
||||
import { randomBytes } from 'node:crypto';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { loadPolicyFromHead, type TrainPolicy } from '../lib/ship-policy';
|
||||
import { NOTES_REF, fetchNotes, nowIso, persist, repoIdentity, type FreezeRecord } from '../lib/ship-queue-train-record';
|
||||
import { E2E_TOUCHFILES, LLM_JUDGE_TOUCHFILES } from '../test/helpers/touchfiles-data';
|
||||
import { DISPATCH_INPUTS, makeNonce, nonceScope } from './eval-bg';
|
||||
import { ghFailureReason, runGh, type GhRunner } from './recover-receipts';
|
||||
import { gitIn, stampFreeTree, type Git, type TrainRecord } from './train-gate';
|
||||
|
||||
const SHA = /^[0-9a-f]{40}$/;
|
||||
const GH_MS = 60_000;
|
||||
|
||||
export type BackstopAction = 'inert' | 'superseded' | 'covered' | 'deferred' | 'gated' | 'frozen' | 'failed';
|
||||
export interface BackstopOptions {
|
||||
repo: string; branch: string; sha: string; repoRoot: string;
|
||||
gh?: GhRunner; git?: Git; now?: () => number; sleep?: (ms: number) => void;
|
||||
settleMs?: number; pollMs?: number; gateDeadlineMs?: number; random?: string;
|
||||
}
|
||||
interface Run { id: number; status: string; conclusion: string | null; head_sha: string; display_title?: string; html_url?: string }
|
||||
|
||||
function json<T>(r: { status: number | null; stdout: string }): T | null {
|
||||
if (r.status !== 0) return null;
|
||||
try { return JSON.parse(r.stdout) as T; } catch { return null; }
|
||||
}
|
||||
|
||||
/** Notes records of one kind for this repository (the local notes ref, after a fetch). */
|
||||
function noteRecords<T extends { kind: string; repo_key?: string }>(git: Git, kind: string, repoKey: string): T[] {
|
||||
const listed = git(['notes', `--ref=${NOTES_REF}`, 'list']);
|
||||
if (!listed.ok || !listed.out.trim()) return [];
|
||||
return listed.out.split('\n').map(line => line.split(' ')[0]!).filter(Boolean).flatMap(blob => {
|
||||
try { const r = JSON.parse(git(['cat-file', 'blob', blob]).out) as T; return r?.kind === kind && r.repo_key === repoKey ? [r] : []; } catch { return []; }
|
||||
});
|
||||
}
|
||||
|
||||
/** The newest green-train record whose prefix trees include this digest, with its run; null when none. */
|
||||
export function coveringRecord(gh: GhRunner, repo: string, train: TrainPolicy, digest: string, lines: string[]): { run: Run; record: TrainRecord; index: number } | null {
|
||||
const runs = json<{ workflow_runs?: Run[] }>(gh(['api', '-X', 'GET', `repos/${repo}/actions/workflows/${train.gate_workflow}/runs`, '-f', 'event=workflow_dispatch', '-f', 'status=success', '-f', 'per_page=50'], GH_MS));
|
||||
if (!runs) { lines.push('cannot list the train gate runs; the backstop gates main'); return null; }
|
||||
const trains = (runs.workflow_runs ?? []).map(run => ({ run, nonce: /\[eval-bg ([A-Za-z0-9._-]+)\]/.exec(run.display_title ?? '')?.[1] ?? '' }))
|
||||
.filter(t => t.run.conclusion === 'success' && nonceScope(t.nonce)?.purpose === 'train').sort((a, b) => b.run.id - a.run.id).slice(0, 20);
|
||||
const scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'train-backstop-'));
|
||||
try {
|
||||
for (const { run, nonce } of trains) {
|
||||
const dir = path.join(scratch, String(run.id));
|
||||
if (gh(['run', 'download', String(run.id), '-R', repo, '-n', train.record_artifact, '-D', dir], GH_MS).status !== 0) continue;
|
||||
const file = fs.existsSync(dir) ? fs.readdirSync(dir).find(name => name.endsWith('.json')) : undefined;
|
||||
let record: TrainRecord | null = null;
|
||||
try { record = file ? JSON.parse(fs.readFileSync(path.join(dir, file), 'utf8')) as TrainRecord : null; } catch { record = null; }
|
||||
if (!record || record.kind !== 'train-gate' || record.verdict !== 'GREEN' || record.census?.complete !== true || record.nonce !== nonce
|
||||
|| record.head_sha !== run.head_sha || !Array.isArray(record.prefixes)) continue;
|
||||
const hit = record.prefixes.find(p => p.stamp_free_tree === digest);
|
||||
if (!hit) continue;
|
||||
const checks = json<{ check_runs?: Array<{ name?: string; conclusion?: string | null }> }>(
|
||||
gh(['api', `repos/${repo}/commits/${run.head_sha}/check-runs?check_name=${encodeURIComponent(train.full_gate_check)}`], GH_MS));
|
||||
if (!checks?.check_runs?.some(c => c.name === train.full_gate_check && c.conclusion === 'success')) {
|
||||
lines.push(`train ${nonce} (run ${run.id}) lists this tree but its ${train.full_gate_check} check run is not a success`);
|
||||
continue;
|
||||
}
|
||||
return { run, record, index: hit.index };
|
||||
}
|
||||
return null;
|
||||
} finally {
|
||||
fs.rmSync(scratch, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/** Red cases of a finished gate run, from its trial outcomes (first attempt), with their declared inputs. */
|
||||
function redCases(gh: GhRunner, repo: string, run: Run): Array<{ id: string; inputs: string[] }> {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'train-backstop-red-'));
|
||||
try {
|
||||
if (gh(['run', 'download', String(run.id), '-R', repo, '-n', 'trial-outcomes-pr-a1', '-D', dir], GH_MS).status !== 0) return [];
|
||||
const text = fs.existsSync(path.join(dir, 'trial-outcomes.jsonl')) ? fs.readFileSync(path.join(dir, 'trial-outcomes.jsonl'), 'utf8') : '';
|
||||
const ids = new Set<string>();
|
||||
for (const line of text.split('\n').filter(Boolean)) {
|
||||
try { const r = JSON.parse(line) as { case?: string; outcome?: string; attempt?: number }; if (r.outcome === 'failed' && (r.attempt ?? 1) === 1 && r.case) ids.add(r.case); } catch { /* skip */ }
|
||||
}
|
||||
return [...ids].sort().map(id => ({ id, inputs: E2E_TOUCHFILES[id] ?? LLM_JUDGE_TOUCHFILES[id] ?? [] }));
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/** Dispatch the full gate on main, find its run by nonce, and follow it to its conclusion. */
|
||||
function gateMain(o: Required<Pick<BackstopOptions, 'repo' | 'branch' | 'sha'>> & { gh: GhRunner; train: TrainPolicy; now: () => number; sleep: (ms: number) => void; pollMs: number; deadline: number; nonce: string },
|
||||
lines: string[]): Run | null {
|
||||
const inputs = { [DISPATCH_INPUTS.evalsAll]: 'true', [DISPATCH_INPUTS.expectedSha]: o.sha, [DISPATCH_INPUTS.nonce]: o.nonce };
|
||||
const sent = o.gh(['workflow', 'run', o.train.gate_workflow, '--repo', o.repo, '--ref', o.branch, ...Object.entries(inputs).flatMap(([k, v]) => ['-f', `${k}=${v}`])], GH_MS);
|
||||
if (sent.status !== 0) { lines.push(`the gate dispatch failed (${ghFailureReason(sent.stderr)})`); return null; }
|
||||
lines.push(`BACKSTOP: dispatched ${o.train.gate_workflow} on ${o.branch}@${o.sha.slice(0, 12)} (nonce ${o.nonce})`);
|
||||
let run: Run | null = null;
|
||||
while (o.now() < o.deadline) {
|
||||
if (!run) {
|
||||
const runs = json<{ workflow_runs?: Run[] }>(o.gh(['api', '-X', 'GET', `repos/${o.repo}/actions/workflows/${o.train.gate_workflow}/runs`, '-f', 'event=workflow_dispatch', '-f', 'per_page=30'], GH_MS));
|
||||
run = runs?.workflow_runs?.find(r => (r.display_title ?? '').includes(`eval-bg ${o.nonce}]`)) ?? null;
|
||||
if (run) lines.push(`BACKSTOP: gate run ${run.id} ${run.html_url ?? ''}`.trim());
|
||||
} else {
|
||||
run = json<Run>(o.gh(['api', `repos/${o.repo}/actions/runs/${run.id}`], GH_MS)) ?? run;
|
||||
if (run.status === 'completed') return run;
|
||||
}
|
||||
o.sleep(o.pollMs);
|
||||
}
|
||||
lines.push(`the gate ${run ? `run ${run.id}` : 'dispatch'} did not finish before the backstop deadline`);
|
||||
return run;
|
||||
}
|
||||
|
||||
export function runBackstop(o: BackstopOptions): { action: BackstopAction; lines: string[] } {
|
||||
const gh = o.gh ?? runGh;
|
||||
const git = o.git ?? gitIn(o.repoRoot);
|
||||
const now = o.now ?? Date.now;
|
||||
const sleep = o.sleep ?? ((ms: number) => Bun.sleepSync(ms));
|
||||
const pollMs = o.pollMs ?? 60_000;
|
||||
const lines: string[] = [];
|
||||
const done = (action: BackstopAction) => ({ action, lines });
|
||||
const loaded = loadPolicyFromHead(o.repoRoot);
|
||||
const train = loaded.policy.train;
|
||||
if (!train) { lines.push(`BACKSTOP: inert (${loaded.source.label} declares no train block; per-PR gates cover main)`); return done('inert'); }
|
||||
if (!SHA.test(o.sha)) { lines.push(`BACKSTOP: ${o.sha} is not a commit SHA`); return done('failed'); }
|
||||
const repoKey = repoIdentity(o.repoRoot).key;
|
||||
const settleUntil = now() + (o.settleMs ?? 60 * 60_000);
|
||||
for (;;) {
|
||||
const fetched = fetchNotes(o.repoRoot);
|
||||
const merging = noteRecords<{ kind: string; repo_key: string; nonce: string; state: string }>(git, 'train', repoKey).filter(t => t.state === 'merging');
|
||||
const tip = git(['ls-remote', 'origin', `refs/heads/${o.branch}`]).out.split(/\s/)[0];
|
||||
if (tip && tip !== o.sha) { lines.push(`BACKSTOP: superseded (${o.branch} moved to ${tip.slice(0, 12)}; that push's run gates it)`); return done('superseded'); }
|
||||
if (!merging.length) break;
|
||||
if (now() >= settleUntil) { lines.push(`BACKSTOP: train(s) ${merging.map(t => t.nonce).join(', ')} still merging after the settle window; gating anyway`); break; }
|
||||
lines.push(`BACKSTOP: waiting for train(s) ${merging.map(t => t.nonce).join(', ')} to finish merging${fetched.ok ? '' : ` (${fetched.detail})`}`);
|
||||
sleep(pollMs);
|
||||
}
|
||||
const digest = stampFreeTree(git, o.sha, o.repoRoot, loaded.policy);
|
||||
if (typeof digest !== 'string') { lines.push(`BACKSTOP: no tree modulo stamps (${digest.error}); gating main`); }
|
||||
const covered = typeof digest === 'string' ? coveringRecord(gh, o.repo, train, digest, lines) : null;
|
||||
if (covered) {
|
||||
lines.push(`BACKSTOP: covered: main ${o.sha.slice(0, 12)} is T${covered.index} of green train ${covered.record.nonce} (run ${covered.run.id}, tree modulo stamps ${String(digest).slice(0, 12)}); no gate`);
|
||||
return done('covered');
|
||||
}
|
||||
const owned = noteRecords<FreezeRecord>(git, 'freeze', repoKey).find(f => !f.cleared_at && f.main.sha === o.sha && f.backstop);
|
||||
if (owned?.backstop) {
|
||||
lines.push(`BACKSTOP: deferred: train ${owned.train} froze the queue on this commit and dispatched backstop run ${owned.backstop.run ?? 'unknown'} (nonce ${owned.backstop.nonce}); no second gate`);
|
||||
return done('deferred');
|
||||
}
|
||||
lines.push(`BACKSTOP: no green-train record lists main's tree modulo stamps${typeof digest === 'string' ? ` ${digest.slice(0, 12)}` : ''}; gating main`);
|
||||
const nonce = makeNonce('gate', o.sha, o.sha, o.random ?? randomBytes(3).toString('hex'), { purpose: 'backstop' });
|
||||
const run = gateMain({ repo: o.repo, branch: o.branch, sha: o.sha, gh, train, now, sleep, pollMs, deadline: now() + (o.gateDeadlineMs ?? 270 * 60_000), nonce }, lines);
|
||||
if (run?.status === 'completed' && run.conclusion === 'success') { lines.push(`BACKSTOP: gated: run ${run.id} is green on main ${o.sha.slice(0, 12)}`); return done('gated'); }
|
||||
return freeze(o, { gh, git, repoKey, nonce, run }, lines);
|
||||
}
|
||||
|
||||
/** Write the freeze record (local state, then refs/notes/gstack-trains) unless a freeze is already active. */
|
||||
function freeze(o: BackstopOptions, c: { gh: GhRunner; git: Git; repoKey: string; nonce: string; run: Run | null }, lines: string[]): { action: BackstopAction; lines: string[] } {
|
||||
const active = noteRecords<FreezeRecord>(c.git, 'freeze', c.repoKey).find(f => !f.cleared_at);
|
||||
if (active) { lines.push(`BACKSTOP: red, and the queue is already frozen (${active.reason}); the existing freeze stands`); return { action: 'frozen', lines }; }
|
||||
const red = c.run?.status === 'completed' ? redCases(c.gh, o.repo, c.run) : [];
|
||||
const runText = c.run ? `run ${c.run.id} (${c.run.status}/${c.run.conclusion ?? '-'})` : 'no run';
|
||||
const cases = red.length ? red.map(r => `${r.id} (inputs ${r.inputs.slice(0, 4).join(', ') || 'none declared'})`).join('; ') : 'no red case recorded';
|
||||
const body = [`The main backstop gate is not green on ${o.sha}: ${runText}.`, '', `Red cases: ${cases}.`, '', 'gstack-ship-queue refuses ordinary trains until a repair train merges (docs/ship-policy.md#run-a-merge-train). Land the revert or fix-forward.'].join('\n');
|
||||
const issue = c.gh(['issue', 'create', '-R', o.repo, '--title', `Main backstop red on ${o.sha.slice(0, 12)}`, '--body', body], GH_MS);
|
||||
const issueUrl = issue.status === 0 ? issue.stdout.trim().split('\n').at(-1) ?? null : null;
|
||||
const at = nowIso();
|
||||
const record: FreezeRecord & { red_cases: typeof red; run_url: string | null; issue: string | null } = {
|
||||
schema: 1, kind: 'freeze', repo_key: c.repoKey, train: 'backstop', merged: [], unmerged: [],
|
||||
main: { sha: o.sha, tree: c.git(['rev-parse', `${o.sha}^{tree}`]).out },
|
||||
reason: `backstop red on main ${o.sha.slice(0, 12)}: ${cases}; ${runText}; fix: land the revert or fix-forward${issueUrl ? ` (${issueUrl})` : ''}`,
|
||||
backstop: { nonce: c.nonce, run: c.run?.id ?? null, state: c.run ? `${c.run.status}/${c.run.conclusion ?? '-'}` : 'unconfirmed' },
|
||||
repair: null, created_at: at, cleared_at: null, cleared_by: null, updated_at: at,
|
||||
red_cases: red, run_url: c.run?.html_url ?? null, issue: issueUrl,
|
||||
};
|
||||
const pushed = persist(o.repoRoot, [record]);
|
||||
lines.push(`BACKSTOP: frozen: ${record.reason}`, `BACKSTOP: freeze record ${pushed.detail}`);
|
||||
return { action: pushed.pushed ? 'frozen' : 'failed', lines };
|
||||
}
|
||||
|
||||
const USAGE = `usage: bun run scripts/train-backstop.ts run --repo <owner/name> --branch <main> --sha <sha>
|
||||
[--settle-minutes 60] [--poll-seconds 60] [--gate-deadline-minutes 270] [--summary <file>]
|
||||
Exit 0 when inert, superseded, covered, deferred to a freeze's own backstop, or gated green; 1 when it froze the queue or could not finish.`;
|
||||
|
||||
if (import.meta.main) {
|
||||
const [command, ...rest] = process.argv.slice(2);
|
||||
const flag = (name: string) => { const i = rest.indexOf(`--${name}`); return i >= 0 ? rest[i + 1] : undefined; };
|
||||
const minutes = (name: string, fallback: number) => Number(flag(name) ?? fallback) * 60_000;
|
||||
const [repo, branch, sha] = [flag('repo'), flag('branch'), flag('sha')];
|
||||
if (command !== 'run' || !repo || !branch || !sha) { console.error(USAGE); process.exit(2); }
|
||||
const { action, lines } = runBackstop({ repo: repo!, branch: branch!, sha: sha!, repoRoot: process.cwd(), settleMs: minutes('settle-minutes', 60),
|
||||
pollMs: Number(flag('poll-seconds') ?? 60) * 1000, gateDeadlineMs: minutes('gate-deadline-minutes', 270) });
|
||||
const summary = flag('summary');
|
||||
for (const line of lines) {
|
||||
console.log(line);
|
||||
if (summary) fs.appendFileSync(summary, `- ${line}\n`);
|
||||
}
|
||||
process.exit(action === 'frozen' || action === 'failed' ? 1 : 0);
|
||||
}
|
||||
@@ -0,0 +1,365 @@
|
||||
/**
|
||||
* The merge-train side of evals.yml (plan GSTA-49 lane T1b;
|
||||
* docs/ship-policy.md#run-a-merge-train). gstack-ship-queue (lib/ship-queue-train*.ts)
|
||||
* dispatches the runs and reads what they publish; this file is the workflow's half.
|
||||
*
|
||||
* route (route job, dispatches) names the run's mode and validates
|
||||
* its inputs: a screen needs a PR, its pinned merge SHA and a
|
||||
* policy; a bisect needs case ids from the full gate's own plan
|
||||
* and prefix SHAs that are ancestors of the train head on
|
||||
* train/<nonce>. Anything else is refused before any spend.
|
||||
* bisect-plan (plan-slices) one executor per prefix, each expecting its SHA.
|
||||
* bisect-prefix (eval-slices) one prefix: every red case once, one trial,
|
||||
* written as the per-prefix record `train bisect` reads.
|
||||
* train-record (slices-report, train runs) the green-train record: the census
|
||||
* from the run's own plan manifest, one panel per planned case,
|
||||
* and the prefix trees T1..Tk modulo stamps. GREEN only for a
|
||||
* complete census: every planned case executed and passed, no
|
||||
* unbound result, partial or cancelled shard, or slice killed
|
||||
* before its upload step.
|
||||
* screen-summary (plan-slices, PR runs under a train policy) the screening
|
||||
* selection and its estimated cost; no paid case runs.
|
||||
*/
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { createHash } from 'node:crypto';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { normalizeChangelog } from '../lib/changelog-check';
|
||||
import { normalizeLockfile } from '../lib/evidence-bundle';
|
||||
import { rewriteStamps, versionAtRef } from '../lib/restamp';
|
||||
import { loadPolicyFromHead, matchesAny, type ShipPolicy } from '../lib/ship-policy';
|
||||
import type { BisectPrefixRecord, TrainGateRecord } from '../lib/ship-queue-train-verdict';
|
||||
import type { CasePanel } from '../lib/ship-queue-train-record';
|
||||
import { E2E_TIERS, LLM_JUDGE_TOUCHFILES } from '../test/helpers/touchfiles-data';
|
||||
import { panelVerdict } from '../test/helpers/eval-store';
|
||||
import { UNRECORDED_SHARD_MS, nonceScope, plannedSerialMs } from './eval-bg';
|
||||
import { caseTrialPlan, fileCaseRegistration, shardCaseId, shardFile } from './lib/paid-cases';
|
||||
import { buildRunManifest, loadPaidTestDurations, parseRunManifest, recordedShardMs, type PaidRunManifest } from './lib/paid-plan';
|
||||
import { loadSliceArtifacts } from './lib/paid-report';
|
||||
import { paidSelectionEnv } from './lib/paid-select';
|
||||
import { ROOT } from './lib/paid-types';
|
||||
|
||||
const SHA = /^[0-9a-f]{40}$/;
|
||||
const CASE_ID = /^[A-Za-z0-9 _.:/-]{1,128}$/;
|
||||
export const JUDGE_FILE = 'test/skill-llm-eval.test.ts';
|
||||
/** T1 caps a train at 6 PRs, so a bisect has at most 7 prefixes (T0..T6). */
|
||||
export const MAX_PREFIXES = 7;
|
||||
export const MAX_BISECT_CASES = 40;
|
||||
/** A full gate's recorded cost (MEASUREMENT.md: $90.93 to $94.70; lib/gate-cost.ts uses the same scale). */
|
||||
export const FULL_GATE_USD = 93.14;
|
||||
/** T1's prefix commits: `train T<i>: merge #<pr> at <sha12>` (lib/ship-queue-train.ts buildChain). */
|
||||
const PREFIX_SUBJECT = /^train T(\d+): merge #(\d+) at [0-9a-f]{12}$/;
|
||||
|
||||
export type Git = (args: string[]) => { ok: boolean; out: string };
|
||||
export const gitIn = (cwd: string): Git => (args) => {
|
||||
const r = spawnSync('git', args, { cwd, encoding: 'utf8', timeout: 60_000, maxBuffer: 256 * 1024 * 1024 });
|
||||
return r.status === 0 ? { ok: true, out: (r.stdout ?? '').replace(/\n$/, '') } : { ok: false, out: (r.stderr || String(r.error ?? '')).trim() };
|
||||
};
|
||||
|
||||
export type RunMode = 'screen' | 'train' | 'bisect' | 'dispatch';
|
||||
export interface RouteInput {
|
||||
nonce: string; refName: string; head: string; expectedSha: string; evalsAll: boolean; trainPolicy: boolean;
|
||||
cases: string; prefixShas: string; screenPr: string; screenMergeSha: string; screenPolicy: string;
|
||||
}
|
||||
export interface Route { mode: RunMode; cases: string[]; prefixes: string[] }
|
||||
|
||||
const list = (text: string) => text.split(',').map(item => item.trim()).filter(Boolean);
|
||||
|
||||
/**
|
||||
* The run's mode from a dispatch's inputs, or why it is refused. Screens,
|
||||
* trains and bisects exist only under a train policy; a bisect's case ids must
|
||||
* be in the full gate's plan and each prefix an ancestor of the train head.
|
||||
*/
|
||||
export function routeDispatch(i: RouteInput, deps: { isAncestor(commit: string, head: string): boolean; gateCases(): Map<string, string> }): Route | { error: string } {
|
||||
const scope = i.nonce ? nonceScope(i.nonce) : null;
|
||||
const trainOnly = (mode: string) => (i.trainPolicy ? null : { error: `a ${mode} dispatch needs a train block in the default branch's .gstack/ship-policy.json` });
|
||||
if (i.screenPr || i.screenMergeSha || i.screenPolicy) {
|
||||
if (!/^[1-9]\d{0,8}$/.test(i.screenPr)) return { error: `screen_pr ${JSON.stringify(i.screenPr)} is not a PR number` };
|
||||
if (!SHA.test(i.screenMergeSha)) return { error: `screen_merge_sha ${JSON.stringify(i.screenMergeSha)} is not a full commit SHA` };
|
||||
if (i.screenPolicy !== 'narrow' && i.screenPolicy !== 'broad') return { error: `screen_policy ${JSON.stringify(i.screenPolicy)} is not narrow or broad` };
|
||||
if (i.cases || i.prefixShas) return { error: 'a screen takes no cases or prefix_shas' };
|
||||
if (i.evalsAll) return { error: 'a screen runs the PR selection: dispatch it with evals_all=false' };
|
||||
if (i.expectedSha && i.expectedSha !== i.screenMergeSha) return { error: `expected_sha ${i.expectedSha.slice(0, 12)} is not the screen's merge SHA ${i.screenMergeSha.slice(0, 12)}` };
|
||||
if (scope && (scope.purpose !== 'screen' || scope.policy !== i.screenPolicy)) return { error: `nonce ${i.nonce} is not a ${i.screenPolicy} screen nonce` };
|
||||
return trainOnly('screen') ?? { mode: 'screen', cases: [], prefixes: [] };
|
||||
}
|
||||
if (i.cases || i.prefixShas || scope?.purpose === 'bisect') {
|
||||
const cases = list(i.cases);
|
||||
const prefixes = list(i.prefixShas);
|
||||
if (scope?.purpose !== 'bisect') return { error: `a bisect needs a bisect nonce (got ${JSON.stringify(i.nonce)})` };
|
||||
if (!i.refName.startsWith('train/')) return { error: `a bisect runs on train/<nonce>, not ${i.refName}` };
|
||||
if (i.expectedSha !== i.head) return { error: `expected_sha ${i.expectedSha.slice(0, 12) || '(empty)'} is not the train head ${i.head.slice(0, 12)} checked out from ${i.refName}` };
|
||||
if (!cases.length || cases.length > MAX_BISECT_CASES || new Set(cases).size !== cases.length) return { error: `cases must name 1 to ${MAX_BISECT_CASES} distinct case ids` };
|
||||
if (!prefixes.length || prefixes.length > MAX_PREFIXES || new Set(prefixes).size !== prefixes.length) return { error: `prefix_shas must name 1 to ${MAX_PREFIXES} distinct commits` };
|
||||
const bad = prefixes.find(sha => !SHA.test(sha));
|
||||
if (bad) return { error: `prefix ${JSON.stringify(bad)} is not a full commit SHA` };
|
||||
const outside = prefixes.find(sha => !deps.isAncestor(sha, i.head));
|
||||
if (outside) return { error: `prefix ${outside.slice(0, 12)} is not an ancestor of the train head ${i.head.slice(0, 12)} on ${i.refName}` };
|
||||
const plan = deps.gateCases();
|
||||
const unknown = cases.filter(id => !CASE_ID.test(id) || !plan.has(id));
|
||||
if (unknown.length) return { error: `case(s) not in the full gate's plan: ${unknown.slice(0, 5).join(', ')}` };
|
||||
return trainOnly('bisect') ?? { mode: 'bisect', cases, prefixes };
|
||||
}
|
||||
if (scope?.purpose === 'train') {
|
||||
if (!i.refName.startsWith('train/')) return { error: `a train gate runs on train/<nonce>, not ${i.refName}` };
|
||||
if (!i.evalsAll) return { error: 'a train gate is the full gate: dispatch it with evals_all=true' };
|
||||
if (i.expectedSha !== i.head) return { error: `expected_sha ${i.expectedSha.slice(0, 12) || '(empty)'} is not the train head ${i.head.slice(0, 12)}` };
|
||||
return trainOnly('train') ?? { mode: 'train', cases: [], prefixes: [] };
|
||||
}
|
||||
return { mode: 'dispatch', cases: [], prefixes: [] };
|
||||
}
|
||||
|
||||
/** Every case a plan manifest plans, with its kind: the census a complete gate must execute. */
|
||||
export function plannedCases(manifest: PaidRunManifest, rootDir = ROOT): Map<string, string> {
|
||||
const cases = new Map<string, string>();
|
||||
const e2e = manifest.selection?.e2e ?? null;
|
||||
for (const entry of manifest.entries.filter(item => item.status === 'planned')) {
|
||||
const file = shardFile(entry.file);
|
||||
if (entry.trial) { cases.set(shardCaseId(entry.file)!, entry.trial.kind); continue; }
|
||||
if (file === JUDGE_FILE) { for (const id of manifest.selection?.judges ?? Object.keys(LLM_JUDGE_TOUCHFILES)) cases.set(id, 'judge'); continue; }
|
||||
const only = shardCaseId(entry.file);
|
||||
const source = fs.readFileSync(path.join(rootDir, file), 'utf8');
|
||||
for (const id of fileCaseRegistration(file, source).registered) {
|
||||
if (E2E_TIERS[id] === manifest.tier && (only === null || id === only) && (e2e === null || e2e.includes(id)) && !(entry.excludeCases ?? []).includes(id)) cases.set(id, 'rule');
|
||||
}
|
||||
}
|
||||
return cases;
|
||||
}
|
||||
|
||||
/** The full gate's planned cases on this checkout (what a bisect may name). */
|
||||
export function fullGateCases(rootDir = ROOT): Map<string, string> {
|
||||
return plannedCases(buildRunManifest({ tier: 'gate', profile: 'full', sliceCount: 1, evalsAll: true, env: { EVALS_ALL: '1' }, rootDir }), rootDir);
|
||||
}
|
||||
|
||||
/**
|
||||
* A commit's tree modulo stamps: every path with its mode and blob, except the
|
||||
* files a restamp rewrites, which contribute their content without the stamp
|
||||
* (the version file nothing, package.json without its version, lockfiles
|
||||
* without their root version, the CHANGELOG with its headings normalized,
|
||||
* release outputs and stamp paths with their own version replaced). Two
|
||||
* commits that differ only by a restamp have the same digest.
|
||||
*/
|
||||
export function stampFreeTree(git: Git, commit: string, repoRoot: string, policy: ShipPolicy): string | { error: string } {
|
||||
const listed = git(['ls-tree', '-r', '-z', '--full-tree', commit]);
|
||||
if (!listed.ok) return { error: `cannot list ${commit.slice(0, 12)} (${listed.out.split('\n')[0]})` };
|
||||
const at = versionAtRef(repoRoot, commit);
|
||||
const versionRel = 'rel' in at ? at.rel : 'VERSION';
|
||||
const version = 'version' in at ? at.version : null;
|
||||
const lines: string[] = [];
|
||||
for (const item of listed.out.split('\0').filter(Boolean)) {
|
||||
const [meta, rel] = [item.slice(0, item.indexOf('\t')), item.slice(item.indexOf('\t') + 1)];
|
||||
const [mode, , blob] = meta.split(' ') as [string, string, string];
|
||||
const base = path.basename(rel);
|
||||
const stamp = rel === versionRel ? 'version' : base === 'package.json' ? 'manifest' : ['bun.lock', 'package-lock.json', 'npm-shrinkwrap.json'].includes(base) ? 'lockfile'
|
||||
: rel === policy.changelog ? 'changelog' : matchesAny([...policy.stamp_paths, ...policy.release_outputs, ...policy.stamp_files], rel) ? 'release' : null;
|
||||
if (!stamp) { lines.push(`${rel}\0${mode}\0${blob}`); continue; }
|
||||
if (stamp === 'version') { lines.push(`${rel}\0${mode}\0(version)`); continue; }
|
||||
const shown = git(['cat-file', 'blob', blob]);
|
||||
if (!shown.ok) return { error: `cannot read ${rel} at ${commit.slice(0, 12)}` };
|
||||
let text = shown.out;
|
||||
if (stamp === 'manifest') { try { const parsed = JSON.parse(text); delete parsed.version; text = JSON.stringify(parsed); } catch { /* not JSON: kept verbatim */ } }
|
||||
else if (stamp === 'lockfile') text = normalizeLockfile(base, text);
|
||||
else if (stamp === 'changelog') text = normalizeChangelog(text);
|
||||
else if (version) text = rewriteStamps(text, version, '\0VERSION', policy.stamp_patterns).text;
|
||||
lines.push(`${rel}\0${mode}\0${createHash('sha256').update(text).digest('hex')}`);
|
||||
}
|
||||
return createHash('sha256').update(lines.sort().join('\n')).digest('hex');
|
||||
}
|
||||
|
||||
export interface TrainPrefixTree { index: number; commit: string; tree: string; stamp_free_tree: string }
|
||||
|
||||
/** T1..Tk from the train head, walking first parents over T1's prefix commits. */
|
||||
export function trainPrefixes(git: Git, head: string, repoRoot: string, policy: ShipPolicy): TrainPrefixTree[] | { error: string } {
|
||||
const log = git(['log', '--first-parent', `--max-count=${MAX_PREFIXES}`, '--format=%H%x00%P%x00%s', head]);
|
||||
if (!log.ok) return { error: `cannot walk ${head.slice(0, 12)}'s first parents (${log.out.split('\n')[0]})` };
|
||||
const found: TrainPrefixTree[] = [];
|
||||
for (const line of log.out.split('\n')) {
|
||||
const [commit, parents, subject] = line.split('\0') as [string, string, string];
|
||||
const m = PREFIX_SUBJECT.exec(subject ?? '');
|
||||
if (!m || parents.split(' ').length !== 2) break;
|
||||
const index = Number(m[1]);
|
||||
if (found.length && index !== found.at(-1)!.index - 1) return { error: `prefix T${index} follows T${found.at(-1)!.index} at ${commit.slice(0, 12)}` };
|
||||
const digest = stampFreeTree(git, commit, repoRoot, policy);
|
||||
if (typeof digest !== 'string') return digest;
|
||||
found.push({ index, commit, tree: git(['rev-parse', `${commit}^{tree}`]).out, stamp_free_tree: digest });
|
||||
}
|
||||
if (!found.length || found.at(-1)!.index !== 1) return { error: `${head.slice(0, 12)} is not a train head (no T1..Tk prefix chain)` };
|
||||
return found.reverse();
|
||||
}
|
||||
|
||||
interface OutcomeLine { case?: string; kind?: string; attempt?: number; outcome?: string; failure_class?: string; cost_usd?: number; cost_known?: boolean; execution?: string }
|
||||
interface CollectorPanel { case: string; status: string; redClass: string | null }
|
||||
|
||||
function readJson<T>(file: string): T | null {
|
||||
try { return JSON.parse(fs.readFileSync(file, 'utf8')) as T; } catch { return null; }
|
||||
}
|
||||
|
||||
/** One panel per planned case from the run's trial outcomes (trial panels from the collector's panelVerdict). */
|
||||
function casePanels(planned: Map<string, string>, outcomes: OutcomeLine[], collector: CollectorPanel[]): CasePanel[] {
|
||||
return [...planned].sort(([a], [b]) => (a < b ? -1 : 1)).map(([id, kind]) => {
|
||||
const mine = outcomes.filter(o => o.case === id && o.execution !== 'reused');
|
||||
const cost = mine.length && mine.every(o => o.cost_known !== false && typeof o.cost_usd === 'number')
|
||||
? Math.round(mine.reduce((sum, o) => sum + o.cost_usd!, 0) * 10_000) / 10_000 : null;
|
||||
const panel = collector.find(p => p.case === id);
|
||||
if (panel) return { case: id, kind, status: panel.status, red_class: panel.redClass, cost_usd: cost };
|
||||
const failed = mine.filter(o => o.outcome === 'failed');
|
||||
if (failed.length) return { case: id, kind, status: 'FAIL', red_class: failed.every(o => o.failure_class === 'infra') ? 'INFRA' : 'VERDICT', cost_usd: cost };
|
||||
if (mine.some(o => o.outcome === 'passed')) return { case: id, kind, status: 'PASS', red_class: null, cost_usd: cost };
|
||||
return { case: id, kind, status: mine.length ? 'SKIPPED' : 'MISSING', red_class: 'INCOMPLETE', cost_usd: cost };
|
||||
});
|
||||
}
|
||||
|
||||
export type TrainRecord = TrainGateRecord & { run: { id: number; attempt: number; url: string }; prefixes: TrainPrefixTree[] };
|
||||
|
||||
/** The green-train record a train's own run publishes (T1's TrainGateRecord plus the run and the prefix trees). */
|
||||
export function buildTrainRecord(o: { reportDir: string; nonce: string; head: string; tree: string; reconcileExit: number; rootDir?: string;
|
||||
prefixes: TrainPrefixTree[]; run: TrainRecord['run'] }): TrainRecord {
|
||||
const manifest = parseRunManifest(fs.readFileSync(path.join(o.reportDir, 'manifest.json'), 'utf8'));
|
||||
const planned = plannedCases(manifest, o.rootDir);
|
||||
let lines: OutcomeLine[] = [];
|
||||
try { lines = fs.readFileSync(path.join(o.reportDir, 'trial-outcomes.jsonl'), 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line) as OutcomeLine); } catch { lines = []; }
|
||||
const primary = Math.min(...lines.map(l => l.attempt ?? 1), Infinity);
|
||||
const collector = readJson<{ verdict?: { verdict?: string }; census?: { unattributed?: unknown[] }; panels?: CollectorPanel[] }>(path.join(o.reportDir, 'collector-outcomes.json'));
|
||||
const panels = casePanels(planned, lines.filter(l => (l.attempt ?? 1) === primary), collector?.panels ?? []);
|
||||
const artifacts = loadSliceArtifacts(o.reportDir);
|
||||
const first = Math.min(...artifacts.map(a => a.result.attempt ?? 1), Infinity);
|
||||
const results = artifacts.filter(a => (a.result.attempt ?? 1) === first).map(a => a.result);
|
||||
const reportedSlices = new Set(results.map(r => r.sliceIndex));
|
||||
const reportedFiles = new Set(results.flatMap(r => r.outcomes.flatMap(outcome => outcome.files)));
|
||||
const count = (judge: boolean, from: Iterable<[string, string]>) => [...from].filter(([, kind]) => (kind === 'judge') === judge).length;
|
||||
const executed = panels.filter(p => p.status === 'PASS' || p.status === 'FAIL').map(p => [p.case, p.kind] as [string, string]);
|
||||
const census = {
|
||||
complete: false,
|
||||
planned: { rule: count(false, planned), judge: count(true, planned) },
|
||||
executed: { rule: count(false, executed), judge: count(true, executed) },
|
||||
unbound: collector?.census?.unattributed?.length ?? 0,
|
||||
partial_shards: results.filter(r => r.checkpoint).length,
|
||||
cancelled_shards: manifest.entries.filter(e => e.status === 'planned' && reportedSlices.has(e.slice) && !reportedFiles.has(e.file)).length,
|
||||
killed_before_upload: Array.from({ length: manifest.sliceCount }, (_, i) => i + 1).filter(index => !reportedSlices.has(index)).length,
|
||||
};
|
||||
census.complete = o.reconcileExit === 0 && !!collector && planned.size > 0 && census.executed.rule === census.planned.rule
|
||||
&& census.executed.judge === census.planned.judge && !census.unbound && !census.partial_shards && !census.cancelled_shards
|
||||
&& !census.killed_before_upload && panels.every(p => p.status === 'PASS');
|
||||
const verdict = census.complete && collector?.verdict?.verdict === 'GREEN' ? 'GREEN' : 'RED';
|
||||
return { schema: 1, kind: 'train-gate', nonce: o.nonce, head_sha: o.head, tree: o.tree, verdict, census, panels, run: o.run, prefixes: o.prefixes };
|
||||
}
|
||||
|
||||
export interface BisectPlan { schema: 1; nonce: string; cases: Array<{ id: string; kind: string }>; prefixes: string[] }
|
||||
|
||||
/** One executor per prefix; each case runs one trial, so a slice's ceiling grows with the case count. */
|
||||
export function bisectMatrix(plan: BisectPlan): { slices: number[]; timeouts: number[]; refs: string[] } {
|
||||
const minutes = Math.min(350, 20 + 30 * plan.cases.length);
|
||||
return { slices: plan.prefixes.map((_, i) => i + 1), timeouts: [0, ...plan.prefixes.map(() => minutes)], refs: ['', ...plan.prefixes] };
|
||||
}
|
||||
|
||||
export type CaseRunner = (id: string, kind: string, evalDir: string) => Promise<{ status: string; redClass: string | null }>;
|
||||
|
||||
/** Run every red case once on the checked-out prefix; a case that leaves no verdict is INCOMPLETE (T1 relaunches it). */
|
||||
export async function runBisectPrefix(plan: BisectPlan, index: number, o: { rootDir: string; evalDir: string; checkedOut: string; runCase?: CaseRunner }): Promise<BisectPrefixRecord> {
|
||||
const sha = plan.prefixes[index - 1];
|
||||
if (!sha) throw new Error(`bisect slice ${index} names no prefix (${plan.prefixes.length} planned)`);
|
||||
if (o.checkedOut !== sha) throw new Error(`bisect slice ${index} checked out ${o.checkedOut.slice(0, 12)}, not its prefix ${sha.slice(0, 12)}`);
|
||||
const runCase = o.runCase ?? ((id, kind, evalDir) => runOneTrial(id, kind, evalDir, o.rootDir));
|
||||
const panels: CasePanel[] = [];
|
||||
for (const c of plan.cases) {
|
||||
let verdict: { status: string; redClass: string | null };
|
||||
try { verdict = await runCase(c.id, c.kind, path.join(o.evalDir, c.id.replace(/[^A-Za-z0-9._-]/g, '_'))); }
|
||||
catch (error) { console.error(`[train-gate] ${c.id}: ${error instanceof Error ? error.message : String(error)}`); verdict = { status: 'INCOMPLETE', redClass: 'INCOMPLETE' }; }
|
||||
panels.push({ case: c.id, kind: c.kind, status: verdict.status, red_class: verdict.redClass, cost_usd: null });
|
||||
console.log(`[train-gate] prefix ${sha.slice(0, 12)} (slice ${index}): ${c.id} ${verdict.status}${verdict.redClass ? ` (${verdict.redClass})` : ''}`);
|
||||
}
|
||||
return { schema: 1, kind: 'train-bisect-prefix', nonce: plan.nonce, prefix_sha: sha, checked_out_sha: o.checkedOut, panels };
|
||||
}
|
||||
|
||||
/** One trial of one case exactly as CI runs it: E2E cases through the panel runner, a judge through its file with only it selected. */
|
||||
async function runOneTrial(id: string, kind: string, evalDir: string, rootDir: string): Promise<{ status: string; redClass: string | null }> {
|
||||
const { runPaidShards } = await import('./test-paid-shards');
|
||||
if (kind !== 'judge') {
|
||||
const { runCaseDiagnosis } = await import('./lib/paid-report');
|
||||
const v = await runCaseDiagnosis(id, { trials: 1, jobs: 1, rootDir, evalDirBase: evalDir, runShards: runPaidShards });
|
||||
return { status: v.status, redClass: v.redClass };
|
||||
}
|
||||
const env = { ...process.env, EVALS: '1', EVALS_TIER: 'gate', EVALS_PREFLIGHT_OK: '1', EVALS_ALL: '1', ...paidSelectionEnv('full', { e2e: [], judges: [id] }, 'train bisect') };
|
||||
const [outcome] = (await runPaidShards([[JUDGE_FILE]], { jobs: 1, rootDir, env, evalDirBase: evalDir })).outcomes;
|
||||
const trials = outcome && outcome.runnerError === undefined && ['passed', 'failed', 'timed-out'].includes(outcome.status)
|
||||
? [{ trial: 1, outcome: outcome.status === 'passed' ? 'passed' as const : 'failed' as const,
|
||||
...(outcome.status === 'passed' ? {} : { failure_class: outcome.status === 'timed-out' ? 'timeout' as const : 'assertion' as const }) }] : [];
|
||||
const v = panelVerdict({ case: id, kind: 'judge', panel: { n: 1, k: 1 }, trials, quarantined: caseTrialPlan(id).quarantined });
|
||||
return { status: v.status, redClass: v.redClass };
|
||||
}
|
||||
|
||||
/** The free summary of a PR push under a train policy: how to ask for the screen, then its selection and estimated cost. */
|
||||
export function screenSummary(manifest: PaidRunManifest, pr: string, rootDir = ROOT): string[] {
|
||||
const planned = plannedCases(manifest, rootDir);
|
||||
const recorded = loadPaidTestDurations(rootDir, manifest.tier);
|
||||
const serial = manifest.entries.filter(e => e.status === 'planned').reduce((sum, e) => sum + (recordedShardMs(recorded, e.file) ?? UNRECORDED_SHARD_MS), 0);
|
||||
const full = plannedSerialMs('gate', rootDir);
|
||||
const usd = full > 0 ? Math.min(FULL_GATE_USD, FULL_GATE_USD * serial / full) : null;
|
||||
const judges = [...planned.values()].filter(kind => kind === 'judge').length;
|
||||
return [
|
||||
`To run the paid screen of this head: \`gstack-ship-queue ready --pr ${pr} --screen\` (add \`--broad\` for today's full PR selection). PR pushes start no paid slices under this repository's train policy; the train's full gate covers every merge.`,
|
||||
`- Screening selection: ${planned.size - judges} case(s) + ${judges} judge(s), PR coverage mode \`${manifest.prCoverage?.mode ?? 'broad'}\``,
|
||||
`- Estimated screen cost: ${usd === null ? 'unknown' : `~$${usd.toFixed(2)}`} (recorded shard durations scaled to the $${FULL_GATE_USD} full gate; shards without a recorded duration count ${UNRECORDED_SHARD_MS / 60_000} min). No paid case ran on this push.`,
|
||||
];
|
||||
}
|
||||
|
||||
const USAGE = `usage:
|
||||
bun run scripts/train-gate.ts route --github-output <file> (inputs from the route job's env)
|
||||
bun run scripts/train-gate.ts bisect-plan --cases <ids> --prefixes <shas> --nonce <nonce> --out <bisect.json> [--github-output <file>]
|
||||
bun run scripts/train-gate.ts bisect-prefix --plan <bisect.json> --index <n> --out <dir>
|
||||
bun run scripts/train-gate.ts train-record <report dir> --nonce <nonce> --head <sha> --reconcile-exit <n> --run-id <id> --run-attempt <n> --repo <owner/name> --out <file>
|
||||
bun run scripts/train-gate.ts screen-summary <manifest.json> --pr <n>`;
|
||||
|
||||
if (import.meta.main) {
|
||||
const [command, ...rest] = process.argv.slice(2);
|
||||
const flag = (name: string) => { const i = rest.indexOf(`--${name}`); return i >= 0 ? rest[i + 1] : undefined; };
|
||||
const fail = (message: string): never => { console.error(`::error::${message}`); process.exit(1); };
|
||||
const usage = (): never => { console.error(USAGE); process.exit(2); };
|
||||
const output = (lines: string[]) => { const file = flag('github-output'); if (file) fs.appendFileSync(file, `${lines.join('\n')}\n`); };
|
||||
const git = gitIn(process.cwd());
|
||||
if (command === 'route') {
|
||||
const env = process.env;
|
||||
const route = routeDispatch({ nonce: env.NONCE ?? '', refName: env.GITHUB_REF_NAME ?? '', head: git(['rev-parse', 'HEAD']).out, expectedSha: env.EXPECTED_SHA ?? '',
|
||||
evalsAll: env.EVALS_ALL === 'true', trainPolicy: env.TRAIN_POLICY === 'true', cases: env.CASES ?? '', prefixShas: env.PREFIX_SHAS ?? '',
|
||||
screenPr: env.SCREEN_PR ?? '', screenMergeSha: env.SCREEN_MERGE_SHA ?? '', screenPolicy: env.SCREEN_POLICY ?? '' },
|
||||
{ isAncestor: (commit, head) => git(['merge-base', '--is-ancestor', commit, head]).ok, gateCases: () => fullGateCases(process.cwd()) });
|
||||
if ('error' in route) fail(`dispatch refused before any paid work: ${route.error}`);
|
||||
const ok = route as Route;
|
||||
output([`mode=${ok.mode}`, `cases=${ok.cases.join(',')}`, `prefixes=${ok.prefixes.join(',')}`]);
|
||||
console.log(`- Run mode: ${ok.mode}${ok.prefixes.length ? ` (${ok.cases.length} case(s) on ${ok.prefixes.length} prefix(es))` : ''}`);
|
||||
} else if (command === 'bisect-plan') {
|
||||
const [cases, prefixes, nonce, out] = [list(flag('cases') ?? ''), list(flag('prefixes') ?? ''), flag('nonce'), flag('out')];
|
||||
if (!cases.length || !prefixes.length || !nonce || !out) usage();
|
||||
const plan = fullGateCases(process.cwd());
|
||||
const bisect: BisectPlan = { schema: 1, nonce: nonce!, cases: cases.map(id => ({ id, kind: plan.get(id) ?? fail(`case ${id} is not in the full gate's plan`) })), prefixes };
|
||||
fs.mkdirSync(path.dirname(out!), { recursive: true });
|
||||
fs.writeFileSync(out!, `${JSON.stringify(bisect, null, 2)}\n`);
|
||||
const m = bisectMatrix(bisect);
|
||||
output([`slices=${JSON.stringify(m.slices)}`, `slice_timeouts=${JSON.stringify(m.timeouts)}`, `prefix_refs=${JSON.stringify(m.refs)}`]);
|
||||
console.log(`- Bisect: ${cases.length} case(s) x ${prefixes.length} prefix(es), one trial each`);
|
||||
} else if (command === 'bisect-prefix') {
|
||||
const [planPath, index, out] = [flag('plan'), Number(flag('index')), flag('out')];
|
||||
if (!planPath || !Number.isSafeInteger(index) || index < 1 || !out) usage();
|
||||
const plan = readJson<BisectPlan>(planPath!) ?? fail(`unreadable bisect plan ${planPath}`);
|
||||
const record = await runBisectPrefix(plan, index, { rootDir: process.cwd(), evalDir: path.join(out!, 'eval'), checkedOut: git(['rev-parse', 'HEAD']).out });
|
||||
fs.mkdirSync(out!, { recursive: true });
|
||||
fs.writeFileSync(path.join(out!, 'bisect-prefix.json'), `${JSON.stringify(record, null, 2)}\n`);
|
||||
} else if (command === 'train-record') {
|
||||
const [reportDir, nonce, head, repo, out] = [rest[0], flag('nonce'), flag('head'), flag('repo'), flag('out')];
|
||||
if (!reportDir || !nonce || !head || !SHA.test(head) || !repo || !out) usage();
|
||||
const prefixes = trainPrefixes(git, head!, process.cwd(), loadPolicyFromHead(process.cwd()).policy);
|
||||
if ('error' in prefixes) fail(`no train record: ${prefixes.error}`);
|
||||
const runId = Number(flag('run-id'));
|
||||
const record = buildTrainRecord({ reportDir: reportDir!, nonce: nonce!, head: head!, tree: git(['rev-parse', `${head}^{tree}`]).out,
|
||||
reconcileExit: /^\d+$/.test(flag('reconcile-exit') ?? '') ? Number(flag('reconcile-exit')) : -1, prefixes: prefixes as TrainPrefixTree[],
|
||||
run: { id: runId, attempt: Number(flag('run-attempt')), url: `https://github.com/${repo}/actions/runs/${runId}` } });
|
||||
fs.mkdirSync(path.dirname(out!), { recursive: true });
|
||||
fs.writeFileSync(out!, `${JSON.stringify(record, null, 2)}\n`);
|
||||
const c = record.census;
|
||||
console.log(`- Train record: ${record.verdict} (rule ${c.executed.rule}/${c.planned.rule}, judge ${c.executed.judge}/${c.planned.judge}, unbound ${c.unbound}, partial ${c.partial_shards}, cancelled ${c.cancelled_shards}, killed before upload ${c.killed_before_upload}); prefixes T1..T${record.prefixes.length}`);
|
||||
} else if (command === 'screen-summary') {
|
||||
const [manifestPath, pr] = [rest[0], flag('pr')];
|
||||
if (!manifestPath || !pr) usage();
|
||||
for (const line of screenSummary(parseRunManifest(fs.readFileSync(manifestPath!, 'utf8')), pr!)) console.log(line);
|
||||
} else usage();
|
||||
}
|
||||
@@ -75,9 +75,9 @@ describe('paid CI coordination stays off the eval image', () => {
|
||||
test(`${name}: planning is independent of image startup and has no dependency install`, () => {
|
||||
const planner = jobs['plan-slices'];
|
||||
if (name === 'evals.yml') {
|
||||
// The PR lane plans after its two tiny coordination jobs (readiness, base-ref receipt recovery), never after the image.
|
||||
expect(planner.needs).toEqual(['readiness', 'recover-receipts']);
|
||||
for (const id of ['readiness', 'recover-receipts']) {
|
||||
// The PR lane plans after its three tiny coordination jobs (route, readiness, base-ref receipt recovery), never after the image.
|
||||
expect(planner.needs).toEqual(['readiness', 'recover-receipts', 'route']);
|
||||
for (const id of ['readiness', 'recover-receipts', 'route']) {
|
||||
const job = jobs[id] as Job & { 'runs-on'?: string };
|
||||
expect(job.container, id).toBeUndefined();
|
||||
expect(job['runs-on'], id).toBe('ubuntu-24.04');
|
||||
@@ -144,7 +144,8 @@ describe('paid CI coordination stays off the eval image', () => {
|
||||
const captures = hiddenUploads.filter(step => step.with?.name === 'native-captures-${{ env.EVALS_RUN_ID }}');
|
||||
// evals-periodic's host-run Codex job mounts $RUNNER_TEMP/eval-home as the container's HOME.
|
||||
const hostHome = (pattern: string) => pattern.replace('${{ runner.temp }}/eval-home/', '~/');
|
||||
expect(captures).toHaveLength(name === 'evals.yml' ? 1 : name === 'evals-periodic.yml' ? 3 : 2);
|
||||
// evals.yml: eval-slices and train-bisect (GSTA-49 T1b), each under its own EVALS_RUN_ID.
|
||||
expect(captures).toHaveLength(name === 'evals.yml' ? 2 : name === 'evals-periodic.yml' ? 3 : 2);
|
||||
for (const capture of captures) {
|
||||
expect(capture.if).toBe('always()');
|
||||
expect(String(capture.with?.path).trim().split('\n').map(hostHome)).toEqual([
|
||||
@@ -159,7 +160,11 @@ describe('paid CI coordination stays off the eval image', () => {
|
||||
// The container sets TMPDIR under HOME, so shard logs land in the mounted HOME's cache.
|
||||
expect(String(codexLogs.with?.path).trim().split('\n').map(hostHome)).toEqual(['~/.cache/gstack-paid-shard-*.log']);
|
||||
}
|
||||
expect(hiddenUploads.filter(step => !captures.includes(step))).toEqual(codexLogs ? [logs!, codexLogs] : [logs!]);
|
||||
// GSTA-49 T1b: train-bisect keeps the same shard-log upload under its own name.
|
||||
const bisectLogs = name === 'evals.yml'
|
||||
? jobs['train-bisect'].steps.find(step => step.with?.name === 'train-bisect-logs-${{ matrix.slice }}-a${{ github.run_attempt }}') : undefined;
|
||||
if (name === 'evals.yml') expect(String(bisectLogs?.with?.path)).toBe(String(logs?.with?.path));
|
||||
expect(hiddenUploads.filter(step => !captures.includes(step))).toEqual([logs!, ...(codexLogs ? [codexLogs] : []), ...(bisectLogs ? [bisectLogs] : [])]);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -168,7 +173,9 @@ describe('paid CI coordination stays off the eval image', () => {
|
||||
const trust = "github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)";
|
||||
// !cancelled() lets planning proceed when readiness was skipped (dispatch) or recovery was skipped or failed (reuse is then off);
|
||||
// a readiness failure (Free Tests red on this head) or a superseded head plans nothing; the trust clause still gates it.
|
||||
expect(jobs['plan-slices'].if).toBe(`\${{ !cancelled() && (needs.readiness.result == 'success' || needs.readiness.result == 'skipped') && needs.readiness.outputs.superseded != 'true' && ${trust} }}`);
|
||||
// GSTA-49 T1b: the route job must have succeeded and named a planned mode (pr, screen, train, dispatch); a superseded screen plans nothing.
|
||||
const route = "needs.route.result == 'success' && needs.route.outputs.superseded != 'true' && contains(fromJSON('[\"pr\",\"screen\",\"train\",\"dispatch\"]'), needs.route.outputs.mode)";
|
||||
expect(jobs['plan-slices'].if).toBe(`\${{ !cancelled() && (needs.readiness.result == 'success' || needs.readiness.result == 'skipped') && needs.readiness.outputs.superseded != 'true' && ${route} && ${trust} }}`);
|
||||
// The coordination jobs never run for Dependabot or for a fork PR either (recovery also serves an explicit pr_receipts dispatch).
|
||||
expect(jobs.readiness.if).toContain("github.actor != 'dependabot[bot]'");
|
||||
expect(jobs.readiness.if).toContain("github.event_name == 'pull_request'");
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
* and a fake clock: no network, no sleeping.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { freeTestsState, readinessDecision, waitForReadiness, type FreeTestsRun } from '../scripts/eval-readiness';
|
||||
import { freeTestsState, readinessDecision, screenTarget, waitForReadiness, type FreeTestsRun } from '../scripts/eval-readiness';
|
||||
import type { GhRunner } from '../scripts/recover-receipts';
|
||||
|
||||
const HEAD = 'a'.repeat(40);
|
||||
@@ -127,3 +127,41 @@ describe('waitForReadiness', () => {
|
||||
expect(c.sleeps[0]).toBe(10 * 60_000);
|
||||
});
|
||||
});
|
||||
|
||||
describe('dispatched screens (GSTA-49 T1b): the merge commit is checked against the PR before Free Tests is read on its head', () => {
|
||||
const MERGE = 'c'.repeat(40);
|
||||
const BASE = 'd'.repeat(40);
|
||||
const pull = (over: Record<string, unknown> = {}) => ({ state: 'open', head: { sha: HEAD, ref: 'feature', repo: { full_name: 'garrytan/gstack' } }, base: { repo: { full_name: 'garrytan/gstack' } }, ...over });
|
||||
const gh = (pr: unknown, parents: string[] | null): GhRunner => (args) => {
|
||||
const target = args.find(a => a.startsWith('repos/')) ?? '';
|
||||
if (target.endsWith('/pulls/42')) return pr ? { status: 0, stdout: JSON.stringify(pr), stderr: '' } : { status: 1, stdout: '', stderr: 'HTTP 404: Not Found' };
|
||||
if (target.endsWith(`/commits/${MERGE}`)) return parents ? { status: 0, stdout: JSON.stringify({ parents: parents.map(sha => ({ sha })) }), stderr: '' } : { status: 1, stdout: '', stderr: 'HTTP 422' };
|
||||
return { status: 1, stdout: '', stderr: 'unexpected' };
|
||||
};
|
||||
|
||||
test("an open same-repository PR's merge commit names its head (second parent) and tested base (first parent)", () => {
|
||||
expect(screenTarget(gh(pull(), [BASE, HEAD]), 'garrytan/gstack', 42, MERGE)).toEqual({ head: HEAD, base: BASE, branch: 'feature' });
|
||||
});
|
||||
|
||||
test('a fork PR, a closed PR, a non-merge commit, a short SHA or an unreadable PR is refused', () => {
|
||||
const refused: Array<[GhRunner, string, string]> = [
|
||||
[gh(pull({ head: { sha: HEAD, ref: 'feature', repo: { full_name: 'fork/gstack' } } }), [BASE, HEAD]), MERGE, 'not a same-repository PR'],
|
||||
[gh(pull({ head: { sha: HEAD, ref: 'feature', repo: null } }), [BASE, HEAD]), MERGE, 'not a same-repository PR'],
|
||||
[gh(pull({ state: 'closed' }), [BASE, HEAD]), MERGE, 'is closed, not open'],
|
||||
[gh(pull(), [BASE]), MERGE, 'is not a two-parent merge commit'],
|
||||
[gh(pull(), null), MERGE, 'cannot read the merge commit'],
|
||||
[gh(null, [BASE, HEAD]), MERGE, 'cannot read PR #42'],
|
||||
[gh(pull(), [BASE, HEAD]), MERGE.slice(0, 12), 'is not a full commit SHA'],
|
||||
];
|
||||
for (const [runner, merge, why] of refused) expect((screenTarget(runner, 'garrytan/gstack', 42, merge) as { error?: string }).error, why).toContain(why);
|
||||
});
|
||||
|
||||
test('the screen then waits on Free Tests for the PR head with no quiet window, and a moved head is superseded', () => {
|
||||
const { gh: ready } = fakeGh([{ free: [free(5, 'completed', 'success')] }]);
|
||||
const c = clock(CREATED);
|
||||
expect(waitForReadiness({ ...base, quietMs: 0, gh: ready, now: c.now, sleep: c.sleep }).decision.kind).toBe('ready');
|
||||
expect(c.sleeps).toEqual([]);
|
||||
const { gh: moved } = fakeGh([{ free: [free(5, 'completed', 'success')], prHead: NEWER }]);
|
||||
expect(waitForReadiness({ ...base, quietMs: 0, gh: moved, now: c.now, sleep: c.sleep }).decision).toEqual({ kind: 'superseded', by: NEWER });
|
||||
});
|
||||
});
|
||||
@@ -44,7 +44,7 @@ describe('evals.yml receipt recovery (CEO-14/34, ENG-7)', () => {
|
||||
expect(recover.permissions).toEqual({ contents: 'read', actions: 'read' });
|
||||
expect(planner.permissions).toEqual({ contents: 'read' });
|
||||
const checkout = recover.steps.find(s => s.uses?.startsWith('actions/checkout@'))!;
|
||||
expect(checkout.with).toMatchObject({ ref: '${{ github.event.pull_request.base.sha || inputs.base_sha || github.event.repository.default_branch }}', 'persist-credentials': false });
|
||||
expect(checkout.with).toMatchObject({ ref: '${{ github.event.pull_request.base.sha || needs.route.outputs.screen_base || inputs.base_sha || github.event.repository.default_branch }}', 'persist-credentials': false });
|
||||
const collect = step('recover-receipts', 'Recover receipts of cancelled runs (base-ref code)')!;
|
||||
expect(collect.env?.GH_TOKEN).toBe('${{ github.token }}');
|
||||
expect(collect.run).toContain('scripts/recover-receipts.ts collect');
|
||||
@@ -105,7 +105,8 @@ describe('evals.yml readiness-gated dispatch (R1, GSTA-48)', () => {
|
||||
});
|
||||
|
||||
test('cancel-in-progress stays keyed on the PR, and evals-fresh turns receipt reuse off', () => {
|
||||
expect(source).toContain('group: evals-${{ github.event.pull_request.number || github.run_id }}');
|
||||
// A dispatched screen (GSTA-49 T1b) is keyed on its PR too: a newer head's screen cancels the older one.
|
||||
expect(source).toContain("group: evals-${{ github.event.pull_request.number || (inputs.screen_pr != '' && format('screen-{0}', inputs.screen_pr)) || github.run_id }}");
|
||||
expect(source).toContain('cancel-in-progress: true');
|
||||
expect(workflow.env.EVALS_FRESH).toContain("contains(github.event.pull_request.labels.*.name, 'evals-fresh')");
|
||||
});
|
||||
@@ -114,12 +115,14 @@ describe('evals.yml readiness-gated dispatch (R1, GSTA-48)', () => {
|
||||
describe('evals.yml gate verdict reuse wiring (R2a restamp-only, R2 main moved; GSTA-48)', () => {
|
||||
test('base-ref recovery proposes a candidate; it never fails the run', () => {
|
||||
const find = step('recover-receipts', 'Find a gate verdict to reuse (R2a/R2, base-ref code)')!;
|
||||
expect(find.if).toBe("github.event_name == 'pull_request'");
|
||||
expect(find.env).toMatchObject({ GH_TOKEN: '${{ github.token }}', HEAD_SHA: '${{ github.event.pull_request.head.sha }}' });
|
||||
// GSTA-49 T1b: a dispatched screen finds too, with its route-checked head and its pinned merge commit as the tested sha.
|
||||
expect(find.if).toBe("github.event_name == 'pull_request' || needs.route.outputs.mode == 'screen'");
|
||||
expect(find.env).toMatchObject({ GH_TOKEN: '${{ github.token }}', HEAD_SHA: '${{ github.event.pull_request.head.sha || needs.route.outputs.screen_head }}',
|
||||
TESTED_SHA: '${{ inputs.screen_merge_sha || github.sha }}' });
|
||||
expect(find.env?.BASE_SHA).toBeUndefined();
|
||||
expect(find.run).toContain('scripts/gate-reuse.ts find');
|
||||
// The base is the tested merge commit's first parent, never the event's base.sha.
|
||||
expect(find.run).toContain('--tested-sha "$GITHUB_SHA"');
|
||||
expect(find.run).toContain('--tested-sha "$TESTED_SHA"');
|
||||
expect(find.run).not.toContain('--base-sha');
|
||||
expect(find.run).toContain('exit 0');
|
||||
expect(find.run).toContain('|| echo');
|
||||
@@ -144,9 +147,11 @@ describe('evals.yml gate verdict reuse wiring (R2a restamp-only, R2 main moved;
|
||||
expect(report.outputs?.['reconcile-exit']).toBe('${{ steps.reconcile.outputs.exit || steps.reused.outputs.exit }}');
|
||||
expect(step('slices-report', 'Fail the workflow when reconciliation failed')!.if).toBe("(steps.reconcile.outputs.exit || steps.reused.outputs.exit) != '0'");
|
||||
const bind = step('slices-report', "Bind this run's verdict for reuse (R2a/R2)")!;
|
||||
expect(bind.if).toBe("always() && github.event_name == 'pull_request' && needs.plan-slices.outputs.gate_reuse != 'true'");
|
||||
// GSTA-49 T1b: a dispatched screen binds too; train, bisect and plain dispatches never bind.
|
||||
expect(bind.if).toBe("always() && (github.event_name == 'pull_request' || needs.plan-slices.outputs.mode == 'screen') && needs.plan-slices.outputs.gate_reuse != 'true'");
|
||||
expect(bind.env?.TESTED_SHA).toBe('${{ inputs.screen_merge_sha || github.sha }}');
|
||||
expect(bind.env?.RECONCILE_EXIT).toBe('${{ steps.reconcile.outputs.exit }}');
|
||||
expect(bind.run).toContain('--tested-sha "$GITHUB_SHA"');
|
||||
expect(bind.run).toContain('--tested-sha "$TESTED_SHA"');
|
||||
expect(bind.env?.BASE_SHA).toBeUndefined();
|
||||
expect(step('slices-report', 'Upload the gate binding')?.with).toMatchObject({ name: 'gate-binding-a${{ github.run_attempt }}', path: '/tmp/gate-binding/gate-binding.json' });
|
||||
});
|
||||
@@ -163,19 +168,22 @@ describe('evals.yml carries no inline planner program (W8f)', () => {
|
||||
describe('evals.yml dispatch contract for eval:bg (CEO-17/29, DX-1, ENG-8/12)', () => {
|
||||
const inputs = workflow.on.workflow_dispatch.inputs as Record<string, { type: string; default: unknown }>;
|
||||
|
||||
test('dispatch inputs: evals_all, base_ref, base_sha, expected_sha, nonce, pr_receipts', () => {
|
||||
expect(Object.keys(inputs).sort()).toEqual(['base_ref', 'base_sha', 'evals_all', 'expected_sha', 'nonce', 'pr_receipts']);
|
||||
test('dispatch inputs: evals_all, base_ref, base_sha, expected_sha, nonce, pr_receipts, and the train inputs', () => {
|
||||
// cases, prefix_shas, screen_pr, screen_merge_sha and screen_policy (GSTA-49 T1b) are validated by the route job.
|
||||
expect(Object.keys(inputs).sort()).toEqual(['base_ref', 'base_sha', 'cases', 'evals_all', 'expected_sha', 'nonce', 'pr_receipts', 'prefix_shas', 'screen_merge_sha', 'screen_policy', 'screen_pr']);
|
||||
expect(inputs.evals_all).toMatchObject({ type: 'boolean', default: true });
|
||||
for (const name of ['base_ref', 'base_sha', 'expected_sha', 'nonce', 'pr_receipts']) expect(inputs[name]).toMatchObject({ type: 'string', default: '' });
|
||||
for (const name of ['base_ref', 'base_sha', 'expected_sha', 'nonce', 'pr_receipts', 'cases', 'prefix_shas', 'screen_pr', 'screen_merge_sha', 'screen_policy']) expect(inputs[name]).toMatchObject({ type: 'string', default: '' });
|
||||
});
|
||||
|
||||
test('the nonce is reflected in the run name so the dispatcher resolves its own run', () => {
|
||||
expect(workflow['run-name']).toBe("${{ inputs.nonce != '' && format('E2E Evals [eval-bg {0}]', inputs.nonce) || '' }}");
|
||||
// A dispatched screen's name also ends ' screen #<pr>' (GSTA-49 T1b); eval-bg's runForNonce still matches 'eval-bg <nonce>]'.
|
||||
expect(workflow['run-name']).toBe("${{ inputs.nonce != '' && format('E2E Evals [eval-bg {0}]{1}', inputs.nonce, inputs.screen_pr != '' && format(' screen #{0}', inputs.screen_pr) || '') || '' }}");
|
||||
});
|
||||
|
||||
test('evals_all=false dispatches run the PR profile on the requested base; evals_all runs the full census', () => {
|
||||
expect(workflow.env.EVALS_PROFILE).toBe("${{ (github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && !inputs.evals_all)) && 'pr' || 'full' }}");
|
||||
expect(step('plan-slices', 'Emit run manifest')?.env?.EVALS_BASE).toBe("${{ inputs.base_sha || (inputs.base_ref != '' && format('origin/{0}', inputs.base_ref)) || '' }}");
|
||||
// A dispatched screen diffs against its merge commit's first parent (route's screen_base; empty for every other run).
|
||||
expect(step('plan-slices', 'Emit run manifest')?.env?.EVALS_BASE).toBe("${{ needs.route.outputs.screen_base || inputs.base_sha || (inputs.base_ref != '' && format('origin/{0}', inputs.base_ref)) || '' }}");
|
||||
});
|
||||
|
||||
test('a dispatch fails fast with a named fix when the checkout is not the expected revision', () => {
|
||||
|
||||
@@ -0,0 +1,353 @@
|
||||
/**
|
||||
* evals.yml and evals-main-backstop.yml under merge trains (plan GSTA-49 T1b).
|
||||
*
|
||||
* The job graph is simulated with a small evaluator of the GitHub expression
|
||||
* subset these workflows use (literals, contexts, ==, !=, &&, ||, !, contains,
|
||||
* fromJSON, format, status functions), so each run mode is checked by which
|
||||
* jobs and steps actually run and what their expressions evaluate to. The
|
||||
* inert case: without a train block a PR push and a plain dispatch run today's
|
||||
* jobs and steps with today's values, plus the free route job.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { bisectArtifact } from '../lib/ship-queue-train-verdict';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
type Step = { id?: string; name?: string; uses?: string; if?: string; run?: string; env?: Record<string, unknown>; with?: Record<string, unknown> };
|
||||
type Job = { if?: string; needs?: string | string[]; outputs?: Record<string, string>; permissions?: Record<string, string>; steps: Step[]; strategy?: { matrix: Record<string, string> } };
|
||||
type Workflow = { 'run-name'?: string; on: Record<string, any>; concurrency: { group: string; 'cancel-in-progress': boolean }; permissions?: unknown; jobs: Record<string, Job> };
|
||||
const load = (rel: string) => Bun.YAML.parse(fs.readFileSync(path.join(ROOT, rel), 'utf8')) as Workflow;
|
||||
const evals = load('.github/workflows/evals.yml');
|
||||
const backstopWf = load('.github/workflows/evals-main-backstop.yml');
|
||||
|
||||
// ── expression subset ────────────────────────────────────────────────────
|
||||
const STAR = Symbol('star');
|
||||
type Star = { [STAR]: unknown[] };
|
||||
const isStar = (v: unknown): v is Star => typeof v === 'object' && v !== null && STAR in v;
|
||||
const truthy = (v: unknown) => !(v === false || v === 0 || v === '' || v === null || v === undefined || Number.isNaN(v));
|
||||
function looseEq(a: unknown, b: unknown): boolean {
|
||||
if (a === undefined) a = null;
|
||||
if (b === undefined) b = null;
|
||||
if (typeof a === 'string' && typeof b === 'string') return a.toLowerCase() === b.toLowerCase();
|
||||
if ((typeof a === 'object' && a !== null) || (typeof b === 'object' && b !== null)) return a === b;
|
||||
if (typeof a === typeof b) return a === b;
|
||||
const num = (v: unknown) => (v === null ? 0 : typeof v === 'boolean' ? Number(v) : typeof v === 'string' ? (v.trim() === '' ? 0 : Number(v)) : Number(v));
|
||||
return num(a) === num(b);
|
||||
}
|
||||
const str = (v: unknown) => (v === null || v === undefined ? '' : typeof v === 'object' ? JSON.stringify(v) : String(v));
|
||||
function evaluate(expr: string, ctx: Record<string, unknown>, status: { success: boolean; failure: boolean }): unknown {
|
||||
const src = expr.trim().replace(/^\$\{\{([\s\S]*)\}\}$/, '$1');
|
||||
const re = /\s*(?:('(?:[^']|'')*')|(\d+(?:\.\d+)?)|(==|!=|&&|\|\||[!()[\],.*])|([A-Za-z_][A-Za-z0-9_-]*))/y;
|
||||
const tokens: string[] = [];
|
||||
while (re.lastIndex < src.length) {
|
||||
const at = re.lastIndex;
|
||||
const m = re.exec(src);
|
||||
if (!m) { if (/^\s*$/.test(src.slice(at))) break; throw new Error(`cannot tokenize ${src.slice(at)}`); }
|
||||
tokens.push(m[1] ?? m[2] ?? m[3] ?? m[4]!);
|
||||
}
|
||||
let i = 0;
|
||||
const take = (want?: string) => { const t = tokens[i++]; if (want !== undefined && t !== want) throw new Error(`expected ${want}, got ${t} in ${src}`); return t!; };
|
||||
const fns: Record<string, (...a: unknown[]) => unknown> = {
|
||||
always: () => true, cancelled: () => false, success: () => status.success, failure: () => status.failure,
|
||||
contains: (hay, needle) => (Array.isArray(hay) || isStar(hay) ? (isStar(hay) ? hay[STAR] : hay as unknown[]).some(v => looseEq(v, needle)) : str(hay).toLowerCase().includes(str(needle).toLowerCase())),
|
||||
fromJSON: (s) => JSON.parse(str(s)),
|
||||
format: (f, ...args) => str(f).replace(/\{(\d+)\}/g, (_, n) => str(args[Number(n)])),
|
||||
startsWith: (a, b) => str(a).toLowerCase().startsWith(str(b).toLowerCase()),
|
||||
hashFiles: () => 'hash',
|
||||
};
|
||||
const member = (v: unknown, key: string): unknown => {
|
||||
if (isStar(v)) return { [STAR]: v[STAR].map(item => member(item, key)) };
|
||||
return v !== null && typeof v === 'object' ? (v as Record<string, unknown>)[key] ?? null : null;
|
||||
};
|
||||
const primary = (): unknown => {
|
||||
const t = take();
|
||||
if (t === '(') { const v = or(); take(')'); return v; }
|
||||
if (t.startsWith("'")) return t.slice(1, -1).replace(/''/g, "'");
|
||||
if (/^\d/.test(t)) return Number(t);
|
||||
if (t === 'true' || t === 'false') return t === 'true';
|
||||
if (t === 'null') return null;
|
||||
if (tokens[i] === '(') {
|
||||
take('(');
|
||||
const args: unknown[] = [];
|
||||
while (tokens[i] !== ')') { args.push(or()); if (tokens[i] === ',') take(','); }
|
||||
take(')');
|
||||
if (!fns[t]) throw new Error(`unknown function ${t}`);
|
||||
return fns[t]!(...args);
|
||||
}
|
||||
return ctx[t] ?? null;
|
||||
};
|
||||
const postfix = (): unknown => {
|
||||
let v = primary();
|
||||
for (;;) {
|
||||
if (tokens[i] === '.') { take('.'); const key = take(); v = key === '*' ? { [STAR]: Array.isArray(v) ? v : v && typeof v === 'object' ? Object.values(v) : [] } : member(v, key); }
|
||||
else if (tokens[i] === '[') { take('['); const k = or(); take(']'); v = member(v, str(k)); }
|
||||
else return v;
|
||||
}
|
||||
};
|
||||
const unary = (): unknown => (tokens[i] === '!' ? (take('!'), !truthy(unary())) : postfix());
|
||||
const eq = (): unknown => {
|
||||
let l = unary();
|
||||
while (tokens[i] === '==' || tokens[i] === '!=') { const op = take(); const r = unary(); l = op === '==' ? looseEq(l, r) : !looseEq(l, r); }
|
||||
return l;
|
||||
};
|
||||
const and = (): unknown => { let l = eq(); while (tokens[i] === '&&') { take(); const r = eq(); l = truthy(l) ? r : l; } return l; };
|
||||
const or = (): unknown => { let l = and(); while (tokens[i] === '||') { take(); const r = and(); l = truthy(l) ? l : r; } return l; };
|
||||
const value = or();
|
||||
if (i !== tokens.length) throw new Error(`trailing tokens in ${src}`);
|
||||
return value;
|
||||
}
|
||||
const interpolate = (text: unknown, ctx: Record<string, unknown>) =>
|
||||
typeof text === 'string' ? text.replace(/\$\{\{([\s\S]*?)\}\}/g, (_, e) => str(evaluate(e, ctx, { success: true, failure: false }))) : text;
|
||||
|
||||
// ── job graph ─────────────────────────────────────────────────────────────
|
||||
const SHA = { main: 'm'.repeat(40), head: 'h'.repeat(40), base: 'b'.repeat(40), merge: 'c'.repeat(40), screenBase: 'e'.repeat(40), screenHead: 'f'.repeat(40) };
|
||||
interface Scenario { github: Record<string, unknown>; inputs: Record<string, unknown>; outputs?: Record<string, Record<string, string>>; results?: Record<string, string> }
|
||||
const ROUTE = { mode: '', record_artifact: '', full_gate_check: '', screen_head: '', screen_base: '', screen_branch: '', superseded: '', bisect_slices: '', bisect_timeouts: '', bisect_refs: '' };
|
||||
function simulate(wf: Workflow, s: Scenario) {
|
||||
const result: Record<string, string> = {};
|
||||
const outputs: Record<string, Record<string, unknown>> = {};
|
||||
const ran: Record<string, { ctx: Record<string, unknown>; steps: Step[] }> = {};
|
||||
const pending = Object.keys(wf.jobs);
|
||||
while (pending.length) {
|
||||
const id = pending.find(j => [wf.jobs[j]!.needs ?? []].flat().every(n => n in result));
|
||||
if (!id) throw new Error('needs cycle');
|
||||
pending.splice(pending.indexOf(id), 1);
|
||||
const job = wf.jobs[id]!;
|
||||
const needs = [job.needs ?? []].flat();
|
||||
const ctx = { github: s.github, inputs: s.inputs, vars: {}, secrets: {}, env: {}, steps: {}, matrix: {}, runner: {},
|
||||
needs: Object.fromEntries(needs.map(n => [n, { result: result[n], outputs: outputs[n] ?? {} }])) };
|
||||
const allOk = needs.every(n => result[n] === 'success');
|
||||
const cond = job.if === undefined ? 'success()' : String(job.if);
|
||||
const value = truthy(evaluate(cond, ctx, { success: allOk, failure: needs.some(n => result[n] === 'failure') }));
|
||||
const runs = /\b(success|failure|always|cancelled)\(\)/.test(cond) ? value : allOk && value;
|
||||
result[id] = runs ? s.results?.[id] ?? 'success' : 'skipped';
|
||||
outputs[id] = runs ? Object.fromEntries(Object.entries(job.outputs ?? {}).map(([k, v]) => [k, s.outputs?.[id]?.[k] ?? interpolate(v, ctx)])) : {};
|
||||
if (runs) ran[id] = { ctx, steps: job.steps.filter(step => step.if === undefined || truthy(evaluate(String(step.if), ctx, { success: true, failure: false }))) };
|
||||
}
|
||||
const step = (job: string, name: string) => {
|
||||
const found = ran[job]?.steps.find(st => st.name === name);
|
||||
return found ? { env: Object.fromEntries(Object.entries(found.env ?? {}).map(([k, v]) => [k, interpolate(v, ran[job]!.ctx)])),
|
||||
with: Object.fromEntries(Object.entries(found.with ?? {}).map(([k, v]) => [k, interpolate(v, ran[job]!.ctx)])), run: found.run } : null;
|
||||
};
|
||||
const checkout = (job: string) => {
|
||||
const found = ran[job]?.steps.find(st => st.uses?.startsWith('actions/checkout@'));
|
||||
return found ? Object.fromEntries(Object.entries(found.with ?? {}).map(([k, v]) => [k, interpolate(v, ran[job]!.ctx)])) : null;
|
||||
};
|
||||
const top = { github: s.github, inputs: s.inputs };
|
||||
return { jobs: Object.keys(ran).sort(), steps: (job: string) => ran[job]?.steps.map(st => st.name ?? st.uses) ?? [], step, checkout, result,
|
||||
group: interpolate(wf.concurrency.group, top), runName: interpolate(wf['run-name'] ?? '', top) };
|
||||
}
|
||||
const pr = (labels: string[] = []) => ({ event_name: 'pull_request', actor: 'garrytan', repository: 'garrytan/gstack', repository_id: 1, sha: SHA.main, run_id: 7, run_attempt: 1, ref_name: '42/merge',
|
||||
event: { pull_request: { number: 42, head: { sha: SHA.head, ref: 'feature', repo: { full_name: 'garrytan/gstack' } }, base: { sha: SHA.base }, labels: labels.map(name => ({ name })) },
|
||||
repository: { default_branch: 'main' } } });
|
||||
const dispatch = (ref = 'main') => ({ event_name: 'workflow_dispatch', actor: 'garrytan', repository: 'garrytan/gstack', repository_id: 1, sha: SHA.main, run_id: 7, run_attempt: 1, ref_name: ref,
|
||||
event: { repository: { default_branch: 'main' } } });
|
||||
const INPUTS = { evals_all: true, base_ref: '', base_sha: '', expected_sha: '', nonce: '', pr_receipts: '', cases: '', prefix_shas: '', screen_pr: '', screen_merge_sha: '', screen_policy: '' };
|
||||
const TODAY_PR_JOBS = ['build-image', 'eval-slices', 'plan-slices', 'readiness', 'recover-receipts', 'slices-comment', 'slices-report'];
|
||||
const TODAY_DISPATCH_JOBS = ['build-image', 'eval-slices', 'plan-slices', 'slices-report'];
|
||||
const TRAIN_STEPS = ['Write the train record', 'Upload the train record', "Check the screen's PR head (Free Tests)", 'Plan the bisect (one executor per prefix)', 'Hand the bisect plan to its executors'];
|
||||
const train = { record_artifact: 'gstack-train-record', full_gate_check: 'gstack/full-gate' };
|
||||
|
||||
describe('without a train block nothing changes (the inert proof)', () => {
|
||||
test("a PR push runs today's jobs and steps with today's values, plus the free route job", () => {
|
||||
const sim = simulate(evals, { github: pr(), inputs: {}, outputs: { route: { ...ROUTE, mode: 'pr' } } });
|
||||
expect(sim.jobs).toEqual([...TODAY_PR_JOBS, 'route'].sort());
|
||||
for (const job of sim.jobs) for (const name of TRAIN_STEPS) expect(sim.steps(job), job).not.toContain(name);
|
||||
expect(sim.group).toBe('evals-42');
|
||||
expect(sim.runName).toBe('');
|
||||
expect(sim.steps('recover-receipts')).toEqual(expect.arrayContaining(["Restore this PR's verified judge and E2E results", 'Recover receipts of cancelled runs (base-ref code)',
|
||||
'Find a gate verdict to reuse (R2a/R2, base-ref code)', 'Hand a gate reuse candidate to plan-slices', 'Hand the receipt store to plan-slices']));
|
||||
expect(sim.checkout('recover-receipts')?.ref).toBe(SHA.base);
|
||||
expect(sim.step('recover-receipts', 'Find a gate verdict to reuse (R2a/R2, base-ref code)')!.env).toMatchObject({
|
||||
PR_NUMBER: '42', HEAD_REPO: 'garrytan/gstack', HEAD_BRANCH: 'feature', HEAD_SHA: SHA.head, TESTED_SHA: SHA.main, SCREEN_POLICY: '' });
|
||||
for (const job of ['plan-slices', 'eval-slices', 'slices-report']) expect(sim.checkout(job)?.ref ?? '', job).toBe('');
|
||||
expect(sim.checkout('slices-report')?.['fetch-depth']).toBe('1');
|
||||
expect(sim.step('plan-slices', 'Emit run manifest')!.env.EVALS_BASE).toBe('');
|
||||
expect(sim.steps('plan-slices')).toEqual(expect.arrayContaining(['Download a gate reuse candidate', 'Confirm gate verdict reuse (R2a/R2)', 'Carry the receipt store to the report']));
|
||||
expect(sim.step('slices-report', "Bind this run's verdict for reuse (R2a/R2)")!.env).toMatchObject({ PR_NUMBER: '42', HEAD_SHA: SHA.head, TESTED_SHA: SHA.main, SCREEN_POLICY: '' });
|
||||
expect(sim.step('slices-comment', 'Post PR comment')!.env).toMatchObject({ PR_NUMBER: '42', SCREEN_POLICY: '' });
|
||||
});
|
||||
|
||||
test("a plain dispatch (eval:bg, backstop) runs today's jobs, plus route", () => {
|
||||
const sim = simulate(evals, { github: dispatch('feature'), inputs: { ...INPUTS, nonce: 'pr-x', evals_all: false, expected_sha: SHA.main }, outputs: { route: { ...ROUTE, mode: 'dispatch' } } });
|
||||
expect(sim.jobs).toEqual([...TODAY_DISPATCH_JOBS, 'route'].sort());
|
||||
expect(sim.group).toBe('evals-7');
|
||||
expect(sim.runName).toBe('E2E Evals [eval-bg pr-x]');
|
||||
for (const job of ['plan-slices', 'eval-slices', 'slices-report']) expect(sim.checkout(job)?.ref ?? '', job).toBe('');
|
||||
expect(sim.steps('slices-report')).not.toContain("Bind this run's verdict for reuse (R2a/R2)");
|
||||
const receipts = simulate(evals, { github: dispatch('feature'), inputs: { ...INPUTS, nonce: 'pr-x', evals_all: false, pr_receipts: '42' }, outputs: { route: { ...ROUTE, mode: 'dispatch' } } });
|
||||
expect(receipts.jobs).toContain('recover-receipts');
|
||||
expect(receipts.steps('recover-receipts')).toContain('Recover receipts of cancelled runs (base-ref code)');
|
||||
});
|
||||
|
||||
test('a fork PR and a Dependabot PR still start nothing paid', () => {
|
||||
const fork = pr();
|
||||
(fork.event.pull_request.head.repo as { full_name: string }).full_name = 'fork/gstack';
|
||||
expect(simulate(evals, { github: fork, inputs: {}, outputs: { route: { ...ROUTE, mode: 'pr' } } }).jobs).toEqual(['build-image', 'route']);
|
||||
expect(simulate(evals, { github: { ...pr(), actor: 'dependabot[bot]' }, inputs: {} }).jobs).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('under a train block (the gate edits, each only on its mode)', () => {
|
||||
test('a PR push starts no paid slice: route, readiness and the free screening summary only', () => {
|
||||
const sim = simulate(evals, { github: pr(), inputs: {}, outputs: { route: { ...ROUTE, ...train, mode: 'pr-summary' } } });
|
||||
expect(sim.jobs).toEqual(['build-image', 'readiness', 'route', 'screen-summary']);
|
||||
const summary = evals.jobs['screen-summary']!;
|
||||
expect(summary.permissions).toEqual({ contents: 'read' });
|
||||
expect(JSON.stringify(summary)).not.toMatch(/secrets\.|restore-deps|bun install/);
|
||||
expect(summary.steps.at(-1)!.run).toContain('--emit-plan /tmp/paid-plan/manifest.json --slices 1');
|
||||
expect(summary.steps.at(-1)!.run).toContain('scripts/train-gate.ts screen-summary /tmp/paid-plan/manifest.json --pr "$PR_NUMBER"');
|
||||
});
|
||||
|
||||
test('a dispatched screen tests its pinned merge commit, reuses and binds as a screen, comments on its PR, and is keyed on the PR', () => {
|
||||
const inputs = { ...INPUTS, evals_all: false, nonce: 'pr.screen.narrow-x', expected_sha: SHA.merge, base_ref: 'main', base_sha: SHA.base, screen_pr: '42', screen_merge_sha: SHA.merge, screen_policy: 'narrow' };
|
||||
const route = { ...ROUTE, ...train, mode: 'screen', screen_head: SHA.screenHead, screen_base: SHA.screenBase, screen_branch: 'feature' };
|
||||
const sim = simulate(evals, { github: dispatch(), inputs, outputs: { route } });
|
||||
expect(sim.jobs).toEqual(['build-image', 'eval-slices', 'plan-slices', 'recover-receipts', 'route', 'slices-comment', 'slices-report']);
|
||||
expect(sim.group).toBe('evals-screen-42');
|
||||
expect(sim.runName).toBe('E2E Evals [eval-bg pr.screen.narrow-x] screen #42');
|
||||
expect(sim.steps('recover-receipts')).not.toContain('Recover receipts of cancelled runs (base-ref code)');
|
||||
expect(sim.checkout('recover-receipts')?.ref).toBe(SHA.screenBase);
|
||||
expect(sim.step('recover-receipts', 'Find a gate verdict to reuse (R2a/R2, base-ref code)')!.env).toMatchObject({
|
||||
PR_NUMBER: '42', HEAD_REPO: 'garrytan/gstack', HEAD_BRANCH: 'feature', HEAD_SHA: SHA.screenHead, TESTED_SHA: SHA.merge, SCREEN_POLICY: 'narrow' });
|
||||
for (const job of ['plan-slices', 'eval-slices', 'slices-report']) expect(sim.checkout(job)?.ref, job).toBe(SHA.merge);
|
||||
expect(sim.step('plan-slices', 'Emit run manifest')!.env.EVALS_BASE).toBe(SHA.screenBase);
|
||||
expect(sim.steps('plan-slices')).toContain('Confirm gate verdict reuse (R2a/R2)');
|
||||
const bind = sim.step('slices-report', "Bind this run's verdict for reuse (R2a/R2)")!;
|
||||
expect(bind.env).toMatchObject({ PR_NUMBER: '42', HEAD_SHA: SHA.screenHead, TESTED_SHA: SHA.merge, SCREEN_POLICY: 'narrow' });
|
||||
expect(bind.run).toContain('${SCREEN_POLICY:+--purpose screen --screen-policy "$SCREEN_POLICY"}');
|
||||
expect(sim.steps('slices-report')).not.toContain('Write the train record');
|
||||
expect(sim.step('slices-comment', 'Post PR comment')!.env).toMatchObject({ PR_NUMBER: '42', SCREEN_POLICY: 'narrow', SCREEN_MERGE_SHA: SHA.merge });
|
||||
const superseded = simulate(evals, { github: dispatch(), inputs, outputs: { route: { ...route, superseded: 'true' } } });
|
||||
expect(superseded.jobs).toEqual(['build-image', 'route']);
|
||||
});
|
||||
|
||||
test('a train gate runs the full gate, writes its record from the report and publishes the check; it never reuses or binds', () => {
|
||||
const inputs = { ...INPUTS, nonce: 'gate.train-x', expected_sha: SHA.head };
|
||||
const sim = simulate(evals, { github: dispatch('train/t1'), inputs, outputs: { route: { ...ROUTE, ...train, mode: 'train' } } });
|
||||
expect(sim.jobs).toEqual(['build-image', 'eval-slices', 'plan-slices', 'route', 'slices-report', 'train-check']);
|
||||
expect(sim.steps('plan-slices')).not.toContain('Confirm gate verdict reuse (R2a/R2)');
|
||||
expect(sim.steps('slices-report')).not.toContain("Bind this run's verdict for reuse (R2a/R2)");
|
||||
expect(sim.steps('slices-report')).toEqual(expect.arrayContaining(['Write the train record', 'Upload the train record']));
|
||||
expect(sim.checkout('slices-report')?.['fetch-depth']).toBe('0');
|
||||
expect(sim.step('slices-report', 'Write the train record')!.run).toContain('scripts/train-gate.ts train-record /tmp/paid-report --nonce "$NONCE" --head "$HEAD_SHA"');
|
||||
expect(sim.step('slices-report', 'Upload the train record')!.with).toMatchObject({ name: 'gstack-train-record', path: '/tmp/train-record', overwrite: true });
|
||||
expect(simulate(evals, { github: dispatch('train/t1'), inputs, outputs: { route: { ...ROUTE, ...train, mode: 'train' } }, results: { 'slices-report': 'failure' } }).jobs).not.toContain('train-check');
|
||||
});
|
||||
|
||||
test('the check run is published only for a complete GREEN census of the train head, by a job with checks: write and no repository code', () => {
|
||||
const check = evals.jobs['train-check']!;
|
||||
expect(check.permissions).toEqual({ checks: 'write' });
|
||||
expect(check.steps.some(st => st.uses?.startsWith('actions/checkout@') || st.uses?.startsWith('oven-sh/setup-bun@'))).toBe(false);
|
||||
const publish = check.steps.find(st => st.name === 'Publish the full-gate check on the train head')!;
|
||||
expect(publish.run).toContain('.verdict == "GREEN" and .census.complete == true and .nonce == $nonce and .head_sha == $head');
|
||||
expect(publish.run).toContain('-f status=completed -f conclusion=success');
|
||||
expect(publish.env).toMatchObject({ CHECK: '${{ needs.plan-slices.outputs.full_gate_check }}', HEAD_SHA: '${{ inputs.expected_sha }}' });
|
||||
});
|
||||
|
||||
test("a bisect runs one executor per prefix only, uploading each prefix's record under the name T1 reads", () => {
|
||||
const inputs = { ...INPUTS, nonce: 'gate.bisect-x', expected_sha: SHA.head, cases: 'case-a', prefix_shas: SHA.base };
|
||||
const sim = simulate(evals, { github: dispatch('train/t1'), inputs, outputs: { route: { ...ROUTE, ...train, mode: 'bisect' } } });
|
||||
expect(sim.jobs).toEqual(['build-image', 'route', 'train-bisect']);
|
||||
const job = evals.jobs['train-bisect']!;
|
||||
expect(job.strategy!.matrix.slice).toBe('${{ fromJSON(needs.route.outputs.bisect_slices) }}');
|
||||
const upload = job.steps.find(st => st.name === 'Upload the prefix record')!;
|
||||
const name = String(upload.with!.name).replace('${{ needs.route.outputs.record_artifact }}', 'gstack-train-record').replace('${{ steps.prefix.outputs.short }}', SHA.base.slice(0, 12));
|
||||
expect(name).toBe(bisectArtifact('gstack-train-record', SHA.base));
|
||||
expect(job.steps.find(st => st.name === 'Verify the prefix checkout')!.run).toContain('echo "short=${PREFIX_SHA:0:12}"');
|
||||
});
|
||||
|
||||
test('a refused dispatch starts nothing paid', () => {
|
||||
const sim = simulate(evals, { github: dispatch('train/t1'), inputs: { ...INPUTS, nonce: 'gate.bisect-x' }, outputs: { route: { ...ROUTE, mode: '' } }, results: { route: 'failure' } });
|
||||
expect(sim.jobs).toEqual(['build-image', 'route']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('GSTA-48 open items', () => {
|
||||
test('a head the readiness job blocked replaces the stale verdict comment and plans nothing (P3-1)', () => {
|
||||
const sim = simulate(evals, { github: pr(), inputs: {}, outputs: { route: { ...ROUTE, mode: 'pr' } }, results: { readiness: 'failure' } });
|
||||
expect(sim.jobs).toEqual(['build-image', 'readiness', 'readiness-comment', 'route']);
|
||||
const comment = evals.jobs['readiness-comment']!;
|
||||
expect(comment.steps.some(st => st.uses)).toBe(false);
|
||||
expect(comment.steps[0]!.run).toContain('if [ -z "$COMMENT_ID" ]; then exit 0; fi');
|
||||
expect(comment.steps[0]!.run).toContain('## E2E Evals: ⛔ NOT RUN on');
|
||||
expect(comment.steps[0]!.run).not.toContain('PASS');
|
||||
expect(simulate(evals, { github: pr(), inputs: {}, outputs: { route: { ...ROUTE, mode: 'pr' } } }).jobs).not.toContain('readiness-comment');
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!Bun.which('jq') || !Bun.which('bash'))('the route job reads the train policy from the base revision with git and jq only', () => {
|
||||
const route = evals.jobs.route!;
|
||||
const policyStep = route.steps.find(st => st.name === 'Read the train policy (base revision)')!;
|
||||
const routeStep = route.steps.find(st => st.name === 'Route this run')!;
|
||||
function fixture(policy: unknown): { dir: string; base: string } {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'evals-route-'));
|
||||
const git = (...args: string[]) => spawnSync('git', ['-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.com', '-c', 'commit.gpgsign=false', ...args], { cwd: dir, encoding: 'utf8', timeout: 30_000 });
|
||||
git('init', '-q', '-b', 'main');
|
||||
if (policy !== undefined) { fs.mkdirSync(path.join(dir, '.gstack')); fs.writeFileSync(path.join(dir, '.gstack/ship-policy.json'), typeof policy === 'string' ? policy : JSON.stringify(policy)); }
|
||||
fs.writeFileSync(path.join(dir, 'README.md'), 'x\n');
|
||||
git('add', '-A');
|
||||
git('commit', '-q', '-m', 'base');
|
||||
return { dir, base: git('rev-parse', 'HEAD').stdout.trim() };
|
||||
}
|
||||
function run(script: string, dir: string, env: Record<string, string>): Record<string, string> {
|
||||
const out = path.join(dir, '.github-output');
|
||||
fs.writeFileSync(out, '');
|
||||
const r = spawnSync('bash', ['-e', '-c', script], { cwd: dir, encoding: 'utf8', timeout: 30_000,
|
||||
env: { PATH: process.env.PATH ?? '', HOME: dir, GITHUB_OUTPUT: out, GITHUB_STEP_SUMMARY: path.join(dir, '.summary'), ...env } });
|
||||
if (r.status !== 0) throw new Error(r.stderr);
|
||||
return Object.fromEntries(fs.readFileSync(out, 'utf8').split('\n').filter(Boolean).map(line => [line.slice(0, line.indexOf('=')), line.slice(line.indexOf('=') + 1)]));
|
||||
}
|
||||
const block = { gate_workflow: 'evals.yml', free_workflow: 'free-tests.yml', full_gate_check: 'gstack/full-gate', record_artifact: 'gstack-train-record', max_size: 5, fill_hours: 3, screen_selection: 'none', trial_reservation_usd: 1.5 };
|
||||
|
||||
test('no policy, no train block, or an invalid one: train=false, and a PR push routes pr', () => {
|
||||
for (const policy of [undefined, { bump: 'patch' }, { bump: 'patch', train: null }, { train: { ...block, record_artifact: 'bad name!' } }, '{not json']) {
|
||||
const fx = fixture(policy);
|
||||
const out = run(policyStep.run!, fx.dir, { BASE_SHA: fx.base, DEFAULT_BRANCH: 'main' });
|
||||
expect(out, JSON.stringify(policy)).toEqual({ train: 'false' });
|
||||
expect(run(routeStep.run!, fx.dir, { GITHUB_EVENT_NAME: 'pull_request', TRAIN_POLICY: out.train! })).toEqual({ mode: 'pr' });
|
||||
fs.rmSync(fx.dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('a train block names the record artifact and check, and a PR push routes pr-summary', () => {
|
||||
const fx = fixture({ bump: 'patch', train: block });
|
||||
const out = run(policyStep.run!, fx.dir, { BASE_SHA: fx.base, DEFAULT_BRANCH: 'main' });
|
||||
expect(out).toEqual({ train: 'true', record_artifact: 'gstack-train-record', full_gate_check: 'gstack/full-gate' });
|
||||
expect(run(routeStep.run!, fx.dir, { GITHUB_EVENT_NAME: 'pull_request', TRAIN_POLICY: 'true' })).toEqual({ mode: 'pr-summary' });
|
||||
fs.rmSync(fx.dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test('a ref without scripts/train-gate.ts routes a plain dispatch as before and refuses train or screen inputs', () => {
|
||||
const fx = fixture(undefined);
|
||||
expect(run(routeStep.run!, fx.dir, { GITHUB_EVENT_NAME: 'workflow_dispatch', GITHUB_REF_NAME: 'old-branch' })).toEqual({ mode: 'dispatch' });
|
||||
expect(() => run(routeStep.run!, fx.dir, { GITHUB_EVENT_NAME: 'workflow_dispatch', GITHUB_REF_NAME: 'old-branch', SCREEN_PR: '42' })).toThrow();
|
||||
fs.rmSync(fx.dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test('the policy is read before any repository code runs', () => {
|
||||
const steps = route.steps.map(st => st.name ?? st.uses ?? '');
|
||||
expect(steps.indexOf('Read the train policy (base revision)')).toBeLessThan(steps.indexOf('Route this run'));
|
||||
expect(policyStep.run).not.toContain('bun');
|
||||
expect(route.permissions).toEqual({ contents: 'read', actions: 'read', 'pull-requests': 'read' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('evals-main-backstop.yml', () => {
|
||||
test('push to main only, one at a time without cancelling, inert unless main declares a train block', () => {
|
||||
expect(Object.keys(backstopWf.on)).toEqual(['push']);
|
||||
expect(backstopWf.on.push).toEqual({ branches: ['main'] });
|
||||
expect(backstopWf.concurrency).toEqual({ group: 'evals-main-backstop', 'cancel-in-progress': false });
|
||||
expect(backstopWf.permissions).toEqual({});
|
||||
const job = backstopWf.jobs.backstop!;
|
||||
expect(job.permissions).toEqual({ contents: 'write', actions: 'write', checks: 'read', issues: 'write' });
|
||||
const policy = job.steps.find(st => st.name === 'Read the train policy (main)')!;
|
||||
expect(policy.run).not.toContain('bun');
|
||||
const gate = job.steps.find(st => st.name === 'Gate main unless a green train covers its tree')!;
|
||||
expect(gate.if).toBe("steps.policy.outputs.train == 'true'");
|
||||
expect(gate.run).toContain('scripts/train-backstop.ts run --repo "$GITHUB_REPOSITORY" --branch "$GITHUB_REF_NAME" --sha "$GITHUB_SHA"');
|
||||
});
|
||||
});
|
||||
@@ -362,7 +362,10 @@ describe('panel verdict surfaces (eval reliability policy)', () => {
|
||||
for (const source of [evalsYml, periodicYml, marathonYml]) {
|
||||
const jobs = jobsOf(source);
|
||||
const uploads = Object.values(jobs).flatMap(job => job.steps).filter(step => step.uses?.startsWith('actions/upload-artifact@'))
|
||||
.map(step => step.with?.name ?? '').filter(name => /slice|census-\$/.test(name));
|
||||
.map(step => step.with?.name ?? '').filter(name => /slice|census-\$/.test(name))
|
||||
// GSTA-49 T1b: the train record keeps the fixed name gstack-ship-queue reads; it is built from the
|
||||
// first attempt's slice results only (scripts/train-gate.ts buildTrainRecord), so a re-run cannot change it.
|
||||
.filter(name => name !== '${{ needs.plan-slices.outputs.record_artifact }}');
|
||||
expect(uploads.length).toBeGreaterThan(0);
|
||||
for (const name of uploads) expect(name, name).toContain('-a${{ github.run_attempt }}');
|
||||
const downloads = Object.values(jobs).flatMap(job => job.steps).filter(step => step.uses?.startsWith('actions/download-artifact@') && step.with?.pattern);
|
||||
|
||||
+74
-4
@@ -129,9 +129,10 @@ type Jobs = Record<number, Array<{ name: string; conclusion: string | null }>>;
|
||||
* slices; an unbound one ran none unless scripted), downloads that write the
|
||||
* scripted binding (or raw text), and merge-bases from the fixture repo.
|
||||
*/
|
||||
function fakeGh(runs: WorkflowRun[], bindings: Record<number, GateBinding | string>, failList = '', jobs: Jobs = {}, failCompare = false): GhRunner {
|
||||
function fakeGh(runs: WorkflowRun[], bindings: Record<number, GateBinding | string>, failList = '', jobs: Jobs = {}, failCompare = false, prCommits: string[] = []): GhRunner {
|
||||
return (args) => {
|
||||
const ok = (body: unknown) => ({ status: 0, stdout: JSON.stringify(body), stderr: '' });
|
||||
if (args.some(a => /\/pulls\/42\/commits$/.test(a))) return { status: 0, stdout: prCommits.join('\n') + '\n', stderr: '' };
|
||||
if (args.some(a => a.endsWith('/workflows/evals.yml/runs'))) return failList ? { status: 1, stdout: '', stderr: failList } : ok({ workflow_runs: runs });
|
||||
const jobList = args.find(a => /\/runs\/\d+\/jobs$/.test(a));
|
||||
if (jobList) {
|
||||
@@ -175,11 +176,11 @@ function tested(base: string, head: string): string {
|
||||
|
||||
let outCount = 0;
|
||||
function find(head: string, runs: WorkflowRun[], bindings: Record<number, GateBinding | string>,
|
||||
over: { baseSha?: string; failList?: string; jobs?: Jobs; failCompare?: boolean; testedSha?: string } = {}) {
|
||||
over: { baseSha?: string; failList?: string; jobs?: Jobs; failCompare?: boolean; testedSha?: string; screen?: { policy: 'narrow' | 'broad' }; prCommits?: string[] } = {}) {
|
||||
const out = path.join(scratch, 'out-' + ++outCount);
|
||||
const result = findReusableGate({ repo: REPO, pr: 42, headRepo: REPO, branch: 'feature', selfRunId: 500, headSha: head,
|
||||
testedSha: over.testedSha ?? tested(over.baseSha ?? sha.base!, head), repoRoot: receipt, out,
|
||||
gh: fakeGh(runs, bindings, over.failList, over.jobs, over.failCompare) });
|
||||
testedSha: over.testedSha ?? tested(over.baseSha ?? sha.base!, head), repoRoot: receipt, out, ...(over.screen ? { screen: over.screen } : {}),
|
||||
gh: fakeGh(runs, bindings, over.failList, over.jobs, over.failCompare, over.prCommits) });
|
||||
return { ...result, out, text: result.lines.join('\n') };
|
||||
}
|
||||
|
||||
@@ -488,3 +489,72 @@ describe.skipIf(!Bun.which('jq') || !Bun.which('bash'))('the PR comment for a re
|
||||
expect(failed.stdout).toContain('❌ FAIL');
|
||||
});
|
||||
});
|
||||
|
||||
describe('dispatched screens (GSTA-49 T1b): authenticated by PR, head, repository and tested base', () => {
|
||||
const title = (policy: string, pr = 42) => `E2E Evals [eval-bg pr.screen.${policy}-${'c'.repeat(12)}-${'d'.repeat(12)}-abc123] screen #${pr}`;
|
||||
// A screen dispatches on main: the run's head_sha is main's tip, never the PR head; the binding carries the PR head and the merge commit.
|
||||
const screenRun = (id: number, policy = 'narrow', over: Partial<WorkflowRun> = {}) => run(id, sha.base!, 'success', { event: 'workflow_dispatch', pull_requests: [], display_title: title(policy), ...over });
|
||||
const screenBinding = (head: string, policy: 'narrow' | 'broad' = 'narrow', over: Partial<GateBinding> = {}) =>
|
||||
binding({ head_sha: head, purpose: 'screen', tested_sha: tested(sha.base!, head), screen_policy: policy, ...over });
|
||||
const commits = () => [sha.gated!, sha.restamp!];
|
||||
const screen = (policy: 'narrow' | 'broad', runs: WorkflowRun[], bindings: Record<number, GateBinding | string>, over: { prCommits?: string[]; jobs?: Jobs } = {}) =>
|
||||
find(sha.restamp!, runs, bindings, { screen: { policy }, prCommits: over.prCommits ?? commits(), jobs: over.jobs });
|
||||
|
||||
test('a screen, a restamp-only push, then a re-screen of the same policy reuses through R2a', () => {
|
||||
const { candidate, text } = screen('narrow', [screenRun(300)], { 300: screenBinding(sha.gated!) });
|
||||
expect(text).toContain('TREE: same-modulo-stamps');
|
||||
expect(candidate).toMatchObject({ kind: 'restamp-only', reused_run_id: 300, gated_head: sha.gated, current_head: sha.restamp, base_sha: sha.base, reused_screen_policy: 'narrow' });
|
||||
expect(reuseReport(candidate!, REPO)).toContain('(a narrow screen)');
|
||||
});
|
||||
|
||||
test("a PR push under a train policy (no paid slice, no binding) is passed over; the PR's earlier screen still reuses", () => {
|
||||
const summary = run(320, sha.restamp!, 'success');
|
||||
const { candidate } = screen('narrow', [summary, screenRun(300)], { 300: screenBinding(sha.gated!) }, { jobs: { 320: [{ name: 'eval-slices', conclusion: 'skipped' }] } });
|
||||
expect(candidate?.reused_run_id).toBe(300);
|
||||
});
|
||||
|
||||
test('a broad screen never adopts a narrow run; a narrow screen never adopts a pull_request gate (broad); a broad screen may', () => {
|
||||
const narrowRun = screen('broad', [screenRun(300)], { 300: screenBinding(sha.gated!) });
|
||||
expect(narrowRun.candidate).toBeNull();
|
||||
expect(narrowRun.text).toContain('gated the narrow selection; a broad screen never adopts it');
|
||||
const prRun = screen('narrow', [run(300, sha.gated!)], { 300: binding() });
|
||||
expect(prRun.candidate).toBeNull();
|
||||
expect(prRun.text).toContain('gated the broad selection; a narrow screen never adopts it');
|
||||
expect(screen('broad', [run(300, sha.gated!)], { 300: binding() }).candidate?.reused_run_id).toBe(300);
|
||||
});
|
||||
|
||||
test('a forged screen is refused: another PR, a head outside the PR, a mismatched binding, a tested commit off its bound base, another repository', () => {
|
||||
const refused: Array<[WorkflowRun[], Record<number, GateBinding | string>, string, string[]?]> = [
|
||||
[[screenRun(300, 'narrow', { display_title: title('narrow', 43) })], { 300: screenBinding(sha.gated!) }, 'no earlier executed gate on this PR bound a verdict'],
|
||||
[[screenRun(300)], { 300: screenBinding(sha.gated!) }, 'is not a commit of PR #42', [sha.restamp!]],
|
||||
[[screenRun(300)], { 300: screenBinding(sha.gated!, 'narrow', { pr: 43 }) }, 'does not match the run it came from'],
|
||||
[[screenRun(300)], { 300: binding() }, 'does not match the run it came from'],
|
||||
[[screenRun(300)], { 300: screenBinding(sha.gated!, 'narrow', { tested_sha: tested(sha.moved!, sha.gated!) }) }, 'does not merge its head into its bound base'],
|
||||
[[screenRun(300, 'narrow', { head_repository: { full_name: 'fork/gstack' } })], { 300: screenBinding(sha.gated!) }, 'no earlier executed gate on this PR bound a verdict'],
|
||||
[[run(300, sha.gated!)], { 300: screenBinding(sha.gated!) }, 'does not match the run it came from'],
|
||||
];
|
||||
for (const [runs, bindings, why, prCommits] of refused) {
|
||||
const { candidate, text } = screen('narrow', runs, bindings, { prCommits });
|
||||
expect(candidate, why).toBeNull();
|
||||
expect(text, why).toContain(why);
|
||||
}
|
||||
});
|
||||
|
||||
test('train, bisect and plain dispatch runs never bind a verdict, so they never reuse or get reused', () => {
|
||||
const report = path.join(scratch, 'bind-purpose');
|
||||
fs.mkdirSync(report, { recursive: true });
|
||||
fs.writeFileSync(path.join(report, 'manifest.json'), JSON.stringify(manifest()));
|
||||
fs.writeFileSync(path.join(report, 'collector-outcomes.json'), JSON.stringify({ verdict: { verdict: 'GREEN' }, totals: { manual_accepted: 0 } }));
|
||||
const args = { repository: REPO, pr: 42, runId: 300, runAttempt: 1, headSha: 'a'.repeat(40), baseSha: 'b'.repeat(40), profile: 'full', reconcileExit: 0, reportDir: report };
|
||||
for (const purpose of ['train', 'bisect', 'dispatch'] as const) {
|
||||
const bound = buildBinding({ ...args, purpose });
|
||||
expect(bound.binding, purpose).toBeNull();
|
||||
expect(bound.reason).toBe(`a ${purpose} run never binds a verdict for reuse`);
|
||||
}
|
||||
expect(buildBinding({ ...args, profile: 'pr', purpose: 'screen', testedSha: 'c'.repeat(40), screenPolicy: 'broad' }).binding)
|
||||
.toMatchObject({ purpose: 'screen', tested_sha: 'c'.repeat(40), screen_policy: 'broad' });
|
||||
// A train dispatch in the history is neither a screen nor a pull_request run of this PR: never listed, never a candidate.
|
||||
const trainRun = run(310, sha.base!, 'success', { event: 'workflow_dispatch', pull_requests: [], display_title: 'E2E Evals [eval-bg gate.train-x]' });
|
||||
expect(screen('narrow', [trainRun, screenRun(300)], { 300: screenBinding(sha.gated!), 310: screenBinding(sha.gated!) }).candidate?.reused_run_id).toBe(300);
|
||||
});
|
||||
});
|
||||
@@ -7,7 +7,7 @@ import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import {
|
||||
collectRecovery, ghFailureReason, readCheckpoint, recoveryDecision, storeRunId,
|
||||
acceptedScreenRun, collectRecovery, ghFailureReason, listGateRuns, readCheckpoint, recoveryDecision, screenRunPr, storeRunId,
|
||||
type GhRunner, type WorkflowRun,
|
||||
} from '../scripts/recover-receipts';
|
||||
import { selectPlanReceipts, writeNegativeReceipt, writePanelReceipt } from '../scripts/e2e-shard-reuse';
|
||||
@@ -196,3 +196,30 @@ describe('dispatch reads PR receipts read-only (ENG-8)', () => {
|
||||
expect(gh.calls.filter(c => c[0] === 'run').map(c => c[2])).toEqual(['300']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('dispatched screens (GSTA-49 T1b): listed beside the PR\'s pull_request runs, filtered by PR and repository', () => {
|
||||
const screen = (id: number, pr = 42, over: Partial<WorkflowRun> = {}) => run(id, 'completed', 'success', { event: 'workflow_dispatch', pull_requests: [],
|
||||
display_title: `E2E Evals [eval-bg pr.screen.narrow-${'c'.repeat(12)}-${'d'.repeat(12)}-abc123] screen #${pr}`, ...over });
|
||||
|
||||
test('the run name names the PR only for a dispatched screen', () => {
|
||||
expect(screenRunPr(screen(1))).toBe(42);
|
||||
expect(screenRunPr({ event: 'pull_request', display_title: screen(1).display_title })).toBeNull();
|
||||
expect(screenRunPr({ event: 'workflow_dispatch', display_title: 'E2E Evals [eval-bg gate.train-x]' })).toBeNull();
|
||||
expect(screenRunPr({ event: 'workflow_dispatch', display_title: `${screen(1).display_title} extra` })).toBeNull();
|
||||
expect(acceptedScreenRun(screen(1, 42, { head_repository: { full_name: 'fork/gstack' } }), { pr: 42, repo: REPO, selfRunId: 9 })).toBe(false);
|
||||
expect(acceptedScreenRun(screen(9), { pr: 42, repo: REPO, selfRunId: 9 })).toBe(false);
|
||||
});
|
||||
|
||||
test('a screen lists both events; a pull_request run lists only its own event, exactly as before', () => {
|
||||
const calls: string[][] = [];
|
||||
const runs = [run(10, 'completed', 'success'), run(11, 'completed', 'success', { pull_requests: [{ number: 43 }] }), screen(12), screen(13, 43), run(14, 'completed', 'success', { event: 'workflow_dispatch' })];
|
||||
const gh: GhRunner = (args) => { calls.push(args); return { status: 0, stdout: JSON.stringify({ workflow_runs: runs }), stderr: '' }; };
|
||||
const o = { repo: REPO, pr: 42, headRepo: REPO, branch: 'feature', selfRunId: 99 };
|
||||
expect((listGateRuns(gh, { ...o, screens: false }, () => 1000) as { runs: WorkflowRun[] }).runs.map(r => r.id)).toEqual([10]);
|
||||
expect(calls).toEqual([['api', '-X', 'GET', `repos/${REPO}/actions/workflows/evals.yml/runs`, '-f', 'event=pull_request', '-f', 'branch=feature', '-f', 'per_page=50']]);
|
||||
expect((listGateRuns(gh, { ...o, screens: true }, () => 1000) as { runs: WorkflowRun[] }).runs.map(r => r.id)).toEqual([10, 12]);
|
||||
expect(calls.at(-1)).toEqual(['api', '-X', 'GET', `repos/${REPO}/actions/workflows/evals.yml/runs`, '-f', 'event=workflow_dispatch', '-f', 'per_page=50']);
|
||||
const down: GhRunner = (args, timeoutMs) => args.includes('event=workflow_dispatch') ? { status: 1, stdout: '', stderr: 'HTTP 502' } : gh(args, timeoutMs);
|
||||
expect(listGateRuns(down, { ...o, screens: true }, () => 1000)).toHaveProperty('error');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,208 @@
|
||||
/**
|
||||
* The main backstop (scripts/train-backstop.ts, evals-main-backstop.yml; plan
|
||||
* GSTA-49 T1b): inert without a train block; a train's merges are covered by
|
||||
* the train's own green record (a five-PR train fires zero backstop gates); a
|
||||
* merge outside the queue is gated, and a red gate writes the freeze record
|
||||
* gstack-ship-queue reads. Fixture repos with a bare origin under os.tmpdir()
|
||||
* and a scripted gh; no network, no API spend.
|
||||
*/
|
||||
import { afterAll, afterEach, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { loadPolicyFromHead } from '../lib/ship-policy';
|
||||
import { NOTES_REF, persist, repoIdentity } from '../lib/ship-queue-train-record';
|
||||
import type { GhRunner } from '../scripts/recover-receipts';
|
||||
import { runBackstop } from '../scripts/train-backstop';
|
||||
import { gitIn, trainPrefixes, type TrainRecord } from '../scripts/train-gate';
|
||||
import { E2E_TOUCHFILES } from './helpers/touchfiles-data';
|
||||
import { BASE_POLICY, cleanup, makeFixtureRepo, type FixtureRepo } from './helpers/restamp-fixture';
|
||||
import { TRAIN_BLOCK, TRAIN_POLICY } from './helpers/ship-queue-train-fixture';
|
||||
|
||||
const REPO = 'fixture/repo';
|
||||
const TRAIN_NONCE = 'gate.train-' + 'a'.repeat(12) + '-' + 'b'.repeat(12) + '-abc123';
|
||||
const RED_CASE = Object.keys(E2E_TOUCHFILES)[0]!;
|
||||
const repos: FixtureRepo[] = [];
|
||||
afterAll(() => { for (const repo of repos) cleanup(repo); });
|
||||
const savedStateRoot = process.env.GSTACK_STATE_ROOT;
|
||||
afterEach(() => { if (savedStateRoot === undefined) delete process.env.GSTACK_STATE_ROOT; else process.env.GSTACK_STATE_ROOT = savedStateRoot; });
|
||||
|
||||
interface Fixture { repo: FixtureRepo; main0: string; prs: string[]; train: string[]; record: TrainRecord }
|
||||
|
||||
/** Five PR branches from main and the train T1..T5 gstack-ship-queue would build from them (lib/ship-queue-train.ts buildChain). */
|
||||
function trainFixture(policy: Record<string, unknown> = TRAIN_POLICY): Fixture {
|
||||
const repo = makeFixtureRepo({ policy, prefix: 'train-backstop-fx-' });
|
||||
repos.push(repo);
|
||||
process.env.GSTACK_STATE_ROOT = path.join(repo.root, 'state');
|
||||
const main0 = repo.git('rev-parse', 'HEAD');
|
||||
const prs = [1, 2, 3, 4, 5].map(i => {
|
||||
repo.git('checkout', '-q', '-B', `pr${i}`, main0);
|
||||
fs.writeFileSync(path.join(repo.work, `pr${i}.txt`), `pr ${i}\n`);
|
||||
repo.git('add', '-A');
|
||||
repo.git('commit', '-q', '-m', `pr ${i}`);
|
||||
return repo.git('rev-parse', 'HEAD');
|
||||
});
|
||||
const train: string[] = [];
|
||||
let parent = main0;
|
||||
prs.forEach((head, i) => {
|
||||
const tree = repo.git('merge-tree', '--write-tree', parent, head);
|
||||
parent = repo.git('commit-tree', tree, '-p', parent, '-p', head, '-m', `train T${i + 1}: merge #${101 + i} at ${head.slice(0, 12)}`);
|
||||
train.push(parent);
|
||||
});
|
||||
repo.git('checkout', '-q', 'main');
|
||||
const prefixes = trainPrefixes(gitIn(repo.work), parent, repo.work, loadPolicyFromHead(repo.work).policy);
|
||||
if (!Array.isArray(prefixes)) throw new Error(prefixes.error);
|
||||
const record: TrainRecord = { schema: 1, kind: 'train-gate', nonce: TRAIN_NONCE, head_sha: parent, tree: repo.git('rev-parse', `${parent}^{tree}`), verdict: 'GREEN',
|
||||
census: { complete: true, planned: { rule: 108, judge: 24 }, executed: { rule: 108, judge: 24 }, unbound: 0, partial_shards: 0, cancelled_shards: 0, killed_before_upload: 0 },
|
||||
panels: [], run: { id: 900, attempt: 1, url: 'https://example.invalid/runs/900' }, prefixes };
|
||||
return { repo, main0, prs, train, record };
|
||||
}
|
||||
|
||||
/** The coordinator's squash merge of PR i onto main, stamped at merge (VERSION, package.json, the release output). */
|
||||
function squashMerge(repo: FixtureRepo, i: number, extra: Record<string, string> = {}): string {
|
||||
repo.git('checkout', '-q', 'main');
|
||||
const version = `1.2.${3 + i}.0`;
|
||||
const files: Record<string, string> = { [`pr${i}.txt`]: `pr ${i}\n`, VERSION: version + '\n', 'generated/digest.md': `# digest v${version}\nbody\n`, ...extra };
|
||||
for (const [rel, text] of Object.entries(files)) fs.writeFileSync(path.join(repo.work, rel), text);
|
||||
const pkg = JSON.parse(fs.readFileSync(path.join(repo.work, 'package.json'), 'utf8'));
|
||||
fs.writeFileSync(path.join(repo.work, 'package.json'), JSON.stringify({ ...pkg, version: version.split('.').slice(0, 3).join('.') }, null, 2) + '\n');
|
||||
repo.git('add', '-A');
|
||||
repo.git('commit', '-q', '-m', `#${100 + i} (squash, stamped ${version})`);
|
||||
repo.git('push', '-q', 'origin', 'main');
|
||||
return repo.git('rev-parse', 'HEAD');
|
||||
}
|
||||
|
||||
/** gh: the train's green run, its record artifact and check run, a gate dispatch whose run concludes as scripted, and an issue. */
|
||||
function fakeGh(fx: Fixture, o: { record?: TrainRecord | null; check?: string; gate?: string } = {}) {
|
||||
const calls: string[][] = [];
|
||||
let dispatched = '';
|
||||
const ok = (body: unknown) => ({ status: 0, stdout: typeof body === 'string' ? body : JSON.stringify(body), stderr: '' });
|
||||
const gh: GhRunner = (args) => {
|
||||
calls.push(args);
|
||||
const joined = args.join(' ');
|
||||
if (args[0] === 'api' && joined.includes('/actions/workflows/evals.yml/runs') && joined.includes('status=success')) {
|
||||
return ok({ workflow_runs: o.record === null ? [] : [{ id: 900, status: 'completed', conclusion: 'success', head_sha: fx.record.head_sha, display_title: `E2E Evals [eval-bg ${TRAIN_NONCE}]` }] });
|
||||
}
|
||||
if (args[0] === 'api' && joined.includes('/actions/workflows/evals.yml/runs')) {
|
||||
return ok({ workflow_runs: dispatched ? [{ id: 901, status: 'queued', conclusion: null, head_sha: '', display_title: `E2E Evals [eval-bg ${dispatched}]`, html_url: 'https://example.invalid/runs/901' }] : [] });
|
||||
}
|
||||
if (args[0] === 'run' && args[1] === 'download') {
|
||||
const dir = args[args.indexOf('-D') + 1]!;
|
||||
const name = args[args.indexOf('-n') + 1];
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
if (name === TRAIN_BLOCK.record_artifact) fs.writeFileSync(path.join(dir, 'record.json'), JSON.stringify(o.record ?? fx.record));
|
||||
if (name === 'trial-outcomes-pr-a1') fs.writeFileSync(path.join(dir, 'trial-outcomes.jsonl'), JSON.stringify({ case: RED_CASE, attempt: 1, outcome: 'failed' }) + '\n');
|
||||
return ok('');
|
||||
}
|
||||
if (args[0] === 'api' && /\/commits\/[0-9a-f]{40}\/check-runs/.test(joined)) return ok({ check_runs: [{ name: TRAIN_BLOCK.full_gate_check, conclusion: o.check ?? 'success' }] });
|
||||
if (args[0] === 'workflow' && args[1] === 'run') {
|
||||
dispatched = args.find(a => a.startsWith('nonce='))!.slice('nonce='.length);
|
||||
return ok('');
|
||||
}
|
||||
if (args[0] === 'api' && /actions\/runs\/901$/.test(joined)) return ok({ id: 901, status: 'completed', conclusion: o.gate ?? 'failure', head_sha: '', html_url: 'https://example.invalid/runs/901' });
|
||||
if (args[0] === 'issue' && args[1] === 'create') return ok('https://github.com/fixture/repo/issues/7\n');
|
||||
return { status: 1, stdout: '', stderr: `unexpected gh ${joined}` };
|
||||
};
|
||||
return { gh, calls, dispatches: () => calls.filter(c => c[0] === 'workflow' && c[1] === 'run') };
|
||||
}
|
||||
|
||||
const backstop = (fx: Fixture, sha: string, gh: GhRunner, over: Partial<Parameters<typeof runBackstop>[0]> = {}) =>
|
||||
runBackstop({ repo: REPO, branch: 'main', sha, repoRoot: fx.repo.work, gh, sleep: () => {}, pollMs: 1, random: 'abc123', ...over });
|
||||
|
||||
function originNotes(fx: Fixture): Array<Record<string, unknown>> {
|
||||
const list = fx.repo.git('--git-dir', fx.repo.origin, 'notes', `--ref=${NOTES_REF}`, 'list');
|
||||
return list.split('\n').filter(Boolean).map(line => JSON.parse(fx.repo.git('--git-dir', fx.repo.origin, 'cat-file', 'blob', line.split(' ')[0]!)));
|
||||
}
|
||||
|
||||
describe('backstop: inert without a train block', () => {
|
||||
test('a push to main under a policy with no train block runs nothing and calls no gh', () => {
|
||||
const fx = trainFixture(BASE_POLICY);
|
||||
const { gh, calls } = fakeGh(fx);
|
||||
const out = backstop(fx, squashMerge(fx.repo, 1), gh);
|
||||
expect(out.action).toBe('inert');
|
||||
expect(out.lines[0]).toContain('declares no train block');
|
||||
expect(calls).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("backstop: a train's merges are covered by the train's own green record", () => {
|
||||
test('a five-PR train fires zero backstop gates', () => {
|
||||
const fx = trainFixture();
|
||||
const { gh, dispatches } = fakeGh(fx);
|
||||
for (let i = 1; i <= 5; i++) {
|
||||
const out = backstop(fx, squashMerge(fx.repo, i), gh);
|
||||
expect(out.action, out.lines.join('\n')).toBe('covered');
|
||||
expect(out.lines.at(-1)).toContain(`is T${i} of green train ${TRAIN_NONCE} (run 900`);
|
||||
}
|
||||
expect(dispatches()).toEqual([]);
|
||||
});
|
||||
|
||||
test('a superseded push hands main to the newer push without a gh call', () => {
|
||||
const fx = trainFixture();
|
||||
const first = squashMerge(fx.repo, 1);
|
||||
squashMerge(fx.repo, 2);
|
||||
const { gh, calls } = fakeGh(fx);
|
||||
expect(backstop(fx, first, gh).action).toBe('superseded');
|
||||
expect(calls).toEqual([]);
|
||||
});
|
||||
|
||||
test('a record without a successful check run, or not GREEN, covers nothing: the backstop gates main', () => {
|
||||
const fx = trainFixture();
|
||||
const sha = squashMerge(fx.repo, 1);
|
||||
for (const o of [{ check: 'failure' }, { record: { ...fx.record, verdict: 'RED' as const } }, { record: { ...fx.record, census: { ...fx.record.census, complete: false } } }]) {
|
||||
const { gh, dispatches } = fakeGh(fx, { ...o, gate: 'success' });
|
||||
const out = backstop(fx, sha, gh);
|
||||
expect(out.action, JSON.stringify(o)).toBe('gated');
|
||||
expect(dispatches()).toHaveLength(1);
|
||||
expect(dispatches()[0]).toEqual(expect.arrayContaining(['evals.yml', '--ref', 'main', 'evals_all=true', `expected_sha=${sha}`]));
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('backstop: a merge outside the queue', () => {
|
||||
test('is gated; a red gate writes the freeze record with its red cases, inputs, run and issue; the next run on that commit defers to it', () => {
|
||||
const fx = trainFixture();
|
||||
squashMerge(fx.repo, 1);
|
||||
const outside = squashMerge(fx.repo, 6, { 'outside.txt': 'merged outside the queue\n' });
|
||||
const { gh, dispatches } = fakeGh(fx);
|
||||
const out = backstop(fx, outside, gh);
|
||||
expect(out.action, out.lines.join('\n')).toBe('frozen');
|
||||
expect(dispatches()).toHaveLength(1);
|
||||
const nonce = dispatches()[0]!.find(a => a.startsWith('nonce='))!.slice('nonce='.length);
|
||||
expect(nonce).toStartWith(`gate.backstop-${outside.slice(0, 12)}-${outside.slice(0, 12)}-`);
|
||||
const freeze = originNotes(fx).find(r => r.kind === 'freeze')!;
|
||||
expect(freeze).toMatchObject({ schema: 1, kind: 'freeze', repo_key: repoIdentity(fx.repo.work).key, train: 'backstop', cleared_at: null,
|
||||
main: { sha: outside }, backstop: { nonce, run: 901, state: 'completed/failure' }, issue: 'https://github.com/fixture/repo/issues/7',
|
||||
red_cases: [{ id: RED_CASE, inputs: E2E_TOUCHFILES[RED_CASE] }] });
|
||||
expect(String(freeze.reason)).toContain(`${RED_CASE} (inputs `);
|
||||
expect(String(freeze.reason)).toContain('fix: land the revert or fix-forward');
|
||||
const again = fakeGh(fx);
|
||||
expect(backstop(fx, outside, again.gh).action).toBe('deferred');
|
||||
expect(again.dispatches()).toEqual([]);
|
||||
});
|
||||
|
||||
test('a green gate on main writes nothing', () => {
|
||||
const fx = trainFixture();
|
||||
const outside = squashMerge(fx.repo, 6, { 'outside.txt': 'merged outside the queue\n' });
|
||||
const { gh, dispatches } = fakeGh(fx, { record: null, gate: 'success' });
|
||||
expect(backstop(fx, outside, gh).action).toBe('gated');
|
||||
expect(dispatches()).toHaveLength(1);
|
||||
expect(originNotes(fx)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('backstop: debounce while a train merges', () => {
|
||||
let fx: Fixture;
|
||||
beforeEach(() => { fx = trainFixture(); });
|
||||
test('waits while a train is merging, then decides once the settle window ends', () => {
|
||||
const at = new Date().toISOString();
|
||||
persist(fx.repo.work, [{ schema: 1, kind: 'train', nonce: TRAIN_NONCE, repo_key: repoIdentity(fx.repo.work).key, state: 'merging', updated_at: at } as never]);
|
||||
const sha = squashMerge(fx.repo, 1);
|
||||
let clock = 0;
|
||||
const { gh, dispatches } = fakeGh(fx);
|
||||
const out = backstop(fx, sha, gh, { now: () => (clock += 20 * 60_000), settleMs: 60 * 60_000 });
|
||||
expect(out.lines.some(line => line.includes(`waiting for train(s) ${TRAIN_NONCE} to finish merging`))).toBe(true);
|
||||
expect(out.lines.some(line => line.includes('still merging after the settle window'))).toBe(true);
|
||||
expect(out.action).toBe('covered');
|
||||
expect(dispatches()).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,250 @@
|
||||
/**
|
||||
* The merge-train side of evals.yml (scripts/train-gate.ts; plan GSTA-49 T1b):
|
||||
* dispatch routing and its refusals, the green-train record (census from the
|
||||
* run's own plan manifest, first attempt only), prefix trees modulo stamps,
|
||||
* the per-prefix bisect record and the free screening summary. Fixture repos
|
||||
* under os.tmpdir() with their own git identity; no network, no API spend.
|
||||
*/
|
||||
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { classifyGate } from '../lib/ship-queue-train-verdict';
|
||||
import { buildRunManifest, type PaidRunManifest } from '../scripts/lib/paid-plan';
|
||||
import { buildReceipt } from '../lib/tree-receipt';
|
||||
import {
|
||||
FULL_GATE_USD, JUDGE_FILE, MAX_BISECT_CASES, MAX_PREFIXES, bisectMatrix, buildTrainRecord, fullGateCases, gitIn, routeDispatch, runBisectPrefix,
|
||||
screenSummary, stampFreeTree, trainPrefixes, type BisectPlan, type RouteInput,
|
||||
} from '../scripts/train-gate';
|
||||
import { loadPolicyFromHead } from '../lib/ship-policy';
|
||||
import { LLM_JUDGE_TOUCHFILES } from './helpers/touchfiles-data';
|
||||
import { cleanup, makeFixtureRepo, type FixtureRepo } from './helpers/restamp-fixture';
|
||||
import { TRAIN_POLICY } from './helpers/ship-queue-train-fixture';
|
||||
|
||||
const scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'train-gate-'));
|
||||
afterAll(() => fs.rmSync(scratch, { recursive: true, force: true }));
|
||||
const A = 'a'.repeat(40);
|
||||
const B = 'b'.repeat(40);
|
||||
const C = 'c'.repeat(40);
|
||||
|
||||
describe('routeDispatch: every run keeps its mode, and train inputs are validated before any paid work', () => {
|
||||
const base: RouteInput = { nonce: '', refName: 'main', head: A, expectedSha: '', evalsAll: true, trainPolicy: false, cases: '', prefixShas: '',
|
||||
screenPr: '', screenMergeSha: '', screenPolicy: '' };
|
||||
const deps = { isAncestor: (commit: string) => commit !== C, gateCases: () => new Map([['case-a', 'rule'], ['review/SKILL.md workflow', 'judge']]) };
|
||||
const route = (over: Partial<RouteInput>) => routeDispatch({ ...base, ...over }, deps);
|
||||
const screenNonce = (policy: string) => `pr.screen.${policy}-${A.slice(0, 12)}-${B.slice(0, 12)}-abc123`;
|
||||
const bisectNonce = `gate.bisect-${A.slice(0, 12)}-${B.slice(0, 12)}-abc123`;
|
||||
const trainNonce = `gate.train-${A.slice(0, 12)}-${B.slice(0, 12)}-abc123`;
|
||||
|
||||
test('a plain eval-bg or backstop dispatch routes as today, with or without a train policy', () => {
|
||||
expect(route({})).toEqual({ mode: 'dispatch', cases: [], prefixes: [] });
|
||||
expect(route({ nonce: `pr-${A.slice(0, 12)}-${B.slice(0, 12)}-abc123`, evalsAll: false })).toMatchObject({ mode: 'dispatch' });
|
||||
expect(route({ nonce: `gate.backstop-${A.slice(0, 12)}-${A.slice(0, 12)}-abc123`, trainPolicy: true })).toMatchObject({ mode: 'dispatch' });
|
||||
});
|
||||
|
||||
test('a screen needs a PR number, a full merge SHA, a policy, evals_all=false, a matching nonce and a train policy', () => {
|
||||
const screen = { screenPr: '42', screenMergeSha: C, screenPolicy: 'narrow', evalsAll: false, expectedSha: C, nonce: screenNonce('narrow'), trainPolicy: true };
|
||||
expect(route(screen)).toEqual({ mode: 'screen', cases: [], prefixes: [] });
|
||||
expect(route({ ...screen, expectedSha: '' })).toMatchObject({ mode: 'screen' });
|
||||
const refused = [
|
||||
{ screenPr: '0' }, { screenPr: '42; rm -rf' }, { screenMergeSha: C.slice(0, 12) }, { screenPolicy: 'wide' }, { evalsAll: true }, { expectedSha: A },
|
||||
{ nonce: screenNonce('broad') }, { nonce: bisectNonce }, { cases: 'case-a' }, { trainPolicy: false },
|
||||
];
|
||||
for (const over of refused) expect(route({ ...screen, ...over }), JSON.stringify(over)).toHaveProperty('error');
|
||||
expect((route({ ...screen, trainPolicy: false }) as { error: string }).error).toContain('needs a train block');
|
||||
});
|
||||
|
||||
test('a bisect needs a bisect nonce on train/<nonce>, the train head, plan case ids and ancestor prefixes', () => {
|
||||
const bisect = { nonce: bisectNonce, refName: 'train/t1', expectedSha: A, cases: 'case-a, review/SKILL.md workflow', prefixShas: `${A},${B}`, trainPolicy: true };
|
||||
expect(route(bisect)).toEqual({ mode: 'bisect', cases: ['case-a', 'review/SKILL.md workflow'], prefixes: [A, B] });
|
||||
const many = (n: number, f: (i: number) => string) => Array.from({ length: n }, (_, i) => f(i)).join(',');
|
||||
const refused: Array<[Partial<RouteInput>, string]> = [
|
||||
[{ nonce: trainNonce }, 'needs a bisect nonce'],
|
||||
[{ refName: 'main' }, 'runs on train/<nonce>'],
|
||||
[{ expectedSha: B }, 'is not the train head'],
|
||||
[{ cases: '' }, 'cases must name'],
|
||||
[{ cases: 'case-a,case-a' }, 'cases must name'],
|
||||
[{ cases: many(MAX_BISECT_CASES + 1, i => `case-${i}`) }, 'cases must name'],
|
||||
[{ prefixShas: many(MAX_PREFIXES + 1, i => i.toString(16).padStart(40, '0')) }, 'prefix_shas must name'],
|
||||
[{ prefixShas: A.slice(0, 12) }, 'is not a full commit SHA'],
|
||||
[{ prefixShas: `${A},${C}` }, 'is not an ancestor of the train head'],
|
||||
[{ cases: 'case-a,not-a-plan-case' }, "not in the full gate's plan"],
|
||||
[{ trainPolicy: false }, 'needs a train block'],
|
||||
];
|
||||
for (const [over, why] of refused) expect((route({ ...bisect, ...over }) as { error?: string }).error, why).toContain(why);
|
||||
expect((route({ cases: 'case-a', prefixShas: A }) as { error?: string }).error).toContain('needs a bisect nonce');
|
||||
});
|
||||
|
||||
test('a train gate is the full gate of the train head on train/<nonce>, under a train policy', () => {
|
||||
const train = { nonce: trainNonce, refName: 'train/t1', expectedSha: A, trainPolicy: true };
|
||||
expect(route(train)).toEqual({ mode: 'train', cases: [], prefixes: [] });
|
||||
for (const over of [{ refName: 'main' }, { evalsAll: false }, { expectedSha: '' }, { trainPolicy: false }]) expect(route({ ...train, ...over })).toHaveProperty('error');
|
||||
});
|
||||
|
||||
test("the bisect's case universe is the full gate's own plan on this checkout", () => {
|
||||
const cases = fullGateCases();
|
||||
expect([...cases.values()].filter(kind => kind === 'judge').length).toBe(Object.keys(LLM_JUDGE_TOUCHFILES).length);
|
||||
expect([...cases.values()].filter(kind => kind !== 'judge').length).toBeGreaterThan(50);
|
||||
});
|
||||
});
|
||||
|
||||
describe('train record: the census comes from the run\'s own manifest, first attempt only', () => {
|
||||
const [j1, j2] = Object.keys(LLM_JUDGE_TOUCHFILES) as [string, string];
|
||||
const behavior = 'shared-libs-opportunity-judgment';
|
||||
const judgeEntry = buildRunManifest({ tier: 'gate', profile: 'full', sliceCount: 1, evalsAll: true, env: { EVALS_ALL: '1' } }).entries.find(e => e.file === JUDGE_FILE)!;
|
||||
const trialFiles = [1, 2, 3].map(t => `test/skill-e2e-shared-libs-periodic.test.ts#${behavior}~t${t}`);
|
||||
const manifest = { version: 1, tier: 'gate', evalsAll: true, sliceCount: 2, selectionReason: 'evals_all', profile: 'full', selection: { e2e: null, judges: [j1, j2] },
|
||||
entries: [
|
||||
{ ...judgeEntry, slice: 1 },
|
||||
...trialFiles.map(file => ({ file, slice: 2, status: 'planned', trial: { kind: 'behavior', panel: { n: 3, k: 2 }, quarantined: false } })),
|
||||
] } as unknown as PaidRunManifest;
|
||||
type Slice = { index: number; attempt?: number; files?: string[]; checkpoint?: true };
|
||||
let n = 0;
|
||||
function report(o: { slices?: Slice[]; outcomes?: Array<Record<string, unknown>>; collector?: Record<string, unknown> | null } = {}): string {
|
||||
const dir = path.join(scratch, `report-${++n}`);
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(path.join(dir, 'manifest.json'), JSON.stringify(manifest));
|
||||
const outcomes = o.outcomes ?? [j1, j2].map(id => ({ case: id, kind: 'judge', attempt: 1, outcome: 'passed', cost_usd: 0.25 }))
|
||||
.concat([1, 2, 3].map(trial => ({ case: behavior, kind: 'behavior', trial, attempt: 1, outcome: 'passed', cost_usd: 1 })));
|
||||
fs.writeFileSync(path.join(dir, 'trial-outcomes.jsonl'), outcomes.map(line => JSON.stringify(line)).join('\n') + '\n');
|
||||
if (o.collector !== null) fs.writeFileSync(path.join(dir, 'collector-outcomes.json'), JSON.stringify(o.collector ?? { verdict: { verdict: 'GREEN' }, census: { unattributed: [] }, panels: [] }));
|
||||
for (const s of o.slices ?? [{ index: 1 }, { index: 2 }]) {
|
||||
const at = path.join(dir, `paid-slice-${s.index}-a${s.attempt ?? 1}`);
|
||||
fs.mkdirSync(at, { recursive: true });
|
||||
fs.writeFileSync(path.join(at, `slice-${s.index}.json`), JSON.stringify({ version: 1, tier: 'gate', sliceIndex: s.index, sliceCount: 2, attempt: s.attempt ?? 1,
|
||||
...(s.checkpoint ? { checkpoint: true } : {}), outcomes: (s.files ?? (s.index === 1 ? [JUDGE_FILE] : trialFiles)).map(file => ({ files: [file], status: 'passed' })) }));
|
||||
}
|
||||
return dir;
|
||||
}
|
||||
const build = (dir: string, reconcileExit = 0) => buildTrainRecord({ reportDir: dir, nonce: 'gate.train-x', head: A, tree: B, reconcileExit,
|
||||
prefixes: [{ index: 1, commit: C, tree: B, stamp_free_tree: 'd'.repeat(64) }], run: { id: 7, attempt: 1, url: 'https://example.invalid/runs/7' } });
|
||||
const classify = (record: unknown) => classifyGate(record, { nonce: 'gate.train-x', head: A, tree: B }, { status: 'completed', conclusion: 'success' },
|
||||
{ status: 'completed', conclusion: 'success' } as never).outcome;
|
||||
|
||||
test('a complete census of passing cases is GREEN, and T1 classifies it GREEN', () => {
|
||||
const record = build(report());
|
||||
expect(record).toMatchObject({ schema: 1, kind: 'train-gate', nonce: 'gate.train-x', head_sha: A, tree: B, verdict: 'GREEN', run: { id: 7 } });
|
||||
expect(record.census).toEqual({ complete: true, planned: { rule: 1, judge: 2 }, executed: { rule: 1, judge: 2 }, unbound: 0, partial_shards: 0, cancelled_shards: 0, killed_before_upload: 0 });
|
||||
expect(record.panels.map(p => [p.case, p.status, p.cost_usd])).toEqual([[j2, 'PASS', 0.25], [j1, 'PASS', 0.25], [behavior, 'PASS', 3]].sort((a, b) => (a[0]! < b[0]! ? -1 : 1)));
|
||||
expect(classify(record)).toBe('GREEN');
|
||||
});
|
||||
|
||||
test('a slice killed before its upload step is incomplete, never passed over, and a re-run attempt cannot fill it', () => {
|
||||
const killed = build(report({ slices: [{ index: 1 }], outcomes: [j1, j2].map(id => ({ case: id, kind: 'judge', attempt: 1, outcome: 'passed' })) }));
|
||||
expect(killed.verdict).toBe('RED');
|
||||
expect(killed.census).toMatchObject({ complete: false, killed_before_upload: 1, executed: { rule: 0, judge: 2 } });
|
||||
expect(classify(killed)).toBe('INCOMPLETE');
|
||||
const rerun = build(report({ slices: [{ index: 1 }, { index: 2, attempt: 2 }], outcomes: [
|
||||
...[j1, j2].map(id => ({ case: id, kind: 'judge', attempt: 1, outcome: 'passed' })), ...[1, 2, 3].map(trial => ({ case: behavior, kind: 'behavior', trial, attempt: 2, outcome: 'passed' }))] }));
|
||||
expect(rerun.census).toMatchObject({ complete: false, killed_before_upload: 1, executed: { rule: 0, judge: 2 } });
|
||||
expect(rerun.verdict).toBe('RED');
|
||||
});
|
||||
|
||||
test('a partial or cancelled shard, a missing collector, a red reconcile or a red collector is never GREEN', () => {
|
||||
expect(build(report({ slices: [{ index: 1 }, { index: 2, checkpoint: true }] })).census.partial_shards).toBe(1);
|
||||
expect(build(report({ slices: [{ index: 1 }, { index: 2, files: [] }] })).census.cancelled_shards).toBe(trialFiles.length);
|
||||
for (const record of [build(report({ slices: [{ index: 1 }, { index: 2, checkpoint: true }] })), build(report({ slices: [{ index: 1 }, { index: 2, files: [] }] })),
|
||||
build(report({ collector: null })), build(report(), 1), build(report({ collector: { verdict: { verdict: 'RED' }, census: { unattributed: [] }, panels: [] } })),
|
||||
build(report({ collector: { verdict: { verdict: 'GREEN' }, census: { unattributed: [{ file: 'x' }] }, panels: [] } }))]) {
|
||||
expect(record.verdict).toBe('RED');
|
||||
}
|
||||
});
|
||||
|
||||
test('a failed case is a red VERDICT panel T1 bisects on', () => {
|
||||
const record = build(report({ outcomes: [{ case: j1, kind: 'judge', attempt: 1, outcome: 'failed', failure_class: 'assertion' },
|
||||
{ case: j2, kind: 'judge', attempt: 1, outcome: 'passed' }, ...[1, 2, 3].map(trial => ({ case: behavior, kind: 'behavior', trial, attempt: 1, outcome: 'passed' }))],
|
||||
collector: { verdict: { verdict: 'RED' }, census: { unattributed: [] }, panels: [] } }));
|
||||
expect(record.verdict).toBe('RED');
|
||||
expect(record.panels.find(p => p.case === j1)).toMatchObject({ status: 'FAIL', red_class: 'VERDICT' });
|
||||
expect(classify(record)).toBe('RED');
|
||||
});
|
||||
});
|
||||
|
||||
describe('prefix trees modulo stamps', () => {
|
||||
let repo: FixtureRepo;
|
||||
const sha: Record<string, string> = {};
|
||||
beforeAll(() => {
|
||||
repo = makeFixtureRepo({ policy: TRAIN_POLICY, prefix: 'train-gate-fx-' });
|
||||
sha.main = repo.git('rev-parse', 'HEAD');
|
||||
const branch = (name: string, file: string) => {
|
||||
repo.git('checkout', '-q', '-B', name, sha.main!);
|
||||
fs.writeFileSync(path.join(repo.work, file), `${name}\n`);
|
||||
repo.git('add', '-A');
|
||||
repo.git('commit', '-q', '-m', name);
|
||||
return repo.git('rev-parse', 'HEAD');
|
||||
};
|
||||
sha.pr1 = branch('pr1', 'one.txt');
|
||||
sha.pr2 = branch('pr2', 'two.txt');
|
||||
const merge = (parent: string, head: string, index: number, pr: number) => {
|
||||
const tree = repo.git('merge-tree', '--write-tree', parent, head);
|
||||
return repo.git('commit-tree', tree, '-p', parent, '-p', head, '-m', `train T${index}: merge #${pr} at ${head.slice(0, 12)}`);
|
||||
};
|
||||
sha.t1 = merge(sha.main!, sha.pr1!, 1, 11);
|
||||
sha.t2 = merge(sha.t1!, sha.pr2!, 2, 12);
|
||||
repo.git('checkout', '-q', '-B', 'squash', sha.main!);
|
||||
fs.writeFileSync(path.join(repo.work, 'one.txt'), 'pr1\n');
|
||||
fs.writeFileSync(path.join(repo.work, 'VERSION'), '1.2.4.0\n');
|
||||
fs.writeFileSync(path.join(repo.work, 'generated', 'digest.md'), '# digest v1.2.4.0\nbody\n');
|
||||
const pkg = JSON.parse(fs.readFileSync(path.join(repo.work, 'package.json'), 'utf8'));
|
||||
fs.writeFileSync(path.join(repo.work, 'package.json'), JSON.stringify({ ...pkg, version: '1.2.4' }, null, 2) + '\n');
|
||||
repo.git('add', '-A');
|
||||
repo.git('commit', '-q', '-m', 'squash #11 (stamped)');
|
||||
sha.squash1 = repo.git('rev-parse', 'HEAD');
|
||||
fs.writeFileSync(path.join(repo.work, 'one.txt'), 'pr1, edited\n');
|
||||
repo.git('commit', '-qam', 'a non-stamp edit');
|
||||
sha.edited = repo.git('rev-parse', 'HEAD');
|
||||
});
|
||||
afterAll(() => cleanup(repo));
|
||||
|
||||
test('a squash merge that differs from its prefix only by stamps has the same tree modulo stamps; a code byte does not', () => {
|
||||
const git = gitIn(repo.work);
|
||||
const policy = loadPolicyFromHead(repo.work).policy;
|
||||
const t1 = stampFreeTree(git, sha.t1!, repo.work, policy);
|
||||
expect(t1).toMatch(/^[0-9a-f]{64}$/);
|
||||
expect(repo.git('rev-parse', `${sha.t1}^{tree}`)).not.toBe(repo.git('rev-parse', `${sha.squash1}^{tree}`));
|
||||
expect(stampFreeTree(git, sha.squash1!, repo.work, policy)).toBe(t1);
|
||||
expect(stampFreeTree(git, sha.edited!, repo.work, policy)).not.toBe(t1);
|
||||
// The same verdicts as the tree receipt verify and gate reuse use (lib/tree-receipt.ts).
|
||||
const receipt = (current: string) => (buildReceipt({ repoRoot: repo.work, gated: sha.t1!, current, policy, policyLabel: 'fixture' }) as { tree: string }).tree;
|
||||
expect(receipt(sha.squash1!)).toBe('same-modulo-stamps');
|
||||
expect(receipt(sha.edited!)).toBe('changed');
|
||||
expect(stampFreeTree(git, 'f'.repeat(40), repo.work, policy)).toHaveProperty('error');
|
||||
});
|
||||
|
||||
test('T1..Tk come from the train head by first parents; a head that is not a train is refused', () => {
|
||||
const git = gitIn(repo.work);
|
||||
const policy = loadPolicyFromHead(repo.work).policy;
|
||||
const prefixes = trainPrefixes(git, sha.t2!, repo.work, policy);
|
||||
expect(Array.isArray(prefixes)).toBe(true);
|
||||
expect((prefixes as Array<{ index: number; commit: string }>).map(p => [p.index, p.commit])).toEqual([[1, sha.t1], [2, sha.t2]]);
|
||||
expect(trainPrefixes(git, sha.main!, repo.work, policy)).toHaveProperty('error');
|
||||
});
|
||||
});
|
||||
|
||||
describe('bisect: one executor per prefix, one trial per case', () => {
|
||||
const plan: BisectPlan = { schema: 1, nonce: 'gate.bisect-x', cases: [{ id: 'case-a', kind: 'rule' }, { id: 'case-b', kind: 'judge' }], prefixes: [A, B] };
|
||||
|
||||
test('the matrix indexes prefixes from 1 and scales its ceiling with the case count', () => {
|
||||
expect(bisectMatrix(plan)).toEqual({ slices: [1, 2], timeouts: [0, 80, 80], refs: ['', A, B] });
|
||||
});
|
||||
|
||||
test('each prefix writes the record T1 reads; a case that leaves no verdict is INCOMPLETE; a wrong checkout refuses', async () => {
|
||||
const record = await runBisectPrefix(plan, 2, { rootDir: scratch, evalDir: path.join(scratch, 'bisect'), checkedOut: B,
|
||||
runCase: async (id) => { if (id === 'case-b') throw new Error('runner crashed'); return { status: 'FAIL', redClass: 'VERDICT' }; } });
|
||||
expect(record).toEqual({ schema: 1, kind: 'train-bisect-prefix', nonce: 'gate.bisect-x', prefix_sha: B, checked_out_sha: B, panels: [
|
||||
{ case: 'case-a', kind: 'rule', status: 'FAIL', red_class: 'VERDICT', cost_usd: null },
|
||||
{ case: 'case-b', kind: 'judge', status: 'INCOMPLETE', red_class: 'INCOMPLETE', cost_usd: null }] });
|
||||
await expect(runBisectPrefix(plan, 1, { rootDir: scratch, evalDir: scratch, checkedOut: B, runCase: async () => ({ status: 'PASS', redClass: null }) })).rejects.toThrow('not its prefix');
|
||||
await expect(runBisectPrefix(plan, 3, { rootDir: scratch, evalDir: scratch, checkedOut: B })).rejects.toThrow('names no prefix');
|
||||
});
|
||||
});
|
||||
|
||||
test('the free screening summary says first how to request the screen, then the selection and its estimated cost', () => {
|
||||
const [j1] = Object.keys(LLM_JUDGE_TOUCHFILES) as [string];
|
||||
const manifest = { version: 1, tier: 'gate', evalsAll: false, sliceCount: 1, profile: 'pr', selection: { e2e: [], judges: [j1] }, prCoverage: { mode: 'narrow' },
|
||||
entries: [{ file: JUDGE_FILE, slice: 1, status: 'planned' }] } as unknown as PaidRunManifest;
|
||||
const lines = screenSummary(manifest, '42');
|
||||
expect(lines[0]).toStartWith('To run the paid screen of this head: `gstack-ship-queue ready --pr 42 --screen`');
|
||||
expect(lines[1]).toBe('- Screening selection: 0 case(s) + 1 judge(s), PR coverage mode `narrow`');
|
||||
expect(lines[2]).toMatch(new RegExp(`^- Estimated screen cost: ~\\$\\d+\\.\\d{2} .*\\$${FULL_GATE_USD} full gate.* No paid case ran on this push\\.$`));
|
||||
});
|
||||
Reference in new issue
Block a user