mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-01 14:34:42 +01:00
Garry Tanandcapy-ai[bot]
95152e3c4c
* feat(evals): the gate reruns failed cases once and reports a flaky green (GSTA-49 F1) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * ci: retrigger after retargeting to main Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test(evals): a case red twice stays a real red for the report, the run, reuse and the train verdict (GSTA-49 F1) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(evals): keep a lone rerun-plan artifact off the plan manifest (run 39363705692) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * chore: stamp v1.91.163.0 Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
1568 lines
81 KiB
YAML
1568 lines
81 KiB
YAML
name: E2E Evals
|
|
# eval-bg (scripts/eval-bg.ts) finds its dispatch by the nonce in the run name (ENG-12).
|
|
# A dispatched screen's name also ends ` screen #<pr>`, which recover-receipts and
|
|
# gate-reuse filter on to list a PR's screens beside its pull_request runs.
|
|
run-name: "${{ inputs.nonce != '' && format('E2E Evals [eval-bg {0}]{1}', inputs.nonce, inputs.screen_pr != '' && format(' screen #{0}', inputs.screen_pr) || '') || '' }}"
|
|
on:
|
|
pull_request:
|
|
branches: [main]
|
|
workflow_dispatch:
|
|
inputs:
|
|
evals_all:
|
|
description: 'Run ALL gate tests in the sliced lane (bypass diff selection; also arms the hollow-shard guard). false = the PR profile on the diff against base_sha/base_ref'
|
|
type: boolean
|
|
default: true
|
|
base_ref:
|
|
description: 'Base branch for diff selection when evals_all is false (default: detected base branch)'
|
|
type: string
|
|
default: ''
|
|
base_sha:
|
|
description: 'Exact merge-base commit for diff selection; overrides base_ref'
|
|
type: string
|
|
default: ''
|
|
expected_sha:
|
|
description: 'Fail fast unless the checked-out commit is this SHA (scripts/eval-bg.ts sets it)'
|
|
type: string
|
|
default: ''
|
|
nonce:
|
|
description: 'Opaque id shown in the run name so the dispatcher can find this run'
|
|
type: string
|
|
default: ''
|
|
pr_receipts:
|
|
description: 'Same-repo PR number whose receipt store this run reads read-only (never saves); empty = fresh'
|
|
type: string
|
|
default: ''
|
|
# Merge trains (docs/ship-policy.md#run-a-merge-train): only under a train
|
|
# block in the default branch's .gstack/ship-policy.json. The route job
|
|
# validates them before any paid work (scripts/train-gate.ts route).
|
|
cases:
|
|
description: 'Train bisect: comma-separated case ids from the full gate plan (gstack-ship-queue train bisect sets it)'
|
|
type: string
|
|
default: ''
|
|
prefix_shas:
|
|
description: 'Train bisect: comma-separated prefix commits, each an ancestor of the train head on train/<nonce>'
|
|
type: string
|
|
default: ''
|
|
screen_pr:
|
|
description: 'Dispatched screen: the same-repo PR number (gstack-ship-queue ready --screen sets it)'
|
|
type: string
|
|
default: ''
|
|
screen_merge_sha:
|
|
description: "Dispatched screen: the PR's refs/pull/<n>/merge commit to check out and test"
|
|
type: string
|
|
default: ''
|
|
screen_policy:
|
|
description: 'Dispatched screen: narrow or broad (a broad screen never adopts a narrow run)'
|
|
type: string
|
|
default: ''
|
|
|
|
# A dispatched screen is keyed on its PR, so a newer head's screen cancels the older one.
|
|
concurrency:
|
|
group: evals-${{ github.event.pull_request.number || (inputs.screen_pr != '' && format('screen-{0}', inputs.screen_pr)) || github.run_id }}
|
|
cancel-in-progress: true
|
|
|
|
env:
|
|
IMAGE: ghcr.io/${{ github.repository }}/ci
|
|
# PRs and evals_all=false dispatches (eval:bg:pr) run the fast PR profile on the diff; evals_all dispatches run the complete gate census.
|
|
EVALS_PROFILE: ${{ (github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && !inputs.evals_all)) && 'pr' || 'full' }}
|
|
# Dispatches validate fresh unless they name pr_receipts (read-only reuse, CEO-29/ENG-8); the evals-fresh PR label opts a PR out of reuse (DX-7).
|
|
EVALS_FRESH: ${{ ((github.event_name == 'workflow_dispatch' && inputs.pr_receipts == '') || contains(github.event.pull_request.labels.*.name, 'evals-fresh')) && '1' || '' }}
|
|
|
|
jobs:
|
|
# Build Docker image with pre-baked toolchain (cached — only rebuilds on Dockerfile/lockfile change)
|
|
build-image:
|
|
# Dependabot-triggered pull_request runs get a read-only GITHUB_TOKEN, so
|
|
# a lockfile bump = new hash = failed ghcr push = permanently red check
|
|
# (EV6, fork port wave 2). Skip the build for dependabot; the evals job's
|
|
# explicit actor guard mirrors it because no eval test selects on a lockfile-only
|
|
# diff — a maintainer's next push rebuilds the image with real perms.
|
|
if: github.actor != 'dependabot[bot]'
|
|
runs-on: ubicloud-standard-8
|
|
timeout-minutes: 15
|
|
permissions:
|
|
contents: read
|
|
packages: write
|
|
outputs:
|
|
image-tag: ${{ steps.meta.outputs.tag }}
|
|
runtime-id: ${{ steps.runtime.outputs.id }}
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
|
|
- id: meta
|
|
# Key on Dockerfile + lockfile only. package.json is deliberately NOT
|
|
# hashed: its version field changes on every ship (60/60 recent commits),
|
|
# which rebuilt the image each time for a dependency set that only
|
|
# bun.lock determines. A stale baked package.json is harmless — checkout
|
|
# overwrites /workspace and node_modules comes from the lockfile.
|
|
run: echo "tag=${{ env.IMAGE }}:${{ hashFiles('.github/docker/Dockerfile.ci', 'bun.lock', 'patches/**') }}" >> "$GITHUB_OUTPUT"
|
|
|
|
- uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
|
|
with:
|
|
registry: ghcr.io
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Check if image exists
|
|
id: check
|
|
run: |
|
|
if docker manifest inspect ${{ steps.meta.outputs.tag }} > /dev/null 2>&1; then
|
|
echo "exists=true" >> "$GITHUB_OUTPUT"
|
|
else
|
|
echo "exists=false" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
|
|
- if: steps.check.outputs.exists == 'false'
|
|
run: cp package.json bun.lock .github/docker/ && cp -R patches .github/docker/patches
|
|
|
|
# A fork PR's GITHUB_TOKEN only has `packages: read`, so pushing fails.
|
|
# Still BUILD (validates Dockerfile.ci changes), just don't publish. This
|
|
# job intentionally keeps no `if:` so fork PRs still get one real, honest
|
|
# green check here instead of a run where every job is grey.
|
|
# Registry cache export needs a docker-container builder — the default
|
|
# `docker` driver hard-errors on cache-to (first live run of the trio).
|
|
- if: steps.check.outputs.exists == 'false'
|
|
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
|
|
|
|
- if: steps.check.outputs.exists == 'false'
|
|
uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
|
|
with:
|
|
context: .github/docker
|
|
file: .github/docker/Dockerfile.ci
|
|
push: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
|
|
# Registry layer cache: reads are safe everywhere; the export is gated
|
|
# to same-repo runs because a fork PR's token can't write GHCR.
|
|
cache-from: type=registry,ref=${{ env.IMAGE }}:buildcache
|
|
cache-to: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && format('type=registry,ref={0}:buildcache,mode=max', env.IMAGE) || '' }}
|
|
tags: |
|
|
${{ steps.meta.outputs.tag }}
|
|
${{ env.IMAGE }}:latest
|
|
|
|
- name: Identify the installed eval runtime
|
|
id: runtime
|
|
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
|
|
env:
|
|
EVAL_IMAGE: ${{ steps.meta.outputs.tag }}
|
|
run: |
|
|
docker manifest inspect "$EVAL_IMAGE" > /tmp/eval-runtime-manifest.json
|
|
echo "id=$(sha256sum /tmp/eval-runtime-manifest.json | cut -d ' ' -f1)" >> "$GITHUB_OUTPUT"
|
|
|
|
# ── Route (merge trains, GSTA-49 T1b; docs/ship-policy.md#run-a-merge-train)
|
|
# Names the run's mode before any paid work, from the train policy of the
|
|
# base revision (the PR's base commit, or the default branch for a
|
|
# dispatch). Without a `train` block every run routes as before: a
|
|
# pull_request run is `pr`, a dispatch is `dispatch`, and no other job
|
|
# changes. Under a train block a PR push is `pr-summary` (screen-summary
|
|
# prints the screening selection and its estimated cost; no paid slice
|
|
# starts), and gstack-ship-queue's dispatches are `screen`, `train` or
|
|
# `bisect` once scripts/train-gate.ts has validated their inputs. A refused
|
|
# dispatch fails here, before any paid work.
|
|
route:
|
|
runs-on: ubuntu-24.04
|
|
if: github.actor != 'dependabot[bot]'
|
|
timeout-minutes: 30
|
|
permissions:
|
|
contents: read
|
|
actions: read
|
|
pull-requests: read
|
|
outputs:
|
|
mode: ${{ steps.route.outputs.mode }}
|
|
record_artifact: ${{ steps.policy.outputs.record_artifact }}
|
|
full_gate_check: ${{ steps.policy.outputs.full_gate_check }}
|
|
screen_head: ${{ steps.screen.outputs.head_sha }}
|
|
screen_base: ${{ steps.screen.outputs.base_sha }}
|
|
screen_branch: ${{ steps.screen.outputs.head_branch }}
|
|
superseded: ${{ steps.screen.outputs.superseded }}
|
|
bisect_slices: ${{ steps.bisect.outputs.slices }}
|
|
bisect_timeouts: ${{ steps.bisect.outputs.slice_timeouts }}
|
|
bisect_refs: ${{ steps.bisect.outputs.prefix_refs }}
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
with:
|
|
fetch-depth: 0
|
|
persist-credentials: false
|
|
|
|
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
|
with:
|
|
bun-version: 1.4.2
|
|
|
|
# Read with git and jq only: no repository code decides whether PR pushes start paid work.
|
|
- name: Read the train policy (base revision)
|
|
id: policy
|
|
env:
|
|
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
|
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
|
run: |
|
|
base="${BASE_SHA:-origin/${DEFAULT_BRANCH}}"
|
|
train=$(git show "${base}:.gstack/ship-policy.json" 2>/dev/null | jq -c 'if (.train | type) == "object" then .train else null end' 2>/dev/null || true)
|
|
artifact=$(jq -r '.record_artifact // ""' <<<"${train:-null}" 2>/dev/null || true)
|
|
check=$(jq -r '.full_gate_check // ""' <<<"${train:-null}" 2>/dev/null || true)
|
|
if [ -z "$train" ] || [ "$train" = null ]; then
|
|
echo "train=false" >> "$GITHUB_OUTPUT"
|
|
echo "- Train policy: none at ${base}; this run routes as before" >> "$GITHUB_STEP_SUMMARY"
|
|
elif [[ "$artifact" =~ ^[A-Za-z0-9_.-]{1,64}$ ]] && [[ "$check" =~ ^[A-Za-z0-9_./-]{1,64}$ ]]; then
|
|
{ echo "train=true"; echo "record_artifact=${artifact}"; echo "full_gate_check=${check}"; } >> "$GITHUB_OUTPUT"
|
|
echo "- Train policy: declared at ${base} (record artifact ${artifact}, check ${check})" >> "$GITHUB_STEP_SUMMARY"
|
|
else
|
|
echo "train=false" >> "$GITHUB_OUTPUT"
|
|
echo "::warning::the train block at ${base} names no valid record_artifact and full_gate_check; this run routes as before"
|
|
fi
|
|
|
|
- name: Route this run
|
|
id: route
|
|
env:
|
|
TRAIN_POLICY: ${{ steps.policy.outputs.train }}
|
|
NONCE: ${{ inputs.nonce }}
|
|
EXPECTED_SHA: ${{ inputs.expected_sha }}
|
|
EVALS_ALL: ${{ inputs.evals_all }}
|
|
CASES: ${{ inputs.cases }}
|
|
PREFIX_SHAS: ${{ inputs.prefix_shas }}
|
|
SCREEN_PR: ${{ inputs.screen_pr }}
|
|
SCREEN_MERGE_SHA: ${{ inputs.screen_merge_sha }}
|
|
SCREEN_POLICY: ${{ inputs.screen_policy }}
|
|
run: |
|
|
if [ "$GITHUB_EVENT_NAME" = pull_request ]; then
|
|
mode='pr'
|
|
if [ "$TRAIN_POLICY" = true ]; then mode=pr-summary; fi
|
|
echo "mode=${mode}" >> "$GITHUB_OUTPUT"
|
|
echo "- Run mode: ${mode}" >> "$GITHUB_STEP_SUMMARY"
|
|
elif [ -f scripts/train-gate.ts ]; then
|
|
bun --no-env-file --config=/dev/null --no-install run scripts/train-gate.ts route --github-output "$GITHUB_OUTPUT" >> "$GITHUB_STEP_SUMMARY"
|
|
elif [ -n "${CASES}${PREFIX_SHAS}${SCREEN_PR}${SCREEN_MERGE_SHA}${SCREEN_POLICY}" ]; then
|
|
echo "::error::${GITHUB_REF_NAME} has no scripts/train-gate.ts to validate the train or screen inputs; dispatch from a ref that has it"
|
|
exit 1
|
|
else
|
|
echo "mode=dispatch" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
|
|
# The screen's merge commit must merge the PR's current head into its
|
|
# base, and Free Tests must be green on that head (base-branch code: a
|
|
# screen dispatches on the default branch). No quiet window: a screen is
|
|
# an explicit request. A moved PR head reports superseded and plans nothing.
|
|
- name: Check the screen's PR head (Free Tests)
|
|
id: screen
|
|
if: steps.route.outputs.mode == 'screen'
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
SCREEN_PR: ${{ inputs.screen_pr }}
|
|
SCREEN_MERGE_SHA: ${{ inputs.screen_merge_sha }}
|
|
run: |
|
|
bun --no-install run scripts/eval-readiness.ts wait --repo "$GITHUB_REPOSITORY" --pr "$SCREEN_PR" --merge-sha "$SCREEN_MERGE_SHA" \
|
|
--run-id "$GITHUB_RUN_ID" --quiet-minutes 0 --deadline-minutes 20 \
|
|
--github-output "$GITHUB_OUTPUT" --summary "$GITHUB_STEP_SUMMARY"
|
|
|
|
- name: Plan the bisect (one executor per prefix)
|
|
id: bisect
|
|
if: steps.route.outputs.mode == 'bisect'
|
|
env:
|
|
CASES: ${{ steps.route.outputs.cases }}
|
|
PREFIXES: ${{ steps.route.outputs.prefixes }}
|
|
NONCE: ${{ inputs.nonce }}
|
|
run: |
|
|
bun --no-env-file --config=/dev/null --no-install run scripts/train-gate.ts bisect-plan --cases "$CASES" --prefixes "$PREFIXES" --nonce "$NONCE" \
|
|
--out /tmp/train-bisect/bisect.json --github-output "$GITHUB_OUTPUT" >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
- name: Hand the bisect plan to its executors
|
|
if: steps.route.outputs.mode == 'bisect'
|
|
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
|
with:
|
|
name: train-bisect-plan
|
|
path: /tmp/train-bisect/bisect.json
|
|
retention-days: 30
|
|
|
|
# ── Readiness-gated dispatch (R1, GSTA-48) ───────────────────────────────
|
|
# Paid slices for a PR head start only after the PR has been quiet for the
|
|
# window (vars.EVALS_QUIET_MINUTES, default 10; the evals-no-debounce label
|
|
# sets it to 0) and Free Tests has passed on the same head SHA. A newer push
|
|
# still cancels this run (cancel-in-progress), so the window is a sleep; the
|
|
# same heads get the same selection and verdict rules, only the start moves.
|
|
# Free Tests red on this head fails the run without paid work (re-run this
|
|
# workflow after a green Free Tests rerun). Dispatches never wait: they are
|
|
# explicit (eval:bg, census). Version Gate is not an input, so a stamp push
|
|
# waits for its own Free Tests only and always gets its gate.
|
|
readiness:
|
|
runs-on: ubuntu-24.04
|
|
if: github.event_name == 'pull_request' && github.actor != 'dependabot[bot]' && github.event.pull_request.head.repo.full_name == github.repository
|
|
timeout-minutes: 90
|
|
permissions:
|
|
contents: read
|
|
actions: read
|
|
pull-requests: read
|
|
outputs:
|
|
superseded: ${{ steps.wait.outputs.superseded }}
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
|
with:
|
|
bun-version: 1.4.2
|
|
|
|
- name: Wait for a quiet PR and green Free Tests on this head
|
|
id: wait
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
PR_NUMBER: ${{ github.event.pull_request.number }}
|
|
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
|
QUIET_MINUTES: ${{ contains(github.event.pull_request.labels.*.name, 'evals-no-debounce') && '0' || vars.EVALS_QUIET_MINUTES || '10' }}
|
|
run: |
|
|
bun --no-install run scripts/eval-readiness.ts wait --repo "$GITHUB_REPOSITORY" --pr "$PR_NUMBER" --head-sha "$HEAD_SHA" \
|
|
--run-id "$GITHUB_RUN_ID" --quiet-minutes "$QUIET_MINUTES" --deadline-minutes 80 \
|
|
--github-output "$GITHUB_OUTPUT" --summary "$GITHUB_STEP_SUMMARY"
|
|
|
|
# GSTA-48 P3-1: a head the readiness job blocked runs no gate, so the PR's
|
|
# last "## E2E Evals" comment would still show an older head's verdict.
|
|
# Replace it (never post a new one) with what happened to this head. Runs no
|
|
# repository code.
|
|
readiness-comment:
|
|
runs-on: ubuntu-24.04
|
|
needs: [readiness]
|
|
if: ${{ !cancelled() && needs.readiness.result == 'failure' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }}
|
|
timeout-minutes: 5
|
|
permissions:
|
|
pull-requests: write
|
|
issues: write
|
|
steps:
|
|
- name: Replace a stale verdict comment
|
|
env:
|
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
PR_NUMBER: ${{ github.event.pull_request.number }}
|
|
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
|
run: |
|
|
COMMENT_ID=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate \
|
|
--jq '.[] | select(.body | startswith("## E2E Evals")) | .id' | tail -1)
|
|
if [ -z "$COMMENT_ID" ]; then exit 0; fi
|
|
BODY="## E2E Evals: ⛔ NOT RUN on \`${HEAD_SHA}\`
|
|
|
|
The readiness job blocked this head (Free Tests did not pass on it, or did not finish within the readiness deadline), so no paid gate ran and no verdict covers it. An earlier verdict in this comment belonged to an older head.
|
|
|
|
Details: [run ${GITHUB_RUN_ID}](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}). Fix and push, or re-run Free Tests and then re-run this workflow."
|
|
gh api "repos/${GITHUB_REPOSITORY}/issues/comments/${COMMENT_ID}" -X PATCH -f body="$BODY"
|
|
|
|
# ── Receipt recovery (W1c: CEO-14/34, ENG-4/5/7) ─────────────────────────
|
|
# A run cancelled by the next push never reaches slices-report, so its
|
|
# finished receipts (passes AND the negatives executors write at the
|
|
# execution boundary) never reach the saved store. This job restores the
|
|
# PR's store, merges the receipts of this PR's completed cancelled runs
|
|
# (oldest first, at most 5, provenance-checked), reconstructs completed
|
|
# panel negatives, writes the recovery.json checkpoint and hands the store
|
|
# to plan-slices as an artifact. It runs scripts from the BASE ref only and
|
|
# alone holds actions: read; plan-slices keeps contents: read. It never
|
|
# fails the run: any gh problem degrades to "no recovered receipts" and
|
|
# plan-slices turns reuse off, so cases execute unreduced.
|
|
recover-receipts:
|
|
runs-on: ubuntu-24.04
|
|
needs: [readiness, route]
|
|
# Runs after readiness settles, so a superseded run's cancellation has usually completed; a head whose Free Tests failed recovers nothing.
|
|
# A dispatch naming pr_receipts reads that PR's receipts from its runs' artifacts (cache entries are ref-scoped).
|
|
# A dispatched screen only looks for a gate verdict to reuse (R2a/R2); a PR push under a train policy (pr-summary) skips this job.
|
|
if: ${{ !cancelled() && (needs.readiness.result == 'success' || needs.readiness.result == 'skipped') && needs.readiness.outputs.superseded != 'true' && needs.route.result == 'success' && needs.route.outputs.superseded != 'true' && github.actor != 'dependabot[bot]' && ((github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && needs.route.outputs.mode == 'pr' && !contains(github.event.pull_request.labels.*.name, 'evals-fresh')) || (github.event_name == 'workflow_dispatch' && inputs.pr_receipts != '') || needs.route.outputs.mode == 'screen') }}
|
|
timeout-minutes: 5
|
|
permissions:
|
|
contents: read
|
|
actions: read
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
with:
|
|
ref: ${{ github.event.pull_request.base.sha || needs.route.outputs.screen_base || inputs.base_sha || github.event.repository.default_branch }}
|
|
persist-credentials: false
|
|
|
|
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
|
with:
|
|
bun-version: 1.4.2
|
|
|
|
- name: Restore this PR's verified judge and E2E results
|
|
id: store
|
|
if: github.event_name == 'pull_request'
|
|
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
|
|
with:
|
|
path: /tmp/gstack-eval-input-cache
|
|
key: eval-input-v1-${{ github.repository_id }}-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }}-plan
|
|
restore-keys: eval-input-v1-${{ github.repository_id }}-pr-${{ github.event.pull_request.number }}-
|
|
|
|
- name: Recover receipts of cancelled runs (base-ref code)
|
|
if: needs.route.outputs.mode != 'screen'
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
STORE_KEY: ${{ steps.store.outputs.cache-matched-key }}
|
|
PR_NUMBER: ${{ github.event.pull_request.number || inputs.pr_receipts }}
|
|
HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
|
|
HEAD_BRANCH: ${{ github.event.pull_request.head.ref || github.ref_name }}
|
|
RECOVERY_MODE: ${{ github.event_name == 'workflow_dispatch' && 'dispatch' || 'pr' }}
|
|
run: |
|
|
mkdir -p /tmp/gstack-eval-input-cache
|
|
if [ ! -f scripts/recover-receipts.ts ]; then
|
|
echo "- Receipt recovery unavailable: the base ref has no scripts/recover-receipts.ts; reuse stays off this run" >> "$GITHUB_STEP_SUMMARY"
|
|
exit 0
|
|
fi
|
|
bun --no-install run scripts/recover-receipts.ts collect --repo "$GITHUB_REPOSITORY" --pr "$PR_NUMBER" \
|
|
--head-repo "$HEAD_REPO" --branch "$HEAD_BRANCH" --run-id "$GITHUB_RUN_ID" \
|
|
--store /tmp/gstack-eval-input-cache --store-key "$STORE_KEY" --mode "$RECOVERY_MODE" --budget-seconds 60 >> "$GITHUB_STEP_SUMMARY" \
|
|
|| echo "- Receipt recovery crashed; reuse stays off this run" >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
# Verdict reuse (GSTA-48), a gate edit: base-ref code proposes reusing
|
|
# the newest executed GREEN gate of this PR when this head differs from
|
|
# it only by stamp files on the same base (R2a: TREE same-modulo-stamps
|
|
# from the tree receipt under the base's ship policy), or when main
|
|
# moved and the PR's own diff from its merge-base, stamp files excluded,
|
|
# is byte-identical (R2). plan-slices confirms the selection is
|
|
# identical. Anything else (a red, partial or cancelled gate, a changed
|
|
# own diff, a non-stamp byte, no binding or receipt, a gh or git error,
|
|
# evals-fresh) leaves no candidate: the full gate runs.
|
|
# A dispatched screen (GSTA-49 T1b) finds the same way, authenticated by
|
|
# PR number, head, repository and tested base (its merge commit's first
|
|
# parent), listing the PR's pull_request runs and earlier screens; a
|
|
# broad screen never adopts a narrow run.
|
|
- name: Find a gate verdict to reuse (R2a/R2, base-ref code)
|
|
if: github.event_name == 'pull_request' || needs.route.outputs.mode == 'screen'
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
PR_NUMBER: ${{ github.event.pull_request.number || inputs.screen_pr }}
|
|
HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
|
|
HEAD_BRANCH: ${{ github.event.pull_request.head.ref || needs.route.outputs.screen_branch }}
|
|
HEAD_SHA: ${{ github.event.pull_request.head.sha || needs.route.outputs.screen_head }}
|
|
TESTED_SHA: ${{ inputs.screen_merge_sha || github.sha }}
|
|
SCREEN_POLICY: ${{ inputs.screen_policy }}
|
|
run: |
|
|
if [ ! -f scripts/gate-reuse.ts ]; then
|
|
echo "- Gate verdict reuse unavailable: the base ref has no scripts/gate-reuse.ts; the full gate runs" >> "$GITHUB_STEP_SUMMARY"
|
|
exit 0
|
|
fi
|
|
bun --no-install run scripts/gate-reuse.ts find --repo "$GITHUB_REPOSITORY" --pr "$PR_NUMBER" --head-repo "$HEAD_REPO" \
|
|
--branch "$HEAD_BRANCH" --run-id "$GITHUB_RUN_ID" --head-sha "$HEAD_SHA" --tested-sha "$TESTED_SHA" \
|
|
${SCREEN_POLICY:+--screen-policy "$SCREEN_POLICY"} --out /tmp/gate-reuse --budget-seconds 90 >> "$GITHUB_STEP_SUMMARY" \
|
|
|| echo "- Gate verdict reuse crashed; the full gate runs" >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
- name: Hand a gate reuse candidate to plan-slices
|
|
if: github.event_name == 'pull_request' || needs.route.outputs.mode == 'screen'
|
|
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
|
with:
|
|
name: gate-reuse-candidate
|
|
path: /tmp/gate-reuse
|
|
if-no-files-found: ignore
|
|
retention-days: 7
|
|
|
|
- name: Hand the receipt store to plan-slices
|
|
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
|
with:
|
|
name: receipt-store
|
|
path: /tmp/gstack-eval-input-cache
|
|
if-no-files-found: ignore
|
|
retention-days: 7
|
|
|
|
# ── PR push under a train policy (GSTA-49 T1b, a gate edit) ─────────────
|
|
# Only when the base revision declares a `train` block: the push starts no
|
|
# paid slice. This job prints how to request the paid screen, the
|
|
# screening selection and its estimated cost, for free; the train's full
|
|
# gate covers every merge. Without a train block it never runs.
|
|
screen-summary:
|
|
runs-on: ubuntu-24.04
|
|
needs: [route]
|
|
if: needs.route.outputs.mode == 'pr-summary' && github.event.pull_request.head.repo.full_name == github.repository
|
|
timeout-minutes: 10
|
|
permissions:
|
|
contents: read
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
with:
|
|
fetch-depth: 0
|
|
persist-credentials: false
|
|
|
|
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
|
with:
|
|
bun-version: 1.4.2
|
|
|
|
- name: Print the screening selection and its estimated cost (no paid case runs)
|
|
env:
|
|
PR_NUMBER: ${{ github.event.pull_request.number }}
|
|
run: |
|
|
EVALS_TIER=gate bun --no-install run scripts/test-paid-shards.ts --tier gate --emit-plan /tmp/paid-plan/manifest.json --slices 1
|
|
bun --no-env-file --config=/dev/null --no-install run scripts/train-gate.ts screen-summary /tmp/paid-plan/manifest.json --pr "$PR_NUMBER" >> "$GITHUB_STEP_SUMMARY"
|
|
bun --no-install run scripts/test-pr-profile.ts summary /tmp/paid-plan/manifest.json >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
# ── Sliced lane (the ONLY paid lane; legacy 17-row matrix deleted) ──────────
|
|
# One PLANNER computes diff selection + the slice plan ONCE (killing
|
|
# per-slice selector divergence); K executors consume the manifest; the
|
|
# report reconciles results against it FAIL-CLOSED (a slice whose artifact
|
|
# never landed is a failure, a planned shard nobody reported is a failure —
|
|
# hollow lanes cannot aggregate green). Engine: scripts/test-paid-shards.ts —
|
|
# the same runner local eval:bg:gate uses, so CI and local share one
|
|
# selection engine, and every gate-tier file is in the census by
|
|
# construction (no hand-enumerated rows to drift). The legacy matrix ran
|
|
# 18 enumerated files for 22.6 min/$21 per PR serialized AHEAD of this
|
|
# lane's 49-file diff-selected census; parity was demonstrated (sliced
|
|
# census ⊇ matrix files) and the matrix deleted — one revert restores it.
|
|
#
|
|
# Fork PRs never receive repository secrets (ANTHROPIC_API_KEY et al), so
|
|
# every API-calling eval fails at SDK auth before a model runs. Skip
|
|
# deterministically; fork work gets real coverage via a trusted base-repo
|
|
# branch (see CLAUDE.md's garrytan-agents workflow).
|
|
plan-slices:
|
|
runs-on: ubicloud-standard-8
|
|
needs: [readiness, recover-receipts, route]
|
|
# !cancelled(): plans even when readiness or recovery was skipped (dispatch, evals-fresh) or recovery failed (reuse is then off); a superseded run, or a head whose readiness failed (Free Tests red), plans nothing.
|
|
# The route job must have named a planned mode: pr-summary (a PR push under a train policy) and bisect plan nothing here, and neither does a refused dispatch.
|
|
if: ${{ !cancelled() && (needs.readiness.result == 'success' || needs.readiness.result == 'skipped') && needs.readiness.outputs.superseded != 'true' && needs.route.result == 'success' && needs.route.outputs.superseded != 'true' && contains(fromJSON('["pr","screen","train","dispatch"]'), needs.route.outputs.mode) && github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }}
|
|
timeout-minutes: 10
|
|
permissions:
|
|
contents: read
|
|
outputs:
|
|
slices: ${{ steps.matrix.outputs.slices }}
|
|
slice_timeouts: ${{ steps.matrix.outputs.slice_timeouts }}
|
|
gate_reuse: ${{ steps.gate_reuse.outputs.gate_reuse }}
|
|
# The route, forwarded: the executors and the report read it from here.
|
|
mode: ${{ needs.route.outputs.mode }}
|
|
record_artifact: ${{ needs.route.outputs.record_artifact }}
|
|
full_gate_check: ${{ needs.route.outputs.full_gate_check }}
|
|
screen_head: ${{ needs.route.outputs.screen_head }}
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
with:
|
|
# A dispatched screen tests the PR's pinned refs/pull/<n>/merge commit; empty for every other run (its own ref).
|
|
ref: ${{ inputs.screen_merge_sha }}
|
|
# Preserve full history for merge-base diff selection. Moving the
|
|
# planner off the eval image must not change its selection inputs.
|
|
fetch-depth: 0
|
|
persist-credentials: false
|
|
|
|
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
|
with:
|
|
bun-version: 1.4.2
|
|
|
|
# DX-1: a dispatch from scripts/eval-bg.ts names the revision it means
|
|
# to test; a different checkout fails here before any paid work.
|
|
- name: Verify the dispatched revision
|
|
if: github.event_name == 'workflow_dispatch'
|
|
env:
|
|
EXPECTED_SHA: ${{ inputs.expected_sha }}
|
|
NONCE: ${{ inputs.nonce }}
|
|
run: |
|
|
actual=$(git rev-parse HEAD)
|
|
echo "- Tested revision: ${actual} (ref ${GITHUB_REF_NAME}${NONCE:+, nonce ${NONCE}})" >> "$GITHUB_STEP_SUMMARY"
|
|
if [ -n "$EXPECTED_SHA" ] && [ "$actual" != "$EXPECTED_SHA" ]; then
|
|
echo "::error::evals.yml checked out ${actual} but the dispatch expected ${EXPECTED_SHA}. Push the branch so its head is ${EXPECTED_SHA} and dispatch again (bun run scripts/eval-bg.ts pr), or run locally with --local."
|
|
exit 1
|
|
fi
|
|
|
|
# Planner-side reuse: take the PR's receipt store from recover-receipts
|
|
# (the restored store plus cancelled runs' receipts) and ship ONE
|
|
# filtered set with the plan, so every trial of a panel sees the same
|
|
# receipts and a newer FAIL blocks any older PASS for the same inputs.
|
|
- name: Download this PR's receipt store
|
|
if: github.event_name == 'pull_request' || inputs.pr_receipts != ''
|
|
continue-on-error: true
|
|
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
|
with:
|
|
name: receipt-store
|
|
path: /tmp/gstack-eval-input-cache
|
|
|
|
- name: Decide receipt reuse
|
|
id: reuse
|
|
if: github.event_name == 'pull_request' || inputs.pr_receipts != ''
|
|
run: |
|
|
mkdir -p /tmp/gstack-eval-input-cache
|
|
bun --no-install run scripts/recover-receipts.ts decide /tmp/gstack-eval-input-cache --github-output "$GITHUB_OUTPUT" >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
- name: Emit run manifest
|
|
env:
|
|
EVALS_ALL: ${{ (github.event_name == 'workflow_dispatch' && inputs.evals_all) && '1' || '' }}
|
|
EVALS_CACHE_DIR: ${{ steps.reuse.outputs.reuse == 'on' && '/tmp/gstack-eval-input-cache' || '' }}
|
|
# Dispatch diff base (CEO-17): a screen's merge-commit first parent, else the exact merge-base, else the named base branch, else detection.
|
|
EVALS_BASE: ${{ needs.route.outputs.screen_base || inputs.base_sha || (inputs.base_ref != '' && format('origin/{0}', inputs.base_ref)) || '' }}
|
|
run: EVALS_TIER=gate bun --no-install run scripts/test-paid-shards.ts --tier gate --emit-plan /tmp/paid-plan/manifest.json --slice-budget 420 --jobs 2 --max-parallel 20
|
|
|
|
# R2a/R2: reuse holds only when this head's own plan selects exactly the
|
|
# cases and trial plans the reused gate ran (selection fingerprint).
|
|
- name: Download a gate reuse candidate
|
|
if: github.event_name == 'pull_request' || needs.route.outputs.mode == 'screen'
|
|
continue-on-error: true
|
|
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
|
with:
|
|
name: gate-reuse-candidate
|
|
path: /tmp/gate-reuse
|
|
|
|
- name: Confirm gate verdict reuse (R2a/R2)
|
|
id: gate_reuse
|
|
if: github.event_name == 'pull_request' || needs.route.outputs.mode == 'screen'
|
|
run: |
|
|
mkdir -p /tmp/gate-reuse
|
|
bun --no-install run scripts/gate-reuse.ts confirm /tmp/gate-reuse /tmp/paid-plan/manifest.json --out /tmp/paid-plan \
|
|
--github-output "$GITHUB_OUTPUT" >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
# CEO-13: the mode, case counts and, on fallback, every file that
|
|
# restored the full gate with the edit that narrows it next time.
|
|
- name: Summarize PR coverage
|
|
run: bun --no-install run scripts/test-pr-profile.ts summary /tmp/paid-plan/manifest.json >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
- name: Derive the executor matrix from the plan
|
|
id: matrix
|
|
run: |
|
|
echo "slices=$(jq -c '[range(1; .sliceCount + 1)]' /tmp/paid-plan/manifest.json)" >> "$GITHUB_OUTPUT"
|
|
echo "slice_timeouts=$(jq -ec '[0] + .plan.sliceCiTimeoutMinutes' /tmp/paid-plan/manifest.json)" >> "$GITHUB_OUTPUT"
|
|
|
|
# The whole store (recovered receipts and the checkpoint included)
|
|
# travels to slices-report, which saves it with this run's receipts, so
|
|
# a receipt reuse turned off for this run is not lost for the next.
|
|
- name: Carry the receipt store to the report
|
|
if: github.event_name == 'pull_request'
|
|
run: mkdir -p /tmp/paid-plan/store && cp -a /tmp/gstack-eval-input-cache/. /tmp/paid-plan/store/
|
|
|
|
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
|
with:
|
|
name: paid-plan
|
|
path: |
|
|
/tmp/paid-plan/manifest.json
|
|
/tmp/paid-plan/receipts
|
|
/tmp/paid-plan/store
|
|
/tmp/paid-plan/gate-reuse.json
|
|
retention-days: 30
|
|
|
|
eval-slices:
|
|
runs-on: ubicloud-standard-8
|
|
needs: [build-image, plan-slices]
|
|
env:
|
|
EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-eval-slices-${{ matrix.slice }}
|
|
# !cancelled(), not always(): still runs when build-image was skipped
|
|
# (image already published), but a newer push's cancel-in-progress stops
|
|
# it instead of letting a superseded run finish its paid slices first.
|
|
# A confirmed verdict reuse (R2a/R2) runs no paid slice.
|
|
if: ${{ !cancelled() && needs.build-image.result == 'success' && needs.plan-slices.result == 'success' && needs.plan-slices.outputs.gate_reuse != 'true' }}
|
|
# The planner packs ~7 minutes of recorded work per slice (EVALS_JOBS=2 x
|
|
# EVALS_CONCURRENCY=2 per runner, never the old 40-way per-row fan-out
|
|
# that queued claude session STARTUP behind 39 siblings). Each slice's job
|
|
# timeout is its own ceiling (W2c): max(2x budget, its longest shard wall,
|
|
# the overlay envelope) + 20 minutes; the executor stops at the slice
|
|
# deadline 5 minutes earlier and reports not_run/hung shards.
|
|
timeout-minutes: ${{ fromJSON(needs.plan-slices.outputs.slice_timeouts)[matrix.slice] }}
|
|
permissions:
|
|
contents: read
|
|
packages: read
|
|
container:
|
|
image: ${{ needs.build-image.outputs.image-tag }}
|
|
credentials:
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
options: --user runner
|
|
strategy:
|
|
fail-fast: false
|
|
max-parallel: 20
|
|
matrix:
|
|
slice: ${{ fromJSON(needs.plan-slices.outputs.slices) }}
|
|
steps:
|
|
- name: Record the job start for the slice deadline
|
|
run: echo "GSTACK_SLICE_JOB_STARTED_AT=$(date +%s)" >> "$GITHUB_ENV"
|
|
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
with:
|
|
# A dispatched screen's pinned merge commit (the planner verified it); empty otherwise.
|
|
ref: ${{ inputs.screen_merge_sha }}
|
|
# Full history: files with SELF-derived selection (the LLM-judge
|
|
# map, routing) walk git at module load, and selection is
|
|
# fail-closed on git errors — a shallow checkout crashed those
|
|
# shards on the lane's first live run ("ambiguous argument
|
|
# 'main...HEAD'"). The manifest still governs WHICH shards run.
|
|
fetch-depth: 0
|
|
persist-credentials: false
|
|
|
|
- name: Fix bun temp
|
|
uses: ./.github/actions/fix-bun-temp
|
|
|
|
- name: Restore deps
|
|
uses: ./.github/actions/restore-deps
|
|
|
|
- run: bun run build
|
|
|
|
# Any slice can host a PTY smoke, so the seed/registration steps run
|
|
# UNCONDITIONALLY (both are idempotent) — the old matrix keyed them on
|
|
# matrix.suite.name, which a sliced lane cannot do. The register
|
|
# composite carries the fail-fast dangling-symlink/frontmatter
|
|
# verification loop, so a moved skill target fails HERE in seconds,
|
|
# not as a wedged PTY session at the shard wall.
|
|
- name: Seed claude interactive config
|
|
uses: ./.github/actions/seed-claude-config
|
|
with:
|
|
anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
|
|
- name: Register gstack skills for PTY smokes
|
|
uses: ./.github/actions/register-gstack-skills
|
|
|
|
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
|
with:
|
|
name: paid-plan
|
|
path: /tmp/paid-plan
|
|
|
|
# Receipts come only from the plan (this PR's store, filtered once by the
|
|
# planner); new receipts land beside the slice results and the report
|
|
# merges them into the next store.
|
|
- name: Seed this slice's receipts from the plan
|
|
run: |
|
|
mkdir -p /tmp/paid-slice-results/receipts
|
|
if [ -d /tmp/paid-plan/receipts ]; then cp -a /tmp/paid-plan/receipts/. /tmp/paid-slice-results/receipts/; fi
|
|
|
|
# The image ships the pinned Codex CLI off PATH and without a
|
|
# credential; codex does not read OPENAI_API_KEY from the environment.
|
|
# Log it in from the secret (same-repo runs only) under a non-default
|
|
# home. The shard runner hands that home and the CLI to the Codex files
|
|
# only (scripts/lib/paid-cases.ts scopeCodexAccess); every other paid case sees no
|
|
# Codex, as before. Without a key the Codex cases fail; they never pass.
|
|
- name: Authenticate the Codex CLI
|
|
shell: bash
|
|
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
|
|
env:
|
|
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
run: |
|
|
set -euo pipefail
|
|
codex="${GSTACK_CI_CODEX_BIN_DIR:?}/codex"
|
|
export CODEX_HOME="$HOME/.gstack-ci-codex"
|
|
mkdir -p "$CODEX_HOME"
|
|
echo "GSTACK_CI_CODEX_HOME=$CODEX_HOME" >> "$GITHUB_ENV"
|
|
if [ -z "${OPENAI_API_KEY:-}" ]; then
|
|
echo "::warning::OPENAI_API_KEY is unavailable; Codex eval cases cannot authenticate and will fail."
|
|
exit 0
|
|
fi
|
|
printenv OPENAI_API_KEY | "$codex" login --with-api-key
|
|
"$codex" login status >/dev/null 2>&1
|
|
|
|
- name: Run slice ${{ matrix.slice }}
|
|
env:
|
|
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
|
|
PLAYWRIGHT_BROWSERS_PATH: /opt/playwright-browsers
|
|
EVALS_JOBS: "2"
|
|
EVALS_CONCURRENCY: "2"
|
|
GSTACK_EVAL_DIR: /tmp/paid-slice-results
|
|
EVALS_CACHE_DIR: /tmp/paid-slice-results/receipts
|
|
EVALS_CACHE_REPOSITORY: ${{ github.repository }}
|
|
EVALS_CACHE_PR: ${{ github.event.pull_request.number || inputs.pr_receipts }}
|
|
EVALS_CACHE_RUNTIME_ID: ${{ needs.build-image.outputs.runtime-id }}
|
|
run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --plan /tmp/paid-plan/manifest.json --slice ${{ matrix.slice }}
|
|
|
|
# Attempt-scoped: a re-run attempt's trials are reported under that
|
|
# attempt and never collide with the first attempt's; the report reads
|
|
# them through the rule-16 case history (scripts/lib/paid-rerun.ts).
|
|
- name: Upload slice results
|
|
if: always()
|
|
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
|
with:
|
|
name: paid-slice-${{ matrix.slice }}-a${{ github.run_attempt }}
|
|
path: /tmp/paid-slice-results
|
|
retention-days: 90
|
|
|
|
- name: Upload native capture evidence
|
|
if: always()
|
|
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
|
with:
|
|
name: native-captures-${{ env.EVALS_RUN_ID }}
|
|
include-hidden-files: true
|
|
path: |
|
|
~/.gstack/projects/*/e2e-runs
|
|
~/.gstack/projects/*/evals/qa-callers
|
|
~/.gstack-dev/e2e-runs
|
|
~/.gstack-dev/evals/qa-callers
|
|
if-no-files-found: ignore
|
|
retention-days: 90
|
|
|
|
# The spooled per-shard full logs — a red weekly/PR lane three weeks
|
|
# later needs more than a summary line.
|
|
# always(), not failure(): a failed behavior trial is a verdict and no
|
|
# longer reds its runner, but its full log is the diagnosis evidence.
|
|
- name: Upload shard logs
|
|
if: always()
|
|
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
|
with:
|
|
name: paid-logs-slice-${{ matrix.slice }}-a${{ github.run_attempt }}
|
|
include-hidden-files: true
|
|
# The Fix-bun-temp step points TMPDIR at /home/runner/.cache, so the
|
|
# runner's spool lands THERE, not /tmp — the original /tmp glob
|
|
# uploaded nothing and a red slice's diagnostics were unreachable.
|
|
path: |
|
|
/home/runner/.cache/gstack-paid-shard-*.log
|
|
/tmp/gstack-paid-shard-*.log
|
|
if-no-files-found: ignore
|
|
retention-days: 30
|
|
|
|
# ── Rule-16 rerun (CREW rule 16, Garry 2026-10-19; a gate edit) ────────
|
|
# "You can just run the failed test again and note that it is flaky... You
|
|
# don't have to do the whole test pass again for one failed test." After the
|
|
# slices, the planner reruns ONCE, at the same head and in this run attempt,
|
|
# only the cases whose verdict failed once with no rerun yet: a red rule or
|
|
# judge case on its own shard (narrowed to it in PR mode; other cases a whole
|
|
# file reruns never count), a behavior FAIL as its whole planned panel.
|
|
# Harness, INFRA, INCOMPLETE and unattributed reds, a case already red twice,
|
|
# or any other problem plan nothing (scripts/lib/paid-rerun.ts planRerun).
|
|
# The rerun runs fresh (no receipts, no reuse) and never changes a trial
|
|
# count, kind or threshold; slices-report decides FLAKY (green, annotated,
|
|
# flake record) or red. Train runs use the same path; bisects rerun inside
|
|
# scripts/train-gate.ts.
|
|
rerun-plan:
|
|
runs-on: ubuntu-24.04
|
|
needs: [plan-slices, eval-slices]
|
|
if: ${{ !cancelled() && needs.plan-slices.result == 'success' && needs.plan-slices.outputs.gate_reuse != 'true' }}
|
|
timeout-minutes: 10
|
|
permissions:
|
|
contents: read
|
|
outputs:
|
|
planned: ${{ steps.plan.outputs.planned }}
|
|
slices: ${{ steps.plan.outputs.slices }}
|
|
slice_timeouts: ${{ steps.plan.outputs.slice_timeouts }}
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
with:
|
|
ref: ${{ inputs.screen_merge_sha }}
|
|
persist-credentials: false
|
|
|
|
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
|
with:
|
|
bun-version: 1.4.2
|
|
|
|
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
|
with:
|
|
name: paid-plan
|
|
path: /tmp/paid-report
|
|
|
|
# Every attempt's slices and earlier attempts' reruns: one history per case.
|
|
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
|
with:
|
|
pattern: paid-slice-*
|
|
path: /tmp/paid-report
|
|
|
|
# A missing rerun artifact leaves its case red (fail-closed), never green.
|
|
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
|
continue-on-error: true
|
|
with:
|
|
pattern: paid-rerun-*
|
|
# Its own directory: download-artifact unpacks a lone match into its path itself, so the
|
|
# report root would otherwise take a rerun slice's files (run 39363705692).
|
|
path: /tmp/paid-report/reruns
|
|
|
|
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
|
continue-on-error: true
|
|
with:
|
|
pattern: rerun-plan-a*
|
|
# Each artifact holds rerun-plan-a<attempt>/manifest.json, so even a lone match never
|
|
# lands on the plan's own manifest.json (run 39363705692).
|
|
path: /tmp/paid-report
|
|
|
|
- name: Plan the rule-16 rerun of this attempt's failed cases
|
|
id: plan
|
|
run: |
|
|
EVALS_TIER=gate bun --no-env-file --config=/dev/null --no-install run scripts/test-paid-shards.ts --tier gate --report /tmp/paid-report \
|
|
--rerun-plan "/tmp/paid-rerun-plan/rerun-plan-a$GITHUB_RUN_ATTEMPT" >> "$GITHUB_STEP_SUMMARY"
|
|
PLAN="/tmp/paid-rerun-plan/rerun-plan-a$GITHUB_RUN_ATTEMPT/manifest.json"
|
|
if [ -f "$PLAN" ]; then
|
|
{ echo "planned=true"
|
|
echo "slices=$(jq -c '[range(1; .sliceCount + 1)]' "$PLAN")"
|
|
echo "slice_timeouts=$(jq -ec '[0] + .plan.sliceCiTimeoutMinutes' "$PLAN")"; } >> "$GITHUB_OUTPUT"
|
|
else
|
|
echo "planned=false" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
|
|
- name: Hand the rerun manifest to its executors and the report
|
|
if: steps.plan.outputs.planned == 'true'
|
|
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
|
with:
|
|
name: rerun-plan-a${{ github.run_attempt }}
|
|
path: /tmp/paid-rerun-plan
|
|
retention-days: 90
|
|
|
|
eval-rerun:
|
|
runs-on: ubicloud-standard-8
|
|
needs: [build-image, rerun-plan]
|
|
if: ${{ !cancelled() && needs.build-image.result == 'success' && needs.rerun-plan.outputs.planned == 'true' }}
|
|
env:
|
|
EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-eval-rerun-${{ matrix.slice }}
|
|
timeout-minutes: ${{ fromJSON(needs.rerun-plan.outputs.slice_timeouts)[matrix.slice] }}
|
|
permissions:
|
|
contents: read
|
|
packages: read
|
|
container:
|
|
image: ${{ needs.build-image.outputs.image-tag }}
|
|
credentials:
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
options: --user runner
|
|
strategy:
|
|
fail-fast: false
|
|
max-parallel: 20
|
|
matrix:
|
|
slice: ${{ fromJSON(needs.rerun-plan.outputs.slices) }}
|
|
steps:
|
|
- name: Record the job start for the slice deadline
|
|
run: echo "GSTACK_SLICE_JOB_STARTED_AT=$(date +%s)" >> "$GITHUB_ENV"
|
|
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
with:
|
|
ref: ${{ inputs.screen_merge_sha }}
|
|
fetch-depth: 0
|
|
persist-credentials: false
|
|
|
|
- name: Fix bun temp
|
|
uses: ./.github/actions/fix-bun-temp
|
|
|
|
- name: Restore deps
|
|
uses: ./.github/actions/restore-deps
|
|
|
|
- run: bun run build
|
|
|
|
- name: Seed claude interactive config
|
|
uses: ./.github/actions/seed-claude-config
|
|
with:
|
|
anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
|
|
- name: Register gstack skills for PTY smokes
|
|
uses: ./.github/actions/register-gstack-skills
|
|
|
|
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
|
with:
|
|
name: rerun-plan-a${{ github.run_attempt }}
|
|
path: /tmp/paid-rerun-plan
|
|
|
|
- name: Authenticate the Codex CLI
|
|
shell: bash
|
|
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
|
|
env:
|
|
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
run: |
|
|
set -euo pipefail
|
|
codex="${GSTACK_CI_CODEX_BIN_DIR:?}/codex"
|
|
export CODEX_HOME="$HOME/.gstack-ci-codex"
|
|
mkdir -p "$CODEX_HOME"
|
|
echo "GSTACK_CI_CODEX_HOME=$CODEX_HOME" >> "$GITHUB_ENV"
|
|
if [ -z "${OPENAI_API_KEY:-}" ]; then
|
|
echo "::warning::OPENAI_API_KEY is unavailable; Codex eval cases cannot authenticate and will fail."
|
|
exit 0
|
|
fi
|
|
printenv OPENAI_API_KEY | "$codex" login --with-api-key
|
|
"$codex" login status >/dev/null 2>&1
|
|
|
|
# No EVALS_CACHE_*: a rerun never reuses a receipt and publishes none.
|
|
- name: Rerun slice ${{ matrix.slice }} (rule 16, once)
|
|
env:
|
|
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
|
|
PLAYWRIGHT_BROWSERS_PATH: /opt/playwright-browsers
|
|
EVALS_JOBS: "2"
|
|
EVALS_CONCURRENCY: "2"
|
|
GSTACK_EVAL_DIR: /tmp/paid-rerun-results
|
|
run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --plan /tmp/paid-rerun-plan/rerun-plan-a${{ github.run_attempt }}/manifest.json --slice ${{ matrix.slice }}
|
|
|
|
- name: Upload rerun results
|
|
if: always()
|
|
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
|
with:
|
|
name: paid-rerun-${{ matrix.slice }}-a${{ github.run_attempt }}
|
|
path: /tmp/paid-rerun-results
|
|
retention-days: 90
|
|
|
|
- name: Upload native capture evidence
|
|
if: always()
|
|
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
|
with:
|
|
name: native-captures-${{ env.EVALS_RUN_ID }}
|
|
include-hidden-files: true
|
|
path: |
|
|
~/.gstack/projects/*/e2e-runs
|
|
~/.gstack/projects/*/evals/qa-callers
|
|
~/.gstack-dev/e2e-runs
|
|
~/.gstack-dev/evals/qa-callers
|
|
if-no-files-found: ignore
|
|
retention-days: 90
|
|
|
|
- name: Upload shard logs
|
|
if: always()
|
|
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
|
with:
|
|
name: rerun-logs-${{ matrix.slice }}-a${{ github.run_attempt }}
|
|
include-hidden-files: true
|
|
path: |
|
|
/home/runner/.cache/gstack-paid-shard-*.log
|
|
/tmp/gstack-paid-shard-*.log
|
|
if-no-files-found: ignore
|
|
retention-days: 30
|
|
|
|
# ── Train bisect (GSTA-49 T1b) ─────────────────────────────────────────
|
|
# One executor per prefix of the train (route's validated prefix_shas, each
|
|
# an ancestor of the train head): every red case once, one trial, on that
|
|
# prefix's own checkout. Each writes the per-prefix record that
|
|
# `gstack-ship-queue train bisect` reads, as <record_artifact>-bisect-<sha12>.
|
|
# A bisect never reuses a result and never binds a verdict.
|
|
train-bisect:
|
|
runs-on: ubicloud-standard-8
|
|
needs: [build-image, route]
|
|
if: ${{ !cancelled() && needs.build-image.result == 'success' && needs.route.result == 'success' && needs.route.outputs.mode == 'bisect' }}
|
|
env:
|
|
EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-train-bisect-${{ matrix.slice }}
|
|
timeout-minutes: ${{ fromJSON(needs.route.outputs.bisect_timeouts)[matrix.slice] }}
|
|
permissions:
|
|
contents: read
|
|
packages: read
|
|
container:
|
|
image: ${{ needs.build-image.outputs.image-tag }}
|
|
credentials:
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
options: --user runner
|
|
strategy:
|
|
fail-fast: false
|
|
max-parallel: 7
|
|
matrix:
|
|
slice: ${{ fromJSON(needs.route.outputs.bisect_slices) }}
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
with:
|
|
ref: ${{ fromJSON(needs.route.outputs.bisect_refs)[matrix.slice] }}
|
|
fetch-depth: 0
|
|
persist-credentials: false
|
|
|
|
- name: Verify the prefix checkout
|
|
id: prefix
|
|
env:
|
|
PREFIX_SHA: ${{ fromJSON(needs.route.outputs.bisect_refs)[matrix.slice] }}
|
|
run: |
|
|
actual=$(git rev-parse HEAD)
|
|
if [ "$actual" != "$PREFIX_SHA" ]; then
|
|
echo "::error::train-bisect checked out ${actual}, not its prefix ${PREFIX_SHA}"
|
|
exit 1
|
|
fi
|
|
echo "short=${PREFIX_SHA:0:12}" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Fix bun temp
|
|
uses: ./.github/actions/fix-bun-temp
|
|
|
|
- name: Restore deps
|
|
uses: ./.github/actions/restore-deps
|
|
|
|
- run: bun run build
|
|
|
|
# Any slice can host a PTY smoke, so the seed/registration steps run
|
|
# UNCONDITIONALLY (both are idempotent) — the old matrix keyed them on
|
|
# matrix.suite.name, which a sliced lane cannot do. The register
|
|
# composite carries the fail-fast dangling-symlink/frontmatter
|
|
# verification loop, so a moved skill target fails HERE in seconds,
|
|
# not as a wedged PTY session at the shard wall.
|
|
- name: Seed claude interactive config
|
|
uses: ./.github/actions/seed-claude-config
|
|
with:
|
|
anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
|
|
- name: Register gstack skills for PTY smokes
|
|
uses: ./.github/actions/register-gstack-skills
|
|
|
|
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
|
with:
|
|
name: train-bisect-plan
|
|
path: /tmp/train-bisect
|
|
|
|
- name: Authenticate the Codex CLI
|
|
shell: bash
|
|
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
|
|
env:
|
|
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
run: |
|
|
set -euo pipefail
|
|
codex="${GSTACK_CI_CODEX_BIN_DIR:?}/codex"
|
|
export CODEX_HOME="$HOME/.gstack-ci-codex"
|
|
mkdir -p "$CODEX_HOME"
|
|
echo "GSTACK_CI_CODEX_HOME=$CODEX_HOME" >> "$GITHUB_ENV"
|
|
if [ -z "${OPENAI_API_KEY:-}" ]; then
|
|
echo "::warning::OPENAI_API_KEY is unavailable; Codex eval cases cannot authenticate and will fail."
|
|
exit 0
|
|
fi
|
|
printenv OPENAI_API_KEY | "$codex" login --with-api-key
|
|
"$codex" login status >/dev/null 2>&1
|
|
|
|
- name: Run the red cases on prefix ${{ matrix.slice }}
|
|
env:
|
|
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
|
|
PLAYWRIGHT_BROWSERS_PATH: /opt/playwright-browsers
|
|
EVALS_JOBS: "1"
|
|
EVALS_CONCURRENCY: "1"
|
|
SLICE: ${{ matrix.slice }}
|
|
run: EVALS_TIER=gate bun --no-env-file --config=/dev/null run scripts/train-gate.ts bisect-prefix --plan /tmp/train-bisect/bisect.json --index "$SLICE" --out /tmp/train-bisect/out
|
|
|
|
- name: Upload the prefix record
|
|
if: always() && steps.prefix.outputs.short != ''
|
|
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
|
with:
|
|
name: ${{ needs.route.outputs.record_artifact }}-bisect-${{ steps.prefix.outputs.short }}
|
|
path: /tmp/train-bisect/out/bisect-prefix.json
|
|
if-no-files-found: ignore
|
|
overwrite: true
|
|
retention-days: 90
|
|
|
|
- name: Upload native capture evidence
|
|
if: always()
|
|
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
|
with:
|
|
name: native-captures-${{ env.EVALS_RUN_ID }}
|
|
include-hidden-files: true
|
|
path: |
|
|
~/.gstack/projects/*/e2e-runs
|
|
~/.gstack/projects/*/evals/qa-callers
|
|
~/.gstack-dev/e2e-runs
|
|
~/.gstack-dev/evals/qa-callers
|
|
if-no-files-found: ignore
|
|
retention-days: 90
|
|
|
|
# The spooled per-shard full logs — a red weekly/PR lane three weeks
|
|
# later needs more than a summary line.
|
|
# always(), not failure(): a failed behavior trial is a verdict and no
|
|
# longer reds its runner, but its full log is the diagnosis evidence.
|
|
- name: Upload shard logs
|
|
if: always()
|
|
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
|
with:
|
|
name: train-bisect-logs-${{ matrix.slice }}-a${{ github.run_attempt }}
|
|
include-hidden-files: true
|
|
# The Fix-bun-temp step points TMPDIR at /home/runner/.cache, so the
|
|
# runner's spool lands THERE, not /tmp — the original /tmp glob
|
|
# uploaded nothing and a red slice's diagnostics were unreachable.
|
|
path: |
|
|
/home/runner/.cache/gstack-paid-shard-*.log
|
|
/tmp/gstack-paid-shard-*.log
|
|
if-no-files-found: ignore
|
|
retention-days: 30
|
|
|
|
slices-report:
|
|
# GitHub-hosted, like recover-receipts: Ubicloud runners send actions/cache
|
|
# to Ubicloud's own transparent cache, which a GitHub-hosted restore never
|
|
# sees, so a store saved on Ubicloud was never restored (GSTA-48: every
|
|
# restore missed, e.g. #3253's restamp gate 3 minutes after a saved store).
|
|
runs-on: ubuntu-24.04
|
|
needs: [plan-slices, eval-slices, rerun-plan, eval-rerun]
|
|
# !cancelled(): the report must run (and FAIL) when an executor died — a
|
|
# missing slice artifact reading as green is the class this lane kills —
|
|
# but a run superseded by a newer push stops here.
|
|
if: ${{ !cancelled() && needs.plan-slices.result == 'success' }}
|
|
timeout-minutes: 5
|
|
# contents:read ONLY — this job executes the PR-authored reconcile
|
|
# runner from the PR checkout, so it
|
|
# must never hold a write-scoped token. The PR comment lives in the
|
|
# separate slices-comment job below, which runs NO repo code: a
|
|
# $GITHUB_ENV/BASH_ENV persistence trick is job-scoped, so the split is
|
|
# the trust boundary (codex adversarial finding, 2026-08-31 — the old
|
|
# matrix-era report job had this separation and the consolidation had
|
|
# regressed it).
|
|
permissions:
|
|
contents: read
|
|
outputs:
|
|
reconcile-exit: ${{ steps.reconcile.outputs.exit || steps.reused.outputs.exit }}
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
with:
|
|
# A screen's pinned merge commit (empty otherwise); a train run needs the
|
|
# history of its prefix commits T1..Tk for the train record.
|
|
ref: ${{ inputs.screen_merge_sha }}
|
|
fetch-depth: ${{ needs.plan-slices.outputs.mode == 'train' && '0' || '1' }}
|
|
persist-credentials: false
|
|
|
|
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
|
with:
|
|
bun-version: 1.4.2
|
|
|
|
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
|
with:
|
|
name: paid-plan
|
|
path: /tmp/paid-report
|
|
|
|
# One directory per attempt-scoped slice artifact (no merge): shard
|
|
# records can never overwrite each other. Attempt 1 decides the lane;
|
|
# a later attempt or a rule-16 rerun resolves a red only by the flaky
|
|
# rule (scripts/lib/paid-rerun.ts) and never hides a new red.
|
|
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
|
with:
|
|
pattern: paid-slice-*
|
|
path: /tmp/paid-report
|
|
|
|
# Rule-16 reruns and their manifests. A missing one leaves its case red.
|
|
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
|
continue-on-error: true
|
|
with:
|
|
pattern: paid-rerun-*
|
|
# Its own directory: download-artifact unpacks a lone match into its path itself, so the
|
|
# report root would otherwise take a rerun slice's files (run 39363705692).
|
|
path: /tmp/paid-report/reruns
|
|
|
|
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
|
continue-on-error: true
|
|
with:
|
|
pattern: rerun-plan-a*
|
|
# Each artifact holds rerun-plan-a<attempt>/manifest.json, so even a lone match never
|
|
# lands on the plan's own manifest.json (run 39363705692).
|
|
path: /tmp/paid-report
|
|
|
|
- name: Reconcile slices against the manifest (fail-closed)
|
|
id: reconcile
|
|
if: needs.plan-slices.outputs.gate_reuse != 'true'
|
|
env:
|
|
# Flake records name the PR and its head (scripts/lib/paid-rerun.ts flakeRecords).
|
|
GSTACK_GATE_PR: ${{ github.event.pull_request.number || inputs.screen_pr }}
|
|
GSTACK_GATE_HEAD_SHA: ${{ github.event.pull_request.head.sha || needs.plan-slices.outputs.screen_head || inputs.expected_sha }}
|
|
run: |
|
|
set +e
|
|
EVALS_TIER=gate bun --no-install run scripts/test-paid-shards.ts --tier gate --report /tmp/paid-report | tee /tmp/report.txt
|
|
# PIPESTATUS[0], NOT $?: GitHub's default run-step shell is
|
|
# `bash -e {0}` with NO pipefail, so $? after the pipe is tee's
|
|
# exit (always 0) — the fail-closed gate was silently fail-open
|
|
# (caught by the ship review army; the wiring test now pins this).
|
|
echo "exit=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT"
|
|
|
|
# R2a/R2: the planner confirmed a reuse, so no slice ran; the verdict is
|
|
# the reused gate's, printed with which run, which head and why. A
|
|
# missing or malformed reuse record exits 1 (fail-closed).
|
|
- name: Report the reused gate verdict (R2a/R2)
|
|
id: reused
|
|
if: needs.plan-slices.outputs.gate_reuse == 'true'
|
|
run: |
|
|
set +e
|
|
bun --no-install run scripts/gate-reuse.ts report /tmp/paid-report/gate-reuse.json --repo "$GITHUB_REPOSITORY" > /tmp/report.txt
|
|
code=$?
|
|
cat /tmp/report.txt
|
|
echo "exit=$code" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Summarize PR coverage and reuse
|
|
if: always()
|
|
run: bun --no-install run scripts/test-pr-profile.ts summary /tmp/paid-report/manifest.json /tmp/paid-report/collector-outcomes.json >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
- name: Stamp trial history series
|
|
if: always()
|
|
run: |
|
|
if [ -f /tmp/paid-report/trial-outcomes.jsonl ]; then
|
|
bun --no-install run scripts/eval-trial-series.ts /tmp/paid-report/trial-outcomes.jsonl
|
|
fi
|
|
|
|
# One merged receipt store per run: the carried store (recovered
|
|
# receipts and checkpoint included), the plan's shipped set, every
|
|
# slice's new pass and negative receipts, and the report's panel and
|
|
# negative receipts. Saved
|
|
# last, so the next planner restores it as the newest prefix match.
|
|
- name: Merge this run's receipts
|
|
if: always() && github.event_name == 'pull_request'
|
|
run: |
|
|
bun --no-install run scripts/e2e-shard-reuse.ts merge /tmp/gstack-eval-input-cache /tmp/paid-report/store \
|
|
/tmp/paid-report/receipts /tmp/paid-report/report-receipts /tmp/paid-report/paid-slice-*/receipts
|
|
|
|
- name: Save this PR's verified judge and E2E results
|
|
if: always() && github.event_name == 'pull_request'
|
|
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
|
|
with:
|
|
path: /tmp/gstack-eval-input-cache
|
|
key: eval-input-v1-${{ github.repository_id }}-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }}-merged
|
|
|
|
- name: Upload reconciliation output for the comment job
|
|
if: always()
|
|
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
|
with:
|
|
name: report-verdict-a${{ github.run_attempt }}
|
|
path: |
|
|
/tmp/report.txt
|
|
/tmp/paid-report/collector-outcomes.json
|
|
/tmp/paid-report/report-summary.md
|
|
/tmp/paid-report/report-receipts
|
|
if-no-files-found: ignore
|
|
retention-days: 30
|
|
|
|
# Rule 16: one JSONL record per case that failed once and passed on its
|
|
# one rerun (case, PR, head, run, both attempts, symptom, rerun cost),
|
|
# read by the daily flaky-fix pass (docs/TESTING_INTERNALS.md#flaky-reruns).
|
|
- name: Upload flake records for the daily flaky-fix pass
|
|
if: always()
|
|
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
|
with:
|
|
name: flake-records-a${{ github.run_attempt }}
|
|
path: /tmp/paid-report/flake-records.jsonl
|
|
if-no-files-found: ignore
|
|
retention-days: 90
|
|
|
|
# R2a/R2: what this executed run proved, read by a later push. The base
|
|
# is the tested merge commit's first parent. Reused runs bind nothing,
|
|
# so reuse never chains off a reuse.
|
|
# A dispatched screen binds too (purpose screen, its tested merge commit
|
|
# and policy). Train, bisect and other dispatches never bind for reuse.
|
|
- name: Bind this run's verdict for reuse (R2a/R2)
|
|
if: always() && (github.event_name == 'pull_request' || needs.plan-slices.outputs.mode == 'screen') && needs.plan-slices.outputs.gate_reuse != 'true'
|
|
env:
|
|
PR_NUMBER: ${{ github.event.pull_request.number || inputs.screen_pr }}
|
|
HEAD_SHA: ${{ github.event.pull_request.head.sha || needs.plan-slices.outputs.screen_head }}
|
|
TESTED_SHA: ${{ inputs.screen_merge_sha || github.sha }}
|
|
SCREEN_POLICY: ${{ inputs.screen_policy }}
|
|
RECONCILE_EXIT: ${{ steps.reconcile.outputs.exit }}
|
|
run: |
|
|
bun --no-install run scripts/gate-reuse.ts bind /tmp/paid-report --out /tmp/gate-binding --repo "$GITHUB_REPOSITORY" \
|
|
--pr "$PR_NUMBER" --run-id "$GITHUB_RUN_ID" --run-attempt "$GITHUB_RUN_ATTEMPT" --head-sha "$HEAD_SHA" \
|
|
--tested-sha "$TESTED_SHA" --profile "$EVALS_PROFILE" --reconcile-exit "$RECONCILE_EXIT" \
|
|
${SCREEN_POLICY:+--purpose screen --screen-policy "$SCREEN_POLICY"} >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
- name: Upload the gate binding
|
|
if: always() && (github.event_name == 'pull_request' || needs.plan-slices.outputs.mode == 'screen') && needs.plan-slices.outputs.gate_reuse != 'true'
|
|
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
|
with:
|
|
name: gate-binding-a${{ github.run_attempt }}
|
|
path: /tmp/gate-binding/gate-binding.json
|
|
if-no-files-found: ignore
|
|
retention-days: 30
|
|
|
|
- name: Upload trial outcomes for pass-rate history
|
|
if: always()
|
|
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
|
with:
|
|
name: trial-outcomes-pr-a${{ github.run_attempt }}
|
|
path: /tmp/paid-report/trial-outcomes.jsonl
|
|
if-no-files-found: ignore
|
|
retention-days: 90
|
|
|
|
# GSTA-49 T1b: a train's own gate run publishes its record, read by
|
|
# gstack-ship-queue's verdict and verify and by the main backstop: the
|
|
# census from this run's plan manifest, one panel per planned case, and
|
|
# the prefix trees T1..Tk modulo stamps. GREEN only for a complete census.
|
|
- name: Write the train record
|
|
if: always() && needs.plan-slices.outputs.mode == 'train'
|
|
env:
|
|
NONCE: ${{ inputs.nonce }}
|
|
HEAD_SHA: ${{ inputs.expected_sha }}
|
|
RECORD: ${{ needs.plan-slices.outputs.record_artifact }}
|
|
RECONCILE_EXIT: ${{ steps.reconcile.outputs.exit }}
|
|
run: |
|
|
bun --no-env-file --config=/dev/null --no-install run scripts/train-gate.ts train-record /tmp/paid-report --nonce "$NONCE" --head "$HEAD_SHA" \
|
|
--reconcile-exit "$RECONCILE_EXIT" --run-id "$GITHUB_RUN_ID" --run-attempt "$GITHUB_RUN_ATTEMPT" --repo "$GITHUB_REPOSITORY" \
|
|
--out "/tmp/train-record/${RECORD}.json" >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
- name: Upload the train record
|
|
if: always() && needs.plan-slices.outputs.mode == 'train'
|
|
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
|
with:
|
|
name: ${{ needs.plan-slices.outputs.record_artifact }}
|
|
path: /tmp/train-record
|
|
if-no-files-found: ignore
|
|
overwrite: true
|
|
retention-days: 90
|
|
|
|
- name: Fail the workflow when reconciliation failed
|
|
if: (steps.reconcile.outputs.exit || steps.reused.outputs.exit) != '0'
|
|
run: exit 1
|
|
|
|
# GSTA-49 T1b: the check run gstack-ship-queue requires on the train head,
|
|
# published only for a complete GREEN census. Its own job with checks:
|
|
# write and no repository code (the same split as slices-comment).
|
|
train-check:
|
|
runs-on: ubuntu-24.04
|
|
needs: [plan-slices, slices-report]
|
|
if: ${{ !cancelled() && needs.plan-slices.outputs.mode == 'train' && needs.slices-report.result == 'success' }}
|
|
timeout-minutes: 5
|
|
permissions:
|
|
checks: write
|
|
steps:
|
|
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
|
with:
|
|
name: ${{ needs.plan-slices.outputs.record_artifact }}
|
|
path: /tmp/train-record
|
|
|
|
- name: Publish the full-gate check on the train head
|
|
env:
|
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
CHECK: ${{ needs.plan-slices.outputs.full_gate_check }}
|
|
RECORD: ${{ needs.plan-slices.outputs.record_artifact }}
|
|
NONCE: ${{ inputs.nonce }}
|
|
HEAD_SHA: ${{ inputs.expected_sha }}
|
|
run: |
|
|
record="/tmp/train-record/${RECORD}.json"
|
|
if ! jq -e --arg nonce "$NONCE" --arg head "$HEAD_SHA" \
|
|
'.schema == 1 and .kind == "train-gate" and .verdict == "GREEN" and .census.complete == true and .nonce == $nonce and .head_sha == $head' \
|
|
"$record" >/dev/null 2>&1; then
|
|
echo "::error::the train record is not a complete GREEN census of ${HEAD_SHA} (nonce ${NONCE}); no ${CHECK} check run"
|
|
exit 1
|
|
fi
|
|
summary=$(jq -r '"rule \(.census.executed.rule)/\(.census.planned.rule), judge \(.census.executed.judge)/\(.census.planned.judge); prefixes T1..T\(.prefixes | length)"' "$record")
|
|
gh api "repos/${GITHUB_REPOSITORY}/check-runs" -X POST -f name="$CHECK" -f head_sha="$HEAD_SHA" -f status=completed -f conclusion=success \
|
|
-f details_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
|
|
-f "output[title]=Train ${NONCE}: complete GREEN census" -f "output[summary]=${summary}" >/dev/null
|
|
echo "- ${CHECK}: success on ${HEAD_SHA} (${summary})" >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
# PR comment in its OWN job with the write token and ZERO repo code: no
|
|
# checkout, no bun install — only downloaded artifacts, jq, and gh. See the
|
|
# trust-boundary note on slices-report.
|
|
slices-comment:
|
|
runs-on: ubicloud-standard-2
|
|
needs: slices-report
|
|
# A dispatched screen comments on its PR too (screen_pr was validated by the route job, or nothing ran).
|
|
if: ${{ !cancelled() && ((github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || inputs.screen_pr != '') && needs.slices-report.result != 'skipped' }}
|
|
timeout-minutes: 5
|
|
permissions:
|
|
pull-requests: write
|
|
# The comment upsert calls the REST `/issues/{n}/comments` endpoints
|
|
# (gh api ... issues/comments). With GITHUB_TOKEN those are gated by the
|
|
# `issues` permission, not `pull-requests` (#1802 CI fix).
|
|
issues: write
|
|
steps:
|
|
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
|
with:
|
|
name: paid-plan
|
|
path: /tmp/paid-report
|
|
|
|
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
|
with:
|
|
name: report-verdict-a${{ github.run_attempt }}
|
|
path: /tmp/verdict
|
|
continue-on-error: true
|
|
|
|
# Every count, verdict and failure line comes from the read-only report
|
|
# job's collector-outcomes v2 (panelVerdict() ran there); this job runs
|
|
# no repo code and never recomputes a verdict. Keeps the
|
|
# "## E2E Evals" marker so the upsert keeps updating the same comment.
|
|
# Runs even when reconciliation failed — a red lane on the PR is the point.
|
|
- name: Post PR comment
|
|
env:
|
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
RECONCILE_EXIT: ${{ needs.slices-report.outputs.reconcile-exit }}
|
|
PR_NUMBER: ${{ github.event.pull_request.number || inputs.screen_pr }}
|
|
SCREEN_POLICY: ${{ inputs.screen_policy }}
|
|
SCREEN_MERGE_SHA: ${{ inputs.screen_merge_sha }}
|
|
run: |
|
|
# shellcheck disable=SC2086,SC2059
|
|
# R2a/R2: a confirmed verdict reuse ran no case; say which run,
|
|
# which head and why, with the receipt or own-diff signature.
|
|
REUSE=/tmp/paid-report/gate-reuse.json
|
|
if [ "${RECONCILE_EXIT:-1}" = "0" ] && [ -f "$REUSE" ] \
|
|
&& jq -e '.schema == 1 and (.kind == "restamp-only" or .kind == "main-moved")' "$REUSE" >/dev/null 2>&1; then
|
|
BODY=$(jq -r --arg repo "$GITHUB_REPOSITORY" '"## E2E Evals: ✅ PASS (reused: \(if .kind == "restamp-only" then "restamp-only, R2a" else "main moved, R2" end))\n\nNo paid case ran on `\(.current_head)`. It carries the GREEN verdict of [run \(.reused_run_id)](https://github.com/\($repo)/actions/runs/\(.reused_run_id)) at `\(.gated_head)`, because \(.why).\(if ((.reused_flaky // []) | length) > 0 then " That gate was green with flaky case(s) that failed once and passed on their one rule-16 rerun: \(.reused_flaky | join(", "))." else "" end)\n\n```\n\(if .kind == "restamp-only" then "TREE: same-modulo-stamps receipt=\(.receipt_sha256)" else "OWN DIFF: identical sha256=\(.own_diff_sha256) (merge-base \(.gated_merge_base) -> \(.merge_base))" end)\nbase: \(.gated_base) -> \(.base_sha)\nselection: \(.selection_sha256) (identical to the reused gate)\n```\n\n---\n*Verdict reuse (R2a restamp-only, R2 main moved): a changed own diff, a non-stamp byte, a changed selection, a newer red or unbound gate, or a missing binding or receipt runs the full gate; the evals-fresh label opts out. Free Tests and every other check still ran on this head.*"' "$REUSE")
|
|
else
|
|
TOTAL=0; PASSED=0; FAILED=0; MANUAL=0; EXECUTED=0; REUSED=0; COST="0"
|
|
SUITE_LINES=""
|
|
VERIFIED=/tmp/verdict/paid-report/collector-outcomes.json
|
|
if ! jq -e '
|
|
. as $summary |
|
|
.version == 2 and (.files | type == "array") and (.totals | type == "object") and
|
|
(.headline | type == "array") and (.failures | type == "array") and (.panels | type == "array") and
|
|
(.verdict.verdict == "GREEN" or .verdict.verdict == "RED") and
|
|
((.flakyCollectorFailures // 0) | type == "number" and . >= 0 and floor == .) and
|
|
((.flakyCollectorFailures // 0) <= .totals.failed) and
|
|
([.files[] | .total == (.passed + .failed + .manual_accepted) and
|
|
(.total == (.executed + .reused)) and
|
|
([.total,.passed,.failed,.manual_accepted,.executed,.reused,.attempts,.flaky] | all(. >= 0 and (floor == .))) ] | all) and
|
|
(.totals | .total == (.passed + .failed + .manual_accepted) and .total == (.executed + .reused)) and
|
|
(["total","passed","failed","manual_accepted","executed","reused","attempts","flaky"] |
|
|
all(. as $key | ([$summary.files[] | .[$key]] | add // 0) == $summary.totals[$key]))
|
|
' "$VERIFIED" >/dev/null 2>&1; then
|
|
VERIFIED=""
|
|
echo 'Verified report summary unavailable; no verdict, counts or manual acceptance can be shown.'
|
|
fi
|
|
HEADLINE='(no verified report headline)'
|
|
FAILURES=""
|
|
FLAKY=""
|
|
FLAKY_FAILED=0
|
|
if [ -n "$VERIFIED" ]; then
|
|
while IFS=$'\t' read -r _FILE T P F M _FLAKY EX RE _ATTEMPTS C TIER SHARD; do
|
|
[ "$T" -eq 0 ] && continue
|
|
TOTAL=$((TOTAL + T)); PASSED=$((PASSED + P)); FAILED=$((FAILED + F))
|
|
MANUAL=$((MANUAL + M))
|
|
EXECUTED=$((EXECUTED + EX)); REUSED=$((REUSED + RE))
|
|
COST=$(echo "$COST + $C" | bc)
|
|
STATUS_ICON="✅"
|
|
[ "$M" -gt 0 ] && STATUS_ICON="⚠ manual/unscored"
|
|
[ "$F" -gt 0 ] && STATUS_ICON="❌"
|
|
SUITE_LINES="${SUITE_LINES}| ${TIER}/${SHARD} | ${P}/${T} | ${M} | ${EX} | ${RE} | ${STATUS_ICON} | \$${C} |\n"
|
|
done < <(jq -r '.files[] | [.file,.total,.passed,.failed,.manual_accepted,.flaky,.executed,.reused,.attempts,.cost,.tier,.shard] | @tsv' "$VERIFIED")
|
|
# Report-sanitized lines (no @-mentions, one capped line each), fenced here.
|
|
HEADLINE=$(jq -r '.headline[]' "$VERIFIED")
|
|
FAILURES=$(jq -r '.failures[]' "$VERIFIED")
|
|
# Rule 16: report-sanitized lines of cases that failed once and passed on their one rerun.
|
|
FLAKY=$(jq -r '(.flakyLines // [])[]' "$VERIFIED")
|
|
# Their first-attempt collector failures, the only failed records a green report may carry.
|
|
FLAKY_FAILED=$(jq -r '.flakyCollectorFailures // 0' "$VERIFIED")
|
|
fi
|
|
|
|
COVERAGE=$(jq -r '"Profile: \(.profile // "full") / mode `\(.prCoverage.mode // "broad")`; selected behaviors: \(.selection.e2e | if . == null then "all" else length end), judges: \(.selection.judges | if . == null then "all" else length end). Deferred to scheduled/release coverage: \(.prCoverage.deferred // [] | length) behaviors and \(.prCoverage.deferredPromptFiles // [] | length) changed prompt files. Deferred checks did not run and receive no PR-pass credit."' /tmp/paid-report/manifest.json) || COVERAGE='Coverage manifest unavailable; no coverage claim.'
|
|
# CEO-13: on fallback, name every file that restored the full gate and its fix.
|
|
FALLBACK=$(jq -r 'select(.prCoverage.mode == "full-fallback") | .prCoverage as $c | ($c.unknownFiles // [])[] as $f
|
|
| (($c.unknownFileLabels // []) | map(select(.file == $f)) | first // {label: "unlabeled", fix: "see docs/TESTING_INTERNALS.md#pr-paid-lane-fallback"}) as $l
|
|
| "- `\($f)` (\($l.label)): \($l.fix)"' /tmp/paid-report/manifest.json 2>/dev/null | head -40 | sed 's/@/@\xe2\x80\x8b/g') || FALLBACK=''
|
|
if [ -n "$FALLBACK" ]; then COVERAGE="${COVERAGE}
|
|
|
|
Full gate restored by:
|
|
${FALLBACK}"; fi
|
|
# Dependents mode: unmapped files placed by derived references, with their dependent counts.
|
|
DERIVED=$(jq -r '(.prCoverage.derivedFiles // [])[:40][] | "- `\(.file)`: \(.e2e) case(s), \(.judges) judge(s)"' /tmp/paid-report/manifest.json 2>/dev/null | sed 's/@/@\xe2\x80\x8b/g') || DERIVED=''
|
|
if [ -n "$DERIVED" ]; then COVERAGE="${COVERAGE}
|
|
|
|
Derived dependents (scripts/pr-dependencies.ts):
|
|
${DERIVED}"; fi
|
|
|
|
STATUS="✅ PASS"
|
|
if [ "${RECONCILE_EXIT:-1}" != "0" ] || [ "$FAILED" -gt "$FLAKY_FAILED" ] \
|
|
|| { [ -n "$VERIFIED" ] && [ "$(jq -r '.verdict.verdict' "$VERIFIED")" != "GREEN" ]; }; then STATUS="❌ FAIL"; fi
|
|
if [ "$STATUS" = '✅ PASS' ] && [ "$MANUAL" -gt 0 ]; then STATUS='⚠ MANUAL ACCEPTED (unscored)'; fi
|
|
if [ "$STATUS" = '✅ PASS' ] && [ -n "$FLAKY" ]; then STATUS="✅ PASS (flaky: $(grep -c . <<<"$FLAKY") case(s) passed on their one rerun)"; fi
|
|
if [ -z "$VERIFIED" ]; then STATUS='❌ FAIL (verified report unavailable)'; fi
|
|
|
|
BODY="## E2E Evals: ${STATUS}
|
|
|
|
\`\`\`
|
|
${HEADLINE}
|
|
\`\`\`
|
|
|
|
**${EXECUTED} executed, ${REUSED} reused** rule/judge records | **${MANUAL} manual accepted (unscored; no score-cache credit)** | **\$${COST}** rule/judge cost | reconcile exit: ${RECONCILE_EXIT:-missing}
|
|
|
|
${COVERAGE}
|
|
|
|
<details><summary>Rule and judge shards</summary>
|
|
|
|
| Shard | Automated result | Manual/unscored | Executed | Reused | Status | Cost |
|
|
|-------|------------------|-----------------|----------|--------|--------|------|
|
|
$(echo -e "$SUITE_LINES")
|
|
</details>
|
|
|
|
<details><summary>Fail-closed reconciliation</summary>
|
|
|
|
\`\`\`
|
|
$(tail -c 4000 /tmp/verdict/report.txt 2>/dev/null | sed 's/@/@\xe2\x80\x8b/g' || echo '(no reconciliation output)')
|
|
\`\`\`
|
|
</details>
|
|
|
|
---
|
|
*Sliced lane: planner → duration-packed executors → fail-closed report. Behavior cases run a pre-registered 3-trial panel (PASS at 2/3 with no contract violation); a PASS 2/3 is shown with its failed trial, never as a clean pass. Reused results retain their original provenance and expiry.*"
|
|
|
|
if [ -n "$FAILURES" ]; then
|
|
BODY="${BODY}
|
|
|
|
### Failures and split verdicts
|
|
\`\`\`
|
|
${FAILURES}
|
|
\`\`\`"
|
|
fi
|
|
|
|
if [ -n "$FLAKY" ]; then
|
|
BODY="${BODY}
|
|
|
|
### Flaky: failed once, passed on its one rerun (rule 16)
|
|
\`\`\`
|
|
${FLAKY}
|
|
\`\`\`
|
|
Each failed and passed on its rerun in [run ${GITHUB_RUN_ID}](https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}) at the attempts named above. Logged for the daily flaky-fix pass in this run's \`flake-records-a${GITHUB_RUN_ATTEMPT}\` artifact (docs/TESTING_INTERNALS.md#flaky-reruns)."
|
|
fi
|
|
|
|
fi
|
|
|
|
if [ -n "$SCREEN_POLICY" ]; then
|
|
BODY="${BODY}
|
|
|
|
*Dispatched ${SCREEN_POLICY} screen (gstack-ship-queue ready --screen) of merge commit \`${SCREEN_MERGE_SHA}\`.*"
|
|
fi
|
|
|
|
COMMENT_ID=$(gh api "repos/${{ github.repository }}/issues/${PR_NUMBER}/comments" \
|
|
--jq '.[] | select(.body | startswith("## E2E Evals")) | .id' | tail -1)
|
|
|
|
if [ -n "$COMMENT_ID" ]; then
|
|
gh api "repos/${{ github.repository }}/issues/comments/${COMMENT_ID}" \
|
|
-X PATCH -f body="$BODY"
|
|
else
|
|
# REST, not gh's pr-comment subcommand: this job runs with NO
|
|
# checkout (the token/exec split), and that subcommand resolves
|
|
# the repo FROM git — it dies with "not a git repository" here.
|
|
gh api "repos/${{ github.repository }}/issues/${PR_NUMBER}/comments" \
|
|
-X POST -f body="$BODY"
|
|
fi
|