mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-31 22:14:25 +01:00
Garry Tanandcapy-ai[bot]
a2a6cabfa6
* feat(evals): readiness-gated paid dispatch (R1), restamp-only gate reuse (R2a), and a receipt store that restores Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * feat(evals): R2 verdict reuse across main moves; review fixes (newer-run classification, receipt parser, manual acceptance) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(evals): R2 own-diff signature covers release-output content and CHANGELOG deletions and preamble Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(evals): R2 normalises release outputs by stamp pattern only; compares stamp-file modes; unreadable blobs refuse Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * chore: stamp v1.91.134.0 Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(evals): autoplan-guard-pty actor continues through the phase boundary; name PHASE_BOUNDARY_STOP Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
861 lines
46 KiB
YAML
861 lines
46 KiB
YAML
name: E2E Evals
|
|
# eval-bg (scripts/eval-bg.ts) finds its dispatch by the nonce in the run name (ENG-12).
|
|
run-name: ${{ inputs.nonce != '' && format('E2E Evals [eval-bg {0}]', inputs.nonce) || '' }}
|
|
on:
|
|
pull_request:
|
|
branches: [main]
|
|
workflow_dispatch:
|
|
inputs:
|
|
evals_all:
|
|
description: 'Run ALL gate tests in the sliced lane (bypass diff selection; also arms the hollow-shard guard). false = the PR profile on the diff against base_sha/base_ref'
|
|
type: boolean
|
|
default: true
|
|
base_ref:
|
|
description: 'Base branch for diff selection when evals_all is false (default: detected base branch)'
|
|
type: string
|
|
default: ''
|
|
base_sha:
|
|
description: 'Exact merge-base commit for diff selection; overrides base_ref'
|
|
type: string
|
|
default: ''
|
|
expected_sha:
|
|
description: 'Fail fast unless the checked-out commit is this SHA (scripts/eval-bg.ts sets it)'
|
|
type: string
|
|
default: ''
|
|
nonce:
|
|
description: 'Opaque id shown in the run name so the dispatcher can find this run'
|
|
type: string
|
|
default: ''
|
|
pr_receipts:
|
|
description: 'Same-repo PR number whose receipt store this run reads read-only (never saves); empty = fresh'
|
|
type: string
|
|
default: ''
|
|
|
|
concurrency:
|
|
group: evals-${{ github.event.pull_request.number || github.run_id }}
|
|
cancel-in-progress: true
|
|
|
|
env:
|
|
IMAGE: ghcr.io/${{ github.repository }}/ci
|
|
# PRs and evals_all=false dispatches (eval:bg:pr) run the fast PR profile on the diff; evals_all dispatches run the complete gate census.
|
|
EVALS_PROFILE: ${{ (github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && !inputs.evals_all)) && 'pr' || 'full' }}
|
|
# Dispatches validate fresh unless they name pr_receipts (read-only reuse, CEO-29/ENG-8); the evals-fresh PR label opts a PR out of reuse (DX-7).
|
|
EVALS_FRESH: ${{ ((github.event_name == 'workflow_dispatch' && inputs.pr_receipts == '') || contains(github.event.pull_request.labels.*.name, 'evals-fresh')) && '1' || '' }}
|
|
|
|
jobs:
|
|
# Build Docker image with pre-baked toolchain (cached — only rebuilds on Dockerfile/lockfile change)
|
|
build-image:
|
|
# Dependabot-triggered pull_request runs get a read-only GITHUB_TOKEN, so
|
|
# a lockfile bump = new hash = failed ghcr push = permanently red check
|
|
# (EV6, fork port wave 2). Skip the build for dependabot; the evals job's
|
|
# explicit actor guard mirrors it because no eval test selects on a lockfile-only
|
|
# diff — a maintainer's next push rebuilds the image with real perms.
|
|
if: github.actor != 'dependabot[bot]'
|
|
runs-on: ubicloud-standard-8
|
|
timeout-minutes: 15
|
|
permissions:
|
|
contents: read
|
|
packages: write
|
|
outputs:
|
|
image-tag: ${{ steps.meta.outputs.tag }}
|
|
runtime-id: ${{ steps.runtime.outputs.id }}
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
|
|
- id: meta
|
|
# Key on Dockerfile + lockfile only. package.json is deliberately NOT
|
|
# hashed: its version field changes on every ship (60/60 recent commits),
|
|
# which rebuilt the image each time for a dependency set that only
|
|
# bun.lock determines. A stale baked package.json is harmless — checkout
|
|
# overwrites /workspace and node_modules comes from the lockfile.
|
|
run: echo "tag=${{ env.IMAGE }}:${{ hashFiles('.github/docker/Dockerfile.ci', 'bun.lock', 'patches/**') }}" >> "$GITHUB_OUTPUT"
|
|
|
|
- uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
|
|
with:
|
|
registry: ghcr.io
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Check if image exists
|
|
id: check
|
|
run: |
|
|
if docker manifest inspect ${{ steps.meta.outputs.tag }} > /dev/null 2>&1; then
|
|
echo "exists=true" >> "$GITHUB_OUTPUT"
|
|
else
|
|
echo "exists=false" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
|
|
- if: steps.check.outputs.exists == 'false'
|
|
run: cp package.json bun.lock .github/docker/ && cp -R patches .github/docker/patches
|
|
|
|
# A fork PR's GITHUB_TOKEN only has `packages: read`, so pushing fails.
|
|
# Still BUILD (validates Dockerfile.ci changes), just don't publish. This
|
|
# job intentionally keeps no `if:` so fork PRs still get one real, honest
|
|
# green check here instead of a run where every job is grey.
|
|
# Registry cache export needs a docker-container builder — the default
|
|
# `docker` driver hard-errors on cache-to (first live run of the trio).
|
|
- if: steps.check.outputs.exists == 'false'
|
|
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
|
|
|
|
- if: steps.check.outputs.exists == 'false'
|
|
uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
|
|
with:
|
|
context: .github/docker
|
|
file: .github/docker/Dockerfile.ci
|
|
push: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
|
|
# Registry layer cache: reads are safe everywhere; the export is gated
|
|
# to same-repo runs because a fork PR's token can't write GHCR.
|
|
cache-from: type=registry,ref=${{ env.IMAGE }}:buildcache
|
|
cache-to: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && format('type=registry,ref={0}:buildcache,mode=max', env.IMAGE) || '' }}
|
|
tags: |
|
|
${{ steps.meta.outputs.tag }}
|
|
${{ env.IMAGE }}:latest
|
|
|
|
- name: Identify the installed eval runtime
|
|
id: runtime
|
|
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
|
|
env:
|
|
EVAL_IMAGE: ${{ steps.meta.outputs.tag }}
|
|
run: |
|
|
docker manifest inspect "$EVAL_IMAGE" > /tmp/eval-runtime-manifest.json
|
|
echo "id=$(sha256sum /tmp/eval-runtime-manifest.json | cut -d ' ' -f1)" >> "$GITHUB_OUTPUT"
|
|
|
|
# ── Readiness-gated dispatch (R1, GSTA-48) ───────────────────────────────
|
|
# Paid slices for a PR head start only after the PR has been quiet for the
|
|
# window (vars.EVALS_QUIET_MINUTES, default 10; the evals-no-debounce label
|
|
# sets it to 0) and Free Tests has passed on the same head SHA. A newer push
|
|
# still cancels this run (cancel-in-progress), so the window is a sleep; the
|
|
# same heads get the same selection and verdict rules, only the start moves.
|
|
# Free Tests red on this head fails the run without paid work (re-run this
|
|
# workflow after a green Free Tests rerun). Dispatches never wait: they are
|
|
# explicit (eval:bg, census). Version Gate is not an input, so a stamp push
|
|
# waits for its own Free Tests only and always gets its gate.
|
|
readiness:
|
|
runs-on: ubuntu-24.04
|
|
if: github.event_name == 'pull_request' && github.actor != 'dependabot[bot]' && github.event.pull_request.head.repo.full_name == github.repository
|
|
timeout-minutes: 90
|
|
permissions:
|
|
contents: read
|
|
actions: read
|
|
pull-requests: read
|
|
outputs:
|
|
superseded: ${{ steps.wait.outputs.superseded }}
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
|
with:
|
|
bun-version: 1.4.2
|
|
|
|
- name: Wait for a quiet PR and green Free Tests on this head
|
|
id: wait
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
PR_NUMBER: ${{ github.event.pull_request.number }}
|
|
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
|
QUIET_MINUTES: ${{ contains(github.event.pull_request.labels.*.name, 'evals-no-debounce') && '0' || vars.EVALS_QUIET_MINUTES || '10' }}
|
|
run: |
|
|
bun --no-install run scripts/eval-readiness.ts wait --repo "$GITHUB_REPOSITORY" --pr "$PR_NUMBER" --head-sha "$HEAD_SHA" \
|
|
--run-id "$GITHUB_RUN_ID" --quiet-minutes "$QUIET_MINUTES" --deadline-minutes 80 \
|
|
--github-output "$GITHUB_OUTPUT" --summary "$GITHUB_STEP_SUMMARY"
|
|
|
|
# ── Receipt recovery (W1c: CEO-14/34, ENG-4/5/7) ─────────────────────────
|
|
# A run cancelled by the next push never reaches slices-report, so its
|
|
# finished receipts (passes AND the negatives executors write at the
|
|
# execution boundary) never reach the saved store. This job restores the
|
|
# PR's store, merges the receipts of this PR's completed cancelled runs
|
|
# (oldest first, at most 5, provenance-checked), reconstructs completed
|
|
# panel negatives, writes the recovery.json checkpoint and hands the store
|
|
# to plan-slices as an artifact. It runs scripts from the BASE ref only and
|
|
# alone holds actions: read; plan-slices keeps contents: read. It never
|
|
# fails the run: any gh problem degrades to "no recovered receipts" and
|
|
# plan-slices turns reuse off, so cases execute unreduced.
|
|
recover-receipts:
|
|
runs-on: ubuntu-24.04
|
|
needs: [readiness]
|
|
# Runs after readiness settles, so a superseded run's cancellation has usually completed; a head whose Free Tests failed recovers nothing.
|
|
# A dispatch naming pr_receipts reads that PR's receipts from its runs' artifacts (cache entries are ref-scoped).
|
|
if: ${{ !cancelled() && (needs.readiness.result == 'success' || needs.readiness.result == 'skipped') && needs.readiness.outputs.superseded != 'true' && github.actor != 'dependabot[bot]' && ((github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && !contains(github.event.pull_request.labels.*.name, 'evals-fresh')) || (github.event_name == 'workflow_dispatch' && inputs.pr_receipts != '')) }}
|
|
timeout-minutes: 5
|
|
permissions:
|
|
contents: read
|
|
actions: read
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
with:
|
|
ref: ${{ github.event.pull_request.base.sha || inputs.base_sha || github.event.repository.default_branch }}
|
|
persist-credentials: false
|
|
|
|
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
|
with:
|
|
bun-version: 1.4.2
|
|
|
|
- name: Restore this PR's verified judge and E2E results
|
|
id: store
|
|
if: github.event_name == 'pull_request'
|
|
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
|
|
with:
|
|
path: /tmp/gstack-eval-input-cache
|
|
key: eval-input-v1-${{ github.repository_id }}-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }}-plan
|
|
restore-keys: eval-input-v1-${{ github.repository_id }}-pr-${{ github.event.pull_request.number }}-
|
|
|
|
- name: Recover receipts of cancelled runs (base-ref code)
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
STORE_KEY: ${{ steps.store.outputs.cache-matched-key }}
|
|
PR_NUMBER: ${{ github.event.pull_request.number || inputs.pr_receipts }}
|
|
HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
|
|
HEAD_BRANCH: ${{ github.event.pull_request.head.ref || github.ref_name }}
|
|
RECOVERY_MODE: ${{ github.event_name == 'workflow_dispatch' && 'dispatch' || 'pr' }}
|
|
run: |
|
|
mkdir -p /tmp/gstack-eval-input-cache
|
|
if [ ! -f scripts/recover-receipts.ts ]; then
|
|
echo "- Receipt recovery unavailable: the base ref has no scripts/recover-receipts.ts; reuse stays off this run" >> "$GITHUB_STEP_SUMMARY"
|
|
exit 0
|
|
fi
|
|
bun --no-install run scripts/recover-receipts.ts collect --repo "$GITHUB_REPOSITORY" --pr "$PR_NUMBER" \
|
|
--head-repo "$HEAD_REPO" --branch "$HEAD_BRANCH" --run-id "$GITHUB_RUN_ID" \
|
|
--store /tmp/gstack-eval-input-cache --store-key "$STORE_KEY" --mode "$RECOVERY_MODE" --budget-seconds 60 >> "$GITHUB_STEP_SUMMARY" \
|
|
|| echo "- Receipt recovery crashed; reuse stays off this run" >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
# Verdict reuse (GSTA-48), a gate edit: base-ref code proposes reusing
|
|
# the newest executed GREEN gate of this PR when this head differs from
|
|
# it only by stamp files on the same base (R2a: TREE same-modulo-stamps
|
|
# from the tree receipt under the base's ship policy), or when main
|
|
# moved and the PR's own diff from its merge-base, stamp files excluded,
|
|
# is byte-identical (R2). plan-slices confirms the selection is
|
|
# identical. Anything else (a red, partial or cancelled gate, a changed
|
|
# own diff, a non-stamp byte, no binding or receipt, a gh or git error,
|
|
# evals-fresh) leaves no candidate: the full gate runs.
|
|
- name: Find a gate verdict to reuse (R2a/R2, base-ref code)
|
|
if: github.event_name == 'pull_request'
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
PR_NUMBER: ${{ github.event.pull_request.number }}
|
|
HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
|
|
HEAD_BRANCH: ${{ github.event.pull_request.head.ref }}
|
|
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
|
run: |
|
|
if [ ! -f scripts/gate-reuse.ts ]; then
|
|
echo "- Gate verdict reuse unavailable: the base ref has no scripts/gate-reuse.ts; the full gate runs" >> "$GITHUB_STEP_SUMMARY"
|
|
exit 0
|
|
fi
|
|
bun --no-install run scripts/gate-reuse.ts find --repo "$GITHUB_REPOSITORY" --pr "$PR_NUMBER" --head-repo "$HEAD_REPO" \
|
|
--branch "$HEAD_BRANCH" --run-id "$GITHUB_RUN_ID" --head-sha "$HEAD_SHA" --tested-sha "$GITHUB_SHA" \
|
|
--out /tmp/gate-reuse --budget-seconds 90 >> "$GITHUB_STEP_SUMMARY" \
|
|
|| echo "- Gate verdict reuse crashed; the full gate runs" >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
- name: Hand a gate reuse candidate to plan-slices
|
|
if: github.event_name == 'pull_request'
|
|
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
|
with:
|
|
name: gate-reuse-candidate
|
|
path: /tmp/gate-reuse
|
|
if-no-files-found: ignore
|
|
retention-days: 7
|
|
|
|
- name: Hand the receipt store to plan-slices
|
|
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
|
with:
|
|
name: receipt-store
|
|
path: /tmp/gstack-eval-input-cache
|
|
if-no-files-found: ignore
|
|
retention-days: 7
|
|
|
|
# ── Sliced lane (the ONLY paid lane; legacy 17-row matrix deleted) ──────────
|
|
# One PLANNER computes diff selection + the slice plan ONCE (killing
|
|
# per-slice selector divergence); K executors consume the manifest; the
|
|
# report reconciles results against it FAIL-CLOSED (a slice whose artifact
|
|
# never landed is a failure, a planned shard nobody reported is a failure —
|
|
# hollow lanes cannot aggregate green). Engine: scripts/test-paid-shards.ts —
|
|
# the same runner local eval:bg:gate uses, so CI and local share one
|
|
# selection engine, and every gate-tier file is in the census by
|
|
# construction (no hand-enumerated rows to drift). The legacy matrix ran
|
|
# 18 enumerated files for 22.6 min/$21 per PR serialized AHEAD of this
|
|
# lane's 49-file diff-selected census; parity was demonstrated (sliced
|
|
# census ⊇ matrix files) and the matrix deleted — one revert restores it.
|
|
#
|
|
# Fork PRs never receive repository secrets (ANTHROPIC_API_KEY et al), so
|
|
# every API-calling eval fails at SDK auth before a model runs. Skip
|
|
# deterministically; fork work gets real coverage via a trusted base-repo
|
|
# branch (see CLAUDE.md's garrytan-agents workflow).
|
|
plan-slices:
|
|
runs-on: ubicloud-standard-8
|
|
needs: [readiness, recover-receipts]
|
|
# !cancelled(): plans even when readiness or recovery was skipped (dispatch, evals-fresh) or recovery failed (reuse is then off); a superseded run, or a head whose readiness failed (Free Tests red), plans nothing.
|
|
if: ${{ !cancelled() && (needs.readiness.result == 'success' || needs.readiness.result == 'skipped') && needs.readiness.outputs.superseded != 'true' && github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }}
|
|
timeout-minutes: 10
|
|
permissions:
|
|
contents: read
|
|
outputs:
|
|
slices: ${{ steps.matrix.outputs.slices }}
|
|
slice_timeouts: ${{ steps.matrix.outputs.slice_timeouts }}
|
|
gate_reuse: ${{ steps.gate_reuse.outputs.gate_reuse }}
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
with:
|
|
# Preserve full history for merge-base diff selection. Moving the
|
|
# planner off the eval image must not change its selection inputs.
|
|
fetch-depth: 0
|
|
persist-credentials: false
|
|
|
|
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
|
with:
|
|
bun-version: 1.4.2
|
|
|
|
# DX-1: a dispatch from scripts/eval-bg.ts names the revision it means
|
|
# to test; a different checkout fails here before any paid work.
|
|
- name: Verify the dispatched revision
|
|
if: github.event_name == 'workflow_dispatch'
|
|
env:
|
|
EXPECTED_SHA: ${{ inputs.expected_sha }}
|
|
NONCE: ${{ inputs.nonce }}
|
|
run: |
|
|
actual=$(git rev-parse HEAD)
|
|
echo "- Tested revision: ${actual} (ref ${GITHUB_REF_NAME}${NONCE:+, nonce ${NONCE}})" >> "$GITHUB_STEP_SUMMARY"
|
|
if [ -n "$EXPECTED_SHA" ] && [ "$actual" != "$EXPECTED_SHA" ]; then
|
|
echo "::error::evals.yml checked out ${actual} but the dispatch expected ${EXPECTED_SHA}. Push the branch so its head is ${EXPECTED_SHA} and dispatch again (bun run scripts/eval-bg.ts pr), or run locally with --local."
|
|
exit 1
|
|
fi
|
|
|
|
# Planner-side reuse: take the PR's receipt store from recover-receipts
|
|
# (the restored store plus cancelled runs' receipts) and ship ONE
|
|
# filtered set with the plan, so every trial of a panel sees the same
|
|
# receipts and a newer FAIL blocks any older PASS for the same inputs.
|
|
- name: Download this PR's receipt store
|
|
if: github.event_name == 'pull_request' || inputs.pr_receipts != ''
|
|
continue-on-error: true
|
|
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
|
with:
|
|
name: receipt-store
|
|
path: /tmp/gstack-eval-input-cache
|
|
|
|
- name: Decide receipt reuse
|
|
id: reuse
|
|
if: github.event_name == 'pull_request' || inputs.pr_receipts != ''
|
|
run: |
|
|
mkdir -p /tmp/gstack-eval-input-cache
|
|
bun --no-install run scripts/recover-receipts.ts decide /tmp/gstack-eval-input-cache --github-output "$GITHUB_OUTPUT" >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
- name: Emit run manifest
|
|
env:
|
|
EVALS_ALL: ${{ (github.event_name == 'workflow_dispatch' && inputs.evals_all) && '1' || '' }}
|
|
EVALS_CACHE_DIR: ${{ steps.reuse.outputs.reuse == 'on' && '/tmp/gstack-eval-input-cache' || '' }}
|
|
# Dispatch diff base (CEO-17): the exact merge-base, else the named base branch, else detection.
|
|
EVALS_BASE: ${{ inputs.base_sha || (inputs.base_ref != '' && format('origin/{0}', inputs.base_ref)) || '' }}
|
|
run: EVALS_TIER=gate bun --no-install run scripts/test-paid-shards.ts --tier gate --emit-plan /tmp/paid-plan/manifest.json --slice-budget 420 --jobs 2 --max-parallel 20
|
|
|
|
# R2a/R2: reuse holds only when this head's own plan selects exactly the
|
|
# cases and trial plans the reused gate ran (selection fingerprint).
|
|
- name: Download a gate reuse candidate
|
|
if: github.event_name == 'pull_request'
|
|
continue-on-error: true
|
|
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
|
with:
|
|
name: gate-reuse-candidate
|
|
path: /tmp/gate-reuse
|
|
|
|
- name: Confirm gate verdict reuse (R2a/R2)
|
|
id: gate_reuse
|
|
if: github.event_name == 'pull_request'
|
|
run: |
|
|
mkdir -p /tmp/gate-reuse
|
|
bun --no-install run scripts/gate-reuse.ts confirm /tmp/gate-reuse /tmp/paid-plan/manifest.json --out /tmp/paid-plan \
|
|
--github-output "$GITHUB_OUTPUT" >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
# CEO-13: the mode, case counts and, on fallback, every file that
|
|
# restored the full gate with the edit that narrows it next time.
|
|
- name: Summarize PR coverage
|
|
run: bun --no-install run scripts/test-pr-profile.ts summary /tmp/paid-plan/manifest.json >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
- name: Derive the executor matrix from the plan
|
|
id: matrix
|
|
run: |
|
|
echo "slices=$(jq -c '[range(1; .sliceCount + 1)]' /tmp/paid-plan/manifest.json)" >> "$GITHUB_OUTPUT"
|
|
echo "slice_timeouts=$(jq -ec '[0] + .plan.sliceCiTimeoutMinutes' /tmp/paid-plan/manifest.json)" >> "$GITHUB_OUTPUT"
|
|
|
|
# The whole store (recovered receipts and the checkpoint included)
|
|
# travels to slices-report, which saves it with this run's receipts, so
|
|
# a receipt reuse turned off for this run is not lost for the next.
|
|
- name: Carry the receipt store to the report
|
|
if: github.event_name == 'pull_request'
|
|
run: mkdir -p /tmp/paid-plan/store && cp -a /tmp/gstack-eval-input-cache/. /tmp/paid-plan/store/
|
|
|
|
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
|
with:
|
|
name: paid-plan
|
|
path: |
|
|
/tmp/paid-plan/manifest.json
|
|
/tmp/paid-plan/receipts
|
|
/tmp/paid-plan/store
|
|
/tmp/paid-plan/gate-reuse.json
|
|
retention-days: 30
|
|
|
|
eval-slices:
|
|
runs-on: ubicloud-standard-8
|
|
needs: [build-image, plan-slices]
|
|
env:
|
|
EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-eval-slices-${{ matrix.slice }}
|
|
# !cancelled(), not always(): still runs when build-image was skipped
|
|
# (image already published), but a newer push's cancel-in-progress stops
|
|
# it instead of letting a superseded run finish its paid slices first.
|
|
# A confirmed verdict reuse (R2a/R2) runs no paid slice.
|
|
if: ${{ !cancelled() && needs.build-image.result == 'success' && needs.plan-slices.result == 'success' && needs.plan-slices.outputs.gate_reuse != 'true' }}
|
|
# The planner packs ~7 minutes of recorded work per slice (EVALS_JOBS=2 x
|
|
# EVALS_CONCURRENCY=2 per runner, never the old 40-way per-row fan-out
|
|
# that queued claude session STARTUP behind 39 siblings). Each slice's job
|
|
# timeout is its own ceiling (W2c): max(2x budget, its longest shard wall,
|
|
# the overlay envelope) + 20 minutes; the executor stops at the slice
|
|
# deadline 5 minutes earlier and reports not_run/hung shards.
|
|
timeout-minutes: ${{ fromJSON(needs.plan-slices.outputs.slice_timeouts)[matrix.slice] }}
|
|
permissions:
|
|
contents: read
|
|
packages: read
|
|
container:
|
|
image: ${{ needs.build-image.outputs.image-tag }}
|
|
credentials:
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
options: --user runner
|
|
strategy:
|
|
fail-fast: false
|
|
max-parallel: 20
|
|
matrix:
|
|
slice: ${{ fromJSON(needs.plan-slices.outputs.slices) }}
|
|
steps:
|
|
- name: Record the job start for the slice deadline
|
|
run: echo "GSTACK_SLICE_JOB_STARTED_AT=$(date +%s)" >> "$GITHUB_ENV"
|
|
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
with:
|
|
# Full history: files with SELF-derived selection (the LLM-judge
|
|
# map, routing) walk git at module load, and selection is
|
|
# fail-closed on git errors — a shallow checkout crashed those
|
|
# shards on the lane's first live run ("ambiguous argument
|
|
# 'main...HEAD'"). The manifest still governs WHICH shards run.
|
|
fetch-depth: 0
|
|
persist-credentials: false
|
|
|
|
- name: Fix bun temp
|
|
uses: ./.github/actions/fix-bun-temp
|
|
|
|
- name: Restore deps
|
|
uses: ./.github/actions/restore-deps
|
|
|
|
- run: bun run build
|
|
|
|
# Any slice can host a PTY smoke, so the seed/registration steps run
|
|
# UNCONDITIONALLY (both are idempotent) — the old matrix keyed them on
|
|
# matrix.suite.name, which a sliced lane cannot do. The register
|
|
# composite carries the fail-fast dangling-symlink/frontmatter
|
|
# verification loop, so a moved skill target fails HERE in seconds,
|
|
# not as a wedged PTY session at the shard wall.
|
|
- name: Seed claude interactive config
|
|
uses: ./.github/actions/seed-claude-config
|
|
with:
|
|
anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
|
|
- name: Register gstack skills for PTY smokes
|
|
uses: ./.github/actions/register-gstack-skills
|
|
|
|
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
|
with:
|
|
name: paid-plan
|
|
path: /tmp/paid-plan
|
|
|
|
# Receipts come only from the plan (this PR's store, filtered once by the
|
|
# planner); new receipts land beside the slice results and the report
|
|
# merges them into the next store.
|
|
- name: Seed this slice's receipts from the plan
|
|
run: |
|
|
mkdir -p /tmp/paid-slice-results/receipts
|
|
if [ -d /tmp/paid-plan/receipts ]; then cp -a /tmp/paid-plan/receipts/. /tmp/paid-slice-results/receipts/; fi
|
|
|
|
# The image ships the pinned Codex CLI off PATH and without a
|
|
# credential; codex does not read OPENAI_API_KEY from the environment.
|
|
# Log it in from the secret (same-repo runs only) under a non-default
|
|
# home. The shard runner hands that home and the CLI to the Codex files
|
|
# only (scripts/lib/paid-cases.ts scopeCodexAccess); every other paid case sees no
|
|
# Codex, as before. Without a key the Codex cases fail; they never pass.
|
|
- name: Authenticate the Codex CLI
|
|
shell: bash
|
|
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
|
|
env:
|
|
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
run: |
|
|
set -euo pipefail
|
|
codex="${GSTACK_CI_CODEX_BIN_DIR:?}/codex"
|
|
export CODEX_HOME="$HOME/.gstack-ci-codex"
|
|
mkdir -p "$CODEX_HOME"
|
|
echo "GSTACK_CI_CODEX_HOME=$CODEX_HOME" >> "$GITHUB_ENV"
|
|
if [ -z "${OPENAI_API_KEY:-}" ]; then
|
|
echo "::warning::OPENAI_API_KEY is unavailable; Codex eval cases cannot authenticate and will fail."
|
|
exit 0
|
|
fi
|
|
printenv OPENAI_API_KEY | "$codex" login --with-api-key
|
|
"$codex" login status >/dev/null 2>&1
|
|
|
|
- name: Run slice ${{ matrix.slice }}
|
|
env:
|
|
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
|
|
PLAYWRIGHT_BROWSERS_PATH: /opt/playwright-browsers
|
|
EVALS_JOBS: "2"
|
|
EVALS_CONCURRENCY: "2"
|
|
GSTACK_EVAL_DIR: /tmp/paid-slice-results
|
|
EVALS_CACHE_DIR: /tmp/paid-slice-results/receipts
|
|
EVALS_CACHE_REPOSITORY: ${{ github.repository }}
|
|
EVALS_CACHE_PR: ${{ github.event.pull_request.number || inputs.pr_receipts }}
|
|
EVALS_CACHE_RUNTIME_ID: ${{ needs.build-image.outputs.runtime-id }}
|
|
run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --plan /tmp/paid-plan/manifest.json --slice ${{ matrix.slice }}
|
|
|
|
# Attempt-scoped: a re-run attempt's trials are reported under that
|
|
# attempt and never replace (or collide with) the first attempt's.
|
|
- name: Upload slice results
|
|
if: always()
|
|
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
|
with:
|
|
name: paid-slice-${{ matrix.slice }}-a${{ github.run_attempt }}
|
|
path: /tmp/paid-slice-results
|
|
retention-days: 90
|
|
|
|
- name: Upload native capture evidence
|
|
if: always()
|
|
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
|
with:
|
|
name: native-captures-${{ env.EVALS_RUN_ID }}
|
|
include-hidden-files: true
|
|
path: |
|
|
~/.gstack/projects/*/e2e-runs
|
|
~/.gstack/projects/*/evals/qa-callers
|
|
~/.gstack-dev/e2e-runs
|
|
~/.gstack-dev/evals/qa-callers
|
|
if-no-files-found: ignore
|
|
retention-days: 90
|
|
|
|
# The spooled per-shard full logs — a red weekly/PR lane three weeks
|
|
# later needs more than a summary line.
|
|
# always(), not failure(): a failed behavior trial is a verdict and no
|
|
# longer reds its runner, but its full log is the diagnosis evidence.
|
|
- name: Upload shard logs
|
|
if: always()
|
|
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
|
with:
|
|
name: paid-logs-slice-${{ matrix.slice }}-a${{ github.run_attempt }}
|
|
include-hidden-files: true
|
|
# The Fix-bun-temp step points TMPDIR at /home/runner/.cache, so the
|
|
# runner's spool lands THERE, not /tmp — the original /tmp glob
|
|
# uploaded nothing and a red slice's diagnostics were unreachable.
|
|
path: |
|
|
/home/runner/.cache/gstack-paid-shard-*.log
|
|
/tmp/gstack-paid-shard-*.log
|
|
if-no-files-found: ignore
|
|
retention-days: 30
|
|
|
|
slices-report:
|
|
# GitHub-hosted, like recover-receipts: Ubicloud runners send actions/cache
|
|
# to Ubicloud's own transparent cache, which a GitHub-hosted restore never
|
|
# sees, so a store saved on Ubicloud was never restored (GSTA-48: every
|
|
# restore missed, e.g. #3253's restamp gate 3 minutes after a saved store).
|
|
runs-on: ubuntu-24.04
|
|
needs: [plan-slices, eval-slices]
|
|
# !cancelled(): the report must run (and FAIL) when an executor died — a
|
|
# missing slice artifact reading as green is the class this lane kills —
|
|
# but a run superseded by a newer push stops here.
|
|
if: ${{ !cancelled() && needs.plan-slices.result == 'success' }}
|
|
timeout-minutes: 5
|
|
# contents:read ONLY — this job executes the PR-authored reconcile
|
|
# runner from the PR checkout, so it
|
|
# must never hold a write-scoped token. The PR comment lives in the
|
|
# separate slices-comment job below, which runs NO repo code: a
|
|
# $GITHUB_ENV/BASH_ENV persistence trick is job-scoped, so the split is
|
|
# the trust boundary (codex adversarial finding, 2026-08-31 — the old
|
|
# matrix-era report job had this separation and the consolidation had
|
|
# regressed it).
|
|
permissions:
|
|
contents: read
|
|
outputs:
|
|
reconcile-exit: ${{ steps.reconcile.outputs.exit || steps.reused.outputs.exit }}
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
|
with:
|
|
bun-version: 1.4.2
|
|
|
|
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
|
with:
|
|
name: paid-plan
|
|
path: /tmp/paid-report
|
|
|
|
# One directory per attempt-scoped slice artifact (no merge): shard
|
|
# records can never overwrite each other, and the report keeps the
|
|
# first attempt's verdict.
|
|
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
|
with:
|
|
pattern: paid-slice-*
|
|
path: /tmp/paid-report
|
|
|
|
- name: Reconcile slices against the manifest (fail-closed)
|
|
id: reconcile
|
|
if: needs.plan-slices.outputs.gate_reuse != 'true'
|
|
run: |
|
|
set +e
|
|
EVALS_TIER=gate bun --no-install run scripts/test-paid-shards.ts --tier gate --report /tmp/paid-report | tee /tmp/report.txt
|
|
# PIPESTATUS[0], NOT $?: GitHub's default run-step shell is
|
|
# `bash -e {0}` with NO pipefail, so $? after the pipe is tee's
|
|
# exit (always 0) — the fail-closed gate was silently fail-open
|
|
# (caught by the ship review army; the wiring test now pins this).
|
|
echo "exit=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT"
|
|
|
|
# R2a/R2: the planner confirmed a reuse, so no slice ran; the verdict is
|
|
# the reused gate's, printed with which run, which head and why. A
|
|
# missing or malformed reuse record exits 1 (fail-closed).
|
|
- name: Report the reused gate verdict (R2a/R2)
|
|
id: reused
|
|
if: needs.plan-slices.outputs.gate_reuse == 'true'
|
|
run: |
|
|
set +e
|
|
bun --no-install run scripts/gate-reuse.ts report /tmp/paid-report/gate-reuse.json --repo "$GITHUB_REPOSITORY" > /tmp/report.txt
|
|
code=$?
|
|
cat /tmp/report.txt
|
|
echo "exit=$code" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Summarize PR coverage and reuse
|
|
if: always()
|
|
run: bun --no-install run scripts/test-pr-profile.ts summary /tmp/paid-report/manifest.json /tmp/paid-report/collector-outcomes.json >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
- name: Stamp trial history series
|
|
if: always()
|
|
run: |
|
|
if [ -f /tmp/paid-report/trial-outcomes.jsonl ]; then
|
|
bun --no-install run scripts/eval-trial-series.ts /tmp/paid-report/trial-outcomes.jsonl
|
|
fi
|
|
|
|
# One merged receipt store per run: the carried store (recovered
|
|
# receipts and checkpoint included), the plan's shipped set, every
|
|
# slice's new pass and negative receipts, and the report's panel and
|
|
# negative receipts. Saved
|
|
# last, so the next planner restores it as the newest prefix match.
|
|
- name: Merge this run's receipts
|
|
if: always() && github.event_name == 'pull_request'
|
|
run: |
|
|
bun --no-install run scripts/e2e-shard-reuse.ts merge /tmp/gstack-eval-input-cache /tmp/paid-report/store \
|
|
/tmp/paid-report/receipts /tmp/paid-report/report-receipts /tmp/paid-report/paid-slice-*/receipts
|
|
|
|
- name: Save this PR's verified judge and E2E results
|
|
if: always() && github.event_name == 'pull_request'
|
|
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
|
|
with:
|
|
path: /tmp/gstack-eval-input-cache
|
|
key: eval-input-v1-${{ github.repository_id }}-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }}-merged
|
|
|
|
- name: Upload reconciliation output for the comment job
|
|
if: always()
|
|
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
|
with:
|
|
name: report-verdict-a${{ github.run_attempt }}
|
|
path: |
|
|
/tmp/report.txt
|
|
/tmp/paid-report/collector-outcomes.json
|
|
/tmp/paid-report/report-summary.md
|
|
/tmp/paid-report/report-receipts
|
|
if-no-files-found: ignore
|
|
retention-days: 30
|
|
|
|
# R2a/R2: what this executed run proved, read by a later push. The base
|
|
# is the tested merge commit's first parent. Reused runs bind nothing,
|
|
# so reuse never chains off a reuse.
|
|
- name: Bind this run's verdict for reuse (R2a/R2)
|
|
if: always() && github.event_name == 'pull_request' && needs.plan-slices.outputs.gate_reuse != 'true'
|
|
env:
|
|
PR_NUMBER: ${{ github.event.pull_request.number }}
|
|
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
|
RECONCILE_EXIT: ${{ steps.reconcile.outputs.exit }}
|
|
run: |
|
|
bun --no-install run scripts/gate-reuse.ts bind /tmp/paid-report --out /tmp/gate-binding --repo "$GITHUB_REPOSITORY" \
|
|
--pr "$PR_NUMBER" --run-id "$GITHUB_RUN_ID" --run-attempt "$GITHUB_RUN_ATTEMPT" --head-sha "$HEAD_SHA" \
|
|
--tested-sha "$GITHUB_SHA" --profile "$EVALS_PROFILE" --reconcile-exit "$RECONCILE_EXIT" >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
- name: Upload the gate binding
|
|
if: always() && github.event_name == 'pull_request' && needs.plan-slices.outputs.gate_reuse != 'true'
|
|
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
|
with:
|
|
name: gate-binding-a${{ github.run_attempt }}
|
|
path: /tmp/gate-binding/gate-binding.json
|
|
if-no-files-found: ignore
|
|
retention-days: 30
|
|
|
|
- name: Upload trial outcomes for pass-rate history
|
|
if: always()
|
|
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
|
with:
|
|
name: trial-outcomes-pr-a${{ github.run_attempt }}
|
|
path: /tmp/paid-report/trial-outcomes.jsonl
|
|
if-no-files-found: ignore
|
|
retention-days: 90
|
|
|
|
- name: Fail the workflow when reconciliation failed
|
|
if: (steps.reconcile.outputs.exit || steps.reused.outputs.exit) != '0'
|
|
run: exit 1
|
|
|
|
# PR comment in its OWN job with the write token and ZERO repo code: no
|
|
# checkout, no bun install — only downloaded artifacts, jq, and gh. See the
|
|
# trust-boundary note on slices-report.
|
|
slices-comment:
|
|
runs-on: ubicloud-standard-2
|
|
needs: slices-report
|
|
if: ${{ !cancelled() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && needs.slices-report.result != 'skipped' }}
|
|
timeout-minutes: 5
|
|
permissions:
|
|
pull-requests: write
|
|
# The comment upsert calls the REST `/issues/{n}/comments` endpoints
|
|
# (gh api ... issues/comments). With GITHUB_TOKEN those are gated by the
|
|
# `issues` permission, not `pull-requests` (#1802 CI fix).
|
|
issues: write
|
|
steps:
|
|
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
|
with:
|
|
name: paid-plan
|
|
path: /tmp/paid-report
|
|
|
|
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
|
with:
|
|
name: report-verdict-a${{ github.run_attempt }}
|
|
path: /tmp/verdict
|
|
continue-on-error: true
|
|
|
|
# Every count, verdict and failure line comes from the read-only report
|
|
# job's collector-outcomes v2 (panelVerdict() ran there); this job runs
|
|
# no repo code and never recomputes a verdict. Keeps the
|
|
# "## E2E Evals" marker so the upsert keeps updating the same comment.
|
|
# Runs even when reconciliation failed — a red lane on the PR is the point.
|
|
- name: Post PR comment
|
|
env:
|
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
RECONCILE_EXIT: ${{ needs.slices-report.outputs.reconcile-exit }}
|
|
run: |
|
|
# shellcheck disable=SC2086,SC2059
|
|
# R2a/R2: a confirmed verdict reuse ran no case; say which run,
|
|
# which head and why, with the receipt or own-diff signature.
|
|
REUSE=/tmp/paid-report/gate-reuse.json
|
|
if [ "${RECONCILE_EXIT:-1}" = "0" ] && [ -f "$REUSE" ] \
|
|
&& jq -e '.schema == 1 and (.kind == "restamp-only" or .kind == "main-moved")' "$REUSE" >/dev/null 2>&1; then
|
|
BODY=$(jq -r --arg repo "$GITHUB_REPOSITORY" '"## E2E Evals: ✅ PASS (reused: \(if .kind == "restamp-only" then "restamp-only, R2a" else "main moved, R2" end))\n\nNo paid case ran on `\(.current_head)`. It carries the GREEN verdict of [run \(.reused_run_id)](https://github.com/\($repo)/actions/runs/\(.reused_run_id)) at `\(.gated_head)`, because \(.why).\n\n```\n\(if .kind == "restamp-only" then "TREE: same-modulo-stamps receipt=\(.receipt_sha256)" else "OWN DIFF: identical sha256=\(.own_diff_sha256) (merge-base \(.gated_merge_base) -> \(.merge_base))" end)\nbase: \(.gated_base) -> \(.base_sha)\nselection: \(.selection_sha256) (identical to the reused gate)\n```\n\n---\n*Verdict reuse (R2a restamp-only, R2 main moved): a changed own diff, a non-stamp byte, a changed selection, a newer red or unbound gate, or a missing binding or receipt runs the full gate; the evals-fresh label opts out. Free Tests and every other check still ran on this head.*"' "$REUSE")
|
|
else
|
|
TOTAL=0; PASSED=0; FAILED=0; MANUAL=0; EXECUTED=0; REUSED=0; COST="0"
|
|
SUITE_LINES=""
|
|
VERIFIED=/tmp/verdict/paid-report/collector-outcomes.json
|
|
if ! jq -e '
|
|
. as $summary |
|
|
.version == 2 and (.files | type == "array") and (.totals | type == "object") and
|
|
(.headline | type == "array") and (.failures | type == "array") and (.panels | type == "array") and
|
|
(.verdict.verdict == "GREEN" or .verdict.verdict == "RED") and
|
|
([.files[] | .total == (.passed + .failed + .manual_accepted) and
|
|
(.total == (.executed + .reused)) and
|
|
([.total,.passed,.failed,.manual_accepted,.executed,.reused,.attempts,.flaky] | all(. >= 0 and (floor == .))) ] | all) and
|
|
(.totals | .total == (.passed + .failed + .manual_accepted) and .total == (.executed + .reused)) and
|
|
(["total","passed","failed","manual_accepted","executed","reused","attempts","flaky"] |
|
|
all(. as $key | ([$summary.files[] | .[$key]] | add // 0) == $summary.totals[$key]))
|
|
' "$VERIFIED" >/dev/null 2>&1; then
|
|
VERIFIED=""
|
|
echo 'Verified report summary unavailable; no verdict, counts or manual acceptance can be shown.'
|
|
fi
|
|
HEADLINE='(no verified report headline)'
|
|
FAILURES=""
|
|
if [ -n "$VERIFIED" ]; then
|
|
while IFS=$'\t' read -r _FILE T P F M _FLAKY EX RE _ATTEMPTS C TIER SHARD; do
|
|
[ "$T" -eq 0 ] && continue
|
|
TOTAL=$((TOTAL + T)); PASSED=$((PASSED + P)); FAILED=$((FAILED + F))
|
|
MANUAL=$((MANUAL + M))
|
|
EXECUTED=$((EXECUTED + EX)); REUSED=$((REUSED + RE))
|
|
COST=$(echo "$COST + $C" | bc)
|
|
STATUS_ICON="✅"
|
|
[ "$M" -gt 0 ] && STATUS_ICON="⚠ manual/unscored"
|
|
[ "$F" -gt 0 ] && STATUS_ICON="❌"
|
|
SUITE_LINES="${SUITE_LINES}| ${TIER}/${SHARD} | ${P}/${T} | ${M} | ${EX} | ${RE} | ${STATUS_ICON} | \$${C} |\n"
|
|
done < <(jq -r '.files[] | [.file,.total,.passed,.failed,.manual_accepted,.flaky,.executed,.reused,.attempts,.cost,.tier,.shard] | @tsv' "$VERIFIED")
|
|
# Report-sanitized lines (no @-mentions, one capped line each), fenced here.
|
|
HEADLINE=$(jq -r '.headline[]' "$VERIFIED")
|
|
FAILURES=$(jq -r '.failures[]' "$VERIFIED")
|
|
fi
|
|
|
|
COVERAGE=$(jq -r '"Profile: \(.profile // "full") / mode `\(.prCoverage.mode // "broad")`; selected behaviors: \(.selection.e2e | if . == null then "all" else length end), judges: \(.selection.judges | if . == null then "all" else length end). Deferred to scheduled/release coverage: \(.prCoverage.deferred // [] | length) behaviors and \(.prCoverage.deferredPromptFiles // [] | length) changed prompt files. Deferred checks did not run and receive no PR-pass credit."' /tmp/paid-report/manifest.json) || COVERAGE='Coverage manifest unavailable; no coverage claim.'
|
|
# CEO-13: on fallback, name every file that restored the full gate and its fix.
|
|
FALLBACK=$(jq -r 'select(.prCoverage.mode == "full-fallback") | .prCoverage as $c | ($c.unknownFiles // [])[] as $f
|
|
| (($c.unknownFileLabels // []) | map(select(.file == $f)) | first // {label: "unlabeled", fix: "see docs/TESTING_INTERNALS.md#pr-paid-lane-fallback"}) as $l
|
|
| "- `\($f)` (\($l.label)): \($l.fix)"' /tmp/paid-report/manifest.json 2>/dev/null | head -40 | sed 's/@/@\xe2\x80\x8b/g') || FALLBACK=''
|
|
if [ -n "$FALLBACK" ]; then COVERAGE="${COVERAGE}
|
|
|
|
Full gate restored by:
|
|
${FALLBACK}"; fi
|
|
# Dependents mode: unmapped files placed by derived references, with their dependent counts.
|
|
DERIVED=$(jq -r '(.prCoverage.derivedFiles // [])[:40][] | "- `\(.file)`: \(.e2e) case(s), \(.judges) judge(s)"' /tmp/paid-report/manifest.json 2>/dev/null | sed 's/@/@\xe2\x80\x8b/g') || DERIVED=''
|
|
if [ -n "$DERIVED" ]; then COVERAGE="${COVERAGE}
|
|
|
|
Derived dependents (scripts/pr-dependencies.ts):
|
|
${DERIVED}"; fi
|
|
|
|
STATUS="✅ PASS"
|
|
if [ "${RECONCILE_EXIT:-1}" != "0" ] || [ "$FAILED" -gt 0 ] \
|
|
|| { [ -n "$VERIFIED" ] && [ "$(jq -r '.verdict.verdict' "$VERIFIED")" != "GREEN" ]; }; then STATUS="❌ FAIL"; fi
|
|
if [ "$STATUS" = '✅ PASS' ] && [ "$MANUAL" -gt 0 ]; then STATUS='⚠ MANUAL ACCEPTED (unscored)'; fi
|
|
if [ -z "$VERIFIED" ]; then STATUS='❌ FAIL (verified report unavailable)'; fi
|
|
|
|
BODY="## E2E Evals: ${STATUS}
|
|
|
|
\`\`\`
|
|
${HEADLINE}
|
|
\`\`\`
|
|
|
|
**${EXECUTED} executed, ${REUSED} reused** rule/judge records | **${MANUAL} manual accepted (unscored; no score-cache credit)** | **\$${COST}** rule/judge cost | reconcile exit: ${RECONCILE_EXIT:-missing}
|
|
|
|
${COVERAGE}
|
|
|
|
<details><summary>Rule and judge shards</summary>
|
|
|
|
| Shard | Automated result | Manual/unscored | Executed | Reused | Status | Cost |
|
|
|-------|------------------|-----------------|----------|--------|--------|------|
|
|
$(echo -e "$SUITE_LINES")
|
|
</details>
|
|
|
|
<details><summary>Fail-closed reconciliation</summary>
|
|
|
|
\`\`\`
|
|
$(tail -c 4000 /tmp/verdict/report.txt 2>/dev/null | sed 's/@/@\xe2\x80\x8b/g' || echo '(no reconciliation output)')
|
|
\`\`\`
|
|
</details>
|
|
|
|
---
|
|
*Sliced lane: planner → duration-packed executors → fail-closed report. Behavior cases run a pre-registered 3-trial panel (PASS at 2/3 with no contract violation); a PASS 2/3 is shown with its failed trial, never as a clean pass. Reused results retain their original provenance and expiry.*"
|
|
|
|
if [ -n "$FAILURES" ]; then
|
|
BODY="${BODY}
|
|
|
|
### Failures and split verdicts
|
|
\`\`\`
|
|
${FAILURES}
|
|
\`\`\`"
|
|
fi
|
|
|
|
fi
|
|
|
|
COMMENT_ID=$(gh api repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments \
|
|
--jq '.[] | select(.body | startswith("## E2E Evals")) | .id' | tail -1)
|
|
|
|
if [ -n "$COMMENT_ID" ]; then
|
|
gh api "repos/${{ github.repository }}/issues/comments/${COMMENT_ID}" \
|
|
-X PATCH -f body="$BODY"
|
|
else
|
|
# REST, not gh's pr-comment subcommand: this job runs with NO
|
|
# checkout (the token/exec split), and that subcommand resolves
|
|
# the repo FROM git — it dies with "not a git repository" here.
|
|
gh api "repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments" \
|
|
-X POST -f body="$BODY"
|
|
fi
|