mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-05 16:34:00 +01:00
Merge remote-tracking branch 'origin/main' into capy/merge-train-workflow-side
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> # Conflicts: # CHANGELOG.md # scripts/harness-version.json
This commit is contained in:
commit
e56e3aa8e2
30 files changed
+1533
-260
No files matched your search
@@ -15,6 +15,10 @@ name: Periodic Evals
|
||||
# diff-billed, so without it the full gate census might never execute
|
||||
# anywhere); the hollow-shard guard (exit 0 + zero executed tests under
|
||||
# EVALS_ALL fails) makes both lanes census-health checks, not just test runs.
|
||||
# The gate census is a matrix of arms (scripts/census-arms.ts): each arm is a
|
||||
# host and model with its own plan. Monday runs claude-opus (the capture
|
||||
# model) and codex-sol (gpt-6.1-sol on the Codex host, adapter cases only);
|
||||
# the monthly cron runs only the census, with claude-fable and codex-astra.
|
||||
# eval-bg (scripts/eval-bg.ts) finds its dispatch by the nonce in the run name.
|
||||
run-name: ${{ inputs.nonce && format('Periodic Evals [eval-bg {0}]', inputs.nonce) || 'Periodic Evals' }}
|
||||
on:
|
||||
@@ -23,6 +27,9 @@ on:
|
||||
# Daily, for autoplan-schema-canary alone (latest Claude Code vs the /autoplan
|
||||
# guard's payload assumptions); every census job below skips this schedule.
|
||||
- cron: '30 7 * * *'
|
||||
# Monthly, the census jobs alone with the claude-fable and codex-astra arms
|
||||
# (the 1st; never combined with a day of week, which cron would OR).
|
||||
- cron: '0 6 1 * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
redispatch_of:
|
||||
@@ -30,9 +37,13 @@ on:
|
||||
type: string
|
||||
default: ''
|
||||
include_gate_census:
|
||||
description: 'Branch dispatch only: also run the weekly gate census (default: periodic lane only; gate cases validate through an evals.yml dispatch with evals_all)'
|
||||
description: 'Branch dispatch only: also run the weekly gate census arms (default: periodic lane only; gate cases validate through an evals.yml dispatch with evals_all)'
|
||||
type: boolean
|
||||
default: false
|
||||
census_arms:
|
||||
description: 'Run only the gate census, with these arms (comma-separated: claude-opus, codex-sol, claude-fable, codex-astra; scripts/census-arms.ts). Empty keeps the periodic lane.'
|
||||
type: string
|
||||
default: ''
|
||||
expected_sha:
|
||||
description: 'eval-bg: the revision the dispatcher tested; the run fails fast when the ref has moved'
|
||||
type: string
|
||||
@@ -51,7 +62,7 @@ on:
|
||||
# runs in its own group so it never cancels the run that dispatched it; both
|
||||
# runs are reported.
|
||||
concurrency:
|
||||
group: evals-periodic-${{ github.ref }}-${{ github.event_name }}${{ inputs.redispatch_of && format('-redispatch-{0}', inputs.redispatch_of) || '' }}${{ (github.event.schedule == '30 7 * * *' || inputs.schema_canary_only) && '-schema-canary' || '' }}
|
||||
group: evals-periodic-${{ github.ref }}-${{ github.event_name }}${{ inputs.redispatch_of && format('-redispatch-{0}', inputs.redispatch_of) || '' }}${{ (github.event.schedule == '30 7 * * *' || inputs.schema_canary_only) && '-schema-canary' || '' }}${{ github.event.schedule == '0 6 1 * *' && '-monthly' || '' }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
@@ -126,8 +137,9 @@ jobs:
|
||||
periodic_slices: ${{ steps.periodic-matrix.outputs.slices }}
|
||||
periodic_codex_slices: ${{ steps.periodic-matrix.outputs.codex_slices }}
|
||||
periodic_slice_timeouts: ${{ steps.periodic-matrix.outputs.slice_timeouts }}
|
||||
gate_slices: ${{ steps.gate-matrix.outputs.slices }}
|
||||
gate_slice_timeouts: ${{ steps.gate-matrix.outputs.slice_timeouts }}
|
||||
periodic_lane: ${{ steps.census.outputs.periodic_lane }}
|
||||
census_arms: ${{ steps.census.outputs.arms }}
|
||||
gate_matrix: ${{ steps.gate-matrix.outputs.gate_matrix }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
@@ -147,7 +159,18 @@ jobs:
|
||||
with:
|
||||
bun-version: 1.4.2
|
||||
|
||||
# Census arms (scripts/census-arms.ts): which host/model arms run the
|
||||
# gate census, and whether the periodic lane runs beside them.
|
||||
- name: Select the census arms
|
||||
id: census
|
||||
env:
|
||||
SCHEDULE: ${{ github.event.schedule }}
|
||||
CENSUS_ARMS: ${{ inputs.census_arms }}
|
||||
INCLUDE_GATE_CENSUS: ${{ github.ref == 'refs/heads/main' || inputs.include_gate_census }}
|
||||
run: bun --no-install run scripts/census-arms.ts --select >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Emit run manifest (ALL periodic tests minus reasoned excludes)
|
||||
if: steps.census.outputs.periodic_lane == 'true'
|
||||
env:
|
||||
EVALS_ALL: "1"
|
||||
run: EVALS_TIER=periodic bun --no-install run scripts/test-paid-shards.ts --tier periodic --emit-plan /tmp/paid-plan/manifest.json --slice-budget 420 --jobs 2 --max-parallel 38
|
||||
@@ -156,6 +179,7 @@ jobs:
|
||||
# container executor runs every other slice.
|
||||
- name: Derive the periodic executor matrix from the plan
|
||||
id: periodic-matrix
|
||||
if: steps.census.outputs.periodic_lane == 'true'
|
||||
run: |
|
||||
{
|
||||
echo "slices=$(jq -c '[range(1; .sliceCount + 1)] - .plan.codexSlices' /tmp/paid-plan/manifest.json)"
|
||||
@@ -164,26 +188,38 @@ jobs:
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
if: steps.census.outputs.periodic_lane == 'true'
|
||||
with:
|
||||
name: paid-plan
|
||||
path: /tmp/paid-plan/manifest.json
|
||||
retention-days: 30
|
||||
|
||||
- name: Emit gate census manifest (ALL gate tests)
|
||||
# One plan per arm: the Codex host plans only its adapter cases and
|
||||
# records every Claude-only case as excluded with its reason.
|
||||
- name: Emit gate census manifests (ALL gate tests, one plan per arm)
|
||||
if: steps.census.outputs.arms != '[]'
|
||||
env:
|
||||
EVALS_ALL: "1"
|
||||
run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --emit-plan /tmp/gate-census-plan/manifest.json --slice-budget 420 --jobs 2 --skip-judges --max-parallel 20
|
||||
|
||||
- name: Derive the gate census executor matrix from the plan
|
||||
id: gate-matrix
|
||||
ARMS: ${{ steps.census.outputs.arms }}
|
||||
run: |
|
||||
echo "slices=$(jq -c '[range(1; .sliceCount + 1)]' /tmp/gate-census-plan/manifest.json)" >> "$GITHUB_OUTPUT"
|
||||
echo "slice_timeouts=$(jq -ec '[0] + .plan.sliceCiTimeoutMinutes' /tmp/gate-census-plan/manifest.json)" >> "$GITHUB_OUTPUT"
|
||||
set -euo pipefail
|
||||
for arm in $(jq -r '.[]' <<<"$ARMS"); do
|
||||
mapfile -t arm_env < <(bun --no-install run scripts/census-arms.ts --env "$arm")
|
||||
env "${arm_env[@]}" EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --emit-plan "/tmp/gate-census-plan/$arm/manifest.json" --slice-budget 420 --jobs 2 --skip-judges --max-parallel 20
|
||||
done
|
||||
|
||||
# One executor per arm and planned slice, each with its slice's ceiling.
|
||||
- name: Derive the gate census executor matrix from the plans
|
||||
id: gate-matrix
|
||||
env:
|
||||
ARMS: ${{ steps.census.outputs.arms }}
|
||||
run: bun --no-install run scripts/census-arms.ts --matrix /tmp/gate-census-plan "$ARMS" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
if: steps.census.outputs.arms != '[]'
|
||||
with:
|
||||
name: gate-census-plan
|
||||
path: /tmp/gate-census-plan/manifest.json
|
||||
path: /tmp/gate-census-plan
|
||||
retention-days: 30
|
||||
|
||||
# autoplan-schema-canary: installs the latest published Claude Code with
|
||||
@@ -237,6 +273,7 @@ jobs:
|
||||
eval-slices:
|
||||
runs-on: ubicloud-standard-8
|
||||
needs: [build-image, plan-slices]
|
||||
if: ${{ needs.plan-slices.outputs.periodic_lane == 'true' }}
|
||||
env:
|
||||
EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-eval-slices-${{ matrix.slice }}
|
||||
# The planner packs ~7 minutes of recorded work per slice; each slice's job
|
||||
@@ -388,7 +425,7 @@ jobs:
|
||||
eval-codex-slices:
|
||||
runs-on: ubicloud-standard-8
|
||||
needs: [build-image, plan-slices]
|
||||
if: ${{ needs.plan-slices.outputs.periodic_codex_slices != '[]' }}
|
||||
if: ${{ needs.plan-slices.outputs.periodic_lane == 'true' && needs.plan-slices.outputs.periodic_codex_slices != '[]' }}
|
||||
env:
|
||||
EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-eval-slices-${{ matrix.slice }}
|
||||
timeout-minutes: ${{ fromJSON(needs.plan-slices.outputs.periodic_slice_timeouts)[matrix.slice] }}
|
||||
@@ -512,16 +549,18 @@ jobs:
|
||||
# blind spots rot invisibly. Census health, not selector correctness —
|
||||
# selector logic has free synthetic-diff contract tests.
|
||||
gate-census:
|
||||
name: gate-census-${{ matrix.arm }} (${{ matrix.slice }})
|
||||
runs-on: ubicloud-standard-8
|
||||
needs: [build-image, plan-slices]
|
||||
# A branch dispatch validates the periodic lane only; gate cases on a
|
||||
# branch go through an evals.yml dispatch with evals_all. Main (schedule
|
||||
# or dispatch) always runs the census; a branch opts in explicitly.
|
||||
if: ${{ github.ref == 'refs/heads/main' || inputs.include_gate_census }}
|
||||
# The planner picks the arms (scripts/census-arms.ts). A branch dispatch
|
||||
# validates the periodic lane only unless it sets include_gate_census or
|
||||
# census_arms; gate cases on a branch also go through an evals.yml
|
||||
# dispatch with evals_all. Main (schedule or dispatch) always runs it.
|
||||
if: ${{ needs.plan-slices.outputs.gate_matrix != '[]' }}
|
||||
env:
|
||||
EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-gate-census-${{ matrix.slice }}
|
||||
# Per-slice job ceiling from the packed plan (W2c/ENG-2), as for eval-slices.
|
||||
timeout-minutes: ${{ fromJSON(needs.plan-slices.outputs.gate_slice_timeouts)[matrix.slice] }}
|
||||
EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-gate-census-${{ matrix.arm }}-${{ matrix.slice }}
|
||||
# Per-slice job ceiling from the arm's packed plan (W2c/ENG-2), as for eval-slices.
|
||||
timeout-minutes: ${{ matrix.timeout }}
|
||||
permissions:
|
||||
contents: read
|
||||
packages: read
|
||||
@@ -536,7 +575,7 @@ jobs:
|
||||
fail-fast: false
|
||||
max-parallel: 20
|
||||
matrix:
|
||||
slice: ${{ fromJSON(needs.plan-slices.outputs.gate_slices) }}
|
||||
include: ${{ fromJSON(needs.plan-slices.outputs.gate_matrix) }}
|
||||
steps:
|
||||
- name: Record the job start for the slice deadline
|
||||
run: echo "GSTACK_SLICE_JOB_STARTED_AT=$(date +%s)" >> "$GITHUB_ENV"
|
||||
@@ -588,7 +627,15 @@ jobs:
|
||||
printenv OPENAI_API_KEY | "$codex" login --with-api-key
|
||||
"$codex" login status >/dev/null 2>&1
|
||||
|
||||
- name: Run gate census slice ${{ matrix.slice }}
|
||||
# The arm's host and model (EVALS_HOST, plus EVALS_MODEL or
|
||||
# GSTACK_CODEX_MODEL). The Codex host runs codex exec with
|
||||
# danger-full-access, so it needs no bubblewrap and runs in this container.
|
||||
- name: Select the census arm's host and model
|
||||
env:
|
||||
ARM: ${{ matrix.arm }}
|
||||
run: bun --no-install run scripts/census-arms.ts --env "$ARM" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Run gate census slice ${{ matrix.slice }} (${{ matrix.arm }})
|
||||
env:
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
@@ -597,13 +644,13 @@ jobs:
|
||||
EVALS_JOBS: "2"
|
||||
EVALS_CONCURRENCY: "2"
|
||||
GSTACK_EVAL_DIR: /tmp/gate-census-results
|
||||
run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --plan /tmp/gate-census-plan/manifest.json --slice ${{ matrix.slice }}
|
||||
run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --plan /tmp/gate-census-plan/${{ matrix.arm }}/manifest.json --slice ${{ matrix.slice }}
|
||||
|
||||
- name: Upload census results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: gate-census-${{ matrix.slice }}-a${{ github.run_attempt }}
|
||||
name: gate-census-${{ matrix.slice }}-a${{ github.run_attempt }}-${{ matrix.arm }}
|
||||
path: /tmp/gate-census-results
|
||||
retention-days: 90
|
||||
|
||||
@@ -638,6 +685,9 @@ jobs:
|
||||
actions: read
|
||||
outputs:
|
||||
redispatch: ${{ steps.verdict.outputs.redispatch }}
|
||||
# A census-only run (monthly cron or census_arms dispatch) re-dispatches
|
||||
# with the same arms; empty keeps the weekly selection.
|
||||
redispatch_census_arms: ${{ needs.plan-slices.outputs.periodic_lane != 'true' && join(fromJSON(needs.plan-slices.outputs.census_arms), ',') || '' }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
@@ -648,6 +698,7 @@ jobs:
|
||||
bun-version: 1.4.2
|
||||
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
||||
if: needs.plan-slices.outputs.periodic_lane == 'true'
|
||||
with:
|
||||
name: paid-plan
|
||||
path: /tmp/paid-report
|
||||
@@ -655,25 +706,35 @@ jobs:
|
||||
# One directory per attempt-scoped slice artifact (no merge): shard
|
||||
# records never overwrite each other and the first attempt decides.
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
||||
if: needs.plan-slices.outputs.periodic_lane == 'true'
|
||||
with:
|
||||
pattern: paid-slice-*
|
||||
path: /tmp/paid-report
|
||||
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
||||
if: needs.plan-slices.outputs.gate_matrix != '[]'
|
||||
with:
|
||||
name: gate-census-plan
|
||||
path: /tmp/gate-census-report
|
||||
path: /tmp/gate-census-plan
|
||||
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
||||
if: needs.plan-slices.outputs.gate_matrix != '[]'
|
||||
with:
|
||||
pattern: gate-census-[0-9]*
|
||||
path: /tmp/gate-census-report
|
||||
path: /tmp/gate-census-slices
|
||||
|
||||
- name: Reconcile slices against the manifest (fail-closed)
|
||||
id: reconcile
|
||||
if: always()
|
||||
env:
|
||||
PERIODIC_LANE: ${{ needs.plan-slices.outputs.periodic_lane }}
|
||||
run: |
|
||||
set +e
|
||||
if [ "$PERIODIC_LANE" != "true" ]; then
|
||||
echo "periodic lane skipped: census-only run (monthly cron or census_arms dispatch)" | tee /tmp/report.txt
|
||||
echo "exit=0" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
EVALS_TIER=periodic bun --no-install run scripts/test-paid-shards.ts --tier periodic --report /tmp/paid-report | tee /tmp/report.txt
|
||||
# PIPESTATUS[0], NOT $?: GitHub's default run-step shell is
|
||||
# `bash -e {0}` with NO pipefail, so $? after the pipe is tee's
|
||||
@@ -686,20 +747,34 @@ jobs:
|
||||
if: always()
|
||||
env:
|
||||
GATE_CENSUS: ${{ needs.gate-census.result }}
|
||||
ARMS: ${{ needs.plan-slices.outputs.census_arms }}
|
||||
# One report per arm: its plan plus its own slice artifacts
|
||||
# (gate-census-<slice>-a<attempt>-<arm>) in /tmp/gate-census-report/<arm>.
|
||||
run: |
|
||||
set +e
|
||||
if [ "$GATE_CENSUS" = "skipped" ]; then
|
||||
echo "gate census skipped: branch dispatch without include_gate_census (periodic lane only)" | tee /tmp/gate-report.txt
|
||||
echo "gate census skipped: no census arm selected (branch dispatch without include_gate_census or census_arms)" | tee /tmp/gate-report.txt
|
||||
echo "exit=0" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --report /tmp/gate-census-report | tee /tmp/gate-report.txt
|
||||
echo "exit=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT"
|
||||
status=0
|
||||
: > /tmp/gate-report.txt
|
||||
for arm in $(jq -r '.[]' <<<"$ARMS"); do
|
||||
dir="/tmp/gate-census-report/$arm"
|
||||
mkdir -p "$dir"
|
||||
cp "/tmp/gate-census-plan/$arm/manifest.json" "$dir/manifest.json" || status=1
|
||||
for slice in /tmp/gate-census-slices/gate-census-*-"$arm"; do if [ -d "$slice" ]; then mv "$slice" "$dir/"; fi; done
|
||||
echo "=== gate census arm: $arm ===" | tee -a /tmp/gate-report.txt
|
||||
EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --report "$dir" | tee -a /tmp/gate-report.txt
|
||||
code=${PIPESTATUS[0]}
|
||||
if [ "$code" -ne 0 ]; then status=$code; fi
|
||||
done
|
||||
echo "exit=$status" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Stamp trial history series
|
||||
if: always()
|
||||
run: |
|
||||
for file in /tmp/paid-report/trial-outcomes.jsonl /tmp/gate-census-report/trial-outcomes.jsonl; do
|
||||
for file in /tmp/paid-report/trial-outcomes.jsonl /tmp/gate-census-report/*/trial-outcomes.jsonl; do
|
||||
if [ -f "$file" ]; then bun --no-install run scripts/eval-trial-series.ts "$file"; fi
|
||||
done
|
||||
|
||||
@@ -719,7 +794,7 @@ jobs:
|
||||
with:
|
||||
name: trial-outcomes-gate-census-a${{ github.run_attempt }}
|
||||
path: |
|
||||
/tmp/gate-census-report/trial-outcomes.jsonl
|
||||
/tmp/gate-census-report/*/trial-outcomes.jsonl
|
||||
if-no-files-found: ignore
|
||||
retention-days: 90
|
||||
|
||||
@@ -732,7 +807,7 @@ jobs:
|
||||
run: |
|
||||
mkdir -p /tmp/exercised-map
|
||||
dirs=()
|
||||
for dir in /tmp/gate-census-report /tmp/paid-report; do [ -d "$dir" ] && dirs+=(--from-dir "$dir"); done
|
||||
for dir in /tmp/gate-census-report/* /tmp/paid-report; do [ -d "$dir" ] && dirs+=(--from-dir "$dir"); done
|
||||
status=0
|
||||
bun --no-install run scripts/exercised-map.ts "${dirs[@]}" \
|
||||
--run-id "$GITHUB_RUN_ID" --out /tmp/exercised-map/exercised-commands.json --report /tmp/exercised-map/report.md || status=$?
|
||||
@@ -779,8 +854,16 @@ jobs:
|
||||
[ "$1" = "0" ] && return 0
|
||||
jq -e '.version == 2 and .verdict.redispatchEligible == true' "$2/collector-outcomes.json" >/dev/null 2>&1
|
||||
}
|
||||
eligible_arms() { # $1 exit, then one report dir per census arm: each must be green or INFRA/INCOMPLETE-only
|
||||
[ "$1" = "0" ] && return 0
|
||||
shift
|
||||
local dir
|
||||
for dir in "$@"; do
|
||||
jq -e '.version == 2 and (.verdict.verdict == "GREEN" or .verdict.redispatchEligible == true)' "$dir/collector-outcomes.json" >/dev/null 2>&1 || return 1
|
||||
done
|
||||
}
|
||||
if [ -z "$REDISPATCH_OF" ] && { [ "$PERIODIC_EXIT" != "0" ] || [ "$GATE_EXIT" != "0" ]; } \
|
||||
&& eligible "$PERIODIC_EXIT" /tmp/paid-report && eligible "$GATE_EXIT" /tmp/gate-census-report; then
|
||||
&& eligible "$PERIODIC_EXIT" /tmp/paid-report && eligible_arms "$GATE_EXIT" /tmp/gate-census-report/*/; then
|
||||
echo "redispatch=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "redispatch=false" >> "$GITHUB_OUTPUT"
|
||||
@@ -793,7 +876,7 @@ jobs:
|
||||
# comments on or closes main's issue. Every report-derived line goes
|
||||
# through the one published-text sanitizer (scripts/lib/published-text.ts).
|
||||
- name: Write the census report
|
||||
if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success')
|
||||
if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || (needs.plan-slices.outputs.periodic_lane == 'true' && needs.eval-slices.result != 'success') || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success')
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
REDISPATCH: ${{ steps.verdict.outputs.redispatch }}
|
||||
@@ -827,7 +910,14 @@ jobs:
|
||||
cat /tmp/paid-report/report-summary.md 2>/dev/null || echo "(no periodic report summary)"
|
||||
echo
|
||||
echo "### Gate census"
|
||||
cat /tmp/gate-census-report/report-summary.md 2>/dev/null || echo "(no gate census report summary)"
|
||||
found=0
|
||||
for summary in /tmp/gate-census-report/*/report-summary.md; do
|
||||
[ -f "$summary" ] || continue
|
||||
found=1
|
||||
echo "#### Arm $(basename "$(dirname "$summary")")"
|
||||
cat "$summary"
|
||||
done
|
||||
[ "$found" = 1 ] || echo "(no gate census report summary)"
|
||||
echo
|
||||
echo "### Pass-rate history (ACTION REQUIRED)"
|
||||
{ awk '/^ACTION REQUIRED/ { on = 1; print; next } on && /^ \[/ { print; next } { on = 0 } /history unavailable/ { print }' /tmp/pass-rates.txt 2>/dev/null \
|
||||
@@ -851,7 +941,7 @@ jobs:
|
||||
cat /tmp/census-report.md >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Upload the census report
|
||||
if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success')
|
||||
if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || (needs.plan-slices.outputs.periodic_lane == 'true' && needs.eval-slices.result != 'success') || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success')
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: census-report-a${{ github.run_attempt }}
|
||||
@@ -860,7 +950,7 @@ jobs:
|
||||
retention-days: 90
|
||||
|
||||
- name: Upsert tracking issue on failure
|
||||
if: always() && github.ref == 'refs/heads/main' && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success')
|
||||
if: always() && github.ref == 'refs/heads/main' && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || (needs.plan-slices.outputs.periodic_lane == 'true' && needs.eval-slices.result != 'success') || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success')
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
@@ -877,7 +967,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Close the tracking issue on a green run
|
||||
if: always() && github.ref == 'refs/heads/main' && steps.reconcile.outputs.exit == '0' && steps.gate-reconcile.outputs.exit == '0' && steps.pass-rates.outputs.exit == '0' && needs.eval-slices.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) && needs.gate-census.result == 'success'
|
||||
if: always() && github.ref == 'refs/heads/main' && steps.reconcile.outputs.exit == '0' && steps.gate-reconcile.outputs.exit == '0' && steps.pass-rates.outputs.exit == '0' && needs.plan-slices.outputs.periodic_lane == 'true' && needs.eval-slices.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) && needs.gate-census.result == 'success'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
REDISPATCH_OF: ${{ inputs.redispatch_of }}
|
||||
@@ -892,7 +982,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Fail the workflow when reconciliation failed
|
||||
if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || !contains(fromJSON('["success","skipped"]'), needs.gate-census.result))
|
||||
if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || (needs.plan-slices.outputs.periodic_lane == 'true' && needs.eval-slices.result != 'success') || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || !contains(fromJSON('["success","skipped"]'), needs.gate-census.result))
|
||||
run: exit 1
|
||||
|
||||
# The one INFRA/INCOMPLETE re-dispatch (UC-E1). Its own job so the report
|
||||
@@ -909,4 +999,5 @@ jobs:
|
||||
- name: Re-dispatch the weekly census once
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: gh workflow run evals-periodic.yml --repo "$GITHUB_REPOSITORY" --ref "$GITHUB_REF_NAME" -f redispatch_of="$GITHUB_RUN_ID"
|
||||
CENSUS_ARMS: ${{ needs.report.outputs.redispatch_census_arms }}
|
||||
run: gh workflow run evals-periodic.yml --repo "$GITHUB_REPOSITORY" --ref "$GITHUB_REF_NAME" -f redispatch_of="$GITHUB_RUN_ID" ${CENSUS_ARMS:+-f census_arms="$CENSUS_ARMS"}
|
||||
Reference in new issue
Block a user