mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-02 17:40:02 +02:00
garrytan
8622535b90
- Slice, census and marathon artifacts carry -a<run_attempt>; reports download them per artifact (no merge), so records never overwrite and a re-run never replaces the first attempt's verdict. - Planners pass --max-parallel for the capacity preflight (24/16 unchanged: the refreshed periodic plan needs 24 slices, the gate census 12). - PR comment: jq-only job reads collector-outcomes v2 (headline, sanitized failure block); the group_by(.name)|last recomputation is gone. - Reports stamp series identities, upload trial-outcomes-* for history, and shard logs upload always (a failed trial no longer reds its runner). - Weekly report: headline + failure block of both lanes in the issue body, the eval:pass-rates --gate step (fails closed without history), close the issue on a green run, and UC-E1: when every red is machine-classified INFRA/INCOMPLETE, one re-dispatch as a new run in its own concurrency group (redispatch_of), both runs reported.
561 lines
25 KiB
YAML
561 lines
25 KiB
YAML
name: Periodic Evals
|
|
# The weekly coverage contract: EVERY periodic-tier paid test runs (EVALS_ALL,
|
|
# minus the reasoned excludes in test/helpers/periodic-exclude-data.ts), so
|
|
# tests can't rot invisibly — the class where the autoplan-dual-voice E2E was
|
|
# silently broken for months until a lucky local diff selected it. Engine:
|
|
# scripts/test-paid-shards.ts (the same runner local eval:bg:periodic uses):
|
|
# one planner manifest packed by recorded durations into as many ~9-minute
|
|
# executors as the work needs (one file, or a tightly packed group, per
|
|
# runner; overlays share one final slice), and a FAIL-CLOSED report — a slice
|
|
# whose artifact never landed is a failure, not an absence. The matrix size
|
|
# and job timeout come from the plan, so they cannot drift from the census.
|
|
# Full end-to-end flows run in the non-blocking marathon lane
|
|
# (evals-marathon.yml), never here. The gate-census
|
|
# job is the weekly EVALS_ALL backstop for the gate tier (PR lanes are
|
|
# diff-billed, so without it the full gate census might never execute
|
|
# anywhere); the hollow-shard guard (exit 0 + zero executed tests under
|
|
# EVALS_ALL fails) makes both lanes census-health checks, not just test runs.
|
|
on:
|
|
schedule:
|
|
- cron: '0 6 * * 1' # Monday 6 AM UTC (ci-image prebuilds at 4 AM)
|
|
workflow_dispatch:
|
|
inputs:
|
|
redispatch_of:
|
|
description: 'Run id this run re-dispatches (the one INFRA/INCOMPLETE-only re-dispatch; set by the report job)'
|
|
type: string
|
|
default: ''
|
|
|
|
# A re-dispatch runs in its own group so it never cancels the run that
|
|
# dispatched it; both runs are reported.
|
|
concurrency:
|
|
group: evals-periodic${{ inputs.redispatch_of && format('-redispatch-{0}', inputs.redispatch_of) || '' }}
|
|
cancel-in-progress: true
|
|
|
|
env:
|
|
IMAGE: ghcr.io/${{ github.repository }}/ci
|
|
EVALS_PROFILE: full
|
|
EVALS_FRESH: "1"
|
|
EVALS_CACHE_PURPOSE: periodic
|
|
|
|
jobs:
|
|
build-image:
|
|
runs-on: ubicloud-standard-8
|
|
timeout-minutes: 15
|
|
permissions:
|
|
contents: read
|
|
packages: write
|
|
outputs:
|
|
image-tag: ${{ steps.meta.outputs.tag }}
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
|
|
- id: meta
|
|
# Keep in sync with evals.yml — key on Dockerfile + lockfile only
|
|
# (package.json's version field would bust the key on every ship).
|
|
# Byte-identity pinned by test/ci-image-tag-binding.test.ts.
|
|
run: echo "tag=${{ env.IMAGE }}:${{ hashFiles('.github/docker/Dockerfile.ci', 'bun.lock', 'patches/**') }}" >> "$GITHUB_OUTPUT"
|
|
|
|
- uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
|
|
with:
|
|
registry: ghcr.io
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Check if image exists
|
|
id: check
|
|
run: |
|
|
if docker manifest inspect ${{ steps.meta.outputs.tag }} > /dev/null 2>&1; then
|
|
echo "exists=true" >> "$GITHUB_OUTPUT"
|
|
else
|
|
echo "exists=false" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
|
|
- if: steps.check.outputs.exists == 'false'
|
|
run: cp package.json bun.lock .github/docker/ && cp -R patches .github/docker/patches
|
|
|
|
# Registry cache export needs a docker-container builder — the default
|
|
# `docker` driver hard-errors on cache-to.
|
|
- if: steps.check.outputs.exists == 'false'
|
|
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
|
|
|
|
- if: steps.check.outputs.exists == 'false'
|
|
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7
|
|
with:
|
|
context: .github/docker
|
|
file: .github/docker/Dockerfile.ci
|
|
push: true
|
|
# Cron-triggered in the base repo only, so cache export is always safe here.
|
|
cache-from: type=registry,ref=${{ env.IMAGE }}:buildcache
|
|
cache-to: type=registry,ref=${{ env.IMAGE }}:buildcache,mode=max
|
|
tags: |
|
|
${{ steps.meta.outputs.tag }}
|
|
${{ env.IMAGE }}:latest
|
|
|
|
plan-slices:
|
|
runs-on: ubicloud-standard-8
|
|
timeout-minutes: 10
|
|
permissions:
|
|
contents: read
|
|
outputs:
|
|
periodic_slices: ${{ steps.periodic-matrix.outputs.slices }}
|
|
periodic_timeout_minutes: ${{ steps.periodic-matrix.outputs.timeout_minutes }}
|
|
gate_slices: ${{ steps.gate-matrix.outputs.slices }}
|
|
gate_timeout_minutes: ${{ steps.gate-matrix.outputs.timeout_minutes }}
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
|
with:
|
|
bun-version: 1.4.0
|
|
|
|
- name: Emit run manifest (ALL periodic tests minus reasoned excludes)
|
|
env:
|
|
EVALS_ALL: "1"
|
|
run: EVALS_TIER=periodic bun --no-install run scripts/test-paid-shards.ts --tier periodic --emit-plan /tmp/paid-plan/manifest.json --slice-budget 540 --jobs 2 --max-parallel 24
|
|
|
|
- name: Derive the periodic executor matrix from the plan
|
|
id: periodic-matrix
|
|
run: |
|
|
echo "slices=$(jq -c '[range(1; .sliceCount + 1)]' /tmp/paid-plan/manifest.json)" >> "$GITHUB_OUTPUT"
|
|
echo "timeout_minutes=$(jq -e '.plan.ciTimeoutMinutes' /tmp/paid-plan/manifest.json)" >> "$GITHUB_OUTPUT"
|
|
|
|
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
with:
|
|
name: paid-plan
|
|
path: /tmp/paid-plan/manifest.json
|
|
retention-days: 30
|
|
|
|
- name: Emit gate census manifest (ALL gate tests)
|
|
env:
|
|
EVALS_ALL: "1"
|
|
run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --emit-plan /tmp/gate-census-plan/manifest.json --slice-budget 540 --jobs 2 --skip-judges --max-parallel 16
|
|
|
|
- name: Derive the gate census executor matrix from the plan
|
|
id: gate-matrix
|
|
run: |
|
|
echo "slices=$(jq -c '[range(1; .sliceCount + 1)]' /tmp/gate-census-plan/manifest.json)" >> "$GITHUB_OUTPUT"
|
|
echo "timeout_minutes=$(jq -e '.plan.ciTimeoutMinutes' /tmp/gate-census-plan/manifest.json)" >> "$GITHUB_OUTPUT"
|
|
|
|
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
with:
|
|
name: gate-census-plan
|
|
path: /tmp/gate-census-plan/manifest.json
|
|
retention-days: 30
|
|
|
|
eval-slices:
|
|
runs-on: ubicloud-standard-8
|
|
needs: [build-image, plan-slices]
|
|
env:
|
|
EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-eval-slices-${{ matrix.slice }}
|
|
# The planner packs ~9 minutes of recorded work per slice; the job timeout
|
|
# is its supervised worst case (every shard at its wall) plus 20 minutes
|
|
# setup/upload, computed from the same manifest the slices execute.
|
|
timeout-minutes: ${{ fromJSON(needs.plan-slices.outputs.periodic_timeout_minutes) }}
|
|
permissions:
|
|
contents: read
|
|
packages: read
|
|
container:
|
|
image: ${{ needs.build-image.outputs.image-tag }}
|
|
credentials:
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
options: --user runner
|
|
strategy:
|
|
fail-fast: false
|
|
# Every planned slice starts at once; test/evals-workflow-wiring.test.ts
|
|
# fails when the live plan outgrows this cap.
|
|
max-parallel: 24
|
|
matrix:
|
|
slice: ${{ fromJSON(needs.plan-slices.outputs.periodic_slices) }}
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
with:
|
|
# Full history: files with SELF-derived selection (the LLM-judge
|
|
# map, routing) walk git at module load, and selection is
|
|
# fail-closed on git errors — a shallow checkout crashed those
|
|
# shards on the lane's first live run ("ambiguous argument
|
|
# 'main...HEAD'"). The manifest still governs WHICH shards run.
|
|
fetch-depth: 0
|
|
persist-credentials: false
|
|
|
|
- name: Fix bun temp
|
|
uses: ./.github/actions/fix-bun-temp
|
|
|
|
- name: Restore deps
|
|
uses: ./.github/actions/restore-deps
|
|
|
|
- run: bun run build
|
|
|
|
# Any slice can host a PTY test — seed + registration run
|
|
# unconditionally (idempotent; mirrors evals.yml's sliced lane). The
|
|
# register composite carries the fail-fast dangling-symlink/frontmatter
|
|
# verification loop — this lane previously LACKED it, so a moved skill
|
|
# target surfaced as a silent "Unknown command" + wedged PTY session.
|
|
- name: Seed claude interactive config
|
|
uses: ./.github/actions/seed-claude-config
|
|
with:
|
|
anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
|
|
- name: Register gstack skills for PTY tests
|
|
uses: ./.github/actions/register-gstack-skills
|
|
|
|
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
|
with:
|
|
name: paid-plan
|
|
path: /tmp/paid-plan
|
|
|
|
- name: Run periodic slice ${{ matrix.slice }}
|
|
env:
|
|
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
|
|
PLAYWRIGHT_BROWSERS_PATH: /opt/playwright-browsers
|
|
EVALS_JOBS: "2"
|
|
EVALS_CONCURRENCY: "2"
|
|
GSTACK_EVAL_DIR: /tmp/paid-slice-results
|
|
run: EVALS_TIER=periodic bun run scripts/test-paid-shards.ts --tier periodic --plan /tmp/paid-plan/manifest.json --slice ${{ matrix.slice }}
|
|
|
|
- name: Upload slice results
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
with:
|
|
name: paid-slice-${{ matrix.slice }}-a${{ github.run_attempt }}
|
|
path: /tmp/paid-slice-results
|
|
retention-days: 90
|
|
|
|
- name: Upload native capture evidence
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
with:
|
|
name: native-captures-${{ env.EVALS_RUN_ID }}
|
|
include-hidden-files: true
|
|
path: |
|
|
~/.gstack/projects/*/e2e-runs
|
|
~/.gstack/projects/*/evals/qa-callers
|
|
~/.gstack-dev/e2e-runs
|
|
~/.gstack-dev/evals/qa-callers
|
|
if-no-files-found: ignore
|
|
retention-days: 90
|
|
|
|
# always(), not failure(): a failed behavior trial is a verdict and no
|
|
# longer reds its runner, but its full log is the diagnosis evidence.
|
|
- name: Upload shard logs
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
with:
|
|
name: paid-logs-slice-${{ matrix.slice }}-a${{ github.run_attempt }}
|
|
include-hidden-files: true
|
|
# The Fix-bun-temp step points TMPDIR at /home/runner/.cache, so the
|
|
# runner's spool lands THERE, not /tmp — the original /tmp glob
|
|
# uploaded nothing and a red slice's diagnostics were unreachable.
|
|
path: |
|
|
/home/runner/.cache/gstack-paid-shard-*.log
|
|
/tmp/gstack-paid-shard-*.log
|
|
if-no-files-found: ignore
|
|
retention-days: 30
|
|
|
|
# Weekly EVALS_ALL gate-tier census: PR lanes are diff-billed, so without
|
|
# this the full gate census might never execute anywhere and the selector's
|
|
# blind spots rot invisibly. Census health, not selector correctness —
|
|
# selector logic has free synthetic-diff contract tests.
|
|
gate-census:
|
|
runs-on: ubicloud-standard-8
|
|
needs: [build-image, plan-slices]
|
|
env:
|
|
EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-gate-census-${{ matrix.slice }}
|
|
# Supervised worst case of the packed plan plus 20 minutes setup/upload.
|
|
timeout-minutes: ${{ fromJSON(needs.plan-slices.outputs.gate_timeout_minutes) }}
|
|
permissions:
|
|
contents: read
|
|
packages: read
|
|
container:
|
|
image: ${{ needs.build-image.outputs.image-tag }}
|
|
credentials:
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
options: --user runner
|
|
strategy:
|
|
# Two file workers per slice, each retaining two in-file case workers.
|
|
fail-fast: false
|
|
max-parallel: 16
|
|
matrix:
|
|
slice: ${{ fromJSON(needs.plan-slices.outputs.gate_slices) }}
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
with:
|
|
# Full history: files with SELF-derived selection (the LLM-judge
|
|
# map, routing) walk git at module load, and selection is
|
|
# fail-closed on git errors — a shallow checkout crashed those
|
|
# shards on the lane's first live run ("ambiguous argument
|
|
# 'main...HEAD'"). The manifest still governs WHICH shards run.
|
|
fetch-depth: 0
|
|
persist-credentials: false
|
|
|
|
- name: Fix bun temp
|
|
uses: ./.github/actions/fix-bun-temp
|
|
|
|
- name: Restore deps
|
|
uses: ./.github/actions/restore-deps
|
|
|
|
- run: bun run build
|
|
|
|
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
|
with:
|
|
name: gate-census-plan
|
|
path: /tmp/gate-census-plan
|
|
|
|
- name: Run gate census slice ${{ matrix.slice }}
|
|
env:
|
|
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
|
|
PLAYWRIGHT_BROWSERS_PATH: /opt/playwright-browsers
|
|
EVALS_JOBS: "2"
|
|
EVALS_CONCURRENCY: "2"
|
|
GSTACK_EVAL_DIR: /tmp/gate-census-results
|
|
run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --plan /tmp/gate-census-plan/manifest.json --slice ${{ matrix.slice }}
|
|
|
|
- name: Upload census results
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
with:
|
|
name: gate-census-${{ matrix.slice }}-a${{ github.run_attempt }}
|
|
path: /tmp/gate-census-results
|
|
retention-days: 90
|
|
|
|
- name: Upload native capture evidence
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
with:
|
|
name: native-captures-${{ env.EVALS_RUN_ID }}
|
|
include-hidden-files: true
|
|
path: |
|
|
~/.gstack/projects/*/e2e-runs
|
|
~/.gstack/projects/*/evals/qa-callers
|
|
~/.gstack-dev/e2e-runs
|
|
~/.gstack-dev/evals/qa-callers
|
|
if-no-files-found: ignore
|
|
retention-days: 90
|
|
|
|
report:
|
|
runs-on: ubicloud-standard-2
|
|
needs: [plan-slices, eval-slices, gate-census]
|
|
# !cancelled(): the report must run (and FAIL) when an executor died — a
|
|
# missing slice artifact reading as green is the class this lane kills —
|
|
# but a cancelled run stops here.
|
|
if: ${{ !cancelled() && needs.plan-slices.result == 'success' }}
|
|
timeout-minutes: 15
|
|
permissions:
|
|
contents: read
|
|
# The notification below upserts (or closes) a tracking issue via
|
|
# `gh issue` — gated by the issues permission.
|
|
issues: write
|
|
# Pass-rate history downloads earlier weekly runs' trial-outcomes.
|
|
actions: read
|
|
outputs:
|
|
redispatch: ${{ steps.verdict.outputs.redispatch }}
|
|
steps:
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
|
with:
|
|
bun-version: 1.4.0
|
|
|
|
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
|
with:
|
|
name: paid-plan
|
|
path: /tmp/paid-report
|
|
|
|
# One directory per attempt-scoped slice artifact (no merge): shard
|
|
# records never overwrite each other and the first attempt decides.
|
|
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
|
with:
|
|
pattern: paid-slice-*
|
|
path: /tmp/paid-report
|
|
|
|
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
|
with:
|
|
name: gate-census-plan
|
|
path: /tmp/gate-census-report
|
|
|
|
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
|
with:
|
|
pattern: gate-census-[0-9]*
|
|
path: /tmp/gate-census-report
|
|
|
|
- name: Reconcile slices against the manifest (fail-closed)
|
|
id: reconcile
|
|
if: always()
|
|
run: |
|
|
set +e
|
|
EVALS_TIER=periodic bun --no-install run scripts/test-paid-shards.ts --tier periodic --report /tmp/paid-report | tee /tmp/report.txt
|
|
# PIPESTATUS[0], NOT $?: GitHub's default run-step shell is
|
|
# `bash -e {0}` with NO pipefail, so $? after the pipe is tee's
|
|
# exit (always 0) — the fail-closed gate was silently fail-open
|
|
# (caught by the ship review army; the wiring test now pins this).
|
|
echo "exit=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Reconcile gate census against the manifest (fail-closed)
|
|
id: gate-reconcile
|
|
if: always()
|
|
run: |
|
|
set +e
|
|
EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --report /tmp/gate-census-report | tee /tmp/gate-report.txt
|
|
echo "exit=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Stamp trial history series
|
|
if: always()
|
|
run: |
|
|
for file in /tmp/paid-report/trial-outcomes.jsonl /tmp/gate-census-report/trial-outcomes.jsonl; do
|
|
if [ -f "$file" ]; then bun --no-install run scripts/eval-trial-series.ts "$file"; fi
|
|
done
|
|
|
|
- name: Upload trial outcomes for pass-rate history
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
with:
|
|
name: trial-outcomes-periodic-a${{ github.run_attempt }}
|
|
path: |
|
|
/tmp/paid-report/trial-outcomes.jsonl
|
|
if-no-files-found: ignore
|
|
retention-days: 90
|
|
|
|
- name: Upload gate census trial outcomes for pass-rate history
|
|
if: always()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
with:
|
|
name: trial-outcomes-gate-census-a${{ github.run_attempt }}
|
|
path: |
|
|
/tmp/gate-census-report/trial-outcomes.jsonl
|
|
if-no-files-found: ignore
|
|
retention-days: 90
|
|
|
|
# Weekly pass-rate gate over the last 10 weekly runs (drift, rule cases
|
|
# behaving like behavior, quarantine exit/expiry/cap). Fails closed when
|
|
# history cannot be fetched.
|
|
- name: Pass-rate history gate
|
|
id: pass-rates
|
|
if: always()
|
|
env:
|
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
run: |
|
|
set +e
|
|
bun run eval:pass-rates --gate --runs 10 > /tmp/pass-rates.txt 2>&1
|
|
echo "exit=$?" >> "$GITHUB_OUTPUT"
|
|
cat /tmp/pass-rates.txt
|
|
|
|
# UC-E1 (approved): a run whose every red verdict is machine-classified
|
|
# INFRA or INCOMPLETE may be re-dispatched ONCE as a new run.
|
|
- name: Classify the census verdict
|
|
id: verdict
|
|
if: always()
|
|
env:
|
|
REDISPATCH_OF: ${{ inputs.redispatch_of }}
|
|
PERIODIC_EXIT: ${{ steps.reconcile.outputs.exit }}
|
|
GATE_EXIT: ${{ steps.gate-reconcile.outputs.exit }}
|
|
run: |
|
|
eligible() { # $1 exit, $2 report dir
|
|
[ "$1" = "0" ] && return 0
|
|
jq -e '.version == 2 and .verdict.redispatchEligible == true' "$2/collector-outcomes.json" >/dev/null 2>&1
|
|
}
|
|
if [ -z "$REDISPATCH_OF" ] && { [ "$PERIODIC_EXIT" != "0" ] || [ "$GATE_EXIT" != "0" ]; } \
|
|
&& eligible "$PERIODIC_EXIT" /tmp/paid-report && eligible "$GATE_EXIT" /tmp/gate-census-report; then
|
|
echo "redispatch=true" >> "$GITHUB_OUTPUT"
|
|
else
|
|
echo "redispatch=false" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
|
|
# A red weekly lane nobody must action is waste — upsert ONE tracking
|
|
# issue (never a new issue per week) with the headline and failure block
|
|
# of both lanes, and close it on the next green run.
|
|
- name: Upsert tracking issue on failure
|
|
if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || needs.gate-census.result != 'success')
|
|
env:
|
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
REDISPATCH: ${{ steps.verdict.outputs.redispatch }}
|
|
REDISPATCH_OF: ${{ inputs.redispatch_of }}
|
|
run: |
|
|
set -euo pipefail
|
|
TITLE="Weekly periodic evals: red lane needs triage"
|
|
BODY_FILE=/tmp/issue-body.md
|
|
RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
|
{
|
|
echo "Automated weekly report — run: ${RUN_URL}"
|
|
if [ -n "$REDISPATCH_OF" ]; then echo; echo "This run is the one INFRA/INCOMPLETE re-dispatch of run ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${REDISPATCH_OF}; both runs are reported."; fi
|
|
if [ "$REDISPATCH" = "true" ]; then echo; echo "Every red verdict is machine-classified INFRA/INCOMPLETE: re-dispatching once as a new run (EVAL_POLICY.infraRedispatch). This run stays red and reported."; fi
|
|
echo
|
|
echo "- periodic reconciliation exit: ${{ steps.reconcile.outputs.exit }} (slices job: ${{ needs.eval-slices.result }})"
|
|
echo "- gate census reconciliation exit: ${{ steps.gate-reconcile.outputs.exit }} (census job: ${{ needs.gate-census.result }})"
|
|
echo "- pass-rate history gate exit: ${{ steps.pass-rates.outputs.exit }}"
|
|
echo
|
|
echo "### Periodic lane"
|
|
cat /tmp/paid-report/report-summary.md 2>/dev/null || echo "(no periodic report summary)"
|
|
echo
|
|
echo "### Gate census"
|
|
cat /tmp/gate-census-report/report-summary.md 2>/dev/null || echo "(no gate census report summary)"
|
|
echo
|
|
echo "### Pass-rate history (ACTION REQUIRED)"
|
|
echo '```'
|
|
{ grep -E 'ACTION REQUIRED|history unavailable' /tmp/pass-rates.txt || echo "(no pass-rate alarms)"; } | sed 's/@/@\xe2\x80\x8b/g' | head -c 6000
|
|
echo '```'
|
|
echo
|
|
echo "<details><summary>Full reconciliation output</summary>"
|
|
echo
|
|
echo '```'
|
|
tail -c 6000 /tmp/report.txt 2>/dev/null | sed 's/@/@\xe2\x80\x8b/g' || echo "(no reconciliation output)"
|
|
echo '```'
|
|
echo
|
|
echo '```'
|
|
tail -c 6000 /tmp/gate-report.txt 2>/dev/null | sed 's/@/@\xe2\x80\x8b/g' || echo "(no gate census reconciliation output)"
|
|
echo '```'
|
|
echo "</details>"
|
|
echo
|
|
echo "Policy: EVAL_POLICY and CASE_QUARANTINE in test/helpers/periodic-exclude-data.ts; history: \`bun run eval:pass-rates\`."
|
|
} > "$BODY_FILE"
|
|
EXISTING=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "in:title \"$TITLE\"" --json number --jq '.[0].number // empty')
|
|
if [ -n "$EXISTING" ]; then
|
|
gh issue comment "$EXISTING" --repo "$GITHUB_REPOSITORY" --body-file "$BODY_FILE"
|
|
echo "commented on #$EXISTING"
|
|
else
|
|
gh issue create --repo "$GITHUB_REPOSITORY" --title "$TITLE" --body-file "$BODY_FILE"
|
|
fi
|
|
|
|
- name: Close the tracking issue on a green run
|
|
if: always() && steps.reconcile.outputs.exit == '0' && steps.gate-reconcile.outputs.exit == '0' && steps.pass-rates.outputs.exit == '0' && needs.eval-slices.result == 'success' && needs.gate-census.result == 'success'
|
|
env:
|
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
REDISPATCH_OF: ${{ inputs.redispatch_of }}
|
|
run: |
|
|
set -euo pipefail
|
|
TITLE="Weekly periodic evals: red lane needs triage"
|
|
EXISTING=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "in:title \"$TITLE\"" --json number --jq '.[0].number // empty')
|
|
if [ -n "$EXISTING" ]; then
|
|
NOTE="Green weekly run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
|
if [ -n "$REDISPATCH_OF" ]; then NOTE="${NOTE} (the INFRA re-dispatch of run ${REDISPATCH_OF}, which stays red and reported)"; fi
|
|
gh issue close "$EXISTING" --repo "$GITHUB_REPOSITORY" --comment "$NOTE"
|
|
fi
|
|
|
|
- name: Fail the workflow when reconciliation failed
|
|
if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || needs.gate-census.result != 'success')
|
|
run: exit 1
|
|
|
|
# The one INFRA/INCOMPLETE re-dispatch (UC-E1). Its own job so the report
|
|
# job keeps no actions:write; the new run's concurrency group differs, so it
|
|
# never cancels this run.
|
|
redispatch:
|
|
runs-on: ubicloud-standard-2
|
|
needs: report
|
|
if: ${{ !cancelled() && needs.report.outputs.redispatch == 'true' }}
|
|
timeout-minutes: 5
|
|
permissions:
|
|
actions: write
|
|
steps:
|
|
- name: Re-dispatch the weekly census once
|
|
env:
|
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
run: gh workflow run evals-periodic.yml --repo "$GITHUB_REPOSITORY" --ref "$GITHUB_REF_NAME" -f redispatch_of="$GITHUB_RUN_ID"
|