From 7814a6d7afd9342cf11b6bcc2d49213395dba5b2 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sun, 4 Oct 2026 18:33:48 +0000 Subject: [PATCH] 0.3: only main posts or closes tracking issues; reports carry alarms, red ledger, headroom and all-green A branch-dispatched periodic or marathon run writes its report to the step summary and a report artifact; the upsert and close steps run only on refs/heads/main. The periodic body now keeps every pass-rate alarm line (the old grep kept only the header), adds the red ledger and session headroom read with an explicit --branch, the all-green estimate and the triage guide link, and routes report text through scripts/lib/published-text.ts. The marathon lane gains the matching close-on-green step. Migrated: evals-workflow-wiring pinned report-summary.md in the upsert step; the body is now written by 'Write the census report'. --- .github/workflows/evals-marathon.yml | 60 +++++++-- .github/workflows/evals-periodic.yml | 73 +++++++--- test/evals-tracking-issue.test.ts | 193 +++++++++++++++++++++++++++ test/evals-workflow-wiring.test.ts | 6 +- 4 files changed, 297 insertions(+), 35 deletions(-) create mode 100644 test/evals-tracking-issue.test.ts diff --git a/.github/workflows/evals-marathon.yml b/.github/workflows/evals-marathon.yml index 9aa0ba98c..430186927 100644 --- a/.github/workflows/evals-marathon.yml +++ b/.github/workflows/evals-marathon.yml @@ -283,25 +283,47 @@ jobs: # PIPESTATUS[0], NOT $?: the default step shell has no pipefail. echo "exit=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT" - # One tracking issue for the whole lane (never one per week). - - name: Upsert tracking issue on failure + # A red marathon run writes its report to the step summary and the + # marathon-report artifact on every ref. Only main upserts the lane's ONE + # tracking issue (never one per week) and closes it on the next green + # run; a branch-dispatched run never touches main's issue. + - name: Write the marathon report if: always() && (steps.reconcile.outputs.exit != '0' || needs.eval-slices.result != 'success') + run: | + set -euo pipefail + { + echo "Marathon report (non-blocking lane): branch ${GITHUB_REF_NAME} @ ${GITHUB_SHA}, event ${GITHUB_EVENT_NAME}, run ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + echo + echo "A census went red: triage with ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/blob/${GITHUB_SHA}/docs/evals/census-red.md" + echo + echo "- reconciliation exit: ${{ steps.reconcile.outputs.exit }}" + echo "- marathon slices job: ${{ needs.eval-slices.result }}" + echo + { tail -c 6000 /tmp/report.txt 2>/dev/null || echo "(no reconciliation output)"; } \ + | bun --no-install -e 'import { publishedFence } from "./scripts/lib/published-text"; + const lines = (await Bun.stdin.text()).split("\n").filter(Boolean); + console.log(publishedFence(lines.length ? lines : ["(none)"]).join("\n"));' + } > /tmp/marathon-report.md + cat /tmp/marathon-report.md >> "$GITHUB_STEP_SUMMARY" + + - name: Upload the marathon report + if: always() && (steps.reconcile.outputs.exit != '0' || needs.eval-slices.result != 'success') + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: marathon-report-a${{ github.run_attempt }} + path: /tmp/marathon-report.md + if-no-files-found: ignore + retention-days: 90 + + - name: Upsert tracking issue on failure + if: always() && github.ref == 'refs/heads/main' && (steps.reconcile.outputs.exit != '0' || needs.eval-slices.result != 'success') env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -euo pipefail TITLE="Weekly marathon evals: red lane needs triage" - BODY_FILE=/tmp/issue-body.md - { - echo "Automated weekly marathon report (non-blocking lane) — run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" - echo - echo "- reconciliation exit: ${{ steps.reconcile.outputs.exit }}" - echo "- marathon slices job: ${{ needs.eval-slices.result }}" - echo - echo '```' - tail -c 6000 /tmp/report.txt 2>/dev/null || echo "(no reconciliation output)" - echo '```' - } > "$BODY_FILE" + BODY_FILE=/tmp/marathon-report.md + [ -s "$BODY_FILE" ] || echo "Marathon report missing; run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" > "$BODY_FILE" EXISTING=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "in:title \"$TITLE\"" --json number --jq '.[0].number // empty') if [ -n "$EXISTING" ]; then gh issue comment "$EXISTING" --repo "$GITHUB_REPOSITORY" --body-file "$BODY_FILE" @@ -310,6 +332,18 @@ jobs: gh issue create --repo "$GITHUB_REPOSITORY" --title "$TITLE" --body-file "$BODY_FILE" fi + - name: Close the tracking issue on a green run + if: always() && github.ref == 'refs/heads/main' && steps.reconcile.outputs.exit == '0' && needs.eval-slices.result == 'success' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + TITLE="Weekly marathon evals: red lane needs triage" + EXISTING=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "in:title \"$TITLE\"" --json number --jq '.[0].number // empty') + if [ -n "$EXISTING" ]; then + gh issue close "$EXISTING" --repo "$GITHUB_REPOSITORY" --comment "Green marathon run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + fi + - name: Fail the workflow when reconciliation failed if: always() && (steps.reconcile.outputs.exit != '0' || needs.eval-slices.result != 'success') run: exit 1 diff --git a/.github/workflows/evals-periodic.yml b/.github/workflows/evals-periodic.yml index 027533301..464b2b45d 100644 --- a/.github/workflows/evals-periodic.yml +++ b/.github/workflows/evals-periodic.yml @@ -627,7 +627,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set +e - bun run eval:pass-rates --gate --runs 10 > /tmp/pass-rates.txt 2>&1 + bun run eval:pass-rates --gate --runs 10 --branch "$GITHUB_REF_NAME" > /tmp/pass-rates.txt 2>&1 echo "exit=$?" >> "$GITHUB_OUTPUT" cat /tmp/pass-rates.txt @@ -652,10 +652,13 @@ jobs: echo "redispatch=false" >> "$GITHUB_OUTPUT" fi - # A red weekly lane nobody must action is waste — upsert ONE tracking - # issue (never a new issue per week) with the headline and failure block - # of both lanes, and close it on the next green run. - - name: Upsert tracking issue on failure + # A red census writes one report on every ref: the step summary and the + # census-report artifact. Only main (scheduled weekly, or dispatched on + # main) upserts ONE tracking issue (never a new issue per week) and + # closes it on the next green run; a branch-dispatched census never + # comments on or closes main's issue. Every report-derived line goes + # through the one published-text sanitizer (scripts/lib/published-text.ts). + - name: Write the census report if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success') env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -663,17 +666,28 @@ jobs: REDISPATCH_OF: ${{ inputs.redispatch_of }} run: | set -euo pipefail - TITLE="Weekly periodic evals: red lane needs triage" - BODY_FILE=/tmp/issue-body.md + published() { # $1 fence|line: stdin lines through the published-text sanitizer + bun --no-install -e 'import { publishedFence, sanitizePublishedText } from "./scripts/lib/published-text"; + const lines = (await Bun.stdin.text()).split("\n").filter(Boolean); + console.log(process.argv[1] === "line" ? sanitizePublishedText(lines.join(" ")) : publishedFence(lines.length ? lines : ["(none)"]).join("\n"));' "$1" + } RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + GUIDE_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/blob/${GITHUB_SHA}/docs/evals/census-red.md" + bun run eval:pass-rates --reds --runs 10 --branch "$GITHUB_REF_NAME" > /tmp/pass-rates-reds.txt 2>&1 \ + || echo "history unavailable: eval:pass-rates --reds exited $?" >> /tmp/pass-rates-reds.txt + bun run eval:pass-rates --headroom --runs 10 --branch "$GITHUB_REF_NAME" > /tmp/pass-rates-headroom.txt 2>&1 \ + || echo "history unavailable: eval:pass-rates --headroom exited $?" >> /tmp/pass-rates-headroom.txt { - echo "Automated weekly report — run: ${RUN_URL}" + echo "Periodic census report: branch ${GITHUB_REF_NAME} @ ${GITHUB_SHA}, event ${GITHUB_EVENT_NAME}, run ${RUN_URL}" + echo + echo "A census went red: triage with ${GUIDE_URL}" if [ -n "$REDISPATCH_OF" ]; then echo; echo "This run is the one INFRA/INCOMPLETE re-dispatch of run ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${REDISPATCH_OF}; both runs are reported."; fi if [ "$REDISPATCH" = "true" ]; then echo; echo "Every red verdict is machine-classified INFRA/INCOMPLETE: re-dispatching once as a new run (EVAL_POLICY.infraRedispatch). This run stays red and reported."; fi echo echo "- periodic reconciliation exit: ${{ steps.reconcile.outputs.exit }} (slices job: ${{ needs.eval-slices.result }}, Codex slices job: ${{ needs.eval-codex-slices.result }})" echo "- gate census reconciliation exit: ${{ steps.gate-reconcile.outputs.exit }} (census job: ${{ needs.gate-census.result }})" echo "- pass-rate history gate exit: ${{ steps.pass-rates.outputs.exit }}" + echo "- $({ grep -E '^all-green probability' /tmp/pass-rates-reds.txt || echo 'all-green probability: unavailable (see the red ledger)'; } | published line)" echo echo "### Periodic lane" cat /tmp/paid-report/report-summary.md 2>/dev/null || echo "(no periodic report summary)" @@ -682,23 +696,44 @@ jobs: cat /tmp/gate-census-report/report-summary.md 2>/dev/null || echo "(no gate census report summary)" echo echo "### Pass-rate history (ACTION REQUIRED)" - echo '```' - { grep -E 'ACTION REQUIRED|history unavailable' /tmp/pass-rates.txt || echo "(no pass-rate alarms)"; } | sed 's/@/@\xe2\x80\x8b/g' | head -c 6000 - echo '```' + { awk '/^ACTION REQUIRED/ { on = 1; print; next } on && /^ \[/ { print; next } { on = 0 } /history unavailable/ { print }' /tmp/pass-rates.txt 2>/dev/null \ + | grep . || echo "(no pass-rate alarms)"; } | head -c 6000 | published fence + echo + echo "### Red ledger: bun run eval:pass-rates --reds --runs 10 --branch ${GITHUB_REF_NAME}" + { head -n 1 /tmp/pass-rates-reds.txt; tail -n +2 /tmp/pass-rates-reds.txt | tail -n 60; } | cut -c1-400 | published fence + echo + echo "### Session headroom: bun run eval:pass-rates --headroom --runs 10 --branch ${GITHUB_REF_NAME}" + head -n 40 /tmp/pass-rates-headroom.txt | cut -c1-400 | published fence echo echo "
Full reconciliation output" echo - echo '```' - tail -c 6000 /tmp/report.txt 2>/dev/null | sed 's/@/@\xe2\x80\x8b/g' || echo "(no reconciliation output)" - echo '```' + { tail -c 6000 /tmp/report.txt 2>/dev/null || echo "(no reconciliation output)"; } | published fence echo - echo '```' - tail -c 6000 /tmp/gate-report.txt 2>/dev/null | sed 's/@/@\xe2\x80\x8b/g' || echo "(no gate census reconciliation output)" - echo '```' + { tail -c 6000 /tmp/gate-report.txt 2>/dev/null || echo "(no gate census reconciliation output)"; } | published fence echo "
" echo echo "Policy: EVAL_POLICY and CASE_QUARANTINE in test/helpers/periodic-exclude-data.ts; history: \`bun run eval:pass-rates\`." - } > "$BODY_FILE" + } > /tmp/census-report.md + cat /tmp/census-report.md >> "$GITHUB_STEP_SUMMARY" + + - name: Upload the census report + if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success') + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: census-report-a${{ github.run_attempt }} + path: /tmp/census-report.md + if-no-files-found: ignore + retention-days: 90 + + - name: Upsert tracking issue on failure + if: always() && github.ref == 'refs/heads/main' && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success') + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + TITLE="Weekly periodic evals: red lane needs triage" + BODY_FILE=/tmp/census-report.md + [ -s "$BODY_FILE" ] || echo "Periodic census report missing; run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" > "$BODY_FILE" EXISTING=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "in:title \"$TITLE\"" --json number --jq '.[0].number // empty') if [ -n "$EXISTING" ]; then gh issue comment "$EXISTING" --repo "$GITHUB_REPOSITORY" --body-file "$BODY_FILE" @@ -708,7 +743,7 @@ jobs: fi - name: Close the tracking issue on a green run - if: always() && steps.reconcile.outputs.exit == '0' && steps.gate-reconcile.outputs.exit == '0' && steps.pass-rates.outputs.exit == '0' && needs.eval-slices.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) && needs.gate-census.result == 'success' + if: always() && github.ref == 'refs/heads/main' && steps.reconcile.outputs.exit == '0' && steps.gate-reconcile.outputs.exit == '0' && steps.pass-rates.outputs.exit == '0' && needs.eval-slices.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) && needs.gate-census.result == 'success' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} REDISPATCH_OF: ${{ inputs.redispatch_of }} diff --git a/test/evals-tracking-issue.test.ts b/test/evals-tracking-issue.test.ts new file mode 100644 index 000000000..0e48535ae --- /dev/null +++ b/test/evals-tracking-issue.test.ts @@ -0,0 +1,193 @@ +/** + * The weekly tracking issues (evals-periodic.yml, evals-marathon.yml). + * + * Only main posts to or closes a tracking issue: a branch-dispatched census + * writes the same report to its step summary and a report artifact instead. + * The periodic report carries the full pass-rate alarm lines, the red ledger, + * session headroom and the all-green estimate, read with an explicit + * --branch, and every report-derived line passes the published-text + * sanitizer. The write steps run here as shell scripts with a fake + * `eval:pass-rates`; the sanitizer is the real one. + */ +import { describe, test, expect } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { spawnSync } from 'child_process'; + +const ROOT = path.join(import.meta.dir, '..'); +type Step = { name?: string; if?: string; run?: string; uses?: string; with?: Record }; +type Workflow = { jobs: Record }; +const WORKFLOWS = ['evals-periodic.yml', 'evals-marathon.yml'] as const; +const parse = (name: string) => Bun.YAML.parse(fs.readFileSync(path.join(ROOT, '.github/workflows', name), 'utf8')) as Workflow; +const MAIN_ONLY = "github.ref == 'refs/heads/main'"; + +describe('tracking issues are main-only', () => { + for (const name of WORKFLOWS) { + test(`${name}: only the upsert and close steps call gh issue, and both run on main only`, () => { + const issueSteps = Object.values(parse(name).jobs).flatMap(job => job.steps ?? []).filter(step => /\bgh issue\b/.test(step.run ?? '')); + expect(issueSteps.map(step => step.name).sort()).toEqual(['Close the tracking issue on a green run', 'Upsert tracking issue on failure']); + for (const step of issueSteps) expect(step.if, step.name).toContain(`always() && ${MAIN_ONLY} && `); + const upsert = issueSteps.find(step => step.name === 'Upsert tracking issue on failure')!; + expect(upsert.run).toMatch(/gh issue comment .*--body-file "\$BODY_FILE"/); + expect(upsert.run).toMatch(/gh issue create .*--body-file "\$BODY_FILE"/); + expect(issueSteps.find(step => step.name === 'Close the tracking issue on a green run')!.run).toContain('gh issue close'); + }); + + test(`${name}: every ref writes the report to the step summary and an artifact`, () => { + const steps = parse(name).jobs.report!.steps!; + const write = steps.find(step => /^Write the (census|marathon) report$/.test(step.name ?? ''))!; + const upload = steps.find(step => /^Upload the (census|marathon) report$/.test(step.name ?? ''))!; + const upsert = steps.find(step => step.name === 'Upsert tracking issue on failure')!; + expect(write.if).not.toContain('github.ref'); + expect(upload.if).toBe(write.if); + expect(upsert.if).toBe(write.if!.replace('always() && ', `always() && ${MAIN_ONLY} && `)); + expect(write.run).toContain('>> "$GITHUB_STEP_SUMMARY"'); + expect(write.run).toContain('docs/evals/census-red.md'); + expect(write.run).toContain('./scripts/lib/published-text'); + const reportFile = /> (\/tmp\/[\w-]+\.md)\n/.exec(write.run!)![1]!; + expect(upload.uses).toStartWith('actions/upload-artifact@'); + expect(upload.with!.path).toBe(reportFile); + expect(upload.with!.name).toMatch(/^(census|marathon)-report-a\$\{\{ github\.run_attempt \}\}$/); + expect(upsert.run).toContain(`BODY_FILE=${reportFile}`); + }); + } + + test('evals-periodic.yml: every pass-rates read names its branch explicitly', () => { + const runs = parse('evals-periodic.yml').jobs.report!.steps!.map(step => step.run ?? '').join('\n'); + const calls = [...runs.matchAll(/^\s*bun run eval:pass-rates [^\n>]*/gm)].map(m => m[0].trim()); + expect(calls.length).toBe(3); + for (const call of calls) expect(call).toContain('--runs 10 --branch "$GITHUB_REF_NAME"'); + expect(calls.some(call => call.includes('--gate'))).toBe(true); + expect(calls.some(call => call.includes('--reds'))).toBe(true); + expect(calls.some(call => call.includes('--headroom'))).toBe(true); + }); +}); + +/** Run a workflow step's script with its expressions and /tmp paths bound to a scratch dir. */ +function runStep(workflow: string, stepName: string, files: Record, expressions: Record) { + const step = parse(workflow).jobs.report!.steps!.find(s => s.name === stepName)!; + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'tracking-issue-')); + const work = path.join(dir, 'tmp'); + const bin = path.join(dir, 'bin'); + fs.mkdirSync(bin); + for (const [rel, body] of Object.entries(files)) { + fs.mkdirSync(path.dirname(path.join(work, rel)), { recursive: true }); + fs.writeFileSync(path.join(work, rel), body); + } + const log = path.join(dir, 'pass-rates-calls.log'); + fs.writeFileSync(path.join(bin, 'bun'), [ + '#!/bin/sh', + 'if [ "$1" = run ] && [ "$2" = eval:pass-rates ]; then', + ` echo "$*" >> '${log}'`, + ` case " $* " in *" --reds "*) cat '${work}/fake-reds.txt';; *" --headroom "*) cat '${work}/fake-headroom.txt';; esac`, + ' exit 0', + 'fi', + `exec '${process.execPath}' "$@"`, + ].join('\n'), { mode: 0o755 }); + const script = step.run! + .replace(/\$\{\{ ([^}]+) \}\}/g, (_, expr: string) => expressions[expr.trim()] ?? `<${expr.trim()}>`) + .replaceAll('/tmp/', `${work}/`); + const summary = path.join(dir, 'step-summary.md'); + const result = spawnSync('bash', ['-c', script], { + cwd: ROOT, encoding: 'utf8', timeout: 60_000, + env: { PATH: `${bin}:${process.env.PATH}`, HOME: dir, GITHUB_STEP_SUMMARY: summary, GITHUB_REF_NAME: 'feature/x', + GITHUB_SHA: 'abc1234', GITHUB_EVENT_NAME: 'workflow_dispatch', GITHUB_SERVER_URL: 'https://github.com', + GITHUB_REPOSITORY: 'garrytan/gstack', GITHUB_RUN_ID: '42', REDISPATCH: 'false', REDISPATCH_OF: '' }, + }); + const read = (file: string) => (fs.existsSync(file) ? fs.readFileSync(file, 'utf8') : ''); + const reports = Object.fromEntries(fs.readdirSync(work).filter(name => name.endsWith('.md')).map(name => [name, read(path.join(work, name))])); + const out = { status: result.status, stderr: result.stderr, summary: read(summary), calls: read(log), body: (name: string) => reports[name] ?? '' }; + fs.rmSync(dir, { recursive: true, force: true }); + return out; +} + +const PERIODIC_EXPRESSIONS = { + 'steps.reconcile.outputs.exit': '1', 'steps.gate-reconcile.outputs.exit': '0', 'steps.pass-rates.outputs.exit': '1', + 'needs.eval-slices.result': 'failure', 'needs.eval-codex-slices.result': 'success', 'needs.gate-census.result': 'success', +}; + +describe('the periodic census report', () => { + const passRates = [ + 'scope: garrytan/gstack evals-periodic.yml on feature/x + main, last 10 completed run(s) per branch: 1, 2', + 'pass-rates: policy v1, 40 post-policy trial(s), 0 pre-policy (display only)', + ' label kind tier current series pre-policy manual case', + ' ACTION rule gate 9/10 [59.6%–98.2%] - 0 noisy-case', + 'ACTION REQUIRED (3):', + ' [drift] noisy-case passes 9/10 (below 95% over >= 10 trials): fix it, or propose a CASE_QUARANTINE entry with a written diagnosis', + ' [headroom] slow-case session sdk:slow-case#1: max 520s of 600s (87%) above the 85% cap; cut work, never raise the budget', + ' [drift] mention @attacker ``` ghp_' + 'a'.repeat(36), + 'free-suite flaky-passes (/home/x/.gstack/flake-ledger.jsonl):', + ' 2x test/unrelated.test.ts', + ].join('\n'); + const reds = [ + 'scope: garrytan/gstack evals-periodic.yml on feature/x + main, last 10 completed run(s) per branch: 1, 2', + 'reds: verdict reds per census lane by failure class and machine cause', + ' 1 periodic/full census: 1 red of 146 verdicts — class: assertion 1; cause: provider_stall 1', + ' ✗ noisy-case FAIL t1 assertion/provider_stall — Error: expect(received).toBe(expected)', + 'pooled: 1 red of 146 verdicts = 0.7% per verdict [95% 0.1%–3.8%]', + 'all-green probability (approximation, assumes independent verdicts): 81.2% for the latest census\'s 146 verdicts [55%–95% with the pooled rate\'s 95% interval as prior]; formula Π(1 - p_i) over its cases; rerun: bun run eval:pass-rates --reds', + ].join('\n'); + const headroom = [ + 'scope: garrytan/gstack evals-periodic.yml on feature/x + main', + 'headroom: slowest armed session per case (warn > 75%, alarm > 85%; insufficient below 3 samples; censored = timed out)', + ' ALARM 520s/600s (87%) 3 540s slow-case', + ].join('\n'); + const files = { + 'pass-rates.txt': passRates, 'fake-reds.txt': reds, 'fake-headroom.txt': headroom, + 'paid-report/report-summary.md': '## Periodic summary\n✗ noisy-case', 'report.txt': 'reconcile tail', + }; + + test('carries the full alarm lines, red ledger, headroom and all-green estimate, sanitized, for an explicit branch', () => { + const run = runStep('evals-periodic.yml', 'Write the census report', files, PERIODIC_EXPRESSIONS); + expect(run.stderr).toBe(''); + expect(run.status).toBe(0); + const body = run.body('census-report.md'); + expect(run.summary).toBe(body); + expect(body).toStartWith('Periodic census report: branch feature/x @ abc1234, event workflow_dispatch, run https://github.com/garrytan/gstack/actions/runs/42'); + expect(body).toContain('triage with https://github.com/garrytan/gstack/blob/abc1234/docs/evals/census-red.md'); + expect(body).toContain('- periodic reconciliation exit: 1 (slices job: failure, Codex slices job: success)'); + expect(body).toContain('- all-green probability (approximation, assumes independent verdicts): 81.2% for the latest census'); + expect(body).toContain('## Periodic summary'); + // Every alarm line, not just the header; the flaky-pass block is not an alarm. + expect(body).toContain('ACTION REQUIRED (3):'); + expect(body).toContain('[drift] noisy-case passes 9/10 (below 95% over >= 10 trials): fix it'); + expect(body).toContain('[headroom] slow-case session sdk:slow-case#1: max 520s of 600s (87%) above the 85% cap'); + expect(body).not.toContain('test/unrelated.test.ts'); + expect(body).toContain('✗ noisy-case FAIL t1 assertion/provider_stall — Error: expect(received).toBe(expected)'); + expect(body).toContain('ALARM 520s/600s (87%)'); + // Hostile alarm text stays literal inside a fence it cannot close; mentions and tokens are neutralized. + const hostile = body.split('\n').findIndex(line => line.includes('mention @')); + const fence = body.split('\n').slice(0, hostile).reverse().find(line => /^`{3,}$/.test(line))!; + expect(fence.length).toBeGreaterThan(3); + expect(body).toContain('@\u200battacker'); + expect(body).not.toContain('ghp_' + 'a'.repeat(36)); + expect(run.calls.trim().split('\n')).toEqual([ + 'run eval:pass-rates --reds --runs 10 --branch feature/x', + 'run eval:pass-rates --headroom --runs 10 --branch feature/x', + ]); + }); + + test('missing history becomes a visible line, never a failed step', () => { + const run = runStep('evals-periodic.yml', 'Write the census report', { 'fake-reds.txt': '', 'fake-headroom.txt': '' }, PERIODIC_EXPRESSIONS); + expect(run.status).toBe(0); + const body = run.body('census-report.md'); + expect(body).toContain('all-green probability: unavailable (see the red ledger)'); + expect(body).toContain('(no pass-rate alarms)'); + expect(body).toContain('(no periodic report summary)'); + }); +}); + +describe('the marathon report', () => { + test('fences the sanitized reconciliation output and links the guide', () => { + const run = runStep('evals-marathon.yml', 'Write the marathon report', { 'report.txt': 'red slice\nping @someone ```' }, + { 'steps.reconcile.outputs.exit': '1', 'needs.eval-slices.result': 'success' }); + expect(run.status).toBe(0); + const body = run.body('marathon-report.md'); + expect(run.summary).toBe(body); + expect(body).toStartWith('Marathon report (non-blocking lane): branch feature/x @ abc1234, event workflow_dispatch'); + expect(body).toContain('docs/evals/census-red.md'); + expect(body).toContain('- reconciliation exit: 1'); + expect(body).toContain('````\nred slice\nping @\u200bsomeone ```\n````'); + }); +}); diff --git a/test/evals-workflow-wiring.test.ts b/test/evals-workflow-wiring.test.ts index 4268c160e..dd816b741 100644 --- a/test/evals-workflow-wiring.test.ts +++ b/test/evals-workflow-wiring.test.ts @@ -377,11 +377,11 @@ describe('panel verdict surfaces (eval reliability policy)', () => { const gate = report.steps.find(step => step.id === 'pass-rates')!; expect(gate.run).toContain('bun run eval:pass-rates --gate --runs 10'); expect(gate.if).toBe('always()'); - for (const name of ['Upsert tracking issue on failure', 'Fail the workflow when reconciliation failed']) { + for (const name of ['Write the census report', 'Upsert tracking issue on failure', 'Fail the workflow when reconciliation failed']) { expect(report.steps.find(step => step.name === name)!.if).toContain("steps.pass-rates.outputs.exit != '0'"); } - const upsert = report.steps.find(step => step.name === 'Upsert tracking issue on failure')!; - expect(upsert.run).toContain('report-summary.md'); + // The report body is written once on every ref; the main-only upsert posts it (test/evals-tracking-issue.test.ts). + expect(report.steps.find(step => step.name === 'Write the census report')!.run).toContain('report-summary.md'); expect(report.steps.find(step => step.name === 'Close the tracking issue on a green run')!.run).toContain('gh issue close'); expect(report.steps.filter(step => step.with?.name?.startsWith('trial-outcomes-')).length).toBe(2); const redispatch = jobs.redispatch!;