CalvinBackup/gstack
1
0 Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-10-31 14:06:50 +01:00
Code Issues Packages Projects Releases Wiki Activity

0.3: only main posts or closes tracking issues; reports carry alarms, red ledger, headroom and all-green

Browse Source 
A branch-dispatched periodic or marathon run writes its report to the step
summary and a report artifact; the upsert and close steps run only on
refs/heads/main. The periodic body now keeps every pass-rate alarm line (the
old grep kept only the header), adds the red ledger and session headroom read
with an explicit --branch, the all-green estimate and the triage guide link,
and routes report text through scripts/lib/published-text.ts. The marathon
lane gains the matching close-on-green step.

Migrated: evals-workflow-wiring pinned report-summary.md in the upsert step;
the body is now written by 'Write the census report'.
This commit is contained in:
garrytan committed 2026-10-04 18:33:48 +00:00
1 parent 694e970c65
commit 7814a6d7af
4 files changed
+297 -35
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/workflows/evals-marathon.yml
+47 -13
View File
@@ -283,25 +283,47 @@ jobs:
# PIPESTATUS[0], NOT $?: the default step shell has no pipefail.
echo "exit=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT"
# One tracking issue for the whole lane (never one per week).
- name: Upsert tracking issue on failure
# A red marathon run writes its report to the step summary and the
# marathon-report artifact on every ref. Only main upserts the lane's ONE
# tracking issue (never one per week) and closes it on the next green
# run; a branch-dispatched run never touches main's issue.
- name: Write the marathon report
if: always() && (steps.reconcile.outputs.exit != '0' || needs.eval-slices.result != 'success')
run: |
set -euo pipefail
{
echo "Marathon report (non-blocking lane): branch ${GITHUB_REF_NAME} @ ${GITHUB_SHA}, event ${GITHUB_EVENT_NAME}, run ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
echo
echo "A census went red: triage with ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/blob/${GITHUB_SHA}/docs/evals/census-red.md"
echo
echo "- reconciliation exit: ${{ steps.reconcile.outputs.exit }}"
echo "- marathon slices job: ${{ needs.eval-slices.result }}"
echo
{ tail -c 6000 /tmp/report.txt 2>/dev/null || echo "(no reconciliation output)"; } \
| bun --no-install -e 'import { publishedFence } from "./scripts/lib/published-text";
const lines = (await Bun.stdin.text()).split("\n").filter(Boolean);
console.log(publishedFence(lines.length ? lines : ["(none)"]).join("\n"));'
} > /tmp/marathon-report.md
cat /tmp/marathon-report.md >> "$GITHUB_STEP_SUMMARY"
- name: Upload the marathon report
if: always() && (steps.reconcile.outputs.exit != '0' || needs.eval-slices.result != 'success')
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: marathon-report-a${{ github.run_attempt }}
path: /tmp/marathon-report.md
if-no-files-found: ignore
retention-days: 90
- name: Upsert tracking issue on failure
if: always() && github.ref == 'refs/heads/main' && (steps.reconcile.outputs.exit != '0' || needs.eval-slices.result != 'success')
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -euo pipefail
TITLE="Weekly marathon evals: red lane needs triage"
BODY_FILE=/tmp/issue-body.md
{
echo "Automated weekly marathon report (non-blocking lane) — run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
echo
echo "- reconciliation exit: ${{ steps.reconcile.outputs.exit }}"
echo "- marathon slices job: ${{ needs.eval-slices.result }}"
echo
echo '```'
tail -c 6000 /tmp/report.txt 2>/dev/null || echo "(no reconciliation output)"
echo '```'
} > "$BODY_FILE"
BODY_FILE=/tmp/marathon-report.md
[ -s "$BODY_FILE" ] || echo "Marathon report missing; run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" > "$BODY_FILE"
EXISTING=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "in:title \"$TITLE\"" --json number --jq '.[0].number // empty')
if [ -n "$EXISTING" ]; then
gh issue comment "$EXISTING" --repo "$GITHUB_REPOSITORY" --body-file "$BODY_FILE"
@@ -310,6 +332,18 @@ jobs:
gh issue create --repo "$GITHUB_REPOSITORY" --title "$TITLE" --body-file "$BODY_FILE"
fi
- name: Close the tracking issue on a green run
if: always() && github.ref == 'refs/heads/main' && steps.reconcile.outputs.exit == '0' && needs.eval-slices.result == 'success'
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -euo pipefail
TITLE="Weekly marathon evals: red lane needs triage"
EXISTING=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "in:title \"$TITLE\"" --json number --jq '.[0].number // empty')
if [ -n "$EXISTING" ]; then
gh issue close "$EXISTING" --repo "$GITHUB_REPOSITORY" --comment "Green marathon run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
fi
- name: Fail the workflow when reconciliation failed
if: always() && (steps.reconcile.outputs.exit != '0' || needs.eval-slices.result != 'success')
run: exit 1
.github/workflows/evals-periodic.yml
+54 -19
View File
@@ -627,7 +627,7 @@ jobs:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set +e
bun run eval:pass-rates --gate --runs 10 > /tmp/pass-rates.txt 2>&1
bun run eval:pass-rates --gate --runs 10 --branch "$GITHUB_REF_NAME" > /tmp/pass-rates.txt 2>&1
echo "exit=$?" >> "$GITHUB_OUTPUT"
cat /tmp/pass-rates.txt
@@ -652,10 +652,13 @@ jobs:
echo "redispatch=false" >> "$GITHUB_OUTPUT"
fi
# A red weekly lane nobody must action is waste — upsert ONE tracking
# issue (never a new issue per week) with the headline and failure block
# of both lanes, and close it on the next green run.
- name: Upsert tracking issue on failure
# A red census writes one report on every ref: the step summary and the
# census-report artifact. Only main (scheduled weekly, or dispatched on
# main) upserts ONE tracking issue (never a new issue per week) and
# closes it on the next green run; a branch-dispatched census never
# comments on or closes main's issue. Every report-derived line goes
# through the one published-text sanitizer (scripts/lib/published-text.ts).
- name: Write the census report
if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success')
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -663,17 +666,28 @@ jobs:
REDISPATCH_OF: ${{ inputs.redispatch_of }}
run: |
set -euo pipefail
TITLE="Weekly periodic evals: red lane needs triage"
BODY_FILE=/tmp/issue-body.md
published() { # $1 fence|line: stdin lines through the published-text sanitizer
bun --no-install -e 'import { publishedFence, sanitizePublishedText } from "./scripts/lib/published-text";
const lines = (await Bun.stdin.text()).split("\n").filter(Boolean);
console.log(process.argv[1] === "line" ? sanitizePublishedText(lines.join(" ")) : publishedFence(lines.length ? lines : ["(none)"]).join("\n"));' "$1"
}
RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
GUIDE_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/blob/${GITHUB_SHA}/docs/evals/census-red.md"
bun run eval:pass-rates --reds --runs 10 --branch "$GITHUB_REF_NAME" > /tmp/pass-rates-reds.txt 2>&1 \
|| echo "history unavailable: eval:pass-rates --reds exited $?" >> /tmp/pass-rates-reds.txt
bun run eval:pass-rates --headroom --runs 10 --branch "$GITHUB_REF_NAME" > /tmp/pass-rates-headroom.txt 2>&1 \
|| echo "history unavailable: eval:pass-rates --headroom exited $?" >> /tmp/pass-rates-headroom.txt
{
echo "Automated weekly report — run: ${RUN_URL}"
echo "Periodic census report: branch ${GITHUB_REF_NAME} @ ${GITHUB_SHA}, event ${GITHUB_EVENT_NAME}, run ${RUN_URL}"
echo
echo "A census went red: triage with ${GUIDE_URL}"
if [ -n "$REDISPATCH_OF" ]; then echo; echo "This run is the one INFRA/INCOMPLETE re-dispatch of run ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${REDISPATCH_OF}; both runs are reported."; fi
if [ "$REDISPATCH" = "true" ]; then echo; echo "Every red verdict is machine-classified INFRA/INCOMPLETE: re-dispatching once as a new run (EVAL_POLICY.infraRedispatch). This run stays red and reported."; fi
echo
echo "- periodic reconciliation exit: ${{ steps.reconcile.outputs.exit }} (slices job: ${{ needs.eval-slices.result }}, Codex slices job: ${{ needs.eval-codex-slices.result }})"
echo "- gate census reconciliation exit: ${{ steps.gate-reconcile.outputs.exit }} (census job: ${{ needs.gate-census.result }})"
echo "- pass-rate history gate exit: ${{ steps.pass-rates.outputs.exit }}"
echo "- $({ grep -E '^all-green probability' /tmp/pass-rates-reds.txt || echo 'all-green probability: unavailable (see the red ledger)'; } | published line)"
echo
echo "### Periodic lane"
cat /tmp/paid-report/report-summary.md 2>/dev/null || echo "(no periodic report summary)"
@@ -682,23 +696,44 @@ jobs:
cat /tmp/gate-census-report/report-summary.md 2>/dev/null || echo "(no gate census report summary)"
echo
echo "### Pass-rate history (ACTION REQUIRED)"
echo '```'
{ grep -E 'ACTION REQUIRED|history unavailable' /tmp/pass-rates.txt || echo "(no pass-rate alarms)"; } | sed 's/@/@\xe2\x80\x8b/g' | head -c 6000
echo '```'
{ awk '/^ACTION REQUIRED/ { on = 1; print; next } on && /^ \[/ { print; next } { on = 0 } /history unavailable/ { print }' /tmp/pass-rates.txt 2>/dev/null \
| grep . || echo "(no pass-rate alarms)"; } | head -c 6000 | published fence
echo
echo "### Red ledger: bun run eval:pass-rates --reds --runs 10 --branch ${GITHUB_REF_NAME}"
{ head -n 1 /tmp/pass-rates-reds.txt; tail -n +2 /tmp/pass-rates-reds.txt | tail -n 60; } | cut -c1-400 | published fence
echo
echo "### Session headroom: bun run eval:pass-rates --headroom --runs 10 --branch ${GITHUB_REF_NAME}"
head -n 40 /tmp/pass-rates-headroom.txt | cut -c1-400 | published fence
echo
echo "<details><summary>Full reconciliation output</summary>"
echo
echo '```'
tail -c 6000 /tmp/report.txt 2>/dev/null | sed 's/@/@\xe2\x80\x8b/g' || echo "(no reconciliation output)"
echo '```'
{ tail -c 6000 /tmp/report.txt 2>/dev/null || echo "(no reconciliation output)"; } | published fence
echo
echo '```'
tail -c 6000 /tmp/gate-report.txt 2>/dev/null | sed 's/@/@\xe2\x80\x8b/g' || echo "(no gate census reconciliation output)"
echo '```'
{ tail -c 6000 /tmp/gate-report.txt 2>/dev/null || echo "(no gate census reconciliation output)"; } | published fence
echo "</details>"
echo
echo "Policy: EVAL_POLICY and CASE_QUARANTINE in test/helpers/periodic-exclude-data.ts; history: \`bun run eval:pass-rates\`."
} > "$BODY_FILE"
} > /tmp/census-report.md
cat /tmp/census-report.md >> "$GITHUB_STEP_SUMMARY"
- name: Upload the census report
if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success')
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: census-report-a${{ github.run_attempt }}
path: /tmp/census-report.md
if-no-files-found: ignore
retention-days: 90
- name: Upsert tracking issue on failure
if: always() && github.ref == 'refs/heads/main' && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success')
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -euo pipefail
TITLE="Weekly periodic evals: red lane needs triage"
BODY_FILE=/tmp/census-report.md
[ -s "$BODY_FILE" ] || echo "Periodic census report missing; run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" > "$BODY_FILE"
EXISTING=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "in:title \"$TITLE\"" --json number --jq '.[0].number // empty')
if [ -n "$EXISTING" ]; then
gh issue comment "$EXISTING" --repo "$GITHUB_REPOSITORY" --body-file "$BODY_FILE"
@@ -708,7 +743,7 @@ jobs:
fi
- name: Close the tracking issue on a green run
if: always() && steps.reconcile.outputs.exit == '0' && steps.gate-reconcile.outputs.exit == '0' && steps.pass-rates.outputs.exit == '0' && needs.eval-slices.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) && needs.gate-census.result == 'success'
if: always() && github.ref == 'refs/heads/main' && steps.reconcile.outputs.exit == '0' && steps.gate-reconcile.outputs.exit == '0' && steps.pass-rates.outputs.exit == '0' && needs.eval-slices.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) && needs.gate-census.result == 'success'
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
REDISPATCH_OF: ${{ inputs.redispatch_of }}
test/evals-tracking-issue.test.ts
+193
View File
@@ -0,0 +1,193 @@
/**
* The weekly tracking issues (evals-periodic.yml, evals-marathon.yml).
*
* Only main posts to or closes a tracking issue: a branch-dispatched census
* writes the same report to its step summary and a report artifact instead.
* The periodic report carries the full pass-rate alarm lines, the red ledger,
* session headroom and the all-green estimate, read with an explicit
* --branch, and every report-derived line passes the published-text
* sanitizer. The write steps run here as shell scripts with a fake
* `eval:pass-rates`; the sanitizer is the real one.
*/
import { describe, test, expect } from 'bun:test';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { spawnSync } from 'child_process';
const ROOT = path.join(import.meta.dir, '..');
type Step = { name?: string; if?: string; run?: string; uses?: string; with?: Record<string, string> };
type Workflow = { jobs: Record<string, { steps?: Step[] }> };
const WORKFLOWS = ['evals-periodic.yml', 'evals-marathon.yml'] as const;
const parse = (name: string) => Bun.YAML.parse(fs.readFileSync(path.join(ROOT, '.github/workflows', name), 'utf8')) as Workflow;
const MAIN_ONLY = "github.ref == 'refs/heads/main'";
describe('tracking issues are main-only', () => {
for (const name of WORKFLOWS) {
test(`${name}: only the upsert and close steps call gh issue, and both run on main only`, () => {
const issueSteps = Object.values(parse(name).jobs).flatMap(job => job.steps ?? []).filter(step => /\bgh issue\b/.test(step.run ?? ''));
expect(issueSteps.map(step => step.name).sort()).toEqual(['Close the tracking issue on a green run', 'Upsert tracking issue on failure']);
for (const step of issueSteps) expect(step.if, step.name).toContain(`always() && ${MAIN_ONLY} && `);
const upsert = issueSteps.find(step => step.name === 'Upsert tracking issue on failure')!;
expect(upsert.run).toMatch(/gh issue comment .*--body-file "\$BODY_FILE"/);
expect(upsert.run).toMatch(/gh issue create .*--body-file "\$BODY_FILE"/);
expect(issueSteps.find(step => step.name === 'Close the tracking issue on a green run')!.run).toContain('gh issue close');
});
test(`${name}: every ref writes the report to the step summary and an artifact`, () => {
const steps = parse(name).jobs.report!.steps!;
const write = steps.find(step => /^Write the (census|marathon) report$/.test(step.name ?? ''))!;
const upload = steps.find(step => /^Upload the (census|marathon) report$/.test(step.name ?? ''))!;
const upsert = steps.find(step => step.name === 'Upsert tracking issue on failure')!;
expect(write.if).not.toContain('github.ref');
expect(upload.if).toBe(write.if);
expect(upsert.if).toBe(write.if!.replace('always() && ', `always() && ${MAIN_ONLY} && `));
expect(write.run).toContain('>> "$GITHUB_STEP_SUMMARY"');
expect(write.run).toContain('docs/evals/census-red.md');
expect(write.run).toContain('./scripts/lib/published-text');
const reportFile = /> (\/tmp\/[\w-]+\.md)\n/.exec(write.run!)![1]!;
expect(upload.uses).toStartWith('actions/upload-artifact@');
expect(upload.with!.path).toBe(reportFile);
expect(upload.with!.name).toMatch(/^(census|marathon)-report-a\$\{\{ github\.run_attempt \}\}$/);
expect(upsert.run).toContain(`BODY_FILE=${reportFile}`);
});
}
test('evals-periodic.yml: every pass-rates read names its branch explicitly', () => {
const runs = parse('evals-periodic.yml').jobs.report!.steps!.map(step => step.run ?? '').join('\n');
const calls = [...runs.matchAll(/^\s*bun run eval:pass-rates [^\n>]*/gm)].map(m => m[0].trim());
expect(calls.length).toBe(3);
for (const call of calls) expect(call).toContain('--runs 10 --branch "$GITHUB_REF_NAME"');
expect(calls.some(call => call.includes('--gate'))).toBe(true);
expect(calls.some(call => call.includes('--reds'))).toBe(true);
expect(calls.some(call => call.includes('--headroom'))).toBe(true);
});
});
/** Run a workflow step's script with its expressions and /tmp paths bound to a scratch dir. */
function runStep(workflow: string, stepName: string, files: Record<string, string>, expressions: Record<string, string>) {
const step = parse(workflow).jobs.report!.steps!.find(s => s.name === stepName)!;
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'tracking-issue-'));
const work = path.join(dir, 'tmp');
const bin = path.join(dir, 'bin');
fs.mkdirSync(bin);
for (const [rel, body] of Object.entries(files)) {
fs.mkdirSync(path.dirname(path.join(work, rel)), { recursive: true });
fs.writeFileSync(path.join(work, rel), body);
}
const log = path.join(dir, 'pass-rates-calls.log');
fs.writeFileSync(path.join(bin, 'bun'), [
'#!/bin/sh',
'if [ "$1" = run ] && [ "$2" = eval:pass-rates ]; then',
` echo "$*" >> '${log}'`,
` case " $* " in *" --reds "*) cat '${work}/fake-reds.txt';; *" --headroom "*) cat '${work}/fake-headroom.txt';; esac`,
' exit 0',
'fi',
`exec '${process.execPath}' "$@"`,
].join('\n'), { mode: 0o755 });
const script = step.run!
.replace(/\$\{\{ ([^}]+) \}\}/g, (_, expr: string) => expressions[expr.trim()] ?? `<${expr.trim()}>`)
.replaceAll('/tmp/', `${work}/`);
const summary = path.join(dir, 'step-summary.md');
const result = spawnSync('bash', ['-c', script], {
cwd: ROOT, encoding: 'utf8', timeout: 60_000,
env: { PATH: `${bin}:${process.env.PATH}`, HOME: dir, GITHUB_STEP_SUMMARY: summary, GITHUB_REF_NAME: 'feature/x',
GITHUB_SHA: 'abc1234', GITHUB_EVENT_NAME: 'workflow_dispatch', GITHUB_SERVER_URL: 'https://github.com',
GITHUB_REPOSITORY: 'garrytan/gstack', GITHUB_RUN_ID: '42', REDISPATCH: 'false', REDISPATCH_OF: '' },
});
const read = (file: string) => (fs.existsSync(file) ? fs.readFileSync(file, 'utf8') : '');
const reports = Object.fromEntries(fs.readdirSync(work).filter(name => name.endsWith('.md')).map(name => [name, read(path.join(work, name))]));
const out = { status: result.status, stderr: result.stderr, summary: read(summary), calls: read(log), body: (name: string) => reports[name] ?? '' };
fs.rmSync(dir, { recursive: true, force: true });
return out;
}
const PERIODIC_EXPRESSIONS = {
'steps.reconcile.outputs.exit': '1', 'steps.gate-reconcile.outputs.exit': '0', 'steps.pass-rates.outputs.exit': '1',
'needs.eval-slices.result': 'failure', 'needs.eval-codex-slices.result': 'success', 'needs.gate-census.result': 'success',
};
describe('the periodic census report', () => {
const passRates = [
'scope: garrytan/gstack evals-periodic.yml on feature/x + main, last 10 completed run(s) per branch: 1, 2',
'pass-rates: policy v1, 40 post-policy trial(s), 0 pre-policy (display only)',
' label kind tier current series pre-policy manual case',
' ACTION rule gate 9/10 [59.6%–98.2%] - 0 noisy-case',
'ACTION REQUIRED (3):',
' [drift] noisy-case passes 9/10 (below 95% over >= 10 trials): fix it, or propose a CASE_QUARANTINE entry with a written diagnosis',
' [headroom] slow-case session sdk:slow-case#1: max 520s of 600s (87%) above the 85% cap; cut work, never raise the budget',
' [drift] mention @attacker ``` </details><img src=x onerror=alert(1)> ghp_' + 'a'.repeat(36),
'free-suite flaky-passes (/home/x/.gstack/flake-ledger.jsonl):',
' 2x test/unrelated.test.ts',
].join('\n');
const reds = [
'scope: garrytan/gstack evals-periodic.yml on feature/x + main, last 10 completed run(s) per branch: 1, 2',
'reds: verdict reds per census lane by failure class and machine cause',
' 1 periodic/full census: 1 red of 146 verdicts — class: assertion 1; cause: provider_stall 1',
' ✗ noisy-case FAIL t1 assertion/provider_stall — Error: expect(received).toBe(expected)',
'pooled: 1 red of 146 verdicts = 0.7% per verdict [95% 0.1%–3.8%]',
'all-green probability (approximation, assumes independent verdicts): 81.2% for the latest census\'s 146 verdicts [55%–95% with the pooled rate\'s 95% interval as prior]; formula Π(1 - p_i) over its cases; rerun: bun run eval:pass-rates --reds',
].join('\n');
const headroom = [
'scope: garrytan/gstack evals-periodic.yml on feature/x + main',
'headroom: slowest armed session per case (warn > 75%, alarm > 85%; insufficient below 3 samples; censored = timed out)',
' ALARM 520s/600s (87%) 3 540s slow-case',
].join('\n');
const files = {
'pass-rates.txt': passRates, 'fake-reds.txt': reds, 'fake-headroom.txt': headroom,
'paid-report/report-summary.md': '## Periodic summary\n✗ noisy-case', 'report.txt': 'reconcile tail',
};
test('carries the full alarm lines, red ledger, headroom and all-green estimate, sanitized, for an explicit branch', () => {
const run = runStep('evals-periodic.yml', 'Write the census report', files, PERIODIC_EXPRESSIONS);
expect(run.stderr).toBe('');
expect(run.status).toBe(0);
const body = run.body('census-report.md');
expect(run.summary).toBe(body);
expect(body).toStartWith('Periodic census report: branch feature/x @ abc1234, event workflow_dispatch, run https://github.com/garrytan/gstack/actions/runs/42');
expect(body).toContain('triage with https://github.com/garrytan/gstack/blob/abc1234/docs/evals/census-red.md');
expect(body).toContain('- periodic reconciliation exit: 1 (slices job: failure, Codex slices job: success)');
expect(body).toContain('- all-green probability (approximation, assumes independent verdicts): 81.2% for the latest census');
expect(body).toContain('## Periodic summary');
// Every alarm line, not just the header; the flaky-pass block is not an alarm.
expect(body).toContain('ACTION REQUIRED (3):');
expect(body).toContain('[drift] noisy-case passes 9/10 (below 95% over >= 10 trials): fix it');
expect(body).toContain('[headroom] slow-case session sdk:slow-case#1: max 520s of 600s (87%) above the 85% cap');
expect(body).not.toContain('test/unrelated.test.ts');
expect(body).toContain('✗ noisy-case FAIL t1 assertion/provider_stall — Error: expect(received).toBe(expected)');
expect(body).toContain('ALARM 520s/600s (87%)');
// Hostile alarm text stays literal inside a fence it cannot close; mentions and tokens are neutralized.
const hostile = body.split('\n').findIndex(line => line.includes('mention @'));
const fence = body.split('\n').slice(0, hostile).reverse().find(line => /^`{3,}$/.test(line))!;
expect(fence.length).toBeGreaterThan(3);
expect(body).toContain('@\u200battacker');
expect(body).not.toContain('ghp_' + 'a'.repeat(36));
expect(run.calls.trim().split('\n')).toEqual([
'run eval:pass-rates --reds --runs 10 --branch feature/x',
'run eval:pass-rates --headroom --runs 10 --branch feature/x',
]);
});
test('missing history becomes a visible line, never a failed step', () => {
const run = runStep('evals-periodic.yml', 'Write the census report', { 'fake-reds.txt': '', 'fake-headroom.txt': '' }, PERIODIC_EXPRESSIONS);
expect(run.status).toBe(0);
const body = run.body('census-report.md');
expect(body).toContain('all-green probability: unavailable (see the red ledger)');
expect(body).toContain('(no pass-rate alarms)');
expect(body).toContain('(no periodic report summary)');
});
});
describe('the marathon report', () => {
test('fences the sanitized reconciliation output and links the guide', () => {
const run = runStep('evals-marathon.yml', 'Write the marathon report', { 'report.txt': 'red slice\nping @someone ```' },
{ 'steps.reconcile.outputs.exit': '1', 'needs.eval-slices.result': 'success' });
expect(run.status).toBe(0);
const body = run.body('marathon-report.md');
expect(run.summary).toBe(body);
expect(body).toStartWith('Marathon report (non-blocking lane): branch feature/x @ abc1234, event workflow_dispatch');
expect(body).toContain('docs/evals/census-red.md');
expect(body).toContain('- reconciliation exit: 1');
expect(body).toContain('````\nred slice\nping @\u200bsomeone ```\n````');
});
});
test/evals-workflow-wiring.test.ts
+3 -3
View File
@@ -377,11 +377,11 @@ describe('panel verdict surfaces (eval reliability policy)', () => {
const gate = report.steps.find(step => step.id === 'pass-rates')!;
expect(gate.run).toContain('bun run eval:pass-rates --gate --runs 10');
expect(gate.if).toBe('always()');
for (const name of ['Upsert tracking issue on failure', 'Fail the workflow when reconciliation failed']) {
for (const name of ['Write the census report', 'Upsert tracking issue on failure', 'Fail the workflow when reconciliation failed']) {
expect(report.steps.find(step => step.name === name)!.if).toContain("steps.pass-rates.outputs.exit != '0'");
}
const upsert = report.steps.find(step => step.name === 'Upsert tracking issue on failure')!;
expect(upsert.run).toContain('report-summary.md');
// The report body is written once on every ref; the main-only upsert posts it (test/evals-tracking-issue.test.ts).
expect(report.steps.find(step => step.name === 'Write the census report')!.run).toContain('report-summary.md');
expect(report.steps.find(step => step.name === 'Close the tracking issue on a green run')!.run).toContain('gh issue close');
expect(report.steps.filter(step => step.with?.name?.startsWith('trial-outcomes-')).length).toBe(2);
const redispatch = jobs.redispatch!;