From 7814a6d7afd9342cf11b6bcc2d49213395dba5b2 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sun, 4 Oct 2026 18:33:48 +0000 Subject: [PATCH 1/4] 0.3: only main posts or closes tracking issues; reports carry alarms, red ledger, headroom and all-green A branch-dispatched periodic or marathon run writes its report to the step summary and a report artifact; the upsert and close steps run only on refs/heads/main. The periodic body now keeps every pass-rate alarm line (the old grep kept only the header), adds the red ledger and session headroom read with an explicit --branch, the all-green estimate and the triage guide link, and routes report text through scripts/lib/published-text.ts. The marathon lane gains the matching close-on-green step. Migrated: evals-workflow-wiring pinned report-summary.md in the upsert step; the body is now written by 'Write the census report'. --- .github/workflows/evals-marathon.yml | 60 +++++++-- .github/workflows/evals-periodic.yml | 73 +++++++--- test/evals-tracking-issue.test.ts | 193 +++++++++++++++++++++++++++ test/evals-workflow-wiring.test.ts | 6 +- 4 files changed, 297 insertions(+), 35 deletions(-) create mode 100644 test/evals-tracking-issue.test.ts diff --git a/.github/workflows/evals-marathon.yml b/.github/workflows/evals-marathon.yml index 9aa0ba98c..430186927 100644 --- a/.github/workflows/evals-marathon.yml +++ b/.github/workflows/evals-marathon.yml @@ -283,25 +283,47 @@ jobs: # PIPESTATUS[0], NOT $?: the default step shell has no pipefail. echo "exit=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT" - # One tracking issue for the whole lane (never one per week). - - name: Upsert tracking issue on failure + # A red marathon run writes its report to the step summary and the + # marathon-report artifact on every ref. Only main upserts the lane's ONE + # tracking issue (never one per week) and closes it on the next green + # run; a branch-dispatched run never touches main's issue. + - name: Write the marathon report if: always() && (steps.reconcile.outputs.exit != '0' || needs.eval-slices.result != 'success') + run: | + set -euo pipefail + { + echo "Marathon report (non-blocking lane): branch ${GITHUB_REF_NAME} @ ${GITHUB_SHA}, event ${GITHUB_EVENT_NAME}, run ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + echo + echo "A census went red: triage with ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/blob/${GITHUB_SHA}/docs/evals/census-red.md" + echo + echo "- reconciliation exit: ${{ steps.reconcile.outputs.exit }}" + echo "- marathon slices job: ${{ needs.eval-slices.result }}" + echo + { tail -c 6000 /tmp/report.txt 2>/dev/null || echo "(no reconciliation output)"; } \ + | bun --no-install -e 'import { publishedFence } from "./scripts/lib/published-text"; + const lines = (await Bun.stdin.text()).split("\n").filter(Boolean); + console.log(publishedFence(lines.length ? lines : ["(none)"]).join("\n"));' + } > /tmp/marathon-report.md + cat /tmp/marathon-report.md >> "$GITHUB_STEP_SUMMARY" + + - name: Upload the marathon report + if: always() && (steps.reconcile.outputs.exit != '0' || needs.eval-slices.result != 'success') + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: marathon-report-a${{ github.run_attempt }} + path: /tmp/marathon-report.md + if-no-files-found: ignore + retention-days: 90 + + - name: Upsert tracking issue on failure + if: always() && github.ref == 'refs/heads/main' && (steps.reconcile.outputs.exit != '0' || needs.eval-slices.result != 'success') env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -euo pipefail TITLE="Weekly marathon evals: red lane needs triage" - BODY_FILE=/tmp/issue-body.md - { - echo "Automated weekly marathon report (non-blocking lane) — run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" - echo - echo "- reconciliation exit: ${{ steps.reconcile.outputs.exit }}" - echo "- marathon slices job: ${{ needs.eval-slices.result }}" - echo - echo '```' - tail -c 6000 /tmp/report.txt 2>/dev/null || echo "(no reconciliation output)" - echo '```' - } > "$BODY_FILE" + BODY_FILE=/tmp/marathon-report.md + [ -s "$BODY_FILE" ] || echo "Marathon report missing; run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" > "$BODY_FILE" EXISTING=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "in:title \"$TITLE\"" --json number --jq '.[0].number // empty') if [ -n "$EXISTING" ]; then gh issue comment "$EXISTING" --repo "$GITHUB_REPOSITORY" --body-file "$BODY_FILE" @@ -310,6 +332,18 @@ jobs: gh issue create --repo "$GITHUB_REPOSITORY" --title "$TITLE" --body-file "$BODY_FILE" fi + - name: Close the tracking issue on a green run + if: always() && github.ref == 'refs/heads/main' && steps.reconcile.outputs.exit == '0' && needs.eval-slices.result == 'success' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + TITLE="Weekly marathon evals: red lane needs triage" + EXISTING=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "in:title \"$TITLE\"" --json number --jq '.[0].number // empty') + if [ -n "$EXISTING" ]; then + gh issue close "$EXISTING" --repo "$GITHUB_REPOSITORY" --comment "Green marathon run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + fi + - name: Fail the workflow when reconciliation failed if: always() && (steps.reconcile.outputs.exit != '0' || needs.eval-slices.result != 'success') run: exit 1 diff --git a/.github/workflows/evals-periodic.yml b/.github/workflows/evals-periodic.yml index 027533301..464b2b45d 100644 --- a/.github/workflows/evals-periodic.yml +++ b/.github/workflows/evals-periodic.yml @@ -627,7 +627,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set +e - bun run eval:pass-rates --gate --runs 10 > /tmp/pass-rates.txt 2>&1 + bun run eval:pass-rates --gate --runs 10 --branch "$GITHUB_REF_NAME" > /tmp/pass-rates.txt 2>&1 echo "exit=$?" >> "$GITHUB_OUTPUT" cat /tmp/pass-rates.txt @@ -652,10 +652,13 @@ jobs: echo "redispatch=false" >> "$GITHUB_OUTPUT" fi - # A red weekly lane nobody must action is waste — upsert ONE tracking - # issue (never a new issue per week) with the headline and failure block - # of both lanes, and close it on the next green run. - - name: Upsert tracking issue on failure + # A red census writes one report on every ref: the step summary and the + # census-report artifact. Only main (scheduled weekly, or dispatched on + # main) upserts ONE tracking issue (never a new issue per week) and + # closes it on the next green run; a branch-dispatched census never + # comments on or closes main's issue. Every report-derived line goes + # through the one published-text sanitizer (scripts/lib/published-text.ts). + - name: Write the census report if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success') env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -663,17 +666,28 @@ jobs: REDISPATCH_OF: ${{ inputs.redispatch_of }} run: | set -euo pipefail - TITLE="Weekly periodic evals: red lane needs triage" - BODY_FILE=/tmp/issue-body.md + published() { # $1 fence|line: stdin lines through the published-text sanitizer + bun --no-install -e 'import { publishedFence, sanitizePublishedText } from "./scripts/lib/published-text"; + const lines = (await Bun.stdin.text()).split("\n").filter(Boolean); + console.log(process.argv[1] === "line" ? sanitizePublishedText(lines.join(" ")) : publishedFence(lines.length ? lines : ["(none)"]).join("\n"));' "$1" + } RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + GUIDE_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/blob/${GITHUB_SHA}/docs/evals/census-red.md" + bun run eval:pass-rates --reds --runs 10 --branch "$GITHUB_REF_NAME" > /tmp/pass-rates-reds.txt 2>&1 \ + || echo "history unavailable: eval:pass-rates --reds exited $?" >> /tmp/pass-rates-reds.txt + bun run eval:pass-rates --headroom --runs 10 --branch "$GITHUB_REF_NAME" > /tmp/pass-rates-headroom.txt 2>&1 \ + || echo "history unavailable: eval:pass-rates --headroom exited $?" >> /tmp/pass-rates-headroom.txt { - echo "Automated weekly report — run: ${RUN_URL}" + echo "Periodic census report: branch ${GITHUB_REF_NAME} @ ${GITHUB_SHA}, event ${GITHUB_EVENT_NAME}, run ${RUN_URL}" + echo + echo "A census went red: triage with ${GUIDE_URL}" if [ -n "$REDISPATCH_OF" ]; then echo; echo "This run is the one INFRA/INCOMPLETE re-dispatch of run ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${REDISPATCH_OF}; both runs are reported."; fi if [ "$REDISPATCH" = "true" ]; then echo; echo "Every red verdict is machine-classified INFRA/INCOMPLETE: re-dispatching once as a new run (EVAL_POLICY.infraRedispatch). This run stays red and reported."; fi echo echo "- periodic reconciliation exit: ${{ steps.reconcile.outputs.exit }} (slices job: ${{ needs.eval-slices.result }}, Codex slices job: ${{ needs.eval-codex-slices.result }})" echo "- gate census reconciliation exit: ${{ steps.gate-reconcile.outputs.exit }} (census job: ${{ needs.gate-census.result }})" echo "- pass-rate history gate exit: ${{ steps.pass-rates.outputs.exit }}" + echo "- $({ grep -E '^all-green probability' /tmp/pass-rates-reds.txt || echo 'all-green probability: unavailable (see the red ledger)'; } | published line)" echo echo "### Periodic lane" cat /tmp/paid-report/report-summary.md 2>/dev/null || echo "(no periodic report summary)" @@ -682,23 +696,44 @@ jobs: cat /tmp/gate-census-report/report-summary.md 2>/dev/null || echo "(no gate census report summary)" echo echo "### Pass-rate history (ACTION REQUIRED)" - echo '```' - { grep -E 'ACTION REQUIRED|history unavailable' /tmp/pass-rates.txt || echo "(no pass-rate alarms)"; } | sed 's/@/@\xe2\x80\x8b/g' | head -c 6000 - echo '```' + { awk '/^ACTION REQUIRED/ { on = 1; print; next } on && /^ \[/ { print; next } { on = 0 } /history unavailable/ { print }' /tmp/pass-rates.txt 2>/dev/null \ + | grep . || echo "(no pass-rate alarms)"; } | head -c 6000 | published fence + echo + echo "### Red ledger: bun run eval:pass-rates --reds --runs 10 --branch ${GITHUB_REF_NAME}" + { head -n 1 /tmp/pass-rates-reds.txt; tail -n +2 /tmp/pass-rates-reds.txt | tail -n 60; } | cut -c1-400 | published fence + echo + echo "### Session headroom: bun run eval:pass-rates --headroom --runs 10 --branch ${GITHUB_REF_NAME}" + head -n 40 /tmp/pass-rates-headroom.txt | cut -c1-400 | published fence echo echo "
Full reconciliation output" echo - echo '```' - tail -c 6000 /tmp/report.txt 2>/dev/null | sed 's/@/@\xe2\x80\x8b/g' || echo "(no reconciliation output)" - echo '```' + { tail -c 6000 /tmp/report.txt 2>/dev/null || echo "(no reconciliation output)"; } | published fence echo - echo '```' - tail -c 6000 /tmp/gate-report.txt 2>/dev/null | sed 's/@/@\xe2\x80\x8b/g' || echo "(no gate census reconciliation output)" - echo '```' + { tail -c 6000 /tmp/gate-report.txt 2>/dev/null || echo "(no gate census reconciliation output)"; } | published fence echo "
" echo echo "Policy: EVAL_POLICY and CASE_QUARANTINE in test/helpers/periodic-exclude-data.ts; history: \`bun run eval:pass-rates\`." - } > "$BODY_FILE" + } > /tmp/census-report.md + cat /tmp/census-report.md >> "$GITHUB_STEP_SUMMARY" + + - name: Upload the census report + if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success') + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: census-report-a${{ github.run_attempt }} + path: /tmp/census-report.md + if-no-files-found: ignore + retention-days: 90 + + - name: Upsert tracking issue on failure + if: always() && github.ref == 'refs/heads/main' && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success') + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + TITLE="Weekly periodic evals: red lane needs triage" + BODY_FILE=/tmp/census-report.md + [ -s "$BODY_FILE" ] || echo "Periodic census report missing; run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" > "$BODY_FILE" EXISTING=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "in:title \"$TITLE\"" --json number --jq '.[0].number // empty') if [ -n "$EXISTING" ]; then gh issue comment "$EXISTING" --repo "$GITHUB_REPOSITORY" --body-file "$BODY_FILE" @@ -708,7 +743,7 @@ jobs: fi - name: Close the tracking issue on a green run - if: always() && steps.reconcile.outputs.exit == '0' && steps.gate-reconcile.outputs.exit == '0' && steps.pass-rates.outputs.exit == '0' && needs.eval-slices.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) && needs.gate-census.result == 'success' + if: always() && github.ref == 'refs/heads/main' && steps.reconcile.outputs.exit == '0' && steps.gate-reconcile.outputs.exit == '0' && steps.pass-rates.outputs.exit == '0' && needs.eval-slices.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) && needs.gate-census.result == 'success' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} REDISPATCH_OF: ${{ inputs.redispatch_of }} diff --git a/test/evals-tracking-issue.test.ts b/test/evals-tracking-issue.test.ts new file mode 100644 index 000000000..0e48535ae --- /dev/null +++ b/test/evals-tracking-issue.test.ts @@ -0,0 +1,193 @@ +/** + * The weekly tracking issues (evals-periodic.yml, evals-marathon.yml). + * + * Only main posts to or closes a tracking issue: a branch-dispatched census + * writes the same report to its step summary and a report artifact instead. + * The periodic report carries the full pass-rate alarm lines, the red ledger, + * session headroom and the all-green estimate, read with an explicit + * --branch, and every report-derived line passes the published-text + * sanitizer. The write steps run here as shell scripts with a fake + * `eval:pass-rates`; the sanitizer is the real one. + */ +import { describe, test, expect } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { spawnSync } from 'child_process'; + +const ROOT = path.join(import.meta.dir, '..'); +type Step = { name?: string; if?: string; run?: string; uses?: string; with?: Record }; +type Workflow = { jobs: Record }; +const WORKFLOWS = ['evals-periodic.yml', 'evals-marathon.yml'] as const; +const parse = (name: string) => Bun.YAML.parse(fs.readFileSync(path.join(ROOT, '.github/workflows', name), 'utf8')) as Workflow; +const MAIN_ONLY = "github.ref == 'refs/heads/main'"; + +describe('tracking issues are main-only', () => { + for (const name of WORKFLOWS) { + test(`${name}: only the upsert and close steps call gh issue, and both run on main only`, () => { + const issueSteps = Object.values(parse(name).jobs).flatMap(job => job.steps ?? []).filter(step => /\bgh issue\b/.test(step.run ?? '')); + expect(issueSteps.map(step => step.name).sort()).toEqual(['Close the tracking issue on a green run', 'Upsert tracking issue on failure']); + for (const step of issueSteps) expect(step.if, step.name).toContain(`always() && ${MAIN_ONLY} && `); + const upsert = issueSteps.find(step => step.name === 'Upsert tracking issue on failure')!; + expect(upsert.run).toMatch(/gh issue comment .*--body-file "\$BODY_FILE"/); + expect(upsert.run).toMatch(/gh issue create .*--body-file "\$BODY_FILE"/); + expect(issueSteps.find(step => step.name === 'Close the tracking issue on a green run')!.run).toContain('gh issue close'); + }); + + test(`${name}: every ref writes the report to the step summary and an artifact`, () => { + const steps = parse(name).jobs.report!.steps!; + const write = steps.find(step => /^Write the (census|marathon) report$/.test(step.name ?? ''))!; + const upload = steps.find(step => /^Upload the (census|marathon) report$/.test(step.name ?? ''))!; + const upsert = steps.find(step => step.name === 'Upsert tracking issue on failure')!; + expect(write.if).not.toContain('github.ref'); + expect(upload.if).toBe(write.if); + expect(upsert.if).toBe(write.if!.replace('always() && ', `always() && ${MAIN_ONLY} && `)); + expect(write.run).toContain('>> "$GITHUB_STEP_SUMMARY"'); + expect(write.run).toContain('docs/evals/census-red.md'); + expect(write.run).toContain('./scripts/lib/published-text'); + const reportFile = /> (\/tmp\/[\w-]+\.md)\n/.exec(write.run!)![1]!; + expect(upload.uses).toStartWith('actions/upload-artifact@'); + expect(upload.with!.path).toBe(reportFile); + expect(upload.with!.name).toMatch(/^(census|marathon)-report-a\$\{\{ github\.run_attempt \}\}$/); + expect(upsert.run).toContain(`BODY_FILE=${reportFile}`); + }); + } + + test('evals-periodic.yml: every pass-rates read names its branch explicitly', () => { + const runs = parse('evals-periodic.yml').jobs.report!.steps!.map(step => step.run ?? '').join('\n'); + const calls = [...runs.matchAll(/^\s*bun run eval:pass-rates [^\n>]*/gm)].map(m => m[0].trim()); + expect(calls.length).toBe(3); + for (const call of calls) expect(call).toContain('--runs 10 --branch "$GITHUB_REF_NAME"'); + expect(calls.some(call => call.includes('--gate'))).toBe(true); + expect(calls.some(call => call.includes('--reds'))).toBe(true); + expect(calls.some(call => call.includes('--headroom'))).toBe(true); + }); +}); + +/** Run a workflow step's script with its expressions and /tmp paths bound to a scratch dir. */ +function runStep(workflow: string, stepName: string, files: Record, expressions: Record) { + const step = parse(workflow).jobs.report!.steps!.find(s => s.name === stepName)!; + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'tracking-issue-')); + const work = path.join(dir, 'tmp'); + const bin = path.join(dir, 'bin'); + fs.mkdirSync(bin); + for (const [rel, body] of Object.entries(files)) { + fs.mkdirSync(path.dirname(path.join(work, rel)), { recursive: true }); + fs.writeFileSync(path.join(work, rel), body); + } + const log = path.join(dir, 'pass-rates-calls.log'); + fs.writeFileSync(path.join(bin, 'bun'), [ + '#!/bin/sh', + 'if [ "$1" = run ] && [ "$2" = eval:pass-rates ]; then', + ` echo "$*" >> '${log}'`, + ` case " $* " in *" --reds "*) cat '${work}/fake-reds.txt';; *" --headroom "*) cat '${work}/fake-headroom.txt';; esac`, + ' exit 0', + 'fi', + `exec '${process.execPath}' "$@"`, + ].join('\n'), { mode: 0o755 }); + const script = step.run! + .replace(/\$\{\{ ([^}]+) \}\}/g, (_, expr: string) => expressions[expr.trim()] ?? `<${expr.trim()}>`) + .replaceAll('/tmp/', `${work}/`); + const summary = path.join(dir, 'step-summary.md'); + const result = spawnSync('bash', ['-c', script], { + cwd: ROOT, encoding: 'utf8', timeout: 60_000, + env: { PATH: `${bin}:${process.env.PATH}`, HOME: dir, GITHUB_STEP_SUMMARY: summary, GITHUB_REF_NAME: 'feature/x', + GITHUB_SHA: 'abc1234', GITHUB_EVENT_NAME: 'workflow_dispatch', GITHUB_SERVER_URL: 'https://github.com', + GITHUB_REPOSITORY: 'garrytan/gstack', GITHUB_RUN_ID: '42', REDISPATCH: 'false', REDISPATCH_OF: '' }, + }); + const read = (file: string) => (fs.existsSync(file) ? fs.readFileSync(file, 'utf8') : ''); + const reports = Object.fromEntries(fs.readdirSync(work).filter(name => name.endsWith('.md')).map(name => [name, read(path.join(work, name))])); + const out = { status: result.status, stderr: result.stderr, summary: read(summary), calls: read(log), body: (name: string) => reports[name] ?? '' }; + fs.rmSync(dir, { recursive: true, force: true }); + return out; +} + +const PERIODIC_EXPRESSIONS = { + 'steps.reconcile.outputs.exit': '1', 'steps.gate-reconcile.outputs.exit': '0', 'steps.pass-rates.outputs.exit': '1', + 'needs.eval-slices.result': 'failure', 'needs.eval-codex-slices.result': 'success', 'needs.gate-census.result': 'success', +}; + +describe('the periodic census report', () => { + const passRates = [ + 'scope: garrytan/gstack evals-periodic.yml on feature/x + main, last 10 completed run(s) per branch: 1, 2', + 'pass-rates: policy v1, 40 post-policy trial(s), 0 pre-policy (display only)', + ' label kind tier current series pre-policy manual case', + ' ACTION rule gate 9/10 [59.6%–98.2%] - 0 noisy-case', + 'ACTION REQUIRED (3):', + ' [drift] noisy-case passes 9/10 (below 95% over >= 10 trials): fix it, or propose a CASE_QUARANTINE entry with a written diagnosis', + ' [headroom] slow-case session sdk:slow-case#1: max 520s of 600s (87%) above the 85% cap; cut work, never raise the budget', + ' [drift] mention @attacker ``` ghp_' + 'a'.repeat(36), + 'free-suite flaky-passes (/home/x/.gstack/flake-ledger.jsonl):', + ' 2x test/unrelated.test.ts', + ].join('\n'); + const reds = [ + 'scope: garrytan/gstack evals-periodic.yml on feature/x + main, last 10 completed run(s) per branch: 1, 2', + 'reds: verdict reds per census lane by failure class and machine cause', + ' 1 periodic/full census: 1 red of 146 verdicts — class: assertion 1; cause: provider_stall 1', + ' ✗ noisy-case FAIL t1 assertion/provider_stall — Error: expect(received).toBe(expected)', + 'pooled: 1 red of 146 verdicts = 0.7% per verdict [95% 0.1%–3.8%]', + 'all-green probability (approximation, assumes independent verdicts): 81.2% for the latest census\'s 146 verdicts [55%–95% with the pooled rate\'s 95% interval as prior]; formula Π(1 - p_i) over its cases; rerun: bun run eval:pass-rates --reds', + ].join('\n'); + const headroom = [ + 'scope: garrytan/gstack evals-periodic.yml on feature/x + main', + 'headroom: slowest armed session per case (warn > 75%, alarm > 85%; insufficient below 3 samples; censored = timed out)', + ' ALARM 520s/600s (87%) 3 540s slow-case', + ].join('\n'); + const files = { + 'pass-rates.txt': passRates, 'fake-reds.txt': reds, 'fake-headroom.txt': headroom, + 'paid-report/report-summary.md': '## Periodic summary\n✗ noisy-case', 'report.txt': 'reconcile tail', + }; + + test('carries the full alarm lines, red ledger, headroom and all-green estimate, sanitized, for an explicit branch', () => { + const run = runStep('evals-periodic.yml', 'Write the census report', files, PERIODIC_EXPRESSIONS); + expect(run.stderr).toBe(''); + expect(run.status).toBe(0); + const body = run.body('census-report.md'); + expect(run.summary).toBe(body); + expect(body).toStartWith('Periodic census report: branch feature/x @ abc1234, event workflow_dispatch, run https://github.com/garrytan/gstack/actions/runs/42'); + expect(body).toContain('triage with https://github.com/garrytan/gstack/blob/abc1234/docs/evals/census-red.md'); + expect(body).toContain('- periodic reconciliation exit: 1 (slices job: failure, Codex slices job: success)'); + expect(body).toContain('- all-green probability (approximation, assumes independent verdicts): 81.2% for the latest census'); + expect(body).toContain('## Periodic summary'); + // Every alarm line, not just the header; the flaky-pass block is not an alarm. + expect(body).toContain('ACTION REQUIRED (3):'); + expect(body).toContain('[drift] noisy-case passes 9/10 (below 95% over >= 10 trials): fix it'); + expect(body).toContain('[headroom] slow-case session sdk:slow-case#1: max 520s of 600s (87%) above the 85% cap'); + expect(body).not.toContain('test/unrelated.test.ts'); + expect(body).toContain('✗ noisy-case FAIL t1 assertion/provider_stall — Error: expect(received).toBe(expected)'); + expect(body).toContain('ALARM 520s/600s (87%)'); + // Hostile alarm text stays literal inside a fence it cannot close; mentions and tokens are neutralized. + const hostile = body.split('\n').findIndex(line => line.includes('mention @')); + const fence = body.split('\n').slice(0, hostile).reverse().find(line => /^`{3,}$/.test(line))!; + expect(fence.length).toBeGreaterThan(3); + expect(body).toContain('@\u200battacker'); + expect(body).not.toContain('ghp_' + 'a'.repeat(36)); + expect(run.calls.trim().split('\n')).toEqual([ + 'run eval:pass-rates --reds --runs 10 --branch feature/x', + 'run eval:pass-rates --headroom --runs 10 --branch feature/x', + ]); + }); + + test('missing history becomes a visible line, never a failed step', () => { + const run = runStep('evals-periodic.yml', 'Write the census report', { 'fake-reds.txt': '', 'fake-headroom.txt': '' }, PERIODIC_EXPRESSIONS); + expect(run.status).toBe(0); + const body = run.body('census-report.md'); + expect(body).toContain('all-green probability: unavailable (see the red ledger)'); + expect(body).toContain('(no pass-rate alarms)'); + expect(body).toContain('(no periodic report summary)'); + }); +}); + +describe('the marathon report', () => { + test('fences the sanitized reconciliation output and links the guide', () => { + const run = runStep('evals-marathon.yml', 'Write the marathon report', { 'report.txt': 'red slice\nping @someone ```' }, + { 'steps.reconcile.outputs.exit': '1', 'needs.eval-slices.result': 'success' }); + expect(run.status).toBe(0); + const body = run.body('marathon-report.md'); + expect(run.summary).toBe(body); + expect(body).toStartWith('Marathon report (non-blocking lane): branch feature/x @ abc1234, event workflow_dispatch'); + expect(body).toContain('docs/evals/census-red.md'); + expect(body).toContain('- reconciliation exit: 1'); + expect(body).toContain('````\nred slice\nping @\u200bsomeone ```\n````'); + }); +}); diff --git a/test/evals-workflow-wiring.test.ts b/test/evals-workflow-wiring.test.ts index 4268c160e..dd816b741 100644 --- a/test/evals-workflow-wiring.test.ts +++ b/test/evals-workflow-wiring.test.ts @@ -377,11 +377,11 @@ describe('panel verdict surfaces (eval reliability policy)', () => { const gate = report.steps.find(step => step.id === 'pass-rates')!; expect(gate.run).toContain('bun run eval:pass-rates --gate --runs 10'); expect(gate.if).toBe('always()'); - for (const name of ['Upsert tracking issue on failure', 'Fail the workflow when reconciliation failed']) { + for (const name of ['Write the census report', 'Upsert tracking issue on failure', 'Fail the workflow when reconciliation failed']) { expect(report.steps.find(step => step.name === name)!.if).toContain("steps.pass-rates.outputs.exit != '0'"); } - const upsert = report.steps.find(step => step.name === 'Upsert tracking issue on failure')!; - expect(upsert.run).toContain('report-summary.md'); + // The report body is written once on every ref; the main-only upsert posts it (test/evals-tracking-issue.test.ts). + expect(report.steps.find(step => step.name === 'Write the census report')!.run).toContain('report-summary.md'); expect(report.steps.find(step => step.name === 'Close the tracking issue on a green run')!.run).toContain('gh issue close'); expect(report.steps.filter(step => step.with?.name?.startsWith('trial-outcomes-')).length).toBe(2); const redispatch = jobs.redispatch!; From 446da35002a7f061c278e34c13d0ccf083178f00 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sun, 4 Oct 2026 18:39:58 +0000 Subject: [PATCH 2/4] E6: live Codex multi-block periodic case in the host-run Codex job test/codex-e2e-multiblock-live.test.ts runs ./setup --host codex once from the job's checkout into a fresh HOME and CODEX_HOME (auth.json copied from the CI login home), asserts codex debug prompt-input lists the gstack router as a real file, seeds one learning, and runs /learn's Show recent block through codex exec: that later block must root itself through its prelude, run the installed gstack-learnings-search and print the seeded learning; no block may report a missing install or an unresolved gstack helper path. The file joins CODEX_CI_FILES so the planner gives it to the host-run Codex job. paid-test-durations.json carries a 180 s packing estimate (refreshed by --write-durations after the first census) so the plan keeps every slice within max-parallel. Oracles live in test/helpers/codex-multiblock.ts with free replays in test/codex-multiblock.test.ts. Registration (E2E_TOUCHFILES, E2E_TIERS periodic, E2E_KINDS rule) is the integrator's. --- scripts/lib/paid-cases.ts | 1 + scripts/paid-test-durations.json | 1 + test/codex-e2e-multiblock-live.test.ts | 116 +++++++++++++++++++++++++ test/codex-multiblock.test.ts | 61 +++++++++++++ test/helpers/codex-multiblock.ts | 68 +++++++++++++++ 5 files changed, 247 insertions(+) create mode 100644 test/codex-e2e-multiblock-live.test.ts create mode 100644 test/codex-multiblock.test.ts create mode 100644 test/helpers/codex-multiblock.ts diff --git a/scripts/lib/paid-cases.ts b/scripts/lib/paid-cases.ts index 93b59280a..eae7a970b 100644 --- a/scripts/lib/paid-cases.ts +++ b/scripts/lib/paid-cases.ts @@ -263,6 +263,7 @@ export const CODEX_CI_FILES: readonly string[] = [ 'test/skill-e2e-outside-voice.test.ts', 'test/skill-e2e-outside-plan-disabled.test.ts', 'test/skill-e2e-safety-codex-boundary.test.ts', + 'test/codex-e2e-multiblock-live.test.ts', ]; /** diff --git a/scripts/paid-test-durations.json b/scripts/paid-test-durations.json index f206fd75f..d120102f4 100644 --- a/scripts/paid-test-durations.json +++ b/scripts/paid-test-durations.json @@ -122,6 +122,7 @@ "test/carve-section-loading-review.test.ts": 247902, "test/carve-section-loading-setup-gbrain.test.ts": 80994, "test/carve-section-loading-spec.test.ts": 176639, + "test/codex-e2e-multiblock-live.test.ts": 180000, "test/codex-e2e-recommendation-substance.test.ts": 17984, "test/codex-e2e-shared-libs.test.ts": 53442, "test/codex-e2e-sol-scope.test.ts": 37063, diff --git a/test/codex-e2e-multiblock-live.test.ts b/test/codex-e2e-multiblock-live.test.ts new file mode 100644 index 000000000..ccb4eebb4 --- /dev/null +++ b/test/codex-e2e-multiblock-live.test.ts @@ -0,0 +1,116 @@ +/** + * Live Codex multi-block check (E6, periodic, host-run Codex job). + * + * Codex runs every shell block of a skill in a fresh shell. This case runs + * `./setup --host codex` once from the job's own checkout into a fresh HOME + * and CODEX_HOME, then checks what a Codex user gets: + * 1. `codex debug prompt-input` lists the `gstack` router, and the router + * it names is a real file (Codex skips a symlinked SKILL.md); + * 2. `codex exec` running /learn's "Show recent" block, a later block that + * only its own prelude can root, runs the installed helper and prints the + * learning seeded before the run; no block reports a missing install or + * an unresolved gstack helper path. + * Auth is the CI login home's auth.json only (scopeCodexAccess hands this + * file the CLI on PATH and that home as CODEX_HOME). + */ +import { describe, test, afterAll, expect } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { spawn, spawnSync } from 'child_process'; +import { CAPTURE_MS } from './helpers/eval-budgets'; +import { e2eTierEnabled } from './helpers/e2e-gate'; +import { hermeticChildEnv } from './helpers/hermetic-env'; +import { parseCodexJSONL, type CodexResult } from './helpers/codex-session-runner'; +import { CODEX_EVAL_FINALIZE_MS, createCodexEvalCollector, runRecordedCodexEval } from './helpers/codex-eval'; +import { MULTIBLOCK_SENTINEL_KEY, multiblockProblems, routerPathFromPromptInput } from './helpers/codex-multiblock'; +import { CODEX_FRONTIER_MODEL } from '../scripts/resolvers/constants'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const CASE = 'codex-multiblock-live'; +const SETUP_MS = 300_000; +const CODEX_AVAILABLE = spawnSync('which', ['codex'], { timeout: 30_000 }).status === 0; +const ENABLED = e2eTierEnabled('periodic') && CODEX_AVAILABLE; +if (e2eTierEnabled('periodic') && !CODEX_AVAILABLE) process.stderr.write(`\n${CASE}: SKIPPED: codex binary not on PATH\n`); + +const collector = ENABLED ? createCodexEvalCollector('codex-e2e-multiblock-live') : null; +afterAll(async () => { await collector?.finalize(); }); + +/** `codex exec --json` with stdout/stderr in files (no pipe can outlive the process group), killed on abort. */ +function codexExec(args: string[], env: NodeJS.ProcessEnv, cwd: string, scratch: string, signal: AbortSignal): Promise { + const started = Date.now(); + const outFile = path.join(scratch, 'exec.jsonl'); + const errFile = path.join(scratch, 'exec.stderr'); + const out = fs.openSync(outFile, 'w'); + const err = fs.openSync(errFile, 'w'); + const proc = spawn('codex', args, { cwd, env, stdio: ['ignore', out, err], detached: true }); + fs.closeSync(out); + fs.closeSync(err); + const kill = () => { try { process.kill(-proc.pid!, 'SIGKILL'); } catch { /* already gone */ } }; + signal.addEventListener('abort', kill, { once: true }); + return new Promise(resolve => { + const finish = (exitCode: number) => { + signal.removeEventListener('abort', kill); + kill(); + const rawLines = fs.readFileSync(outFile, 'utf8').split('\n').filter(line => line.trim()); + resolve({ ...parseCodexJSONL(rawLines), rawLines, exitCode: signal.aborted ? 124 : exitCode, + durationMs: Date.now() - started, stderr: fs.readFileSync(errFile, 'utf8') }); + }; + proc.once('error', () => finish(1)); + proc.once('exit', (code, sig) => finish(code ?? (sig ? 128 + (os.constants.signals[sig] ?? 0) : 1))); + }); +} + +(ENABLED ? describe : describe.skip)('live Codex multi-block install (E6)', () => { + test(CASE, async () => { + const scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'e6-')); + try { + const home = path.join(scratch, 'home'); + const codexHome = path.join(home, '.codex'); + const repo = path.join(scratch, 'repo'); + // Codex will not create its sandbox helper aliases under its temp dir, so TMPDIR sits beside CODEX_HOME. + const tmp = path.join(home, 'tmp'); + for (const dir of [codexHome, repo, tmp]) fs.mkdirSync(dir, { recursive: true }); + const ciHome = process.env.CODEX_HOME ?? path.join(os.homedir(), '.codex'); + if (fs.existsSync(path.join(ciHome, 'auth.json'))) fs.copyFileSync(path.join(ciHome, 'auth.json'), path.join(codexHome, 'auth.json')); + const env = hermeticChildEnv({ HOME: home, CODEX_HOME: codexHome, TMPDIR: tmp }, { extraAllow: ['OPENAI_API_KEY', 'CODEX_*'] }); + + const setupStarted = Date.now(); + const setup = spawnSync('./setup', ['--host', 'codex'], { cwd: ROOT, env, encoding: 'utf8', timeout: SETUP_MS }); + console.log(`${CASE}: ./setup --host codex exit ${setup.status} in ${Math.round((Date.now() - setupStarted) / 1000)}s`); + expect(setup.status, `${setup.stdout.slice(-2000)}\n${setup.stderr.slice(-2000)}`).toBe(0); + + const promptInput = spawnSync('codex', ['debug', 'prompt-input'], { cwd: repo, env, encoding: 'utf8', timeout: 60_000 }); + expect(promptInput.status, promptInput.stderr).toBe(0); + const router = routerPathFromPromptInput(promptInput.stdout); + expect(router, 'codex debug prompt-input does not list the gstack router').toBe(path.join(codexHome, 'skills', 'gstack', 'SKILL.md')); + expect(fs.lstatSync(router!).isFile()).toBe(true); + + for (const args of [['init', '-q'], ['remote', 'add', 'origin', 'https://github.com/example/e6-fixture.git']]) { + expect(spawnSync('git', args, { cwd: repo, env, timeout: 30_000 }).status).toBe(0); + } + const seed = spawnSync(path.join(codexHome, 'skills', 'gstack', 'bin', 'gstack-learnings-log'), [JSON.stringify({ skill: 'learn', + type: 'pitfall', key: MULTIBLOCK_SENTINEL_KEY, insight: 'Seeded by the E6 live Codex check', confidence: 9, source: 'user-stated' })], + { cwd: repo, env, encoding: 'utf8', timeout: 30_000 }); + expect(seed.status, seed.stderr).toBe(0); + + const result = await runRecordedCodexEval({ + name: CASE, + suite: 'codex-e2e-multiblock-live', + budgetMs: CAPTURE_MS, + run: signal => codexExec(['exec', '--json', '-s', 'read-only', '--skip-git-repo-check', + '--model', process.env.GSTACK_CODEX_MODEL ?? CODEX_FRONTIER_MODEL, + 'Use the gstack learn skill to show this project\'s recent learnings. Run its "Show recent" step, then list what it printed.'], + env, repo, scratch, signal), + validate: run => { + const problems = multiblockProblems(run.rawLines); + if (problems.length) throw new Error(problems.join('\n')); + }, + record: entry => collector?.addTest(entry), + }); + console.log(`${CASE}: ${result.toolCalls.length} commands, ${result.tokens} tokens, ${Math.round(result.durationMs / 1000)}s`); + } finally { + fs.rmSync(scratch, { recursive: true, force: true }); + } + }, SETUP_MS + 120_000 + CAPTURE_MS + CODEX_EVAL_FINALIZE_MS); +}); diff --git a/test/codex-multiblock.test.ts b/test/codex-multiblock.test.ts new file mode 100644 index 000000000..c96ca181b --- /dev/null +++ b/test/codex-multiblock.test.ts @@ -0,0 +1,61 @@ +/** + * Free replays for the live Codex multi-block oracles (E6): synthetic + * `codex exec --json` events in the shape codex-cli 0.160 writes (see + * test/fixtures/codex-sandbox/exec-json-healthy.jsonl) and a + * `codex debug prompt-input` skills block. + */ +import { describe, test, expect } from 'bun:test'; +import { MULTIBLOCK_SENTINEL_KEY, multiblockProblems, routerPathFromPromptInput } from './helpers/codex-multiblock'; + +const PRELUDE = '[ -d "${GSTACK_ROOT:-/-}/bin" ]&&[ -d "$GSTACK_ROOT/lib" ]||{ _r=$(git rev-parse --show-toplevel 2>/dev/null)/.agents/skills/gstack;' + + '[ -d "$_r/bin" ]||_r=${CODEX_HOME:-~/.codex}/skills/gstack;[ -d "$_r/bin" ]||{ echo "gstack: no install found (tried $_r). Fix: ./setup --host codex from your gstack checkout; ./setup --status shows it.">&2;exit 1;};GSTACK_ROOT=$_r;}'; +const SHOW_RECENT = `${PRELUDE}\nSLUG=$($GSTACK_ROOT/bin/gstack-slug --get SLUG 2>/dev/null)\n$GSTACK_ROOT/bin/gstack-learnings-search --limit 20 2>/dev/null || echo "No learnings yet."`; +const LISTING = `LEARNINGS: 1 loaded (1 pitfall)\n\n## Pitfalls\n- [${MULTIBLOCK_SENTINEL_KEY}] (confidence: 9/10, user-stated, 2026-10-04)\n Seeded by the E6 live Codex check\n`; + +const command = (cmd: string, output: string, exitCode: number) => JSON.stringify({ type: 'item.completed', + item: { id: 'item_2', type: 'command_execution', command: `/bin/bash -lc ${JSON.stringify(cmd)}`, aggregated_output: output, exit_code: exitCode, status: exitCode === 0 ? 'completed' : 'failed' } }); +const run = (...lines: string[]) => ['{"type":"thread.started","thread_id":"t"}', ...lines, '{"type":"turn.completed","usage":{"input_tokens":1,"output_tokens":1}}']; +const preamble = command(`${PRELUDE}\nGSTACK_BIN=$GSTACK_ROOT/bin\n"$GSTACK_BIN/gstack-skill-start" --skill "learn"`, 'BRANCH: main\n', 0); + +describe('multiblockProblems', () => { + test('a later block rooted by its own prelude that prints the seeded learning passes', () => { + expect(multiblockProblems(run(preamble, command(SHOW_RECENT, LISTING, 0)))).toEqual([]); + }); + + test('a block that cannot find the install fails, even when another block succeeded', () => { + const missing = command(SHOW_RECENT, 'gstack: no install found (tried /root/.codex/skills/gstack). Fix: ./setup --host codex', 1); + const problems = multiblockProblems(run(preamble, missing, command(SHOW_RECENT, LISTING, 0))); + expect(problems).toHaveLength(1); + expect(problems[0]).toStartWith('a block could not find the install'); + }); + + test('the pre-prelude symptom (an empty root, /bin/gstack-slug) fails', () => { + const problems = multiblockProblems(run(command('SLUG=$($GSTACK_ROOT/bin/gstack-slug --get SLUG)\n$GSTACK_ROOT/bin/gstack-learnings-search', + 'bash: line 1: /bin/gstack-slug: No such file or directory\nbash: line 2: /bin/gstack-learnings-search: No such file or directory\n', 127))); + expect(problems.some(p => p.startsWith('a gstack helper path did not resolve: /bin/gstack-slug: No such file or directory'))).toBe(true); + expect(problems.some(p => p.startsWith('no learnings-search block rooted by its own prelude printed the seeded learning'))).toBe(true); + }); + + test('a preamble-only run, a run without the seeded learning, or a hand-set root does not count', () => { + expect(multiblockProblems(run(preamble))).toEqual(['no block ran gstack-learnings-search (the skill\'s "Show recent" block)']); + expect(multiblockProblems(run(preamble, command(SHOW_RECENT, 'No learnings yet.\n', 0)))[0]).toContain('exit 0, sentinel absent'); + const handSet = command('GSTACK_ROOT=/home/u/.codex/skills/gstack\n$GSTACK_ROOT/bin/gstack-learnings-search --limit 20', LISTING, 0); + expect(multiblockProblems(run(preamble, handSet))[0]).toContain('GSTACK_ROOT set by hand'); + }); +}); + +describe('routerPathFromPromptInput', () => { + const promptInput = (entries: string[]) => JSON.stringify([{ type: 'message', role: 'developer', content: [{ type: 'input_text', + text: ['', '## Skills', '### Skill roots', '- `r0` = `/h/.codex/skills`', '- `r1` = `/h/.codex/skills/.system`', + '### Available skills', '- skill-creator: Create or update a Codex skill. (file: r1/skill-creator/SKILL.md)', ...entries].join('\n') }] }], null, 2); + + test('names the router file through the skill roots table', () => { + const listed = promptInput(['- gstack: Route a task to the right gstack skill. (gstack) (file: r0/gstack/SKILL.md)', + '- learn: Manage project learnings. (gstack) (file: r0/gstack-learn/SKILL.md)']); + expect(routerPathFromPromptInput(`WARNING: proceeding without PATH aliases\n${listed}`)).toBe('/h/.codex/skills/gstack/SKILL.md'); + }); + + test('is null when Codex lists gstack skills but not the router', () => { + expect(routerPathFromPromptInput(promptInput(['- learn: Manage project learnings. (gstack) (file: r0/gstack-learn/SKILL.md)']))).toBeNull(); + }); +}); diff --git a/test/helpers/codex-multiblock.ts b/test/helpers/codex-multiblock.ts new file mode 100644 index 000000000..eaef4edf9 --- /dev/null +++ b/test/helpers/codex-multiblock.ts @@ -0,0 +1,68 @@ +/** + * Oracles for the live Codex multi-block case (E6, + * test/codex-e2e-multiblock-live.test.ts). Codex runs every shell block in a + * fresh shell, so a gstack skill works there only when each later block + * resolves the install on its own (the env-var host prelude). The case + * installs gstack with `./setup --host codex` into a fresh CODEX_HOME, seeds + * one learning, and runs /learn's "Show recent" block through `codex exec`. + * Pure functions: the free test replays synthetic `exec --json` events. + */ + +export const MULTIBLOCK_SENTINEL_KEY = 'e6-multiblock-sentinel'; + +interface CommandEvent { command: string; output: string; exitCode: number | null } + +function completedCommands(rawLines: readonly string[]): CommandEvent[] { + const commands: CommandEvent[] = []; + for (const line of rawLines) { + try { + const event = JSON.parse(line); + if (event?.type !== 'item.completed' || event.item?.type !== 'command_execution') continue; + commands.push({ command: String(event.item.command ?? ''), output: String(event.item.aggregated_output ?? ''), + exitCode: typeof event.item.exit_code === 'number' ? event.item.exit_code : null }); + } catch { /* not an event line */ } + } + return commands; +} + +/** + * Problems with a live run: any block that could not find the install, a + * gstack helper path that did not resolve, or no later block (the skill's + * learnings-search block, which only the prelude can root) that ran the + * installed helper and printed the seeded learning. + */ +export function multiblockProblems(rawLines: readonly string[], sentinel = MULTIBLOCK_SENTINEL_KEY): string[] { + const commands = completedCommands(rawLines); + const problems: string[] = []; + for (const { command, output } of commands) { + if (output.includes('gstack: no install found')) problems.push(`a block could not find the install: ${output.trim().slice(0, 200)}`); + const unresolved = /\S*\/bin\/gstack-[\w-]+: (?:No such file or directory|command not found|not found)/.exec(output); + if (unresolved) problems.push(`a gstack helper path did not resolve: ${unresolved[0]} (in: ${command.slice(0, 120)})`); + } + const search = commands.filter(c => c.command.includes('gstack-learnings-search')); + if (search.length === 0) problems.push('no block ran gstack-learnings-search (the skill\'s "Show recent" block)'); + else if (!search.some(c => c.exitCode === 0 && c.output.includes(sentinel) && !/GSTACK_ROOT=["']?\//.test(c.command))) { + problems.push(`no learnings-search block rooted by its own prelude printed the seeded learning ${sentinel}: ` + + search.map(c => `exit ${c.exitCode}, ${c.output.includes(sentinel) ? 'sentinel present' : 'sentinel absent'}` + + `${/GSTACK_ROOT=["']?\//.test(c.command) ? ', GSTACK_ROOT set by hand' : ''}`).join('; ')); + } + return problems; +} + +/** + * The `gstack` router entry from `codex debug prompt-input` output (a JSON + * array of input items): the skill path it names, resolved through the skill + * roots table, or null when Codex does not list the router. + */ +export function routerPathFromPromptInput(promptInput: string): string | null { + let text = promptInput; + try { + const items = JSON.parse(promptInput.slice(promptInput.indexOf('['))); + text = (Array.isArray(items) ? items : []).flatMap((item: { content?: Array<{ text?: string }> }) => item.content ?? []) + .map(part => part.text ?? '').join('\n'); + } catch { /* scan the raw text */ } + const entry = /^- gstack: .*\(file: (r\d+)\/gstack\/SKILL\.md\)$/m.exec(text); + if (!entry) return null; + const root = new RegExp(`^- \`${entry[1]}\` = \`([^\`]+)\`$`, 'm').exec(text); + return root ? `${root[1]}/gstack/SKILL.md` : null; +} From a0939d6593776a40d46121c08eedd978a70dde84 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sun, 4 Oct 2026 18:40:06 +0000 Subject: [PATCH 3/4] E5: design live-model check as a periodic case test/skill-e2e-design-model-smoke.test.ts runs design/scripts/live-model-check.ts (default models only) under a temp HOME. Without OPENAI_API_KEY the case is a Bun skip that the census lists as SKIPPED with no coverage credit; with a key, exit 0 passes and any other exit fails with the script output (exit 2 under a present key fails as 'nothing was checked'). paid-test-durations.json carries a 90 s packing estimate. Registration (E2E_TOUCHFILES, E2E_TIERS periodic, E2E_KINDS rule) is the integrator's. --- scripts/paid-test-durations.json | 1 + test/skill-e2e-design-model-smoke.test.ts | 41 +++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 test/skill-e2e-design-model-smoke.test.ts diff --git a/scripts/paid-test-durations.json b/scripts/paid-test-durations.json index d120102f4..bc02a74d5 100644 --- a/scripts/paid-test-durations.json +++ b/scripts/paid-test-durations.json @@ -141,6 +141,7 @@ "test/skill-e2e-coverage-audit.test.ts": 1000, "test/skill-e2e-cso.test.ts": 253498, "test/skill-e2e-deploy.test.ts": 1000, + "test/skill-e2e-design-model-smoke.test.ts": 90000, "test/skill-e2e-design.test.ts": 817000, "test/skill-e2e-design.test.ts#design-consultation-core": 159257, "test/skill-e2e-design.test.ts#design-consultation-existing": 175607, diff --git a/test/skill-e2e-design-model-smoke.test.ts b/test/skill-e2e-design-model-smoke.test.ts new file mode 100644 index 000000000..4f8c9a57e --- /dev/null +++ b/test/skill-e2e-design-model-smoke.test.ts @@ -0,0 +1,41 @@ +/** + * Design model smoke (E5, periodic): runs design/scripts/live-model-check.ts, + * which sends the design binary's real request bodies with the DEFAULT + * models (overrides are ignored), so a retired or unentitled default fails + * here before users hit it. Without OPENAI_API_KEY the case is a Bun skip: + * the report lists it as SKIPPED with zero coverage credit, never a pass. + * With a key, exit 0 passes; exit 2 (the script saw no key) and every other + * exit fail with the script's output attached. + */ +import { describe, test, expect } from 'bun:test'; +import { spawnSync } from 'child_process'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { e2eTierEnabled } from './helpers/e2e-gate'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const CASE = 'design-model-smoke'; +const KEY_PRESENT = !!process.env.OPENAI_API_KEY?.trim(); +const CHECK_MS = 400_000; +if (e2eTierEnabled('periodic') && !KEY_PRESENT) { + process.stderr.write(`\n${CASE}: SKIPPED: OPENAI_API_KEY is not set; the design model defaults were not checked (no coverage)\n`); +} + +describe.skipIf(!e2eTierEnabled('periodic') || !KEY_PRESENT)('design live model defaults (E5)', () => { + test(CASE, () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), 'design-model-smoke-')); + try { + const run = spawnSync(process.execPath, ['run', 'design/scripts/live-model-check.ts'], { + cwd: ROOT, encoding: 'utf8', timeout: CHECK_MS, + env: { ...process.env, HOME: home, GSTACK_HOME: path.join(home, '.gstack') }, + }); + const output = `${run.stdout ?? ''}${run.stderr ?? ''}`.trim(); + console.log(output); + if (run.status === 2) throw new Error(`live-model-check saw no OPENAI_API_KEY although the test had one; nothing was checked:\n${output}`); + expect(run.status, output).toBe(0); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }, CHECK_MS + 10_000); +}); From a1162e3d94125ab896c5efb536e9c26710185dcb Mon Sep 17 00:00:00 2001 From: garrytan Date: Sun, 4 Oct 2026 19:02:14 +0000 Subject: [PATCH 4/4] 0.4: 'A census went red' guide; docs for causes, session ledger, headroom and new messages docs/evals/census-red.md walks one red line to a diagnosis: prerequisites, the report fields, eval:pass-rates --run/--reds/--headroom with expected output, what unknown/insufficient/unrecorded data means, free replay, and the single 'after a repair' paid run allowed by TESTING_INTERNALS. Linked from CONTRIBUTING, TESTING_INTERNALS and both census reports. TESTING_INTERNALS documents the failure_cause glossary (one row per FAILURE_CAUSES value, diagnostic only under EVAL_POLICY v1), failure_detail, sessions[], cost_known, the session ledger, the pass-rates views and the 85% headroom alarm, main-only tracking issues, overlay contract v4, auq-matrix's 3-sample substance panel and the E5/E6 periodic cases. troubleshooting.md gains #bun-too-old and an anchor per new user-facing message from lanes P, D and F. README names GSTACK_DESIGN_IMAGE_MODEL. test/census-red-docs.test.ts pins the glossary to FAILURE_CAUSES, parses every guide command and checks the links. --- CONTRIBUTING.md | 10 +++ README.md | 8 +- docs/TESTING_INTERNALS.md | 115 ++++++++++++++++++++++-- docs/evals/census-red.md | 134 +++++++++++++++++++++++++++ docs/troubleshooting.md | 169 +++++++++++++++++++++++++++++++++++ test/census-red-docs.test.ts | 45 ++++++++++ 6 files changed, 471 insertions(+), 10 deletions(-) create mode 100644 docs/evals/census-red.md create mode 100644 test/census-red-docs.test.ts diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4ba76804b..bb6ddc4cb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -274,6 +274,13 @@ this policy. Quarantine (`CASE_QUARANTINE`) and history are described in `docs/TESTING_INTERNALS.md`; `bun run eval:pass-rates --case ` shows a case's per-trial pass rate with its Wilson interval. +**A census went red?** Follow [docs/evals/census-red.md](docs/evals/census-red.md): +read the red line, inspect its evidence with `bun run eval:pass-rates --run `, +check `--reds` and `--headroom`, repair with a free regression test, and only +then spend one paid run with the line's `after a repair:` command. A red census +on `main` is reported on the weekly tracking issue; a census dispatched on a +branch writes the same report to its run summary and `census-report` artifact. + CI enables verified first-attempt reuse for 16 workflow quality judges for 24 hours within the same PR. The cookie workflow's custom input and the other 11 quality cases stay fresh. PR-profile E2E shards that run once (no retry, so the @@ -446,6 +453,9 @@ bun run eval:list # list all eval runs (turns, duration, cost per run bun run eval:compare # compare two runs — shows per-test deltas + Takeaway commentary bun run eval:summary # aggregate stats + per-test efficiency averages across runs bun run eval:pass-rates # per-case trial pass rates + Wilson intervals from recent weekly runs (--case, --runs, --dir, --backfill, --json, --gate); eval:flake-rank is an alias +bun run eval:pass-rates --reds # verdict reds per census by failure class and cause, all-green probability +bun run eval:pass-rates --headroom # slowest session per case vs its armed budget (alarm above 85% in --gate) +bun run eval:pass-rates --run # one census's reds with their values and fetched transcript evidence ``` **Detached runs for agents and long suites.** When an agent (or you, for a run diff --git a/README.md b/README.md index 0ad7765c5..846b35751 100644 --- a/README.md +++ b/README.md @@ -186,8 +186,12 @@ for capture, judge, and benchmark model selection. The design binary (`$D`) uses OpenAI's `gpt-5.5` by default, with image generation through `gpt-image-2`. Set `GSTACK_DESIGN_MODEL=` to use another gpt-5-class model for both image generation and screenshot analysis; -if OpenAI rejects it, the error names `GSTACK_DESIGN_MODEL`. Check a key -against the defaults with `bun run design/scripts/live-model-check.ts`. +if OpenAI rejects it, the error names `GSTACK_DESIGN_MODEL`. Set +`GSTACK_DESIGN_IMAGE_MODEL=` to change the image tool model; a +value that is not a gpt-image model name is refused before any request. Check a +key against the defaults with `bun run design/scripts/live-model-check.ts`, +which always tests the default models and ignores both overrides; the weekly +periodic census runs the same check. **Want to add support for another agent?** See [docs/ADDING_A_HOST.md](docs/ADDING_A_HOST.md). Rendering a new agent is one TypeScript config file; installing it also needs a diff --git a/docs/TESTING_INTERNALS.md b/docs/TESTING_INTERNALS.md index 2b4559e61..80e308ef7 100644 --- a/docs/TESTING_INTERNALS.md +++ b/docs/TESTING_INTERNALS.md @@ -96,9 +96,13 @@ against the previous finalized run (in-flight `_partial` files are never used as a baseline, so a run can't compare against itself). The periodic overlay fixtures use a versioned behavior gate with efficacy -reported separately. See [Overlay benchmark contract v2](OVERLAY_BENCHMARK_CONTRACT.md) +reported separately. See [Overlay benchmark contract v4](OVERLAY_BENCHMARK_CONTRACT.md) for exact correctness requirements, retired fanout cases, immutable evidence, -and the limits of a passing result. +and the limits of a passing result. Under v4 a completed wrong answer is a +measured correctness result in either arm (`taskCorrect: false` with +`answerError`); only an overlay-ON wrong answer fails the case, so an +overlay-OFF wrong answer stays in the comparison. v3 verdicts keep their +original meaning. ## Coverage ownership @@ -365,7 +369,22 @@ report and tracking issue, and nothing requires it. evals-periodic.yml runs ALL periodic-tier files weekly (the coverage contract) minus the reasoned exclusions in `test/helpers/periodic-exclude-data.ts` (reason + tracking required per entry; removal re-activates the file), plus a weekly -`EVALS_ALL` gate census, plus a tracking-issue UPSERT on red weeks. The CI +`EVALS_ALL` gate census. A red census writes one report on every ref, to the +step summary and the `census-report-a` artifact: the run's branch, sha +and event, both lanes' summaries, every pass-rate alarm line, the red ledger +and session headroom (`eval:pass-rates --reds` / `--headroom --runs 10 --branch +`), the all-green estimate and a link to the red-census guide, with +report-derived text passed through `scripts/lib/published-text.ts`. Only a run +on `refs/heads/main` comments on (or opens) the one tracking issue and closes it +on the next green run; the marathon lane does the same with its own issue and +`marathon-report-a` artifact. `test/evals-tracking-issue.test.ts` +pins the main-only guard and runs both report scripts. The host-run Codex job +also carries the live Codex multi-block case (`codex-multiblock-live`: `./setup +--host codex` from the job's checkout into a fresh `CODEX_HOME`, the router in +`codex debug prompt-input`, and a later /learn block rooted by its own +prelude), and the periodic lane runs the design default-model smoke +(`design-model-smoke`), a visible SKIPPED with no coverage credit when +`OPENAI_API_KEY` is absent. The CI image pins the claude CLI to an exact version (`.github/docker/Dockerfile.ci`, enforced by `test/ci-image-cli-pin.test.ts` — bumps ride PRs that run the PTY gate), and every eval-store run records `claude --version`, resolved once in @@ -373,6 +392,7 @@ the runner parent and handed to shard children as `GSTACK_CLAUDE_CLI_VERSION` (never spawned on a test thread), so a TUI-drift flake hunt is a grep, not archaeology. + **Eval verdict policy** (`EVAL_POLICY` version 1 in `test/helpers/periodic-exclude-data.ts`, pre-registered 2026-09-29). Paid evals never retry. Each live case has exactly one kind in `E2E_KINDS` @@ -397,6 +417,11 @@ and the kind fixes its trials before the run: refusal counts as an unscored panel only when every sample refused. `callJudge`'s 429 backoff happens before any model output and is transport, not a verdict retry. The workflow-judge cache stores whole panels only. + The `rule` case auq-matrix uses the same panel inside its one execution: each + skill's single native capture has its recommendation substance scored by 3 + judge samples, gated on their mean against the unchanged minimum of 4, and a + failed sample fails the panel without resampling (approved 2026-10-04 for + censuses after the change). `panelVerdict()` (`test/helpers/eval-store.ts`) is the single verdict function the report, `collector-outcomes.json`, the PR comment and pass-rates @@ -414,6 +439,58 @@ reported. Changing any `EVAL_POLICY` constant after seeing census results needs Garry's re-approval, a `version` bump and a fresh census; `test/periodic-exclude-policy.test.ts` pins the approved values. + +**Failure causes and details** (diagnostic only; verdicts unchanged under v1). +Every failed trial record keeps `failure_class` (`assertion`, `contract`, +`timeout`, `infra`: the field verdicts, INFRA re-dispatch and pass rates read) +and its 300-character `error` exactly as before. `failureCauseOf()` +(`test/helpers/eval-store.ts`, called by both record builders, including the +JUnit census for PTY and JUnit-only cases) adds an optional `failure_cause` +with a one-line `failure_cause_evidence`. It never changes `failure_class`. +The highest-precedence cause with evidence wins: + +| `failure_cause` | Recorded when | First step | +|---|---|---| +| `contract` | `expectContract()` stamped a contract violation | fix the product; a violation fails a behavior panel outright | +| `pre_turn_infra` | the trial failed before the first model turn (`failure_class: 'infra'`) | check the runner and API status; an all-INFRA census is re-dispatched once | +| `api_error` | a session ended on an API error (for PTY, the `API Error:` panel on an idle turn) or the runner's exit reason was `error_api` | read the evidence line; a provider outage is a named red | +| `refusal` | the structured stop reason was a refusal (phrases only as a last resort; quoted refusal text in a tool result does not count) | check whether the fixture prompt invites the refusal | +| `provider_stall` | a session that streamed partial messages saw no event of any kind, thinking deltas included, for `STALL_WINDOW_MS` (120 s) while a model request was in flight and no tool, permission prompt, hook or subagent was outstanding; sessions without partial messages are never stalls | read the session ledger row; still a failed trial under v1 | +| `session_timeout` | a runner's armed session timeout fired | `eval:pass-rates --headroom`; cut work, never raise the budget | +| `observer_timeout` | the test's observer gave up waiting for an outcome (`outcome=timeout`) | read the transcript tail: a missed detector or a product that never answered | +| `assertion` | a check failed after the session ran (`session completed; check failed` when it ended normally) | the Expected/Received values or judge dimensions | +| `unknown` | the trial failed with no evidence for any cause above | inspect the shard directory | + +A detector, harness or product fault is a human diagnosis, never a cause. +Readers before this field accept the new records (a frozen v1 reader runs over +them in `test/eval-store.test.ts`); an unknown cause from a newer gstack is +rejected with `unknown failure_cause "" (written by a newer gstack; update +this checkout to read it)`. + +`failure_detail` carries the values `error` cannot hold: a failed Bun matcher's +`{expected, received}`, or for a judge one `{dimension, mean, threshold, +samples, rationale}` entry per failing dimension, each part capped at 200 +characters. `sessions[]` (at most 32) summarizes the trial's session-ledger +rows. `cost_known: false` marks a record or trial whose harness captured no +billing: PTY and Codex sessions never bill, and an SDK session bills only with a +terminal `total_cost_usd`. An absent field means known. Census and marathon +cost lines print the known sum and `N of M trial(s) cost unknown`, never a bare +total over unknowns. + + +**Session ledger.** Every runner (session-runner, the Agent SDK runner +including `runSharedInteractive`, the PTY session and `runRecordedCodexEval`) +appends one best-effort line per armed session to +`$GSTACK_EVAL_DIR/session-ledger.jsonl` (`test/helpers/session-ledger.ts`): +a stable key `: