0.3: only main posts or closes tracking issues; reports carry alarms, red ledger, headroom and all-green

A branch-dispatched periodic or marathon run writes its report to the step summary and a report artifact; the upsert and close steps run only on refs/heads/main. The periodic body now keeps every pass-rate alarm line (the old grep kept only the header), adds the red ledger and session headroom read with an explicit --branch, the all-green estimate and the triage guide link, and routes report text through scripts/lib/published-text.ts. The marathon lane gains the matching close-on-green step. Migrated: evals-workflow-wiring pinned report-summary.md in the upsert step; the body is now written by 'Write the census report'.
2026-11-11 19:20:26 +01:00 · 2026-10-04 18:33:48 +00:00
1 parent 694e970c65
commit 7814a6d7af
@@ -283,25 +283,47 @@ jobs:
          # PIPESTATUS[0], NOT $?: the default step shell has no pipefail.
          echo "exit=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT"

-      # One tracking issue for the whole lane (never one per week).
-      - name: Upsert tracking issue on failure
+      # A red marathon run writes its report to the step summary and the
+      # marathon-report artifact on every ref. Only main upserts the lane's ONE
+      # tracking issue (never one per week) and closes it on the next green
+      # run; a branch-dispatched run never touches main's issue.
+      - name: Write the marathon report
        if: always() && (steps.reconcile.outputs.exit != '0' || needs.eval-slices.result != 'success')
+        run: |
+          set -euo pipefail
+          {
+            echo "Marathon report (non-blocking lane): branch ${GITHUB_REF_NAME} @ ${GITHUB_SHA}, event ${GITHUB_EVENT_NAME}, run ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+            echo
+            echo "A census went red: triage with ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/blob/${GITHUB_SHA}/docs/evals/census-red.md"
+            echo
+            echo "- reconciliation exit: ${{ steps.reconcile.outputs.exit }}"
+            echo "- marathon slices job: ${{ needs.eval-slices.result }}"
+            echo
+            { tail -c 6000 /tmp/report.txt 2>/dev/null || echo "(no reconciliation output)"; } \
+              | bun --no-install -e 'import { publishedFence } from "./scripts/lib/published-text";
+                  const lines = (await Bun.stdin.text()).split("\n").filter(Boolean);
+                  console.log(publishedFence(lines.length ? lines : ["(none)"]).join("\n"));'
+          } > /tmp/marathon-report.md
+          cat /tmp/marathon-report.md >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload the marathon report
+        if: always() && (steps.reconcile.outputs.exit != '0' || needs.eval-slices.result != 'success')
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: marathon-report-a${{ github.run_attempt }}
+          path: /tmp/marathon-report.md
+          if-no-files-found: ignore
+          retention-days: 90
+
+      - name: Upsert tracking issue on failure
+        if: always() && github.ref == 'refs/heads/main' && (steps.reconcile.outputs.exit != '0' || needs.eval-slices.result != 'success')
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set -euo pipefail
          TITLE="Weekly marathon evals: red lane needs triage"
-          BODY_FILE=/tmp/issue-body.md
-          {
-            echo "Automated weekly marathon report (non-blocking lane) — run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
-            echo
-            echo "- reconciliation exit: ${{ steps.reconcile.outputs.exit }}"
-            echo "- marathon slices job: ${{ needs.eval-slices.result }}"
-            echo
-            echo '```'
-            tail -c 6000 /tmp/report.txt 2>/dev/null || echo "(no reconciliation output)"
-            echo '```'
-          } > "$BODY_FILE"
+          BODY_FILE=/tmp/marathon-report.md
+          [ -s "$BODY_FILE" ] || echo "Marathon report missing; run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" > "$BODY_FILE"
          EXISTING=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "in:title \"$TITLE\"" --json number --jq '.[0].number // empty')
          if [ -n "$EXISTING" ]; then
            gh issue comment "$EXISTING" --repo "$GITHUB_REPOSITORY" --body-file "$BODY_FILE"
@@ -310,6 +332,18 @@ jobs:
            gh issue create --repo "$GITHUB_REPOSITORY" --title "$TITLE" --body-file "$BODY_FILE"
          fi

+      - name: Close the tracking issue on a green run
+        if: always() && github.ref == 'refs/heads/main' && steps.reconcile.outputs.exit == '0' && needs.eval-slices.result == 'success'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -euo pipefail
+          TITLE="Weekly marathon evals: red lane needs triage"
+          EXISTING=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "in:title \"$TITLE\"" --json number --jq '.[0].number // empty')
+          if [ -n "$EXISTING" ]; then
+            gh issue close "$EXISTING" --repo "$GITHUB_REPOSITORY" --comment "Green marathon run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+          fi
+
      - name: Fail the workflow when reconciliation failed
        if: always() && (steps.reconcile.outputs.exit != '0' || needs.eval-slices.result != 'success')
        run: exit 1
@@ -627,7 +627,7 @@ jobs:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set +e
-          bun run eval:pass-rates --gate --runs 10 > /tmp/pass-rates.txt 2>&1
+          bun run eval:pass-rates --gate --runs 10 --branch "$GITHUB_REF_NAME" > /tmp/pass-rates.txt 2>&1
          echo "exit=$?" >> "$GITHUB_OUTPUT"
          cat /tmp/pass-rates.txt

@@ -652,10 +652,13 @@ jobs:
            echo "redispatch=false" >> "$GITHUB_OUTPUT"
          fi

-      # A red weekly lane nobody must action is waste — upsert ONE tracking
-      # issue (never a new issue per week) with the headline and failure block
-      # of both lanes, and close it on the next green run.
-      - name: Upsert tracking issue on failure
+      # A red census writes one report on every ref: the step summary and the
+      # census-report artifact. Only main (scheduled weekly, or dispatched on
+      # main) upserts ONE tracking issue (never a new issue per week) and
+      # closes it on the next green run; a branch-dispatched census never
+      # comments on or closes main's issue. Every report-derived line goes
+      # through the one published-text sanitizer (scripts/lib/published-text.ts).
+      - name: Write the census report
        if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success')
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -663,17 +666,28 @@ jobs:
          REDISPATCH_OF: ${{ inputs.redispatch_of }}
        run: |
          set -euo pipefail
-          TITLE="Weekly periodic evals: red lane needs triage"
-          BODY_FILE=/tmp/issue-body.md
+          published() { # $1 fence|line: stdin lines through the published-text sanitizer
+            bun --no-install -e 'import { publishedFence, sanitizePublishedText } from "./scripts/lib/published-text";
+              const lines = (await Bun.stdin.text()).split("\n").filter(Boolean);
+              console.log(process.argv[1] === "line" ? sanitizePublishedText(lines.join(" ")) : publishedFence(lines.length ? lines : ["(none)"]).join("\n"));' "$1"
+          }
          RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+          GUIDE_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/blob/${GITHUB_SHA}/docs/evals/census-red.md"
+          bun run eval:pass-rates --reds --runs 10 --branch "$GITHUB_REF_NAME" > /tmp/pass-rates-reds.txt 2>&1 \
+            || echo "history unavailable: eval:pass-rates --reds exited $?" >> /tmp/pass-rates-reds.txt
+          bun run eval:pass-rates --headroom --runs 10 --branch "$GITHUB_REF_NAME" > /tmp/pass-rates-headroom.txt 2>&1 \
+            || echo "history unavailable: eval:pass-rates --headroom exited $?" >> /tmp/pass-rates-headroom.txt
          {
-            echo "Automated weekly report — run: ${RUN_URL}"
+            echo "Periodic census report: branch ${GITHUB_REF_NAME} @ ${GITHUB_SHA}, event ${GITHUB_EVENT_NAME}, run ${RUN_URL}"
+            echo
+            echo "A census went red: triage with ${GUIDE_URL}"
            if [ -n "$REDISPATCH_OF" ]; then echo; echo "This run is the one INFRA/INCOMPLETE re-dispatch of run ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${REDISPATCH_OF}; both runs are reported."; fi
            if [ "$REDISPATCH" = "true" ]; then echo; echo "Every red verdict is machine-classified INFRA/INCOMPLETE: re-dispatching once as a new run (EVAL_POLICY.infraRedispatch). This run stays red and reported."; fi
            echo
            echo "- periodic reconciliation exit: ${{ steps.reconcile.outputs.exit }} (slices job: ${{ needs.eval-slices.result }}, Codex slices job: ${{ needs.eval-codex-slices.result }})"
            echo "- gate census reconciliation exit: ${{ steps.gate-reconcile.outputs.exit }} (census job: ${{ needs.gate-census.result }})"
            echo "- pass-rate history gate exit: ${{ steps.pass-rates.outputs.exit }}"
+            echo "- $({ grep -E '^all-green probability' /tmp/pass-rates-reds.txt || echo 'all-green probability: unavailable (see the red ledger)'; } | published line)"
            echo
            echo "### Periodic lane"
            cat /tmp/paid-report/report-summary.md 2>/dev/null || echo "(no periodic report summary)"
@@ -682,23 +696,44 @@ jobs:
            cat /tmp/gate-census-report/report-summary.md 2>/dev/null || echo "(no gate census report summary)"
            echo
            echo "### Pass-rate history (ACTION REQUIRED)"
-            echo '```'
-            { grep -E 'ACTION REQUIRED|history unavailable' /tmp/pass-rates.txt || echo "(no pass-rate alarms)"; } | sed 's/@/@\xe2\x80\x8b/g' | head -c 6000
-            echo '```'
+            { awk '/^ACTION REQUIRED/ { on = 1; print; next } on && /^  \[/ { print; next } { on = 0 } /history unavailable/ { print }' /tmp/pass-rates.txt 2>/dev/null \
+              | grep . || echo "(no pass-rate alarms)"; } | head -c 6000 | published fence
+            echo
+            echo "### Red ledger: bun run eval:pass-rates --reds --runs 10 --branch ${GITHUB_REF_NAME}"
+            { head -n 1 /tmp/pass-rates-reds.txt; tail -n +2 /tmp/pass-rates-reds.txt | tail -n 60; } | cut -c1-400 | published fence
+            echo
+            echo "### Session headroom: bun run eval:pass-rates --headroom --runs 10 --branch ${GITHUB_REF_NAME}"
+            head -n 40 /tmp/pass-rates-headroom.txt | cut -c1-400 | published fence
            echo
            echo "<details><summary>Full reconciliation output</summary>"
            echo
-            echo '```'
-            tail -c 6000 /tmp/report.txt 2>/dev/null | sed 's/@/@\xe2\x80\x8b/g' || echo "(no reconciliation output)"
-            echo '```'
+            { tail -c 6000 /tmp/report.txt 2>/dev/null || echo "(no reconciliation output)"; } | published fence
            echo
-            echo '```'
-            tail -c 6000 /tmp/gate-report.txt 2>/dev/null | sed 's/@/@\xe2\x80\x8b/g' || echo "(no gate census reconciliation output)"
-            echo '```'
+            { tail -c 6000 /tmp/gate-report.txt 2>/dev/null || echo "(no gate census reconciliation output)"; } | published fence
            echo "</details>"
            echo
            echo "Policy: EVAL_POLICY and CASE_QUARANTINE in test/helpers/periodic-exclude-data.ts; history: \`bun run eval:pass-rates\`."
-          } > "$BODY_FILE"
+          } > /tmp/census-report.md
+          cat /tmp/census-report.md >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload the census report
+        if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success')
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: census-report-a${{ github.run_attempt }}
+          path: /tmp/census-report.md
+          if-no-files-found: ignore
+          retention-days: 90
+
+      - name: Upsert tracking issue on failure
+        if: always() && github.ref == 'refs/heads/main' && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success')
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -euo pipefail
+          TITLE="Weekly periodic evals: red lane needs triage"
+          BODY_FILE=/tmp/census-report.md
+          [ -s "$BODY_FILE" ] || echo "Periodic census report missing; run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" > "$BODY_FILE"
          EXISTING=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "in:title \"$TITLE\"" --json number --jq '.[0].number // empty')
          if [ -n "$EXISTING" ]; then
            gh issue comment "$EXISTING" --repo "$GITHUB_REPOSITORY" --body-file "$BODY_FILE"
@@ -708,7 +743,7 @@ jobs:
          fi

      - name: Close the tracking issue on a green run
-        if: always() && steps.reconcile.outputs.exit == '0' && steps.gate-reconcile.outputs.exit == '0' && steps.pass-rates.outputs.exit == '0' && needs.eval-slices.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) && needs.gate-census.result == 'success'
+        if: always() && github.ref == 'refs/heads/main' && steps.reconcile.outputs.exit == '0' && steps.gate-reconcile.outputs.exit == '0' && steps.pass-rates.outputs.exit == '0' && needs.eval-slices.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) && needs.gate-census.result == 'success'
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          REDISPATCH_OF: ${{ inputs.redispatch_of }}