From 95152e3c4c04c61d320e1f6602a8af3ef7b6f769 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Mon, 19 Oct 2026 21:47:11 -0700 Subject: [PATCH] v1.91.163.0 feat(evals): the gate reruns failed cases once and reports a flaky green (GSTA-49 F1) (#3315) * feat(evals): the gate reruns failed cases once and reports a flaky green (GSTA-49 F1) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * ci: retrigger after retargeting to main Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test(evals): a case red twice stays a real red for the report, the run, reuse and the train verdict (GSTA-49 F1) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(evals): keep a lone rerun-plan artifact off the plan manifest (run 39363705692) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * chore: stamp v1.91.163.0 Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- .github/workflows/evals.yml | 266 ++++++++++++- AGENTS.md | 7 +- CHANGELOG.md | 14 + CONTRIBUTING.md | 4 +- VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- docs/TESTING_INTERNALS.md | 58 ++- lib/ship-queue-train-record.ts | 6 +- package.json | 2 +- scripts/gate-reuse.ts | 25 +- scripts/harness-version.json | 6 +- scripts/lib/paid-plan.ts | 44 ++- scripts/lib/paid-report.ts | 76 ++-- scripts/lib/paid-rerun.ts | 396 ++++++++++++++++++++ scripts/test-paid-shards.ts | 20 +- scripts/train-gate.ts | 38 +- test/ci-native-evidence.test.ts | 4 +- test/ci-paid-coordination.test.ts | 10 +- test/evals-train-workflow.test.ts | 95 ++++- test/fixtures/shard-cli-parity/results.json | 8 + test/gate-reuse.test.ts | 73 +++- test/helpers/periodic-exclude-data.ts | 7 + test/paid-rerun.test.ts | 327 ++++++++++++++++ test/train-gate.test.ts | 31 ++ 24 files changed, 1441 insertions(+), 80 deletions(-) create mode 100644 scripts/lib/paid-rerun.ts create mode 100644 test/paid-rerun.test.ts diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml index 4c0066fe1..ec8d4cb29 100644 --- a/.github/workflows/evals.yml +++ b/.github/workflows/evals.yml @@ -739,7 +739,8 @@ jobs: run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --plan /tmp/paid-plan/manifest.json --slice ${{ matrix.slice }} # Attempt-scoped: a re-run attempt's trials are reported under that - # attempt and never replace (or collide with) the first attempt's. + # attempt and never collide with the first attempt's; the report reads + # them through the rule-16 case history (scripts/lib/paid-rerun.ts). - name: Upload slice results if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 @@ -781,6 +782,206 @@ jobs: if-no-files-found: ignore retention-days: 30 + # ── Rule-16 rerun (CREW rule 16, Garry 2026-10-19; a gate edit) ──────── + # "You can just run the failed test again and note that it is flaky... You + # don't have to do the whole test pass again for one failed test." After the + # slices, the planner reruns ONCE, at the same head and in this run attempt, + # only the cases whose verdict failed once with no rerun yet: a red rule or + # judge case on its own shard (narrowed to it in PR mode; other cases a whole + # file reruns never count), a behavior FAIL as its whole planned panel. + # Harness, INFRA, INCOMPLETE and unattributed reds, a case already red twice, + # or any other problem plan nothing (scripts/lib/paid-rerun.ts planRerun). + # The rerun runs fresh (no receipts, no reuse) and never changes a trial + # count, kind or threshold; slices-report decides FLAKY (green, annotated, + # flake record) or red. Train runs use the same path; bisects rerun inside + # scripts/train-gate.ts. + rerun-plan: + runs-on: ubuntu-24.04 + needs: [plan-slices, eval-slices] + if: ${{ !cancelled() && needs.plan-slices.result == 'success' && needs.plan-slices.outputs.gate_reuse != 'true' }} + timeout-minutes: 10 + permissions: + contents: read + outputs: + planned: ${{ steps.plan.outputs.planned }} + slices: ${{ steps.plan.outputs.slices }} + slice_timeouts: ${{ steps.plan.outputs.slice_timeouts }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ inputs.screen_merge_sha }} + persist-credentials: false + + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.4.2 + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + name: paid-plan + path: /tmp/paid-report + + # Every attempt's slices and earlier attempts' reruns: one history per case. + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + pattern: paid-slice-* + path: /tmp/paid-report + + # A missing rerun artifact leaves its case red (fail-closed), never green. + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + continue-on-error: true + with: + pattern: paid-rerun-* + # Its own directory: download-artifact unpacks a lone match into its path itself, so the + # report root would otherwise take a rerun slice's files (run 39363705692). + path: /tmp/paid-report/reruns + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + continue-on-error: true + with: + pattern: rerun-plan-a* + # Each artifact holds rerun-plan-a/manifest.json, so even a lone match never + # lands on the plan's own manifest.json (run 39363705692). + path: /tmp/paid-report + + - name: Plan the rule-16 rerun of this attempt's failed cases + id: plan + run: | + EVALS_TIER=gate bun --no-env-file --config=/dev/null --no-install run scripts/test-paid-shards.ts --tier gate --report /tmp/paid-report \ + --rerun-plan "/tmp/paid-rerun-plan/rerun-plan-a$GITHUB_RUN_ATTEMPT" >> "$GITHUB_STEP_SUMMARY" + PLAN="/tmp/paid-rerun-plan/rerun-plan-a$GITHUB_RUN_ATTEMPT/manifest.json" + if [ -f "$PLAN" ]; then + { echo "planned=true" + echo "slices=$(jq -c '[range(1; .sliceCount + 1)]' "$PLAN")" + echo "slice_timeouts=$(jq -ec '[0] + .plan.sliceCiTimeoutMinutes' "$PLAN")"; } >> "$GITHUB_OUTPUT" + else + echo "planned=false" >> "$GITHUB_OUTPUT" + fi + + - name: Hand the rerun manifest to its executors and the report + if: steps.plan.outputs.planned == 'true' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: rerun-plan-a${{ github.run_attempt }} + path: /tmp/paid-rerun-plan + retention-days: 90 + + eval-rerun: + runs-on: ubicloud-standard-8 + needs: [build-image, rerun-plan] + if: ${{ !cancelled() && needs.build-image.result == 'success' && needs.rerun-plan.outputs.planned == 'true' }} + env: + EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-eval-rerun-${{ matrix.slice }} + timeout-minutes: ${{ fromJSON(needs.rerun-plan.outputs.slice_timeouts)[matrix.slice] }} + permissions: + contents: read + packages: read + container: + image: ${{ needs.build-image.outputs.image-tag }} + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + options: --user runner + strategy: + fail-fast: false + max-parallel: 20 + matrix: + slice: ${{ fromJSON(needs.rerun-plan.outputs.slices) }} + steps: + - name: Record the job start for the slice deadline + run: echo "GSTACK_SLICE_JOB_STARTED_AT=$(date +%s)" >> "$GITHUB_ENV" + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ inputs.screen_merge_sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Fix bun temp + uses: ./.github/actions/fix-bun-temp + + - name: Restore deps + uses: ./.github/actions/restore-deps + + - run: bun run build + + - name: Seed claude interactive config + uses: ./.github/actions/seed-claude-config + with: + anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} + + - name: Register gstack skills for PTY smokes + uses: ./.github/actions/register-gstack-skills + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + name: rerun-plan-a${{ github.run_attempt }} + path: /tmp/paid-rerun-plan + + - name: Authenticate the Codex CLI + shell: bash + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + run: | + set -euo pipefail + codex="${GSTACK_CI_CODEX_BIN_DIR:?}/codex" + export CODEX_HOME="$HOME/.gstack-ci-codex" + mkdir -p "$CODEX_HOME" + echo "GSTACK_CI_CODEX_HOME=$CODEX_HOME" >> "$GITHUB_ENV" + if [ -z "${OPENAI_API_KEY:-}" ]; then + echo "::warning::OPENAI_API_KEY is unavailable; Codex eval cases cannot authenticate and will fail." + exit 0 + fi + printenv OPENAI_API_KEY | "$codex" login --with-api-key + "$codex" login status >/dev/null 2>&1 + + # No EVALS_CACHE_*: a rerun never reuses a receipt and publishes none. + - name: Rerun slice ${{ matrix.slice }} (rule 16, once) + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + PLAYWRIGHT_BROWSERS_PATH: /opt/playwright-browsers + EVALS_JOBS: "2" + EVALS_CONCURRENCY: "2" + GSTACK_EVAL_DIR: /tmp/paid-rerun-results + run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --plan /tmp/paid-rerun-plan/rerun-plan-a${{ github.run_attempt }}/manifest.json --slice ${{ matrix.slice }} + + - name: Upload rerun results + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: paid-rerun-${{ matrix.slice }}-a${{ github.run_attempt }} + path: /tmp/paid-rerun-results + retention-days: 90 + + - name: Upload native capture evidence + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: native-captures-${{ env.EVALS_RUN_ID }} + include-hidden-files: true + path: | + ~/.gstack/projects/*/e2e-runs + ~/.gstack/projects/*/evals/qa-callers + ~/.gstack-dev/e2e-runs + ~/.gstack-dev/evals/qa-callers + if-no-files-found: ignore + retention-days: 90 + + - name: Upload shard logs + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: rerun-logs-${{ matrix.slice }}-a${{ github.run_attempt }} + include-hidden-files: true + path: | + /home/runner/.cache/gstack-paid-shard-*.log + /tmp/gstack-paid-shard-*.log + if-no-files-found: ignore + retention-days: 30 + # ── Train bisect (GSTA-49 T1b) ───────────────────────────────────────── # One executor per prefix of the train (route's validated prefix_shas, each # an ancestor of the train head): every red case once, one trial, on that @@ -932,7 +1133,7 @@ jobs: # sees, so a store saved on Ubicloud was never restored (GSTA-48: every # restore missed, e.g. #3253's restamp gate 3 minutes after a saved store). runs-on: ubuntu-24.04 - needs: [plan-slices, eval-slices] + needs: [plan-slices, eval-slices, rerun-plan, eval-rerun] # !cancelled(): the report must run (and FAIL) when an executor died — a # missing slice artifact reading as green is the class this lane kills — # but a run superseded by a newer push stops here. @@ -969,16 +1170,38 @@ jobs: path: /tmp/paid-report # One directory per attempt-scoped slice artifact (no merge): shard - # records can never overwrite each other, and the report keeps the - # first attempt's verdict. + # records can never overwrite each other. Attempt 1 decides the lane; + # a later attempt or a rule-16 rerun resolves a red only by the flaky + # rule (scripts/lib/paid-rerun.ts) and never hides a new red. - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 with: pattern: paid-slice-* path: /tmp/paid-report + # Rule-16 reruns and their manifests. A missing one leaves its case red. + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + continue-on-error: true + with: + pattern: paid-rerun-* + # Its own directory: download-artifact unpacks a lone match into its path itself, so the + # report root would otherwise take a rerun slice's files (run 39363705692). + path: /tmp/paid-report/reruns + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + continue-on-error: true + with: + pattern: rerun-plan-a* + # Each artifact holds rerun-plan-a/manifest.json, so even a lone match never + # lands on the plan's own manifest.json (run 39363705692). + path: /tmp/paid-report + - name: Reconcile slices against the manifest (fail-closed) id: reconcile if: needs.plan-slices.outputs.gate_reuse != 'true' + env: + # Flake records name the PR and its head (scripts/lib/paid-rerun.ts flakeRecords). + GSTACK_GATE_PR: ${{ github.event.pull_request.number || inputs.screen_pr }} + GSTACK_GATE_HEAD_SHA: ${{ github.event.pull_request.head.sha || needs.plan-slices.outputs.screen_head || inputs.expected_sha }} run: | set +e EVALS_TIER=gate bun --no-install run scripts/test-paid-shards.ts --tier gate --report /tmp/paid-report | tee /tmp/report.txt @@ -1043,6 +1266,18 @@ jobs: if-no-files-found: ignore retention-days: 30 + # Rule 16: one JSONL record per case that failed once and passed on its + # one rerun (case, PR, head, run, both attempts, symptom, rerun cost), + # read by the daily flaky-fix pass (docs/TESTING_INTERNALS.md#flaky-reruns). + - name: Upload flake records for the daily flaky-fix pass + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: flake-records-a${{ github.run_attempt }} + path: /tmp/paid-report/flake-records.jsonl + if-no-files-found: ignore + retention-days: 90 + # R2a/R2: what this executed run proved, read by a later push. The base # is the tested merge commit's first parent. Reused runs bind nothing, # so reuse never chains off a reuse. @@ -1193,7 +1428,7 @@ jobs: REUSE=/tmp/paid-report/gate-reuse.json if [ "${RECONCILE_EXIT:-1}" = "0" ] && [ -f "$REUSE" ] \ && jq -e '.schema == 1 and (.kind == "restamp-only" or .kind == "main-moved")' "$REUSE" >/dev/null 2>&1; then - BODY=$(jq -r --arg repo "$GITHUB_REPOSITORY" '"## E2E Evals: ✅ PASS (reused: \(if .kind == "restamp-only" then "restamp-only, R2a" else "main moved, R2" end))\n\nNo paid case ran on `\(.current_head)`. It carries the GREEN verdict of [run \(.reused_run_id)](https://github.com/\($repo)/actions/runs/\(.reused_run_id)) at `\(.gated_head)`, because \(.why).\n\n```\n\(if .kind == "restamp-only" then "TREE: same-modulo-stamps receipt=\(.receipt_sha256)" else "OWN DIFF: identical sha256=\(.own_diff_sha256) (merge-base \(.gated_merge_base) -> \(.merge_base))" end)\nbase: \(.gated_base) -> \(.base_sha)\nselection: \(.selection_sha256) (identical to the reused gate)\n```\n\n---\n*Verdict reuse (R2a restamp-only, R2 main moved): a changed own diff, a non-stamp byte, a changed selection, a newer red or unbound gate, or a missing binding or receipt runs the full gate; the evals-fresh label opts out. Free Tests and every other check still ran on this head.*"' "$REUSE") + BODY=$(jq -r --arg repo "$GITHUB_REPOSITORY" '"## E2E Evals: ✅ PASS (reused: \(if .kind == "restamp-only" then "restamp-only, R2a" else "main moved, R2" end))\n\nNo paid case ran on `\(.current_head)`. It carries the GREEN verdict of [run \(.reused_run_id)](https://github.com/\($repo)/actions/runs/\(.reused_run_id)) at `\(.gated_head)`, because \(.why).\(if ((.reused_flaky // []) | length) > 0 then " That gate was green with flaky case(s) that failed once and passed on their one rule-16 rerun: \(.reused_flaky | join(", "))." else "" end)\n\n```\n\(if .kind == "restamp-only" then "TREE: same-modulo-stamps receipt=\(.receipt_sha256)" else "OWN DIFF: identical sha256=\(.own_diff_sha256) (merge-base \(.gated_merge_base) -> \(.merge_base))" end)\nbase: \(.gated_base) -> \(.base_sha)\nselection: \(.selection_sha256) (identical to the reused gate)\n```\n\n---\n*Verdict reuse (R2a restamp-only, R2 main moved): a changed own diff, a non-stamp byte, a changed selection, a newer red or unbound gate, or a missing binding or receipt runs the full gate; the evals-fresh label opts out. Free Tests and every other check still ran on this head.*"' "$REUSE") else TOTAL=0; PASSED=0; FAILED=0; MANUAL=0; EXECUTED=0; REUSED=0; COST="0" SUITE_LINES="" @@ -1203,6 +1438,8 @@ jobs: .version == 2 and (.files | type == "array") and (.totals | type == "object") and (.headline | type == "array") and (.failures | type == "array") and (.panels | type == "array") and (.verdict.verdict == "GREEN" or .verdict.verdict == "RED") and + ((.flakyCollectorFailures // 0) | type == "number" and . >= 0 and floor == .) and + ((.flakyCollectorFailures // 0) <= .totals.failed) and ([.files[] | .total == (.passed + .failed + .manual_accepted) and (.total == (.executed + .reused)) and ([.total,.passed,.failed,.manual_accepted,.executed,.reused,.attempts,.flaky] | all(. >= 0 and (floor == .))) ] | all) and @@ -1215,6 +1452,8 @@ jobs: fi HEADLINE='(no verified report headline)' FAILURES="" + FLAKY="" + FLAKY_FAILED=0 if [ -n "$VERIFIED" ]; then while IFS=$'\t' read -r _FILE T P F M _FLAKY EX RE _ATTEMPTS C TIER SHARD; do [ "$T" -eq 0 ] && continue @@ -1230,6 +1469,10 @@ jobs: # Report-sanitized lines (no @-mentions, one capped line each), fenced here. HEADLINE=$(jq -r '.headline[]' "$VERIFIED") FAILURES=$(jq -r '.failures[]' "$VERIFIED") + # Rule 16: report-sanitized lines of cases that failed once and passed on their one rerun. + FLAKY=$(jq -r '(.flakyLines // [])[]' "$VERIFIED") + # Their first-attempt collector failures, the only failed records a green report may carry. + FLAKY_FAILED=$(jq -r '.flakyCollectorFailures // 0' "$VERIFIED") fi COVERAGE=$(jq -r '"Profile: \(.profile // "full") / mode `\(.prCoverage.mode // "broad")`; selected behaviors: \(.selection.e2e | if . == null then "all" else length end), judges: \(.selection.judges | if . == null then "all" else length end). Deferred to scheduled/release coverage: \(.prCoverage.deferred // [] | length) behaviors and \(.prCoverage.deferredPromptFiles // [] | length) changed prompt files. Deferred checks did not run and receive no PR-pass credit."' /tmp/paid-report/manifest.json) || COVERAGE='Coverage manifest unavailable; no coverage claim.' @@ -1249,9 +1492,10 @@ jobs: ${DERIVED}"; fi STATUS="✅ PASS" - if [ "${RECONCILE_EXIT:-1}" != "0" ] || [ "$FAILED" -gt 0 ] \ + if [ "${RECONCILE_EXIT:-1}" != "0" ] || [ "$FAILED" -gt "$FLAKY_FAILED" ] \ || { [ -n "$VERIFIED" ] && [ "$(jq -r '.verdict.verdict' "$VERIFIED")" != "GREEN" ]; }; then STATUS="❌ FAIL"; fi if [ "$STATUS" = '✅ PASS' ] && [ "$MANUAL" -gt 0 ]; then STATUS='⚠ MANUAL ACCEPTED (unscored)'; fi + if [ "$STATUS" = '✅ PASS' ] && [ -n "$FLAKY" ]; then STATUS="✅ PASS (flaky: $(grep -c . <<<"$FLAKY") case(s) passed on their one rerun)"; fi if [ -z "$VERIFIED" ]; then STATUS='❌ FAIL (verified report unavailable)'; fi BODY="## E2E Evals: ${STATUS} @@ -1290,6 +1534,16 @@ jobs: \`\`\`" fi + if [ -n "$FLAKY" ]; then + BODY="${BODY} + + ### Flaky: failed once, passed on its one rerun (rule 16) + \`\`\` + ${FLAKY} + \`\`\` + Each failed and passed on its rerun in [run ${GITHUB_RUN_ID}](https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}) at the attempts named above. Logged for the daily flaky-fix pass in this run's \`flake-records-a${GITHUB_RUN_ATTEMPT}\` artifact (docs/TESTING_INTERNALS.md#flaky-reruns)." + fi + fi if [ -n "$SCREEN_POLICY" ]; then diff --git a/AGENTS.md b/AGENTS.md index ee8777385..04ac4b7c0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -211,7 +211,12 @@ When fixing failures or preparing `/ship`, follow this order: result and pending permission state; diagnose a blocked actor before waiting through its deadline. Preserve cancellation separately from a test verdict. Skipped or unstarted cases - do not satisfy coverage; preserve every attempt. Paid evals never retry. Each + do not satisfy coverage; preserve every attempt. Paid evals never retry, with + one exception: the gate reruns a red rule, behavior or judge case verdict once + at the same head (Garry, 2026-10-19: "you can just run the failed test again + and note that it is flaky"); a pass is green with a flaky annotation and a + flake record, a second failure is a real red, and harness, INFRA, INCOMPLETE + or unattributed reds never rerun (`docs/TESTING_INTERNALS.md#flaky-reruns`). Each case's kind (`E2E_KINDS`) fixes its trials before the run: `rule` one trial; `behavior` a panel of 3 independent trials, PASS at >= 2 with no contract violation; `judge` 3 samples on one output, each dimension gated on its diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a0cfded6..dbd6ef057 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## [1.91.163.0] - 2026-10-19 + +**A gate red on one test now reruns that test once instead of the whole gate, and a pass is green with a flaky note.** Garry, 2026-10-19 (CREW rule 16): "you can just run the failed test again and note that it is flaky... You don't have to do the whole test pass again for one failed test." Plan GSTA-49, lane F1; the #3308 reuse miss is GSTA-53. + +### Added +- **In-run flaky rerun (`rerun-plan` and `eval-rerun` in `evals.yml`, `scripts/lib/paid-rerun.ts`).** After the slices, a red rule, behavior or judge case verdict reruns once at the same head in the same run attempt: a rule or judge case on its own shard (only the rerun cases count), a behavior case as its whole 3-trial panel, a judge case with its 3 samples. It runs fresh (no receipts in or out). If it passes, the gate is GREEN with a `FLAKY N` headline line and rerun spend, a `⚑` line per case in the report and PR comment, and one record per case in the run's `flake-records-a` artifact for the daily flaky-fix pass. If it fails again, it is a real red (`FAILED TWICE`). +- **Manual `gh run rerun --failed` attempts follow the same rule.** An attempt that passes every earlier red is GREEN with the same flaky annotation. A red that first appears in a later attempt stays red unless its own next run passes, so a later attempt never hides a new red. +- **Flaky GREENs are reusable (GSTA-53).** `gate-reuse` binds a flaky GREEN with its `flaky_cases`, reads each run's newest attempt binding, and R2a and R2 reuse it with `reused_flaky` named in the log and PR comment. On #3308 (run 39315041557), both attempts bound RED because the report kept attempt 1's verdict, so the next push paid for a full gate. A run still red after its rerun binds RED and is never reused. + +### Changed +- **Gate integrity (a gate edit):** harness, INFRA, INCOMPLETE and unattributed reds never rerun and are never flaky. Neither is a case already red twice, or a case whose rerun could not turn the gate green. No threshold, trial count, case kind, judge model, `EVAL_POLICY` constant or `HARNESS_VERSION` changed (recorded non-behavioral). Pass-rate history still records each run's first execution unchanged; a rerun's trials stay in its `paid-rerun-*` artifacts and the flake record. On the gate lane, a rule shard that ran to completion with a red case no longer fails its slice job. The report still fails closed on it, so the run's conclusion follows the report and a flaky GREEN run concludes `success`. Execution failures (timeouts, runner errors, hollow or unfinished shards) still fail the slice. Train gates take the same rerun path. A bisect reruns a VERDICT FAIL once on its prefix (`scripts/train-gate.ts`), and a pass there is a flaky PASS naming both runs. The PR comment's verified-summary check also bounds `flakyCollectorFailures`, so a failed collector record that is not a flaky case still fails the comment. Negative control for the slice-exit change: a case red on its first run and red again on its rerun keeps the real report CLI at exit 1 and `VERDICT RED`; `slices-report` still runs after every rerun path and its last step fails the job on reconcile exit 1, no job carries `continue-on-error`, so the run concludes `failure`; the binding is RED and `gate-reuse` refuses it; and the train record is RED, which T1 classifies RED whatever the run concluded (`test/paid-rerun.test.ts`, `test/evals-train-workflow.test.ts`). +- **Free regressions:** `test/paid-rerun.test.ts` replays a trimmed capture of run 39293281037 through the real `--report` and `--rerun-plan` CLI. Its negative controls cover red twice, harness, INCOMPLETE and checkpoint reds, a new red on a later attempt, a first-attempt pass, rerun results with a missing or mismatched manifest, and reruns of unplanned cases. `test/gate-reuse.test.ts` covers R2a and R2 reuse of a flaky GREEN, a manual-attempt binding, and a run still red after its rerun. `test/train-gate.test.ts` covers the train record and the bisect rerun. Workflow wiring tests are in `test/evals-train-workflow.test.ts` and `test/ci-paid-coordination.test.ts`. +- **Rerun artifact layout:** the first gate of this change (run 39363705692) reran its one red case and passed, but `download-artifact` unpacked the lone `rerun-plan-a1` match into the report root, over the plan's own `manifest.json`, so the report read every case as unplanned and stayed red. The plan artifact now holds `rerun-plan-a/manifest.json`, rerun results download under `reruns/`, the rerun logs artifact is `rerun-logs-*` so it no longer matches `paid-rerun-*`, and the report finds the rerun manifest in either layout `download-artifact` produces. That run's artifacts, replayed in the new layout, report `VERDICT GREEN` with `FLAKY 1`. + ## [1.91.162.0] - 2026-10-19 **The evals workflow carries merge trains: a train's own gate publishes its record and the `gstack/full-gate` check, bisects run one executor per prefix, `ready --screen` dispatches a real screen, and a main backstop gates any tree no green train covered.** All of it stays off unless the base revision's `.gstack/ship-policy.json` declares a `train` block. Without one, a PR push and every eval-bg dispatch run today's jobs and steps with today's values. The one addition is a free `route` job that reads the policy with git and jq. gstack's own policy still has no `train` block; turning it on is a separate one-line policy PR. Plan GSTA-49, lane T1b. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a00e4474b..ee101e0eb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -279,7 +279,9 @@ Complete start-to-finish flows belong to the `marathon` tier (`describeE2ETier('marathon')`), which runs only in the non-blocking `evals-marathon.yml` lane (weekly and on dispatch) and never gates a merge. -Verdicts: paid evals never retry. Each case's kind in `E2E_KINDS` +Verdicts: paid evals never retry, except the gate's one flaky rerun of a red +case verdict at the same head ([flaky reruns](docs/TESTING_INTERNALS.md#flaky-reruns)). +Each case's kind in `E2E_KINDS` (`test/helpers/touchfiles-data.ts`) fixes its trials before the run, from the constants in `EVAL_POLICY` (`test/helpers/periodic-exclude-data.ts`): diff --git a/VERSION b/VERSION index de430fa60..c21a1c317 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.162.0 +1.91.163.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 5b0af3a1d..3d318789a 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.162.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.163.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/docs/TESTING_INTERNALS.md b/docs/TESTING_INTERNALS.md index 3aec0d114..aec69f4e7 100644 --- a/docs/TESTING_INTERNALS.md +++ b/docs/TESTING_INTERNALS.md @@ -256,8 +256,8 @@ full-stream spooling to per-shard log files (path printed at START and on failure), never-started/timed-out taxonomy, and parent-computed diff selection propagated to children via `EVALS_SELECTION_JSON` (fail-open: a child that can't parse it recomputes locally with one warning). Paid evals -never retry; each case's kind fixes its trials before the run (see "Eval verdict -policy" below). Files in `CASE_SHARDED_FILES` run one registered case per +never retry, except the gate's one [flaky rerun](#flaky-reruns); each case's kind +fixes its trials before the run (see "Eval verdict policy" below). Files in `CASE_SHARDED_FILES` run one registered case per process (`#`, an exact `--test-name-pattern`, exactly one executed case), so a long file of short cases spreads across runners and each case gets its own SDK semaphore. `test/carve-section-loading.test.ts` is one of them: it registers @@ -411,7 +411,7 @@ archaeology. `test/helpers/periodic-exclude-data.ts`; version 1 pre-registered 2026-09-29, version 2 approved 2026-10-04 with the decision memo in [test-audit-2026-10.md](test-audit-2026-10.md)). Paid evals -never retry. Each live case has exactly one kind in `E2E_KINDS` +never retry, except the gate's one [flaky rerun](#flaky-reruns). Each live case has exactly one kind in `E2E_KINDS` (`test/helpers/touchfiles-data.ts`; `test/eval-kinds.test.ts` enforces coverage), and the kind fixes its trials before the run: @@ -448,7 +448,8 @@ all use. A timed-out, crashed or infrastructure-failed trial is a failed trial recorded with its class; a missing or duplicate trial record makes the verdict INCOMPLETE, which fails the lane; a 2/3 PASS is shown as `PASS 2/3` with the failed trial's cause. A manual re-run adds trials under a new run attempt and -never replaces the first attempt's verdict. A red census is never rerun on +never replaces the first attempt's verdict; on the gate it resolves a red only +by the [flaky rule](#flaky-reruns). A red census is never rerun on unchanged inputs: each red is diagnosed as product, test/detector, harness or infra and resolved by a concrete repair and a fresh census, or listed as a named red. The one exception: a census whose every red verdict is machine-classified @@ -458,6 +459,51 @@ reported. Changing any `EVAL_POLICY` constant after seeing census results needs Garry's re-approval, a `version` bump and a fresh census; `test/periodic-exclude-policy.test.ts` pins the approved values. +### Flaky reruns + +**CREW rule 16; a gate edit.** Garry, 2026-10-19: "you can just +run the failed test again and note that it is flaky... You don't have to do the +whole test pass again for one failed test." After the gate's slices, +`rerun-plan` in `evals.yml` (`scripts/test-paid-shards.ts --report DIR +--rerun-plan OUT`, `scripts/lib/paid-rerun.ts`) reruns, once, at the same head +and in the same run attempt, every case whose verdict failed and that nothing +has re-executed yet: a red rule or judge case on its own shard (narrowed to that +case in PR mode; when a dependents or full-fallback plan reruns the whole file, +only the planned cases count), a behavior FAIL as its whole planned panel of 3 trials, a judge case with +its 3 samples. Kinds, trial counts, thresholds and the judge model never change, +and the rerun runs fresh (`eval-rerun`: no receipts in, none published). It +plans nothing when the red is a harness failure (runner error, slice deadline, +unfinalized checkpoint, a different executed case count, a reused result), an +INFRA or INCOMPLETE panel, an INFRA-class or unattributed testcase, a case +already red twice, or when any other problem would keep the gate red anyway. + +`slices-report` reads every attempt's slices (`paid-slice-*`), each attempt's +rerun (`paid-rerun-*`, bound to its own `rerun-plan-a` manifest) and +orders them into one history per case. A case is **FLAKY** when it failed +exactly once, that failure was eligible, and its very next observation passed: +the gate is GREEN with a `FLAKY N` headline line, a `⚑` line per case in the +report and PR comment, and one JSONL record per case in the run's +`flake-records-a` artifact (`flake-records.jsonl`: case, kind, PR, +head, run, where it failed and passed, symptom, rerun cost) for the daily +flaky-fix pass. A case that fails twice is a real red (`FAILED TWICE`). Rerun +spend is reported on the same headline line. Pass-rate history +(`trial-outcomes.jsonl`) still records each run's first execution unchanged; a +rerun's trials stay in its `paid-rerun-*` artifacts and the flake record. + +A manual `gh run rerun --failed` attempt follows the same history, so an +attempt that passes every earlier red is GREEN with the same annotation, while +a red that first appears in a later attempt stays red unless its own next +observation passes; a later attempt never hides a new red. On the gate lane a +completed rule shard's red case no longer fails its slice job (`sliceExitCode` +`caseVerdicts`); the report still fails closed on it, so the run's conclusion +follows the report, and a flaky GREEN run concludes `success`. A rerun that +`--failed` re-triggers (because an earlier problem blocked the in-run rerun) +still reruns a case only once. A flaky GREEN binds as GREEN with its +`flaky_cases` and is reusable by R2a and R2 (`reused_flaky`); a run still red +after its rerun binds RED and is never reused. Train runs take the same +`rerun-plan` path; a bisect reruns a VERDICT FAIL once on its prefix inside +`scripts/train-gate.ts`, and a pass there is a flaky PASS naming both runs. + ### Ship measure **Measure-then-fix: diagnostic trials are not verdicts** (`scripts/ship-measure.ts`). @@ -1164,7 +1210,9 @@ or cancelled before a slice finished); a newer run that is still running, finished a slice, or holds a `paid-slice-*` artifact without a bound verdict blocks reuse, so a newer red or partial result is never skipped. The gate is GREEN with reconcile exit 0, a successful run and no manually accepted -(unscored) case; it ran the PR profile on a different head. Then either: +(unscored) case; it ran the PR profile on a different head. Each run's newest +attempt binding counts, and a [flaky](#flaky-reruns) GREEN is a GREEN whose +reuse names its flaky cases. Then either: - **R2a, restamp-only:** it ran on the same base (the tested merge commit's first parent), the tree receipt (`lib/tree-receipt.ts` under the base's ship diff --git a/lib/ship-queue-train-record.ts b/lib/ship-queue-train-record.ts index 66c62b150..9d0670b01 100644 --- a/lib/ship-queue-train-record.ts +++ b/lib/ship-queue-train-record.ts @@ -37,7 +37,11 @@ export interface TrainRecord { } export type AttemptPurpose = 'gate' | 'rebuild' | 'infra-redispatch' | 'bisect'; export type GateOutcome = 'PENDING' | 'GREEN' | 'RED' | 'INCOMPLETE'; -export interface CasePanel { case: string; kind: string; status: string; red_class: string | null; cost_usd: number | null } +export interface CasePanel { + case: string; kind: string; status: string; red_class: string | null; cost_usd: number | null; + /** Rule 16: a PASS that failed once and passed on its one rerun, with where each ran. */ + flaky?: { first: string; rerun: string }; +} export interface AttemptRecord { schema: 1; kind: 'train-attempt'; train: string; id: string; purpose: AttemptPurpose; nonce: string; workflow: string; ref: string; inputs: Record; state: 'dispatching' | 'unknown' | 'absent' | 'dispatched' | 'completed'; run: { id: number; url: string } | null; diff --git a/package.json b/package.json index e9e7fc750..ac986a554 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.162", + "version": "1.91.163", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/scripts/gate-reuse.ts b/scripts/gate-reuse.ts index 4bc4a633c..4a7dccb15 100644 --- a/scripts/gate-reuse.ts +++ b/scripts/gate-reuse.ts @@ -9,9 +9,13 @@ * report verdict and its plan's selection fingerprint. Reused runs * bind nothing, so reuse never chains off a reuse. * find (recover-receipts, BASE-REF code, actions: read) takes the newest - * earlier completed run of this PR that bound a verdict. It must be - * GREEN with reconcile exit 0 and a successful conclusion, ran the - * PR profile, and gated a different head. Then one of: + * earlier completed run of this PR that bound a verdict (its + * newest attempt's binding). It must be GREEN with reconcile exit 0 + * and a successful conclusion, ran the PR profile, and gated a + * different head. A flaky-annotated GREEN (rule 16: a red that + * passed its one in-run rerun, or a `gh run rerun --failed` + * attempt that passed every earlier red) is such a GREEN; a run + * still red after its rerun binds RED and is never reused. Then one of: * restamp-only (R2a): same base; the tree receipt * (lib/tree-receipt.ts, the base's ship policy) says * `TREE: same-modulo-stamps` and every differing file is a stamp @@ -78,6 +82,8 @@ export interface GateBinding { /** Cases accepted manually (unscored); a gate with any is never reused as a clean pass. */ manual_accepted: number; selection_sha256: string; + /** Rule 16: cases that failed once and passed on their one rerun; the GREEN is reusable and carries their names. */ + flaky_cases?: string[]; /** Dispatched screens only: the purpose, the refs/pull//merge commit it tested, and the screen policy. */ purpose?: 'screen'; tested_sha?: string; @@ -112,6 +118,8 @@ export interface GateReuse { own_diff_sha256?: string; /** The reused run was a dispatched screen with this policy. */ reused_screen_policy?: ScreenPolicy; + /** The reused GREEN was flaky-annotated (rule 16): these cases failed once and passed on their one rerun. */ + reused_flaky?: string[]; } /** The cases, trial plans and verdict-relevant plan fields a run executes; slice packing and receipts are excluded. */ @@ -133,6 +141,7 @@ export function parseBinding(value: unknown): GateBinding | null { if (!b || b.schema !== 1 || typeof b.repository !== 'string' || !Number.isSafeInteger(b.pr) || !Number.isSafeInteger(b.run_id) || !Number.isSafeInteger(b.run_attempt) || !SHA.test(b.head_sha ?? '') || !SHA.test(b.base_sha ?? '') || typeof b.profile !== 'string' || !Number.isSafeInteger(b.reconcile_exit) || typeof b.verdict !== 'string' || !Number.isSafeInteger(b.manual_accepted) || b.manual_accepted < 0 || !SHA256.test(b.selection_sha256 ?? '')) return null; + if (b.flaky_cases !== undefined && (!Array.isArray(b.flaky_cases) || b.flaky_cases.some(id => typeof id !== 'string' || !id))) return null; if (b.purpose !== undefined && (b.purpose !== 'screen' || !SHA.test(b.tested_sha ?? '') || (b.screen_policy !== 'narrow' && b.screen_policy !== 'broad'))) return null; return b; } @@ -278,13 +287,15 @@ export function buildBinding(o: { repository: string; pr: number; runId: number; if (!o.baseSha) return { binding: null, reason: 'the tested base is unknown' }; const manifest = readJson(path.join(o.reportDir, 'manifest.json')); if (!manifest?.entries) return { binding: null, reason: 'no readable manifest' }; - const outcomes = readJson<{ verdict?: { verdict?: string }; totals?: { manual_accepted?: number } }>(path.join(o.reportDir, 'collector-outcomes.json')); + const outcomes = readJson<{ verdict?: { verdict?: string }; totals?: { manual_accepted?: number }; flaky?: Array<{ case?: string }> }>(path.join(o.reportDir, 'collector-outcomes.json')); + const flaky = (outcomes?.flaky ?? []).map(record => record.case ?? ''); const binding: GateBinding = { schema: 1, repository: o.repository, pr: o.pr, run_id: o.runId, run_attempt: o.runAttempt, head_sha: o.headSha, base_sha: o.baseSha, profile: o.profile, reconcile_exit: o.reconcileExit, verdict: outcomes?.verdict?.verdict ?? 'MISSING', manual_accepted: outcomes?.totals?.manual_accepted ?? -1, selection_sha256: selectionFingerprint(manifest), + ...(flaky.length ? { flaky_cases: flaky } : {}), ...(o.purpose === 'screen' ? { purpose: 'screen' as const, tested_sha: o.testedSha ?? '', screen_policy: o.screenPolicy as ScreenPolicy } : {}) }; return parseBinding(binding) - ? { binding, reason: `bound ${binding.verdict} at ${binding.head_sha.slice(0, 12)} on base ${binding.base_sha.slice(0, 12)} (reconcile exit ${binding.reconcile_exit})` } + ? { binding, reason: `bound ${binding.verdict}${binding.flaky_cases ? ` (flaky: ${binding.flaky_cases.join(', ')})` : ''} at ${binding.head_sha.slice(0, 12)} on base ${binding.base_sha.slice(0, 12)} (reconcile exit ${binding.reconcile_exit})` } : { binding: null, reason: 'binding fields are incomplete' }; } @@ -419,7 +430,8 @@ export function findReusableGate(o: FindOptions): { candidate: GateReuse | null; if (policy.errors.length) return refuse(`the base's ship policy is invalid (${policy.errors[0]!.message})`); const common = { schema: 1 as const, pr: o.pr, reused_run_id: run.id, reused_run_attempt: binding.run_attempt, gated_head: binding.head_sha, current_head: o.headSha, gated_base: binding.base_sha, base_sha: baseSha, selection_sha256: binding.selection_sha256, - ...(binding.screen_policy ? { reused_screen_policy: binding.screen_policy } : {}) }; + ...(binding.screen_policy ? { reused_screen_policy: binding.screen_policy } : {}), + ...(binding.flaky_cases?.length ? { reused_flaky: binding.flaky_cases } : {}) }; let candidate: GateReuse; if (binding.base_sha === baseSha) { @@ -484,6 +496,7 @@ export function reuseReport(reuse: GateReuse, repo: string): string { `GATE REUSED (${reuse.kind === 'restamp-only' ? 'R2a, restamp-only' : 'R2, main moved'}): no paid case ran on ${reuse.current_head}`, `reused verdict: GREEN from run ${reuse.reused_run_id} attempt ${reuse.reused_run_attempt}${reuse.reused_screen_policy ? ` (a ${reuse.reused_screen_policy} screen)` : ''} at ${reuse.gated_head} (https://github.com/${repo}/actions/runs/${reuse.reused_run_id})`, `why: ${reuse.why}`, + ...(reuse.reused_flaky?.length ? [`flaky (rule 16): ${reuse.reused_flaky.join(', ')} failed once and passed on their one rerun in the reused gate`] : []), `base: ${reuse.gated_base} -> ${reuse.base_sha}`, ...evidence, `selection: ${reuse.selection_sha256} (identical to the reused gate)`, diff --git a/scripts/harness-version.json b/scripts/harness-version.json index 0c9832bef..779878132 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -12,11 +12,11 @@ ], "files": { "scripts/lib/paid-cases.ts": {"blob":"f862cf5241580c7593b9ee62eacf5ef046248d26","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, - "scripts/lib/paid-plan.ts": {"blob":"95e1798e7e2495ace350d2624e59f83412f341bc","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, - "scripts/lib/paid-report.ts": {"blob":"f92c6bb723b158e328aa040fa5de6a0ee6d3916d","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, + "scripts/lib/paid-plan.ts": {"blob":"e8ce3cb98474dded38c5128a631d283b0ca35b63","since":6,"nonBehavioral":"F1 rule-16 flaky rerun: report aggregation, rerun planning and the gate slice exit code; no case's execution, trial outcome, trial record or panel verdict changes, so no pass-rate series resets"}, + "scripts/lib/paid-report.ts": {"blob":"df19605183b8c6bfabd01bbe23fbe821d97a405e","since":6,"nonBehavioral":"F1 rule-16 flaky rerun: report aggregation, rerun planning and the gate slice exit code; no case's execution, trial outcome, trial record or panel verdict changes, so no pass-rate series resets"}, "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6}, "scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":6}, - "scripts/test-paid-shards.ts": {"blob":"7a362c457cd0536dad60aecc675d67599759c8a3","since":6,"nonBehavioral":"GSTA-56: only the shebang changed, to the isolated form (--no-env-file --config=/dev/null); the paid runner is launched via bun run, which ignores the shebang, so no verdict can change"}, + "scripts/test-paid-shards.ts": {"blob":"b3b4786e406ca6793df2eb24fafac27cfb99a48e","since":6,"nonBehavioral":"F1 rule-16 flaky rerun: report aggregation, rerun planning and the gate slice exit code; no case's execution, trial outcome, trial record or panel verdict changes, so no pass-rate series resets"}, "scripts/test-pr-profile.ts": {"blob":"d334195b0d0393622e2c380dc3e2feebe099b6e6","since":6,"nonBehavioral":"classify the new evals-main-backstop.yml workflow as free-lane only: it dispatches evals.yml on main and runs inside no case, so no case's selection on a PR, input or verdict changes"}, "scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":6}, "test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":6}, diff --git a/scripts/lib/paid-plan.ts b/scripts/lib/paid-plan.ts index 95e1798e7..e8ce3cb98 100644 --- a/scripts/lib/paid-plan.ts +++ b/scripts/lib/paid-plan.ts @@ -66,14 +66,17 @@ export interface PaidRunManifest { selection?: PaidCaseSelection; prCoverage?: PrProfileSelection; plan?: PaidSlicePlan; + /** A rule-16 rerun manifest (scripts/lib/paid-rerun.ts): the run attempt whose reds it reruns once, and those cases. */ + rerun?: { attempt: number; cases: string[] }; entries: ManifestEntry[]; } /** - * Paid evals never retry (approved 2026-09-29): a failed verdict is final for - * its run, and trials are fixed by kind before the run (EVAL_POLICY). The - * function stays the single statement of that policy for the Bun arguments - * and the reuse identity. + * Bun never retries a paid case inside a shard (approved 2026-09-29): trials + * are fixed by kind before the run (EVAL_POLICY). The one rule-16 rerun of a + * red case (CREW rule 16, scripts/lib/paid-rerun.ts) is a separate, + * reported execution, never a Bun retry. The function stays the single + * statement of that policy for the Bun arguments and the reuse identity. */ export function retriesForFiles(_files: string[]): number { return 0; @@ -514,6 +517,10 @@ export function parseRunManifest(raw: string): PaidRunManifest { throw new Error('manifest deferred case is outside the broad census'); } } + if (parsed.rerun !== undefined && (!Number.isSafeInteger(parsed.rerun?.attempt) || parsed.rerun.attempt < 1 || !Array.isArray(parsed.rerun.cases) + || parsed.rerun.cases.length === 0 || parsed.rerun.cases.some(id => typeof id !== 'string') || new Set(parsed.rerun.cases).size !== parsed.rerun.cases.length)) { + throw new Error('manifest rerun block invalid'); + } if (!Number.isInteger(parsed.sliceCount) || parsed.sliceCount <= 0) throw new Error('manifest sliceCount invalid'); if (!Array.isArray(parsed.entries)) throw new Error('manifest entries missing'); for (const entry of parsed.entries) { @@ -665,6 +672,8 @@ export interface SliceResult { finishedAt?: number; /** A progress checkpoint the executor never replaced: the job ended (ceiling or cancellation) before its final result. */ checkpoint?: true; + /** Produced from a rule-16 rerun manifest (`manifest.rerun`), never a first-attempt slice. */ + rerun?: true; outcomes: Array>; } @@ -674,11 +683,18 @@ export interface SliceResult { * isolated trial shard fails it only when the harness produced no trial * record. Failed, timed-out or crashed trials are verdict input for the * report's panelVerdict(), so a 2/3 PASS panel never reds its runner. + * `caseVerdicts` (the gate lane's CI slices, rule 16): a rule shard that ran + * to completion and reported executed cases is verdict input too, so its red + * cases stay in the report (which still fails closed on them) and the run's + * conclusion follows the report, where a flaky case can resolve green. */ -export function sliceExitCode(outcomes: ReadonlyArray>): number { +export function sliceExitCode(outcomes: ReadonlyArray & Partial>>, + options: { caseVerdicts?: boolean } = {}): number { + const completedRed = (o: (typeof outcomes)[number]) => !!options.caseVerdicts && o.status === 'failed' && o.exitCode !== null && o.exitCode !== undefined + && typeof o.executedTests === 'number' && o.executedTests - (o.skippedTests ?? 0) > 0 && o.runnerError === undefined && !o.sliceDeadline; return outcomes.every(outcome => outcome.trial !== undefined ? outcome.trial.outcome !== null - : outcome.status === 'passed' || outcome.status === 'skipped-by-diff') ? 0 : 1; + : outcome.status === 'passed' || outcome.status === 'skipped-by-diff' || completedRed(outcome)) ? 0 : 1; } /** A hollow-guarded trial shard has no trial record: the guard's verdict is a harness problem. */ @@ -697,8 +713,16 @@ export function guardTrialRecords; partial?: boolean } = {}, ): { ok: boolean; problems: string[] } { const problems: string[] = []; + const exitOk = (file: string, exitCode: number | null) => exitCode === 0 || !!options.flaky?.has(file); try { parseRunManifest(JSON.stringify(manifest)); } catch (error) { problems.push(`Invalid run manifest: ${error instanceof Error ? error.message : String(error)}`); } const byIndex = new Map(); @@ -713,7 +737,7 @@ export function verifySliceResults( byIndex.set(result.sliceIndex, result); } for (let index = 1; index <= manifest.sliceCount; index += 1) { - if (!byIndex.has(index)) problems.push(`slice ${index}/${manifest.sliceCount} reported NO result — cancelled/crashed executor, not a pass`); + if (!byIndex.has(index) && !options.partial) problems.push(`slice ${index}/${manifest.sliceCount} reported NO result — cancelled/crashed executor, not a pass`); } const reported = new Map(); @@ -731,7 +755,7 @@ export function verifySliceResults( ? -1 : outcome.executedTests - outcome.skippedTests; // A trial's exit status is its verdict (panelVerdict decides); its // completeness is the trial record checked below. - if ((!trialPlan && outcome.exitCode !== 0) || expected < 1 || (!trialPlan && executed !== expected)) { + if ((!trialPlan && !exitOk(file, outcome.exitCode)) || expected < 1 || (!trialPlan && executed !== expected)) { problems.push(`PR profile expected ${expected} executed cases in ${file}, received ${executed}`); } } @@ -767,7 +791,7 @@ export function verifySliceResults( if (finding) { // Full-census runs must account for every registered case. A manifest // explicitly marked selective may report its executed subset. - if (outcome.exitCode !== 0 || !Number.isInteger(outcome.executedTests) || + if (!exitOk(file, outcome.exitCode) || !Number.isInteger(outcome.executedTests) || outcome.executedTests! < 1 || outcome.executedTests! > finding.cases || (manifest.evalsAll !== false && outcome.executedTests !== finding.cases) || outcome.skippedTests !== 0) { problems.push(`Finding workflow must execute real unskipped cases with exit zero: ${file}`); @@ -794,7 +818,7 @@ export function verifySliceResults( // Isolated trial shards: harness health only; the panel verdict gates. if (entry.trial) { if (got.trial?.outcome === null) problems.push(`${entry.file}: no trial record (${got.trial.harness ?? 'unknown'})`); - } else if (got.status !== 'passed') { + } else if (got.status !== 'passed' && !(got.status === 'failed' && options.flaky?.has(normalizeRelativePath(entry.file)))) { // The slice deadline distinguishes work that never started (INFRA) from work killed in flight (TIMEOUT). const why = got.sliceDeadline === 'not_run' ? ' (not_run: the slice deadline passed before it started)' : got.sliceDeadline === 'hung' ? ' (hung: killed at the slice deadline)' : ''; diff --git a/scripts/lib/paid-report.ts b/scripts/lib/paid-report.ts index f92c6bb72..df1960518 100644 --- a/scripts/lib/paid-report.ts +++ b/scripts/lib/paid-report.ts @@ -19,13 +19,14 @@ import { writeNegativeReceipt, writePanelReceipt } from '../e2e-shard-reuse'; import { E2E_TOUCHFILES, E2E_TIERS, LLM_JUDGE_TOUCHFILES } from '../../test/helpers/touchfiles'; import { type EvalHost, hostSkipReason } from '../../test/helpers/e2e-hosts'; import { CASE_TEST_NAMES, type CaseTrialPlan, caseTestNamePattern, caseTrialPlan, fileCaseRegistration, shardCaseId, shardFile, trialShardKey } from './paid-cases'; -import { type ManifestEntry, PAID_TEST_DURATIONS_FILE, type PaidRunManifest, type SliceResult, collectorOutcomeCounts, formatProfileCoverage, loadPaidTestDurations, mergePaidTestDurations, parseRunManifest, trialPanelKey, verifySliceResults, writePaidTestDurations } from './paid-plan'; +import { type ManifestEntry, PAID_TEST_DURATIONS_FILE, type PaidRunManifest, type SliceResult, collectorOutcomeCounts, formatProfileCoverage, loadPaidTestDurations, mergePaidTestDurations, parseRunManifest, trialPanelKey, writePaidTestDurations } from './paid-plan'; import { DEFAULT_JOBS, type PaidCaseSelection, type PaidTier, ROOT, type ShardTrialRecord } from './paid-types'; import { collectPaidTestFiles, isAllSkippedPass, shardSlug, paidSelectionEnv } from './paid-select'; import type { RunShardsOptions } from '../test-paid-shards'; import { attributeCommands, formatMapGap, mapGaps, readShardCommands } from '../exercised-map'; import { checkoutExercisedConsumers } from '../pr-dependencies'; import { PR_FULL_GATE_FILES, PR_PROFILE_MAPS } from '../test-pr-profile'; +import { FLAKE_RECORDS_FILE, flakeRecords, flakyShards, formatFlakyLine, gateProblems, rerunSpend, rerunState } from './paid-rerun'; // ─── Local diagnosis: one case through the CI panel runner (A9) ──────────── @@ -215,7 +216,7 @@ export function attributeJUnitCase(tc: JUnitCase, ctx: JUnitShardContext, return { kind: 'case', id }; } -interface RuleCase { id: string; kind: EvalCaseKind; outcome: TrialOutcome; line?: string } +export interface RuleCase { id: string; kind: EvalCaseKind; outcome: TrialOutcome; key: string; slice: number; failureClass?: TrialFailureClass; error?: string; costUsd: number; costKnown: boolean; line?: string } /** JUnit evidence of the non-trial shards of one attempt. */ export interface JUnitCensus { @@ -246,7 +247,7 @@ export const CENSUS_RED_GUIDE = 'docs/evals/census-red.md'; export function sliceArtifactName(root: string, slice: number, attempt: number): string { const name = path.basename(root); - return /^(?:paid-slice|gate-census)-\d+-a\d+(?:-[a-z0-9-]+)?$/.test(name) ? name : `slice ${slice} attempt ${attempt}`; + return /^(?:paid-slice|paid-rerun|gate-census)-\d+-a\d+(?:-[a-z0-9-]+)?$/.test(name) ? name : `slice ${slice} attempt ${attempt}`; } export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact[], recordsByShard: Map, @@ -288,7 +289,8 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact bump(reasons, RUNTIME_SKIP_REASON); census.skipped.push(`◌ ${id} ${key} (${cases.map(tc => tc.name).join('; ')}): ${RUNTIME_SKIP_REASON}`); } - census.ruleCases.push({ id, kind, outcome: caseOutcome, + census.ruleCases.push({ id, kind, outcome: caseOutcome, key, slice: result.sliceIndex, ...(failureClass ? { failureClass } : {}), ...(error && failed ? { error } : {}), + costUsd: Math.round(mine.reduce((sum: number, r: any) => sum + (Number(r.cost_usd) || 0), 0) * 100) / 100, costKnown: trialCostKnown(mine, sessions).cost_known !== false, ...(failed ? { line: `✗ ${id} ${kind} FAIL ${failureClass}${failedRecord?.exit_reason === 'timeout' && failedRecord?.timeout_at_turn !== undefined ? ` at turn ${failedRecord.timeout_at_turn}` : ''}${error ? ` — ${error}` : ''}` + `${diagnosisText(diagnosis, error)} [slice ${result.sliceIndex}, attempt ${result.attempt ?? 1}]` + `${evidencePointer(sliceArtifactName(root, result.sliceIndex, result.attempt ?? 1), key, typeof base.run_id === 'string' ? base.run_id : undefined, id)}` @@ -318,15 +320,20 @@ export function formatJUnitCensus(census: JUnitCensus): string[] { ]; } -interface ReportArtifact { root: string; result: SliceResult } +export interface ReportArtifact { root: string; result: SliceResult } -/** Every slice result under the report dir: flat (merged) or one directory per attempt-scoped artifact. */ -export function loadSliceArtifacts(reportDir: string): ReportArtifact[] { +/** + * Every slice result under the report dir: flat (merged) or one directory per + * attempt-scoped artifact. Rule-16 rerun results (scripts/lib/paid-rerun.ts) + * are returned only with `{ reruns: true }`, and then only they are. + */ +export function loadSliceArtifacts(reportDir: string, options: { reruns?: boolean } = {}): ReportArtifact[] { const found: ReportArtifact[] = []; for (const name of fs.readdirSync(reportDir, { recursive: true }) as string[]) { const rel = normalizeRelativePath(name); if (!/^slice-\d+\.json$/.test(path.basename(rel)) || rel.split('/').includes('shards') || rel.split('/').includes('receipts')) continue; - found.push({ root: path.join(reportDir, path.dirname(rel)), result: JSON.parse(fs.readFileSync(path.join(reportDir, rel), 'utf8')) as SliceResult }); + const result = JSON.parse(fs.readFileSync(path.join(reportDir, rel), 'utf8')) as SliceResult; + if ((result.rerun === true) === (options.reruns === true)) found.push({ root: path.join(reportDir, path.dirname(rel)), result }); } return found.sort((a, b) => (a.result.attempt ?? 1) - (b.result.attempt ?? 1) || a.result.sliceIndex - b.result.sliceIndex); } @@ -452,7 +459,11 @@ export interface ReportHeadline { infra: number; incomplete: number; unattributed: number; + /** Cases that failed once and passed on their one rule-16 rerun (scripts/lib/paid-rerun.ts). */ + flaky?: number; }; + /** Re-executions of once-red cases (rule-16 reruns and later attempts) and their cost, when any ran. */ + reruns?: { executions: number; usd: number; unknown: number }; actionRequired: number; wallMs: number | null; costUsd: number; @@ -474,6 +485,8 @@ export function formatHeadline(h: ReportHeadline, cost: { unknownTrials: number; + ` · judge ${c.judge.passed}/${c.judge.total} · quarantined ${c.quarantined.total} (${c.quarantined.failingLane} failing the lane)`, ` SKIPPED ${c.skipped} · INFRA ${c.infra} · INCOMPLETE ${c.incomplete} · unattributed ${c.unattributed} · ACTION REQUIRED ${h.actionRequired}`, ` wall ${minutes} · ${formatCost(h.costUsd, cost.unknownTrials, cost.trials)}${h.redispatchEligible ? ' · every red is machine-classified INFRA/INCOMPLETE: eligible for ONE re-dispatch as a new run (EVAL_POLICY.infraRedispatch); report both runs' : ''}`, + ...(c.flaky || h.reruns ? [` FLAKY ${c.flaky ?? 0} (failed once, passed on its one rule-16 rerun) · ${h.reruns?.executions ?? 0} rerun execution(s), ` + + `${formatCost(h.reruns?.usd ?? 0, h.reruns?.unknown ?? 0, h.reruns?.executions ?? 0)}`] : []), ]; } @@ -527,16 +540,20 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo const summaryPath = path.join(reportDir, 'collector-outcomes.json'); const summaryMdPath = path.join(reportDir, 'report-summary.md'); const trialOutcomesPath = path.join(reportDir, TRIAL_OUTCOMES_FILE); - for (const file of [summaryPath, summaryMdPath, trialOutcomesPath]) fs.rmSync(file, { force: true }); + for (const file of [summaryPath, summaryMdPath, trialOutcomesPath, path.join(reportDir, FLAKE_RECORDS_FILE)]) fs.rmSync(file, { force: true }); const manifest = parseRunManifest(fs.readFileSync(path.join(reportDir, 'manifest.json'), 'utf-8')); const artifacts = loadSliceArtifacts(reportDir); const attempts = [...new Set(artifacts.map(a => a.result.attempt ?? 1))].sort((a, b) => a - b); const primary = attempts[0] ?? 1; const results = artifacts.filter(a => (a.result.attempt ?? 1) === primary).map(a => a.result); - const verdict = verifySliceResults(manifest, results); + // Rule 16 (scripts/lib/paid-rerun.ts): every attempt and each attempt's one rerun form one history per case; a flaky case's own red is no problem. + const rerun = rerunState(reportDir, manifest, artifacts); + const flakyFirst = flakyShards(rerun.observations.filter(o => o.attempt === primary && o.phase === 'first'), rerun.flaky); + const verdict = { problems: gateProblems(rerun.sets, rerun.observations, rerun.flaky) }; const planned = manifest.entries.filter((e) => e.status === 'planned').length; const lane = `${manifest.tier}/${manifest.profile ?? 'full'}${manifest.evalsAll ? ' census' : ''}${manifest.host ? ` on ${manifest.host}` : ''}`; - console.log(`[test:paid] report: ${results.length}/${manifest.sliceCount} slices, ${planned} planned shards, tier=${manifest.tier}, attempt ${primary}${attempts.length > 1 ? ` (later attempts ${attempts.slice(1).join(', ')} reported, never replacing it)` : ''}`); + console.log(`[test:paid] report: ${results.length}/${manifest.sliceCount} slices, ${planned} planned shards, tier=${manifest.tier}, attempt ${primary}${attempts.length > 1 ? ` (later attempts ${attempts.slice(1).join(', ')} reported, never replacing it); a later attempt resolves a red only by the rule-16 flaky rule and never hides a new red` : ''}`); + for (const line of rerun.ignored) console.log(`[test:paid] rule-16 rerun ignored: ${line}`); for (const line of formatProfileCoverage(manifest)) console.log(line); for (const result of [...results].sort((a, b) => a.sliceIndex - b.sliceIndex)) { for (const outcome of result.outcomes) { @@ -555,6 +572,8 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo const roots = artifacts.map(a => ({ root: path.resolve(a.root), attempt: a.result.attempt ?? 1 })) .sort((a, b) => b.root.length - a.root.length); const attemptOf = (abs: string) => roots.find(r => abs === r.root || abs.startsWith(r.root + path.sep))?.attempt ?? primary; + const rerunRoots = loadSliceArtifacts(reportDir, { reruns: true }).map(a => path.resolve(a.root)); + const inRerun = (abs: string) => rerunRoots.some(root => abs === root || abs.startsWith(root + path.sep)); const entryBySlug = new Map(manifest.entries.map(entry => [shardSlug([entry.file]), entry])); const shardOf = (rel: string) => { const parts = normalizeRelativePath(rel).split('/'); @@ -571,10 +590,11 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo const manualClaims = new Map(); const recordsByShard = new Map(); let costUsd = 0; + let flakyCollectorFailures = 0; for (const name of fs.readdirSync(reportDir, { recursive: true }) as string[]) { const rel = normalizeRelativePath(name); if (!isFinalizedEvalResultFile(rel) || rel.split('/').includes('receipts') || rel === 'collector-outcomes.json') continue; - if (attemptOf(path.resolve(reportDir, rel)) !== primary) continue; + if (inRerun(path.resolve(reportDir, rel)) || attemptOf(path.resolve(reportDir, rel)) !== primary) continue; const shard = shardOf(rel); try { const parsed = JSON.parse(fs.readFileSync(path.join(reportDir, rel), 'utf-8')); @@ -611,6 +631,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo } collectors.push(parsed); const counts = collectorOutcomeCounts([parsed]); + if (shard && flakyFirst.has(normalizeRelativePath(shard.file))) flakyCollectorFailures += counts.failed; files.push({ file: rel, tier: parsed.tier ?? 'unknown', shard: parsed.shard ?? '-', cost: parsed.total_cost_usd ?? 0, flaky: parsed.flaky_retries?.length ?? 0, total: counts.passed + counts.failed + counts.manual_accepted, ...counts }); @@ -626,7 +647,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo for (const f of flaky) console.log(` ⚠ ${f.name} (x${f.attempts}) — ${f.file}`); } if (manualProblems.length) verdict.problems.push(...manualProblems); - if (evidence.failed > 0) verdict.problems.push(`${evidence.failed} unapproved final collector failure(s)`); + if (evidence.failed > flakyCollectorFailures) verdict.problems.push(`${evidence.failed - flakyCollectorFailures} unapproved final collector failure(s)`); if (files.reduce((sum, file) => sum + file.total, 0) !== evidence.passed + evidence.failed + evidence.manual_accepted || files.reduce((sum, file) => sum + file.executed + file.reused, 0) !== evidence.executed + evidence.reused) { verdict.problems.push('Collector summary totals are inconsistent'); @@ -634,9 +655,6 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo // Panel verdicts: one function, computed here only. const panels = panelReports(manifest, results, primary); - for (const panel of panels.filter(p => p.failsLane)) { - verdict.problems.push(`PANEL ${panel.case} ${panel.status}${panel.redClass === 'INFRA' ? ' (INFRA)' : ''} ${panel.passed}/${panel.panel.n} (${panel.marks}): ${panel.reason}`); - } const laterPanels = attempts.slice(1).flatMap(attempt => panelReports(manifest, artifacts.map(a => a.result), attempt) .filter(panel => panel.trials.length > 0)); @@ -719,17 +737,22 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo const behaviorPanels = panels.filter(p => !p.quarantined && p.kind === 'behavior'); const lanePanels = panels.filter(p => !p.quarantined); const count = (kind: EvalCaseKind) => ({ - passed: ruleCases.filter(c => c.kind === kind && c.outcome === 'passed').length - + lanePanels.filter(p => p.kind === kind && p.status === 'PASS').length, + passed: ruleCases.filter(c => c.kind === kind && (c.outcome === 'passed' || (c.outcome === 'failed' && rerun.flaky.has(c.id)))).length + + lanePanels.filter(p => p.kind === kind && (p.status === 'PASS' || rerun.flaky.has(p.case))).length, total: ruleCases.filter(c => c.kind === kind).length + lanePanels.filter(p => p.kind === kind).length, }); const primaryTrials = results.flatMap(r => r.outcomes.map(o => o.trial)).filter((t): t is ShardTrialRecord => !!t); const wall = results.filter(r => Number.isSafeInteger(r.startedAt) && Number.isSafeInteger(r.finishedAt)); const failureLines = [ ...ruleShardFailures, - ...ruleCases.filter(c => c.line).map(c => c.line!), - ...panels.filter(p => p.status !== 'PASS' || p.split).map(p => formatPanelLine(p, manifest.tier, { artifacts: sliceArtifacts, runId })), + ...ruleCases.filter(c => c.line && !rerun.flaky.has(c.id)).map(c => c.line!), + ...panels.filter(p => (p.status !== 'PASS' || p.split) && !rerun.flaky.has(p.case)).map(p => formatPanelLine(p, manifest.tier, { artifacts: sliceArtifacts, runId })), + ...[...rerun.resolutions.values()].filter(r => r.status === 'red' && r.next) + .map(r => `✗ ${r.case} ${r.kind} FAILED TWICE (rule 16: a real red): ${r.first.where}; again in ${r.next!.where}${r.next!.symptom ? ` — ${r.next!.symptom}` : ''}`), ]; + const flakes = flakeRecords(rerun.resolutions.values(), { lane, env }); + const flakyLines = flakes.map(formatFlakyLine); + const spend = rerunSpend(rerun.observations); const red = verdict.problems.length > 0; const headline: ReportHeadline = { lane, verdict: red ? 'RED' : 'GREEN', attempt: primary, @@ -743,7 +766,9 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo + results.flatMap(r => r.outcomes).filter(o => !o.trial && o.runnerError !== undefined).length, incomplete: panels.filter(p => p.status === 'INCOMPLETE').length, unattributed: census.unattributed.length, + ...(flakes.length ? { flaky: flakes.length } : {}), }, + ...(spend.executions ? { reruns: spend } : {}), actionRequired: verdict.problems.length, wallMs: wall.length ? Math.max(...wall.map(r => r.finishedAt!)) - Math.min(...wall.map(r => r.startedAt!)) : null, costUsd: Math.round(costUsd * 100) / 100, @@ -756,11 +781,17 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo console.log('[test:paid] failures and split verdicts:'); for (const line of failureLines) console.log(` ${line}`); } - for (const panel of laterPanels) console.log(` attempt ${panel.attempt} (re-run; reported, never replacing attempt ${primary}): ${formatPanelLine(panel, manifest.tier)}`); + if (flakyLines.length) { + console.log('[test:paid] flaky (rule 16: failed once, passed on its one rerun; green with this annotation):'); + for (const line of flakyLines) console.log(` ${line}`); + } + for (const panel of laterPanels) console.log(` attempt ${panel.attempt} (re-run; resolves a red only by the rule-16 flaky rule): ${formatPanelLine(panel, manifest.tier)}`); + if (flakes.length) fs.writeFileSync(path.join(reportDir, FLAKE_RECORDS_FILE), flakes.map(record => JSON.stringify(record)).join('\n') + '\n'); const fence = (lines: string[]) => publishedFence(lines); fs.writeFileSync(summaryMdPath, [ ...fence(headlineLines), ...(failureLines.length ? ['', '**Failures and split verdicts**', '', ...fence(failureLines)] : []), + ...(flakyLines.length ? ['', '**Flaky: failed once, passed on its one rule-16 rerun**', '', ...fence(flakyLines)] : []), ...(censusLines.length ? ['', '**Skipped, deselected and unattributed testcases**', '', ...fence(censusLines)] : []), ...(verdict.problems.length ? ['', `**ACTION REQUIRED (${verdict.problems.length})**`, '', ...fence(verdict.problems.map(p => sanitizeTrialError(p) ?? p)), '', `Guide: ${CENSUS_RED_GUIDE}`] : []), ].join('\n') + '\n'); @@ -774,7 +805,8 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo trials: p.trials.map(t => ({ trial: t.trial, outcome: t.outcome, ...(t.failure_class ? { failure_class: t.failure_class } : {}), ...(t.exit_reason ? { exit_reason: t.exit_reason } : {}), ...(t.error ? { error: t.error } : {}) })) })), // evals.yml wraps these in a fixed fence for the PR comment. - failures: failureLines.map(line => sanitizeFixedFenceLine(line)) }, null, 2) + '\n'); + failures: failureLines.map(line => sanitizeFixedFenceLine(line)), + flaky: flakes, flakyLines: flakyLines.map(line => sanitizeFixedFenceLine(line)), flakyCollectorFailures }, null, 2) + '\n'); reportMapGaps(reportDir, new Set(executed.map(record => record.case)), rootDir); if (verdict.problems.length) { console.error(`[test:paid] report: ${verdict.problems.length} problem(s):`); diff --git a/scripts/lib/paid-rerun.ts b/scripts/lib/paid-rerun.ts new file mode 100644 index 000000000..d73a1204b --- /dev/null +++ b/scripts/lib/paid-rerun.ts @@ -0,0 +1,396 @@ +/** + * Rule-16 flaky reruns for the paid gate (CREW rule 16, Garry 2026-10-19: + * "you can just run the failed test again and note that it is flaky... You + * don't have to do the whole test pass again for one failed test."). + * + * Every executed case leaves an ordered history of observations: each run + * attempt's first execution, then that attempt's one in-run rerun. A case is + * - FLAKY (green, annotated) when it failed exactly once, that failure was a + * case verdict (a red rule or judge case, or a behavior panel FAIL whose + * red class is VERDICT), and the very next observation of it passed; + * - RED when it failed twice, or its failure was harness, INFRA, + * INCOMPLETE or unattributed, or nothing has re-executed it yet. + * A case that passed is never rerun and leaves no flake record. + * + * The rerun planner (evals.yml `rerun-plan`) reruns, once and in the same + * run attempt, exactly the cases whose only failure is a case verdict that + * nothing has re-executed yet, and only when their passing would turn the + * gate green. On the gate lane a completed rule shard's red case no longer + * fails its slice job (sliceExitCode caseVerdicts), so the run's conclusion + * follows this report and a flaky green concludes success. The + * report (runPaidReport) applies the same history to every attempt, so a + * manual `gh run rerun --failed` attempt that passes every earlier red is + * green with the same annotation, while a new red on any attempt stays red + * unless its own next observation passes. Trials, kinds, thresholds and + * panels never change: a behavior case reruns its whole planned panel, a + * judge case its judge's samples, a rule case its one trial. + */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { normalizeRelativePath } from './shard-engine'; +import { E2E_TOUCHFILES, LLM_JUDGE_TOUCHFILES } from '../../test/helpers/touchfiles'; +import { isFinalizedEvalResultFile, sanitizeTrialError, type EvalCaseKind } from '../../test/helpers/eval-store'; +import { shardTrial } from './paid-cases'; +import { type PaidRunManifest, type SliceResult, codexSlicesOf, isCodexShard, loadPaidTestDurations, packBySliceBudget, parseRunManifest, trialPanelKey, verifySliceResults } from './paid-plan'; +import { type ReportArtifact, junitCensus, loadSliceArtifacts, panelReports, sliceArtifactName } from './paid-report'; +import { expectedPrCaseCount, shardSlug } from './paid-select'; +import { ROOT } from './paid-types'; + +export const FLAKE_RECORDS_FILE = 'flake-records.jsonl'; +/** Each attempt's rerun manifest lands in the report dir as `-a/manifest.json` (one level deeper when several attempts' plans download). */ +export const RERUN_PLAN_ARTIFACT = 'rerun-plan'; + +export type Phase = 'first' | 'rerun'; + +export interface CaseObservation { + case: string; + kind: EvalCaseKind; + attempt: number; + phase: Phase; + outcome: 'passed' | 'failed'; + /** A failed case verdict the one rerun may resolve; harness, INFRA, INCOMPLETE and unattributed failures never are. */ + eligible: boolean; + /** Why a failure is not eligible. */ + reason?: string; + /** The manifest keys that executed it: its non-trial shard, or its panel's trial shards. */ + keys: string[]; + /** Where it ran: attempt, phase and slice artifacts. */ + where: string; + symptom?: string; + costUsd: number; + /** False when some execution captured no billing (PTY sessions): costUsd is then a lower bound. */ + costKnown: boolean; +} + +export interface CaseResolution { + case: string; + kind: EvalCaseKind; + status: 'flaky' | 'red'; + first: CaseObservation; + /** flaky: the passing rerun; red: the second failure, when there is one. */ + next?: CaseObservation; + reason: string; +} + +export interface ObservationSet { + attempt: number; + phase: Phase; + manifest: PaidRunManifest; + artifacts: ReportArtifact[]; + /** A later attempt or a rerun that ran only some slices: panels count only when all their trials ran in it. */ + partial: boolean; +} + +const phaseOrder = (phase: Phase) => (phase === 'first' ? 0 : 1); +const registered = (id: string) => Object.hasOwn(E2E_TOUCHFILES, id) || Object.hasOwn(LLM_JUDGE_TOUCHFILES, id); +function where(set: ObservationSet, slices: number[]): string { + const names = [...new Set(slices)].sort((a, b) => a - b) + .map(slice => sliceArtifactName(set.artifacts.find(a => a.result.sliceIndex === slice)?.root ?? '', slice, set.attempt)); + return `attempt ${set.attempt}${set.phase === 'rerun' ? ' rerun' : ''} (${names.join(', ')})`; +} + +/** Collector records under each reported shard's own eval dir, keyed by shard. */ +function shardRecords(artifacts: ReportArtifact[]): Map { + const records = new Map(); + for (const { root, result } of artifacts) { + for (const outcome of result.outcomes) { + const key = normalizeRelativePath(outcome.files[0] ?? ''); + const dir = path.join(root, 'shards', shardSlug([key])); + if (!fs.existsSync(dir)) continue; + for (const name of fs.readdirSync(dir, { recursive: true }) as string[]) { + if (!isFinalizedEvalResultFile(name)) continue; + try { + const parsed = JSON.parse(fs.readFileSync(path.join(dir, name), 'utf8')); + if (Array.isArray(parsed.tests)) records.set(key, [...(records.get(key) ?? []), ...parsed.tests]); + } catch { /* a malformed collector is the report's problem, not an observation */ } + } + } + } + return records; +} + +type Outcome = SliceResult['outcomes'][number]; + +/** Why a non-trial shard's case results cannot count as a verdict observation, else null. */ +export function shardProblem(manifest: PaidRunManifest, key: string, got: { outcome: Outcome; checkpoint: boolean } | undefined): string | null { + const entry = manifest.entries.find(e => normalizeRelativePath(e.file) === key); + if (!entry || entry.status !== 'planned') return 'not a planned shard'; + if (!got) return 'never reported'; + const o = got.outcome; + if (got.checkpoint) return 'its slice ended before its final result'; + if (o.reused) return 'a reused result'; + if (o.runnerError !== undefined) return 'runner error'; + if (o.sliceDeadline) return `slice deadline (${o.sliceDeadline})`; + if (o.status !== 'passed' && o.status !== 'failed') return `shard ${o.status}`; + if (o.executedTests === null || o.skippedTests === null) return 'no test summary'; + if (manifest.prCoverage?.mode === 'pr' && o.executedTests - o.skippedTests !== expectedPrCaseCount(entry.file, manifest.selection!, entry.excludeCases)) { + return 'executed a different case count than planned'; + } + return null; +} + +/** Every case verdict one set of slice results holds: JUnit rule and judge cases, and complete trial panels. */ +export function observeCases(set: ObservationSet): CaseObservation[] { + const { manifest, artifacts, attempt, phase } = set; + const results = artifacts.map(a => a.result); + const reported = new Map(); + for (const result of results) { + for (const outcome of result.outcomes) reported.set(normalizeRelativePath(outcome.files[0] ?? ''), { outcome, checkpoint: !!result.checkpoint }); + } + const census = junitCensus(manifest, artifacts, shardRecords(artifacts), {}); + const unattributed = new Set(census.unattributed.map(line => line.split(' :: ')[0])); + const observations: CaseObservation[] = []; + for (const c of census.ruleCases) { + if (c.outcome === 'skipped') continue; + const unsound = shardProblem(manifest, c.key, reported.get(c.key)) ?? (unattributed.has(c.key) ? 'an unattributed testcase in its shard' : null); + if (c.outcome === 'passed' && unsound) continue; + const reason = c.outcome !== 'failed' ? undefined : unsound ?? (c.failureClass === 'infra' ? 'INFRA failure class' + : !registered(c.id) ? 'no registry case id (unattributed)' : undefined); + observations.push({ case: c.id, kind: c.kind, attempt, phase, outcome: c.outcome, eligible: c.outcome === 'failed' && !reason, ...(reason ? { reason } : {}), + keys: [c.key], where: where(set, [c.slice]), + ...(c.outcome === 'failed' ? { symptom: sanitizeTrialError(`${c.failureClass ?? 'assertion'}${c.error ? `: ${c.error}` : ''}`) } : {}), + costUsd: c.costUsd, costKnown: c.costKnown }); + } + const slices = new Set(results.map(r => r.sliceIndex)); + for (const panel of panelReports(manifest, results, attempt)) { + const entries = manifest.entries.filter(e => e.status === 'planned' && e.trial && trialPanelKey(e.file) === `${panel.file}#${panel.case}`); + if (set.partial && !entries.every(e => slices.has(e.slice))) continue; + if (panel.status === 'SKIPPED' || (panel.status === 'FAIL' && !panel.failsLane)) continue; + const outcomes = entries.map(e => reported.get(normalizeRelativePath(e.file))?.outcome); + if (phase === 'rerun' && outcomes.some(o => o?.reused)) continue; + const passed = panel.status === 'PASS'; + const reason = passed ? undefined : panel.status === 'INCOMPLETE' ? `INCOMPLETE: ${panel.reason}` + : panel.redClass === 'INFRA' ? 'INFRA: every failed trial is infra-class' : panel.redClass !== 'VERDICT' ? `red class ${panel.redClass}` : undefined; + const failed = panel.trials.filter(t => t.outcome === 'failed').map(t => `t${t.trial} ${t.failure_class ?? 'assertion'}${t.error ? ` ${t.error}` : ''}`); + observations.push({ case: panel.case, kind: panel.kind, attempt, phase, outcome: passed ? 'passed' : 'failed', eligible: !passed && !reason, ...(reason ? { reason } : {}), + keys: entries.map(e => normalizeRelativePath(e.file)), where: where(set, Object.values(panel.slices)), + ...(passed ? {} : { symptom: sanitizeTrialError(`${panel.status} ${panel.passed}/${panel.panel.n} (${panel.marks}): ${failed.join('; ')}`) }), + costUsd: Math.round(outcomes.reduce((sum, o) => sum + (o?.trial?.cost_usd ?? 0), 0) * 100) / 100, costKnown: outcomes.every(o => o?.trial && o.trial.cost_known !== false) }); + } + return observations; +} + +/** + * The observation sets of a report dir: each attempt's first execution + * (attempt 1 complete, later attempts partial) and each attempt's rerun, + * bound to its own rerun manifest. A rerun result that does not match its + * manifest is set aside and named. + */ +export function observationSets(reportDir: string, manifest: PaidRunManifest, firstArtifacts: ReportArtifact[]): { sets: ObservationSet[]; ignored: string[] } { + const reported = [...new Set(firstArtifacts.map(a => a.result.attempt ?? 1))].sort((a, b) => a - b); + const attempts = reported.length ? reported : [1]; + const sets: ObservationSet[] = attempts.map((attempt, i) => ({ attempt, phase: 'first' as const, manifest, partial: i > 0, + artifacts: firstArtifacts.filter(a => (a.result.attempt ?? 1) === attempt) })); + const ignored: string[] = []; + const reruns = loadSliceArtifacts(reportDir, { reruns: true }); + for (const attempt of [...new Set(reruns.map(a => a.result.attempt ?? 1))].sort((a, b) => a - b)) { + let rerunManifest: PaidRunManifest; + // download-artifact unpacks a lone pattern match into the report dir and each of several into its own directory. + const dir = path.join(reportDir, `${RERUN_PLAN_ARTIFACT}-a${attempt}`); + const file = [path.join(dir, 'manifest.json'), path.join(dir, path.basename(dir), 'manifest.json')].find(candidate => fs.existsSync(candidate)) ?? path.join(dir, 'manifest.json'); + try { rerunManifest = parseRunManifest(fs.readFileSync(file, 'utf8')); } catch (error) { + ignored.push(`attempt ${attempt}: rerun results without a readable rerun manifest (${error instanceof Error ? error.message : String(error)})`); + continue; + } + const mine = reruns.filter(a => (a.result.attempt ?? 1) === attempt); + const bound = mine.filter(a => rerunManifest.rerun?.attempt === attempt && a.result.sliceCount === rerunManifest.sliceCount + && a.result.tier === rerunManifest.tier && JSON.stringify(a.result.selection ?? null) === JSON.stringify(rerunManifest.selection ?? null)); + if (bound.length !== mine.length) ignored.push(`attempt ${attempt}: ${mine.length - bound.length} rerun result(s) do not match the rerun manifest`); + sets.push({ attempt, phase: 'rerun', manifest: rerunManifest, partial: true, artifacts: bound }); + } + return { sets, ignored }; +} + +/** Every observation in order: attempt, then first execution before its rerun. A rerun counts only for the cases it was planned for. */ +export function caseHistory(sets: ObservationSet[]): CaseObservation[] { + return sets.flatMap(set => observeCases(set).filter(o => set.phase === 'first' || (set.manifest.rerun?.cases ?? []).includes(o.case))) + .map((o, index) => ({ o, index })) + .sort((a, b) => a.o.attempt - b.o.attempt || phaseOrder(a.o.phase) - phaseOrder(b.o.phase) || a.index - b.index) + .map(({ o }) => o); +} + +/** The rule-16 status of every case that failed at least once. */ +export function resolveCases(history: CaseObservation[]): Map { + const resolved = new Map(); + const byCase = new Map(); + for (const o of history) byCase.set(o.case, [...(byCase.get(o.case) ?? []), o]); + for (const [id, list] of byCase) { + const fails = list.filter(o => o.outcome === 'failed'); + if (!fails.length) continue; + const first = fails[0]!; + const next = list[list.indexOf(first) + 1]; + if (fails.length === 1 && first.eligible && next?.outcome === 'passed') { + resolved.set(id, { case: id, kind: first.kind, status: 'flaky', first, next, reason: `failed once in ${first.where}, passed on rerun in ${next.where}` }); + continue; + } + const reason = fails.length > 1 ? `failed twice (${first.where}; ${fails[1]!.where})` + : !first.eligible ? `never rerun: ${first.reason}` : 'no rerun verdict'; + resolved.set(id, { case: id, kind: first.kind, status: 'red', first, ...(fails[1] ? { next: fails[1] } : {}), reason }); + } + return resolved; +} + +/** + * The cases to rerun once now: their only failure is a case verdict and + * nothing has executed them since. Usually this attempt's own reds; on a + * `gh run rerun --failed` attempt also an earlier attempt's red that its + * in-run rerun never reached (another problem blocked it then). + */ +export function rerunCandidates(history: CaseObservation[]): CaseObservation[] { + return [...resolveCases(history).values()].filter(r => r.status === 'red' && !r.next && r.first.eligible + && history.filter(o => o.case === r.case).at(-1) === r.first).map(r => r.first); +} + +/** Non-trial shards whose every failed case in `observations` is flaky: their non-zero exit and failed status are explained. */ +export function flakyShards(observations: CaseObservation[], flaky: ReadonlySet): Set { + const failed = observations.filter(o => o.outcome === 'failed' && o.keys.length === 1 && shardTrial(o.keys[0]!) === null); + const keys = new Set(failed.map(o => o.keys[0]!)); + return new Set([...keys].filter(key => failed.filter(o => o.keys[0] === key).every(o => flaky.has(o.case)))); +} + +/** + * The rerun manifest: the candidates' own shards from the plan (a behavior + * case's whole panel, a rule or judge case's own shard, narrowed to it in PR + * mode; otherwise the file reruns and only the planned cases count), packed + * with the plan's budget. It runs fresh: the rerun job + * gets no receipts and the executor never reuses for a rerun manifest. + */ +export function buildRerunManifest(manifest: PaidRunManifest, candidates: CaseObservation[], attempt: number, + durations: Record = loadPaidTestDurations(ROOT, manifest.tier)): PaidRunManifest { + const ids = [...new Set(candidates.map(c => c.case))].sort(); + const keys = [...new Set(candidates.flatMap(c => c.keys))].sort(); + const panels = new Set(candidates.filter(c => c.keys.some(key => shardTrial(key) !== null)).map(c => c.case)); + const selection = { e2e: ids.filter(id => Object.hasOwn(E2E_TOUCHFILES, id)), + judges: ids.filter(id => !Object.hasOwn(E2E_TOUCHFILES, id) && Object.hasOwn(LLM_JUDGE_TOUCHFILES, id)) }; + const budget = manifest.plan ?? { sliceBudgetMs: 420_000, jobs: 2 }; + const packed = packBySliceBudget(keys, budget.sliceBudgetMs, budget.jobs, durations, undefined, file => isCodexShard(file, manifest.tier)); + const entries = packed.slices.flatMap((files, index) => files.map(file => { + const source = manifest.entries.find(e => normalizeRelativePath(e.file) === file)!; + const exclude = (source.excludeCases ?? []).filter(id => panels.has(id)); + return { file: source.file, slice: index + 1, status: 'planned' as const, estimatedMs: packed.estimates[file]!, + ...(source.budget ? { budget: source.budget } : {}), ...(source.trial ? { trial: source.trial } : {}), ...(exclude.length ? { excludeCases: exclude } : {}) }; + })).sort((a, b) => (a.file < b.file ? -1 : 1)); + return parseRunManifest(JSON.stringify({ + version: 1, tier: manifest.tier, evalsAll: false, sliceCount: packed.slices.length, + selectionReason: `rule-16 rerun of attempt ${attempt}: ${ids.join(', ')}`, ...(manifest.host ? { host: manifest.host } : {}), + profile: manifest.profile ?? 'full', selection, + ...(manifest.prCoverage ? { prCoverage: { ...manifest.prCoverage, e2e: selection.e2e, judges: selection.judges } } : {}), + plan: { sliceBudgetMs: budget.sliceBudgetMs, jobs: budget.jobs, estimatedSliceMs: packed.estimatedSliceMs, ciTimeoutMinutes: packed.ciTimeoutMinutes, + sliceCiTimeoutMinutes: packed.sliceCiTimeoutMinutes, codexSlices: codexSlicesOf(manifest.tier, entries) }, + rerun: { attempt, cases: ids }, entries, + } satisfies PaidRunManifest)); +} + +export interface FlakeRecord { + schema: 1; + kind: 'gate-flake'; + case: string; + case_kind: EvalCaseKind; + repository: string | null; + pr: number | null; + head_sha: string | null; + tested_sha: string | null; + lane: string; + run_id: string | null; + first: { attempt: number; phase: Phase; where: string }; + rerun: { attempt: number; phase: Phase; where: string }; + symptom: string; + /** Null when the rerun captured no billing (PTY sessions). */ + rerun_cost_usd: number | null; + recorded_at: string; +} + +/** One JSONL record per flaky case, read by the daily flaky-fix pass from the run's `flake-records-a` artifact. */ +export function flakeRecords(resolutions: Iterable, ctx: { lane: string; env: NodeJS.ProcessEnv; now?: Date }): FlakeRecord[] { + const { env } = ctx; + const pr = Number(env.GSTACK_GATE_PR); + const sha = (value: string | undefined) => (/^[0-9a-f]{40}$/.test(value ?? '') ? value! : null); + return [...resolutions].filter(r => r.status === 'flaky').sort((a, b) => (a.case < b.case ? -1 : 1)).map(r => ({ + schema: 1, kind: 'gate-flake', case: r.case, case_kind: r.kind, repository: env.GITHUB_REPOSITORY ?? null, + pr: Number.isSafeInteger(pr) && pr > 0 ? pr : null, head_sha: sha(env.GSTACK_GATE_HEAD_SHA) ?? sha(env.GITHUB_SHA), tested_sha: sha(env.GITHUB_SHA), + lane: ctx.lane, run_id: env.GITHUB_RUN_ID ?? null, + first: { attempt: r.first.attempt, phase: r.first.phase, where: r.first.where }, + rerun: { attempt: r.next!.attempt, phase: r.next!.phase, where: r.next!.where }, + symptom: r.first.symptom ?? 'failed', rerun_cost_usd: r.next!.costKnown ? r.next!.costUsd : null, recorded_at: (ctx.now ?? new Date()).toISOString(), + })); +} + +/** The report and PR comment line of one flaky case (the run id stays in the record: the fence sanitizer redacts long digit runs). */ +export function formatFlakyLine(record: FlakeRecord): string { + return `⚑ ${record.case} ${record.case_kind} FLAKY: failed in ${record.first.where}, passed on its one rerun in ${record.rerun.where}; ` + + `rerun cost ${record.rerun_cost_usd === null ? 'unknown (no billing captured)' : `$${record.rerun_cost_usd.toFixed(2)}`}; first failure: ${record.symptom}`; +} + +/** What re-executing once-red cases cost: every observation after a case's first failure, in-run reruns and later attempts alike. */ +export function rerunSpend(history: CaseObservation[]): { executions: number; usd: number; unknown: number } { + const firstFail = new Map(); + history.forEach((o, index) => { if (o.outcome === 'failed' && !firstFail.has(o.case)) firstFail.set(o.case, index); }); + const after = history.filter((o, index) => firstFail.has(o.case) && index > firstFail.get(o.case)!); + return { executions: after.length, usd: Math.round(after.reduce((sum, o) => sum + o.costUsd, 0) * 100) / 100, unknown: after.filter(o => !o.costKnown).length }; +} + +/** A panel whose every planned trial ran in this set (a later attempt or rerun may hold only some). */ +function completePanel(set: ObservationSet, file: string, id: string): boolean { + const slices = new Set(set.artifacts.map(a => a.result.sliceIndex)); + return set.manifest.entries.filter(e => e.status === 'planned' && e.trial && trialPanelKey(e.file) === `${file}#${id}`).every(e => slices.has(e.slice)); +} + +/** + * The slice and panel problems of every first-execution set, with each flaky + * case's own red taken out: its shard's non-zero exit and failed status and + * its panel's FAIL. A later attempt's problems are prefixed with it, so a new + * red there is never hidden. Collector and manual-review checks stay in the + * report. + */ +export function gateProblems(sets: ObservationSet[], observations: CaseObservation[], flaky: ReadonlySet): string[] { + const problems: string[] = []; + for (const set of sets.filter(s => s.phase === 'first')) { + const results = set.artifacts.map(a => a.result); + const prefix = set.partial ? `attempt ${set.attempt}: ` : ''; + const shards = flakyShards(observations.filter(o => o.attempt === set.attempt && o.phase === 'first'), flaky); + problems.push(...verifySliceResults(set.manifest, results, { flaky: shards, partial: set.partial }).problems.map(problem => prefix + problem)); + for (const panel of panelReports(set.manifest, results, set.attempt)) { + if (!panel.failsLane || flaky.has(panel.case) || (set.partial && !completePanel(set, panel.file, panel.case))) continue; + problems.push(`${prefix}PANEL ${panel.case} ${panel.status}${panel.redClass === 'INFRA' ? ' (INFRA)' : ''} ${panel.passed}/${panel.panel.n} (${panel.marks}): ${panel.reason}`); + } + } + return problems; +} + +/** Everything a report dir proves about rule-16 reruns: the ordered history, each failed case's resolution and the flaky set. */ +export function rerunState(reportDir: string, manifest: PaidRunManifest, firstArtifacts: ReportArtifact[]) { + const { sets, ignored } = observationSets(reportDir, manifest, firstArtifacts); + const observations = caseHistory(sets); + const resolutions = resolveCases(observations); + const flaky = new Set([...resolutions.values()].filter(r => r.status === 'flaky').map(r => r.case)); + return { sets, ignored, observations, resolutions, flaky }; +} + +/** + * evals.yml `rerun-plan`: write this attempt's rerun manifest to + * `/manifest.json` when some case verdict failed once in this attempt + * and those cases passing would leave the gate with no other problem. A + * harness, INFRA, INCOMPLETE or unattributed red, a case already red twice, + * or any other problem plans nothing: the gate keeps its red. + */ +export function planRerun(reportDir: string, attempt: number, out: string, durations?: Record): { manifest: PaidRunManifest | null; lines: string[] } { + const manifest = parseRunManifest(fs.readFileSync(path.join(reportDir, 'manifest.json'), 'utf8')); + const state = rerunState(reportDir, manifest, loadSliceArtifacts(reportDir)); + const candidates = rerunCandidates(state.observations); + const reds = [...state.resolutions.values()].filter(r => r.status === 'red' && !candidates.includes(r.first)); + const lines = [...state.ignored.map(line => `ignored: ${line}`), ...reds.map(r => `not rerun: ${r.case} (${r.reason})`)]; + if (!candidates.length) return { manifest: null, lines: [`no rule-16 rerun: no case verdict failed once with no rerun yet`, ...lines] }; + const remaining = gateProblems(state.sets, state.observations, new Set([...state.flaky, ...candidates.map(c => c.case)])); + if (remaining.length) { + return { manifest: null, lines: [`no rule-16 rerun: rerunning ${candidates.map(c => c.case).join(', ')} cannot turn this gate green; ${remaining.length} other problem(s), first: ${remaining[0]}`, ...lines] }; + } + let rerun: PaidRunManifest; + try { rerun = buildRerunManifest(manifest, candidates, attempt, durations); } catch (error) { + return { manifest: null, lines: [`no rule-16 rerun: the rerun manifest is invalid (${error instanceof Error ? error.message : String(error)})`, ...lines] }; + } + fs.mkdirSync(out, { recursive: true }); + fs.writeFileSync(path.join(out, 'manifest.json'), `${JSON.stringify(rerun, null, 2)}\n`); + return { manifest: rerun, lines: [ + `rule-16 rerun of attempt ${attempt}: ${candidates.length} case(s) once, on ${rerun.sliceCount} slice(s), same head, fresh (no reuse)`, + ...candidates.map(c => ` ${c.case} ${c.kind} failed in ${c.where}: ${c.symptom ?? 'failed'}; reruns ${c.keys.join(' ')}`), ...lines] }; +} diff --git a/scripts/test-paid-shards.ts b/scripts/test-paid-shards.ts index 7a362c457..b3b4786e4 100644 --- a/scripts/test-paid-shards.ts +++ b/scripts/test-paid-shards.ts @@ -77,6 +77,7 @@ import { resolveEvalHost } from '../test/helpers/e2e-hosts'; import { scopeCodexAccess, shardFile, shardCaseId, shardTrial, trialShardKey, type CaseTrialPlan, caseTrialPlan, excludedCasesNamePattern, caseTestNamePattern, expandCaseShards, expandTrialShards, fileCaseRegistration, partitionCaseExclusions } from './lib/paid-cases'; import { retriesForFiles, trialPanelKey, sliceExecutionOrder, buildRunManifest, parseRunManifest, type SliceResult, sliceExitCode, guardTrialRecords, formatSlicePlan, formatCapacityPreflight, sliceDeadlineMs } from './lib/paid-plan'; import { caseSelection, runCaseDiagnosis, formatPanelLine, runPaidReport } from './lib/paid-report'; +import { planRerun } from './lib/paid-rerun'; import { DEFAULT_JOBS, DEFAULT_MAX_FILES_PER_SHARD, DEFAULT_SHARD_TIMEOUT_MS, DEFAULT_TIER, DEFAULT_WITHIN_SHARD_CONCURRENCY, OVERLAY_MAX_ACTIVE_SHARDS, PAID_LANE_POLICY, PAID_TIERS, ROOT, SLICE_JOB_STARTED_AT_ENV, SLICE_UPLOAD_RESERVE_MS, isOverlayTestFile, @@ -97,6 +98,7 @@ export * from './lib/paid-select'; export * from './lib/paid-cases'; export * from './lib/paid-plan'; export * from './lib/paid-report'; +export * from './lib/paid-rerun'; type TrialEvidence = { records: any[]; contract: string | null; sessions?: SessionLedgerRow[] }; @@ -722,6 +724,8 @@ type CliOptions = { reportDir: string | null; /** Report mode: merge executed shard wall times into the duration seed. */ writeDurations: boolean; + /** With --report: plan this attempt's rule-16 rerun into this directory instead of reporting. */ + rerunPlanDir: string | null; /** Planner: the workflow matrix cap, for the capacity preflight's wave count. */ maxParallel: number | null; /** Local diagnosis: run one case through the panel runner CI uses (never read by CI). */ @@ -764,6 +768,7 @@ CI modes (the eval workflows): --emit-plan PATH (--slices K | --slice-budget SECS --jobs N) [--skip-judges] [--max-parallel N] --plan PATH --slice I execute one slice of a manifest --report DIR [--write-durations] reconcile slice results against the manifest + --report DIR --rerun-plan OUT plan this attempt's one rule-16 rerun of its failed cases (scripts/lib/paid-rerun.ts) Background runs and CI dispatch: bun run eval:bg: (scripts/eval-bg.ts --help).`; @@ -795,6 +800,7 @@ export function parseCliOptions(argv: string[], env: NodeJS.ProcessEnv = process sliceIndex: null, reportDir: null, writeDurations: false, + rerunPlanDir: null, maxParallel: null, caseId: null, trials: null, @@ -822,6 +828,7 @@ export function parseCliOptions(argv: string[], env: NodeJS.ProcessEnv = process '--slice': (next) => { options.sliceIndex = parsePositiveInt(next(), '--slice'); }, '--report': pathValue('--report needs a directory', (value) => { options.reportDir = value; }), '--write-durations': () => { options.writeDurations = true; }, + '--rerun-plan': pathValue('--rerun-plan needs a directory', (value) => { options.rerunPlanDir = value; }), '--max-parallel': (next) => { options.maxParallel = parsePositiveInt(next(), '--max-parallel'); }, '--case': (next) => { const value = next(); @@ -831,6 +838,7 @@ export function parseCliOptions(argv: string[], env: NodeJS.ProcessEnv = process '--trials': (next) => { options.trials = parsePositiveInt(next(), '--trials'); }, }, PAID_USAGE); if (options.writeDurations && !options.reportDir) throw new Error('--write-durations requires --report'); + if (options.rerunPlanDir && (!options.reportDir || options.writeDurations)) throw new Error('--rerun-plan requires --report and writes no durations'); if (options.trials !== null && options.caseId === null) throw new Error('--trials requires --case'); if (options.caseId !== null && (options.emitPlanPath || options.planPath || options.reportDir || options.sliceIndex !== null)) { throw new Error('--case is local diagnosis; it cannot combine with --emit-plan, --plan/--slice or --report'); @@ -880,6 +888,12 @@ async function main(): Promise { // ── Report mode: reconcile slice artifacts against the manifest. Fail-closed: // a slice whose artifact never landed is a FAILURE, not an absence. + if (options.reportDir && options.rerunPlanDir) { + const attempt = Number(process.env.GITHUB_RUN_ATTEMPT); + const { lines } = planRerun(options.reportDir, Number.isSafeInteger(attempt) && attempt > 0 ? attempt : 1, options.rerunPlanDir); + for (const line of lines) console.log(`- ${line}`); + return 0; + } if (options.reportDir) return runPaidReport(options.reportDir, { writeDurations: options.writeDurations }); if (options.caseId && options.listOnly) { @@ -960,6 +974,7 @@ async function main(): Promise { startedAt, finishedAt: Date.now(), ...(checkpoint ? { checkpoint: true as const } : {}), + ...(manifest.rerun ? { rerun: true as const } : {}), outcomes: outcomes.map(({ files, status, exitCode, elapsedMs, executedTests, skippedTests, budget, reused, runnerError, trial, inputKey, sliceDeadline }) => ({ files, status, exitCode, elapsedMs, executedTests, skippedTests, ...(budget ? { budget } : {}), ...(reused ? { reused } : {}), ...(runnerError !== undefined ? { runnerError } : {}), ...(trial ? { trial } : {}), ...(inputKey ? { inputKey } : {}), @@ -989,7 +1004,8 @@ async function main(): Promise { expectedCases: Object.fromEntries(mine.map(entry => [entry.file, expectedPrCaseCount(entry.file, manifest.selection!, entry.excludeCases)])), casePatterns: Object.fromEntries(mine.map(entry => [entry.file, prProfileTestNamePattern(entry.file, manifest.selection!, entry.excludeCases)])), expectedCaseIds: Object.fromEntries(mine.map(entry => [entry.file, prProfileShardIds(entry.file, manifest.selection!, entry.excludeCases)])), - reuseFor: e2eReuseLaneProblem(process.env, manifest.prCoverage.mode) !== null ? undefined : (files, env, budget) => { + // A rule-16 rerun executes fresh: a receipt would replay the verdict it is rerunning. + reuseFor: manifest.rerun || e2eReuseLaneProblem(process.env, manifest.prCoverage.mode) !== null ? undefined : (files, env, budget) => { const key = files[0]!; const file = shardFile(key); if (files.length !== 1 || !/^test\/skill-e2e-/.test(file)) return null; @@ -1027,7 +1043,7 @@ async function main(): Promise { const t = outcome.trial!; console.log(` trial ${t.case} t${t.trial}/${t.panel.n}: ${t.outcome ?? `NO RECORD (${t.harness})`}${t.failure_class ? ` [${t.failure_class}]` : ''}`); } - return sliceExitCode(guarded); + return sliceExitCode(guarded, { caseVerdicts: manifest.tier === 'gate' }); } if (options.listOnly && options.sliceBudgetMs !== null) { diff --git a/scripts/train-gate.ts b/scripts/train-gate.ts index 79b07d6b7..8c38c2fd0 100644 --- a/scripts/train-gate.ts +++ b/scripts/train-gate.ts @@ -36,6 +36,7 @@ import { UNRECORDED_SHARD_MS, nonceScope, plannedSerialMs } from './eval-bg'; import { caseTrialPlan, fileCaseRegistration, shardCaseId, shardFile } from './lib/paid-cases'; import { buildRunManifest, loadPaidTestDurations, parseRunManifest, recordedShardMs, type PaidRunManifest } from './lib/paid-plan'; import { loadSliceArtifacts } from './lib/paid-report'; +import type { FlakeRecord } from './lib/paid-rerun'; import { paidSelectionEnv } from './lib/paid-select'; import { ROOT } from './lib/paid-types'; @@ -195,11 +196,14 @@ function readJson(file: string): T | null { } /** One panel per planned case from the run's trial outcomes (trial panels from the collector's panelVerdict). */ -function casePanels(planned: Map, outcomes: OutcomeLine[], collector: CollectorPanel[]): CasePanel[] { +function casePanels(planned: Map, outcomes: OutcomeLine[], collector: CollectorPanel[], flaky: FlakeRecord[] = []): CasePanel[] { return [...planned].sort(([a], [b]) => (a < b ? -1 : 1)).map(([id, kind]) => { const mine = outcomes.filter(o => o.case === id && o.execution !== 'reused'); const cost = mine.length && mine.every(o => o.cost_known !== false && typeof o.cost_usd === 'number') ? Math.round(mine.reduce((sum, o) => sum + o.cost_usd!, 0) * 10_000) / 10_000 : null; + // Rule 16: the report resolved this case FLAKY (failed once, passed its one rerun): a PASS that names both runs. + const flake = flaky.find(f => f.case === id); + if (flake) return { case: id, kind, status: 'PASS', red_class: null, cost_usd: cost, flaky: { first: flake.first.where, rerun: flake.rerun.where } }; const panel = collector.find(p => p.case === id); if (panel) return { case: id, kind, status: panel.status, red_class: panel.redClass, cost_usd: cost }; const failed = mine.filter(o => o.outcome === 'failed'); @@ -219,8 +223,8 @@ export function buildTrainRecord(o: { reportDir: string; nonce: string; head: st let lines: OutcomeLine[] = []; try { lines = fs.readFileSync(path.join(o.reportDir, 'trial-outcomes.jsonl'), 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line) as OutcomeLine); } catch { lines = []; } const primary = Math.min(...lines.map(l => l.attempt ?? 1), Infinity); - const collector = readJson<{ verdict?: { verdict?: string }; census?: { unattributed?: unknown[] }; panels?: CollectorPanel[] }>(path.join(o.reportDir, 'collector-outcomes.json')); - const panels = casePanels(planned, lines.filter(l => (l.attempt ?? 1) === primary), collector?.panels ?? []); + const collector = readJson<{ verdict?: { verdict?: string }; census?: { unattributed?: unknown[] }; panels?: CollectorPanel[]; flaky?: FlakeRecord[] }>(path.join(o.reportDir, 'collector-outcomes.json')); + const panels = casePanels(planned, lines.filter(l => (l.attempt ?? 1) === primary), collector?.panels ?? [], collector?.flaky ?? []); const artifacts = loadSliceArtifacts(o.reportDir); const first = Math.min(...artifacts.map(a => a.result.attempt ?? 1), Infinity); const results = artifacts.filter(a => (a.result.attempt ?? 1) === first).map(a => a.result); @@ -246,7 +250,7 @@ export function buildTrainRecord(o: { reportDir: string; nonce: string; head: st export interface BisectPlan { schema: 1; nonce: string; cases: Array<{ id: string; kind: string }>; prefixes: string[] } -/** One executor per prefix; each case runs one trial, so a slice's ceiling grows with the case count. */ +/** One executor per prefix; each case runs one trial (and at most one rule-16 rerun inside its 30 minutes), so a slice's ceiling grows with the case count. */ export function bisectMatrix(plan: BisectPlan): { slices: number[]; timeouts: number[]; refs: string[] } { const minutes = Math.min(350, 20 + 30 * plan.cases.length); return { slices: plan.prefixes.map((_, i) => i + 1), timeouts: [0, ...plan.prefixes.map(() => minutes)], refs: ['', ...plan.prefixes] }; @@ -254,7 +258,13 @@ export function bisectMatrix(plan: BisectPlan): { slices: number[]; timeouts: nu export type CaseRunner = (id: string, kind: string, evalDir: string) => Promise<{ status: string; redClass: string | null }>; -/** Run every red case once on the checked-out prefix; a case that leaves no verdict is INCOMPLETE (T1 relaunches it). */ +/** + * Run every red case once on the checked-out prefix; a case that leaves no + * verdict is INCOMPLETE (T1 relaunches it). Rule 16 holds here as in the + * gate: a case verdict that fails (red class VERDICT) reruns once on the same + * prefix; passing makes it a flaky PASS naming both runs, failing again keeps + * it red. INFRA and INCOMPLETE are never rerun. + */ export async function runBisectPrefix(plan: BisectPlan, index: number, o: { rootDir: string; evalDir: string; checkedOut: string; runCase?: CaseRunner }): Promise { const sha = plan.prefixes[index - 1]; if (!sha) throw new Error(`bisect slice ${index} names no prefix (${plan.prefixes.length} planned)`); @@ -262,11 +272,19 @@ export async function runBisectPrefix(plan: BisectPlan, index: number, o: { root const runCase = o.runCase ?? ((id, kind, evalDir) => runOneTrial(id, kind, evalDir, o.rootDir)); const panels: CasePanel[] = []; for (const c of plan.cases) { - let verdict: { status: string; redClass: string | null }; - try { verdict = await runCase(c.id, c.kind, path.join(o.evalDir, c.id.replace(/[^A-Za-z0-9._-]/g, '_'))); } - catch (error) { console.error(`[train-gate] ${c.id}: ${error instanceof Error ? error.message : String(error)}`); verdict = { status: 'INCOMPLETE', redClass: 'INCOMPLETE' }; } - panels.push({ case: c.id, kind: c.kind, status: verdict.status, red_class: verdict.redClass, cost_usd: null }); - console.log(`[train-gate] prefix ${sha.slice(0, 12)} (slice ${index}): ${c.id} ${verdict.status}${verdict.redClass ? ` (${verdict.redClass})` : ''}`); + const dir = path.join(o.evalDir, c.id.replace(/[^A-Za-z0-9._-]/g, '_')); + const run = async (evalDir: string): Promise<{ status: string; redClass: string | null }> => { + try { return await runCase(c.id, c.kind, evalDir); } + catch (error) { console.error(`[train-gate] ${c.id}: ${error instanceof Error ? error.message : String(error)}`); return { status: 'INCOMPLETE', redClass: 'INCOMPLETE' }; } + }; + const verdict = await run(dir); + const rerun = verdict.status === 'FAIL' && verdict.redClass === 'VERDICT' ? await run(`${dir}.rerun`) : null; + const flaky = rerun?.status === 'PASS'; + const final = flaky ? rerun! : verdict; + panels.push({ case: c.id, kind: c.kind, status: final.status, red_class: final.redClass, cost_usd: null, + ...(flaky ? { flaky: { first: `prefix ${sha.slice(0, 12)} slice ${index}`, rerun: `prefix ${sha.slice(0, 12)} slice ${index} rerun` } } : {}) }); + console.log(`[train-gate] prefix ${sha.slice(0, 12)} (slice ${index}): ${c.id} ${verdict.status}${verdict.redClass ? ` (${verdict.redClass})` : ''}` + + `${rerun ? `; rule-16 rerun ${rerun.status}${rerun.redClass ? ` (${rerun.redClass})` : ''}${flaky ? ': FLAKY, counted as PASS' : ': failed twice, red'}` : ''}`); } return { schema: 1, kind: 'train-bisect-prefix', nonce: plan.nonce, prefix_sha: sha, checked_out_sha: o.checkedOut, panels }; } diff --git a/test/ci-native-evidence.test.ts b/test/ci-native-evidence.test.ts index c7da89696..22bf68bab 100644 --- a/test/ci-native-evidence.test.ts +++ b/test/ci-native-evidence.test.ts @@ -44,7 +44,7 @@ function render(template: string, fields: Record): string { test('every direct CI paid executor binds a safe unique run/attempt/job/slice identity', () => { expect(executors.map(({ name, jobName }) => `${name}:${jobName}`)).toEqual([ - 'evals.yml:eval-slices', 'evals-periodic.yml:eval-slices', 'evals-periodic.yml:eval-codex-slices', 'evals-periodic.yml:gate-census', 'evals-marathon.yml:eval-slices', + 'evals.yml:eval-slices', 'evals.yml:eval-rerun', 'evals-periodic.yml:eval-slices', 'evals-periodic.yml:eval-codex-slices', 'evals-periodic.yml:gate-census', 'evals-marathon.yml:eval-slices', ]); const ids = new Set(); for (const [workflowIndex, { job, step }] of executors.entries()) { @@ -59,7 +59,7 @@ test('every direct CI paid executor binds a safe unique run/attempt/job/slice id for (const attempt of ['1', '2']) { // The planner sizes the matrix; cover more slices than any live plan. expect(armed ? job.strategy.matrix.include : job.strategy.matrix.slice) - .toMatch(armed ? /^\$\{\{ fromJSON\(needs\.plan-slices\.outputs\.gate_matrix\) \}\}$/ : /^\$\{\{ fromJSON\(needs\.plan-slices\.outputs\.(?:[a-z]+_)*slices\) \}\}$/); + .toMatch(armed ? /^\$\{\{ fromJSON\(needs\.plan-slices\.outputs\.gate_matrix\) \}\}$/ : /^\$\{\{ fromJSON\(needs\.(?:plan-slices|rerun-plan)\.outputs\.(?:[a-z]+_)*slices\) \}\}$/); for (const arm of armed ? Object.keys(CENSUS_ARMS) : ['']) { for (let slice = 1; slice <= 64; slice++) { const id = render(env.EVALS_RUN_ID, { diff --git a/test/ci-paid-coordination.test.ts b/test/ci-paid-coordination.test.ts index b99b9fc25..0bb517205 100644 --- a/test/ci-paid-coordination.test.ts +++ b/test/ci-paid-coordination.test.ts @@ -145,8 +145,8 @@ describe('paid CI coordination stays off the eval image', () => { const captures = hiddenUploads.filter(step => step.with?.name === 'native-captures-${{ env.EVALS_RUN_ID }}'); // evals-periodic's host-run Codex job mounts $RUNNER_TEMP/eval-home as the container's HOME. const hostHome = (pattern: string) => pattern.replace('${{ runner.temp }}/eval-home/', '~/'); - // evals.yml: eval-slices and train-bisect (GSTA-49 T1b), each under its own EVALS_RUN_ID. - expect(captures).toHaveLength(name === 'evals.yml' ? 2 : name === 'evals-periodic.yml' ? 3 : 2); + // evals.yml: eval-slices, the rule-16 eval-rerun and train-bisect (GSTA-49 T1b), each under its own EVALS_RUN_ID. + expect(captures).toHaveLength(name === 'evals.yml' ? 3 : name === 'evals-periodic.yml' ? 3 : 2); for (const capture of captures) { expect(capture.if).toBe('always()'); expect(String(capture.with?.path).trim().split('\n').map(hostHome)).toEqual([ @@ -165,7 +165,11 @@ describe('paid CI coordination stays off the eval image', () => { const bisectLogs = name === 'evals.yml' ? jobs['train-bisect'].steps.find(step => step.with?.name === 'train-bisect-logs-${{ matrix.slice }}-a${{ github.run_attempt }}') : undefined; if (name === 'evals.yml') expect(String(bisectLogs?.with?.path)).toBe(String(logs?.with?.path)); - expect(hiddenUploads.filter(step => !captures.includes(step))).toEqual([logs!, ...(codexLogs ? [codexLogs] : []), ...(bisectLogs ? [bisectLogs] : [])]); + // F1: the rule-16 eval-rerun keeps the same shard-log upload under its own name. + const rerunLogs = name === 'evals.yml' + ? jobs['eval-rerun'].steps.find(step => step.with?.name === 'rerun-logs-${{ matrix.slice }}-a${{ github.run_attempt }}') : undefined; + if (name === 'evals.yml') expect(String(rerunLogs?.with?.path)).toBe(String(logs?.with?.path)); + expect(hiddenUploads.filter(step => !captures.includes(step))).toEqual([logs!, ...(codexLogs ? [codexLogs] : []), ...(rerunLogs ? [rerunLogs] : []), ...(bisectLogs ? [bisectLogs] : [])]); }); } diff --git a/test/evals-train-workflow.test.ts b/test/evals-train-workflow.test.ts index 6f2a7a090..e52fd1986 100644 --- a/test/evals-train-workflow.test.ts +++ b/test/evals-train-workflow.test.ts @@ -146,8 +146,9 @@ const pr = (labels: string[] = []) => ({ event_name: 'pull_request', actor: 'gar const dispatch = (ref = 'main') => ({ event_name: 'workflow_dispatch', actor: 'garrytan', repository: 'garrytan/gstack', repository_id: 1, sha: SHA.main, run_id: 7, run_attempt: 1, ref_name: ref, event: { repository: { default_branch: 'main' } } }); const INPUTS = { evals_all: true, base_ref: '', base_sha: '', expected_sha: '', nonce: '', pr_receipts: '', cases: '', prefix_shas: '', screen_pr: '', screen_merge_sha: '', screen_policy: '' }; -const TODAY_PR_JOBS = ['build-image', 'eval-slices', 'plan-slices', 'readiness', 'recover-receipts', 'slices-comment', 'slices-report']; -const TODAY_DISPATCH_JOBS = ['build-image', 'eval-slices', 'plan-slices', 'slices-report']; +// rerun-plan is the free rule-16 planner (F1); its paid eval-rerun runs only when it plans a rerun. +const TODAY_PR_JOBS = ['build-image', 'eval-slices', 'plan-slices', 'readiness', 'recover-receipts', 'rerun-plan', 'slices-comment', 'slices-report']; +const TODAY_DISPATCH_JOBS = ['build-image', 'eval-slices', 'plan-slices', 'rerun-plan', 'slices-report']; const TRAIN_STEPS = ['Write the train record', 'Upload the train record', "Check the screen's PR head (Free Tests)", 'Plan the bisect (one executor per prefix)', 'Hand the bisect plan to its executors']; const train = { record_artifact: 'gstack-train-record', full_gate_check: 'gstack/full-gate' }; @@ -206,7 +207,7 @@ describe('under a train block (the gate edits, each only on its mode)', () => { const inputs = { ...INPUTS, evals_all: false, nonce: 'pr.screen.narrow-x', expected_sha: SHA.merge, base_ref: 'main', base_sha: SHA.base, screen_pr: '42', screen_merge_sha: SHA.merge, screen_policy: 'narrow' }; const route = { ...ROUTE, ...train, mode: 'screen', screen_head: SHA.screenHead, screen_base: SHA.screenBase, screen_branch: 'feature' }; const sim = simulate(evals, { github: dispatch(), inputs, outputs: { route } }); - expect(sim.jobs).toEqual(['build-image', 'eval-slices', 'plan-slices', 'recover-receipts', 'route', 'slices-comment', 'slices-report']); + expect(sim.jobs).toEqual(['build-image', 'eval-slices', 'plan-slices', 'recover-receipts', 'rerun-plan', 'route', 'slices-comment', 'slices-report']); expect(sim.group).toBe('evals-screen-42'); expect(sim.runName).toBe('E2E Evals [eval-bg pr.screen.narrow-x] screen #42'); expect(sim.steps('recover-receipts')).not.toContain('Recover receipts of cancelled runs (base-ref code)'); @@ -228,7 +229,7 @@ describe('under a train block (the gate edits, each only on its mode)', () => { test('a train gate runs the full gate, writes its record from the report and publishes the check; it never reuses or binds', () => { const inputs = { ...INPUTS, nonce: 'gate.train-x', expected_sha: SHA.head }; const sim = simulate(evals, { github: dispatch('train/t1'), inputs, outputs: { route: { ...ROUTE, ...train, mode: 'train' } } }); - expect(sim.jobs).toEqual(['build-image', 'eval-slices', 'plan-slices', 'route', 'slices-report', 'train-check']); + expect(sim.jobs).toEqual(['build-image', 'eval-slices', 'plan-slices', 'rerun-plan', 'route', 'slices-report', 'train-check']); expect(sim.steps('plan-slices')).not.toContain('Confirm gate verdict reuse (R2a/R2)'); expect(sim.steps('slices-report')).not.toContain("Bind this run's verdict for reuse (R2a/R2)"); expect(sim.steps('slices-report')).toEqual(expect.arrayContaining(['Write the train record', 'Upload the train record'])); @@ -279,6 +280,92 @@ describe('GSTA-48 open items', () => { }); }); +describe('rule-16 rerun wiring (F1)', () => { + const planned = { planned: 'true', slices: '[1]', slice_timeouts: '[0,40]' }; + test('the paid rerun runs only when rerun-plan planned one, on the same checkout as the slices, and the report waits for it', () => { + const quiet = simulate(evals, { github: pr(), inputs: {}, outputs: { route: { ...ROUTE, mode: 'pr' }, 'rerun-plan': { planned: 'false', slices: '', slice_timeouts: '' } } }); + expect(quiet.jobs).not.toContain('eval-rerun'); + const sim = simulate(evals, { github: pr(), inputs: {}, outputs: { route: { ...ROUTE, mode: 'pr' }, 'rerun-plan': planned } }); + expect(sim.jobs).toEqual([...TODAY_PR_JOBS, 'eval-rerun', 'route'].sort()); + for (const job of ['rerun-plan', 'eval-rerun']) expect(sim.checkout(job)?.ref ?? '', job).toBe(sim.checkout('eval-slices')?.ref ?? ''); + expect([evals.jobs['slices-report']!.needs].flat()).toEqual(expect.arrayContaining(['rerun-plan', 'eval-rerun'])); + const screen = simulate(evals, { github: dispatch(), inputs: { ...INPUTS, evals_all: false, nonce: 'pr.screen.narrow-x', expected_sha: SHA.merge, base_ref: 'main', base_sha: SHA.base, screen_pr: '42', screen_merge_sha: SHA.merge, screen_policy: 'narrow' }, + outputs: { route: { ...ROUTE, ...train, mode: 'screen', screen_head: SHA.screenHead, screen_base: SHA.screenBase, screen_branch: 'feature' }, 'rerun-plan': planned } }); + for (const job of ['rerun-plan', 'eval-rerun']) expect(screen.checkout(job)?.ref, job).toBe(SHA.merge); + const trainRun = simulate(evals, { github: dispatch('train/t1'), inputs: { ...INPUTS, nonce: 'gate.train-x', expected_sha: SHA.head }, outputs: { route: { ...ROUTE, ...train, mode: 'train' }, 'rerun-plan': planned } }); + expect(trainRun.jobs).toEqual(expect.arrayContaining(['rerun-plan', 'eval-rerun', 'slices-report', 'train-check'])); + }); + + test('a real red (reconcile exit 1, e.g. a case red twice) fails slices-report and so the run, after every rerun path', () => { + // slices-report runs whether the slices and the rerun passed, failed or were skipped, and no job may swallow its failure. + for (const results of [{}, { 'eval-slices': 'failure' }, { 'eval-rerun': 'failure' }, { 'eval-rerun': 'skipped' }] as Array>) { + const sim = simulate(evals, { github: pr(), inputs: {}, outputs: { route: { ...ROUTE, mode: 'pr' }, 'rerun-plan': planned }, results }); + expect(sim.jobs, JSON.stringify(results)).toContain('slices-report'); + } + for (const [id, job] of Object.entries(evals.jobs)) expect((job as Record)['continue-on-error'], id).toBeUndefined(); + const report = evals.jobs['slices-report']!; + const reconcile = report.steps.find(st => st.id === 'reconcile')!; + expect(reconcile.run).toContain('echo "exit=${PIPESTATUS[0]}"'); + const fail = report.steps.at(-1)!; + expect(fail.name).toBe('Fail the workflow when reconciliation failed'); + expect(fail.run).toBe('exit 1'); + const fires = (exit: string | null, reused: string | null = null) => truthy(evaluate(String(fail.if), { steps: { reconcile: { outputs: { exit } }, reused: { outputs: { exit: reused } } } }, { success: true, failure: false })); + expect(fires('1')).toBe(true); + expect(fires('0')).toBe(false); + // A red report also withholds the train check (the train verdict reads the run's failure too). + const trainRun = simulate(evals, { github: dispatch('train/t1'), inputs: { ...INPUTS, nonce: 'gate.train-x', expected_sha: SHA.head }, outputs: { route: { ...ROUTE, ...train, mode: 'train' }, 'rerun-plan': planned }, results: { 'slices-report': 'failure' } }); + expect(trainRun.jobs).not.toContain('train-check'); + }); + + test('a confirmed verdict reuse plans and runs no rerun', () => { + const sim = simulate(evals, { github: pr(), inputs: {}, outputs: { route: { ...ROUTE, mode: 'pr' }, 'plan-slices': { gate_reuse: 'true' }, 'rerun-plan': planned } }); + expect(sim.jobs).not.toContain('rerun-plan'); + expect(sim.jobs).not.toContain('eval-rerun'); + expect(sim.jobs).not.toContain('eval-slices'); + }); + + test('the rerun runs its own attempt-scoped manifest fresh: no receipts in, none out, results under paid-rerun-*', () => { + const job = evals.jobs['eval-rerun']!; + expect(job.strategy!.matrix.slice).toBe('${{ fromJSON(needs.rerun-plan.outputs.slices) }}'); + expect(job.steps.find(st => st.uses?.startsWith('actions/download-artifact@'))!.with).toEqual({ name: 'rerun-plan-a${{ github.run_attempt }}', path: '/tmp/paid-rerun-plan' }); + const run = job.steps.find(st => st.name === 'Rerun slice ${{ matrix.slice }} (rule 16, once)')!; + expect(run.run).toContain('--plan /tmp/paid-rerun-plan/rerun-plan-a${{ github.run_attempt }}/manifest.json --slice ${{ matrix.slice }}'); + expect(Object.keys(run.env!).filter(key => key.startsWith('EVALS_CACHE'))).toEqual([]); + expect(JSON.stringify(job)).not.toContain('receipts'); + expect(job.steps.find(st => st.name === 'Upload rerun results')!.with).toMatchObject({ name: 'paid-rerun-${{ matrix.slice }}-a${{ github.run_attempt }}', path: run.env!.GSTACK_EVAL_DIR }); + const uploads = Object.values(evals.jobs).flatMap(j => j.steps).map(st => String(st.with?.name ?? '')).filter(Boolean); + expect(uploads.filter(name => name.startsWith('paid-rerun-'))).toEqual(['paid-rerun-${{ matrix.slice }}-a${{ github.run_attempt }}']); + const plan = evals.jobs['rerun-plan']!; + expect(plan.permissions).toEqual({ contents: 'read' }); + expect(JSON.stringify(plan)).not.toMatch(/secrets\.|restore-deps|bun install/); + expect(plan.steps.find(st => st.name === "Plan the rule-16 rerun of this attempt's failed cases")!.run!.replace(/\s*\\\n\s*/g, ' ')) + .toContain('--report /tmp/paid-report --rerun-plan "/tmp/paid-rerun-plan/rerun-plan-a$GITHUB_RUN_ATTEMPT"'); + expect(plan.steps.find(st => st.name === 'Hand the rerun manifest to its executors and the report')!.with).toMatchObject({ name: 'rerun-plan-a${{ github.run_attempt }}', path: '/tmp/paid-rerun-plan' }); + }); + + test('a lone rerun artifact never unpacks over the report root or the plan manifest (run 39363705692 did)', () => { + for (const id of ['rerun-plan', 'slices-report']) { + const downloads = evals.jobs[id]!.steps.filter(st => st.uses?.startsWith('actions/download-artifact@') && /^(paid-rerun|rerun-plan)-/.test(String(st.with?.pattern ?? ''))); + expect(downloads.map(st => st.with), id).toEqual([{ pattern: 'paid-rerun-*', path: '/tmp/paid-report/reruns' }, { pattern: 'rerun-plan-a*', path: '/tmp/paid-report' }]); + } + // The plan artifact's root is a directory holding rerun-plan-a/manifest.json, written there by the planner. + const plan = evals.jobs['rerun-plan']!; + expect(plan.steps.find(st => st.name === 'Hand the rerun manifest to its executors and the report')!.with!.path).toBe('/tmp/paid-rerun-plan'); + expect(plan.steps.find(st => st.name === "Plan the rule-16 rerun of this attempt's failed cases")!.run).toContain('PLAN="/tmp/paid-rerun-plan/rerun-plan-a$GITHUB_RUN_ATTEMPT/manifest.json"'); + }); + + test('the report reads every attempt and rerun, publishes flake records, and the comment never hides a non-flaky failed record', () => { + const report = evals.jobs['slices-report']!; + const patterns = report.steps.filter(st => st.uses?.startsWith('actions/download-artifact@')).map(st => st.with?.pattern ?? st.with?.name); + expect(patterns).toEqual(expect.arrayContaining(['paid-slice-*', 'paid-rerun-*', 'rerun-plan-a*'])); + expect(report.steps.find(st => st.name === 'Upload flake records for the daily flaky-fix pass')!.with).toMatchObject({ name: 'flake-records-a${{ github.run_attempt }}', path: '/tmp/paid-report/flake-records.jsonl' }); + const comment = evals.jobs['slices-comment']!.steps.find(st => st.name === 'Post PR comment')!.run!; + expect(comment).toContain('[ "$FAILED" -gt "$FLAKY_FAILED" ]'); + expect(comment).toContain("FLAKY_FAILED=$(jq -r '.flakyCollectorFailures // 0' \"$VERIFIED\")"); + expect(comment).toContain('((.flakyCollectorFailures // 0) <= .totals.failed)'); + }); +}); + describe.skipIf(!Bun.which('jq') || !Bun.which('bash'))('the route job reads the train policy from the base revision with git and jq only', () => { const route = evals.jobs.route!; const policyStep = route.steps.find(st => st.name === 'Read the train policy (base revision)')!; diff --git a/test/fixtures/shard-cli-parity/results.json b/test/fixtures/shard-cli-parity/results.json index f8986404f..cb5bc092d 100644 --- a/test/fixtures/shard-cli-parity/results.json +++ b/test/fixtures/shard-cli-parity/results.json @@ -461,6 +461,7 @@ "planPath": null, "sliceIndex": null, "reportDir": null, + "rerunPlanDir": null, "writeDurations": false, "maxParallel": null, "caseId": null, @@ -495,6 +496,7 @@ "planPath": null, "sliceIndex": null, "reportDir": null, + "rerunPlanDir": null, "writeDurations": false, "maxParallel": null, "caseId": null, @@ -529,6 +531,7 @@ "planPath": null, "sliceIndex": null, "reportDir": null, + "rerunPlanDir": null, "writeDurations": false, "maxParallel": null, "caseId": null, @@ -564,6 +567,7 @@ "planPath": null, "sliceIndex": null, "reportDir": null, + "rerunPlanDir": null, "writeDurations": false, "maxParallel": null, "caseId": null, @@ -598,6 +602,7 @@ "planPath": null, "sliceIndex": null, "reportDir": null, + "rerunPlanDir": null, "writeDurations": false, "maxParallel": null, "caseId": null, @@ -631,6 +636,7 @@ "planPath": "manifest.json", "sliceIndex": 2, "reportDir": null, + "rerunPlanDir": null, "writeDurations": false, "maxParallel": null, "caseId": null, @@ -663,6 +669,7 @@ "planPath": null, "sliceIndex": null, "reportDir": "reports", + "rerunPlanDir": null, "writeDurations": true, "maxParallel": null, "caseId": null, @@ -698,6 +705,7 @@ "planPath": null, "sliceIndex": null, "reportDir": null, + "rerunPlanDir": null, "writeDurations": false, "maxParallel": null, "caseId": null, diff --git a/test/gate-reuse.test.ts b/test/gate-reuse.test.ts index 0e9442c8b..994c33fed 100644 --- a/test/gate-reuse.test.ts +++ b/test/gate-reuse.test.ts @@ -9,7 +9,7 @@ import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { - BINDING_FILE, CANDIDATE_FILE, REUSE_FILE, buildBinding, confirmReuse, findReusableGate, parseReuse, reuseReport, selectionFingerprint, + BINDING_FILE, CANDIDATE_FILE, REUSE_FILE, buildBinding, confirmReuse, findReusableGate, parseBinding, parseReuse, reuseReport, selectionFingerprint, type GateBinding, type GateReuse, } from '../scripts/gate-reuse'; import type { GhRunner, WorkflowRun } from '../scripts/recover-receipts'; @@ -558,3 +558,74 @@ describe('dispatched screens (GSTA-49 T1b): authenticated by PR, head, repositor expect(screen('narrow', [trainRun, screenRun(300)], { 300: screenBinding(sha.gated!), 310: screenBinding(sha.gated!) }).candidate?.reused_run_id).toBe(300); }); }); + +describe('rule-16 flaky GREENs are reusable (GSTA-53, F1)', () => { + const flaky = (over: Partial = {}) => binding({ flaky_cases: ['tpa-absent-darwin'], ...over }); + /** A run that bound once per attempt: gate-binding-a each, download by name (a `gh run rerun --failed` attempt binds again). */ + function attemptsGh(runs: WorkflowRun[], perAttempt: Record): GhRunner { + const inner = fakeGh(runs, { 300: perAttempt[1]! }); + return (args, timeoutMs) => { + if (args.some(a => /\/runs\/300\/artifacts$/.test(a))) { + return { status: 0, stdout: JSON.stringify({ artifacts: [...Object.keys(perAttempt).map(n => ({ name: `gate-binding-a${n}`, expired: false })), { name: 'paid-slice-1-a1', expired: false }] }), stderr: '' }; + } + if (args[0] === 'run' && args[1] === 'download' && args[2] === '300') { + const dir = args[args.indexOf('-D') + 1]!; + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, BINDING_FILE), JSON.stringify(perAttempt[Number(args[args.indexOf('-n') + 1]!.slice('gate-binding-a'.length))])); + return { status: 0, stdout: '', stderr: '' }; + } + return inner(args, timeoutMs); + }; + } + const findWith = (head: string, gh: GhRunner, baseSha = sha.base!) => { + const out = path.join(scratch, 'out-' + ++outCount); + const result = findReusableGate({ repo: REPO, pr: 42, headRepo: REPO, branch: 'feature', selfRunId: 500, headSha: head, testedSha: tested(baseSha, head), repoRoot: receipt, out, gh }); + return { ...result, out, text: result.lines.join('\n') }; + }; + + test('R2a: a restamp of a flaky-annotated GREEN reuses it and carries the flaky case names', () => { + const { candidate } = find(sha.restamp!, [run(300, sha.gated!)], { 300: flaky() }); + expect(candidate).toMatchObject({ kind: 'restamp-only', reused_run_id: 300, reused_flaky: ['tpa-absent-darwin'] }); + expect(parseReuse(candidate)).toEqual(candidate); + expect(reuseReport(candidate!, REPO)).toContain('flaky (rule 16): tpa-absent-darwin failed once and passed on their one rerun in the reused gate'); + }); + + test('R2: main moved with an identical own diff reuses a flaky-annotated GREEN', () => { + const { candidate } = find(sha.rebased!, [run(300, sha.gated!)], { 300: flaky() }, { baseSha: sha.moved }); + expect(candidate).toMatchObject({ kind: 'main-moved', reused_run_id: 300, reused_flaky: ['tpa-absent-darwin'] }); + }); + + test('a manual --failed attempt that went green: its newest binding is reused (R2a and R2), never the red first attempt', () => { + const attempts = { 1: binding({ verdict: 'RED', reconcile_exit: 1 }), 2: flaky({ run_attempt: 2 }) }; + const restamp = findWith(sha.restamp!, attemptsGh([run(300, sha.gated!)], attempts)); + expect(restamp.candidate, restamp.text).toMatchObject({ kind: 'restamp-only', reused_run_attempt: 2, reused_flaky: ['tpa-absent-darwin'] }); + const moved = findWith(sha.rebased!, attemptsGh([run(300, sha.gated!)], attempts), sha.moved!); + expect(moved.candidate, moved.text).toMatchObject({ kind: 'main-moved', reused_run_attempt: 2 }); + }); + + test('negative control: a run still red after its rerun is never reused, whatever it calls flaky', () => { + const stillRed = find(sha.restamp!, [run(300, sha.gated!, 'failure')], { 300: flaky({ verdict: 'RED', reconcile_exit: 1 }) }); + expect(stillRed.candidate).toBeNull(); + expect(stillRed.text).toContain('is RED, reconcile exit 1'); + const moved = find(sha.rebased!, [run(300, sha.gated!, 'failure')], { 300: flaky({ verdict: 'RED', reconcile_exit: 1 }) }, { baseSha: sha.moved }); + expect(moved.candidate).toBeNull(); + const newestRed = findWith(sha.restamp!, attemptsGh([run(300, sha.gated!, 'failure')], { 1: flaky(), 2: binding({ run_attempt: 2, verdict: 'RED', reconcile_exit: 1 }) })); + expect(newestRed.candidate).toBeNull(); + expect(newestRed.text).toContain('is RED'); + }); + + test('bind records the report\'s flaky cases; a malformed flaky list is no binding', () => { + const dir = path.join(scratch, 'bind-flaky'); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, 'manifest.json'), JSON.stringify(manifest())); + fs.writeFileSync(path.join(dir, 'collector-outcomes.json'), JSON.stringify({ verdict: { verdict: 'GREEN' }, totals: { manual_accepted: 0 }, + flaky: [{ case: 'tpa-absent-darwin' }, { case: 'plan-design-with-ui-scope' }] })); + const { binding: bound, reason } = buildBinding({ repository: REPO, pr: 42, runId: 300, runAttempt: 2, headSha: 'a'.repeat(40), baseSha: 'b'.repeat(40), profile: 'pr', reconcileExit: 0, reportDir: dir }); + expect(bound).toMatchObject({ verdict: 'GREEN', run_attempt: 2, flaky_cases: ['tpa-absent-darwin', 'plan-design-with-ui-scope'] }); + expect(reason).toContain('bound GREEN (flaky: tpa-absent-darwin, plan-design-with-ui-scope)'); + expect(parseBinding({ ...binding(), flaky_cases: 'x' })).toBeNull(); + expect(parseBinding({ ...binding(), flaky_cases: [''] })).toBeNull(); + expect(parseBinding(binding())?.flaky_cases).toBeUndefined(); + }); +}); + diff --git a/test/helpers/periodic-exclude-data.ts b/test/helpers/periodic-exclude-data.ts index 2d7693fbb..14cb3fdb3 100644 --- a/test/helpers/periodic-exclude-data.ts +++ b/test/helpers/periodic-exclude-data.ts @@ -75,6 +75,13 @@ export const CASE_CI_EXCLUDE: Record 1)', profile: 'pr', selection, + prCoverage: { mode: 'dependents', e2e: [CASE], judges: [], deferred: [], unknownFiles: [], unknownFileLabels: [], directCases: [], derivedFiles: [], noConsumerFiles: [], + deferredPromptFiles: [], missingCoverage: [], needsFullValidation: false, reasons: [] }, + plan: { sliceBudgetMs: 420000, jobs: 2, estimatedSliceMs: [418862], ciTimeoutMinutes: 50, sliceCiTimeoutMinutes: [50], codexSlices: [] }, + entries: [{ file: SHARD, slice: 1, status: 'planned', estimatedMs: 248087, budget }] }; +const junit = (failure: string | null, seconds: string) => ` + + + + \n \n ` : ' />'} + + + +`; +/** The two captured attempts of slice 6: the slice result, the shard's JUnit and its session ledger. */ +const CAPTURE: Record; junit: string; ledger: Record }> = { + [FAIL]: { + slice: { version: 1, tier: 'gate', profile: 'pr', selection, sliceIndex: 1, sliceCount: 1, attempt: 1, startedAt: 1792431197455, finishedAt: 1792431811508, + outcomes: [{ files: [SHARD], status: 'failed', exitCode: 1, elapsedMs: 610510, executedTests: 1, skippedTests: 0, budget }] }, + junit: junit('floor test FAILED: outcome=timeout elapsed=609738ms summary: no qualifying finding question within 600000ms', '610.414157'), + ledger: { key: 'pty:session#1', runner: 'pty', started_at: '2026-10-19T17:33:21.040Z', budget_ms: 600000, elapsed_ms: 609726, end: 'observer_timeout', billed: false }, + }, + [PASS]: { + slice: { version: 1, tier: 'gate', profile: 'pr', selection, sliceIndex: 1, sliceCount: 1, attempt: 2, startedAt: 1792433125271, finishedAt: 1792433342167, + outcomes: [{ files: [SHARD], status: 'passed', exitCode: 0, elapsedMs: 117689, executedTests: 1, skippedTests: 0, budget }] }, + junit: junit(null, '117.609903'), + ledger: { key: 'pty:session#1', runner: 'pty', started_at: '2026-10-19T18:05:41.000Z', budget_ms: 600000, elapsed_ms: 117600, end: 'completed', billed: false }, + }, +}; + +let base: string; +beforeAll(() => { base = fs.mkdtempSync(path.join(os.tmpdir(), 'paid-rerun-')); }); +afterAll(() => { fs.rmSync(base, { recursive: true, force: true }); }); + +type Placement = { from: string; to: string; attempt: number; edit?: (slice: any) => void }; +function place(dir: string, p: Placement, extra: Record = {}) { + const capture = CAPTURE[p.from]!; + const shard = path.join(dir, p.to, 'shards', 'skill-e2e-plan-ceo-finding-floor'); + fs.mkdirSync(shard, { recursive: true }); + fs.writeFileSync(path.join(shard, 'junit.xml'), capture.junit); + fs.writeFileSync(path.join(shard, 'session-ledger.jsonl'), JSON.stringify(capture.ledger) + '\n'); + const slice = structuredClone({ ...capture.slice, attempt: p.attempt, ...extra }) as any; + p.edit?.(slice); + fs.writeFileSync(path.join(dir, p.to, 'slice-1.json'), JSON.stringify(slice)); +} +let counter = 0; +/** A report dir holding the fixture manifest and the named first-execution slice artifacts. */ +function reportDir(placements: Placement[]): string { + const dir = path.join(base, `case-${++counter}`); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, 'manifest.json'), JSON.stringify(MANIFEST)); + for (const p of placements) place(dir, p); + return dir; +} + +const env = (extra: NodeJS.ProcessEnv = {}) => ({ ...process.env, EVALS_TIER: 'gate', GITHUB_RUN_ID: '39293281037', GITHUB_SHA: '', GSTACK_GATE_PR: '', GSTACK_GATE_HEAD_SHA: '', ...extra }); +const cli = (args: string[], extra: NodeJS.ProcessEnv = {}) => spawnSync(process.execPath, ['--no-env-file', '--config=/dev/null', path.join(ROOT, 'scripts/test-paid-shards.ts'), '--tier', 'gate', ...args], + { cwd: ROOT, encoding: 'utf8', timeout: 60_000, env: env(extra) }); +function report(dir: string) { + const r = cli(['--report', dir]); + const collector = JSON.parse(fs.readFileSync(path.join(dir, 'collector-outcomes.json'), 'utf8')); + const flakes = fs.existsSync(path.join(dir, FLAKE_RECORDS_FILE)) + ? fs.readFileSync(path.join(dir, FLAKE_RECORDS_FILE), 'utf8').trim().split('\n').map(line => JSON.parse(line)) : []; + return { status: r.status, out: `${r.stdout}\n${r.stderr}`, collector, flakes }; +} +/** The rerun-plan job: writes `/rerun-plan-a/manifest.json` when it plans a rerun. */ +function planRerun(dir: string, attempt: number) { + const out = path.join(dir, `${RERUN_PLAN_ARTIFACT}-a${attempt}`); + const r = cli(['--report', dir, '--rerun-plan', out], { GITHUB_RUN_ATTEMPT: String(attempt) }); + expect(r.status, r.stdout + r.stderr).toBe(0); + const file = path.join(out, 'manifest.json'); + return { out: r.stdout, manifest: fs.existsSync(file) ? JSON.parse(fs.readFileSync(file, 'utf8')) : null }; +} +/** Attempt 1 failed, the real planner wrote its rerun manifest, and the eval-rerun job's result is `from` (as paid-rerun-1-a1). */ +function withRerun(from: string, edit?: (slice: any) => void): string { + const dir = reportDir([{ from: FAIL, to: FAIL, attempt: 1 }]); + const plan = planRerun(dir, 1).manifest; + expect(plan).not.toBeNull(); + fs.rmSync(path.join(dir, 'collector-outcomes.json'), { force: true }); + place(dir, { from, to: 'paid-rerun-1-a1', attempt: 1, edit }, { rerun: true, sliceCount: plan.sliceCount, selection: plan.selection }); + return dir; +} + +describe('replay of run 39293281037 (trimmed capture)', () => { + test('attempt 1 alone is RED, and the planner reruns exactly the failed case once, fresh', () => { + const dir = reportDir([{ from: FAIL, to: FAIL, attempt: 1 }]); + const r = report(dir); + expect(r.status, r.out).toBe(1); + expect(r.out).toContain('VERDICT RED'); + expect(r.flakes).toEqual([]); + const plan = planRerun(dir, 1); + expect(plan.out).toContain('rule-16 rerun of attempt 1: 1 case(s) once'); + expect(plan.manifest.rerun).toEqual({ attempt: 1, cases: [CASE] }); + expect(plan.manifest.entries.map((e: any) => e.file)).toEqual([SHARD]); + expect(plan.manifest.selection).toEqual({ e2e: [CASE], judges: [] }); + expect(plan.manifest.tier).toBe('gate'); + }); + + test('a manual `gh run rerun --failed` attempt that passes the red: GREEN, flaky-annotated, one flake record', () => { + const r = report(reportDir([{ from: FAIL, to: FAIL, attempt: 1 }, { from: PASS, to: PASS, attempt: 2 }])); + expect(r.status, r.out).toBe(0); + expect(r.out).toContain('VERDICT GREEN'); + expect(r.out).toContain('FLAKY 1 (failed once, passed on its one rule-16 rerun)'); + expect(r.collector.verdict.verdict).toBe('GREEN'); + expect(r.collector.flakyLines).toHaveLength(1); + expect(r.collector.flakyLines[0]).toContain(`⚑ ${CASE} rule FLAKY: failed in attempt 1 (paid-slice-1-a1), passed on its one rerun in attempt 2 (paid-slice-1-a2); rerun cost unknown (no billing captured); first failure: assertion: floor test FAILED: outcome=timeout`); + expect(r.flakes).toEqual([expect.objectContaining({ schema: 1, kind: 'gate-flake', case: CASE, case_kind: 'rule', run_id: '39293281037', + first: { attempt: 1, phase: 'first', where: 'attempt 1 (paid-slice-1-a1)' }, rerun: { attempt: 2, phase: 'first', where: 'attempt 2 (paid-slice-1-a2)' }, + symptom: expect.stringContaining('outcome=timeout elapsed=609738ms'), rerun_cost_usd: null })]); + }); + + test('the in-run rerun that passes: GREEN, flaky-annotated, its record names the rerun, and nothing reruns again', () => { + const dir = withRerun(PASS); + const r = report(dir); + expect(r.status, r.out).toBe(0); + expect(r.flakes).toEqual([expect.objectContaining({ case: CASE, first: { attempt: 1, phase: 'first', where: 'attempt 1 (paid-slice-1-a1)' }, + rerun: { attempt: 1, phase: 'rerun', where: 'attempt 1 rerun (paid-rerun-1-a1)' } })]); + expect(planRerun(dir, 2).manifest).toBeNull(); + }); +}); + +describe('negative controls: rule 16 never turns a real red green', () => { + test('red twice stays RED, is named a real red, and is never rerun again (one rerun per case)', () => { + const dir = withRerun(FAIL); + const r = report(dir); + expect(r.status, r.out).toBe(1); + expect(r.out).toContain(`✗ ${CASE} rule FAILED TWICE (rule 16: a real red): attempt 1 (paid-slice-1-a1); again in attempt 1 rerun (paid-rerun-1-a1)`); + expect(r.flakes).toEqual([]); + expect(r.collector.flakyLines).toEqual([]); + const again = planRerun(dir, 2); + expect(again.manifest).toBeNull(); + expect(again.out).toContain(`not rerun: ${CASE} (failed twice`); + }); + + test('red twice is a real red everywhere it is read: the report, the reuse binding and lookup, and the train verdict', () => { + const dir = withRerun(FAIL); + const r = report(dir); + expect(r.status, r.out).toBe(1); + expect(r.out).toContain('VERDICT RED'); + expect(r.collector.verdict.verdict).toBe('RED'); + // gate-reuse: slices-report binds what the report proved, and find refuses it (evals.yml also fails the run on reconcile exit 1). + const head = 'a'.repeat(40); + const { binding } = buildBinding({ repository: 'garrytan/gstack', pr: 42, runId: 300, runAttempt: 1, headSha: head, baseSha: 'b'.repeat(40), profile: 'pr', reconcileExit: r.status!, reportDir: dir }); + expect(binding).toMatchObject({ verdict: 'RED', reconcile_exit: 1 }); + expect(binding!.flaky_cases).toBeUndefined(); + const gh: GhRunner = (args) => { + const ok = (body: unknown) => ({ status: 0, stdout: JSON.stringify(body), stderr: '' }); + if (args.some(a => a.endsWith('/workflows/evals.yml/runs'))) { + return ok({ workflow_runs: [{ id: 300, status: 'completed', conclusion: 'failure', event: 'pull_request', head_sha: head, run_attempt: 1, + head_repository: { full_name: 'garrytan/gstack' }, pull_requests: [{ number: 42 }] }] }); + } + if (args.some(a => /\/runs\/300\/artifacts$/.test(a))) return ok({ artifacts: [{ name: 'gate-binding-a1', expired: false }, { name: 'paid-slice-1-a1', expired: false }] }); + if (args[0] === 'run' && args[1] === 'download') { + const out = args[args.indexOf('-D') + 1]!; + fs.mkdirSync(out, { recursive: true }); + fs.writeFileSync(path.join(out, BINDING_FILE), JSON.stringify(binding)); + return { status: 0, stdout: '', stderr: '' }; + } + return { status: 1, stdout: '', stderr: `unexpected gh ${args.join(' ')}` }; + }; + const found = findReusableGate({ repo: 'garrytan/gstack', pr: 42, headRepo: 'garrytan/gstack', branch: 'feature', selfRunId: 500, headSha: 'c'.repeat(40), + testedSha: 'd'.repeat(40), repoRoot: dir, out: path.join(dir, 'reuse-out'), gh }); + expect(found.candidate).toBeNull(); + expect(found.lines.join('\n')).toContain('is RED, reconcile exit 1, conclusion failure'); + // The train verdict: its record is RED with the case a VERDICT red, and T1 classifies it RED whatever the run concluded. + const record = buildTrainRecord({ reportDir: dir, nonce: 'gate.train-x', head, tree: 'e'.repeat(40), reconcileExit: r.status!, + prefixes: [], run: { id: 300, attempt: 1, url: 'https://example.invalid/runs/300' } }); + expect(record.verdict).toBe('RED'); + expect(record.panels.find(p => p.case === CASE)).toMatchObject({ status: 'FAIL', red_class: 'VERDICT' }); + expect(record.panels.find(p => p.case === CASE)!.flaky).toBeUndefined(); + for (const conclusion of ['failure', 'success']) { + expect(classifyGate(record, { nonce: 'gate.train-x', head, tree: 'e'.repeat(40) }, { status: 'completed', conclusion }, { status: 'completed', conclusion: 'success' } as never).outcome).toBe('RED'); + } + }); + + test('a manual attempt after a failed rerun never makes it flaky: two failures stay RED', () => { + const dir = withRerun(FAIL); + place(dir, { from: PASS, to: PASS, attempt: 2 }); + const r = report(dir); + expect(r.status, r.out).toBe(1); + expect(r.flakes).toEqual([]); + }); + + test('a later attempt never hides a new red: attempt 1 passed, attempt 2 failed, RED', () => { + const r = report(reportDir([{ from: PASS, to: FAIL, attempt: 1 }, { from: FAIL, to: PASS, attempt: 2 }])); + expect(r.status, r.out).toBe(1); + expect(r.out).toContain('VERDICT RED'); + expect(r.flakes).toEqual([]); + }); + + test('a first-attempt pass: GREEN with no flake record and nothing to rerun', () => { + const dir = reportDir([{ from: PASS, to: PASS, attempt: 1 }]); + const r = report(dir); + expect(r.status, r.out).toBe(0); + expect(r.flakes).toEqual([]); + expect(r.out).not.toContain('FLAKY'); + expect(fs.existsSync(path.join(dir, FLAKE_RECORDS_FILE))).toBe(false); + expect(planRerun(dir, 1).manifest).toBeNull(); + }); + + for (const [label, edit] of [ + ['a runner error (harness)', (s: any) => { s.outcomes[0].runnerError = 'spawn failed'; }], + ['a slice-deadline hang (INCOMPLETE)', (s: any) => { Object.assign(s.outcomes[0], { sliceDeadline: 'hung', status: 'timed-out', exitCode: null }); }], + ['a checkpoint the slice never finalized', (s: any) => { s.checkpoint = true; }], + ] as const) { + test(`${label} is never rerun and never flaky, even when a later attempt passes`, () => { + expect(planRerun(reportDir([{ from: FAIL, to: FAIL, attempt: 1, edit }]), 1).manifest).toBeNull(); + const r = report(reportDir([{ from: FAIL, to: FAIL, attempt: 1, edit }, { from: PASS, to: PASS, attempt: 2 }])); + expect(r.status, r.out).toBe(1); + expect(r.flakes).toEqual([]); + }); + } + + test('rerun results without their rerun manifest are set aside and the red stays', () => { + const dir = withRerun(PASS); + fs.rmSync(path.join(dir, `${RERUN_PLAN_ARTIFACT}-a1`), { recursive: true }); + const r = report(dir); + expect(r.status, r.out).toBe(1); + expect(r.out).toContain('rule-16 rerun ignored: attempt 1: rerun results without a readable rerun manifest'); + expect(r.flakes).toEqual([]); + }); + + test('the rerun reads the same in both download-artifact layouts: a lone match unpacked into its path, or each in its own directory', () => { + const lone = withRerun(PASS); + fs.mkdirSync(path.join(lone, 'reruns')); + fs.renameSync(path.join(lone, 'paid-rerun-1-a1'), path.join(lone, 'reruns', 'only')); + const several = withRerun(PASS); + const planDir = path.join(several, `${RERUN_PLAN_ARTIFACT}-a1`); + fs.renameSync(planDir, `${planDir}.tmp`); + fs.mkdirSync(planDir); + fs.renameSync(`${planDir}.tmp`, path.join(planDir, `${RERUN_PLAN_ARTIFACT}-a1`)); + for (const dir of [lone, several]) { + const r = report(dir); + expect(r.status, r.out).toBe(0); + expect(r.out).toContain('FLAKY 1'); + expect(r.flakes.map((f: any) => f.case)).toEqual([CASE]); + } + }); + + test('a rerun result that does not match its manifest is set aside and the red stays', () => { + const r = report(withRerun(PASS, slice => { slice.sliceCount = 9; })); + expect(r.status, r.out).toBe(1); + expect(r.out).toContain('1 rerun result(s) do not match the rerun manifest'); + expect(r.flakes).toEqual([]); + }); + + test('a rerun counts only for the cases its manifest planned', () => { + const dir = withRerun(PASS); + const planFile = path.join(dir, `${RERUN_PLAN_ARTIFACT}-a1`, 'manifest.json'); + fs.writeFileSync(planFile, JSON.stringify({ ...JSON.parse(fs.readFileSync(planFile, 'utf8')), rerun: { attempt: 1, cases: ['some-other-case'] } })); + const r = report(dir); + expect(r.status, r.out).toBe(1); + expect(r.flakes).toEqual([]); + }); +}); + +describe('case history resolution (pure)', () => { + const o = (attempt: number, phase: 'first' | 'rerun', outcome: 'passed' | 'failed', extra: Partial = {}): CaseObservation => ({ + case: 'c', kind: 'rule', attempt, phase, outcome, eligible: outcome === 'failed', keys: ['test/x.test.ts'], where: `a${attempt} ${phase}`, + costUsd: 0.5, costKnown: true, ...extra }); + + test('flaky only when the one failure is eligible and the very next observation passed', () => { + expect(resolveCases([o(1, 'first', 'failed'), o(1, 'rerun', 'passed')]).get('c')!.status).toBe('flaky'); + expect(resolveCases([o(1, 'first', 'failed'), o(1, 'rerun', 'failed')]).get('c')!.status).toBe('red'); + expect(resolveCases([o(1, 'first', 'failed', { eligible: false, reason: 'INFRA failure class' }), o(2, 'first', 'passed')]).get('c')!.reason).toBe('never rerun: INFRA failure class'); + expect(resolveCases([o(1, 'first', 'failed')]).get('c')!.reason).toBe('no rerun verdict'); + expect(resolveCases([o(1, 'first', 'passed')]).has('c')).toBe(false); + expect(resolveCases([o(1, 'first', 'passed'), o(2, 'first', 'failed')]).get('c')!.status).toBe('red'); + }); + + test('candidates: only an eligible single failure nothing has re-executed, including an earlier attempt never reached', () => { + expect(rerunCandidates([o(1, 'first', 'failed')])).toHaveLength(1); + expect(rerunCandidates([o(1, 'first', 'failed'), o(2, 'first', 'passed')])).toEqual([]); + expect(rerunCandidates([o(1, 'first', 'failed'), o(1, 'rerun', 'failed')])).toEqual([]); + expect(rerunCandidates([o(1, 'first', 'failed', { eligible: false, reason: 'runner error' })])).toEqual([]); + expect(rerunCandidates([o(1, 'first', 'passed')])).toEqual([]); + }); + + test('rerun spend counts every execution after a case first failed', () => { + expect(rerunSpend([o(1, 'first', 'failed'), o(1, 'rerun', 'passed', { costKnown: false })])).toEqual({ executions: 1, usd: 0.5, unknown: 1 }); + expect(rerunSpend([o(1, 'first', 'passed')])).toEqual({ executions: 0, usd: 0, unknown: 0 }); + }); +}); + +describe('gate slice exit (case verdicts)', () => { + const ran = { status: 'failed' as const, exitCode: 1, executedTests: 2, skippedTests: 0 }; + test('a completed rule shard red is report input on the gate lane, so the run concludes as the report does', () => { + expect(sliceExitCode([ran], { caseVerdicts: true })).toBe(0); + expect(sliceExitCode([ran])).toBe(1); + }); + test('execution failures still red the slice', () => { + for (const bad of [{ ...ran, runnerError: 'x' }, { ...ran, sliceDeadline: 'hung' as const }, { ...ran, executedTests: null }, { ...ran, skippedTests: 2 }, + { ...ran, exitCode: null }, { ...ran, status: 'timed-out' as const }, { status: 'never-started' as const, exitCode: null, executedTests: null, skippedTests: null }]) { + expect(sliceExitCode([bad], { caseVerdicts: true }), JSON.stringify(bad)).toBe(1); + } + }); +}); diff --git a/test/train-gate.test.ts b/test/train-gate.test.ts index cd7d5564f..e0dfa4cd7 100644 --- a/test/train-gate.test.ts +++ b/test/train-gate.test.ts @@ -158,6 +158,19 @@ describe('train record: the census comes from the run\'s own manifest, first att expect(record.panels.find(p => p.case === j1)).toMatchObject({ status: 'FAIL', red_class: 'VERDICT' }); expect(classify(record)).toBe('RED'); }); + + test('rule 16: a case the report resolved FLAKY is a PASS naming both runs, and the train is GREEN; without the record it stays red', () => { + const outcomes = [{ case: j1, kind: 'judge', attempt: 1, outcome: 'failed', failure_class: 'assertion' }, + { case: j2, kind: 'judge', attempt: 1, outcome: 'passed' }, ...[1, 2, 3].map(trial => ({ case: behavior, kind: 'behavior', trial, attempt: 1, outcome: 'passed' }))]; + const flake = { schema: 1, kind: 'gate-flake', case: j1, case_kind: 'judge', first: { attempt: 1, phase: 'first', where: 'attempt 1 (paid-slice-1-a1)' }, + rerun: { attempt: 1, phase: 'rerun', where: 'attempt 1 rerun (paid-rerun-1-a1)' } }; + const record = build(report({ outcomes, collector: { verdict: { verdict: 'GREEN' }, census: { unattributed: [] }, panels: [], flaky: [flake] } })); + expect(record.panels.find(p => p.case === j1)).toEqual({ case: j1, kind: 'judge', status: 'PASS', red_class: null, cost_usd: null, + flaky: { first: 'attempt 1 (paid-slice-1-a1)', rerun: 'attempt 1 rerun (paid-rerun-1-a1)' } }); + expect(record.verdict).toBe('GREEN'); + expect(classify(record)).toBe('GREEN'); + expect(build(report({ outcomes, collector: { verdict: { verdict: 'RED' }, census: { unattributed: [] }, panels: [] } })).verdict).toBe('RED'); + }); }); describe('prefix trees modulo stamps', () => { @@ -237,6 +250,24 @@ describe('bisect: one executor per prefix, one trial per case', () => { await expect(runBisectPrefix(plan, 1, { rootDir: scratch, evalDir: scratch, checkedOut: B, runCase: async () => ({ status: 'PASS', redClass: null }) })).rejects.toThrow('not its prefix'); await expect(runBisectPrefix(plan, 3, { rootDir: scratch, evalDir: scratch, checkedOut: B })).rejects.toThrow('names no prefix'); }); + + test('rule 16 in a bisect: a case verdict FAIL reruns once on the prefix; a pass is a flaky PASS, a second FAIL stays red, INFRA and INCOMPLETE never rerun', async () => { + const script = (verdicts: Record>) => { + const calls: string[] = []; + return { calls, runCase: async (id: string, _kind: string, evalDir: string) => { calls.push(`${id} ${path.basename(evalDir)}`); return verdicts[id]!.shift()!; } }; + }; + const fail = { status: 'FAIL', redClass: 'VERDICT' }, pass = { status: 'PASS', redClass: null }; + const flaky = script({ 'case-a': [fail, pass], 'case-b': [fail, fail] }); + const record = await runBisectPrefix(plan, 1, { rootDir: scratch, evalDir: path.join(scratch, 'bisect-flaky'), checkedOut: A, runCase: flaky.runCase }); + expect(record.panels).toEqual([ + { case: 'case-a', kind: 'rule', status: 'PASS', red_class: null, cost_usd: null, flaky: { first: `prefix ${A.slice(0, 12)} slice 1`, rerun: `prefix ${A.slice(0, 12)} slice 1 rerun` } }, + { case: 'case-b', kind: 'judge', status: 'FAIL', red_class: 'VERDICT', cost_usd: null }]); + expect(flaky.calls).toEqual(['case-a case-a', 'case-a case-a.rerun', 'case-b case-b', 'case-b case-b.rerun']); + const never = script({ 'case-a': [{ status: 'FAIL', redClass: 'INFRA' }], 'case-b': [{ status: 'INCOMPLETE', redClass: 'INCOMPLETE' }] }); + const kept = await runBisectPrefix(plan, 1, { rootDir: scratch, evalDir: path.join(scratch, 'bisect-never'), checkedOut: A, runCase: never.runCase }); + expect(kept.panels.map(p => [p.case, p.status, p.red_class])).toEqual([['case-a', 'FAIL', 'INFRA'], ['case-b', 'INCOMPLETE', 'INCOMPLETE']]); + expect(never.calls).toHaveLength(2); + }); }); test('the free screening summary says first how to request the screen, then the selection and its estimated cost', () => {