From 9cf105ccffe11f414f9572a68551db6a0ebb16a6 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Mon, 19 Oct 2026 11:26:20 -0700 Subject: [PATCH] v1.91.149.0 feat(evals): Codex host for gate cases, sol weekly and Fable/Astra monthly census arms (GSTA-49 TM2) (#3292) * feat(evals): Opus 5.5 is the paid gate's model under test; judge stays Fable (GSTA-49 TM1) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * v1.91.140.0 chore: stamp TM1 (Opus 5.5 model under test) and CHANGELOG Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * feat(evals): Codex host adapter, real Codex cost, two-arm weekly and monthly census (GSTA-49 TM2) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * chore: TM2 CHANGELOG under [Unreleased]; re-pin codex-host-runner in the harness manifest after the main merge Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * ci: retrigger PR workflows (base changed to main after open; no pull_request run fired) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * v1.91.149.0 chore: stamp TM2 (Codex host, census arms) and CHANGELOG Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- .github/workflows/evals-periodic.yml | 171 ++++++-- CHANGELOG.md | 27 ++ VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- package.json | 2 +- scripts/census-arms.ts | 81 ++++ scripts/e2e-shard-reuse.ts | 1 + scripts/harness-version.json | 20 +- scripts/lib/paid-cases.ts | 37 +- scripts/lib/paid-plan.ts | 15 +- scripts/lib/paid-report.ts | 11 +- scripts/test-paid-shards.ts | 4 + test/ci-native-evidence.test.ts | 28 +- test/ci-paid-coordination.test.ts | 3 +- test/codex-host.test.ts | 287 +++++++++++++ test/evals-workflow-wiring.test.ts | 159 ++++++- test/fixtures/codex-host/exec-commands.jsonl | 7 + .../codex-host/exec-file-change.jsonl | 9 + test/helpers/anthropic-preflight.ts | 7 +- test/helpers/codex-eval.ts | 12 +- test/helpers/codex-host-runner.ts | 306 ++++++++++++++ test/helpers/codex-session-runner.ts | 399 +++++++++++------- test/helpers/e2e-helpers.ts | 14 + test/helpers/e2e-hosts.ts | 126 ++++++ test/helpers/eval-store.ts | 3 + test/helpers/free-fixtures-data.ts | 1 + test/helpers/session-ledger.ts | 5 + test/helpers/session-runner.ts | 23 +- test/paid-retry-supervision.test.ts | 29 +- test/paid-shards.test.ts | 2 +- 30 files changed, 1533 insertions(+), 260 deletions(-) create mode 100644 scripts/census-arms.ts create mode 100644 test/codex-host.test.ts create mode 100644 test/fixtures/codex-host/exec-commands.jsonl create mode 100644 test/fixtures/codex-host/exec-file-change.jsonl create mode 100644 test/helpers/codex-host-runner.ts create mode 100644 test/helpers/e2e-hosts.ts diff --git a/.github/workflows/evals-periodic.yml b/.github/workflows/evals-periodic.yml index da9f8dde6..f1951fffb 100644 --- a/.github/workflows/evals-periodic.yml +++ b/.github/workflows/evals-periodic.yml @@ -15,6 +15,10 @@ name: Periodic Evals # diff-billed, so without it the full gate census might never execute # anywhere); the hollow-shard guard (exit 0 + zero executed tests under # EVALS_ALL fails) makes both lanes census-health checks, not just test runs. +# The gate census is a matrix of arms (scripts/census-arms.ts): each arm is a +# host and model with its own plan. Monday runs claude-opus (the capture +# model) and codex-sol (gpt-6.1-sol on the Codex host, adapter cases only); +# the monthly cron runs only the census, with claude-fable and codex-astra. # eval-bg (scripts/eval-bg.ts) finds its dispatch by the nonce in the run name. run-name: ${{ inputs.nonce && format('Periodic Evals [eval-bg {0}]', inputs.nonce) || 'Periodic Evals' }} on: @@ -23,6 +27,9 @@ on: # Daily, for autoplan-schema-canary alone (latest Claude Code vs the /autoplan # guard's payload assumptions); every census job below skips this schedule. - cron: '30 7 * * *' + # Monthly, the census jobs alone with the claude-fable and codex-astra arms + # (the 1st; never combined with a day of week, which cron would OR). + - cron: '0 6 1 * *' workflow_dispatch: inputs: redispatch_of: @@ -30,9 +37,13 @@ on: type: string default: '' include_gate_census: - description: 'Branch dispatch only: also run the weekly gate census (default: periodic lane only; gate cases validate through an evals.yml dispatch with evals_all)' + description: 'Branch dispatch only: also run the weekly gate census arms (default: periodic lane only; gate cases validate through an evals.yml dispatch with evals_all)' type: boolean default: false + census_arms: + description: 'Run only the gate census, with these arms (comma-separated: claude-opus, codex-sol, claude-fable, codex-astra; scripts/census-arms.ts). Empty keeps the periodic lane.' + type: string + default: '' expected_sha: description: 'eval-bg: the revision the dispatcher tested; the run fails fast when the ref has moved' type: string @@ -51,7 +62,7 @@ on: # runs in its own group so it never cancels the run that dispatched it; both # runs are reported. concurrency: - group: evals-periodic-${{ github.ref }}-${{ github.event_name }}${{ inputs.redispatch_of && format('-redispatch-{0}', inputs.redispatch_of) || '' }}${{ (github.event.schedule == '30 7 * * *' || inputs.schema_canary_only) && '-schema-canary' || '' }} + group: evals-periodic-${{ github.ref }}-${{ github.event_name }}${{ inputs.redispatch_of && format('-redispatch-{0}', inputs.redispatch_of) || '' }}${{ (github.event.schedule == '30 7 * * *' || inputs.schema_canary_only) && '-schema-canary' || '' }}${{ github.event.schedule == '0 6 1 * *' && '-monthly' || '' }} cancel-in-progress: true env: @@ -126,8 +137,9 @@ jobs: periodic_slices: ${{ steps.periodic-matrix.outputs.slices }} periodic_codex_slices: ${{ steps.periodic-matrix.outputs.codex_slices }} periodic_slice_timeouts: ${{ steps.periodic-matrix.outputs.slice_timeouts }} - gate_slices: ${{ steps.gate-matrix.outputs.slices }} - gate_slice_timeouts: ${{ steps.gate-matrix.outputs.slice_timeouts }} + periodic_lane: ${{ steps.census.outputs.periodic_lane }} + census_arms: ${{ steps.census.outputs.arms }} + gate_matrix: ${{ steps.gate-matrix.outputs.gate_matrix }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: @@ -147,7 +159,18 @@ jobs: with: bun-version: 1.4.2 + # Census arms (scripts/census-arms.ts): which host/model arms run the + # gate census, and whether the periodic lane runs beside them. + - name: Select the census arms + id: census + env: + SCHEDULE: ${{ github.event.schedule }} + CENSUS_ARMS: ${{ inputs.census_arms }} + INCLUDE_GATE_CENSUS: ${{ github.ref == 'refs/heads/main' || inputs.include_gate_census }} + run: bun --no-install run scripts/census-arms.ts --select >> "$GITHUB_OUTPUT" + - name: Emit run manifest (ALL periodic tests minus reasoned excludes) + if: steps.census.outputs.periodic_lane == 'true' env: EVALS_ALL: "1" run: EVALS_TIER=periodic bun --no-install run scripts/test-paid-shards.ts --tier periodic --emit-plan /tmp/paid-plan/manifest.json --slice-budget 420 --jobs 2 --max-parallel 38 @@ -156,6 +179,7 @@ jobs: # container executor runs every other slice. - name: Derive the periodic executor matrix from the plan id: periodic-matrix + if: steps.census.outputs.periodic_lane == 'true' run: | { echo "slices=$(jq -c '[range(1; .sliceCount + 1)] - .plan.codexSlices' /tmp/paid-plan/manifest.json)" @@ -164,26 +188,38 @@ jobs: } >> "$GITHUB_OUTPUT" - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + if: steps.census.outputs.periodic_lane == 'true' with: name: paid-plan path: /tmp/paid-plan/manifest.json retention-days: 30 - - name: Emit gate census manifest (ALL gate tests) + # One plan per arm: the Codex host plans only its adapter cases and + # records every Claude-only case as excluded with its reason. + - name: Emit gate census manifests (ALL gate tests, one plan per arm) + if: steps.census.outputs.arms != '[]' env: EVALS_ALL: "1" - run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --emit-plan /tmp/gate-census-plan/manifest.json --slice-budget 420 --jobs 2 --skip-judges --max-parallel 20 - - - name: Derive the gate census executor matrix from the plan - id: gate-matrix + ARMS: ${{ steps.census.outputs.arms }} run: | - echo "slices=$(jq -c '[range(1; .sliceCount + 1)]' /tmp/gate-census-plan/manifest.json)" >> "$GITHUB_OUTPUT" - echo "slice_timeouts=$(jq -ec '[0] + .plan.sliceCiTimeoutMinutes' /tmp/gate-census-plan/manifest.json)" >> "$GITHUB_OUTPUT" + set -euo pipefail + for arm in $(jq -r '.[]' <<<"$ARMS"); do + mapfile -t arm_env < <(bun --no-install run scripts/census-arms.ts --env "$arm") + env "${arm_env[@]}" EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --emit-plan "/tmp/gate-census-plan/$arm/manifest.json" --slice-budget 420 --jobs 2 --skip-judges --max-parallel 20 + done + + # One executor per arm and planned slice, each with its slice's ceiling. + - name: Derive the gate census executor matrix from the plans + id: gate-matrix + env: + ARMS: ${{ steps.census.outputs.arms }} + run: bun --no-install run scripts/census-arms.ts --matrix /tmp/gate-census-plan "$ARMS" >> "$GITHUB_OUTPUT" - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + if: steps.census.outputs.arms != '[]' with: name: gate-census-plan - path: /tmp/gate-census-plan/manifest.json + path: /tmp/gate-census-plan retention-days: 30 # autoplan-schema-canary: installs the latest published Claude Code with @@ -237,6 +273,7 @@ jobs: eval-slices: runs-on: ubicloud-standard-8 needs: [build-image, plan-slices] + if: ${{ needs.plan-slices.outputs.periodic_lane == 'true' }} env: EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-eval-slices-${{ matrix.slice }} # The planner packs ~7 minutes of recorded work per slice; each slice's job @@ -388,7 +425,7 @@ jobs: eval-codex-slices: runs-on: ubicloud-standard-8 needs: [build-image, plan-slices] - if: ${{ needs.plan-slices.outputs.periodic_codex_slices != '[]' }} + if: ${{ needs.plan-slices.outputs.periodic_lane == 'true' && needs.plan-slices.outputs.periodic_codex_slices != '[]' }} env: EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-eval-slices-${{ matrix.slice }} timeout-minutes: ${{ fromJSON(needs.plan-slices.outputs.periodic_slice_timeouts)[matrix.slice] }} @@ -512,16 +549,18 @@ jobs: # blind spots rot invisibly. Census health, not selector correctness — # selector logic has free synthetic-diff contract tests. gate-census: + name: gate-census-${{ matrix.arm }} (${{ matrix.slice }}) runs-on: ubicloud-standard-8 needs: [build-image, plan-slices] - # A branch dispatch validates the periodic lane only; gate cases on a - # branch go through an evals.yml dispatch with evals_all. Main (schedule - # or dispatch) always runs the census; a branch opts in explicitly. - if: ${{ github.ref == 'refs/heads/main' || inputs.include_gate_census }} + # The planner picks the arms (scripts/census-arms.ts). A branch dispatch + # validates the periodic lane only unless it sets include_gate_census or + # census_arms; gate cases on a branch also go through an evals.yml + # dispatch with evals_all. Main (schedule or dispatch) always runs it. + if: ${{ needs.plan-slices.outputs.gate_matrix != '[]' }} env: - EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-gate-census-${{ matrix.slice }} - # Per-slice job ceiling from the packed plan (W2c/ENG-2), as for eval-slices. - timeout-minutes: ${{ fromJSON(needs.plan-slices.outputs.gate_slice_timeouts)[matrix.slice] }} + EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-gate-census-${{ matrix.arm }}-${{ matrix.slice }} + # Per-slice job ceiling from the arm's packed plan (W2c/ENG-2), as for eval-slices. + timeout-minutes: ${{ matrix.timeout }} permissions: contents: read packages: read @@ -536,7 +575,7 @@ jobs: fail-fast: false max-parallel: 20 matrix: - slice: ${{ fromJSON(needs.plan-slices.outputs.gate_slices) }} + include: ${{ fromJSON(needs.plan-slices.outputs.gate_matrix) }} steps: - name: Record the job start for the slice deadline run: echo "GSTACK_SLICE_JOB_STARTED_AT=$(date +%s)" >> "$GITHUB_ENV" @@ -588,7 +627,15 @@ jobs: printenv OPENAI_API_KEY | "$codex" login --with-api-key "$codex" login status >/dev/null 2>&1 - - name: Run gate census slice ${{ matrix.slice }} + # The arm's host and model (EVALS_HOST, plus EVALS_MODEL or + # GSTACK_CODEX_MODEL). The Codex host runs codex exec with + # danger-full-access, so it needs no bubblewrap and runs in this container. + - name: Select the census arm's host and model + env: + ARM: ${{ matrix.arm }} + run: bun --no-install run scripts/census-arms.ts --env "$ARM" >> "$GITHUB_ENV" + + - name: Run gate census slice ${{ matrix.slice }} (${{ matrix.arm }}) env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} @@ -597,13 +644,13 @@ jobs: EVALS_JOBS: "2" EVALS_CONCURRENCY: "2" GSTACK_EVAL_DIR: /tmp/gate-census-results - run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --plan /tmp/gate-census-plan/manifest.json --slice ${{ matrix.slice }} + run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --plan /tmp/gate-census-plan/${{ matrix.arm }}/manifest.json --slice ${{ matrix.slice }} - name: Upload census results if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: - name: gate-census-${{ matrix.slice }}-a${{ github.run_attempt }} + name: gate-census-${{ matrix.slice }}-a${{ github.run_attempt }}-${{ matrix.arm }} path: /tmp/gate-census-results retention-days: 90 @@ -638,6 +685,9 @@ jobs: actions: read outputs: redispatch: ${{ steps.verdict.outputs.redispatch }} + # A census-only run (monthly cron or census_arms dispatch) re-dispatches + # with the same arms; empty keeps the weekly selection. + redispatch_census_arms: ${{ needs.plan-slices.outputs.periodic_lane != 'true' && join(fromJSON(needs.plan-slices.outputs.census_arms), ',') || '' }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: @@ -648,6 +698,7 @@ jobs: bun-version: 1.4.2 - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + if: needs.plan-slices.outputs.periodic_lane == 'true' with: name: paid-plan path: /tmp/paid-report @@ -655,25 +706,35 @@ jobs: # One directory per attempt-scoped slice artifact (no merge): shard # records never overwrite each other and the first attempt decides. - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + if: needs.plan-slices.outputs.periodic_lane == 'true' with: pattern: paid-slice-* path: /tmp/paid-report - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + if: needs.plan-slices.outputs.gate_matrix != '[]' with: name: gate-census-plan - path: /tmp/gate-census-report + path: /tmp/gate-census-plan - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + if: needs.plan-slices.outputs.gate_matrix != '[]' with: pattern: gate-census-[0-9]* - path: /tmp/gate-census-report + path: /tmp/gate-census-slices - name: Reconcile slices against the manifest (fail-closed) id: reconcile if: always() + env: + PERIODIC_LANE: ${{ needs.plan-slices.outputs.periodic_lane }} run: | set +e + if [ "$PERIODIC_LANE" != "true" ]; then + echo "periodic lane skipped: census-only run (monthly cron or census_arms dispatch)" | tee /tmp/report.txt + echo "exit=0" >> "$GITHUB_OUTPUT" + exit 0 + fi EVALS_TIER=periodic bun --no-install run scripts/test-paid-shards.ts --tier periodic --report /tmp/paid-report | tee /tmp/report.txt # PIPESTATUS[0], NOT $?: GitHub's default run-step shell is # `bash -e {0}` with NO pipefail, so $? after the pipe is tee's @@ -686,20 +747,34 @@ jobs: if: always() env: GATE_CENSUS: ${{ needs.gate-census.result }} + ARMS: ${{ needs.plan-slices.outputs.census_arms }} + # One report per arm: its plan plus its own slice artifacts + # (gate-census--a-) in /tmp/gate-census-report/. run: | set +e if [ "$GATE_CENSUS" = "skipped" ]; then - echo "gate census skipped: branch dispatch without include_gate_census (periodic lane only)" | tee /tmp/gate-report.txt + echo "gate census skipped: no census arm selected (branch dispatch without include_gate_census or census_arms)" | tee /tmp/gate-report.txt echo "exit=0" >> "$GITHUB_OUTPUT" exit 0 fi - EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --report /tmp/gate-census-report | tee /tmp/gate-report.txt - echo "exit=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT" + status=0 + : > /tmp/gate-report.txt + for arm in $(jq -r '.[]' <<<"$ARMS"); do + dir="/tmp/gate-census-report/$arm" + mkdir -p "$dir" + cp "/tmp/gate-census-plan/$arm/manifest.json" "$dir/manifest.json" || status=1 + for slice in /tmp/gate-census-slices/gate-census-*-"$arm"; do if [ -d "$slice" ]; then mv "$slice" "$dir/"; fi; done + echo "=== gate census arm: $arm ===" | tee -a /tmp/gate-report.txt + EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --report "$dir" | tee -a /tmp/gate-report.txt + code=${PIPESTATUS[0]} + if [ "$code" -ne 0 ]; then status=$code; fi + done + echo "exit=$status" >> "$GITHUB_OUTPUT" - name: Stamp trial history series if: always() run: | - for file in /tmp/paid-report/trial-outcomes.jsonl /tmp/gate-census-report/trial-outcomes.jsonl; do + for file in /tmp/paid-report/trial-outcomes.jsonl /tmp/gate-census-report/*/trial-outcomes.jsonl; do if [ -f "$file" ]; then bun --no-install run scripts/eval-trial-series.ts "$file"; fi done @@ -719,7 +794,7 @@ jobs: with: name: trial-outcomes-gate-census-a${{ github.run_attempt }} path: | - /tmp/gate-census-report/trial-outcomes.jsonl + /tmp/gate-census-report/*/trial-outcomes.jsonl if-no-files-found: ignore retention-days: 90 @@ -732,7 +807,7 @@ jobs: run: | mkdir -p /tmp/exercised-map dirs=() - for dir in /tmp/gate-census-report /tmp/paid-report; do [ -d "$dir" ] && dirs+=(--from-dir "$dir"); done + for dir in /tmp/gate-census-report/* /tmp/paid-report; do [ -d "$dir" ] && dirs+=(--from-dir "$dir"); done status=0 bun --no-install run scripts/exercised-map.ts "${dirs[@]}" \ --run-id "$GITHUB_RUN_ID" --out /tmp/exercised-map/exercised-commands.json --report /tmp/exercised-map/report.md || status=$? @@ -779,8 +854,16 @@ jobs: [ "$1" = "0" ] && return 0 jq -e '.version == 2 and .verdict.redispatchEligible == true' "$2/collector-outcomes.json" >/dev/null 2>&1 } + eligible_arms() { # $1 exit, then one report dir per census arm: each must be green or INFRA/INCOMPLETE-only + [ "$1" = "0" ] && return 0 + shift + local dir + for dir in "$@"; do + jq -e '.version == 2 and (.verdict.verdict == "GREEN" or .verdict.redispatchEligible == true)' "$dir/collector-outcomes.json" >/dev/null 2>&1 || return 1 + done + } if [ -z "$REDISPATCH_OF" ] && { [ "$PERIODIC_EXIT" != "0" ] || [ "$GATE_EXIT" != "0" ]; } \ - && eligible "$PERIODIC_EXIT" /tmp/paid-report && eligible "$GATE_EXIT" /tmp/gate-census-report; then + && eligible "$PERIODIC_EXIT" /tmp/paid-report && eligible_arms "$GATE_EXIT" /tmp/gate-census-report/*/; then echo "redispatch=true" >> "$GITHUB_OUTPUT" else echo "redispatch=false" >> "$GITHUB_OUTPUT" @@ -793,7 +876,7 @@ jobs: # comments on or closes main's issue. Every report-derived line goes # through the one published-text sanitizer (scripts/lib/published-text.ts). - name: Write the census report - if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success') + if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || (needs.plan-slices.outputs.periodic_lane == 'true' && needs.eval-slices.result != 'success') || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success') env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} REDISPATCH: ${{ steps.verdict.outputs.redispatch }} @@ -827,7 +910,14 @@ jobs: cat /tmp/paid-report/report-summary.md 2>/dev/null || echo "(no periodic report summary)" echo echo "### Gate census" - cat /tmp/gate-census-report/report-summary.md 2>/dev/null || echo "(no gate census report summary)" + found=0 + for summary in /tmp/gate-census-report/*/report-summary.md; do + [ -f "$summary" ] || continue + found=1 + echo "#### Arm $(basename "$(dirname "$summary")")" + cat "$summary" + done + [ "$found" = 1 ] || echo "(no gate census report summary)" echo echo "### Pass-rate history (ACTION REQUIRED)" { awk '/^ACTION REQUIRED/ { on = 1; print; next } on && /^ \[/ { print; next } { on = 0 } /history unavailable/ { print }' /tmp/pass-rates.txt 2>/dev/null \ @@ -851,7 +941,7 @@ jobs: cat /tmp/census-report.md >> "$GITHUB_STEP_SUMMARY" - name: Upload the census report - if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success') + if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || (needs.plan-slices.outputs.periodic_lane == 'true' && needs.eval-slices.result != 'success') || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success') uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: census-report-a${{ github.run_attempt }} @@ -860,7 +950,7 @@ jobs: retention-days: 90 - name: Upsert tracking issue on failure - if: always() && github.ref == 'refs/heads/main' && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success') + if: always() && github.ref == 'refs/heads/main' && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || (needs.plan-slices.outputs.periodic_lane == 'true' && needs.eval-slices.result != 'success') || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success') env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -877,7 +967,7 @@ jobs: fi - name: Close the tracking issue on a green run - if: always() && github.ref == 'refs/heads/main' && steps.reconcile.outputs.exit == '0' && steps.gate-reconcile.outputs.exit == '0' && steps.pass-rates.outputs.exit == '0' && needs.eval-slices.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) && needs.gate-census.result == 'success' + if: always() && github.ref == 'refs/heads/main' && steps.reconcile.outputs.exit == '0' && steps.gate-reconcile.outputs.exit == '0' && steps.pass-rates.outputs.exit == '0' && needs.plan-slices.outputs.periodic_lane == 'true' && needs.eval-slices.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) && needs.gate-census.result == 'success' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} REDISPATCH_OF: ${{ inputs.redispatch_of }} @@ -892,7 +982,7 @@ jobs: fi - name: Fail the workflow when reconciliation failed - if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || !contains(fromJSON('["success","skipped"]'), needs.gate-census.result)) + if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || (needs.plan-slices.outputs.periodic_lane == 'true' && needs.eval-slices.result != 'success') || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || !contains(fromJSON('["success","skipped"]'), needs.gate-census.result)) run: exit 1 # The one INFRA/INCOMPLETE re-dispatch (UC-E1). Its own job so the report @@ -909,4 +999,5 @@ jobs: - name: Re-dispatch the weekly census once env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: gh workflow run evals-periodic.yml --repo "$GITHUB_REPOSITORY" --ref "$GITHUB_REF_NAME" -f redispatch_of="$GITHUB_RUN_ID" + CENSUS_ARMS: ${{ needs.report.outputs.redispatch_census_arms }} + run: gh workflow run evals-periodic.yml --repo "$GITHUB_REPOSITORY" --ref "$GITHUB_REF_NAME" -f redispatch_of="$GITHUB_RUN_ID" ${CENSUS_ARMS:+-f census_arms="$CENSUS_ARMS"} diff --git a/CHANGELOG.md b/CHANGELOG.md index b130dfee2..71b30aaaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,32 @@ # Changelog +## [1.91.149.0] - 2026-10-19 + +**Paid eval cases can now run on the Codex host, and the Monday census runs two arms: Claude Opus 5.5 and Codex gpt-6.1-sol.** `EVALS_HOST=codex` runs the 52 adapter-only gate cases through a full Codex runtime per case. On its first pilot gpt-6.1-sol passed 48 of 52 with no retries, at $4.63 for the 52 cases, against $28.93 for the same cases on Fable (0.16×). Claude-only cases are skipped on Codex with a recorded reason and never count as failed. GSTA-49 TM2. + +### Added +- **Codex host adapter:** `test/helpers/e2e-hosts.ts` holds the `EVALS_HOST` switch (`claude` default, `codex`) and the per-case host metadata. `test/helpers/codex-host-runner.ts` adds: + - the case runner, with sandbox settings and a reasoning effort (the model default unless set); + - file-change event mapping; + - a transcript translated into the Claude stream-json shape that existing assertions read; + - the native-lifecycle hooks. + + The 26 rework cases stay Claude-only. +- **Census arms (`scripts/census-arms.ts`):** `evals-periodic.yml` gets two census schedules. + - **Monday:** a two-arm gate census, `claude-opus` and `codex-sol`. + - **Monthly (`0 6 1 * *`):** census jobs only, with `claude-fable` (claude-fable-5-1) and `codex-astra` (gpt-6-astra) arms and a `-monthly` concurrency key. + + A `census_arms` dispatch input picks arms by hand. Artifact names and run IDs carry the arm, and a census-only run executes no periodic slice. Every arm keeps the census's `--skip-judges`. + +### Changed +- **Codex cost is real:** `cost_known: false` is gone. Codex sessions price their `turn.completed` usage (input, cached input, cache-write input and output tokens) through `lib/pricing.ts`. The shared `execCodex` keeps the old runner's events and verdicts. +- **Plans, reports and the ledger carry the host** when it is not Claude. On Codex, the Anthropic preflight skips its `claude -p` ping. +- **Gate integrity:** no threshold, trial count, case kind, judge or Claude-host verdict changed. `HARNESS_VERSION` stays 6, and the changed harness files are recorded as non-behavioral. Five workflow tests now assert the census matrix per arm (one row per arm and slice, per-arm plan and report paths, a `periodic_lane` guard) in place of the single-arm shape; no assertion was dropped. + +### If you… +- **…want one case on Codex:** `EVALS_HOST=codex GSTACK_CODEX_MODEL=gpt-6.1-sol bun run scripts/test-paid-shards.ts --tier gate --case --trials 1`. +- **…want the monthly arms now:** dispatch `evals-periodic.yml` with `census_arms=claude-fable,codex-astra`. + ## [1.91.148.0] - 2026-10-19 **Weekly seed PRs now get the credential scan as well as the free suites.** A pull request the recorder opens with the workflow token never starts its own `pull_request` checks. GitHub holds them as "action required". So the recorder dispatches its checks on the bot branch instead. diff --git a/VERSION b/VERSION index 427e8cae8..bdda80070 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.148.0 +1.91.149.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 7c7dab179..f819c371f 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.148.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.149.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/package.json b/package.json index 61f535db9..0a91faf86 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.148", + "version": "1.91.149", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/scripts/census-arms.ts b/scripts/census-arms.ts new file mode 100644 index 000000000..9ede1799a --- /dev/null +++ b/scripts/census-arms.ts @@ -0,0 +1,81 @@ +#!/usr/bin/env bun +/** + * Gate census arms for evals-periodic.yml (GSTA-49 TM2). Each arm is one host + * and model running the weekly gate census plan: + * - Monday cron (0 6 * * 1): claude-opus (the capture model, lib/eval-model.ts) + * and codex-sol (gpt-6.1-sol on the Codex host, adapter cases only); + * - monthly cron (0 6 1 * *): claude-fable and codex-astra, census jobs only; + * - dispatch: `census_arms` runs only the census with the named arms; without + * it a dispatch keeps the periodic lane plus the Monday arms when the census + * runs (main, or include_gate_census). + * Judges stay in the Claude lanes: every census plan skips them. + * + * CLI (GitHub step outputs and env lines): + * --select env SCHEDULE, CENSUS_ARMS, INCLUDE_GATE_CENSUS -> arms=, periodic_lane= + * --env KEY=VALUE lines selecting the arm's host and model + * --matrix gate_matrix= from //manifest.json + */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import type { EvalHost } from '../test/helpers/e2e-hosts'; + +export interface CensusArm { host: EvalHost; model?: string } + +export const CENSUS_ARMS: Readonly> = Object.freeze({ + // No model: the Claude runners' capture default (claude-opus-5-5 since TM1). + 'claude-opus': { host: 'claude' }, + 'codex-sol': { host: 'codex', model: 'gpt-6.1-sol' }, + 'claude-fable': { host: 'claude', model: 'claude-fable-5-1' }, + 'codex-astra': { host: 'codex', model: 'gpt-6-astra' }, +}); + +export const WEEKLY_CENSUS_SCHEDULE = '0 6 * * 1'; +export const MONTHLY_CENSUS_SCHEDULE = '0 6 1 * *'; +export const WEEKLY_CENSUS_ARMS: readonly string[] = ['claude-opus', 'codex-sol']; +export const MONTHLY_CENSUS_ARMS: readonly string[] = ['claude-fable', 'codex-astra']; + +/** Which census arms run, and whether the periodic lane runs beside them. */ +export function selectCensus(opts: { schedule?: string; censusArms?: string; includeGateCensus: boolean }): { arms: string[]; periodicLane: boolean } { + if (opts.schedule === MONTHLY_CENSUS_SCHEDULE) return { arms: [...MONTHLY_CENSUS_ARMS], periodicLane: false }; + const named = (opts.censusArms ?? '').split(/[\s,]+/).filter(Boolean); + const unknown = named.filter(arm => !Object.hasOwn(CENSUS_ARMS, arm)); + if (unknown.length) throw new Error(`unknown census arm(s): ${unknown.join(', ')}; known: ${Object.keys(CENSUS_ARMS).join(', ')}`); + if (named.length) return { arms: [...new Set(named)], periodicLane: false }; + return { arms: opts.includeGateCensus ? [...WEEKLY_CENSUS_ARMS] : [], periodicLane: true }; +} + +/** The env one arm's planner and executor run with. */ +export function censusArmEnv(arm: string): Record { + const spec = CENSUS_ARMS[arm]; + if (!spec) throw new Error(`unknown census arm: ${arm}`); + if (!spec.model) return { EVALS_HOST: spec.host }; + return { EVALS_HOST: spec.host, [spec.host === 'codex' ? 'GSTACK_CODEX_MODEL' : 'EVALS_MODEL']: spec.model }; +} + +/** One executor per arm and planned slice, with that slice's planned CI ceiling. */ +export function censusMatrix(arms: readonly string[], planDir: string): Array<{ arm: string; slice: number; timeout: number }> { + return arms.flatMap(arm => { + const manifest = JSON.parse(fs.readFileSync(path.join(planDir, arm, 'manifest.json'), 'utf8')) as + { sliceCount: number; plan?: { ciTimeoutMinutes: number; sliceCiTimeoutMinutes?: number[] } }; + if (!manifest.plan) throw new Error(`${arm}: census manifest has no budget plan`); + return Array.from({ length: manifest.sliceCount }, (_, i) => ({ + arm, slice: i + 1, timeout: manifest.plan!.sliceCiTimeoutMinutes?.[i] ?? manifest.plan!.ciTimeoutMinutes, + })); + }); +} + +if (import.meta.main) { + const [mode, ...args] = process.argv.slice(2); + if (mode === '--select') { + const { arms, periodicLane } = selectCensus({ schedule: process.env.SCHEDULE, censusArms: process.env.CENSUS_ARMS, + includeGateCensus: process.env.INCLUDE_GATE_CENSUS === 'true' }); + console.log(`arms=${JSON.stringify(arms)}\nperiodic_lane=${periodicLane}`); + } else if (mode === '--env' && args.length === 1) { + for (const [key, value] of Object.entries(censusArmEnv(args[0]!))) console.log(`${key}=${value}`); + } else if (mode === '--matrix' && args.length === 2) { + console.log(`gate_matrix=${JSON.stringify(censusMatrix(JSON.parse(args[1]!) as string[], args[0]!))}`); + } else { + console.error('usage: census-arms.ts --select | --env | --matrix '); + process.exit(2); + } +} diff --git a/scripts/e2e-shard-reuse.ts b/scripts/e2e-shard-reuse.ts index 64839e98f..e6ffcc176 100644 --- a/scripts/e2e-shard-reuse.ts +++ b/scripts/e2e-shard-reuse.ts @@ -83,6 +83,7 @@ export function e2eReuseLaneProblem(env: NodeJS.ProcessEnv, profileMode: string if (!/^(?:sha256:)?[a-f0-9]{64}$/.test(env.EVALS_CACHE_RUNTIME_ID ?? '')) return 'No immutable runtime identity'; if (env.EVALS_TIER !== 'gate' || env.EVALS_FRESH === '1') return 'Fresh validation requested'; if (['release', 'periodic', 'marathon'].includes(env.EVALS_CACHE_PURPOSE ?? '')) return 'Scheduled and release lanes execute fresh'; + if ((env.EVALS_HOST ?? '').trim() && env.EVALS_HOST!.trim() !== 'claude') return 'Non-Claude hosts execute fresh'; if (env.NODE_OPTIONS || env.BUN_OPTIONS) return 'Preload options change execution outside the consumed source'; if (env.ANTHROPIC_BASE_URL && env.ANTHROPIC_BASE_URL !== 'https://api.anthropic.com') return 'Custom model endpoint'; return null; diff --git a/scripts/harness-version.json b/scripts/harness-version.json index dc18269d4..4e9d5794e 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -3,6 +3,7 @@ "patterns": [ "test/helpers/agent-sdk-runner.ts", "test/helpers/codex-session-runner.ts", + "test/helpers/codex-host-runner.ts", "test/helpers/claude-pty-runner.ts", "test/helpers/pty/**", "test/helpers/pty-screen.ts", @@ -10,22 +11,23 @@ "test/helpers/llm-judge.ts" ], "files": { - "scripts/lib/paid-cases.ts": {"blob":"9bacd52446192a7ecf3dfaae901ea443cf9c9ae2","since":6}, - "scripts/lib/paid-plan.ts": {"blob":"45721ce128a112666a0caed6c13c66637b74ca54","since":6}, - "scripts/lib/paid-report.ts": {"blob":"b66c41237611ccecaaa7bd6a4d7b7aac8dc548cd","since":6}, + "scripts/lib/paid-cases.ts": {"blob":"f862cf5241580c7593b9ee62eacf5ef046248d26","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, + "scripts/lib/paid-plan.ts": {"blob":"95e1798e7e2495ace350d2624e59f83412f341bc","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, + "scripts/lib/paid-report.ts": {"blob":"f92c6bb723b158e328aa040fa5de6a0ee6d3916d","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6}, "scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":6}, - "scripts/test-paid-shards.ts": {"blob":"8ddee5f11ed7856876135520dd8025ab989e2db9","since":6}, + "scripts/test-paid-shards.ts": {"blob":"19c9b2376d09090aee1a8acacfe721d66f55b0fc","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "scripts/test-pr-profile.ts": {"blob":"579ca1bbd590f709c1ea3d6f28fa6c629688c79f","since":6,"nonBehavioral":"Classifies scripts/free-test-seed-allowlist.json as free-only; only the free seed ratchet reads it, so no verdict can change"}, "scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":6}, "test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":6}, "test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":6}, "test/helpers/claude-pty-runner.ts": {"blob":"20cc5894532873aa6145fc6617fdeec8cef94a2d","since":6}, - "test/helpers/codex-session-runner.ts": {"blob":"4eea35931a35a0afb006363402e3d4191e379737","since":6}, + "test/helpers/codex-host-runner.ts": {"blob":"bff225017e235f0be6ece29f04971ac04b95b544","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, + "test/helpers/codex-session-runner.ts": {"blob":"c040ebbfa815b8cce19ef80c6133bd5a5474149a","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "test/helpers/e2e-gate.ts": {"blob":"efb7e197dd5f66c3ad0e34074b14e640754f7a45","since":6}, - "test/helpers/e2e-helpers.ts": {"blob":"2d4a2a53063cfc1f003db366d2c861ab1e379e6e","since":6}, + "test/helpers/e2e-helpers.ts": {"blob":"6ee6937e6c3ca081c77131b5a755278c7b714834","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "test/helpers/eval-budgets.ts": {"blob":"667d0e09b20fe5d7366f57b77b5fb9b04baa37ef","since":6}, - "test/helpers/eval-store.ts": {"blob":"609c6d099108ff2696eb014b47aa318a618dd2af","since":6}, + "test/helpers/eval-store.ts": {"blob":"ed8721f13d39f4ef62e0a1d814efbe9352da781a","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "test/helpers/hermetic-env.ts": {"blob":"cc072895461dc52ea317d1beff72121342420d30","since":6}, "test/helpers/llm-judge.ts": {"blob":"d0bc6bbbe9f478603d11a1ece95e66c812ad34e6","since":6}, "test/helpers/paid-test-set.ts": {"blob":"ff9bfe43c43ec2bdd11114f7b8ec7c5736116f75","since":6}, @@ -44,8 +46,8 @@ "test/helpers/pty/screen.ts": {"blob":"ba7a80bb68b1959296ab2a827cfacdaa46f4ba2d","since":6}, "test/helpers/pty/session.ts": {"blob":"58533cea9d81cc87bb3b499bdfe0db8e3a8709c5","since":6}, "test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":6}, - "test/helpers/session-ledger.ts": {"blob":"6e3c9e837e46f98addba2d12eaebff802da320bb","since":6}, - "test/helpers/session-runner.ts": {"blob":"d40eb29e8b8fdc0675b511693e4a90823b5963fc","since":6}, + "test/helpers/session-ledger.ts": {"blob":"425323284a368f5e77937e03884db4dea4e6134e","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, + "test/helpers/session-runner.ts": {"blob":"cc65a66adafa9c053e3da3c6f17b5ac86c0eade7","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "test/helpers/skill-fixture.ts": {"blob":"b188064594eb4201a54a80ba1bc6f28e31d7f49e","since":6}, "test/helpers/test-selection.ts": {"blob":"51655f89f66be7a5ec487e2abfe4744300ebd5f6","since":6}, "test/helpers/touchfiles.ts": {"blob":"15f369056d41b9dc42dcd86f81f60c878074704c","since":6} diff --git a/scripts/lib/paid-cases.ts b/scripts/lib/paid-cases.ts index 9bacd5244..f862cf524 100644 --- a/scripts/lib/paid-cases.ts +++ b/scripts/lib/paid-cases.ts @@ -9,6 +9,7 @@ import type { EvalCaseKind, PanelShape } from '../../test/helpers/eval-store'; import { E2E_KINDS } from '../../test/helpers/touchfiles-data'; import { E2E_TOUCHFILES, E2E_TIERS } from '../../test/helpers/touchfiles'; import { type PaidTier, ROOT } from './paid-types'; +import { type EvalHost, hostSkipReason, resolveEvalHost } from '../../test/helpers/e2e-hosts'; /** * The E2E ids a paid file registers: the touchfile registrations that list the @@ -226,12 +227,38 @@ export function partitionCaseExclusions(keys: string[]): { runnable: string[]; e return { runnable, excluded }; } +/** + * Host exclusions (EVALS_HOST, test/helpers/e2e-hosts.ts): on a non-Claude + * host a case key whose case is Claude-only, or a file key none of whose + * `tier` cases runs on the host, is excluded with its reason. A file that + * keeps one host case stays runnable; its Claude-only cases skip at module + * load and the report deselects them. The Claude host excludes nothing. + */ +export function partitionHostExclusions(keys: string[], tier: PaidTier, host: EvalHost, rootDir = ROOT): { runnable: string[]; excluded: Array<{ file: string; reason: string }> } { + if (host === 'claude') return { runnable: keys, excluded: [] }; + const excluded: Array<{ file: string; reason: string }> = []; + const runnable = keys.filter(key => { + const caseId = shardCaseId(key); + const ids = caseId !== null ? [caseId] + : fileCaseRegistration(shardFile(key), fs.readFileSync(path.join(rootDir, shardFile(key)), 'utf8')).registered.filter(id => E2E_TIERS[id] === tier); + if (ids.some(id => hostSkipReason(id, host) === null)) return true; + const reason = ids.length === 1 ? hostSkipReason(ids[0]!, host)! + : ids.length === 0 ? `Claude-only on the ${host} host: no Codex host metadata (no ${tier} case registered)` + : `Claude-only on the ${host} host: no case runs on it (${ids.join(', ')})`; + excluded.push({ file: key, reason: `skipped: ${reason}` }); + return false; + }); + return { runnable, excluded }; +} + /** * Codex access in the CI eval image, scoped per paid shard. * * The image installs the pinned Codex CLI off PATH (GSTACK_CI_CODEX_BIN_DIR) * and the eval workflows log it in under a non-default home - * (GSTACK_CI_CODEX_HOME). A gate shard never sees either. A non-gate shard + * (GSTACK_CI_CODEX_HOME). On the Codex host (EVALS_HOST=codex) Codex is the + * agent, so every shard gets the CLI on PATH and the home as CODEX_HOME. + * Otherwise a gate shard never sees either. A non-gate shard * whose every file exercises Codex gets the CLI on PATH and the home as * CODEX_HOME. A file that mixes one Codex case with other cases keeps only * the two variables, and that case opts in on its own session; hermetic @@ -266,8 +293,10 @@ export const CODEX_CI_CASES: Readonly> = { export type CodexShardAccess = 'none' | 'case-opt-in' | 'path'; /** What one shard, given the test files behind its keys, may see of the CI Codex install in `tier`. */ -export function codexShardAccess(files: readonly string[], tier: string | undefined): CodexShardAccess { - if (tier === 'gate' || files.length === 0) return 'none'; +export function codexShardAccess(files: readonly string[], tier: string | undefined, host: EvalHost = 'claude'): CodexShardAccess { + if (files.length === 0) return 'none'; + if (host === 'codex') return 'path'; + if (tier === 'gate') return 'none'; if (files.every(file => CODEX_CI_FILES.includes(file))) return 'path'; return files.every(file => file in CODEX_CI_CASES) ? 'case-opt-in' : 'none'; } @@ -277,7 +306,7 @@ export function scopeCodexAccess(env: NodeJS.ProcessEnv, files: readonly string[ const binDir = env[CODEX_CI_ENV.binDir]; if (!binDir) return; const home = env[CODEX_CI_ENV.home]; - const access = codexShardAccess(files, env.EVALS_TIER); + const access = codexShardAccess(files, env.EVALS_TIER, resolveEvalHost(env)); delete env.CODEX_HOME; if (access === 'case-opt-in') return; delete env[CODEX_CI_ENV.binDir]; diff --git a/scripts/lib/paid-plan.ts b/scripts/lib/paid-plan.ts index 45721ce12..95e1798e7 100644 --- a/scripts/lib/paid-plan.ts +++ b/scripts/lib/paid-plan.ts @@ -10,7 +10,8 @@ import { evalEntryOutcome, type EvalCaseKind } from '../../test/helpers/eval-sto import { E2E_KINDS } from '../../test/helpers/touchfiles-data'; import { prProfileCaseAllowed, prProfileFileMap, type PrProfileSelection } from '../test-pr-profile'; import { E2E_TOUCHFILES, E2E_TIERS, LLM_JUDGE_TOUCHFILES } from '../../test/helpers/touchfiles'; -import { CASE_KEY_SEPARATOR, CASE_SHARDED_FILES, type CaseTrialPlan, caseTrialPlan, codexShardAccess, expandCaseShards, expandTrialShards, isIsolatedCase, partitionCaseExclusions, sameTrialPlan, shardCaseId, shardFile, shardTrial } from './paid-cases'; +import { CASE_KEY_SEPARATOR, CASE_SHARDED_FILES, type CaseTrialPlan, caseTrialPlan, codexShardAccess, expandCaseShards, expandTrialShards, isIsolatedCase, partitionCaseExclusions, partitionHostExclusions, sameTrialPlan, shardCaseId, shardFile, shardTrial } from './paid-cases'; +import { EVAL_HOSTS, type EvalHost, resolveEvalHost } from '../../test/helpers/e2e-hosts'; import { DEFAULT_JOBS, OVERLAY_MAX_ACTIVE_SHARDS, PAID_TIERS, SLICE_UPLOAD_RESERVE_MS, type PaidCaseSelection, type PaidProfile, type PaidShardBudget, type PaidTier, ROOT, type ShardOutcome, type ShardStatus, type ShardTrialRecord, isOverlayTestFile } from './paid-types'; import { collectPaidTestFiles, computePaidCaseSelection, expectedPrCaseCount, isAllSkippedPass, paidShardWallUpperBoundMs, partitionShardsByDiffSelection, planPaidShards, prProfileFileSelected, resolvePaidShardBudget, resolvePaidShardTimeoutMs, sameBudget, selectPaidTestFiles, shardSlug, validatedProfile } from './paid-select'; @@ -58,6 +59,8 @@ export interface PaidRunManifest { evalsAll: boolean; sliceCount: number; selectionReason: string; + /** The agent host every session runs on (EVALS_HOST); absent means claude. */ + host?: EvalHost; /** Legacy v1 manifests omit these; new plans bind case-level execution. */ profile?: PaidProfile; selection?: PaidCaseSelection; @@ -314,9 +317,11 @@ export function buildRunManifest(opts: { const quarantine = opts.quarantine ?? CASE_QUARANTINE; const notLive = [...Object.keys(kinds).filter(id => kinds[id] === 'behavior'), ...Object.keys(quarantine)].filter(id => !Object.hasOwn(E2E_TIERS, id)); if (notLive.length) throw new Error(`Only live E2E cases can be behavior or quarantined (judges sample their panel inside the case): ${notLive.join(', ')}`); - const caseKeys = partitionCaseExclusions(expandCaseShards(selected, opts.tier, rootDir)); + const host = resolveEvalHost(env); + const ciKeys = partitionCaseExclusions(expandCaseShards(selected, opts.tier, rootDir)); + const caseKeys = partitionHostExclusions(ciKeys.runnable, opts.tier, host, rootDir); const excluded = [...tierSelection.excluded, ...(opts.skipJudges ? tierSelection.selected.filter(judge) - .map(file => ({ file, reason: 'skipped: LLM judges run in the periodic census and PR gate lanes' })) : []), ...caseKeys.excluded]; + .map(file => ({ file, reason: 'skipped: LLM judges run in the periodic census and PR gate lanes' })) : []), ...ciKeys.excluded, ...caseKeys.excluded]; const expansion = expandTrialShards(caseKeys.runnable, opts.tier, rootDir, { kinds, quarantine }); const excludeOf = (key: string) => expansion.excludeCases[normalizeRelativePath(key)] ?? []; const shards = planPaidShards(expansion.keys, { maxFilesPerShard: 1 }); @@ -346,7 +351,7 @@ export function buildRunManifest(opts: { entries.sort((a, b) => (a.file < b.file ? -1 : 1)); return parseRunManifest(JSON.stringify({ version: 1, tier: opts.tier, evalsAll: opts.evalsAll, sliceCount: plan.slices.length, - selectionReason: cases.reason, profile, selection: cases.selection, + selectionReason: cases.reason, ...(host !== 'claude' ? { host } : {}), profile, selection: cases.selection, ...(cases.coverage ? { prCoverage: cases.coverage } : {}), plan: { sliceBudgetMs: opts.sliceBudgetMs!, jobs: opts.jobs!, estimatedSliceMs: plan.estimatedSliceMs, ciTimeoutMinutes: plan.ciTimeoutMinutes, sliceCiTimeoutMinutes: plan.sliceCiTimeoutMinutes, codexSlices: codexSlicesOf(opts.tier, entries) }, @@ -466,6 +471,7 @@ export function buildRunManifest(opts: { evalsAll: opts.evalsAll, sliceCount, selectionReason: cases.reason, + ...(host !== 'claude' ? { host } : {}), profile, selection: cases.selection, ...(cases.coverage ? { prCoverage: cases.coverage } : {}), @@ -479,6 +485,7 @@ export function parseRunManifest(raw: string): PaidRunManifest { if (parsed.version !== 1) throw new Error(`unsupported manifest version: ${(parsed as { version?: unknown }).version}`); if (!PAID_TIERS.includes(parsed.tier)) throw new Error(`manifest tier invalid: ${parsed.tier}`); if (parsed.profile !== undefined && parsed.profile !== 'pr' && parsed.profile !== 'full') throw new Error('manifest profile invalid'); + if (parsed.host !== undefined && (parsed.host === 'claude' || !EVAL_HOSTS.includes(parsed.host))) throw new Error(`manifest host invalid: ${parsed.host}`); if (parsed.selection !== undefined) { for (const [key, inventory] of [['e2e', E2E_TOUCHFILES], ['judges', LLM_JUDGE_TOUCHFILES]] as const) { const ids = parsed.selection?.[key]; diff --git a/scripts/lib/paid-report.ts b/scripts/lib/paid-report.ts index b66c41237..f92c6bb72 100644 --- a/scripts/lib/paid-report.ts +++ b/scripts/lib/paid-report.ts @@ -17,6 +17,7 @@ import { E2E_KINDS } from '../../test/helpers/touchfiles-data'; import { manualReviewProblem } from '../../test/helpers/cookie-workflow-manual-review'; import { writeNegativeReceipt, writePanelReceipt } from '../e2e-shard-reuse'; import { E2E_TOUCHFILES, E2E_TIERS, LLM_JUDGE_TOUCHFILES } from '../../test/helpers/touchfiles'; +import { type EvalHost, hostSkipReason } from '../../test/helpers/e2e-hosts'; import { CASE_TEST_NAMES, type CaseTrialPlan, caseTestNamePattern, caseTrialPlan, fileCaseRegistration, shardCaseId, shardFile, trialShardKey } from './paid-cases'; import { type ManifestEntry, PAID_TEST_DURATIONS_FILE, type PaidRunManifest, type SliceResult, collectorOutcomeCounts, formatProfileCoverage, loadPaidTestDurations, mergePaidTestDurations, parseRunManifest, trialPanelKey, verifySliceResults, writePaidTestDurations } from './paid-plan'; import { DEFAULT_JOBS, type PaidCaseSelection, type PaidTier, ROOT, type ShardTrialRecord } from './paid-types'; @@ -170,6 +171,8 @@ export interface JUnitShardContext { selection?: PaidCaseSelection; /** Ids this file shard leaves to their isolated trial shards. */ excludeCases?: readonly string[]; + /** The run's agent host (manifest.host); absent means claude. */ + host?: EvalHost; } export const DESELECTION_REASONS = { @@ -205,6 +208,8 @@ export function attributeJUnitCase(tc: JUnitCase, ctx: JUnitShardContext, const tier = tiers[id]; if (tier !== undefined && tier !== ctx.tier) return { kind: 'deselected', reason: `${tier}-tier case (this lane runs ${ctx.tier})` }; if (ctx.excludeCases?.includes(id)) return { kind: 'deselected', reason: DESELECTION_REASONS.isolated }; + const hostReason = hostSkipReason(id, ctx.host ?? 'claude'); + if (hostReason) return { kind: 'deselected', reason: hostReason }; const pool = tier !== undefined ? ctx.selection?.e2e : Object.hasOwn(LLM_JUDGE_TOUCHFILES, id) ? ctx.selection?.judges : null; if (pool && !pool.includes(id)) return { kind: 'deselected', reason: DESELECTION_REASONS.selection }; return { kind: 'case', id }; @@ -241,7 +246,7 @@ export const CENSUS_RED_GUIDE = 'docs/evals/census-red.md'; export function sliceArtifactName(root: string, slice: number, attempt: number): string { const name = path.basename(root); - return /^(?:paid-slice|gate-census)-\d+-a\d+$/.test(name) ? name : `slice ${slice} attempt ${attempt}`; + return /^(?:paid-slice|gate-census)-\d+-a\d+(?:-[a-z0-9-]+)?$/.test(name) ? name : `slice ${slice} attempt ${attempt}`; } export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact[], recordsByShard: Map, @@ -254,7 +259,7 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact if (outcome.trial || outcome.files.length !== 1) continue; let xml = ''; try { xml = fs.readFileSync(path.join(root, 'shards', shardSlug([key]), 'junit.xml'), 'utf8'); } catch { continue; } - const ctx: JUnitShardContext = { key, tier: manifest.tier, selection: manifest.selection, excludeCases: entries.get(key)?.excludeCases }; + const ctx: JUnitShardContext = { key, tier: manifest.tier, selection: manifest.selection, excludeCases: entries.get(key)?.excludeCases, host: manifest.host }; const byCase = new Map(); const reasons = new Map(); for (const tc of parseJUnitCases(xml)) { @@ -530,7 +535,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo const results = artifacts.filter(a => (a.result.attempt ?? 1) === primary).map(a => a.result); const verdict = verifySliceResults(manifest, results); const planned = manifest.entries.filter((e) => e.status === 'planned').length; - const lane = `${manifest.tier}/${manifest.profile ?? 'full'}${manifest.evalsAll ? ' census' : ''}`; + const lane = `${manifest.tier}/${manifest.profile ?? 'full'}${manifest.evalsAll ? ' census' : ''}${manifest.host ? ` on ${manifest.host}` : ''}`; console.log(`[test:paid] report: ${results.length}/${manifest.sliceCount} slices, ${planned} planned shards, tier=${manifest.tier}, attempt ${primary}${attempts.length > 1 ? ` (later attempts ${attempts.slice(1).join(', ')} reported, never replacing it)` : ''}`); for (const line of formatProfileCoverage(manifest)) console.log(line); for (const result of [...results].sort((a, b) => a.sliceIndex - b.sliceIndex)) { diff --git a/scripts/test-paid-shards.ts b/scripts/test-paid-shards.ts index 8ddee5f11..19c9b2376 100644 --- a/scripts/test-paid-shards.ts +++ b/scripts/test-paid-shards.ts @@ -73,6 +73,7 @@ import { readSessionLedger, type SessionLedgerRow } from '../test/helpers/sessio import { preflightAnthropicApi } from '../test/helpers/anthropic-preflight'; import { e2eReuseLaneProblem, prepareE2EShardReuse, selectPlanReceipts } from './e2e-shard-reuse'; import { E2E_TOUCHFILES, E2E_TIERS } from '../test/helpers/touchfiles'; +import { resolveEvalHost } from '../test/helpers/e2e-hosts'; import { scopeCodexAccess, shardFile, shardCaseId, shardTrial, trialShardKey, type CaseTrialPlan, caseTrialPlan, excludedCasesNamePattern, caseTestNamePattern, expandCaseShards, expandTrialShards, fileCaseRegistration, partitionCaseExclusions } from './lib/paid-cases'; import { retriesForFiles, trialPanelKey, sliceExecutionOrder, buildRunManifest, parseRunManifest, type SliceResult, sliceExitCode, guardTrialRecords, formatSlicePlan, formatCapacityPreflight, sliceDeadlineMs } from './lib/paid-plan'; import { caseSelection, runCaseDiagnosis, formatPanelLine, runPaidReport } from './lib/paid-report'; @@ -909,6 +910,9 @@ async function main(): Promise { if (manifest.tier !== options.tier) { throw new Error(`manifest tier ${manifest.tier} != requested tier ${options.tier} — refusing a cross-tier run`); } + if ((manifest.host ?? 'claude') !== resolveEvalHost()) { + throw new Error(`manifest host ${manifest.host ?? 'claude'} != EVALS_HOST ${resolveEvalHost()} — refusing a cross-host run`); + } const profile = manifest.profile ?? 'full'; if (options.profileExplicit && options.profile !== profile) throw new Error(`manifest profile ${profile} != requested profile ${options.profile}`); if (options.sliceIndex > manifest.sliceCount) { diff --git a/test/ci-native-evidence.test.ts b/test/ci-native-evidence.test.ts index 318d5a9f3..c7da89696 100644 --- a/test/ci-native-evidence.test.ts +++ b/test/ci-native-evidence.test.ts @@ -3,6 +3,7 @@ import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { runPaidShard, shardSlug } from '../scripts/test-paid-shards'; +import { CENSUS_ARMS } from '../scripts/census-arms'; const ROOT = path.resolve(import.meta.dir, '..'); const workflows = ['evals.yml', 'evals-periodic.yml', 'evals-marathon.yml'].map(name => ({ @@ -52,19 +53,24 @@ test('every direct CI paid executor binds a safe unique run/attempt/job/slice id else expect(step.run).toContain('--user "$(id -u):$(id -g)"'); const env = executorEnv(job, step); expect(env.EVALS_RUN_ID).toBeString(); + // The gate census matrix is one row per census arm and planned slice (scripts/census-arms.ts). + const armed = job.strategy.matrix.include !== undefined; for (const run of ['36302678692', '36302678693']) { for (const attempt of ['1', '2']) { // The planner sizes the matrix; cover more slices than any live plan. - expect(job.strategy.matrix.slice).toMatch(/^\$\{\{ fromJSON\(needs\.plan-slices\.outputs\.(?:[a-z]+_)*slices\) \}\}$/); - for (let slice = 1; slice <= 64; slice++) { - const id = render(env.EVALS_RUN_ID, { - 'github.run_id': `${run}${workflowIndex}`, - 'github.run_attempt': attempt, 'matrix.slice': String(slice), - }); - expect(id).toMatch(/^[A-Za-z0-9_-]+$/); - expect(id.length).toBeLessThan(120); - expect(ids.has(id)).toBe(false); - ids.add(id); + expect(armed ? job.strategy.matrix.include : job.strategy.matrix.slice) + .toMatch(armed ? /^\$\{\{ fromJSON\(needs\.plan-slices\.outputs\.gate_matrix\) \}\}$/ : /^\$\{\{ fromJSON\(needs\.plan-slices\.outputs\.(?:[a-z]+_)*slices\) \}\}$/); + for (const arm of armed ? Object.keys(CENSUS_ARMS) : ['']) { + for (let slice = 1; slice <= 64; slice++) { + const id = render(env.EVALS_RUN_ID, { + 'github.run_id': `${run}${workflowIndex}`, + 'github.run_attempt': attempt, 'matrix.slice': String(slice), 'matrix.arm': arm, + }); + expect(id).toMatch(/^[A-Za-z0-9_-]+$/); + expect(id.length).toBeLessThan(120); + expect(ids.has(id)).toBe(false); + ids.add(id); + } } } } @@ -79,7 +85,7 @@ for (const { name, jobName, job, step } of executors) { fs.mkdirSync(home); fs.mkdirSync(bin); fs.mkdirSync(path.join(root, 'test')); const configured = executorEnv(job, step); const runId = render(configured.EVALS_RUN_ID, { - 'github.run_id': '36302678692', 'github.run_attempt': '2', 'matrix.slice': '4', + 'github.run_id': '36302678692', 'github.run_attempt': '2', 'matrix.slice': '4', 'matrix.arm': 'codex-sol', }); const evalDir = path.join(root, path.basename(configured.GSTACK_EVAL_DIR)); const file = 'test/native-launch.test.ts'; diff --git a/test/ci-paid-coordination.test.ts b/test/ci-paid-coordination.test.ts index b92375e11..f248d76a9 100644 --- a/test/ci-paid-coordination.test.ts +++ b/test/ci-paid-coordination.test.ts @@ -113,7 +113,8 @@ describe('paid CI coordination stays off the eval image', () => { // A confirmed verdict reuse (R2a/R2) is the one PR run that executes no slice. expect(executor.if).toBe("${{ !cancelled() && needs.build-image.result == 'success' && needs.plan-slices.result == 'success' && needs.plan-slices.outputs.gate_reuse != 'true' }}"); } else { - expect(executor.if).toBeUndefined(); + // A census-only run (monthly cron or census_arms dispatch) executes no periodic slice. + expect(executor.if).toBe("${{ needs.plan-slices.outputs.periodic_lane == 'true' }}"); } expect(executor.steps.some(step => step.run === 'bun run build')).toBe(true); expect(executor.steps.some(step => step.uses === './.github/actions/restore-deps')).toBe(true); diff --git a/test/codex-host.test.ts b/test/codex-host.test.ts new file mode 100644 index 000000000..0976a72e6 --- /dev/null +++ b/test/codex-host.test.ts @@ -0,0 +1,287 @@ +/** + * The Codex host for paid gate cases (GSTA-49 TM2): host selection with + * skip-with-reason (never a failure), per-case host metadata over every gate + * case, real Codex cost accounting against captured `codex exec --json` + * usage events (Codex CLI 0.160.0, gpt-6.1-sol), and runner parity. + */ +import { afterEach, describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { CLAUDE_ONLY_REASONS, E2E_HOSTS, hostSkipReason, resolveEvalHost } from './helpers/e2e-hosts'; +import { E2E_TIERS, E2E_TOUCHFILES } from './helpers/touchfiles'; +import { addCodexTurnUsage, codexUsageCostUsd, NO_CODEX_USAGE, parseCodexJSONL, type CodexResult } from './helpers/codex-session-runner'; +import { runRecordedCodexEval } from './helpers/codex-eval'; +import { codexHostExecArgs, codexHostModel, codexHostToolCalls, codexHostTranscript, unwrapShellCommand } from './helpers/codex-host-runner'; +import { coverageAuditReadEvidence } from './helpers/coverage-audit-evidence'; +import { runSkillTest } from './helpers/session-runner'; +import { readSessionLedger } from './helpers/session-ledger'; +import { preflightAnthropicApi } from './helpers/anthropic-preflight'; +import type { EvalTestEntry } from './helpers/eval-store'; +import { buildRunManifest } from '../scripts/test-paid-shards'; +import { codexShardAccess, fileCaseRegistration, partitionHostExclusions, scopeCodexAccess, shardCaseId, shardFile } from '../scripts/lib/paid-cases'; +import { parseRunManifest } from '../scripts/lib/paid-plan'; +import { attributeJUnitCase } from '../scripts/lib/paid-report'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const fixture = (name: string) => fs.readFileSync(path.join(ROOT, 'test/fixtures/codex-host', name), 'utf8').split('\n').filter(Boolean); +const COMMANDS = fixture('exec-commands.jsonl'); +const FILE_CHANGE = fixture('exec-file-change.jsonl'); +const gateCases = Object.keys(E2E_TIERS).filter(id => E2E_TIERS[id] === 'gate').sort(); +const adapterCases = Object.keys(E2E_HOSTS).filter(id => E2E_HOSTS[id] === 'both'); +const scratch: string[] = []; +const tempDir = (prefix: string) => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); scratch.push(dir); return dir; }; +afterEach(() => { for (const dir of scratch.splice(0)) fs.rmSync(dir, { recursive: true, force: true }); }); + +describe('host selection: Claude-only cases skip with a reason, never fail', () => { + test('EVALS_HOST defaults to claude, accepts codex and refuses anything else', () => { + expect(resolveEvalHost({})).toBe('claude'); + expect(resolveEvalHost({ EVALS_HOST: ' ' })).toBe('claude'); + expect(resolveEvalHost({ EVALS_HOST: 'codex' })).toBe('codex'); + expect(() => resolveEvalHost({ EVALS_HOST: 'gemini' })).toThrow('EVALS_HOST must be one of claude, codex'); + }); + + test('the Anthropic preflight never pings claude on the Codex host', () => { + const pinged: string[] = []; + const spawn = ((cmd: string) => { pinged.push(cmd); return { status: 127, stdout: '', stderr: '' }; }) as unknown as typeof spawnSync; + expect(preflightAnthropicApi({ EVALS_HOST: 'codex' }, spawn)).toBe('skipped'); + expect(pinged).toEqual([]); + expect(() => preflightAnthropicApi({}, spawn)).toThrow('`claude` not found on PATH'); + expect(pinged).toEqual(['sh']); + }); + + test('every case runs on Claude; on Codex only adapter cases run and the rest carry their reason', () => { + for (const id of gateCases) expect(hostSkipReason(id, 'claude')).toBeNull(); + for (const id of adapterCases) expect(hostSkipReason(id, 'codex')).toBeNull(); + expect(hostSkipReason('plan-mode-no-op', 'codex')).toBe(`Claude-only on the codex host: ${CLAUDE_ONLY_REASONS.pty}`); + expect(hostSkipReason('ship-docsync-store', 'codex')).toContain('not ported to the Codex host'); + expect(hostSkipReason('plan-ceo-review', 'codex')).toContain('no Codex host metadata'); + }); + + test('the module selection drops Claude-only cases on the Codex host', () => { + const root = tempDir('codex-host-root-'); + fs.mkdirSync(path.join(root, 'home', '.codex'), { recursive: true }); + const run = spawnSync('bun', ['-e', "const h = await import('./test/helpers/e2e-helpers'); console.log(JSON.stringify(h.selectedTests))"], { + cwd: ROOT, encoding: 'utf8', timeout: 60_000, + env: { ...process.env, EVALS: '1', EVALS_TIER: 'gate', EVALS_HOST: 'codex', EVALS_PREFLIGHT_OK: '1', GSTACK_CODEX_HOST_ROOT: root, + EVALS_SELECTION_JSON: JSON.stringify({ selected: null, reason: 'test' }) }, + }); + expect(run.status, run.stderr).toBe(0); + expect((JSON.parse(run.stdout.trim().split('\n').at(-1)!) as string[]).sort()).toEqual([...adapterCases].sort()); + expect(run.stderr).toContain(`EVALS_HOST=codex: ${adapterCases.length} tests; ${gateCases.length - adapterCases.length} Claude-only skipped`); + }); + + test('the planner excludes Claude-only shards with their reason and plans every adapter case', () => { + const manifest = buildRunManifest({ tier: 'gate', profile: 'full', sliceBudgetMs: 420_000, jobs: 2, evalsAll: true, skipJudges: true, + env: { EVALS_ALL: '1', EVALS_HOST: 'codex' }, rootDir: ROOT }); + expect(manifest.host).toBe('codex'); + const planned = manifest.entries.filter(entry => entry.status === 'planned'); + const casesOf = (key: string) => { + const id = shardCaseId(key); + if (id !== null) return [id]; + return fileCaseRegistration(shardFile(key), fs.readFileSync(path.join(ROOT, shardFile(key)), 'utf8')).registered.filter(c => E2E_TIERS[c] === 'gate'); + }; + const reachable = new Set(planned.flatMap(entry => casesOf(entry.file)).filter(id => E2E_HOSTS[id] === 'both')); + expect([...reachable].sort()).toEqual([...adapterCases].sort()); + for (const entry of planned) expect(casesOf(entry.file).some(id => E2E_HOSTS[id] === 'both'), entry.file).toBe(true); + const hostExcluded = manifest.entries.filter(entry => entry.status === 'excluded' && entry.reason?.includes('Claude-only on the codex host')); + expect(hostExcluded.length).toBeGreaterThan(0); + for (const entry of hostExcluded) expect(entry.reason, entry.file).toStartWith('skipped: '); + expect(manifest.entries.some(entry => entry.file === 'test/skill-e2e-ship-docsync.test.ts#ship-docsync-store' && entry.status === 'excluded')).toBe(true); + }); + + test('the Claude host plans exactly as before, with no host on the manifest', () => { + const manifest = buildRunManifest({ tier: 'gate', profile: 'full', sliceBudgetMs: 420_000, jobs: 2, evalsAll: true, skipJudges: true, + env: { EVALS_ALL: '1' }, rootDir: ROOT }); + expect(manifest.host).toBeUndefined(); + expect(manifest.entries.some(entry => entry.reason?.includes('Claude-only'))).toBe(false); + expect(partitionHostExclusions(['a.test.ts', 'b.test.ts#x'], 'gate', 'claude')).toEqual({ runnable: ['a.test.ts', 'b.test.ts#x'], excluded: [] }); + }); + + test('the report deselects a skipped Claude-only testcase on the Codex host (zero credit, never SKIPPED or failed)', () => { + const tc = { name: 'review-dashboard-via', classname: '', outcome: 'skipped' as const, timeMs: 0, line: 10 }; + const key = 'test/skill-e2e-review-attribution.test.ts'; + expect(attributeJUnitCase(tc, { key, tier: 'gate', host: 'codex' })).toEqual({ kind: 'deselected', reason: hostSkipReason('review-dashboard-via', 'codex')! }); + expect(attributeJUnitCase(tc, { key, tier: 'gate' })).toEqual({ kind: 'case', id: 'review-dashboard-via' }); + expect(attributeJUnitCase({ ...tc, name: 'review-base-branch' }, { key, tier: 'gate', host: 'codex' })).toEqual({ kind: 'case', id: 'review-base-branch' }); + }); + + test('a manifest names only a non-default host, and the executor refuses a cross-host run', () => { + const base = { version: 1, tier: 'gate', evalsAll: true, sliceCount: 1, selectionReason: 'x', entries: [] }; + expect(parseRunManifest(JSON.stringify({ ...base, host: 'codex' })).host).toBe('codex'); + expect(() => parseRunManifest(JSON.stringify({ ...base, host: 'claude' }))).toThrow('manifest host invalid'); + expect(() => parseRunManifest(JSON.stringify({ ...base, host: 'gemini' }))).toThrow('manifest host invalid'); + const plan = path.join(tempDir('codex-host-plan-'), 'manifest.json'); + fs.writeFileSync(plan, JSON.stringify({ ...base, host: 'codex' })); + const run = spawnSync('bun', ['run', 'scripts/test-paid-shards.ts', '--tier', 'gate', '--plan', plan, '--slice', '1', '--list'], { + cwd: ROOT, encoding: 'utf8', timeout: 60_000, env: { ...process.env, EVALS_HOST: 'claude', EVALS_TIER: 'gate' }, + }); + expect(run.status).not.toBe(0); + expect(run.stderr + run.stdout).toContain('manifest host codex != EVALS_HOST claude'); + }); +}); + +describe('per-case host metadata', () => { + test('covers every gate case exactly once, and nothing else', () => { + expect(Object.keys(E2E_HOSTS).sort()).toEqual(gateCases); + for (const id of Object.keys(E2E_HOSTS)) expect(Object.hasOwn(E2E_TOUCHFILES, id), id).toBe(true); + }); + + test('classifies 52 adapter-only, 29 Claude-specific, 26 rework and 1 host-neutral case', () => { + const counts: Record = {}; + for (const support of Object.values(E2E_HOSTS)) counts[support] = (counts[support] ?? 0) + 1; + expect(counts.both).toBe(52); + expect(counts.rework).toBe(26); + expect(counts['host-neutral']).toBe(1); + expect(['pty', 'sdk', 'subagents', 'skill-tool', 'hermetic', 'self-host'].reduce((sum, reason) => sum + (counts[reason] ?? 0), 0)).toBe(29); + }); +}); + +describe('Codex cost accounting from captured usage events', () => { + test('parses every token class of turn.completed (cache reads and writes are subsets of input)', () => { + const commands = parseCodexJSONL(COMMANDS); + expect(commands.usage).toEqual({ input: 25009, cached_input: 12437, cache_write_input: 12566, output: 86, reasoning_output: 0 }); + expect(commands.tokens).toBe(25095); + expect(parseCodexJSONL(FILE_CHANGE).usage).toEqual({ input: 25020, cached_input: 12440, cache_write_input: 12574, output: 137, reasoning_output: 0 }); + expect(parseCodexJSONL(['{"type":"thread.started","thread_id":"t"}']).usage).toBeNull(); + const twoTurns = [COMMANDS, FILE_CHANGE].map(lines => JSON.parse(lines.find(line => line.includes('turn.completed'))!).usage) + .reduce(addCodexTurnUsage, NO_CODEX_USAGE); + expect(twoTurns).toEqual({ input: 50029, cached_input: 24877, cache_write_input: 25140, output: 223, reasoning_output: 0 }); + }); + + test('prices with lib/pricing.ts: uncached input (cache writes included) at input, cache reads at the cached rate', () => { + const usage = parseCodexJSONL(COMMANDS).usage!; + // gpt-6.1-sol: 12,572 x $2 + 12,437 x $0.10 + 86 x $10 per million. + expect(codexUsageCostUsd(usage, 'gpt-6.1-sol')).toBe(0.027248); + expect(codexUsageCostUsd(parseCodexJSONL(FILE_CHANGE).usage!, 'gpt-6.1-sol')).toBe(0.027774); + // gpt-6-astra: 12,572 x $10 + 12,437 x $1 + 86 x $50 per million. + expect(codexUsageCostUsd(usage, 'gpt-6-astra')).toBe(0.142457); + expect(codexUsageCostUsd(usage, 'gpt-0-unpriced')).toBeNull(); + }); + + test('runRecordedCodexEval records real cost and billing instead of cost_known: false', async () => { + const evalDir = tempDir('codex-host-ledger-'); + const previous = process.env.GSTACK_EVAL_DIR; + process.env.GSTACK_EVAL_DIR = evalDir; + try { + const recorded: EvalTestEntry[] = []; + const result: CodexResult = { ...parseCodexJSONL(COMMANDS), exitCode: 0, durationMs: 5, rawLines: COMMANDS, stderr: '', model: 'gpt-6.1-sol' }; + await runRecordedCodexEval({ name: 'cost-case', suite: 's', budgetMs: 10_000, run: async () => result, validate: () => {}, record: entry => recorded.push(entry) }); + expect(recorded[0]).toMatchObject({ passed: true, cost_usd: 0.027248, model: 'gpt-6.1-sol', token_usage: result.usage }); + expect(recorded[0]!.cost_known).toBeUndefined(); + const noTurn: CodexResult = { ...parseCodexJSONL([]), exitCode: 0, durationMs: 5, rawLines: [], stderr: '' }; + await runRecordedCodexEval({ name: 'no-turn', suite: 's', budgetMs: 10_000, run: async () => noTurn, validate: () => {}, record: entry => recorded.push(entry) }); + expect(recorded[1]).toMatchObject({ cost_usd: 0, cost_known: false }); + const rows = readSessionLedger(evalDir); + expect(rows.map(row => [row.test_name, row.billed])).toEqual([['cost-case', true], ['no-turn', false]]); + expect(rows[0]).toMatchObject({ runner: 'codex', model: 'gpt-6.1-sol', cost_usd: 0.027248, tokens: result.usage }); + } finally { + if (previous === undefined) delete process.env.GSTACK_EVAL_DIR; else process.env.GSTACK_EVAL_DIR = previous; + } + }); +}); + +describe('Codex runner parity', () => { + test('maps command_execution to Bash (unwrapped, with output) and file_change to Write/Edit', () => { + expect(codexHostToolCalls(COMMANDS)).toEqual([ + { tool: 'Bash', input: { command: "printf 'hi' > hello.txt\nprintf 'hello' > hello.txt\ncat hello.txt" }, output: 'hello' }, + ]); + expect(codexHostToolCalls(FILE_CHANGE)).toEqual([ + { tool: 'Write', input: { file_path: '/tmp/codex-host-fixture/notes.md', kind: 'add' }, output: '' }, + { tool: 'Edit', input: { file_path: '/tmp/codex-host-fixture/notes.md', kind: 'update' }, output: '' }, + ]); + }); + + test('the transcript is the Claude stream-json shape the case observers read', () => { + const output = '`cat hello.txt` output: `hello`'; + const transcript = codexHostTranscript(COMMANDS, '/tmp/codex-host-fixture', output); + expect(transcript[0]).toEqual({ type: 'system', subtype: 'init', cwd: '/tmp/codex-host-fixture', session_id: '01a150b3-ae5a-7932-ba51-4c295c66adf4', tools: [] }); + expect(transcript.slice(1).map(event => [event.type, event.message?.content[0].type ?? event.subtype])).toEqual([ + ['assistant', 'text'], ['assistant', 'tool_use'], ['user', 'tool_result'], ['assistant', 'text'], ['result', 'success']]); + expect(transcript[2].message.content[0]).toEqual({ type: 'tool_use', id: 'item_1', name: 'Bash', input: { command: "printf 'hi' > hello.txt\nprintf 'hello' > hello.txt\ncat hello.txt" } }); + expect(transcript[3].message.content[0]).toEqual({ type: 'tool_result', tool_use_id: 'item_1', content: 'hello', is_error: false }); + expect(transcript.at(-1)).toMatchObject({ type: 'result', result: output }); + const edits = codexHostTranscript(FILE_CHANGE, '/tmp/codex-host-fixture', '').filter(event => event.type === 'assistant' && event.message.content[0].type === 'tool_use'); + expect(edits.map(event => [event.message.content[0].name, event.message.content[0].input.file_path])).toEqual([ + ['Write', '/tmp/codex-host-fixture/notes.md'], ['Edit', '/tmp/codex-host-fixture/notes.md']]); + // A Codex `cat` of a fixture file is a read the coverage-audit evidence credits, as a native Bash read is. + const source = { path: '/w/src/billing.ts', content: 'export const a = 1;\n' }; + const tests = { path: '/w/test/billing.test.ts', content: 'test("a", () => {});\n' }; + const read = (id: string, file: string, out: string) => JSON.stringify({ type: 'item.completed', item: { id, type: 'command_execution', command: `/bin/bash -lc 'cat ${file}'`, aggregated_output: out, exit_code: 0, status: 'completed' } }); + const lines = ['{"type":"thread.started","thread_id":"t1"}', read('item_0', 'src/billing.ts', source.content), read('item_1', 'test/billing.test.ts', tests.content)]; + expect(coverageAuditReadEvidence(codexHostTranscript(lines, '/w', ''), { cwd: '/w', source, tests })).toEqual({ sourceRead: true, testsRead: true }); + const failed = lines.map(line => line.replace('"exit_code":0', '"exit_code":1')); + expect(coverageAuditReadEvidence(codexHostTranscript(failed, '/w', ''), { cwd: '/w', source, tests })).toEqual({ sourceRead: false, testsRead: false }); + }); + + test('unwraps only a single well-formed bash -lc word', () => { + expect(unwrapShellCommand("/bin/bash -lc 'git diff main...HEAD'")).toBe('git diff main...HEAD'); + expect(unwrapShellCommand("/bin/bash -lc 'echo '\"'\"'x'\"'\"''")).toBe("echo 'x'"); + expect(unwrapShellCommand('/bin/bash -lc "echo \\"\\$HOME\\""')).toBe('echo "$HOME"'); + expect(unwrapShellCommand('bash -lc ls')).toBe('ls'); + expect(unwrapShellCommand('git status')).toBe('git status'); + expect(unwrapShellCommand("/bin/bash -lc 'unterminated")).toBe("/bin/bash -lc 'unterminated"); + expect(unwrapShellCommand("/bin/bash -lc 'a' 'b'")).toBe("/bin/bash -lc 'a' 'b'"); + }); + + test('sandbox, effort and model settings', () => { + expect(codexHostExecArgs('gpt-6.1-sol', 'go', {})).toEqual(['exec', '--json', '-s', 'danger-full-access', '--skip-git-repo-check', '--model', 'gpt-6.1-sol', 'go']); + expect(codexHostExecArgs('gpt-6.1-sol', 'go', { GSTACK_CODEX_EFFORT: 'high' })).toContain('model_reasoning_effort="high"'); + expect(codexHostExecArgs('m', 'go', { GSTACK_CODEX_SANDBOX: 'workspace-write' })).toEqual( + ['exec', '--json', '-s', 'workspace-write', '--skip-git-repo-check', '--model', 'm', '-c', 'sandbox_workspace_write.network_access=true', 'go']); + expect(() => codexHostExecArgs('m', 'go', { GSTACK_CODEX_SANDBOX: 'none' })).toThrow('GSTACK_CODEX_SANDBOX'); + expect(() => codexHostExecArgs('m', 'go', { GSTACK_CODEX_EFFORT: 'high" -c x="y' })).toThrow('GSTACK_CODEX_EFFORT'); + expect(codexHostModel('claude-opus-4-7', { GSTACK_CODEX_MODEL: 'gpt-6.1-sol' })).toBe('gpt-6.1-sol'); + expect(codexHostModel('gpt-6-astra', { GSTACK_CODEX_MODEL: 'gpt-6.1-sol' })).toBe('gpt-6-astra'); + expect(codexHostModel(undefined, {})).toBe('gpt-6-astra'); + }); + + test('every shard on the Codex host gets the CI Codex install', () => { + expect(codexShardAccess(['test/skill-e2e-review.test.ts'], 'gate')).toBe('none'); + expect(codexShardAccess(['test/skill-e2e-review.test.ts'], 'gate', 'codex')).toBe('path'); + const env: NodeJS.ProcessEnv = { EVALS_HOST: 'codex', EVALS_TIER: 'gate', PATH: '/usr/bin', GSTACK_CI_CODEX_BIN_DIR: '/opt/codex/bin', GSTACK_CI_CODEX_HOME: '/home/runner/.ci-codex' }; + scopeCodexAccess(env, ['test/skill-e2e-review.test.ts']); + expect(env.PATH).toBe(`/opt/codex/bin${path.delimiter}/usr/bin`); + expect(env.CODEX_HOME).toBe('/home/runner/.ci-codex'); + }); + + test('runSkillTest on EVALS_HOST=codex runs codex exec and returns a priced SkillTestResult', async () => { + const dir = tempDir('codex-host-run-'); + const root = path.join(dir, 'runtime'); + fs.mkdirSync(path.join(root, 'home', '.codex'), { recursive: true }); + const bin = path.join(dir, 'bin'); + fs.mkdirSync(bin); + const argsFile = path.join(dir, 'args.txt'); + fs.writeFileSync(path.join(bin, 'codex'), `#!/bin/sh\nprintf '%s\\n' "$@" > '${argsFile}'\nprintf '%s\\n' "$CODEX_HOME" >> '${argsFile}'\ncat '${path.join(ROOT, 'test/fixtures/codex-host/exec-commands.jsonl')}'\n`, { mode: 0o755 }); + const evalDir = path.join(dir, 'evals'); + const saved = { ...process.env }; + Object.assign(process.env, { EVALS_HOST: 'codex', GSTACK_CODEX_MODEL: 'gpt-6.1-sol', GSTACK_CODEX_HOST_ROOT: root, GSTACK_EVAL_DIR: evalDir, PATH: `${bin}${path.delimiter}${process.env.PATH}` }); + try { + const result = await runSkillTest({ prompt: 'Run the fixture.', workingDirectory: dir, model: 'claude-opus-4-7', timeout: 30_000, testName: 'host-switch' }); + expect(result).toMatchObject({ exitReason: 'success', model: 'gpt-6.1-sol', output: '`cat hello.txt` output: `hello`', browseErrors: [] }); + expect(result.toolCalls.map(call => call.input.command)).toEqual(["printf 'hi' > hello.txt\nprintf 'hello' > hello.txt\ncat hello.txt"]); + expect(result.costEstimate).toMatchObject({ estimatedCost: 0.027248, estimatedTokens: 25095, turnsUsed: 3 }); + expect(result.tokenUsage).toEqual(parseCodexJSONL(COMMANDS).usage!); + const args = fs.readFileSync(argsFile, 'utf8').trim().split('\n'); + expect(args.slice(0, 7)).toEqual(['exec', '--json', '-s', 'danger-full-access', '--skip-git-repo-check', '--model', 'gpt-6.1-sol']); + expect(args.at(-2)).toBe('Run the fixture.'); + expect(args.at(-1)).toBe(path.join(root, 'home', '.codex')); + expect(readSessionLedger(evalDir)).toEqual([expect.objectContaining({ runner: 'codex', test_name: 'host-switch', end: 'completed', billed: true, cost_usd: 0.027248 })]); + // The native lifecycle sees the codex group owner at spawn and settles after its exit. + const seen: string[] = []; + await runSkillTest({ prompt: 'Run the fixture.', workingDirectory: dir, timeout: 30_000, testName: 'host-lifecycle', nativeLifecycle: { + onSpawn: pid => { seen.push(`spawn ${pid > 0}`); }, + onSettled: async ({ deadline, exited }) => { seen.push(`settled ${exited} ${deadline > Date.now()}`); }, + } }); + expect(seen).toEqual(['spawn true', 'settled true true']); + await expect(runSkillTest({ prompt: 'Run the fixture.', workingDirectory: dir, timeout: 30_000, testName: 'host-lifecycle-refused', nativeLifecycle: { + onSpawn: () => { throw new Error('native process is not group owner'); }, + onSettled: async () => {}, + } })).rejects.toThrow('native process is not group owner'); + } finally { + for (const key of Object.keys(process.env)) if (!(key in saved)) delete process.env[key]; + Object.assign(process.env, saved); + } + }); +}); diff --git a/test/evals-workflow-wiring.test.ts b/test/evals-workflow-wiring.test.ts index 6cfc3217a..013fc621c 100644 --- a/test/evals-workflow-wiring.test.ts +++ b/test/evals-workflow-wiring.test.ts @@ -22,7 +22,10 @@ import { describe, test, expect } from 'bun:test'; import * as fs from 'fs'; import * as path from 'path'; +import * as os from 'os'; +import { spawnSync } from 'child_process'; import { buildRunManifest, isCodexShard, isOverlayTestFile, parseCliOptions, resolvePaidShardTimeoutMs, sliceExecutionOrder, sliceSupervisedWallMs, CI_SETUP_ALLOWANCE_MINUTES } from '../scripts/test-paid-shards'; +import { CENSUS_ARMS, MONTHLY_CENSUS_ARMS, MONTHLY_CENSUS_SCHEDULE, WEEKLY_CENSUS_ARMS, WEEKLY_CENSUS_SCHEDULE, censusArmEnv, censusMatrix, selectCensus } from '../scripts/census-arms'; const ROOT = path.join(import.meta.dir, '..'); const read = (rel: string) => fs.readFileSync(path.join(ROOT, rel), 'utf-8'); @@ -40,7 +43,7 @@ function plannerSites(source: string): Array<{ manifest: string; budgetSeconds: type Step = { id?: string; name?: string; run?: string; uses?: string; env?: Record; with?: Record }; type Job = { needs?: string[]; env?: Record; outputs?: Record; 'timeout-minutes': string | number; - strategy?: { 'max-parallel': number; matrix: { slice: string } }; steps: Step[] }; + strategy?: { 'max-parallel': number; matrix: { slice?: string; include?: string } }; steps: Step[] }; /** * An executor's matrix and timeout must come from the planner step that wrote @@ -159,7 +162,6 @@ describe('evals.yml sliced-lane wiring (post-matrix)', () => { describe('evals-periodic.yml sliced-lane wiring', () => { const lanes = [ { source: periodicYml, name: 'evals-periodic.yml', executor: 'eval-slices', prefix: 'periodic_', tier: 'periodic' }, - { source: periodicYml, name: 'evals-periodic.yml', executor: 'gate-census', prefix: 'gate_', tier: 'gate' }, { source: evalsYml, name: 'evals.yml', executor: 'eval-slices', prefix: '', tier: 'gate' }, { source: marathonYml, name: 'evals-marathon.yml', executor: 'eval-slices', prefix: '', tier: 'marathon' }, ] as const; @@ -200,6 +202,64 @@ describe('evals-periodic.yml sliced-lane wiring', () => { }); } + test('evals-periodic.yml:gate-census — one row per census arm and planned slice with its plan\'s ceiling, and every row of a scheduled census starts at once', () => { + const workflow = Bun.YAML.parse(periodicYml) as { env?: Record; jobs: Record }; + const planner = workflow.jobs['plan-slices']!; + const executor = workflow.jobs['gate-census']!; + expect(executor.needs).toContain('plan-slices'); + expect(executor.strategy!.matrix).toEqual({ include: '${{ fromJSON(needs.plan-slices.outputs.gate_matrix) }}' }); + expect(executor['timeout-minutes']).toBe('${{ matrix.timeout }}'); + expect(executor.steps[0]!.run).toBe('echo "GSTACK_SLICE_JOB_STARTED_AT=$(date +%s)" >> "$GITHUB_ENV"'); + expect(planner.outputs!.gate_matrix).toBe('${{ steps.gate-matrix.outputs.gate_matrix }}'); + expect(planner.steps.find(step => step.id === 'gate-matrix')!.run).toBe('bun --no-install run scripts/census-arms.ts --matrix /tmp/gate-census-plan "$ARMS" >> "$GITHUB_OUTPUT"'); + const emit = planner.steps.filter(step => step.run?.includes('--emit-plan "/tmp/gate-census-plan/$arm/manifest.json" ')); + expect(emit).toHaveLength(1); + expect(emit[0]!.run).toContain('mapfile -t arm_env < <(bun --no-install run scripts/census-arms.ts --env "$arm")'); + expect(emit[0]!.run).toContain('env "${arm_env[@]}" EVALS_TIER=gate bun run scripts/test-paid-shards.ts'); + const execute = executor.steps.filter(step => step.run?.includes('--plan ')); + expect(execute).toHaveLength(1); + expect(execute[0]!.run).toContain('--plan /tmp/gate-census-plan/${{ matrix.arm }}/manifest.json --slice ${{ matrix.slice }}'); + // The executor runs with its arm's host and model, and downloads every arm's plan. + expect(executor.steps.find(step => step.name === "Select the census arm's host and model")!.run).toBe('bun --no-install run scripts/census-arms.ts --env "$ARM" >> "$GITHUB_ENV"'); + expect(executor.steps.some(step => step.with?.path === '/tmp/gate-census-plan')).toBe(true); + const cliArgs = (run: string) => { + const command = /\bbun(?: --no-install)? run scripts\/test-paid-shards\.ts (.*)$/m.exec(run)!; + return command[1]!.replace(/\$\{\{\s*matrix\.slice\s*\}\}/g, '1').replace(/\$\{\{\s*matrix\.arm\s*\}\}/g, 'claude-opus').trim().split(/\s+/); + }; + const planned = parseCliOptions(cliArgs(emit[0]!.run!), { ...workflow.env, ...emit[0]!.env, EVALS_ALL: '1', EVALS_PROFILE: 'full' }); + const active = parseCliOptions(cliArgs(execute[0]!.run!), { ...workflow.env, ...executor.env, ...execute[0]!.env, EVALS_PROFILE: 'full' }); + expect(planned.tier).toBe('gate'); + expect(active.tier).toBe('gate'); + expect(active.jobs).toBe(planned.jobs); + expect(execute[0]!.env?.EVALS_JOBS).toBe(String(planned.jobs)); + const planDir = fs.mkdtempSync(path.join(os.tmpdir(), 'census-arms-')); + try { + for (const arm of Object.keys(CENSUS_ARMS)) { + const env = { ...workflow.env, ...emit[0]!.env, EVALS_ALL: '1', EVALS_PROFILE: 'full', ...censusArmEnv(arm) }; + const manifest = buildRunManifest({ tier: 'gate', profile: 'full', sliceBudgetMs: planned.sliceBudgetMs!, jobs: planned.jobs, + evalsAll: true, env, rootDir: ROOT, skipJudges: planned.skipJudges }); + expect(manifest.host ?? 'claude').toBe(CENSUS_ARMS[arm]!.host); + const sliceFiles = Array.from({ length: manifest.sliceCount }, (_, i) => sliceExecutionOrder( + manifest.entries.filter(entry => entry.status === 'planned' && entry.slice === i + 1)).map(entry => entry.file)); + const ceilings = sliceFiles.map(files => Math.ceil(Math.max(2 * planned.sliceBudgetMs!, + ...files.map(file => resolvePaidShardTimeoutMs([file])), + files.some(isOverlayTestFile) ? sliceSupervisedWallMs(files, planned.jobs) : 0) / 60_000) + CI_SETUP_ALLOWANCE_MINUTES); + expect(manifest.plan!.sliceCiTimeoutMinutes).toEqual(ceilings); + expect(manifest.plan!.ciTimeoutMinutes).toBeLessThanOrEqual(360); + fs.mkdirSync(path.join(planDir, arm)); + fs.writeFileSync(path.join(planDir, arm, 'manifest.json'), JSON.stringify(manifest)); + expect(censusMatrix([arm], planDir)).toEqual(ceilings.map((timeout, i) => ({ arm, slice: i + 1, timeout }))); + } + for (const arms of [WEEKLY_CENSUS_ARMS, MONTHLY_CENSUS_ARMS]) { + const rows = censusMatrix(arms, planDir); + expect(new Set(rows.map(row => `${row.arm}/${row.slice}`)).size).toBe(rows.length); + expect(rows.length, `${arms.join('+')} rows exceed max-parallel`).toBeLessThanOrEqual(executor.strategy!['max-parallel']); + } + } finally { + fs.rmSync(planDir, { recursive: true, force: true }); + } + }); + test('planner/executor/report tier=periodic agree and plan with the ~7-minute budget (W5c)', () => { expect(periodicYml).toMatch(/EVALS_TIER=periodic bun --no-install run scripts\/test-paid-shards\.ts --tier periodic --emit-plan/); expect(periodicYml).toMatch(/EVALS_TIER=periodic bun run scripts\/test-paid-shards\.ts --tier periodic --plan .* --slice /); @@ -207,7 +267,7 @@ describe('evals-periodic.yml sliced-lane wiring', () => { // Periodic work and the full gate census have distinct immutable plans. expect(plannerSites(periodicYml)).toEqual([ { manifest: '/tmp/paid-plan/manifest.json', budgetSeconds: 420, jobs: 2 }, - { manifest: '/tmp/gate-census-plan/manifest.json', budgetSeconds: 420, jobs: 2 }, + { manifest: '"/tmp/gate-census-plan/$arm/manifest.json"', budgetSeconds: 420, jobs: 2 }, ]); }); }); @@ -228,7 +288,7 @@ describe('evals-periodic.yml host-run Codex slices', () => { expect(matrixStep.run).toContain('} >> "$GITHUB_OUTPUT"'); expect(planner.outputs!.periodic_codex_slices).toBe('${{ steps.periodic-matrix.outputs.codex_slices }}'); expect(codex.needs).toEqual(['build-image', 'plan-slices']); - expect(codex.if).toBe("${{ needs.plan-slices.outputs.periodic_codex_slices != '[]' }}"); + expect(codex.if).toBe("${{ needs.plan-slices.outputs.periodic_lane == 'true' && needs.plan-slices.outputs.periodic_codex_slices != '[]' }}"); expect(codex.strategy!.matrix.slice).toBe('${{ fromJSON(needs.plan-slices.outputs.periodic_codex_slices) }}'); expect(codex['timeout-minutes']).toBe('${{ fromJSON(needs.plan-slices.outputs.periodic_slice_timeouts)[matrix.slice] }}'); expect(codex.steps[0]!.run).toBe('echo "GSTACK_SLICE_JOB_STARTED_AT=$(date +%s)" >> "$GITHUB_ENV"'); @@ -350,7 +410,7 @@ describe('panel verdict surfaces (eval reliability policy)', () => { test('planners size the capacity preflight with their executor cap', () => { for (const [source, executor, manifest] of [[evalsYml, 'eval-slices', '/tmp/paid-plan/manifest.json'], - [periodicYml, 'eval-slices', '/tmp/paid-plan/manifest.json'], [periodicYml, 'gate-census', '/tmp/gate-census-plan/manifest.json']] as const) { + [periodicYml, 'eval-slices', '/tmp/paid-plan/manifest.json'], [periodicYml, 'gate-census', '"/tmp/gate-census-plan/$arm/manifest.json"']] as const) { const jobs = jobsOf(source); const emit = jobs['plan-slices']!.steps.find(step => step.run?.includes(`--emit-plan ${manifest} `))!; const cap = Number(/--max-parallel (\d+)/.exec(emit.run!)?.[1]); @@ -401,6 +461,10 @@ describe('panel verdict surfaces (eval reliability policy)', () => { expect(redispatch.permissions).toEqual({ actions: 'write' }); expect(redispatch.if).toBe("${{ !cancelled() && needs.report.outputs.redispatch == 'true' }}"); expect(redispatch.steps[0]!.run).toContain('-f redispatch_of="$GITHUB_RUN_ID"'); + // A census-only run re-dispatches with its own arms, never as a weekly run. + expect(redispatch.steps[0]!.run).toContain('${CENSUS_ARMS:+-f census_arms="$CENSUS_ARMS"}'); + expect(redispatch.steps[0]!.env?.CENSUS_ARMS).toBe('${{ needs.report.outputs.redispatch_census_arms }}'); + expect(report.outputs!.redispatch_census_arms).toBe("${{ needs.plan-slices.outputs.periodic_lane != 'true' && join(fromJSON(needs.plan-slices.outputs.census_arms), ',') || '' }}"); const classify = report.steps.find(step => step.id === 'verdict')!; expect(classify.run).toContain('.verdict.redispatchEligible == true'); expect(classify.run).toContain('[ -z "$REDISPATCH_OF" ]'); @@ -426,7 +490,13 @@ describe('scheduled paid lanes: concurrency and branch dispatch scope', () => { test('a branch dispatch of evals-periodic runs the periodic lane only unless it opts into the gate census', () => { const wf = parse(periodicYml); expect(wf.on.workflow_dispatch!.inputs!.include_gate_census).toMatchObject({ type: 'boolean', default: false }); - expect(wf.jobs['gate-census']!.if).toBe("${{ github.ref == 'refs/heads/main' || inputs.include_gate_census }}"); + // The planner selects the census arms (scripts/census-arms.ts); no arm, no census job. + expect(wf.jobs['gate-census']!.if).toBe("${{ needs.plan-slices.outputs.gate_matrix != '[]' }}"); + const select = wf.jobs['plan-slices']!.steps.find(step => step.id === 'census')!; + expect(select.env?.INCLUDE_GATE_CENSUS).toBe("${{ github.ref == 'refs/heads/main' || inputs.include_gate_census }}"); + expect(select.run).toBe('bun --no-install run scripts/census-arms.ts --select >> "$GITHUB_OUTPUT"'); + expect(selectCensus({ schedule: '', censusArms: '', includeGateCensus: false })).toEqual({ arms: [], periodicLane: true }); + expect(selectCensus({ schedule: WEEKLY_CENSUS_SCHEDULE, includeGateCensus: true })).toEqual({ arms: [...WEEKLY_CENSUS_ARMS], periodicLane: true }); const report = wf.jobs.report!.steps; const reconcile = report.find(step => step.name === 'Reconcile gate census against the manifest (fail-closed)')!; expect(reconcile.env?.GATE_CENSUS).toBe('${{ needs.gate-census.result }}'); @@ -436,3 +506,80 @@ describe('scheduled paid lanes: concurrency and branch dispatch scope', () => { expect(fail.if).toContain(`!contains(fromJSON('["success","skipped"]'), needs.gate-census.result)`); }); }); + +describe('evals-periodic.yml census arms: Monday two-arm census, monthly Fable/Astra census', () => { + type CensusStep = Step & { if?: string }; + type CensusWf = { on: { schedule: Array<{ cron: string }>; workflow_dispatch: { inputs: Record } }; + concurrency: { group: string }; jobs: Record; steps: CensusStep[] }> }; + const wf = Bun.YAML.parse(periodicYml) as CensusWf; + const planner = wf.jobs['plan-slices']!; + const census = wf.jobs['gate-census']!; + const report = wf.jobs.report!; + + test('each arm is one host and model: the capture default and gpt-6.1-sol weekly, claude-fable-5-1 and gpt-6-astra monthly', () => { + expect(WEEKLY_CENSUS_ARMS).toEqual(['claude-opus', 'codex-sol']); + expect(MONTHLY_CENSUS_ARMS).toEqual(['claude-fable', 'codex-astra']); + // claude-opus pins no model: the Claude runners' capture default (lib/eval-model.ts) applies. + expect(censusArmEnv('claude-opus')).toEqual({ EVALS_HOST: 'claude' }); + expect(censusArmEnv('codex-sol')).toEqual({ EVALS_HOST: 'codex', GSTACK_CODEX_MODEL: 'gpt-6.1-sol' }); + expect(censusArmEnv('claude-fable')).toEqual({ EVALS_HOST: 'claude', EVALS_MODEL: 'claude-fable-5-1' }); + expect(censusArmEnv('codex-astra')).toEqual({ EVALS_HOST: 'codex', GSTACK_CODEX_MODEL: 'gpt-6-astra' }); + expect(() => censusArmEnv('codex-luna')).toThrow('unknown census arm'); + // The report files slice artifacts by arm suffix: no arm id may end another one. + const ids = Object.keys(CENSUS_ARMS); + for (const a of ids) for (const b of ids) if (a !== b) expect(a.endsWith(`-${b}`)).toBe(false); + for (const id of ids) expect(id).toMatch(/^[a-z]+-[a-z]+$/); + }); + + test('the monthly cron runs only the census, under its own concurrency group; the Monday and daily canary schedules are unchanged', () => { + expect(wf.on.schedule.map(entry => entry.cron)).toEqual([WEEKLY_CENSUS_SCHEDULE, '30 7 * * *', MONTHLY_CENSUS_SCHEDULE]); + expect(MONTHLY_CENSUS_SCHEDULE).toBe('0 6 1 * *'); + expect(wf.concurrency.group.endsWith("${{ github.event.schedule == '0 6 1 * *' && '-monthly' || '' }}")).toBe(true); + expect(wf.concurrency.group).toContain("(github.event.schedule == '30 7 * * *' || inputs.schema_canary_only) && '-schema-canary'"); + expect(selectCensus({ schedule: MONTHLY_CENSUS_SCHEDULE, includeGateCensus: true })).toEqual({ arms: [...MONTHLY_CENSUS_ARMS], periodicLane: false }); + const select = planner.steps.find(step => step.id === 'census')!; + expect(select.env).toMatchObject({ SCHEDULE: '${{ github.event.schedule }}', CENSUS_ARMS: '${{ inputs.census_arms }}' }); + expect(planner.outputs).toMatchObject({ periodic_lane: '${{ steps.census.outputs.periodic_lane }}', census_arms: '${{ steps.census.outputs.arms }}' }); + // Without the periodic lane no periodic plan is emitted, executed or reconciled. + for (const step of planner.steps.filter(step => step.run?.includes('/tmp/paid-plan/manifest.json') || step.with?.name === 'paid-plan')) { + expect(step.if, step.name ?? step.with?.name).toBe("steps.census.outputs.periodic_lane == 'true'"); + } + const reconcile = report.steps.find(step => step.id === 'reconcile')!; + expect(reconcile.env?.PERIODIC_LANE).toBe('${{ needs.plan-slices.outputs.periodic_lane }}'); + expect(reconcile.run).toContain('if [ "$PERIODIC_LANE" != "true" ]; then'); + // A census-only green run never closes the weekly tracking issue. + expect(report.steps.find(step => step.name === 'Close the tracking issue on a green run')!.if) + .toContain("needs.plan-slices.outputs.periodic_lane == 'true' && needs.eval-slices.result == 'success'"); + }); + + test('a census_arms dispatch runs only the named arms and refuses unknown ones', () => { + expect(wf.on.workflow_dispatch.inputs.census_arms).toMatchObject({ type: 'string', default: '' }); + expect(selectCensus({ censusArms: 'codex-sol', includeGateCensus: true })).toEqual({ arms: ['codex-sol'], periodicLane: false }); + expect(selectCensus({ censusArms: 'claude-opus, codex-sol,codex-sol', includeGateCensus: false })).toEqual({ arms: ['claude-opus', 'codex-sol'], periodicLane: false }); + expect(() => selectCensus({ censusArms: 'codex-sol,gpt', includeGateCensus: false })).toThrow('unknown census arm(s): gpt'); + const cli = (args: string[], env: Record) => { + const proc = spawnSync(process.execPath, ['--no-install', 'run', 'scripts/census-arms.ts', ...args], + { cwd: ROOT, env: { PATH: process.env.PATH ?? '', ...env }, encoding: 'utf8', timeout: 15_000 }); + expect(proc.status, proc.stderr).toBe(0); + return proc.stdout; + }; + expect(cli(['--select'], { SCHEDULE: MONTHLY_CENSUS_SCHEDULE })).toBe('arms=["claude-fable","codex-astra"]\nperiodic_lane=false\n'); + expect(cli(['--select'], { CENSUS_ARMS: 'codex-sol', INCLUDE_GATE_CENSUS: 'true' })).toBe('arms=["codex-sol"]\nperiodic_lane=false\n'); + expect(cli(['--env', 'codex-sol'], {})).toBe('EVALS_HOST=codex\nGSTACK_CODEX_MODEL=gpt-6.1-sol\n'); + }); + + test('census artifacts and reconciliation are per arm', () => { + const upload = census.steps.find(step => step.name === 'Upload census results')!; + expect(upload.with!.name).toBe('gate-census-${{ matrix.slice }}-a${{ github.run_attempt }}-${{ matrix.arm }}'); + expect(census.steps.find(step => step.name?.startsWith('Run gate census slice'))!.name).toContain('(${{ matrix.arm }})'); + expect(report.steps.some(step => step.with?.pattern === 'gate-census-[0-9]*' && step.with?.path === '/tmp/gate-census-slices')).toBe(true); + const reconcile = report.steps.find(step => step.id === 'gate-reconcile')!; + expect(reconcile.env?.ARMS).toBe('${{ needs.plan-slices.outputs.census_arms }}'); + expect(reconcile.run).toContain('cp "/tmp/gate-census-plan/$arm/manifest.json" "$dir/manifest.json" || status=1'); + expect(reconcile.run).toContain('for slice in /tmp/gate-census-slices/gate-census-*-"$arm"; do'); + expect(reconcile.run).toContain('EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --report "$dir"'); + // Every arm's report counts: one red arm fails the census. + expect(reconcile.run).toContain('if [ "$code" -ne 0 ]; then status=$code; fi'); + expect(reconcile.run).toContain('echo "exit=$status" >> "$GITHUB_OUTPUT"'); + }); +}); diff --git a/test/fixtures/codex-host/exec-commands.jsonl b/test/fixtures/codex-host/exec-commands.jsonl new file mode 100644 index 000000000..94518ca86 --- /dev/null +++ b/test/fixtures/codex-host/exec-commands.jsonl @@ -0,0 +1,7 @@ +{"type":"thread.started","thread_id":"01a150b3-ae5a-7932-ba51-4c295c66adf4"} +{"type":"turn.started"} +{"type":"item.completed","item":{"id":"item_0","type":"agent_message","text":"I’ll write and overwrite `hello.txt` with `printf`, then read it with `cat`.\n"}} +{"type":"item.started","item":{"id":"item_1","type":"command_execution","command":"/bin/bash -lc \"printf 'hi' > hello.txt\nprintf 'hello' > hello.txt\ncat hello.txt\"","aggregated_output":"","exit_code":null,"status":"in_progress"}} +{"type":"item.completed","item":{"id":"item_1","type":"command_execution","command":"/bin/bash -lc \"printf 'hi' > hello.txt\nprintf 'hello' > hello.txt\ncat hello.txt\"","aggregated_output":"hello","exit_code":0,"status":"completed"}} +{"type":"item.completed","item":{"id":"item_2","type":"agent_message","text":"`cat hello.txt` output: `hello`"}} +{"type":"turn.completed","usage":{"input_tokens":25009,"cached_input_tokens":12437,"cache_write_input_tokens":12566,"output_tokens":86,"reasoning_output_tokens":0}} diff --git a/test/fixtures/codex-host/exec-file-change.jsonl b/test/fixtures/codex-host/exec-file-change.jsonl new file mode 100644 index 000000000..41647909f --- /dev/null +++ b/test/fixtures/codex-host/exec-file-change.jsonl @@ -0,0 +1,9 @@ +{"type":"thread.started","thread_id":"01a150b3-be24-7f22-aa81-7eb4f79018e0"} +{"type":"turn.started"} +{"type":"item.completed","item":{"id":"item_0","type":"agent_message","text":"I’ll create `notes.md` with `draft`, then change it to `final` using two patches.\n"}} +{"type":"item.started","item":{"id":"item_1","type":"file_change","changes":[{"path":"/tmp/codex-host-fixture/notes.md","kind":"add"}],"status":"in_progress"}} +{"type":"item.completed","item":{"id":"item_1","type":"file_change","changes":[{"path":"/tmp/codex-host-fixture/notes.md","kind":"add"}],"status":"completed"}} +{"type":"item.started","item":{"id":"item_2","type":"file_change","changes":[{"path":"/tmp/codex-host-fixture/notes.md","kind":"update"}],"status":"in_progress"}} +{"type":"item.completed","item":{"id":"item_2","type":"file_change","changes":[{"path":"/tmp/codex-host-fixture/notes.md","kind":"update"}],"status":"completed"}} +{"type":"item.completed","item":{"id":"item_3","type":"agent_message","text":"Created `notes.md` with `draft`, then changed it to `final` using two `apply_patch` calls."}} +{"type":"turn.completed","usage":{"input_tokens":25020,"cached_input_tokens":12440,"cache_write_input_tokens":12574,"output_tokens":137,"reasoning_output_tokens":0}} diff --git a/test/helpers/anthropic-preflight.ts b/test/helpers/anthropic-preflight.ts index 9ca9f1b76..522090c12 100644 --- a/test/helpers/anthropic-preflight.ts +++ b/test/helpers/anthropic-preflight.ts @@ -11,9 +11,14 @@ * * Lives in its own module (not e2e-helpers) so the runner can import it * without dragging in bun:test. + * + * On the Codex host (EVALS_HOST=codex) Codex is the agent and no session + * spawns `claude`, so the ping is skipped; Codex API failures surface per + * session as error_api. */ import { spawnSync } from 'child_process'; +import { resolveEvalHost } from './e2e-hosts'; export type PreflightResult = 'skipped' | 'ok'; @@ -21,7 +26,7 @@ export function preflightAnthropicApi( env: NodeJS.ProcessEnv = process.env, spawn: typeof spawnSync = spawnSync, ): PreflightResult { - if (env.EVALS_PREFLIGHT_OK === '1') return 'skipped'; + if (env.EVALS_PREFLIGHT_OK === '1' || resolveEvalHost(env) === 'codex') return 'skipped'; const check = spawn( 'sh', ['-c', 'echo "ping" | claude -p --max-turns 1 --output-format stream-json --verbose --dangerously-skip-permissions'], diff --git a/test/helpers/codex-eval.ts b/test/helpers/codex-eval.ts index 1a402da51..02dc35297 100644 --- a/test/helpers/codex-eval.ts +++ b/test/helpers/codex-eval.ts @@ -1,5 +1,5 @@ /** One terminal record per Codex eval attempt, using the assertions as the oracle. */ -import { CODEX_DRAIN_GRACE_MS, CodexHarnessError, type CodexResult } from './codex-session-runner'; +import { CODEX_DRAIN_GRACE_MS, CodexHarnessError, codexUsageCostUsd, type CodexResult } from './codex-session-runner'; import { EvalCollector, getProjectEvalDir, shardSlugOfEvalDir, type EvalTestEntry } from './eval-store'; import * as path from 'node:path'; import * as fs from 'node:fs'; @@ -198,10 +198,13 @@ export async function runRecordedCodexEval(opts: CodexEvalOptions): Promise Bash ({ command } unwrapped from Codex's + * `bash -lc`, its output); file_change -> Write (add), Edit (update) or + * Delete per path; web_search -> WebSearch; mcp_tool_call -> mcp__server__tool; + * the last agent message is the output; + * - transcript: the Claude stream-json shape the cases' observers read: one + * system/init event (cwd, the Codex thread id as session_id), assistant text + * and tool_use events, user tool_result events (is_error from the exit code) + * and a final result event; + * - sandbox: GSTACK_CODEX_SANDBOX, default danger-full-access (the counterpart + * of claude -p --dangerously-skip-permissions; cases write outside their + * working directory, start loopback servers and browsers); workspace-write + * gets network access; + * - effort: GSTACK_CODEX_EFFORT sets model_reasoning_effort; unset keeps the + * model's default; + * - model: a non-Claude options.model, else GSTACK_CODEX_MODEL, else + * CODEX_FRONTIER_MODEL; a case's Claude model pin does not apply here; + * - cost: real accounting from `turn.completed` usage at lib/pricing.ts rates, + * recorded on the result, the eval record and the session ledger. + * - nativeLifecycle: onSpawn gets the codex process-group owner right after + * spawn; onSettled runs after its exit and group kill, within the drain grace. + * Not translated: maxTurns (Codex has no turn cap), allowedTools/tools (Codex + * has its own tool surface) and startupGraceMs (the timeout bounds the session). + */ +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { spawnSync } from 'child_process'; +import { CODEX_DRAIN_GRACE_MS, CodexHarnessError, codexUsageCostUsd, execCodex, type CodexResult } from './codex-session-runner'; +import { getProjectEvalDir } from './eval-store'; +import { hermeticChildEnv } from './hermetic-env'; +import { appendSessionLedger, sessionKey, type SessionEnd } from './session-ledger'; +import { sanitizeTestName, scanBrowseErrors, type SkillTestResult } from './session-runner'; +import { CODEX_FRONTIER_MODEL } from '../../scripts/resolvers/constants'; + +const ROOT = path.resolve(import.meta.dir, '..', '..'); +const SETUP_TIMEOUT_MS = 300_000; +export const CODEX_SANDBOXES = ['read-only', 'workspace-write', 'danger-full-access'] as const; +const EFFORT = /^[a-z]+$/; + +/** The Codex model for a session: a Claude pin never reaches `codex --model`. */ +export function codexHostModel(explicit: string | undefined, env: NodeJS.ProcessEnv = process.env): string { + if (explicit && !/^claude-/.test(explicit)) return explicit; + return env.GSTACK_CODEX_MODEL || CODEX_FRONTIER_MODEL; +} + +/** `codex exec` arguments for one session; the prompt is the last argument. */ +export function codexHostExecArgs(model: string, prompt: string, env: NodeJS.ProcessEnv = process.env): string[] { + const sandbox = env.GSTACK_CODEX_SANDBOX || 'danger-full-access'; + if (!(CODEX_SANDBOXES as readonly string[]).includes(sandbox)) { + throw new Error(`GSTACK_CODEX_SANDBOX must be one of ${CODEX_SANDBOXES.join(', ')}; got "${sandbox}"`); + } + const effort = env.GSTACK_CODEX_EFFORT; + if (effort && !EFFORT.test(effort)) throw new Error(`GSTACK_CODEX_EFFORT must be a bare effort name; got "${effort}"`); + return ['exec', '--json', '-s', sandbox, '--skip-git-repo-check', '--model', model, + ...(sandbox === 'workspace-write' ? ['-c', 'sandbox_workspace_write.network_access=true'] : []), + ...(effort ? ['-c', `model_reasoning_effort="${effort}"`] : []), + prompt]; +} + +/** The command inside Codex's `/bin/bash -lc ` wrapper, else the command unchanged. */ +export function unwrapShellCommand(command: string): string { + const match = /^(?:\/(?:usr\/)?bin\/)?(?:ba|z)?sh -lc (.+)$/s.exec(command); + if (!match) return command; + const word = match[1]!; + let out = ''; + let i = 0; + while (i < word.length) { + const c = word[i]!; + if (c === "'") { + const end = word.indexOf("'", i + 1); + if (end < 0) return command; + out += word.slice(i + 1, end); + i = end + 1; + } else if (c === '"') { + i++; + let closed = false; + while (i < word.length) { + if (word[i] === '\\' && i + 1 < word.length && '"\\$`\n'.includes(word[i + 1]!)) { out += word[i + 1]; i += 2; } + else if (word[i] === '"') { i++; closed = true; break; } + else out += word[i++]; + } + if (!closed) return command; + } else if (c === '\\' && i + 1 < word.length) { + out += word[i + 1]; + i += 2; + } else if (/\s/.test(c)) { + return command; + } else { + out += c; + i++; + } + } + return out; +} + +export type HostToolCall = SkillTestResult['toolCalls'][number]; + +const FILE_CHANGE_TOOLS: Record = { add: 'Write', update: 'Edit', delete: 'Delete' }; + +/** Claude-shaped tool calls from completed Codex items, in order. */ +export function codexHostToolCalls(rawLines: readonly string[]): HostToolCall[] { + const calls: HostToolCall[] = []; + for (const line of rawLines) { + let event: any; + try { event = JSON.parse(line); } catch { continue; } + if (event?.type !== 'item.completed' || !event.item) continue; + const item = event.item; + if (item.type === 'command_execution') { + calls.push({ tool: 'Bash', input: { command: unwrapShellCommand(String(item.command ?? '')) }, output: String(item.aggregated_output ?? '') }); + } else if (item.type === 'file_change') { + for (const change of Array.isArray(item.changes) ? item.changes : []) { + calls.push({ tool: FILE_CHANGE_TOOLS[change?.kind] ?? 'Edit', input: { file_path: String(change?.path ?? ''), kind: change?.kind }, output: '' }); + } + } else if (item.type === 'web_search') { + calls.push({ tool: 'WebSearch', input: { query: item.query }, output: '' }); + } else if (item.type === 'mcp_tool_call') { + calls.push({ tool: `mcp__${item.server}__${item.tool}`, input: item.arguments ?? {}, output: typeof item.result === 'string' ? item.result : JSON.stringify(item.result ?? '') }); + } + } + return calls; +} + +/** + * Codex events as the Claude stream-json transcript the cases' observers read + * (system/init, assistant text and tool_use, user tool_result, result). + */ +export function codexHostTranscript(rawLines: readonly string[], cwd: string, output: string): any[] { + const events: any[] = []; + let session = ''; + const say = (role: 'assistant' | 'user', content: any[]) => events.push({ type: role, session_id: session, parent_tool_use_id: null, message: { role, content } }); + for (const line of rawLines) { + let event: any; + try { event = JSON.parse(line); } catch { continue; } + if (event?.type === 'thread.started') { + session = String(event.thread_id ?? ''); + events.push({ type: 'system', subtype: 'init', cwd, session_id: session, tools: [] }); + continue; + } + if (event?.type !== 'item.completed' || !event.item) continue; + const item = event.item; + const id = String(item.id ?? `item_${events.length}`); + if (item.type === 'agent_message' && item.text) say('assistant', [{ type: 'text', text: String(item.text) }]); + else if (item.type === 'command_execution') { + say('assistant', [{ type: 'tool_use', id, name: 'Bash', input: { command: unwrapShellCommand(String(item.command ?? '')) } }]); + say('user', [{ type: 'tool_result', tool_use_id: id, content: String(item.aggregated_output ?? ''), is_error: item.exit_code !== 0 }]); + } else if (item.type === 'file_change') { + (Array.isArray(item.changes) ? item.changes : []).forEach((change: any, index: number) => { + const useId = `${id}_${index}`; + say('assistant', [{ type: 'tool_use', id: useId, name: FILE_CHANGE_TOOLS[change?.kind] ?? 'Edit', input: { file_path: String(change?.path ?? '') } }]); + say('user', [{ type: 'tool_result', tool_use_id: useId, content: '', is_error: item.status !== 'completed' }]); + }); + } + } + events.push({ type: 'result', subtype: 'success', session_id: session, result: output }); + return events; +} + +let runtimeRoot: string | undefined; + +/** + * Install the full gstack Codex runtime once per process: `./setup --host codex` + * into a private HOME/CODEX_HOME, with the login home's auth.json copied in. + * Returns the root holding home/.codex and tmp/ (Codex will not create its + * helper aliases when CODEX_HOME sits under its own TMPDIR). A caller that + * already prepared one passes it as GSTACK_CODEX_HOST_ROOT. + */ +export function prepareCodexHostRuntime(env: NodeJS.ProcessEnv = process.env): string { + if (runtimeRoot) return runtimeRoot; + const preset = env.GSTACK_CODEX_HOST_ROOT; + if (preset) { + if (!fs.existsSync(path.join(preset, 'home', '.codex'))) throw new Error(`GSTACK_CODEX_HOST_ROOT ${preset} has no home/.codex`); + fs.mkdirSync(path.join(preset, 'tmp'), { recursive: true }); + return runtimeRoot = preset; + } + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'codex-host-')); + const home = path.join(root, 'home'); + const codexHome = path.join(home, '.codex'); + const tmp = path.join(root, 'tmp'); + for (const dir of [codexHome, tmp]) fs.mkdirSync(dir, { recursive: true }); + const auth = path.join(env.CODEX_HOME || path.join(os.homedir(), '.codex'), 'auth.json'); + if (fs.existsSync(auth)) fs.copyFileSync(auth, path.join(codexHome, 'auth.json')); + const setup = spawnSync(path.join(ROOT, 'setup'), ['--host', 'codex'], { + cwd: ROOT, encoding: 'utf8', timeout: SETUP_TIMEOUT_MS, + env: hermeticChildEnv({ HOME: home, CODEX_HOME: codexHome, TMPDIR: tmp }, { extraAllow: ['OPENAI_API_KEY', 'CODEX_*'] }), + }); + if (setup.status !== 0) { + fs.rmSync(root, { recursive: true, force: true }); + throw new Error(`./setup --host codex failed (exit ${setup.status}${setup.error ? `, ${setup.error.message}` : ''}): ${(setup.stderr || setup.stdout || '').slice(-2000)}`); + } + process.once('exit', () => { try { fs.rmSync(root, { recursive: true, force: true }); } catch { /* best effort */ } }); + runtimeRoot = root; + return root; +} + +type RunSkillTestOptions = Parameters[0]; + +function exitReasonOf(result: CodexResult, aborted: boolean): string { + if (result.exitCode === 124) return 'timeout'; + const failedTurn = result.rawLines.some(line => /"type":"(?:turn\.failed|error)"/.test(line)); + if (result.exitCode === 0) return failedTurn ? 'error_api' : 'success'; + if (aborted) return 'timeout'; + return failedTurn && !result.rawLines.some(line => line.includes('"type":"item.completed"')) ? 'error_api' : `exit_code_${result.exitCode}`; +} + +const SESSION_ENDS: Record = { success: 'completed', timeout: 'session_timeout', error_api: 'api_error' }; + +/** runSkillTest on the Codex host. `appendSystemPrompt` arrives with any completion notice already applied. */ +export async function runCodexHostSkillTest(options: RunSkillTestOptions): Promise { + const startTime = Date.now(); + const startMono = performance.now(); + const { workingDirectory, timeout = 120_000, testName, runId, signal } = options; + const deadline = startTime + timeout; + const model = codexHostModel(options.model); + const prompt = options.appendSystemPrompt ? `${options.appendSystemPrompt}\n\n${options.prompt}` : options.prompt; + const root = prepareCodexHostRuntime(); + const safeName = testName ? sanitizeTestName(testName) : null; + let runDir: string | null = null; + if (runId) { + try { + runDir = path.join(path.dirname(getProjectEvalDir()), 'e2e-runs', runId); + fs.mkdirSync(runDir, { recursive: true }); + } catch { runDir = null; } + } + let firstResponseMs = 0; + let lastItemMs = 0; + let maxInterTurnMs = 0; + const onLine = (line: string, elapsedMs: number) => { + if (!firstResponseMs) firstResponseMs = Math.max(1, elapsedMs); + if (runDir && safeName) { try { fs.appendFileSync(path.join(runDir, `${safeName}.ndjson`), line + '\n'); } catch { /* non-fatal */ } } + if (!line.includes('"type":"item.completed"') || !/"type":"(?:command_execution|file_change)"/.test(line)) return; + if (lastItemMs) maxInterTurnMs = Math.max(maxInterTurnMs, elapsedMs - lastItemMs); + lastItemMs = elapsedMs; + }; + + let result: CodexResult; + let harnessReason: string | undefined; + let started = true; + const lifecycle = options.nativeLifecycle; + try { + result = await execCodex(codexHostExecArgs(model, prompt), { + cwd: workingDirectory, startTime, deadline, signal, onLine, ...(lifecycle ? { onSpawn: (pid: number) => lifecycle.onSpawn(pid) } : {}), + env: hermeticChildEnv( + { GSTACK_HEADLESS: '1', ...options.env, CODEX_HOME: path.join(root, 'home', '.codex'), TMPDIR: options.env?.TMPDIR ?? path.join(root, 'tmp') }, + { extraAllow: ['OPENAI_API_KEY', 'CODEX_*'] }, + ), + }); + } catch (error) { + if (!(error instanceof CodexHarnessError) || !error.result) throw error; + result = error.result; + started = !error.message.startsWith('Could not start Codex'); + harnessReason = /drain/.test(error.message) ? 'error_output_drain' : 'error_output_stream'; + } + if (lifecycle) { + const settleDeadline = Date.now() + CODEX_DRAIN_GRACE_MS; + let hookTimer: ReturnType | undefined; + try { + await Promise.race([ + lifecycle.onSettled({ deadline: settleDeadline, exited: started }), + new Promise((_, reject) => { hookTimer = setTimeout(() => reject(new Error('native lifecycle settlement deadline exceeded')), CODEX_DRAIN_GRACE_MS); }), + ]); + } finally { clearTimeout(hookTimer); } + } + + const exitReason = harnessReason ?? exitReasonOf(result, !!signal?.aborted); + const toolCalls = codexHostToolCalls(result.rawLines); + const events = result.rawLines.flatMap(line => { try { return [JSON.parse(line)]; } catch { return []; } }); + const messages = events.filter(event => event?.type === 'item.completed' && event.item?.type === 'agent_message'); + const output = String(messages.at(-1)?.item?.text ?? ''); + const turnsUsed = events.filter(event => event?.type === 'item.completed' && event.item?.type !== 'reasoning').length; + const transcript = codexHostTranscript(result.rawLines, workingDirectory, output); + const cost = result.usage ? codexUsageCostUsd(result.usage, model) : null; + const browseErrors = scanBrowseErrors(toolCalls.filter(call => call.tool === 'Bash').map(call => call.output).join('\n') + '\n' + result.stderr); + const duration = Date.now() - startTime; + + appendSessionLedger({ key: sessionKey('codex', testName), ...(testName ? { test_name: testName } : {}), runner: 'codex', + started_at: new Date(startTime).toISOString(), budget_ms: timeout, elapsed_ms: performance.now() - startMono, + end: exitReason === 'timeout' && signal?.aborted ? 'aborted' : SESSION_ENDS[exitReason] ?? 'error', + ...(exitReason === 'timeout' ? { evidence: `armed ${timeout}ms session timeout fired` } : exitReason !== 'success' && result.stderr ? { evidence: result.stderr } : {}), + billed: cost !== null, model, ...(cost !== null ? { cost_usd: cost, tokens: result.usage! } : {}) }); + + if (exitReason !== 'success' || browseErrors.length > 0) { + try { + const failureDir = runDir || path.join(workingDirectory, '.gstack', 'test-transcripts'); + fs.mkdirSync(failureDir, { recursive: true }); + fs.writeFileSync(path.join(failureDir, safeName ? `${safeName}-failure.json` : `e2e-${new Date().toISOString().replace(/[:.]/g, '-')}.json`), + JSON.stringify({ host: 'codex', model, prompt: prompt.slice(0, 500), testName: testName || 'unknown', exitReason, browseErrors, duration, + stderr: result.stderr.slice(0, 2000), output: output.slice(0, 500) }, null, 2)); + } catch { /* non-fatal */ } + } + + return { + toolCalls, browseErrors, exitReason, duration, output, transcript, model, firstResponseMs, maxInterTurnMs, + costEstimate: { inputChars: prompt.length, outputChars: output.length, + estimatedTokens: (result.usage?.input ?? 0) + (result.usage?.output ?? 0), estimatedCost: cost ?? 0, turnsUsed }, + ...(result.usage ? { tokenUsage: result.usage } : {}), + ...(cost === null ? { costKnown: false } : {}), + }; +} diff --git a/test/helpers/codex-session-runner.ts b/test/helpers/codex-session-runner.ts index 4eea35931..c040ebbfa 100644 --- a/test/helpers/codex-session-runner.ts +++ b/test/helpers/codex-session-runner.ts @@ -21,6 +21,7 @@ import { hermeticChildEnv } from './hermetic-env'; import { extractSkillSections } from './skill-fixture'; import { killProcessGroup } from '../../scripts/test-strict-output'; import { CODEX_FRONTIER_MODEL } from '../../scripts/resolvers/constants'; +import { estimateCostUsd, hasPricing } from '../../lib/pricing'; // --- Interfaces --- @@ -34,6 +35,47 @@ export interface CodexResult { sessionId: string | null; // Thread ID for session continuity rawLines: string[]; // Raw JSONL lines for debugging stderr: string; // Stderr output (skill loading errors, auth failures) + usage?: CodexTokenUsage | null; // Summed turn.completed usage; null when no turn completed + model?: string; // The --model this session ran +} + +/** + * Token classes of Codex `turn.completed` usage. `cached_input` (cache reads) + * and `cache_write_input` are subsets of `input`; `reasoning_output` is a + * subset of `output`. + */ +export interface CodexTokenUsage { + input: number; + cached_input: number; + cache_write_input: number; + output: number; + reasoning_output: number; +} + +export const NO_CODEX_USAGE: CodexTokenUsage = Object.freeze({ input: 0, cached_input: 0, cache_write_input: 0, output: 0, reasoning_output: 0 }); + +const count = (value: unknown) => (typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : 0); + +/** Add one `turn.completed` usage object to a running total. */ +export function addCodexTurnUsage(total: CodexTokenUsage, usage: Record | undefined): CodexTokenUsage { + return { + input: total.input + count(usage?.input_tokens), + cached_input: total.cached_input + count(usage?.cached_input_tokens), + cache_write_input: total.cache_write_input + count(usage?.cache_write_input_tokens), + output: total.output + count(usage?.output_tokens), + reasoning_output: total.reasoning_output + count(usage?.reasoning_output_tokens), + }; +} + +/** + * A session's cost at lib/pricing.ts rates: cache reads at the cached-input + * rate, all other input (cache writes included) at the input rate, output at + * the output rate. Null when the model has no price row. + */ +export function codexUsageCostUsd(usage: CodexTokenUsage, model: string | undefined): number | null { + if (!hasPricing(model)) return null; + const cached = Math.min(usage.cached_input, usage.input); + return estimateCostUsd({ input: usage.input - cached, cached, output: usage.output }, model); } /** Existing pipe-drain allowance, separate from the model's work budget. */ @@ -54,6 +96,7 @@ export interface ParsedCodexJSONL { toolCalls: string[]; tokens: number; sessionId: string | null; + usage: CodexTokenUsage | null; } /** @@ -71,6 +114,7 @@ export function parseCodexJSONL(lines: string[]): ParsedCodexJSONL { const toolCalls: string[] = []; let tokens = 0; let sessionId: string | null = null; + let usage: CodexTokenUsage | null = null; for (const line of lines) { if (!line.trim()) continue; @@ -95,9 +139,9 @@ export function parseCodexJSONL(lines: string[]): ParsedCodexJSONL { if (cmd) toolCalls.push(cmd); } } else if (t === 'turn.completed') { - const usage = obj.usage || {}; - const turnTokens = (usage.input_tokens || 0) + (usage.output_tokens || 0); - tokens += turnTokens; + const turn = obj.usage || {}; + tokens += (turn.input_tokens || 0) + (turn.output_tokens || 0); + usage = addCodexTurnUsage(usage ?? NO_CODEX_USAGE, turn); } } catch { /* skip malformed lines */ } } @@ -108,6 +152,7 @@ export function parseCodexJSONL(lines: string[]): ParsedCodexJSONL { toolCalls, tokens, sessionId, + usage, }; } @@ -166,6 +211,192 @@ export function installSkillToTempHome( return home; } +// --- Spawn core --- + +/** + * Spawn `codex ` in its own process group and collect its JSONL until + * exit, the deadline or an abort, with the existing drain grace. Shared by + * runCodexSkill and the Codex host (codex-host-runner.ts). `onLine` sees each + * JSONL line as it arrives with the elapsed milliseconds since `startTime`. + */ +export async function execCodex(args: string[], opts: { + cwd: string; + env: Record; + startTime: number; + deadline: number; + signal?: AbortSignal; + onLine?: (line: string, elapsedMs: number) => void; + /** Called with the group-owning pid right after spawn; a throw kills the group. */ + onSpawn?: (pid: number) => void; +}): Promise { + const { cwd, env, startTime, deadline, signal, onLine } = opts; + const proc = spawn('codex', args, { + cwd, + stdio: ['ignore', 'pipe', 'pipe'], + detached: process.platform !== 'win32', + env, + }); + if (opts.onSpawn) { + try { + if (proc.pid === undefined) throw new Error('Codex did not start'); + opts.onSpawn(proc.pid); + } catch (error) { + killProcessGroup(proc, 'SIGKILL'); + proc.stdout!.destroy(); + proc.stderr!.destroy(); + throw error; + } + } + const collectedLines: string[] = []; + // work --exit--> drain (<=5s) --> result + // \--timeout/abort--> kill group + close pipes --> timeout result + // Exit status comes from 'exit'; a forced drain after exit 0 is an error. + let stdoutBuffer = ''; + let stderr = ''; + let exitCode: number | undefined; + let timedOut = false; + let drainExpired = false; + let streamError: { stream: 'stdout' | 'stderr'; error: Error } | undefined; + let spawnError: Error | undefined; + let stdoutDone = false; + let stderrDone = false; + let stdoutEnded = false; + let stderrEnded = false; + let finalized = false; + const stdoutDecoder = new StringDecoder('utf8'); + const stderrDecoder = new StringDecoder('utf8'); + let workTimer: ReturnType | undefined; + let drainTimer: ReturnType | undefined; + let finish!: () => void; + const finished = new Promise((resolve) => { finish = resolve; }); + + const maybeFinish = () => { + if (exitCode !== undefined && stdoutDone && stderrDone) finish(); + }; + const closePipes = () => { + // Destroy both streams: canceling stdout alone cannot release an + // inherited stderr pipe held by a descendant in another process group. + proc.stdout!.destroy(); + proc.stderr!.destroy(); + }; + const expireDrain = () => { + drainExpired = !stdoutDone || !stderrDone; + killProcessGroup(proc, 'SIGKILL'); + closePipes(); + // Also bound the rare case where a failed group kill produces no exit. + finish(); + }; + const stopRun = () => { + // A real auth/crash exit that preceded a pipe stall stays a real exit. + if (exitCode === undefined) timedOut = true; + else drainExpired ||= !stdoutDone || !stderrDone; + killProcessGroup(proc, 'SIGKILL'); + closePipes(); + clearTimeout(drainTimer); + drainTimer = setTimeout(expireDrain, CODEX_DRAIN_GRACE_MS); + maybeFinish(); + }; + const onExit = (code: number | null, exitSignal: NodeJS.Signals | null) => { + exitCode = code ?? (exitSignal ? 128 + (os.constants.signals[exitSignal] ?? 0) : 1); + clearTimeout(workTimer); + clearTimeout(drainTimer); + // 'exit' means the child is gone; 'close' also waits for descendant + // pipes. Start the drain allowance at actual exit, never at close. + drainTimer = setTimeout(expireDrain, CODEX_DRAIN_GRACE_MS); + maybeFinish(); + }; + const onSpawnError = (error: Error) => { + if (finalized) return; + spawnError = error; + exitCode = 1; + closePipes(); + finish(); + }; + // Closure releases lifecycle waits, but only 'end' proves all bytes were + // drained. A destroyed pipe can emit 'close' without either EOF or error. + const onStdoutDone = () => { if (!finalized) { stdoutDone = true; maybeFinish(); } }; + const onStderrDone = () => { if (!finalized) { stderrDone = true; maybeFinish(); } }; + const onStdoutEnd = () => { if (!finalized) { stdoutBuffer += stdoutDecoder.end(); stdoutEnded = true; onStdoutDone(); } }; + const onStderrEnd = () => { if (!finalized) { stderr += stderrDecoder.end(); stderrEnded = true; onStderrDone(); } }; + const onStreamError = (stream: 'stdout' | 'stderr', error: Error) => { + if (!finalized) streamError ??= { stream, error }; + }; + const onStdout = (chunk: Buffer) => { + if (finalized) return; + stdoutBuffer += stdoutDecoder.write(chunk); + const lines = stdoutBuffer.split('\n'); + stdoutBuffer = lines.pop() || ''; + for (const line of lines) { + if (!line.trim()) continue; + collectedLines.push(line); + onLine?.(line, Date.now() - startTime); + try { + const event = JSON.parse(line); + if (event.type === 'item.completed' && event.item) { + const item = event.item; + const elapsed = Math.round((Date.now() - startTime) / 1000); + if (item.type === 'command_execution' && item.command) { + process.stderr.write(` [codex ${elapsed}s] ran: ${item.command.slice(0, 100)}\n`); + } else if (item.type === 'agent_message' && item.text) { + process.stderr.write(` [codex ${elapsed}s] message: ${item.text.slice(0, 100)}\n`); + } + } + } catch { /* malformed JSONL is ignored by parseCodexJSONL too */ } + } + }; + const onStderr = (chunk: Buffer) => { if (!finalized) stderr += stderrDecoder.write(chunk); }; + + proc.on('exit', onExit); + proc.on('error', onSpawnError); + proc.stdout!.on('data', onStdout); + proc.stderr!.on('data', onStderr); + proc.stdout!.on('end', onStdoutEnd).on('close', onStdoutDone).on('error', error => onStreamError('stdout', error)); + proc.stderr!.on('end', onStderrEnd).on('close', onStderrDone).on('error', error => onStreamError('stderr', error)); + signal?.addEventListener('abort', stopRun, { once: true }); + workTimer = setTimeout(stopRun, Math.max(0, deadline - Date.now())); + if (signal?.aborted || Date.now() >= deadline) stopRun(); + + try { + await finished; + if (stdoutBuffer.trim()) collectedLines.push(stdoutBuffer); + const parsed = parseCodexJSONL(collectedLines); + const result: CodexResult = { + ...parsed, + exitCode: timedOut ? 124 : exitCode ?? 1, + durationMs: Date.now() - startTime, + rawLines: collectedLines, + stderr, + }; + if (stderr.trim()) process.stderr.write(` [codex stderr] ${stderr.trim().slice(0, 200)}\n`); + if (spawnError) throw new CodexHarnessError(`Could not start Codex: ${spawnError.message}`, result); + const incompleteStreams = [!stdoutEnded && 'stdout', !stderrEnded && 'stderr'].filter(Boolean).join(' and '); + if (result.exitCode === 0 && (drainExpired || streamError || incompleteStreams)) { + throw new CodexHarnessError( + drainExpired ? `Codex output drain exceeded ${CODEX_DRAIN_GRACE_MS}ms after exit 0` + : streamError ? `Codex ${streamError.stream} stream failed: ${streamError.error.message}` + : `Codex ${incompleteStreams} closed before EOF after exit 0`, + result, + ); + } + return result; + } finally { + finalized = true; + clearTimeout(workTimer); + clearTimeout(drainTimer); + signal?.removeEventListener('abort', stopRun); + // Child exit and closed pipes do not prove its descendants exited: + // background tools may redirect both streams. Reap the owned group on + // every result, including ordinary success and genuine process failure. + killProcessGroup(proc, 'SIGKILL'); + closePipes(); + proc.removeListener('exit', onExit); + // Retain the harmless error listener through stream teardown so a late + // OS spawn error cannot turn into an unhandled event after cleanup. + proc.stdout!.removeListener('data', onStdout); + proc.stderr!.removeListener('data', onStderr); + } +} + // --- Main runner --- /** @@ -209,7 +440,7 @@ export async function runCodexSkill(opts: { const emptyResult = (exitCode: number, output = ''): CodexResult => ({ output, reasoning: [], toolCalls: [], tokens: 0, exitCode, - durationMs: Date.now() - startTime, sessionId: null, rawLines: [], stderr: '', + durationMs: Date.now() - startTime, sessionId: null, rawLines: [], stderr: '', usage: null, }); if (signal?.aborted || Date.now() >= deadline) return emptyResult(124); @@ -257,7 +488,8 @@ export async function runCodexSkill(opts: { // exactly that. Empirically verified against codex on this machine. const args = ['exec', '--json', '-s', sandbox, '--skip-git-repo-check']; if (ignoreUserConfig) args.push('--ignore-user-config'); - args.push('--model', model ?? process.env.GSTACK_CODEX_MODEL ?? CODEX_FRONTIER_MODEL); + const sessionModel = model ?? process.env.GSTACK_CODEX_MODEL ?? CODEX_FRONTIER_MODEL; + args.push('--model', sessionModel); for (const override of configOverrides) args.push('-c', override); args.push(prompt); @@ -275,162 +507,17 @@ export async function runCodexSkill(opts: { // dir beside CODEX_HOME instead of the shard TMPDIR that holds tempHome. const tempTmpDir = path.join(tempHome, 'tmp'); fs.mkdirSync(tempTmpDir, { recursive: true }); - const proc = spawn('codex', args, { - cwd: cwd || skillDir, - stdio: ['ignore', 'pipe', 'pipe'], - detached: process.platform !== 'win32', + const result = await execCodex(args, { + cwd: cwd || skillDir, startTime, deadline, signal, env: hermeticChildEnv( { HOME: tempHome, CODEX_HOME: tempCodexDir, TMPDIR: tempTmpDir }, { extraAllow: ['OPENAI_API_KEY', 'CODEX_*'] }, ), + }).catch((error: unknown) => { + if (error instanceof CodexHarnessError && error.result) error.result.model = sessionModel; + throw error; }); - const collectedLines: string[] = []; - // work --exit--> drain (<=5s) --> result - // \--timeout/abort--> kill group + close pipes --> timeout result - // Exit status comes from 'exit'; a forced drain after exit 0 is an error. - let stdoutBuffer = ''; - let stderr = ''; - let exitCode: number | undefined; - let timedOut = false; - let drainExpired = false; - let streamError: { stream: 'stdout' | 'stderr'; error: Error } | undefined; - let spawnError: Error | undefined; - let stdoutDone = false; - let stderrDone = false; - let stdoutEnded = false; - let stderrEnded = false; - let finalized = false; - const stdoutDecoder = new StringDecoder('utf8'); - const stderrDecoder = new StringDecoder('utf8'); - let workTimer: ReturnType | undefined; - let drainTimer: ReturnType | undefined; - let finish!: () => void; - const finished = new Promise((resolve) => { finish = resolve; }); - - const maybeFinish = () => { - if (exitCode !== undefined && stdoutDone && stderrDone) finish(); - }; - const closePipes = () => { - // Destroy both streams: canceling stdout alone cannot release an - // inherited stderr pipe held by a descendant in another process group. - proc.stdout!.destroy(); - proc.stderr!.destroy(); - }; - const expireDrain = () => { - drainExpired = !stdoutDone || !stderrDone; - killProcessGroup(proc, 'SIGKILL'); - closePipes(); - // Also bound the rare case where a failed group kill produces no exit. - finish(); - }; - const stopRun = () => { - // A real auth/crash exit that preceded a pipe stall stays a real exit. - if (exitCode === undefined) timedOut = true; - else drainExpired ||= !stdoutDone || !stderrDone; - killProcessGroup(proc, 'SIGKILL'); - closePipes(); - clearTimeout(drainTimer); - drainTimer = setTimeout(expireDrain, CODEX_DRAIN_GRACE_MS); - maybeFinish(); - }; - const onExit = (code: number | null, exitSignal: NodeJS.Signals | null) => { - exitCode = code ?? (exitSignal ? 128 + (os.constants.signals[exitSignal] ?? 0) : 1); - clearTimeout(workTimer); - clearTimeout(drainTimer); - // 'exit' means the child is gone; 'close' also waits for descendant - // pipes. Start the drain allowance at actual exit, never at close. - drainTimer = setTimeout(expireDrain, CODEX_DRAIN_GRACE_MS); - maybeFinish(); - }; - const onSpawnError = (error: Error) => { - if (finalized) return; - spawnError = error; - exitCode = 1; - closePipes(); - finish(); - }; - // Closure releases lifecycle waits, but only 'end' proves all bytes were - // drained. A destroyed pipe can emit 'close' without either EOF or error. - const onStdoutDone = () => { if (!finalized) { stdoutDone = true; maybeFinish(); } }; - const onStderrDone = () => { if (!finalized) { stderrDone = true; maybeFinish(); } }; - const onStdoutEnd = () => { if (!finalized) { stdoutBuffer += stdoutDecoder.end(); stdoutEnded = true; onStdoutDone(); } }; - const onStderrEnd = () => { if (!finalized) { stderr += stderrDecoder.end(); stderrEnded = true; onStderrDone(); } }; - const onStreamError = (stream: 'stdout' | 'stderr', error: Error) => { - if (!finalized) streamError ??= { stream, error }; - }; - const onStdout = (chunk: Buffer) => { - if (finalized) return; - stdoutBuffer += stdoutDecoder.write(chunk); - const lines = stdoutBuffer.split('\n'); - stdoutBuffer = lines.pop() || ''; - for (const line of lines) { - if (!line.trim()) continue; - collectedLines.push(line); - try { - const event = JSON.parse(line); - if (event.type === 'item.completed' && event.item) { - const item = event.item; - const elapsed = Math.round((Date.now() - startTime) / 1000); - if (item.type === 'command_execution' && item.command) { - process.stderr.write(` [codex ${elapsed}s] ran: ${item.command.slice(0, 100)}\n`); - } else if (item.type === 'agent_message' && item.text) { - process.stderr.write(` [codex ${elapsed}s] message: ${item.text.slice(0, 100)}\n`); - } - } - } catch { /* malformed JSONL is ignored by parseCodexJSONL too */ } - } - }; - const onStderr = (chunk: Buffer) => { if (!finalized) stderr += stderrDecoder.write(chunk); }; - - proc.on('exit', onExit); - proc.on('error', onSpawnError); - proc.stdout!.on('data', onStdout); - proc.stderr!.on('data', onStderr); - proc.stdout!.on('end', onStdoutEnd).on('close', onStdoutDone).on('error', error => onStreamError('stdout', error)); - proc.stderr!.on('end', onStderrEnd).on('close', onStderrDone).on('error', error => onStreamError('stderr', error)); - signal?.addEventListener('abort', stopRun, { once: true }); - workTimer = setTimeout(stopRun, Math.max(0, deadline - Date.now())); - if (signal?.aborted || Date.now() >= deadline) stopRun(); - - try { - await finished; - if (stdoutBuffer.trim()) collectedLines.push(stdoutBuffer); - const parsed = parseCodexJSONL(collectedLines); - const result: CodexResult = { - ...parsed, - exitCode: timedOut ? 124 : exitCode ?? 1, - durationMs: Date.now() - startTime, - rawLines: collectedLines, - stderr, - }; - if (stderr.trim()) process.stderr.write(` [codex stderr] ${stderr.trim().slice(0, 200)}\n`); - if (spawnError) throw new CodexHarnessError(`Could not start Codex: ${spawnError.message}`, result); - const incompleteStreams = [!stdoutEnded && 'stdout', !stderrEnded && 'stderr'].filter(Boolean).join(' and '); - if (result.exitCode === 0 && (drainExpired || streamError || incompleteStreams)) { - throw new CodexHarnessError( - drainExpired ? `Codex output drain exceeded ${CODEX_DRAIN_GRACE_MS}ms after exit 0` - : streamError ? `Codex ${streamError.stream} stream failed: ${streamError.error.message}` - : `Codex ${incompleteStreams} closed before EOF after exit 0`, - result, - ); - } - return result; - } finally { - finalized = true; - clearTimeout(workTimer); - clearTimeout(drainTimer); - signal?.removeEventListener('abort', stopRun); - // Child exit and closed pipes do not prove its descendants exited: - // background tools may redirect both streams. Reap the owned group on - // every result, including ordinary success and genuine process failure. - killProcessGroup(proc, 'SIGKILL'); - closePipes(); - proc.removeListener('exit', onExit); - // Retain the harmless error listener through stream teardown so a late - // OS spawn error cannot turn into an unhandled event after cleanup. - proc.stdout!.removeListener('data', onStdout); - proc.stderr!.removeListener('data', onStderr); - } + return { ...result, model: sessionModel }; } finally { // Clean up temp HOME diff --git a/test/helpers/e2e-helpers.ts b/test/helpers/e2e-helpers.ts index 2d4a2a530..6ee6937e6 100644 --- a/test/helpers/e2e-helpers.ts +++ b/test/helpers/e2e-helpers.ts @@ -16,6 +16,7 @@ import { WorktreeManager } from '../../lib/worktree'; import type { HarvestResult } from '../../lib/worktree'; import { preflightAnthropicApi } from './anthropic-preflight'; import { isHermeticEnabled } from './hermetic-env'; +import { hostSkipReason, resolveEvalHost } from './e2e-hosts'; import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; @@ -133,6 +134,17 @@ if (evalsEnabled && process.env.EVALS_TIER) { process.stderr.write(`EVALS_TIER=${tier}: ${selectedTests.length} tests\n\n`); } +// EVALS_HOST=codex: Claude-only cases are skipped with their reason, never run +// (and never failed); the report deselects them (test/helpers/e2e-hosts.ts). +if (evalsEnabled && resolveEvalHost() !== 'claude') { + const host = resolveEvalHost(); + const candidates = selectedTests ?? Object.keys(E2E_TIERS); + const skipped = candidates.filter(id => hostSkipReason(id, host)); + selectedTests = candidates.filter(id => !hostSkipReason(id, host)); + process.stderr.write(`EVALS_HOST=${host}: ${selectedTests.length} tests; ${skipped.length} Claude-only skipped (${skipped.join(', ') || 'none'})\n\n`); + (await import('./codex-host-runner')).prepareCodexHostRuntime(); +} + export const describeE2E = evalsEnabled ? describe : describe.skip; /** @@ -261,6 +273,8 @@ export function recordE2E( ...(!passed && isPreTurnInfraFailure(result) ? { failure_class: 'infra' as const } : {}), duration_ms: result.duration, cost_usd: result.costEstimate.estimatedCost, + ...(result.costKnown === false ? { cost_known: false } : {}), + ...(result.tokenUsage ? { token_usage: result.tokenUsage } : {}), transcript: result.transcript, output: result.output?.slice(0, 2000), turns_used: result.costEstimate.turnsUsed, diff --git a/test/helpers/e2e-hosts.ts b/test/helpers/e2e-hosts.ts new file mode 100644 index 000000000..622c3756d --- /dev/null +++ b/test/helpers/e2e-hosts.ts @@ -0,0 +1,126 @@ +/** + * Agent host selection for paid E2E cases (GSTA-49 TM2). EVALS_HOST picks the + * agent every runSkillTest session runs on: `claude` (default) or `codex` + * (codex exec, test/helpers/codex-host-runner.ts). E2E_HOSTS is the per-case + * host metadata over every gate case (classification in the GSTA-49 TM2 + * scoping): `both` runs on either host with the shared adapter; every other + * value is the reason the case is Claude-only. On the Codex host a Claude-only + * case is SKIPPED with its reason, planned as excluded and deselected by the + * report with zero credit, never counted as failed. A case without metadata + * (every non-gate case) is Claude-only. + */ + +export type EvalHost = 'claude' | 'codex'; +export const EVAL_HOSTS: readonly EvalHost[] = ['claude', 'codex']; + +export function resolveEvalHost(env: NodeJS.ProcessEnv = process.env): EvalHost { + const raw = (env.EVALS_HOST ?? '').trim(); + if (!raw) return 'claude'; + if ((EVAL_HOSTS as readonly string[]).includes(raw)) return raw as EvalHost; + throw new Error(`EVALS_HOST must be one of ${EVAL_HOSTS.join(', ')}; got "${raw}"`); +} + +export const CLAUDE_ONLY_REASONS = { + pty: 'drives the interactive Claude Code TUI (plan mode, native AskUserQuestion) through the PTY harness', + sdk: 'uses the Claude Agent SDK (canUseTool answers and PreToolUse hooks)', + subagents: 'asserts Claude Agent/Task subagent events', + 'skill-tool': 'depends on the Claude Skill tool load and registration', + hermetic: 'a claude -p hermetic canary; the Codex host needs its own canary twin', + 'self-host': 'the Claude host calling /codex; on the Codex host /codex is the agent itself', + rework: 'parses Claude stream-json tool events or prescribes native Claude tools; not ported to the Codex host', + 'host-neutral': 'runs no model session, so a second host adds nothing', +} as const; + +export type ClaudeOnlyReason = keyof typeof CLAUDE_ONLY_REASONS; +export type HostSupport = 'both' | ClaudeOnlyReason; + +/** Adapter-only cases: final output, files, git state, Bash commands, server logs or browseErrors. */ +const ADAPTER_ONLY: readonly string[] = [ + 'benchmark-workflow', + 'browse-basic', + 'browse-snapshot', + 'canary-workflow', + 'context-recovery-artifacts', + 'context-restore-loads-latest', + 'context-save-writes-file', + 'cso-diff-mode', + 'design-review-detector-shim', + 'design-review-detector-shim-dom', + 'design-review-plugin-handoff', + 'diagram-triplet', + 'document-release', + 'gstack-upgrade-happy-path', + 'land-and-deploy-first-run', + 'land-and-deploy-review-gate', + 'land-and-deploy-workflow', + 'learnings-show', + 'operational-learning', + 'plan-design-review-no-ui-scope', + 'plan-eng-coverage-audit', + 'plan-review-report', + 'plan-tune-inspect', + 'qa-bootstrap', + 'qa-quick', + 'retro-base-branch', + 'review-army-delivery-audit', + 'review-army-json-findings', + 'review-army-migration-safety', + 'review-army-quality-score', + 'review-base-branch', + 'review-coverage-audit', + 'review-enum-completeness', + 'review-sql-injection', + 'review-test-value', + 'safety-pair-agent-block', + 'safety-ship-stale-evidence', + 'setup-deploy-workflow', + 'ship-base-branch', + 'ship-coverage-audit', + 'ship-coverage-value', + 'ship-local-workflow', + 'ship-triage', + 'skillmd-no-local-binary', + 'skillmd-outside-git', + 'skillmd-setup-discovery', + 'test-audit-report-only', + 'tpa-absent-darwin', + 'tpa-absent-linux', + 'tpa-apple-ban', + 'tpa-broken', + 'tpa-present' +]; + +const CLAUDE_ONLY: Readonly> = { + pty: ['autoplan-guard-pty', 'office-hours-auto-mode', 'plan-ceo-finding-floor', 'plan-ceo-review-plan-mode', + 'plan-design-with-ui-scope', 'plan-devex-finding-floor', 'plan-devex-review-plan-mode', 'plan-mode-no-op'], + sdk: ['auq-format-gate', 'investigate-owned-abort', 'investigate-owned-completion', 'investigate-owned-ending-error', + 'shared-libs-review-index-flags', 'shared-libs-review-lifecycle', 'shared-libs-review-path-eligibility', + 'shared-libs-review-prior-coverage', 'shared-libs-review-revalidation', 'ship-local-hook-preservation', + 'ship-managed-hook-refresh', 'ship-skipped-queued-finding', 'ship-unmanaged-hook-consent'], + subagents: ['docsync-spawned', 'review-army-perf-n-plus-one'], + 'skill-tool': ['skillify-approval-reject', 'skillify-happy-path', 'skillify-provenance-refusal'], + hermetic: ['hermetic-canary', 'hermetic-sentinel'], + 'self-host': ['safety-codex-consult-embed'], + rework: ['qa-functional-cli-fix', 'qa-functional-cli-report', 'qa-functional-webhook-fix', 'qa-functional-webhook-report', + 'qa-only-no-fix', 'review-dashboard-via', 'review-exploratory-small-cli', 'review-gate-integrity', + 'shared-libs-read-only', 'shared-libs-unsupported-git', 'ship-docsync-completion', 'ship-docsync-current', + 'ship-docsync-failure', 'ship-docsync-late-result', 'ship-docsync-launch-failure', 'ship-docsync-missing-asset', + 'ship-docsync-missing-marker', 'ship-docsync-recovery', 'ship-docsync-stale-after', 'ship-docsync-stale-before', + 'ship-docsync-store', 'ship-docsync-timeout-unsettled', 'ship-exploratory-late-input', 'ship-exploratory-plan-checks', + 'ship-exploratory-small-cli', 'ship-exploratory-unavailable'], + 'host-neutral': ['timeline-event-flow'], +}; + +export const E2E_HOSTS: Readonly> = Object.freeze(Object.fromEntries([ + ...ADAPTER_ONLY.map(id => [id, 'both'] as const), + ...(Object.entries(CLAUDE_ONLY) as Array<[ClaudeOnlyReason, readonly string[]]>).flatMap(([reason, ids]) => ids.map(id => [id, reason] as const)), +])); + +/** Why a case does not run on `host`, or null when it does. */ +export function hostSkipReason(id: string, host: EvalHost, hosts: Readonly> = E2E_HOSTS): string | null { + if (host === 'claude') return null; + const support = hosts[id]; + if (support === 'both') return null; + if (support === undefined) return `Claude-only on the ${host} host: no Codex host metadata (E2E_HOSTS covers the gate cases)`; + return `Claude-only on the ${host} host: ${CLAUDE_ONLY_REASONS[support]}`; +} diff --git a/test/helpers/eval-store.ts b/test/helpers/eval-store.ts index 609c6d099..ed8721f13 100644 --- a/test/helpers/eval-store.ts +++ b/test/helpers/eval-store.ts @@ -14,6 +14,7 @@ import * as os from 'os'; import { spawnSync } from 'child_process'; import { isManualReviewEntry } from './cookie-workflow-manual-review'; import type { ManualJudgeReview } from './cookie-workflow-manual-review'; +import type { CodexTokenUsage } from './codex-session-runner'; // v2: EvalTestEntry.harvest gains optional {insertions, deletions, net} and // may be explicitly null (arm-benchmark harvest-failure taxonomy). Readers @@ -67,6 +68,8 @@ export interface EvalTestEntry { cost_usd: number; /** False when the harness captured no billing, so cost_usd 0 means unknown. Absent means known. */ cost_known?: boolean; + /** Codex sessions: the token classes behind cost_usd. */ + token_usage?: CodexTokenUsage; /** Absent in older records means executed; reuse is never a new model run. */ execution?: 'executed' | 'reused'; reused_from?: { input_key: string; run_id: string; revision: string; completed_at: string }; diff --git a/test/helpers/free-fixtures-data.ts b/test/helpers/free-fixtures-data.ts index 7c724f1c6..317e26b5d 100644 --- a/test/helpers/free-fixtures-data.ts +++ b/test/helpers/free-fixtures-data.ts @@ -54,6 +54,7 @@ export const FREE_FIXTURES: Readonly> = { 'test/fixtures/claude-agent-payload-2.1.292.json': ['test/autoplan-publication-payload.test.ts', 'test/autoplan-schema-canary-compare.test.ts'], 'test/fixtures/claude-native-journal-roots-2.1.284.json': ['test/autoplan-publication-hook.test.ts', 'test/plan-count-transcript.test.ts'], 'test/fixtures/codex-boundary-commands.json': ['test/codex-boundary-evidence.test.ts'], + 'test/fixtures/codex-host/**': ['test/codex-host.test.ts'], 'test/fixtures/codex-review-sandbox-unavailable-37158847998.json': ['test/outside-voice-evidence.test.ts'], 'test/fixtures/codex-review/**': ['test/codex-hardening.test.ts', 'test/outside-review-severity.test.ts', 'test/outside-voice-evidence.test.ts', 'test/skill-validation.test.ts'], 'test/fixtures/codex-sandbox/**': ['test/codex-eval-recording.test.ts', 'test/codex-model-probe.test.ts', 'test/outside-review-result.test.ts', 'test/outside-voice-invocation.test.ts'], diff --git a/test/helpers/session-ledger.ts b/test/helpers/session-ledger.ts index 6e3c9e837..425323284 100644 --- a/test/helpers/session-ledger.ts +++ b/test/helpers/session-ledger.ts @@ -12,6 +12,7 @@ */ import * as fs from 'fs'; import * as path from 'path'; +import type { CodexTokenUsage } from './codex-session-runner'; export const SESSION_LEDGER_FILE = 'session-ledger.jsonl'; export const SESSION_EVIDENCE_MAX = 200; @@ -48,6 +49,10 @@ export interface SessionLedgerRow { liveness?: LivenessSummary; /** Whether the runner captured this session's billing (a terminal result with total_cost_usd). */ billed?: boolean; + /** Codex sessions: the model, its billed cost and token classes. */ + model?: string; + cost_usd?: number; + tokens?: CodexTokenUsage; } /** One line, control characters stripped, mentions neutralized, capped. */ diff --git a/test/helpers/session-runner.ts b/test/helpers/session-runner.ts index d40eb29e8..cc65a66ad 100644 --- a/test/helpers/session-runner.ts +++ b/test/helpers/session-runner.ts @@ -17,6 +17,8 @@ import { hermeticChildEnv, isHermeticEnabled } from './hermetic-env'; import { killProcessGroup } from '../../scripts/test-strict-output'; import { resolveEvalModel } from '../../lib/eval-model'; import { SessionObserver, appendSessionLedger, sessionKey, type SessionEnd } from './session-ledger'; +import { resolveEvalHost } from './e2e-hosts'; +import type { CodexTokenUsage } from './codex-session-runner'; const GSTACK_DEV_DIR = path.join(os.homedir(), '.gstack-dev'); const HEARTBEAT_PATH = path.join(GSTACK_DEV_DIR, 'e2e-live.json'); // heartbeat stays global @@ -55,6 +57,10 @@ export interface SkillTestResult { firstResponseMs: number; /** Peak latency between consecutive tool calls, in ms */ maxInterTurnMs: number; + /** Codex host: token classes behind costEstimate.estimatedCost. */ + tokenUsage?: CodexTokenUsage; + /** Codex host: false when the session completed no turn or its model has no price row. */ + costKnown?: boolean; } /** Local default startup grace: 90s covers observed API queue latency @@ -76,6 +82,14 @@ const BROWSE_ERROR_PATTERNS = [ /no such file or directory.*\bbrowse(?:\.exe)?(?=$|[\s'":),])/i, ]; +/** First match of each browse failure pattern in tool output plus stderr. */ +export function scanBrowseErrors(text: string): string[] { + return BROWSE_ERROR_PATTERNS.flatMap(pattern => { + const match = text.match(pattern); + return match ? [match[0].slice(0, 200)] : []; + }); +} + // --- Testable NDJSON parser --- export interface ParsedNDJSON { @@ -300,6 +314,7 @@ Setup, CLI startup and API queueing consume this same window; it never resets. Before source Reads, use Bash to run exactly \`date -u +%Y-%m-%dT%H:%M:%SZ\`. After each saved checkpoint, compare the latest evidence capture's printed completedAt with the times above; run that clock read again only when no capture has completed since your last clock read. When remaining time is at most ${reserve / 1000} seconds, prioritize the remaining required completion outputs and verification. No required content or gate may be skipped. If the clock read fails, report timing unavailable; do not invent remaining time or restart the deadline.`; systemPrompt = systemPrompt ? `${systemPrompt}\n\n${notice}` : notice; } + if (resolveEvalHost() === 'codex') return (await import('./codex-host-runner')).runCodexHostSkillTest({ ...options, appendSystemPrompt: systemPrompt, completionReserveMs: undefined }); // Set up per-run log directory if runId is provided let runDir: string | null = null; @@ -661,13 +676,7 @@ Before source Reads, use Bash to run exactly \`date -u +%Y-%m-%dT%H:%M:%SZ\`. Af const { transcript, resultLine, toolCalls } = parsed; const browseErrors: string[] = []; - const allText = toolCalls.filter(call => call.tool === 'Bash').map(call => call.output).join('\n') + '\n' + stderr; - for (const pattern of BROWSE_ERROR_PATTERNS) { - const match = allText.match(pattern); - if (match) { - browseErrors.push(match[0].slice(0, 200)); - } - } + browseErrors.push(...scanBrowseErrors(toolCalls.filter(call => call.tool === 'Bash').map(call => call.output).join('\n') + '\n' + stderr)); // Native Claude uses exit 1 for its structured max-turns result. Preserve // that existing semantic outcome only after a complete, uncancelled drain; diff --git a/test/paid-retry-supervision.test.ts b/test/paid-retry-supervision.test.ts index a132bd53a..95c5fd3b3 100644 --- a/test/paid-retry-supervision.test.ts +++ b/test/paid-retry-supervision.test.ts @@ -217,8 +217,9 @@ test('eval:bg runs the PR and release commands with their declared workers (cap: const cliOptions = (step: { run: string; env?: Record }) => { const prefix = /\bbun(?: --no-install)? run scripts\/test-paid-shards\.ts /.exec(step.run); expect(prefix).not.toBeNull(); - const args = step.run.slice(prefix!.index + prefix![0].length) - .replace(/\$\{\{\s*matrix\.slice\s*\}\}/g, '1').trim().split(/\s+/); + // One command line: a census arm loop continues past it. + const args = step.run.slice(prefix!.index + prefix![0].length).split('\n')[0]! + .replace(/\$\{\{\s*matrix\.slice\s*\}\}/g, '1').replace(/\$\{\{\s*matrix\.arm\s*\}\}/g, 'claude-opus').trim().split(/\s+/); return parseCliOptions(args, step.env ?? {}); }; @@ -242,10 +243,16 @@ test('both gate executors plan the complete census and supervise every planned s expect(planned.skipJudges).toBe(skipJudges); expect(execute[0].env.EVALS_CONCURRENCY).toBe('2'); expect(executor.strategy['fail-fast']).toBe(false); - expect(executor.strategy.matrix.slice).toBe(`\${{ fromJSON(needs.plan-slices.outputs.${prefix}slices) }}`); - // Per-slice ceilings (W2c/ENG-2), or the largest of them as one job cap. - expect([`\${{ fromJSON(needs.plan-slices.outputs.${prefix}slice_timeouts)[matrix.slice] }}`, - `\${{ fromJSON(needs.plan-slices.outputs.${prefix}timeout_minutes) }}`]).toContain(executor['timeout-minutes']); + if (jobName === 'gate-census') { + // One row per census arm and planned slice, each with its arm's planned slice ceiling (scripts/census-arms.ts). + expect(executor.strategy.matrix.include).toBe('${{ fromJSON(needs.plan-slices.outputs.gate_matrix) }}'); + expect(executor['timeout-minutes']).toBe('${{ matrix.timeout }}'); + } else { + expect(executor.strategy.matrix.slice).toBe(`\${{ fromJSON(needs.plan-slices.outputs.${prefix}slices) }}`); + // Per-slice ceilings (W2c/ENG-2), or the largest of them as one job cap. + expect([`\${{ fromJSON(needs.plan-slices.outputs.${prefix}slice_timeouts)[matrix.slice] }}`, + `\${{ fromJSON(needs.plan-slices.outputs.${prefix}timeout_minutes) }}`]).toContain(executor['timeout-minutes']); + } const manifest = buildRunManifest({ tier: 'gate', sliceBudgetMs: planned.sliceBudgetMs!, jobs: planned.jobs, evalsAll: true, env: { EVALS_ALL: '1' }, skipJudges }); const files = [...new Set(manifest.entries.filter(row => row.status === 'planned').map(row => shardFile(row.file)))]; expect(files).toContain('test/skill-e2e-ship-skip.test.ts'); @@ -262,8 +269,8 @@ test('both gate executors plan the complete census and supervise every planned s expect(manifest.sliceCount).toBeLessThanOrEqual(executor.strategy['max-parallel']); if (jobName === 'gate-census') { expect(emit[0].env.EVALS_ALL).toBe('1'); - expect(emit[0].run).toContain('--emit-plan /tmp/gate-census-plan/manifest.json'); - expect(execute[0].run).toContain('--plan /tmp/gate-census-plan/manifest.json --slice ${{ matrix.slice }}'); + expect(emit[0].run).toContain('--emit-plan "/tmp/gate-census-plan/$arm/manifest.json"'); + expect(execute[0].run).toContain('--plan /tmp/gate-census-plan/${{ matrix.arm }}/manifest.json --slice ${{ matrix.slice }}'); expect(executor.steps.some((step: any) => step.with?.name === 'gate-census-plan')).toBe(true); expect(executor.steps.filter((step: any) => step.run?.includes('--emit-plan '))).toHaveLength(0); } @@ -305,8 +312,10 @@ test('gate census requires all seven distinct slice results and its own reconcil const reconcile = report.steps.find((step: any) => step.id === 'gate-reconcile'); expect(report.needs).toContain('gate-census'); expect(reconcile.if).toBe('always()'); - expect(reconcile.run).toContain('EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --report /tmp/gate-census-report'); - expect(reconcile.run).toContain('exit=${PIPESTATUS[0]}'); + // One reconciliation per census arm, each against its own plan and slices. + expect(reconcile.run).toContain('dir="/tmp/gate-census-report/$arm"'); + expect(reconcile.run).toContain('EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --report "$dir"'); + expect(reconcile.run).toContain('code=${PIPESTATUS[0]}'); const failureGuards = report.steps.filter((step: any) => /Upsert tracking|Fail the workflow/.test(step.name ?? '')); expect(failureGuards).toHaveLength(2); for (const step of failureGuards) { diff --git a/test/paid-shards.test.ts b/test/paid-shards.test.ts index 78d3b00d9..08f8629ac 100644 --- a/test/paid-shards.test.ts +++ b/test/paid-shards.test.ts @@ -178,7 +178,7 @@ describe('tier lane skip (B5)', () => { } const workflow = fs.readFileSync(path.join(ROOT, '.github/workflows/evals-periodic.yml'), 'utf8'); expect(workflow.match(/--skip-judges/g)).toHaveLength(1); - expect(workflow).toMatch(/--tier gate --emit-plan \/tmp\/gate-census-plan\/manifest\.json --slice-budget 420 --jobs 2 --skip-judges/); + expect(workflow).toMatch(/--tier gate --emit-plan "\/tmp\/gate-census-plan\/\$arm\/manifest\.json" --slice-budget 420 --jobs 2 --skip-judges/); }); });