diff --git a/.github/workflows/evals-periodic.yml b/.github/workflows/evals-periodic.yml index da9f8dde6..f1951fffb 100644 --- a/.github/workflows/evals-periodic.yml +++ b/.github/workflows/evals-periodic.yml @@ -15,6 +15,10 @@ name: Periodic Evals # diff-billed, so without it the full gate census might never execute # anywhere); the hollow-shard guard (exit 0 + zero executed tests under # EVALS_ALL fails) makes both lanes census-health checks, not just test runs. +# The gate census is a matrix of arms (scripts/census-arms.ts): each arm is a +# host and model with its own plan. Monday runs claude-opus (the capture +# model) and codex-sol (gpt-6.1-sol on the Codex host, adapter cases only); +# the monthly cron runs only the census, with claude-fable and codex-astra. # eval-bg (scripts/eval-bg.ts) finds its dispatch by the nonce in the run name. run-name: ${{ inputs.nonce && format('Periodic Evals [eval-bg {0}]', inputs.nonce) || 'Periodic Evals' }} on: @@ -23,6 +27,9 @@ on: # Daily, for autoplan-schema-canary alone (latest Claude Code vs the /autoplan # guard's payload assumptions); every census job below skips this schedule. - cron: '30 7 * * *' + # Monthly, the census jobs alone with the claude-fable and codex-astra arms + # (the 1st; never combined with a day of week, which cron would OR). + - cron: '0 6 1 * *' workflow_dispatch: inputs: redispatch_of: @@ -30,9 +37,13 @@ on: type: string default: '' include_gate_census: - description: 'Branch dispatch only: also run the weekly gate census (default: periodic lane only; gate cases validate through an evals.yml dispatch with evals_all)' + description: 'Branch dispatch only: also run the weekly gate census arms (default: periodic lane only; gate cases validate through an evals.yml dispatch with evals_all)' type: boolean default: false + census_arms: + description: 'Run only the gate census, with these arms (comma-separated: claude-opus, codex-sol, claude-fable, codex-astra; scripts/census-arms.ts). Empty keeps the periodic lane.' + type: string + default: '' expected_sha: description: 'eval-bg: the revision the dispatcher tested; the run fails fast when the ref has moved' type: string @@ -51,7 +62,7 @@ on: # runs in its own group so it never cancels the run that dispatched it; both # runs are reported. concurrency: - group: evals-periodic-${{ github.ref }}-${{ github.event_name }}${{ inputs.redispatch_of && format('-redispatch-{0}', inputs.redispatch_of) || '' }}${{ (github.event.schedule == '30 7 * * *' || inputs.schema_canary_only) && '-schema-canary' || '' }} + group: evals-periodic-${{ github.ref }}-${{ github.event_name }}${{ inputs.redispatch_of && format('-redispatch-{0}', inputs.redispatch_of) || '' }}${{ (github.event.schedule == '30 7 * * *' || inputs.schema_canary_only) && '-schema-canary' || '' }}${{ github.event.schedule == '0 6 1 * *' && '-monthly' || '' }} cancel-in-progress: true env: @@ -126,8 +137,9 @@ jobs: periodic_slices: ${{ steps.periodic-matrix.outputs.slices }} periodic_codex_slices: ${{ steps.periodic-matrix.outputs.codex_slices }} periodic_slice_timeouts: ${{ steps.periodic-matrix.outputs.slice_timeouts }} - gate_slices: ${{ steps.gate-matrix.outputs.slices }} - gate_slice_timeouts: ${{ steps.gate-matrix.outputs.slice_timeouts }} + periodic_lane: ${{ steps.census.outputs.periodic_lane }} + census_arms: ${{ steps.census.outputs.arms }} + gate_matrix: ${{ steps.gate-matrix.outputs.gate_matrix }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: @@ -147,7 +159,18 @@ jobs: with: bun-version: 1.4.2 + # Census arms (scripts/census-arms.ts): which host/model arms run the + # gate census, and whether the periodic lane runs beside them. + - name: Select the census arms + id: census + env: + SCHEDULE: ${{ github.event.schedule }} + CENSUS_ARMS: ${{ inputs.census_arms }} + INCLUDE_GATE_CENSUS: ${{ github.ref == 'refs/heads/main' || inputs.include_gate_census }} + run: bun --no-install run scripts/census-arms.ts --select >> "$GITHUB_OUTPUT" + - name: Emit run manifest (ALL periodic tests minus reasoned excludes) + if: steps.census.outputs.periodic_lane == 'true' env: EVALS_ALL: "1" run: EVALS_TIER=periodic bun --no-install run scripts/test-paid-shards.ts --tier periodic --emit-plan /tmp/paid-plan/manifest.json --slice-budget 420 --jobs 2 --max-parallel 38 @@ -156,6 +179,7 @@ jobs: # container executor runs every other slice. - name: Derive the periodic executor matrix from the plan id: periodic-matrix + if: steps.census.outputs.periodic_lane == 'true' run: | { echo "slices=$(jq -c '[range(1; .sliceCount + 1)] - .plan.codexSlices' /tmp/paid-plan/manifest.json)" @@ -164,26 +188,38 @@ jobs: } >> "$GITHUB_OUTPUT" - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + if: steps.census.outputs.periodic_lane == 'true' with: name: paid-plan path: /tmp/paid-plan/manifest.json retention-days: 30 - - name: Emit gate census manifest (ALL gate tests) + # One plan per arm: the Codex host plans only its adapter cases and + # records every Claude-only case as excluded with its reason. + - name: Emit gate census manifests (ALL gate tests, one plan per arm) + if: steps.census.outputs.arms != '[]' env: EVALS_ALL: "1" - run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --emit-plan /tmp/gate-census-plan/manifest.json --slice-budget 420 --jobs 2 --skip-judges --max-parallel 20 - - - name: Derive the gate census executor matrix from the plan - id: gate-matrix + ARMS: ${{ steps.census.outputs.arms }} run: | - echo "slices=$(jq -c '[range(1; .sliceCount + 1)]' /tmp/gate-census-plan/manifest.json)" >> "$GITHUB_OUTPUT" - echo "slice_timeouts=$(jq -ec '[0] + .plan.sliceCiTimeoutMinutes' /tmp/gate-census-plan/manifest.json)" >> "$GITHUB_OUTPUT" + set -euo pipefail + for arm in $(jq -r '.[]' <<<"$ARMS"); do + mapfile -t arm_env < <(bun --no-install run scripts/census-arms.ts --env "$arm") + env "${arm_env[@]}" EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --emit-plan "/tmp/gate-census-plan/$arm/manifest.json" --slice-budget 420 --jobs 2 --skip-judges --max-parallel 20 + done + + # One executor per arm and planned slice, each with its slice's ceiling. + - name: Derive the gate census executor matrix from the plans + id: gate-matrix + env: + ARMS: ${{ steps.census.outputs.arms }} + run: bun --no-install run scripts/census-arms.ts --matrix /tmp/gate-census-plan "$ARMS" >> "$GITHUB_OUTPUT" - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + if: steps.census.outputs.arms != '[]' with: name: gate-census-plan - path: /tmp/gate-census-plan/manifest.json + path: /tmp/gate-census-plan retention-days: 30 # autoplan-schema-canary: installs the latest published Claude Code with @@ -237,6 +273,7 @@ jobs: eval-slices: runs-on: ubicloud-standard-8 needs: [build-image, plan-slices] + if: ${{ needs.plan-slices.outputs.periodic_lane == 'true' }} env: EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-eval-slices-${{ matrix.slice }} # The planner packs ~7 minutes of recorded work per slice; each slice's job @@ -388,7 +425,7 @@ jobs: eval-codex-slices: runs-on: ubicloud-standard-8 needs: [build-image, plan-slices] - if: ${{ needs.plan-slices.outputs.periodic_codex_slices != '[]' }} + if: ${{ needs.plan-slices.outputs.periodic_lane == 'true' && needs.plan-slices.outputs.periodic_codex_slices != '[]' }} env: EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-eval-slices-${{ matrix.slice }} timeout-minutes: ${{ fromJSON(needs.plan-slices.outputs.periodic_slice_timeouts)[matrix.slice] }} @@ -512,16 +549,18 @@ jobs: # blind spots rot invisibly. Census health, not selector correctness — # selector logic has free synthetic-diff contract tests. gate-census: + name: gate-census-${{ matrix.arm }} (${{ matrix.slice }}) runs-on: ubicloud-standard-8 needs: [build-image, plan-slices] - # A branch dispatch validates the periodic lane only; gate cases on a - # branch go through an evals.yml dispatch with evals_all. Main (schedule - # or dispatch) always runs the census; a branch opts in explicitly. - if: ${{ github.ref == 'refs/heads/main' || inputs.include_gate_census }} + # The planner picks the arms (scripts/census-arms.ts). A branch dispatch + # validates the periodic lane only unless it sets include_gate_census or + # census_arms; gate cases on a branch also go through an evals.yml + # dispatch with evals_all. Main (schedule or dispatch) always runs it. + if: ${{ needs.plan-slices.outputs.gate_matrix != '[]' }} env: - EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-gate-census-${{ matrix.slice }} - # Per-slice job ceiling from the packed plan (W2c/ENG-2), as for eval-slices. - timeout-minutes: ${{ fromJSON(needs.plan-slices.outputs.gate_slice_timeouts)[matrix.slice] }} + EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-gate-census-${{ matrix.arm }}-${{ matrix.slice }} + # Per-slice job ceiling from the arm's packed plan (W2c/ENG-2), as for eval-slices. + timeout-minutes: ${{ matrix.timeout }} permissions: contents: read packages: read @@ -536,7 +575,7 @@ jobs: fail-fast: false max-parallel: 20 matrix: - slice: ${{ fromJSON(needs.plan-slices.outputs.gate_slices) }} + include: ${{ fromJSON(needs.plan-slices.outputs.gate_matrix) }} steps: - name: Record the job start for the slice deadline run: echo "GSTACK_SLICE_JOB_STARTED_AT=$(date +%s)" >> "$GITHUB_ENV" @@ -588,7 +627,15 @@ jobs: printenv OPENAI_API_KEY | "$codex" login --with-api-key "$codex" login status >/dev/null 2>&1 - - name: Run gate census slice ${{ matrix.slice }} + # The arm's host and model (EVALS_HOST, plus EVALS_MODEL or + # GSTACK_CODEX_MODEL). The Codex host runs codex exec with + # danger-full-access, so it needs no bubblewrap and runs in this container. + - name: Select the census arm's host and model + env: + ARM: ${{ matrix.arm }} + run: bun --no-install run scripts/census-arms.ts --env "$ARM" >> "$GITHUB_ENV" + + - name: Run gate census slice ${{ matrix.slice }} (${{ matrix.arm }}) env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} @@ -597,13 +644,13 @@ jobs: EVALS_JOBS: "2" EVALS_CONCURRENCY: "2" GSTACK_EVAL_DIR: /tmp/gate-census-results - run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --plan /tmp/gate-census-plan/manifest.json --slice ${{ matrix.slice }} + run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --plan /tmp/gate-census-plan/${{ matrix.arm }}/manifest.json --slice ${{ matrix.slice }} - name: Upload census results if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: - name: gate-census-${{ matrix.slice }}-a${{ github.run_attempt }} + name: gate-census-${{ matrix.slice }}-a${{ github.run_attempt }}-${{ matrix.arm }} path: /tmp/gate-census-results retention-days: 90 @@ -638,6 +685,9 @@ jobs: actions: read outputs: redispatch: ${{ steps.verdict.outputs.redispatch }} + # A census-only run (monthly cron or census_arms dispatch) re-dispatches + # with the same arms; empty keeps the weekly selection. + redispatch_census_arms: ${{ needs.plan-slices.outputs.periodic_lane != 'true' && join(fromJSON(needs.plan-slices.outputs.census_arms), ',') || '' }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: @@ -648,6 +698,7 @@ jobs: bun-version: 1.4.2 - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + if: needs.plan-slices.outputs.periodic_lane == 'true' with: name: paid-plan path: /tmp/paid-report @@ -655,25 +706,35 @@ jobs: # One directory per attempt-scoped slice artifact (no merge): shard # records never overwrite each other and the first attempt decides. - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + if: needs.plan-slices.outputs.periodic_lane == 'true' with: pattern: paid-slice-* path: /tmp/paid-report - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + if: needs.plan-slices.outputs.gate_matrix != '[]' with: name: gate-census-plan - path: /tmp/gate-census-report + path: /tmp/gate-census-plan - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + if: needs.plan-slices.outputs.gate_matrix != '[]' with: pattern: gate-census-[0-9]* - path: /tmp/gate-census-report + path: /tmp/gate-census-slices - name: Reconcile slices against the manifest (fail-closed) id: reconcile if: always() + env: + PERIODIC_LANE: ${{ needs.plan-slices.outputs.periodic_lane }} run: | set +e + if [ "$PERIODIC_LANE" != "true" ]; then + echo "periodic lane skipped: census-only run (monthly cron or census_arms dispatch)" | tee /tmp/report.txt + echo "exit=0" >> "$GITHUB_OUTPUT" + exit 0 + fi EVALS_TIER=periodic bun --no-install run scripts/test-paid-shards.ts --tier periodic --report /tmp/paid-report | tee /tmp/report.txt # PIPESTATUS[0], NOT $?: GitHub's default run-step shell is # `bash -e {0}` with NO pipefail, so $? after the pipe is tee's @@ -686,20 +747,34 @@ jobs: if: always() env: GATE_CENSUS: ${{ needs.gate-census.result }} + ARMS: ${{ needs.plan-slices.outputs.census_arms }} + # One report per arm: its plan plus its own slice artifacts + # (gate-census--a-) in /tmp/gate-census-report/. run: | set +e if [ "$GATE_CENSUS" = "skipped" ]; then - echo "gate census skipped: branch dispatch without include_gate_census (periodic lane only)" | tee /tmp/gate-report.txt + echo "gate census skipped: no census arm selected (branch dispatch without include_gate_census or census_arms)" | tee /tmp/gate-report.txt echo "exit=0" >> "$GITHUB_OUTPUT" exit 0 fi - EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --report /tmp/gate-census-report | tee /tmp/gate-report.txt - echo "exit=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT" + status=0 + : > /tmp/gate-report.txt + for arm in $(jq -r '.[]' <<<"$ARMS"); do + dir="/tmp/gate-census-report/$arm" + mkdir -p "$dir" + cp "/tmp/gate-census-plan/$arm/manifest.json" "$dir/manifest.json" || status=1 + for slice in /tmp/gate-census-slices/gate-census-*-"$arm"; do if [ -d "$slice" ]; then mv "$slice" "$dir/"; fi; done + echo "=== gate census arm: $arm ===" | tee -a /tmp/gate-report.txt + EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --report "$dir" | tee -a /tmp/gate-report.txt + code=${PIPESTATUS[0]} + if [ "$code" -ne 0 ]; then status=$code; fi + done + echo "exit=$status" >> "$GITHUB_OUTPUT" - name: Stamp trial history series if: always() run: | - for file in /tmp/paid-report/trial-outcomes.jsonl /tmp/gate-census-report/trial-outcomes.jsonl; do + for file in /tmp/paid-report/trial-outcomes.jsonl /tmp/gate-census-report/*/trial-outcomes.jsonl; do if [ -f "$file" ]; then bun --no-install run scripts/eval-trial-series.ts "$file"; fi done @@ -719,7 +794,7 @@ jobs: with: name: trial-outcomes-gate-census-a${{ github.run_attempt }} path: | - /tmp/gate-census-report/trial-outcomes.jsonl + /tmp/gate-census-report/*/trial-outcomes.jsonl if-no-files-found: ignore retention-days: 90 @@ -732,7 +807,7 @@ jobs: run: | mkdir -p /tmp/exercised-map dirs=() - for dir in /tmp/gate-census-report /tmp/paid-report; do [ -d "$dir" ] && dirs+=(--from-dir "$dir"); done + for dir in /tmp/gate-census-report/* /tmp/paid-report; do [ -d "$dir" ] && dirs+=(--from-dir "$dir"); done status=0 bun --no-install run scripts/exercised-map.ts "${dirs[@]}" \ --run-id "$GITHUB_RUN_ID" --out /tmp/exercised-map/exercised-commands.json --report /tmp/exercised-map/report.md || status=$? @@ -779,8 +854,16 @@ jobs: [ "$1" = "0" ] && return 0 jq -e '.version == 2 and .verdict.redispatchEligible == true' "$2/collector-outcomes.json" >/dev/null 2>&1 } + eligible_arms() { # $1 exit, then one report dir per census arm: each must be green or INFRA/INCOMPLETE-only + [ "$1" = "0" ] && return 0 + shift + local dir + for dir in "$@"; do + jq -e '.version == 2 and (.verdict.verdict == "GREEN" or .verdict.redispatchEligible == true)' "$dir/collector-outcomes.json" >/dev/null 2>&1 || return 1 + done + } if [ -z "$REDISPATCH_OF" ] && { [ "$PERIODIC_EXIT" != "0" ] || [ "$GATE_EXIT" != "0" ]; } \ - && eligible "$PERIODIC_EXIT" /tmp/paid-report && eligible "$GATE_EXIT" /tmp/gate-census-report; then + && eligible "$PERIODIC_EXIT" /tmp/paid-report && eligible_arms "$GATE_EXIT" /tmp/gate-census-report/*/; then echo "redispatch=true" >> "$GITHUB_OUTPUT" else echo "redispatch=false" >> "$GITHUB_OUTPUT" @@ -793,7 +876,7 @@ jobs: # comments on or closes main's issue. Every report-derived line goes # through the one published-text sanitizer (scripts/lib/published-text.ts). - name: Write the census report - if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success') + if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || (needs.plan-slices.outputs.periodic_lane == 'true' && needs.eval-slices.result != 'success') || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success') env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} REDISPATCH: ${{ steps.verdict.outputs.redispatch }} @@ -827,7 +910,14 @@ jobs: cat /tmp/paid-report/report-summary.md 2>/dev/null || echo "(no periodic report summary)" echo echo "### Gate census" - cat /tmp/gate-census-report/report-summary.md 2>/dev/null || echo "(no gate census report summary)" + found=0 + for summary in /tmp/gate-census-report/*/report-summary.md; do + [ -f "$summary" ] || continue + found=1 + echo "#### Arm $(basename "$(dirname "$summary")")" + cat "$summary" + done + [ "$found" = 1 ] || echo "(no gate census report summary)" echo echo "### Pass-rate history (ACTION REQUIRED)" { awk '/^ACTION REQUIRED/ { on = 1; print; next } on && /^ \[/ { print; next } { on = 0 } /history unavailable/ { print }' /tmp/pass-rates.txt 2>/dev/null \ @@ -851,7 +941,7 @@ jobs: cat /tmp/census-report.md >> "$GITHUB_STEP_SUMMARY" - name: Upload the census report - if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success') + if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || (needs.plan-slices.outputs.periodic_lane == 'true' && needs.eval-slices.result != 'success') || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success') uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: census-report-a${{ github.run_attempt }} @@ -860,7 +950,7 @@ jobs: retention-days: 90 - name: Upsert tracking issue on failure - if: always() && github.ref == 'refs/heads/main' && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success') + if: always() && github.ref == 'refs/heads/main' && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || (needs.plan-slices.outputs.periodic_lane == 'true' && needs.eval-slices.result != 'success') || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success') env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -877,7 +967,7 @@ jobs: fi - name: Close the tracking issue on a green run - if: always() && github.ref == 'refs/heads/main' && steps.reconcile.outputs.exit == '0' && steps.gate-reconcile.outputs.exit == '0' && steps.pass-rates.outputs.exit == '0' && needs.eval-slices.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) && needs.gate-census.result == 'success' + if: always() && github.ref == 'refs/heads/main' && steps.reconcile.outputs.exit == '0' && steps.gate-reconcile.outputs.exit == '0' && steps.pass-rates.outputs.exit == '0' && needs.plan-slices.outputs.periodic_lane == 'true' && needs.eval-slices.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) && needs.gate-census.result == 'success' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} REDISPATCH_OF: ${{ inputs.redispatch_of }} @@ -892,7 +982,7 @@ jobs: fi - name: Fail the workflow when reconciliation failed - if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || !contains(fromJSON('["success","skipped"]'), needs.gate-census.result)) + if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || (needs.plan-slices.outputs.periodic_lane == 'true' && needs.eval-slices.result != 'success') || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || !contains(fromJSON('["success","skipped"]'), needs.gate-census.result)) run: exit 1 # The one INFRA/INCOMPLETE re-dispatch (UC-E1). Its own job so the report @@ -909,4 +999,5 @@ jobs: - name: Re-dispatch the weekly census once env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: gh workflow run evals-periodic.yml --repo "$GITHUB_REPOSITORY" --ref "$GITHUB_REF_NAME" -f redispatch_of="$GITHUB_RUN_ID" + CENSUS_ARMS: ${{ needs.report.outputs.redispatch_census_arms }} + run: gh workflow run evals-periodic.yml --repo "$GITHUB_REPOSITORY" --ref "$GITHUB_REF_NAME" -f redispatch_of="$GITHUB_RUN_ID" ${CENSUS_ARMS:+-f census_arms="$CENSUS_ARMS"} diff --git a/BROWSER.md b/BROWSER.md index 75cd99b9e..396bb6e9b 100644 --- a/BROWSER.md +++ b/BROWSER.md @@ -584,7 +584,7 @@ SITE=app.example.com DOMAIN=example.com $B goto "https://$SITE" $B cookie-import-browser opera-gx --domain "$DOMAIN" --profile Default # or: opera -$B reload # confirm the intended account +$B reload # view the page; only --verify-auth proves sign-in ``` Run `$B cookie-import-browser` with no flags to see which browsers were detected. `--profile` can be omitted when only one profile has cookies for the domain. If the receipt reports App-Bound Encryption, run `$B handoff`, sign in to the intended account in the window that opens, then `$B resume` (needs a display). diff --git a/CHANGELOG.md b/CHANGELOG.md index c977413a5..3a65b1e76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,46 @@ ### Gate integrity - The structured and review gates' no-findings check (`classifyOutsideReview`) is stricter about position and wider in vocabulary. New free regressions in `test/outside-review-result.test.ts` cover the reporter's exact reply through their CLI invocation, two clean `codex review --uncommitted` replies captured from codex-cli 0.160.0 with gpt-6-sol (both read `unverified` before), fifteen mixed replies that must stay `unverified` under both gates (the two strings from the issue, the phrase before, after and inside untagged findings, exception conclusions, conclusions scoped to one place or to the tests, a quoted `NO_FINDINGS`, a no-issues line inside a code fence), and Recommendation-clause cases including a wrapped clause with "but". A GPT-6.1 Sol outside review probed the classifier; its false passes are among those controls. A known residual, unchanged from before: untagged findings followed by "No issues found in the changed parser." still read `clean`, because "changed" names the change and the parser cannot tell a finding from a summary without tags. All four new tests fail on the previous classifier. Every existing negative control passes unchanged, including the stored Codex clean review that still reads `unverified` ("no actionable correctness issues"). No threshold, trial count or case kind changed. +## [1.91.150.0] - 2026-10-19 + +**The paid evals that still pinned older models now run on the current generation, and two cases that went red on the move are repaired at their cause.** Garry's direction (2026-10-18) is to stop testing older frontier models, so these files are re-baselined as a fresh measured reference rather than held to a threshold change. + +### Changed +- **Old-model pins:** the `claude-opus-4-7` pins in `skill-e2e-plan`, `skill-e2e-plan-prosons`, `skill-e2e-design`, `skill-e2e-office-hours-phase4` and `skill-e2e-qa-bugs` now follow `resolveEvalModel('capture')` (Opus 5.5). The overlay benchmark fixtures pin `claude-opus-5-5` and `claude-sonnet-5-5`, the cookie workflow judge pins `claude-sonnet-5-5`, and `codex-sol-scope-termination` runs on `gpt-6.1-sol` while still rendering the `gpt-5.6-sol` overlay. +- **/setup-browser-cookies:** the workflow is now five ordered steps. It checks for Aside first and ends there on `READY`, since no import is needed. It then asks every decision (source and profile, domains and target, verification, storage reset, all cookies) before any `$B` command, prepares gstack's browser (verification variables before the daemon starts, the fallback, the `Mode: cdp` stop), imports, and reports on three lines: Imported, Storage reset, Sign-in. The `{{BROWSE_SETUP}}` block, which contradicted the Aside probe, is gone from this skill. +- **Gate integrity:** no threshold, trial count, case kind or assertion changed. Free-test edits follow the new cookie template: two phrase and order assertions, the approval fixture's reverse edits, and the positional-literals checksum list, which no longer includes this skill. + +### Fixed +- **plan-review-prosons-hardstop-neg:** on Opus 5.5 one trial in three stopped with a provider refusal (`stop_reason: refusal`, category `reasoning_extraction`) on the turn that writes the captured question. Replays of the refused request were stopped 7 of 16 times with the capture prompt's "verbatim" and "Do NOT paraphrase. This is a format-capture test." wording and 0 of 24 without it. The shared prosons capture prompt now asks for "the text of the single AskUserQuestion you would have made", with the same required parts. The decision content and every assertion are unchanged; all four prosons cases pass 3/3. +- **Cookie workflow judge:** on `claude-sonnet-5-5` it scored clarity 3, completeness 4, actionability 3 against the old template. The rewritten template scores 4/4/4 with unchanged thresholds. + +## [1.91.149.0] - 2026-10-19 + +**Paid eval cases can now run on the Codex host, and the Monday census runs two arms: Claude Opus 5.5 and Codex gpt-6.1-sol.** `EVALS_HOST=codex` runs the 52 adapter-only gate cases through a full Codex runtime per case. On its first pilot gpt-6.1-sol passed 48 of 52 with no retries, at $4.63 for the 52 cases, against $28.93 for the same cases on Fable (0.16×). Claude-only cases are skipped on Codex with a recorded reason and never count as failed. GSTA-49 TM2. + +### Added +- **Codex host adapter:** `test/helpers/e2e-hosts.ts` holds the `EVALS_HOST` switch (`claude` default, `codex`) and the per-case host metadata. `test/helpers/codex-host-runner.ts` adds: + - the case runner, with sandbox settings and a reasoning effort (the model default unless set); + - file-change event mapping; + - a transcript translated into the Claude stream-json shape that existing assertions read; + - the native-lifecycle hooks. + + The 26 rework cases stay Claude-only. +- **Census arms (`scripts/census-arms.ts`):** `evals-periodic.yml` gets two census schedules. + - **Monday:** a two-arm gate census, `claude-opus` and `codex-sol`. + - **Monthly (`0 6 1 * *`):** census jobs only, with `claude-fable` (claude-fable-5-1) and `codex-astra` (gpt-6-astra) arms and a `-monthly` concurrency key. + + A `census_arms` dispatch input picks arms by hand. Artifact names and run IDs carry the arm, and a census-only run executes no periodic slice. Every arm keeps the census's `--skip-judges`. + +### Changed +- **Codex cost is real:** `cost_known: false` is gone. Codex sessions price their `turn.completed` usage (input, cached input, cache-write input and output tokens) through `lib/pricing.ts`. The shared `execCodex` keeps the old runner's events and verdicts. +- **Plans, reports and the ledger carry the host** when it is not Claude. On Codex, the Anthropic preflight skips its `claude -p` ping. +- **Gate integrity:** no threshold, trial count, case kind, judge or Claude-host verdict changed. `HARNESS_VERSION` stays 6, and the changed harness files are recorded as non-behavioral. Five workflow tests now assert the census matrix per arm (one row per arm and slice, per-arm plan and report paths, a `periodic_lane` guard) in place of the single-arm shape; no assertion was dropped. + +### If you… +- **…want one case on Codex:** `EVALS_HOST=codex GSTACK_CODEX_MODEL=gpt-6.1-sol bun run scripts/test-paid-shards.ts --tier gate --case --trials 1`. +- **…want the monthly arms now:** dispatch `evals-periodic.yml` with `census_arms=claude-fable,codex-astra`. + ## [1.91.148.0] - 2026-10-19 **Weekly seed PRs now get the credential scan as well as the free suites.** A pull request the recorder opens with the workflow token never starts its own `pull_request` checks. GitHub holds them as "action required". So the recorder dispatches its checks on the bot branch instead. diff --git a/TODOS.md b/TODOS.md index 25de1dcf9..8453a0b26 100644 --- a/TODOS.md +++ b/TODOS.md @@ -1645,19 +1645,24 @@ and `test/dx-selected-navigation-ap.test.ts`. One shared table run once against only after `engFirstReviewAUQ` checks native completion once at entry; today each branch gates it separately, so the change alters a paid verdict and needs its own paid run. -### P3: Re-pin the five remaining claude-opus-4-7 paid files +### P3: Follow-ups from re-baselining the old-model paid pins on the current generation -**What:** The 2026-09 audit moved seven paid evals to the default capture model (`resolveEvalModel('capture')`). -`skill-e2e-design`, `skill-e2e-office-hours-phase4`, `skill-e2e-plan-prosons` and `skill-e2e-plan` keep -`claude-opus-4-7` because six cases failed on the default model in one run (plan-design-review-plan-mode timeout, -office-hours-phase4-fork format, plan-review-prosons-neutral-neg missing output, plan-ceo-review-selective and -plan-eng-review 600 s timeouts, plan-ceo-review-expansion-energy posture score 3). plan-review-report, the one PR-gate -case in `skill-e2e-plan`, left the pin on 2026-10-18 and follows the capture default. `skill-e2e-qa-bugs` returned -to `claude-opus-4-7` after `qa-b6-static` timed out on the default model in two of three runs (census 36597762183 -and a targeted local rerun): each time the stream stopped mid-message, with no pending tool, right after the model -found the disabled submit button, and emitted nothing until the 300 s case deadline. They measure an old model. +**What:** GSTA-49 TM3 (2026-10-18) moved every remaining old-model paid pin to the current generation: the +`claude-opus-4-7` pins in `skill-e2e-{plan,plan-prosons,design,office-hours-phase4,qa-bugs}` follow +`resolveEvalModel('capture')`, the overlay fixtures pin `claude-opus-5-5` and `claude-sonnet-5-5`, the cookie workflow +judge pins `claude-sonnet-5-5`, and `codex-sol-scope-termination` runs on `gpt-6.1-sol` (still rendering the +`gpt-5.6-sol` overlay). Three things remain: -**Re-entry:** fix the prompt, budget or rubric so each case passes on the default model in one run, then drop the pin. +- The base `claude-dedicated-tools-vs-bash` fixture now duplicates `claude-dedicated-tools-vs-bash-opus-5-5`. +- The `opus-4-7-*` fixtures measure `model-overlays/opus-4-7.md` on Opus 5.5, which gets the generic `claude` + overlay in product. +- Opus 5.5 refuses (stop category `reasoning_extraction`) capture prompts that ask for the "verbatim text" of a + question "you would have made" with "Do NOT paraphrase"; that wording stopped 7 of 16 replays of a prosons + request and 0 of 24 without it. `skill-e2e-plan-prosons` dropped it; `skill-e2e-plan-format` and + `skill-e2e-office-hours-phase4` still use it. + +**Re-entry:** retire one dedicated-tools fixture, decide the opus-4-7 overlay's future, and drop the extraction +wording from the remaining capture prompts with a paid run each. ### P3: Retire the unused CEO payment seeder diff --git a/VERSION b/VERSION index 427e8cae8..d31a4e90c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.148.0 +1.91.150.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 7c7dab179..8a1c9f422 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.148.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.150.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/docs/OVERLAY_BENCHMARK_CONTRACT.md b/docs/OVERLAY_BENCHMARK_CONTRACT.md index d6f3cc38e..84f87b4d6 100644 --- a/docs/OVERLAY_BENCHMARK_CONTRACT.md +++ b/docs/OVERLAY_BENCHMARK_CONTRACT.md @@ -14,8 +14,10 @@ Selection IDs for the six retained paid cases stay unchanged. ## Blocking checks Both arms use the real Claude Code preset; ON appends the resolved overlay and -OFF does not. Each retained fixture runs ten samples per arm, separately for -`claude-opus-4-7` and `claude-sonnet-4-6`. Each model keeps its own results. +OFF does not. Each retained fixture runs ten samples per arm on its pinned +model: `claude-opus-5-5`, plus `claude-sonnet-5-5` for the dedicated-tools `-sonnet` case. +Each model keeps its own results. The `opus-4-7-*` cases keep their overlay +(`model-overlays/opus-4-7.md`) but run on Opus 5.5 since GSTA-49. Every planned sample in both arms must execute successfully, produce exactly one successful native terminal result without an error flag, preserve workspace @@ -38,7 +40,7 @@ are not reinterpreted. | `opus-4-7-effort-match-trivial` | Read-only workspace; native final JSON contains the exact version string | Correct answer in every ON sample | Native `usage.output_tokens_details.thinking_tokens` | | `opus-4-7-literal-interpretation` | Only `src/auth.ts`, `src/billing.ts`, and `src/notifications.ts` may change; public tests stay frozen; independent behavior checks run outside the writable fixture | All three target behaviors pass in every ON sample | Correct target behaviors, 0..3 | -Each family also has a `-sonnet` case. Output validation checks the native final +The dedicated-tools family also has `-sonnet` and `-opus-5-5` cases. Output validation checks the native final answer, so a correct word mentioned earlier cannot conceal a wrong final answer. The read-only prompts explicitly request the JSON shape. These exact-answer prompts are new in v2: compare ON/OFF within the same contract and fixture, not diff --git a/package.json b/package.json index 61f535db9..302f0c4c1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.148", + "version": "1.91.150", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/scripts/census-arms.ts b/scripts/census-arms.ts new file mode 100644 index 000000000..9ede1799a --- /dev/null +++ b/scripts/census-arms.ts @@ -0,0 +1,81 @@ +#!/usr/bin/env bun +/** + * Gate census arms for evals-periodic.yml (GSTA-49 TM2). Each arm is one host + * and model running the weekly gate census plan: + * - Monday cron (0 6 * * 1): claude-opus (the capture model, lib/eval-model.ts) + * and codex-sol (gpt-6.1-sol on the Codex host, adapter cases only); + * - monthly cron (0 6 1 * *): claude-fable and codex-astra, census jobs only; + * - dispatch: `census_arms` runs only the census with the named arms; without + * it a dispatch keeps the periodic lane plus the Monday arms when the census + * runs (main, or include_gate_census). + * Judges stay in the Claude lanes: every census plan skips them. + * + * CLI (GitHub step outputs and env lines): + * --select env SCHEDULE, CENSUS_ARMS, INCLUDE_GATE_CENSUS -> arms=, periodic_lane= + * --env KEY=VALUE lines selecting the arm's host and model + * --matrix gate_matrix= from //manifest.json + */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import type { EvalHost } from '../test/helpers/e2e-hosts'; + +export interface CensusArm { host: EvalHost; model?: string } + +export const CENSUS_ARMS: Readonly> = Object.freeze({ + // No model: the Claude runners' capture default (claude-opus-5-5 since TM1). + 'claude-opus': { host: 'claude' }, + 'codex-sol': { host: 'codex', model: 'gpt-6.1-sol' }, + 'claude-fable': { host: 'claude', model: 'claude-fable-5-1' }, + 'codex-astra': { host: 'codex', model: 'gpt-6-astra' }, +}); + +export const WEEKLY_CENSUS_SCHEDULE = '0 6 * * 1'; +export const MONTHLY_CENSUS_SCHEDULE = '0 6 1 * *'; +export const WEEKLY_CENSUS_ARMS: readonly string[] = ['claude-opus', 'codex-sol']; +export const MONTHLY_CENSUS_ARMS: readonly string[] = ['claude-fable', 'codex-astra']; + +/** Which census arms run, and whether the periodic lane runs beside them. */ +export function selectCensus(opts: { schedule?: string; censusArms?: string; includeGateCensus: boolean }): { arms: string[]; periodicLane: boolean } { + if (opts.schedule === MONTHLY_CENSUS_SCHEDULE) return { arms: [...MONTHLY_CENSUS_ARMS], periodicLane: false }; + const named = (opts.censusArms ?? '').split(/[\s,]+/).filter(Boolean); + const unknown = named.filter(arm => !Object.hasOwn(CENSUS_ARMS, arm)); + if (unknown.length) throw new Error(`unknown census arm(s): ${unknown.join(', ')}; known: ${Object.keys(CENSUS_ARMS).join(', ')}`); + if (named.length) return { arms: [...new Set(named)], periodicLane: false }; + return { arms: opts.includeGateCensus ? [...WEEKLY_CENSUS_ARMS] : [], periodicLane: true }; +} + +/** The env one arm's planner and executor run with. */ +export function censusArmEnv(arm: string): Record { + const spec = CENSUS_ARMS[arm]; + if (!spec) throw new Error(`unknown census arm: ${arm}`); + if (!spec.model) return { EVALS_HOST: spec.host }; + return { EVALS_HOST: spec.host, [spec.host === 'codex' ? 'GSTACK_CODEX_MODEL' : 'EVALS_MODEL']: spec.model }; +} + +/** One executor per arm and planned slice, with that slice's planned CI ceiling. */ +export function censusMatrix(arms: readonly string[], planDir: string): Array<{ arm: string; slice: number; timeout: number }> { + return arms.flatMap(arm => { + const manifest = JSON.parse(fs.readFileSync(path.join(planDir, arm, 'manifest.json'), 'utf8')) as + { sliceCount: number; plan?: { ciTimeoutMinutes: number; sliceCiTimeoutMinutes?: number[] } }; + if (!manifest.plan) throw new Error(`${arm}: census manifest has no budget plan`); + return Array.from({ length: manifest.sliceCount }, (_, i) => ({ + arm, slice: i + 1, timeout: manifest.plan!.sliceCiTimeoutMinutes?.[i] ?? manifest.plan!.ciTimeoutMinutes, + })); + }); +} + +if (import.meta.main) { + const [mode, ...args] = process.argv.slice(2); + if (mode === '--select') { + const { arms, periodicLane } = selectCensus({ schedule: process.env.SCHEDULE, censusArms: process.env.CENSUS_ARMS, + includeGateCensus: process.env.INCLUDE_GATE_CENSUS === 'true' }); + console.log(`arms=${JSON.stringify(arms)}\nperiodic_lane=${periodicLane}`); + } else if (mode === '--env' && args.length === 1) { + for (const [key, value] of Object.entries(censusArmEnv(args[0]!))) console.log(`${key}=${value}`); + } else if (mode === '--matrix' && args.length === 2) { + console.log(`gate_matrix=${JSON.stringify(censusMatrix(JSON.parse(args[1]!) as string[], args[0]!))}`); + } else { + console.error('usage: census-arms.ts --select | --env | --matrix '); + process.exit(2); + } +} diff --git a/scripts/e2e-shard-reuse.ts b/scripts/e2e-shard-reuse.ts index 64839e98f..e6ffcc176 100644 --- a/scripts/e2e-shard-reuse.ts +++ b/scripts/e2e-shard-reuse.ts @@ -83,6 +83,7 @@ export function e2eReuseLaneProblem(env: NodeJS.ProcessEnv, profileMode: string if (!/^(?:sha256:)?[a-f0-9]{64}$/.test(env.EVALS_CACHE_RUNTIME_ID ?? '')) return 'No immutable runtime identity'; if (env.EVALS_TIER !== 'gate' || env.EVALS_FRESH === '1') return 'Fresh validation requested'; if (['release', 'periodic', 'marathon'].includes(env.EVALS_CACHE_PURPOSE ?? '')) return 'Scheduled and release lanes execute fresh'; + if ((env.EVALS_HOST ?? '').trim() && env.EVALS_HOST!.trim() !== 'claude') return 'Non-Claude hosts execute fresh'; if (env.NODE_OPTIONS || env.BUN_OPTIONS) return 'Preload options change execution outside the consumed source'; if (env.ANTHROPIC_BASE_URL && env.ANTHROPIC_BASE_URL !== 'https://api.anthropic.com') return 'Custom model endpoint'; return null; diff --git a/scripts/harness-version.json b/scripts/harness-version.json index dc18269d4..4e9d5794e 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -3,6 +3,7 @@ "patterns": [ "test/helpers/agent-sdk-runner.ts", "test/helpers/codex-session-runner.ts", + "test/helpers/codex-host-runner.ts", "test/helpers/claude-pty-runner.ts", "test/helpers/pty/**", "test/helpers/pty-screen.ts", @@ -10,22 +11,23 @@ "test/helpers/llm-judge.ts" ], "files": { - "scripts/lib/paid-cases.ts": {"blob":"9bacd52446192a7ecf3dfaae901ea443cf9c9ae2","since":6}, - "scripts/lib/paid-plan.ts": {"blob":"45721ce128a112666a0caed6c13c66637b74ca54","since":6}, - "scripts/lib/paid-report.ts": {"blob":"b66c41237611ccecaaa7bd6a4d7b7aac8dc548cd","since":6}, + "scripts/lib/paid-cases.ts": {"blob":"f862cf5241580c7593b9ee62eacf5ef046248d26","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, + "scripts/lib/paid-plan.ts": {"blob":"95e1798e7e2495ace350d2624e59f83412f341bc","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, + "scripts/lib/paid-report.ts": {"blob":"f92c6bb723b158e328aa040fa5de6a0ee6d3916d","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6}, "scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":6}, - "scripts/test-paid-shards.ts": {"blob":"8ddee5f11ed7856876135520dd8025ab989e2db9","since":6}, + "scripts/test-paid-shards.ts": {"blob":"19c9b2376d09090aee1a8acacfe721d66f55b0fc","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "scripts/test-pr-profile.ts": {"blob":"579ca1bbd590f709c1ea3d6f28fa6c629688c79f","since":6,"nonBehavioral":"Classifies scripts/free-test-seed-allowlist.json as free-only; only the free seed ratchet reads it, so no verdict can change"}, "scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":6}, "test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":6}, "test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":6}, "test/helpers/claude-pty-runner.ts": {"blob":"20cc5894532873aa6145fc6617fdeec8cef94a2d","since":6}, - "test/helpers/codex-session-runner.ts": {"blob":"4eea35931a35a0afb006363402e3d4191e379737","since":6}, + "test/helpers/codex-host-runner.ts": {"blob":"bff225017e235f0be6ece29f04971ac04b95b544","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, + "test/helpers/codex-session-runner.ts": {"blob":"c040ebbfa815b8cce19ef80c6133bd5a5474149a","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "test/helpers/e2e-gate.ts": {"blob":"efb7e197dd5f66c3ad0e34074b14e640754f7a45","since":6}, - "test/helpers/e2e-helpers.ts": {"blob":"2d4a2a53063cfc1f003db366d2c861ab1e379e6e","since":6}, + "test/helpers/e2e-helpers.ts": {"blob":"6ee6937e6c3ca081c77131b5a755278c7b714834","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "test/helpers/eval-budgets.ts": {"blob":"667d0e09b20fe5d7366f57b77b5fb9b04baa37ef","since":6}, - "test/helpers/eval-store.ts": {"blob":"609c6d099108ff2696eb014b47aa318a618dd2af","since":6}, + "test/helpers/eval-store.ts": {"blob":"ed8721f13d39f4ef62e0a1d814efbe9352da781a","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "test/helpers/hermetic-env.ts": {"blob":"cc072895461dc52ea317d1beff72121342420d30","since":6}, "test/helpers/llm-judge.ts": {"blob":"d0bc6bbbe9f478603d11a1ece95e66c812ad34e6","since":6}, "test/helpers/paid-test-set.ts": {"blob":"ff9bfe43c43ec2bdd11114f7b8ec7c5736116f75","since":6}, @@ -44,8 +46,8 @@ "test/helpers/pty/screen.ts": {"blob":"ba7a80bb68b1959296ab2a827cfacdaa46f4ba2d","since":6}, "test/helpers/pty/session.ts": {"blob":"58533cea9d81cc87bb3b499bdfe0db8e3a8709c5","since":6}, "test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":6}, - "test/helpers/session-ledger.ts": {"blob":"6e3c9e837e46f98addba2d12eaebff802da320bb","since":6}, - "test/helpers/session-runner.ts": {"blob":"d40eb29e8b8fdc0675b511693e4a90823b5963fc","since":6}, + "test/helpers/session-ledger.ts": {"blob":"425323284a368f5e77937e03884db4dea4e6134e","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, + "test/helpers/session-runner.ts": {"blob":"cc65a66adafa9c053e3da3c6f17b5ac86c0eade7","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "test/helpers/skill-fixture.ts": {"blob":"b188064594eb4201a54a80ba1bc6f28e31d7f49e","since":6}, "test/helpers/test-selection.ts": {"blob":"51655f89f66be7a5ec487e2abfe4744300ebd5f6","since":6}, "test/helpers/touchfiles.ts": {"blob":"15f369056d41b9dc42dcd86f81f60c878074704c","since":6} diff --git a/scripts/lib/paid-cases.ts b/scripts/lib/paid-cases.ts index 9bacd5244..f862cf524 100644 --- a/scripts/lib/paid-cases.ts +++ b/scripts/lib/paid-cases.ts @@ -9,6 +9,7 @@ import type { EvalCaseKind, PanelShape } from '../../test/helpers/eval-store'; import { E2E_KINDS } from '../../test/helpers/touchfiles-data'; import { E2E_TOUCHFILES, E2E_TIERS } from '../../test/helpers/touchfiles'; import { type PaidTier, ROOT } from './paid-types'; +import { type EvalHost, hostSkipReason, resolveEvalHost } from '../../test/helpers/e2e-hosts'; /** * The E2E ids a paid file registers: the touchfile registrations that list the @@ -226,12 +227,38 @@ export function partitionCaseExclusions(keys: string[]): { runnable: string[]; e return { runnable, excluded }; } +/** + * Host exclusions (EVALS_HOST, test/helpers/e2e-hosts.ts): on a non-Claude + * host a case key whose case is Claude-only, or a file key none of whose + * `tier` cases runs on the host, is excluded with its reason. A file that + * keeps one host case stays runnable; its Claude-only cases skip at module + * load and the report deselects them. The Claude host excludes nothing. + */ +export function partitionHostExclusions(keys: string[], tier: PaidTier, host: EvalHost, rootDir = ROOT): { runnable: string[]; excluded: Array<{ file: string; reason: string }> } { + if (host === 'claude') return { runnable: keys, excluded: [] }; + const excluded: Array<{ file: string; reason: string }> = []; + const runnable = keys.filter(key => { + const caseId = shardCaseId(key); + const ids = caseId !== null ? [caseId] + : fileCaseRegistration(shardFile(key), fs.readFileSync(path.join(rootDir, shardFile(key)), 'utf8')).registered.filter(id => E2E_TIERS[id] === tier); + if (ids.some(id => hostSkipReason(id, host) === null)) return true; + const reason = ids.length === 1 ? hostSkipReason(ids[0]!, host)! + : ids.length === 0 ? `Claude-only on the ${host} host: no Codex host metadata (no ${tier} case registered)` + : `Claude-only on the ${host} host: no case runs on it (${ids.join(', ')})`; + excluded.push({ file: key, reason: `skipped: ${reason}` }); + return false; + }); + return { runnable, excluded }; +} + /** * Codex access in the CI eval image, scoped per paid shard. * * The image installs the pinned Codex CLI off PATH (GSTACK_CI_CODEX_BIN_DIR) * and the eval workflows log it in under a non-default home - * (GSTACK_CI_CODEX_HOME). A gate shard never sees either. A non-gate shard + * (GSTACK_CI_CODEX_HOME). On the Codex host (EVALS_HOST=codex) Codex is the + * agent, so every shard gets the CLI on PATH and the home as CODEX_HOME. + * Otherwise a gate shard never sees either. A non-gate shard * whose every file exercises Codex gets the CLI on PATH and the home as * CODEX_HOME. A file that mixes one Codex case with other cases keeps only * the two variables, and that case opts in on its own session; hermetic @@ -266,8 +293,10 @@ export const CODEX_CI_CASES: Readonly> = { export type CodexShardAccess = 'none' | 'case-opt-in' | 'path'; /** What one shard, given the test files behind its keys, may see of the CI Codex install in `tier`. */ -export function codexShardAccess(files: readonly string[], tier: string | undefined): CodexShardAccess { - if (tier === 'gate' || files.length === 0) return 'none'; +export function codexShardAccess(files: readonly string[], tier: string | undefined, host: EvalHost = 'claude'): CodexShardAccess { + if (files.length === 0) return 'none'; + if (host === 'codex') return 'path'; + if (tier === 'gate') return 'none'; if (files.every(file => CODEX_CI_FILES.includes(file))) return 'path'; return files.every(file => file in CODEX_CI_CASES) ? 'case-opt-in' : 'none'; } @@ -277,7 +306,7 @@ export function scopeCodexAccess(env: NodeJS.ProcessEnv, files: readonly string[ const binDir = env[CODEX_CI_ENV.binDir]; if (!binDir) return; const home = env[CODEX_CI_ENV.home]; - const access = codexShardAccess(files, env.EVALS_TIER); + const access = codexShardAccess(files, env.EVALS_TIER, resolveEvalHost(env)); delete env.CODEX_HOME; if (access === 'case-opt-in') return; delete env[CODEX_CI_ENV.binDir]; diff --git a/scripts/lib/paid-plan.ts b/scripts/lib/paid-plan.ts index 45721ce12..95e1798e7 100644 --- a/scripts/lib/paid-plan.ts +++ b/scripts/lib/paid-plan.ts @@ -10,7 +10,8 @@ import { evalEntryOutcome, type EvalCaseKind } from '../../test/helpers/eval-sto import { E2E_KINDS } from '../../test/helpers/touchfiles-data'; import { prProfileCaseAllowed, prProfileFileMap, type PrProfileSelection } from '../test-pr-profile'; import { E2E_TOUCHFILES, E2E_TIERS, LLM_JUDGE_TOUCHFILES } from '../../test/helpers/touchfiles'; -import { CASE_KEY_SEPARATOR, CASE_SHARDED_FILES, type CaseTrialPlan, caseTrialPlan, codexShardAccess, expandCaseShards, expandTrialShards, isIsolatedCase, partitionCaseExclusions, sameTrialPlan, shardCaseId, shardFile, shardTrial } from './paid-cases'; +import { CASE_KEY_SEPARATOR, CASE_SHARDED_FILES, type CaseTrialPlan, caseTrialPlan, codexShardAccess, expandCaseShards, expandTrialShards, isIsolatedCase, partitionCaseExclusions, partitionHostExclusions, sameTrialPlan, shardCaseId, shardFile, shardTrial } from './paid-cases'; +import { EVAL_HOSTS, type EvalHost, resolveEvalHost } from '../../test/helpers/e2e-hosts'; import { DEFAULT_JOBS, OVERLAY_MAX_ACTIVE_SHARDS, PAID_TIERS, SLICE_UPLOAD_RESERVE_MS, type PaidCaseSelection, type PaidProfile, type PaidShardBudget, type PaidTier, ROOT, type ShardOutcome, type ShardStatus, type ShardTrialRecord, isOverlayTestFile } from './paid-types'; import { collectPaidTestFiles, computePaidCaseSelection, expectedPrCaseCount, isAllSkippedPass, paidShardWallUpperBoundMs, partitionShardsByDiffSelection, planPaidShards, prProfileFileSelected, resolvePaidShardBudget, resolvePaidShardTimeoutMs, sameBudget, selectPaidTestFiles, shardSlug, validatedProfile } from './paid-select'; @@ -58,6 +59,8 @@ export interface PaidRunManifest { evalsAll: boolean; sliceCount: number; selectionReason: string; + /** The agent host every session runs on (EVALS_HOST); absent means claude. */ + host?: EvalHost; /** Legacy v1 manifests omit these; new plans bind case-level execution. */ profile?: PaidProfile; selection?: PaidCaseSelection; @@ -314,9 +317,11 @@ export function buildRunManifest(opts: { const quarantine = opts.quarantine ?? CASE_QUARANTINE; const notLive = [...Object.keys(kinds).filter(id => kinds[id] === 'behavior'), ...Object.keys(quarantine)].filter(id => !Object.hasOwn(E2E_TIERS, id)); if (notLive.length) throw new Error(`Only live E2E cases can be behavior or quarantined (judges sample their panel inside the case): ${notLive.join(', ')}`); - const caseKeys = partitionCaseExclusions(expandCaseShards(selected, opts.tier, rootDir)); + const host = resolveEvalHost(env); + const ciKeys = partitionCaseExclusions(expandCaseShards(selected, opts.tier, rootDir)); + const caseKeys = partitionHostExclusions(ciKeys.runnable, opts.tier, host, rootDir); const excluded = [...tierSelection.excluded, ...(opts.skipJudges ? tierSelection.selected.filter(judge) - .map(file => ({ file, reason: 'skipped: LLM judges run in the periodic census and PR gate lanes' })) : []), ...caseKeys.excluded]; + .map(file => ({ file, reason: 'skipped: LLM judges run in the periodic census and PR gate lanes' })) : []), ...ciKeys.excluded, ...caseKeys.excluded]; const expansion = expandTrialShards(caseKeys.runnable, opts.tier, rootDir, { kinds, quarantine }); const excludeOf = (key: string) => expansion.excludeCases[normalizeRelativePath(key)] ?? []; const shards = planPaidShards(expansion.keys, { maxFilesPerShard: 1 }); @@ -346,7 +351,7 @@ export function buildRunManifest(opts: { entries.sort((a, b) => (a.file < b.file ? -1 : 1)); return parseRunManifest(JSON.stringify({ version: 1, tier: opts.tier, evalsAll: opts.evalsAll, sliceCount: plan.slices.length, - selectionReason: cases.reason, profile, selection: cases.selection, + selectionReason: cases.reason, ...(host !== 'claude' ? { host } : {}), profile, selection: cases.selection, ...(cases.coverage ? { prCoverage: cases.coverage } : {}), plan: { sliceBudgetMs: opts.sliceBudgetMs!, jobs: opts.jobs!, estimatedSliceMs: plan.estimatedSliceMs, ciTimeoutMinutes: plan.ciTimeoutMinutes, sliceCiTimeoutMinutes: plan.sliceCiTimeoutMinutes, codexSlices: codexSlicesOf(opts.tier, entries) }, @@ -466,6 +471,7 @@ export function buildRunManifest(opts: { evalsAll: opts.evalsAll, sliceCount, selectionReason: cases.reason, + ...(host !== 'claude' ? { host } : {}), profile, selection: cases.selection, ...(cases.coverage ? { prCoverage: cases.coverage } : {}), @@ -479,6 +485,7 @@ export function parseRunManifest(raw: string): PaidRunManifest { if (parsed.version !== 1) throw new Error(`unsupported manifest version: ${(parsed as { version?: unknown }).version}`); if (!PAID_TIERS.includes(parsed.tier)) throw new Error(`manifest tier invalid: ${parsed.tier}`); if (parsed.profile !== undefined && parsed.profile !== 'pr' && parsed.profile !== 'full') throw new Error('manifest profile invalid'); + if (parsed.host !== undefined && (parsed.host === 'claude' || !EVAL_HOSTS.includes(parsed.host))) throw new Error(`manifest host invalid: ${parsed.host}`); if (parsed.selection !== undefined) { for (const [key, inventory] of [['e2e', E2E_TOUCHFILES], ['judges', LLM_JUDGE_TOUCHFILES]] as const) { const ids = parsed.selection?.[key]; diff --git a/scripts/lib/paid-report.ts b/scripts/lib/paid-report.ts index b66c41237..f92c6bb72 100644 --- a/scripts/lib/paid-report.ts +++ b/scripts/lib/paid-report.ts @@ -17,6 +17,7 @@ import { E2E_KINDS } from '../../test/helpers/touchfiles-data'; import { manualReviewProblem } from '../../test/helpers/cookie-workflow-manual-review'; import { writeNegativeReceipt, writePanelReceipt } from '../e2e-shard-reuse'; import { E2E_TOUCHFILES, E2E_TIERS, LLM_JUDGE_TOUCHFILES } from '../../test/helpers/touchfiles'; +import { type EvalHost, hostSkipReason } from '../../test/helpers/e2e-hosts'; import { CASE_TEST_NAMES, type CaseTrialPlan, caseTestNamePattern, caseTrialPlan, fileCaseRegistration, shardCaseId, shardFile, trialShardKey } from './paid-cases'; import { type ManifestEntry, PAID_TEST_DURATIONS_FILE, type PaidRunManifest, type SliceResult, collectorOutcomeCounts, formatProfileCoverage, loadPaidTestDurations, mergePaidTestDurations, parseRunManifest, trialPanelKey, verifySliceResults, writePaidTestDurations } from './paid-plan'; import { DEFAULT_JOBS, type PaidCaseSelection, type PaidTier, ROOT, type ShardTrialRecord } from './paid-types'; @@ -170,6 +171,8 @@ export interface JUnitShardContext { selection?: PaidCaseSelection; /** Ids this file shard leaves to their isolated trial shards. */ excludeCases?: readonly string[]; + /** The run's agent host (manifest.host); absent means claude. */ + host?: EvalHost; } export const DESELECTION_REASONS = { @@ -205,6 +208,8 @@ export function attributeJUnitCase(tc: JUnitCase, ctx: JUnitShardContext, const tier = tiers[id]; if (tier !== undefined && tier !== ctx.tier) return { kind: 'deselected', reason: `${tier}-tier case (this lane runs ${ctx.tier})` }; if (ctx.excludeCases?.includes(id)) return { kind: 'deselected', reason: DESELECTION_REASONS.isolated }; + const hostReason = hostSkipReason(id, ctx.host ?? 'claude'); + if (hostReason) return { kind: 'deselected', reason: hostReason }; const pool = tier !== undefined ? ctx.selection?.e2e : Object.hasOwn(LLM_JUDGE_TOUCHFILES, id) ? ctx.selection?.judges : null; if (pool && !pool.includes(id)) return { kind: 'deselected', reason: DESELECTION_REASONS.selection }; return { kind: 'case', id }; @@ -241,7 +246,7 @@ export const CENSUS_RED_GUIDE = 'docs/evals/census-red.md'; export function sliceArtifactName(root: string, slice: number, attempt: number): string { const name = path.basename(root); - return /^(?:paid-slice|gate-census)-\d+-a\d+$/.test(name) ? name : `slice ${slice} attempt ${attempt}`; + return /^(?:paid-slice|gate-census)-\d+-a\d+(?:-[a-z0-9-]+)?$/.test(name) ? name : `slice ${slice} attempt ${attempt}`; } export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact[], recordsByShard: Map, @@ -254,7 +259,7 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact if (outcome.trial || outcome.files.length !== 1) continue; let xml = ''; try { xml = fs.readFileSync(path.join(root, 'shards', shardSlug([key]), 'junit.xml'), 'utf8'); } catch { continue; } - const ctx: JUnitShardContext = { key, tier: manifest.tier, selection: manifest.selection, excludeCases: entries.get(key)?.excludeCases }; + const ctx: JUnitShardContext = { key, tier: manifest.tier, selection: manifest.selection, excludeCases: entries.get(key)?.excludeCases, host: manifest.host }; const byCase = new Map(); const reasons = new Map(); for (const tc of parseJUnitCases(xml)) { @@ -530,7 +535,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo const results = artifacts.filter(a => (a.result.attempt ?? 1) === primary).map(a => a.result); const verdict = verifySliceResults(manifest, results); const planned = manifest.entries.filter((e) => e.status === 'planned').length; - const lane = `${manifest.tier}/${manifest.profile ?? 'full'}${manifest.evalsAll ? ' census' : ''}`; + const lane = `${manifest.tier}/${manifest.profile ?? 'full'}${manifest.evalsAll ? ' census' : ''}${manifest.host ? ` on ${manifest.host}` : ''}`; console.log(`[test:paid] report: ${results.length}/${manifest.sliceCount} slices, ${planned} planned shards, tier=${manifest.tier}, attempt ${primary}${attempts.length > 1 ? ` (later attempts ${attempts.slice(1).join(', ')} reported, never replacing it)` : ''}`); for (const line of formatProfileCoverage(manifest)) console.log(line); for (const result of [...results].sort((a, b) => a.sliceIndex - b.sliceIndex)) { diff --git a/scripts/test-paid-shards.ts b/scripts/test-paid-shards.ts index 8ddee5f11..19c9b2376 100644 --- a/scripts/test-paid-shards.ts +++ b/scripts/test-paid-shards.ts @@ -73,6 +73,7 @@ import { readSessionLedger, type SessionLedgerRow } from '../test/helpers/sessio import { preflightAnthropicApi } from '../test/helpers/anthropic-preflight'; import { e2eReuseLaneProblem, prepareE2EShardReuse, selectPlanReceipts } from './e2e-shard-reuse'; import { E2E_TOUCHFILES, E2E_TIERS } from '../test/helpers/touchfiles'; +import { resolveEvalHost } from '../test/helpers/e2e-hosts'; import { scopeCodexAccess, shardFile, shardCaseId, shardTrial, trialShardKey, type CaseTrialPlan, caseTrialPlan, excludedCasesNamePattern, caseTestNamePattern, expandCaseShards, expandTrialShards, fileCaseRegistration, partitionCaseExclusions } from './lib/paid-cases'; import { retriesForFiles, trialPanelKey, sliceExecutionOrder, buildRunManifest, parseRunManifest, type SliceResult, sliceExitCode, guardTrialRecords, formatSlicePlan, formatCapacityPreflight, sliceDeadlineMs } from './lib/paid-plan'; import { caseSelection, runCaseDiagnosis, formatPanelLine, runPaidReport } from './lib/paid-report'; @@ -909,6 +910,9 @@ async function main(): Promise { if (manifest.tier !== options.tier) { throw new Error(`manifest tier ${manifest.tier} != requested tier ${options.tier} — refusing a cross-tier run`); } + if ((manifest.host ?? 'claude') !== resolveEvalHost()) { + throw new Error(`manifest host ${manifest.host ?? 'claude'} != EVALS_HOST ${resolveEvalHost()} — refusing a cross-host run`); + } const profile = manifest.profile ?? 'full'; if (options.profileExplicit && options.profile !== profile) throw new Error(`manifest profile ${profile} != requested profile ${options.profile}`); if (options.sliceIndex > manifest.sliceCount) { diff --git a/setup-browser-cookies/SKILL.md b/setup-browser-cookies/SKILL.md index 1aa2a0433..830f9c7c2 100644 --- a/setup-browser-cookies/SKILL.md +++ b/setup-browser-cookies/SKILL.md @@ -152,83 +152,59 @@ Skills that run plan reviews (`/plan-*-review`, `/codex review`) include the EXI # Setup Browser Cookies -## 1. Choose the browser +## 1. Check for Aside -Use this checkout as the gstack root if it contains `BROWSER.md` and `browse/SKILL.md`; otherwise use the installed root containing `bin/gstack-skill-start`, never a generated host stub. Read that root's `browse/SKILL.md` **BROWSER SETUP** section and run its probe first. On `READY`, stop importing: use Aside's sessions or ask the user to sign in there. Otherwise follow the probe's fallback handling, then continue below. +Use this checkout as the gstack root if it contains `BROWSER.md` and `browse/SKILL.md`; otherwise use the installed root containing `bin/gstack-skill-start`, never a generated host stub. Read that root's `browse/SKILL.md` **BROWSER SETUP** section and run its Aside probe. -## SETUP (run this check BEFORE any browse command) +- **Aside probe prints `READY`:** no import is needed, because Aside already has the user's signed-in sessions. Tell the user to use Aside, or to sign in there, and end this skill. +- **Any other result:** continue with gstack's own headless browser (`$B`). + +## 2. Get every decision from the user first + +Ask these together, before running any `$B` command. Never guess an answer or treat the default Comet browser as consent. + +1. **Source browser and profile.** `--profile` takes the profile's directory (`Default`, `Profile 2`), not its display name. A profile that cannot be read is unknown, not empty. +2. **Domains to import,** and the **target**: one page on a chosen domain that the session should open, by URL. +3. **Sign-in verification (`--verify-auth`)?** Default no. It reloads the target and reports **verified** only after a successful same-origin response with exactly one visible element, found by a CSS selector, whose whitespace-normalized text equals the expected identity. If yes, the user supplies the selector and the expected identity privately. +4. **Clear stale storage (`--clear-storage`)?** Default no; needs explicit approval. Chromium only. It clears localStorage for the target's origin (shared by that origin's tabs) and sessionStorage for the target tab, and nothing else; other browser engines reject the reset but still import. A failed reset may already have cleared some storage and is not rolled back. +5. **Every cookie in the profile (`--all`)?** Default no; needs explicit consent. It cannot be combined with a domain or with clearing storage. + +Use the picker when any of answers 1-2 is still open; it asks for them. Use direct import when the user has named the browser, the profile directory and the domains. + +## 3. Prepare gstack's browser + +1. If the user wants verification, have them set `GSTACK_COOKIE_AUTH_SELECTOR` and `GSTACK_COOKIE_AUTH_EXPECTED_IDENTITY` privately in the shell that runs `$B`. The browser daemon reads them only when it starts, so if a gstack daemon is already running, run `$B stop` first. Never invent the values. +2. Follow `browse/SKILL.md`'s **Browser fallback** section to find `$B`, building it if the check prints `NEEDS_SETUP`. +3. Run `$B status`. If it reports `Mode: cdp`, `$B` is attached to the user's real browser and already has its sessions: tell the user no import is needed and end this skill. +4. Open the target URL from answer 2 and leave that tab alone until the import finishes: ```bash -_ROOT=$(git rev-parse --show-toplevel 2>/dev/null) -B="" -[ -n "$_ROOT" ] && [ -x "$_ROOT/.claude/skills/gstack/browse/dist/browse" ] && B="$_ROOT/.claude/skills/gstack/browse/dist/browse" -[ -z "$B" ] && B="$HOME/.claude/skills/gstack/browse/dist/browse" -if [ -x "$B" ] && "${B%/browse/*}/bin/gstack-browser-ensure"; then - echo "READY: $B" -else - echo "NEEDS_SETUP" -fi +$B goto "" ``` -`gstack-browser-ensure` installs Chromium on first use when the install skipped it (lazy browser); it prints `BROWSER_OK` or `BROWSER_UNAVAILABLE ` first. +## 4. Import -If `NEEDS_SETUP`: -1. Tell the user: "gstack browse needs a one-time build (~10 seconds). OK to proceed?" Then STOP and wait. -2. Run: `cd && ./setup` -3. If `bun` is not installed: - ```bash - if ! command -v bun >/dev/null 2>&1; then - BUN_VERSION="1.4.2" - BUN_INSTALL_SHA="bab8acfb046aac8c72407bdcce903957665d655d7acaa3e11c7c4616beae68dd" - tmpfile=$(mktemp "${TMPDIR:-/tmp}/bun-install.XXXXXX") - curl -fsSL "https://bun.sh/install" -o "$tmpfile" - # shasum is macOS/perl; coreutils-only Linux ships sha256sum instead — - # resolve whichever exists so the verify never fails on a missing tool. - if command -v sha256sum >/dev/null 2>&1; then - actual_sha=$(sha256sum < "$tmpfile" | awk '{print $(1)}') - else - actual_sha=$(shasum -a 256 < "$tmpfile" | awk '{print $(1)}') - fi - if [ "$actual_sha" != "$BUN_INSTALL_SHA" ]; then - echo "ERROR: bun install script checksum mismatch" >&2 - echo " expected: $BUN_INSTALL_SHA" >&2 - echo " got: $actual_sha" >&2 - rm "$tmpfile"; exit 1 - fi - bash "$tmpfile" "bun-v$BUN_VERSION" - rm "$tmpfile" - fi - ``` - -```bash -$B status -``` -If status says `Mode: cdp`, stop: the real browser already has sessions. - -## 2. Confirm options before import - -Open the known target and keep its tab unchanged. Both options default off and require explicit request: - -- **`--verify-auth` / picker checkbox:** reloads the target. Have the user privately configure daemon `GSTACK_COOKIE_AUTH_SELECTOR` and `GSTACK_COOKIE_AUTH_EXPECTED_IDENTITY` **before startup**. Never invent values or assume CLI env reconfigures an existing daemon. Missing config rejects before mutation. Require a successful same-origin response and exactly one visible element whose whitespace-normalized text exactly matches the expected identity. -- **`--clear-storage`:** for suspected stale storage, obtain explicit approval. Chromium only: clears captured-origin localStorage (shared across same-origin context tabs) and target-tab sessionStorage. Other origins, other tabs' sessionStorage, IndexedDB, and service workers stay intact. It uses an isolated world/native deadline; other engines reject reset, not imports/auth checks. Never auto-approve or claim rollback after partial failure. - -## 3. Select source and scope +Picker (the user confirms the browser, profile and domains there, ticks verification if they asked for it, then tells you when done; the link works once, for five minutes, so rerun the command if it expires): ```bash $B cookie-import-browser ``` -Ask the user to choose browser, account/profile, and domains, then say when done. Never guess accounts or treat default Comet as consent. Unreadable profiles are unknown, not empty. Rerun for an expired five-minute one-use link. +Direct import: run `$B cookie-import-browser` with the chosen browser name, one `--domain` per run and `--profile` set to the profile directory (not its display name); add `--verify-auth` or `--clear-storage` only if the user said yes in step 2. -Direct import: pass the chosen browser and `--domain` after navigating to a matching target. `--profile` takes a directory, not a display name; omit only for an unambiguous relevant profile, otherwise use the picker. `--all` requires consent for all non-expired profile cookies; it cannot accompany `--domain` or `--clear-storage`. +If the import rejects because verification settings are missing, nothing changed: tell the user which variable is missing and return to step 3. Read that same root's `BROWSER.md`, **Choosing a source and checking sign-in**, for other errors and receipt failure reasons; it has the error tables, examples, profile labels, supported sources and platform setup. -Read that same root's `BROWSER.md`, **Choosing a source and checking sign-in**, for examples, profile labels, supported sources and platform setup. +## 5. Report honestly -## 4. Report honestly +Report from the import receipt, not from raw `$B cookies`, as three lines: -Report receipt/picker counts, partial/zero/error and reset outcomes, not raw `$B cookies`. Imports affect the context, not one tab. **Not checked** means no requested check; **not verified** means it failed; **verified** requires positive target evidence. Zero imports, counts, or HTTP 200 never prove login. +- `Imported: (, as the receipt says) from for ` +- `Storage reset: ` +- `Sign-in: ` -Never request/publish cookie values, passwords, identity/profile text, session details, or raw errors in public logs. +**Not checked** means verification was not requested; **not verified** means it ran and failed; **verified** needs the identity match. Cookie counts, zero imports and HTTP 200 never prove sign-in. Imported cookies apply to the whole browser context, not one tab. + +Never request or publish cookie values, passwords, identity or profile text, session details, or raw errors in public logs. ## Platform boundaries diff --git a/setup-browser-cookies/SKILL.md.tmpl b/setup-browser-cookies/SKILL.md.tmpl index f6ed3f61e..82313c668 100644 --- a/setup-browser-cookies/SKILL.md.tmpl +++ b/setup-browser-cookies/SKILL.md.tmpl @@ -21,41 +21,59 @@ allowed-tools: # Setup Browser Cookies -## 1. Choose the browser +## 1. Check for Aside -Use this checkout as the gstack root if it contains `BROWSER.md` and `browse/SKILL.md`; otherwise use the installed root containing `bin/gstack-skill-start`, never a generated host stub. Read that root's `browse/SKILL.md` **BROWSER SETUP** section and run its probe first. On `READY`, stop importing: use Aside's sessions or ask the user to sign in there. Otherwise follow the probe's fallback handling, then continue below. +Use this checkout as the gstack root if it contains `BROWSER.md` and `browse/SKILL.md`; otherwise use the installed root containing `bin/gstack-skill-start`, never a generated host stub. Read that root's `browse/SKILL.md` **BROWSER SETUP** section and run its Aside probe. -{{BROWSE_SETUP}} +- **Aside probe prints `READY`:** no import is needed, because Aside already has the user's signed-in sessions. Tell the user to use Aside, or to sign in there, and end this skill. +- **Any other result:** continue with gstack's own headless browser (`$B`). + +## 2. Get every decision from the user first + +Ask these together, before running any `$B` command. Never guess an answer or treat the default Comet browser as consent. + +1. **Source browser and profile.** `--profile` takes the profile's directory (`Default`, `Profile 2`), not its display name. A profile that cannot be read is unknown, not empty. +2. **Domains to import,** and the **target**: one page on a chosen domain that the session should open, by URL. +3. **Sign-in verification (`--verify-auth`)?** Default no. It reloads the target and reports **verified** only after a successful same-origin response with exactly one visible element, found by a CSS selector, whose whitespace-normalized text equals the expected identity. If yes, the user supplies the selector and the expected identity privately. +4. **Clear stale storage (`--clear-storage`)?** Default no; needs explicit approval. Chromium only. It clears localStorage for the target's origin (shared by that origin's tabs) and sessionStorage for the target tab, and nothing else; other browser engines reject the reset but still import. A failed reset may already have cleared some storage and is not rolled back. +5. **Every cookie in the profile (`--all`)?** Default no; needs explicit consent. It cannot be combined with a domain or with clearing storage. + +Use the picker when any of answers 1-2 is still open; it asks for them. Use direct import when the user has named the browser, the profile directory and the domains. + +## 3. Prepare gstack's browser + +1. If the user wants verification, have them set `GSTACK_COOKIE_AUTH_SELECTOR` and `GSTACK_COOKIE_AUTH_EXPECTED_IDENTITY` privately in the shell that runs `$B`. The browser daemon reads them only when it starts, so if a gstack daemon is already running, run `$B stop` first. Never invent the values. +2. Follow `browse/SKILL.md`'s **Browser fallback** section to find `$B`, building it if the check prints `NEEDS_SETUP`. +3. Run `$B status`. If it reports `Mode: cdp`, `$B` is attached to the user's real browser and already has its sessions: tell the user no import is needed and end this skill. +4. Open the target URL from answer 2 and leave that tab alone until the import finishes: ```bash -$B status +$B goto "" ``` -If status says `Mode: cdp`, stop: the real browser already has sessions. -## 2. Confirm options before import +## 4. Import -Open the known target and keep its tab unchanged. Both options default off and require explicit request: - -- **`--verify-auth` / picker checkbox:** reloads the target. Have the user privately configure daemon `GSTACK_COOKIE_AUTH_SELECTOR` and `GSTACK_COOKIE_AUTH_EXPECTED_IDENTITY` **before startup**. Never invent values or assume CLI env reconfigures an existing daemon. Missing config rejects before mutation. Require a successful same-origin response and exactly one visible element whose whitespace-normalized text exactly matches the expected identity. -- **`--clear-storage`:** for suspected stale storage, obtain explicit approval. Chromium only: clears captured-origin localStorage (shared across same-origin context tabs) and target-tab sessionStorage. Other origins, other tabs' sessionStorage, IndexedDB, and service workers stay intact. It uses an isolated world/native deadline; other engines reject reset, not imports/auth checks. Never auto-approve or claim rollback after partial failure. - -## 3. Select source and scope +Picker (the user confirms the browser, profile and domains there, ticks verification if they asked for it, then tells you when done; the link works once, for five minutes, so rerun the command if it expires): ```bash $B cookie-import-browser ``` -Ask the user to choose browser, account/profile, and domains, then say when done. Never guess accounts or treat default Comet as consent. Unreadable profiles are unknown, not empty. Rerun for an expired five-minute one-use link. +Direct import: run `$B cookie-import-browser` with the chosen browser name, one `--domain` per run and `--profile` set to the profile directory (not its display name); add `--verify-auth` or `--clear-storage` only if the user said yes in step 2. -Direct import: pass the chosen browser and `--domain` after navigating to a matching target. `--profile` takes a directory, not a display name; omit only for an unambiguous relevant profile, otherwise use the picker. `--all` requires consent for all non-expired profile cookies; it cannot accompany `--domain` or `--clear-storage`. +If the import rejects because verification settings are missing, nothing changed: tell the user which variable is missing and return to step 3. Read that same root's `BROWSER.md`, **Choosing a source and checking sign-in**, for other errors and receipt failure reasons; it has the error tables, examples, profile labels, supported sources and platform setup. -Read that same root's `BROWSER.md`, **Choosing a source and checking sign-in**, for examples, profile labels, supported sources and platform setup. +## 5. Report honestly -## 4. Report honestly +Report from the import receipt, not from raw `$B cookies`, as three lines: -Report receipt/picker counts, partial/zero/error and reset outcomes, not raw `$B cookies`. Imports affect the context, not one tab. **Not checked** means no requested check; **not verified** means it failed; **verified** requires positive target evidence. Zero imports, counts, or HTTP 200 never prove login. +- `Imported: (, as the receipt says) from for ` +- `Storage reset: ` +- `Sign-in: ` -Never request/publish cookie values, passwords, identity/profile text, session details, or raw errors in public logs. +**Not checked** means verification was not requested; **not verified** means it ran and failed; **verified** needs the identity match. Cookie counts, zero imports and HTTP 200 never prove sign-in. Imported cookies apply to the whole browser context, not one tab. + +Never request or publish cookie values, passwords, identity or profile text, session details, or raw errors in public logs. ## Platform boundaries diff --git a/test/ci-native-evidence.test.ts b/test/ci-native-evidence.test.ts index 318d5a9f3..c7da89696 100644 --- a/test/ci-native-evidence.test.ts +++ b/test/ci-native-evidence.test.ts @@ -3,6 +3,7 @@ import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { runPaidShard, shardSlug } from '../scripts/test-paid-shards'; +import { CENSUS_ARMS } from '../scripts/census-arms'; const ROOT = path.resolve(import.meta.dir, '..'); const workflows = ['evals.yml', 'evals-periodic.yml', 'evals-marathon.yml'].map(name => ({ @@ -52,19 +53,24 @@ test('every direct CI paid executor binds a safe unique run/attempt/job/slice id else expect(step.run).toContain('--user "$(id -u):$(id -g)"'); const env = executorEnv(job, step); expect(env.EVALS_RUN_ID).toBeString(); + // The gate census matrix is one row per census arm and planned slice (scripts/census-arms.ts). + const armed = job.strategy.matrix.include !== undefined; for (const run of ['36302678692', '36302678693']) { for (const attempt of ['1', '2']) { // The planner sizes the matrix; cover more slices than any live plan. - expect(job.strategy.matrix.slice).toMatch(/^\$\{\{ fromJSON\(needs\.plan-slices\.outputs\.(?:[a-z]+_)*slices\) \}\}$/); - for (let slice = 1; slice <= 64; slice++) { - const id = render(env.EVALS_RUN_ID, { - 'github.run_id': `${run}${workflowIndex}`, - 'github.run_attempt': attempt, 'matrix.slice': String(slice), - }); - expect(id).toMatch(/^[A-Za-z0-9_-]+$/); - expect(id.length).toBeLessThan(120); - expect(ids.has(id)).toBe(false); - ids.add(id); + expect(armed ? job.strategy.matrix.include : job.strategy.matrix.slice) + .toMatch(armed ? /^\$\{\{ fromJSON\(needs\.plan-slices\.outputs\.gate_matrix\) \}\}$/ : /^\$\{\{ fromJSON\(needs\.plan-slices\.outputs\.(?:[a-z]+_)*slices\) \}\}$/); + for (const arm of armed ? Object.keys(CENSUS_ARMS) : ['']) { + for (let slice = 1; slice <= 64; slice++) { + const id = render(env.EVALS_RUN_ID, { + 'github.run_id': `${run}${workflowIndex}`, + 'github.run_attempt': attempt, 'matrix.slice': String(slice), 'matrix.arm': arm, + }); + expect(id).toMatch(/^[A-Za-z0-9_-]+$/); + expect(id.length).toBeLessThan(120); + expect(ids.has(id)).toBe(false); + ids.add(id); + } } } } @@ -79,7 +85,7 @@ for (const { name, jobName, job, step } of executors) { fs.mkdirSync(home); fs.mkdirSync(bin); fs.mkdirSync(path.join(root, 'test')); const configured = executorEnv(job, step); const runId = render(configured.EVALS_RUN_ID, { - 'github.run_id': '36302678692', 'github.run_attempt': '2', 'matrix.slice': '4', + 'github.run_id': '36302678692', 'github.run_attempt': '2', 'matrix.slice': '4', 'matrix.arm': 'codex-sol', }); const evalDir = path.join(root, path.basename(configured.GSTACK_EVAL_DIR)); const file = 'test/native-launch.test.ts'; diff --git a/test/ci-paid-coordination.test.ts b/test/ci-paid-coordination.test.ts index b92375e11..f248d76a9 100644 --- a/test/ci-paid-coordination.test.ts +++ b/test/ci-paid-coordination.test.ts @@ -113,7 +113,8 @@ describe('paid CI coordination stays off the eval image', () => { // A confirmed verdict reuse (R2a/R2) is the one PR run that executes no slice. expect(executor.if).toBe("${{ !cancelled() && needs.build-image.result == 'success' && needs.plan-slices.result == 'success' && needs.plan-slices.outputs.gate_reuse != 'true' }}"); } else { - expect(executor.if).toBeUndefined(); + // A census-only run (monthly cron or census_arms dispatch) executes no periodic slice. + expect(executor.if).toBe("${{ needs.plan-slices.outputs.periodic_lane == 'true' }}"); } expect(executor.steps.some(step => step.run === 'bun run build')).toBe(true); expect(executor.steps.some(step => step.uses === './.github/actions/restore-deps')).toBe(true); diff --git a/test/codex-e2e-sol-scope.test.ts b/test/codex-e2e-sol-scope.test.ts index 862f7b8de..c3f59a619 100644 --- a/test/codex-e2e-sol-scope.test.ts +++ b/test/codex-e2e-sol-scope.test.ts @@ -180,14 +180,14 @@ TODO: consider migrating this example to a larger configuration framework. name: 'codex-sol-scope-termination', suite: 'codex-e2e-sol-scope', budgetMs: CAPTURE_MS, - model: 'gpt-5.6-sol', + model: 'gpt-6.1-sol', outputLimit: Infinity, run: (signal) => runCodexSkill({ skillDir, skillName: 'gstack-investigate', cwd: scratch, sandbox: 'workspace-write', - model: 'gpt-5.6-sol', + model: 'gpt-6.1-sol', configOverrides: ['model_reasoning_effort="high"'], ignoreUserConfig: true, timeoutMs: CODEX_TIMEOUT_MS, diff --git a/test/codex-host.test.ts b/test/codex-host.test.ts new file mode 100644 index 000000000..0976a72e6 --- /dev/null +++ b/test/codex-host.test.ts @@ -0,0 +1,287 @@ +/** + * The Codex host for paid gate cases (GSTA-49 TM2): host selection with + * skip-with-reason (never a failure), per-case host metadata over every gate + * case, real Codex cost accounting against captured `codex exec --json` + * usage events (Codex CLI 0.160.0, gpt-6.1-sol), and runner parity. + */ +import { afterEach, describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { CLAUDE_ONLY_REASONS, E2E_HOSTS, hostSkipReason, resolveEvalHost } from './helpers/e2e-hosts'; +import { E2E_TIERS, E2E_TOUCHFILES } from './helpers/touchfiles'; +import { addCodexTurnUsage, codexUsageCostUsd, NO_CODEX_USAGE, parseCodexJSONL, type CodexResult } from './helpers/codex-session-runner'; +import { runRecordedCodexEval } from './helpers/codex-eval'; +import { codexHostExecArgs, codexHostModel, codexHostToolCalls, codexHostTranscript, unwrapShellCommand } from './helpers/codex-host-runner'; +import { coverageAuditReadEvidence } from './helpers/coverage-audit-evidence'; +import { runSkillTest } from './helpers/session-runner'; +import { readSessionLedger } from './helpers/session-ledger'; +import { preflightAnthropicApi } from './helpers/anthropic-preflight'; +import type { EvalTestEntry } from './helpers/eval-store'; +import { buildRunManifest } from '../scripts/test-paid-shards'; +import { codexShardAccess, fileCaseRegistration, partitionHostExclusions, scopeCodexAccess, shardCaseId, shardFile } from '../scripts/lib/paid-cases'; +import { parseRunManifest } from '../scripts/lib/paid-plan'; +import { attributeJUnitCase } from '../scripts/lib/paid-report'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const fixture = (name: string) => fs.readFileSync(path.join(ROOT, 'test/fixtures/codex-host', name), 'utf8').split('\n').filter(Boolean); +const COMMANDS = fixture('exec-commands.jsonl'); +const FILE_CHANGE = fixture('exec-file-change.jsonl'); +const gateCases = Object.keys(E2E_TIERS).filter(id => E2E_TIERS[id] === 'gate').sort(); +const adapterCases = Object.keys(E2E_HOSTS).filter(id => E2E_HOSTS[id] === 'both'); +const scratch: string[] = []; +const tempDir = (prefix: string) => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); scratch.push(dir); return dir; }; +afterEach(() => { for (const dir of scratch.splice(0)) fs.rmSync(dir, { recursive: true, force: true }); }); + +describe('host selection: Claude-only cases skip with a reason, never fail', () => { + test('EVALS_HOST defaults to claude, accepts codex and refuses anything else', () => { + expect(resolveEvalHost({})).toBe('claude'); + expect(resolveEvalHost({ EVALS_HOST: ' ' })).toBe('claude'); + expect(resolveEvalHost({ EVALS_HOST: 'codex' })).toBe('codex'); + expect(() => resolveEvalHost({ EVALS_HOST: 'gemini' })).toThrow('EVALS_HOST must be one of claude, codex'); + }); + + test('the Anthropic preflight never pings claude on the Codex host', () => { + const pinged: string[] = []; + const spawn = ((cmd: string) => { pinged.push(cmd); return { status: 127, stdout: '', stderr: '' }; }) as unknown as typeof spawnSync; + expect(preflightAnthropicApi({ EVALS_HOST: 'codex' }, spawn)).toBe('skipped'); + expect(pinged).toEqual([]); + expect(() => preflightAnthropicApi({}, spawn)).toThrow('`claude` not found on PATH'); + expect(pinged).toEqual(['sh']); + }); + + test('every case runs on Claude; on Codex only adapter cases run and the rest carry their reason', () => { + for (const id of gateCases) expect(hostSkipReason(id, 'claude')).toBeNull(); + for (const id of adapterCases) expect(hostSkipReason(id, 'codex')).toBeNull(); + expect(hostSkipReason('plan-mode-no-op', 'codex')).toBe(`Claude-only on the codex host: ${CLAUDE_ONLY_REASONS.pty}`); + expect(hostSkipReason('ship-docsync-store', 'codex')).toContain('not ported to the Codex host'); + expect(hostSkipReason('plan-ceo-review', 'codex')).toContain('no Codex host metadata'); + }); + + test('the module selection drops Claude-only cases on the Codex host', () => { + const root = tempDir('codex-host-root-'); + fs.mkdirSync(path.join(root, 'home', '.codex'), { recursive: true }); + const run = spawnSync('bun', ['-e', "const h = await import('./test/helpers/e2e-helpers'); console.log(JSON.stringify(h.selectedTests))"], { + cwd: ROOT, encoding: 'utf8', timeout: 60_000, + env: { ...process.env, EVALS: '1', EVALS_TIER: 'gate', EVALS_HOST: 'codex', EVALS_PREFLIGHT_OK: '1', GSTACK_CODEX_HOST_ROOT: root, + EVALS_SELECTION_JSON: JSON.stringify({ selected: null, reason: 'test' }) }, + }); + expect(run.status, run.stderr).toBe(0); + expect((JSON.parse(run.stdout.trim().split('\n').at(-1)!) as string[]).sort()).toEqual([...adapterCases].sort()); + expect(run.stderr).toContain(`EVALS_HOST=codex: ${adapterCases.length} tests; ${gateCases.length - adapterCases.length} Claude-only skipped`); + }); + + test('the planner excludes Claude-only shards with their reason and plans every adapter case', () => { + const manifest = buildRunManifest({ tier: 'gate', profile: 'full', sliceBudgetMs: 420_000, jobs: 2, evalsAll: true, skipJudges: true, + env: { EVALS_ALL: '1', EVALS_HOST: 'codex' }, rootDir: ROOT }); + expect(manifest.host).toBe('codex'); + const planned = manifest.entries.filter(entry => entry.status === 'planned'); + const casesOf = (key: string) => { + const id = shardCaseId(key); + if (id !== null) return [id]; + return fileCaseRegistration(shardFile(key), fs.readFileSync(path.join(ROOT, shardFile(key)), 'utf8')).registered.filter(c => E2E_TIERS[c] === 'gate'); + }; + const reachable = new Set(planned.flatMap(entry => casesOf(entry.file)).filter(id => E2E_HOSTS[id] === 'both')); + expect([...reachable].sort()).toEqual([...adapterCases].sort()); + for (const entry of planned) expect(casesOf(entry.file).some(id => E2E_HOSTS[id] === 'both'), entry.file).toBe(true); + const hostExcluded = manifest.entries.filter(entry => entry.status === 'excluded' && entry.reason?.includes('Claude-only on the codex host')); + expect(hostExcluded.length).toBeGreaterThan(0); + for (const entry of hostExcluded) expect(entry.reason, entry.file).toStartWith('skipped: '); + expect(manifest.entries.some(entry => entry.file === 'test/skill-e2e-ship-docsync.test.ts#ship-docsync-store' && entry.status === 'excluded')).toBe(true); + }); + + test('the Claude host plans exactly as before, with no host on the manifest', () => { + const manifest = buildRunManifest({ tier: 'gate', profile: 'full', sliceBudgetMs: 420_000, jobs: 2, evalsAll: true, skipJudges: true, + env: { EVALS_ALL: '1' }, rootDir: ROOT }); + expect(manifest.host).toBeUndefined(); + expect(manifest.entries.some(entry => entry.reason?.includes('Claude-only'))).toBe(false); + expect(partitionHostExclusions(['a.test.ts', 'b.test.ts#x'], 'gate', 'claude')).toEqual({ runnable: ['a.test.ts', 'b.test.ts#x'], excluded: [] }); + }); + + test('the report deselects a skipped Claude-only testcase on the Codex host (zero credit, never SKIPPED or failed)', () => { + const tc = { name: 'review-dashboard-via', classname: '', outcome: 'skipped' as const, timeMs: 0, line: 10 }; + const key = 'test/skill-e2e-review-attribution.test.ts'; + expect(attributeJUnitCase(tc, { key, tier: 'gate', host: 'codex' })).toEqual({ kind: 'deselected', reason: hostSkipReason('review-dashboard-via', 'codex')! }); + expect(attributeJUnitCase(tc, { key, tier: 'gate' })).toEqual({ kind: 'case', id: 'review-dashboard-via' }); + expect(attributeJUnitCase({ ...tc, name: 'review-base-branch' }, { key, tier: 'gate', host: 'codex' })).toEqual({ kind: 'case', id: 'review-base-branch' }); + }); + + test('a manifest names only a non-default host, and the executor refuses a cross-host run', () => { + const base = { version: 1, tier: 'gate', evalsAll: true, sliceCount: 1, selectionReason: 'x', entries: [] }; + expect(parseRunManifest(JSON.stringify({ ...base, host: 'codex' })).host).toBe('codex'); + expect(() => parseRunManifest(JSON.stringify({ ...base, host: 'claude' }))).toThrow('manifest host invalid'); + expect(() => parseRunManifest(JSON.stringify({ ...base, host: 'gemini' }))).toThrow('manifest host invalid'); + const plan = path.join(tempDir('codex-host-plan-'), 'manifest.json'); + fs.writeFileSync(plan, JSON.stringify({ ...base, host: 'codex' })); + const run = spawnSync('bun', ['run', 'scripts/test-paid-shards.ts', '--tier', 'gate', '--plan', plan, '--slice', '1', '--list'], { + cwd: ROOT, encoding: 'utf8', timeout: 60_000, env: { ...process.env, EVALS_HOST: 'claude', EVALS_TIER: 'gate' }, + }); + expect(run.status).not.toBe(0); + expect(run.stderr + run.stdout).toContain('manifest host codex != EVALS_HOST claude'); + }); +}); + +describe('per-case host metadata', () => { + test('covers every gate case exactly once, and nothing else', () => { + expect(Object.keys(E2E_HOSTS).sort()).toEqual(gateCases); + for (const id of Object.keys(E2E_HOSTS)) expect(Object.hasOwn(E2E_TOUCHFILES, id), id).toBe(true); + }); + + test('classifies 52 adapter-only, 29 Claude-specific, 26 rework and 1 host-neutral case', () => { + const counts: Record = {}; + for (const support of Object.values(E2E_HOSTS)) counts[support] = (counts[support] ?? 0) + 1; + expect(counts.both).toBe(52); + expect(counts.rework).toBe(26); + expect(counts['host-neutral']).toBe(1); + expect(['pty', 'sdk', 'subagents', 'skill-tool', 'hermetic', 'self-host'].reduce((sum, reason) => sum + (counts[reason] ?? 0), 0)).toBe(29); + }); +}); + +describe('Codex cost accounting from captured usage events', () => { + test('parses every token class of turn.completed (cache reads and writes are subsets of input)', () => { + const commands = parseCodexJSONL(COMMANDS); + expect(commands.usage).toEqual({ input: 25009, cached_input: 12437, cache_write_input: 12566, output: 86, reasoning_output: 0 }); + expect(commands.tokens).toBe(25095); + expect(parseCodexJSONL(FILE_CHANGE).usage).toEqual({ input: 25020, cached_input: 12440, cache_write_input: 12574, output: 137, reasoning_output: 0 }); + expect(parseCodexJSONL(['{"type":"thread.started","thread_id":"t"}']).usage).toBeNull(); + const twoTurns = [COMMANDS, FILE_CHANGE].map(lines => JSON.parse(lines.find(line => line.includes('turn.completed'))!).usage) + .reduce(addCodexTurnUsage, NO_CODEX_USAGE); + expect(twoTurns).toEqual({ input: 50029, cached_input: 24877, cache_write_input: 25140, output: 223, reasoning_output: 0 }); + }); + + test('prices with lib/pricing.ts: uncached input (cache writes included) at input, cache reads at the cached rate', () => { + const usage = parseCodexJSONL(COMMANDS).usage!; + // gpt-6.1-sol: 12,572 x $2 + 12,437 x $0.10 + 86 x $10 per million. + expect(codexUsageCostUsd(usage, 'gpt-6.1-sol')).toBe(0.027248); + expect(codexUsageCostUsd(parseCodexJSONL(FILE_CHANGE).usage!, 'gpt-6.1-sol')).toBe(0.027774); + // gpt-6-astra: 12,572 x $10 + 12,437 x $1 + 86 x $50 per million. + expect(codexUsageCostUsd(usage, 'gpt-6-astra')).toBe(0.142457); + expect(codexUsageCostUsd(usage, 'gpt-0-unpriced')).toBeNull(); + }); + + test('runRecordedCodexEval records real cost and billing instead of cost_known: false', async () => { + const evalDir = tempDir('codex-host-ledger-'); + const previous = process.env.GSTACK_EVAL_DIR; + process.env.GSTACK_EVAL_DIR = evalDir; + try { + const recorded: EvalTestEntry[] = []; + const result: CodexResult = { ...parseCodexJSONL(COMMANDS), exitCode: 0, durationMs: 5, rawLines: COMMANDS, stderr: '', model: 'gpt-6.1-sol' }; + await runRecordedCodexEval({ name: 'cost-case', suite: 's', budgetMs: 10_000, run: async () => result, validate: () => {}, record: entry => recorded.push(entry) }); + expect(recorded[0]).toMatchObject({ passed: true, cost_usd: 0.027248, model: 'gpt-6.1-sol', token_usage: result.usage }); + expect(recorded[0]!.cost_known).toBeUndefined(); + const noTurn: CodexResult = { ...parseCodexJSONL([]), exitCode: 0, durationMs: 5, rawLines: [], stderr: '' }; + await runRecordedCodexEval({ name: 'no-turn', suite: 's', budgetMs: 10_000, run: async () => noTurn, validate: () => {}, record: entry => recorded.push(entry) }); + expect(recorded[1]).toMatchObject({ cost_usd: 0, cost_known: false }); + const rows = readSessionLedger(evalDir); + expect(rows.map(row => [row.test_name, row.billed])).toEqual([['cost-case', true], ['no-turn', false]]); + expect(rows[0]).toMatchObject({ runner: 'codex', model: 'gpt-6.1-sol', cost_usd: 0.027248, tokens: result.usage }); + } finally { + if (previous === undefined) delete process.env.GSTACK_EVAL_DIR; else process.env.GSTACK_EVAL_DIR = previous; + } + }); +}); + +describe('Codex runner parity', () => { + test('maps command_execution to Bash (unwrapped, with output) and file_change to Write/Edit', () => { + expect(codexHostToolCalls(COMMANDS)).toEqual([ + { tool: 'Bash', input: { command: "printf 'hi' > hello.txt\nprintf 'hello' > hello.txt\ncat hello.txt" }, output: 'hello' }, + ]); + expect(codexHostToolCalls(FILE_CHANGE)).toEqual([ + { tool: 'Write', input: { file_path: '/tmp/codex-host-fixture/notes.md', kind: 'add' }, output: '' }, + { tool: 'Edit', input: { file_path: '/tmp/codex-host-fixture/notes.md', kind: 'update' }, output: '' }, + ]); + }); + + test('the transcript is the Claude stream-json shape the case observers read', () => { + const output = '`cat hello.txt` output: `hello`'; + const transcript = codexHostTranscript(COMMANDS, '/tmp/codex-host-fixture', output); + expect(transcript[0]).toEqual({ type: 'system', subtype: 'init', cwd: '/tmp/codex-host-fixture', session_id: '01a150b3-ae5a-7932-ba51-4c295c66adf4', tools: [] }); + expect(transcript.slice(1).map(event => [event.type, event.message?.content[0].type ?? event.subtype])).toEqual([ + ['assistant', 'text'], ['assistant', 'tool_use'], ['user', 'tool_result'], ['assistant', 'text'], ['result', 'success']]); + expect(transcript[2].message.content[0]).toEqual({ type: 'tool_use', id: 'item_1', name: 'Bash', input: { command: "printf 'hi' > hello.txt\nprintf 'hello' > hello.txt\ncat hello.txt" } }); + expect(transcript[3].message.content[0]).toEqual({ type: 'tool_result', tool_use_id: 'item_1', content: 'hello', is_error: false }); + expect(transcript.at(-1)).toMatchObject({ type: 'result', result: output }); + const edits = codexHostTranscript(FILE_CHANGE, '/tmp/codex-host-fixture', '').filter(event => event.type === 'assistant' && event.message.content[0].type === 'tool_use'); + expect(edits.map(event => [event.message.content[0].name, event.message.content[0].input.file_path])).toEqual([ + ['Write', '/tmp/codex-host-fixture/notes.md'], ['Edit', '/tmp/codex-host-fixture/notes.md']]); + // A Codex `cat` of a fixture file is a read the coverage-audit evidence credits, as a native Bash read is. + const source = { path: '/w/src/billing.ts', content: 'export const a = 1;\n' }; + const tests = { path: '/w/test/billing.test.ts', content: 'test("a", () => {});\n' }; + const read = (id: string, file: string, out: string) => JSON.stringify({ type: 'item.completed', item: { id, type: 'command_execution', command: `/bin/bash -lc 'cat ${file}'`, aggregated_output: out, exit_code: 0, status: 'completed' } }); + const lines = ['{"type":"thread.started","thread_id":"t1"}', read('item_0', 'src/billing.ts', source.content), read('item_1', 'test/billing.test.ts', tests.content)]; + expect(coverageAuditReadEvidence(codexHostTranscript(lines, '/w', ''), { cwd: '/w', source, tests })).toEqual({ sourceRead: true, testsRead: true }); + const failed = lines.map(line => line.replace('"exit_code":0', '"exit_code":1')); + expect(coverageAuditReadEvidence(codexHostTranscript(failed, '/w', ''), { cwd: '/w', source, tests })).toEqual({ sourceRead: false, testsRead: false }); + }); + + test('unwraps only a single well-formed bash -lc word', () => { + expect(unwrapShellCommand("/bin/bash -lc 'git diff main...HEAD'")).toBe('git diff main...HEAD'); + expect(unwrapShellCommand("/bin/bash -lc 'echo '\"'\"'x'\"'\"''")).toBe("echo 'x'"); + expect(unwrapShellCommand('/bin/bash -lc "echo \\"\\$HOME\\""')).toBe('echo "$HOME"'); + expect(unwrapShellCommand('bash -lc ls')).toBe('ls'); + expect(unwrapShellCommand('git status')).toBe('git status'); + expect(unwrapShellCommand("/bin/bash -lc 'unterminated")).toBe("/bin/bash -lc 'unterminated"); + expect(unwrapShellCommand("/bin/bash -lc 'a' 'b'")).toBe("/bin/bash -lc 'a' 'b'"); + }); + + test('sandbox, effort and model settings', () => { + expect(codexHostExecArgs('gpt-6.1-sol', 'go', {})).toEqual(['exec', '--json', '-s', 'danger-full-access', '--skip-git-repo-check', '--model', 'gpt-6.1-sol', 'go']); + expect(codexHostExecArgs('gpt-6.1-sol', 'go', { GSTACK_CODEX_EFFORT: 'high' })).toContain('model_reasoning_effort="high"'); + expect(codexHostExecArgs('m', 'go', { GSTACK_CODEX_SANDBOX: 'workspace-write' })).toEqual( + ['exec', '--json', '-s', 'workspace-write', '--skip-git-repo-check', '--model', 'm', '-c', 'sandbox_workspace_write.network_access=true', 'go']); + expect(() => codexHostExecArgs('m', 'go', { GSTACK_CODEX_SANDBOX: 'none' })).toThrow('GSTACK_CODEX_SANDBOX'); + expect(() => codexHostExecArgs('m', 'go', { GSTACK_CODEX_EFFORT: 'high" -c x="y' })).toThrow('GSTACK_CODEX_EFFORT'); + expect(codexHostModel('claude-opus-4-7', { GSTACK_CODEX_MODEL: 'gpt-6.1-sol' })).toBe('gpt-6.1-sol'); + expect(codexHostModel('gpt-6-astra', { GSTACK_CODEX_MODEL: 'gpt-6.1-sol' })).toBe('gpt-6-astra'); + expect(codexHostModel(undefined, {})).toBe('gpt-6-astra'); + }); + + test('every shard on the Codex host gets the CI Codex install', () => { + expect(codexShardAccess(['test/skill-e2e-review.test.ts'], 'gate')).toBe('none'); + expect(codexShardAccess(['test/skill-e2e-review.test.ts'], 'gate', 'codex')).toBe('path'); + const env: NodeJS.ProcessEnv = { EVALS_HOST: 'codex', EVALS_TIER: 'gate', PATH: '/usr/bin', GSTACK_CI_CODEX_BIN_DIR: '/opt/codex/bin', GSTACK_CI_CODEX_HOME: '/home/runner/.ci-codex' }; + scopeCodexAccess(env, ['test/skill-e2e-review.test.ts']); + expect(env.PATH).toBe(`/opt/codex/bin${path.delimiter}/usr/bin`); + expect(env.CODEX_HOME).toBe('/home/runner/.ci-codex'); + }); + + test('runSkillTest on EVALS_HOST=codex runs codex exec and returns a priced SkillTestResult', async () => { + const dir = tempDir('codex-host-run-'); + const root = path.join(dir, 'runtime'); + fs.mkdirSync(path.join(root, 'home', '.codex'), { recursive: true }); + const bin = path.join(dir, 'bin'); + fs.mkdirSync(bin); + const argsFile = path.join(dir, 'args.txt'); + fs.writeFileSync(path.join(bin, 'codex'), `#!/bin/sh\nprintf '%s\\n' "$@" > '${argsFile}'\nprintf '%s\\n' "$CODEX_HOME" >> '${argsFile}'\ncat '${path.join(ROOT, 'test/fixtures/codex-host/exec-commands.jsonl')}'\n`, { mode: 0o755 }); + const evalDir = path.join(dir, 'evals'); + const saved = { ...process.env }; + Object.assign(process.env, { EVALS_HOST: 'codex', GSTACK_CODEX_MODEL: 'gpt-6.1-sol', GSTACK_CODEX_HOST_ROOT: root, GSTACK_EVAL_DIR: evalDir, PATH: `${bin}${path.delimiter}${process.env.PATH}` }); + try { + const result = await runSkillTest({ prompt: 'Run the fixture.', workingDirectory: dir, model: 'claude-opus-4-7', timeout: 30_000, testName: 'host-switch' }); + expect(result).toMatchObject({ exitReason: 'success', model: 'gpt-6.1-sol', output: '`cat hello.txt` output: `hello`', browseErrors: [] }); + expect(result.toolCalls.map(call => call.input.command)).toEqual(["printf 'hi' > hello.txt\nprintf 'hello' > hello.txt\ncat hello.txt"]); + expect(result.costEstimate).toMatchObject({ estimatedCost: 0.027248, estimatedTokens: 25095, turnsUsed: 3 }); + expect(result.tokenUsage).toEqual(parseCodexJSONL(COMMANDS).usage!); + const args = fs.readFileSync(argsFile, 'utf8').trim().split('\n'); + expect(args.slice(0, 7)).toEqual(['exec', '--json', '-s', 'danger-full-access', '--skip-git-repo-check', '--model', 'gpt-6.1-sol']); + expect(args.at(-2)).toBe('Run the fixture.'); + expect(args.at(-1)).toBe(path.join(root, 'home', '.codex')); + expect(readSessionLedger(evalDir)).toEqual([expect.objectContaining({ runner: 'codex', test_name: 'host-switch', end: 'completed', billed: true, cost_usd: 0.027248 })]); + // The native lifecycle sees the codex group owner at spawn and settles after its exit. + const seen: string[] = []; + await runSkillTest({ prompt: 'Run the fixture.', workingDirectory: dir, timeout: 30_000, testName: 'host-lifecycle', nativeLifecycle: { + onSpawn: pid => { seen.push(`spawn ${pid > 0}`); }, + onSettled: async ({ deadline, exited }) => { seen.push(`settled ${exited} ${deadline > Date.now()}`); }, + } }); + expect(seen).toEqual(['spawn true', 'settled true true']); + await expect(runSkillTest({ prompt: 'Run the fixture.', workingDirectory: dir, timeout: 30_000, testName: 'host-lifecycle-refused', nativeLifecycle: { + onSpawn: () => { throw new Error('native process is not group owner'); }, + onSettled: async () => {}, + } })).rejects.toThrow('native process is not group owner'); + } finally { + for (const key of Object.keys(process.env)) if (!(key in saved)) delete process.env[key]; + Object.assign(process.env, saved); + } + }); +}); diff --git a/test/cookie-workflow-judge-input.test.ts b/test/cookie-workflow-judge-input.test.ts index c245e04ab..a8a268a3b 100644 --- a/test/cookie-workflow-judge-input.test.ts +++ b/test/cookie-workflow-judge-input.test.ts @@ -244,8 +244,8 @@ describe('cookie workflow judge input', () => { expect(body).toContain('Use this checkout as the gstack root if it contains `BROWSER.md` and `browse/SKILL.md`'); expect(body).toContain('the installed root containing `bin/gstack-skill-start`, never a generated host stub'); expect(body).toContain("Read that root's `browse/SKILL.md`"); - expect(body).toContain('On `READY`, stop importing'); - expect(body.indexOf('**BROWSER SETUP**')).toBeLessThan(body.indexOf('## SETUP')); + expect(body).toContain('**Aside probe prints `READY`:** no import is needed'); + expect(body.indexOf('**BROWSER SETUP**')).toBeLessThan(body.indexOf('**Browser fallback**')); expect(body).toContain("Read that same root's `BROWSER.md`, **Choosing a source and checking sign-in**"); expect(body).not.toContain('../BROWSER.md'); } diff --git a/test/evals-workflow-wiring.test.ts b/test/evals-workflow-wiring.test.ts index 6cfc3217a..013fc621c 100644 --- a/test/evals-workflow-wiring.test.ts +++ b/test/evals-workflow-wiring.test.ts @@ -22,7 +22,10 @@ import { describe, test, expect } from 'bun:test'; import * as fs from 'fs'; import * as path from 'path'; +import * as os from 'os'; +import { spawnSync } from 'child_process'; import { buildRunManifest, isCodexShard, isOverlayTestFile, parseCliOptions, resolvePaidShardTimeoutMs, sliceExecutionOrder, sliceSupervisedWallMs, CI_SETUP_ALLOWANCE_MINUTES } from '../scripts/test-paid-shards'; +import { CENSUS_ARMS, MONTHLY_CENSUS_ARMS, MONTHLY_CENSUS_SCHEDULE, WEEKLY_CENSUS_ARMS, WEEKLY_CENSUS_SCHEDULE, censusArmEnv, censusMatrix, selectCensus } from '../scripts/census-arms'; const ROOT = path.join(import.meta.dir, '..'); const read = (rel: string) => fs.readFileSync(path.join(ROOT, rel), 'utf-8'); @@ -40,7 +43,7 @@ function plannerSites(source: string): Array<{ manifest: string; budgetSeconds: type Step = { id?: string; name?: string; run?: string; uses?: string; env?: Record; with?: Record }; type Job = { needs?: string[]; env?: Record; outputs?: Record; 'timeout-minutes': string | number; - strategy?: { 'max-parallel': number; matrix: { slice: string } }; steps: Step[] }; + strategy?: { 'max-parallel': number; matrix: { slice?: string; include?: string } }; steps: Step[] }; /** * An executor's matrix and timeout must come from the planner step that wrote @@ -159,7 +162,6 @@ describe('evals.yml sliced-lane wiring (post-matrix)', () => { describe('evals-periodic.yml sliced-lane wiring', () => { const lanes = [ { source: periodicYml, name: 'evals-periodic.yml', executor: 'eval-slices', prefix: 'periodic_', tier: 'periodic' }, - { source: periodicYml, name: 'evals-periodic.yml', executor: 'gate-census', prefix: 'gate_', tier: 'gate' }, { source: evalsYml, name: 'evals.yml', executor: 'eval-slices', prefix: '', tier: 'gate' }, { source: marathonYml, name: 'evals-marathon.yml', executor: 'eval-slices', prefix: '', tier: 'marathon' }, ] as const; @@ -200,6 +202,64 @@ describe('evals-periodic.yml sliced-lane wiring', () => { }); } + test('evals-periodic.yml:gate-census — one row per census arm and planned slice with its plan\'s ceiling, and every row of a scheduled census starts at once', () => { + const workflow = Bun.YAML.parse(periodicYml) as { env?: Record; jobs: Record }; + const planner = workflow.jobs['plan-slices']!; + const executor = workflow.jobs['gate-census']!; + expect(executor.needs).toContain('plan-slices'); + expect(executor.strategy!.matrix).toEqual({ include: '${{ fromJSON(needs.plan-slices.outputs.gate_matrix) }}' }); + expect(executor['timeout-minutes']).toBe('${{ matrix.timeout }}'); + expect(executor.steps[0]!.run).toBe('echo "GSTACK_SLICE_JOB_STARTED_AT=$(date +%s)" >> "$GITHUB_ENV"'); + expect(planner.outputs!.gate_matrix).toBe('${{ steps.gate-matrix.outputs.gate_matrix }}'); + expect(planner.steps.find(step => step.id === 'gate-matrix')!.run).toBe('bun --no-install run scripts/census-arms.ts --matrix /tmp/gate-census-plan "$ARMS" >> "$GITHUB_OUTPUT"'); + const emit = planner.steps.filter(step => step.run?.includes('--emit-plan "/tmp/gate-census-plan/$arm/manifest.json" ')); + expect(emit).toHaveLength(1); + expect(emit[0]!.run).toContain('mapfile -t arm_env < <(bun --no-install run scripts/census-arms.ts --env "$arm")'); + expect(emit[0]!.run).toContain('env "${arm_env[@]}" EVALS_TIER=gate bun run scripts/test-paid-shards.ts'); + const execute = executor.steps.filter(step => step.run?.includes('--plan ')); + expect(execute).toHaveLength(1); + expect(execute[0]!.run).toContain('--plan /tmp/gate-census-plan/${{ matrix.arm }}/manifest.json --slice ${{ matrix.slice }}'); + // The executor runs with its arm's host and model, and downloads every arm's plan. + expect(executor.steps.find(step => step.name === "Select the census arm's host and model")!.run).toBe('bun --no-install run scripts/census-arms.ts --env "$ARM" >> "$GITHUB_ENV"'); + expect(executor.steps.some(step => step.with?.path === '/tmp/gate-census-plan')).toBe(true); + const cliArgs = (run: string) => { + const command = /\bbun(?: --no-install)? run scripts\/test-paid-shards\.ts (.*)$/m.exec(run)!; + return command[1]!.replace(/\$\{\{\s*matrix\.slice\s*\}\}/g, '1').replace(/\$\{\{\s*matrix\.arm\s*\}\}/g, 'claude-opus').trim().split(/\s+/); + }; + const planned = parseCliOptions(cliArgs(emit[0]!.run!), { ...workflow.env, ...emit[0]!.env, EVALS_ALL: '1', EVALS_PROFILE: 'full' }); + const active = parseCliOptions(cliArgs(execute[0]!.run!), { ...workflow.env, ...executor.env, ...execute[0]!.env, EVALS_PROFILE: 'full' }); + expect(planned.tier).toBe('gate'); + expect(active.tier).toBe('gate'); + expect(active.jobs).toBe(planned.jobs); + expect(execute[0]!.env?.EVALS_JOBS).toBe(String(planned.jobs)); + const planDir = fs.mkdtempSync(path.join(os.tmpdir(), 'census-arms-')); + try { + for (const arm of Object.keys(CENSUS_ARMS)) { + const env = { ...workflow.env, ...emit[0]!.env, EVALS_ALL: '1', EVALS_PROFILE: 'full', ...censusArmEnv(arm) }; + const manifest = buildRunManifest({ tier: 'gate', profile: 'full', sliceBudgetMs: planned.sliceBudgetMs!, jobs: planned.jobs, + evalsAll: true, env, rootDir: ROOT, skipJudges: planned.skipJudges }); + expect(manifest.host ?? 'claude').toBe(CENSUS_ARMS[arm]!.host); + const sliceFiles = Array.from({ length: manifest.sliceCount }, (_, i) => sliceExecutionOrder( + manifest.entries.filter(entry => entry.status === 'planned' && entry.slice === i + 1)).map(entry => entry.file)); + const ceilings = sliceFiles.map(files => Math.ceil(Math.max(2 * planned.sliceBudgetMs!, + ...files.map(file => resolvePaidShardTimeoutMs([file])), + files.some(isOverlayTestFile) ? sliceSupervisedWallMs(files, planned.jobs) : 0) / 60_000) + CI_SETUP_ALLOWANCE_MINUTES); + expect(manifest.plan!.sliceCiTimeoutMinutes).toEqual(ceilings); + expect(manifest.plan!.ciTimeoutMinutes).toBeLessThanOrEqual(360); + fs.mkdirSync(path.join(planDir, arm)); + fs.writeFileSync(path.join(planDir, arm, 'manifest.json'), JSON.stringify(manifest)); + expect(censusMatrix([arm], planDir)).toEqual(ceilings.map((timeout, i) => ({ arm, slice: i + 1, timeout }))); + } + for (const arms of [WEEKLY_CENSUS_ARMS, MONTHLY_CENSUS_ARMS]) { + const rows = censusMatrix(arms, planDir); + expect(new Set(rows.map(row => `${row.arm}/${row.slice}`)).size).toBe(rows.length); + expect(rows.length, `${arms.join('+')} rows exceed max-parallel`).toBeLessThanOrEqual(executor.strategy!['max-parallel']); + } + } finally { + fs.rmSync(planDir, { recursive: true, force: true }); + } + }); + test('planner/executor/report tier=periodic agree and plan with the ~7-minute budget (W5c)', () => { expect(periodicYml).toMatch(/EVALS_TIER=periodic bun --no-install run scripts\/test-paid-shards\.ts --tier periodic --emit-plan/); expect(periodicYml).toMatch(/EVALS_TIER=periodic bun run scripts\/test-paid-shards\.ts --tier periodic --plan .* --slice /); @@ -207,7 +267,7 @@ describe('evals-periodic.yml sliced-lane wiring', () => { // Periodic work and the full gate census have distinct immutable plans. expect(plannerSites(periodicYml)).toEqual([ { manifest: '/tmp/paid-plan/manifest.json', budgetSeconds: 420, jobs: 2 }, - { manifest: '/tmp/gate-census-plan/manifest.json', budgetSeconds: 420, jobs: 2 }, + { manifest: '"/tmp/gate-census-plan/$arm/manifest.json"', budgetSeconds: 420, jobs: 2 }, ]); }); }); @@ -228,7 +288,7 @@ describe('evals-periodic.yml host-run Codex slices', () => { expect(matrixStep.run).toContain('} >> "$GITHUB_OUTPUT"'); expect(planner.outputs!.periodic_codex_slices).toBe('${{ steps.periodic-matrix.outputs.codex_slices }}'); expect(codex.needs).toEqual(['build-image', 'plan-slices']); - expect(codex.if).toBe("${{ needs.plan-slices.outputs.periodic_codex_slices != '[]' }}"); + expect(codex.if).toBe("${{ needs.plan-slices.outputs.periodic_lane == 'true' && needs.plan-slices.outputs.periodic_codex_slices != '[]' }}"); expect(codex.strategy!.matrix.slice).toBe('${{ fromJSON(needs.plan-slices.outputs.periodic_codex_slices) }}'); expect(codex['timeout-minutes']).toBe('${{ fromJSON(needs.plan-slices.outputs.periodic_slice_timeouts)[matrix.slice] }}'); expect(codex.steps[0]!.run).toBe('echo "GSTACK_SLICE_JOB_STARTED_AT=$(date +%s)" >> "$GITHUB_ENV"'); @@ -350,7 +410,7 @@ describe('panel verdict surfaces (eval reliability policy)', () => { test('planners size the capacity preflight with their executor cap', () => { for (const [source, executor, manifest] of [[evalsYml, 'eval-slices', '/tmp/paid-plan/manifest.json'], - [periodicYml, 'eval-slices', '/tmp/paid-plan/manifest.json'], [periodicYml, 'gate-census', '/tmp/gate-census-plan/manifest.json']] as const) { + [periodicYml, 'eval-slices', '/tmp/paid-plan/manifest.json'], [periodicYml, 'gate-census', '"/tmp/gate-census-plan/$arm/manifest.json"']] as const) { const jobs = jobsOf(source); const emit = jobs['plan-slices']!.steps.find(step => step.run?.includes(`--emit-plan ${manifest} `))!; const cap = Number(/--max-parallel (\d+)/.exec(emit.run!)?.[1]); @@ -401,6 +461,10 @@ describe('panel verdict surfaces (eval reliability policy)', () => { expect(redispatch.permissions).toEqual({ actions: 'write' }); expect(redispatch.if).toBe("${{ !cancelled() && needs.report.outputs.redispatch == 'true' }}"); expect(redispatch.steps[0]!.run).toContain('-f redispatch_of="$GITHUB_RUN_ID"'); + // A census-only run re-dispatches with its own arms, never as a weekly run. + expect(redispatch.steps[0]!.run).toContain('${CENSUS_ARMS:+-f census_arms="$CENSUS_ARMS"}'); + expect(redispatch.steps[0]!.env?.CENSUS_ARMS).toBe('${{ needs.report.outputs.redispatch_census_arms }}'); + expect(report.outputs!.redispatch_census_arms).toBe("${{ needs.plan-slices.outputs.periodic_lane != 'true' && join(fromJSON(needs.plan-slices.outputs.census_arms), ',') || '' }}"); const classify = report.steps.find(step => step.id === 'verdict')!; expect(classify.run).toContain('.verdict.redispatchEligible == true'); expect(classify.run).toContain('[ -z "$REDISPATCH_OF" ]'); @@ -426,7 +490,13 @@ describe('scheduled paid lanes: concurrency and branch dispatch scope', () => { test('a branch dispatch of evals-periodic runs the periodic lane only unless it opts into the gate census', () => { const wf = parse(periodicYml); expect(wf.on.workflow_dispatch!.inputs!.include_gate_census).toMatchObject({ type: 'boolean', default: false }); - expect(wf.jobs['gate-census']!.if).toBe("${{ github.ref == 'refs/heads/main' || inputs.include_gate_census }}"); + // The planner selects the census arms (scripts/census-arms.ts); no arm, no census job. + expect(wf.jobs['gate-census']!.if).toBe("${{ needs.plan-slices.outputs.gate_matrix != '[]' }}"); + const select = wf.jobs['plan-slices']!.steps.find(step => step.id === 'census')!; + expect(select.env?.INCLUDE_GATE_CENSUS).toBe("${{ github.ref == 'refs/heads/main' || inputs.include_gate_census }}"); + expect(select.run).toBe('bun --no-install run scripts/census-arms.ts --select >> "$GITHUB_OUTPUT"'); + expect(selectCensus({ schedule: '', censusArms: '', includeGateCensus: false })).toEqual({ arms: [], periodicLane: true }); + expect(selectCensus({ schedule: WEEKLY_CENSUS_SCHEDULE, includeGateCensus: true })).toEqual({ arms: [...WEEKLY_CENSUS_ARMS], periodicLane: true }); const report = wf.jobs.report!.steps; const reconcile = report.find(step => step.name === 'Reconcile gate census against the manifest (fail-closed)')!; expect(reconcile.env?.GATE_CENSUS).toBe('${{ needs.gate-census.result }}'); @@ -436,3 +506,80 @@ describe('scheduled paid lanes: concurrency and branch dispatch scope', () => { expect(fail.if).toContain(`!contains(fromJSON('["success","skipped"]'), needs.gate-census.result)`); }); }); + +describe('evals-periodic.yml census arms: Monday two-arm census, monthly Fable/Astra census', () => { + type CensusStep = Step & { if?: string }; + type CensusWf = { on: { schedule: Array<{ cron: string }>; workflow_dispatch: { inputs: Record } }; + concurrency: { group: string }; jobs: Record; steps: CensusStep[] }> }; + const wf = Bun.YAML.parse(periodicYml) as CensusWf; + const planner = wf.jobs['plan-slices']!; + const census = wf.jobs['gate-census']!; + const report = wf.jobs.report!; + + test('each arm is one host and model: the capture default and gpt-6.1-sol weekly, claude-fable-5-1 and gpt-6-astra monthly', () => { + expect(WEEKLY_CENSUS_ARMS).toEqual(['claude-opus', 'codex-sol']); + expect(MONTHLY_CENSUS_ARMS).toEqual(['claude-fable', 'codex-astra']); + // claude-opus pins no model: the Claude runners' capture default (lib/eval-model.ts) applies. + expect(censusArmEnv('claude-opus')).toEqual({ EVALS_HOST: 'claude' }); + expect(censusArmEnv('codex-sol')).toEqual({ EVALS_HOST: 'codex', GSTACK_CODEX_MODEL: 'gpt-6.1-sol' }); + expect(censusArmEnv('claude-fable')).toEqual({ EVALS_HOST: 'claude', EVALS_MODEL: 'claude-fable-5-1' }); + expect(censusArmEnv('codex-astra')).toEqual({ EVALS_HOST: 'codex', GSTACK_CODEX_MODEL: 'gpt-6-astra' }); + expect(() => censusArmEnv('codex-luna')).toThrow('unknown census arm'); + // The report files slice artifacts by arm suffix: no arm id may end another one. + const ids = Object.keys(CENSUS_ARMS); + for (const a of ids) for (const b of ids) if (a !== b) expect(a.endsWith(`-${b}`)).toBe(false); + for (const id of ids) expect(id).toMatch(/^[a-z]+-[a-z]+$/); + }); + + test('the monthly cron runs only the census, under its own concurrency group; the Monday and daily canary schedules are unchanged', () => { + expect(wf.on.schedule.map(entry => entry.cron)).toEqual([WEEKLY_CENSUS_SCHEDULE, '30 7 * * *', MONTHLY_CENSUS_SCHEDULE]); + expect(MONTHLY_CENSUS_SCHEDULE).toBe('0 6 1 * *'); + expect(wf.concurrency.group.endsWith("${{ github.event.schedule == '0 6 1 * *' && '-monthly' || '' }}")).toBe(true); + expect(wf.concurrency.group).toContain("(github.event.schedule == '30 7 * * *' || inputs.schema_canary_only) && '-schema-canary'"); + expect(selectCensus({ schedule: MONTHLY_CENSUS_SCHEDULE, includeGateCensus: true })).toEqual({ arms: [...MONTHLY_CENSUS_ARMS], periodicLane: false }); + const select = planner.steps.find(step => step.id === 'census')!; + expect(select.env).toMatchObject({ SCHEDULE: '${{ github.event.schedule }}', CENSUS_ARMS: '${{ inputs.census_arms }}' }); + expect(planner.outputs).toMatchObject({ periodic_lane: '${{ steps.census.outputs.periodic_lane }}', census_arms: '${{ steps.census.outputs.arms }}' }); + // Without the periodic lane no periodic plan is emitted, executed or reconciled. + for (const step of planner.steps.filter(step => step.run?.includes('/tmp/paid-plan/manifest.json') || step.with?.name === 'paid-plan')) { + expect(step.if, step.name ?? step.with?.name).toBe("steps.census.outputs.periodic_lane == 'true'"); + } + const reconcile = report.steps.find(step => step.id === 'reconcile')!; + expect(reconcile.env?.PERIODIC_LANE).toBe('${{ needs.plan-slices.outputs.periodic_lane }}'); + expect(reconcile.run).toContain('if [ "$PERIODIC_LANE" != "true" ]; then'); + // A census-only green run never closes the weekly tracking issue. + expect(report.steps.find(step => step.name === 'Close the tracking issue on a green run')!.if) + .toContain("needs.plan-slices.outputs.periodic_lane == 'true' && needs.eval-slices.result == 'success'"); + }); + + test('a census_arms dispatch runs only the named arms and refuses unknown ones', () => { + expect(wf.on.workflow_dispatch.inputs.census_arms).toMatchObject({ type: 'string', default: '' }); + expect(selectCensus({ censusArms: 'codex-sol', includeGateCensus: true })).toEqual({ arms: ['codex-sol'], periodicLane: false }); + expect(selectCensus({ censusArms: 'claude-opus, codex-sol,codex-sol', includeGateCensus: false })).toEqual({ arms: ['claude-opus', 'codex-sol'], periodicLane: false }); + expect(() => selectCensus({ censusArms: 'codex-sol,gpt', includeGateCensus: false })).toThrow('unknown census arm(s): gpt'); + const cli = (args: string[], env: Record) => { + const proc = spawnSync(process.execPath, ['--no-install', 'run', 'scripts/census-arms.ts', ...args], + { cwd: ROOT, env: { PATH: process.env.PATH ?? '', ...env }, encoding: 'utf8', timeout: 15_000 }); + expect(proc.status, proc.stderr).toBe(0); + return proc.stdout; + }; + expect(cli(['--select'], { SCHEDULE: MONTHLY_CENSUS_SCHEDULE })).toBe('arms=["claude-fable","codex-astra"]\nperiodic_lane=false\n'); + expect(cli(['--select'], { CENSUS_ARMS: 'codex-sol', INCLUDE_GATE_CENSUS: 'true' })).toBe('arms=["codex-sol"]\nperiodic_lane=false\n'); + expect(cli(['--env', 'codex-sol'], {})).toBe('EVALS_HOST=codex\nGSTACK_CODEX_MODEL=gpt-6.1-sol\n'); + }); + + test('census artifacts and reconciliation are per arm', () => { + const upload = census.steps.find(step => step.name === 'Upload census results')!; + expect(upload.with!.name).toBe('gate-census-${{ matrix.slice }}-a${{ github.run_attempt }}-${{ matrix.arm }}'); + expect(census.steps.find(step => step.name?.startsWith('Run gate census slice'))!.name).toContain('(${{ matrix.arm }})'); + expect(report.steps.some(step => step.with?.pattern === 'gate-census-[0-9]*' && step.with?.path === '/tmp/gate-census-slices')).toBe(true); + const reconcile = report.steps.find(step => step.id === 'gate-reconcile')!; + expect(reconcile.env?.ARMS).toBe('${{ needs.plan-slices.outputs.census_arms }}'); + expect(reconcile.run).toContain('cp "/tmp/gate-census-plan/$arm/manifest.json" "$dir/manifest.json" || status=1'); + expect(reconcile.run).toContain('for slice in /tmp/gate-census-slices/gate-census-*-"$arm"; do'); + expect(reconcile.run).toContain('EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --report "$dir"'); + // Every arm's report counts: one red arm fails the census. + expect(reconcile.run).toContain('if [ "$code" -ne 0 ]; then status=$code; fi'); + expect(reconcile.run).toContain('echo "exit=$status" >> "$GITHUB_OUTPUT"'); + }); +}); diff --git a/test/fixtures/codex-host/exec-commands.jsonl b/test/fixtures/codex-host/exec-commands.jsonl new file mode 100644 index 000000000..94518ca86 --- /dev/null +++ b/test/fixtures/codex-host/exec-commands.jsonl @@ -0,0 +1,7 @@ +{"type":"thread.started","thread_id":"01a150b3-ae5a-7932-ba51-4c295c66adf4"} +{"type":"turn.started"} +{"type":"item.completed","item":{"id":"item_0","type":"agent_message","text":"I’ll write and overwrite `hello.txt` with `printf`, then read it with `cat`.\n"}} +{"type":"item.started","item":{"id":"item_1","type":"command_execution","command":"/bin/bash -lc \"printf 'hi' > hello.txt\nprintf 'hello' > hello.txt\ncat hello.txt\"","aggregated_output":"","exit_code":null,"status":"in_progress"}} +{"type":"item.completed","item":{"id":"item_1","type":"command_execution","command":"/bin/bash -lc \"printf 'hi' > hello.txt\nprintf 'hello' > hello.txt\ncat hello.txt\"","aggregated_output":"hello","exit_code":0,"status":"completed"}} +{"type":"item.completed","item":{"id":"item_2","type":"agent_message","text":"`cat hello.txt` output: `hello`"}} +{"type":"turn.completed","usage":{"input_tokens":25009,"cached_input_tokens":12437,"cache_write_input_tokens":12566,"output_tokens":86,"reasoning_output_tokens":0}} diff --git a/test/fixtures/codex-host/exec-file-change.jsonl b/test/fixtures/codex-host/exec-file-change.jsonl new file mode 100644 index 000000000..41647909f --- /dev/null +++ b/test/fixtures/codex-host/exec-file-change.jsonl @@ -0,0 +1,9 @@ +{"type":"thread.started","thread_id":"01a150b3-be24-7f22-aa81-7eb4f79018e0"} +{"type":"turn.started"} +{"type":"item.completed","item":{"id":"item_0","type":"agent_message","text":"I’ll create `notes.md` with `draft`, then change it to `final` using two patches.\n"}} +{"type":"item.started","item":{"id":"item_1","type":"file_change","changes":[{"path":"/tmp/codex-host-fixture/notes.md","kind":"add"}],"status":"in_progress"}} +{"type":"item.completed","item":{"id":"item_1","type":"file_change","changes":[{"path":"/tmp/codex-host-fixture/notes.md","kind":"add"}],"status":"completed"}} +{"type":"item.started","item":{"id":"item_2","type":"file_change","changes":[{"path":"/tmp/codex-host-fixture/notes.md","kind":"update"}],"status":"in_progress"}} +{"type":"item.completed","item":{"id":"item_2","type":"file_change","changes":[{"path":"/tmp/codex-host-fixture/notes.md","kind":"update"}],"status":"completed"}} +{"type":"item.completed","item":{"id":"item_3","type":"agent_message","text":"Created `notes.md` with `draft`, then changed it to `final` using two `apply_patch` calls."}} +{"type":"turn.completed","usage":{"input_tokens":25020,"cached_input_tokens":12440,"cache_write_input_tokens":12574,"output_tokens":137,"reasoning_output_tokens":0}} diff --git a/test/fixtures/overlay-nudges.ts b/test/fixtures/overlay-nudges.ts index 0b499d785..5baac1656 100644 --- a/test/fixtures/overlay-nudges.ts +++ b/test/fixtures/overlay-nudges.ts @@ -202,7 +202,7 @@ export const OVERLAY_FIXTURES: OverlayFixture[] = [ { id: 'claude-dedicated-tools-vs-bash', overlayPath: 'model-overlays/claude.md', - model: 'claude-opus-4-7', + model: 'claude-opus-5-5', trials: 10, concurrency: 3, direction: 'lower_is_better', @@ -241,7 +241,7 @@ export const OVERLAY_FIXTURES: OverlayFixture[] = [ { id: 'opus-4-7-effort-match-trivial', overlayPath: 'model-overlays/opus-4-7.md', - model: 'claude-opus-4-7', + model: 'claude-opus-5-5', trials: 10, concurrency: 3, direction: 'lower_is_better', @@ -269,7 +269,7 @@ export const OVERLAY_FIXTURES: OverlayFixture[] = [ { id: 'opus-4-7-literal-interpretation', overlayPath: 'model-overlays/opus-4-7.md', - model: 'claude-opus-4-7', + model: 'claude-opus-5-5', trials: 10, concurrency: 3, direction: 'higher_is_better', @@ -290,16 +290,16 @@ export const OVERLAY_FIXTURES: OverlayFixture[] = [ }, // ========================================================================= - // Sonnet 4.6 variants of the Opus-4.7 fixtures. + // Sonnet 5.5 variant of the Opus 5.5 dedicated-tools fixture. // - // Same overlays, prompts, metrics and trial counts; different pinned model. + // Same overlay, prompt, metric and trial count; different pinned model. // Model effects remain separate instead of pooling their observations. // ========================================================================= { id: 'claude-dedicated-tools-vs-bash-sonnet', overlayPath: 'model-overlays/claude.md', - model: 'claude-sonnet-4-6', + model: 'claude-sonnet-5-5', trials: 10, concurrency: 3, direction: 'lower_is_better', diff --git a/test/helpers/anthropic-preflight.ts b/test/helpers/anthropic-preflight.ts index 9ca9f1b76..522090c12 100644 --- a/test/helpers/anthropic-preflight.ts +++ b/test/helpers/anthropic-preflight.ts @@ -11,9 +11,14 @@ * * Lives in its own module (not e2e-helpers) so the runner can import it * without dragging in bun:test. + * + * On the Codex host (EVALS_HOST=codex) Codex is the agent and no session + * spawns `claude`, so the ping is skipped; Codex API failures surface per + * session as error_api. */ import { spawnSync } from 'child_process'; +import { resolveEvalHost } from './e2e-hosts'; export type PreflightResult = 'skipped' | 'ok'; @@ -21,7 +26,7 @@ export function preflightAnthropicApi( env: NodeJS.ProcessEnv = process.env, spawn: typeof spawnSync = spawnSync, ): PreflightResult { - if (env.EVALS_PREFLIGHT_OK === '1') return 'skipped'; + if (env.EVALS_PREFLIGHT_OK === '1' || resolveEvalHost(env) === 'codex') return 'skipped'; const check = spawn( 'sh', ['-c', 'echo "ping" | claude -p --max-turns 1 --output-format stream-json --verbose --dangerously-skip-permissions'], diff --git a/test/helpers/codex-eval.ts b/test/helpers/codex-eval.ts index 1a402da51..02dc35297 100644 --- a/test/helpers/codex-eval.ts +++ b/test/helpers/codex-eval.ts @@ -1,5 +1,5 @@ /** One terminal record per Codex eval attempt, using the assertions as the oracle. */ -import { CODEX_DRAIN_GRACE_MS, CodexHarnessError, type CodexResult } from './codex-session-runner'; +import { CODEX_DRAIN_GRACE_MS, CodexHarnessError, codexUsageCostUsd, type CodexResult } from './codex-session-runner'; import { EvalCollector, getProjectEvalDir, shardSlugOfEvalDir, type EvalTestEntry } from './eval-store'; import * as path from 'node:path'; import * as fs from 'node:fs'; @@ -198,10 +198,13 @@ export async function runRecordedCodexEval(opts: CodexEvalOptions): Promise Bash ({ command } unwrapped from Codex's + * `bash -lc`, its output); file_change -> Write (add), Edit (update) or + * Delete per path; web_search -> WebSearch; mcp_tool_call -> mcp__server__tool; + * the last agent message is the output; + * - transcript: the Claude stream-json shape the cases' observers read: one + * system/init event (cwd, the Codex thread id as session_id), assistant text + * and tool_use events, user tool_result events (is_error from the exit code) + * and a final result event; + * - sandbox: GSTACK_CODEX_SANDBOX, default danger-full-access (the counterpart + * of claude -p --dangerously-skip-permissions; cases write outside their + * working directory, start loopback servers and browsers); workspace-write + * gets network access; + * - effort: GSTACK_CODEX_EFFORT sets model_reasoning_effort; unset keeps the + * model's default; + * - model: a non-Claude options.model, else GSTACK_CODEX_MODEL, else + * CODEX_FRONTIER_MODEL; a case's Claude model pin does not apply here; + * - cost: real accounting from `turn.completed` usage at lib/pricing.ts rates, + * recorded on the result, the eval record and the session ledger. + * - nativeLifecycle: onSpawn gets the codex process-group owner right after + * spawn; onSettled runs after its exit and group kill, within the drain grace. + * Not translated: maxTurns (Codex has no turn cap), allowedTools/tools (Codex + * has its own tool surface) and startupGraceMs (the timeout bounds the session). + */ +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { spawnSync } from 'child_process'; +import { CODEX_DRAIN_GRACE_MS, CodexHarnessError, codexUsageCostUsd, execCodex, type CodexResult } from './codex-session-runner'; +import { getProjectEvalDir } from './eval-store'; +import { hermeticChildEnv } from './hermetic-env'; +import { appendSessionLedger, sessionKey, type SessionEnd } from './session-ledger'; +import { sanitizeTestName, scanBrowseErrors, type SkillTestResult } from './session-runner'; +import { CODEX_FRONTIER_MODEL } from '../../scripts/resolvers/constants'; + +const ROOT = path.resolve(import.meta.dir, '..', '..'); +const SETUP_TIMEOUT_MS = 300_000; +export const CODEX_SANDBOXES = ['read-only', 'workspace-write', 'danger-full-access'] as const; +const EFFORT = /^[a-z]+$/; + +/** The Codex model for a session: a Claude pin never reaches `codex --model`. */ +export function codexHostModel(explicit: string | undefined, env: NodeJS.ProcessEnv = process.env): string { + if (explicit && !/^claude-/.test(explicit)) return explicit; + return env.GSTACK_CODEX_MODEL || CODEX_FRONTIER_MODEL; +} + +/** `codex exec` arguments for one session; the prompt is the last argument. */ +export function codexHostExecArgs(model: string, prompt: string, env: NodeJS.ProcessEnv = process.env): string[] { + const sandbox = env.GSTACK_CODEX_SANDBOX || 'danger-full-access'; + if (!(CODEX_SANDBOXES as readonly string[]).includes(sandbox)) { + throw new Error(`GSTACK_CODEX_SANDBOX must be one of ${CODEX_SANDBOXES.join(', ')}; got "${sandbox}"`); + } + const effort = env.GSTACK_CODEX_EFFORT; + if (effort && !EFFORT.test(effort)) throw new Error(`GSTACK_CODEX_EFFORT must be a bare effort name; got "${effort}"`); + return ['exec', '--json', '-s', sandbox, '--skip-git-repo-check', '--model', model, + ...(sandbox === 'workspace-write' ? ['-c', 'sandbox_workspace_write.network_access=true'] : []), + ...(effort ? ['-c', `model_reasoning_effort="${effort}"`] : []), + prompt]; +} + +/** The command inside Codex's `/bin/bash -lc ` wrapper, else the command unchanged. */ +export function unwrapShellCommand(command: string): string { + const match = /^(?:\/(?:usr\/)?bin\/)?(?:ba|z)?sh -lc (.+)$/s.exec(command); + if (!match) return command; + const word = match[1]!; + let out = ''; + let i = 0; + while (i < word.length) { + const c = word[i]!; + if (c === "'") { + const end = word.indexOf("'", i + 1); + if (end < 0) return command; + out += word.slice(i + 1, end); + i = end + 1; + } else if (c === '"') { + i++; + let closed = false; + while (i < word.length) { + if (word[i] === '\\' && i + 1 < word.length && '"\\$`\n'.includes(word[i + 1]!)) { out += word[i + 1]; i += 2; } + else if (word[i] === '"') { i++; closed = true; break; } + else out += word[i++]; + } + if (!closed) return command; + } else if (c === '\\' && i + 1 < word.length) { + out += word[i + 1]; + i += 2; + } else if (/\s/.test(c)) { + return command; + } else { + out += c; + i++; + } + } + return out; +} + +export type HostToolCall = SkillTestResult['toolCalls'][number]; + +const FILE_CHANGE_TOOLS: Record = { add: 'Write', update: 'Edit', delete: 'Delete' }; + +/** Claude-shaped tool calls from completed Codex items, in order. */ +export function codexHostToolCalls(rawLines: readonly string[]): HostToolCall[] { + const calls: HostToolCall[] = []; + for (const line of rawLines) { + let event: any; + try { event = JSON.parse(line); } catch { continue; } + if (event?.type !== 'item.completed' || !event.item) continue; + const item = event.item; + if (item.type === 'command_execution') { + calls.push({ tool: 'Bash', input: { command: unwrapShellCommand(String(item.command ?? '')) }, output: String(item.aggregated_output ?? '') }); + } else if (item.type === 'file_change') { + for (const change of Array.isArray(item.changes) ? item.changes : []) { + calls.push({ tool: FILE_CHANGE_TOOLS[change?.kind] ?? 'Edit', input: { file_path: String(change?.path ?? ''), kind: change?.kind }, output: '' }); + } + } else if (item.type === 'web_search') { + calls.push({ tool: 'WebSearch', input: { query: item.query }, output: '' }); + } else if (item.type === 'mcp_tool_call') { + calls.push({ tool: `mcp__${item.server}__${item.tool}`, input: item.arguments ?? {}, output: typeof item.result === 'string' ? item.result : JSON.stringify(item.result ?? '') }); + } + } + return calls; +} + +/** + * Codex events as the Claude stream-json transcript the cases' observers read + * (system/init, assistant text and tool_use, user tool_result, result). + */ +export function codexHostTranscript(rawLines: readonly string[], cwd: string, output: string): any[] { + const events: any[] = []; + let session = ''; + const say = (role: 'assistant' | 'user', content: any[]) => events.push({ type: role, session_id: session, parent_tool_use_id: null, message: { role, content } }); + for (const line of rawLines) { + let event: any; + try { event = JSON.parse(line); } catch { continue; } + if (event?.type === 'thread.started') { + session = String(event.thread_id ?? ''); + events.push({ type: 'system', subtype: 'init', cwd, session_id: session, tools: [] }); + continue; + } + if (event?.type !== 'item.completed' || !event.item) continue; + const item = event.item; + const id = String(item.id ?? `item_${events.length}`); + if (item.type === 'agent_message' && item.text) say('assistant', [{ type: 'text', text: String(item.text) }]); + else if (item.type === 'command_execution') { + say('assistant', [{ type: 'tool_use', id, name: 'Bash', input: { command: unwrapShellCommand(String(item.command ?? '')) } }]); + say('user', [{ type: 'tool_result', tool_use_id: id, content: String(item.aggregated_output ?? ''), is_error: item.exit_code !== 0 }]); + } else if (item.type === 'file_change') { + (Array.isArray(item.changes) ? item.changes : []).forEach((change: any, index: number) => { + const useId = `${id}_${index}`; + say('assistant', [{ type: 'tool_use', id: useId, name: FILE_CHANGE_TOOLS[change?.kind] ?? 'Edit', input: { file_path: String(change?.path ?? '') } }]); + say('user', [{ type: 'tool_result', tool_use_id: useId, content: '', is_error: item.status !== 'completed' }]); + }); + } + } + events.push({ type: 'result', subtype: 'success', session_id: session, result: output }); + return events; +} + +let runtimeRoot: string | undefined; + +/** + * Install the full gstack Codex runtime once per process: `./setup --host codex` + * into a private HOME/CODEX_HOME, with the login home's auth.json copied in. + * Returns the root holding home/.codex and tmp/ (Codex will not create its + * helper aliases when CODEX_HOME sits under its own TMPDIR). A caller that + * already prepared one passes it as GSTACK_CODEX_HOST_ROOT. + */ +export function prepareCodexHostRuntime(env: NodeJS.ProcessEnv = process.env): string { + if (runtimeRoot) return runtimeRoot; + const preset = env.GSTACK_CODEX_HOST_ROOT; + if (preset) { + if (!fs.existsSync(path.join(preset, 'home', '.codex'))) throw new Error(`GSTACK_CODEX_HOST_ROOT ${preset} has no home/.codex`); + fs.mkdirSync(path.join(preset, 'tmp'), { recursive: true }); + return runtimeRoot = preset; + } + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'codex-host-')); + const home = path.join(root, 'home'); + const codexHome = path.join(home, '.codex'); + const tmp = path.join(root, 'tmp'); + for (const dir of [codexHome, tmp]) fs.mkdirSync(dir, { recursive: true }); + const auth = path.join(env.CODEX_HOME || path.join(os.homedir(), '.codex'), 'auth.json'); + if (fs.existsSync(auth)) fs.copyFileSync(auth, path.join(codexHome, 'auth.json')); + const setup = spawnSync(path.join(ROOT, 'setup'), ['--host', 'codex'], { + cwd: ROOT, encoding: 'utf8', timeout: SETUP_TIMEOUT_MS, + env: hermeticChildEnv({ HOME: home, CODEX_HOME: codexHome, TMPDIR: tmp }, { extraAllow: ['OPENAI_API_KEY', 'CODEX_*'] }), + }); + if (setup.status !== 0) { + fs.rmSync(root, { recursive: true, force: true }); + throw new Error(`./setup --host codex failed (exit ${setup.status}${setup.error ? `, ${setup.error.message}` : ''}): ${(setup.stderr || setup.stdout || '').slice(-2000)}`); + } + process.once('exit', () => { try { fs.rmSync(root, { recursive: true, force: true }); } catch { /* best effort */ } }); + runtimeRoot = root; + return root; +} + +type RunSkillTestOptions = Parameters[0]; + +function exitReasonOf(result: CodexResult, aborted: boolean): string { + if (result.exitCode === 124) return 'timeout'; + const failedTurn = result.rawLines.some(line => /"type":"(?:turn\.failed|error)"/.test(line)); + if (result.exitCode === 0) return failedTurn ? 'error_api' : 'success'; + if (aborted) return 'timeout'; + return failedTurn && !result.rawLines.some(line => line.includes('"type":"item.completed"')) ? 'error_api' : `exit_code_${result.exitCode}`; +} + +const SESSION_ENDS: Record = { success: 'completed', timeout: 'session_timeout', error_api: 'api_error' }; + +/** runSkillTest on the Codex host. `appendSystemPrompt` arrives with any completion notice already applied. */ +export async function runCodexHostSkillTest(options: RunSkillTestOptions): Promise { + const startTime = Date.now(); + const startMono = performance.now(); + const { workingDirectory, timeout = 120_000, testName, runId, signal } = options; + const deadline = startTime + timeout; + const model = codexHostModel(options.model); + const prompt = options.appendSystemPrompt ? `${options.appendSystemPrompt}\n\n${options.prompt}` : options.prompt; + const root = prepareCodexHostRuntime(); + const safeName = testName ? sanitizeTestName(testName) : null; + let runDir: string | null = null; + if (runId) { + try { + runDir = path.join(path.dirname(getProjectEvalDir()), 'e2e-runs', runId); + fs.mkdirSync(runDir, { recursive: true }); + } catch { runDir = null; } + } + let firstResponseMs = 0; + let lastItemMs = 0; + let maxInterTurnMs = 0; + const onLine = (line: string, elapsedMs: number) => { + if (!firstResponseMs) firstResponseMs = Math.max(1, elapsedMs); + if (runDir && safeName) { try { fs.appendFileSync(path.join(runDir, `${safeName}.ndjson`), line + '\n'); } catch { /* non-fatal */ } } + if (!line.includes('"type":"item.completed"') || !/"type":"(?:command_execution|file_change)"/.test(line)) return; + if (lastItemMs) maxInterTurnMs = Math.max(maxInterTurnMs, elapsedMs - lastItemMs); + lastItemMs = elapsedMs; + }; + + let result: CodexResult; + let harnessReason: string | undefined; + let started = true; + const lifecycle = options.nativeLifecycle; + try { + result = await execCodex(codexHostExecArgs(model, prompt), { + cwd: workingDirectory, startTime, deadline, signal, onLine, ...(lifecycle ? { onSpawn: (pid: number) => lifecycle.onSpawn(pid) } : {}), + env: hermeticChildEnv( + { GSTACK_HEADLESS: '1', ...options.env, CODEX_HOME: path.join(root, 'home', '.codex'), TMPDIR: options.env?.TMPDIR ?? path.join(root, 'tmp') }, + { extraAllow: ['OPENAI_API_KEY', 'CODEX_*'] }, + ), + }); + } catch (error) { + if (!(error instanceof CodexHarnessError) || !error.result) throw error; + result = error.result; + started = !error.message.startsWith('Could not start Codex'); + harnessReason = /drain/.test(error.message) ? 'error_output_drain' : 'error_output_stream'; + } + if (lifecycle) { + const settleDeadline = Date.now() + CODEX_DRAIN_GRACE_MS; + let hookTimer: ReturnType | undefined; + try { + await Promise.race([ + lifecycle.onSettled({ deadline: settleDeadline, exited: started }), + new Promise((_, reject) => { hookTimer = setTimeout(() => reject(new Error('native lifecycle settlement deadline exceeded')), CODEX_DRAIN_GRACE_MS); }), + ]); + } finally { clearTimeout(hookTimer); } + } + + const exitReason = harnessReason ?? exitReasonOf(result, !!signal?.aborted); + const toolCalls = codexHostToolCalls(result.rawLines); + const events = result.rawLines.flatMap(line => { try { return [JSON.parse(line)]; } catch { return []; } }); + const messages = events.filter(event => event?.type === 'item.completed' && event.item?.type === 'agent_message'); + const output = String(messages.at(-1)?.item?.text ?? ''); + const turnsUsed = events.filter(event => event?.type === 'item.completed' && event.item?.type !== 'reasoning').length; + const transcript = codexHostTranscript(result.rawLines, workingDirectory, output); + const cost = result.usage ? codexUsageCostUsd(result.usage, model) : null; + const browseErrors = scanBrowseErrors(toolCalls.filter(call => call.tool === 'Bash').map(call => call.output).join('\n') + '\n' + result.stderr); + const duration = Date.now() - startTime; + + appendSessionLedger({ key: sessionKey('codex', testName), ...(testName ? { test_name: testName } : {}), runner: 'codex', + started_at: new Date(startTime).toISOString(), budget_ms: timeout, elapsed_ms: performance.now() - startMono, + end: exitReason === 'timeout' && signal?.aborted ? 'aborted' : SESSION_ENDS[exitReason] ?? 'error', + ...(exitReason === 'timeout' ? { evidence: `armed ${timeout}ms session timeout fired` } : exitReason !== 'success' && result.stderr ? { evidence: result.stderr } : {}), + billed: cost !== null, model, ...(cost !== null ? { cost_usd: cost, tokens: result.usage! } : {}) }); + + if (exitReason !== 'success' || browseErrors.length > 0) { + try { + const failureDir = runDir || path.join(workingDirectory, '.gstack', 'test-transcripts'); + fs.mkdirSync(failureDir, { recursive: true }); + fs.writeFileSync(path.join(failureDir, safeName ? `${safeName}-failure.json` : `e2e-${new Date().toISOString().replace(/[:.]/g, '-')}.json`), + JSON.stringify({ host: 'codex', model, prompt: prompt.slice(0, 500), testName: testName || 'unknown', exitReason, browseErrors, duration, + stderr: result.stderr.slice(0, 2000), output: output.slice(0, 500) }, null, 2)); + } catch { /* non-fatal */ } + } + + return { + toolCalls, browseErrors, exitReason, duration, output, transcript, model, firstResponseMs, maxInterTurnMs, + costEstimate: { inputChars: prompt.length, outputChars: output.length, + estimatedTokens: (result.usage?.input ?? 0) + (result.usage?.output ?? 0), estimatedCost: cost ?? 0, turnsUsed }, + ...(result.usage ? { tokenUsage: result.usage } : {}), + ...(cost === null ? { costKnown: false } : {}), + }; +} diff --git a/test/helpers/codex-session-runner.ts b/test/helpers/codex-session-runner.ts index 4eea35931..c040ebbfa 100644 --- a/test/helpers/codex-session-runner.ts +++ b/test/helpers/codex-session-runner.ts @@ -21,6 +21,7 @@ import { hermeticChildEnv } from './hermetic-env'; import { extractSkillSections } from './skill-fixture'; import { killProcessGroup } from '../../scripts/test-strict-output'; import { CODEX_FRONTIER_MODEL } from '../../scripts/resolvers/constants'; +import { estimateCostUsd, hasPricing } from '../../lib/pricing'; // --- Interfaces --- @@ -34,6 +35,47 @@ export interface CodexResult { sessionId: string | null; // Thread ID for session continuity rawLines: string[]; // Raw JSONL lines for debugging stderr: string; // Stderr output (skill loading errors, auth failures) + usage?: CodexTokenUsage | null; // Summed turn.completed usage; null when no turn completed + model?: string; // The --model this session ran +} + +/** + * Token classes of Codex `turn.completed` usage. `cached_input` (cache reads) + * and `cache_write_input` are subsets of `input`; `reasoning_output` is a + * subset of `output`. + */ +export interface CodexTokenUsage { + input: number; + cached_input: number; + cache_write_input: number; + output: number; + reasoning_output: number; +} + +export const NO_CODEX_USAGE: CodexTokenUsage = Object.freeze({ input: 0, cached_input: 0, cache_write_input: 0, output: 0, reasoning_output: 0 }); + +const count = (value: unknown) => (typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : 0); + +/** Add one `turn.completed` usage object to a running total. */ +export function addCodexTurnUsage(total: CodexTokenUsage, usage: Record | undefined): CodexTokenUsage { + return { + input: total.input + count(usage?.input_tokens), + cached_input: total.cached_input + count(usage?.cached_input_tokens), + cache_write_input: total.cache_write_input + count(usage?.cache_write_input_tokens), + output: total.output + count(usage?.output_tokens), + reasoning_output: total.reasoning_output + count(usage?.reasoning_output_tokens), + }; +} + +/** + * A session's cost at lib/pricing.ts rates: cache reads at the cached-input + * rate, all other input (cache writes included) at the input rate, output at + * the output rate. Null when the model has no price row. + */ +export function codexUsageCostUsd(usage: CodexTokenUsage, model: string | undefined): number | null { + if (!hasPricing(model)) return null; + const cached = Math.min(usage.cached_input, usage.input); + return estimateCostUsd({ input: usage.input - cached, cached, output: usage.output }, model); } /** Existing pipe-drain allowance, separate from the model's work budget. */ @@ -54,6 +96,7 @@ export interface ParsedCodexJSONL { toolCalls: string[]; tokens: number; sessionId: string | null; + usage: CodexTokenUsage | null; } /** @@ -71,6 +114,7 @@ export function parseCodexJSONL(lines: string[]): ParsedCodexJSONL { const toolCalls: string[] = []; let tokens = 0; let sessionId: string | null = null; + let usage: CodexTokenUsage | null = null; for (const line of lines) { if (!line.trim()) continue; @@ -95,9 +139,9 @@ export function parseCodexJSONL(lines: string[]): ParsedCodexJSONL { if (cmd) toolCalls.push(cmd); } } else if (t === 'turn.completed') { - const usage = obj.usage || {}; - const turnTokens = (usage.input_tokens || 0) + (usage.output_tokens || 0); - tokens += turnTokens; + const turn = obj.usage || {}; + tokens += (turn.input_tokens || 0) + (turn.output_tokens || 0); + usage = addCodexTurnUsage(usage ?? NO_CODEX_USAGE, turn); } } catch { /* skip malformed lines */ } } @@ -108,6 +152,7 @@ export function parseCodexJSONL(lines: string[]): ParsedCodexJSONL { toolCalls, tokens, sessionId, + usage, }; } @@ -166,6 +211,192 @@ export function installSkillToTempHome( return home; } +// --- Spawn core --- + +/** + * Spawn `codex ` in its own process group and collect its JSONL until + * exit, the deadline or an abort, with the existing drain grace. Shared by + * runCodexSkill and the Codex host (codex-host-runner.ts). `onLine` sees each + * JSONL line as it arrives with the elapsed milliseconds since `startTime`. + */ +export async function execCodex(args: string[], opts: { + cwd: string; + env: Record; + startTime: number; + deadline: number; + signal?: AbortSignal; + onLine?: (line: string, elapsedMs: number) => void; + /** Called with the group-owning pid right after spawn; a throw kills the group. */ + onSpawn?: (pid: number) => void; +}): Promise { + const { cwd, env, startTime, deadline, signal, onLine } = opts; + const proc = spawn('codex', args, { + cwd, + stdio: ['ignore', 'pipe', 'pipe'], + detached: process.platform !== 'win32', + env, + }); + if (opts.onSpawn) { + try { + if (proc.pid === undefined) throw new Error('Codex did not start'); + opts.onSpawn(proc.pid); + } catch (error) { + killProcessGroup(proc, 'SIGKILL'); + proc.stdout!.destroy(); + proc.stderr!.destroy(); + throw error; + } + } + const collectedLines: string[] = []; + // work --exit--> drain (<=5s) --> result + // \--timeout/abort--> kill group + close pipes --> timeout result + // Exit status comes from 'exit'; a forced drain after exit 0 is an error. + let stdoutBuffer = ''; + let stderr = ''; + let exitCode: number | undefined; + let timedOut = false; + let drainExpired = false; + let streamError: { stream: 'stdout' | 'stderr'; error: Error } | undefined; + let spawnError: Error | undefined; + let stdoutDone = false; + let stderrDone = false; + let stdoutEnded = false; + let stderrEnded = false; + let finalized = false; + const stdoutDecoder = new StringDecoder('utf8'); + const stderrDecoder = new StringDecoder('utf8'); + let workTimer: ReturnType | undefined; + let drainTimer: ReturnType | undefined; + let finish!: () => void; + const finished = new Promise((resolve) => { finish = resolve; }); + + const maybeFinish = () => { + if (exitCode !== undefined && stdoutDone && stderrDone) finish(); + }; + const closePipes = () => { + // Destroy both streams: canceling stdout alone cannot release an + // inherited stderr pipe held by a descendant in another process group. + proc.stdout!.destroy(); + proc.stderr!.destroy(); + }; + const expireDrain = () => { + drainExpired = !stdoutDone || !stderrDone; + killProcessGroup(proc, 'SIGKILL'); + closePipes(); + // Also bound the rare case where a failed group kill produces no exit. + finish(); + }; + const stopRun = () => { + // A real auth/crash exit that preceded a pipe stall stays a real exit. + if (exitCode === undefined) timedOut = true; + else drainExpired ||= !stdoutDone || !stderrDone; + killProcessGroup(proc, 'SIGKILL'); + closePipes(); + clearTimeout(drainTimer); + drainTimer = setTimeout(expireDrain, CODEX_DRAIN_GRACE_MS); + maybeFinish(); + }; + const onExit = (code: number | null, exitSignal: NodeJS.Signals | null) => { + exitCode = code ?? (exitSignal ? 128 + (os.constants.signals[exitSignal] ?? 0) : 1); + clearTimeout(workTimer); + clearTimeout(drainTimer); + // 'exit' means the child is gone; 'close' also waits for descendant + // pipes. Start the drain allowance at actual exit, never at close. + drainTimer = setTimeout(expireDrain, CODEX_DRAIN_GRACE_MS); + maybeFinish(); + }; + const onSpawnError = (error: Error) => { + if (finalized) return; + spawnError = error; + exitCode = 1; + closePipes(); + finish(); + }; + // Closure releases lifecycle waits, but only 'end' proves all bytes were + // drained. A destroyed pipe can emit 'close' without either EOF or error. + const onStdoutDone = () => { if (!finalized) { stdoutDone = true; maybeFinish(); } }; + const onStderrDone = () => { if (!finalized) { stderrDone = true; maybeFinish(); } }; + const onStdoutEnd = () => { if (!finalized) { stdoutBuffer += stdoutDecoder.end(); stdoutEnded = true; onStdoutDone(); } }; + const onStderrEnd = () => { if (!finalized) { stderr += stderrDecoder.end(); stderrEnded = true; onStderrDone(); } }; + const onStreamError = (stream: 'stdout' | 'stderr', error: Error) => { + if (!finalized) streamError ??= { stream, error }; + }; + const onStdout = (chunk: Buffer) => { + if (finalized) return; + stdoutBuffer += stdoutDecoder.write(chunk); + const lines = stdoutBuffer.split('\n'); + stdoutBuffer = lines.pop() || ''; + for (const line of lines) { + if (!line.trim()) continue; + collectedLines.push(line); + onLine?.(line, Date.now() - startTime); + try { + const event = JSON.parse(line); + if (event.type === 'item.completed' && event.item) { + const item = event.item; + const elapsed = Math.round((Date.now() - startTime) / 1000); + if (item.type === 'command_execution' && item.command) { + process.stderr.write(` [codex ${elapsed}s] ran: ${item.command.slice(0, 100)}\n`); + } else if (item.type === 'agent_message' && item.text) { + process.stderr.write(` [codex ${elapsed}s] message: ${item.text.slice(0, 100)}\n`); + } + } + } catch { /* malformed JSONL is ignored by parseCodexJSONL too */ } + } + }; + const onStderr = (chunk: Buffer) => { if (!finalized) stderr += stderrDecoder.write(chunk); }; + + proc.on('exit', onExit); + proc.on('error', onSpawnError); + proc.stdout!.on('data', onStdout); + proc.stderr!.on('data', onStderr); + proc.stdout!.on('end', onStdoutEnd).on('close', onStdoutDone).on('error', error => onStreamError('stdout', error)); + proc.stderr!.on('end', onStderrEnd).on('close', onStderrDone).on('error', error => onStreamError('stderr', error)); + signal?.addEventListener('abort', stopRun, { once: true }); + workTimer = setTimeout(stopRun, Math.max(0, deadline - Date.now())); + if (signal?.aborted || Date.now() >= deadline) stopRun(); + + try { + await finished; + if (stdoutBuffer.trim()) collectedLines.push(stdoutBuffer); + const parsed = parseCodexJSONL(collectedLines); + const result: CodexResult = { + ...parsed, + exitCode: timedOut ? 124 : exitCode ?? 1, + durationMs: Date.now() - startTime, + rawLines: collectedLines, + stderr, + }; + if (stderr.trim()) process.stderr.write(` [codex stderr] ${stderr.trim().slice(0, 200)}\n`); + if (spawnError) throw new CodexHarnessError(`Could not start Codex: ${spawnError.message}`, result); + const incompleteStreams = [!stdoutEnded && 'stdout', !stderrEnded && 'stderr'].filter(Boolean).join(' and '); + if (result.exitCode === 0 && (drainExpired || streamError || incompleteStreams)) { + throw new CodexHarnessError( + drainExpired ? `Codex output drain exceeded ${CODEX_DRAIN_GRACE_MS}ms after exit 0` + : streamError ? `Codex ${streamError.stream} stream failed: ${streamError.error.message}` + : `Codex ${incompleteStreams} closed before EOF after exit 0`, + result, + ); + } + return result; + } finally { + finalized = true; + clearTimeout(workTimer); + clearTimeout(drainTimer); + signal?.removeEventListener('abort', stopRun); + // Child exit and closed pipes do not prove its descendants exited: + // background tools may redirect both streams. Reap the owned group on + // every result, including ordinary success and genuine process failure. + killProcessGroup(proc, 'SIGKILL'); + closePipes(); + proc.removeListener('exit', onExit); + // Retain the harmless error listener through stream teardown so a late + // OS spawn error cannot turn into an unhandled event after cleanup. + proc.stdout!.removeListener('data', onStdout); + proc.stderr!.removeListener('data', onStderr); + } +} + // --- Main runner --- /** @@ -209,7 +440,7 @@ export async function runCodexSkill(opts: { const emptyResult = (exitCode: number, output = ''): CodexResult => ({ output, reasoning: [], toolCalls: [], tokens: 0, exitCode, - durationMs: Date.now() - startTime, sessionId: null, rawLines: [], stderr: '', + durationMs: Date.now() - startTime, sessionId: null, rawLines: [], stderr: '', usage: null, }); if (signal?.aborted || Date.now() >= deadline) return emptyResult(124); @@ -257,7 +488,8 @@ export async function runCodexSkill(opts: { // exactly that. Empirically verified against codex on this machine. const args = ['exec', '--json', '-s', sandbox, '--skip-git-repo-check']; if (ignoreUserConfig) args.push('--ignore-user-config'); - args.push('--model', model ?? process.env.GSTACK_CODEX_MODEL ?? CODEX_FRONTIER_MODEL); + const sessionModel = model ?? process.env.GSTACK_CODEX_MODEL ?? CODEX_FRONTIER_MODEL; + args.push('--model', sessionModel); for (const override of configOverrides) args.push('-c', override); args.push(prompt); @@ -275,162 +507,17 @@ export async function runCodexSkill(opts: { // dir beside CODEX_HOME instead of the shard TMPDIR that holds tempHome. const tempTmpDir = path.join(tempHome, 'tmp'); fs.mkdirSync(tempTmpDir, { recursive: true }); - const proc = spawn('codex', args, { - cwd: cwd || skillDir, - stdio: ['ignore', 'pipe', 'pipe'], - detached: process.platform !== 'win32', + const result = await execCodex(args, { + cwd: cwd || skillDir, startTime, deadline, signal, env: hermeticChildEnv( { HOME: tempHome, CODEX_HOME: tempCodexDir, TMPDIR: tempTmpDir }, { extraAllow: ['OPENAI_API_KEY', 'CODEX_*'] }, ), + }).catch((error: unknown) => { + if (error instanceof CodexHarnessError && error.result) error.result.model = sessionModel; + throw error; }); - const collectedLines: string[] = []; - // work --exit--> drain (<=5s) --> result - // \--timeout/abort--> kill group + close pipes --> timeout result - // Exit status comes from 'exit'; a forced drain after exit 0 is an error. - let stdoutBuffer = ''; - let stderr = ''; - let exitCode: number | undefined; - let timedOut = false; - let drainExpired = false; - let streamError: { stream: 'stdout' | 'stderr'; error: Error } | undefined; - let spawnError: Error | undefined; - let stdoutDone = false; - let stderrDone = false; - let stdoutEnded = false; - let stderrEnded = false; - let finalized = false; - const stdoutDecoder = new StringDecoder('utf8'); - const stderrDecoder = new StringDecoder('utf8'); - let workTimer: ReturnType | undefined; - let drainTimer: ReturnType | undefined; - let finish!: () => void; - const finished = new Promise((resolve) => { finish = resolve; }); - - const maybeFinish = () => { - if (exitCode !== undefined && stdoutDone && stderrDone) finish(); - }; - const closePipes = () => { - // Destroy both streams: canceling stdout alone cannot release an - // inherited stderr pipe held by a descendant in another process group. - proc.stdout!.destroy(); - proc.stderr!.destroy(); - }; - const expireDrain = () => { - drainExpired = !stdoutDone || !stderrDone; - killProcessGroup(proc, 'SIGKILL'); - closePipes(); - // Also bound the rare case where a failed group kill produces no exit. - finish(); - }; - const stopRun = () => { - // A real auth/crash exit that preceded a pipe stall stays a real exit. - if (exitCode === undefined) timedOut = true; - else drainExpired ||= !stdoutDone || !stderrDone; - killProcessGroup(proc, 'SIGKILL'); - closePipes(); - clearTimeout(drainTimer); - drainTimer = setTimeout(expireDrain, CODEX_DRAIN_GRACE_MS); - maybeFinish(); - }; - const onExit = (code: number | null, exitSignal: NodeJS.Signals | null) => { - exitCode = code ?? (exitSignal ? 128 + (os.constants.signals[exitSignal] ?? 0) : 1); - clearTimeout(workTimer); - clearTimeout(drainTimer); - // 'exit' means the child is gone; 'close' also waits for descendant - // pipes. Start the drain allowance at actual exit, never at close. - drainTimer = setTimeout(expireDrain, CODEX_DRAIN_GRACE_MS); - maybeFinish(); - }; - const onSpawnError = (error: Error) => { - if (finalized) return; - spawnError = error; - exitCode = 1; - closePipes(); - finish(); - }; - // Closure releases lifecycle waits, but only 'end' proves all bytes were - // drained. A destroyed pipe can emit 'close' without either EOF or error. - const onStdoutDone = () => { if (!finalized) { stdoutDone = true; maybeFinish(); } }; - const onStderrDone = () => { if (!finalized) { stderrDone = true; maybeFinish(); } }; - const onStdoutEnd = () => { if (!finalized) { stdoutBuffer += stdoutDecoder.end(); stdoutEnded = true; onStdoutDone(); } }; - const onStderrEnd = () => { if (!finalized) { stderr += stderrDecoder.end(); stderrEnded = true; onStderrDone(); } }; - const onStreamError = (stream: 'stdout' | 'stderr', error: Error) => { - if (!finalized) streamError ??= { stream, error }; - }; - const onStdout = (chunk: Buffer) => { - if (finalized) return; - stdoutBuffer += stdoutDecoder.write(chunk); - const lines = stdoutBuffer.split('\n'); - stdoutBuffer = lines.pop() || ''; - for (const line of lines) { - if (!line.trim()) continue; - collectedLines.push(line); - try { - const event = JSON.parse(line); - if (event.type === 'item.completed' && event.item) { - const item = event.item; - const elapsed = Math.round((Date.now() - startTime) / 1000); - if (item.type === 'command_execution' && item.command) { - process.stderr.write(` [codex ${elapsed}s] ran: ${item.command.slice(0, 100)}\n`); - } else if (item.type === 'agent_message' && item.text) { - process.stderr.write(` [codex ${elapsed}s] message: ${item.text.slice(0, 100)}\n`); - } - } - } catch { /* malformed JSONL is ignored by parseCodexJSONL too */ } - } - }; - const onStderr = (chunk: Buffer) => { if (!finalized) stderr += stderrDecoder.write(chunk); }; - - proc.on('exit', onExit); - proc.on('error', onSpawnError); - proc.stdout!.on('data', onStdout); - proc.stderr!.on('data', onStderr); - proc.stdout!.on('end', onStdoutEnd).on('close', onStdoutDone).on('error', error => onStreamError('stdout', error)); - proc.stderr!.on('end', onStderrEnd).on('close', onStderrDone).on('error', error => onStreamError('stderr', error)); - signal?.addEventListener('abort', stopRun, { once: true }); - workTimer = setTimeout(stopRun, Math.max(0, deadline - Date.now())); - if (signal?.aborted || Date.now() >= deadline) stopRun(); - - try { - await finished; - if (stdoutBuffer.trim()) collectedLines.push(stdoutBuffer); - const parsed = parseCodexJSONL(collectedLines); - const result: CodexResult = { - ...parsed, - exitCode: timedOut ? 124 : exitCode ?? 1, - durationMs: Date.now() - startTime, - rawLines: collectedLines, - stderr, - }; - if (stderr.trim()) process.stderr.write(` [codex stderr] ${stderr.trim().slice(0, 200)}\n`); - if (spawnError) throw new CodexHarnessError(`Could not start Codex: ${spawnError.message}`, result); - const incompleteStreams = [!stdoutEnded && 'stdout', !stderrEnded && 'stderr'].filter(Boolean).join(' and '); - if (result.exitCode === 0 && (drainExpired || streamError || incompleteStreams)) { - throw new CodexHarnessError( - drainExpired ? `Codex output drain exceeded ${CODEX_DRAIN_GRACE_MS}ms after exit 0` - : streamError ? `Codex ${streamError.stream} stream failed: ${streamError.error.message}` - : `Codex ${incompleteStreams} closed before EOF after exit 0`, - result, - ); - } - return result; - } finally { - finalized = true; - clearTimeout(workTimer); - clearTimeout(drainTimer); - signal?.removeEventListener('abort', stopRun); - // Child exit and closed pipes do not prove its descendants exited: - // background tools may redirect both streams. Reap the owned group on - // every result, including ordinary success and genuine process failure. - killProcessGroup(proc, 'SIGKILL'); - closePipes(); - proc.removeListener('exit', onExit); - // Retain the harmless error listener through stream teardown so a late - // OS spawn error cannot turn into an unhandled event after cleanup. - proc.stdout!.removeListener('data', onStdout); - proc.stderr!.removeListener('data', onStderr); - } + return { ...result, model: sessionModel }; } finally { // Clean up temp HOME diff --git a/test/helpers/cookie-workflow-judge-input.ts b/test/helpers/cookie-workflow-judge-input.ts index 5e595f32d..814d66892 100644 --- a/test/helpers/cookie-workflow-judge-input.ts +++ b/test/helpers/cookie-workflow-judge-input.ts @@ -4,7 +4,7 @@ import { join } from 'node:path'; import { buildWorkflowJudgePrompt, type WorkflowJudgeFile, type WorkflowJudgeInput } from './workflow-judge-input'; export const COOKIE_WORKFLOW_JUDGE = { - model: 'claude-sonnet-4-6', + model: 'claude-sonnet-5-5', judgeContext: 'a fallback-browser cookie import workflow', judgeGoal: 'how to select an authorized source browser, profile, and domain without guessing an account; configure optional authentication verification before mutation; obtain explicit consent for precisely scoped storage reset; distinguish copied cookies from positive sign-in evidence; and recover within the documented platform and privacy boundaries', thresholds: { clarity: 4, completeness: 3, actionability: 4 }, diff --git a/test/helpers/e2e-helpers.ts b/test/helpers/e2e-helpers.ts index 2d4a2a530..6ee6937e6 100644 --- a/test/helpers/e2e-helpers.ts +++ b/test/helpers/e2e-helpers.ts @@ -16,6 +16,7 @@ import { WorktreeManager } from '../../lib/worktree'; import type { HarvestResult } from '../../lib/worktree'; import { preflightAnthropicApi } from './anthropic-preflight'; import { isHermeticEnabled } from './hermetic-env'; +import { hostSkipReason, resolveEvalHost } from './e2e-hosts'; import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; @@ -133,6 +134,17 @@ if (evalsEnabled && process.env.EVALS_TIER) { process.stderr.write(`EVALS_TIER=${tier}: ${selectedTests.length} tests\n\n`); } +// EVALS_HOST=codex: Claude-only cases are skipped with their reason, never run +// (and never failed); the report deselects them (test/helpers/e2e-hosts.ts). +if (evalsEnabled && resolveEvalHost() !== 'claude') { + const host = resolveEvalHost(); + const candidates = selectedTests ?? Object.keys(E2E_TIERS); + const skipped = candidates.filter(id => hostSkipReason(id, host)); + selectedTests = candidates.filter(id => !hostSkipReason(id, host)); + process.stderr.write(`EVALS_HOST=${host}: ${selectedTests.length} tests; ${skipped.length} Claude-only skipped (${skipped.join(', ') || 'none'})\n\n`); + (await import('./codex-host-runner')).prepareCodexHostRuntime(); +} + export const describeE2E = evalsEnabled ? describe : describe.skip; /** @@ -261,6 +273,8 @@ export function recordE2E( ...(!passed && isPreTurnInfraFailure(result) ? { failure_class: 'infra' as const } : {}), duration_ms: result.duration, cost_usd: result.costEstimate.estimatedCost, + ...(result.costKnown === false ? { cost_known: false } : {}), + ...(result.tokenUsage ? { token_usage: result.tokenUsage } : {}), transcript: result.transcript, output: result.output?.slice(0, 2000), turns_used: result.costEstimate.turnsUsed, diff --git a/test/helpers/e2e-hosts.ts b/test/helpers/e2e-hosts.ts new file mode 100644 index 000000000..622c3756d --- /dev/null +++ b/test/helpers/e2e-hosts.ts @@ -0,0 +1,126 @@ +/** + * Agent host selection for paid E2E cases (GSTA-49 TM2). EVALS_HOST picks the + * agent every runSkillTest session runs on: `claude` (default) or `codex` + * (codex exec, test/helpers/codex-host-runner.ts). E2E_HOSTS is the per-case + * host metadata over every gate case (classification in the GSTA-49 TM2 + * scoping): `both` runs on either host with the shared adapter; every other + * value is the reason the case is Claude-only. On the Codex host a Claude-only + * case is SKIPPED with its reason, planned as excluded and deselected by the + * report with zero credit, never counted as failed. A case without metadata + * (every non-gate case) is Claude-only. + */ + +export type EvalHost = 'claude' | 'codex'; +export const EVAL_HOSTS: readonly EvalHost[] = ['claude', 'codex']; + +export function resolveEvalHost(env: NodeJS.ProcessEnv = process.env): EvalHost { + const raw = (env.EVALS_HOST ?? '').trim(); + if (!raw) return 'claude'; + if ((EVAL_HOSTS as readonly string[]).includes(raw)) return raw as EvalHost; + throw new Error(`EVALS_HOST must be one of ${EVAL_HOSTS.join(', ')}; got "${raw}"`); +} + +export const CLAUDE_ONLY_REASONS = { + pty: 'drives the interactive Claude Code TUI (plan mode, native AskUserQuestion) through the PTY harness', + sdk: 'uses the Claude Agent SDK (canUseTool answers and PreToolUse hooks)', + subagents: 'asserts Claude Agent/Task subagent events', + 'skill-tool': 'depends on the Claude Skill tool load and registration', + hermetic: 'a claude -p hermetic canary; the Codex host needs its own canary twin', + 'self-host': 'the Claude host calling /codex; on the Codex host /codex is the agent itself', + rework: 'parses Claude stream-json tool events or prescribes native Claude tools; not ported to the Codex host', + 'host-neutral': 'runs no model session, so a second host adds nothing', +} as const; + +export type ClaudeOnlyReason = keyof typeof CLAUDE_ONLY_REASONS; +export type HostSupport = 'both' | ClaudeOnlyReason; + +/** Adapter-only cases: final output, files, git state, Bash commands, server logs or browseErrors. */ +const ADAPTER_ONLY: readonly string[] = [ + 'benchmark-workflow', + 'browse-basic', + 'browse-snapshot', + 'canary-workflow', + 'context-recovery-artifacts', + 'context-restore-loads-latest', + 'context-save-writes-file', + 'cso-diff-mode', + 'design-review-detector-shim', + 'design-review-detector-shim-dom', + 'design-review-plugin-handoff', + 'diagram-triplet', + 'document-release', + 'gstack-upgrade-happy-path', + 'land-and-deploy-first-run', + 'land-and-deploy-review-gate', + 'land-and-deploy-workflow', + 'learnings-show', + 'operational-learning', + 'plan-design-review-no-ui-scope', + 'plan-eng-coverage-audit', + 'plan-review-report', + 'plan-tune-inspect', + 'qa-bootstrap', + 'qa-quick', + 'retro-base-branch', + 'review-army-delivery-audit', + 'review-army-json-findings', + 'review-army-migration-safety', + 'review-army-quality-score', + 'review-base-branch', + 'review-coverage-audit', + 'review-enum-completeness', + 'review-sql-injection', + 'review-test-value', + 'safety-pair-agent-block', + 'safety-ship-stale-evidence', + 'setup-deploy-workflow', + 'ship-base-branch', + 'ship-coverage-audit', + 'ship-coverage-value', + 'ship-local-workflow', + 'ship-triage', + 'skillmd-no-local-binary', + 'skillmd-outside-git', + 'skillmd-setup-discovery', + 'test-audit-report-only', + 'tpa-absent-darwin', + 'tpa-absent-linux', + 'tpa-apple-ban', + 'tpa-broken', + 'tpa-present' +]; + +const CLAUDE_ONLY: Readonly> = { + pty: ['autoplan-guard-pty', 'office-hours-auto-mode', 'plan-ceo-finding-floor', 'plan-ceo-review-plan-mode', + 'plan-design-with-ui-scope', 'plan-devex-finding-floor', 'plan-devex-review-plan-mode', 'plan-mode-no-op'], + sdk: ['auq-format-gate', 'investigate-owned-abort', 'investigate-owned-completion', 'investigate-owned-ending-error', + 'shared-libs-review-index-flags', 'shared-libs-review-lifecycle', 'shared-libs-review-path-eligibility', + 'shared-libs-review-prior-coverage', 'shared-libs-review-revalidation', 'ship-local-hook-preservation', + 'ship-managed-hook-refresh', 'ship-skipped-queued-finding', 'ship-unmanaged-hook-consent'], + subagents: ['docsync-spawned', 'review-army-perf-n-plus-one'], + 'skill-tool': ['skillify-approval-reject', 'skillify-happy-path', 'skillify-provenance-refusal'], + hermetic: ['hermetic-canary', 'hermetic-sentinel'], + 'self-host': ['safety-codex-consult-embed'], + rework: ['qa-functional-cli-fix', 'qa-functional-cli-report', 'qa-functional-webhook-fix', 'qa-functional-webhook-report', + 'qa-only-no-fix', 'review-dashboard-via', 'review-exploratory-small-cli', 'review-gate-integrity', + 'shared-libs-read-only', 'shared-libs-unsupported-git', 'ship-docsync-completion', 'ship-docsync-current', + 'ship-docsync-failure', 'ship-docsync-late-result', 'ship-docsync-launch-failure', 'ship-docsync-missing-asset', + 'ship-docsync-missing-marker', 'ship-docsync-recovery', 'ship-docsync-stale-after', 'ship-docsync-stale-before', + 'ship-docsync-store', 'ship-docsync-timeout-unsettled', 'ship-exploratory-late-input', 'ship-exploratory-plan-checks', + 'ship-exploratory-small-cli', 'ship-exploratory-unavailable'], + 'host-neutral': ['timeline-event-flow'], +}; + +export const E2E_HOSTS: Readonly> = Object.freeze(Object.fromEntries([ + ...ADAPTER_ONLY.map(id => [id, 'both'] as const), + ...(Object.entries(CLAUDE_ONLY) as Array<[ClaudeOnlyReason, readonly string[]]>).flatMap(([reason, ids]) => ids.map(id => [id, reason] as const)), +])); + +/** Why a case does not run on `host`, or null when it does. */ +export function hostSkipReason(id: string, host: EvalHost, hosts: Readonly> = E2E_HOSTS): string | null { + if (host === 'claude') return null; + const support = hosts[id]; + if (support === 'both') return null; + if (support === undefined) return `Claude-only on the ${host} host: no Codex host metadata (E2E_HOSTS covers the gate cases)`; + return `Claude-only on the ${host} host: ${CLAUDE_ONLY_REASONS[support]}`; +} diff --git a/test/helpers/eval-store.ts b/test/helpers/eval-store.ts index 609c6d099..ed8721f13 100644 --- a/test/helpers/eval-store.ts +++ b/test/helpers/eval-store.ts @@ -14,6 +14,7 @@ import * as os from 'os'; import { spawnSync } from 'child_process'; import { isManualReviewEntry } from './cookie-workflow-manual-review'; import type { ManualJudgeReview } from './cookie-workflow-manual-review'; +import type { CodexTokenUsage } from './codex-session-runner'; // v2: EvalTestEntry.harvest gains optional {insertions, deletions, net} and // may be explicitly null (arm-benchmark harvest-failure taxonomy). Readers @@ -67,6 +68,8 @@ export interface EvalTestEntry { cost_usd: number; /** False when the harness captured no billing, so cost_usd 0 means unknown. Absent means known. */ cost_known?: boolean; + /** Codex sessions: the token classes behind cost_usd. */ + token_usage?: CodexTokenUsage; /** Absent in older records means executed; reuse is never a new model run. */ execution?: 'executed' | 'reused'; reused_from?: { input_key: string; run_id: string; revision: string; completed_at: string }; diff --git a/test/helpers/free-fixtures-data.ts b/test/helpers/free-fixtures-data.ts index 7c724f1c6..317e26b5d 100644 --- a/test/helpers/free-fixtures-data.ts +++ b/test/helpers/free-fixtures-data.ts @@ -54,6 +54,7 @@ export const FREE_FIXTURES: Readonly> = { 'test/fixtures/claude-agent-payload-2.1.292.json': ['test/autoplan-publication-payload.test.ts', 'test/autoplan-schema-canary-compare.test.ts'], 'test/fixtures/claude-native-journal-roots-2.1.284.json': ['test/autoplan-publication-hook.test.ts', 'test/plan-count-transcript.test.ts'], 'test/fixtures/codex-boundary-commands.json': ['test/codex-boundary-evidence.test.ts'], + 'test/fixtures/codex-host/**': ['test/codex-host.test.ts'], 'test/fixtures/codex-review-sandbox-unavailable-37158847998.json': ['test/outside-voice-evidence.test.ts'], 'test/fixtures/codex-review/**': ['test/codex-hardening.test.ts', 'test/outside-review-severity.test.ts', 'test/outside-voice-evidence.test.ts', 'test/skill-validation.test.ts'], 'test/fixtures/codex-sandbox/**': ['test/codex-eval-recording.test.ts', 'test/codex-model-probe.test.ts', 'test/outside-review-result.test.ts', 'test/outside-voice-invocation.test.ts'], diff --git a/test/helpers/manual-judge-review-fixture.ts b/test/helpers/manual-judge-review-fixture.ts index ded8a3508..0acb4f5d5 100644 --- a/test/helpers/manual-judge-review-fixture.ts +++ b/test/helpers/manual-judge-review-fixture.ts @@ -46,14 +46,31 @@ const LATER_ENTRYPOINT_EDITS: Array<[string, string]> = [ ['bash "$tmpfile" "bun-v$BUN_VERSION"', 'BUN_VERSION="$BUN_VERSION" bash "$tmpfile"'], ]; +// The GSTA-49 cookie-workflow repair rewrote the entrypoint's steps 1-5 (split +// the two probe READY outcomes, dropped the duplicate SETUP block for the browse +// fallback, moved source selection before the target, asked about verification +// before the daemon starts, added a report shape) and one BROWSER.md comment. +// Reversing it first restores the pre-repair text, which the edits below then +// take back further. +const LATER_COOKIE_ORDER_EDITS: Array<[string, string]> = [ + ["## 1. Check for Aside\n\nUse this checkout as the gstack root if it contains `BROWSER.md` and `browse/SKILL.md`; otherwise use the installed root containing `bin/gstack-skill-start`, never a generated host stub. Read that root's `browse/SKILL.md` **BROWSER SETUP** section and run its Aside probe.\n\n- **Aside probe prints `READY`:** no import is needed, because Aside already has the user's signed-in sessions. Tell the user to use Aside, or to sign in there, and end this skill.\n- **Any other result:** continue with gstack's own headless browser (`$B`).\n\n## 2. Get every decision from the user first\n\nAsk these together, before running any `$B` command. Never guess an answer or treat the default Comet browser as consent.\n\n1. **Source browser and profile.** `--profile` takes the profile's directory (`Default`, `Profile 2`), not its display name. A profile that cannot be read is unknown, not empty.\n2. **Domains to import,** and the **target**: one page on a chosen domain that the session should open, by URL.\n3. **Sign-in verification (`--verify-auth`)?** Default no. It reloads the target and reports **verified** only after a successful same-origin response with exactly one visible element, found by a CSS selector, whose whitespace-normalized text equals the expected identity. If yes, the user supplies the selector and the expected identity privately.\n4. **Clear stale storage (`--clear-storage`)?** Default no; needs explicit approval. Chromium only. It clears localStorage for the target's origin (shared by that origin's tabs) and sessionStorage for the target tab, and nothing else; other browser engines reject the reset but still import. A failed reset may already have cleared some storage and is not rolled back.\n5. **Every cookie in the profile (`--all`)?** Default no; needs explicit consent. It cannot be combined with a domain or with clearing storage.\n\nUse the picker when any of answers 1-2 is still open; it asks for them. Use direct import when the user has named the browser, the profile directory and the domains.\n\n## 3. Prepare gstack's browser\n\n1. If the user wants verification, have them set `GSTACK_COOKIE_AUTH_SELECTOR` and `GSTACK_COOKIE_AUTH_EXPECTED_IDENTITY` privately in the shell that runs `$B`. The browser daemon reads them only when it starts, so if a gstack daemon is already running, run `$B stop` first. Never invent the values.\n2. Follow `browse/SKILL.md`'s **Browser fallback** section to find `$B`, building it if the check prints `NEEDS_SETUP`.\n3. Run `$B status`. If it reports `Mode: cdp`, `$B` is attached to the user's real browser and already has its sessions: tell the user no import is needed and end this skill.\n4. Open the target URL from answer 2 and leave that tab alone until the import finishes:\n\n```bash\n$B goto \"\"\n```\n\n## 4. Import\n\nPicker (the user confirms the browser, profile and domains there, ticks verification if they asked for it, then tells you when done; the link works once, for five minutes, so rerun the command if it expires):\n\n```bash\n$B cookie-import-browser\n```\n\nDirect import: run `$B cookie-import-browser` with the chosen browser name, one `--domain` per run and `--profile` set to the profile directory (not its display name); add `--verify-auth` or `--clear-storage` only if the user said yes in step 2.\n\nIf the import rejects because verification settings are missing, nothing changed: tell the user which variable is missing and return to step 3. Read that same root's `BROWSER.md`, **Choosing a source and checking sign-in**, for other errors and receipt failure reasons; it has the error tables, examples, profile labels, supported sources and platform setup.\n\n## 5. Report honestly\n\nReport from the import receipt, not from raw `$B cookies`, as three lines:\n\n- `Imported: (, as the receipt says) from for `\n- `Storage reset: `\n- `Sign-in: `\n\n**Not checked** means verification was not requested; **not verified** means it ran and failed; **verified** needs the identity match. Cookie counts, zero imports and HTTP 200 never prove sign-in. Imported cookies apply to the whole browser context, not one tab.\n\nNever request or publish cookie values, passwords, identity or profile text, session details, or raw errors in public logs.\n\n## Platform boundaries", "## 1. Choose the browser\n\nUse this checkout as the gstack root if it contains `BROWSER.md` and `browse/SKILL.md`; otherwise use the installed root containing `bin/gstack-skill-start`, never a generated host stub. Read that root's `browse/SKILL.md` **BROWSER SETUP** section and run its probe first. On `READY`, stop importing: use Aside's sessions or ask the user to sign in there. Otherwise follow the probe's fallback handling, then continue below.\n\n## SETUP (run this check BEFORE any browse command)\n\n```bash\n_ROOT=$(git rev-parse --show-toplevel 2>/dev/null)\nB=\"\"\n[ -n \"$_ROOT\" ] && [ -x \"$_ROOT/.claude/skills/gstack/browse/dist/browse\" ] && B=\"$_ROOT/.claude/skills/gstack/browse/dist/browse\"\n[ -z \"$B\" ] && B=\"$HOME/.claude/skills/gstack/browse/dist/browse\"\nif [ -x \"$B\" ] && \"${B%/browse/*}/bin/gstack-browser-ensure\"; then\n echo \"READY: $B\"\nelse\n echo \"NEEDS_SETUP\"\nfi\n```\n\n`gstack-browser-ensure` installs Chromium on first use when the install skipped it (lazy browser); it prints `BROWSER_OK` or `BROWSER_UNAVAILABLE ` first.\n\nIf `NEEDS_SETUP`:\n1. Tell the user: \"gstack browse needs a one-time build (~10 seconds). OK to proceed?\" Then STOP and wait.\n2. Run: `cd && ./setup`\n3. If `bun` is not installed:\n ```bash\n if ! command -v bun >/dev/null 2>&1; then\n BUN_VERSION=\"1.4.2\"\n BUN_INSTALL_SHA=\"bab8acfb046aac8c72407bdcce903957665d655d7acaa3e11c7c4616beae68dd\"\n tmpfile=$(mktemp \"${TMPDIR:-/tmp}/bun-install.XXXXXX\")\n curl -fsSL \"https://bun.sh/install\" -o \"$tmpfile\"\n # shasum is macOS/perl; coreutils-only Linux ships sha256sum instead —\n # resolve whichever exists so the verify never fails on a missing tool.\n if command -v sha256sum >/dev/null 2>&1; then\n actual_sha=$(sha256sum < \"$tmpfile\" | awk '{print $(1)}')\n else\n actual_sha=$(shasum -a 256 < \"$tmpfile\" | awk '{print $(1)}')\n fi\n if [ \"$actual_sha\" != \"$BUN_INSTALL_SHA\" ]; then\n echo \"ERROR: bun install script checksum mismatch\" >&2\n echo \" expected: $BUN_INSTALL_SHA\" >&2\n echo \" got: $actual_sha\" >&2\n rm \"$tmpfile\"; exit 1\n fi\n bash \"$tmpfile\" \"bun-v$BUN_VERSION\"\n rm \"$tmpfile\"\n fi\n ```\n\n```bash\n$B status\n```\nIf status says `Mode: cdp`, stop: the real browser already has sessions.\n\n## 2. Confirm options before import\n\nOpen the known target and keep its tab unchanged. Both options default off and require explicit request:\n\n- **`--verify-auth` / picker checkbox:** reloads the target. Have the user privately configure daemon `GSTACK_COOKIE_AUTH_SELECTOR` and `GSTACK_COOKIE_AUTH_EXPECTED_IDENTITY` **before startup**. Never invent values or assume CLI env reconfigures an existing daemon. Missing config rejects before mutation. Require a successful same-origin response and exactly one visible element whose whitespace-normalized text exactly matches the expected identity.\n- **`--clear-storage`:** for suspected stale storage, obtain explicit approval. Chromium only: clears captured-origin localStorage (shared across same-origin context tabs) and target-tab sessionStorage. Other origins, other tabs' sessionStorage, IndexedDB, and service workers stay intact. It uses an isolated world/native deadline; other engines reject reset, not imports/auth checks. Never auto-approve or claim rollback after partial failure.\n\n## 3. Select source and scope\n\n```bash\n$B cookie-import-browser\n```\n\nAsk the user to choose browser, account/profile, and domains, then say when done. Never guess accounts or treat default Comet as consent. Unreadable profiles are unknown, not empty. Rerun for an expired five-minute one-use link.\n\nDirect import: pass the chosen browser and `--domain` after navigating to a matching target. `--profile` takes a directory, not a display name; omit only for an unambiguous relevant profile, otherwise use the picker. `--all` requires consent for all non-expired profile cookies; it cannot accompany `--domain` or `--clear-storage`.\n\nRead that same root's `BROWSER.md`, **Choosing a source and checking sign-in**, for examples, profile labels, supported sources and platform setup.\n\n## 4. Report honestly\n\nReport receipt/picker counts, partial/zero/error and reset outcomes, not raw `$B cookies`. Imports affect the context, not one tab. **Not checked** means no requested check; **not verified** means it failed; **verified** requires positive target evidence. Zero imports, counts, or HTTP 200 never prove login.\n\nNever request/publish cookie values, passwords, identity/profile text, session details, or raw errors in public logs.\n\n## Platform boundaries"], + ["# view the page; only --verify-auth proves sign-in", "# confirm the intended account"], +]; + export function approvedCookieWorkflowSource(source: string): string { let removedLines = 0; let removedEntrypointLines = 0; - let historical = source + let reordered = source; + for (const [later, earlier] of LATER_COOKIE_ORDER_EDITS) { + if (!reordered.includes(later)) continue; + removedEntrypointLines += later.split('\n').length - earlier.split('\n').length; + reordered = reordered.replace(later, earlier); + } + let historical = reordered .replace('sha256sum < "$tmpfile" | awk \'{print $(1)}\'', 'sha256sum "$tmpfile" | awk \'{print $1}\'') .replace('shasum -a 256 < "$tmpfile" | awk \'{print $(1)}\'', 'shasum -a 256 "$tmpfile" | awk \'{print $1}\'') .replace(LATER_BROWSER_BLOCK, block => { removedLines = block.split('\n').length - 1; return ''; }) - .replace(LATER_ENTRYPOINT_BLOCK, block => { removedEntrypointLines = block.split('\n').length - 1; return ''; }); + .replace(LATER_ENTRYPOINT_BLOCK, block => { removedEntrypointLines += block.split('\n').length - 1; return ''; }); for (const [later, earlier] of [...LATER_BROWSER_EDITS, ...LATER_PREAMBLE_EDITS, ...LATER_WAVE_EDITS, ...LATER_ENTRYPOINT_EDITS]) historical = historical.replace(later, earlier); return historical .replace(/(--- BEGIN FILE "BROWSER\.md" \(lines \d+-)(\d+)(; section\) ---)/, (_, head, end, tail) => `${head}${Number(end) - removedLines}${tail}`) diff --git a/test/helpers/session-ledger.ts b/test/helpers/session-ledger.ts index 6e3c9e837..425323284 100644 --- a/test/helpers/session-ledger.ts +++ b/test/helpers/session-ledger.ts @@ -12,6 +12,7 @@ */ import * as fs from 'fs'; import * as path from 'path'; +import type { CodexTokenUsage } from './codex-session-runner'; export const SESSION_LEDGER_FILE = 'session-ledger.jsonl'; export const SESSION_EVIDENCE_MAX = 200; @@ -48,6 +49,10 @@ export interface SessionLedgerRow { liveness?: LivenessSummary; /** Whether the runner captured this session's billing (a terminal result with total_cost_usd). */ billed?: boolean; + /** Codex sessions: the model, its billed cost and token classes. */ + model?: string; + cost_usd?: number; + tokens?: CodexTokenUsage; } /** One line, control characters stripped, mentions neutralized, capped. */ diff --git a/test/helpers/session-runner.ts b/test/helpers/session-runner.ts index d40eb29e8..cc65a66ad 100644 --- a/test/helpers/session-runner.ts +++ b/test/helpers/session-runner.ts @@ -17,6 +17,8 @@ import { hermeticChildEnv, isHermeticEnabled } from './hermetic-env'; import { killProcessGroup } from '../../scripts/test-strict-output'; import { resolveEvalModel } from '../../lib/eval-model'; import { SessionObserver, appendSessionLedger, sessionKey, type SessionEnd } from './session-ledger'; +import { resolveEvalHost } from './e2e-hosts'; +import type { CodexTokenUsage } from './codex-session-runner'; const GSTACK_DEV_DIR = path.join(os.homedir(), '.gstack-dev'); const HEARTBEAT_PATH = path.join(GSTACK_DEV_DIR, 'e2e-live.json'); // heartbeat stays global @@ -55,6 +57,10 @@ export interface SkillTestResult { firstResponseMs: number; /** Peak latency between consecutive tool calls, in ms */ maxInterTurnMs: number; + /** Codex host: token classes behind costEstimate.estimatedCost. */ + tokenUsage?: CodexTokenUsage; + /** Codex host: false when the session completed no turn or its model has no price row. */ + costKnown?: boolean; } /** Local default startup grace: 90s covers observed API queue latency @@ -76,6 +82,14 @@ const BROWSE_ERROR_PATTERNS = [ /no such file or directory.*\bbrowse(?:\.exe)?(?=$|[\s'":),])/i, ]; +/** First match of each browse failure pattern in tool output plus stderr. */ +export function scanBrowseErrors(text: string): string[] { + return BROWSE_ERROR_PATTERNS.flatMap(pattern => { + const match = text.match(pattern); + return match ? [match[0].slice(0, 200)] : []; + }); +} + // --- Testable NDJSON parser --- export interface ParsedNDJSON { @@ -300,6 +314,7 @@ Setup, CLI startup and API queueing consume this same window; it never resets. Before source Reads, use Bash to run exactly \`date -u +%Y-%m-%dT%H:%M:%SZ\`. After each saved checkpoint, compare the latest evidence capture's printed completedAt with the times above; run that clock read again only when no capture has completed since your last clock read. When remaining time is at most ${reserve / 1000} seconds, prioritize the remaining required completion outputs and verification. No required content or gate may be skipped. If the clock read fails, report timing unavailable; do not invent remaining time or restart the deadline.`; systemPrompt = systemPrompt ? `${systemPrompt}\n\n${notice}` : notice; } + if (resolveEvalHost() === 'codex') return (await import('./codex-host-runner')).runCodexHostSkillTest({ ...options, appendSystemPrompt: systemPrompt, completionReserveMs: undefined }); // Set up per-run log directory if runId is provided let runDir: string | null = null; @@ -661,13 +676,7 @@ Before source Reads, use Bash to run exactly \`date -u +%Y-%m-%dT%H:%M:%SZ\`. Af const { transcript, resultLine, toolCalls } = parsed; const browseErrors: string[] = []; - const allText = toolCalls.filter(call => call.tool === 'Bash').map(call => call.output).join('\n') + '\n' + stderr; - for (const pattern of BROWSE_ERROR_PATTERNS) { - const match = allText.match(pattern); - if (match) { - browseErrors.push(match[0].slice(0, 200)); - } - } + browseErrors.push(...scanBrowseErrors(toolCalls.filter(call => call.tool === 'Bash').map(call => call.output).join('\n') + '\n' + stderr)); // Native Claude uses exit 1 for its structured max-turns result. Preserve // that existing semantic outcome only after a complete, uncancelled drain; diff --git a/test/office-hours-attempt.test.ts b/test/office-hours-attempt.test.ts index a5345a0f8..29946a398 100644 --- a/test/office-hours-attempt.test.ts +++ b/test/office-hours-attempt.test.ts @@ -654,7 +654,7 @@ describe('Plan format actual capture and judge lifecycle', () => { const starts = events.filter(event => event.kind === 'start'); expect(starts.map(({ timeout, maxTurns, model }) => ({ timeout, maxTurns, model }))) .toEqual([1, 2].map(() => ({ timeout: 300, maxTurns: 10, - model: file.includes('plan-prosons') ? 'claude-opus-4-7' : resolveEvalModel('capture') }))); + model: resolveEvalModel('capture') }))); expect(events.filter(event => event.kind === 'ready').map(event => event.fixtureExists)).toEqual([true, true]); expect(entries).toHaveLength(2); expect(entries.map(entry => entry.attempt)).toEqual([1, 2]); diff --git a/test/office-hours-phase4-caller.test.ts b/test/office-hours-phase4-caller.test.ts index 43c00f767..199ff2476 100644 --- a/test/office-hours-phase4-caller.test.ts +++ b/test/office-hours-phase4-caller.test.ts @@ -2,6 +2,7 @@ import { expect, test } from 'bun:test'; import * as fs from 'node:fs'; import * as path from 'node:path'; import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets'; +import { resolveEvalModel } from '../lib/eval-model'; const source = fs.readFileSync(path.join(import.meta.dir, 'skill-e2e-office-hours-phase4.test.ts'), 'utf8'); const question = (word = 'architectural', options = 'A) Put retrieval on the server\nB) Put retrieval on the client') => `Where should retrieval live?\nThis ${word} choice decides which component owns the retrieval algorithm and the cross-host API contract.\n${options}\nRecommendation: A because all hosts need a consistent algorithm with one owner.\n`; @@ -19,7 +20,7 @@ async function runCaller(captured: string | undefined, scenario: 'success' | 'ti const result = { exitReason: scenario === 'timeout' ? 'timeout' : 'success', browseErrors: [], durationMs: 123, costUsd: 0.1, output: 'public completion' }; const args: Record = { expect, beforeAll: (fn: () => void) => setups.push(fn), afterAll: (fn: () => Promise) => finalizers.push(fn), - CAPTURE_MS, CAPTURE_LONG_MS, ROOT: sourceRoot, runId: 'synthetic-run', + CAPTURE_MS, CAPTURE_LONG_MS, resolveEvalModel, ROOT: sourceRoot, runId: 'synthetic-run', describeIfSelected: (_title: string, _names: string[], fn: () => void) => fn(), testConcurrentIfSelected: (name: string, fn: () => Promise, timeout: number) => { expect(name).toBe('office-hours-phase4-fork'); expect(timeout).toBe(CAPTURE_LONG_MS); callbacks.push(fn); }, createEvalCollector: () => collector, finalizeEvalCollector: async () => {}, logCost: () => {}, @@ -35,7 +36,7 @@ async function runCaller(captured: string | undefined, scenario: 'success' | 'ti fs: { mkdtempSync:(prefix:string)=>prefix+'owned', mkdirSync:()=>{}, rmSync:()=>{}, existsSync:(name:string)=>files.has(name), writeFileSync:(name:string,body:string)=>files.set(name,body), readFileSync:(name:string)=>name===fixturePath.join(sourceRoot,'office-hours','SKILL.md') ? '## AskUserQuestion Format\nformat\n## Phase 4: Alternatives Generation\nworkflow\n## Phase 4.5\nnext' : files.get(name) }, runSkillTest: async (opts: any) => { - calls.push(opts); expect(opts.timeout).toBe(CAPTURE_MS); expect(opts.maxTurns).toBe(12); expect(opts.model).toBe('claude-opus-4-7'); + calls.push(opts); expect(opts.timeout).toBe(CAPTURE_MS); expect(opts.maxTurns).toBe(12); expect(opts.model).toBe(resolveEvalModel('capture')); expect(opts.prompt).toContain('Do NOT call any tool to ask the user.'); if (captured !== undefined) files.set(fixturePath.join(opts.workingDirectory, 'phase4-capture.md'), captured); return result; diff --git a/test/overlay-lifecycle.test.ts b/test/overlay-lifecycle.test.ts index b27124611..66232ce59 100644 --- a/test/overlay-lifecycle.test.ts +++ b/test/overlay-lifecycle.test.ts @@ -28,7 +28,7 @@ test('every fixture owns one paid wrapper; public models/trials/concurrency/turn expect(isPaidTestFile('test/overlay-lifecycle.test.ts')).toBe(false); expect(OVERLAY_FIXTURES.every(f => f.trials === 10 && f.concurrency === 3)).toBe(true); expect(OVERLAY_FIXTURES.map(f => f.maxTurns ?? 5)).toEqual([15,8,15,15,15]); - expect(OVERLAY_FIXTURES.map(f => f.model)).toEqual([...Array(3).fill('claude-opus-4-7'), 'claude-sonnet-4-6', 'claude-opus-5-5']); + expect(OVERLAY_FIXTURES.map(f => f.model)).toEqual([...Array(3).fill('claude-opus-5-5'), 'claude-sonnet-5-5', 'claude-opus-5-5']); expect([OVERLAY_CASE_WORK_MS, OVERLAY_RECORD_GRACE_MS, OVERLAY_CASE_OUTER_MS, OVERLAY_MIN_FILE_WALL_MS]).toEqual([1_800_000, 5_000, 1_810_000, 1_830_000]); }); diff --git a/test/overlay-measurement.test.ts b/test/overlay-measurement.test.ts index 273647113..1f965442e 100644 --- a/test/overlay-measurement.test.ts +++ b/test/overlay-measurement.test.ts @@ -65,7 +65,7 @@ describe('SDK overlay measurements', () => { }); describe('correctness versus efficacy', () => { - describe.each(['claude-opus-4-7', 'claude-sonnet-4-6'])('dedicated-tool gate for %s', (model) => { + describe.each(['claude-opus-5-5', 'claude-sonnet-5-5'])('dedicated-tool gate for %s', (model) => { const f = { ...OVERLAY_FIXTURES.find(f => f.metricName === 'bash_tool_calls' && f.model === model)!, trials: 3 }; // Contract v4: a trial whose final JSON fails the fixture's verify is a valid // measurement with a wrong answer, not an invalid measurement. @@ -276,9 +276,8 @@ describe('literal fixture task correctness', () => { expect(OVERLAY_FIXTURES).toHaveLength(5); expect(OVERLAY_FIXTURES.some((f) => f.id.includes('fanout') || f.comparison?.unsupportedHypothesis)).toBe(false); expect(OVERLAY_FIXTURES.every((f) => f.trials === 10 && f.comparison)).toBe(true); - expect(OVERLAY_FIXTURES.filter((f) => f.model === 'claude-opus-4-7')).toHaveLength(3); - expect(OVERLAY_FIXTURES.filter((f) => f.model === 'claude-sonnet-4-6')).toHaveLength(1); - expect(OVERLAY_FIXTURES.filter((f) => f.model === 'claude-opus-5-5')).toHaveLength(1); + expect(OVERLAY_FIXTURES.filter((f) => f.model === 'claude-opus-5-5')).toHaveLength(4); + expect(OVERLAY_FIXTURES.filter((f) => f.model === 'claude-sonnet-5-5')).toHaveLength(1); }); }); diff --git a/test/paid-retry-supervision.test.ts b/test/paid-retry-supervision.test.ts index a132bd53a..95c5fd3b3 100644 --- a/test/paid-retry-supervision.test.ts +++ b/test/paid-retry-supervision.test.ts @@ -217,8 +217,9 @@ test('eval:bg runs the PR and release commands with their declared workers (cap: const cliOptions = (step: { run: string; env?: Record }) => { const prefix = /\bbun(?: --no-install)? run scripts\/test-paid-shards\.ts /.exec(step.run); expect(prefix).not.toBeNull(); - const args = step.run.slice(prefix!.index + prefix![0].length) - .replace(/\$\{\{\s*matrix\.slice\s*\}\}/g, '1').trim().split(/\s+/); + // One command line: a census arm loop continues past it. + const args = step.run.slice(prefix!.index + prefix![0].length).split('\n')[0]! + .replace(/\$\{\{\s*matrix\.slice\s*\}\}/g, '1').replace(/\$\{\{\s*matrix\.arm\s*\}\}/g, 'claude-opus').trim().split(/\s+/); return parseCliOptions(args, step.env ?? {}); }; @@ -242,10 +243,16 @@ test('both gate executors plan the complete census and supervise every planned s expect(planned.skipJudges).toBe(skipJudges); expect(execute[0].env.EVALS_CONCURRENCY).toBe('2'); expect(executor.strategy['fail-fast']).toBe(false); - expect(executor.strategy.matrix.slice).toBe(`\${{ fromJSON(needs.plan-slices.outputs.${prefix}slices) }}`); - // Per-slice ceilings (W2c/ENG-2), or the largest of them as one job cap. - expect([`\${{ fromJSON(needs.plan-slices.outputs.${prefix}slice_timeouts)[matrix.slice] }}`, - `\${{ fromJSON(needs.plan-slices.outputs.${prefix}timeout_minutes) }}`]).toContain(executor['timeout-minutes']); + if (jobName === 'gate-census') { + // One row per census arm and planned slice, each with its arm's planned slice ceiling (scripts/census-arms.ts). + expect(executor.strategy.matrix.include).toBe('${{ fromJSON(needs.plan-slices.outputs.gate_matrix) }}'); + expect(executor['timeout-minutes']).toBe('${{ matrix.timeout }}'); + } else { + expect(executor.strategy.matrix.slice).toBe(`\${{ fromJSON(needs.plan-slices.outputs.${prefix}slices) }}`); + // Per-slice ceilings (W2c/ENG-2), or the largest of them as one job cap. + expect([`\${{ fromJSON(needs.plan-slices.outputs.${prefix}slice_timeouts)[matrix.slice] }}`, + `\${{ fromJSON(needs.plan-slices.outputs.${prefix}timeout_minutes) }}`]).toContain(executor['timeout-minutes']); + } const manifest = buildRunManifest({ tier: 'gate', sliceBudgetMs: planned.sliceBudgetMs!, jobs: planned.jobs, evalsAll: true, env: { EVALS_ALL: '1' }, skipJudges }); const files = [...new Set(manifest.entries.filter(row => row.status === 'planned').map(row => shardFile(row.file)))]; expect(files).toContain('test/skill-e2e-ship-skip.test.ts'); @@ -262,8 +269,8 @@ test('both gate executors plan the complete census and supervise every planned s expect(manifest.sliceCount).toBeLessThanOrEqual(executor.strategy['max-parallel']); if (jobName === 'gate-census') { expect(emit[0].env.EVALS_ALL).toBe('1'); - expect(emit[0].run).toContain('--emit-plan /tmp/gate-census-plan/manifest.json'); - expect(execute[0].run).toContain('--plan /tmp/gate-census-plan/manifest.json --slice ${{ matrix.slice }}'); + expect(emit[0].run).toContain('--emit-plan "/tmp/gate-census-plan/$arm/manifest.json"'); + expect(execute[0].run).toContain('--plan /tmp/gate-census-plan/${{ matrix.arm }}/manifest.json --slice ${{ matrix.slice }}'); expect(executor.steps.some((step: any) => step.with?.name === 'gate-census-plan')).toBe(true); expect(executor.steps.filter((step: any) => step.run?.includes('--emit-plan '))).toHaveLength(0); } @@ -305,8 +312,10 @@ test('gate census requires all seven distinct slice results and its own reconcil const reconcile = report.steps.find((step: any) => step.id === 'gate-reconcile'); expect(report.needs).toContain('gate-census'); expect(reconcile.if).toBe('always()'); - expect(reconcile.run).toContain('EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --report /tmp/gate-census-report'); - expect(reconcile.run).toContain('exit=${PIPESTATUS[0]}'); + // One reconciliation per census arm, each against its own plan and slices. + expect(reconcile.run).toContain('dir="/tmp/gate-census-report/$arm"'); + expect(reconcile.run).toContain('EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --report "$dir"'); + expect(reconcile.run).toContain('code=${PIPESTATUS[0]}'); const failureGuards = report.steps.filter((step: any) => /Upsert tracking|Fail the workflow/.test(step.name ?? '')); expect(failureGuards).toHaveLength(2); for (const step of failureGuards) { diff --git a/test/paid-shards.test.ts b/test/paid-shards.test.ts index 78d3b00d9..08f8629ac 100644 --- a/test/paid-shards.test.ts +++ b/test/paid-shards.test.ts @@ -178,7 +178,7 @@ describe('tier lane skip (B5)', () => { } const workflow = fs.readFileSync(path.join(ROOT, '.github/workflows/evals-periodic.yml'), 'utf8'); expect(workflow.match(/--skip-judges/g)).toHaveLength(1); - expect(workflow).toMatch(/--tier gate --emit-plan \/tmp\/gate-census-plan\/manifest\.json --slice-budget 420 --jobs 2 --skip-judges/); + expect(workflow).toMatch(/--tier gate --emit-plan "\/tmp\/gate-census-plan\/\$arm\/manifest\.json" --slice-budget 420 --jobs 2 --skip-judges/); }); }); diff --git a/test/skill-e2e-design.test.ts b/test/skill-e2e-design.test.ts index 10c0fe72c..8f078f9ce 100644 --- a/test/skill-e2e-design.test.ts +++ b/test/skill-e2e-design.test.ts @@ -124,7 +124,7 @@ Write DESIGN.md and CLAUDE.md (or update it) in the working directory.`, timeout: CAPTURE_LONG_MS, testName: 'design-consultation-core', runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }); logCost('/design-consultation core', result); @@ -298,7 +298,7 @@ Run Phase 0's DESIGN.md format check exactly as written (the gstack bin director timeout: CAPTURE_LONG_MS, testName: 'design-consultation-existing', runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }); logCost('/design-consultation existing', result); diff --git a/test/skill-e2e-office-hours-phase4.test.ts b/test/skill-e2e-office-hours-phase4.test.ts index 2043f4a55..eb7f35433 100644 --- a/test/skill-e2e-office-hours-phase4.test.ts +++ b/test/skill-e2e-office-hours-phase4.test.ts @@ -22,6 +22,7 @@ import { expect, beforeAll, afterAll } from 'bun:test'; import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; +import { resolveEvalModel } from '../lib/eval-model'; import type { EvalTestEntry } from './helpers/eval-store'; import { ROOT, runId, @@ -166,7 +167,7 @@ After writing the file with that ONE Phase 4 question, stop. Do not continue to timeout: CAPTURE_MS, testName: 'office-hours-phase4-fork', runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }); logCost('/office-hours Phase 4 fork', result); diff --git a/test/skill-e2e-plan-prosons.test.ts b/test/skill-e2e-plan-prosons.test.ts index ab41f9606..7100850d0 100644 --- a/test/skill-e2e-plan-prosons.test.ts +++ b/test/skill-e2e-plan-prosons.test.ts @@ -30,6 +30,7 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { CAPTURE_MS } from './helpers/eval-budgets'; import { NEUTRAL_POSTURE_RE } from './helpers/prosons-posture'; import { runSkillTest } from './helpers/session-runner'; +import { resolveEvalModel } from '../lib/eval-model'; import { runRecordedOfficeHoursAttempt, OFFICE_HOURS_BUN_GRACE_MS } from './helpers/office-hours-attempt'; import { ROOT, runId, @@ -134,7 +135,7 @@ function setupPlanDir(tmpPrefix: string, planContent: string, skillName: string) } function captureInstruction(outFile: string): string { - return `Write the verbatim text of the single AskUserQuestion you would have made to ${outFile} (full text including D header, ELI10, Stakes, Recommendation, Pros/cons, and Net line — the complete rich markdown body). Do NOT call any tool to ask the user. Do NOT paraphrase. This is a format-capture test.`; + return `Write the text of the single AskUserQuestion you would have made to ${outFile} (full text including D header, ELI10, Stakes, Recommendation, Pros/cons, and Net line — the complete rich markdown body). Do NOT call any tool to ask the user.`; } // --- Case 1: Format positive — all v1.7.0.0 tokens present --- @@ -155,7 +156,7 @@ describeIfSelected('Plan Prosons — Format Positive', ['plan-review-prosons-for testConcurrentIfSelected('plan-review-prosons-format', async () => { await runRecordedOfficeHoursAttempt({ collector: evalCollector, name: '/plan-review-prosons-format', suite: 'Plan Prosons — Format Positive', - model: 'claude-opus-4-7', budgetMs: CAPTURE_MS, + model: resolveEvalModel('capture'), budgetMs: CAPTURE_MS, run: signal => runSkillTest({ signal, prompt: `Read plan-ceo-review/SKILL.md for the review workflow. @@ -170,7 +171,7 @@ After writing the file, stop.`, timeout: CAPTURE_MS, testName: 'plan-review-prosons-format', runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }), validate: async (result, signal) => { logCost('/plan-review prosons format positive', result); @@ -217,7 +218,7 @@ describeIfSelected('Plan Prosons — Hard-stop Negative', ['plan-review-prosons- testConcurrentIfSelected('plan-review-prosons-hardstop-neg', async () => { await runRecordedOfficeHoursAttempt({ collector: evalCollector, name: '/plan-review-prosons-hardstop-neg', suite: 'Plan Prosons — Hard-stop Negative', - model: 'claude-opus-4-7', budgetMs: CAPTURE_MS, + model: resolveEvalModel('capture'), budgetMs: CAPTURE_MS, run: signal => runSkillTest({ signal, prompt: `Read plan-ceo-review/SKILL.md. @@ -232,7 +233,7 @@ After writing the file, stop.`, timeout: CAPTURE_MS, testName: 'plan-review-prosons-hardstop-neg', runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }), validate: async (result, signal) => { logCost('/plan-review prosons hard-stop negative', result); @@ -270,7 +271,7 @@ describeIfSelected('Plan Prosons — Neutral-posture Negative', ['plan-review-pr testConcurrentIfSelected('plan-review-prosons-neutral-neg', async () => { await runRecordedOfficeHoursAttempt({ collector: evalCollector, name: '/plan-review-prosons-neutral-neg', suite: 'Plan Prosons — Neutral Negative', - model: 'claude-opus-4-7', budgetMs: CAPTURE_MS, + model: resolveEvalModel('capture'), budgetMs: CAPTURE_MS, run: signal => runSkillTest({ signal, prompt: `Read plan-ceo-review/SKILL.md. @@ -285,7 +286,7 @@ After writing the file, stop.`, timeout: CAPTURE_MS, testName: 'plan-review-prosons-neutral-neg', runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }), validate: async (result, signal) => { logCost('/plan-review prosons neutral negative', result); @@ -324,7 +325,7 @@ describeIfSelected('Plan Prosons — Hard-stop Positive', ['plan-ceo-review-pros testConcurrentIfSelected('plan-ceo-review-prosons-cadence', async () => { await runRecordedOfficeHoursAttempt({ collector: evalCollector, name: '/plan-ceo-review-prosons-cadence', suite: 'Plan Prosons — Hard-stop Positive', - model: 'claude-opus-4-7', budgetMs: CAPTURE_MS, + model: resolveEvalModel('capture'), budgetMs: CAPTURE_MS, run: signal => runSkillTest({ signal, prompt: `Read plan-ceo-review/SKILL.md. @@ -339,7 +340,7 @@ After writing the file, stop.`, timeout: CAPTURE_MS, testName: 'plan-ceo-review-prosons-cadence', runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }), validate: async (result, signal) => { logCost('/plan-review prosons hard-stop positive', result); diff --git a/test/skill-e2e-plan.test.ts b/test/skill-e2e-plan.test.ts index 92b79175c..2f7a7eec8 100644 --- a/test/skill-e2e-plan.test.ts +++ b/test/skill-e2e-plan.test.ts @@ -91,7 +91,7 @@ Focus on reviewing the plan content: architecture, error handling, security, and timeout: 540_000, testName: 'plan-ceo-review', runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }); logCost('/plan-ceo-review', result); @@ -178,7 +178,7 @@ Focus on reviewing the plan content: architecture, error handling, security, and timeout: CAPTURE_LONG_MS, testName: 'plan-ceo-review-selective', runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }); logCost('/plan-ceo-review (SELECTIVE)', result); @@ -246,7 +246,7 @@ Write your expansion proposals to ${planDir}/proposals.md with ONLY the proposal timeout: CAPTURE_LONG_MS, testName: 'plan-ceo-review-expansion-energy', runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }); logCost('/plan-ceo-review (EXPANSION ENERGY)', result); @@ -355,7 +355,7 @@ Focus on architecture, code quality, tests, and performance sections.`, timeout: CAPTURE_LONG_MS, testName: 'plan-eng-review', runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }); logCost('/plan-eng-review', result); diff --git a/test/skill-e2e-qa-bugs.test.ts b/test/skill-e2e-qa-bugs.test.ts index 14db97bf6..1a59acf33 100644 --- a/test/skill-e2e-qa-bugs.test.ts +++ b/test/skill-e2e-qa-bugs.test.ts @@ -1,6 +1,7 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; +import { resolveEvalModel } from '../lib/eval-model'; import { outcomeJudge } from './helpers/llm-judge'; import { judgePassed } from './helpers/eval-store'; import { @@ -108,7 +109,7 @@ CRITICAL RULES: timeout: CAPTURE_MS, testName: `qa-${label}`, runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }); logCost(`/qa ${label}`, result); diff --git a/test/skill-positional-literals.test.ts b/test/skill-positional-literals.test.ts index e8280a397..e691697c4 100644 --- a/test/skill-positional-literals.test.ts +++ b/test/skill-positional-literals.test.ts @@ -75,7 +75,7 @@ for (const host of ['claude', 'codex'] as const) for (const args of [[], tenArgu const file = backslashPath ? join(root, 'checksum\\path', 'install-script') : join(root, 'install-script'); mkdirSync(dirname(file), { recursive: true }); writeFileSync(file, 'synthetic installer bytes\n'); - for (const name of ['open-gstack-browser', 'pair-agent', 'setup-browser-cookies']) { + for (const name of ['open-gstack-browser', 'pair-agent']) { const lines = apply(skill(host, name)).split('\n').filter(line => line.includes('actual_sha=$(')); expect(lines.map(line => line.trim())).toEqual([ literals.checksum,