mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-31 14:06:50 +01:00
v1.91.149.0 feat(evals): Codex host for gate cases, sol weekly and Fable/Astra monthly census arms (GSTA-49 TM2) (#3292)
* feat(evals): Opus 5.5 is the paid gate's model under test; judge stays Fable (GSTA-49 TM1) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * v1.91.140.0 chore: stamp TM1 (Opus 5.5 model under test) and CHANGELOG Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * feat(evals): Codex host adapter, real Codex cost, two-arm weekly and monthly census (GSTA-49 TM2) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * chore: TM2 CHANGELOG under [Unreleased]; re-pin codex-host-runner in the harness manifest after the main merge Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * ci: retrigger PR workflows (base changed to main after open; no pull_request run fired) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * v1.91.149.0 chore: stamp TM2 (Codex host, census arms) and CHANGELOG Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
9665621d18
commit
9cf105ccff
30 files changed
+1533
-260
No files matched your search
@@ -15,6 +15,10 @@ name: Periodic Evals
|
||||
# diff-billed, so without it the full gate census might never execute
|
||||
# anywhere); the hollow-shard guard (exit 0 + zero executed tests under
|
||||
# EVALS_ALL fails) makes both lanes census-health checks, not just test runs.
|
||||
# The gate census is a matrix of arms (scripts/census-arms.ts): each arm is a
|
||||
# host and model with its own plan. Monday runs claude-opus (the capture
|
||||
# model) and codex-sol (gpt-6.1-sol on the Codex host, adapter cases only);
|
||||
# the monthly cron runs only the census, with claude-fable and codex-astra.
|
||||
# eval-bg (scripts/eval-bg.ts) finds its dispatch by the nonce in the run name.
|
||||
run-name: ${{ inputs.nonce && format('Periodic Evals [eval-bg {0}]', inputs.nonce) || 'Periodic Evals' }}
|
||||
on:
|
||||
@@ -23,6 +27,9 @@ on:
|
||||
# Daily, for autoplan-schema-canary alone (latest Claude Code vs the /autoplan
|
||||
# guard's payload assumptions); every census job below skips this schedule.
|
||||
- cron: '30 7 * * *'
|
||||
# Monthly, the census jobs alone with the claude-fable and codex-astra arms
|
||||
# (the 1st; never combined with a day of week, which cron would OR).
|
||||
- cron: '0 6 1 * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
redispatch_of:
|
||||
@@ -30,9 +37,13 @@ on:
|
||||
type: string
|
||||
default: ''
|
||||
include_gate_census:
|
||||
description: 'Branch dispatch only: also run the weekly gate census (default: periodic lane only; gate cases validate through an evals.yml dispatch with evals_all)'
|
||||
description: 'Branch dispatch only: also run the weekly gate census arms (default: periodic lane only; gate cases validate through an evals.yml dispatch with evals_all)'
|
||||
type: boolean
|
||||
default: false
|
||||
census_arms:
|
||||
description: 'Run only the gate census, with these arms (comma-separated: claude-opus, codex-sol, claude-fable, codex-astra; scripts/census-arms.ts). Empty keeps the periodic lane.'
|
||||
type: string
|
||||
default: ''
|
||||
expected_sha:
|
||||
description: 'eval-bg: the revision the dispatcher tested; the run fails fast when the ref has moved'
|
||||
type: string
|
||||
@@ -51,7 +62,7 @@ on:
|
||||
# runs in its own group so it never cancels the run that dispatched it; both
|
||||
# runs are reported.
|
||||
concurrency:
|
||||
group: evals-periodic-${{ github.ref }}-${{ github.event_name }}${{ inputs.redispatch_of && format('-redispatch-{0}', inputs.redispatch_of) || '' }}${{ (github.event.schedule == '30 7 * * *' || inputs.schema_canary_only) && '-schema-canary' || '' }}
|
||||
group: evals-periodic-${{ github.ref }}-${{ github.event_name }}${{ inputs.redispatch_of && format('-redispatch-{0}', inputs.redispatch_of) || '' }}${{ (github.event.schedule == '30 7 * * *' || inputs.schema_canary_only) && '-schema-canary' || '' }}${{ github.event.schedule == '0 6 1 * *' && '-monthly' || '' }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
@@ -126,8 +137,9 @@ jobs:
|
||||
periodic_slices: ${{ steps.periodic-matrix.outputs.slices }}
|
||||
periodic_codex_slices: ${{ steps.periodic-matrix.outputs.codex_slices }}
|
||||
periodic_slice_timeouts: ${{ steps.periodic-matrix.outputs.slice_timeouts }}
|
||||
gate_slices: ${{ steps.gate-matrix.outputs.slices }}
|
||||
gate_slice_timeouts: ${{ steps.gate-matrix.outputs.slice_timeouts }}
|
||||
periodic_lane: ${{ steps.census.outputs.periodic_lane }}
|
||||
census_arms: ${{ steps.census.outputs.arms }}
|
||||
gate_matrix: ${{ steps.gate-matrix.outputs.gate_matrix }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
@@ -147,7 +159,18 @@ jobs:
|
||||
with:
|
||||
bun-version: 1.4.2
|
||||
|
||||
# Census arms (scripts/census-arms.ts): which host/model arms run the
|
||||
# gate census, and whether the periodic lane runs beside them.
|
||||
- name: Select the census arms
|
||||
id: census
|
||||
env:
|
||||
SCHEDULE: ${{ github.event.schedule }}
|
||||
CENSUS_ARMS: ${{ inputs.census_arms }}
|
||||
INCLUDE_GATE_CENSUS: ${{ github.ref == 'refs/heads/main' || inputs.include_gate_census }}
|
||||
run: bun --no-install run scripts/census-arms.ts --select >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Emit run manifest (ALL periodic tests minus reasoned excludes)
|
||||
if: steps.census.outputs.periodic_lane == 'true'
|
||||
env:
|
||||
EVALS_ALL: "1"
|
||||
run: EVALS_TIER=periodic bun --no-install run scripts/test-paid-shards.ts --tier periodic --emit-plan /tmp/paid-plan/manifest.json --slice-budget 420 --jobs 2 --max-parallel 38
|
||||
@@ -156,6 +179,7 @@ jobs:
|
||||
# container executor runs every other slice.
|
||||
- name: Derive the periodic executor matrix from the plan
|
||||
id: periodic-matrix
|
||||
if: steps.census.outputs.periodic_lane == 'true'
|
||||
run: |
|
||||
{
|
||||
echo "slices=$(jq -c '[range(1; .sliceCount + 1)] - .plan.codexSlices' /tmp/paid-plan/manifest.json)"
|
||||
@@ -164,26 +188,38 @@ jobs:
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
if: steps.census.outputs.periodic_lane == 'true'
|
||||
with:
|
||||
name: paid-plan
|
||||
path: /tmp/paid-plan/manifest.json
|
||||
retention-days: 30
|
||||
|
||||
- name: Emit gate census manifest (ALL gate tests)
|
||||
# One plan per arm: the Codex host plans only its adapter cases and
|
||||
# records every Claude-only case as excluded with its reason.
|
||||
- name: Emit gate census manifests (ALL gate tests, one plan per arm)
|
||||
if: steps.census.outputs.arms != '[]'
|
||||
env:
|
||||
EVALS_ALL: "1"
|
||||
run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --emit-plan /tmp/gate-census-plan/manifest.json --slice-budget 420 --jobs 2 --skip-judges --max-parallel 20
|
||||
|
||||
- name: Derive the gate census executor matrix from the plan
|
||||
id: gate-matrix
|
||||
ARMS: ${{ steps.census.outputs.arms }}
|
||||
run: |
|
||||
echo "slices=$(jq -c '[range(1; .sliceCount + 1)]' /tmp/gate-census-plan/manifest.json)" >> "$GITHUB_OUTPUT"
|
||||
echo "slice_timeouts=$(jq -ec '[0] + .plan.sliceCiTimeoutMinutes' /tmp/gate-census-plan/manifest.json)" >> "$GITHUB_OUTPUT"
|
||||
set -euo pipefail
|
||||
for arm in $(jq -r '.[]' <<<"$ARMS"); do
|
||||
mapfile -t arm_env < <(bun --no-install run scripts/census-arms.ts --env "$arm")
|
||||
env "${arm_env[@]}" EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --emit-plan "/tmp/gate-census-plan/$arm/manifest.json" --slice-budget 420 --jobs 2 --skip-judges --max-parallel 20
|
||||
done
|
||||
|
||||
# One executor per arm and planned slice, each with its slice's ceiling.
|
||||
- name: Derive the gate census executor matrix from the plans
|
||||
id: gate-matrix
|
||||
env:
|
||||
ARMS: ${{ steps.census.outputs.arms }}
|
||||
run: bun --no-install run scripts/census-arms.ts --matrix /tmp/gate-census-plan "$ARMS" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
if: steps.census.outputs.arms != '[]'
|
||||
with:
|
||||
name: gate-census-plan
|
||||
path: /tmp/gate-census-plan/manifest.json
|
||||
path: /tmp/gate-census-plan
|
||||
retention-days: 30
|
||||
|
||||
# autoplan-schema-canary: installs the latest published Claude Code with
|
||||
@@ -237,6 +273,7 @@ jobs:
|
||||
eval-slices:
|
||||
runs-on: ubicloud-standard-8
|
||||
needs: [build-image, plan-slices]
|
||||
if: ${{ needs.plan-slices.outputs.periodic_lane == 'true' }}
|
||||
env:
|
||||
EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-eval-slices-${{ matrix.slice }}
|
||||
# The planner packs ~7 minutes of recorded work per slice; each slice's job
|
||||
@@ -388,7 +425,7 @@ jobs:
|
||||
eval-codex-slices:
|
||||
runs-on: ubicloud-standard-8
|
||||
needs: [build-image, plan-slices]
|
||||
if: ${{ needs.plan-slices.outputs.periodic_codex_slices != '[]' }}
|
||||
if: ${{ needs.plan-slices.outputs.periodic_lane == 'true' && needs.plan-slices.outputs.periodic_codex_slices != '[]' }}
|
||||
env:
|
||||
EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-eval-slices-${{ matrix.slice }}
|
||||
timeout-minutes: ${{ fromJSON(needs.plan-slices.outputs.periodic_slice_timeouts)[matrix.slice] }}
|
||||
@@ -512,16 +549,18 @@ jobs:
|
||||
# blind spots rot invisibly. Census health, not selector correctness —
|
||||
# selector logic has free synthetic-diff contract tests.
|
||||
gate-census:
|
||||
name: gate-census-${{ matrix.arm }} (${{ matrix.slice }})
|
||||
runs-on: ubicloud-standard-8
|
||||
needs: [build-image, plan-slices]
|
||||
# A branch dispatch validates the periodic lane only; gate cases on a
|
||||
# branch go through an evals.yml dispatch with evals_all. Main (schedule
|
||||
# or dispatch) always runs the census; a branch opts in explicitly.
|
||||
if: ${{ github.ref == 'refs/heads/main' || inputs.include_gate_census }}
|
||||
# The planner picks the arms (scripts/census-arms.ts). A branch dispatch
|
||||
# validates the periodic lane only unless it sets include_gate_census or
|
||||
# census_arms; gate cases on a branch also go through an evals.yml
|
||||
# dispatch with evals_all. Main (schedule or dispatch) always runs it.
|
||||
if: ${{ needs.plan-slices.outputs.gate_matrix != '[]' }}
|
||||
env:
|
||||
EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-gate-census-${{ matrix.slice }}
|
||||
# Per-slice job ceiling from the packed plan (W2c/ENG-2), as for eval-slices.
|
||||
timeout-minutes: ${{ fromJSON(needs.plan-slices.outputs.gate_slice_timeouts)[matrix.slice] }}
|
||||
EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-gate-census-${{ matrix.arm }}-${{ matrix.slice }}
|
||||
# Per-slice job ceiling from the arm's packed plan (W2c/ENG-2), as for eval-slices.
|
||||
timeout-minutes: ${{ matrix.timeout }}
|
||||
permissions:
|
||||
contents: read
|
||||
packages: read
|
||||
@@ -536,7 +575,7 @@ jobs:
|
||||
fail-fast: false
|
||||
max-parallel: 20
|
||||
matrix:
|
||||
slice: ${{ fromJSON(needs.plan-slices.outputs.gate_slices) }}
|
||||
include: ${{ fromJSON(needs.plan-slices.outputs.gate_matrix) }}
|
||||
steps:
|
||||
- name: Record the job start for the slice deadline
|
||||
run: echo "GSTACK_SLICE_JOB_STARTED_AT=$(date +%s)" >> "$GITHUB_ENV"
|
||||
@@ -588,7 +627,15 @@ jobs:
|
||||
printenv OPENAI_API_KEY | "$codex" login --with-api-key
|
||||
"$codex" login status >/dev/null 2>&1
|
||||
|
||||
- name: Run gate census slice ${{ matrix.slice }}
|
||||
# The arm's host and model (EVALS_HOST, plus EVALS_MODEL or
|
||||
# GSTACK_CODEX_MODEL). The Codex host runs codex exec with
|
||||
# danger-full-access, so it needs no bubblewrap and runs in this container.
|
||||
- name: Select the census arm's host and model
|
||||
env:
|
||||
ARM: ${{ matrix.arm }}
|
||||
run: bun --no-install run scripts/census-arms.ts --env "$ARM" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Run gate census slice ${{ matrix.slice }} (${{ matrix.arm }})
|
||||
env:
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
@@ -597,13 +644,13 @@ jobs:
|
||||
EVALS_JOBS: "2"
|
||||
EVALS_CONCURRENCY: "2"
|
||||
GSTACK_EVAL_DIR: /tmp/gate-census-results
|
||||
run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --plan /tmp/gate-census-plan/manifest.json --slice ${{ matrix.slice }}
|
||||
run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --plan /tmp/gate-census-plan/${{ matrix.arm }}/manifest.json --slice ${{ matrix.slice }}
|
||||
|
||||
- name: Upload census results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: gate-census-${{ matrix.slice }}-a${{ github.run_attempt }}
|
||||
name: gate-census-${{ matrix.slice }}-a${{ github.run_attempt }}-${{ matrix.arm }}
|
||||
path: /tmp/gate-census-results
|
||||
retention-days: 90
|
||||
|
||||
@@ -638,6 +685,9 @@ jobs:
|
||||
actions: read
|
||||
outputs:
|
||||
redispatch: ${{ steps.verdict.outputs.redispatch }}
|
||||
# A census-only run (monthly cron or census_arms dispatch) re-dispatches
|
||||
# with the same arms; empty keeps the weekly selection.
|
||||
redispatch_census_arms: ${{ needs.plan-slices.outputs.periodic_lane != 'true' && join(fromJSON(needs.plan-slices.outputs.census_arms), ',') || '' }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
@@ -648,6 +698,7 @@ jobs:
|
||||
bun-version: 1.4.2
|
||||
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
||||
if: needs.plan-slices.outputs.periodic_lane == 'true'
|
||||
with:
|
||||
name: paid-plan
|
||||
path: /tmp/paid-report
|
||||
@@ -655,25 +706,35 @@ jobs:
|
||||
# One directory per attempt-scoped slice artifact (no merge): shard
|
||||
# records never overwrite each other and the first attempt decides.
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
||||
if: needs.plan-slices.outputs.periodic_lane == 'true'
|
||||
with:
|
||||
pattern: paid-slice-*
|
||||
path: /tmp/paid-report
|
||||
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
||||
if: needs.plan-slices.outputs.gate_matrix != '[]'
|
||||
with:
|
||||
name: gate-census-plan
|
||||
path: /tmp/gate-census-report
|
||||
path: /tmp/gate-census-plan
|
||||
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
||||
if: needs.plan-slices.outputs.gate_matrix != '[]'
|
||||
with:
|
||||
pattern: gate-census-[0-9]*
|
||||
path: /tmp/gate-census-report
|
||||
path: /tmp/gate-census-slices
|
||||
|
||||
- name: Reconcile slices against the manifest (fail-closed)
|
||||
id: reconcile
|
||||
if: always()
|
||||
env:
|
||||
PERIODIC_LANE: ${{ needs.plan-slices.outputs.periodic_lane }}
|
||||
run: |
|
||||
set +e
|
||||
if [ "$PERIODIC_LANE" != "true" ]; then
|
||||
echo "periodic lane skipped: census-only run (monthly cron or census_arms dispatch)" | tee /tmp/report.txt
|
||||
echo "exit=0" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
EVALS_TIER=periodic bun --no-install run scripts/test-paid-shards.ts --tier periodic --report /tmp/paid-report | tee /tmp/report.txt
|
||||
# PIPESTATUS[0], NOT $?: GitHub's default run-step shell is
|
||||
# `bash -e {0}` with NO pipefail, so $? after the pipe is tee's
|
||||
@@ -686,20 +747,34 @@ jobs:
|
||||
if: always()
|
||||
env:
|
||||
GATE_CENSUS: ${{ needs.gate-census.result }}
|
||||
ARMS: ${{ needs.plan-slices.outputs.census_arms }}
|
||||
# One report per arm: its plan plus its own slice artifacts
|
||||
# (gate-census-<slice>-a<attempt>-<arm>) in /tmp/gate-census-report/<arm>.
|
||||
run: |
|
||||
set +e
|
||||
if [ "$GATE_CENSUS" = "skipped" ]; then
|
||||
echo "gate census skipped: branch dispatch without include_gate_census (periodic lane only)" | tee /tmp/gate-report.txt
|
||||
echo "gate census skipped: no census arm selected (branch dispatch without include_gate_census or census_arms)" | tee /tmp/gate-report.txt
|
||||
echo "exit=0" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --report /tmp/gate-census-report | tee /tmp/gate-report.txt
|
||||
echo "exit=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT"
|
||||
status=0
|
||||
: > /tmp/gate-report.txt
|
||||
for arm in $(jq -r '.[]' <<<"$ARMS"); do
|
||||
dir="/tmp/gate-census-report/$arm"
|
||||
mkdir -p "$dir"
|
||||
cp "/tmp/gate-census-plan/$arm/manifest.json" "$dir/manifest.json" || status=1
|
||||
for slice in /tmp/gate-census-slices/gate-census-*-"$arm"; do if [ -d "$slice" ]; then mv "$slice" "$dir/"; fi; done
|
||||
echo "=== gate census arm: $arm ===" | tee -a /tmp/gate-report.txt
|
||||
EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --report "$dir" | tee -a /tmp/gate-report.txt
|
||||
code=${PIPESTATUS[0]}
|
||||
if [ "$code" -ne 0 ]; then status=$code; fi
|
||||
done
|
||||
echo "exit=$status" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Stamp trial history series
|
||||
if: always()
|
||||
run: |
|
||||
for file in /tmp/paid-report/trial-outcomes.jsonl /tmp/gate-census-report/trial-outcomes.jsonl; do
|
||||
for file in /tmp/paid-report/trial-outcomes.jsonl /tmp/gate-census-report/*/trial-outcomes.jsonl; do
|
||||
if [ -f "$file" ]; then bun --no-install run scripts/eval-trial-series.ts "$file"; fi
|
||||
done
|
||||
|
||||
@@ -719,7 +794,7 @@ jobs:
|
||||
with:
|
||||
name: trial-outcomes-gate-census-a${{ github.run_attempt }}
|
||||
path: |
|
||||
/tmp/gate-census-report/trial-outcomes.jsonl
|
||||
/tmp/gate-census-report/*/trial-outcomes.jsonl
|
||||
if-no-files-found: ignore
|
||||
retention-days: 90
|
||||
|
||||
@@ -732,7 +807,7 @@ jobs:
|
||||
run: |
|
||||
mkdir -p /tmp/exercised-map
|
||||
dirs=()
|
||||
for dir in /tmp/gate-census-report /tmp/paid-report; do [ -d "$dir" ] && dirs+=(--from-dir "$dir"); done
|
||||
for dir in /tmp/gate-census-report/* /tmp/paid-report; do [ -d "$dir" ] && dirs+=(--from-dir "$dir"); done
|
||||
status=0
|
||||
bun --no-install run scripts/exercised-map.ts "${dirs[@]}" \
|
||||
--run-id "$GITHUB_RUN_ID" --out /tmp/exercised-map/exercised-commands.json --report /tmp/exercised-map/report.md || status=$?
|
||||
@@ -779,8 +854,16 @@ jobs:
|
||||
[ "$1" = "0" ] && return 0
|
||||
jq -e '.version == 2 and .verdict.redispatchEligible == true' "$2/collector-outcomes.json" >/dev/null 2>&1
|
||||
}
|
||||
eligible_arms() { # $1 exit, then one report dir per census arm: each must be green or INFRA/INCOMPLETE-only
|
||||
[ "$1" = "0" ] && return 0
|
||||
shift
|
||||
local dir
|
||||
for dir in "$@"; do
|
||||
jq -e '.version == 2 and (.verdict.verdict == "GREEN" or .verdict.redispatchEligible == true)' "$dir/collector-outcomes.json" >/dev/null 2>&1 || return 1
|
||||
done
|
||||
}
|
||||
if [ -z "$REDISPATCH_OF" ] && { [ "$PERIODIC_EXIT" != "0" ] || [ "$GATE_EXIT" != "0" ]; } \
|
||||
&& eligible "$PERIODIC_EXIT" /tmp/paid-report && eligible "$GATE_EXIT" /tmp/gate-census-report; then
|
||||
&& eligible "$PERIODIC_EXIT" /tmp/paid-report && eligible_arms "$GATE_EXIT" /tmp/gate-census-report/*/; then
|
||||
echo "redispatch=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "redispatch=false" >> "$GITHUB_OUTPUT"
|
||||
@@ -793,7 +876,7 @@ jobs:
|
||||
# comments on or closes main's issue. Every report-derived line goes
|
||||
# through the one published-text sanitizer (scripts/lib/published-text.ts).
|
||||
- name: Write the census report
|
||||
if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success')
|
||||
if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || (needs.plan-slices.outputs.periodic_lane == 'true' && needs.eval-slices.result != 'success') || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success')
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
REDISPATCH: ${{ steps.verdict.outputs.redispatch }}
|
||||
@@ -827,7 +910,14 @@ jobs:
|
||||
cat /tmp/paid-report/report-summary.md 2>/dev/null || echo "(no periodic report summary)"
|
||||
echo
|
||||
echo "### Gate census"
|
||||
cat /tmp/gate-census-report/report-summary.md 2>/dev/null || echo "(no gate census report summary)"
|
||||
found=0
|
||||
for summary in /tmp/gate-census-report/*/report-summary.md; do
|
||||
[ -f "$summary" ] || continue
|
||||
found=1
|
||||
echo "#### Arm $(basename "$(dirname "$summary")")"
|
||||
cat "$summary"
|
||||
done
|
||||
[ "$found" = 1 ] || echo "(no gate census report summary)"
|
||||
echo
|
||||
echo "### Pass-rate history (ACTION REQUIRED)"
|
||||
{ awk '/^ACTION REQUIRED/ { on = 1; print; next } on && /^ \[/ { print; next } { on = 0 } /history unavailable/ { print }' /tmp/pass-rates.txt 2>/dev/null \
|
||||
@@ -851,7 +941,7 @@ jobs:
|
||||
cat /tmp/census-report.md >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Upload the census report
|
||||
if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success')
|
||||
if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || (needs.plan-slices.outputs.periodic_lane == 'true' && needs.eval-slices.result != 'success') || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success')
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: census-report-a${{ github.run_attempt }}
|
||||
@@ -860,7 +950,7 @@ jobs:
|
||||
retention-days: 90
|
||||
|
||||
- name: Upsert tracking issue on failure
|
||||
if: always() && github.ref == 'refs/heads/main' && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success')
|
||||
if: always() && github.ref == 'refs/heads/main' && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || (needs.plan-slices.outputs.periodic_lane == 'true' && needs.eval-slices.result != 'success') || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success')
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
@@ -877,7 +967,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Close the tracking issue on a green run
|
||||
if: always() && github.ref == 'refs/heads/main' && steps.reconcile.outputs.exit == '0' && steps.gate-reconcile.outputs.exit == '0' && steps.pass-rates.outputs.exit == '0' && needs.eval-slices.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) && needs.gate-census.result == 'success'
|
||||
if: always() && github.ref == 'refs/heads/main' && steps.reconcile.outputs.exit == '0' && steps.gate-reconcile.outputs.exit == '0' && steps.pass-rates.outputs.exit == '0' && needs.plan-slices.outputs.periodic_lane == 'true' && needs.eval-slices.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) && needs.gate-census.result == 'success'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
REDISPATCH_OF: ${{ inputs.redispatch_of }}
|
||||
@@ -892,7 +982,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Fail the workflow when reconciliation failed
|
||||
if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || !contains(fromJSON('["success","skipped"]'), needs.gate-census.result))
|
||||
if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || (needs.plan-slices.outputs.periodic_lane == 'true' && needs.eval-slices.result != 'success') || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || !contains(fromJSON('["success","skipped"]'), needs.gate-census.result))
|
||||
run: exit 1
|
||||
|
||||
# The one INFRA/INCOMPLETE re-dispatch (UC-E1). Its own job so the report
|
||||
@@ -909,4 +999,5 @@ jobs:
|
||||
- name: Re-dispatch the weekly census once
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: gh workflow run evals-periodic.yml --repo "$GITHUB_REPOSITORY" --ref "$GITHUB_REF_NAME" -f redispatch_of="$GITHUB_RUN_ID"
|
||||
CENSUS_ARMS: ${{ needs.report.outputs.redispatch_census_arms }}
|
||||
run: gh workflow run evals-periodic.yml --repo "$GITHUB_REPOSITORY" --ref "$GITHUB_REF_NAME" -f redispatch_of="$GITHUB_RUN_ID" ${CENSUS_ARMS:+-f census_arms="$CENSUS_ARMS"}
|
||||
@@ -1,5 +1,32 @@
|
||||
# Changelog
|
||||
|
||||
## [1.91.149.0] - 2026-10-19
|
||||
|
||||
**Paid eval cases can now run on the Codex host, and the Monday census runs two arms: Claude Opus 5.5 and Codex gpt-6.1-sol.** `EVALS_HOST=codex` runs the 52 adapter-only gate cases through a full Codex runtime per case. On its first pilot gpt-6.1-sol passed 48 of 52 with no retries, at $4.63 for the 52 cases, against $28.93 for the same cases on Fable (0.16×). Claude-only cases are skipped on Codex with a recorded reason and never count as failed. GSTA-49 TM2.
|
||||
|
||||
### Added
|
||||
- **Codex host adapter:** `test/helpers/e2e-hosts.ts` holds the `EVALS_HOST` switch (`claude` default, `codex`) and the per-case host metadata. `test/helpers/codex-host-runner.ts` adds:
|
||||
- the case runner, with sandbox settings and a reasoning effort (the model default unless set);
|
||||
- file-change event mapping;
|
||||
- a transcript translated into the Claude stream-json shape that existing assertions read;
|
||||
- the native-lifecycle hooks.
|
||||
|
||||
The 26 rework cases stay Claude-only.
|
||||
- **Census arms (`scripts/census-arms.ts`):** `evals-periodic.yml` gets two census schedules.
|
||||
- **Monday:** a two-arm gate census, `claude-opus` and `codex-sol`.
|
||||
- **Monthly (`0 6 1 * *`):** census jobs only, with `claude-fable` (claude-fable-5-1) and `codex-astra` (gpt-6-astra) arms and a `-monthly` concurrency key.
|
||||
|
||||
A `census_arms` dispatch input picks arms by hand. Artifact names and run IDs carry the arm, and a census-only run executes no periodic slice. Every arm keeps the census's `--skip-judges`.
|
||||
|
||||
### Changed
|
||||
- **Codex cost is real:** `cost_known: false` is gone. Codex sessions price their `turn.completed` usage (input, cached input, cache-write input and output tokens) through `lib/pricing.ts`. The shared `execCodex` keeps the old runner's events and verdicts.
|
||||
- **Plans, reports and the ledger carry the host** when it is not Claude. On Codex, the Anthropic preflight skips its `claude -p` ping.
|
||||
- **Gate integrity:** no threshold, trial count, case kind, judge or Claude-host verdict changed. `HARNESS_VERSION` stays 6, and the changed harness files are recorded as non-behavioral. Five workflow tests now assert the census matrix per arm (one row per arm and slice, per-arm plan and report paths, a `periodic_lane` guard) in place of the single-arm shape; no assertion was dropped.
|
||||
|
||||
### If you…
|
||||
- **…want one case on Codex:** `EVALS_HOST=codex GSTACK_CODEX_MODEL=gpt-6.1-sol bun run scripts/test-paid-shards.ts --tier gate --case <id> --trials 1`.
|
||||
- **…want the monthly arms now:** dispatch `evals-periodic.yml` with `census_arms=claude-fable,codex-astra`.
|
||||
|
||||
## [1.91.148.0] - 2026-10-19
|
||||
|
||||
**Weekly seed PRs now get the credential scan as well as the free suites.** A pull request the recorder opens with the workflow token never starts its own `pull_request` checks. GitHub holds them as "action required". So the recorder dispatches its checks on the bot branch instead.
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# gstack digest v1.91.148.0 — regenerate/re-copy after upgrading gstack
|
||||
# gstack digest v1.91.149.0 — regenerate/re-copy after upgrading gstack
|
||||
|
||||
Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed
|
||||
for agent hosts without a full skill install. The full skills add workflows,
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "gstack",
|
||||
"version": "1.91.148",
|
||||
"version": "1.91.149",
|
||||
"description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.",
|
||||
"license": "MIT",
|
||||
"type": "module",
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env bun
|
||||
/**
|
||||
* Gate census arms for evals-periodic.yml (GSTA-49 TM2). Each arm is one host
|
||||
* and model running the weekly gate census plan:
|
||||
* - Monday cron (0 6 * * 1): claude-opus (the capture model, lib/eval-model.ts)
|
||||
* and codex-sol (gpt-6.1-sol on the Codex host, adapter cases only);
|
||||
* - monthly cron (0 6 1 * *): claude-fable and codex-astra, census jobs only;
|
||||
* - dispatch: `census_arms` runs only the census with the named arms; without
|
||||
* it a dispatch keeps the periodic lane plus the Monday arms when the census
|
||||
* runs (main, or include_gate_census).
|
||||
* Judges stay in the Claude lanes: every census plan skips them.
|
||||
*
|
||||
* CLI (GitHub step outputs and env lines):
|
||||
* --select env SCHEDULE, CENSUS_ARMS, INCLUDE_GATE_CENSUS -> arms=<json>, periodic_lane=<bool>
|
||||
* --env <arm> KEY=VALUE lines selecting the arm's host and model
|
||||
* --matrix <dir> <arms> gate_matrix=<json> from <dir>/<arm>/manifest.json
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import type { EvalHost } from '../test/helpers/e2e-hosts';
|
||||
|
||||
export interface CensusArm { host: EvalHost; model?: string }
|
||||
|
||||
export const CENSUS_ARMS: Readonly<Record<string, CensusArm>> = Object.freeze({
|
||||
// No model: the Claude runners' capture default (claude-opus-5-5 since TM1).
|
||||
'claude-opus': { host: 'claude' },
|
||||
'codex-sol': { host: 'codex', model: 'gpt-6.1-sol' },
|
||||
'claude-fable': { host: 'claude', model: 'claude-fable-5-1' },
|
||||
'codex-astra': { host: 'codex', model: 'gpt-6-astra' },
|
||||
});
|
||||
|
||||
export const WEEKLY_CENSUS_SCHEDULE = '0 6 * * 1';
|
||||
export const MONTHLY_CENSUS_SCHEDULE = '0 6 1 * *';
|
||||
export const WEEKLY_CENSUS_ARMS: readonly string[] = ['claude-opus', 'codex-sol'];
|
||||
export const MONTHLY_CENSUS_ARMS: readonly string[] = ['claude-fable', 'codex-astra'];
|
||||
|
||||
/** Which census arms run, and whether the periodic lane runs beside them. */
|
||||
export function selectCensus(opts: { schedule?: string; censusArms?: string; includeGateCensus: boolean }): { arms: string[]; periodicLane: boolean } {
|
||||
if (opts.schedule === MONTHLY_CENSUS_SCHEDULE) return { arms: [...MONTHLY_CENSUS_ARMS], periodicLane: false };
|
||||
const named = (opts.censusArms ?? '').split(/[\s,]+/).filter(Boolean);
|
||||
const unknown = named.filter(arm => !Object.hasOwn(CENSUS_ARMS, arm));
|
||||
if (unknown.length) throw new Error(`unknown census arm(s): ${unknown.join(', ')}; known: ${Object.keys(CENSUS_ARMS).join(', ')}`);
|
||||
if (named.length) return { arms: [...new Set(named)], periodicLane: false };
|
||||
return { arms: opts.includeGateCensus ? [...WEEKLY_CENSUS_ARMS] : [], periodicLane: true };
|
||||
}
|
||||
|
||||
/** The env one arm's planner and executor run with. */
|
||||
export function censusArmEnv(arm: string): Record<string, string> {
|
||||
const spec = CENSUS_ARMS[arm];
|
||||
if (!spec) throw new Error(`unknown census arm: ${arm}`);
|
||||
if (!spec.model) return { EVALS_HOST: spec.host };
|
||||
return { EVALS_HOST: spec.host, [spec.host === 'codex' ? 'GSTACK_CODEX_MODEL' : 'EVALS_MODEL']: spec.model };
|
||||
}
|
||||
|
||||
/** One executor per arm and planned slice, with that slice's planned CI ceiling. */
|
||||
export function censusMatrix(arms: readonly string[], planDir: string): Array<{ arm: string; slice: number; timeout: number }> {
|
||||
return arms.flatMap(arm => {
|
||||
const manifest = JSON.parse(fs.readFileSync(path.join(planDir, arm, 'manifest.json'), 'utf8')) as
|
||||
{ sliceCount: number; plan?: { ciTimeoutMinutes: number; sliceCiTimeoutMinutes?: number[] } };
|
||||
if (!manifest.plan) throw new Error(`${arm}: census manifest has no budget plan`);
|
||||
return Array.from({ length: manifest.sliceCount }, (_, i) => ({
|
||||
arm, slice: i + 1, timeout: manifest.plan!.sliceCiTimeoutMinutes?.[i] ?? manifest.plan!.ciTimeoutMinutes,
|
||||
}));
|
||||
});
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
const [mode, ...args] = process.argv.slice(2);
|
||||
if (mode === '--select') {
|
||||
const { arms, periodicLane } = selectCensus({ schedule: process.env.SCHEDULE, censusArms: process.env.CENSUS_ARMS,
|
||||
includeGateCensus: process.env.INCLUDE_GATE_CENSUS === 'true' });
|
||||
console.log(`arms=${JSON.stringify(arms)}\nperiodic_lane=${periodicLane}`);
|
||||
} else if (mode === '--env' && args.length === 1) {
|
||||
for (const [key, value] of Object.entries(censusArmEnv(args[0]!))) console.log(`${key}=${value}`);
|
||||
} else if (mode === '--matrix' && args.length === 2) {
|
||||
console.log(`gate_matrix=${JSON.stringify(censusMatrix(JSON.parse(args[1]!) as string[], args[0]!))}`);
|
||||
} else {
|
||||
console.error('usage: census-arms.ts --select | --env <arm> | --matrix <plan dir> <arms json>');
|
||||
process.exit(2);
|
||||
}
|
||||
}
|
||||
@@ -83,6 +83,7 @@ export function e2eReuseLaneProblem(env: NodeJS.ProcessEnv, profileMode: string
|
||||
if (!/^(?:sha256:)?[a-f0-9]{64}$/.test(env.EVALS_CACHE_RUNTIME_ID ?? '')) return 'No immutable runtime identity';
|
||||
if (env.EVALS_TIER !== 'gate' || env.EVALS_FRESH === '1') return 'Fresh validation requested';
|
||||
if (['release', 'periodic', 'marathon'].includes(env.EVALS_CACHE_PURPOSE ?? '')) return 'Scheduled and release lanes execute fresh';
|
||||
if ((env.EVALS_HOST ?? '').trim() && env.EVALS_HOST!.trim() !== 'claude') return 'Non-Claude hosts execute fresh';
|
||||
if (env.NODE_OPTIONS || env.BUN_OPTIONS) return 'Preload options change execution outside the consumed source';
|
||||
if (env.ANTHROPIC_BASE_URL && env.ANTHROPIC_BASE_URL !== 'https://api.anthropic.com') return 'Custom model endpoint';
|
||||
return null;
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
"patterns": [
|
||||
"test/helpers/agent-sdk-runner.ts",
|
||||
"test/helpers/codex-session-runner.ts",
|
||||
"test/helpers/codex-host-runner.ts",
|
||||
"test/helpers/claude-pty-runner.ts",
|
||||
"test/helpers/pty/**",
|
||||
"test/helpers/pty-screen.ts",
|
||||
@@ -10,22 +11,23 @@
|
||||
"test/helpers/llm-judge.ts"
|
||||
],
|
||||
"files": {
|
||||
"scripts/lib/paid-cases.ts": {"blob":"9bacd52446192a7ecf3dfaae901ea443cf9c9ae2","since":6},
|
||||
"scripts/lib/paid-plan.ts": {"blob":"45721ce128a112666a0caed6c13c66637b74ca54","since":6},
|
||||
"scripts/lib/paid-report.ts": {"blob":"b66c41237611ccecaaa7bd6a4d7b7aac8dc548cd","since":6},
|
||||
"scripts/lib/paid-cases.ts": {"blob":"f862cf5241580c7593b9ee62eacf5ef046248d26","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
|
||||
"scripts/lib/paid-plan.ts": {"blob":"95e1798e7e2495ace350d2624e59f83412f341bc","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
|
||||
"scripts/lib/paid-report.ts": {"blob":"f92c6bb723b158e328aa040fa5de6a0ee6d3916d","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
|
||||
"scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6},
|
||||
"scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":6},
|
||||
"scripts/test-paid-shards.ts": {"blob":"8ddee5f11ed7856876135520dd8025ab989e2db9","since":6},
|
||||
"scripts/test-paid-shards.ts": {"blob":"19c9b2376d09090aee1a8acacfe721d66f55b0fc","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
|
||||
"scripts/test-pr-profile.ts": {"blob":"579ca1bbd590f709c1ea3d6f28fa6c629688c79f","since":6,"nonBehavioral":"Classifies scripts/free-test-seed-allowlist.json as free-only; only the free seed ratchet reads it, so no verdict can change"},
|
||||
"scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":6},
|
||||
"test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":6},
|
||||
"test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":6},
|
||||
"test/helpers/claude-pty-runner.ts": {"blob":"20cc5894532873aa6145fc6617fdeec8cef94a2d","since":6},
|
||||
"test/helpers/codex-session-runner.ts": {"blob":"4eea35931a35a0afb006363402e3d4191e379737","since":6},
|
||||
"test/helpers/codex-host-runner.ts": {"blob":"bff225017e235f0be6ece29f04971ac04b95b544","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
|
||||
"test/helpers/codex-session-runner.ts": {"blob":"c040ebbfa815b8cce19ef80c6133bd5a5474149a","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
|
||||
"test/helpers/e2e-gate.ts": {"blob":"efb7e197dd5f66c3ad0e34074b14e640754f7a45","since":6},
|
||||
"test/helpers/e2e-helpers.ts": {"blob":"2d4a2a53063cfc1f003db366d2c861ab1e379e6e","since":6},
|
||||
"test/helpers/e2e-helpers.ts": {"blob":"6ee6937e6c3ca081c77131b5a755278c7b714834","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
|
||||
"test/helpers/eval-budgets.ts": {"blob":"667d0e09b20fe5d7366f57b77b5fb9b04baa37ef","since":6},
|
||||
"test/helpers/eval-store.ts": {"blob":"609c6d099108ff2696eb014b47aa318a618dd2af","since":6},
|
||||
"test/helpers/eval-store.ts": {"blob":"ed8721f13d39f4ef62e0a1d814efbe9352da781a","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
|
||||
"test/helpers/hermetic-env.ts": {"blob":"cc072895461dc52ea317d1beff72121342420d30","since":6},
|
||||
"test/helpers/llm-judge.ts": {"blob":"d0bc6bbbe9f478603d11a1ece95e66c812ad34e6","since":6},
|
||||
"test/helpers/paid-test-set.ts": {"blob":"ff9bfe43c43ec2bdd11114f7b8ec7c5736116f75","since":6},
|
||||
@@ -44,8 +46,8 @@
|
||||
"test/helpers/pty/screen.ts": {"blob":"ba7a80bb68b1959296ab2a827cfacdaa46f4ba2d","since":6},
|
||||
"test/helpers/pty/session.ts": {"blob":"58533cea9d81cc87bb3b499bdfe0db8e3a8709c5","since":6},
|
||||
"test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":6},
|
||||
"test/helpers/session-ledger.ts": {"blob":"6e3c9e837e46f98addba2d12eaebff802da320bb","since":6},
|
||||
"test/helpers/session-runner.ts": {"blob":"d40eb29e8b8fdc0675b511693e4a90823b5963fc","since":6},
|
||||
"test/helpers/session-ledger.ts": {"blob":"425323284a368f5e77937e03884db4dea4e6134e","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
|
||||
"test/helpers/session-runner.ts": {"blob":"cc65a66adafa9c053e3da3c6f17b5ac86c0eade7","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
|
||||
"test/helpers/skill-fixture.ts": {"blob":"b188064594eb4201a54a80ba1bc6f28e31d7f49e","since":6},
|
||||
"test/helpers/test-selection.ts": {"blob":"51655f89f66be7a5ec487e2abfe4744300ebd5f6","since":6},
|
||||
"test/helpers/touchfiles.ts": {"blob":"15f369056d41b9dc42dcd86f81f60c878074704c","since":6}
|
||||
|
||||
@@ -9,6 +9,7 @@ import type { EvalCaseKind, PanelShape } from '../../test/helpers/eval-store';
|
||||
import { E2E_KINDS } from '../../test/helpers/touchfiles-data';
|
||||
import { E2E_TOUCHFILES, E2E_TIERS } from '../../test/helpers/touchfiles';
|
||||
import { type PaidTier, ROOT } from './paid-types';
|
||||
import { type EvalHost, hostSkipReason, resolveEvalHost } from '../../test/helpers/e2e-hosts';
|
||||
|
||||
/**
|
||||
* The E2E ids a paid file registers: the touchfile registrations that list the
|
||||
@@ -226,12 +227,38 @@ export function partitionCaseExclusions(keys: string[]): { runnable: string[]; e
|
||||
return { runnable, excluded };
|
||||
}
|
||||
|
||||
/**
|
||||
* Host exclusions (EVALS_HOST, test/helpers/e2e-hosts.ts): on a non-Claude
|
||||
* host a case key whose case is Claude-only, or a file key none of whose
|
||||
* `tier` cases runs on the host, is excluded with its reason. A file that
|
||||
* keeps one host case stays runnable; its Claude-only cases skip at module
|
||||
* load and the report deselects them. The Claude host excludes nothing.
|
||||
*/
|
||||
export function partitionHostExclusions(keys: string[], tier: PaidTier, host: EvalHost, rootDir = ROOT): { runnable: string[]; excluded: Array<{ file: string; reason: string }> } {
|
||||
if (host === 'claude') return { runnable: keys, excluded: [] };
|
||||
const excluded: Array<{ file: string; reason: string }> = [];
|
||||
const runnable = keys.filter(key => {
|
||||
const caseId = shardCaseId(key);
|
||||
const ids = caseId !== null ? [caseId]
|
||||
: fileCaseRegistration(shardFile(key), fs.readFileSync(path.join(rootDir, shardFile(key)), 'utf8')).registered.filter(id => E2E_TIERS[id] === tier);
|
||||
if (ids.some(id => hostSkipReason(id, host) === null)) return true;
|
||||
const reason = ids.length === 1 ? hostSkipReason(ids[0]!, host)!
|
||||
: ids.length === 0 ? `Claude-only on the ${host} host: no Codex host metadata (no ${tier} case registered)`
|
||||
: `Claude-only on the ${host} host: no case runs on it (${ids.join(', ')})`;
|
||||
excluded.push({ file: key, reason: `skipped: ${reason}` });
|
||||
return false;
|
||||
});
|
||||
return { runnable, excluded };
|
||||
}
|
||||
|
||||
/**
|
||||
* Codex access in the CI eval image, scoped per paid shard.
|
||||
*
|
||||
* The image installs the pinned Codex CLI off PATH (GSTACK_CI_CODEX_BIN_DIR)
|
||||
* and the eval workflows log it in under a non-default home
|
||||
* (GSTACK_CI_CODEX_HOME). A gate shard never sees either. A non-gate shard
|
||||
* (GSTACK_CI_CODEX_HOME). On the Codex host (EVALS_HOST=codex) Codex is the
|
||||
* agent, so every shard gets the CLI on PATH and the home as CODEX_HOME.
|
||||
* Otherwise a gate shard never sees either. A non-gate shard
|
||||
* whose every file exercises Codex gets the CLI on PATH and the home as
|
||||
* CODEX_HOME. A file that mixes one Codex case with other cases keeps only
|
||||
* the two variables, and that case opts in on its own session; hermetic
|
||||
@@ -266,8 +293,10 @@ export const CODEX_CI_CASES: Readonly<Record<string, readonly string[]>> = {
|
||||
export type CodexShardAccess = 'none' | 'case-opt-in' | 'path';
|
||||
|
||||
/** What one shard, given the test files behind its keys, may see of the CI Codex install in `tier`. */
|
||||
export function codexShardAccess(files: readonly string[], tier: string | undefined): CodexShardAccess {
|
||||
if (tier === 'gate' || files.length === 0) return 'none';
|
||||
export function codexShardAccess(files: readonly string[], tier: string | undefined, host: EvalHost = 'claude'): CodexShardAccess {
|
||||
if (files.length === 0) return 'none';
|
||||
if (host === 'codex') return 'path';
|
||||
if (tier === 'gate') return 'none';
|
||||
if (files.every(file => CODEX_CI_FILES.includes(file))) return 'path';
|
||||
return files.every(file => file in CODEX_CI_CASES) ? 'case-opt-in' : 'none';
|
||||
}
|
||||
@@ -277,7 +306,7 @@ export function scopeCodexAccess(env: NodeJS.ProcessEnv, files: readonly string[
|
||||
const binDir = env[CODEX_CI_ENV.binDir];
|
||||
if (!binDir) return;
|
||||
const home = env[CODEX_CI_ENV.home];
|
||||
const access = codexShardAccess(files, env.EVALS_TIER);
|
||||
const access = codexShardAccess(files, env.EVALS_TIER, resolveEvalHost(env));
|
||||
delete env.CODEX_HOME;
|
||||
if (access === 'case-opt-in') return;
|
||||
delete env[CODEX_CI_ENV.binDir];
|
||||
|
||||
@@ -10,7 +10,8 @@ import { evalEntryOutcome, type EvalCaseKind } from '../../test/helpers/eval-sto
|
||||
import { E2E_KINDS } from '../../test/helpers/touchfiles-data';
|
||||
import { prProfileCaseAllowed, prProfileFileMap, type PrProfileSelection } from '../test-pr-profile';
|
||||
import { E2E_TOUCHFILES, E2E_TIERS, LLM_JUDGE_TOUCHFILES } from '../../test/helpers/touchfiles';
|
||||
import { CASE_KEY_SEPARATOR, CASE_SHARDED_FILES, type CaseTrialPlan, caseTrialPlan, codexShardAccess, expandCaseShards, expandTrialShards, isIsolatedCase, partitionCaseExclusions, sameTrialPlan, shardCaseId, shardFile, shardTrial } from './paid-cases';
|
||||
import { CASE_KEY_SEPARATOR, CASE_SHARDED_FILES, type CaseTrialPlan, caseTrialPlan, codexShardAccess, expandCaseShards, expandTrialShards, isIsolatedCase, partitionCaseExclusions, partitionHostExclusions, sameTrialPlan, shardCaseId, shardFile, shardTrial } from './paid-cases';
|
||||
import { EVAL_HOSTS, type EvalHost, resolveEvalHost } from '../../test/helpers/e2e-hosts';
|
||||
import { DEFAULT_JOBS, OVERLAY_MAX_ACTIVE_SHARDS, PAID_TIERS, SLICE_UPLOAD_RESERVE_MS, type PaidCaseSelection, type PaidProfile, type PaidShardBudget, type PaidTier, ROOT, type ShardOutcome, type ShardStatus, type ShardTrialRecord, isOverlayTestFile } from './paid-types';
|
||||
import { collectPaidTestFiles, computePaidCaseSelection, expectedPrCaseCount, isAllSkippedPass, paidShardWallUpperBoundMs, partitionShardsByDiffSelection, planPaidShards, prProfileFileSelected, resolvePaidShardBudget, resolvePaidShardTimeoutMs, sameBudget, selectPaidTestFiles, shardSlug, validatedProfile } from './paid-select';
|
||||
|
||||
@@ -58,6 +59,8 @@ export interface PaidRunManifest {
|
||||
evalsAll: boolean;
|
||||
sliceCount: number;
|
||||
selectionReason: string;
|
||||
/** The agent host every session runs on (EVALS_HOST); absent means claude. */
|
||||
host?: EvalHost;
|
||||
/** Legacy v1 manifests omit these; new plans bind case-level execution. */
|
||||
profile?: PaidProfile;
|
||||
selection?: PaidCaseSelection;
|
||||
@@ -314,9 +317,11 @@ export function buildRunManifest(opts: {
|
||||
const quarantine = opts.quarantine ?? CASE_QUARANTINE;
|
||||
const notLive = [...Object.keys(kinds).filter(id => kinds[id] === 'behavior'), ...Object.keys(quarantine)].filter(id => !Object.hasOwn(E2E_TIERS, id));
|
||||
if (notLive.length) throw new Error(`Only live E2E cases can be behavior or quarantined (judges sample their panel inside the case): ${notLive.join(', ')}`);
|
||||
const caseKeys = partitionCaseExclusions(expandCaseShards(selected, opts.tier, rootDir));
|
||||
const host = resolveEvalHost(env);
|
||||
const ciKeys = partitionCaseExclusions(expandCaseShards(selected, opts.tier, rootDir));
|
||||
const caseKeys = partitionHostExclusions(ciKeys.runnable, opts.tier, host, rootDir);
|
||||
const excluded = [...tierSelection.excluded, ...(opts.skipJudges ? tierSelection.selected.filter(judge)
|
||||
.map(file => ({ file, reason: 'skipped: LLM judges run in the periodic census and PR gate lanes' })) : []), ...caseKeys.excluded];
|
||||
.map(file => ({ file, reason: 'skipped: LLM judges run in the periodic census and PR gate lanes' })) : []), ...ciKeys.excluded, ...caseKeys.excluded];
|
||||
const expansion = expandTrialShards(caseKeys.runnable, opts.tier, rootDir, { kinds, quarantine });
|
||||
const excludeOf = (key: string) => expansion.excludeCases[normalizeRelativePath(key)] ?? [];
|
||||
const shards = planPaidShards(expansion.keys, { maxFilesPerShard: 1 });
|
||||
@@ -346,7 +351,7 @@ export function buildRunManifest(opts: {
|
||||
entries.sort((a, b) => (a.file < b.file ? -1 : 1));
|
||||
return parseRunManifest(JSON.stringify({
|
||||
version: 1, tier: opts.tier, evalsAll: opts.evalsAll, sliceCount: plan.slices.length,
|
||||
selectionReason: cases.reason, profile, selection: cases.selection,
|
||||
selectionReason: cases.reason, ...(host !== 'claude' ? { host } : {}), profile, selection: cases.selection,
|
||||
...(cases.coverage ? { prCoverage: cases.coverage } : {}),
|
||||
plan: { sliceBudgetMs: opts.sliceBudgetMs!, jobs: opts.jobs!, estimatedSliceMs: plan.estimatedSliceMs, ciTimeoutMinutes: plan.ciTimeoutMinutes,
|
||||
sliceCiTimeoutMinutes: plan.sliceCiTimeoutMinutes, codexSlices: codexSlicesOf(opts.tier, entries) },
|
||||
@@ -466,6 +471,7 @@ export function buildRunManifest(opts: {
|
||||
evalsAll: opts.evalsAll,
|
||||
sliceCount,
|
||||
selectionReason: cases.reason,
|
||||
...(host !== 'claude' ? { host } : {}),
|
||||
profile,
|
||||
selection: cases.selection,
|
||||
...(cases.coverage ? { prCoverage: cases.coverage } : {}),
|
||||
@@ -479,6 +485,7 @@ export function parseRunManifest(raw: string): PaidRunManifest {
|
||||
if (parsed.version !== 1) throw new Error(`unsupported manifest version: ${(parsed as { version?: unknown }).version}`);
|
||||
if (!PAID_TIERS.includes(parsed.tier)) throw new Error(`manifest tier invalid: ${parsed.tier}`);
|
||||
if (parsed.profile !== undefined && parsed.profile !== 'pr' && parsed.profile !== 'full') throw new Error('manifest profile invalid');
|
||||
if (parsed.host !== undefined && (parsed.host === 'claude' || !EVAL_HOSTS.includes(parsed.host))) throw new Error(`manifest host invalid: ${parsed.host}`);
|
||||
if (parsed.selection !== undefined) {
|
||||
for (const [key, inventory] of [['e2e', E2E_TOUCHFILES], ['judges', LLM_JUDGE_TOUCHFILES]] as const) {
|
||||
const ids = parsed.selection?.[key];
|
||||
|
||||
@@ -17,6 +17,7 @@ import { E2E_KINDS } from '../../test/helpers/touchfiles-data';
|
||||
import { manualReviewProblem } from '../../test/helpers/cookie-workflow-manual-review';
|
||||
import { writeNegativeReceipt, writePanelReceipt } from '../e2e-shard-reuse';
|
||||
import { E2E_TOUCHFILES, E2E_TIERS, LLM_JUDGE_TOUCHFILES } from '../../test/helpers/touchfiles';
|
||||
import { type EvalHost, hostSkipReason } from '../../test/helpers/e2e-hosts';
|
||||
import { CASE_TEST_NAMES, type CaseTrialPlan, caseTestNamePattern, caseTrialPlan, fileCaseRegistration, shardCaseId, shardFile, trialShardKey } from './paid-cases';
|
||||
import { type ManifestEntry, PAID_TEST_DURATIONS_FILE, type PaidRunManifest, type SliceResult, collectorOutcomeCounts, formatProfileCoverage, loadPaidTestDurations, mergePaidTestDurations, parseRunManifest, trialPanelKey, verifySliceResults, writePaidTestDurations } from './paid-plan';
|
||||
import { DEFAULT_JOBS, type PaidCaseSelection, type PaidTier, ROOT, type ShardTrialRecord } from './paid-types';
|
||||
@@ -170,6 +171,8 @@ export interface JUnitShardContext {
|
||||
selection?: PaidCaseSelection;
|
||||
/** Ids this file shard leaves to their isolated trial shards. */
|
||||
excludeCases?: readonly string[];
|
||||
/** The run's agent host (manifest.host); absent means claude. */
|
||||
host?: EvalHost;
|
||||
}
|
||||
|
||||
export const DESELECTION_REASONS = {
|
||||
@@ -205,6 +208,8 @@ export function attributeJUnitCase(tc: JUnitCase, ctx: JUnitShardContext,
|
||||
const tier = tiers[id];
|
||||
if (tier !== undefined && tier !== ctx.tier) return { kind: 'deselected', reason: `${tier}-tier case (this lane runs ${ctx.tier})` };
|
||||
if (ctx.excludeCases?.includes(id)) return { kind: 'deselected', reason: DESELECTION_REASONS.isolated };
|
||||
const hostReason = hostSkipReason(id, ctx.host ?? 'claude');
|
||||
if (hostReason) return { kind: 'deselected', reason: hostReason };
|
||||
const pool = tier !== undefined ? ctx.selection?.e2e : Object.hasOwn(LLM_JUDGE_TOUCHFILES, id) ? ctx.selection?.judges : null;
|
||||
if (pool && !pool.includes(id)) return { kind: 'deselected', reason: DESELECTION_REASONS.selection };
|
||||
return { kind: 'case', id };
|
||||
@@ -241,7 +246,7 @@ export const CENSUS_RED_GUIDE = 'docs/evals/census-red.md';
|
||||
|
||||
export function sliceArtifactName(root: string, slice: number, attempt: number): string {
|
||||
const name = path.basename(root);
|
||||
return /^(?:paid-slice|gate-census)-\d+-a\d+$/.test(name) ? name : `slice ${slice} attempt ${attempt}`;
|
||||
return /^(?:paid-slice|gate-census)-\d+-a\d+(?:-[a-z0-9-]+)?$/.test(name) ? name : `slice ${slice} attempt ${attempt}`;
|
||||
}
|
||||
|
||||
export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact[], recordsByShard: Map<string, any[]>,
|
||||
@@ -254,7 +259,7 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact
|
||||
if (outcome.trial || outcome.files.length !== 1) continue;
|
||||
let xml = '';
|
||||
try { xml = fs.readFileSync(path.join(root, 'shards', shardSlug([key]), 'junit.xml'), 'utf8'); } catch { continue; }
|
||||
const ctx: JUnitShardContext = { key, tier: manifest.tier, selection: manifest.selection, excludeCases: entries.get(key)?.excludeCases };
|
||||
const ctx: JUnitShardContext = { key, tier: manifest.tier, selection: manifest.selection, excludeCases: entries.get(key)?.excludeCases, host: manifest.host };
|
||||
const byCase = new Map<string, JUnitCase[]>();
|
||||
const reasons = new Map<string, number>();
|
||||
for (const tc of parseJUnitCases(xml)) {
|
||||
@@ -530,7 +535,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
|
||||
const results = artifacts.filter(a => (a.result.attempt ?? 1) === primary).map(a => a.result);
|
||||
const verdict = verifySliceResults(manifest, results);
|
||||
const planned = manifest.entries.filter((e) => e.status === 'planned').length;
|
||||
const lane = `${manifest.tier}/${manifest.profile ?? 'full'}${manifest.evalsAll ? ' census' : ''}`;
|
||||
const lane = `${manifest.tier}/${manifest.profile ?? 'full'}${manifest.evalsAll ? ' census' : ''}${manifest.host ? ` on ${manifest.host}` : ''}`;
|
||||
console.log(`[test:paid] report: ${results.length}/${manifest.sliceCount} slices, ${planned} planned shards, tier=${manifest.tier}, attempt ${primary}${attempts.length > 1 ? ` (later attempts ${attempts.slice(1).join(', ')} reported, never replacing it)` : ''}`);
|
||||
for (const line of formatProfileCoverage(manifest)) console.log(line);
|
||||
for (const result of [...results].sort((a, b) => a.sliceIndex - b.sliceIndex)) {
|
||||
|
||||
@@ -73,6 +73,7 @@ import { readSessionLedger, type SessionLedgerRow } from '../test/helpers/sessio
|
||||
import { preflightAnthropicApi } from '../test/helpers/anthropic-preflight';
|
||||
import { e2eReuseLaneProblem, prepareE2EShardReuse, selectPlanReceipts } from './e2e-shard-reuse';
|
||||
import { E2E_TOUCHFILES, E2E_TIERS } from '../test/helpers/touchfiles';
|
||||
import { resolveEvalHost } from '../test/helpers/e2e-hosts';
|
||||
import { scopeCodexAccess, shardFile, shardCaseId, shardTrial, trialShardKey, type CaseTrialPlan, caseTrialPlan, excludedCasesNamePattern, caseTestNamePattern, expandCaseShards, expandTrialShards, fileCaseRegistration, partitionCaseExclusions } from './lib/paid-cases';
|
||||
import { retriesForFiles, trialPanelKey, sliceExecutionOrder, buildRunManifest, parseRunManifest, type SliceResult, sliceExitCode, guardTrialRecords, formatSlicePlan, formatCapacityPreflight, sliceDeadlineMs } from './lib/paid-plan';
|
||||
import { caseSelection, runCaseDiagnosis, formatPanelLine, runPaidReport } from './lib/paid-report';
|
||||
@@ -909,6 +910,9 @@ async function main(): Promise<number> {
|
||||
if (manifest.tier !== options.tier) {
|
||||
throw new Error(`manifest tier ${manifest.tier} != requested tier ${options.tier} — refusing a cross-tier run`);
|
||||
}
|
||||
if ((manifest.host ?? 'claude') !== resolveEvalHost()) {
|
||||
throw new Error(`manifest host ${manifest.host ?? 'claude'} != EVALS_HOST ${resolveEvalHost()} — refusing a cross-host run`);
|
||||
}
|
||||
const profile = manifest.profile ?? 'full';
|
||||
if (options.profileExplicit && options.profile !== profile) throw new Error(`manifest profile ${profile} != requested profile ${options.profile}`);
|
||||
if (options.sliceIndex > manifest.sliceCount) {
|
||||
|
||||
@@ -3,6 +3,7 @@ import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { runPaidShard, shardSlug } from '../scripts/test-paid-shards';
|
||||
import { CENSUS_ARMS } from '../scripts/census-arms';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const workflows = ['evals.yml', 'evals-periodic.yml', 'evals-marathon.yml'].map(name => ({
|
||||
@@ -52,19 +53,24 @@ test('every direct CI paid executor binds a safe unique run/attempt/job/slice id
|
||||
else expect(step.run).toContain('--user "$(id -u):$(id -g)"');
|
||||
const env = executorEnv(job, step);
|
||||
expect(env.EVALS_RUN_ID).toBeString();
|
||||
// The gate census matrix is one row per census arm and planned slice (scripts/census-arms.ts).
|
||||
const armed = job.strategy.matrix.include !== undefined;
|
||||
for (const run of ['36302678692', '36302678693']) {
|
||||
for (const attempt of ['1', '2']) {
|
||||
// The planner sizes the matrix; cover more slices than any live plan.
|
||||
expect(job.strategy.matrix.slice).toMatch(/^\$\{\{ fromJSON\(needs\.plan-slices\.outputs\.(?:[a-z]+_)*slices\) \}\}$/);
|
||||
for (let slice = 1; slice <= 64; slice++) {
|
||||
const id = render(env.EVALS_RUN_ID, {
|
||||
'github.run_id': `${run}${workflowIndex}`,
|
||||
'github.run_attempt': attempt, 'matrix.slice': String(slice),
|
||||
});
|
||||
expect(id).toMatch(/^[A-Za-z0-9_-]+$/);
|
||||
expect(id.length).toBeLessThan(120);
|
||||
expect(ids.has(id)).toBe(false);
|
||||
ids.add(id);
|
||||
expect(armed ? job.strategy.matrix.include : job.strategy.matrix.slice)
|
||||
.toMatch(armed ? /^\$\{\{ fromJSON\(needs\.plan-slices\.outputs\.gate_matrix\) \}\}$/ : /^\$\{\{ fromJSON\(needs\.plan-slices\.outputs\.(?:[a-z]+_)*slices\) \}\}$/);
|
||||
for (const arm of armed ? Object.keys(CENSUS_ARMS) : ['']) {
|
||||
for (let slice = 1; slice <= 64; slice++) {
|
||||
const id = render(env.EVALS_RUN_ID, {
|
||||
'github.run_id': `${run}${workflowIndex}`,
|
||||
'github.run_attempt': attempt, 'matrix.slice': String(slice), 'matrix.arm': arm,
|
||||
});
|
||||
expect(id).toMatch(/^[A-Za-z0-9_-]+$/);
|
||||
expect(id.length).toBeLessThan(120);
|
||||
expect(ids.has(id)).toBe(false);
|
||||
ids.add(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -79,7 +85,7 @@ for (const { name, jobName, job, step } of executors) {
|
||||
fs.mkdirSync(home); fs.mkdirSync(bin); fs.mkdirSync(path.join(root, 'test'));
|
||||
const configured = executorEnv(job, step);
|
||||
const runId = render(configured.EVALS_RUN_ID, {
|
||||
'github.run_id': '36302678692', 'github.run_attempt': '2', 'matrix.slice': '4',
|
||||
'github.run_id': '36302678692', 'github.run_attempt': '2', 'matrix.slice': '4', 'matrix.arm': 'codex-sol',
|
||||
});
|
||||
const evalDir = path.join(root, path.basename(configured.GSTACK_EVAL_DIR));
|
||||
const file = 'test/native-launch.test.ts';
|
||||
|
||||
@@ -113,7 +113,8 @@ describe('paid CI coordination stays off the eval image', () => {
|
||||
// A confirmed verdict reuse (R2a/R2) is the one PR run that executes no slice.
|
||||
expect(executor.if).toBe("${{ !cancelled() && needs.build-image.result == 'success' && needs.plan-slices.result == 'success' && needs.plan-slices.outputs.gate_reuse != 'true' }}");
|
||||
} else {
|
||||
expect(executor.if).toBeUndefined();
|
||||
// A census-only run (monthly cron or census_arms dispatch) executes no periodic slice.
|
||||
expect(executor.if).toBe("${{ needs.plan-slices.outputs.periodic_lane == 'true' }}");
|
||||
}
|
||||
expect(executor.steps.some(step => step.run === 'bun run build')).toBe(true);
|
||||
expect(executor.steps.some(step => step.uses === './.github/actions/restore-deps')).toBe(true);
|
||||
|
||||
@@ -0,0 +1,287 @@
|
||||
/**
|
||||
* The Codex host for paid gate cases (GSTA-49 TM2): host selection with
|
||||
* skip-with-reason (never a failure), per-case host metadata over every gate
|
||||
* case, real Codex cost accounting against captured `codex exec --json`
|
||||
* usage events (Codex CLI 0.160.0, gpt-6.1-sol), and runner parity.
|
||||
*/
|
||||
import { afterEach, describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { CLAUDE_ONLY_REASONS, E2E_HOSTS, hostSkipReason, resolveEvalHost } from './helpers/e2e-hosts';
|
||||
import { E2E_TIERS, E2E_TOUCHFILES } from './helpers/touchfiles';
|
||||
import { addCodexTurnUsage, codexUsageCostUsd, NO_CODEX_USAGE, parseCodexJSONL, type CodexResult } from './helpers/codex-session-runner';
|
||||
import { runRecordedCodexEval } from './helpers/codex-eval';
|
||||
import { codexHostExecArgs, codexHostModel, codexHostToolCalls, codexHostTranscript, unwrapShellCommand } from './helpers/codex-host-runner';
|
||||
import { coverageAuditReadEvidence } from './helpers/coverage-audit-evidence';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import { readSessionLedger } from './helpers/session-ledger';
|
||||
import { preflightAnthropicApi } from './helpers/anthropic-preflight';
|
||||
import type { EvalTestEntry } from './helpers/eval-store';
|
||||
import { buildRunManifest } from '../scripts/test-paid-shards';
|
||||
import { codexShardAccess, fileCaseRegistration, partitionHostExclusions, scopeCodexAccess, shardCaseId, shardFile } from '../scripts/lib/paid-cases';
|
||||
import { parseRunManifest } from '../scripts/lib/paid-plan';
|
||||
import { attributeJUnitCase } from '../scripts/lib/paid-report';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const fixture = (name: string) => fs.readFileSync(path.join(ROOT, 'test/fixtures/codex-host', name), 'utf8').split('\n').filter(Boolean);
|
||||
const COMMANDS = fixture('exec-commands.jsonl');
|
||||
const FILE_CHANGE = fixture('exec-file-change.jsonl');
|
||||
const gateCases = Object.keys(E2E_TIERS).filter(id => E2E_TIERS[id] === 'gate').sort();
|
||||
const adapterCases = Object.keys(E2E_HOSTS).filter(id => E2E_HOSTS[id] === 'both');
|
||||
const scratch: string[] = [];
|
||||
const tempDir = (prefix: string) => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); scratch.push(dir); return dir; };
|
||||
afterEach(() => { for (const dir of scratch.splice(0)) fs.rmSync(dir, { recursive: true, force: true }); });
|
||||
|
||||
describe('host selection: Claude-only cases skip with a reason, never fail', () => {
|
||||
test('EVALS_HOST defaults to claude, accepts codex and refuses anything else', () => {
|
||||
expect(resolveEvalHost({})).toBe('claude');
|
||||
expect(resolveEvalHost({ EVALS_HOST: ' ' })).toBe('claude');
|
||||
expect(resolveEvalHost({ EVALS_HOST: 'codex' })).toBe('codex');
|
||||
expect(() => resolveEvalHost({ EVALS_HOST: 'gemini' })).toThrow('EVALS_HOST must be one of claude, codex');
|
||||
});
|
||||
|
||||
test('the Anthropic preflight never pings claude on the Codex host', () => {
|
||||
const pinged: string[] = [];
|
||||
const spawn = ((cmd: string) => { pinged.push(cmd); return { status: 127, stdout: '', stderr: '' }; }) as unknown as typeof spawnSync;
|
||||
expect(preflightAnthropicApi({ EVALS_HOST: 'codex' }, spawn)).toBe('skipped');
|
||||
expect(pinged).toEqual([]);
|
||||
expect(() => preflightAnthropicApi({}, spawn)).toThrow('`claude` not found on PATH');
|
||||
expect(pinged).toEqual(['sh']);
|
||||
});
|
||||
|
||||
test('every case runs on Claude; on Codex only adapter cases run and the rest carry their reason', () => {
|
||||
for (const id of gateCases) expect(hostSkipReason(id, 'claude')).toBeNull();
|
||||
for (const id of adapterCases) expect(hostSkipReason(id, 'codex')).toBeNull();
|
||||
expect(hostSkipReason('plan-mode-no-op', 'codex')).toBe(`Claude-only on the codex host: ${CLAUDE_ONLY_REASONS.pty}`);
|
||||
expect(hostSkipReason('ship-docsync-store', 'codex')).toContain('not ported to the Codex host');
|
||||
expect(hostSkipReason('plan-ceo-review', 'codex')).toContain('no Codex host metadata');
|
||||
});
|
||||
|
||||
test('the module selection drops Claude-only cases on the Codex host', () => {
|
||||
const root = tempDir('codex-host-root-');
|
||||
fs.mkdirSync(path.join(root, 'home', '.codex'), { recursive: true });
|
||||
const run = spawnSync('bun', ['-e', "const h = await import('./test/helpers/e2e-helpers'); console.log(JSON.stringify(h.selectedTests))"], {
|
||||
cwd: ROOT, encoding: 'utf8', timeout: 60_000,
|
||||
env: { ...process.env, EVALS: '1', EVALS_TIER: 'gate', EVALS_HOST: 'codex', EVALS_PREFLIGHT_OK: '1', GSTACK_CODEX_HOST_ROOT: root,
|
||||
EVALS_SELECTION_JSON: JSON.stringify({ selected: null, reason: 'test' }) },
|
||||
});
|
||||
expect(run.status, run.stderr).toBe(0);
|
||||
expect((JSON.parse(run.stdout.trim().split('\n').at(-1)!) as string[]).sort()).toEqual([...adapterCases].sort());
|
||||
expect(run.stderr).toContain(`EVALS_HOST=codex: ${adapterCases.length} tests; ${gateCases.length - adapterCases.length} Claude-only skipped`);
|
||||
});
|
||||
|
||||
test('the planner excludes Claude-only shards with their reason and plans every adapter case', () => {
|
||||
const manifest = buildRunManifest({ tier: 'gate', profile: 'full', sliceBudgetMs: 420_000, jobs: 2, evalsAll: true, skipJudges: true,
|
||||
env: { EVALS_ALL: '1', EVALS_HOST: 'codex' }, rootDir: ROOT });
|
||||
expect(manifest.host).toBe('codex');
|
||||
const planned = manifest.entries.filter(entry => entry.status === 'planned');
|
||||
const casesOf = (key: string) => {
|
||||
const id = shardCaseId(key);
|
||||
if (id !== null) return [id];
|
||||
return fileCaseRegistration(shardFile(key), fs.readFileSync(path.join(ROOT, shardFile(key)), 'utf8')).registered.filter(c => E2E_TIERS[c] === 'gate');
|
||||
};
|
||||
const reachable = new Set(planned.flatMap(entry => casesOf(entry.file)).filter(id => E2E_HOSTS[id] === 'both'));
|
||||
expect([...reachable].sort()).toEqual([...adapterCases].sort());
|
||||
for (const entry of planned) expect(casesOf(entry.file).some(id => E2E_HOSTS[id] === 'both'), entry.file).toBe(true);
|
||||
const hostExcluded = manifest.entries.filter(entry => entry.status === 'excluded' && entry.reason?.includes('Claude-only on the codex host'));
|
||||
expect(hostExcluded.length).toBeGreaterThan(0);
|
||||
for (const entry of hostExcluded) expect(entry.reason, entry.file).toStartWith('skipped: ');
|
||||
expect(manifest.entries.some(entry => entry.file === 'test/skill-e2e-ship-docsync.test.ts#ship-docsync-store' && entry.status === 'excluded')).toBe(true);
|
||||
});
|
||||
|
||||
test('the Claude host plans exactly as before, with no host on the manifest', () => {
|
||||
const manifest = buildRunManifest({ tier: 'gate', profile: 'full', sliceBudgetMs: 420_000, jobs: 2, evalsAll: true, skipJudges: true,
|
||||
env: { EVALS_ALL: '1' }, rootDir: ROOT });
|
||||
expect(manifest.host).toBeUndefined();
|
||||
expect(manifest.entries.some(entry => entry.reason?.includes('Claude-only'))).toBe(false);
|
||||
expect(partitionHostExclusions(['a.test.ts', 'b.test.ts#x'], 'gate', 'claude')).toEqual({ runnable: ['a.test.ts', 'b.test.ts#x'], excluded: [] });
|
||||
});
|
||||
|
||||
test('the report deselects a skipped Claude-only testcase on the Codex host (zero credit, never SKIPPED or failed)', () => {
|
||||
const tc = { name: 'review-dashboard-via', classname: '', outcome: 'skipped' as const, timeMs: 0, line: 10 };
|
||||
const key = 'test/skill-e2e-review-attribution.test.ts';
|
||||
expect(attributeJUnitCase(tc, { key, tier: 'gate', host: 'codex' })).toEqual({ kind: 'deselected', reason: hostSkipReason('review-dashboard-via', 'codex')! });
|
||||
expect(attributeJUnitCase(tc, { key, tier: 'gate' })).toEqual({ kind: 'case', id: 'review-dashboard-via' });
|
||||
expect(attributeJUnitCase({ ...tc, name: 'review-base-branch' }, { key, tier: 'gate', host: 'codex' })).toEqual({ kind: 'case', id: 'review-base-branch' });
|
||||
});
|
||||
|
||||
test('a manifest names only a non-default host, and the executor refuses a cross-host run', () => {
|
||||
const base = { version: 1, tier: 'gate', evalsAll: true, sliceCount: 1, selectionReason: 'x', entries: [] };
|
||||
expect(parseRunManifest(JSON.stringify({ ...base, host: 'codex' })).host).toBe('codex');
|
||||
expect(() => parseRunManifest(JSON.stringify({ ...base, host: 'claude' }))).toThrow('manifest host invalid');
|
||||
expect(() => parseRunManifest(JSON.stringify({ ...base, host: 'gemini' }))).toThrow('manifest host invalid');
|
||||
const plan = path.join(tempDir('codex-host-plan-'), 'manifest.json');
|
||||
fs.writeFileSync(plan, JSON.stringify({ ...base, host: 'codex' }));
|
||||
const run = spawnSync('bun', ['run', 'scripts/test-paid-shards.ts', '--tier', 'gate', '--plan', plan, '--slice', '1', '--list'], {
|
||||
cwd: ROOT, encoding: 'utf8', timeout: 60_000, env: { ...process.env, EVALS_HOST: 'claude', EVALS_TIER: 'gate' },
|
||||
});
|
||||
expect(run.status).not.toBe(0);
|
||||
expect(run.stderr + run.stdout).toContain('manifest host codex != EVALS_HOST claude');
|
||||
});
|
||||
});
|
||||
|
||||
describe('per-case host metadata', () => {
|
||||
test('covers every gate case exactly once, and nothing else', () => {
|
||||
expect(Object.keys(E2E_HOSTS).sort()).toEqual(gateCases);
|
||||
for (const id of Object.keys(E2E_HOSTS)) expect(Object.hasOwn(E2E_TOUCHFILES, id), id).toBe(true);
|
||||
});
|
||||
|
||||
test('classifies 52 adapter-only, 29 Claude-specific, 26 rework and 1 host-neutral case', () => {
|
||||
const counts: Record<string, number> = {};
|
||||
for (const support of Object.values(E2E_HOSTS)) counts[support] = (counts[support] ?? 0) + 1;
|
||||
expect(counts.both).toBe(52);
|
||||
expect(counts.rework).toBe(26);
|
||||
expect(counts['host-neutral']).toBe(1);
|
||||
expect(['pty', 'sdk', 'subagents', 'skill-tool', 'hermetic', 'self-host'].reduce((sum, reason) => sum + (counts[reason] ?? 0), 0)).toBe(29);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Codex cost accounting from captured usage events', () => {
|
||||
test('parses every token class of turn.completed (cache reads and writes are subsets of input)', () => {
|
||||
const commands = parseCodexJSONL(COMMANDS);
|
||||
expect(commands.usage).toEqual({ input: 25009, cached_input: 12437, cache_write_input: 12566, output: 86, reasoning_output: 0 });
|
||||
expect(commands.tokens).toBe(25095);
|
||||
expect(parseCodexJSONL(FILE_CHANGE).usage).toEqual({ input: 25020, cached_input: 12440, cache_write_input: 12574, output: 137, reasoning_output: 0 });
|
||||
expect(parseCodexJSONL(['{"type":"thread.started","thread_id":"t"}']).usage).toBeNull();
|
||||
const twoTurns = [COMMANDS, FILE_CHANGE].map(lines => JSON.parse(lines.find(line => line.includes('turn.completed'))!).usage)
|
||||
.reduce(addCodexTurnUsage, NO_CODEX_USAGE);
|
||||
expect(twoTurns).toEqual({ input: 50029, cached_input: 24877, cache_write_input: 25140, output: 223, reasoning_output: 0 });
|
||||
});
|
||||
|
||||
test('prices with lib/pricing.ts: uncached input (cache writes included) at input, cache reads at the cached rate', () => {
|
||||
const usage = parseCodexJSONL(COMMANDS).usage!;
|
||||
// gpt-6.1-sol: 12,572 x $2 + 12,437 x $0.10 + 86 x $10 per million.
|
||||
expect(codexUsageCostUsd(usage, 'gpt-6.1-sol')).toBe(0.027248);
|
||||
expect(codexUsageCostUsd(parseCodexJSONL(FILE_CHANGE).usage!, 'gpt-6.1-sol')).toBe(0.027774);
|
||||
// gpt-6-astra: 12,572 x $10 + 12,437 x $1 + 86 x $50 per million.
|
||||
expect(codexUsageCostUsd(usage, 'gpt-6-astra')).toBe(0.142457);
|
||||
expect(codexUsageCostUsd(usage, 'gpt-0-unpriced')).toBeNull();
|
||||
});
|
||||
|
||||
test('runRecordedCodexEval records real cost and billing instead of cost_known: false', async () => {
|
||||
const evalDir = tempDir('codex-host-ledger-');
|
||||
const previous = process.env.GSTACK_EVAL_DIR;
|
||||
process.env.GSTACK_EVAL_DIR = evalDir;
|
||||
try {
|
||||
const recorded: EvalTestEntry[] = [];
|
||||
const result: CodexResult = { ...parseCodexJSONL(COMMANDS), exitCode: 0, durationMs: 5, rawLines: COMMANDS, stderr: '', model: 'gpt-6.1-sol' };
|
||||
await runRecordedCodexEval({ name: 'cost-case', suite: 's', budgetMs: 10_000, run: async () => result, validate: () => {}, record: entry => recorded.push(entry) });
|
||||
expect(recorded[0]).toMatchObject({ passed: true, cost_usd: 0.027248, model: 'gpt-6.1-sol', token_usage: result.usage });
|
||||
expect(recorded[0]!.cost_known).toBeUndefined();
|
||||
const noTurn: CodexResult = { ...parseCodexJSONL([]), exitCode: 0, durationMs: 5, rawLines: [], stderr: '' };
|
||||
await runRecordedCodexEval({ name: 'no-turn', suite: 's', budgetMs: 10_000, run: async () => noTurn, validate: () => {}, record: entry => recorded.push(entry) });
|
||||
expect(recorded[1]).toMatchObject({ cost_usd: 0, cost_known: false });
|
||||
const rows = readSessionLedger(evalDir);
|
||||
expect(rows.map(row => [row.test_name, row.billed])).toEqual([['cost-case', true], ['no-turn', false]]);
|
||||
expect(rows[0]).toMatchObject({ runner: 'codex', model: 'gpt-6.1-sol', cost_usd: 0.027248, tokens: result.usage });
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.GSTACK_EVAL_DIR; else process.env.GSTACK_EVAL_DIR = previous;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('Codex runner parity', () => {
|
||||
test('maps command_execution to Bash (unwrapped, with output) and file_change to Write/Edit', () => {
|
||||
expect(codexHostToolCalls(COMMANDS)).toEqual([
|
||||
{ tool: 'Bash', input: { command: "printf 'hi' > hello.txt\nprintf 'hello' > hello.txt\ncat hello.txt" }, output: 'hello' },
|
||||
]);
|
||||
expect(codexHostToolCalls(FILE_CHANGE)).toEqual([
|
||||
{ tool: 'Write', input: { file_path: '/tmp/codex-host-fixture/notes.md', kind: 'add' }, output: '' },
|
||||
{ tool: 'Edit', input: { file_path: '/tmp/codex-host-fixture/notes.md', kind: 'update' }, output: '' },
|
||||
]);
|
||||
});
|
||||
|
||||
test('the transcript is the Claude stream-json shape the case observers read', () => {
|
||||
const output = '`cat hello.txt` output: `hello`';
|
||||
const transcript = codexHostTranscript(COMMANDS, '/tmp/codex-host-fixture', output);
|
||||
expect(transcript[0]).toEqual({ type: 'system', subtype: 'init', cwd: '/tmp/codex-host-fixture', session_id: '01a150b3-ae5a-7932-ba51-4c295c66adf4', tools: [] });
|
||||
expect(transcript.slice(1).map(event => [event.type, event.message?.content[0].type ?? event.subtype])).toEqual([
|
||||
['assistant', 'text'], ['assistant', 'tool_use'], ['user', 'tool_result'], ['assistant', 'text'], ['result', 'success']]);
|
||||
expect(transcript[2].message.content[0]).toEqual({ type: 'tool_use', id: 'item_1', name: 'Bash', input: { command: "printf 'hi' > hello.txt\nprintf 'hello' > hello.txt\ncat hello.txt" } });
|
||||
expect(transcript[3].message.content[0]).toEqual({ type: 'tool_result', tool_use_id: 'item_1', content: 'hello', is_error: false });
|
||||
expect(transcript.at(-1)).toMatchObject({ type: 'result', result: output });
|
||||
const edits = codexHostTranscript(FILE_CHANGE, '/tmp/codex-host-fixture', '').filter(event => event.type === 'assistant' && event.message.content[0].type === 'tool_use');
|
||||
expect(edits.map(event => [event.message.content[0].name, event.message.content[0].input.file_path])).toEqual([
|
||||
['Write', '/tmp/codex-host-fixture/notes.md'], ['Edit', '/tmp/codex-host-fixture/notes.md']]);
|
||||
// A Codex `cat` of a fixture file is a read the coverage-audit evidence credits, as a native Bash read is.
|
||||
const source = { path: '/w/src/billing.ts', content: 'export const a = 1;\n' };
|
||||
const tests = { path: '/w/test/billing.test.ts', content: 'test("a", () => {});\n' };
|
||||
const read = (id: string, file: string, out: string) => JSON.stringify({ type: 'item.completed', item: { id, type: 'command_execution', command: `/bin/bash -lc 'cat ${file}'`, aggregated_output: out, exit_code: 0, status: 'completed' } });
|
||||
const lines = ['{"type":"thread.started","thread_id":"t1"}', read('item_0', 'src/billing.ts', source.content), read('item_1', 'test/billing.test.ts', tests.content)];
|
||||
expect(coverageAuditReadEvidence(codexHostTranscript(lines, '/w', ''), { cwd: '/w', source, tests })).toEqual({ sourceRead: true, testsRead: true });
|
||||
const failed = lines.map(line => line.replace('"exit_code":0', '"exit_code":1'));
|
||||
expect(coverageAuditReadEvidence(codexHostTranscript(failed, '/w', ''), { cwd: '/w', source, tests })).toEqual({ sourceRead: false, testsRead: false });
|
||||
});
|
||||
|
||||
test('unwraps only a single well-formed bash -lc word', () => {
|
||||
expect(unwrapShellCommand("/bin/bash -lc 'git diff main...HEAD'")).toBe('git diff main...HEAD');
|
||||
expect(unwrapShellCommand("/bin/bash -lc 'echo '\"'\"'x'\"'\"''")).toBe("echo 'x'");
|
||||
expect(unwrapShellCommand('/bin/bash -lc "echo \\"\\$HOME\\""')).toBe('echo "$HOME"');
|
||||
expect(unwrapShellCommand('bash -lc ls')).toBe('ls');
|
||||
expect(unwrapShellCommand('git status')).toBe('git status');
|
||||
expect(unwrapShellCommand("/bin/bash -lc 'unterminated")).toBe("/bin/bash -lc 'unterminated");
|
||||
expect(unwrapShellCommand("/bin/bash -lc 'a' 'b'")).toBe("/bin/bash -lc 'a' 'b'");
|
||||
});
|
||||
|
||||
test('sandbox, effort and model settings', () => {
|
||||
expect(codexHostExecArgs('gpt-6.1-sol', 'go', {})).toEqual(['exec', '--json', '-s', 'danger-full-access', '--skip-git-repo-check', '--model', 'gpt-6.1-sol', 'go']);
|
||||
expect(codexHostExecArgs('gpt-6.1-sol', 'go', { GSTACK_CODEX_EFFORT: 'high' })).toContain('model_reasoning_effort="high"');
|
||||
expect(codexHostExecArgs('m', 'go', { GSTACK_CODEX_SANDBOX: 'workspace-write' })).toEqual(
|
||||
['exec', '--json', '-s', 'workspace-write', '--skip-git-repo-check', '--model', 'm', '-c', 'sandbox_workspace_write.network_access=true', 'go']);
|
||||
expect(() => codexHostExecArgs('m', 'go', { GSTACK_CODEX_SANDBOX: 'none' })).toThrow('GSTACK_CODEX_SANDBOX');
|
||||
expect(() => codexHostExecArgs('m', 'go', { GSTACK_CODEX_EFFORT: 'high" -c x="y' })).toThrow('GSTACK_CODEX_EFFORT');
|
||||
expect(codexHostModel('claude-opus-4-7', { GSTACK_CODEX_MODEL: 'gpt-6.1-sol' })).toBe('gpt-6.1-sol');
|
||||
expect(codexHostModel('gpt-6-astra', { GSTACK_CODEX_MODEL: 'gpt-6.1-sol' })).toBe('gpt-6-astra');
|
||||
expect(codexHostModel(undefined, {})).toBe('gpt-6-astra');
|
||||
});
|
||||
|
||||
test('every shard on the Codex host gets the CI Codex install', () => {
|
||||
expect(codexShardAccess(['test/skill-e2e-review.test.ts'], 'gate')).toBe('none');
|
||||
expect(codexShardAccess(['test/skill-e2e-review.test.ts'], 'gate', 'codex')).toBe('path');
|
||||
const env: NodeJS.ProcessEnv = { EVALS_HOST: 'codex', EVALS_TIER: 'gate', PATH: '/usr/bin', GSTACK_CI_CODEX_BIN_DIR: '/opt/codex/bin', GSTACK_CI_CODEX_HOME: '/home/runner/.ci-codex' };
|
||||
scopeCodexAccess(env, ['test/skill-e2e-review.test.ts']);
|
||||
expect(env.PATH).toBe(`/opt/codex/bin${path.delimiter}/usr/bin`);
|
||||
expect(env.CODEX_HOME).toBe('/home/runner/.ci-codex');
|
||||
});
|
||||
|
||||
test('runSkillTest on EVALS_HOST=codex runs codex exec and returns a priced SkillTestResult', async () => {
|
||||
const dir = tempDir('codex-host-run-');
|
||||
const root = path.join(dir, 'runtime');
|
||||
fs.mkdirSync(path.join(root, 'home', '.codex'), { recursive: true });
|
||||
const bin = path.join(dir, 'bin');
|
||||
fs.mkdirSync(bin);
|
||||
const argsFile = path.join(dir, 'args.txt');
|
||||
fs.writeFileSync(path.join(bin, 'codex'), `#!/bin/sh\nprintf '%s\\n' "$@" > '${argsFile}'\nprintf '%s\\n' "$CODEX_HOME" >> '${argsFile}'\ncat '${path.join(ROOT, 'test/fixtures/codex-host/exec-commands.jsonl')}'\n`, { mode: 0o755 });
|
||||
const evalDir = path.join(dir, 'evals');
|
||||
const saved = { ...process.env };
|
||||
Object.assign(process.env, { EVALS_HOST: 'codex', GSTACK_CODEX_MODEL: 'gpt-6.1-sol', GSTACK_CODEX_HOST_ROOT: root, GSTACK_EVAL_DIR: evalDir, PATH: `${bin}${path.delimiter}${process.env.PATH}` });
|
||||
try {
|
||||
const result = await runSkillTest({ prompt: 'Run the fixture.', workingDirectory: dir, model: 'claude-opus-4-7', timeout: 30_000, testName: 'host-switch' });
|
||||
expect(result).toMatchObject({ exitReason: 'success', model: 'gpt-6.1-sol', output: '`cat hello.txt` output: `hello`', browseErrors: [] });
|
||||
expect(result.toolCalls.map(call => call.input.command)).toEqual(["printf 'hi' > hello.txt\nprintf 'hello' > hello.txt\ncat hello.txt"]);
|
||||
expect(result.costEstimate).toMatchObject({ estimatedCost: 0.027248, estimatedTokens: 25095, turnsUsed: 3 });
|
||||
expect(result.tokenUsage).toEqual(parseCodexJSONL(COMMANDS).usage!);
|
||||
const args = fs.readFileSync(argsFile, 'utf8').trim().split('\n');
|
||||
expect(args.slice(0, 7)).toEqual(['exec', '--json', '-s', 'danger-full-access', '--skip-git-repo-check', '--model', 'gpt-6.1-sol']);
|
||||
expect(args.at(-2)).toBe('Run the fixture.');
|
||||
expect(args.at(-1)).toBe(path.join(root, 'home', '.codex'));
|
||||
expect(readSessionLedger(evalDir)).toEqual([expect.objectContaining({ runner: 'codex', test_name: 'host-switch', end: 'completed', billed: true, cost_usd: 0.027248 })]);
|
||||
// The native lifecycle sees the codex group owner at spawn and settles after its exit.
|
||||
const seen: string[] = [];
|
||||
await runSkillTest({ prompt: 'Run the fixture.', workingDirectory: dir, timeout: 30_000, testName: 'host-lifecycle', nativeLifecycle: {
|
||||
onSpawn: pid => { seen.push(`spawn ${pid > 0}`); },
|
||||
onSettled: async ({ deadline, exited }) => { seen.push(`settled ${exited} ${deadline > Date.now()}`); },
|
||||
} });
|
||||
expect(seen).toEqual(['spawn true', 'settled true true']);
|
||||
await expect(runSkillTest({ prompt: 'Run the fixture.', workingDirectory: dir, timeout: 30_000, testName: 'host-lifecycle-refused', nativeLifecycle: {
|
||||
onSpawn: () => { throw new Error('native process is not group owner'); },
|
||||
onSettled: async () => {},
|
||||
} })).rejects.toThrow('native process is not group owner');
|
||||
} finally {
|
||||
for (const key of Object.keys(process.env)) if (!(key in saved)) delete process.env[key];
|
||||
Object.assign(process.env, saved);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -22,7 +22,10 @@
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
import { spawnSync } from 'child_process';
|
||||
import { buildRunManifest, isCodexShard, isOverlayTestFile, parseCliOptions, resolvePaidShardTimeoutMs, sliceExecutionOrder, sliceSupervisedWallMs, CI_SETUP_ALLOWANCE_MINUTES } from '../scripts/test-paid-shards';
|
||||
import { CENSUS_ARMS, MONTHLY_CENSUS_ARMS, MONTHLY_CENSUS_SCHEDULE, WEEKLY_CENSUS_ARMS, WEEKLY_CENSUS_SCHEDULE, censusArmEnv, censusMatrix, selectCensus } from '../scripts/census-arms';
|
||||
|
||||
const ROOT = path.join(import.meta.dir, '..');
|
||||
const read = (rel: string) => fs.readFileSync(path.join(ROOT, rel), 'utf-8');
|
||||
@@ -40,7 +43,7 @@ function plannerSites(source: string): Array<{ manifest: string; budgetSeconds:
|
||||
|
||||
type Step = { id?: string; name?: string; run?: string; uses?: string; env?: Record<string, string>; with?: Record<string, string> };
|
||||
type Job = { needs?: string[]; env?: Record<string, string>; outputs?: Record<string, string>; 'timeout-minutes': string | number;
|
||||
strategy?: { 'max-parallel': number; matrix: { slice: string } }; steps: Step[] };
|
||||
strategy?: { 'max-parallel': number; matrix: { slice?: string; include?: string } }; steps: Step[] };
|
||||
|
||||
/**
|
||||
* An executor's matrix and timeout must come from the planner step that wrote
|
||||
@@ -159,7 +162,6 @@ describe('evals.yml sliced-lane wiring (post-matrix)', () => {
|
||||
describe('evals-periodic.yml sliced-lane wiring', () => {
|
||||
const lanes = [
|
||||
{ source: periodicYml, name: 'evals-periodic.yml', executor: 'eval-slices', prefix: 'periodic_', tier: 'periodic' },
|
||||
{ source: periodicYml, name: 'evals-periodic.yml', executor: 'gate-census', prefix: 'gate_', tier: 'gate' },
|
||||
{ source: evalsYml, name: 'evals.yml', executor: 'eval-slices', prefix: '', tier: 'gate' },
|
||||
{ source: marathonYml, name: 'evals-marathon.yml', executor: 'eval-slices', prefix: '', tier: 'marathon' },
|
||||
] as const;
|
||||
@@ -200,6 +202,64 @@ describe('evals-periodic.yml sliced-lane wiring', () => {
|
||||
});
|
||||
}
|
||||
|
||||
test('evals-periodic.yml:gate-census — one row per census arm and planned slice with its plan\'s ceiling, and every row of a scheduled census starts at once', () => {
|
||||
const workflow = Bun.YAML.parse(periodicYml) as { env?: Record<string, string>; jobs: Record<string, Job> };
|
||||
const planner = workflow.jobs['plan-slices']!;
|
||||
const executor = workflow.jobs['gate-census']!;
|
||||
expect(executor.needs).toContain('plan-slices');
|
||||
expect(executor.strategy!.matrix).toEqual({ include: '${{ fromJSON(needs.plan-slices.outputs.gate_matrix) }}' });
|
||||
expect(executor['timeout-minutes']).toBe('${{ matrix.timeout }}');
|
||||
expect(executor.steps[0]!.run).toBe('echo "GSTACK_SLICE_JOB_STARTED_AT=$(date +%s)" >> "$GITHUB_ENV"');
|
||||
expect(planner.outputs!.gate_matrix).toBe('${{ steps.gate-matrix.outputs.gate_matrix }}');
|
||||
expect(planner.steps.find(step => step.id === 'gate-matrix')!.run).toBe('bun --no-install run scripts/census-arms.ts --matrix /tmp/gate-census-plan "$ARMS" >> "$GITHUB_OUTPUT"');
|
||||
const emit = planner.steps.filter(step => step.run?.includes('--emit-plan "/tmp/gate-census-plan/$arm/manifest.json" '));
|
||||
expect(emit).toHaveLength(1);
|
||||
expect(emit[0]!.run).toContain('mapfile -t arm_env < <(bun --no-install run scripts/census-arms.ts --env "$arm")');
|
||||
expect(emit[0]!.run).toContain('env "${arm_env[@]}" EVALS_TIER=gate bun run scripts/test-paid-shards.ts');
|
||||
const execute = executor.steps.filter(step => step.run?.includes('--plan '));
|
||||
expect(execute).toHaveLength(1);
|
||||
expect(execute[0]!.run).toContain('--plan /tmp/gate-census-plan/${{ matrix.arm }}/manifest.json --slice ${{ matrix.slice }}');
|
||||
// The executor runs with its arm's host and model, and downloads every arm's plan.
|
||||
expect(executor.steps.find(step => step.name === "Select the census arm's host and model")!.run).toBe('bun --no-install run scripts/census-arms.ts --env "$ARM" >> "$GITHUB_ENV"');
|
||||
expect(executor.steps.some(step => step.with?.path === '/tmp/gate-census-plan')).toBe(true);
|
||||
const cliArgs = (run: string) => {
|
||||
const command = /\bbun(?: --no-install)? run scripts\/test-paid-shards\.ts (.*)$/m.exec(run)!;
|
||||
return command[1]!.replace(/\$\{\{\s*matrix\.slice\s*\}\}/g, '1').replace(/\$\{\{\s*matrix\.arm\s*\}\}/g, 'claude-opus').trim().split(/\s+/);
|
||||
};
|
||||
const planned = parseCliOptions(cliArgs(emit[0]!.run!), { ...workflow.env, ...emit[0]!.env, EVALS_ALL: '1', EVALS_PROFILE: 'full' });
|
||||
const active = parseCliOptions(cliArgs(execute[0]!.run!), { ...workflow.env, ...executor.env, ...execute[0]!.env, EVALS_PROFILE: 'full' });
|
||||
expect(planned.tier).toBe('gate');
|
||||
expect(active.tier).toBe('gate');
|
||||
expect(active.jobs).toBe(planned.jobs);
|
||||
expect(execute[0]!.env?.EVALS_JOBS).toBe(String(planned.jobs));
|
||||
const planDir = fs.mkdtempSync(path.join(os.tmpdir(), 'census-arms-'));
|
||||
try {
|
||||
for (const arm of Object.keys(CENSUS_ARMS)) {
|
||||
const env = { ...workflow.env, ...emit[0]!.env, EVALS_ALL: '1', EVALS_PROFILE: 'full', ...censusArmEnv(arm) };
|
||||
const manifest = buildRunManifest({ tier: 'gate', profile: 'full', sliceBudgetMs: planned.sliceBudgetMs!, jobs: planned.jobs,
|
||||
evalsAll: true, env, rootDir: ROOT, skipJudges: planned.skipJudges });
|
||||
expect(manifest.host ?? 'claude').toBe(CENSUS_ARMS[arm]!.host);
|
||||
const sliceFiles = Array.from({ length: manifest.sliceCount }, (_, i) => sliceExecutionOrder(
|
||||
manifest.entries.filter(entry => entry.status === 'planned' && entry.slice === i + 1)).map(entry => entry.file));
|
||||
const ceilings = sliceFiles.map(files => Math.ceil(Math.max(2 * planned.sliceBudgetMs!,
|
||||
...files.map(file => resolvePaidShardTimeoutMs([file])),
|
||||
files.some(isOverlayTestFile) ? sliceSupervisedWallMs(files, planned.jobs) : 0) / 60_000) + CI_SETUP_ALLOWANCE_MINUTES);
|
||||
expect(manifest.plan!.sliceCiTimeoutMinutes).toEqual(ceilings);
|
||||
expect(manifest.plan!.ciTimeoutMinutes).toBeLessThanOrEqual(360);
|
||||
fs.mkdirSync(path.join(planDir, arm));
|
||||
fs.writeFileSync(path.join(planDir, arm, 'manifest.json'), JSON.stringify(manifest));
|
||||
expect(censusMatrix([arm], planDir)).toEqual(ceilings.map((timeout, i) => ({ arm, slice: i + 1, timeout })));
|
||||
}
|
||||
for (const arms of [WEEKLY_CENSUS_ARMS, MONTHLY_CENSUS_ARMS]) {
|
||||
const rows = censusMatrix(arms, planDir);
|
||||
expect(new Set(rows.map(row => `${row.arm}/${row.slice}`)).size).toBe(rows.length);
|
||||
expect(rows.length, `${arms.join('+')} rows exceed max-parallel`).toBeLessThanOrEqual(executor.strategy!['max-parallel']);
|
||||
}
|
||||
} finally {
|
||||
fs.rmSync(planDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('planner/executor/report tier=periodic agree and plan with the ~7-minute budget (W5c)', () => {
|
||||
expect(periodicYml).toMatch(/EVALS_TIER=periodic bun --no-install run scripts\/test-paid-shards\.ts --tier periodic --emit-plan/);
|
||||
expect(periodicYml).toMatch(/EVALS_TIER=periodic bun run scripts\/test-paid-shards\.ts --tier periodic --plan .* --slice /);
|
||||
@@ -207,7 +267,7 @@ describe('evals-periodic.yml sliced-lane wiring', () => {
|
||||
// Periodic work and the full gate census have distinct immutable plans.
|
||||
expect(plannerSites(periodicYml)).toEqual([
|
||||
{ manifest: '/tmp/paid-plan/manifest.json', budgetSeconds: 420, jobs: 2 },
|
||||
{ manifest: '/tmp/gate-census-plan/manifest.json', budgetSeconds: 420, jobs: 2 },
|
||||
{ manifest: '"/tmp/gate-census-plan/$arm/manifest.json"', budgetSeconds: 420, jobs: 2 },
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -228,7 +288,7 @@ describe('evals-periodic.yml host-run Codex slices', () => {
|
||||
expect(matrixStep.run).toContain('} >> "$GITHUB_OUTPUT"');
|
||||
expect(planner.outputs!.periodic_codex_slices).toBe('${{ steps.periodic-matrix.outputs.codex_slices }}');
|
||||
expect(codex.needs).toEqual(['build-image', 'plan-slices']);
|
||||
expect(codex.if).toBe("${{ needs.plan-slices.outputs.periodic_codex_slices != '[]' }}");
|
||||
expect(codex.if).toBe("${{ needs.plan-slices.outputs.periodic_lane == 'true' && needs.plan-slices.outputs.periodic_codex_slices != '[]' }}");
|
||||
expect(codex.strategy!.matrix.slice).toBe('${{ fromJSON(needs.plan-slices.outputs.periodic_codex_slices) }}');
|
||||
expect(codex['timeout-minutes']).toBe('${{ fromJSON(needs.plan-slices.outputs.periodic_slice_timeouts)[matrix.slice] }}');
|
||||
expect(codex.steps[0]!.run).toBe('echo "GSTACK_SLICE_JOB_STARTED_AT=$(date +%s)" >> "$GITHUB_ENV"');
|
||||
@@ -350,7 +410,7 @@ describe('panel verdict surfaces (eval reliability policy)', () => {
|
||||
|
||||
test('planners size the capacity preflight with their executor cap', () => {
|
||||
for (const [source, executor, manifest] of [[evalsYml, 'eval-slices', '/tmp/paid-plan/manifest.json'],
|
||||
[periodicYml, 'eval-slices', '/tmp/paid-plan/manifest.json'], [periodicYml, 'gate-census', '/tmp/gate-census-plan/manifest.json']] as const) {
|
||||
[periodicYml, 'eval-slices', '/tmp/paid-plan/manifest.json'], [periodicYml, 'gate-census', '"/tmp/gate-census-plan/$arm/manifest.json"']] as const) {
|
||||
const jobs = jobsOf(source);
|
||||
const emit = jobs['plan-slices']!.steps.find(step => step.run?.includes(`--emit-plan ${manifest} `))!;
|
||||
const cap = Number(/--max-parallel (\d+)/.exec(emit.run!)?.[1]);
|
||||
@@ -401,6 +461,10 @@ describe('panel verdict surfaces (eval reliability policy)', () => {
|
||||
expect(redispatch.permissions).toEqual({ actions: 'write' });
|
||||
expect(redispatch.if).toBe("${{ !cancelled() && needs.report.outputs.redispatch == 'true' }}");
|
||||
expect(redispatch.steps[0]!.run).toContain('-f redispatch_of="$GITHUB_RUN_ID"');
|
||||
// A census-only run re-dispatches with its own arms, never as a weekly run.
|
||||
expect(redispatch.steps[0]!.run).toContain('${CENSUS_ARMS:+-f census_arms="$CENSUS_ARMS"}');
|
||||
expect(redispatch.steps[0]!.env?.CENSUS_ARMS).toBe('${{ needs.report.outputs.redispatch_census_arms }}');
|
||||
expect(report.outputs!.redispatch_census_arms).toBe("${{ needs.plan-slices.outputs.periodic_lane != 'true' && join(fromJSON(needs.plan-slices.outputs.census_arms), ',') || '' }}");
|
||||
const classify = report.steps.find(step => step.id === 'verdict')!;
|
||||
expect(classify.run).toContain('.verdict.redispatchEligible == true');
|
||||
expect(classify.run).toContain('[ -z "$REDISPATCH_OF" ]');
|
||||
@@ -426,7 +490,13 @@ describe('scheduled paid lanes: concurrency and branch dispatch scope', () => {
|
||||
test('a branch dispatch of evals-periodic runs the periodic lane only unless it opts into the gate census', () => {
|
||||
const wf = parse(periodicYml);
|
||||
expect(wf.on.workflow_dispatch!.inputs!.include_gate_census).toMatchObject({ type: 'boolean', default: false });
|
||||
expect(wf.jobs['gate-census']!.if).toBe("${{ github.ref == 'refs/heads/main' || inputs.include_gate_census }}");
|
||||
// The planner selects the census arms (scripts/census-arms.ts); no arm, no census job.
|
||||
expect(wf.jobs['gate-census']!.if).toBe("${{ needs.plan-slices.outputs.gate_matrix != '[]' }}");
|
||||
const select = wf.jobs['plan-slices']!.steps.find(step => step.id === 'census')!;
|
||||
expect(select.env?.INCLUDE_GATE_CENSUS).toBe("${{ github.ref == 'refs/heads/main' || inputs.include_gate_census }}");
|
||||
expect(select.run).toBe('bun --no-install run scripts/census-arms.ts --select >> "$GITHUB_OUTPUT"');
|
||||
expect(selectCensus({ schedule: '', censusArms: '', includeGateCensus: false })).toEqual({ arms: [], periodicLane: true });
|
||||
expect(selectCensus({ schedule: WEEKLY_CENSUS_SCHEDULE, includeGateCensus: true })).toEqual({ arms: [...WEEKLY_CENSUS_ARMS], periodicLane: true });
|
||||
const report = wf.jobs.report!.steps;
|
||||
const reconcile = report.find(step => step.name === 'Reconcile gate census against the manifest (fail-closed)')!;
|
||||
expect(reconcile.env?.GATE_CENSUS).toBe('${{ needs.gate-census.result }}');
|
||||
@@ -436,3 +506,80 @@ describe('scheduled paid lanes: concurrency and branch dispatch scope', () => {
|
||||
expect(fail.if).toContain(`!contains(fromJSON('["success","skipped"]'), needs.gate-census.result)`);
|
||||
});
|
||||
});
|
||||
|
||||
describe('evals-periodic.yml census arms: Monday two-arm census, monthly Fable/Astra census', () => {
|
||||
type CensusStep = Step & { if?: string };
|
||||
type CensusWf = { on: { schedule: Array<{ cron: string }>; workflow_dispatch: { inputs: Record<string, { type?: string; default?: unknown }> } };
|
||||
concurrency: { group: string }; jobs: Record<string, { if?: string; outputs?: Record<string, string>; steps: CensusStep[] }> };
|
||||
const wf = Bun.YAML.parse(periodicYml) as CensusWf;
|
||||
const planner = wf.jobs['plan-slices']!;
|
||||
const census = wf.jobs['gate-census']!;
|
||||
const report = wf.jobs.report!;
|
||||
|
||||
test('each arm is one host and model: the capture default and gpt-6.1-sol weekly, claude-fable-5-1 and gpt-6-astra monthly', () => {
|
||||
expect(WEEKLY_CENSUS_ARMS).toEqual(['claude-opus', 'codex-sol']);
|
||||
expect(MONTHLY_CENSUS_ARMS).toEqual(['claude-fable', 'codex-astra']);
|
||||
// claude-opus pins no model: the Claude runners' capture default (lib/eval-model.ts) applies.
|
||||
expect(censusArmEnv('claude-opus')).toEqual({ EVALS_HOST: 'claude' });
|
||||
expect(censusArmEnv('codex-sol')).toEqual({ EVALS_HOST: 'codex', GSTACK_CODEX_MODEL: 'gpt-6.1-sol' });
|
||||
expect(censusArmEnv('claude-fable')).toEqual({ EVALS_HOST: 'claude', EVALS_MODEL: 'claude-fable-5-1' });
|
||||
expect(censusArmEnv('codex-astra')).toEqual({ EVALS_HOST: 'codex', GSTACK_CODEX_MODEL: 'gpt-6-astra' });
|
||||
expect(() => censusArmEnv('codex-luna')).toThrow('unknown census arm');
|
||||
// The report files slice artifacts by arm suffix: no arm id may end another one.
|
||||
const ids = Object.keys(CENSUS_ARMS);
|
||||
for (const a of ids) for (const b of ids) if (a !== b) expect(a.endsWith(`-${b}`)).toBe(false);
|
||||
for (const id of ids) expect(id).toMatch(/^[a-z]+-[a-z]+$/);
|
||||
});
|
||||
|
||||
test('the monthly cron runs only the census, under its own concurrency group; the Monday and daily canary schedules are unchanged', () => {
|
||||
expect(wf.on.schedule.map(entry => entry.cron)).toEqual([WEEKLY_CENSUS_SCHEDULE, '30 7 * * *', MONTHLY_CENSUS_SCHEDULE]);
|
||||
expect(MONTHLY_CENSUS_SCHEDULE).toBe('0 6 1 * *');
|
||||
expect(wf.concurrency.group.endsWith("${{ github.event.schedule == '0 6 1 * *' && '-monthly' || '' }}")).toBe(true);
|
||||
expect(wf.concurrency.group).toContain("(github.event.schedule == '30 7 * * *' || inputs.schema_canary_only) && '-schema-canary'");
|
||||
expect(selectCensus({ schedule: MONTHLY_CENSUS_SCHEDULE, includeGateCensus: true })).toEqual({ arms: [...MONTHLY_CENSUS_ARMS], periodicLane: false });
|
||||
const select = planner.steps.find(step => step.id === 'census')!;
|
||||
expect(select.env).toMatchObject({ SCHEDULE: '${{ github.event.schedule }}', CENSUS_ARMS: '${{ inputs.census_arms }}' });
|
||||
expect(planner.outputs).toMatchObject({ periodic_lane: '${{ steps.census.outputs.periodic_lane }}', census_arms: '${{ steps.census.outputs.arms }}' });
|
||||
// Without the periodic lane no periodic plan is emitted, executed or reconciled.
|
||||
for (const step of planner.steps.filter(step => step.run?.includes('/tmp/paid-plan/manifest.json') || step.with?.name === 'paid-plan')) {
|
||||
expect(step.if, step.name ?? step.with?.name).toBe("steps.census.outputs.periodic_lane == 'true'");
|
||||
}
|
||||
const reconcile = report.steps.find(step => step.id === 'reconcile')!;
|
||||
expect(reconcile.env?.PERIODIC_LANE).toBe('${{ needs.plan-slices.outputs.periodic_lane }}');
|
||||
expect(reconcile.run).toContain('if [ "$PERIODIC_LANE" != "true" ]; then');
|
||||
// A census-only green run never closes the weekly tracking issue.
|
||||
expect(report.steps.find(step => step.name === 'Close the tracking issue on a green run')!.if)
|
||||
.toContain("needs.plan-slices.outputs.periodic_lane == 'true' && needs.eval-slices.result == 'success'");
|
||||
});
|
||||
|
||||
test('a census_arms dispatch runs only the named arms and refuses unknown ones', () => {
|
||||
expect(wf.on.workflow_dispatch.inputs.census_arms).toMatchObject({ type: 'string', default: '' });
|
||||
expect(selectCensus({ censusArms: 'codex-sol', includeGateCensus: true })).toEqual({ arms: ['codex-sol'], periodicLane: false });
|
||||
expect(selectCensus({ censusArms: 'claude-opus, codex-sol,codex-sol', includeGateCensus: false })).toEqual({ arms: ['claude-opus', 'codex-sol'], periodicLane: false });
|
||||
expect(() => selectCensus({ censusArms: 'codex-sol,gpt', includeGateCensus: false })).toThrow('unknown census arm(s): gpt');
|
||||
const cli = (args: string[], env: Record<string, string>) => {
|
||||
const proc = spawnSync(process.execPath, ['--no-install', 'run', 'scripts/census-arms.ts', ...args],
|
||||
{ cwd: ROOT, env: { PATH: process.env.PATH ?? '', ...env }, encoding: 'utf8', timeout: 15_000 });
|
||||
expect(proc.status, proc.stderr).toBe(0);
|
||||
return proc.stdout;
|
||||
};
|
||||
expect(cli(['--select'], { SCHEDULE: MONTHLY_CENSUS_SCHEDULE })).toBe('arms=["claude-fable","codex-astra"]\nperiodic_lane=false\n');
|
||||
expect(cli(['--select'], { CENSUS_ARMS: 'codex-sol', INCLUDE_GATE_CENSUS: 'true' })).toBe('arms=["codex-sol"]\nperiodic_lane=false\n');
|
||||
expect(cli(['--env', 'codex-sol'], {})).toBe('EVALS_HOST=codex\nGSTACK_CODEX_MODEL=gpt-6.1-sol\n');
|
||||
});
|
||||
|
||||
test('census artifacts and reconciliation are per arm', () => {
|
||||
const upload = census.steps.find(step => step.name === 'Upload census results')!;
|
||||
expect(upload.with!.name).toBe('gate-census-${{ matrix.slice }}-a${{ github.run_attempt }}-${{ matrix.arm }}');
|
||||
expect(census.steps.find(step => step.name?.startsWith('Run gate census slice'))!.name).toContain('(${{ matrix.arm }})');
|
||||
expect(report.steps.some(step => step.with?.pattern === 'gate-census-[0-9]*' && step.with?.path === '/tmp/gate-census-slices')).toBe(true);
|
||||
const reconcile = report.steps.find(step => step.id === 'gate-reconcile')!;
|
||||
expect(reconcile.env?.ARMS).toBe('${{ needs.plan-slices.outputs.census_arms }}');
|
||||
expect(reconcile.run).toContain('cp "/tmp/gate-census-plan/$arm/manifest.json" "$dir/manifest.json" || status=1');
|
||||
expect(reconcile.run).toContain('for slice in /tmp/gate-census-slices/gate-census-*-"$arm"; do');
|
||||
expect(reconcile.run).toContain('EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --report "$dir"');
|
||||
// Every arm's report counts: one red arm fails the census.
|
||||
expect(reconcile.run).toContain('if [ "$code" -ne 0 ]; then status=$code; fi');
|
||||
expect(reconcile.run).toContain('echo "exit=$status" >> "$GITHUB_OUTPUT"');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,7 @@
|
||||
{"type":"thread.started","thread_id":"01a150b3-ae5a-7932-ba51-4c295c66adf4"}
|
||||
{"type":"turn.started"}
|
||||
{"type":"item.completed","item":{"id":"item_0","type":"agent_message","text":"I’ll write and overwrite `hello.txt` with `printf`, then read it with `cat`.\n"}}
|
||||
{"type":"item.started","item":{"id":"item_1","type":"command_execution","command":"/bin/bash -lc \"printf 'hi' > hello.txt\nprintf 'hello' > hello.txt\ncat hello.txt\"","aggregated_output":"","exit_code":null,"status":"in_progress"}}
|
||||
{"type":"item.completed","item":{"id":"item_1","type":"command_execution","command":"/bin/bash -lc \"printf 'hi' > hello.txt\nprintf 'hello' > hello.txt\ncat hello.txt\"","aggregated_output":"hello","exit_code":0,"status":"completed"}}
|
||||
{"type":"item.completed","item":{"id":"item_2","type":"agent_message","text":"`cat hello.txt` output: `hello`"}}
|
||||
{"type":"turn.completed","usage":{"input_tokens":25009,"cached_input_tokens":12437,"cache_write_input_tokens":12566,"output_tokens":86,"reasoning_output_tokens":0}}
|
||||
@@ -0,0 +1,9 @@
|
||||
{"type":"thread.started","thread_id":"01a150b3-be24-7f22-aa81-7eb4f79018e0"}
|
||||
{"type":"turn.started"}
|
||||
{"type":"item.completed","item":{"id":"item_0","type":"agent_message","text":"I’ll create `notes.md` with `draft`, then change it to `final` using two patches.\n"}}
|
||||
{"type":"item.started","item":{"id":"item_1","type":"file_change","changes":[{"path":"/tmp/codex-host-fixture/notes.md","kind":"add"}],"status":"in_progress"}}
|
||||
{"type":"item.completed","item":{"id":"item_1","type":"file_change","changes":[{"path":"/tmp/codex-host-fixture/notes.md","kind":"add"}],"status":"completed"}}
|
||||
{"type":"item.started","item":{"id":"item_2","type":"file_change","changes":[{"path":"/tmp/codex-host-fixture/notes.md","kind":"update"}],"status":"in_progress"}}
|
||||
{"type":"item.completed","item":{"id":"item_2","type":"file_change","changes":[{"path":"/tmp/codex-host-fixture/notes.md","kind":"update"}],"status":"completed"}}
|
||||
{"type":"item.completed","item":{"id":"item_3","type":"agent_message","text":"Created `notes.md` with `draft`, then changed it to `final` using two `apply_patch` calls."}}
|
||||
{"type":"turn.completed","usage":{"input_tokens":25020,"cached_input_tokens":12440,"cache_write_input_tokens":12574,"output_tokens":137,"reasoning_output_tokens":0}}
|
||||
@@ -11,9 +11,14 @@
|
||||
*
|
||||
* Lives in its own module (not e2e-helpers) so the runner can import it
|
||||
* without dragging in bun:test.
|
||||
*
|
||||
* On the Codex host (EVALS_HOST=codex) Codex is the agent and no session
|
||||
* spawns `claude`, so the ping is skipped; Codex API failures surface per
|
||||
* session as error_api.
|
||||
*/
|
||||
|
||||
import { spawnSync } from 'child_process';
|
||||
import { resolveEvalHost } from './e2e-hosts';
|
||||
|
||||
export type PreflightResult = 'skipped' | 'ok';
|
||||
|
||||
@@ -21,7 +26,7 @@ export function preflightAnthropicApi(
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
spawn: typeof spawnSync = spawnSync,
|
||||
): PreflightResult {
|
||||
if (env.EVALS_PREFLIGHT_OK === '1') return 'skipped';
|
||||
if (env.EVALS_PREFLIGHT_OK === '1' || resolveEvalHost(env) === 'codex') return 'skipped';
|
||||
const check = spawn(
|
||||
'sh',
|
||||
['-c', 'echo "ping" | claude -p --max-turns 1 --output-format stream-json --verbose --dangerously-skip-permissions'],
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/** One terminal record per Codex eval attempt, using the assertions as the oracle. */
|
||||
import { CODEX_DRAIN_GRACE_MS, CodexHarnessError, type CodexResult } from './codex-session-runner';
|
||||
import { CODEX_DRAIN_GRACE_MS, CodexHarnessError, codexUsageCostUsd, type CodexResult } from './codex-session-runner';
|
||||
import { EvalCollector, getProjectEvalDir, shardSlugOfEvalDir, type EvalTestEntry } from './eval-store';
|
||||
import * as path from 'node:path';
|
||||
import * as fs from 'node:fs';
|
||||
@@ -198,10 +198,13 @@ export async function runRecordedCodexEval(opts: CodexEvalOptions): Promise<Code
|
||||
: 'validation_failed';
|
||||
const message = failure instanceof Error ? failure.message : String(failure);
|
||||
const diagnostic = result?.stderr && !message.includes(result.stderr) ? `${message}\n${result.stderr}` : message;
|
||||
const model = result?.model ?? opts.model;
|
||||
const cost = result?.usage ? codexUsageCostUsd(result.usage, model) : null;
|
||||
appendSessionLedger({ key: sessionKey('codex', opts.name), test_name: opts.name, runner: 'codex', started_at: new Date(started).toISOString(),
|
||||
budget_ms: opts.budgetMs, elapsed_ms: performance.now() - startMono,
|
||||
end: exitReason === 'timeout' ? 'session_timeout' : passed || exitReason === 'validation_failed' ? 'completed' : 'error',
|
||||
...(exitReason === 'timeout' ? { evidence: `armed ${opts.budgetMs}ms Codex budget expired` } : {}), billed: false });
|
||||
...(exitReason === 'timeout' ? { evidence: `armed ${opts.budgetMs}ms Codex budget expired` } : {}), billed: cost !== null,
|
||||
...(model ? { model } : {}), ...(cost !== null ? { cost_usd: cost, tokens: result!.usage! } : {}) });
|
||||
|
||||
// addTest appends retry attempts. Never pre-record a failure and then
|
||||
// overwrite it: that would manufacture two attempts from one execution.
|
||||
@@ -211,14 +214,15 @@ export async function runRecordedCodexEval(opts: CodexEvalOptions): Promise<Code
|
||||
tier: 'e2e',
|
||||
passed,
|
||||
duration_ms: failure instanceof CodexEvalTimeout ? Date.now() - started : result?.durationMs ?? Date.now() - started,
|
||||
cost_usd: 0, cost_known: false,
|
||||
cost_usd: cost ?? 0, ...(cost === null ? { cost_known: false } : {}),
|
||||
...(result?.usage ? { token_usage: result.usage } : {}),
|
||||
...(result ? {
|
||||
output: result.output.slice(0, opts.outputLimit ?? 2000),
|
||||
turns_used: result.toolCalls.length,
|
||||
tokens_used: result.tokens,
|
||||
last_tool_call: result.toolCalls.at(-1),
|
||||
} : {}),
|
||||
...(opts.model ? { model: opts.model } : {}),
|
||||
...(model ? { model } : {}),
|
||||
exit_reason: exitReason,
|
||||
...(failure !== undefined ? { error: diagnostic }
|
||||
: result?.stderr ? { error: result.stderr } : {}),
|
||||
|
||||
@@ -0,0 +1,306 @@
|
||||
/**
|
||||
* The Codex host for runSkillTest (EVALS_HOST=codex, GSTA-49 TM2): runs a
|
||||
* Claude-authored gate case through `codex exec --json` and returns the same
|
||||
* SkillTestResult shape, so the case's own assertions judge the outcome.
|
||||
*
|
||||
* Parity with the Claude runner (session-runner.ts):
|
||||
* - runtime: the full gstack Codex runtime (`./setup --host codex`), installed
|
||||
* once per process into a private CODEX_HOME holding the login's auth.json;
|
||||
* - events: command_execution -> Bash ({ command } unwrapped from Codex's
|
||||
* `bash -lc`, its output); file_change -> Write (add), Edit (update) or
|
||||
* Delete per path; web_search -> WebSearch; mcp_tool_call -> mcp__server__tool;
|
||||
* the last agent message is the output;
|
||||
* - transcript: the Claude stream-json shape the cases' observers read: one
|
||||
* system/init event (cwd, the Codex thread id as session_id), assistant text
|
||||
* and tool_use events, user tool_result events (is_error from the exit code)
|
||||
* and a final result event;
|
||||
* - sandbox: GSTACK_CODEX_SANDBOX, default danger-full-access (the counterpart
|
||||
* of claude -p --dangerously-skip-permissions; cases write outside their
|
||||
* working directory, start loopback servers and browsers); workspace-write
|
||||
* gets network access;
|
||||
* - effort: GSTACK_CODEX_EFFORT sets model_reasoning_effort; unset keeps the
|
||||
* model's default;
|
||||
* - model: a non-Claude options.model, else GSTACK_CODEX_MODEL, else
|
||||
* CODEX_FRONTIER_MODEL; a case's Claude model pin does not apply here;
|
||||
* - cost: real accounting from `turn.completed` usage at lib/pricing.ts rates,
|
||||
* recorded on the result, the eval record and the session ledger.
|
||||
* - nativeLifecycle: onSpawn gets the codex process-group owner right after
|
||||
* spawn; onSettled runs after its exit and group kill, within the drain grace.
|
||||
* Not translated: maxTurns (Codex has no turn cap), allowedTools/tools (Codex
|
||||
* has its own tool surface) and startupGraceMs (the timeout bounds the session).
|
||||
*/
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { spawnSync } from 'child_process';
|
||||
import { CODEX_DRAIN_GRACE_MS, CodexHarnessError, codexUsageCostUsd, execCodex, type CodexResult } from './codex-session-runner';
|
||||
import { getProjectEvalDir } from './eval-store';
|
||||
import { hermeticChildEnv } from './hermetic-env';
|
||||
import { appendSessionLedger, sessionKey, type SessionEnd } from './session-ledger';
|
||||
import { sanitizeTestName, scanBrowseErrors, type SkillTestResult } from './session-runner';
|
||||
import { CODEX_FRONTIER_MODEL } from '../../scripts/resolvers/constants';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..', '..');
|
||||
const SETUP_TIMEOUT_MS = 300_000;
|
||||
export const CODEX_SANDBOXES = ['read-only', 'workspace-write', 'danger-full-access'] as const;
|
||||
const EFFORT = /^[a-z]+$/;
|
||||
|
||||
/** The Codex model for a session: a Claude pin never reaches `codex --model`. */
|
||||
export function codexHostModel(explicit: string | undefined, env: NodeJS.ProcessEnv = process.env): string {
|
||||
if (explicit && !/^claude-/.test(explicit)) return explicit;
|
||||
return env.GSTACK_CODEX_MODEL || CODEX_FRONTIER_MODEL;
|
||||
}
|
||||
|
||||
/** `codex exec` arguments for one session; the prompt is the last argument. */
|
||||
export function codexHostExecArgs(model: string, prompt: string, env: NodeJS.ProcessEnv = process.env): string[] {
|
||||
const sandbox = env.GSTACK_CODEX_SANDBOX || 'danger-full-access';
|
||||
if (!(CODEX_SANDBOXES as readonly string[]).includes(sandbox)) {
|
||||
throw new Error(`GSTACK_CODEX_SANDBOX must be one of ${CODEX_SANDBOXES.join(', ')}; got "${sandbox}"`);
|
||||
}
|
||||
const effort = env.GSTACK_CODEX_EFFORT;
|
||||
if (effort && !EFFORT.test(effort)) throw new Error(`GSTACK_CODEX_EFFORT must be a bare effort name; got "${effort}"`);
|
||||
return ['exec', '--json', '-s', sandbox, '--skip-git-repo-check', '--model', model,
|
||||
...(sandbox === 'workspace-write' ? ['-c', 'sandbox_workspace_write.network_access=true'] : []),
|
||||
...(effort ? ['-c', `model_reasoning_effort="${effort}"`] : []),
|
||||
prompt];
|
||||
}
|
||||
|
||||
/** The command inside Codex's `/bin/bash -lc <word>` wrapper, else the command unchanged. */
|
||||
export function unwrapShellCommand(command: string): string {
|
||||
const match = /^(?:\/(?:usr\/)?bin\/)?(?:ba|z)?sh -lc (.+)$/s.exec(command);
|
||||
if (!match) return command;
|
||||
const word = match[1]!;
|
||||
let out = '';
|
||||
let i = 0;
|
||||
while (i < word.length) {
|
||||
const c = word[i]!;
|
||||
if (c === "'") {
|
||||
const end = word.indexOf("'", i + 1);
|
||||
if (end < 0) return command;
|
||||
out += word.slice(i + 1, end);
|
||||
i = end + 1;
|
||||
} else if (c === '"') {
|
||||
i++;
|
||||
let closed = false;
|
||||
while (i < word.length) {
|
||||
if (word[i] === '\\' && i + 1 < word.length && '"\\$`\n'.includes(word[i + 1]!)) { out += word[i + 1]; i += 2; }
|
||||
else if (word[i] === '"') { i++; closed = true; break; }
|
||||
else out += word[i++];
|
||||
}
|
||||
if (!closed) return command;
|
||||
} else if (c === '\\' && i + 1 < word.length) {
|
||||
out += word[i + 1];
|
||||
i += 2;
|
||||
} else if (/\s/.test(c)) {
|
||||
return command;
|
||||
} else {
|
||||
out += c;
|
||||
i++;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export type HostToolCall = SkillTestResult['toolCalls'][number];
|
||||
|
||||
const FILE_CHANGE_TOOLS: Record<string, string> = { add: 'Write', update: 'Edit', delete: 'Delete' };
|
||||
|
||||
/** Claude-shaped tool calls from completed Codex items, in order. */
|
||||
export function codexHostToolCalls(rawLines: readonly string[]): HostToolCall[] {
|
||||
const calls: HostToolCall[] = [];
|
||||
for (const line of rawLines) {
|
||||
let event: any;
|
||||
try { event = JSON.parse(line); } catch { continue; }
|
||||
if (event?.type !== 'item.completed' || !event.item) continue;
|
||||
const item = event.item;
|
||||
if (item.type === 'command_execution') {
|
||||
calls.push({ tool: 'Bash', input: { command: unwrapShellCommand(String(item.command ?? '')) }, output: String(item.aggregated_output ?? '') });
|
||||
} else if (item.type === 'file_change') {
|
||||
for (const change of Array.isArray(item.changes) ? item.changes : []) {
|
||||
calls.push({ tool: FILE_CHANGE_TOOLS[change?.kind] ?? 'Edit', input: { file_path: String(change?.path ?? ''), kind: change?.kind }, output: '' });
|
||||
}
|
||||
} else if (item.type === 'web_search') {
|
||||
calls.push({ tool: 'WebSearch', input: { query: item.query }, output: '' });
|
||||
} else if (item.type === 'mcp_tool_call') {
|
||||
calls.push({ tool: `mcp__${item.server}__${item.tool}`, input: item.arguments ?? {}, output: typeof item.result === 'string' ? item.result : JSON.stringify(item.result ?? '') });
|
||||
}
|
||||
}
|
||||
return calls;
|
||||
}
|
||||
|
||||
/**
|
||||
* Codex events as the Claude stream-json transcript the cases' observers read
|
||||
* (system/init, assistant text and tool_use, user tool_result, result).
|
||||
*/
|
||||
export function codexHostTranscript(rawLines: readonly string[], cwd: string, output: string): any[] {
|
||||
const events: any[] = [];
|
||||
let session = '';
|
||||
const say = (role: 'assistant' | 'user', content: any[]) => events.push({ type: role, session_id: session, parent_tool_use_id: null, message: { role, content } });
|
||||
for (const line of rawLines) {
|
||||
let event: any;
|
||||
try { event = JSON.parse(line); } catch { continue; }
|
||||
if (event?.type === 'thread.started') {
|
||||
session = String(event.thread_id ?? '');
|
||||
events.push({ type: 'system', subtype: 'init', cwd, session_id: session, tools: [] });
|
||||
continue;
|
||||
}
|
||||
if (event?.type !== 'item.completed' || !event.item) continue;
|
||||
const item = event.item;
|
||||
const id = String(item.id ?? `item_${events.length}`);
|
||||
if (item.type === 'agent_message' && item.text) say('assistant', [{ type: 'text', text: String(item.text) }]);
|
||||
else if (item.type === 'command_execution') {
|
||||
say('assistant', [{ type: 'tool_use', id, name: 'Bash', input: { command: unwrapShellCommand(String(item.command ?? '')) } }]);
|
||||
say('user', [{ type: 'tool_result', tool_use_id: id, content: String(item.aggregated_output ?? ''), is_error: item.exit_code !== 0 }]);
|
||||
} else if (item.type === 'file_change') {
|
||||
(Array.isArray(item.changes) ? item.changes : []).forEach((change: any, index: number) => {
|
||||
const useId = `${id}_${index}`;
|
||||
say('assistant', [{ type: 'tool_use', id: useId, name: FILE_CHANGE_TOOLS[change?.kind] ?? 'Edit', input: { file_path: String(change?.path ?? '') } }]);
|
||||
say('user', [{ type: 'tool_result', tool_use_id: useId, content: '', is_error: item.status !== 'completed' }]);
|
||||
});
|
||||
}
|
||||
}
|
||||
events.push({ type: 'result', subtype: 'success', session_id: session, result: output });
|
||||
return events;
|
||||
}
|
||||
|
||||
let runtimeRoot: string | undefined;
|
||||
|
||||
/**
|
||||
* Install the full gstack Codex runtime once per process: `./setup --host codex`
|
||||
* into a private HOME/CODEX_HOME, with the login home's auth.json copied in.
|
||||
* Returns the root holding home/.codex and tmp/ (Codex will not create its
|
||||
* helper aliases when CODEX_HOME sits under its own TMPDIR). A caller that
|
||||
* already prepared one passes it as GSTACK_CODEX_HOST_ROOT.
|
||||
*/
|
||||
export function prepareCodexHostRuntime(env: NodeJS.ProcessEnv = process.env): string {
|
||||
if (runtimeRoot) return runtimeRoot;
|
||||
const preset = env.GSTACK_CODEX_HOST_ROOT;
|
||||
if (preset) {
|
||||
if (!fs.existsSync(path.join(preset, 'home', '.codex'))) throw new Error(`GSTACK_CODEX_HOST_ROOT ${preset} has no home/.codex`);
|
||||
fs.mkdirSync(path.join(preset, 'tmp'), { recursive: true });
|
||||
return runtimeRoot = preset;
|
||||
}
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'codex-host-'));
|
||||
const home = path.join(root, 'home');
|
||||
const codexHome = path.join(home, '.codex');
|
||||
const tmp = path.join(root, 'tmp');
|
||||
for (const dir of [codexHome, tmp]) fs.mkdirSync(dir, { recursive: true });
|
||||
const auth = path.join(env.CODEX_HOME || path.join(os.homedir(), '.codex'), 'auth.json');
|
||||
if (fs.existsSync(auth)) fs.copyFileSync(auth, path.join(codexHome, 'auth.json'));
|
||||
const setup = spawnSync(path.join(ROOT, 'setup'), ['--host', 'codex'], {
|
||||
cwd: ROOT, encoding: 'utf8', timeout: SETUP_TIMEOUT_MS,
|
||||
env: hermeticChildEnv({ HOME: home, CODEX_HOME: codexHome, TMPDIR: tmp }, { extraAllow: ['OPENAI_API_KEY', 'CODEX_*'] }),
|
||||
});
|
||||
if (setup.status !== 0) {
|
||||
fs.rmSync(root, { recursive: true, force: true });
|
||||
throw new Error(`./setup --host codex failed (exit ${setup.status}${setup.error ? `, ${setup.error.message}` : ''}): ${(setup.stderr || setup.stdout || '').slice(-2000)}`);
|
||||
}
|
||||
process.once('exit', () => { try { fs.rmSync(root, { recursive: true, force: true }); } catch { /* best effort */ } });
|
||||
runtimeRoot = root;
|
||||
return root;
|
||||
}
|
||||
|
||||
type RunSkillTestOptions = Parameters<typeof import('./session-runner').runSkillTest>[0];
|
||||
|
||||
function exitReasonOf(result: CodexResult, aborted: boolean): string {
|
||||
if (result.exitCode === 124) return 'timeout';
|
||||
const failedTurn = result.rawLines.some(line => /"type":"(?:turn\.failed|error)"/.test(line));
|
||||
if (result.exitCode === 0) return failedTurn ? 'error_api' : 'success';
|
||||
if (aborted) return 'timeout';
|
||||
return failedTurn && !result.rawLines.some(line => line.includes('"type":"item.completed"')) ? 'error_api' : `exit_code_${result.exitCode}`;
|
||||
}
|
||||
|
||||
const SESSION_ENDS: Record<string, SessionEnd> = { success: 'completed', timeout: 'session_timeout', error_api: 'api_error' };
|
||||
|
||||
/** runSkillTest on the Codex host. `appendSystemPrompt` arrives with any completion notice already applied. */
|
||||
export async function runCodexHostSkillTest(options: RunSkillTestOptions): Promise<SkillTestResult> {
|
||||
const startTime = Date.now();
|
||||
const startMono = performance.now();
|
||||
const { workingDirectory, timeout = 120_000, testName, runId, signal } = options;
|
||||
const deadline = startTime + timeout;
|
||||
const model = codexHostModel(options.model);
|
||||
const prompt = options.appendSystemPrompt ? `${options.appendSystemPrompt}\n\n${options.prompt}` : options.prompt;
|
||||
const root = prepareCodexHostRuntime();
|
||||
const safeName = testName ? sanitizeTestName(testName) : null;
|
||||
let runDir: string | null = null;
|
||||
if (runId) {
|
||||
try {
|
||||
runDir = path.join(path.dirname(getProjectEvalDir()), 'e2e-runs', runId);
|
||||
fs.mkdirSync(runDir, { recursive: true });
|
||||
} catch { runDir = null; }
|
||||
}
|
||||
let firstResponseMs = 0;
|
||||
let lastItemMs = 0;
|
||||
let maxInterTurnMs = 0;
|
||||
const onLine = (line: string, elapsedMs: number) => {
|
||||
if (!firstResponseMs) firstResponseMs = Math.max(1, elapsedMs);
|
||||
if (runDir && safeName) { try { fs.appendFileSync(path.join(runDir, `${safeName}.ndjson`), line + '\n'); } catch { /* non-fatal */ } }
|
||||
if (!line.includes('"type":"item.completed"') || !/"type":"(?:command_execution|file_change)"/.test(line)) return;
|
||||
if (lastItemMs) maxInterTurnMs = Math.max(maxInterTurnMs, elapsedMs - lastItemMs);
|
||||
lastItemMs = elapsedMs;
|
||||
};
|
||||
|
||||
let result: CodexResult;
|
||||
let harnessReason: string | undefined;
|
||||
let started = true;
|
||||
const lifecycle = options.nativeLifecycle;
|
||||
try {
|
||||
result = await execCodex(codexHostExecArgs(model, prompt), {
|
||||
cwd: workingDirectory, startTime, deadline, signal, onLine, ...(lifecycle ? { onSpawn: (pid: number) => lifecycle.onSpawn(pid) } : {}),
|
||||
env: hermeticChildEnv(
|
||||
{ GSTACK_HEADLESS: '1', ...options.env, CODEX_HOME: path.join(root, 'home', '.codex'), TMPDIR: options.env?.TMPDIR ?? path.join(root, 'tmp') },
|
||||
{ extraAllow: ['OPENAI_API_KEY', 'CODEX_*'] },
|
||||
),
|
||||
});
|
||||
} catch (error) {
|
||||
if (!(error instanceof CodexHarnessError) || !error.result) throw error;
|
||||
result = error.result;
|
||||
started = !error.message.startsWith('Could not start Codex');
|
||||
harnessReason = /drain/.test(error.message) ? 'error_output_drain' : 'error_output_stream';
|
||||
}
|
||||
if (lifecycle) {
|
||||
const settleDeadline = Date.now() + CODEX_DRAIN_GRACE_MS;
|
||||
let hookTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
try {
|
||||
await Promise.race([
|
||||
lifecycle.onSettled({ deadline: settleDeadline, exited: started }),
|
||||
new Promise<never>((_, reject) => { hookTimer = setTimeout(() => reject(new Error('native lifecycle settlement deadline exceeded')), CODEX_DRAIN_GRACE_MS); }),
|
||||
]);
|
||||
} finally { clearTimeout(hookTimer); }
|
||||
}
|
||||
|
||||
const exitReason = harnessReason ?? exitReasonOf(result, !!signal?.aborted);
|
||||
const toolCalls = codexHostToolCalls(result.rawLines);
|
||||
const events = result.rawLines.flatMap(line => { try { return [JSON.parse(line)]; } catch { return []; } });
|
||||
const messages = events.filter(event => event?.type === 'item.completed' && event.item?.type === 'agent_message');
|
||||
const output = String(messages.at(-1)?.item?.text ?? '');
|
||||
const turnsUsed = events.filter(event => event?.type === 'item.completed' && event.item?.type !== 'reasoning').length;
|
||||
const transcript = codexHostTranscript(result.rawLines, workingDirectory, output);
|
||||
const cost = result.usage ? codexUsageCostUsd(result.usage, model) : null;
|
||||
const browseErrors = scanBrowseErrors(toolCalls.filter(call => call.tool === 'Bash').map(call => call.output).join('\n') + '\n' + result.stderr);
|
||||
const duration = Date.now() - startTime;
|
||||
|
||||
appendSessionLedger({ key: sessionKey('codex', testName), ...(testName ? { test_name: testName } : {}), runner: 'codex',
|
||||
started_at: new Date(startTime).toISOString(), budget_ms: timeout, elapsed_ms: performance.now() - startMono,
|
||||
end: exitReason === 'timeout' && signal?.aborted ? 'aborted' : SESSION_ENDS[exitReason] ?? 'error',
|
||||
...(exitReason === 'timeout' ? { evidence: `armed ${timeout}ms session timeout fired` } : exitReason !== 'success' && result.stderr ? { evidence: result.stderr } : {}),
|
||||
billed: cost !== null, model, ...(cost !== null ? { cost_usd: cost, tokens: result.usage! } : {}) });
|
||||
|
||||
if (exitReason !== 'success' || browseErrors.length > 0) {
|
||||
try {
|
||||
const failureDir = runDir || path.join(workingDirectory, '.gstack', 'test-transcripts');
|
||||
fs.mkdirSync(failureDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(failureDir, safeName ? `${safeName}-failure.json` : `e2e-${new Date().toISOString().replace(/[:.]/g, '-')}.json`),
|
||||
JSON.stringify({ host: 'codex', model, prompt: prompt.slice(0, 500), testName: testName || 'unknown', exitReason, browseErrors, duration,
|
||||
stderr: result.stderr.slice(0, 2000), output: output.slice(0, 500) }, null, 2));
|
||||
} catch { /* non-fatal */ }
|
||||
}
|
||||
|
||||
return {
|
||||
toolCalls, browseErrors, exitReason, duration, output, transcript, model, firstResponseMs, maxInterTurnMs,
|
||||
costEstimate: { inputChars: prompt.length, outputChars: output.length,
|
||||
estimatedTokens: (result.usage?.input ?? 0) + (result.usage?.output ?? 0), estimatedCost: cost ?? 0, turnsUsed },
|
||||
...(result.usage ? { tokenUsage: result.usage } : {}),
|
||||
...(cost === null ? { costKnown: false } : {}),
|
||||
};
|
||||
}
|
||||
@@ -21,6 +21,7 @@ import { hermeticChildEnv } from './hermetic-env';
|
||||
import { extractSkillSections } from './skill-fixture';
|
||||
import { killProcessGroup } from '../../scripts/test-strict-output';
|
||||
import { CODEX_FRONTIER_MODEL } from '../../scripts/resolvers/constants';
|
||||
import { estimateCostUsd, hasPricing } from '../../lib/pricing';
|
||||
|
||||
// --- Interfaces ---
|
||||
|
||||
@@ -34,6 +35,47 @@ export interface CodexResult {
|
||||
sessionId: string | null; // Thread ID for session continuity
|
||||
rawLines: string[]; // Raw JSONL lines for debugging
|
||||
stderr: string; // Stderr output (skill loading errors, auth failures)
|
||||
usage?: CodexTokenUsage | null; // Summed turn.completed usage; null when no turn completed
|
||||
model?: string; // The --model this session ran
|
||||
}
|
||||
|
||||
/**
|
||||
* Token classes of Codex `turn.completed` usage. `cached_input` (cache reads)
|
||||
* and `cache_write_input` are subsets of `input`; `reasoning_output` is a
|
||||
* subset of `output`.
|
||||
*/
|
||||
export interface CodexTokenUsage {
|
||||
input: number;
|
||||
cached_input: number;
|
||||
cache_write_input: number;
|
||||
output: number;
|
||||
reasoning_output: number;
|
||||
}
|
||||
|
||||
export const NO_CODEX_USAGE: CodexTokenUsage = Object.freeze({ input: 0, cached_input: 0, cache_write_input: 0, output: 0, reasoning_output: 0 });
|
||||
|
||||
const count = (value: unknown) => (typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : 0);
|
||||
|
||||
/** Add one `turn.completed` usage object to a running total. */
|
||||
export function addCodexTurnUsage(total: CodexTokenUsage, usage: Record<string, unknown> | undefined): CodexTokenUsage {
|
||||
return {
|
||||
input: total.input + count(usage?.input_tokens),
|
||||
cached_input: total.cached_input + count(usage?.cached_input_tokens),
|
||||
cache_write_input: total.cache_write_input + count(usage?.cache_write_input_tokens),
|
||||
output: total.output + count(usage?.output_tokens),
|
||||
reasoning_output: total.reasoning_output + count(usage?.reasoning_output_tokens),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A session's cost at lib/pricing.ts rates: cache reads at the cached-input
|
||||
* rate, all other input (cache writes included) at the input rate, output at
|
||||
* the output rate. Null when the model has no price row.
|
||||
*/
|
||||
export function codexUsageCostUsd(usage: CodexTokenUsage, model: string | undefined): number | null {
|
||||
if (!hasPricing(model)) return null;
|
||||
const cached = Math.min(usage.cached_input, usage.input);
|
||||
return estimateCostUsd({ input: usage.input - cached, cached, output: usage.output }, model);
|
||||
}
|
||||
|
||||
/** Existing pipe-drain allowance, separate from the model's work budget. */
|
||||
@@ -54,6 +96,7 @@ export interface ParsedCodexJSONL {
|
||||
toolCalls: string[];
|
||||
tokens: number;
|
||||
sessionId: string | null;
|
||||
usage: CodexTokenUsage | null;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -71,6 +114,7 @@ export function parseCodexJSONL(lines: string[]): ParsedCodexJSONL {
|
||||
const toolCalls: string[] = [];
|
||||
let tokens = 0;
|
||||
let sessionId: string | null = null;
|
||||
let usage: CodexTokenUsage | null = null;
|
||||
|
||||
for (const line of lines) {
|
||||
if (!line.trim()) continue;
|
||||
@@ -95,9 +139,9 @@ export function parseCodexJSONL(lines: string[]): ParsedCodexJSONL {
|
||||
if (cmd) toolCalls.push(cmd);
|
||||
}
|
||||
} else if (t === 'turn.completed') {
|
||||
const usage = obj.usage || {};
|
||||
const turnTokens = (usage.input_tokens || 0) + (usage.output_tokens || 0);
|
||||
tokens += turnTokens;
|
||||
const turn = obj.usage || {};
|
||||
tokens += (turn.input_tokens || 0) + (turn.output_tokens || 0);
|
||||
usage = addCodexTurnUsage(usage ?? NO_CODEX_USAGE, turn);
|
||||
}
|
||||
} catch { /* skip malformed lines */ }
|
||||
}
|
||||
@@ -108,6 +152,7 @@ export function parseCodexJSONL(lines: string[]): ParsedCodexJSONL {
|
||||
toolCalls,
|
||||
tokens,
|
||||
sessionId,
|
||||
usage,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -166,6 +211,192 @@ export function installSkillToTempHome(
|
||||
return home;
|
||||
}
|
||||
|
||||
// --- Spawn core ---
|
||||
|
||||
/**
|
||||
* Spawn `codex <args>` in its own process group and collect its JSONL until
|
||||
* exit, the deadline or an abort, with the existing drain grace. Shared by
|
||||
* runCodexSkill and the Codex host (codex-host-runner.ts). `onLine` sees each
|
||||
* JSONL line as it arrives with the elapsed milliseconds since `startTime`.
|
||||
*/
|
||||
export async function execCodex(args: string[], opts: {
|
||||
cwd: string;
|
||||
env: Record<string, string>;
|
||||
startTime: number;
|
||||
deadline: number;
|
||||
signal?: AbortSignal;
|
||||
onLine?: (line: string, elapsedMs: number) => void;
|
||||
/** Called with the group-owning pid right after spawn; a throw kills the group. */
|
||||
onSpawn?: (pid: number) => void;
|
||||
}): Promise<CodexResult> {
|
||||
const { cwd, env, startTime, deadline, signal, onLine } = opts;
|
||||
const proc = spawn('codex', args, {
|
||||
cwd,
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
detached: process.platform !== 'win32',
|
||||
env,
|
||||
});
|
||||
if (opts.onSpawn) {
|
||||
try {
|
||||
if (proc.pid === undefined) throw new Error('Codex did not start');
|
||||
opts.onSpawn(proc.pid);
|
||||
} catch (error) {
|
||||
killProcessGroup(proc, 'SIGKILL');
|
||||
proc.stdout!.destroy();
|
||||
proc.stderr!.destroy();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
const collectedLines: string[] = [];
|
||||
// work --exit--> drain (<=5s) --> result
|
||||
// \--timeout/abort--> kill group + close pipes --> timeout result
|
||||
// Exit status comes from 'exit'; a forced drain after exit 0 is an error.
|
||||
let stdoutBuffer = '';
|
||||
let stderr = '';
|
||||
let exitCode: number | undefined;
|
||||
let timedOut = false;
|
||||
let drainExpired = false;
|
||||
let streamError: { stream: 'stdout' | 'stderr'; error: Error } | undefined;
|
||||
let spawnError: Error | undefined;
|
||||
let stdoutDone = false;
|
||||
let stderrDone = false;
|
||||
let stdoutEnded = false;
|
||||
let stderrEnded = false;
|
||||
let finalized = false;
|
||||
const stdoutDecoder = new StringDecoder('utf8');
|
||||
const stderrDecoder = new StringDecoder('utf8');
|
||||
let workTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
let drainTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
let finish!: () => void;
|
||||
const finished = new Promise<void>((resolve) => { finish = resolve; });
|
||||
|
||||
const maybeFinish = () => {
|
||||
if (exitCode !== undefined && stdoutDone && stderrDone) finish();
|
||||
};
|
||||
const closePipes = () => {
|
||||
// Destroy both streams: canceling stdout alone cannot release an
|
||||
// inherited stderr pipe held by a descendant in another process group.
|
||||
proc.stdout!.destroy();
|
||||
proc.stderr!.destroy();
|
||||
};
|
||||
const expireDrain = () => {
|
||||
drainExpired = !stdoutDone || !stderrDone;
|
||||
killProcessGroup(proc, 'SIGKILL');
|
||||
closePipes();
|
||||
// Also bound the rare case where a failed group kill produces no exit.
|
||||
finish();
|
||||
};
|
||||
const stopRun = () => {
|
||||
// A real auth/crash exit that preceded a pipe stall stays a real exit.
|
||||
if (exitCode === undefined) timedOut = true;
|
||||
else drainExpired ||= !stdoutDone || !stderrDone;
|
||||
killProcessGroup(proc, 'SIGKILL');
|
||||
closePipes();
|
||||
clearTimeout(drainTimer);
|
||||
drainTimer = setTimeout(expireDrain, CODEX_DRAIN_GRACE_MS);
|
||||
maybeFinish();
|
||||
};
|
||||
const onExit = (code: number | null, exitSignal: NodeJS.Signals | null) => {
|
||||
exitCode = code ?? (exitSignal ? 128 + (os.constants.signals[exitSignal] ?? 0) : 1);
|
||||
clearTimeout(workTimer);
|
||||
clearTimeout(drainTimer);
|
||||
// 'exit' means the child is gone; 'close' also waits for descendant
|
||||
// pipes. Start the drain allowance at actual exit, never at close.
|
||||
drainTimer = setTimeout(expireDrain, CODEX_DRAIN_GRACE_MS);
|
||||
maybeFinish();
|
||||
};
|
||||
const onSpawnError = (error: Error) => {
|
||||
if (finalized) return;
|
||||
spawnError = error;
|
||||
exitCode = 1;
|
||||
closePipes();
|
||||
finish();
|
||||
};
|
||||
// Closure releases lifecycle waits, but only 'end' proves all bytes were
|
||||
// drained. A destroyed pipe can emit 'close' without either EOF or error.
|
||||
const onStdoutDone = () => { if (!finalized) { stdoutDone = true; maybeFinish(); } };
|
||||
const onStderrDone = () => { if (!finalized) { stderrDone = true; maybeFinish(); } };
|
||||
const onStdoutEnd = () => { if (!finalized) { stdoutBuffer += stdoutDecoder.end(); stdoutEnded = true; onStdoutDone(); } };
|
||||
const onStderrEnd = () => { if (!finalized) { stderr += stderrDecoder.end(); stderrEnded = true; onStderrDone(); } };
|
||||
const onStreamError = (stream: 'stdout' | 'stderr', error: Error) => {
|
||||
if (!finalized) streamError ??= { stream, error };
|
||||
};
|
||||
const onStdout = (chunk: Buffer) => {
|
||||
if (finalized) return;
|
||||
stdoutBuffer += stdoutDecoder.write(chunk);
|
||||
const lines = stdoutBuffer.split('\n');
|
||||
stdoutBuffer = lines.pop() || '';
|
||||
for (const line of lines) {
|
||||
if (!line.trim()) continue;
|
||||
collectedLines.push(line);
|
||||
onLine?.(line, Date.now() - startTime);
|
||||
try {
|
||||
const event = JSON.parse(line);
|
||||
if (event.type === 'item.completed' && event.item) {
|
||||
const item = event.item;
|
||||
const elapsed = Math.round((Date.now() - startTime) / 1000);
|
||||
if (item.type === 'command_execution' && item.command) {
|
||||
process.stderr.write(` [codex ${elapsed}s] ran: ${item.command.slice(0, 100)}\n`);
|
||||
} else if (item.type === 'agent_message' && item.text) {
|
||||
process.stderr.write(` [codex ${elapsed}s] message: ${item.text.slice(0, 100)}\n`);
|
||||
}
|
||||
}
|
||||
} catch { /* malformed JSONL is ignored by parseCodexJSONL too */ }
|
||||
}
|
||||
};
|
||||
const onStderr = (chunk: Buffer) => { if (!finalized) stderr += stderrDecoder.write(chunk); };
|
||||
|
||||
proc.on('exit', onExit);
|
||||
proc.on('error', onSpawnError);
|
||||
proc.stdout!.on('data', onStdout);
|
||||
proc.stderr!.on('data', onStderr);
|
||||
proc.stdout!.on('end', onStdoutEnd).on('close', onStdoutDone).on('error', error => onStreamError('stdout', error));
|
||||
proc.stderr!.on('end', onStderrEnd).on('close', onStderrDone).on('error', error => onStreamError('stderr', error));
|
||||
signal?.addEventListener('abort', stopRun, { once: true });
|
||||
workTimer = setTimeout(stopRun, Math.max(0, deadline - Date.now()));
|
||||
if (signal?.aborted || Date.now() >= deadline) stopRun();
|
||||
|
||||
try {
|
||||
await finished;
|
||||
if (stdoutBuffer.trim()) collectedLines.push(stdoutBuffer);
|
||||
const parsed = parseCodexJSONL(collectedLines);
|
||||
const result: CodexResult = {
|
||||
...parsed,
|
||||
exitCode: timedOut ? 124 : exitCode ?? 1,
|
||||
durationMs: Date.now() - startTime,
|
||||
rawLines: collectedLines,
|
||||
stderr,
|
||||
};
|
||||
if (stderr.trim()) process.stderr.write(` [codex stderr] ${stderr.trim().slice(0, 200)}\n`);
|
||||
if (spawnError) throw new CodexHarnessError(`Could not start Codex: ${spawnError.message}`, result);
|
||||
const incompleteStreams = [!stdoutEnded && 'stdout', !stderrEnded && 'stderr'].filter(Boolean).join(' and ');
|
||||
if (result.exitCode === 0 && (drainExpired || streamError || incompleteStreams)) {
|
||||
throw new CodexHarnessError(
|
||||
drainExpired ? `Codex output drain exceeded ${CODEX_DRAIN_GRACE_MS}ms after exit 0`
|
||||
: streamError ? `Codex ${streamError.stream} stream failed: ${streamError.error.message}`
|
||||
: `Codex ${incompleteStreams} closed before EOF after exit 0`,
|
||||
result,
|
||||
);
|
||||
}
|
||||
return result;
|
||||
} finally {
|
||||
finalized = true;
|
||||
clearTimeout(workTimer);
|
||||
clearTimeout(drainTimer);
|
||||
signal?.removeEventListener('abort', stopRun);
|
||||
// Child exit and closed pipes do not prove its descendants exited:
|
||||
// background tools may redirect both streams. Reap the owned group on
|
||||
// every result, including ordinary success and genuine process failure.
|
||||
killProcessGroup(proc, 'SIGKILL');
|
||||
closePipes();
|
||||
proc.removeListener('exit', onExit);
|
||||
// Retain the harmless error listener through stream teardown so a late
|
||||
// OS spawn error cannot turn into an unhandled event after cleanup.
|
||||
proc.stdout!.removeListener('data', onStdout);
|
||||
proc.stderr!.removeListener('data', onStderr);
|
||||
}
|
||||
}
|
||||
|
||||
// --- Main runner ---
|
||||
|
||||
/**
|
||||
@@ -209,7 +440,7 @@ export async function runCodexSkill(opts: {
|
||||
|
||||
const emptyResult = (exitCode: number, output = ''): CodexResult => ({
|
||||
output, reasoning: [], toolCalls: [], tokens: 0, exitCode,
|
||||
durationMs: Date.now() - startTime, sessionId: null, rawLines: [], stderr: '',
|
||||
durationMs: Date.now() - startTime, sessionId: null, rawLines: [], stderr: '', usage: null,
|
||||
});
|
||||
if (signal?.aborted || Date.now() >= deadline) return emptyResult(124);
|
||||
|
||||
@@ -257,7 +488,8 @@ export async function runCodexSkill(opts: {
|
||||
// exactly that. Empirically verified against codex on this machine.
|
||||
const args = ['exec', '--json', '-s', sandbox, '--skip-git-repo-check'];
|
||||
if (ignoreUserConfig) args.push('--ignore-user-config');
|
||||
args.push('--model', model ?? process.env.GSTACK_CODEX_MODEL ?? CODEX_FRONTIER_MODEL);
|
||||
const sessionModel = model ?? process.env.GSTACK_CODEX_MODEL ?? CODEX_FRONTIER_MODEL;
|
||||
args.push('--model', sessionModel);
|
||||
for (const override of configOverrides) args.push('-c', override);
|
||||
args.push(prompt);
|
||||
|
||||
@@ -275,162 +507,17 @@ export async function runCodexSkill(opts: {
|
||||
// dir beside CODEX_HOME instead of the shard TMPDIR that holds tempHome.
|
||||
const tempTmpDir = path.join(tempHome, 'tmp');
|
||||
fs.mkdirSync(tempTmpDir, { recursive: true });
|
||||
const proc = spawn('codex', args, {
|
||||
cwd: cwd || skillDir,
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
detached: process.platform !== 'win32',
|
||||
const result = await execCodex(args, {
|
||||
cwd: cwd || skillDir, startTime, deadline, signal,
|
||||
env: hermeticChildEnv(
|
||||
{ HOME: tempHome, CODEX_HOME: tempCodexDir, TMPDIR: tempTmpDir },
|
||||
{ extraAllow: ['OPENAI_API_KEY', 'CODEX_*'] },
|
||||
),
|
||||
}).catch((error: unknown) => {
|
||||
if (error instanceof CodexHarnessError && error.result) error.result.model = sessionModel;
|
||||
throw error;
|
||||
});
|
||||
const collectedLines: string[] = [];
|
||||
// work --exit--> drain (<=5s) --> result
|
||||
// \--timeout/abort--> kill group + close pipes --> timeout result
|
||||
// Exit status comes from 'exit'; a forced drain after exit 0 is an error.
|
||||
let stdoutBuffer = '';
|
||||
let stderr = '';
|
||||
let exitCode: number | undefined;
|
||||
let timedOut = false;
|
||||
let drainExpired = false;
|
||||
let streamError: { stream: 'stdout' | 'stderr'; error: Error } | undefined;
|
||||
let spawnError: Error | undefined;
|
||||
let stdoutDone = false;
|
||||
let stderrDone = false;
|
||||
let stdoutEnded = false;
|
||||
let stderrEnded = false;
|
||||
let finalized = false;
|
||||
const stdoutDecoder = new StringDecoder('utf8');
|
||||
const stderrDecoder = new StringDecoder('utf8');
|
||||
let workTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
let drainTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
let finish!: () => void;
|
||||
const finished = new Promise<void>((resolve) => { finish = resolve; });
|
||||
|
||||
const maybeFinish = () => {
|
||||
if (exitCode !== undefined && stdoutDone && stderrDone) finish();
|
||||
};
|
||||
const closePipes = () => {
|
||||
// Destroy both streams: canceling stdout alone cannot release an
|
||||
// inherited stderr pipe held by a descendant in another process group.
|
||||
proc.stdout!.destroy();
|
||||
proc.stderr!.destroy();
|
||||
};
|
||||
const expireDrain = () => {
|
||||
drainExpired = !stdoutDone || !stderrDone;
|
||||
killProcessGroup(proc, 'SIGKILL');
|
||||
closePipes();
|
||||
// Also bound the rare case where a failed group kill produces no exit.
|
||||
finish();
|
||||
};
|
||||
const stopRun = () => {
|
||||
// A real auth/crash exit that preceded a pipe stall stays a real exit.
|
||||
if (exitCode === undefined) timedOut = true;
|
||||
else drainExpired ||= !stdoutDone || !stderrDone;
|
||||
killProcessGroup(proc, 'SIGKILL');
|
||||
closePipes();
|
||||
clearTimeout(drainTimer);
|
||||
drainTimer = setTimeout(expireDrain, CODEX_DRAIN_GRACE_MS);
|
||||
maybeFinish();
|
||||
};
|
||||
const onExit = (code: number | null, exitSignal: NodeJS.Signals | null) => {
|
||||
exitCode = code ?? (exitSignal ? 128 + (os.constants.signals[exitSignal] ?? 0) : 1);
|
||||
clearTimeout(workTimer);
|
||||
clearTimeout(drainTimer);
|
||||
// 'exit' means the child is gone; 'close' also waits for descendant
|
||||
// pipes. Start the drain allowance at actual exit, never at close.
|
||||
drainTimer = setTimeout(expireDrain, CODEX_DRAIN_GRACE_MS);
|
||||
maybeFinish();
|
||||
};
|
||||
const onSpawnError = (error: Error) => {
|
||||
if (finalized) return;
|
||||
spawnError = error;
|
||||
exitCode = 1;
|
||||
closePipes();
|
||||
finish();
|
||||
};
|
||||
// Closure releases lifecycle waits, but only 'end' proves all bytes were
|
||||
// drained. A destroyed pipe can emit 'close' without either EOF or error.
|
||||
const onStdoutDone = () => { if (!finalized) { stdoutDone = true; maybeFinish(); } };
|
||||
const onStderrDone = () => { if (!finalized) { stderrDone = true; maybeFinish(); } };
|
||||
const onStdoutEnd = () => { if (!finalized) { stdoutBuffer += stdoutDecoder.end(); stdoutEnded = true; onStdoutDone(); } };
|
||||
const onStderrEnd = () => { if (!finalized) { stderr += stderrDecoder.end(); stderrEnded = true; onStderrDone(); } };
|
||||
const onStreamError = (stream: 'stdout' | 'stderr', error: Error) => {
|
||||
if (!finalized) streamError ??= { stream, error };
|
||||
};
|
||||
const onStdout = (chunk: Buffer) => {
|
||||
if (finalized) return;
|
||||
stdoutBuffer += stdoutDecoder.write(chunk);
|
||||
const lines = stdoutBuffer.split('\n');
|
||||
stdoutBuffer = lines.pop() || '';
|
||||
for (const line of lines) {
|
||||
if (!line.trim()) continue;
|
||||
collectedLines.push(line);
|
||||
try {
|
||||
const event = JSON.parse(line);
|
||||
if (event.type === 'item.completed' && event.item) {
|
||||
const item = event.item;
|
||||
const elapsed = Math.round((Date.now() - startTime) / 1000);
|
||||
if (item.type === 'command_execution' && item.command) {
|
||||
process.stderr.write(` [codex ${elapsed}s] ran: ${item.command.slice(0, 100)}\n`);
|
||||
} else if (item.type === 'agent_message' && item.text) {
|
||||
process.stderr.write(` [codex ${elapsed}s] message: ${item.text.slice(0, 100)}\n`);
|
||||
}
|
||||
}
|
||||
} catch { /* malformed JSONL is ignored by parseCodexJSONL too */ }
|
||||
}
|
||||
};
|
||||
const onStderr = (chunk: Buffer) => { if (!finalized) stderr += stderrDecoder.write(chunk); };
|
||||
|
||||
proc.on('exit', onExit);
|
||||
proc.on('error', onSpawnError);
|
||||
proc.stdout!.on('data', onStdout);
|
||||
proc.stderr!.on('data', onStderr);
|
||||
proc.stdout!.on('end', onStdoutEnd).on('close', onStdoutDone).on('error', error => onStreamError('stdout', error));
|
||||
proc.stderr!.on('end', onStderrEnd).on('close', onStderrDone).on('error', error => onStreamError('stderr', error));
|
||||
signal?.addEventListener('abort', stopRun, { once: true });
|
||||
workTimer = setTimeout(stopRun, Math.max(0, deadline - Date.now()));
|
||||
if (signal?.aborted || Date.now() >= deadline) stopRun();
|
||||
|
||||
try {
|
||||
await finished;
|
||||
if (stdoutBuffer.trim()) collectedLines.push(stdoutBuffer);
|
||||
const parsed = parseCodexJSONL(collectedLines);
|
||||
const result: CodexResult = {
|
||||
...parsed,
|
||||
exitCode: timedOut ? 124 : exitCode ?? 1,
|
||||
durationMs: Date.now() - startTime,
|
||||
rawLines: collectedLines,
|
||||
stderr,
|
||||
};
|
||||
if (stderr.trim()) process.stderr.write(` [codex stderr] ${stderr.trim().slice(0, 200)}\n`);
|
||||
if (spawnError) throw new CodexHarnessError(`Could not start Codex: ${spawnError.message}`, result);
|
||||
const incompleteStreams = [!stdoutEnded && 'stdout', !stderrEnded && 'stderr'].filter(Boolean).join(' and ');
|
||||
if (result.exitCode === 0 && (drainExpired || streamError || incompleteStreams)) {
|
||||
throw new CodexHarnessError(
|
||||
drainExpired ? `Codex output drain exceeded ${CODEX_DRAIN_GRACE_MS}ms after exit 0`
|
||||
: streamError ? `Codex ${streamError.stream} stream failed: ${streamError.error.message}`
|
||||
: `Codex ${incompleteStreams} closed before EOF after exit 0`,
|
||||
result,
|
||||
);
|
||||
}
|
||||
return result;
|
||||
} finally {
|
||||
finalized = true;
|
||||
clearTimeout(workTimer);
|
||||
clearTimeout(drainTimer);
|
||||
signal?.removeEventListener('abort', stopRun);
|
||||
// Child exit and closed pipes do not prove its descendants exited:
|
||||
// background tools may redirect both streams. Reap the owned group on
|
||||
// every result, including ordinary success and genuine process failure.
|
||||
killProcessGroup(proc, 'SIGKILL');
|
||||
closePipes();
|
||||
proc.removeListener('exit', onExit);
|
||||
// Retain the harmless error listener through stream teardown so a late
|
||||
// OS spawn error cannot turn into an unhandled event after cleanup.
|
||||
proc.stdout!.removeListener('data', onStdout);
|
||||
proc.stderr!.removeListener('data', onStderr);
|
||||
}
|
||||
return { ...result, model: sessionModel };
|
||||
|
||||
} finally {
|
||||
// Clean up temp HOME
|
||||
|
||||
@@ -16,6 +16,7 @@ import { WorktreeManager } from '../../lib/worktree';
|
||||
import type { HarvestResult } from '../../lib/worktree';
|
||||
import { preflightAnthropicApi } from './anthropic-preflight';
|
||||
import { isHermeticEnabled } from './hermetic-env';
|
||||
import { hostSkipReason, resolveEvalHost } from './e2e-hosts';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
@@ -133,6 +134,17 @@ if (evalsEnabled && process.env.EVALS_TIER) {
|
||||
process.stderr.write(`EVALS_TIER=${tier}: ${selectedTests.length} tests\n\n`);
|
||||
}
|
||||
|
||||
// EVALS_HOST=codex: Claude-only cases are skipped with their reason, never run
|
||||
// (and never failed); the report deselects them (test/helpers/e2e-hosts.ts).
|
||||
if (evalsEnabled && resolveEvalHost() !== 'claude') {
|
||||
const host = resolveEvalHost();
|
||||
const candidates = selectedTests ?? Object.keys(E2E_TIERS);
|
||||
const skipped = candidates.filter(id => hostSkipReason(id, host));
|
||||
selectedTests = candidates.filter(id => !hostSkipReason(id, host));
|
||||
process.stderr.write(`EVALS_HOST=${host}: ${selectedTests.length} tests; ${skipped.length} Claude-only skipped (${skipped.join(', ') || 'none'})\n\n`);
|
||||
(await import('./codex-host-runner')).prepareCodexHostRuntime();
|
||||
}
|
||||
|
||||
export const describeE2E = evalsEnabled ? describe : describe.skip;
|
||||
|
||||
/**
|
||||
@@ -261,6 +273,8 @@ export function recordE2E(
|
||||
...(!passed && isPreTurnInfraFailure(result) ? { failure_class: 'infra' as const } : {}),
|
||||
duration_ms: result.duration,
|
||||
cost_usd: result.costEstimate.estimatedCost,
|
||||
...(result.costKnown === false ? { cost_known: false } : {}),
|
||||
...(result.tokenUsage ? { token_usage: result.tokenUsage } : {}),
|
||||
transcript: result.transcript,
|
||||
output: result.output?.slice(0, 2000),
|
||||
turns_used: result.costEstimate.turnsUsed,
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
/**
|
||||
* Agent host selection for paid E2E cases (GSTA-49 TM2). EVALS_HOST picks the
|
||||
* agent every runSkillTest session runs on: `claude` (default) or `codex`
|
||||
* (codex exec, test/helpers/codex-host-runner.ts). E2E_HOSTS is the per-case
|
||||
* host metadata over every gate case (classification in the GSTA-49 TM2
|
||||
* scoping): `both` runs on either host with the shared adapter; every other
|
||||
* value is the reason the case is Claude-only. On the Codex host a Claude-only
|
||||
* case is SKIPPED with its reason, planned as excluded and deselected by the
|
||||
* report with zero credit, never counted as failed. A case without metadata
|
||||
* (every non-gate case) is Claude-only.
|
||||
*/
|
||||
|
||||
export type EvalHost = 'claude' | 'codex';
|
||||
export const EVAL_HOSTS: readonly EvalHost[] = ['claude', 'codex'];
|
||||
|
||||
export function resolveEvalHost(env: NodeJS.ProcessEnv = process.env): EvalHost {
|
||||
const raw = (env.EVALS_HOST ?? '').trim();
|
||||
if (!raw) return 'claude';
|
||||
if ((EVAL_HOSTS as readonly string[]).includes(raw)) return raw as EvalHost;
|
||||
throw new Error(`EVALS_HOST must be one of ${EVAL_HOSTS.join(', ')}; got "${raw}"`);
|
||||
}
|
||||
|
||||
export const CLAUDE_ONLY_REASONS = {
|
||||
pty: 'drives the interactive Claude Code TUI (plan mode, native AskUserQuestion) through the PTY harness',
|
||||
sdk: 'uses the Claude Agent SDK (canUseTool answers and PreToolUse hooks)',
|
||||
subagents: 'asserts Claude Agent/Task subagent events',
|
||||
'skill-tool': 'depends on the Claude Skill tool load and registration',
|
||||
hermetic: 'a claude -p hermetic canary; the Codex host needs its own canary twin',
|
||||
'self-host': 'the Claude host calling /codex; on the Codex host /codex is the agent itself',
|
||||
rework: 'parses Claude stream-json tool events or prescribes native Claude tools; not ported to the Codex host',
|
||||
'host-neutral': 'runs no model session, so a second host adds nothing',
|
||||
} as const;
|
||||
|
||||
export type ClaudeOnlyReason = keyof typeof CLAUDE_ONLY_REASONS;
|
||||
export type HostSupport = 'both' | ClaudeOnlyReason;
|
||||
|
||||
/** Adapter-only cases: final output, files, git state, Bash commands, server logs or browseErrors. */
|
||||
const ADAPTER_ONLY: readonly string[] = [
|
||||
'benchmark-workflow',
|
||||
'browse-basic',
|
||||
'browse-snapshot',
|
||||
'canary-workflow',
|
||||
'context-recovery-artifacts',
|
||||
'context-restore-loads-latest',
|
||||
'context-save-writes-file',
|
||||
'cso-diff-mode',
|
||||
'design-review-detector-shim',
|
||||
'design-review-detector-shim-dom',
|
||||
'design-review-plugin-handoff',
|
||||
'diagram-triplet',
|
||||
'document-release',
|
||||
'gstack-upgrade-happy-path',
|
||||
'land-and-deploy-first-run',
|
||||
'land-and-deploy-review-gate',
|
||||
'land-and-deploy-workflow',
|
||||
'learnings-show',
|
||||
'operational-learning',
|
||||
'plan-design-review-no-ui-scope',
|
||||
'plan-eng-coverage-audit',
|
||||
'plan-review-report',
|
||||
'plan-tune-inspect',
|
||||
'qa-bootstrap',
|
||||
'qa-quick',
|
||||
'retro-base-branch',
|
||||
'review-army-delivery-audit',
|
||||
'review-army-json-findings',
|
||||
'review-army-migration-safety',
|
||||
'review-army-quality-score',
|
||||
'review-base-branch',
|
||||
'review-coverage-audit',
|
||||
'review-enum-completeness',
|
||||
'review-sql-injection',
|
||||
'review-test-value',
|
||||
'safety-pair-agent-block',
|
||||
'safety-ship-stale-evidence',
|
||||
'setup-deploy-workflow',
|
||||
'ship-base-branch',
|
||||
'ship-coverage-audit',
|
||||
'ship-coverage-value',
|
||||
'ship-local-workflow',
|
||||
'ship-triage',
|
||||
'skillmd-no-local-binary',
|
||||
'skillmd-outside-git',
|
||||
'skillmd-setup-discovery',
|
||||
'test-audit-report-only',
|
||||
'tpa-absent-darwin',
|
||||
'tpa-absent-linux',
|
||||
'tpa-apple-ban',
|
||||
'tpa-broken',
|
||||
'tpa-present'
|
||||
];
|
||||
|
||||
const CLAUDE_ONLY: Readonly<Record<ClaudeOnlyReason, readonly string[]>> = {
|
||||
pty: ['autoplan-guard-pty', 'office-hours-auto-mode', 'plan-ceo-finding-floor', 'plan-ceo-review-plan-mode',
|
||||
'plan-design-with-ui-scope', 'plan-devex-finding-floor', 'plan-devex-review-plan-mode', 'plan-mode-no-op'],
|
||||
sdk: ['auq-format-gate', 'investigate-owned-abort', 'investigate-owned-completion', 'investigate-owned-ending-error',
|
||||
'shared-libs-review-index-flags', 'shared-libs-review-lifecycle', 'shared-libs-review-path-eligibility',
|
||||
'shared-libs-review-prior-coverage', 'shared-libs-review-revalidation', 'ship-local-hook-preservation',
|
||||
'ship-managed-hook-refresh', 'ship-skipped-queued-finding', 'ship-unmanaged-hook-consent'],
|
||||
subagents: ['docsync-spawned', 'review-army-perf-n-plus-one'],
|
||||
'skill-tool': ['skillify-approval-reject', 'skillify-happy-path', 'skillify-provenance-refusal'],
|
||||
hermetic: ['hermetic-canary', 'hermetic-sentinel'],
|
||||
'self-host': ['safety-codex-consult-embed'],
|
||||
rework: ['qa-functional-cli-fix', 'qa-functional-cli-report', 'qa-functional-webhook-fix', 'qa-functional-webhook-report',
|
||||
'qa-only-no-fix', 'review-dashboard-via', 'review-exploratory-small-cli', 'review-gate-integrity',
|
||||
'shared-libs-read-only', 'shared-libs-unsupported-git', 'ship-docsync-completion', 'ship-docsync-current',
|
||||
'ship-docsync-failure', 'ship-docsync-late-result', 'ship-docsync-launch-failure', 'ship-docsync-missing-asset',
|
||||
'ship-docsync-missing-marker', 'ship-docsync-recovery', 'ship-docsync-stale-after', 'ship-docsync-stale-before',
|
||||
'ship-docsync-store', 'ship-docsync-timeout-unsettled', 'ship-exploratory-late-input', 'ship-exploratory-plan-checks',
|
||||
'ship-exploratory-small-cli', 'ship-exploratory-unavailable'],
|
||||
'host-neutral': ['timeline-event-flow'],
|
||||
};
|
||||
|
||||
export const E2E_HOSTS: Readonly<Record<string, HostSupport>> = Object.freeze(Object.fromEntries([
|
||||
...ADAPTER_ONLY.map(id => [id, 'both'] as const),
|
||||
...(Object.entries(CLAUDE_ONLY) as Array<[ClaudeOnlyReason, readonly string[]]>).flatMap(([reason, ids]) => ids.map(id => [id, reason] as const)),
|
||||
]));
|
||||
|
||||
/** Why a case does not run on `host`, or null when it does. */
|
||||
export function hostSkipReason(id: string, host: EvalHost, hosts: Readonly<Record<string, HostSupport>> = E2E_HOSTS): string | null {
|
||||
if (host === 'claude') return null;
|
||||
const support = hosts[id];
|
||||
if (support === 'both') return null;
|
||||
if (support === undefined) return `Claude-only on the ${host} host: no Codex host metadata (E2E_HOSTS covers the gate cases)`;
|
||||
return `Claude-only on the ${host} host: ${CLAUDE_ONLY_REASONS[support]}`;
|
||||
}
|
||||
@@ -14,6 +14,7 @@ import * as os from 'os';
|
||||
import { spawnSync } from 'child_process';
|
||||
import { isManualReviewEntry } from './cookie-workflow-manual-review';
|
||||
import type { ManualJudgeReview } from './cookie-workflow-manual-review';
|
||||
import type { CodexTokenUsage } from './codex-session-runner';
|
||||
|
||||
// v2: EvalTestEntry.harvest gains optional {insertions, deletions, net} and
|
||||
// may be explicitly null (arm-benchmark harvest-failure taxonomy). Readers
|
||||
@@ -67,6 +68,8 @@ export interface EvalTestEntry {
|
||||
cost_usd: number;
|
||||
/** False when the harness captured no billing, so cost_usd 0 means unknown. Absent means known. */
|
||||
cost_known?: boolean;
|
||||
/** Codex sessions: the token classes behind cost_usd. */
|
||||
token_usage?: CodexTokenUsage;
|
||||
/** Absent in older records means executed; reuse is never a new model run. */
|
||||
execution?: 'executed' | 'reused';
|
||||
reused_from?: { input_key: string; run_id: string; revision: string; completed_at: string };
|
||||
|
||||
@@ -54,6 +54,7 @@ export const FREE_FIXTURES: Readonly<Record<string, readonly string[]>> = {
|
||||
'test/fixtures/claude-agent-payload-2.1.292.json': ['test/autoplan-publication-payload.test.ts', 'test/autoplan-schema-canary-compare.test.ts'],
|
||||
'test/fixtures/claude-native-journal-roots-2.1.284.json': ['test/autoplan-publication-hook.test.ts', 'test/plan-count-transcript.test.ts'],
|
||||
'test/fixtures/codex-boundary-commands.json': ['test/codex-boundary-evidence.test.ts'],
|
||||
'test/fixtures/codex-host/**': ['test/codex-host.test.ts'],
|
||||
'test/fixtures/codex-review-sandbox-unavailable-37158847998.json': ['test/outside-voice-evidence.test.ts'],
|
||||
'test/fixtures/codex-review/**': ['test/codex-hardening.test.ts', 'test/outside-review-severity.test.ts', 'test/outside-voice-evidence.test.ts', 'test/skill-validation.test.ts'],
|
||||
'test/fixtures/codex-sandbox/**': ['test/codex-eval-recording.test.ts', 'test/codex-model-probe.test.ts', 'test/outside-review-result.test.ts', 'test/outside-voice-invocation.test.ts'],
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
*/
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import type { CodexTokenUsage } from './codex-session-runner';
|
||||
|
||||
export const SESSION_LEDGER_FILE = 'session-ledger.jsonl';
|
||||
export const SESSION_EVIDENCE_MAX = 200;
|
||||
@@ -48,6 +49,10 @@ export interface SessionLedgerRow {
|
||||
liveness?: LivenessSummary;
|
||||
/** Whether the runner captured this session's billing (a terminal result with total_cost_usd). */
|
||||
billed?: boolean;
|
||||
/** Codex sessions: the model, its billed cost and token classes. */
|
||||
model?: string;
|
||||
cost_usd?: number;
|
||||
tokens?: CodexTokenUsage;
|
||||
}
|
||||
|
||||
/** One line, control characters stripped, mentions neutralized, capped. */
|
||||
|
||||
@@ -17,6 +17,8 @@ import { hermeticChildEnv, isHermeticEnabled } from './hermetic-env';
|
||||
import { killProcessGroup } from '../../scripts/test-strict-output';
|
||||
import { resolveEvalModel } from '../../lib/eval-model';
|
||||
import { SessionObserver, appendSessionLedger, sessionKey, type SessionEnd } from './session-ledger';
|
||||
import { resolveEvalHost } from './e2e-hosts';
|
||||
import type { CodexTokenUsage } from './codex-session-runner';
|
||||
|
||||
const GSTACK_DEV_DIR = path.join(os.homedir(), '.gstack-dev');
|
||||
const HEARTBEAT_PATH = path.join(GSTACK_DEV_DIR, 'e2e-live.json'); // heartbeat stays global
|
||||
@@ -55,6 +57,10 @@ export interface SkillTestResult {
|
||||
firstResponseMs: number;
|
||||
/** Peak latency between consecutive tool calls, in ms */
|
||||
maxInterTurnMs: number;
|
||||
/** Codex host: token classes behind costEstimate.estimatedCost. */
|
||||
tokenUsage?: CodexTokenUsage;
|
||||
/** Codex host: false when the session completed no turn or its model has no price row. */
|
||||
costKnown?: boolean;
|
||||
}
|
||||
|
||||
/** Local default startup grace: 90s covers observed API queue latency
|
||||
@@ -76,6 +82,14 @@ const BROWSE_ERROR_PATTERNS = [
|
||||
/no such file or directory.*\bbrowse(?:\.exe)?(?=$|[\s'":),])/i,
|
||||
];
|
||||
|
||||
/** First match of each browse failure pattern in tool output plus stderr. */
|
||||
export function scanBrowseErrors(text: string): string[] {
|
||||
return BROWSE_ERROR_PATTERNS.flatMap(pattern => {
|
||||
const match = text.match(pattern);
|
||||
return match ? [match[0].slice(0, 200)] : [];
|
||||
});
|
||||
}
|
||||
|
||||
// --- Testable NDJSON parser ---
|
||||
|
||||
export interface ParsedNDJSON {
|
||||
@@ -300,6 +314,7 @@ Setup, CLI startup and API queueing consume this same window; it never resets.
|
||||
Before source Reads, use Bash to run exactly \`date -u +%Y-%m-%dT%H:%M:%SZ\`. After each saved checkpoint, compare the latest evidence capture's printed completedAt with the times above; run that clock read again only when no capture has completed since your last clock read. When remaining time is at most ${reserve / 1000} seconds, prioritize the remaining required completion outputs and verification. No required content or gate may be skipped. If the clock read fails, report timing unavailable; do not invent remaining time or restart the deadline.`;
|
||||
systemPrompt = systemPrompt ? `${systemPrompt}\n\n${notice}` : notice;
|
||||
}
|
||||
if (resolveEvalHost() === 'codex') return (await import('./codex-host-runner')).runCodexHostSkillTest({ ...options, appendSystemPrompt: systemPrompt, completionReserveMs: undefined });
|
||||
|
||||
// Set up per-run log directory if runId is provided
|
||||
let runDir: string | null = null;
|
||||
@@ -661,13 +676,7 @@ Before source Reads, use Bash to run exactly \`date -u +%Y-%m-%dT%H:%M:%SZ\`. Af
|
||||
const { transcript, resultLine, toolCalls } = parsed;
|
||||
const browseErrors: string[] = [];
|
||||
|
||||
const allText = toolCalls.filter(call => call.tool === 'Bash').map(call => call.output).join('\n') + '\n' + stderr;
|
||||
for (const pattern of BROWSE_ERROR_PATTERNS) {
|
||||
const match = allText.match(pattern);
|
||||
if (match) {
|
||||
browseErrors.push(match[0].slice(0, 200));
|
||||
}
|
||||
}
|
||||
browseErrors.push(...scanBrowseErrors(toolCalls.filter(call => call.tool === 'Bash').map(call => call.output).join('\n') + '\n' + stderr));
|
||||
|
||||
// Native Claude uses exit 1 for its structured max-turns result. Preserve
|
||||
// that existing semantic outcome only after a complete, uncancelled drain;
|
||||
|
||||
@@ -217,8 +217,9 @@ test('eval:bg runs the PR and release commands with their declared workers (cap:
|
||||
const cliOptions = (step: { run: string; env?: Record<string, string> }) => {
|
||||
const prefix = /\bbun(?: --no-install)? run scripts\/test-paid-shards\.ts /.exec(step.run);
|
||||
expect(prefix).not.toBeNull();
|
||||
const args = step.run.slice(prefix!.index + prefix![0].length)
|
||||
.replace(/\$\{\{\s*matrix\.slice\s*\}\}/g, '1').trim().split(/\s+/);
|
||||
// One command line: a census arm loop continues past it.
|
||||
const args = step.run.slice(prefix!.index + prefix![0].length).split('\n')[0]!
|
||||
.replace(/\$\{\{\s*matrix\.slice\s*\}\}/g, '1').replace(/\$\{\{\s*matrix\.arm\s*\}\}/g, 'claude-opus').trim().split(/\s+/);
|
||||
return parseCliOptions(args, step.env ?? {});
|
||||
};
|
||||
|
||||
@@ -242,10 +243,16 @@ test('both gate executors plan the complete census and supervise every planned s
|
||||
expect(planned.skipJudges).toBe(skipJudges);
|
||||
expect(execute[0].env.EVALS_CONCURRENCY).toBe('2');
|
||||
expect(executor.strategy['fail-fast']).toBe(false);
|
||||
expect(executor.strategy.matrix.slice).toBe(`\${{ fromJSON(needs.plan-slices.outputs.${prefix}slices) }}`);
|
||||
// Per-slice ceilings (W2c/ENG-2), or the largest of them as one job cap.
|
||||
expect([`\${{ fromJSON(needs.plan-slices.outputs.${prefix}slice_timeouts)[matrix.slice] }}`,
|
||||
`\${{ fromJSON(needs.plan-slices.outputs.${prefix}timeout_minutes) }}`]).toContain(executor['timeout-minutes']);
|
||||
if (jobName === 'gate-census') {
|
||||
// One row per census arm and planned slice, each with its arm's planned slice ceiling (scripts/census-arms.ts).
|
||||
expect(executor.strategy.matrix.include).toBe('${{ fromJSON(needs.plan-slices.outputs.gate_matrix) }}');
|
||||
expect(executor['timeout-minutes']).toBe('${{ matrix.timeout }}');
|
||||
} else {
|
||||
expect(executor.strategy.matrix.slice).toBe(`\${{ fromJSON(needs.plan-slices.outputs.${prefix}slices) }}`);
|
||||
// Per-slice ceilings (W2c/ENG-2), or the largest of them as one job cap.
|
||||
expect([`\${{ fromJSON(needs.plan-slices.outputs.${prefix}slice_timeouts)[matrix.slice] }}`,
|
||||
`\${{ fromJSON(needs.plan-slices.outputs.${prefix}timeout_minutes) }}`]).toContain(executor['timeout-minutes']);
|
||||
}
|
||||
const manifest = buildRunManifest({ tier: 'gate', sliceBudgetMs: planned.sliceBudgetMs!, jobs: planned.jobs, evalsAll: true, env: { EVALS_ALL: '1' }, skipJudges });
|
||||
const files = [...new Set(manifest.entries.filter(row => row.status === 'planned').map(row => shardFile(row.file)))];
|
||||
expect(files).toContain('test/skill-e2e-ship-skip.test.ts');
|
||||
@@ -262,8 +269,8 @@ test('both gate executors plan the complete census and supervise every planned s
|
||||
expect(manifest.sliceCount).toBeLessThanOrEqual(executor.strategy['max-parallel']);
|
||||
if (jobName === 'gate-census') {
|
||||
expect(emit[0].env.EVALS_ALL).toBe('1');
|
||||
expect(emit[0].run).toContain('--emit-plan /tmp/gate-census-plan/manifest.json');
|
||||
expect(execute[0].run).toContain('--plan /tmp/gate-census-plan/manifest.json --slice ${{ matrix.slice }}');
|
||||
expect(emit[0].run).toContain('--emit-plan "/tmp/gate-census-plan/$arm/manifest.json"');
|
||||
expect(execute[0].run).toContain('--plan /tmp/gate-census-plan/${{ matrix.arm }}/manifest.json --slice ${{ matrix.slice }}');
|
||||
expect(executor.steps.some((step: any) => step.with?.name === 'gate-census-plan')).toBe(true);
|
||||
expect(executor.steps.filter((step: any) => step.run?.includes('--emit-plan '))).toHaveLength(0);
|
||||
}
|
||||
@@ -305,8 +312,10 @@ test('gate census requires all seven distinct slice results and its own reconcil
|
||||
const reconcile = report.steps.find((step: any) => step.id === 'gate-reconcile');
|
||||
expect(report.needs).toContain('gate-census');
|
||||
expect(reconcile.if).toBe('always()');
|
||||
expect(reconcile.run).toContain('EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --report /tmp/gate-census-report');
|
||||
expect(reconcile.run).toContain('exit=${PIPESTATUS[0]}');
|
||||
// One reconciliation per census arm, each against its own plan and slices.
|
||||
expect(reconcile.run).toContain('dir="/tmp/gate-census-report/$arm"');
|
||||
expect(reconcile.run).toContain('EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --report "$dir"');
|
||||
expect(reconcile.run).toContain('code=${PIPESTATUS[0]}');
|
||||
const failureGuards = report.steps.filter((step: any) => /Upsert tracking|Fail the workflow/.test(step.name ?? ''));
|
||||
expect(failureGuards).toHaveLength(2);
|
||||
for (const step of failureGuards) {
|
||||
|
||||
@@ -178,7 +178,7 @@ describe('tier lane skip (B5)', () => {
|
||||
}
|
||||
const workflow = fs.readFileSync(path.join(ROOT, '.github/workflows/evals-periodic.yml'), 'utf8');
|
||||
expect(workflow.match(/--skip-judges/g)).toHaveLength(1);
|
||||
expect(workflow).toMatch(/--tier gate --emit-plan \/tmp\/gate-census-plan\/manifest\.json --slice-budget 420 --jobs 2 --skip-judges/);
|
||||
expect(workflow).toMatch(/--tier gate --emit-plan "\/tmp\/gate-census-plan\/\$arm\/manifest\.json" --slice-budget 420 --jobs 2 --skip-judges/);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
Reference in new issue
Block a user