From be05f299bc9739f4cf2dae021080bcfba9b208c3 Mon Sep 17 00:00:00 2001 From: garrytan Date: Mon, 19 Oct 2026 07:29:18 +0000 Subject: [PATCH] chore: TM2 CHANGELOG under [Unreleased]; re-pin codex-host-runner in the harness manifest after the main merge Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 27 +++++++++++++++++++++++++++ scripts/harness-version.json | 20 +++++++++++--------- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 566320d96..534b9910d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,32 @@ # Changelog +## [Unreleased] + +**Paid eval cases can now run on the Codex host, and the Monday census runs two arms: Claude Opus 5.5 and Codex gpt-6.1-sol.** `EVALS_HOST=codex` runs the 52 adapter-only gate cases through a full Codex runtime per case. On its first pilot gpt-6.1-sol passed 48 of 52 with no retries, at $4.63 for the 52 cases, against $28.93 for the same cases on Fable (0.16×). Claude-only cases are skipped on Codex with a recorded reason and never count as failed. GSTA-49 TM2. + +### Added +- **Codex host adapter:** `test/helpers/e2e-hosts.ts` holds the `EVALS_HOST` switch (`claude` default, `codex`) and the per-case host metadata. `test/helpers/codex-host-runner.ts` adds: + - the case runner, with sandbox settings and a reasoning effort (the model default unless set); + - file-change event mapping; + - a transcript translated into the Claude stream-json shape that existing assertions read; + - the native-lifecycle hooks. + + The 26 rework cases stay Claude-only. +- **Census arms (`scripts/census-arms.ts`):** `evals-periodic.yml` gets two census schedules. + - **Monday:** a two-arm gate census, `claude-opus` and `codex-sol`. + - **Monthly (`0 6 1 * *`):** census jobs only, with `claude-fable` (claude-fable-5-1) and `codex-astra` (gpt-6-astra) arms and a `-monthly` concurrency key. + + A `census_arms` dispatch input picks arms by hand. Artifact names and run IDs carry the arm, and a census-only run executes no periodic slice. Every arm keeps the census's `--skip-judges`. + +### Changed +- **Codex cost is real:** `cost_known: false` is gone. Codex sessions price their `turn.completed` usage (input, cached input, cache-write input and output tokens) through `lib/pricing.ts`. The shared `execCodex` keeps the old runner's events and verdicts. +- **Plans, reports and the ledger carry the host** when it is not Claude. On Codex, the Anthropic preflight skips its `claude -p` ping. +- **Gate integrity:** no threshold, trial count, case kind, judge or Claude-host verdict changed. `HARNESS_VERSION` stays 6, and the changed harness files are recorded as non-behavioral. Five workflow tests now assert the census matrix per arm (one row per arm and slice, per-arm plan and report paths, a `periodic_lane` guard) in place of the single-arm shape; no assertion was dropped. + +### If you… +- **…want one case on Codex:** `EVALS_HOST=codex GSTACK_CODEX_MODEL=gpt-6.1-sol bun run scripts/test-paid-shards.ts --tier gate --case --trials 1`. +- **…want the monthly arms now:** dispatch `evals-periodic.yml` with `census_arms=claude-fable,codex-astra`. + ## [1.91.144.0] - 2026-10-18 **The paid PR gate now tests Claude Opus 5.5 instead of Fable 5.1. Its first full Opus gate went 132/132 green and recorded $32.33, against Fable's $93.16 mean over ten full gates.** Garry's direction (2026-10-18) is to test each lab's best code model. Calibration run 39157502890 passed all 108 rule cases and 24 judges on one attempt each, with no reused receipts, on the same case set every Fable gate passed. Opus 5.5 sessions cost $30.86 (Fable: $90.49), judges $1.44 and Haiku side calls $0.08; with the unrecorded PTY sessions estimated, the full-gate scale is $37.13 against $106.17. Output tokens fell 37% (thinking 69%) and session time from 11,859 s to 6,159 s. The Claude CLI runs Opus 5.5 at medium effort by default and Fable at high. diff --git a/scripts/harness-version.json b/scripts/harness-version.json index 50da17512..a41f4702f 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -3,6 +3,7 @@ "patterns": [ "test/helpers/agent-sdk-runner.ts", "test/helpers/codex-session-runner.ts", + "test/helpers/codex-host-runner.ts", "test/helpers/claude-pty-runner.ts", "test/helpers/pty/**", "test/helpers/pty-screen.ts", @@ -10,22 +11,23 @@ "test/helpers/llm-judge.ts" ], "files": { - "scripts/lib/paid-cases.ts": {"blob":"9bacd52446192a7ecf3dfaae901ea443cf9c9ae2","since":6}, - "scripts/lib/paid-plan.ts": {"blob":"45721ce128a112666a0caed6c13c66637b74ca54","since":6}, - "scripts/lib/paid-report.ts": {"blob":"b66c41237611ccecaaa7bd6a4d7b7aac8dc548cd","since":6}, + "scripts/lib/paid-cases.ts": {"blob":"f862cf5241580c7593b9ee62eacf5ef046248d26","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, + "scripts/lib/paid-plan.ts": {"blob":"95e1798e7e2495ace350d2624e59f83412f341bc","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, + "scripts/lib/paid-report.ts": {"blob":"f92c6bb723b158e328aa040fa5de6a0ee6d3916d","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6}, "scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":6}, - "scripts/test-paid-shards.ts": {"blob":"8ddee5f11ed7856876135520dd8025ab989e2db9","since":6}, + "scripts/test-paid-shards.ts": {"blob":"19c9b2376d09090aee1a8acacfe721d66f55b0fc","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "scripts/test-pr-profile.ts": {"blob":"2029a89e14f511866cc91bebfee00b4fe9e7cf63","since":6}, "scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":6}, "test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":6}, "test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":6}, "test/helpers/claude-pty-runner.ts": {"blob":"20cc5894532873aa6145fc6617fdeec8cef94a2d","since":6}, - "test/helpers/codex-session-runner.ts": {"blob":"4eea35931a35a0afb006363402e3d4191e379737","since":6}, + "test/helpers/codex-host-runner.ts": {"blob":"bff225017e235f0be6ece29f04971ac04b95b544","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, + "test/helpers/codex-session-runner.ts": {"blob":"c040ebbfa815b8cce19ef80c6133bd5a5474149a","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "test/helpers/e2e-gate.ts": {"blob":"efb7e197dd5f66c3ad0e34074b14e640754f7a45","since":6}, - "test/helpers/e2e-helpers.ts": {"blob":"2d4a2a53063cfc1f003db366d2c861ab1e379e6e","since":6}, + "test/helpers/e2e-helpers.ts": {"blob":"6ee6937e6c3ca081c77131b5a755278c7b714834","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "test/helpers/eval-budgets.ts": {"blob":"667d0e09b20fe5d7366f57b77b5fb9b04baa37ef","since":6}, - "test/helpers/eval-store.ts": {"blob":"609c6d099108ff2696eb014b47aa318a618dd2af","since":6}, + "test/helpers/eval-store.ts": {"blob":"ed8721f13d39f4ef62e0a1d814efbe9352da781a","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "test/helpers/hermetic-env.ts": {"blob":"cc072895461dc52ea317d1beff72121342420d30","since":6}, "test/helpers/llm-judge.ts": {"blob":"d0bc6bbbe9f478603d11a1ece95e66c812ad34e6","since":6}, "test/helpers/paid-test-set.ts": {"blob":"ff9bfe43c43ec2bdd11114f7b8ec7c5736116f75","since":6}, @@ -44,8 +46,8 @@ "test/helpers/pty/screen.ts": {"blob":"ba7a80bb68b1959296ab2a827cfacdaa46f4ba2d","since":6}, "test/helpers/pty/session.ts": {"blob":"58533cea9d81cc87bb3b499bdfe0db8e3a8709c5","since":6}, "test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":6}, - "test/helpers/session-ledger.ts": {"blob":"6e3c9e837e46f98addba2d12eaebff802da320bb","since":6}, - "test/helpers/session-runner.ts": {"blob":"d40eb29e8b8fdc0675b511693e4a90823b5963fc","since":6}, + "test/helpers/session-ledger.ts": {"blob":"425323284a368f5e77937e03884db4dea4e6134e","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, + "test/helpers/session-runner.ts": {"blob":"cc65a66adafa9c053e3da3c6f17b5ac86c0eade7","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "test/helpers/skill-fixture.ts": {"blob":"b188064594eb4201a54a80ba1bc6f28e31d7f49e","since":6}, "test/helpers/test-selection.ts": {"blob":"51655f89f66be7a5ec487e2abfe4744300ebd5f6","since":6}, "test/helpers/touchfiles.ts": {"blob":"15f369056d41b9dc42dcd86f81f60c878074704c","since":6}