From c205e226a1139f98add5dc3a67804c780e926aaf Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:17:57 +0000 Subject: [PATCH 1/3] v1.91.147.0 test(seed): refresh the free-suite duration seeds (#3289) * test(seed): refresh the free-suite duration seeds from the weekly recording * test(seed): allowlist brain-sync and setup-codex-scope with the split todo; drop host-renders-prune The 2026-10-19 weekly recording put test/brain-sync.test.ts at 89s (#3254's case-collision suite) and test/setup-codex-scope.test.ts at 64s, over the 60s ceiling; test/host-renders-prune.test.ts fell to 57s, so its entry is stale. Both new entries carry the existing TODOS.md split todo. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test(seed): classify the seed allowlist as free-only so editing it selects no paid case Only test/free-seed-ratchet.test.ts reads scripts/free-test-seed-allowlist.json, but eval selection derived all 246 gate cases from it, so #3289's allowlist repair paid for a full gate. The new free-fixtures case shows seed data and the allowlist select nothing, with a negative control: design/src/evolve.ts still selects its derived dependents. Non-behavioral HARNESS_VERSION decision. Gate integrity: scripts/test-pr-profile.ts FREE_ONLY_PR_FILES gains one entry. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * chore: stamp v1.91.147.0 Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: garrytan Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 8 + TODOS.md | 11 +- VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- package.json | 2 +- scripts/free-test-durations-windows.json | 1478 ++++++------- scripts/free-test-durations.json | 2524 +++++++++++----------- scripts/free-test-seed-allowlist.json | 3 +- scripts/harness-version.json | 2 +- scripts/test-pr-profile.ts | 1 + test/free-fixtures.test.ts | 10 + 11 files changed, 2042 insertions(+), 2001 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c8a1988a..1232dd584 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [1.91.147.0] - 2026-10-19 + +**The free-suite duration seeds are current again.** This is the first refresh from the weekly recorder. It adds 14 Linux and 6 Windows test files that the shard packers had been placing at the 99th-percentile estimate. + +### Changed +- **Seed allowlist:** `test/brain-sync.test.ts` (89 s) and `test/setup-codex-scope.test.ts` (64 s) are allowlisted with the existing split todo. `test/host-renders-prune.test.ts` (now 57 s) leaves the allowlist. +- **Editing the seed allowlist no longer selects paid evals.** Only the free seed ratchet reads `scripts/free-test-seed-allowlist.json`, but an edit to it used to pull in all 246 dependent gate cases. It is now classified free-only, like the two seeds. + ## [1.91.146.0] - 2026-10-19 **`gstack-ship-queue` can run merge trains: several READY PRs share one full gate, merge one by one only while main's tree matches the prefix that gate tested, and a red train is bisected on its red cases alone.** The surface is off unless the base revision's `.gstack/ship-policy.json` declares a `train` block. Without one, `ready --pr ` prints exactly the block it prints today. gstack's own policy does not turn trains on yet: that waits for the workflow side (the train-nonce record, the `cases` and `prefix_shas` inputs and the dispatched screen) and one dry-run train. Walkthrough: [docs/ship-policy.md#run-a-merge-train](docs/ship-policy.md#run-a-merge-train). Plan GSTA-49, lane T1. diff --git a/TODOS.md b/TODOS.md index f76d0d5a4..25de1dcf9 100644 --- a/TODOS.md +++ b/TODOS.md @@ -5471,10 +5471,11 @@ Garry for each; revisit 2026-10-29 unless a date says otherwise. 16-worker duration recording on 2026-10-15 the event had not landed (`['paste', 'enter', 'end_turn']`). Wait for the CLI's own write instead of a timer (docs/TESTING_INTERNALS.md#flake-ledger). **Effort:** S. **Priority:** P2. -- **Split the free files the 2026-10-15 seed put over 60 s** — - `test/setup-install-registry.test.ts` (82 s), `test/host-renders-prune.test.ts` - (64 s) and `test/setup-codex-scope.test.ts` (60.4 s) are allowlisted with this - todo in `scripts/free-test-seed-allowlist.json`; `test/brain-sync.test.ts` - (57 s) and `test/setup-codex-scope-namespaces.test.ts` (57 s) are next. Split +- **Split the free files the weekly seed puts over 60 s** — + `test/setup-install-registry.test.ts` (84 s), `test/brain-sync.test.ts` (89 s) + and `test/setup-codex-scope.test.ts` (64 s) are allowlisted with this todo in + `scripts/free-test-seed-allowlist.json` (2026-10-19 recording); + `test/host-renders-prune.test.ts` (57 s) and + `test/setup-codex-scope-namespaces.test.ts` are next. Split each by describe block, rerun `bun run test:ubicloud --record-durations` and delete the entries. **Effort:** S each. **Priority:** P2. diff --git a/VERSION b/VERSION index 45b7708e8..ae26945c3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.146.0 +1.91.147.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 4ca41e67d..656c5d1e5 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.146.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.147.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/package.json b/package.json index d3122c016..ff953935f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.146", + "version": "1.91.147", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/scripts/free-test-durations-windows.json b/scripts/free-test-durations-windows.json index 8f2bc3798..8be52d252 100644 --- a/scripts/free-test-durations-windows.json +++ b/scripts/free-test-durations-windows.json @@ -1,744 +1,750 @@ { "version": 1, - "recordedAt": "2026-10-17T06:23:07.787Z", + "recordedAt": "2026-10-19T04:37:35.081Z", "durations": { - "browse/test/activity.test.ts": 78, - "browse/test/adversarial-security.test.ts": 41, - "browse/test/batch.test.ts": 5919, - "browse/test/bridge-chromium-e2e.test.ts": 4106, - "browse/test/browse-client.test.ts": 291, - "browse/test/browse-eval-wrapping.test.ts": 150, - "browse/test/browser-manager-custom-chromium.test.ts": 620, - "browse/test/browser-skills-storage.test.ts": 115, - "browse/test/build-command-response.test.ts": 704, - "browse/test/build.test.ts": 111, - "browse/test/bun-child-config.test.ts": 67, - "browse/test/bun-polyfill.test.ts": 1473, - "browse/test/busy-daemon-iron-rule.test.ts": 7808, - "browse/test/busy-daemon-recovery.test.ts": 600, - "browse/test/cdp-allowlist.test.ts": 50, - "browse/test/cdp-e2e.test.ts": 1315, - "browse/test/cdp-inspector-history-cap.test.ts": 34, - "browse/test/cdp-mutex.test.ts": 817, - "browse/test/cdp-session-cleanup.test.ts": 52, - "browse/test/claude-bin.test.ts": 31, - "browse/test/cli-chain-stdin.test.ts": 153, - "browse/test/cli-config-mismatch-message.test.ts": 86, - "browse/test/cli-lock.test.ts": 49, - "browse/test/cli-loopback-no-proxy.test.ts": 39, - "browse/test/cli-setsid-daemonize.test.ts": 25, - "browse/test/cli-start-final-healthcheck.test.ts": 37, - "browse/test/cli-supervisor.test.ts": 52, - "browse/test/content-security.test.ts": 1229, - "browse/test/cookie-auth-verification.test.ts": 14212, - "browse/test/cookie-credential-deadline.test.ts": 230, - "browse/test/cookie-database.test.ts": 451, - "browse/test/cookie-fixture-delete-lease.test.ts": 1124, - "browse/test/cookie-import-browser.test.ts": 74, - "browse/test/cookie-import-native-job.test.ts": 118070, - "browse/test/cookie-import-native.test.ts": 4727, - "browse/test/cookie-import-node.test.ts": 4003, - "browse/test/cookie-import-operation.test.ts": 441, - "browse/test/cookie-import-reliability.test.ts": 1230, - "browse/test/cookie-import-transport.test.ts": 38, - "browse/test/cookie-picker-binding-browser.test.ts": 2699, - "browse/test/cookie-picker-binding.test.ts": 359, - "browse/test/cookie-picker-csrf-browser.test.ts": 695, - "browse/test/cookie-picker-routes.test.ts": 369, - "browse/test/cookie-picker-ui.test.ts": 33245, - "browse/test/daemon-log-hygiene.test.ts": 208, - "browse/test/daemon-mismatch-refuse.test.ts": 228, - "browse/test/data-platform.test.ts": 56, - "browse/test/domain-skills-e2e.test.ts": 546, - "browse/test/domain-skills-storage.test.ts": 84, - "browse/test/dotenv-isolation.test.ts": 555, - "browse/test/dual-listener.test.ts": 615, - "browse/test/dx-polish.test.ts": 51, - "browse/test/error-handling.test.ts": 35, - "browse/test/extension-sender-auth.test.ts": 69, - "browse/test/extension-token.test.ts": 1570, - "browse/test/file-drop.test.ts": 59, - "browse/test/file-permissions.test.ts": 896, - "browse/test/fill-change-event.test.ts": 2053, + "browse/test/activity.test.ts": 126, + "browse/test/adversarial-security.test.ts": 99, + "browse/test/batch.test.ts": 6701, + "browse/test/bridge-chromium-e2e.test.ts": 4235, + "browse/test/browse-client.test.ts": 162, + "browse/test/browse-eval-wrapping.test.ts": 77, + "browse/test/browser-manager-custom-chromium.test.ts": 1019, + "browse/test/browser-skills-storage.test.ts": 210, + "browse/test/build-command-response.test.ts": 899, + "browse/test/build.test.ts": 173, + "browse/test/bun-child-config.test.ts": 109, + "browse/test/bun-polyfill.test.ts": 2432, + "browse/test/busy-daemon-iron-rule.test.ts": 7824, + "browse/test/busy-daemon-recovery.test.ts": 707, + "browse/test/cdp-allowlist.test.ts": 61, + "browse/test/cdp-e2e.test.ts": 1145, + "browse/test/cdp-inspector-history-cap.test.ts": 63, + "browse/test/cdp-mutex.test.ts": 1151, + "browse/test/cdp-session-cleanup.test.ts": 64, + "browse/test/claude-bin.test.ts": 56, + "browse/test/cli-chain-stdin.test.ts": 365, + "browse/test/cli-config-mismatch-message.test.ts": 215, + "browse/test/cli-lock.test.ts": 85, + "browse/test/cli-loopback-no-proxy.test.ts": 76, + "browse/test/cli-setsid-daemonize.test.ts": 114, + "browse/test/cli-start-final-healthcheck.test.ts": 139, + "browse/test/cli-supervisor.test.ts": 70, + "browse/test/content-security.test.ts": 1551, + "browse/test/cookie-auth-verification.test.ts": 18096, + "browse/test/cookie-credential-deadline.test.ts": 710, + "browse/test/cookie-database.test.ts": 735, + "browse/test/cookie-fixture-delete-lease.test.ts": 1273, + "browse/test/cookie-import-browser.test.ts": 98, + "browse/test/cookie-import-native-job.test.ts": 127403, + "browse/test/cookie-import-native.test.ts": 9044, + "browse/test/cookie-import-node.test.ts": 5402, + "browse/test/cookie-import-operation.test.ts": 2846, + "browse/test/cookie-import-reliability.test.ts": 1849, + "browse/test/cookie-import-transport.test.ts": 73, + "browse/test/cookie-picker-binding-browser.test.ts": 3657, + "browse/test/cookie-picker-binding.test.ts": 866, + "browse/test/cookie-picker-csrf-browser.test.ts": 1082, + "browse/test/cookie-picker-routes.test.ts": 570, + "browse/test/cookie-picker-ui.test.ts": 42854, + "browse/test/daemon-log-hygiene.test.ts": 85, + "browse/test/daemon-mismatch-refuse.test.ts": 259, + "browse/test/data-platform.test.ts": 110, + "browse/test/domain-skills-e2e.test.ts": 1046, + "browse/test/domain-skills-storage.test.ts": 159, + "browse/test/dotenv-isolation.test.ts": 483, + "browse/test/dual-listener.test.ts": 1209, + "browse/test/dx-polish.test.ts": 62, + "browse/test/error-handling.test.ts": 49, + "browse/test/extension-sender-auth.test.ts": 84, + "browse/test/extension-token.test.ts": 2607, + "browse/test/file-drop.test.ts": 70, + "browse/test/file-permissions.test.ts": 991, + "browse/test/fill-change-event.test.ts": 3014, "browse/test/find-browse.test.ts": 205, - "browse/test/from-file-path-validation.test.ts": 75, - "browse/test/handoff.test.ts": 5180, - "browse/test/headless-gpu-and-reap.test.ts": 558, - "browse/test/headless-launch-options.test.ts": 326, - "browse/test/launch-signal-flags.test.ts": 85, - "browse/test/learnings-injection.test.ts": 1372, - "browse/test/media-extract-unit.test.ts": 82, - "browse/test/memory-command.test.ts": 502, - "browse/test/memory-leak-reproducer.test.ts": 500, - "browse/test/navigation-guard.test.ts": 3054, - "browse/test/output-path-symlinks.test.ts": 32, - "browse/test/pair-agent-optin-gate.test.ts": 387, - "browse/test/path-security-realpath.test.ts": 437, - "browse/test/pdf-flags.test.ts": 32, - "browse/test/playwright-core-patch.test.ts": 27, - "browse/test/poisoned-bundle-probe.test.ts": 250, - "browse/test/process-liveness-windows.test.ts": 182, - "browse/test/proxy-config.test.ts": 56, - "browse/test/proxy-env-daemon.test.ts": 64, - "browse/test/proxy-env.test.ts": 48, - "browse/test/proxy-redact.test.ts": 34, - "browse/test/pty-inject-scan.test.ts": 773, - "browse/test/pty-session-lease.test.ts": 36, - "browse/test/rebrand-signed-bundle.test.ts": 31, - "browse/test/regression-pr1169-pdf-from-file-invalid-json.test.ts": 56, - "browse/test/restart-env.test.ts": 35, - "browse/test/runtime-root-server-bundle.test.ts": 48, - "browse/test/sanitize.test.ts": 30, - "browse/test/screenshot-size-guard.test.ts": 260, - "browse/test/security-adversarial-fixes.test.ts": 41, + "browse/test/from-file-path-validation.test.ts": 102, + "browse/test/handoff.test.ts": 7216, + "browse/test/headless-gpu-and-reap.test.ts": 1282, + "browse/test/headless-launch-options.test.ts": 571, + "browse/test/launch-signal-flags.test.ts": 38, + "browse/test/learnings-injection.test.ts": 2242, + "browse/test/media-extract-unit.test.ts": 61, + "browse/test/memory-command.test.ts": 1103, + "browse/test/memory-leak-reproducer.test.ts": 577, + "browse/test/navigation-guard.test.ts": 3749, + "browse/test/output-path-symlinks.test.ts": 41, + "browse/test/pair-agent-optin-gate.test.ts": 788, + "browse/test/path-security-realpath.test.ts": 681, + "browse/test/pdf-flags.test.ts": 72, + "browse/test/playwright-core-patch.test.ts": 58, + "browse/test/poisoned-bundle-probe.test.ts": 443, + "browse/test/process-liveness-windows.test.ts": 243, + "browse/test/proxy-config.test.ts": 58, + "browse/test/proxy-env-daemon.test.ts": 144, + "browse/test/proxy-env.test.ts": 65, + "browse/test/proxy-redact.test.ts": 40, + "browse/test/pty-inject-scan.test.ts": 1533, + "browse/test/pty-session-lease.test.ts": 51, + "browse/test/rebrand-signed-bundle.test.ts": 55, + "browse/test/regression-pr1169-pdf-from-file-invalid-json.test.ts": 107, + "browse/test/restart-env.test.ts": 73, + "browse/test/runtime-root-server-bundle.test.ts": 96, + "browse/test/sanitize.test.ts": 56, + "browse/test/screenshot-size-guard.test.ts": 436, + "browse/test/security-adversarial-fixes.test.ts": 58, "browse/test/security-adversarial.test.ts": 54, - "browse/test/security-bench.test.ts": 52, - "browse/test/security-classifier-download-cleanup.test.ts": 50, - "browse/test/security-classifier.test.ts": 30, - "browse/test/security-integration.test.ts": 36, - "browse/test/security-live-playwright.test.ts": 951, - "browse/test/security-sidecar-client.test.ts": 59, - "browse/test/security.test.ts": 43, - "browse/test/server-auth.test.ts": 508, - "browse/test/server-factory.test.ts": 2024, - "browse/test/server-flush-trackers.test.ts": 27, - "browse/test/server-launch-resolution.test.ts": 38, - "browse/test/server-lock-errors.test.ts": 81, - "browse/test/server-no-import-side-effects.test.ts": 520, - "browse/test/server-pty-lease-routes.test.ts": 403, - "browse/test/server-route-auth-blackbox.test.ts": 1573, - "browse/test/server-route-dispatch-ratchet.test.ts": 317, - "browse/test/server-sanitize-surrogates.test.ts": 354, - "browse/test/server-tmp-state-path.test.ts": 34, - "browse/test/session-cookie-store.test.ts": 86, - "browse/test/sidebar-tabs.test.ts": 471, - "browse/test/sidebar-ux.test.ts": 412, - "browse/test/sidepanel-patient-autoconnect.test.ts": 30, - "browse/test/sidepanel-reattach.test.ts": 32, - "browse/test/sidepanel-restart-dispose.test.ts": 32, - "browse/test/skill-token.test.ts": 34, - "browse/test/socks-bridge.test.ts": 506, - "browse/test/sse-helpers.test.ts": 175, - "browse/test/sse-session-cookie.test.ts": 34, - "browse/test/state-ttl.test.ts": 25, - "browse/test/stealth-extended.test.ts": 32, - "browse/test/stealth-layer-c.test.ts": 34, - "browse/test/stealth-webdriver.test.ts": 8809, - "browse/test/stop-ack-before-shutdown.test.ts": 177, - "browse/test/stop-dead-daemon.test.ts": 305, - "browse/test/tab-each.test.ts": 50, - "browse/test/tab-external-close-dispatch.test.ts": 816, - "browse/test/tab-external-close.test.ts": 508, - "browse/test/tab-guardrail.test.ts": 565, - "browse/test/tab-isolation.test.ts": 763, - "browse/test/tab-session-frame-detach.test.ts": 35, - "browse/test/telemetry-optout.test.ts": 67, - "browse/test/telemetry.test.ts": 39, - "browse/test/temp-dirs.test.ts": 45, - "browse/test/terminal-agent-detach-reattach.test.ts": 28, - "browse/test/terminal-agent-import.test.ts": 4415, - "browse/test/terminal-agent-keepalive.test.ts": 31, - "browse/test/terminal-agent-native-observation.test.ts": 38, - "browse/test/terminal-agent-owner-watchdog.test.ts": 2297, - "browse/test/terminal-agent-pid-identity.test.ts": 91, - "browse/test/terminal-agent-port-range.test.ts": 41, - "browse/test/terminal-agent-ring-buffer-runtime.test.ts": 50, - "browse/test/terminal-agent-session-routing.test.ts": 33, - "browse/test/terminal-agent-signal-cleanup.test.ts": 39, - "browse/test/terminal-agent-watchdog.test.ts": 34, - "browse/test/terminal-agent.test.ts": 380, - "browse/test/terminal-pty-lifecycle.test.ts": 37, - "browse/test/token-registry.test.ts": 44, - "browse/test/tunnel-gate-unit.test.ts": 345, - "browse/test/watch.test.ts": 365, - "browse/test/welcome-page.test.ts": 37, - "browse/test/windows-spawn-hide.test.ts": 52, - "browse/test/xprotect-heal.test.ts": 301, - "browser-skills/hackernews-frontpage/script.test.ts": 34, - "design/test/brief.test.ts": 28, - "design/test/cli-persistence.test.ts": 6766, - "design/test/cli-version.test.ts": 57, - "design/test/feedback-roundtrip-daemon.test.ts": 171, - "design/test/feedback-roundtrip.test.ts": 3395, - "design/test/image-args.test.ts": 118, - "design/test/image-gen-pairing.test.ts": 133, - "design/test/receipted-fetch.test.ts": 45, - "ios-qa/daemon/test/allowlist.test.ts": 47, - "ios-qa/daemon/test/audit.test.ts": 53, - "ios-qa/daemon/test/auth-mint.test.ts": 41, - "ios-qa/daemon/test/daemon-integration.test.ts": 3980, - "ios-qa/daemon/test/proxy-classify.test.ts": 90, - "ios-qa/daemon/test/proxy-transport.test.ts": 1756, - "ios-qa/daemon/test/session-tokens.test.ts": 31, - "ios-qa/daemon/test/single-instance.test.ts": 30, - "ios-qa/daemon/test/tailscale-localapi.test.ts": 42, - "ios-qa/scripts/gen-accessors.test.ts": 87, - "make-pdf/test/asideClient.test.ts": 47, - "make-pdf/test/cli-args.test.ts": 51, - "make-pdf/test/cli-exit-codes.test.ts": 297, - "make-pdf/test/diagram-prepass.test.ts": 112, - "make-pdf/test/e2e/ci-prereqs.test.ts": 28, - "make-pdf/test/e2e/combined-gate.test.ts": 32, - "make-pdf/test/e2e/diagram-gate.test.ts": 37, - "make-pdf/test/e2e/emoji-gate.test.ts": 37, - "make-pdf/test/e2e/format-gate.test.ts": 40, - "make-pdf/test/e2e/landscape-gate.test.ts": 38, - "make-pdf/test/e2e/toc-gate.test.ts": 37, - "make-pdf/test/image-policy.test.ts": 29, - "make-pdf/test/render-offline-sanitize.test.ts": 47, - "make-pdf/test/render.test.ts": 58, - "make-pdf/test/setup-smoke.test.ts": 54, - "make-pdf/test/toc-anchors.test.ts": 46, - "make-pdf/test/toc-pages.test.ts": 49, - "test/agents-digest.test.ts": 37, - "test/analytics.test.ts": 55, - "test/anthropic-preflight.test.ts": 25, - "test/archaeology-lint.test.ts": 2035, - "test/arm-benchmark-selftest.test.ts": 2788, - "test/arm-setup-smoke-workflow.test.ts": 135, - "test/auq-format-always-loaded.test.ts": 76, - "test/auq-native-capture.test.ts": 323, - "test/auq-parallel.test.ts": 27784, - "test/auq-substance-panel.test.ts": 109, - "test/autoplan-admission.test.ts": 1629, - "test/autoplan-artifact-windows-argv.test.ts": 240, - "test/autoplan-bounded-journal.test.ts": 566, - "test/autoplan-budget.test.ts": 10875, - "test/autoplan-capy-e2e.test.ts": 10262, - "test/autoplan-compaction-copies.test.ts": 282, - "test/autoplan-compaction-preserved-segment.test.ts": 1283, - "test/autoplan-guard-log.test.ts": 72, - "test/autoplan-guard-reasons.test.ts": 28, - "test/autoplan-init.test.ts": 1486, - "test/autoplan-long-session-observer.test.ts": 105, - "test/autoplan-method-read-audit.test.ts": 332, - "test/autoplan-owned-read.test.ts": 4598, - "test/autoplan-owned-state.test.ts": 535, - "test/autoplan-pairing.test.ts": 49, - "test/autoplan-pending-artifact.test.ts": 42, - "test/autoplan-pending-question.test.ts": 4439, - "test/autoplan-phase-handoff.test.ts": 137, - "test/autoplan-phase-observer.test.ts": 46, - "test/autoplan-phase-order.test.ts": 100, - "test/autoplan-public-narration.test.ts": 70, - "test/autoplan-publication-generation.test.ts": 117, - "test/autoplan-publication-guard.test.ts": 5524, - "test/autoplan-publication-hook.test.ts": 3401, - "test/autoplan-publication-payload.test.ts": 2730, - "test/autoplan-record.test.ts": 3410, - "test/autoplan-revise.test.ts": 10957, - "test/autoplan-schema-canary-compare.test.ts": 44, - "test/benchmark-runner.test.ts": 60, - "test/binary-version-probe.test.ts": 260, - "test/binding-template-drift.test.ts": 225, - "test/bootstrap-retention-shard.test.ts": 34, - "test/bootstrap-session-lifecycle.test.ts": 34, - "test/brain-cache-roundtrip.test.ts": 310, - "test/brain-cache-spec.test.ts": 47, - "test/brain-preflight.test.ts": 64, - "test/build-script-shell-compat.test.ts": 36, - "test/bun-subprocess-fd-lifetime.test.ts": 32, - "test/cache-concurrent-refresh.test.ts": 80, - "test/carve-guard-completeness.test.ts": 39, - "test/carve-guards-negative.test.ts": 41, - "test/carve-plan-fixture.test.ts": 510, - "test/carve-section-ordering.test.ts": 52, - "test/carve-section-sharding.test.ts": 2218, - "test/catalog-budget.test.ts": 71, - "test/catalog-mode-full.test.ts": 1095, - "test/catalog-trim.test.ts": 62, - "test/census-red-docs.test.ts": 59, - "test/ceo-barless-submit.test.ts": 114, - "test/ceo-count-ad-v2.test.ts": 62, - "test/ceo-expansion-auq.test.ts": 105, - "test/ceo-expansion-pacing-native.test.ts": 98, - "test/ceo-hold-posture-review.test.ts": 144, - "test/ceo-mode-expansion-disposition.test.ts": 89, - "test/ceo-mode-handoff-hook.test.ts": 204, - "test/ceo-mode-labels-native.test.ts": 68, - "test/ceo-mode-option.test.ts": 444, - "test/ceo-mode-pending-submit.test.ts": 75, - "test/ceo-mode-posture-native.test.ts": 73, - "test/ceo-mode-prerequisite.test.ts": 85, - "test/ceo-mode-routing-fixture.test.ts": 4832, - "test/ceo-plan-mode-fixture.test.ts": 3740, - "test/ceo-posture-packet.test.ts": 88, - "test/ceo-section-loading-fixture.test.ts": 386, - "test/ceo-split-collection.test.ts": 514, - "test/ceo-split-question-policy.test.ts": 417, - "test/ceo-stale-fill-decision.test.ts": 102, - "test/ceo-workflow-clarity.test.ts": 48, - "test/changed-files-union.test.ts": 2889, - "test/ci-image-cli-pin.test.ts": 52, - "test/ci-image-tag-binding.test.ts": 32, - "test/ci-paid-coordination.test.ts": 12119, - "test/claude-code-runner.test.ts": 3912, - "test/claude-code-windows-job.test.ts": 2926, - "test/claude-md-size.test.ts": 28, - "test/claude-overlay-models.test.ts": 61, - "test/claude-producer-stream.test.ts": 34, - "test/claude-provider-keychain.test.ts": 141, - "test/codex-carve-fixture.test.ts": 2111, - "test/codex-generation-model.test.ts": 210, - "test/codex-invocation-sites.test.ts": 438, - "test/codex-resume-flag-semantics.test.ts": 81, - "test/conductor-env-shim.test.ts": 35, - "test/context-budget-ratchet.test.ts": 194, - "test/context-remaining-work-provenance.test.ts": 32, - "test/cookie-workflow-judge-input.test.ts": 1009, - "test/cookie-workflow-manual-review.test.ts": 139, - "test/copilot-windows-bash.test.ts": 40, - "test/cost-known.test.ts": 102, - "test/coverage-audit.test.ts": 6956, - "test/cso-bounded-file.test.ts": 46, - "test/cso-docker-policy.test.ts": 58, - "test/cso-docker-socket-realpath.test.ts": 49, - "test/cso-git-global-config.test.ts": 86, - "test/cso-git-hardening.test.ts": 60, - "test/cso-history.test.ts": 35, - "test/cso-image-provisioning.test.ts": 249, - "test/cso-lease-identity.test.ts": 209, - "test/cso-native-evidence.test.ts": 59, - "test/cso-node-lifecycle-integration.test.ts": 60, - "test/cso-ntfs-fixture.test.ts": 340, - "test/cso-postgresql-policy.test.ts": 50, - "test/cso-preparation-java.test.ts": 47, - "test/cso-preparation.test.ts": 116, - "test/cso-preserved.test.ts": 42, - "test/cso-public-ghcr.test.ts": 43, - "test/cso-requalification.test.ts": 223, - "test/cso-runtime-promotion.test.ts": 270, - "test/cso-scanner-docker-integration.test.ts": 81, - "test/cso-scanners.test.ts": 101, - "test/cso-snapshot-disappearance-race.test.ts": 49, - "test/cso-spec-taxonomy-alignment.test.ts": 46, - "test/cso-staging-run.test.ts": 165, - "test/cso-verifier.test.ts": 63, - "test/cso-windows-build-contract.test.ts": 45, - "test/cso-windows-docker.test.ts": 52, - "test/declared-annotation.test.ts": 61, - "test/defensive-review-framing.test.ts": 35, - "test/dependency-security.test.ts": 137, - "test/deps-smoke.test.ts": 113, - "test/design-board-reload.test.ts": 349, - "test/design-catalog.test.ts": 99, - "test/design-checklist-sync.test.ts": 1801, - "test/design-completion-handoff-scored.test.ts": 61, - "test/design-consultation-contract.test.ts": 54, - "test/design-daemon-windows-identity.test.ts": 93, - "test/design-detect-contract.test.ts": 218, - "test/design-doc-data-guard.test.ts": 59, - "test/design-flag-utils.test.ts": 76, - "test/design-html-section-completion.test.ts": 5695, - "test/design-md.test.ts": 793, - "test/design-printed-paths.test.ts": 109, - "test/design-research-fixture.test.ts": 37, - "test/detector-corpora.test.ts": 423, - "test/detector-corpus-auto-decide-preserved.test.ts": 80, - "test/detector-corpus-plan-ceo-section-loading.test.ts": 75, - "test/detector-corpus-plan-design-review-plan-mode.test.ts": 1420, - "test/detector-corpus-plan-eng-multi-finding-batching.test.ts": 211, - "test/detector-corpus-shared-libs-plan-callers.test.ts": 46, - "test/detector-corpus-ship-docsync-late-result.test.ts": 83, - "test/detector-corpus-ship-docsync-launch-failure.test.ts": 62, - "test/devex-finding-fixture.test.ts": 1778, - "test/devex-peer-comparison-calibration.test.ts": 596, - "test/diagram-render-drift.test.ts": 129, - "test/disabled-dated-record-at.test.ts": 116, - "test/discover-section-templates.test.ts": 36, - "test/dispatch-rules.test.ts": 63, - "test/docsync-authority.test.ts": 3365, - "test/docsync-child-marker.test.ts": 51, - "test/docsync-lifecycle-interface.test.ts": 594, - "test/doctor-components.test.ts": 71, - "test/document-skills-redaction.test.ts": 65, - "test/dom-dump-hygiene.test.ts": 315, - "test/dx-error-contract-grader.test.ts": 64, - "test/dx-error-contract.test.ts": 57, - "test/dx-selected-navigation-ap.test.ts": 131, - "test/e2e-harness-audit.test.ts": 49, - "test/e2e-tier-alignment.test.ts": 172, - "test/eng-batching-current-ledger.test.ts": 248, - "test/eng-batching-native-replay.test.ts": 97, - "test/eng-batching-saved-ledger.test.ts": 684, - "test/eng-count-question-policy.test.ts": 79, - "test/eng-devex-s-count.test.ts": 70, - "test/eng-finding-retry-budget.test.ts": 1506, - "test/eng-first-review.test.ts": 290, - "test/eng-published-navigation.test.ts": 85, - "test/eng-resolution-block-position.test.ts": 265, - "test/eng-review-routing.test.ts": 52, - "test/eng-seeded-completion-ai.test.ts": 16200, - "test/eng-seeded-coverage.test.ts": 1338, - "test/eng-semantic-terminal.test.ts": 110, - "test/env-host-runtime-assets.test.ts": 2602, - "test/eval-budgets-policy.test.ts": 97, - "test/eval-flake-rank.test.ts": 663, - "test/eval-history.test.ts": 2055, - "test/eval-kinds.test.ts": 136, - "test/eval-model.test.ts": 39, - "test/eval-sweep-workflow.test.ts": 105, - "test/evals-workflow-wiring.test.ts": 848, - "test/exit-propagation.test.ts": 413, - "test/extension-pty-inject-invariant.test.ts": 34, - "test/external-host-carve.test.ts": 3163, - "test/flake-ledger.test.ts": 74, - "test/free-home-guard.test.ts": 59, - "test/free-seed-ratchet.test.ts": 305, - "test/free-tests-workflow-wiring.test.ts": 101, - "test/gate-integrity-templates.test.ts": 81, - "test/gbrain-detection-override.test.ts": 1436, - "test/gbrain-exec-invariant.test.ts": 35, - "test/gbrain-guards.test.ts": 56, - "test/gbrain-source-gitignore.test.ts": 582, - "test/gbrain-sources-parse.test.ts": 50, - "test/gbrain-spawn-windows-shell.test.ts": 141, - "test/gbrain-sync-voyage-code-3-integration.test.ts": 55, - "test/gen-skill-docs-checks.test.ts": 18267, - "test/gen-skill-docs-idempotency.test.ts": 3568, - "test/gen-skill-docs-import-purity.test.ts": 76, - "test/gen-skill-docs-prune-stale.test.ts": 688, - "test/generated-docs-fences.test.ts": 1228, - "test/generator-eexist.test.ts": 443, - "test/git-ref-fixture-tripwire.test.ts": 151, - "test/gitattributes-eol.test.ts": 1232, - "test/global-discover.test.ts": 3257, - "test/greptile-early.test.ts": 77, - "test/gstack-ci-gate.test.ts": 65, - "test/gstack-config-gbrain-refresh.test.ts": 1474, - "test/gstack-design-detect-plugins.test.ts": 674, - "test/gstack-doctor.test.ts": 392261, - "test/gstack-home-module-scope.test.ts": 162, - "test/gstack-launch-probe.test.ts": 42, - "test/gstack-post.test.ts": 79, - "test/gstack-regen.test.ts": 16822, - "test/gstack-render-cli.test.ts": 680, - "test/gstack-schema-pack.test.ts": 47, - "test/gstack-upgrade-migration-v1_78_0_0.test.ts": 411, - "test/gstack-wtree-windows-paths.test.ts": 716, - "test/harness-version.test.ts": 292, - "test/helpers-unit.test.ts": 89, - "test/helpers/budget-override.test.ts": 66, - "test/helpers/capture-parity-baseline.test.ts": 333, - "test/helpers/claude-pty-runner.auq.unit.test.ts": 94, - "test/helpers/claude-pty-runner.boundaries.unit.test.ts": 99, - "test/helpers/claude-pty-runner.classify.unit.test.ts": 101, - "test/helpers/claude-pty-runner.launch.unit.test.ts": 48, - "test/helpers/claude-pty-runner.plan-native.unit.test.ts": 85, - "test/helpers/claude-pty-runner.runners.unit.test.ts": 1612, - "test/helpers/claude-pty-runner.scope-gate-floor.unit.test.ts": 81, - "test/helpers/claude-pty-runner.screen.unit.test.ts": 78, - "test/helpers/e2e-gate.unit.test.ts": 46, - "test/helpers/eval-store.test.ts": 2538, - "test/helpers/hermetic-env.test.ts": 12214, - "test/helpers/observability.test.ts": 403, - "test/helpers/providers/gemini.test.ts": 46, - "test/helpers/run-bin.test.ts": 147, - "test/heredoc-pipe-deadlock.test.ts": 526, - "test/honesty-kernel.test.ts": 63, - "test/hook-scripts.test.ts": 135136, - "test/instruction-facts.test.ts": 134, - "test/investigate-freeze-path.test.ts": 20245, - "test/investigate-recipes.test.ts": 51, - "test/ios-debug-bridge-release-guard.test.ts": 43, - "test/ios-qa-daemon-cli.test.ts": 128, - "test/ios-qa-stateserver-hardening.test.ts": 43, - "test/ios-qa-swift-build.test.ts": 49, - "test/ios-qa.test.ts": 411, - "test/is-conductor.test.ts": 39, - "test/jargon-list.test.ts": 46, - "test/judge-calibration.test.ts": 92, - "test/judge-panel-median.test.ts": 89, - "test/land-and-deploy-command-trigger.test.ts": 40, - "test/land-and-deploy-flow.test.ts": 10910, - "test/land-and-deploy-postfail.test.ts": 46, - "test/llm-judge-abort.test.ts": 57, - "test/llm-judge-frontier.test.ts": 158, - "test/llm-judge-stream.test.ts": 75, - "test/llms-txt-shape.test.ts": 73, - "test/maintenance-cli-args.test.ts": 316, - "test/memory-ingest-timeout.test.ts": 62, - "test/migration-checkpoint-ownership.test.ts": 1518, - "test/mktemp-portability.test.ts": 593, - "test/model-catalog-isolation.test.ts": 158, - "test/model-overlays.test.ts": 49, - "test/model-policy-freshness.test.ts": 250, - "test/model-policy.test.ts": 134, - "test/module-size-ratchet.test.ts": 280, - "test/native-auto-decide-pty.test.ts": 12802, - "test/native-auto-decide.test.ts": 356, - "test/no-quoted-tilde-assignments.test.ts": 55, - "test/no-suicide-exit.test.ts": 181, - "test/office-hours-budget.test.ts": 75, - "test/office-hours-open-question-form.test.ts": 37, - "test/office-hours-phase4-caller.test.ts": 81, - "test/office-hours-review.test.ts": 1891, - "test/office-posture-recording.test.ts": 85, - "test/one-way-doors.test.ts": 47, - "test/open-gstack-browser-liveness.test.ts": 34, - "test/openclaw-native-skills.test.ts": 30, - "test/osv-config-wiring.test.ts": 48, - "test/osv-ignore-expiry.test.ts": 61, - "test/outside-review-severity.test.ts": 168, - "test/outside-review-unparsed-severity.test.ts": 292, - "test/outside-send-boundary.test.ts": 499, - "test/outside-voice-evidence.test.ts": 200, - "test/outside-voice-failure-policy.test.ts": 233, - "test/outside-voice-fixture.test.ts": 1585, - "test/outside-voice-native-deadline.test.ts": 69, - "test/outside-voice-routing.test.ts": 1710, - "test/overlay-lifecycle.test.ts": 402, - "test/overlay-measurement.test.ts": 631, - "test/overlay-recording-order.test.ts": 53, - "test/overlay-sdk-cancel-eof.test.ts": 115, - "test/paid-free-boundary.test.ts": 9506, - "test/paid-orphan-tripwire.test.ts": 174, - "test/paid-overlay-scheduling.test.ts": 1395, - "test/paid-pr-profile.test.ts": 3674, - "test/paid-report-census.test.ts": 237, - "test/paid-report-fail-open.test.ts": 2098, - "test/paid-retry-supervision.test.ts": 2530, - "test/paid-run-manifest.test.ts": 5052, - "test/paid-selection-propagation.test.ts": 88, - "test/paid-shard-settlement.test.ts": 1565, - "test/paid-shards.test.ts": 9945, - "test/paid-slice-deadline.test.ts": 1594, - "test/pair-agent-token-hygiene.test.ts": 44, - "test/parity-baseline-integrity.test.ts": 53, - "test/parity-sectioned.test.ts": 54, - "test/parity-suite.test.ts": 233, - "test/path-spelling.test.ts": 1422, - "test/pending-question-completion.test.ts": 276, - "test/periodic-exclude-policy.test.ts": 278, - "test/plan-count-checkbox.test.ts": 74, - "test/plan-count-completion.test.ts": 440, - "test/plan-count-cropped-wrap.test.ts": 140, - "test/plan-count-cross-cwd-ancestry.test.ts": 249, - "test/plan-count-design-ui-recovery.test.ts": 93, - "test/plan-count-dx-handoff.test.ts": 201, - "test/plan-count-empty-review.test.ts": 74, - "test/plan-count-history.test.ts": 46, - "test/plan-count-long-edit.test.ts": 87, - "test/plan-count-native-input.test.ts": 95, - "test/plan-count-owned-permission.test.ts": 88, - "test/plan-count-pending-exit.test.ts": 166, - "test/plan-count-prerequisite.test.ts": 102, - "test/plan-count-preview-footer.test.ts": 82, - "test/plan-count-session-cwd.test.ts": 162, - "test/plan-count-timeout.test.ts": 47, - "test/plan-count-transcript.test.ts": 145, - "test/plan-count-truncated-border.test.ts": 77, - "test/plan-count-truncated-question.test.ts": 76, - "test/plan-create-combined-permission.test.ts": 242, - "test/plan-create-permission.test.ts": 153, - "test/plan-design-floor-fixture.test.ts": 1393, - "test/plan-design-sdk-fixture.test.ts": 2646, - "test/plan-design-with-ui-fixture.test.ts": 6444, - "test/plan-edit-cropped-permission.test.ts": 270, - "test/plan-eng-resume.test.ts": 44, - "test/plan-floor-dx-actor.test.ts": 144, - "test/plan-floor-edit-wrap.test.ts": 113, - "test/plan-floor-review.test.ts": 80, - "test/plan-floor-target.test.ts": 14476, - "test/plan-format-approach-prompt.test.ts": 34, - "test/plan-format-kind-note.test.ts": 37, - "test/plan-pending-question-pty.test.ts": 76, - "test/plan-review-board-feedback.test.ts": 10440, - "test/plan-review-calibration.test.ts": 688, - "test/plan-review-decisions.test.ts": 335, - "test/plan-review-report-recording.test.ts": 8758, - "test/plan-scope-selection.test.ts": 3675, - "test/plan-skill-question-hook-scope.test.ts": 6962, - "test/plan-skill-read-permission.test.ts": 110, - "test/plan-skill-webfetch-permission.test.ts": 3898, - "test/platform-qualification-workflow.test.ts": 40, - "test/post-rename-doc-regen.test.ts": 59, - "test/pr-direct-cases.test.ts": 5718, - "test/pr-shared-input-selection.test.ts": 1298, - "test/pr-title-sync-workflow-safety.test.ts": 47, - "test/pregate-import-graph-paths.test.ts": 8100, - "test/pregate-manifest-paths.test.ts": 6068, - "test/pregate-wtree-refusal.test.ts": 6553, - "test/prompt-audit-manifest.test.ts": 536, - "test/prosons-neutral-posture.test.ts": 40, - "test/provider-model-defaults.test.ts": 64, - "test/pty-askuserquestion-single-line.test.ts": 79, - "test/pty-numbered-option-indent-native.test.ts": 86, - "test/pty-option-selection.test.ts": 47, - "test/pty-output-wake.test.ts": 3285, - "test/pty-screen-session.test.ts": 42, - "test/pty-screen-supervision.test.ts": 10134, - "test/pty-screen-unicode-ap.test.ts": 174, - "test/pty-screen.test.ts": 144, - "test/pty-skill-seeding-wiring.test.ts": 391, - "test/pty-trust-dialog.test.ts": 44, - "test/published-text.test.ts": 58, - "test/qa-browser-preservation.test.ts": 10434, - "test/qa-bugs-fixture.test.ts": 62, - "test/qa-caller-authority.test.ts": 176, - "test/qa-caller-freshness-order.test.ts": 103, - "test/qa-caller-report-observer.test.ts": 57, - "test/qa-checkpoint-evidence.test.ts": 239, - "test/qa-deadline-publication-observer.test.ts": 63, - "test/qa-deadline-selection.test.ts": 81, - "test/qa-deadline.test.ts": 21660, - "test/qa-evidence-selection.test.ts": 125, - "test/qa-evidence.test.ts": 13989, - "test/qa-fix-loop-fixture.test.ts": 5560, - "test/qa-health-rubric.test.ts": 36, - "test/qa-only-browser-probe.test.ts": 1977, - "test/qa-probe-gates.test.ts": 106, - "test/qa-report-fields.test.ts": 43, - "test/qa-supervision-selection.test.ts": 69, - "test/question-tuning-registry-path.test.ts": 49, - "test/readme-skill-lists.test.ts": 43, - "test/redact-context-exemptions.test.ts": 77, - "test/redact-doc-resolver.test.ts": 94, - "test/redact-dotenv-filename-false-positive.test.ts": 77, - "test/redact-engine-autoredact.test.ts": 93, - "test/redact-parcel-id-false-positive.test.ts": 58, - "test/redact-pattern-lint.test.ts": 52, - "test/redact-prepush-emails.test.ts": 37019, - "test/redact-prepush-target.test.ts": 42318, - "test/redact-span-binding.test.ts": 81, - "test/redact-unicode-offsets.test.ts": 162, - "test/regression-1539-review-self-verify.test.ts": 41, - "test/regression-1834-retro-recommendations.test.ts": 31, - "test/regression-issue2091-bsd-mktemp.test.ts": 513, - "test/regression-pr1169-mktemp-fallbacks.test.ts": 130, - "test/regression-transcript-frontmatter-fence.test.ts": 74, - "test/remote-identity-vectors.test.ts": 212, - "test/replay-pr-selection.test.ts": 1401, - "test/resolvers-gbrain-put-rewrite.test.ts": 75, - "test/resolvers-gbrain-save-results.test.ts": 73, - "test/retired-commands.test.ts": 88, - "test/review-army-budget.test.ts": 4143, - "test/review-base-refresh.test.ts": 1423, - "test/review-consensus-lifecycle.test.ts": 78, - "test/review-count-markdown.test.ts": 163, - "test/review-entry-and-design-clarity-au.test.ts": 106, - "test/review-enum-lifecycle.test.ts": 73, - "test/review-finalization-budget.test.ts": 5566, - "test/review-n-plus-one-contract.test.ts": 3812, - "test/review-quality-provenance.test.ts": 76, - "test/router-disabled-skills.test.ts": 1094, - "test/run-shard-child.test.ts": 1345, - "test/runtime-pins.test.ts": 172, - "test/safety-rule-registry.test.ts": 74, - "test/salience-allowlist.test.ts": 61, - "test/schema-version-migration.test.ts": 290, - "test/section-manifest-consistency.test.ts": 58, - "test/session-runner-browse-errors.test.ts": 1498, - "test/session-runner-stream-lifecycle.test.ts": 338, - "test/session-runner-timeout.test.ts": 54, - "test/session-runner-tools.test.ts": 161, - "test/setup-alias-name-uniqueness.test.ts": 17731, - "test/setup-browser-hint.test.ts": 567, - "test/setup-bun-cmd-and-pipe-bugs.test.ts": 499, - "test/setup-cleanup-orphans.test.ts": 306, - "test/setup-codex-design-pdf-runtime.test.ts": 81, - "test/setup-codex-model.test.ts": 93, - "test/setup-codex-scope-review-boundaries.test.ts": 473, - "test/setup-cso-windows-probe.test.ts": 357, - "test/setup-cursor-legacy-skills.test.ts": 474, - "test/setup-deploy-key-presence.test.ts": 148, - "test/setup-deploy-not-deployed.test.ts": 41, - "test/setup-deploy-vercel-trigger.test.ts": 57, - "test/setup-gbrain-bin-invocation-paths.test.ts": 79, - "test/setup-gbrain-path4-caller.test.ts": 3645, - "test/setup-gbrain-path4-structure.test.ts": 75, - "test/setup-gbrain-remote-caller.test.ts": 9128, - "test/setup-help.test.ts": 515, - "test/setup-hook-syntax-gate.test.ts": 98800, - "test/setup-playwright-platform.test.ts": 6847, - "test/setup-prune-stale-generated.test.ts": 2751, - "test/setup-sections-linking.test.ts": 46, - "test/setup-skill-copies.test.ts": 129, - "test/setup-staged-copy.test.ts": 456, - "test/setup-windows-fallback.test.ts": 141, - "test/setup-windows-rerun-refresh.test.ts": 1408, - "test/shard-cli-parity.test.ts": 220, - "test/shared-libs-cancellation.test.ts": 257, - "test/shared-libs-evidence.test.ts": 89, - "test/shared-libs-plan-actor.test.ts": 139, - "test/shared-libs-rendering.test.ts": 2425, - "test/shared-libs-source-reads.test.ts": 15268, - "test/ship-apple-gate.test.ts": 64, - "test/ship-control-flow.test.ts": 3997, - "test/ship-coverage-audit-af.test.ts": 115, - "test/ship-greptile-early.test.ts": 314, - "test/ship-greptile-pr-check.test.ts": 1145, - "test/ship-hook-windows-paths.test.ts": 2185, - "test/ship-measure-bar.test.ts": 222, - "test/ship-measure-loop-skill.test.ts": 70, - "test/ship-pr-liveness-policy.test.ts": 50, - "test/ship-publication-gates.test.ts": 18722, - "test/ship-reentry-gates.test.ts": 65, - "test/ship-review-loop.test.ts": 59, - "test/ship-section-fixture.test.ts": 1638, - "test/ship-skip-requeue.test.ts": 318, - "test/ship-template-redaction.test.ts": 42, - "test/ship-test-detection-markers.test.ts": 1429, - "test/ship-triage-labels.test.ts": 40, - "test/ship-workflow-clarity.test.ts": 92, - "test/shortcut-debt-ledger.test.ts": 180, - "test/sideways-review.test.ts": 72, - "test/skill-browse-state-extraction.test.ts": 215, - "test/skill-budget-regression.test.ts": 407, - "test/skill-check-driver.test.ts": 403, + "browse/test/security-bench.test.ts": 45, + "browse/test/security-classifier-download-cleanup.test.ts": 68, + "browse/test/security-classifier.test.ts": 46, + "browse/test/security-integration.test.ts": 67, + "browse/test/security-live-playwright.test.ts": 1859, + "browse/test/security-sidecar-client.test.ts": 85, + "browse/test/security.test.ts": 52, + "browse/test/server-auth.test.ts": 1043, + "browse/test/server-factory.test.ts": 3576, + "browse/test/server-flush-trackers.test.ts": 43, + "browse/test/server-launch-resolution.test.ts": 61, + "browse/test/server-lock-errors.test.ts": 121, + "browse/test/server-no-import-side-effects.test.ts": 729, + "browse/test/server-pty-lease-routes.test.ts": 765, + "browse/test/server-route-auth-blackbox.test.ts": 3016, + "browse/test/server-route-dispatch-ratchet.test.ts": 643, + "browse/test/server-sanitize-surrogates.test.ts": 780, + "browse/test/server-tmp-state-path.test.ts": 57, + "browse/test/session-cookie-store.test.ts": 138, + "browse/test/sidebar-tabs.test.ts": 842, + "browse/test/sidebar-ux.test.ts": 768, + "browse/test/sidepanel-patient-autoconnect.test.ts": 54, + "browse/test/sidepanel-reattach.test.ts": 57, + "browse/test/sidepanel-restart-dispose.test.ts": 54, + "browse/test/skill-token.test.ts": 66, + "browse/test/socks-bridge.test.ts": 605, + "browse/test/sse-helpers.test.ts": 205, + "browse/test/sse-session-cookie.test.ts": 63, + "browse/test/state-ttl.test.ts": 47, + "browse/test/stealth-extended.test.ts": 45, + "browse/test/stealth-layer-c.test.ts": 55, + "browse/test/stealth-webdriver.test.ts": 12124, + "browse/test/stop-ack-before-shutdown.test.ts": 174, + "browse/test/stop-dead-daemon.test.ts": 529, + "browse/test/tab-each.test.ts": 69, + "browse/test/tab-external-close-dispatch.test.ts": 1471, + "browse/test/tab-external-close.test.ts": 1241, + "browse/test/tab-guardrail.test.ts": 1097, + "browse/test/tab-isolation.test.ts": 800, + "browse/test/tab-session-frame-detach.test.ts": 55, + "browse/test/telemetry-optout.test.ts": 149, + "browse/test/telemetry.test.ts": 96, + "browse/test/temp-dirs.test.ts": 60, + "browse/test/terminal-agent-detach-reattach.test.ts": 50, + "browse/test/terminal-agent-import.test.ts": 4478, + "browse/test/terminal-agent-keepalive.test.ts": 56, + "browse/test/terminal-agent-native-observation.test.ts": 55, + "browse/test/terminal-agent-owner-watchdog.test.ts": 3577, + "browse/test/terminal-agent-pid-identity.test.ts": 137, + "browse/test/terminal-agent-port-range.test.ts": 52, + "browse/test/terminal-agent-ring-buffer-runtime.test.ts": 81, + "browse/test/terminal-agent-session-routing.test.ts": 41, + "browse/test/terminal-agent-signal-cleanup.test.ts": 59, + "browse/test/terminal-agent-watchdog.test.ts": 41, + "browse/test/terminal-agent.test.ts": 827, + "browse/test/terminal-pty-lifecycle.test.ts": 45, + "browse/test/token-registry.test.ts": 59, + "browse/test/tunnel-gate-unit.test.ts": 715, + "browse/test/watch.test.ts": 584, + "browse/test/welcome-page.test.ts": 49, + "browse/test/windows-spawn-hide.test.ts": 76, + "browse/test/xprotect-heal.test.ts": 590, + "browser-skills/hackernews-frontpage/script.test.ts": 62, + "design/test/brief.test.ts": 95, + "design/test/cli-persistence.test.ts": 7092, + "design/test/cli-version.test.ts": 103, + "design/test/feedback-roundtrip-daemon.test.ts": 404, + "design/test/feedback-roundtrip.test.ts": 3654, + "design/test/image-args.test.ts": 82, + "design/test/image-gen-pairing.test.ts": 118, + "design/test/receipted-fetch.test.ts": 53, + "ios-qa/daemon/test/allowlist.test.ts": 73, + "ios-qa/daemon/test/audit.test.ts": 50, + "ios-qa/daemon/test/auth-mint.test.ts": 60, + "ios-qa/daemon/test/daemon-integration.test.ts": 4014, + "ios-qa/daemon/test/proxy-classify.test.ts": 52, + "ios-qa/daemon/test/proxy-transport.test.ts": 1854, + "ios-qa/daemon/test/session-tokens.test.ts": 47, + "ios-qa/daemon/test/single-instance.test.ts": 50, + "ios-qa/daemon/test/tailscale-localapi.test.ts": 63, + "ios-qa/scripts/gen-accessors.test.ts": 142, + "make-pdf/test/asideClient.test.ts": 54, + "make-pdf/test/cli-args.test.ts": 40, + "make-pdf/test/cli-exit-codes.test.ts": 596, + "make-pdf/test/diagram-prepass.test.ts": 205, + "make-pdf/test/docx-image-size.test.ts": 1418, + "make-pdf/test/e2e/ci-prereqs.test.ts": 54, + "make-pdf/test/e2e/combined-gate.test.ts": 71, + "make-pdf/test/e2e/diagram-gate.test.ts": 67, + "make-pdf/test/e2e/emoji-gate.test.ts": 61, + "make-pdf/test/e2e/format-gate.test.ts": 63, + "make-pdf/test/e2e/landscape-gate.test.ts": 64, + "make-pdf/test/e2e/toc-gate.test.ts": 68, + "make-pdf/test/image-policy.test.ts": 55, + "make-pdf/test/render-offline-sanitize.test.ts": 86, + "make-pdf/test/render.test.ts": 108, + "make-pdf/test/setup-smoke.test.ts": 127, + "make-pdf/test/toc-anchors.test.ts": 76, + "make-pdf/test/toc-pages.test.ts": 130, + "test/agents-digest.test.ts": 93, + "test/analytics.test.ts": 72, + "test/anthropic-preflight.test.ts": 54, + "test/archaeology-lint.test.ts": 3760, + "test/arm-benchmark-selftest.test.ts": 4277, + "test/arm-setup-smoke-workflow.test.ts": 200, + "test/auq-format-always-loaded.test.ts": 105, + "test/auq-native-capture.test.ts": 588, + "test/auq-parallel.test.ts": 42691, + "test/auq-substance-panel.test.ts": 226, + "test/autoplan-admission.test.ts": 2528, + "test/autoplan-artifact-windows-argv.test.ts": 352, + "test/autoplan-bounded-journal.test.ts": 904, + "test/autoplan-budget.test.ts": 20449, + "test/autoplan-capy-e2e.test.ts": 19388, + "test/autoplan-compaction-copies.test.ts": 465, + "test/autoplan-compaction-preserved-segment.test.ts": 2089, + "test/autoplan-guard-log.test.ts": 132, + "test/autoplan-guard-pty-fixture.test.ts": 72, + "test/autoplan-guard-reasons.test.ts": 48, + "test/autoplan-init.test.ts": 2154, + "test/autoplan-long-session-observer.test.ts": 165, + "test/autoplan-method-read-audit.test.ts": 642, + "test/autoplan-owned-read.test.ts": 5990, + "test/autoplan-owned-state.test.ts": 913, + "test/autoplan-pairing.test.ts": 79, + "test/autoplan-pending-artifact.test.ts": 46, + "test/autoplan-pending-question.test.ts": 4700, + "test/autoplan-phase-handoff.test.ts": 244, + "test/autoplan-phase-observer.test.ts": 84, + "test/autoplan-phase-order.test.ts": 177, + "test/autoplan-public-narration.test.ts": 150, + "test/autoplan-publication-generation.test.ts": 263, + "test/autoplan-publication-guard.test.ts": 8675, + "test/autoplan-publication-hook.test.ts": 3772, + "test/autoplan-publication-payload.test.ts": 3045, + "test/autoplan-record.test.ts": 6031, + "test/autoplan-revise.test.ts": 18079, + "test/autoplan-schema-canary-compare.test.ts": 63, + "test/benchmark-runner.test.ts": 82, + "test/binary-version-probe.test.ts": 377, + "test/binding-template-drift.test.ts": 311, + "test/bootstrap-retention-shard.test.ts": 55, + "test/bootstrap-session-lifecycle.test.ts": 59, + "test/brain-cache-roundtrip.test.ts": 690, + "test/brain-cache-spec.test.ts": 78, + "test/brain-preflight.test.ts": 95, + "test/build-script-shell-compat.test.ts": 52, + "test/bun-subprocess-fd-lifetime.test.ts": 45, + "test/cache-concurrent-refresh.test.ts": 125, + "test/carve-guard-completeness.test.ts": 60, + "test/carve-guards-negative.test.ts": 68, + "test/carve-plan-fixture.test.ts": 769, + "test/carve-section-ordering.test.ts": 84, + "test/carve-section-sharding.test.ts": 3822, + "test/catalog-budget.test.ts": 121, + "test/catalog-mode-full.test.ts": 1860, + "test/catalog-trim.test.ts": 104, + "test/census-red-docs.test.ts": 98, + "test/ceo-barless-submit.test.ts": 168, + "test/ceo-count-ad-v2.test.ts": 77, + "test/ceo-expansion-auq.test.ts": 182, + "test/ceo-expansion-pacing-native.test.ts": 151, + "test/ceo-hold-posture-review.test.ts": 259, + "test/ceo-mode-expansion-disposition.test.ts": 146, + "test/ceo-mode-handoff-hook.test.ts": 318, + "test/ceo-mode-labels-native.test.ts": 115, + "test/ceo-mode-option.test.ts": 741, + "test/ceo-mode-pending-submit.test.ts": 114, + "test/ceo-mode-posture-native.test.ts": 120, + "test/ceo-mode-prerequisite.test.ts": 115, + "test/ceo-mode-routing-fixture.test.ts": 8146, + "test/ceo-plan-mode-fixture.test.ts": 6398, + "test/ceo-posture-packet.test.ts": 135, + "test/ceo-section-loading-fixture.test.ts": 727, + "test/ceo-split-collection.test.ts": 838, + "test/ceo-split-question-policy.test.ts": 712, + "test/ceo-stale-fill-decision.test.ts": 166, + "test/ceo-workflow-clarity.test.ts": 65, + "test/changed-files-union.test.ts": 5182, + "test/ci-image-cli-pin.test.ts": 78, + "test/ci-image-tag-binding.test.ts": 49, + "test/ci-paid-coordination.test.ts": 27144, + "test/claude-code-runner.test.ts": 5500, + "test/claude-code-windows-job.test.ts": 3345, + "test/claude-md-size.test.ts": 53, + "test/claude-overlay-models.test.ts": 173, + "test/claude-producer-stream.test.ts": 100, + "test/claude-provider-keychain.test.ts": 189, + "test/codex-carve-fixture.test.ts": 4731, + "test/codex-generation-model.test.ts": 370, + "test/codex-invocation-sites.test.ts": 783, + "test/codex-resume-flag-semantics.test.ts": 103, + "test/compare-pr-version.test.ts": 476, + "test/conductor-env-shim.test.ts": 62, + "test/context-budget-ratchet.test.ts": 415, + "test/context-remaining-work-provenance.test.ts": 59, + "test/cookie-workflow-judge-input.test.ts": 4451, + "test/cookie-workflow-manual-review.test.ts": 279, + "test/copilot-windows-bash.test.ts": 53, + "test/cost-known.test.ts": 172, + "test/coverage-audit.test.ts": 11104, + "test/cso-bounded-file.test.ts": 72, + "test/cso-docker-policy.test.ts": 114, + "test/cso-docker-socket-realpath.test.ts": 141, + "test/cso-git-global-config.test.ts": 176, + "test/cso-git-hardening.test.ts": 94, + "test/cso-history.test.ts": 56, + "test/cso-image-provisioning.test.ts": 333, + "test/cso-lease-identity.test.ts": 486, + "test/cso-native-evidence.test.ts": 129, + "test/cso-node-lifecycle-integration.test.ts": 134, + "test/cso-ntfs-fixture.test.ts": 824, + "test/cso-postgresql-policy.test.ts": 98, + "test/cso-preparation-java.test.ts": 90, + "test/cso-preparation.test.ts": 244, + "test/cso-preserved.test.ts": 74, + "test/cso-public-ghcr.test.ts": 80, + "test/cso-requalification.test.ts": 231, + "test/cso-runtime-promotion.test.ts": 453, + "test/cso-scanner-docker-integration.test.ts": 155, + "test/cso-scanners.test.ts": 190, + "test/cso-snapshot-disappearance-race.test.ts": 96, + "test/cso-spec-taxonomy-alignment.test.ts": 58, + "test/cso-staging-run.test.ts": 271, + "test/cso-verifier.test.ts": 151, + "test/cso-windows-build-contract.test.ts": 86, + "test/cso-windows-docker.test.ts": 81, + "test/declared-annotation.test.ts": 81, + "test/defensive-review-framing.test.ts": 71, + "test/dependency-security.test.ts": 196, + "test/deps-smoke.test.ts": 202, + "test/design-board-reload.test.ts": 350, + "test/design-catalog.test.ts": 161, + "test/design-checklist-sync.test.ts": 2242, + "test/design-completion-handoff-scored.test.ts": 133, + "test/design-consultation-contract.test.ts": 173, + "test/design-daemon-windows-identity.test.ts": 188, + "test/design-detect-contract.test.ts": 320, + "test/design-doc-data-guard.test.ts": 64, + "test/design-flag-utils.test.ts": 138, + "test/design-html-section-completion.test.ts": 11306, + "test/design-md.test.ts": 1597, + "test/design-printed-paths.test.ts": 242, + "test/design-research-fixture.test.ts": 67, + "test/detector-corpora.test.ts": 1058, + "test/detector-corpus-auto-decide-preserved.test.ts": 128, + "test/detector-corpus-plan-ceo-section-loading.test.ts": 152, + "test/detector-corpus-plan-design-review-plan-mode.test.ts": 2707, + "test/detector-corpus-plan-eng-multi-finding-batching.test.ts": 687, + "test/detector-corpus-shared-libs-plan-callers.test.ts": 91, + "test/detector-corpus-ship-docsync-late-result.test.ts": 147, + "test/detector-corpus-ship-docsync-launch-failure.test.ts": 142, + "test/devex-finding-fixture.test.ts": 2989, + "test/devex-peer-comparison-calibration.test.ts": 924, + "test/diagram-render-drift.test.ts": 173, + "test/disabled-dated-record-at.test.ts": 194, + "test/discover-section-templates.test.ts": 66, + "test/dispatch-rules.test.ts": 109, + "test/docsync-authority.test.ts": 6355, + "test/docsync-child-marker.test.ts": 99, + "test/docsync-lifecycle-interface.test.ts": 1149, + "test/doctor-components.test.ts": 128, + "test/document-skills-redaction.test.ts": 130, + "test/dom-dump-hygiene.test.ts": 654, + "test/dx-error-contract-grader.test.ts": 77, + "test/dx-error-contract.test.ts": 102, + "test/dx-selected-navigation-ap.test.ts": 224, + "test/e2e-harness-audit.test.ts": 102, + "test/e2e-tier-alignment.test.ts": 516, + "test/eng-batching-current-ledger.test.ts": 607, + "test/eng-batching-native-replay.test.ts": 206, + "test/eng-batching-saved-ledger.test.ts": 1530, + "test/eng-count-question-policy.test.ts": 133, + "test/eng-devex-s-count.test.ts": 144, + "test/eng-finding-retry-budget.test.ts": 1916, + "test/eng-first-review.test.ts": 665, + "test/eng-published-navigation.test.ts": 179, + "test/eng-resolution-block-position.test.ts": 658, + "test/eng-review-routing.test.ts": 116, + "test/eng-seeded-completion-ai.test.ts": 15107, + "test/eng-seeded-coverage.test.ts": 4674, + "test/eng-semantic-terminal.test.ts": 272, + "test/env-host-runtime-assets.test.ts": 4927, + "test/eval-budgets-policy.test.ts": 471, + "test/eval-flake-rank.test.ts": 1938, + "test/eval-history.test.ts": 3522, + "test/eval-kinds.test.ts": 122, + "test/eval-model.test.ts": 47, + "test/eval-sweep-workflow.test.ts": 55, + "test/evals-workflow-wiring.test.ts": 694, + "test/exit-propagation.test.ts": 446, + "test/extension-pty-inject-invariant.test.ts": 52, + "test/external-host-carve.test.ts": 6155, + "test/flake-ledger.test.ts": 161, + "test/free-home-guard.test.ts": 88, + "test/free-seed-ratchet.test.ts": 788, + "test/free-tests-workflow-wiring.test.ts": 172, + "test/gate-cost.test.ts": 4110, + "test/gate-integrity-templates.test.ts": 97, + "test/gbrain-detection-override.test.ts": 2586, + "test/gbrain-exec-invariant.test.ts": 59, + "test/gbrain-guards.test.ts": 86, + "test/gbrain-source-gitignore.test.ts": 1151, + "test/gbrain-sources-parse.test.ts": 72, + "test/gbrain-spawn-windows-shell.test.ts": 211, + "test/gbrain-sync-voyage-code-3-integration.test.ts": 86, + "test/gen-skill-docs-checks.test.ts": 40966, + "test/gen-skill-docs-idempotency.test.ts": 6547, + "test/gen-skill-docs-import-purity.test.ts": 143, + "test/gen-skill-docs-prune-stale.test.ts": 1297, + "test/generated-docs-fences.test.ts": 1489, + "test/generator-eexist.test.ts": 954, + "test/git-ref-fixture-tripwire.test.ts": 367, + "test/gitattributes-eol.test.ts": 3037, + "test/global-discover.test.ts": 6207, + "test/greptile-early.test.ts": 110, + "test/gstack-ci-gate.test.ts": 99, + "test/gstack-config-gbrain-refresh.test.ts": 2123, + "test/gstack-design-detect-plugins.test.ts": 1090, + "test/gstack-doctor.test.ts": 647123, + "test/gstack-home-module-scope.test.ts": 318, + "test/gstack-launch-probe.test.ts": 62, + "test/gstack-post.test.ts": 128, + "test/gstack-regen.test.ts": 38964, + "test/gstack-render-cli.test.ts": 1029, + "test/gstack-schema-pack.test.ts": 64, + "test/gstack-upgrade-migration-v1_78_0_0.test.ts": 634, + "test/gstack-wtree-windows-paths.test.ts": 1325, + "test/harness-version.test.ts": 492, + "test/helpers-unit.test.ts": 124, + "test/helpers/budget-override.test.ts": 80, + "test/helpers/capture-parity-baseline.test.ts": 555, + "test/helpers/claude-pty-runner.auq.unit.test.ts": 165, + "test/helpers/claude-pty-runner.boundaries.unit.test.ts": 178, + "test/helpers/claude-pty-runner.classify.unit.test.ts": 154, + "test/helpers/claude-pty-runner.launch.unit.test.ts": 64, + "test/helpers/claude-pty-runner.plan-native.unit.test.ts": 124, + "test/helpers/claude-pty-runner.runners.unit.test.ts": 2609, + "test/helpers/claude-pty-runner.scope-gate-floor.unit.test.ts": 150, + "test/helpers/claude-pty-runner.screen.unit.test.ts": 149, + "test/helpers/e2e-gate.unit.test.ts": 83, + "test/helpers/eval-store.test.ts": 4830, + "test/helpers/hermetic-env.test.ts": 20821, + "test/helpers/observability.test.ts": 814, + "test/helpers/providers/gemini.test.ts": 90, + "test/helpers/run-bin.test.ts": 250, + "test/heredoc-pipe-deadlock.test.ts": 995, + "test/honesty-kernel.test.ts": 120, + "test/hook-scripts.test.ts": 233104, + "test/instruction-facts.test.ts": 178, + "test/investigate-freeze-path.test.ts": 33705, + "test/investigate-recipes.test.ts": 69, + "test/ios-debug-bridge-release-guard.test.ts": 56, + "test/ios-qa-daemon-cli.test.ts": 249, + "test/ios-qa-stateserver-hardening.test.ts": 61, + "test/ios-qa-swift-build.test.ts": 84, + "test/ios-qa.test.ts": 326, + "test/is-conductor.test.ts": 57, + "test/jargon-list.test.ts": 67, + "test/judge-calibration.test.ts": 164, + "test/judge-panel-median.test.ts": 115, + "test/land-and-deploy-command-trigger.test.ts": 53, + "test/land-and-deploy-flow.test.ts": 17992, + "test/land-and-deploy-postfail.test.ts": 71, + "test/llm-judge-abort.test.ts": 91, + "test/llm-judge-frontier.test.ts": 328, + "test/llm-judge-stream.test.ts": 128, + "test/llms-txt-shape.test.ts": 107, + "test/maintenance-cli-args.test.ts": 635, + "test/memory-ingest-timeout.test.ts": 101, + "test/migration-checkpoint-ownership.test.ts": 2348, + "test/mktemp-portability.test.ts": 987, + "test/model-catalog-isolation.test.ts": 289, + "test/model-overlays.test.ts": 94, + "test/model-policy-freshness.test.ts": 403, + "test/model-policy.test.ts": 241, + "test/module-size-ratchet.test.ts": 607, + "test/native-auto-decide-pty.test.ts": 13810, + "test/native-auto-decide.test.ts": 697, + "test/no-quoted-tilde-assignments.test.ts": 78, + "test/no-suicide-exit.test.ts": 371, + "test/office-hours-budget.test.ts": 120, + "test/office-hours-open-question-form.test.ts": 59, + "test/office-hours-phase4-caller.test.ts": 114, + "test/office-hours-review.test.ts": 3336, + "test/office-posture-recording.test.ts": 134, + "test/one-way-doors.test.ts": 56, + "test/open-gstack-browser-liveness.test.ts": 50, + "test/openclaw-native-skills.test.ts": 51, + "test/osv-config-wiring.test.ts": 64, + "test/osv-ignore-expiry.test.ts": 68, + "test/outside-review-severity.test.ts": 253, + "test/outside-review-unparsed-severity.test.ts": 506, + "test/outside-send-boundary.test.ts": 1049, + "test/outside-voice-evidence.test.ts": 360, + "test/outside-voice-failure-policy.test.ts": 470, + "test/outside-voice-fixture.test.ts": 2897, + "test/outside-voice-native-deadline.test.ts": 129, + "test/outside-voice-routing.test.ts": 3475, + "test/overlay-lifecycle.test.ts": 618, + "test/overlay-measurement.test.ts": 1097, + "test/overlay-recording-order.test.ts": 156, + "test/overlay-sdk-cancel-eof.test.ts": 233, + "test/paid-free-boundary.test.ts": 12473, + "test/paid-orphan-tripwire.test.ts": 386, + "test/paid-overlay-scheduling.test.ts": 2314, + "test/paid-pr-profile.test.ts": 7684, + "test/paid-report-census.test.ts": 494, + "test/paid-report-fail-open.test.ts": 3816, + "test/paid-retry-supervision.test.ts": 5816, + "test/paid-run-manifest.test.ts": 6396, + "test/paid-selection-propagation.test.ts": 184, + "test/paid-shard-settlement.test.ts": 1986, + "test/paid-shards.test.ts": 19222, + "test/paid-slice-deadline.test.ts": 1658, + "test/pair-agent-token-hygiene.test.ts": 57, + "test/parity-baseline-integrity.test.ts": 86, + "test/parity-sectioned.test.ts": 75, + "test/parity-suite.test.ts": 410, + "test/path-spelling.test.ts": 2158, + "test/pending-question-completion.test.ts": 500, + "test/periodic-exclude-policy.test.ts": 382, + "test/plan-count-checkbox.test.ts": 145, + "test/plan-count-completion.test.ts": 742, + "test/plan-count-cropped-wrap.test.ts": 280, + "test/plan-count-cross-cwd-ancestry.test.ts": 453, + "test/plan-count-design-ui-recovery.test.ts": 178, + "test/plan-count-dx-handoff.test.ts": 294, + "test/plan-count-empty-review.test.ts": 147, + "test/plan-count-history.test.ts": 86, + "test/plan-count-long-edit.test.ts": 152, + "test/plan-count-native-input.test.ts": 127, + "test/plan-count-owned-permission.test.ts": 121, + "test/plan-count-pending-exit.test.ts": 223, + "test/plan-count-prerequisite.test.ts": 161, + "test/plan-count-preview-footer.test.ts": 124, + "test/plan-count-session-cwd.test.ts": 313, + "test/plan-count-timeout.test.ts": 68, + "test/plan-count-transcript.test.ts": 256, + "test/plan-count-truncated-border.test.ts": 158, + "test/plan-count-truncated-question.test.ts": 140, + "test/plan-create-combined-permission.test.ts": 457, + "test/plan-create-permission.test.ts": 254, + "test/plan-design-floor-fixture.test.ts": 2448, + "test/plan-design-sdk-fixture.test.ts": 4508, + "test/plan-design-with-ui-fixture.test.ts": 10957, + "test/plan-edit-cropped-permission.test.ts": 501, + "test/plan-eng-resume.test.ts": 69, + "test/plan-floor-dx-actor.test.ts": 270, + "test/plan-floor-edit-wrap.test.ts": 198, + "test/plan-floor-review.test.ts": 114, + "test/plan-floor-target.test.ts": 15199, + "test/plan-format-approach-prompt.test.ts": 46, + "test/plan-format-kind-note.test.ts": 64, + "test/plan-pending-question-pty.test.ts": 125, + "test/plan-review-board-feedback.test.ts": 16253, + "test/plan-review-calibration.test.ts": 1141, + "test/plan-review-decisions.test.ts": 492, + "test/plan-review-report-recording.test.ts": 11843, + "test/plan-scope-selection.test.ts": 4032, + "test/plan-skill-question-hook-scope.test.ts": 11531, + "test/plan-skill-read-permission.test.ts": 174, + "test/plan-skill-webfetch-permission.test.ts": 6455, + "test/platform-qualification-workflow.test.ts": 57, + "test/post-rename-doc-regen.test.ts": 107, + "test/pr-direct-cases.test.ts": 8938, + "test/pr-shared-input-selection.test.ts": 4713, + "test/pr-title-sync-workflow-safety.test.ts": 61, + "test/pregate-import-graph-paths.test.ts": 13703, + "test/pregate-manifest-paths.test.ts": 8174, + "test/pregate-wtree-refusal.test.ts": 12209, + "test/prompt-audit-manifest.test.ts": 1048, + "test/prosons-neutral-posture.test.ts": 56, + "test/provider-model-defaults.test.ts": 105, + "test/pty-askuserquestion-single-line.test.ts": 116, + "test/pty-numbered-option-indent-native.test.ts": 150, + "test/pty-option-selection.test.ts": 70, + "test/pty-output-wake.test.ts": 3387, + "test/pty-screen-session.test.ts": 66, + "test/pty-screen-supervision.test.ts": 10816, + "test/pty-screen-unicode-ap.test.ts": 219, + "test/pty-screen.test.ts": 228, + "test/pty-skill-seeding-wiring.test.ts": 725, + "test/pty-trust-dialog.test.ts": 67, + "test/published-text.test.ts": 79, + "test/qa-browser-preservation.test.ts": 17642, + "test/qa-bugs-fixture.test.ts": 110, + "test/qa-caller-authority.test.ts": 356, + "test/qa-caller-freshness-order.test.ts": 192, + "test/qa-caller-report-observer.test.ts": 100, + "test/qa-checkpoint-evidence.test.ts": 426, + "test/qa-deadline-publication-observer.test.ts": 106, + "test/qa-deadline-selection.test.ts": 117, + "test/qa-deadline.test.ts": 24851, + "test/qa-evidence-selection.test.ts": 182, + "test/qa-evidence.test.ts": 21011, + "test/qa-fix-loop-fixture.test.ts": 9304, + "test/qa-health-rubric.test.ts": 69, + "test/qa-only-browser-probe.test.ts": 1717, + "test/qa-probe-gates.test.ts": 217, + "test/qa-report-fields.test.ts": 60, + "test/qa-supervision-selection.test.ts": 109, + "test/question-tuning-registry-path.test.ts": 62, + "test/read-only-command-grammar.test.ts": 103, + "test/readme-skill-lists.test.ts": 82, + "test/redact-context-exemptions.test.ts": 127, + "test/redact-doc-resolver.test.ts": 143, + "test/redact-dotenv-filename-false-positive.test.ts": 70, + "test/redact-engine-autoredact.test.ts": 73, + "test/redact-parcel-id-false-positive.test.ts": 81, + "test/redact-pattern-lint.test.ts": 94, + "test/redact-prepush-emails.test.ts": 70990, + "test/redact-prepush-target.test.ts": 82963, + "test/redact-span-binding.test.ts": 144, + "test/redact-unicode-offsets.test.ts": 230, + "test/regression-1539-review-self-verify.test.ts": 64, + "test/regression-1834-retro-recommendations.test.ts": 60, + "test/regression-issue2091-bsd-mktemp.test.ts": 960, + "test/regression-pr1169-mktemp-fallbacks.test.ts": 257, + "test/regression-transcript-frontmatter-fence.test.ts": 136, + "test/remote-identity-vectors.test.ts": 360, + "test/replay-pr-selection.test.ts": 4979, + "test/resolvers-gbrain-put-rewrite.test.ts": 134, + "test/resolvers-gbrain-save-results.test.ts": 70, + "test/retired-commands.test.ts": 141, + "test/review-army-budget.test.ts": 6890, + "test/review-base-refresh.test.ts": 2405, + "test/review-consensus-lifecycle.test.ts": 125, + "test/review-count-markdown.test.ts": 301, + "test/review-entry-and-design-clarity-au.test.ts": 176, + "test/review-enum-lifecycle.test.ts": 102, + "test/review-finalization-budget.test.ts": 8384, + "test/review-n-plus-one-contract.test.ts": 6711, + "test/review-quality-provenance.test.ts": 117, + "test/router-disabled-skills.test.ts": 1691, + "test/run-shard-child.test.ts": 1420, + "test/runtime-pins.test.ts": 276, + "test/safety-rule-registry.test.ts": 112, + "test/salience-allowlist.test.ts": 134, + "test/schema-version-migration.test.ts": 551, + "test/section-manifest-consistency.test.ts": 99, + "test/session-runner-browse-errors.test.ts": 4838, + "test/session-runner-stream-lifecycle.test.ts": 410, + "test/session-runner-timeout.test.ts": 88, + "test/session-runner-tools.test.ts": 267, + "test/setup-alias-name-uniqueness.test.ts": 30161, + "test/setup-browser-hint.test.ts": 764, + "test/setup-bun-cmd-and-pipe-bugs.test.ts": 443, + "test/setup-cleanup-orphans.test.ts": 75, + "test/setup-codex-design-pdf-runtime.test.ts": 90, + "test/setup-codex-model.test.ts": 67, + "test/setup-codex-scope-review-boundaries.test.ts": 206, + "test/setup-cso-windows-probe.test.ts": 65, + "test/setup-cursor-legacy-skills.test.ts": 171, + "test/setup-deploy-key-presence.test.ts": 153, + "test/setup-deploy-not-deployed.test.ts": 64, + "test/setup-deploy-vercel-trigger.test.ts": 66, + "test/setup-gbrain-bin-invocation-paths.test.ts": 84, + "test/setup-gbrain-path4-caller.test.ts": 8187, + "test/setup-gbrain-path4-structure.test.ts": 263, + "test/setup-gbrain-remote-caller.test.ts": 11356, + "test/setup-help.test.ts": 849, + "test/setup-hook-syntax-gate.test.ts": 166453, + "test/setup-playwright-platform.test.ts": 12030, + "test/setup-prune-stale-generated.test.ts": 3660, + "test/setup-sections-linking.test.ts": 82, + "test/setup-skill-copies.test.ts": 94, + "test/setup-staged-copy.test.ts": 604, + "test/setup-windows-fallback.test.ts": 72, + "test/setup-windows-rerun-refresh.test.ts": 2592, + "test/shard-cli-parity.test.ts": 377, + "test/shared-libs-cancellation.test.ts": 416, + "test/shared-libs-evidence.test.ts": 217, + "test/shared-libs-plan-actor.test.ts": 291, + "test/shared-libs-rendering.test.ts": 4300, + "test/shared-libs-source-reads.test.ts": 28516, + "test/ship-apple-gate.test.ts": 79, + "test/ship-control-flow.test.ts": 7090, + "test/ship-coverage-audit-af.test.ts": 278, + "test/ship-greptile-early.test.ts": 108, + "test/ship-greptile-pr-check.test.ts": 2116, + "test/ship-hook-windows-paths.test.ts": 4202, + "test/ship-measure-bar.test.ts": 264, + "test/ship-measure-loop-skill.test.ts": 131, + "test/ship-pr-liveness-policy.test.ts": 88, + "test/ship-publication-gates.test.ts": 19864, + "test/ship-reentry-gates.test.ts": 154, + "test/ship-review-loop.test.ts": 85, + "test/ship-section-fixture.test.ts": 2515, + "test/ship-skip-requeue.test.ts": 173, + "test/ship-template-redaction.test.ts": 76, + "test/ship-test-detection-markers.test.ts": 2597, + "test/ship-triage-labels.test.ts": 91, + "test/ship-workflow-clarity.test.ts": 168, + "test/shortcut-debt-ledger.test.ts": 80, + "test/sideways-review.test.ts": 59, + "test/skill-browse-state-extraction.test.ts": 430, + "test/skill-budget-regression.test.ts": 651, + "test/skill-check-driver.test.ts": 408, "test/skill-collision-sentinel.test.ts": 79, - "test/skill-coverage-floor.test.ts": 115, - "test/skill-cross-model-recommendation-emit.test.ts": 52, - "test/skill-md-size-guard.test.ts": 2496, - "test/skill-parser.test.ts": 86, - "test/skill-positional-literals.test.ts": 5711, - "test/skill-preflight-budget.test.ts": 53, - "test/skillify-host-paths.test.ts": 2098, - "test/sol-skill-fixture.test.ts": 3451, - "test/spawnsync-timeout-tripwire.test.ts": 381, - "test/spec-template-invariants.test.ts": 80, - "test/spend-ledger.test.ts": 365, - "test/state-root-parity.test.ts": 1999, - "test/strict-output-capture.test.ts": 212, - "test/strict-output-formats.test.ts": 2337, - "test/strict-output-settlement.test.ts": 4253, - "test/strict-output.test.ts": 51, - "test/sync-gbrain-readiness-fixture.test.ts": 475, - "test/takes-fence-fallback.test.ts": 39, - "test/tasks-section-jq.test.ts": 173, - "test/team-init-hook-shell-neutral.test.ts": 1055, - "test/template-context-parity.test.ts": 36, - "test/terse-build.test.ts": 48, - "test/test-free-shards-git-maintenance.test.ts": 708, - "test/test-free-shards-sandbox-knobs.test.ts": 407, - "test/test-free-shards-settlement-lifecycle.test.ts": 62, - "test/test-free-shards-settlement-ownership.test.ts": 71, - "test/test-health-report.test.ts": 949, - "test/test-health-workflow.test.ts": 43, - "test/test-of-test-ratchet.test.ts": 389, - "test/test-pr-profile.test.ts": 84, - "test/test-value-bar.test.ts": 562, - "test/third-party-actions-recording.test.ts": 639, - "test/third-party-actions.test.ts": 441, - "test/timeline-stop-hook-slug.test.ts": 4770, - "test/touchfiles-facade.test.ts": 67, - "test/touchfiles-map-diff.test.ts": 1611, - "test/tracker-guard-wiring.test.ts": 122, - "test/trial-cause.test.ts": 1074, - "test/typecheck-test-ratchet.test.ts": 76, - "test/uninstall-windows-copies.test.ts": 7745, - "test/v0-dormancy.test.ts": 73, - "test/version-source.test.ts": 60, - "test/voice-directive.test.ts": 110, - "test/windows-free-selection-autoplan-guard.test.ts": 81, - "test/windows-native-workflows.test.ts": 450, - "test/workflow-action-pins.test.ts": 71, - "test/workflow-boundaries-fixture.test.ts": 15322, - "test/workflow-concurrency.test.ts": 48, - "test/workflow-excerpt.test.ts": 170, - "test/workflow-judge-cache.test.ts": 15259, - "test/workflow-required-steps.test.ts": 76, - "test/worktree-isolated-shapes.test.ts": 69, - "test/worktree.test.ts": 5238 + "test/skill-coverage-floor.test.ts": 163, + "test/skill-cross-model-recommendation-emit.test.ts": 85, + "test/skill-md-size-guard.test.ts": 4454, + "test/skill-parser.test.ts": 111, + "test/skill-positional-literals.test.ts": 9956, + "test/skill-preflight-budget.test.ts": 81, + "test/skillify-host-paths.test.ts": 3766, + "test/sol-skill-fixture.test.ts": 4905, + "test/spawnsync-timeout-tripwire.test.ts": 618, + "test/spec-template-invariants.test.ts": 78, + "test/spend-ledger.test.ts": 402, + "test/state-root-parity.test.ts": 3083, + "test/strict-output-capture.test.ts": 271, + "test/strict-output-formats.test.ts": 3508, + "test/strict-output-settlement.test.ts": 4344, + "test/strict-output.test.ts": 64, + "test/sync-gbrain-readiness-fixture.test.ts": 745, + "test/takes-fence-fallback.test.ts": 59, + "test/tasks-section-jq.test.ts": 259, + "test/team-init-hook-shell-neutral.test.ts": 1758, + "test/template-context-parity.test.ts": 92, + "test/terse-build.test.ts": 80, + "test/test-free-shards-files-from.test.ts": 400, + "test/test-free-shards-git-maintenance.test.ts": 1296, + "test/test-free-shards-sandbox-knobs.test.ts": 593, + "test/test-free-shards-settlement-lifecycle.test.ts": 111, + "test/test-free-shards-settlement-ownership.test.ts": 94, + "test/test-health-report.test.ts": 1977, + "test/test-health-workflow.test.ts": 70, + "test/test-of-test-ratchet.test.ts": 696, + "test/test-pr-profile.test.ts": 175, + "test/test-value-bar.test.ts": 889, + "test/third-party-actions-recording.test.ts": 909, + "test/third-party-actions.test.ts": 748, + "test/timeline-stop-hook-slug.test.ts": 5216, + "test/touchfiles-facade.test.ts": 89, + "test/touchfiles-map-diff.test.ts": 1942, + "test/tracker-guard-wiring.test.ts": 187, + "test/trial-cause.test.ts": 1253, + "test/typecheck-test-ratchet.test.ts": 139, + "test/uninstall-windows-copies.test.ts": 13420, + "test/v0-dormancy.test.ts": 139, + "test/version-source.test.ts": 69, + "test/voice-directive.test.ts": 160, + "test/windows-free-selection-autoplan-guard.test.ts": 103, + "test/windows-native-workflows.test.ts": 858, + "test/workflow-action-pins.test.ts": 109, + "test/workflow-boundaries-fixture.test.ts": 28350, + "test/workflow-concurrency.test.ts": 61, + "test/workflow-excerpt.test.ts": 350, + "test/workflow-judge-cache.test.ts": 25059, + "test/workflow-required-steps.test.ts": 128, + "test/worktree-isolated-shapes.test.ts": 127, + "test/worktree.test.ts": 9019 } } diff --git a/scripts/free-test-durations.json b/scripts/free-test-durations.json index 1401d7962..06b704c3e 100644 --- a/scripts/free-test-durations.json +++ b/scripts/free-test-durations.json @@ -1,1279 +1,1293 @@ { "version": 1, - "recordedAt": "2026-10-17T04:13:10.728Z", + "recordedAt": "2026-10-19T04:31:41.807Z", "durations": { - "browse/test/activity.test.ts": 62, - "browse/test/adversarial-security.test.ts": 51, - "browse/test/batch.test.ts": 1780, - "browse/test/bridge-chromium-e2e.test.ts": 812, - "browse/test/browse-client.test.ts": 82, - "browse/test/browse-eval-wrapping.test.ts": 64, - "browse/test/browser-manager-custom-chromium.test.ts": 310, - "browse/test/browser-manager-unit.test.ts": 461, - "browse/test/browser-skill-commands.test.ts": 1178, - "browse/test/browser-skill-write.test.ts": 85, - "browse/test/browser-skills-e2e.test.ts": 96, - "browse/test/browser-skills-storage.test.ts": 85, - "browse/test/build-command-response.test.ts": 341, - "browse/test/build.test.ts": 103, - "browse/test/bun-child-config.test.ts": 79, - "browse/test/bun-polyfill.test.ts": 1200, - "browse/test/busy-daemon-iron-rule.test.ts": 15865, - "browse/test/busy-daemon-recovery.test.ts": 617, - "browse/test/cdp-allowlist.test.ts": 20, - "browse/test/cdp-e2e.test.ts": 580, - "browse/test/cdp-inspector-history-cap.test.ts": 26, - "browse/test/cdp-mutex.test.ts": 660, - "browse/test/cdp-session-cleanup.test.ts": 34, - "browse/test/chromium-profile-isolation.test.ts": 13228, - "browse/test/claude-bin.test.ts": 14, - "browse/test/cli-chain-stdin.test.ts": 151, - "browse/test/cli-config-mismatch-message.test.ts": 75, - "browse/test/cli-lock.test.ts": 23, - "browse/test/cli-loopback-no-proxy.test.ts": 93, - "browse/test/cli-setsid-daemonize.test.ts": 13, - "browse/test/cli-start-final-healthcheck.test.ts": 31, - "browse/test/cli-supervisor.test.ts": 28, - "browse/test/commands.test.ts": 32240, - "browse/test/compare-board.test.ts": 1162, - "browse/test/config.test.ts": 123, - "browse/test/content-security.test.ts": 918, - "browse/test/cookie-auth-verification.test.ts": 7671, - "browse/test/cookie-credential-deadline.test.ts": 401, - "browse/test/cookie-database.test.ts": 474, - "browse/test/cookie-fixture-delete-lease.test.ts": 43, - "browse/test/cookie-import-browser.test.ts": 198, - "browse/test/cookie-import-native-job.test.ts": 602, - "browse/test/cookie-import-native.test.ts": 3853, - "browse/test/cookie-import-node.test.ts": 345, - "browse/test/cookie-import-operation.test.ts": 1204, - "browse/test/cookie-import-reliability.test.ts": 1855, - "browse/test/cookie-import-transport.test.ts": 24, - "browse/test/cookie-picker-binding-browser.test.ts": 2424, - "browse/test/cookie-picker-binding.test.ts": 313, - "browse/test/cookie-picker-csrf-browser.test.ts": 579, - "browse/test/cookie-picker-routes.test.ts": 302, - "browse/test/cookie-picker-ui.test.ts": 11344, - "browse/test/daemon-log-hygiene.test.ts": 49, - "browse/test/daemon-mismatch-refuse.test.ts": 176, - "browse/test/data-platform.test.ts": 53, - "browse/test/dia-gui-readiness.test.ts": 613, - "browse/test/dia-launch-comparison.test.ts": 2680, - "browse/test/dia-macos-qualification.test.ts": 2987, - "browse/test/domain-skills-e2e.test.ts": 487, - "browse/test/domain-skills-storage.test.ts": 220, - "browse/test/dotenv-isolation.test.ts": 354, - "browse/test/dual-listener.test.ts": 327, - "browse/test/dx-polish.test.ts": 29, - "browse/test/error-handling.test.ts": 10, - "browse/test/extension-sender-auth.test.ts": 34, - "browse/test/extension-token.test.ts": 339, - "browse/test/file-drop.test.ts": 32, - "browse/test/file-permissions.test.ts": 43, - "browse/test/fill-change-event.test.ts": 1770, - "browse/test/find-browse.test.ts": 14, - "browse/test/findport.test.ts": 371, - "browse/test/from-file-path-validation.test.ts": 13, - "browse/test/gstack-config.test.ts": 1453, - "browse/test/gstack-update-check.test.ts": 3815, - "browse/test/handoff.test.ts": 8712, - "browse/test/headless-gpu-and-reap.test.ts": 1608, - "browse/test/headless-launch-options.test.ts": 1848, - "browse/test/launch-signal-flags.test.ts": 12, - "browse/test/learnings-injection.test.ts": 134, - "browse/test/media-extract-unit.test.ts": 19, - "browse/test/memory-command.test.ts": 339, - "browse/test/memory-leak-reproducer.test.ts": 338, - "browse/test/navigation-guard.test.ts": 6700, - "browse/test/output-path-symlinks.test.ts": 76, - "browse/test/pair-agent-e2e.test.ts": 1829, - "browse/test/pair-agent-optin-gate.test.ts": 348, - "browse/test/pair-agent-tunnel-eval.test.ts": 7609, - "browse/test/path-security-realpath.test.ts": 72, - "browse/test/path-validation.test.ts": 1822, - "browse/test/pdf-flags.test.ts": 23, - "browse/test/platform.test.ts": 17, - "browse/test/playwright-core-patch.test.ts": 40, - "browse/test/poisoned-bundle-probe.test.ts": 267, - "browse/test/process-liveness-windows.test.ts": 62, - "browse/test/proxy-config.test.ts": 31, - "browse/test/proxy-env-daemon.test.ts": 2762, - "browse/test/proxy-env.test.ts": 46, - "browse/test/proxy-redact.test.ts": 10, - "browse/test/pty-inject-scan.test.ts": 615, - "browse/test/pty-session-lease.test.ts": 31, - "browse/test/rebrand-signed-bundle.test.ts": 13, - "browse/test/regression-pr1169-pdf-from-file-invalid-json.test.ts": 35, - "browse/test/restart-env.test.ts": 20, - "browse/test/runtime-root-server-bundle.test.ts": 32, - "browse/test/sanitize.test.ts": 14, - "browse/test/screenshot-size-guard.test.ts": 498, - "browse/test/security-adversarial-fixes.test.ts": 16, - "browse/test/security-adversarial.test.ts": 23, - "browse/test/security-audit-r2.test.ts": 271, + "browse/test/activity.test.ts": 37, + "browse/test/adversarial-security.test.ts": 26, + "browse/test/batch.test.ts": 1675, + "browse/test/bridge-chromium-e2e.test.ts": 826, + "browse/test/browse-client.test.ts": 106, + "browse/test/browse-eval-wrapping.test.ts": 19, + "browse/test/browser-manager-custom-chromium.test.ts": 367, + "browse/test/browser-manager-unit.test.ts": 511, + "browse/test/browser-skill-commands.test.ts": 1162, + "browse/test/browser-skill-write.test.ts": 93, + "browse/test/browser-skills-e2e.test.ts": 355, + "browse/test/browser-skills-storage.test.ts": 273, + "browse/test/build-command-response.test.ts": 542, + "browse/test/build.test.ts": 107, + "browse/test/bun-child-config.test.ts": 98, + "browse/test/bun-polyfill.test.ts": 1428, + "browse/test/busy-daemon-iron-rule.test.ts": 15956, + "browse/test/busy-daemon-recovery.test.ts": 564, + "browse/test/cdp-allowlist.test.ts": 21, + "browse/test/cdp-e2e.test.ts": 520, + "browse/test/cdp-inspector-history-cap.test.ts": 24, + "browse/test/cdp-mutex.test.ts": 654, + "browse/test/cdp-session-cleanup.test.ts": 43, + "browse/test/chromium-profile-isolation.test.ts": 13717, + "browse/test/claude-bin.test.ts": 9, + "browse/test/cli-chain-stdin.test.ts": 147, + "browse/test/cli-config-mismatch-message.test.ts": 100, + "browse/test/cli-lock.test.ts": 29, + "browse/test/cli-loopback-no-proxy.test.ts": 205, + "browse/test/cli-setsid-daemonize.test.ts": 10, + "browse/test/cli-start-final-healthcheck.test.ts": 53, + "browse/test/cli-supervisor.test.ts": 39, + "browse/test/commands.test.ts": 31274, + "browse/test/compare-board.test.ts": 904, + "browse/test/config.test.ts": 163, + "browse/test/content-security.test.ts": 771, + "browse/test/cookie-auth-verification.test.ts": 7627, + "browse/test/cookie-credential-deadline.test.ts": 381, + "browse/test/cookie-database.test.ts": 465, + "browse/test/cookie-fixture-delete-lease.test.ts": 50, + "browse/test/cookie-import-browser.test.ts": 201, + "browse/test/cookie-import-native-job.test.ts": 461, + "browse/test/cookie-import-native.test.ts": 3243, + "browse/test/cookie-import-node.test.ts": 191, + "browse/test/cookie-import-operation.test.ts": 828, + "browse/test/cookie-import-reliability.test.ts": 1536, + "browse/test/cookie-import-transport.test.ts": 29, + "browse/test/cookie-picker-binding-browser.test.ts": 1812, + "browse/test/cookie-picker-binding.test.ts": 224, + "browse/test/cookie-picker-csrf-browser.test.ts": 424, + "browse/test/cookie-picker-routes.test.ts": 257, + "browse/test/cookie-picker-ui.test.ts": 12684, + "browse/test/daemon-log-hygiene.test.ts": 32, + "browse/test/daemon-mismatch-refuse.test.ts": 105, + "browse/test/data-platform.test.ts": 21, + "browse/test/dia-gui-readiness.test.ts": 895, + "browse/test/dia-launch-comparison.test.ts": 2807, + "browse/test/dia-macos-qualification.test.ts": 2829, + "browse/test/domain-skills-e2e.test.ts": 520, + "browse/test/domain-skills-storage.test.ts": 232, + "browse/test/dotenv-isolation.test.ts": 405, + "browse/test/dual-listener.test.ts": 470, + "browse/test/dx-polish.test.ts": 25, + "browse/test/error-handling.test.ts": 21, + "browse/test/extension-sender-auth.test.ts": 52, + "browse/test/extension-token.test.ts": 460, + "browse/test/file-drop.test.ts": 35, + "browse/test/file-permissions.test.ts": 26, + "browse/test/fill-change-event.test.ts": 1533, + "browse/test/find-browse.test.ts": 27, + "browse/test/findport.test.ts": 419, + "browse/test/from-file-path-validation.test.ts": 27, + "browse/test/gstack-config.test.ts": 929, + "browse/test/gstack-update-check.test.ts": 3133, + "browse/test/handoff.test.ts": 10217, + "browse/test/headless-gpu-and-reap.test.ts": 1756, + "browse/test/headless-launch-options.test.ts": 3549, + "browse/test/launch-signal-flags.test.ts": 9, + "browse/test/learnings-injection.test.ts": 82, + "browse/test/media-extract-unit.test.ts": 11, + "browse/test/memory-command.test.ts": 319, + "browse/test/memory-leak-reproducer.test.ts": 330, + "browse/test/navigation-guard.test.ts": 6572, + "browse/test/output-path-symlinks.test.ts": 64, + "browse/test/pair-agent-e2e.test.ts": 1663, + "browse/test/pair-agent-optin-gate.test.ts": 291, + "browse/test/pair-agent-tunnel-eval.test.ts": 7572, + "browse/test/path-security-realpath.test.ts": 44, + "browse/test/path-validation.test.ts": 1580, + "browse/test/pdf-flags.test.ts": 24, + "browse/test/platform.test.ts": 11, + "browse/test/playwright-core-patch.test.ts": 12, + "browse/test/poisoned-bundle-probe.test.ts": 215, + "browse/test/process-liveness-windows.test.ts": 69, + "browse/test/proxy-config.test.ts": 19, + "browse/test/proxy-env-daemon.test.ts": 2566, + "browse/test/proxy-env.test.ts": 26, + "browse/test/proxy-redact.test.ts": 11, + "browse/test/pty-inject-scan.test.ts": 554, + "browse/test/pty-session-lease.test.ts": 23, + "browse/test/rebrand-signed-bundle.test.ts": 10, + "browse/test/regression-pr1169-pdf-from-file-invalid-json.test.ts": 26, + "browse/test/restart-env.test.ts": 23, + "browse/test/runtime-root-server-bundle.test.ts": 45, + "browse/test/sanitize.test.ts": 13, + "browse/test/screenshot-size-guard.test.ts": 426, + "browse/test/security-adversarial-fixes.test.ts": 27, + "browse/test/security-adversarial.test.ts": 17, + "browse/test/security-audit-r2.test.ts": 370, "browse/test/security-bench.test.ts": 14, - "browse/test/security-classifier-download-cleanup.test.ts": 89, - "browse/test/security-classifier.test.ts": 25, - "browse/test/security-integration.test.ts": 21, - "browse/test/security-live-playwright.test.ts": 681, - "browse/test/security-sidecar-client.test.ts": 29, - "browse/test/security.test.ts": 27, - "browse/test/server-auth.test.ts": 381, - "browse/test/server-embedder-terminal-port.test.ts": 4014, - "browse/test/server-factory.test.ts": 974, - "browse/test/server-flush-trackers.test.ts": 14, - "browse/test/server-launch-resolution.test.ts": 20, - "browse/test/server-lock-errors.test.ts": 28, - "browse/test/server-no-import-side-effects.test.ts": 591, - "browse/test/server-proxy-fail-fast.test.ts": 1372, - "browse/test/server-pty-lease-routes.test.ts": 316, - "browse/test/server-route-auth-blackbox.test.ts": 401, - "browse/test/server-route-dispatch-ratchet.test.ts": 264, - "browse/test/server-route-table.test.ts": 331, - "browse/test/server-sanitize-surrogates.test.ts": 291, - "browse/test/server-tmp-state-path.test.ts": 12, - "browse/test/session-cookie-store.test.ts": 51, - "browse/test/session-persist.test.ts": 650, - "browse/test/sidebar-tabs.test.ts": 308, - "browse/test/sidebar-ux.test.ts": 292, - "browse/test/sidepanel-font-size.test.ts": 16, - "browse/test/sidepanel-patient-autoconnect.test.ts": 14, - "browse/test/sidepanel-reattach.test.ts": 21, - "browse/test/sidepanel-restart-dispose.test.ts": 19, + "browse/test/security-classifier-download-cleanup.test.ts": 48, + "browse/test/security-classifier.test.ts": 13, + "browse/test/security-integration.test.ts": 19, + "browse/test/security-live-playwright.test.ts": 606, + "browse/test/security-sidecar-client.test.ts": 33, + "browse/test/security.test.ts": 28, + "browse/test/server-auth.test.ts": 354, + "browse/test/server-embedder-terminal-port.test.ts": 4027, + "browse/test/server-factory.test.ts": 926, + "browse/test/server-flush-trackers.test.ts": 10, + "browse/test/server-launch-resolution.test.ts": 22, + "browse/test/server-lock-errors.test.ts": 38, + "browse/test/server-no-import-side-effects.test.ts": 548, + "browse/test/server-proxy-fail-fast.test.ts": 1297, + "browse/test/server-pty-lease-routes.test.ts": 233, + "browse/test/server-route-auth-blackbox.test.ts": 328, + "browse/test/server-route-dispatch-ratchet.test.ts": 235, + "browse/test/server-route-table.test.ts": 296, + "browse/test/server-sanitize-surrogates.test.ts": 227, + "browse/test/server-tmp-state-path.test.ts": 9, + "browse/test/session-cookie-store.test.ts": 39, + "browse/test/session-persist.test.ts": 753, + "browse/test/sidebar-tabs.test.ts": 279, + "browse/test/sidebar-ux.test.ts": 304, + "browse/test/sidepanel-font-size.test.ts": 15, + "browse/test/sidepanel-patient-autoconnect.test.ts": 20, + "browse/test/sidepanel-reattach.test.ts": 18, + "browse/test/sidepanel-restart-dispose.test.ts": 20, "browse/test/skill-token.test.ts": 20, - "browse/test/snapshot.test.ts": 8283, - "browse/test/socks-bridge.test.ts": 471, + "browse/test/snapshot.test.ts": 7715, + "browse/test/socks-bridge.test.ts": 498, "browse/test/sse-helpers.test.ts": 156, - "browse/test/sse-session-cookie.test.ts": 17, - "browse/test/state-ttl.test.ts": 12, - "browse/test/stealth-extended.test.ts": 15, - "browse/test/stealth-layer-c.test.ts": 12, - "browse/test/stealth-webdriver.test.ts": 1640, - "browse/test/stop-ack-before-shutdown.test.ts": 129, - "browse/test/stop-dead-daemon.test.ts": 182, - "browse/test/tab-each.test.ts": 30, - "browse/test/tab-external-close-dispatch.test.ts": 652, - "browse/test/tab-external-close.test.ts": 496, - "browse/test/tab-guardrail.test.ts": 284, - "browse/test/tab-isolation.test.ts": 277, + "browse/test/sse-session-cookie.test.ts": 19, + "browse/test/state-ttl.test.ts": 10, + "browse/test/stealth-extended.test.ts": 7, + "browse/test/stealth-layer-c.test.ts": 10, + "browse/test/stealth-webdriver.test.ts": 1567, + "browse/test/stop-ack-before-shutdown.test.ts": 133, + "browse/test/stop-dead-daemon.test.ts": 229, + "browse/test/tab-each.test.ts": 25, + "browse/test/tab-external-close-dispatch.test.ts": 701, + "browse/test/tab-external-close.test.ts": 566, + "browse/test/tab-guardrail.test.ts": 334, + "browse/test/tab-isolation.test.ts": 316, "browse/test/tab-session-frame-detach.test.ts": 9, - "browse/test/telemetry-optout.test.ts": 23, + "browse/test/telemetry-optout.test.ts": 54, "browse/test/telemetry.test.ts": 24, - "browse/test/temp-dirs.test.ts": 42, - "browse/test/terminal-agent-detach-reattach.test.ts": 13, - "browse/test/terminal-agent-import.test.ts": 4376, - "browse/test/terminal-agent-integration.test.ts": 790, - "browse/test/terminal-agent-keepalive.test.ts": 13, - "browse/test/terminal-agent-lifecycle.test.ts": 5047, - "browse/test/terminal-agent-native-observation.test.ts": 51, - "browse/test/terminal-agent-owner-watchdog.test.ts": 114, - "browse/test/terminal-agent-pid-identity.test.ts": 26, - "browse/test/terminal-agent-port-range.test.ts": 22, - "browse/test/terminal-agent-publication-lock.test.ts": 1689, - "browse/test/terminal-agent-ring-buffer-runtime.test.ts": 93, - "browse/test/terminal-agent-session-routing.test.ts": 11, - "browse/test/terminal-agent-signal-cleanup.test.ts": 15589, + "browse/test/temp-dirs.test.ts": 51, + "browse/test/terminal-agent-detach-reattach.test.ts": 10, + "browse/test/terminal-agent-import.test.ts": 4362, + "browse/test/terminal-agent-integration.test.ts": 777, + "browse/test/terminal-agent-keepalive.test.ts": 14, + "browse/test/terminal-agent-lifecycle.test.ts": 4251, + "browse/test/terminal-agent-native-observation.test.ts": 13, + "browse/test/terminal-agent-owner-watchdog.test.ts": 85, + "browse/test/terminal-agent-pid-identity.test.ts": 29, + "browse/test/terminal-agent-port-range.test.ts": 19, + "browse/test/terminal-agent-publication-lock.test.ts": 1147, + "browse/test/terminal-agent-ring-buffer-runtime.test.ts": 30, + "browse/test/terminal-agent-session-routing.test.ts": 9, + "browse/test/terminal-agent-signal-cleanup.test.ts": 15627, "browse/test/terminal-agent-watchdog.test.ts": 10, - "browse/test/terminal-agent.test.ts": 270, - "browse/test/terminal-pty-lifecycle.test.ts": 16, - "browse/test/token-registry.test.ts": 23, - "browse/test/tunnel-gate-unit.test.ts": 267, - "browse/test/tunnel-revoke-cli.test.ts": 907, - "browse/test/url-validation.test.ts": 56, - "browse/test/watch.test.ts": 290, - "browse/test/watchdog.test.ts": 3042, - "browse/test/welcome-page.test.ts": 13, - "browse/test/windows-spawn-hide.test.ts": 25, - "browse/test/xprotect-heal.test.ts": 778, - "browse/test/xvfb.test.ts": 16129, - "browser-skills/hackernews-frontpage/script.test.ts": 13, - "design/test/approval.test.ts": 30, - "design/test/auth.test.ts": 15, - "design/test/brief.test.ts": 9, - "design/test/cli-persistence.test.ts": 6971, - "design/test/cli-version.test.ts": 32, - "design/test/daemon-discovery.test.ts": 11676, - "design/test/daemon.test.ts": 154, - "design/test/feedback-roundtrip-daemon.test.ts": 388, - "design/test/feedback-roundtrip.test.ts": 2922, - "design/test/gallery.test.ts": 16, - "design/test/image-args.test.ts": 35, - "design/test/image-gen-pairing.test.ts": 53, - "design/test/models.test.ts": 55, - "design/test/persist.test.ts": 41, - "design/test/receipted-fetch.test.ts": 23, - "design/test/serve.test.ts": 177, - "design/test/variants-briefs.test.ts": 3575, - "design/test/variants-retry-after.test.ts": 8381, - "ios-qa/daemon/test/allowlist.test.ts": 34, - "ios-qa/daemon/test/audit.test.ts": 29, - "ios-qa/daemon/test/auth-mint.test.ts": 38, - "ios-qa/daemon/test/cli-mint.test.ts": 219, - "ios-qa/daemon/test/daemon-integration.test.ts": 3909, - "ios-qa/daemon/test/devicectl-launch-env.test.ts": 27, - "ios-qa/daemon/test/proxy-classify.test.ts": 41, - "ios-qa/daemon/test/proxy-transport.test.ts": 1759, - "ios-qa/daemon/test/session-tokens.test.ts": 15, - "ios-qa/daemon/test/single-instance.test.ts": 16, - "ios-qa/daemon/test/tailscale-localapi.test.ts": 25, - "ios-qa/daemon/test/tunnel-bootstrap.test.ts": 420, - "ios-qa/scripts/gen-accessors.test.ts": 50, - "make-pdf/test/asideClient.test.ts": 19, - "make-pdf/test/cli-args.test.ts": 10, - "make-pdf/test/cli-exit-codes.test.ts": 322, - "make-pdf/test/diagram-prepass.test.ts": 98, - "make-pdf/test/e2e/ci-prereqs.test.ts": 37, - "make-pdf/test/e2e/combined-gate.test.ts": 926, - "make-pdf/test/e2e/diagram-gate.test.ts": 2673, - "make-pdf/test/e2e/emoji-gate.test.ts": 891, - "make-pdf/test/e2e/format-gate.test.ts": 4823, - "make-pdf/test/e2e/landscape-gate.test.ts": 3251, - "make-pdf/test/e2e/toc-gate.test.ts": 5810, - "make-pdf/test/image-policy.test.ts": 19, - "make-pdf/test/pdftotext.test.ts": 36, - "make-pdf/test/render-offline-sanitize.test.ts": 32, - "make-pdf/test/render.test.ts": 40, - "make-pdf/test/setup-smoke.test.ts": 104, - "make-pdf/test/toc-anchors.test.ts": 28, - "make-pdf/test/toc-pages.test.ts": 56, - "test/agent-sdk-runner.test.ts": 8251, - "test/agents-digest.test.ts": 20, - "test/analytics.test.ts": 29, - "test/anthropic-preflight.test.ts": 11, - "test/approval-classes.test.ts": 7087, - "test/archaeology-lint.test.ts": 1758, - "test/arm-benchmark-selftest.test.ts": 415, - "test/arm-setup-smoke-workflow.test.ts": 20, - "test/artifacts-allowlist-decisions.test.ts": 11, - "test/artifacts-init-migration.test.ts": 251, - "test/aside-driver.test.ts": 150, - "test/aside-probe-shell.test.ts": 9593, - "test/aside-render.test.ts": 10224, - "test/audit-compliance.test.ts": 22, - "test/auq-error-fallback-hook.test.ts": 170, - "test/auq-format-always-loaded.test.ts": 45, - "test/auq-mode-capture.test.ts": 147, - "test/auq-native-capture.test.ts": 2302, - "test/auq-parallel.test.ts": 12051, - "test/auq-substance-panel.test.ts": 102, - "test/auto-decide-fixture.test.ts": 2984, - "test/autoplan-admission.test.ts": 958, - "test/autoplan-amend-input.test.ts": 999, - "test/autoplan-answer-invalidation.test.ts": 8280, - "test/autoplan-artifact-recorder.test.ts": 5705, - "test/autoplan-artifact-windows-argv.test.ts": 80, - "test/autoplan-bounded-journal.test.ts": 376, - "test/autoplan-budget.test.ts": 9339, - "test/autoplan-capy-e2e.test.ts": 8983, - "test/autoplan-checkpoint.test.ts": 6677, - "test/autoplan-compaction-copies.test.ts": 117, - "test/autoplan-compaction-preserved-segment.test.ts": 1266, - "test/autoplan-driver.test.ts": 2085, - "test/autoplan-dual-voice-evidence.test.ts": 1255, - "test/autoplan-dual-voice-fixture.test.ts": 3240, - "test/autoplan-durability.test.ts": 1740, - "test/autoplan-edit-digests-al.test.ts": 287, - "test/autoplan-gate-artifacts.test.ts": 1959, - "test/autoplan-gate-judgment.test.ts": 17866, - "test/autoplan-guard-log.test.ts": 62, - "test/autoplan-guard-reasons.test.ts": 12, - "test/autoplan-init.test.ts": 957, - "test/autoplan-long-session-observer.test.ts": 72, - "test/autoplan-method-read-audit.test.ts": 274, - "test/autoplan-methodology-names.test.ts": 1136, - "test/autoplan-obligations.test.ts": 2439, - "test/autoplan-owned-read.test.ts": 2432, - "test/autoplan-owned-state.test.ts": 1704, - "test/autoplan-pairing.test.ts": 35, - "test/autoplan-pending-artifact.test.ts": 904, - "test/autoplan-pending-question.test.ts": 4167, - "test/autoplan-phase-handoff.test.ts": 129, - "test/autoplan-phase-observer.test.ts": 102, - "test/autoplan-phase-order.test.ts": 65, - "test/autoplan-preconfigured-onboarding-ar.test.ts": 988, - "test/autoplan-public-export.test.ts": 5753, - "test/autoplan-public-narration.test.ts": 70, - "test/autoplan-publication-generation.test.ts": 25, - "test/autoplan-publication-guard.test.ts": 1625, - "test/autoplan-publication-hook.test.ts": 2553, - "test/autoplan-publication-payload.test.ts": 2216, - "test/autoplan-record.test.ts": 2925, - "test/autoplan-residuals.test.ts": 6404, - "test/autoplan-review-discovery.test.ts": 11020, - "test/autoplan-revise.test.ts": 10146, - "test/autoplan-run-concurrency.test.ts": 3664, - "test/autoplan-run-recovery.test.ts": 8822, - "test/autoplan-schema-canary-compare.test.ts": 29, - "test/autoplan-scope.test.ts": 711, - "test/autoplan-snapshot.test.ts": 2015, - "test/autoplan-timing.test.ts": 353, - "test/benchmark-cli.test.ts": 516, - "test/benchmark-provider-adapters.test.ts": 61, - "test/benchmark-runner.test.ts": 23, - "test/bin-context-windows-slug.test.ts": 2898, - "test/bin-windows-bun-import-paths.test.ts": 935, - "test/binary-version-probe.test.ts": 132, - "test/binding-template-drift.test.ts": 39, - "test/bootstrap-retention-shard.test.ts": 3133, - "test/bootstrap-retention.test.ts": 12024, - "test/bootstrap-session-lifecycle.test.ts": 5096, - "test/brain-cache-roundtrip.test.ts": 153, - "test/brain-cache-spec.test.ts": 24, - "test/brain-preflight.test.ts": 43, - "test/brain-sync-removals.test.ts": 9851, - "test/brain-sync-windows-paths.test.ts": 32, - "test/brain-sync.test.ts": 58148, - "test/branch-slug-hygiene.test.ts": 723, - "test/build-cso-nonfatal.test.ts": 51, - "test/build-cso-result-record.test.ts": 660, - "test/build-gbrain-env.test.ts": 90, + "browse/test/terminal-agent.test.ts": 230, + "browse/test/terminal-pty-lifecycle.test.ts": 13, + "browse/test/token-registry.test.ts": 24, + "browse/test/tunnel-gate-unit.test.ts": 254, + "browse/test/tunnel-revoke-cli.test.ts": 858, + "browse/test/url-validation.test.ts": 66, + "browse/test/watch.test.ts": 305, + "browse/test/watchdog.test.ts": 2841, + "browse/test/welcome-page.test.ts": 10, + "browse/test/windows-spawn-hide.test.ts": 20, + "browse/test/xprotect-heal.test.ts": 722, + "browse/test/xvfb.test.ts": 15970, + "browser-skills/hackernews-frontpage/script.test.ts": 10, + "design/test/approval.test.ts": 31, + "design/test/auth.test.ts": 14, + "design/test/brief.test.ts": 8, + "design/test/cli-persistence.test.ts": 6474, + "design/test/cli-version.test.ts": 26, + "design/test/daemon-discovery.test.ts": 11208, + "design/test/daemon.test.ts": 155, + "design/test/feedback-roundtrip-daemon.test.ts": 288, + "design/test/feedback-roundtrip.test.ts": 2796, + "design/test/gallery.test.ts": 11, + "design/test/image-args.test.ts": 25, + "design/test/image-gen-pairing.test.ts": 34, + "design/test/models.test.ts": 62, + "design/test/persist.test.ts": 55, + "design/test/receipted-fetch.test.ts": 16, + "design/test/serve.test.ts": 138, + "design/test/variants-briefs.test.ts": 3520, + "design/test/variants-retry-after.test.ts": 8258, + "ios-qa/daemon/test/allowlist.test.ts": 12, + "ios-qa/daemon/test/audit.test.ts": 12, + "ios-qa/daemon/test/auth-mint.test.ts": 16, + "ios-qa/daemon/test/cli-mint.test.ts": 169, + "ios-qa/daemon/test/daemon-integration.test.ts": 3821, + "ios-qa/daemon/test/devicectl-launch-env.test.ts": 14, + "ios-qa/daemon/test/proxy-classify.test.ts": 14, + "ios-qa/daemon/test/proxy-transport.test.ts": 1728, + "ios-qa/daemon/test/session-tokens.test.ts": 35, + "ios-qa/daemon/test/single-instance.test.ts": 21, + "ios-qa/daemon/test/tailscale-localapi.test.ts": 28, + "ios-qa/daemon/test/tunnel-bootstrap.test.ts": 407, + "ios-qa/scripts/gen-accessors.test.ts": 2527, + "make-pdf/test/asideClient.test.ts": 28, + "make-pdf/test/cli-args.test.ts": 13, + "make-pdf/test/cli-exit-codes.test.ts": 361, + "make-pdf/test/diagram-prepass.test.ts": 52, + "make-pdf/test/docx-image-size.test.ts": 485, + "make-pdf/test/e2e/ci-prereqs.test.ts": 22, + "make-pdf/test/e2e/combined-gate.test.ts": 933, + "make-pdf/test/e2e/diagram-gate.test.ts": 2783, + "make-pdf/test/e2e/emoji-gate.test.ts": 936, + "make-pdf/test/e2e/format-gate.test.ts": 4960, + "make-pdf/test/e2e/landscape-gate.test.ts": 3795, + "make-pdf/test/e2e/toc-gate.test.ts": 6374, + "make-pdf/test/image-policy.test.ts": 35, + "make-pdf/test/pdftotext.test.ts": 64, + "make-pdf/test/render-offline-sanitize.test.ts": 38, + "make-pdf/test/render.test.ts": 47, + "make-pdf/test/setup-smoke.test.ts": 209, + "make-pdf/test/toc-anchors.test.ts": 36, + "make-pdf/test/toc-pages.test.ts": 35, + "test/agent-sdk-runner.test.ts": 8308, + "test/agents-digest.test.ts": 40, + "test/analytics.test.ts": 50, + "test/anthropic-preflight.test.ts": 40, + "test/approval-classes.test.ts": 6647, + "test/archaeology-lint.test.ts": 1727, + "test/arm-benchmark-selftest.test.ts": 410, + "test/arm-setup-smoke-workflow.test.ts": 28, + "test/artifacts-allowlist-decisions.test.ts": 13, + "test/artifacts-init-migration.test.ts": 237, + "test/aside-driver.test.ts": 151, + "test/aside-probe-shell.test.ts": 9418, + "test/aside-render.test.ts": 10140, + "test/audit-compliance.test.ts": 24, + "test/auq-error-fallback-hook.test.ts": 203, + "test/auq-format-always-loaded.test.ts": 77, + "test/auq-mode-capture.test.ts": 126, + "test/auq-native-capture.test.ts": 2288, + "test/auq-parallel.test.ts": 10877, + "test/auq-substance-panel.test.ts": 119, + "test/auto-decide-fixture.test.ts": 2298, + "test/autoplan-admission.test.ts": 769, + "test/autoplan-amend-input.test.ts": 774, + "test/autoplan-answer-invalidation.test.ts": 6424, + "test/autoplan-artifact-recorder.test.ts": 5347, + "test/autoplan-artifact-windows-argv.test.ts": 62, + "test/autoplan-bounded-journal.test.ts": 307, + "test/autoplan-budget.test.ts": 7678, + "test/autoplan-capy-e2e.test.ts": 7548, + "test/autoplan-checkpoint.test.ts": 5891, + "test/autoplan-compaction-copies.test.ts": 138, + "test/autoplan-compaction-preserved-segment.test.ts": 1402, + "test/autoplan-driver.test.ts": 1904, + "test/autoplan-dual-voice-evidence.test.ts": 1131, + "test/autoplan-dual-voice-fixture.test.ts": 2757, + "test/autoplan-durability.test.ts": 1731, + "test/autoplan-edit-digests-al.test.ts": 268, + "test/autoplan-gate-artifacts.test.ts": 1954, + "test/autoplan-gate-judgment.test.ts": 14990, + "test/autoplan-guard-log.test.ts": 49, + "test/autoplan-guard-pty-fixture.test.ts": 18, + "test/autoplan-guard-reasons.test.ts": 13, + "test/autoplan-init.test.ts": 866, + "test/autoplan-long-session-observer.test.ts": 82, + "test/autoplan-method-read-audit.test.ts": 252, + "test/autoplan-methodology-names.test.ts": 897, + "test/autoplan-obligations.test.ts": 2163, + "test/autoplan-owned-read.test.ts": 2493, + "test/autoplan-owned-state.test.ts": 1471, + "test/autoplan-pairing.test.ts": 32, + "test/autoplan-pending-artifact.test.ts": 825, + "test/autoplan-pending-question.test.ts": 4151, + "test/autoplan-phase-handoff.test.ts": 94, + "test/autoplan-phase-observer.test.ts": 106, + "test/autoplan-phase-order.test.ts": 69, + "test/autoplan-preconfigured-onboarding-ar.test.ts": 804, + "test/autoplan-public-export.test.ts": 4410, + "test/autoplan-public-narration.test.ts": 41, + "test/autoplan-publication-generation.test.ts": 23, + "test/autoplan-publication-guard.test.ts": 1451, + "test/autoplan-publication-hook.test.ts": 2530, + "test/autoplan-publication-payload.test.ts": 2219, + "test/autoplan-record.test.ts": 2230, + "test/autoplan-residuals.test.ts": 5130, + "test/autoplan-review-discovery.test.ts": 9403, + "test/autoplan-revise.test.ts": 8220, + "test/autoplan-run-concurrency.test.ts": 3115, + "test/autoplan-run-recovery.test.ts": 7433, + "test/autoplan-schema-canary-compare.test.ts": 18, + "test/autoplan-scope.test.ts": 739, + "test/autoplan-snapshot.test.ts": 1765, + "test/autoplan-timing.test.ts": 279, + "test/benchmark-cli.test.ts": 531, + "test/benchmark-provider-adapters.test.ts": 53, + "test/benchmark-runner.test.ts": 19, + "test/bin-context-windows-slug.test.ts": 1812, + "test/bin-windows-bun-import-paths.test.ts": 874, + "test/binary-version-probe.test.ts": 134, + "test/binding-template-drift.test.ts": 41, + "test/bootstrap-retention-shard.test.ts": 1876, + "test/bootstrap-retention.test.ts": 10115, + "test/bootstrap-session-lifecycle.test.ts": 5082, + "test/brain-cache-roundtrip.test.ts": 80, + "test/brain-cache-spec.test.ts": 22, + "test/brain-preflight.test.ts": 22, + "test/brain-sync-removals.test.ts": 11559, + "test/brain-sync-windows-paths.test.ts": 10, + "test/brain-sync.test.ts": 88583, + "test/branch-slug-hygiene.test.ts": 620, + "test/build-cso-nonfatal.test.ts": 42, + "test/build-cso-result-record.test.ts": 407, + "test/build-gbrain-env.test.ts": 65, "test/build-script-shell-compat.test.ts": 15, - "test/builder-profile.test.ts": 4068, - "test/bun-resolve.test.ts": 687, - "test/bun-subprocess-fd-lifetime.test.ts": 78, - "test/bun-version-drift.test.ts": 532, - "test/bun-version-shells.test.ts": 849, - "test/cache-concurrent-refresh.test.ts": 143, - "test/capy-review-voice.test.ts": 9959, - "test/capy-skill-wrapper.test.ts": 7846, - "test/carve-guard-completeness.test.ts": 15, - "test/carve-guards-negative.test.ts": 34, - "test/carve-plan-fixture.test.ts": 464, - "test/carve-section-ordering.test.ts": 28, - "test/carve-section-sharding.test.ts": 1499, - "test/catalog-budget.test.ts": 49, - "test/catalog-mode-full.test.ts": 1085, - "test/catalog-trim.test.ts": 55, - "test/census-red-docs.test.ts": 38, - "test/ceo-barless-submit.test.ts": 114, - "test/ceo-count-ad-v2.test.ts": 37, - "test/ceo-expansion-auq.test.ts": 89, - "test/ceo-expansion-pacing-native.test.ts": 65, - "test/ceo-finding-fixture.test.ts": 842, - "test/ceo-hold-posture-review.test.ts": 130, - "test/ceo-mode-expansion-disposition.test.ts": 64, - "test/ceo-mode-handoff-hook.test.ts": 54, - "test/ceo-mode-handoff.test.ts": 80, - "test/ceo-mode-labels-native.test.ts": 45, - "test/ceo-mode-option.test.ts": 1644, - "test/ceo-mode-pending-submit.test.ts": 48, - "test/ceo-mode-posture-native.test.ts": 41, - "test/ceo-mode-preference-al.test.ts": 2215, - "test/ceo-mode-prerequisite.test.ts": 1149, - "test/ceo-mode-routing-fixture.test.ts": 944, - "test/ceo-plan-find-cross-branch.test.ts": 163, - "test/ceo-plan-mode-fixture.test.ts": 881, - "test/ceo-posture-packet.test.ts": 322, - "test/ceo-section-loading-fixture.test.ts": 358, - "test/ceo-split-collection.test.ts": 548, - "test/ceo-split-question-policy.test.ts": 442, - "test/ceo-stale-fill-decision.test.ts": 84, - "test/ceo-workflow-clarity.test.ts": 47, - "test/changed-files-union.test.ts": 380, - "test/chromium-sandbox-ci.test.ts": 174, - "test/ci-eval-cache.test.ts": 273, - "test/ci-image-cli-pin.test.ts": 33, - "test/ci-image-tag-binding.test.ts": 18, - "test/ci-native-evidence.test.ts": 782, - "test/ci-paid-coordination.test.ts": 6665, - "test/claude-code-migration.test.ts": 190, - "test/claude-code-runner.test.ts": 3027, - "test/claude-code-skill.test.ts": 1515, - "test/claude-code-windows-job.test.ts": 49, - "test/claude-md-size.test.ts": 10, - "test/claude-overlay-models.test.ts": 30, - "test/claude-overlay-setup-default.test.ts": 34280, - "test/claude-overlay-setup-installs.test.ts": 47530, - "test/claude-overlay-setup-lifecycle.test.ts": 53306, - "test/claude-producer-stream.test.ts": 3537, - "test/claude-provider-keychain.test.ts": 77, - "test/code-intelligence-cli.test.ts": 662, - "test/code-intelligence.test.ts": 4561, - "test/codex-boundary-evidence.test.ts": 12, - "test/codex-carve-fixture.test.ts": 929, - "test/codex-ci-access.test.ts": 238, - "test/codex-effort.test.ts": 1821, - "test/codex-eval-recording.test.ts": 318, - "test/codex-generation-model.test.ts": 115, - "test/codex-hardening.test.ts": 10324, - "test/codex-invocation-sites.test.ts": 418, - "test/codex-model-probe.test.ts": 5344, - "test/codex-multiblock.test.ts": 13, - "test/codex-probe-cache.test.ts": 12745, - "test/codex-probe-subcommands.test.ts": 5118, - "test/codex-resume-flag-semantics.test.ts": 16, - "test/codex-session-lifecycle.test.ts": 12019, - "test/codex-skill-isolation.test.ts": 145, - "test/codex-under-codex-detection.test.ts": 29, - "test/codex-web-search-flag.test.ts": 374, + "test/builder-profile.test.ts": 3308, + "test/bun-resolve.test.ts": 421, + "test/bun-subprocess-fd-lifetime.test.ts": 58, + "test/bun-version-drift.test.ts": 447, + "test/bun-version-shells.test.ts": 354, + "test/cache-concurrent-refresh.test.ts": 44, + "test/capy-review-voice.test.ts": 9422, + "test/capy-skill-wrapper.test.ts": 7874, + "test/carve-guard-completeness.test.ts": 13, + "test/carve-guards-negative.test.ts": 16, + "test/carve-plan-fixture.test.ts": 167, + "test/carve-section-ordering.test.ts": 27, + "test/carve-section-sharding.test.ts": 1515, + "test/catalog-budget.test.ts": 48, + "test/catalog-mode-full.test.ts": 803, + "test/catalog-trim.test.ts": 105, + "test/census-red-docs.test.ts": 37, + "test/ceo-barless-submit.test.ts": 104, + "test/ceo-count-ad-v2.test.ts": 31, + "test/ceo-expansion-auq.test.ts": 79, + "test/ceo-expansion-pacing-native.test.ts": 71, + "test/ceo-finding-fixture.test.ts": 958, + "test/ceo-hold-posture-review.test.ts": 168, + "test/ceo-mode-expansion-disposition.test.ts": 80, + "test/ceo-mode-handoff-hook.test.ts": 64, + "test/ceo-mode-handoff.test.ts": 89, + "test/ceo-mode-labels-native.test.ts": 57, + "test/ceo-mode-option.test.ts": 1751, + "test/ceo-mode-pending-submit.test.ts": 45, + "test/ceo-mode-posture-native.test.ts": 83, + "test/ceo-mode-preference-al.test.ts": 2755, + "test/ceo-mode-prerequisite.test.ts": 1450, + "test/ceo-mode-routing-fixture.test.ts": 1181, + "test/ceo-plan-find-cross-branch.test.ts": 168, + "test/ceo-plan-mode-fixture.test.ts": 953, + "test/ceo-posture-packet.test.ts": 334, + "test/ceo-section-loading-fixture.test.ts": 414, + "test/ceo-split-collection.test.ts": 622, + "test/ceo-split-question-policy.test.ts": 486, + "test/ceo-stale-fill-decision.test.ts": 96, + "test/ceo-workflow-clarity.test.ts": 44, + "test/changed-files-union.test.ts": 468, + "test/chromium-sandbox-ci.test.ts": 233, + "test/ci-eval-cache.test.ts": 307, + "test/ci-image-cli-pin.test.ts": 26, + "test/ci-image-tag-binding.test.ts": 12, + "test/ci-native-evidence.test.ts": 838, + "test/ci-paid-coordination.test.ts": 7431, + "test/claude-code-migration.test.ts": 177, + "test/claude-code-runner.test.ts": 3111, + "test/claude-code-skill.test.ts": 1506, + "test/claude-code-windows-job.test.ts": 50, + "test/claude-md-size.test.ts": 11, + "test/claude-overlay-models.test.ts": 21, + "test/claude-overlay-setup-default.test.ts": 29082, + "test/claude-overlay-setup-installs.test.ts": 43402, + "test/claude-overlay-setup-lifecycle.test.ts": 50190, + "test/claude-producer-stream.test.ts": 3548, + "test/claude-provider-keychain.test.ts": 76, + "test/code-intelligence-cli.test.ts": 638, + "test/code-intelligence.test.ts": 4335, + "test/codex-boundary-evidence.test.ts": 10, + "test/codex-carve-fixture.test.ts": 895, + "test/codex-ci-access.test.ts": 226, + "test/codex-effort.test.ts": 1458, + "test/codex-eval-recording.test.ts": 301, + "test/codex-generation-model.test.ts": 111, + "test/codex-hardening.test.ts": 9961, + "test/codex-invocation-sites.test.ts": 311, + "test/codex-model-probe.test.ts": 4423, + "test/codex-multiblock.test.ts": 11, + "test/codex-probe-cache.test.ts": 11411, + "test/codex-probe-subcommands.test.ts": 4930, + "test/codex-resume-flag-semantics.test.ts": 13, + "test/codex-session-lifecycle.test.ts": 12022, + "test/codex-skill-isolation.test.ts": 135, + "test/codex-under-codex-detection.test.ts": 20, + "test/codex-web-search-flag.test.ts": 299, + "test/compare-pr-version.test.ts": 77, "test/conductor-env-shim.test.ts": 9, - "test/context-bill.test.ts": 90, - "test/context-budget-ratchet.test.ts": 109, - "test/context-identity.test.ts": 25065, - "test/context-remaining-work-provenance.test.ts": 12, - "test/context-save-hardening.test.ts": 380, - "test/cookie-workflow-judge-input.test.ts": 774, - "test/cookie-workflow-manual-review.test.ts": 89, - "test/copilot-windows-bash.test.ts": 10, - "test/cost-known.test.ts": 51, - "test/coverage-audit-evidence.test.ts": 177, - "test/coverage-audit.test.ts": 1066, - "test/cso-audit-paths.test.ts": 17621, - "test/cso-bounded-file.test.ts": 21, - "test/cso-cache.test.ts": 320, - "test/cso-cli-lifecycle.test.ts": 4902, - "test/cso-cli-recheck.test.ts": 17452, - "test/cso-cli.test.ts": 29390, - "test/cso-contracts.test.ts": 241, - "test/cso-distribution.test.ts": 2968, - "test/cso-docker-argv0.test.ts": 939, - "test/cso-docker-integration.test.ts": 23, - "test/cso-docker-mounts.test.ts": 40, - "test/cso-docker-policy.test.ts": 24, - "test/cso-docker-socket-realpath.test.ts": 30, - "test/cso-eval.test.ts": 7893, - "test/cso-file-ceiling.test.ts": 30942, - "test/cso-git-global-config.test.ts": 33, - "test/cso-git-hardening.test.ts": 192, - "test/cso-history.test.ts": 11, - "test/cso-image-provisioning.test.ts": 264, - "test/cso-json-redaction.test.ts": 263, - "test/cso-launcher-generation.test.ts": 1547, - "test/cso-lease-identity.test.ts": 492, - "test/cso-macos-launcher.test.ts": 14, - "test/cso-native-evidence.test.ts": 85, - "test/cso-node-lifecycle-integration.test.ts": 35, - "test/cso-ntfs-fixture.test.ts": 159, - "test/cso-postgresql-policy.test.ts": 31, - "test/cso-preparation-adversarial.test.ts": 76, - "test/cso-preparation-container.test.ts": 59, - "test/cso-preparation-executor.test.ts": 707, - "test/cso-preparation-java.test.ts": 21, - "test/cso-preparation.test.ts": 84, - "test/cso-preserved.test.ts": 19, - "test/cso-public-ghcr.test.ts": 38, - "test/cso-python-runner-shadow.test.ts": 2567, - "test/cso-rails-verification.test.ts": 34, - "test/cso-registry-socket.test.ts": 2372, - "test/cso-requalification.test.ts": 3251, - "test/cso-runtime-promotion.test.ts": 142, - "test/cso-scanner-cli.test.ts": 7357, - "test/cso-scanner-docker-integration.test.ts": 36, - "test/cso-scanner-executor.test.ts": 165, - "test/cso-scanner-release.test.ts": 98, - "test/cso-scanners.test.ts": 84, - "test/cso-snapshot-disappearance-race.test.ts": 155, - "test/cso-snapshot-identity.test.ts": 491, - "test/cso-snapshot-state.test.ts": 3688, - "test/cso-spec-taxonomy-alignment.test.ts": 13, - "test/cso-stack-cold-integration.test.ts": 29, - "test/cso-staging-run.test.ts": 79, - "test/cso-verification-cleanup.test.ts": 1086, - "test/cso-verifier.test.ts": 32, - "test/cso-watchdog.test.ts": 8268, - "test/cso-windows-build-contract.test.ts": 17, - "test/cso-windows-docker.test.ts": 19, - "test/cso-windows-launcher.test.ts": 19, - "test/cso-witness.test.ts": 1037, - "test/declared-annotation.test.ts": 18, - "test/defensive-review-framing.test.ts": 15, - "test/dependency-security.test.ts": 55, - "test/deps-smoke.test.ts": 55, - "test/design-approval-flow.test.ts": 6353, - "test/design-board-reload.test.ts": 25, - "test/design-catalog.test.ts": 25, - "test/design-checklist-sync.test.ts": 882, - "test/design-completion-handoff-scored.test.ts": 38, - "test/design-consultation-command-contract.test.ts": 314, - "test/design-consultation-contract.test.ts": 27, - "test/design-daemon-windows-identity.test.ts": 35, - "test/design-detect-contract.test.ts": 134, - "test/design-detector-source-fixture.test.ts": 5858, - "test/design-doc-data-guard.test.ts": 14, - "test/design-flag-utils.test.ts": 27, - "test/design-html-section-completion.test.ts": 3129, - "test/design-md.test.ts": 495, - "test/design-printed-paths.test.ts": 94, - "test/design-research-fixture.test.ts": 61, - "test/design-setup-probe.test.ts": 11230, - "test/detector-corpora.test.ts": 392, - "test/detector-corpus-auto-decide-preserved.test.ts": 41, - "test/detector-corpus-plan-ceo-section-loading.test.ts": 41, - "test/detector-corpus-plan-design-review-plan-mode.test.ts": 241, - "test/detector-corpus-plan-eng-multi-finding-batching.test.ts": 215, - "test/detector-corpus-shared-libs-plan-callers.test.ts": 26, - "test/detector-corpus-ship-docsync-late-result.test.ts": 37, - "test/detector-corpus-ship-docsync-launch-failure.test.ts": 38, - "test/dev-setup-render-isolation.test.ts": 136, - "test/devex-finding-fixture.test.ts": 993, - "test/devex-peer-comparison-calibration.test.ts": 297, - "test/diagram-render-drift.test.ts": 312, - "test/diff-scope.test.ts": 1772, - "test/disabled-dated-record-at.test.ts": 69, - "test/disabled-plan-review-evidence.test.ts": 1267, - "test/discover-section-templates.test.ts": 13, - "test/dispatch-rules.test.ts": 85, - "test/distill-apply.test.ts": 914, - "test/distill-free-text.test.ts": 1038, - "test/docs-config-keys.test.ts": 155, - "test/docsync-atomic-writes.test.ts": 1686, - "test/docsync-authority.test.ts": 567, - "test/docsync-child-marker.test.ts": 23, - "test/docsync-command-grammar.test.ts": 303, - "test/docsync-fault-interface.test.ts": 27471, - "test/docsync-lifecycle-interface.test.ts": 124, - "test/docsync-nested-writes.test.ts": 994, - "test/docsync-report-interface.test.ts": 547, - "test/doctor-components.test.ts": 23, - "test/document-skills-redaction.test.ts": 17, - "test/dom-dump-hygiene.test.ts": 875, - "test/dx-error-capture.test.ts": 4905, - "test/dx-error-contract-grader.test.ts": 24, - "test/dx-error-contract.test.ts": 59, - "test/dx-selected-navigation-ap.test.ts": 112, - "test/e2e-harness-audit.test.ts": 27, - "test/e2e-shard-reuse.test.ts": 2398, - "test/e2e-tier-alignment.test.ts": 133, - "test/egress-lib.test.ts": 298, - "test/egress-receipt-wiring.test.ts": 197, - "test/egress-receipt.test.ts": 5416, - "test/empty-find-fallthrough.test.ts": 254, - "test/eng-batching-current-ledger.test.ts": 226, - "test/eng-batching-native-replay.test.ts": 63, - "test/eng-batching-saved-ledger.test.ts": 655, - "test/eng-count-question-policy.test.ts": 32, - "test/eng-devex-s-count.test.ts": 40, - "test/eng-finding-retry-budget.test.ts": 505, - "test/eng-first-review.test.ts": 205, - "test/eng-published-navigation.test.ts": 57, - "test/eng-resolution-block-position.test.ts": 313, - "test/eng-review-routing.test.ts": 28, - "test/eng-scope-entry-ap.test.ts": 85, - "test/eng-seeded-completion-ai.test.ts": 12146, - "test/eng-seeded-coverage.test.ts": 1357, - "test/eng-semantic-terminal.test.ts": 73, - "test/eng-test-plan-edit-approval.test.ts": 1092, - "test/env-host-fence-conformance.test.ts": 7437, - "test/env-host-runtime-assets.test.ts": 1372, - "test/eval-budgets-policy.test.ts": 47, - "test/eval-cli-family.test.ts": 1449, - "test/eval-detach-timeout-floor.test.ts": 1045, - "test/eval-flake-rank.test.ts": 263, - "test/eval-history.test.ts": 1756, - "test/eval-input-cache.test.ts": 76, - "test/eval-kinds.test.ts": 48, - "test/eval-list-cli.test.ts": 217, - "test/eval-model.test.ts": 10, - "test/eval-models.test.ts": 781, - "test/eval-plan-prereg-diagnostics.test.ts": 413, - "test/eval-sweep-workflow.test.ts": 12, - "test/evals-pr-lane-workflow.test.ts": 16, - "test/evals-tracking-issue.test.ts": 230, - "test/evals-workflow-wiring.test.ts": 255, - "test/evidence-bundle-portability.test.ts": 4311, - "test/evidence.test.ts": 13723, - "test/exit-propagation.test.ts": 338, - "test/explain-level-config.test.ts": 615, - "test/extension-pty-inject-invariant.test.ts": 13, - "test/external-host-carve.test.ts": 2109, - "test/flake-ledger.test.ts": 35, - "test/founder-resources-optout.test.ts": 204, - "test/free-fixtures.test.ts": 1247, - "test/free-home-guard.test.ts": 73, - "test/free-seed-ratchet.test.ts": 85, - "test/free-tests-workflow-wiring.test.ts": 29, - "test/freeze-owned-lifecycle.test.ts": 3130, - "test/frontend-scope.test.ts": 1249, - "test/fs-atomic.test.ts": 84, - "test/fs-utils.test.ts": 238, - "test/gate-diff-cli.test.ts": 3258, - "test/gate-diff.test.ts": 32, - "test/gate-integrity-templates.test.ts": 33, - "test/gate-list.test.ts": 258, - "test/gate-secret-scan.test.ts": 553, - "test/gate-tail-lint.test.ts": 1525, - "test/gbrain-cycle-completed.test.ts": 38, - "test/gbrain-detect-install.test.ts": 665, - "test/gbrain-detect-shape.test.ts": 489, - "test/gbrain-detection-override.test.ts": 1105, - "test/gbrain-dream-stage.test.ts": 289, - "test/gbrain-exec-invariant.test.ts": 13, - "test/gbrain-guards.test.ts": 26, - "test/gbrain-init-voyage-code-3.test.ts": 179, - "test/gbrain-lib-validate-varname.test.ts": 51, - "test/gbrain-lib-verify.test.ts": 166, - "test/gbrain-local-status.test.ts": 4029, - "test/gbrain-prune-gone-worktrees.test.ts": 375, - "test/gbrain-read-capability.test.ts": 1889, - "test/gbrain-refresh-install-render.test.ts": 12, - "test/gbrain-repo-policy-client.test.ts": 958, - "test/gbrain-repo-policy.test.ts": 1588, - "test/gbrain-source-gitignore.test.ts": 123, - "test/gbrain-source-worktree-advance.test.ts": 845, - "test/gbrain-sources-parse.test.ts": 21, - "test/gbrain-sources.test.ts": 102, - "test/gbrain-spawn-windows-shell.test.ts": 16, - "test/gbrain-structured-busy.test.ts": 1865, - "test/gbrain-supabase-provision.test.ts": 131, - "test/gbrain-sync-no-bash-message.test.ts": 23, - "test/gbrain-sync-skip.test.ts": 10871, - "test/gbrain-sync-voyage-code-3-integration.test.ts": 14, - "test/gen-skill-docs-checks.test.ts": 11740, - "test/gen-skill-docs-idempotency.test.ts": 2727, - "test/gen-skill-docs-import-purity.test.ts": 42, - "test/gen-skill-docs-out-dir.test.ts": 2461, - "test/gen-skill-docs-prune-stale.test.ts": 713, - "test/gen-skill-docs.test.ts": 6685, - "test/generated-bash-placeholders.test.ts": 764, - "test/generated-docs-fences.test.ts": 197, - "test/generator-eexist.test.ts": 389, - "test/git-ref-fixture-tripwire.test.ts": 112, - "test/gitattributes-eol.test.ts": 911, - "test/global-discover.test.ts": 769, - "test/greptile-early-cli.test.ts": 241, - "test/greptile-early.test.ts": 24, - "test/gstack-analytics-avg-duration.test.ts": 204, - "test/gstack-artifacts-init.test.ts": 6640, - "test/gstack-artifacts-url.test.ts": 119, - "test/gstack-brain-context-load.test.ts": 1004, - "test/gstack-browser-ensure.test.ts": 5469, - "test/gstack-capy-install.test.ts": 30245, + "test/context-bill.test.ts": 87, + "test/context-budget-ratchet.test.ts": 90, + "test/context-identity.test.ts": 24661, + "test/context-remaining-work-provenance.test.ts": 10, + "test/context-save-hardening.test.ts": 409, + "test/cookie-workflow-judge-input.test.ts": 1735, + "test/cookie-workflow-manual-review.test.ts": 93, + "test/copilot-windows-bash.test.ts": 11, + "test/cost-known.test.ts": 60, + "test/coverage-audit-evidence.test.ts": 197, + "test/coverage-audit.test.ts": 1061, + "test/cso-audit-paths.test.ts": 19830, + "test/cso-bounded-file.test.ts": 22, + "test/cso-cache.test.ts": 226, + "test/cso-cli-lifecycle.test.ts": 6493, + "test/cso-cli-recheck.test.ts": 20426, + "test/cso-cli.test.ts": 35485, + "test/cso-contracts.test.ts": 327, + "test/cso-distribution.test.ts": 3215, + "test/cso-docker-argv0.test.ts": 1018, + "test/cso-docker-integration.test.ts": 25, + "test/cso-docker-mounts.test.ts": 47, + "test/cso-docker-policy.test.ts": 23, + "test/cso-docker-socket-realpath.test.ts": 40, + "test/cso-eval.test.ts": 10845, + "test/cso-file-ceiling.test.ts": 31474, + "test/cso-git-global-config.test.ts": 38, + "test/cso-git-hardening.test.ts": 243, + "test/cso-history.test.ts": 18, + "test/cso-image-provisioning.test.ts": 317, + "test/cso-json-redaction.test.ts": 277, + "test/cso-launcher-generation.test.ts": 1825, + "test/cso-lease-identity.test.ts": 537, + "test/cso-macos-launcher.test.ts": 28, + "test/cso-native-evidence.test.ts": 122, + "test/cso-node-lifecycle-integration.test.ts": 49, + "test/cso-ntfs-fixture.test.ts": 209, + "test/cso-postgresql-policy.test.ts": 30, + "test/cso-preparation-adversarial.test.ts": 147, + "test/cso-preparation-container.test.ts": 105, + "test/cso-preparation-executor.test.ts": 914, + "test/cso-preparation-java.test.ts": 38, + "test/cso-preparation.test.ts": 115, + "test/cso-preserved.test.ts": 31, + "test/cso-public-ghcr.test.ts": 34, + "test/cso-python-runner-shadow.test.ts": 3438, + "test/cso-rails-verification.test.ts": 42, + "test/cso-registry-socket.test.ts": 1643, + "test/cso-requalification.test.ts": 2788, + "test/cso-runtime-promotion.test.ts": 146, + "test/cso-scanner-cli.test.ts": 8578, + "test/cso-scanner-docker-integration.test.ts": 41, + "test/cso-scanner-executor.test.ts": 171, + "test/cso-scanner-release.test.ts": 121, + "test/cso-scanners.test.ts": 83, + "test/cso-snapshot-disappearance-race.test.ts": 203, + "test/cso-snapshot-identity.test.ts": 634, + "test/cso-snapshot-state.test.ts": 4346, + "test/cso-spec-taxonomy-alignment.test.ts": 14, + "test/cso-stack-cold-integration.test.ts": 30, + "test/cso-staging-run.test.ts": 114, + "test/cso-verification-cleanup.test.ts": 1125, + "test/cso-verifier.test.ts": 42, + "test/cso-watchdog.test.ts": 8346, + "test/cso-windows-build-contract.test.ts": 14, + "test/cso-windows-docker.test.ts": 24, + "test/cso-windows-launcher.test.ts": 17, + "test/cso-witness.test.ts": 1151, + "test/declared-annotation.test.ts": 20, + "test/defensive-review-framing.test.ts": 19, + "test/dependency-security.test.ts": 70, + "test/deps-smoke.test.ts": 60, + "test/design-approval-flow.test.ts": 6280, + "test/design-board-reload.test.ts": 21, + "test/design-catalog.test.ts": 68, + "test/design-checklist-sync.test.ts": 994, + "test/design-completion-handoff-scored.test.ts": 42, + "test/design-consultation-command-contract.test.ts": 307, + "test/design-consultation-contract.test.ts": 30, + "test/design-daemon-windows-identity.test.ts": 55, + "test/design-detect-contract.test.ts": 152, + "test/design-detector-source-fixture.test.ts": 6738, + "test/design-doc-data-guard.test.ts": 12, + "test/design-flag-utils.test.ts": 26, + "test/design-html-section-completion.test.ts": 3508, + "test/design-md.test.ts": 639, + "test/design-printed-paths.test.ts": 99, + "test/design-research-fixture.test.ts": 12, + "test/design-setup-probe.test.ts": 11257, + "test/detector-corpora.test.ts": 571, + "test/detector-corpus-auto-decide-preserved.test.ts": 49, + "test/detector-corpus-plan-ceo-section-loading.test.ts": 51, + "test/detector-corpus-plan-design-review-plan-mode.test.ts": 246, + "test/detector-corpus-plan-eng-multi-finding-batching.test.ts": 255, + "test/detector-corpus-shared-libs-plan-callers.test.ts": 20, + "test/detector-corpus-ship-docsync-late-result.test.ts": 50, + "test/detector-corpus-ship-docsync-launch-failure.test.ts": 51, + "test/dev-setup-render-isolation.test.ts": 143, + "test/devex-finding-fixture.test.ts": 1032, + "test/devex-peer-comparison-calibration.test.ts": 378, + "test/diagram-render-drift.test.ts": 408, + "test/diff-scope.test.ts": 2326, + "test/disabled-dated-record-at.test.ts": 75, + "test/disabled-plan-review-evidence.test.ts": 1386, + "test/discover-section-templates.test.ts": 10, + "test/dispatch-rules.test.ts": 38, + "test/distill-apply.test.ts": 1091, + "test/distill-free-text.test.ts": 1114, + "test/docs-config-keys.test.ts": 143, + "test/docsync-atomic-writes.test.ts": 1725, + "test/docsync-authority.test.ts": 699, + "test/docsync-child-marker.test.ts": 18, + "test/docsync-command-grammar.test.ts": 714, + "test/docsync-fault-interface.test.ts": 37247, + "test/docsync-lifecycle-interface.test.ts": 155, + "test/docsync-nested-writes.test.ts": 856, + "test/docsync-report-interface.test.ts": 473, + "test/doctor-components.test.ts": 20, + "test/document-skills-redaction.test.ts": 26, + "test/dom-dump-hygiene.test.ts": 870, + "test/dx-error-capture.test.ts": 5039, + "test/dx-error-contract-grader.test.ts": 21, + "test/dx-error-contract.test.ts": 44, + "test/dx-selected-navigation-ap.test.ts": 151, + "test/e2e-harness-audit.test.ts": 20, + "test/e2e-shard-reuse.test.ts": 2497, + "test/e2e-tier-alignment.test.ts": 148, + "test/egress-lib.test.ts": 282, + "test/egress-receipt-wiring.test.ts": 193, + "test/egress-receipt.test.ts": 5414, + "test/empty-find-fallthrough.test.ts": 222, + "test/eng-batching-current-ledger.test.ts": 208, + "test/eng-batching-native-replay.test.ts": 59, + "test/eng-batching-saved-ledger.test.ts": 582, + "test/eng-count-question-policy.test.ts": 31, + "test/eng-devex-s-count.test.ts": 29, + "test/eng-finding-retry-budget.test.ts": 469, + "test/eng-first-review.test.ts": 295, + "test/eng-published-navigation.test.ts": 45, + "test/eng-resolution-block-position.test.ts": 242, + "test/eng-review-routing.test.ts": 21, + "test/eng-scope-entry-ap.test.ts": 79, + "test/eng-seeded-completion-ai.test.ts": 12139, + "test/eng-seeded-coverage.test.ts": 1676, + "test/eng-semantic-terminal.test.ts": 72, + "test/eng-test-plan-edit-approval.test.ts": 986, + "test/env-host-fence-conformance.test.ts": 8225, + "test/env-host-runtime-assets.test.ts": 1402, + "test/eval-budgets-policy.test.ts": 38, + "test/eval-cli-family.test.ts": 3123, + "test/eval-detach-timeout-floor.test.ts": 1199, + "test/eval-flake-rank.test.ts": 278, + "test/eval-history.test.ts": 1919, + "test/eval-input-cache.test.ts": 59, + "test/eval-kinds.test.ts": 67, + "test/eval-list-cli.test.ts": 284, + "test/eval-model.test.ts": 33, + "test/eval-models.test.ts": 1233, + "test/eval-plan-prereg-diagnostics.test.ts": 547, + "test/eval-readiness.test.ts": 66, + "test/eval-sweep-workflow.test.ts": 14, + "test/evals-pr-lane-workflow.test.ts": 29, + "test/evals-tracking-issue.test.ts": 327, + "test/evals-workflow-wiring.test.ts": 239, + "test/evidence-bundle-portability.test.ts": 5008, + "test/evidence.test.ts": 15876, + "test/exercised-map.test.ts": 14205, + "test/exit-propagation.test.ts": 351, + "test/explain-level-config.test.ts": 866, + "test/extension-pty-inject-invariant.test.ts": 18, + "test/external-host-carve.test.ts": 2256, + "test/flake-ledger.test.ts": 23, + "test/founder-resources-optout.test.ts": 164, + "test/free-durations-record-workflow.test.ts": 364, + "test/free-fixtures.test.ts": 2204, + "test/free-home-guard.test.ts": 92, + "test/free-seed-ratchet.test.ts": 92, + "test/free-tests-workflow-wiring.test.ts": 20, + "test/freeze-owned-lifecycle.test.ts": 3234, + "test/frontend-scope.test.ts": 1312, + "test/fs-atomic.test.ts": 39, + "test/fs-utils.test.ts": 265, + "test/gate-cost.test.ts": 2107, + "test/gate-diff-cli.test.ts": 3892, + "test/gate-diff.test.ts": 49, + "test/gate-integrity-templates.test.ts": 45, + "test/gate-list.test.ts": 312, + "test/gate-reuse.test.ts": 6384, + "test/gate-secret-scan.test.ts": 603, + "test/gate-tail-lint.test.ts": 1603, + "test/gbrain-cycle-completed.test.ts": 84, + "test/gbrain-detect-install.test.ts": 778, + "test/gbrain-detect-shape.test.ts": 552, + "test/gbrain-detection-override.test.ts": 1112, + "test/gbrain-dream-stage.test.ts": 309, + "test/gbrain-exec-invariant.test.ts": 20, + "test/gbrain-guards.test.ts": 33, + "test/gbrain-init-voyage-code-3.test.ts": 201, + "test/gbrain-lib-validate-varname.test.ts": 47, + "test/gbrain-lib-verify.test.ts": 168, + "test/gbrain-local-status.test.ts": 3942, + "test/gbrain-prune-gone-worktrees.test.ts": 310, + "test/gbrain-read-capability.test.ts": 1900, + "test/gbrain-refresh-install-render.test.ts": 21, + "test/gbrain-repo-policy-client.test.ts": 835, + "test/gbrain-repo-policy.test.ts": 1620, + "test/gbrain-source-gitignore.test.ts": 86, + "test/gbrain-source-worktree-advance.test.ts": 1072, + "test/gbrain-sources-parse.test.ts": 18, + "test/gbrain-sources.test.ts": 128, + "test/gbrain-spawn-windows-shell.test.ts": 13, + "test/gbrain-structured-busy.test.ts": 1773, + "test/gbrain-supabase-provision.test.ts": 126, + "test/gbrain-sync-no-bash-message.test.ts": 19, + "test/gbrain-sync-skip.test.ts": 10846, + "test/gbrain-sync-voyage-code-3-integration.test.ts": 15, + "test/gen-skill-docs-checks.test.ts": 12316, + "test/gen-skill-docs-idempotency.test.ts": 2543, + "test/gen-skill-docs-import-purity.test.ts": 47, + "test/gen-skill-docs-out-dir.test.ts": 2503, + "test/gen-skill-docs-prune-stale.test.ts": 893, + "test/gen-skill-docs.test.ts": 7015, + "test/generated-bash-placeholders.test.ts": 848, + "test/generated-docs-fences.test.ts": 331, + "test/generator-eexist.test.ts": 405, + "test/git-ref-fixture-tripwire.test.ts": 139, + "test/gitattributes-eol.test.ts": 822, + "test/global-discover.test.ts": 987, + "test/greptile-early-cli.test.ts": 288, + "test/greptile-early.test.ts": 36, + "test/gstack-analytics-avg-duration.test.ts": 249, + "test/gstack-artifacts-init.test.ts": 6375, + "test/gstack-artifacts-url.test.ts": 195, + "test/gstack-brain-context-load.test.ts": 1007, + "test/gstack-browser-ensure.test.ts": 5528, + "test/gstack-capy-install.test.ts": 26225, "test/gstack-ceo-plan-find.test.ts": 227, - "test/gstack-ci-gate.test.ts": 25, - "test/gstack-codex-session-import.test.ts": 696, - "test/gstack-config-browse-extension-id.test.ts": 375, - "test/gstack-config-cross-project.test.ts": 284, - "test/gstack-config-defaults.test.ts": 6231, - "test/gstack-config-gbrain-refresh.test.ts": 137, - "test/gstack-config-key-locale.test.ts": 139, - "test/gstack-config-memorable-key.test.ts": 351, - "test/gstack-config-redact-keys.test.ts": 243, - "test/gstack-config-transcript-scope.test.ts": 2709, - "test/gstack-context-recovery.test.ts": 110, - "test/gstack-decision-bins.test.ts": 5874, - "test/gstack-decision-semantic.test.ts": 39, - "test/gstack-decision.test.ts": 29, - "test/gstack-design-claim.test.ts": 644, - "test/gstack-design-detect-plugins.test.ts": 407, - "test/gstack-design-detect.test.ts": 15669, - "test/gstack-design-doc-find.test.ts": 348, - "test/gstack-detach.test.ts": 23812, - "test/gstack-developer-profile.test.ts": 13090, - "test/gstack-docs-candidate.test.ts": 770, - "test/gstack-doctor.test.ts": 28549, - "test/gstack-egress-cli.test.ts": 546, - "test/gstack-ff-update.test.ts": 15528, - "test/gstack-gbrain-detect-mcp-mode.test.ts": 1044, - "test/gstack-gbrain-mcp-verify.test.ts": 1312, - "test/gstack-gbrain-source-wireup.test.ts": 3055, - "test/gstack-gbrain-sync-transcript-mode.test.ts": 2557, - "test/gstack-gbrain-sync.test.ts": 2053, - "test/gstack-home-module-scope.test.ts": 75, - "test/gstack-launch-probe.test.ts": 103, - "test/gstack-learnings-search.test.ts": 1729, - "test/gstack-lease.test.ts": 4117, - "test/gstack-memorable.test.ts": 8662, - "test/gstack-memory-helpers.test.ts": 11106, - "test/gstack-memory-ingest.test.ts": 13044, - "test/gstack-next-version.test.ts": 1435, - "test/gstack-paths.test.ts": 918, - "test/gstack-post-cli.test.ts": 447, - "test/gstack-post.test.ts": 27, - "test/gstack-question-log.test.ts": 2562, - "test/gstack-question-preference.test.ts": 8575, - "test/gstack-redact-cli.test.ts": 3346, - "test/gstack-regen.test.ts": 2003, - "test/gstack-render-cli.test.ts": 392, - "test/gstack-repo-mode.test.ts": 1337, - "test/gstack-restamp-queue.test.ts": 9214, - "test/gstack-retro-metrics.test.ts": 818, - "test/gstack-safe-git.test.ts": 1090, - "test/gstack-schema-pack.test.ts": 12, - "test/gstack-session-kind.test.ts": 118, - "test/gstack-settings-hook-schema-aware.test.ts": 4992, - "test/gstack-settings-hook-symlink.test.ts": 1263, - "test/gstack-ship-policy.test.ts": 1290, - "test/gstack-ship-queue.test.ts": 11840, - "test/gstack-ship-receipt.test.ts": 1833, - "test/gstack-skill-start.test.ts": 24611, - "test/gstack-slug-cwd-walk-up.test.ts": 898, - "test/gstack-slug-get.test.ts": 521, - "test/gstack-slug-parity.test.ts": 5082, - "test/gstack-slug-sanitize.test.ts": 162, - "test/gstack-state-root-override.test.ts": 552, - "test/gstack-team-init-hook-schema.test.ts": 194, - "test/gstack-upgrade-bin.test.ts": 11370, - "test/gstack-upgrade-fresh-shell.test.ts": 6111, - "test/gstack-upgrade-migration-v1_17_0_0.test.ts": 53, - "test/gstack-upgrade-migration-v1_37_0_0.test.ts": 119, - "test/gstack-upgrade-migration-v1_40_0_0.test.ts": 202, - "test/gstack-upgrade-migration-v1_78_0_0.test.ts": 39, - "test/gstack-upgrade-migration-v1_91_19_0.test.ts": 177, - "test/gstack-upgrade-path-spelling.test.ts": 530, - "test/gstack-version-bump.test.ts": 1721, - "test/gstack-wtree-windows-paths.test.ts": 162, - "test/harness-version.test.ts": 106, - "test/headless-artifacts.test.ts": 792, - "test/health-capture.test.ts": 158, - "test/health-eval-fixture.test.ts": 196, - "test/held-install-recovery.test.ts": 32501, - "test/helpers-unit.test.ts": 40, - "test/helpers/budget-override.test.ts": 71, - "test/helpers/capture-parity-baseline.test.ts": 163, - "test/helpers/claude-pty-runner.auq.unit.test.ts": 55, - "test/helpers/claude-pty-runner.boundaries.unit.test.ts": 62, - "test/helpers/claude-pty-runner.classify.unit.test.ts": 46, - "test/helpers/claude-pty-runner.launch.unit.test.ts": 313, - "test/helpers/claude-pty-runner.plan-native.unit.test.ts": 41, - "test/helpers/claude-pty-runner.runners.unit.test.ts": 277, - "test/helpers/claude-pty-runner.scope-gate-floor.unit.test.ts": 45, - "test/helpers/claude-pty-runner.screen.unit.test.ts": 42, - "test/helpers/e2e-gate.unit.test.ts": 18, - "test/helpers/eval-store.test.ts": 303, - "test/helpers/hermetic-env.test.ts": 1393, - "test/helpers/observability.test.ts": 77, - "test/helpers/providers/gemini.test.ts": 21, - "test/helpers/resolve-repo-path.test.ts": 13, - "test/helpers/run-bin.test.ts": 23, - "test/helpers/session-runner.test.ts": 98, - "test/helpers/sync-command-capture.test.ts": 1065, - "test/heredoc-pipe-deadlock.test.ts": 144, - "test/hermetic-skill-runtime.test.ts": 2605, - "test/hermetic-skills-seeding.test.ts": 204, - "test/hermetic-wiring.test.ts": 176, + "test/gstack-ci-gate.test.ts": 29, + "test/gstack-codex-session-import.test.ts": 613, + "test/gstack-config-browse-extension-id.test.ts": 361, + "test/gstack-config-cross-project.test.ts": 240, + "test/gstack-config-defaults.test.ts": 5462, + "test/gstack-config-gbrain-refresh.test.ts": 149, + "test/gstack-config-key-locale.test.ts": 128, + "test/gstack-config-memorable-key.test.ts": 342, + "test/gstack-config-redact-keys.test.ts": 214, + "test/gstack-config-transcript-scope.test.ts": 2451, + "test/gstack-context-recovery.test.ts": 123, + "test/gstack-decision-bins.test.ts": 6341, + "test/gstack-decision-semantic.test.ts": 44, + "test/gstack-decision.test.ts": 41, + "test/gstack-design-claim.test.ts": 852, + "test/gstack-design-detect-plugins.test.ts": 427, + "test/gstack-design-detect.test.ts": 15747, + "test/gstack-design-doc-find.test.ts": 464, + "test/gstack-detach.test.ts": 23575, + "test/gstack-developer-profile.test.ts": 12010, + "test/gstack-docs-candidate.test.ts": 1064, + "test/gstack-doctor.test.ts": 26676, + "test/gstack-egress-cli.test.ts": 628, + "test/gstack-ff-update.test.ts": 14842, + "test/gstack-gbrain-detect-mcp-mode.test.ts": 1037, + "test/gstack-gbrain-mcp-verify.test.ts": 1126, + "test/gstack-gbrain-source-wireup.test.ts": 2813, + "test/gstack-gbrain-sync-transcript-mode.test.ts": 2607, + "test/gstack-gbrain-sync.test.ts": 1740, + "test/gstack-home-module-scope.test.ts": 64, + "test/gstack-launch-probe.test.ts": 74, + "test/gstack-learnings-search.test.ts": 1364, + "test/gstack-lease.test.ts": 3330, + "test/gstack-memorable.test.ts": 7491, + "test/gstack-memory-helpers.test.ts": 11108, + "test/gstack-memory-ingest.test.ts": 10632, + "test/gstack-next-version.test.ts": 18668, + "test/gstack-paths.test.ts": 724, + "test/gstack-post-cli.test.ts": 379, + "test/gstack-post.test.ts": 20, + "test/gstack-question-log.test.ts": 2265, + "test/gstack-question-preference.test.ts": 7053, + "test/gstack-redact-cli.test.ts": 3269, + "test/gstack-regen.test.ts": 2361, + "test/gstack-render-cli.test.ts": 371, + "test/gstack-repo-mode.test.ts": 1034, + "test/gstack-restamp-queue.test.ts": 9311, + "test/gstack-retro-metrics.test.ts": 727, + "test/gstack-safe-git.test.ts": 1146, + "test/gstack-schema-pack.test.ts": 8, + "test/gstack-session-kind.test.ts": 104, + "test/gstack-settings-hook-schema-aware.test.ts": 5006, + "test/gstack-settings-hook-symlink.test.ts": 1301, + "test/gstack-ship-policy.test.ts": 1407, + "test/gstack-ship-queue.test.ts": 12022, + "test/gstack-ship-receipt.test.ts": 1772, + "test/gstack-skill-start.test.ts": 26358, + "test/gstack-slug-cwd-walk-up.test.ts": 907, + "test/gstack-slug-get.test.ts": 512, + "test/gstack-slug-parity.test.ts": 4206, + "test/gstack-slug-sanitize.test.ts": 137, + "test/gstack-state-root-override.test.ts": 606, + "test/gstack-team-init-hook-schema.test.ts": 156, + "test/gstack-upgrade-bin.test.ts": 11332, + "test/gstack-upgrade-fresh-shell.test.ts": 5572, + "test/gstack-upgrade-migration-v1_17_0_0.test.ts": 56, + "test/gstack-upgrade-migration-v1_37_0_0.test.ts": 103, + "test/gstack-upgrade-migration-v1_40_0_0.test.ts": 182, + "test/gstack-upgrade-migration-v1_78_0_0.test.ts": 37, + "test/gstack-upgrade-migration-v1_91_19_0.test.ts": 135, + "test/gstack-upgrade-path-spelling.test.ts": 595, + "test/gstack-version-bump.test.ts": 2219, + "test/gstack-wtree-windows-paths.test.ts": 229, + "test/harness-version.test.ts": 127, + "test/headless-artifacts.test.ts": 911, + "test/health-capture.test.ts": 154, + "test/health-eval-fixture.test.ts": 332, + "test/held-install-recovery.test.ts": 30520, + "test/helpers-unit.test.ts": 66, + "test/helpers/budget-override.test.ts": 20, + "test/helpers/capture-parity-baseline.test.ts": 152, + "test/helpers/claude-pty-runner.auq.unit.test.ts": 60, + "test/helpers/claude-pty-runner.boundaries.unit.test.ts": 67, + "test/helpers/claude-pty-runner.classify.unit.test.ts": 50, + "test/helpers/claude-pty-runner.launch.unit.test.ts": 336, + "test/helpers/claude-pty-runner.plan-native.unit.test.ts": 30, + "test/helpers/claude-pty-runner.runners.unit.test.ts": 426, + "test/helpers/claude-pty-runner.scope-gate-floor.unit.test.ts": 40, + "test/helpers/claude-pty-runner.screen.unit.test.ts": 32, + "test/helpers/e2e-gate.unit.test.ts": 11, + "test/helpers/eval-store.test.ts": 261, + "test/helpers/hermetic-env.test.ts": 951, + "test/helpers/observability.test.ts": 74, + "test/helpers/providers/gemini.test.ts": 25, + "test/helpers/resolve-repo-path.test.ts": 11, + "test/helpers/run-bin.test.ts": 21, + "test/helpers/session-runner.test.ts": 114, + "test/helpers/sync-command-capture.test.ts": 1056, + "test/heredoc-pipe-deadlock.test.ts": 109, + "test/hermetic-skill-runtime.test.ts": 2211, + "test/hermetic-skills-seeding.test.ts": 138, + "test/hermetic-wiring.test.ts": 169, "test/honesty-kernel.test.ts": 28, - "test/hook-log.test.ts": 269, - "test/hook-scripts.test.ts": 26813, - "test/hooks-windows-paths.test.ts": 164, - "test/host-config.test.ts": 2995, - "test/host-conformance.test.ts": 86601, - "test/host-renders-prune.test.ts": 63485, - "test/host-renders.test.ts": 25268, - "test/hostile-path-writers.test.ts": 2348, - "test/impeccable-fixtures.test.ts": 13, - "test/install-fixture.test.ts": 1153, - "test/install-verdict.test.ts": 1544, - "test/instruction-facts.test.ts": 54, - "test/investigate-freeze-path.test.ts": 5496, - "test/investigate-recipes.test.ts": 16, - "test/ios-debug-bridge-release-guard.test.ts": 24, + "test/hook-log.test.ts": 232, + "test/hook-scripts.test.ts": 28307, + "test/hooks-windows-paths.test.ts": 192, + "test/host-config.test.ts": 2874, + "test/host-conformance.test.ts": 73521, + "test/host-renders-prune.test.ts": 57494, + "test/host-renders.test.ts": 26926, + "test/hostile-path-writers.test.ts": 1951, + "test/impeccable-fixtures.test.ts": 14, + "test/install-fixture.test.ts": 903, + "test/install-verdict.test.ts": 1435, + "test/instruction-facts.test.ts": 68, + "test/investigate-freeze-path.test.ts": 5568, + "test/investigate-recipes.test.ts": 15, + "test/ios-debug-bridge-release-guard.test.ts": 13, "test/ios-qa-daemon-cli.test.ts": 18, - "test/ios-qa-regen.test.ts": 327, - "test/ios-qa-stateserver-hardening.test.ts": 19, - "test/ios-qa-swift-build.test.ts": 20, - "test/ios-qa.test.ts": 201, - "test/is-conductor.test.ts": 14, - "test/jargon-list.test.ts": 13, - "test/jsonl-merge.test.ts": 314, - "test/jsonl-store.test.ts": 14, - "test/judge-calibration.test.ts": 45, - "test/judge-panel-median.test.ts": 28, - "test/land-and-deploy-ci-gate.test.ts": 3161, - "test/land-and-deploy-command-trigger.test.ts": 11, - "test/land-and-deploy-flow.test.ts": 660, - "test/land-and-deploy-postfail.test.ts": 14, - "test/lane-coordination.test.ts": 1839, - "test/learnings-callers-unavailable.test.ts": 1418, - "test/learnings-injection.test.ts": 12, - "test/learnings.test.ts": 5358, - "test/llm-judge-abort.test.ts": 28, - "test/llm-judge-frontier.test.ts": 107, - "test/llm-judge-stream.test.ts": 35, - "test/llms-txt-shape.test.ts": 28, - "test/maintenance-cli-args.test.ts": 176, - "test/memorable-user-prompt-hook.test.ts": 11489, - "test/memory-cache-injection.test.ts": 282, - "test/memory-ingest-include-gitignored.test.ts": 59, - "test/memory-ingest-landing.test.ts": 1745, - "test/memory-ingest-sources.test.ts": 2691, - "test/memory-ingest-timeout.test.ts": 27, - "test/memory-pipeline.test.ts": 656, - "test/migration-checkpoint-ownership.test.ts": 152, - "test/migrations-v1.27.0.0.test.ts": 827, - "test/migrations-v1.65.0.0.test.ts": 242, - "test/mktemp-portability.test.ts": 81, - "test/model-catalog-isolation.test.ts": 69, - "test/model-overlays.test.ts": 19, - "test/model-policy-cli.test.ts": 3220, - "test/model-policy-freshness.test.ts": 155, - "test/model-policy-workflow.test.ts": 280, - "test/model-policy.test.ts": 70, - "test/model-role-invocation.test.ts": 35711, - "test/module-size-ratchet.test.ts": 185, - "test/native-auto-decide-pty.test.ts": 10148, - "test/native-auto-decide.test.ts": 261, - "test/no-quoted-tilde-assignments.test.ts": 29, - "test/no-stale-gstack-brain-refs.test.ts": 350, - "test/no-suicide-exit.test.ts": 115, - "test/office-hours-attempt.test.ts": 39882, - "test/office-hours-budget.test.ts": 38, - "test/office-hours-completion.test.ts": 128, - "test/office-hours-open-question-form.test.ts": 15, - "test/office-hours-phase4-caller.test.ts": 29, - "test/office-hours-review.test.ts": 997, - "test/office-hours-writeback-env.test.ts": 452, - "test/office-posture-recording.test.ts": 46, - "test/onboarding-moved-literals.test.ts": 40, - "test/one-way-doors.test.ts": 11, + "test/ios-qa-regen.test.ts": 2085, + "test/ios-qa-stateserver-hardening.test.ts": 15, + "test/ios-qa-swift-build.test.ts": 15, + "test/ios-qa.test.ts": 192, + "test/is-conductor.test.ts": 13, + "test/jargon-list.test.ts": 17, + "test/jsonl-merge.test.ts": 425, + "test/jsonl-store.test.ts": 26, + "test/judge-calibration.test.ts": 43, + "test/judge-panel-median.test.ts": 36, + "test/land-and-deploy-ci-gate.test.ts": 3252, + "test/land-and-deploy-command-trigger.test.ts": 14, + "test/land-and-deploy-flow.test.ts": 696, + "test/land-and-deploy-postfail.test.ts": 16, + "test/lane-coordination.test.ts": 2106, + "test/learnings-callers-unavailable.test.ts": 1491, + "test/learnings-injection.test.ts": 10, + "test/learnings.test.ts": 4875, + "test/llm-judge-abort.test.ts": 30, + "test/llm-judge-frontier.test.ts": 82, + "test/llm-judge-stream.test.ts": 32, + "test/llms-txt-shape.test.ts": 32, + "test/maintenance-cli-args.test.ts": 151, + "test/memorable-user-prompt-hook.test.ts": 12006, + "test/memory-cache-injection.test.ts": 272, + "test/memory-ingest-include-gitignored.test.ts": 41, + "test/memory-ingest-landing.test.ts": 1418, + "test/memory-ingest-sources.test.ts": 2812, + "test/memory-ingest-timeout.test.ts": 28, + "test/memory-pipeline.test.ts": 618, + "test/migration-checkpoint-ownership.test.ts": 144, + "test/migrations-v1.27.0.0.test.ts": 693, + "test/migrations-v1.65.0.0.test.ts": 218, + "test/mktemp-portability.test.ts": 79, + "test/model-catalog-isolation.test.ts": 79, + "test/model-overlays.test.ts": 22, + "test/model-policy-cli.test.ts": 3631, + "test/model-policy-freshness.test.ts": 243, + "test/model-policy-workflow.test.ts": 386, + "test/model-policy.test.ts": 84, + "test/model-role-invocation.test.ts": 35850, + "test/module-size-ratchet.test.ts": 261, + "test/native-auto-decide-pty.test.ts": 10224, + "test/native-auto-decide.test.ts": 279, + "test/no-quoted-tilde-assignments.test.ts": 26, + "test/no-stale-gstack-brain-refs.test.ts": 323, + "test/no-suicide-exit.test.ts": 105, + "test/office-hours-attempt.test.ts": 38967, + "test/office-hours-budget.test.ts": 41, + "test/office-hours-completion.test.ts": 146, + "test/office-hours-open-question-form.test.ts": 16, + "test/office-hours-phase4-caller.test.ts": 34, + "test/office-hours-review.test.ts": 951, + "test/office-hours-writeback-env.test.ts": 385, + "test/office-posture-recording.test.ts": 37, + "test/onboarding-moved-literals.test.ts": 32, + "test/one-way-doors.test.ts": 10, "test/open-gstack-browser-liveness.test.ts": 10, - "test/openclaw-native-skills.test.ts": 12, - "test/osv-config-wiring.test.ts": 33, - "test/osv-ignore-expiry.test.ts": 12, - "test/outside-consent.test.ts": 1038, - "test/outside-review-result.test.ts": 674, - "test/outside-review-severity.test.ts": 68, - "test/outside-review-unparsed-severity.test.ts": 139, - "test/outside-runner-contract.test.ts": 739, - "test/outside-send-boundary.test.ts": 380, - "test/outside-voice-evidence.test.ts": 44, - "test/outside-voice-failure-policy.test.ts": 121, - "test/outside-voice-fixture.test.ts": 142, - "test/outside-voice-invocation.test.ts": 9113, - "test/outside-voice-native-deadline.test.ts": 41, - "test/outside-voice-preflight.test.ts": 405, - "test/outside-voice-provenance.test.ts": 1103, - "test/outside-voice-receipt.test.ts": 358, - "test/outside-voice-routing.test.ts": 1110, - "test/overlay-lifecycle.test.ts": 341, - "test/overlay-measurement.test.ts": 283, - "test/overlay-recording-order.test.ts": 24, - "test/overlay-sdk-cancel-eof.test.ts": 137, - "test/owner-brief-contributor-banned.test.ts": 462, - "test/paid-free-boundary.test.ts": 2384, - "test/paid-orphan-tripwire.test.ts": 86, - "test/paid-overlay-scheduling.test.ts": 406, - "test/paid-pr-profile.test.ts": 1429, - "test/paid-report-census.test.ts": 139, - "test/paid-report-fail-open.test.ts": 1208, - "test/paid-retry-supervision.test.ts": 1091, - "test/paid-run-manifest.test.ts": 1630, - "test/paid-selection-propagation.test.ts": 63, - "test/paid-shard-settlement.test.ts": 1274, - "test/paid-shards.test.ts": 6029, - "test/paid-slice-deadline.test.ts": 1599, - "test/pair-agent-token-hygiene.test.ts": 11, - "test/parity-baseline-integrity.test.ts": 20, - "test/parity-sectioned.test.ts": 15, - "test/parity-suite.test.ts": 102, - "test/path-spelling.test.ts": 325, - "test/pending-question-completion.test.ts": 114, - "test/periodic-exclude-policy.test.ts": 88, - "test/plan-ceo-floor-interactive-boundary.test.ts": 1273, - "test/plan-count-artifacts.test.ts": 43, - "test/plan-count-checkbox.test.ts": 15295, - "test/plan-count-clipped-elision.test.ts": 60, - "test/plan-count-collection-completion.test.ts": 30506, - "test/plan-count-completion.test.ts": 13994, - "test/plan-count-cropped-wrap.test.ts": 123, - "test/plan-count-cross-cwd-ancestry.test.ts": 141, - "test/plan-count-design-ui-recovery.test.ts": 15966, - "test/plan-count-dx-handoff.test.ts": 77, - "test/plan-count-empty-review.test.ts": 1450, - "test/plan-count-file-permission.test.ts": 44707, - "test/plan-count-fixture.test.ts": 15071, - "test/plan-count-history.test.ts": 711, - "test/plan-count-long-edit.test.ts": 38, - "test/plan-count-native-input.test.ts": 11607, - "test/plan-count-owned-permission.test.ts": 11344, - "test/plan-count-pending-exit.test.ts": 296, - "test/plan-count-prerequisite.test.ts": 45, - "test/plan-count-preview-footer.test.ts": 3458, - "test/plan-count-session-cwd.test.ts": 66, - "test/plan-count-timeout.test.ts": 9464, - "test/plan-count-transcript.test.ts": 58, - "test/plan-count-truncated-border.test.ts": 48, - "test/plan-count-truncated-question.test.ts": 36, - "test/plan-create-combined-permission.test.ts": 91, - "test/plan-create-permission.test.ts": 55, - "test/plan-create-prepublication.test.ts": 84, - "test/plan-design-floor-fixture.test.ts": 199, - "test/plan-design-sdk-fixture.test.ts": 470, - "test/plan-design-with-ui-fixture.test.ts": 1604, - "test/plan-edit-cropped-permission.test.ts": 113, + "test/openclaw-native-skills.test.ts": 8, + "test/osv-config-wiring.test.ts": 7, + "test/osv-ignore-expiry.test.ts": 9, + "test/outside-consent.test.ts": 985, + "test/outside-review-result.test.ts": 608, + "test/outside-review-severity.test.ts": 67, + "test/outside-review-unparsed-severity.test.ts": 108, + "test/outside-runner-contract.test.ts": 781, + "test/outside-send-boundary.test.ts": 342, + "test/outside-voice-evidence.test.ts": 67, + "test/outside-voice-failure-policy.test.ts": 119, + "test/outside-voice-fixture.test.ts": 211, + "test/outside-voice-invocation.test.ts": 10044, + "test/outside-voice-native-deadline.test.ts": 56, + "test/outside-voice-preflight.test.ts": 352, + "test/outside-voice-provenance.test.ts": 869, + "test/outside-voice-receipt.test.ts": 362, + "test/outside-voice-routing.test.ts": 1066, + "test/overlay-lifecycle.test.ts": 331, + "test/overlay-measurement.test.ts": 230, + "test/overlay-recording-order.test.ts": 22, + "test/overlay-sdk-cancel-eof.test.ts": 138, + "test/owner-brief-contributor-banned.test.ts": 445, + "test/paid-free-boundary.test.ts": 3912, + "test/paid-orphan-tripwire.test.ts": 72, + "test/paid-overlay-scheduling.test.ts": 557, + "test/paid-pr-profile.test.ts": 3056, + "test/paid-report-census.test.ts": 141, + "test/paid-report-fail-open.test.ts": 1680, + "test/paid-retry-supervision.test.ts": 2379, + "test/paid-run-manifest.test.ts": 2183, + "test/paid-selection-propagation.test.ts": 45, + "test/paid-shard-settlement.test.ts": 1372, + "test/paid-shards.test.ts": 6919, + "test/paid-slice-deadline.test.ts": 1548, + "test/pair-agent-token-hygiene.test.ts": 25, + "test/parity-baseline-integrity.test.ts": 17, + "test/parity-sectioned.test.ts": 13, + "test/parity-suite.test.ts": 100, + "test/path-spelling.test.ts": 308, + "test/pending-question-completion.test.ts": 133, + "test/periodic-exclude-policy.test.ts": 129, + "test/plan-ceo-floor-interactive-boundary.test.ts": 1489, + "test/plan-count-artifacts.test.ts": 31, + "test/plan-count-checkbox.test.ts": 15372, + "test/plan-count-clipped-elision.test.ts": 61, + "test/plan-count-collection-completion.test.ts": 30036, + "test/plan-count-completion.test.ts": 14184, + "test/plan-count-cropped-wrap.test.ts": 64, + "test/plan-count-cross-cwd-ancestry.test.ts": 88, + "test/plan-count-design-ui-recovery.test.ts": 15758, + "test/plan-count-dx-handoff.test.ts": 88, + "test/plan-count-empty-review.test.ts": 1523, + "test/plan-count-file-permission.test.ts": 44425, + "test/plan-count-fixture.test.ts": 15542, + "test/plan-count-history.test.ts": 718, + "test/plan-count-long-edit.test.ts": 49, + "test/plan-count-native-input.test.ts": 11550, + "test/plan-count-owned-permission.test.ts": 11376, + "test/plan-count-pending-exit.test.ts": 248, + "test/plan-count-prerequisite.test.ts": 41, + "test/plan-count-preview-footer.test.ts": 3446, + "test/plan-count-session-cwd.test.ts": 55, + "test/plan-count-timeout.test.ts": 9490, + "test/plan-count-transcript.test.ts": 139, + "test/plan-count-truncated-border.test.ts": 46, + "test/plan-count-truncated-question.test.ts": 29, + "test/plan-create-combined-permission.test.ts": 110, + "test/plan-create-permission.test.ts": 52, + "test/plan-create-prepublication.test.ts": 94, + "test/plan-design-floor-fixture.test.ts": 196, + "test/plan-design-sdk-fixture.test.ts": 393, + "test/plan-design-with-ui-fixture.test.ts": 1411, + "test/plan-edit-cropped-permission.test.ts": 90, "test/plan-eng-resume.test.ts": 13, - "test/plan-floor-dx-actor.test.ts": 74, - "test/plan-floor-edit-wrap.test.ts": 48, - "test/plan-floor-permission.test.ts": 1660, - "test/plan-floor-review.test.ts": 32, - "test/plan-floor-target.test.ts": 12508, - "test/plan-format-approach-prompt.test.ts": 18, - "test/plan-format-kind-note.test.ts": 12, - "test/plan-mode-evidence.test.ts": 798, - "test/plan-pending-question-pty.test.ts": 509, - "test/plan-reality.test.ts": 386, - "test/plan-review-board-feedback.test.ts": 4290, - "test/plan-review-calibration.test.ts": 320, - "test/plan-review-cases.test.ts": 1046, - "test/plan-review-decisions.test.ts": 272, - "test/plan-review-report-recording.test.ts": 5405, - "test/plan-scope-selection.test.ts": 2745, - "test/plan-seed-submission.test.ts": 53872, - "test/plan-skill-question-events.test.ts": 2626, - "test/plan-skill-question-hook-scope.test.ts": 3480, - "test/plan-skill-questions.test.ts": 16397, - "test/plan-skill-read-permission.test.ts": 71, - "test/plan-skill-webfetch-permission.test.ts": 1293, - "test/plan-tune-cathedral.test.ts": 672, - "test/plan-tune-gates.test.ts": 1178, - "test/plan-tune.test.ts": 1324, - "test/platform-qualification-workflow.test.ts": 10, - "test/post-rename-doc-regen.test.ts": 22, - "test/pr-dependencies.test.ts": 2792, - "test/pr-direct-cases.test.ts": 2038, - "test/pr-shared-input-selection.test.ts": 772, - "test/pr-title-rewrite.test.ts": 116, - "test/pr-title-sync-workflow-safety.test.ts": 11, - "test/preamble-compose.test.ts": 16, - "test/preamble-first-task-scaffold.test.ts": 1822, - "test/pregate-docs-lane-review.test.ts": 2119, - "test/pregate-guards.test.ts": 6930, - "test/pregate-import-graph-paths.test.ts": 2060, - "test/pregate-invalidation.test.ts": 4593, - "test/pregate-manifest-paths.test.ts": 650, - "test/pregate-regate.test.ts": 9394, - "test/pregate-remote-gbrain.test.ts": 852, - "test/pregate-remote.test.ts": 3561, - "test/pregate-secrets-source-context.test.ts": 904, - "test/pregate-selection.test.ts": 8149, - "test/pregate-supervise.test.ts": 29119, - "test/pregate-timing.test.ts": 1574, - "test/pricing-eval-plan.test.ts": 1135, - "test/prompt-audit-manifest.test.ts": 259, - "test/prosons-neutral-posture.test.ts": 11, - "test/provider-model-defaults.test.ts": 246, - "test/pty-askuserquestion-single-line.test.ts": 31, - "test/pty-numbered-option-indent-native.test.ts": 42, - "test/pty-option-selection.test.ts": 594, - "test/pty-output-wake.test.ts": 10763, - "test/pty-screen-session.test.ts": 8855, - "test/pty-screen-supervision.test.ts": 10434, - "test/pty-screen-unicode-ap.test.ts": 115, - "test/pty-screen.test.ts": 141, - "test/pty-skill-seeding-wiring.test.ts": 463, - "test/pty-trust-dialog.test.ts": 3060, - "test/pty-upgrade-isolation.test.ts": 859, - "test/pty-workspace-trust.test.ts": 363, - "test/published-text.test.ts": 21, - "test/qa-browser-deadline-evidence.test.ts": 4066, - "test/qa-browser-preservation.test.ts": 1431, - "test/qa-bugs-fixture.test.ts": 17, - "test/qa-caller-authority.test.ts": 82, - "test/qa-caller-freshness-order.test.ts": 63, - "test/qa-caller-report-observer.test.ts": 159, - "test/qa-checkpoint-evidence.test.ts": 100, - "test/qa-deadline-publication-observer.test.ts": 258, - "test/qa-deadline-selection.test.ts": 29, - "test/qa-deadline.test.ts": 19447, - "test/qa-evidence-producer.test.ts": 9186, - "test/qa-evidence-selection.test.ts": 60, - "test/qa-evidence.test.ts": 10282, - "test/qa-exploratory-callers.test.ts": 13744, - "test/qa-fix-loop-fixture.test.ts": 2414, - "test/qa-functional-evidence.test.ts": 4831, - "test/qa-functional-fixture.test.ts": 882, - "test/qa-functional-observer-atomic.test.ts": 454, - "test/qa-functional-observer.test.ts": 1127, - "test/qa-functional-prompt.test.ts": 944, - "test/qa-health-rubric.test.ts": 13, - "test/qa-lazy-sections.test.ts": 9027, - "test/qa-only-browser-probe.test.ts": 751, - "test/qa-only-capability.test.ts": 1478, - "test/qa-only-cleanup.test.ts": 7773, - "test/qa-only-fixture.test.ts": 10410, - "test/qa-probe-gates.test.ts": 48, - "test/qa-report-fields.test.ts": 24, - "test/qa-supervision-selection.test.ts": 29, - "test/question-log-hook.test.ts": 1454, - "test/question-preference-hook.test.ts": 2748, - "test/question-tuning-registry-path.test.ts": 14, - "test/readme-skill-lists.test.ts": 18, - "test/recover-receipts.test.ts": 695, - "test/redact-audit-log.test.ts": 33, - "test/redact-context-exemptions.test.ts": 30, - "test/redact-doc-resolver.test.ts": 23, - "test/redact-dotenv-filename-false-positive.test.ts": 15, - "test/redact-engine-autoredact.test.ts": 14, - "test/redact-engine.test.ts": 82, - "test/redact-parcel-id-false-positive.test.ts": 15, - "test/redact-pattern-lint.test.ts": 19, - "test/redact-prepush-emails.test.ts": 14736, - "test/redact-prepush-hook.test.ts": 3766, - "test/redact-prepush-rebase-force-push.test.ts": 1065, - "test/redact-prepush-scan-range.test.ts": 1884, - "test/redact-prepush-target.test.ts": 8369, - "test/redact-span-binding.test.ts": 52, - "test/redact-unicode-offsets.test.ts": 65, - "test/regression-1539-review-self-verify.test.ts": 14, - "test/regression-1611-gbrain-sync-resume.test.ts": 34, - "test/regression-1624-retro-stale-base.test.ts": 12, - "test/regression-1834-retro-recommendations.test.ts": 10, - "test/regression-2922-memory-ingest-sources.test.ts": 530, - "test/regression-dot-leading-slug.test.ts": 111, - "test/regression-issue2091-bsd-mktemp.test.ts": 166, - "test/regression-issue3024-zsh-sourced-helpers.test.ts": 1143, - "test/regression-pr1169-build-app-sed.test.ts": 56, - "test/regression-pr1169-mktemp-fallbacks.test.ts": 36, - "test/regression-transcript-frontmatter-fence.test.ts": 24, + "test/plan-floor-dx-actor.test.ts": 94, + "test/plan-floor-edit-wrap.test.ts": 46, + "test/plan-floor-permission.test.ts": 1415, + "test/plan-floor-review.test.ts": 49, + "test/plan-floor-target.test.ts": 12547, + "test/plan-format-approach-prompt.test.ts": 11, + "test/plan-format-kind-note.test.ts": 14, + "test/plan-mode-evidence.test.ts": 992, + "test/plan-pending-question-pty.test.ts": 455, + "test/plan-reality.test.ts": 573, + "test/plan-review-board-feedback.test.ts": 4413, + "test/plan-review-calibration.test.ts": 476, + "test/plan-review-cases.test.ts": 1120, + "test/plan-review-decisions.test.ts": 305, + "test/plan-review-report-recording.test.ts": 5473, + "test/plan-scope-selection.test.ts": 2767, + "test/plan-seed-submission.test.ts": 54200, + "test/plan-skill-question-events.test.ts": 2124, + "test/plan-skill-question-hook-scope.test.ts": 2601, + "test/plan-skill-questions.test.ts": 17186, + "test/plan-skill-read-permission.test.ts": 76, + "test/plan-skill-webfetch-permission.test.ts": 1111, + "test/plan-tune-cathedral.test.ts": 577, + "test/plan-tune-gates.test.ts": 1150, + "test/plan-tune.test.ts": 1276, + "test/platform-qualification-workflow.test.ts": 8, + "test/post-rename-doc-regen.test.ts": 20, + "test/pr-dependencies.test.ts": 3629, + "test/pr-direct-cases.test.ts": 3220, + "test/pr-shared-input-selection.test.ts": 1818, + "test/pr-title-rewrite.test.ts": 181, + "test/pr-title-sync-workflow-safety.test.ts": 16, + "test/preamble-compose.test.ts": 15, + "test/preamble-first-task-scaffold.test.ts": 869, + "test/pregate-bats-nested.test.ts": 1307, + "test/pregate-docs-lane-review.test.ts": 1103, + "test/pregate-guards.test.ts": 6109, + "test/pregate-import-graph-paths.test.ts": 1012, + "test/pregate-invalidation.test.ts": 3781, + "test/pregate-manifest-paths.test.ts": 410, + "test/pregate-regate.test.ts": 9722, + "test/pregate-remote-gbrain.test.ts": 701, + "test/pregate-remote.test.ts": 3677, + "test/pregate-secrets-source-context.test.ts": 923, + "test/pregate-selection.test.ts": 9239, + "test/pregate-supervise.test.ts": 29677, + "test/pregate-timing.test.ts": 1929, + "test/pregate-wtree-refusal.test.ts": 982, + "test/pricing-eval-plan.test.ts": 1443, + "test/prompt-audit-manifest.test.ts": 329, + "test/prosons-neutral-posture.test.ts": 15, + "test/provider-model-defaults.test.ts": 406, + "test/pty-askuserquestion-single-line.test.ts": 34, + "test/pty-numbered-option-indent-native.test.ts": 55, + "test/pty-option-selection.test.ts": 644, + "test/pty-output-wake.test.ts": 10958, + "test/pty-screen-session.test.ts": 9363, + "test/pty-screen-supervision.test.ts": 10484, + "test/pty-screen-unicode-ap.test.ts": 131, + "test/pty-screen.test.ts": 110, + "test/pty-skill-seeding-wiring.test.ts": 479, + "test/pty-trust-dialog.test.ts": 3061, + "test/pty-upgrade-isolation.test.ts": 637, + "test/pty-workspace-trust.test.ts": 371, + "test/published-text.test.ts": 19, + "test/qa-browser-deadline-evidence.test.ts": 3691, + "test/qa-browser-preservation.test.ts": 992, + "test/qa-bugs-fixture.test.ts": 19, + "test/qa-caller-authority.test.ts": 88, + "test/qa-caller-freshness-order.test.ts": 65, + "test/qa-caller-report-observer.test.ts": 142, + "test/qa-checkpoint-evidence.test.ts": 68, + "test/qa-deadline-publication-observer.test.ts": 158, + "test/qa-deadline-selection.test.ts": 24, + "test/qa-deadline.test.ts": 19552, + "test/qa-evidence-producer.test.ts": 8179, + "test/qa-evidence-selection.test.ts": 64, + "test/qa-evidence.test.ts": 10945, + "test/qa-exploratory-callers.test.ts": 13065, + "test/qa-fix-loop-fixture.test.ts": 2088, + "test/qa-functional-evidence.test.ts": 6146, + "test/qa-functional-fixture.test.ts": 873, + "test/qa-functional-observer-atomic.test.ts": 226, + "test/qa-functional-observer.test.ts": 1108, + "test/qa-functional-prompt.test.ts": 1918, + "test/qa-health-rubric.test.ts": 30, + "test/qa-lazy-sections.test.ts": 8813, + "test/qa-only-browser-probe.test.ts": 704, + "test/qa-only-capability.test.ts": 2014, + "test/qa-only-cleanup.test.ts": 7703, + "test/qa-only-fixture.test.ts": 12325, + "test/qa-probe-gates.test.ts": 83, + "test/qa-report-fields.test.ts": 10, + "test/qa-supervision-selection.test.ts": 27, + "test/question-log-hook.test.ts": 1615, + "test/question-preference-hook.test.ts": 2366, + "test/question-tuning-registry-path.test.ts": 11, + "test/read-only-command-grammar.test.ts": 454, + "test/readme-skill-lists.test.ts": 16, + "test/recover-receipts.test.ts": 902, + "test/redact-audit-log.test.ts": 60, + "test/redact-context-exemptions.test.ts": 35, + "test/redact-doc-resolver.test.ts": 17, + "test/redact-dotenv-filename-false-positive.test.ts": 18, + "test/redact-engine-autoredact.test.ts": 42, + "test/redact-engine.test.ts": 113, + "test/redact-parcel-id-false-positive.test.ts": 18, + "test/redact-pattern-lint.test.ts": 24, + "test/redact-prepush-emails.test.ts": 15127, + "test/redact-prepush-hook.test.ts": 4128, + "test/redact-prepush-rebase-force-push.test.ts": 1078, + "test/redact-prepush-scan-range.test.ts": 1821, + "test/redact-prepush-target.test.ts": 10881, + "test/redact-span-binding.test.ts": 29, + "test/redact-unicode-offsets.test.ts": 82, + "test/regression-1539-review-self-verify.test.ts": 12, + "test/regression-1611-gbrain-sync-resume.test.ts": 46, + "test/regression-1624-retro-stale-base.test.ts": 25, + "test/regression-1834-retro-recommendations.test.ts": 12, + "test/regression-2922-memory-ingest-sources.test.ts": 515, + "test/regression-dot-leading-slug.test.ts": 72, + "test/regression-issue2091-bsd-mktemp.test.ts": 127, + "test/regression-issue3024-zsh-sourced-helpers.test.ts": 1120, + "test/regression-pr1169-build-app-sed.test.ts": 66, + "test/regression-pr1169-mktemp-fallbacks.test.ts": 34, + "test/regression-transcript-frontmatter-fence.test.ts": 28, "test/regression-transcript-slug-collision.test.ts": 24, - "test/relink-disabled-router.test.ts": 846, - "test/relink.test.ts": 9618, - "test/remote-identity-vectors.test.ts": 572, - "test/render-sections-provenance.test.ts": 278, - "test/replay-pr-selection.test.ts": 754, - "test/resolver-ask-user-format.test.ts": 25, - "test/resolvers-gbrain-put-rewrite.test.ts": 24, - "test/resolvers-gbrain-save-results.test.ts": 17, - "test/restamp-command-boundary.test.ts": 3674, - "test/restamp-gate-ahead-changelog.test.ts": 2122, - "test/retired-commands.test.ts": 22, - "test/review-army-budget.test.ts": 1879, - "test/review-base-refresh.test.ts": 116, - "test/review-checklists.test.ts": 22, - "test/review-consensus-lifecycle.test.ts": 30, - "test/review-count-markdown.test.ts": 95, - "test/review-entry-and-design-clarity-au.test.ts": 50, - "test/review-enum-lifecycle.test.ts": 27, - "test/review-finalization-budget.test.ts": 2506, - "test/review-log-findings.test.ts": 4910, - "test/review-log.test.ts": 2689, - "test/review-n-plus-one-contract.test.ts": 605, - "test/review-quality-provenance.test.ts": 38, - "test/review-start-evidence.test.ts": 21394, - "test/review-workflow-clarity.test.ts": 37, - "test/review-workflow-fixture.test.ts": 23, - "test/router-disabled-skills.test.ts": 564, - "test/routing-probe.test.ts": 22, - "test/run-in-background-guidance.test.ts": 1797, - "test/run-shard-child.test.ts": 1232, - "test/runtime-pins.test.ts": 61, - "test/runtime-root-assets.test.ts": 1538, - "test/safety-rule-registry.test.ts": 30, - "test/salience-allowlist.test.ts": 25, - "test/sandbox-doctor-shell.test.ts": 16, - "test/sanitize-echo.test.ts": 319, - "test/schema-version-migration.test.ts": 59, - "test/script-invocation-tripwire.test.ts": 1271, - "test/script-invocation.test.ts": 232, - "test/sealed-guard.test.ts": 2152, - "test/secret-sink-harness.test.ts": 47, - "test/section-capture-native-tools.test.ts": 3810, - "test/section-manifest-consistency.test.ts": 18, - "test/security-dashboard-fallback.test.ts": 1320, - "test/session-runner-browse-errors.test.ts": 778, - "test/session-runner-groupkill.test.ts": 4090, - "test/session-runner-startup-grace.test.ts": 13087, - "test/session-runner-stream-lifecycle.test.ts": 262, - "test/session-runner-timeout.test.ts": 3027, - "test/session-runner-tools.test.ts": 5121, - "test/session-update-autostash.test.ts": 11251, - "test/session-update-heartbeat.test.ts": 257, - "test/session-update-held-banner.test.ts": 1639, - "test/session-update-hook-hold.test.ts": 5260, - "test/session-update-stages.test.ts": 5384, - "test/setup-alias-name-uniqueness.test.ts": 1311, - "test/setup-browser-hint.test.ts": 43, + "test/relink-disabled-router.test.ts": 1219, + "test/relink.test.ts": 11775, + "test/remote-identity-vectors.test.ts": 817, + "test/render-sections-provenance.test.ts": 308, + "test/replay-pr-selection.test.ts": 1871, + "test/resolver-ask-user-format.test.ts": 43, + "test/resolvers-gbrain-put-rewrite.test.ts": 19, + "test/resolvers-gbrain-save-results.test.ts": 23, + "test/restamp-command-boundary.test.ts": 5076, + "test/restamp-gate-ahead-changelog.test.ts": 3035, + "test/retired-commands.test.ts": 18, + "test/review-army-budget.test.ts": 2175, + "test/review-base-refresh.test.ts": 212, + "test/review-checklists.test.ts": 26, + "test/review-consensus-lifecycle.test.ts": 32, + "test/review-count-markdown.test.ts": 133, + "test/review-entry-and-design-clarity-au.test.ts": 54, + "test/review-enum-lifecycle.test.ts": 23, + "test/review-finalization-budget.test.ts": 2709, + "test/review-log-findings.test.ts": 5531, + "test/review-log.test.ts": 2612, + "test/review-n-plus-one-contract.test.ts": 733, + "test/review-quality-provenance.test.ts": 36, + "test/review-start-evidence.test.ts": 22384, + "test/review-workflow-clarity.test.ts": 204, + "test/review-workflow-fixture.test.ts": 27, + "test/router-disabled-skills.test.ts": 511, + "test/routing-probe.test.ts": 36, + "test/run-in-background-guidance.test.ts": 2144, + "test/run-shard-child.test.ts": 1243, + "test/runtime-pins.test.ts": 60, + "test/runtime-root-assets.test.ts": 1631, + "test/safety-rule-registry.test.ts": 44, + "test/salience-allowlist.test.ts": 23, + "test/sandbox-doctor-shell.test.ts": 12, + "test/sanitize-echo.test.ts": 378, + "test/schema-version-migration.test.ts": 74, + "test/script-invocation-tripwire.test.ts": 1339, + "test/script-invocation.test.ts": 224, + "test/sealed-guard.test.ts": 2663, + "test/secret-sink-harness.test.ts": 55, + "test/section-capture-native-tools.test.ts": 4167, + "test/section-manifest-consistency.test.ts": 28, + "test/security-dashboard-fallback.test.ts": 1700, + "test/session-runner-browse-errors.test.ts": 1991, + "test/session-runner-groupkill.test.ts": 4073, + "test/session-runner-startup-grace.test.ts": 13088, + "test/session-runner-stream-lifecycle.test.ts": 303, + "test/session-runner-timeout.test.ts": 3031, + "test/session-runner-tools.test.ts": 5188, + "test/session-update-autostash.test.ts": 11360, + "test/session-update-heartbeat.test.ts": 208, + "test/session-update-held-banner.test.ts": 1462, + "test/session-update-hook-hold.test.ts": 5312, + "test/session-update-stages.test.ts": 5546, + "test/setup-alias-name-uniqueness.test.ts": 1347, + "test/setup-browser-hint.test.ts": 39, "test/setup-bun-cmd-and-pipe-bugs.test.ts": 26, - "test/setup-bun-floor.test.ts": 182, - "test/setup-claude-code-migration.test.ts": 15950, - "test/setup-claude-skill-assets.test.ts": 954, - "test/setup-cleanup-orphans.test.ts": 168, - "test/setup-codesign-restore.test.ts": 290, - "test/setup-codesign.test.ts": 19, - "test/setup-codex-design-pdf-runtime.test.ts": 112, - "test/setup-codex-model.test.ts": 12, - "test/setup-codex-scope-migrations.test.ts": 32725, - "test/setup-codex-scope-namespaces.test.ts": 54477, - "test/setup-codex-scope-review-boundaries.test.ts": 49800, - "test/setup-codex-scope.test.ts": 58903, - "test/setup-conductor-auq-hook.test.ts": 21509, - "test/setup-conductor-worktree.test.ts": 29, - "test/setup-crlf-repair.test.ts": 241, - "test/setup-cso-degrade.test.ts": 169, - "test/setup-cso-windows-probe.test.ts": 44, - "test/setup-cursor-legacy-skills.test.ts": 4340, - "test/setup-deploy-key-presence.test.ts": 15, - "test/setup-deploy-not-deployed.test.ts": 10, - "test/setup-deploy-vercel-trigger.test.ts": 10, - "test/setup-emoji-font.test.ts": 65, - "test/setup-gbrain-bin-invocation-paths.test.ts": 14, - "test/setup-gbrain-fixture.test.ts": 2386, - "test/setup-gbrain-path4-caller.test.ts": 2564, - "test/setup-gbrain-path4-structure.test.ts": 12, - "test/setup-gbrain-remote-caller.test.ts": 8895, - "test/setup-gbrain-windows-mcp.test.ts": 155, - "test/setup-help.test.ts": 58, - "test/setup-hook-canonical-paths.test.ts": 39, - "test/setup-hook-syntax-gate.test.ts": 8335, - "test/setup-install-registry.test.ts": 86484, - "test/setup-kiro-native.test.ts": 575, - "test/setup-link-ownership.test.ts": 718, - "test/setup-needs-build.test.ts": 254, - "test/setup-plan-tune-hooks-noninteractive.test.ts": 439, - "test/setup-playwright-best-effort.test.ts": 19882, - "test/setup-playwright-platform.test.ts": 5076, - "test/setup-prune-stale-generated.test.ts": 210, - "test/setup-ps1.test.ts": 15, - "test/setup-runtime-lib-command.test.ts": 13833, - "test/setup-runtime-root-guard.test.ts": 16604, - "test/setup-sections-linking.test.ts": 14, - "test/setup-skill-copies.test.ts": 410, - "test/setup-staged-copy.test.ts": 55, - "test/setup-timeline-hook-gate.test.ts": 355, - "test/setup-windows-fallback.test.ts": 42, - "test/setup-windows-launch-hint.test.ts": 155, - "test/setup-windows-rerun-refresh.test.ts": 142, - "test/shard-cli-parity.test.ts": 87, - "test/shard-engine-equivalence.test.ts": 3137, - "test/shard-engine.test.ts": 3709, - "test/shared-libs-cancellation.test.ts": 113, - "test/shared-libs-checker-interface-evidence.test.ts": 55560, - "test/shared-libs-evidence.test.ts": 24, - "test/shared-libs-fixture.test.ts": 8339, - "test/shared-libs-plan-actor.test.ts": 61, - "test/shared-libs-rendering.test.ts": 1139, - "test/shared-libs-revalidation-prompt.test.ts": 13480, - "test/shared-libs-review-start-evidence.test.ts": 3820, - "test/shared-libs-snapshot-check.test.ts": 32403, - "test/shared-libs-source-reads.test.ts": 1347, - "test/shared-libs-stage-actor.test.ts": 127424, - "test/ship-apple-gate.test.ts": 16, - "test/ship-control-flow.test.ts": 299, - "test/ship-coverage-audit-af.test.ts": 59, - "test/ship-document-release-dispatch.test.ts": 4781, - "test/ship-greptile-early.test.ts": 13, - "test/ship-greptile-pr-check.test.ts": 81, - "test/ship-hook-actor.test.ts": 2041, - "test/ship-hook-refresh.test.ts": 2588, - "test/ship-hook-windows-paths.test.ts": 386, - "test/ship-issue-intent.test.ts": 3730, - "test/ship-measure-bar.test.ts": 47, - "test/ship-measure-loop-skill.test.ts": 13, - "test/ship-measure-seeded-fixture.test.ts": 1314, - "test/ship-measure-sweep.test.ts": 916, - "test/ship-measure.test.ts": 2175, - "test/ship-plan-completion-invariants.test.ts": 1161, - "test/ship-policy-compat.test.ts": 462, - "test/ship-pr-liveness-policy.test.ts": 14, - "test/ship-publication-gates.test.ts": 550, - "test/ship-reentry-gates.test.ts": 23, - "test/ship-review-loop.test.ts": 22, - "test/ship-section-fixture.test.ts": 360, - "test/ship-skip-actor.test.ts": 32083, - "test/ship-skip-requeue.test.ts": 31, - "test/ship-skip-selection.test.ts": 897, - "test/ship-template-redaction.test.ts": 24, - "test/ship-test-detection-markers.test.ts": 153, - "test/ship-triage-labels.test.ts": 17, - "test/ship-version-sync.test.ts": 546, - "test/ship-workflow-clarity.test.ts": 49, - "test/shortcut-debt-ledger.test.ts": 19, + "test/setup-bun-floor.test.ts": 198, + "test/setup-claude-code-migration.test.ts": 15393, + "test/setup-claude-skill-assets.test.ts": 1217, + "test/setup-cleanup-orphans.test.ts": 214, + "test/setup-codesign-restore.test.ts": 389, + "test/setup-codesign.test.ts": 22, + "test/setup-codex-design-pdf-runtime.test.ts": 121, + "test/setup-codex-model.test.ts": 50, + "test/setup-codex-scope-migrations.test.ts": 32108, + "test/setup-codex-scope-namespaces.test.ts": 59048, + "test/setup-codex-scope-review-boundaries.test.ts": 52917, + "test/setup-codex-scope.test.ts": 64112, + "test/setup-conductor-auq-hook.test.ts": 22140, + "test/setup-conductor-worktree.test.ts": 33, + "test/setup-crlf-repair.test.ts": 367, + "test/setup-cso-degrade.test.ts": 246, + "test/setup-cso-windows-probe.test.ts": 45, + "test/setup-cursor-legacy-skills.test.ts": 4552, + "test/setup-deploy-key-presence.test.ts": 21, + "test/setup-deploy-not-deployed.test.ts": 9, + "test/setup-deploy-vercel-trigger.test.ts": 14, + "test/setup-emoji-font.test.ts": 57, + "test/setup-gbrain-bin-invocation-paths.test.ts": 12, + "test/setup-gbrain-fixture.test.ts": 2784, + "test/setup-gbrain-path4-caller.test.ts": 2931, + "test/setup-gbrain-path4-structure.test.ts": 11, + "test/setup-gbrain-remote-caller.test.ts": 9141, + "test/setup-gbrain-windows-mcp.test.ts": 84, + "test/setup-help.test.ts": 50, + "test/setup-hook-canonical-paths.test.ts": 67, + "test/setup-hook-syntax-gate.test.ts": 9913, + "test/setup-install-registry.test.ts": 84467, + "test/setup-kiro-native.test.ts": 789, + "test/setup-link-ownership.test.ts": 1061, + "test/setup-needs-build.test.ts": 259, + "test/setup-plan-tune-hooks-noninteractive.test.ts": 520, + "test/setup-playwright-best-effort.test.ts": 20960, + "test/setup-playwright-platform.test.ts": 5339, + "test/setup-prune-stale-generated.test.ts": 162, + "test/setup-ps1.test.ts": 37512, + "test/setup-runtime-lib-command.test.ts": 12800, + "test/setup-runtime-root-guard.test.ts": 16309, + "test/setup-sections-linking.test.ts": 15, + "test/setup-skill-copies.test.ts": 350, + "test/setup-staged-copy.test.ts": 52, + "test/setup-timeline-hook-gate.test.ts": 308, + "test/setup-windows-fallback.test.ts": 56, + "test/setup-windows-launch-hint.test.ts": 245, + "test/setup-windows-rerun-refresh.test.ts": 167, + "test/shard-cli-parity.test.ts": 119, + "test/shard-engine-equivalence.test.ts": 3227, + "test/shard-engine.test.ts": 3783, + "test/shared-libs-cancellation.test.ts": 292, + "test/shared-libs-checker-interface-evidence.test.ts": 55934, + "test/shared-libs-evidence.test.ts": 40, + "test/shared-libs-fixture.test.ts": 9847, + "test/shared-libs-plan-actor.test.ts": 67, + "test/shared-libs-rendering.test.ts": 1590, + "test/shared-libs-revalidation-prompt.test.ts": 13573, + "test/shared-libs-review-start-evidence.test.ts": 3344, + "test/shared-libs-snapshot-check.test.ts": 34909, + "test/shared-libs-source-reads.test.ts": 1468, + "test/shared-libs-stage-actor.test.ts": 83451, + "test/ship-apple-gate.test.ts": 31, + "test/ship-control-flow.test.ts": 314, + "test/ship-coverage-audit-af.test.ts": 76, + "test/ship-document-release-dispatch.test.ts": 6173, + "test/ship-greptile-early.test.ts": 55, + "test/ship-greptile-pr-check.test.ts": 111, + "test/ship-hook-actor.test.ts": 2196, + "test/ship-hook-refresh.test.ts": 3122, + "test/ship-hook-windows-paths.test.ts": 445, + "test/ship-issue-intent.test.ts": 4171, + "test/ship-measure-bar.test.ts": 147, + "test/ship-measure-loop-skill.test.ts": 11, + "test/ship-measure-seeded-fixture.test.ts": 1617, + "test/ship-measure-sweep.test.ts": 1205, + "test/ship-measure.test.ts": 2585, + "test/ship-plan-completion-invariants.test.ts": 1371, + "test/ship-policy-compat.test.ts": 551, + "test/ship-pr-liveness-policy.test.ts": 15, + "test/ship-publication-gates.test.ts": 809, + "test/ship-reentry-gates.test.ts": 30, + "test/ship-review-loop.test.ts": 16, + "test/ship-section-fixture.test.ts": 547, + "test/ship-skip-actor.test.ts": 34256, + "test/ship-skip-requeue.test.ts": 75, + "test/ship-skip-selection.test.ts": 2052, + "test/ship-template-redaction.test.ts": 16, + "test/ship-test-detection-markers.test.ts": 169, + "test/ship-triage-labels.test.ts": 16, + "test/ship-version-sync.test.ts": 863, + "test/ship-workflow-clarity.test.ts": 35, + "test/shortcut-debt-ledger.test.ts": 14, "test/sideways-review.test.ts": 13, - "test/skill-browse-state-extraction.test.ts": 22, - "test/skill-budget-regression.test.ts": 41, - "test/skill-census.test.ts": 24, - "test/skill-ceo-section-ordering.test.ts": 2222, - "test/skill-check-driver.test.ts": 98, - "test/skill-collision-sentinel.test.ts": 16, - "test/skill-coverage-floor.test.ts": 51, - "test/skill-cross-model-recommendation-emit.test.ts": 24, - "test/skill-end-completion-durability.test.ts": 8482, - "test/skill-fixture.test.ts": 52, + "test/skill-browse-state-extraction.test.ts": 43, + "test/skill-budget-regression.test.ts": 43, + "test/skill-census.test.ts": 43, + "test/skill-ceo-section-ordering.test.ts": 2342, + "test/skill-check-driver.test.ts": 199, + "test/skill-collision-sentinel.test.ts": 18, + "test/skill-coverage-floor.test.ts": 80, + "test/skill-cross-model-recommendation-emit.test.ts": 21, + "test/skill-end-completion-durability.test.ts": 10111, + "test/skill-fixture.test.ts": 69, + "test/skill-md-size-guard.test.ts": 1474, "test/skill-parser.test.ts": 26, - "test/skill-positional-literals.test.ts": 1081, - "test/skill-preflight-budget.test.ts": 16, - "test/skill-size-budget.test.ts": 463, - "test/skill-validation.test.ts": 1085, - "test/skillify-host-paths.test.ts": 1103, - "test/slop-diff-cli.test.ts": 340, - "test/sol-skill-fixture.test.ts": 822, - "test/spawned-consent-rule.test.ts": 25, - "test/spawnsync-timeout-tripwire.test.ts": 176, - "test/spec-quality-gate-secret-sink.test.ts": 1571, - "test/spec-template-invariants.test.ts": 19, - "test/spend-ledger.test.ts": 127, - "test/stamp-templates.test.ts": 439, - "test/state-matrix.test.ts": 975, - "test/state-root-config-merge.test.ts": 1117, - "test/state-root-one-variable.test.ts": 4741, - "test/state-root-parity.test.ts": 172, - "test/state-root-ratchet.test.ts": 325, - "test/state-root-resolver-failure.test.ts": 31, - "test/static-no-legacy-writes.test.ts": 1097, - "test/strict-output-capture.test.ts": 85, - "test/strict-output-formats.test.ts": 781, - "test/strict-output-settlement.test.ts": 4273, - "test/strict-output.test.ts": 16, - "test/sync-gbrain-code-scope.test.ts": 108, - "test/sync-gbrain-cycle-freshness.test.ts": 83, - "test/sync-gbrain-readiness-fixture.test.ts": 144, - "test/sync-gbrain-source-probe.test.ts": 496, - "test/takes-fence-fallback.test.ts": 14, - "test/tasks-section-jq.test.ts": 26, - "test/taste-engine.test.ts": 775, - "test/team-init-hook-shell-neutral.test.ts": 108, - "test/team-mode.test.ts": 2001, - "test/telemetry-repo-strip.test.ts": 29, - "test/telemetry.test.ts": 7181, - "test/template-context-parity.test.ts": 16, - "test/terse-build.test.ts": 22, - "test/test-free-shards-capture.test.ts": 1242, - "test/test-free-shards-git-maintenance.test.ts": 125, - "test/test-free-shards-sandbox-knobs.test.ts": 97, - "test/test-free-shards-settlement-lifecycle.test.ts": 46203, - "test/test-free-shards-settlement-ownership.test.ts": 37739, - "test/test-free-shards.test.ts": 9725, - "test/test-health-report.test.ts": 249, - "test/test-health-workflow.test.ts": 11, - "test/test-of-test-ratchet.test.ts": 181, - "test/test-pr-profile.test.ts": 43, - "test/test-value-bar.test.ts": 93, - "test/testing-internals-anchors.test.ts": 711, - "test/third-party-actions-recording.test.ts": 108, - "test/third-party-actions.test.ts": 254, - "test/timeline-stop-hook-slug.test.ts": 4382, - "test/timeline-stop-hook.test.ts": 536, - "test/timeline.test.ts": 11030, - "test/touchfiles-facade.test.ts": 20, - "test/touchfiles-map-diff.test.ts": 152, - "test/touchfiles.test.ts": 1064, - "test/tracker-guard-wiring.test.ts": 46, - "test/tracker-guard.test.ts": 198, - "test/transcript-consent.test.ts": 967, - "test/transcript-gate-values.test.ts": 1478, - "test/transcript-scope-brain-sync.test.ts": 4537, - "test/transcript-scope-ingest.test.ts": 886, - "test/tree-receipt-gbrain-pair.test.ts": 809, - "test/tree-receipt-stamp-sensitive.test.ts": 9714, - "test/trial-cause.test.ts": 985, - "test/troubleshooting-anchors.test.ts": 1306, - "test/troubleshooting-windows-recovery.test.ts": 14, + "test/skill-positional-literals.test.ts": 1004, + "test/skill-preflight-budget.test.ts": 11, + "test/skill-size-budget.test.ts": 520, + "test/skill-validation.test.ts": 1049, + "test/skillify-host-paths.test.ts": 1232, + "test/slop-diff-cli.test.ts": 318, + "test/sol-skill-fixture.test.ts": 884, + "test/spawned-consent-rule.test.ts": 29, + "test/spawnsync-timeout-tripwire.test.ts": 175, + "test/spec-quality-gate-secret-sink.test.ts": 1578, + "test/spec-template-invariants.test.ts": 18, + "test/spend-ledger.test.ts": 98, + "test/stamp-templates.test.ts": 405, + "test/state-matrix.test.ts": 1065, + "test/state-root-config-merge.test.ts": 1020, + "test/state-root-one-variable.test.ts": 5011, + "test/state-root-parity.test.ts": 134, + "test/state-root-ratchet.test.ts": 337, + "test/state-root-resolver-failure.test.ts": 28, + "test/static-no-legacy-writes.test.ts": 971, + "test/strict-output-capture.test.ts": 83, + "test/strict-output-formats.test.ts": 804, + "test/strict-output-settlement.test.ts": 4301, + "test/strict-output.test.ts": 13, + "test/sync-gbrain-code-scope.test.ts": 143, + "test/sync-gbrain-cycle-freshness.test.ts": 73, + "test/sync-gbrain-readiness-fixture.test.ts": 122, + "test/sync-gbrain-source-probe.test.ts": 649, + "test/takes-fence-fallback.test.ts": 41, + "test/tasks-section-jq.test.ts": 27, + "test/taste-engine.test.ts": 830, + "test/team-init-hook-shell-neutral.test.ts": 935, + "test/team-mode.test.ts": 1926, + "test/telemetry-repo-strip.test.ts": 23, + "test/telemetry.test.ts": 7424, + "test/template-context-parity.test.ts": 22, + "test/terse-build.test.ts": 28, + "test/test-free-shards-capture.test.ts": 1513, + "test/test-free-shards-files-from.test.ts": 254, + "test/test-free-shards-git-maintenance.test.ts": 277, + "test/test-free-shards-sandbox-knobs.test.ts": 136, + "test/test-free-shards-settlement-lifecycle.test.ts": 46142, + "test/test-free-shards-settlement-ownership.test.ts": 37520, + "test/test-free-shards.test.ts": 9740, + "test/test-health-report.test.ts": 252, + "test/test-health-workflow.test.ts": 14, + "test/test-of-test-ratchet.test.ts": 185, + "test/test-pr-profile.test.ts": 41, + "test/test-value-bar.test.ts": 71, + "test/testing-internals-anchors.test.ts": 671, + "test/third-party-actions-recording.test.ts": 83, + "test/third-party-actions.test.ts": 275, + "test/timeline-stop-hook-slug.test.ts": 4368, + "test/timeline-stop-hook.test.ts": 482, + "test/timeline.test.ts": 9809, + "test/touchfiles-facade.test.ts": 23, + "test/touchfiles-map-diff.test.ts": 207, + "test/touchfiles.test.ts": 1323, + "test/tracker-guard-wiring.test.ts": 77, + "test/tracker-guard.test.ts": 161, + "test/transcript-consent.test.ts": 865, + "test/transcript-gate-values.test.ts": 1386, + "test/transcript-scope-brain-sync.test.ts": 5566, + "test/transcript-scope-ingest.test.ts": 855, + "test/tree-receipt-gbrain-pair.test.ts": 673, + "test/tree-receipt-stamp-sensitive.test.ts": 8817, + "test/trial-cause.test.ts": 978, + "test/troubleshooting-anchors.test.ts": 1475, + "test/troubleshooting-windows-recovery.test.ts": 17, "test/typecheck-test-ratchet.test.ts": 36, - "test/ubicloud-runner.test.ts": 2533, - "test/uninstall-windows-copies.test.ts": 1740, - "test/uninstall.test.ts": 5543, - "test/update-check-crash-sentinel.test.ts": 610, - "test/update-check-main-moved.test.ts": 3617, - "test/upgrade-migration-v1.test.ts": 38, - "test/upgrade-setup-recovery.test.ts": 444, - "test/upgrade-template-pins.test.ts": 11, - "test/user-render-out-dir-install.test.ts": 128, - "test/user-slug-fallback.test.ts": 644, - "test/v0-dormancy.test.ts": 31, - "test/verify-gate.test.ts": 1994, + "test/ubicloud-pregate-remote.test.ts": 32047, + "test/ubicloud-runner.test.ts": 2498, + "test/uninstall-windows-copies.test.ts": 1562, + "test/uninstall.test.ts": 5274, + "test/update-check-crash-sentinel.test.ts": 644, + "test/update-check-main-moved.test.ts": 3233, + "test/upgrade-migration-v1.test.ts": 36, + "test/upgrade-setup-recovery.test.ts": 424, + "test/upgrade-template-pins.test.ts": 9, + "test/user-render-out-dir-install.test.ts": 126, + "test/user-slug-fallback.test.ts": 572, + "test/v0-dormancy.test.ts": 29, + "test/verify-gate.test.ts": 1793, "test/version-source.test.ts": 13, - "test/voice-directive.test.ts": 85, - "test/windows-free-selection-autoplan-guard.test.ts": 31, - "test/windows-native-workflows.test.ts": 243, - "test/workflow-action-pins.test.ts": 20, - "test/workflow-boundaries-fixture.test.ts": 2673, - "test/workflow-concurrency.test.ts": 13, - "test/workflow-excerpt.test.ts": 103, - "test/workflow-judge-cache.test.ts": 1780, - "test/workflow-judge-input.test.ts": 50, - "test/workflow-required-steps.test.ts": 25, - "test/worktree-isolated-shapes.test.ts": 33, - "test/worktree.test.ts": 510, - "test/writing-style-resolver.test.ts": 17, - "test/wtree-consumers.test.ts": 1013 + "test/voice-directive.test.ts": 65, + "test/windows-free-selection-autoplan-guard.test.ts": 23, + "test/windows-native-workflows.test.ts": 206, + "test/workflow-action-pins.test.ts": 16, + "test/workflow-boundaries-fixture.test.ts": 3086, + "test/workflow-concurrency.test.ts": 9, + "test/workflow-excerpt.test.ts": 118, + "test/workflow-judge-cache.test.ts": 1467, + "test/workflow-judge-input.test.ts": 45, + "test/workflow-required-steps.test.ts": 33, + "test/worktree-isolated-shapes.test.ts": 26, + "test/worktree.test.ts": 364, + "test/writing-style-resolver.test.ts": 14, + "test/wtree-consumers.test.ts": 790 } } diff --git a/scripts/free-test-seed-allowlist.json b/scripts/free-test-seed-allowlist.json index 4af44717a..c2492267e 100644 --- a/scripts/free-test-seed-allowlist.json +++ b/scripts/free-test-seed-allowlist.json @@ -3,6 +3,7 @@ {"file": "test/host-conformance.test.ts", "reason": "real ./setup per host and layout; the time is setup's own skill-linking loop, not fixture copies (shared fixture landed in W6c)"}, {"file": "test/shared-libs-stage-actor.test.ts", "reason": "re-runs the paid shared-libs runner body against git fixture repos per test; owned by the paid-harness value-bar review"}, {"file": "test/setup-install-registry.test.ts", "reason": "82s in the 2026-10-15 recording (58s on 2026-10-04): 11 cases each run the real ./setup, gstack-relink, gstack-config and gstack-uninstall against throwaway checkouts", "todo": "split by describe block so the packer can balance it, then rerun bun run test:ubicloud --record-durations (TODOS.md, fix wave 2 follow-ups)"}, - {"file": "test/host-renders-prune.test.ts", "reason": "64s in the 2026-10-15 recording (unseeded before): every case runs a real ./setup or the gstack-upgrade migration over host render dirs", "todo": "split by describe block so the packer can balance it, then rerun bun run test:ubicloud --record-durations (TODOS.md, fix wave 2 follow-ups)"} + {"file": "test/brain-sync.test.ts", "reason": "89s in the 2026-10-19 weekly recording (58s on 2026-10-15): #3254 added the case-collision suite, and every case drains a real gstack-brain-sync against a local bare remote", "todo": "split by describe block so the packer can balance it, then rerun bun run test:ubicloud --record-durations (TODOS.md, fix wave 2 follow-ups)"}, + {"file": "test/setup-codex-scope.test.ts", "reason": "64s in the 2026-10-19 weekly recording (59s on 2026-10-15): each case runs the real ./setup twice against a Codex install fixture", "todo": "split by describe block so the packer can balance it, then rerun bun run test:ubicloud --record-durations (TODOS.md, fix wave 2 follow-ups)"} ] } diff --git a/scripts/harness-version.json b/scripts/harness-version.json index 50da17512..dc18269d4 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -16,7 +16,7 @@ "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6}, "scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":6}, "scripts/test-paid-shards.ts": {"blob":"8ddee5f11ed7856876135520dd8025ab989e2db9","since":6}, - "scripts/test-pr-profile.ts": {"blob":"2029a89e14f511866cc91bebfee00b4fe9e7cf63","since":6}, + "scripts/test-pr-profile.ts": {"blob":"579ca1bbd590f709c1ea3d6f28fa6c629688c79f","since":6,"nonBehavioral":"Classifies scripts/free-test-seed-allowlist.json as free-only; only the free seed ratchet reads it, so no verdict can change"}, "scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":6}, "test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":6}, "test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":6}, diff --git a/scripts/test-pr-profile.ts b/scripts/test-pr-profile.ts index 2029a89e1..579ca1bbd 100644 --- a/scripts/test-pr-profile.ts +++ b/scripts/test-pr-profile.ts @@ -231,6 +231,7 @@ export const FREE_ONLY_PR_FILES = [ 'test/helpers/auq-parallel-worker.ts', 'test/helpers/stored-zip.ts', // Read only by the free test-health report tests. 'scripts/free-test-durations.json', + 'scripts/free-test-seed-allowlist.json', // Read only by the free seed ratchet (test/free-seed-ratchet.test.ts). 'scripts/paid-test-durations.json', 'scripts/ubicloud/**', 'capy/skill/**', // The vendored Capy skill (Drive bootstrapper): run by Capy agents, read by no paid case; test/capy-skill-wrapper.test.ts covers it. diff --git a/test/free-fixtures.test.ts b/test/free-fixtures.test.ts index 4feeb8c37..dbfa225d0 100644 --- a/test/free-fixtures.test.ts +++ b/test/free-fixtures.test.ts @@ -107,6 +107,16 @@ describe('PR selection over the W1a single-file fallbacks', () => { } }); + test('seed data and the seed allowlist select no paid case; a derived paid-surface file still selects its dependents (#3289)', () => { + // #3289: the allowlist edit derived 246 gate cases, though only test/free-seed-ratchet.test.ts reads it. + const seed = select(['scripts/free-test-seed-allowlist.json', 'scripts/free-test-durations.json', 'scripts/free-test-durations-windows.json']); + expect({ mode: seed.mode, unknown: seed.unknownFiles, derived: seed.derivedFiles, e2e: seed.e2e, judges: seed.judges }) + .toEqual({ mode: 'pr', unknown: [], derived: [], e2e: [], judges: [] }); + const paid = select(['design/src/evolve.ts']); + expect(paid.derivedFiles.map(entry => entry.file)).toEqual(['design/src/evolve.ts']); + expect(paid.e2e.length).toBeGreaterThan(0); + }); + test('the paid workflow itself still restores the full gate (36493452542)', () => { const result = select(['.github/workflows/evals.yml']); expect(result.mode).toBe('full-fallback'); From 9665621d18086efc3c755d1052dc103f076846a5 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Mon, 19 Oct 2026 10:13:58 -0700 Subject: [PATCH 2/3] v1.91.148.0 ci(seed): seed PRs also dispatch Quality gate, and say which runs are their checks (#3297) * ci(seed): seed PRs also dispatch Quality gate, and say which runs are their checks Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * chore: stamp v1.91.148.0 Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- .github/workflows/free-durations-record.yml | 3 ++- CHANGELOG.md | 8 ++++++++ VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- docs/TESTING_INTERNALS.md | 5 ++++- package.json | 2 +- test/free-durations-record-workflow.test.ts | 6 +++++- 7 files changed, 22 insertions(+), 6 deletions(-) diff --git a/.github/workflows/free-durations-record.yml b/.github/workflows/free-durations-record.yml index 848be5dfc..9101104d3 100644 --- a/.github/workflows/free-durations-record.yml +++ b/.github/workflows/free-durations-record.yml @@ -161,7 +161,7 @@ jobs: - Files that failed while recording (durations kept; this PR's own free checks decide): $failed_line - Seed growth ratchet (60 s ceiling, shrink-only allowlist): $ratchet_line - Data only. Stamp and merge per docs/ship-policy.md; the stamp push runs the full PR checks. + Data only. This run dispatched Free Tests (the required free-tests check), Windows Free Tests and Quality gate on this branch; those are this PR's checks. The pull_request runs wait as "action required" because a token-opened PR never starts them without approval. Stamp and merge per docs/ship-policy.md; the stamp push runs the full PR checks. EOF if [ -n "$number" ]; then gh pr edit "$number" --body-file pr-body.md @@ -170,3 +170,4 @@ jobs: fi gh workflow run free-tests.yml --repo "$GITHUB_REPOSITORY" --ref "$branch" gh workflow run windows-free-tests.yml --repo "$GITHUB_REPOSITORY" --ref "$branch" + gh workflow run quality-gate.yml --repo "$GITHUB_REPOSITORY" --ref "$branch" diff --git a/CHANGELOG.md b/CHANGELOG.md index 1232dd584..b130dfee2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [1.91.148.0] - 2026-10-19 + +**Weekly seed PRs now get the credential scan as well as the free suites.** A pull request the recorder opens with the workflow token never starts its own `pull_request` checks. GitHub holds them as "action required". So the recorder dispatches its checks on the bot branch instead. + +### Changed +- **`free-durations-record.yml` also dispatches Quality gate** on `bot/free-duration-seeds`, next to Free Tests (the required `free-tests` check) and Windows Free Tests. +- **The seed PR body says the dispatched runs are its checks** and explains why its pull_request runs wait for approval. + ## [1.91.147.0] - 2026-10-19 **The free-suite duration seeds are current again.** This is the first refresh from the weekly recorder. It adds 14 Linux and 6 Windows test files that the shard packers had been placing at the 99th-percentile estimate. diff --git a/VERSION b/VERSION index ae26945c3..427e8cae8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.147.0 +1.91.148.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 656c5d1e5..7c7dab179 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.147.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.148.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/docs/TESTING_INTERNALS.md b/docs/TESTING_INTERNALS.md index 9f2d95941..3aec0d114 100644 --- a/docs/TESTING_INTERNALS.md +++ b/docs/TESTING_INTERNALS.md @@ -1379,7 +1379,10 @@ refresh to a maintainer. `windows-free-tests.yml`'s `record_durations` path. When either recording adds or drops a test file, it force-updates the `bot/free-duration-seeds` branch, opens or refreshes one seed PR whose body states the ratchet result, and - dispatches the free lanes on it. A file that fails while recording turns its + dispatches Free Tests, Windows Free Tests and Quality gate on it. Those + dispatched runs are the seed PR's checks, and Free Tests supplies the required + `free-tests` check. GitHub holds the pull_request runs of a token-opened PR as + "action required" until someone approves them. A file that fails while recording turns its job red but keeps its duration, and the PR body names it; the seed PR's own free checks decide whether the tree is healthy. An interrupted recording uploads no seed, and more than 5 red files on either OS proposes nothing. The diff --git a/package.json b/package.json index ff953935f..61f535db9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.147", + "version": "1.91.148", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/test/free-durations-record-workflow.test.ts b/test/free-durations-record-workflow.test.ts index 7a61d7c6b..a86d0fcbc 100644 --- a/test/free-durations-record-workflow.test.ts +++ b/test/free-durations-record-workflow.test.ts @@ -81,7 +81,11 @@ test('the propose step touches only the two seeds, never the allowlist, on one b expect(script).not.toContain('free-test-seed-allowlist.json'); expect(script).toContain('branch=bot/free-duration-seeds'); expect(script).toContain('bun test test/free-seed-ratchet.test.ts'); - expect(script).toContain('gh workflow run free-tests.yml'); + for (const workflow of ['free-tests.yml', 'windows-free-tests.yml', 'quality-gate.yml']) { + expect(script).toContain(`gh workflow run ${workflow} --repo "$GITHUB_REPOSITORY" --ref "$branch"`); + expect(load(workflow).on).toHaveProperty('workflow_dispatch'); + } + expect(script).toContain('those are this PR\'s checks'); expect(proposeStep.env.OPEN_PR).toBe("${{ github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || inputs.open_pr) }}"); expect(script.indexOf('if [ "$OPEN_PR" != true ]')).toBeLessThan(script.indexOf('git push')); expect(recorder.on.workflow_dispatch.inputs.open_pr).toMatchObject({ type: 'boolean', default: true }); From 9cf105ccffe11f414f9572a68551db6a0ebb16a6 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Mon, 19 Oct 2026 11:26:20 -0700 Subject: [PATCH 3/3] v1.91.149.0 feat(evals): Codex host for gate cases, sol weekly and Fable/Astra monthly census arms (GSTA-49 TM2) (#3292) * feat(evals): Opus 5.5 is the paid gate's model under test; judge stays Fable (GSTA-49 TM1) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * v1.91.140.0 chore: stamp TM1 (Opus 5.5 model under test) and CHANGELOG Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * feat(evals): Codex host adapter, real Codex cost, two-arm weekly and monthly census (GSTA-49 TM2) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * chore: TM2 CHANGELOG under [Unreleased]; re-pin codex-host-runner in the harness manifest after the main merge Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * ci: retrigger PR workflows (base changed to main after open; no pull_request run fired) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * v1.91.149.0 chore: stamp TM2 (Codex host, census arms) and CHANGELOG Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- .github/workflows/evals-periodic.yml | 171 ++++++-- CHANGELOG.md | 27 ++ VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- package.json | 2 +- scripts/census-arms.ts | 81 ++++ scripts/e2e-shard-reuse.ts | 1 + scripts/harness-version.json | 20 +- scripts/lib/paid-cases.ts | 37 +- scripts/lib/paid-plan.ts | 15 +- scripts/lib/paid-report.ts | 11 +- scripts/test-paid-shards.ts | 4 + test/ci-native-evidence.test.ts | 28 +- test/ci-paid-coordination.test.ts | 3 +- test/codex-host.test.ts | 287 +++++++++++++ test/evals-workflow-wiring.test.ts | 159 ++++++- test/fixtures/codex-host/exec-commands.jsonl | 7 + .../codex-host/exec-file-change.jsonl | 9 + test/helpers/anthropic-preflight.ts | 7 +- test/helpers/codex-eval.ts | 12 +- test/helpers/codex-host-runner.ts | 306 ++++++++++++++ test/helpers/codex-session-runner.ts | 399 +++++++++++------- test/helpers/e2e-helpers.ts | 14 + test/helpers/e2e-hosts.ts | 126 ++++++ test/helpers/eval-store.ts | 3 + test/helpers/free-fixtures-data.ts | 1 + test/helpers/session-ledger.ts | 5 + test/helpers/session-runner.ts | 23 +- test/paid-retry-supervision.test.ts | 29 +- test/paid-shards.test.ts | 2 +- 30 files changed, 1533 insertions(+), 260 deletions(-) create mode 100644 scripts/census-arms.ts create mode 100644 test/codex-host.test.ts create mode 100644 test/fixtures/codex-host/exec-commands.jsonl create mode 100644 test/fixtures/codex-host/exec-file-change.jsonl create mode 100644 test/helpers/codex-host-runner.ts create mode 100644 test/helpers/e2e-hosts.ts diff --git a/.github/workflows/evals-periodic.yml b/.github/workflows/evals-periodic.yml index da9f8dde6..f1951fffb 100644 --- a/.github/workflows/evals-periodic.yml +++ b/.github/workflows/evals-periodic.yml @@ -15,6 +15,10 @@ name: Periodic Evals # diff-billed, so without it the full gate census might never execute # anywhere); the hollow-shard guard (exit 0 + zero executed tests under # EVALS_ALL fails) makes both lanes census-health checks, not just test runs. +# The gate census is a matrix of arms (scripts/census-arms.ts): each arm is a +# host and model with its own plan. Monday runs claude-opus (the capture +# model) and codex-sol (gpt-6.1-sol on the Codex host, adapter cases only); +# the monthly cron runs only the census, with claude-fable and codex-astra. # eval-bg (scripts/eval-bg.ts) finds its dispatch by the nonce in the run name. run-name: ${{ inputs.nonce && format('Periodic Evals [eval-bg {0}]', inputs.nonce) || 'Periodic Evals' }} on: @@ -23,6 +27,9 @@ on: # Daily, for autoplan-schema-canary alone (latest Claude Code vs the /autoplan # guard's payload assumptions); every census job below skips this schedule. - cron: '30 7 * * *' + # Monthly, the census jobs alone with the claude-fable and codex-astra arms + # (the 1st; never combined with a day of week, which cron would OR). + - cron: '0 6 1 * *' workflow_dispatch: inputs: redispatch_of: @@ -30,9 +37,13 @@ on: type: string default: '' include_gate_census: - description: 'Branch dispatch only: also run the weekly gate census (default: periodic lane only; gate cases validate through an evals.yml dispatch with evals_all)' + description: 'Branch dispatch only: also run the weekly gate census arms (default: periodic lane only; gate cases validate through an evals.yml dispatch with evals_all)' type: boolean default: false + census_arms: + description: 'Run only the gate census, with these arms (comma-separated: claude-opus, codex-sol, claude-fable, codex-astra; scripts/census-arms.ts). Empty keeps the periodic lane.' + type: string + default: '' expected_sha: description: 'eval-bg: the revision the dispatcher tested; the run fails fast when the ref has moved' type: string @@ -51,7 +62,7 @@ on: # runs in its own group so it never cancels the run that dispatched it; both # runs are reported. concurrency: - group: evals-periodic-${{ github.ref }}-${{ github.event_name }}${{ inputs.redispatch_of && format('-redispatch-{0}', inputs.redispatch_of) || '' }}${{ (github.event.schedule == '30 7 * * *' || inputs.schema_canary_only) && '-schema-canary' || '' }} + group: evals-periodic-${{ github.ref }}-${{ github.event_name }}${{ inputs.redispatch_of && format('-redispatch-{0}', inputs.redispatch_of) || '' }}${{ (github.event.schedule == '30 7 * * *' || inputs.schema_canary_only) && '-schema-canary' || '' }}${{ github.event.schedule == '0 6 1 * *' && '-monthly' || '' }} cancel-in-progress: true env: @@ -126,8 +137,9 @@ jobs: periodic_slices: ${{ steps.periodic-matrix.outputs.slices }} periodic_codex_slices: ${{ steps.periodic-matrix.outputs.codex_slices }} periodic_slice_timeouts: ${{ steps.periodic-matrix.outputs.slice_timeouts }} - gate_slices: ${{ steps.gate-matrix.outputs.slices }} - gate_slice_timeouts: ${{ steps.gate-matrix.outputs.slice_timeouts }} + periodic_lane: ${{ steps.census.outputs.periodic_lane }} + census_arms: ${{ steps.census.outputs.arms }} + gate_matrix: ${{ steps.gate-matrix.outputs.gate_matrix }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: @@ -147,7 +159,18 @@ jobs: with: bun-version: 1.4.2 + # Census arms (scripts/census-arms.ts): which host/model arms run the + # gate census, and whether the periodic lane runs beside them. + - name: Select the census arms + id: census + env: + SCHEDULE: ${{ github.event.schedule }} + CENSUS_ARMS: ${{ inputs.census_arms }} + INCLUDE_GATE_CENSUS: ${{ github.ref == 'refs/heads/main' || inputs.include_gate_census }} + run: bun --no-install run scripts/census-arms.ts --select >> "$GITHUB_OUTPUT" + - name: Emit run manifest (ALL periodic tests minus reasoned excludes) + if: steps.census.outputs.periodic_lane == 'true' env: EVALS_ALL: "1" run: EVALS_TIER=periodic bun --no-install run scripts/test-paid-shards.ts --tier periodic --emit-plan /tmp/paid-plan/manifest.json --slice-budget 420 --jobs 2 --max-parallel 38 @@ -156,6 +179,7 @@ jobs: # container executor runs every other slice. - name: Derive the periodic executor matrix from the plan id: periodic-matrix + if: steps.census.outputs.periodic_lane == 'true' run: | { echo "slices=$(jq -c '[range(1; .sliceCount + 1)] - .plan.codexSlices' /tmp/paid-plan/manifest.json)" @@ -164,26 +188,38 @@ jobs: } >> "$GITHUB_OUTPUT" - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + if: steps.census.outputs.periodic_lane == 'true' with: name: paid-plan path: /tmp/paid-plan/manifest.json retention-days: 30 - - name: Emit gate census manifest (ALL gate tests) + # One plan per arm: the Codex host plans only its adapter cases and + # records every Claude-only case as excluded with its reason. + - name: Emit gate census manifests (ALL gate tests, one plan per arm) + if: steps.census.outputs.arms != '[]' env: EVALS_ALL: "1" - run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --emit-plan /tmp/gate-census-plan/manifest.json --slice-budget 420 --jobs 2 --skip-judges --max-parallel 20 - - - name: Derive the gate census executor matrix from the plan - id: gate-matrix + ARMS: ${{ steps.census.outputs.arms }} run: | - echo "slices=$(jq -c '[range(1; .sliceCount + 1)]' /tmp/gate-census-plan/manifest.json)" >> "$GITHUB_OUTPUT" - echo "slice_timeouts=$(jq -ec '[0] + .plan.sliceCiTimeoutMinutes' /tmp/gate-census-plan/manifest.json)" >> "$GITHUB_OUTPUT" + set -euo pipefail + for arm in $(jq -r '.[]' <<<"$ARMS"); do + mapfile -t arm_env < <(bun --no-install run scripts/census-arms.ts --env "$arm") + env "${arm_env[@]}" EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --emit-plan "/tmp/gate-census-plan/$arm/manifest.json" --slice-budget 420 --jobs 2 --skip-judges --max-parallel 20 + done + + # One executor per arm and planned slice, each with its slice's ceiling. + - name: Derive the gate census executor matrix from the plans + id: gate-matrix + env: + ARMS: ${{ steps.census.outputs.arms }} + run: bun --no-install run scripts/census-arms.ts --matrix /tmp/gate-census-plan "$ARMS" >> "$GITHUB_OUTPUT" - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + if: steps.census.outputs.arms != '[]' with: name: gate-census-plan - path: /tmp/gate-census-plan/manifest.json + path: /tmp/gate-census-plan retention-days: 30 # autoplan-schema-canary: installs the latest published Claude Code with @@ -237,6 +273,7 @@ jobs: eval-slices: runs-on: ubicloud-standard-8 needs: [build-image, plan-slices] + if: ${{ needs.plan-slices.outputs.periodic_lane == 'true' }} env: EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-eval-slices-${{ matrix.slice }} # The planner packs ~7 minutes of recorded work per slice; each slice's job @@ -388,7 +425,7 @@ jobs: eval-codex-slices: runs-on: ubicloud-standard-8 needs: [build-image, plan-slices] - if: ${{ needs.plan-slices.outputs.periodic_codex_slices != '[]' }} + if: ${{ needs.plan-slices.outputs.periodic_lane == 'true' && needs.plan-slices.outputs.periodic_codex_slices != '[]' }} env: EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-eval-slices-${{ matrix.slice }} timeout-minutes: ${{ fromJSON(needs.plan-slices.outputs.periodic_slice_timeouts)[matrix.slice] }} @@ -512,16 +549,18 @@ jobs: # blind spots rot invisibly. Census health, not selector correctness — # selector logic has free synthetic-diff contract tests. gate-census: + name: gate-census-${{ matrix.arm }} (${{ matrix.slice }}) runs-on: ubicloud-standard-8 needs: [build-image, plan-slices] - # A branch dispatch validates the periodic lane only; gate cases on a - # branch go through an evals.yml dispatch with evals_all. Main (schedule - # or dispatch) always runs the census; a branch opts in explicitly. - if: ${{ github.ref == 'refs/heads/main' || inputs.include_gate_census }} + # The planner picks the arms (scripts/census-arms.ts). A branch dispatch + # validates the periodic lane only unless it sets include_gate_census or + # census_arms; gate cases on a branch also go through an evals.yml + # dispatch with evals_all. Main (schedule or dispatch) always runs it. + if: ${{ needs.plan-slices.outputs.gate_matrix != '[]' }} env: - EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-gate-census-${{ matrix.slice }} - # Per-slice job ceiling from the packed plan (W2c/ENG-2), as for eval-slices. - timeout-minutes: ${{ fromJSON(needs.plan-slices.outputs.gate_slice_timeouts)[matrix.slice] }} + EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-gate-census-${{ matrix.arm }}-${{ matrix.slice }} + # Per-slice job ceiling from the arm's packed plan (W2c/ENG-2), as for eval-slices. + timeout-minutes: ${{ matrix.timeout }} permissions: contents: read packages: read @@ -536,7 +575,7 @@ jobs: fail-fast: false max-parallel: 20 matrix: - slice: ${{ fromJSON(needs.plan-slices.outputs.gate_slices) }} + include: ${{ fromJSON(needs.plan-slices.outputs.gate_matrix) }} steps: - name: Record the job start for the slice deadline run: echo "GSTACK_SLICE_JOB_STARTED_AT=$(date +%s)" >> "$GITHUB_ENV" @@ -588,7 +627,15 @@ jobs: printenv OPENAI_API_KEY | "$codex" login --with-api-key "$codex" login status >/dev/null 2>&1 - - name: Run gate census slice ${{ matrix.slice }} + # The arm's host and model (EVALS_HOST, plus EVALS_MODEL or + # GSTACK_CODEX_MODEL). The Codex host runs codex exec with + # danger-full-access, so it needs no bubblewrap and runs in this container. + - name: Select the census arm's host and model + env: + ARM: ${{ matrix.arm }} + run: bun --no-install run scripts/census-arms.ts --env "$ARM" >> "$GITHUB_ENV" + + - name: Run gate census slice ${{ matrix.slice }} (${{ matrix.arm }}) env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} @@ -597,13 +644,13 @@ jobs: EVALS_JOBS: "2" EVALS_CONCURRENCY: "2" GSTACK_EVAL_DIR: /tmp/gate-census-results - run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --plan /tmp/gate-census-plan/manifest.json --slice ${{ matrix.slice }} + run: EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --plan /tmp/gate-census-plan/${{ matrix.arm }}/manifest.json --slice ${{ matrix.slice }} - name: Upload census results if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: - name: gate-census-${{ matrix.slice }}-a${{ github.run_attempt }} + name: gate-census-${{ matrix.slice }}-a${{ github.run_attempt }}-${{ matrix.arm }} path: /tmp/gate-census-results retention-days: 90 @@ -638,6 +685,9 @@ jobs: actions: read outputs: redispatch: ${{ steps.verdict.outputs.redispatch }} + # A census-only run (monthly cron or census_arms dispatch) re-dispatches + # with the same arms; empty keeps the weekly selection. + redispatch_census_arms: ${{ needs.plan-slices.outputs.periodic_lane != 'true' && join(fromJSON(needs.plan-slices.outputs.census_arms), ',') || '' }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: @@ -648,6 +698,7 @@ jobs: bun-version: 1.4.2 - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + if: needs.plan-slices.outputs.periodic_lane == 'true' with: name: paid-plan path: /tmp/paid-report @@ -655,25 +706,35 @@ jobs: # One directory per attempt-scoped slice artifact (no merge): shard # records never overwrite each other and the first attempt decides. - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + if: needs.plan-slices.outputs.periodic_lane == 'true' with: pattern: paid-slice-* path: /tmp/paid-report - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + if: needs.plan-slices.outputs.gate_matrix != '[]' with: name: gate-census-plan - path: /tmp/gate-census-report + path: /tmp/gate-census-plan - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + if: needs.plan-slices.outputs.gate_matrix != '[]' with: pattern: gate-census-[0-9]* - path: /tmp/gate-census-report + path: /tmp/gate-census-slices - name: Reconcile slices against the manifest (fail-closed) id: reconcile if: always() + env: + PERIODIC_LANE: ${{ needs.plan-slices.outputs.periodic_lane }} run: | set +e + if [ "$PERIODIC_LANE" != "true" ]; then + echo "periodic lane skipped: census-only run (monthly cron or census_arms dispatch)" | tee /tmp/report.txt + echo "exit=0" >> "$GITHUB_OUTPUT" + exit 0 + fi EVALS_TIER=periodic bun --no-install run scripts/test-paid-shards.ts --tier periodic --report /tmp/paid-report | tee /tmp/report.txt # PIPESTATUS[0], NOT $?: GitHub's default run-step shell is # `bash -e {0}` with NO pipefail, so $? after the pipe is tee's @@ -686,20 +747,34 @@ jobs: if: always() env: GATE_CENSUS: ${{ needs.gate-census.result }} + ARMS: ${{ needs.plan-slices.outputs.census_arms }} + # One report per arm: its plan plus its own slice artifacts + # (gate-census--a-) in /tmp/gate-census-report/. run: | set +e if [ "$GATE_CENSUS" = "skipped" ]; then - echo "gate census skipped: branch dispatch without include_gate_census (periodic lane only)" | tee /tmp/gate-report.txt + echo "gate census skipped: no census arm selected (branch dispatch without include_gate_census or census_arms)" | tee /tmp/gate-report.txt echo "exit=0" >> "$GITHUB_OUTPUT" exit 0 fi - EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --report /tmp/gate-census-report | tee /tmp/gate-report.txt - echo "exit=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT" + status=0 + : > /tmp/gate-report.txt + for arm in $(jq -r '.[]' <<<"$ARMS"); do + dir="/tmp/gate-census-report/$arm" + mkdir -p "$dir" + cp "/tmp/gate-census-plan/$arm/manifest.json" "$dir/manifest.json" || status=1 + for slice in /tmp/gate-census-slices/gate-census-*-"$arm"; do if [ -d "$slice" ]; then mv "$slice" "$dir/"; fi; done + echo "=== gate census arm: $arm ===" | tee -a /tmp/gate-report.txt + EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --report "$dir" | tee -a /tmp/gate-report.txt + code=${PIPESTATUS[0]} + if [ "$code" -ne 0 ]; then status=$code; fi + done + echo "exit=$status" >> "$GITHUB_OUTPUT" - name: Stamp trial history series if: always() run: | - for file in /tmp/paid-report/trial-outcomes.jsonl /tmp/gate-census-report/trial-outcomes.jsonl; do + for file in /tmp/paid-report/trial-outcomes.jsonl /tmp/gate-census-report/*/trial-outcomes.jsonl; do if [ -f "$file" ]; then bun --no-install run scripts/eval-trial-series.ts "$file"; fi done @@ -719,7 +794,7 @@ jobs: with: name: trial-outcomes-gate-census-a${{ github.run_attempt }} path: | - /tmp/gate-census-report/trial-outcomes.jsonl + /tmp/gate-census-report/*/trial-outcomes.jsonl if-no-files-found: ignore retention-days: 90 @@ -732,7 +807,7 @@ jobs: run: | mkdir -p /tmp/exercised-map dirs=() - for dir in /tmp/gate-census-report /tmp/paid-report; do [ -d "$dir" ] && dirs+=(--from-dir "$dir"); done + for dir in /tmp/gate-census-report/* /tmp/paid-report; do [ -d "$dir" ] && dirs+=(--from-dir "$dir"); done status=0 bun --no-install run scripts/exercised-map.ts "${dirs[@]}" \ --run-id "$GITHUB_RUN_ID" --out /tmp/exercised-map/exercised-commands.json --report /tmp/exercised-map/report.md || status=$? @@ -779,8 +854,16 @@ jobs: [ "$1" = "0" ] && return 0 jq -e '.version == 2 and .verdict.redispatchEligible == true' "$2/collector-outcomes.json" >/dev/null 2>&1 } + eligible_arms() { # $1 exit, then one report dir per census arm: each must be green or INFRA/INCOMPLETE-only + [ "$1" = "0" ] && return 0 + shift + local dir + for dir in "$@"; do + jq -e '.version == 2 and (.verdict.verdict == "GREEN" or .verdict.redispatchEligible == true)' "$dir/collector-outcomes.json" >/dev/null 2>&1 || return 1 + done + } if [ -z "$REDISPATCH_OF" ] && { [ "$PERIODIC_EXIT" != "0" ] || [ "$GATE_EXIT" != "0" ]; } \ - && eligible "$PERIODIC_EXIT" /tmp/paid-report && eligible "$GATE_EXIT" /tmp/gate-census-report; then + && eligible "$PERIODIC_EXIT" /tmp/paid-report && eligible_arms "$GATE_EXIT" /tmp/gate-census-report/*/; then echo "redispatch=true" >> "$GITHUB_OUTPUT" else echo "redispatch=false" >> "$GITHUB_OUTPUT" @@ -793,7 +876,7 @@ jobs: # comments on or closes main's issue. Every report-derived line goes # through the one published-text sanitizer (scripts/lib/published-text.ts). - name: Write the census report - if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success') + if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || (needs.plan-slices.outputs.periodic_lane == 'true' && needs.eval-slices.result != 'success') || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success') env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} REDISPATCH: ${{ steps.verdict.outputs.redispatch }} @@ -827,7 +910,14 @@ jobs: cat /tmp/paid-report/report-summary.md 2>/dev/null || echo "(no periodic report summary)" echo echo "### Gate census" - cat /tmp/gate-census-report/report-summary.md 2>/dev/null || echo "(no gate census report summary)" + found=0 + for summary in /tmp/gate-census-report/*/report-summary.md; do + [ -f "$summary" ] || continue + found=1 + echo "#### Arm $(basename "$(dirname "$summary")")" + cat "$summary" + done + [ "$found" = 1 ] || echo "(no gate census report summary)" echo echo "### Pass-rate history (ACTION REQUIRED)" { awk '/^ACTION REQUIRED/ { on = 1; print; next } on && /^ \[/ { print; next } { on = 0 } /history unavailable/ { print }' /tmp/pass-rates.txt 2>/dev/null \ @@ -851,7 +941,7 @@ jobs: cat /tmp/census-report.md >> "$GITHUB_STEP_SUMMARY" - name: Upload the census report - if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success') + if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || (needs.plan-slices.outputs.periodic_lane == 'true' && needs.eval-slices.result != 'success') || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success') uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: census-report-a${{ github.run_attempt }} @@ -860,7 +950,7 @@ jobs: retention-days: 90 - name: Upsert tracking issue on failure - if: always() && github.ref == 'refs/heads/main' && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success') + if: always() && github.ref == 'refs/heads/main' && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || (needs.plan-slices.outputs.periodic_lane == 'true' && needs.eval-slices.result != 'success') || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success') env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -877,7 +967,7 @@ jobs: fi - name: Close the tracking issue on a green run - if: always() && github.ref == 'refs/heads/main' && steps.reconcile.outputs.exit == '0' && steps.gate-reconcile.outputs.exit == '0' && steps.pass-rates.outputs.exit == '0' && needs.eval-slices.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) && needs.gate-census.result == 'success' + if: always() && github.ref == 'refs/heads/main' && steps.reconcile.outputs.exit == '0' && steps.gate-reconcile.outputs.exit == '0' && steps.pass-rates.outputs.exit == '0' && needs.plan-slices.outputs.periodic_lane == 'true' && needs.eval-slices.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) && needs.gate-census.result == 'success' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} REDISPATCH_OF: ${{ inputs.redispatch_of }} @@ -892,7 +982,7 @@ jobs: fi - name: Fail the workflow when reconciliation failed - if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || !contains(fromJSON('["success","skipped"]'), needs.gate-census.result)) + if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || (needs.plan-slices.outputs.periodic_lane == 'true' && needs.eval-slices.result != 'success') || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || !contains(fromJSON('["success","skipped"]'), needs.gate-census.result)) run: exit 1 # The one INFRA/INCOMPLETE re-dispatch (UC-E1). Its own job so the report @@ -909,4 +999,5 @@ jobs: - name: Re-dispatch the weekly census once env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: gh workflow run evals-periodic.yml --repo "$GITHUB_REPOSITORY" --ref "$GITHUB_REF_NAME" -f redispatch_of="$GITHUB_RUN_ID" + CENSUS_ARMS: ${{ needs.report.outputs.redispatch_census_arms }} + run: gh workflow run evals-periodic.yml --repo "$GITHUB_REPOSITORY" --ref "$GITHUB_REF_NAME" -f redispatch_of="$GITHUB_RUN_ID" ${CENSUS_ARMS:+-f census_arms="$CENSUS_ARMS"} diff --git a/CHANGELOG.md b/CHANGELOG.md index b130dfee2..71b30aaaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,32 @@ # Changelog +## [1.91.149.0] - 2026-10-19 + +**Paid eval cases can now run on the Codex host, and the Monday census runs two arms: Claude Opus 5.5 and Codex gpt-6.1-sol.** `EVALS_HOST=codex` runs the 52 adapter-only gate cases through a full Codex runtime per case. On its first pilot gpt-6.1-sol passed 48 of 52 with no retries, at $4.63 for the 52 cases, against $28.93 for the same cases on Fable (0.16×). Claude-only cases are skipped on Codex with a recorded reason and never count as failed. GSTA-49 TM2. + +### Added +- **Codex host adapter:** `test/helpers/e2e-hosts.ts` holds the `EVALS_HOST` switch (`claude` default, `codex`) and the per-case host metadata. `test/helpers/codex-host-runner.ts` adds: + - the case runner, with sandbox settings and a reasoning effort (the model default unless set); + - file-change event mapping; + - a transcript translated into the Claude stream-json shape that existing assertions read; + - the native-lifecycle hooks. + + The 26 rework cases stay Claude-only. +- **Census arms (`scripts/census-arms.ts`):** `evals-periodic.yml` gets two census schedules. + - **Monday:** a two-arm gate census, `claude-opus` and `codex-sol`. + - **Monthly (`0 6 1 * *`):** census jobs only, with `claude-fable` (claude-fable-5-1) and `codex-astra` (gpt-6-astra) arms and a `-monthly` concurrency key. + + A `census_arms` dispatch input picks arms by hand. Artifact names and run IDs carry the arm, and a census-only run executes no periodic slice. Every arm keeps the census's `--skip-judges`. + +### Changed +- **Codex cost is real:** `cost_known: false` is gone. Codex sessions price their `turn.completed` usage (input, cached input, cache-write input and output tokens) through `lib/pricing.ts`. The shared `execCodex` keeps the old runner's events and verdicts. +- **Plans, reports and the ledger carry the host** when it is not Claude. On Codex, the Anthropic preflight skips its `claude -p` ping. +- **Gate integrity:** no threshold, trial count, case kind, judge or Claude-host verdict changed. `HARNESS_VERSION` stays 6, and the changed harness files are recorded as non-behavioral. Five workflow tests now assert the census matrix per arm (one row per arm and slice, per-arm plan and report paths, a `periodic_lane` guard) in place of the single-arm shape; no assertion was dropped. + +### If you… +- **…want one case on Codex:** `EVALS_HOST=codex GSTACK_CODEX_MODEL=gpt-6.1-sol bun run scripts/test-paid-shards.ts --tier gate --case --trials 1`. +- **…want the monthly arms now:** dispatch `evals-periodic.yml` with `census_arms=claude-fable,codex-astra`. + ## [1.91.148.0] - 2026-10-19 **Weekly seed PRs now get the credential scan as well as the free suites.** A pull request the recorder opens with the workflow token never starts its own `pull_request` checks. GitHub holds them as "action required". So the recorder dispatches its checks on the bot branch instead. diff --git a/VERSION b/VERSION index 427e8cae8..bdda80070 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.148.0 +1.91.149.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 7c7dab179..f819c371f 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.148.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.149.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/package.json b/package.json index 61f535db9..0a91faf86 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.148", + "version": "1.91.149", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/scripts/census-arms.ts b/scripts/census-arms.ts new file mode 100644 index 000000000..9ede1799a --- /dev/null +++ b/scripts/census-arms.ts @@ -0,0 +1,81 @@ +#!/usr/bin/env bun +/** + * Gate census arms for evals-periodic.yml (GSTA-49 TM2). Each arm is one host + * and model running the weekly gate census plan: + * - Monday cron (0 6 * * 1): claude-opus (the capture model, lib/eval-model.ts) + * and codex-sol (gpt-6.1-sol on the Codex host, adapter cases only); + * - monthly cron (0 6 1 * *): claude-fable and codex-astra, census jobs only; + * - dispatch: `census_arms` runs only the census with the named arms; without + * it a dispatch keeps the periodic lane plus the Monday arms when the census + * runs (main, or include_gate_census). + * Judges stay in the Claude lanes: every census plan skips them. + * + * CLI (GitHub step outputs and env lines): + * --select env SCHEDULE, CENSUS_ARMS, INCLUDE_GATE_CENSUS -> arms=, periodic_lane= + * --env KEY=VALUE lines selecting the arm's host and model + * --matrix gate_matrix= from //manifest.json + */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import type { EvalHost } from '../test/helpers/e2e-hosts'; + +export interface CensusArm { host: EvalHost; model?: string } + +export const CENSUS_ARMS: Readonly> = Object.freeze({ + // No model: the Claude runners' capture default (claude-opus-5-5 since TM1). + 'claude-opus': { host: 'claude' }, + 'codex-sol': { host: 'codex', model: 'gpt-6.1-sol' }, + 'claude-fable': { host: 'claude', model: 'claude-fable-5-1' }, + 'codex-astra': { host: 'codex', model: 'gpt-6-astra' }, +}); + +export const WEEKLY_CENSUS_SCHEDULE = '0 6 * * 1'; +export const MONTHLY_CENSUS_SCHEDULE = '0 6 1 * *'; +export const WEEKLY_CENSUS_ARMS: readonly string[] = ['claude-opus', 'codex-sol']; +export const MONTHLY_CENSUS_ARMS: readonly string[] = ['claude-fable', 'codex-astra']; + +/** Which census arms run, and whether the periodic lane runs beside them. */ +export function selectCensus(opts: { schedule?: string; censusArms?: string; includeGateCensus: boolean }): { arms: string[]; periodicLane: boolean } { + if (opts.schedule === MONTHLY_CENSUS_SCHEDULE) return { arms: [...MONTHLY_CENSUS_ARMS], periodicLane: false }; + const named = (opts.censusArms ?? '').split(/[\s,]+/).filter(Boolean); + const unknown = named.filter(arm => !Object.hasOwn(CENSUS_ARMS, arm)); + if (unknown.length) throw new Error(`unknown census arm(s): ${unknown.join(', ')}; known: ${Object.keys(CENSUS_ARMS).join(', ')}`); + if (named.length) return { arms: [...new Set(named)], periodicLane: false }; + return { arms: opts.includeGateCensus ? [...WEEKLY_CENSUS_ARMS] : [], periodicLane: true }; +} + +/** The env one arm's planner and executor run with. */ +export function censusArmEnv(arm: string): Record { + const spec = CENSUS_ARMS[arm]; + if (!spec) throw new Error(`unknown census arm: ${arm}`); + if (!spec.model) return { EVALS_HOST: spec.host }; + return { EVALS_HOST: spec.host, [spec.host === 'codex' ? 'GSTACK_CODEX_MODEL' : 'EVALS_MODEL']: spec.model }; +} + +/** One executor per arm and planned slice, with that slice's planned CI ceiling. */ +export function censusMatrix(arms: readonly string[], planDir: string): Array<{ arm: string; slice: number; timeout: number }> { + return arms.flatMap(arm => { + const manifest = JSON.parse(fs.readFileSync(path.join(planDir, arm, 'manifest.json'), 'utf8')) as + { sliceCount: number; plan?: { ciTimeoutMinutes: number; sliceCiTimeoutMinutes?: number[] } }; + if (!manifest.plan) throw new Error(`${arm}: census manifest has no budget plan`); + return Array.from({ length: manifest.sliceCount }, (_, i) => ({ + arm, slice: i + 1, timeout: manifest.plan!.sliceCiTimeoutMinutes?.[i] ?? manifest.plan!.ciTimeoutMinutes, + })); + }); +} + +if (import.meta.main) { + const [mode, ...args] = process.argv.slice(2); + if (mode === '--select') { + const { arms, periodicLane } = selectCensus({ schedule: process.env.SCHEDULE, censusArms: process.env.CENSUS_ARMS, + includeGateCensus: process.env.INCLUDE_GATE_CENSUS === 'true' }); + console.log(`arms=${JSON.stringify(arms)}\nperiodic_lane=${periodicLane}`); + } else if (mode === '--env' && args.length === 1) { + for (const [key, value] of Object.entries(censusArmEnv(args[0]!))) console.log(`${key}=${value}`); + } else if (mode === '--matrix' && args.length === 2) { + console.log(`gate_matrix=${JSON.stringify(censusMatrix(JSON.parse(args[1]!) as string[], args[0]!))}`); + } else { + console.error('usage: census-arms.ts --select | --env | --matrix '); + process.exit(2); + } +} diff --git a/scripts/e2e-shard-reuse.ts b/scripts/e2e-shard-reuse.ts index 64839e98f..e6ffcc176 100644 --- a/scripts/e2e-shard-reuse.ts +++ b/scripts/e2e-shard-reuse.ts @@ -83,6 +83,7 @@ export function e2eReuseLaneProblem(env: NodeJS.ProcessEnv, profileMode: string if (!/^(?:sha256:)?[a-f0-9]{64}$/.test(env.EVALS_CACHE_RUNTIME_ID ?? '')) return 'No immutable runtime identity'; if (env.EVALS_TIER !== 'gate' || env.EVALS_FRESH === '1') return 'Fresh validation requested'; if (['release', 'periodic', 'marathon'].includes(env.EVALS_CACHE_PURPOSE ?? '')) return 'Scheduled and release lanes execute fresh'; + if ((env.EVALS_HOST ?? '').trim() && env.EVALS_HOST!.trim() !== 'claude') return 'Non-Claude hosts execute fresh'; if (env.NODE_OPTIONS || env.BUN_OPTIONS) return 'Preload options change execution outside the consumed source'; if (env.ANTHROPIC_BASE_URL && env.ANTHROPIC_BASE_URL !== 'https://api.anthropic.com') return 'Custom model endpoint'; return null; diff --git a/scripts/harness-version.json b/scripts/harness-version.json index dc18269d4..4e9d5794e 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -3,6 +3,7 @@ "patterns": [ "test/helpers/agent-sdk-runner.ts", "test/helpers/codex-session-runner.ts", + "test/helpers/codex-host-runner.ts", "test/helpers/claude-pty-runner.ts", "test/helpers/pty/**", "test/helpers/pty-screen.ts", @@ -10,22 +11,23 @@ "test/helpers/llm-judge.ts" ], "files": { - "scripts/lib/paid-cases.ts": {"blob":"9bacd52446192a7ecf3dfaae901ea443cf9c9ae2","since":6}, - "scripts/lib/paid-plan.ts": {"blob":"45721ce128a112666a0caed6c13c66637b74ca54","since":6}, - "scripts/lib/paid-report.ts": {"blob":"b66c41237611ccecaaa7bd6a4d7b7aac8dc548cd","since":6}, + "scripts/lib/paid-cases.ts": {"blob":"f862cf5241580c7593b9ee62eacf5ef046248d26","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, + "scripts/lib/paid-plan.ts": {"blob":"95e1798e7e2495ace350d2624e59f83412f341bc","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, + "scripts/lib/paid-report.ts": {"blob":"f92c6bb723b158e328aa040fa5de6a0ee6d3916d","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6}, "scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":6}, - "scripts/test-paid-shards.ts": {"blob":"8ddee5f11ed7856876135520dd8025ab989e2db9","since":6}, + "scripts/test-paid-shards.ts": {"blob":"19c9b2376d09090aee1a8acacfe721d66f55b0fc","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "scripts/test-pr-profile.ts": {"blob":"579ca1bbd590f709c1ea3d6f28fa6c629688c79f","since":6,"nonBehavioral":"Classifies scripts/free-test-seed-allowlist.json as free-only; only the free seed ratchet reads it, so no verdict can change"}, "scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":6}, "test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":6}, "test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":6}, "test/helpers/claude-pty-runner.ts": {"blob":"20cc5894532873aa6145fc6617fdeec8cef94a2d","since":6}, - "test/helpers/codex-session-runner.ts": {"blob":"4eea35931a35a0afb006363402e3d4191e379737","since":6}, + "test/helpers/codex-host-runner.ts": {"blob":"bff225017e235f0be6ece29f04971ac04b95b544","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, + "test/helpers/codex-session-runner.ts": {"blob":"c040ebbfa815b8cce19ef80c6133bd5a5474149a","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "test/helpers/e2e-gate.ts": {"blob":"efb7e197dd5f66c3ad0e34074b14e640754f7a45","since":6}, - "test/helpers/e2e-helpers.ts": {"blob":"2d4a2a53063cfc1f003db366d2c861ab1e379e6e","since":6}, + "test/helpers/e2e-helpers.ts": {"blob":"6ee6937e6c3ca081c77131b5a755278c7b714834","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "test/helpers/eval-budgets.ts": {"blob":"667d0e09b20fe5d7366f57b77b5fb9b04baa37ef","since":6}, - "test/helpers/eval-store.ts": {"blob":"609c6d099108ff2696eb014b47aa318a618dd2af","since":6}, + "test/helpers/eval-store.ts": {"blob":"ed8721f13d39f4ef62e0a1d814efbe9352da781a","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "test/helpers/hermetic-env.ts": {"blob":"cc072895461dc52ea317d1beff72121342420d30","since":6}, "test/helpers/llm-judge.ts": {"blob":"d0bc6bbbe9f478603d11a1ece95e66c812ad34e6","since":6}, "test/helpers/paid-test-set.ts": {"blob":"ff9bfe43c43ec2bdd11114f7b8ec7c5736116f75","since":6}, @@ -44,8 +46,8 @@ "test/helpers/pty/screen.ts": {"blob":"ba7a80bb68b1959296ab2a827cfacdaa46f4ba2d","since":6}, "test/helpers/pty/session.ts": {"blob":"58533cea9d81cc87bb3b499bdfe0db8e3a8709c5","since":6}, "test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":6}, - "test/helpers/session-ledger.ts": {"blob":"6e3c9e837e46f98addba2d12eaebff802da320bb","since":6}, - "test/helpers/session-runner.ts": {"blob":"d40eb29e8b8fdc0675b511693e4a90823b5963fc","since":6}, + "test/helpers/session-ledger.ts": {"blob":"425323284a368f5e77937e03884db4dea4e6134e","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, + "test/helpers/session-runner.ts": {"blob":"cc65a66adafa9c053e3da3c6f17b5ac86c0eade7","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"}, "test/helpers/skill-fixture.ts": {"blob":"b188064594eb4201a54a80ba1bc6f28e31d7f49e","since":6}, "test/helpers/test-selection.ts": {"blob":"51655f89f66be7a5ec487e2abfe4744300ebd5f6","since":6}, "test/helpers/touchfiles.ts": {"blob":"15f369056d41b9dc42dcd86f81f60c878074704c","since":6} diff --git a/scripts/lib/paid-cases.ts b/scripts/lib/paid-cases.ts index 9bacd5244..f862cf524 100644 --- a/scripts/lib/paid-cases.ts +++ b/scripts/lib/paid-cases.ts @@ -9,6 +9,7 @@ import type { EvalCaseKind, PanelShape } from '../../test/helpers/eval-store'; import { E2E_KINDS } from '../../test/helpers/touchfiles-data'; import { E2E_TOUCHFILES, E2E_TIERS } from '../../test/helpers/touchfiles'; import { type PaidTier, ROOT } from './paid-types'; +import { type EvalHost, hostSkipReason, resolveEvalHost } from '../../test/helpers/e2e-hosts'; /** * The E2E ids a paid file registers: the touchfile registrations that list the @@ -226,12 +227,38 @@ export function partitionCaseExclusions(keys: string[]): { runnable: string[]; e return { runnable, excluded }; } +/** + * Host exclusions (EVALS_HOST, test/helpers/e2e-hosts.ts): on a non-Claude + * host a case key whose case is Claude-only, or a file key none of whose + * `tier` cases runs on the host, is excluded with its reason. A file that + * keeps one host case stays runnable; its Claude-only cases skip at module + * load and the report deselects them. The Claude host excludes nothing. + */ +export function partitionHostExclusions(keys: string[], tier: PaidTier, host: EvalHost, rootDir = ROOT): { runnable: string[]; excluded: Array<{ file: string; reason: string }> } { + if (host === 'claude') return { runnable: keys, excluded: [] }; + const excluded: Array<{ file: string; reason: string }> = []; + const runnable = keys.filter(key => { + const caseId = shardCaseId(key); + const ids = caseId !== null ? [caseId] + : fileCaseRegistration(shardFile(key), fs.readFileSync(path.join(rootDir, shardFile(key)), 'utf8')).registered.filter(id => E2E_TIERS[id] === tier); + if (ids.some(id => hostSkipReason(id, host) === null)) return true; + const reason = ids.length === 1 ? hostSkipReason(ids[0]!, host)! + : ids.length === 0 ? `Claude-only on the ${host} host: no Codex host metadata (no ${tier} case registered)` + : `Claude-only on the ${host} host: no case runs on it (${ids.join(', ')})`; + excluded.push({ file: key, reason: `skipped: ${reason}` }); + return false; + }); + return { runnable, excluded }; +} + /** * Codex access in the CI eval image, scoped per paid shard. * * The image installs the pinned Codex CLI off PATH (GSTACK_CI_CODEX_BIN_DIR) * and the eval workflows log it in under a non-default home - * (GSTACK_CI_CODEX_HOME). A gate shard never sees either. A non-gate shard + * (GSTACK_CI_CODEX_HOME). On the Codex host (EVALS_HOST=codex) Codex is the + * agent, so every shard gets the CLI on PATH and the home as CODEX_HOME. + * Otherwise a gate shard never sees either. A non-gate shard * whose every file exercises Codex gets the CLI on PATH and the home as * CODEX_HOME. A file that mixes one Codex case with other cases keeps only * the two variables, and that case opts in on its own session; hermetic @@ -266,8 +293,10 @@ export const CODEX_CI_CASES: Readonly> = { export type CodexShardAccess = 'none' | 'case-opt-in' | 'path'; /** What one shard, given the test files behind its keys, may see of the CI Codex install in `tier`. */ -export function codexShardAccess(files: readonly string[], tier: string | undefined): CodexShardAccess { - if (tier === 'gate' || files.length === 0) return 'none'; +export function codexShardAccess(files: readonly string[], tier: string | undefined, host: EvalHost = 'claude'): CodexShardAccess { + if (files.length === 0) return 'none'; + if (host === 'codex') return 'path'; + if (tier === 'gate') return 'none'; if (files.every(file => CODEX_CI_FILES.includes(file))) return 'path'; return files.every(file => file in CODEX_CI_CASES) ? 'case-opt-in' : 'none'; } @@ -277,7 +306,7 @@ export function scopeCodexAccess(env: NodeJS.ProcessEnv, files: readonly string[ const binDir = env[CODEX_CI_ENV.binDir]; if (!binDir) return; const home = env[CODEX_CI_ENV.home]; - const access = codexShardAccess(files, env.EVALS_TIER); + const access = codexShardAccess(files, env.EVALS_TIER, resolveEvalHost(env)); delete env.CODEX_HOME; if (access === 'case-opt-in') return; delete env[CODEX_CI_ENV.binDir]; diff --git a/scripts/lib/paid-plan.ts b/scripts/lib/paid-plan.ts index 45721ce12..95e1798e7 100644 --- a/scripts/lib/paid-plan.ts +++ b/scripts/lib/paid-plan.ts @@ -10,7 +10,8 @@ import { evalEntryOutcome, type EvalCaseKind } from '../../test/helpers/eval-sto import { E2E_KINDS } from '../../test/helpers/touchfiles-data'; import { prProfileCaseAllowed, prProfileFileMap, type PrProfileSelection } from '../test-pr-profile'; import { E2E_TOUCHFILES, E2E_TIERS, LLM_JUDGE_TOUCHFILES } from '../../test/helpers/touchfiles'; -import { CASE_KEY_SEPARATOR, CASE_SHARDED_FILES, type CaseTrialPlan, caseTrialPlan, codexShardAccess, expandCaseShards, expandTrialShards, isIsolatedCase, partitionCaseExclusions, sameTrialPlan, shardCaseId, shardFile, shardTrial } from './paid-cases'; +import { CASE_KEY_SEPARATOR, CASE_SHARDED_FILES, type CaseTrialPlan, caseTrialPlan, codexShardAccess, expandCaseShards, expandTrialShards, isIsolatedCase, partitionCaseExclusions, partitionHostExclusions, sameTrialPlan, shardCaseId, shardFile, shardTrial } from './paid-cases'; +import { EVAL_HOSTS, type EvalHost, resolveEvalHost } from '../../test/helpers/e2e-hosts'; import { DEFAULT_JOBS, OVERLAY_MAX_ACTIVE_SHARDS, PAID_TIERS, SLICE_UPLOAD_RESERVE_MS, type PaidCaseSelection, type PaidProfile, type PaidShardBudget, type PaidTier, ROOT, type ShardOutcome, type ShardStatus, type ShardTrialRecord, isOverlayTestFile } from './paid-types'; import { collectPaidTestFiles, computePaidCaseSelection, expectedPrCaseCount, isAllSkippedPass, paidShardWallUpperBoundMs, partitionShardsByDiffSelection, planPaidShards, prProfileFileSelected, resolvePaidShardBudget, resolvePaidShardTimeoutMs, sameBudget, selectPaidTestFiles, shardSlug, validatedProfile } from './paid-select'; @@ -58,6 +59,8 @@ export interface PaidRunManifest { evalsAll: boolean; sliceCount: number; selectionReason: string; + /** The agent host every session runs on (EVALS_HOST); absent means claude. */ + host?: EvalHost; /** Legacy v1 manifests omit these; new plans bind case-level execution. */ profile?: PaidProfile; selection?: PaidCaseSelection; @@ -314,9 +317,11 @@ export function buildRunManifest(opts: { const quarantine = opts.quarantine ?? CASE_QUARANTINE; const notLive = [...Object.keys(kinds).filter(id => kinds[id] === 'behavior'), ...Object.keys(quarantine)].filter(id => !Object.hasOwn(E2E_TIERS, id)); if (notLive.length) throw new Error(`Only live E2E cases can be behavior or quarantined (judges sample their panel inside the case): ${notLive.join(', ')}`); - const caseKeys = partitionCaseExclusions(expandCaseShards(selected, opts.tier, rootDir)); + const host = resolveEvalHost(env); + const ciKeys = partitionCaseExclusions(expandCaseShards(selected, opts.tier, rootDir)); + const caseKeys = partitionHostExclusions(ciKeys.runnable, opts.tier, host, rootDir); const excluded = [...tierSelection.excluded, ...(opts.skipJudges ? tierSelection.selected.filter(judge) - .map(file => ({ file, reason: 'skipped: LLM judges run in the periodic census and PR gate lanes' })) : []), ...caseKeys.excluded]; + .map(file => ({ file, reason: 'skipped: LLM judges run in the periodic census and PR gate lanes' })) : []), ...ciKeys.excluded, ...caseKeys.excluded]; const expansion = expandTrialShards(caseKeys.runnable, opts.tier, rootDir, { kinds, quarantine }); const excludeOf = (key: string) => expansion.excludeCases[normalizeRelativePath(key)] ?? []; const shards = planPaidShards(expansion.keys, { maxFilesPerShard: 1 }); @@ -346,7 +351,7 @@ export function buildRunManifest(opts: { entries.sort((a, b) => (a.file < b.file ? -1 : 1)); return parseRunManifest(JSON.stringify({ version: 1, tier: opts.tier, evalsAll: opts.evalsAll, sliceCount: plan.slices.length, - selectionReason: cases.reason, profile, selection: cases.selection, + selectionReason: cases.reason, ...(host !== 'claude' ? { host } : {}), profile, selection: cases.selection, ...(cases.coverage ? { prCoverage: cases.coverage } : {}), plan: { sliceBudgetMs: opts.sliceBudgetMs!, jobs: opts.jobs!, estimatedSliceMs: plan.estimatedSliceMs, ciTimeoutMinutes: plan.ciTimeoutMinutes, sliceCiTimeoutMinutes: plan.sliceCiTimeoutMinutes, codexSlices: codexSlicesOf(opts.tier, entries) }, @@ -466,6 +471,7 @@ export function buildRunManifest(opts: { evalsAll: opts.evalsAll, sliceCount, selectionReason: cases.reason, + ...(host !== 'claude' ? { host } : {}), profile, selection: cases.selection, ...(cases.coverage ? { prCoverage: cases.coverage } : {}), @@ -479,6 +485,7 @@ export function parseRunManifest(raw: string): PaidRunManifest { if (parsed.version !== 1) throw new Error(`unsupported manifest version: ${(parsed as { version?: unknown }).version}`); if (!PAID_TIERS.includes(parsed.tier)) throw new Error(`manifest tier invalid: ${parsed.tier}`); if (parsed.profile !== undefined && parsed.profile !== 'pr' && parsed.profile !== 'full') throw new Error('manifest profile invalid'); + if (parsed.host !== undefined && (parsed.host === 'claude' || !EVAL_HOSTS.includes(parsed.host))) throw new Error(`manifest host invalid: ${parsed.host}`); if (parsed.selection !== undefined) { for (const [key, inventory] of [['e2e', E2E_TOUCHFILES], ['judges', LLM_JUDGE_TOUCHFILES]] as const) { const ids = parsed.selection?.[key]; diff --git a/scripts/lib/paid-report.ts b/scripts/lib/paid-report.ts index b66c41237..f92c6bb72 100644 --- a/scripts/lib/paid-report.ts +++ b/scripts/lib/paid-report.ts @@ -17,6 +17,7 @@ import { E2E_KINDS } from '../../test/helpers/touchfiles-data'; import { manualReviewProblem } from '../../test/helpers/cookie-workflow-manual-review'; import { writeNegativeReceipt, writePanelReceipt } from '../e2e-shard-reuse'; import { E2E_TOUCHFILES, E2E_TIERS, LLM_JUDGE_TOUCHFILES } from '../../test/helpers/touchfiles'; +import { type EvalHost, hostSkipReason } from '../../test/helpers/e2e-hosts'; import { CASE_TEST_NAMES, type CaseTrialPlan, caseTestNamePattern, caseTrialPlan, fileCaseRegistration, shardCaseId, shardFile, trialShardKey } from './paid-cases'; import { type ManifestEntry, PAID_TEST_DURATIONS_FILE, type PaidRunManifest, type SliceResult, collectorOutcomeCounts, formatProfileCoverage, loadPaidTestDurations, mergePaidTestDurations, parseRunManifest, trialPanelKey, verifySliceResults, writePaidTestDurations } from './paid-plan'; import { DEFAULT_JOBS, type PaidCaseSelection, type PaidTier, ROOT, type ShardTrialRecord } from './paid-types'; @@ -170,6 +171,8 @@ export interface JUnitShardContext { selection?: PaidCaseSelection; /** Ids this file shard leaves to their isolated trial shards. */ excludeCases?: readonly string[]; + /** The run's agent host (manifest.host); absent means claude. */ + host?: EvalHost; } export const DESELECTION_REASONS = { @@ -205,6 +208,8 @@ export function attributeJUnitCase(tc: JUnitCase, ctx: JUnitShardContext, const tier = tiers[id]; if (tier !== undefined && tier !== ctx.tier) return { kind: 'deselected', reason: `${tier}-tier case (this lane runs ${ctx.tier})` }; if (ctx.excludeCases?.includes(id)) return { kind: 'deselected', reason: DESELECTION_REASONS.isolated }; + const hostReason = hostSkipReason(id, ctx.host ?? 'claude'); + if (hostReason) return { kind: 'deselected', reason: hostReason }; const pool = tier !== undefined ? ctx.selection?.e2e : Object.hasOwn(LLM_JUDGE_TOUCHFILES, id) ? ctx.selection?.judges : null; if (pool && !pool.includes(id)) return { kind: 'deselected', reason: DESELECTION_REASONS.selection }; return { kind: 'case', id }; @@ -241,7 +246,7 @@ export const CENSUS_RED_GUIDE = 'docs/evals/census-red.md'; export function sliceArtifactName(root: string, slice: number, attempt: number): string { const name = path.basename(root); - return /^(?:paid-slice|gate-census)-\d+-a\d+$/.test(name) ? name : `slice ${slice} attempt ${attempt}`; + return /^(?:paid-slice|gate-census)-\d+-a\d+(?:-[a-z0-9-]+)?$/.test(name) ? name : `slice ${slice} attempt ${attempt}`; } export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact[], recordsByShard: Map, @@ -254,7 +259,7 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact if (outcome.trial || outcome.files.length !== 1) continue; let xml = ''; try { xml = fs.readFileSync(path.join(root, 'shards', shardSlug([key]), 'junit.xml'), 'utf8'); } catch { continue; } - const ctx: JUnitShardContext = { key, tier: manifest.tier, selection: manifest.selection, excludeCases: entries.get(key)?.excludeCases }; + const ctx: JUnitShardContext = { key, tier: manifest.tier, selection: manifest.selection, excludeCases: entries.get(key)?.excludeCases, host: manifest.host }; const byCase = new Map(); const reasons = new Map(); for (const tc of parseJUnitCases(xml)) { @@ -530,7 +535,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo const results = artifacts.filter(a => (a.result.attempt ?? 1) === primary).map(a => a.result); const verdict = verifySliceResults(manifest, results); const planned = manifest.entries.filter((e) => e.status === 'planned').length; - const lane = `${manifest.tier}/${manifest.profile ?? 'full'}${manifest.evalsAll ? ' census' : ''}`; + const lane = `${manifest.tier}/${manifest.profile ?? 'full'}${manifest.evalsAll ? ' census' : ''}${manifest.host ? ` on ${manifest.host}` : ''}`; console.log(`[test:paid] report: ${results.length}/${manifest.sliceCount} slices, ${planned} planned shards, tier=${manifest.tier}, attempt ${primary}${attempts.length > 1 ? ` (later attempts ${attempts.slice(1).join(', ')} reported, never replacing it)` : ''}`); for (const line of formatProfileCoverage(manifest)) console.log(line); for (const result of [...results].sort((a, b) => a.sliceIndex - b.sliceIndex)) { diff --git a/scripts/test-paid-shards.ts b/scripts/test-paid-shards.ts index 8ddee5f11..19c9b2376 100644 --- a/scripts/test-paid-shards.ts +++ b/scripts/test-paid-shards.ts @@ -73,6 +73,7 @@ import { readSessionLedger, type SessionLedgerRow } from '../test/helpers/sessio import { preflightAnthropicApi } from '../test/helpers/anthropic-preflight'; import { e2eReuseLaneProblem, prepareE2EShardReuse, selectPlanReceipts } from './e2e-shard-reuse'; import { E2E_TOUCHFILES, E2E_TIERS } from '../test/helpers/touchfiles'; +import { resolveEvalHost } from '../test/helpers/e2e-hosts'; import { scopeCodexAccess, shardFile, shardCaseId, shardTrial, trialShardKey, type CaseTrialPlan, caseTrialPlan, excludedCasesNamePattern, caseTestNamePattern, expandCaseShards, expandTrialShards, fileCaseRegistration, partitionCaseExclusions } from './lib/paid-cases'; import { retriesForFiles, trialPanelKey, sliceExecutionOrder, buildRunManifest, parseRunManifest, type SliceResult, sliceExitCode, guardTrialRecords, formatSlicePlan, formatCapacityPreflight, sliceDeadlineMs } from './lib/paid-plan'; import { caseSelection, runCaseDiagnosis, formatPanelLine, runPaidReport } from './lib/paid-report'; @@ -909,6 +910,9 @@ async function main(): Promise { if (manifest.tier !== options.tier) { throw new Error(`manifest tier ${manifest.tier} != requested tier ${options.tier} — refusing a cross-tier run`); } + if ((manifest.host ?? 'claude') !== resolveEvalHost()) { + throw new Error(`manifest host ${manifest.host ?? 'claude'} != EVALS_HOST ${resolveEvalHost()} — refusing a cross-host run`); + } const profile = manifest.profile ?? 'full'; if (options.profileExplicit && options.profile !== profile) throw new Error(`manifest profile ${profile} != requested profile ${options.profile}`); if (options.sliceIndex > manifest.sliceCount) { diff --git a/test/ci-native-evidence.test.ts b/test/ci-native-evidence.test.ts index 318d5a9f3..c7da89696 100644 --- a/test/ci-native-evidence.test.ts +++ b/test/ci-native-evidence.test.ts @@ -3,6 +3,7 @@ import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { runPaidShard, shardSlug } from '../scripts/test-paid-shards'; +import { CENSUS_ARMS } from '../scripts/census-arms'; const ROOT = path.resolve(import.meta.dir, '..'); const workflows = ['evals.yml', 'evals-periodic.yml', 'evals-marathon.yml'].map(name => ({ @@ -52,19 +53,24 @@ test('every direct CI paid executor binds a safe unique run/attempt/job/slice id else expect(step.run).toContain('--user "$(id -u):$(id -g)"'); const env = executorEnv(job, step); expect(env.EVALS_RUN_ID).toBeString(); + // The gate census matrix is one row per census arm and planned slice (scripts/census-arms.ts). + const armed = job.strategy.matrix.include !== undefined; for (const run of ['36302678692', '36302678693']) { for (const attempt of ['1', '2']) { // The planner sizes the matrix; cover more slices than any live plan. - expect(job.strategy.matrix.slice).toMatch(/^\$\{\{ fromJSON\(needs\.plan-slices\.outputs\.(?:[a-z]+_)*slices\) \}\}$/); - for (let slice = 1; slice <= 64; slice++) { - const id = render(env.EVALS_RUN_ID, { - 'github.run_id': `${run}${workflowIndex}`, - 'github.run_attempt': attempt, 'matrix.slice': String(slice), - }); - expect(id).toMatch(/^[A-Za-z0-9_-]+$/); - expect(id.length).toBeLessThan(120); - expect(ids.has(id)).toBe(false); - ids.add(id); + expect(armed ? job.strategy.matrix.include : job.strategy.matrix.slice) + .toMatch(armed ? /^\$\{\{ fromJSON\(needs\.plan-slices\.outputs\.gate_matrix\) \}\}$/ : /^\$\{\{ fromJSON\(needs\.plan-slices\.outputs\.(?:[a-z]+_)*slices\) \}\}$/); + for (const arm of armed ? Object.keys(CENSUS_ARMS) : ['']) { + for (let slice = 1; slice <= 64; slice++) { + const id = render(env.EVALS_RUN_ID, { + 'github.run_id': `${run}${workflowIndex}`, + 'github.run_attempt': attempt, 'matrix.slice': String(slice), 'matrix.arm': arm, + }); + expect(id).toMatch(/^[A-Za-z0-9_-]+$/); + expect(id.length).toBeLessThan(120); + expect(ids.has(id)).toBe(false); + ids.add(id); + } } } } @@ -79,7 +85,7 @@ for (const { name, jobName, job, step } of executors) { fs.mkdirSync(home); fs.mkdirSync(bin); fs.mkdirSync(path.join(root, 'test')); const configured = executorEnv(job, step); const runId = render(configured.EVALS_RUN_ID, { - 'github.run_id': '36302678692', 'github.run_attempt': '2', 'matrix.slice': '4', + 'github.run_id': '36302678692', 'github.run_attempt': '2', 'matrix.slice': '4', 'matrix.arm': 'codex-sol', }); const evalDir = path.join(root, path.basename(configured.GSTACK_EVAL_DIR)); const file = 'test/native-launch.test.ts'; diff --git a/test/ci-paid-coordination.test.ts b/test/ci-paid-coordination.test.ts index b92375e11..f248d76a9 100644 --- a/test/ci-paid-coordination.test.ts +++ b/test/ci-paid-coordination.test.ts @@ -113,7 +113,8 @@ describe('paid CI coordination stays off the eval image', () => { // A confirmed verdict reuse (R2a/R2) is the one PR run that executes no slice. expect(executor.if).toBe("${{ !cancelled() && needs.build-image.result == 'success' && needs.plan-slices.result == 'success' && needs.plan-slices.outputs.gate_reuse != 'true' }}"); } else { - expect(executor.if).toBeUndefined(); + // A census-only run (monthly cron or census_arms dispatch) executes no periodic slice. + expect(executor.if).toBe("${{ needs.plan-slices.outputs.periodic_lane == 'true' }}"); } expect(executor.steps.some(step => step.run === 'bun run build')).toBe(true); expect(executor.steps.some(step => step.uses === './.github/actions/restore-deps')).toBe(true); diff --git a/test/codex-host.test.ts b/test/codex-host.test.ts new file mode 100644 index 000000000..0976a72e6 --- /dev/null +++ b/test/codex-host.test.ts @@ -0,0 +1,287 @@ +/** + * The Codex host for paid gate cases (GSTA-49 TM2): host selection with + * skip-with-reason (never a failure), per-case host metadata over every gate + * case, real Codex cost accounting against captured `codex exec --json` + * usage events (Codex CLI 0.160.0, gpt-6.1-sol), and runner parity. + */ +import { afterEach, describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { CLAUDE_ONLY_REASONS, E2E_HOSTS, hostSkipReason, resolveEvalHost } from './helpers/e2e-hosts'; +import { E2E_TIERS, E2E_TOUCHFILES } from './helpers/touchfiles'; +import { addCodexTurnUsage, codexUsageCostUsd, NO_CODEX_USAGE, parseCodexJSONL, type CodexResult } from './helpers/codex-session-runner'; +import { runRecordedCodexEval } from './helpers/codex-eval'; +import { codexHostExecArgs, codexHostModel, codexHostToolCalls, codexHostTranscript, unwrapShellCommand } from './helpers/codex-host-runner'; +import { coverageAuditReadEvidence } from './helpers/coverage-audit-evidence'; +import { runSkillTest } from './helpers/session-runner'; +import { readSessionLedger } from './helpers/session-ledger'; +import { preflightAnthropicApi } from './helpers/anthropic-preflight'; +import type { EvalTestEntry } from './helpers/eval-store'; +import { buildRunManifest } from '../scripts/test-paid-shards'; +import { codexShardAccess, fileCaseRegistration, partitionHostExclusions, scopeCodexAccess, shardCaseId, shardFile } from '../scripts/lib/paid-cases'; +import { parseRunManifest } from '../scripts/lib/paid-plan'; +import { attributeJUnitCase } from '../scripts/lib/paid-report'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const fixture = (name: string) => fs.readFileSync(path.join(ROOT, 'test/fixtures/codex-host', name), 'utf8').split('\n').filter(Boolean); +const COMMANDS = fixture('exec-commands.jsonl'); +const FILE_CHANGE = fixture('exec-file-change.jsonl'); +const gateCases = Object.keys(E2E_TIERS).filter(id => E2E_TIERS[id] === 'gate').sort(); +const adapterCases = Object.keys(E2E_HOSTS).filter(id => E2E_HOSTS[id] === 'both'); +const scratch: string[] = []; +const tempDir = (prefix: string) => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); scratch.push(dir); return dir; }; +afterEach(() => { for (const dir of scratch.splice(0)) fs.rmSync(dir, { recursive: true, force: true }); }); + +describe('host selection: Claude-only cases skip with a reason, never fail', () => { + test('EVALS_HOST defaults to claude, accepts codex and refuses anything else', () => { + expect(resolveEvalHost({})).toBe('claude'); + expect(resolveEvalHost({ EVALS_HOST: ' ' })).toBe('claude'); + expect(resolveEvalHost({ EVALS_HOST: 'codex' })).toBe('codex'); + expect(() => resolveEvalHost({ EVALS_HOST: 'gemini' })).toThrow('EVALS_HOST must be one of claude, codex'); + }); + + test('the Anthropic preflight never pings claude on the Codex host', () => { + const pinged: string[] = []; + const spawn = ((cmd: string) => { pinged.push(cmd); return { status: 127, stdout: '', stderr: '' }; }) as unknown as typeof spawnSync; + expect(preflightAnthropicApi({ EVALS_HOST: 'codex' }, spawn)).toBe('skipped'); + expect(pinged).toEqual([]); + expect(() => preflightAnthropicApi({}, spawn)).toThrow('`claude` not found on PATH'); + expect(pinged).toEqual(['sh']); + }); + + test('every case runs on Claude; on Codex only adapter cases run and the rest carry their reason', () => { + for (const id of gateCases) expect(hostSkipReason(id, 'claude')).toBeNull(); + for (const id of adapterCases) expect(hostSkipReason(id, 'codex')).toBeNull(); + expect(hostSkipReason('plan-mode-no-op', 'codex')).toBe(`Claude-only on the codex host: ${CLAUDE_ONLY_REASONS.pty}`); + expect(hostSkipReason('ship-docsync-store', 'codex')).toContain('not ported to the Codex host'); + expect(hostSkipReason('plan-ceo-review', 'codex')).toContain('no Codex host metadata'); + }); + + test('the module selection drops Claude-only cases on the Codex host', () => { + const root = tempDir('codex-host-root-'); + fs.mkdirSync(path.join(root, 'home', '.codex'), { recursive: true }); + const run = spawnSync('bun', ['-e', "const h = await import('./test/helpers/e2e-helpers'); console.log(JSON.stringify(h.selectedTests))"], { + cwd: ROOT, encoding: 'utf8', timeout: 60_000, + env: { ...process.env, EVALS: '1', EVALS_TIER: 'gate', EVALS_HOST: 'codex', EVALS_PREFLIGHT_OK: '1', GSTACK_CODEX_HOST_ROOT: root, + EVALS_SELECTION_JSON: JSON.stringify({ selected: null, reason: 'test' }) }, + }); + expect(run.status, run.stderr).toBe(0); + expect((JSON.parse(run.stdout.trim().split('\n').at(-1)!) as string[]).sort()).toEqual([...adapterCases].sort()); + expect(run.stderr).toContain(`EVALS_HOST=codex: ${adapterCases.length} tests; ${gateCases.length - adapterCases.length} Claude-only skipped`); + }); + + test('the planner excludes Claude-only shards with their reason and plans every adapter case', () => { + const manifest = buildRunManifest({ tier: 'gate', profile: 'full', sliceBudgetMs: 420_000, jobs: 2, evalsAll: true, skipJudges: true, + env: { EVALS_ALL: '1', EVALS_HOST: 'codex' }, rootDir: ROOT }); + expect(manifest.host).toBe('codex'); + const planned = manifest.entries.filter(entry => entry.status === 'planned'); + const casesOf = (key: string) => { + const id = shardCaseId(key); + if (id !== null) return [id]; + return fileCaseRegistration(shardFile(key), fs.readFileSync(path.join(ROOT, shardFile(key)), 'utf8')).registered.filter(c => E2E_TIERS[c] === 'gate'); + }; + const reachable = new Set(planned.flatMap(entry => casesOf(entry.file)).filter(id => E2E_HOSTS[id] === 'both')); + expect([...reachable].sort()).toEqual([...adapterCases].sort()); + for (const entry of planned) expect(casesOf(entry.file).some(id => E2E_HOSTS[id] === 'both'), entry.file).toBe(true); + const hostExcluded = manifest.entries.filter(entry => entry.status === 'excluded' && entry.reason?.includes('Claude-only on the codex host')); + expect(hostExcluded.length).toBeGreaterThan(0); + for (const entry of hostExcluded) expect(entry.reason, entry.file).toStartWith('skipped: '); + expect(manifest.entries.some(entry => entry.file === 'test/skill-e2e-ship-docsync.test.ts#ship-docsync-store' && entry.status === 'excluded')).toBe(true); + }); + + test('the Claude host plans exactly as before, with no host on the manifest', () => { + const manifest = buildRunManifest({ tier: 'gate', profile: 'full', sliceBudgetMs: 420_000, jobs: 2, evalsAll: true, skipJudges: true, + env: { EVALS_ALL: '1' }, rootDir: ROOT }); + expect(manifest.host).toBeUndefined(); + expect(manifest.entries.some(entry => entry.reason?.includes('Claude-only'))).toBe(false); + expect(partitionHostExclusions(['a.test.ts', 'b.test.ts#x'], 'gate', 'claude')).toEqual({ runnable: ['a.test.ts', 'b.test.ts#x'], excluded: [] }); + }); + + test('the report deselects a skipped Claude-only testcase on the Codex host (zero credit, never SKIPPED or failed)', () => { + const tc = { name: 'review-dashboard-via', classname: '', outcome: 'skipped' as const, timeMs: 0, line: 10 }; + const key = 'test/skill-e2e-review-attribution.test.ts'; + expect(attributeJUnitCase(tc, { key, tier: 'gate', host: 'codex' })).toEqual({ kind: 'deselected', reason: hostSkipReason('review-dashboard-via', 'codex')! }); + expect(attributeJUnitCase(tc, { key, tier: 'gate' })).toEqual({ kind: 'case', id: 'review-dashboard-via' }); + expect(attributeJUnitCase({ ...tc, name: 'review-base-branch' }, { key, tier: 'gate', host: 'codex' })).toEqual({ kind: 'case', id: 'review-base-branch' }); + }); + + test('a manifest names only a non-default host, and the executor refuses a cross-host run', () => { + const base = { version: 1, tier: 'gate', evalsAll: true, sliceCount: 1, selectionReason: 'x', entries: [] }; + expect(parseRunManifest(JSON.stringify({ ...base, host: 'codex' })).host).toBe('codex'); + expect(() => parseRunManifest(JSON.stringify({ ...base, host: 'claude' }))).toThrow('manifest host invalid'); + expect(() => parseRunManifest(JSON.stringify({ ...base, host: 'gemini' }))).toThrow('manifest host invalid'); + const plan = path.join(tempDir('codex-host-plan-'), 'manifest.json'); + fs.writeFileSync(plan, JSON.stringify({ ...base, host: 'codex' })); + const run = spawnSync('bun', ['run', 'scripts/test-paid-shards.ts', '--tier', 'gate', '--plan', plan, '--slice', '1', '--list'], { + cwd: ROOT, encoding: 'utf8', timeout: 60_000, env: { ...process.env, EVALS_HOST: 'claude', EVALS_TIER: 'gate' }, + }); + expect(run.status).not.toBe(0); + expect(run.stderr + run.stdout).toContain('manifest host codex != EVALS_HOST claude'); + }); +}); + +describe('per-case host metadata', () => { + test('covers every gate case exactly once, and nothing else', () => { + expect(Object.keys(E2E_HOSTS).sort()).toEqual(gateCases); + for (const id of Object.keys(E2E_HOSTS)) expect(Object.hasOwn(E2E_TOUCHFILES, id), id).toBe(true); + }); + + test('classifies 52 adapter-only, 29 Claude-specific, 26 rework and 1 host-neutral case', () => { + const counts: Record = {}; + for (const support of Object.values(E2E_HOSTS)) counts[support] = (counts[support] ?? 0) + 1; + expect(counts.both).toBe(52); + expect(counts.rework).toBe(26); + expect(counts['host-neutral']).toBe(1); + expect(['pty', 'sdk', 'subagents', 'skill-tool', 'hermetic', 'self-host'].reduce((sum, reason) => sum + (counts[reason] ?? 0), 0)).toBe(29); + }); +}); + +describe('Codex cost accounting from captured usage events', () => { + test('parses every token class of turn.completed (cache reads and writes are subsets of input)', () => { + const commands = parseCodexJSONL(COMMANDS); + expect(commands.usage).toEqual({ input: 25009, cached_input: 12437, cache_write_input: 12566, output: 86, reasoning_output: 0 }); + expect(commands.tokens).toBe(25095); + expect(parseCodexJSONL(FILE_CHANGE).usage).toEqual({ input: 25020, cached_input: 12440, cache_write_input: 12574, output: 137, reasoning_output: 0 }); + expect(parseCodexJSONL(['{"type":"thread.started","thread_id":"t"}']).usage).toBeNull(); + const twoTurns = [COMMANDS, FILE_CHANGE].map(lines => JSON.parse(lines.find(line => line.includes('turn.completed'))!).usage) + .reduce(addCodexTurnUsage, NO_CODEX_USAGE); + expect(twoTurns).toEqual({ input: 50029, cached_input: 24877, cache_write_input: 25140, output: 223, reasoning_output: 0 }); + }); + + test('prices with lib/pricing.ts: uncached input (cache writes included) at input, cache reads at the cached rate', () => { + const usage = parseCodexJSONL(COMMANDS).usage!; + // gpt-6.1-sol: 12,572 x $2 + 12,437 x $0.10 + 86 x $10 per million. + expect(codexUsageCostUsd(usage, 'gpt-6.1-sol')).toBe(0.027248); + expect(codexUsageCostUsd(parseCodexJSONL(FILE_CHANGE).usage!, 'gpt-6.1-sol')).toBe(0.027774); + // gpt-6-astra: 12,572 x $10 + 12,437 x $1 + 86 x $50 per million. + expect(codexUsageCostUsd(usage, 'gpt-6-astra')).toBe(0.142457); + expect(codexUsageCostUsd(usage, 'gpt-0-unpriced')).toBeNull(); + }); + + test('runRecordedCodexEval records real cost and billing instead of cost_known: false', async () => { + const evalDir = tempDir('codex-host-ledger-'); + const previous = process.env.GSTACK_EVAL_DIR; + process.env.GSTACK_EVAL_DIR = evalDir; + try { + const recorded: EvalTestEntry[] = []; + const result: CodexResult = { ...parseCodexJSONL(COMMANDS), exitCode: 0, durationMs: 5, rawLines: COMMANDS, stderr: '', model: 'gpt-6.1-sol' }; + await runRecordedCodexEval({ name: 'cost-case', suite: 's', budgetMs: 10_000, run: async () => result, validate: () => {}, record: entry => recorded.push(entry) }); + expect(recorded[0]).toMatchObject({ passed: true, cost_usd: 0.027248, model: 'gpt-6.1-sol', token_usage: result.usage }); + expect(recorded[0]!.cost_known).toBeUndefined(); + const noTurn: CodexResult = { ...parseCodexJSONL([]), exitCode: 0, durationMs: 5, rawLines: [], stderr: '' }; + await runRecordedCodexEval({ name: 'no-turn', suite: 's', budgetMs: 10_000, run: async () => noTurn, validate: () => {}, record: entry => recorded.push(entry) }); + expect(recorded[1]).toMatchObject({ cost_usd: 0, cost_known: false }); + const rows = readSessionLedger(evalDir); + expect(rows.map(row => [row.test_name, row.billed])).toEqual([['cost-case', true], ['no-turn', false]]); + expect(rows[0]).toMatchObject({ runner: 'codex', model: 'gpt-6.1-sol', cost_usd: 0.027248, tokens: result.usage }); + } finally { + if (previous === undefined) delete process.env.GSTACK_EVAL_DIR; else process.env.GSTACK_EVAL_DIR = previous; + } + }); +}); + +describe('Codex runner parity', () => { + test('maps command_execution to Bash (unwrapped, with output) and file_change to Write/Edit', () => { + expect(codexHostToolCalls(COMMANDS)).toEqual([ + { tool: 'Bash', input: { command: "printf 'hi' > hello.txt\nprintf 'hello' > hello.txt\ncat hello.txt" }, output: 'hello' }, + ]); + expect(codexHostToolCalls(FILE_CHANGE)).toEqual([ + { tool: 'Write', input: { file_path: '/tmp/codex-host-fixture/notes.md', kind: 'add' }, output: '' }, + { tool: 'Edit', input: { file_path: '/tmp/codex-host-fixture/notes.md', kind: 'update' }, output: '' }, + ]); + }); + + test('the transcript is the Claude stream-json shape the case observers read', () => { + const output = '`cat hello.txt` output: `hello`'; + const transcript = codexHostTranscript(COMMANDS, '/tmp/codex-host-fixture', output); + expect(transcript[0]).toEqual({ type: 'system', subtype: 'init', cwd: '/tmp/codex-host-fixture', session_id: '01a150b3-ae5a-7932-ba51-4c295c66adf4', tools: [] }); + expect(transcript.slice(1).map(event => [event.type, event.message?.content[0].type ?? event.subtype])).toEqual([ + ['assistant', 'text'], ['assistant', 'tool_use'], ['user', 'tool_result'], ['assistant', 'text'], ['result', 'success']]); + expect(transcript[2].message.content[0]).toEqual({ type: 'tool_use', id: 'item_1', name: 'Bash', input: { command: "printf 'hi' > hello.txt\nprintf 'hello' > hello.txt\ncat hello.txt" } }); + expect(transcript[3].message.content[0]).toEqual({ type: 'tool_result', tool_use_id: 'item_1', content: 'hello', is_error: false }); + expect(transcript.at(-1)).toMatchObject({ type: 'result', result: output }); + const edits = codexHostTranscript(FILE_CHANGE, '/tmp/codex-host-fixture', '').filter(event => event.type === 'assistant' && event.message.content[0].type === 'tool_use'); + expect(edits.map(event => [event.message.content[0].name, event.message.content[0].input.file_path])).toEqual([ + ['Write', '/tmp/codex-host-fixture/notes.md'], ['Edit', '/tmp/codex-host-fixture/notes.md']]); + // A Codex `cat` of a fixture file is a read the coverage-audit evidence credits, as a native Bash read is. + const source = { path: '/w/src/billing.ts', content: 'export const a = 1;\n' }; + const tests = { path: '/w/test/billing.test.ts', content: 'test("a", () => {});\n' }; + const read = (id: string, file: string, out: string) => JSON.stringify({ type: 'item.completed', item: { id, type: 'command_execution', command: `/bin/bash -lc 'cat ${file}'`, aggregated_output: out, exit_code: 0, status: 'completed' } }); + const lines = ['{"type":"thread.started","thread_id":"t1"}', read('item_0', 'src/billing.ts', source.content), read('item_1', 'test/billing.test.ts', tests.content)]; + expect(coverageAuditReadEvidence(codexHostTranscript(lines, '/w', ''), { cwd: '/w', source, tests })).toEqual({ sourceRead: true, testsRead: true }); + const failed = lines.map(line => line.replace('"exit_code":0', '"exit_code":1')); + expect(coverageAuditReadEvidence(codexHostTranscript(failed, '/w', ''), { cwd: '/w', source, tests })).toEqual({ sourceRead: false, testsRead: false }); + }); + + test('unwraps only a single well-formed bash -lc word', () => { + expect(unwrapShellCommand("/bin/bash -lc 'git diff main...HEAD'")).toBe('git diff main...HEAD'); + expect(unwrapShellCommand("/bin/bash -lc 'echo '\"'\"'x'\"'\"''")).toBe("echo 'x'"); + expect(unwrapShellCommand('/bin/bash -lc "echo \\"\\$HOME\\""')).toBe('echo "$HOME"'); + expect(unwrapShellCommand('bash -lc ls')).toBe('ls'); + expect(unwrapShellCommand('git status')).toBe('git status'); + expect(unwrapShellCommand("/bin/bash -lc 'unterminated")).toBe("/bin/bash -lc 'unterminated"); + expect(unwrapShellCommand("/bin/bash -lc 'a' 'b'")).toBe("/bin/bash -lc 'a' 'b'"); + }); + + test('sandbox, effort and model settings', () => { + expect(codexHostExecArgs('gpt-6.1-sol', 'go', {})).toEqual(['exec', '--json', '-s', 'danger-full-access', '--skip-git-repo-check', '--model', 'gpt-6.1-sol', 'go']); + expect(codexHostExecArgs('gpt-6.1-sol', 'go', { GSTACK_CODEX_EFFORT: 'high' })).toContain('model_reasoning_effort="high"'); + expect(codexHostExecArgs('m', 'go', { GSTACK_CODEX_SANDBOX: 'workspace-write' })).toEqual( + ['exec', '--json', '-s', 'workspace-write', '--skip-git-repo-check', '--model', 'm', '-c', 'sandbox_workspace_write.network_access=true', 'go']); + expect(() => codexHostExecArgs('m', 'go', { GSTACK_CODEX_SANDBOX: 'none' })).toThrow('GSTACK_CODEX_SANDBOX'); + expect(() => codexHostExecArgs('m', 'go', { GSTACK_CODEX_EFFORT: 'high" -c x="y' })).toThrow('GSTACK_CODEX_EFFORT'); + expect(codexHostModel('claude-opus-4-7', { GSTACK_CODEX_MODEL: 'gpt-6.1-sol' })).toBe('gpt-6.1-sol'); + expect(codexHostModel('gpt-6-astra', { GSTACK_CODEX_MODEL: 'gpt-6.1-sol' })).toBe('gpt-6-astra'); + expect(codexHostModel(undefined, {})).toBe('gpt-6-astra'); + }); + + test('every shard on the Codex host gets the CI Codex install', () => { + expect(codexShardAccess(['test/skill-e2e-review.test.ts'], 'gate')).toBe('none'); + expect(codexShardAccess(['test/skill-e2e-review.test.ts'], 'gate', 'codex')).toBe('path'); + const env: NodeJS.ProcessEnv = { EVALS_HOST: 'codex', EVALS_TIER: 'gate', PATH: '/usr/bin', GSTACK_CI_CODEX_BIN_DIR: '/opt/codex/bin', GSTACK_CI_CODEX_HOME: '/home/runner/.ci-codex' }; + scopeCodexAccess(env, ['test/skill-e2e-review.test.ts']); + expect(env.PATH).toBe(`/opt/codex/bin${path.delimiter}/usr/bin`); + expect(env.CODEX_HOME).toBe('/home/runner/.ci-codex'); + }); + + test('runSkillTest on EVALS_HOST=codex runs codex exec and returns a priced SkillTestResult', async () => { + const dir = tempDir('codex-host-run-'); + const root = path.join(dir, 'runtime'); + fs.mkdirSync(path.join(root, 'home', '.codex'), { recursive: true }); + const bin = path.join(dir, 'bin'); + fs.mkdirSync(bin); + const argsFile = path.join(dir, 'args.txt'); + fs.writeFileSync(path.join(bin, 'codex'), `#!/bin/sh\nprintf '%s\\n' "$@" > '${argsFile}'\nprintf '%s\\n' "$CODEX_HOME" >> '${argsFile}'\ncat '${path.join(ROOT, 'test/fixtures/codex-host/exec-commands.jsonl')}'\n`, { mode: 0o755 }); + const evalDir = path.join(dir, 'evals'); + const saved = { ...process.env }; + Object.assign(process.env, { EVALS_HOST: 'codex', GSTACK_CODEX_MODEL: 'gpt-6.1-sol', GSTACK_CODEX_HOST_ROOT: root, GSTACK_EVAL_DIR: evalDir, PATH: `${bin}${path.delimiter}${process.env.PATH}` }); + try { + const result = await runSkillTest({ prompt: 'Run the fixture.', workingDirectory: dir, model: 'claude-opus-4-7', timeout: 30_000, testName: 'host-switch' }); + expect(result).toMatchObject({ exitReason: 'success', model: 'gpt-6.1-sol', output: '`cat hello.txt` output: `hello`', browseErrors: [] }); + expect(result.toolCalls.map(call => call.input.command)).toEqual(["printf 'hi' > hello.txt\nprintf 'hello' > hello.txt\ncat hello.txt"]); + expect(result.costEstimate).toMatchObject({ estimatedCost: 0.027248, estimatedTokens: 25095, turnsUsed: 3 }); + expect(result.tokenUsage).toEqual(parseCodexJSONL(COMMANDS).usage!); + const args = fs.readFileSync(argsFile, 'utf8').trim().split('\n'); + expect(args.slice(0, 7)).toEqual(['exec', '--json', '-s', 'danger-full-access', '--skip-git-repo-check', '--model', 'gpt-6.1-sol']); + expect(args.at(-2)).toBe('Run the fixture.'); + expect(args.at(-1)).toBe(path.join(root, 'home', '.codex')); + expect(readSessionLedger(evalDir)).toEqual([expect.objectContaining({ runner: 'codex', test_name: 'host-switch', end: 'completed', billed: true, cost_usd: 0.027248 })]); + // The native lifecycle sees the codex group owner at spawn and settles after its exit. + const seen: string[] = []; + await runSkillTest({ prompt: 'Run the fixture.', workingDirectory: dir, timeout: 30_000, testName: 'host-lifecycle', nativeLifecycle: { + onSpawn: pid => { seen.push(`spawn ${pid > 0}`); }, + onSettled: async ({ deadline, exited }) => { seen.push(`settled ${exited} ${deadline > Date.now()}`); }, + } }); + expect(seen).toEqual(['spawn true', 'settled true true']); + await expect(runSkillTest({ prompt: 'Run the fixture.', workingDirectory: dir, timeout: 30_000, testName: 'host-lifecycle-refused', nativeLifecycle: { + onSpawn: () => { throw new Error('native process is not group owner'); }, + onSettled: async () => {}, + } })).rejects.toThrow('native process is not group owner'); + } finally { + for (const key of Object.keys(process.env)) if (!(key in saved)) delete process.env[key]; + Object.assign(process.env, saved); + } + }); +}); diff --git a/test/evals-workflow-wiring.test.ts b/test/evals-workflow-wiring.test.ts index 6cfc3217a..013fc621c 100644 --- a/test/evals-workflow-wiring.test.ts +++ b/test/evals-workflow-wiring.test.ts @@ -22,7 +22,10 @@ import { describe, test, expect } from 'bun:test'; import * as fs from 'fs'; import * as path from 'path'; +import * as os from 'os'; +import { spawnSync } from 'child_process'; import { buildRunManifest, isCodexShard, isOverlayTestFile, parseCliOptions, resolvePaidShardTimeoutMs, sliceExecutionOrder, sliceSupervisedWallMs, CI_SETUP_ALLOWANCE_MINUTES } from '../scripts/test-paid-shards'; +import { CENSUS_ARMS, MONTHLY_CENSUS_ARMS, MONTHLY_CENSUS_SCHEDULE, WEEKLY_CENSUS_ARMS, WEEKLY_CENSUS_SCHEDULE, censusArmEnv, censusMatrix, selectCensus } from '../scripts/census-arms'; const ROOT = path.join(import.meta.dir, '..'); const read = (rel: string) => fs.readFileSync(path.join(ROOT, rel), 'utf-8'); @@ -40,7 +43,7 @@ function plannerSites(source: string): Array<{ manifest: string; budgetSeconds: type Step = { id?: string; name?: string; run?: string; uses?: string; env?: Record; with?: Record }; type Job = { needs?: string[]; env?: Record; outputs?: Record; 'timeout-minutes': string | number; - strategy?: { 'max-parallel': number; matrix: { slice: string } }; steps: Step[] }; + strategy?: { 'max-parallel': number; matrix: { slice?: string; include?: string } }; steps: Step[] }; /** * An executor's matrix and timeout must come from the planner step that wrote @@ -159,7 +162,6 @@ describe('evals.yml sliced-lane wiring (post-matrix)', () => { describe('evals-periodic.yml sliced-lane wiring', () => { const lanes = [ { source: periodicYml, name: 'evals-periodic.yml', executor: 'eval-slices', prefix: 'periodic_', tier: 'periodic' }, - { source: periodicYml, name: 'evals-periodic.yml', executor: 'gate-census', prefix: 'gate_', tier: 'gate' }, { source: evalsYml, name: 'evals.yml', executor: 'eval-slices', prefix: '', tier: 'gate' }, { source: marathonYml, name: 'evals-marathon.yml', executor: 'eval-slices', prefix: '', tier: 'marathon' }, ] as const; @@ -200,6 +202,64 @@ describe('evals-periodic.yml sliced-lane wiring', () => { }); } + test('evals-periodic.yml:gate-census — one row per census arm and planned slice with its plan\'s ceiling, and every row of a scheduled census starts at once', () => { + const workflow = Bun.YAML.parse(periodicYml) as { env?: Record; jobs: Record }; + const planner = workflow.jobs['plan-slices']!; + const executor = workflow.jobs['gate-census']!; + expect(executor.needs).toContain('plan-slices'); + expect(executor.strategy!.matrix).toEqual({ include: '${{ fromJSON(needs.plan-slices.outputs.gate_matrix) }}' }); + expect(executor['timeout-minutes']).toBe('${{ matrix.timeout }}'); + expect(executor.steps[0]!.run).toBe('echo "GSTACK_SLICE_JOB_STARTED_AT=$(date +%s)" >> "$GITHUB_ENV"'); + expect(planner.outputs!.gate_matrix).toBe('${{ steps.gate-matrix.outputs.gate_matrix }}'); + expect(planner.steps.find(step => step.id === 'gate-matrix')!.run).toBe('bun --no-install run scripts/census-arms.ts --matrix /tmp/gate-census-plan "$ARMS" >> "$GITHUB_OUTPUT"'); + const emit = planner.steps.filter(step => step.run?.includes('--emit-plan "/tmp/gate-census-plan/$arm/manifest.json" ')); + expect(emit).toHaveLength(1); + expect(emit[0]!.run).toContain('mapfile -t arm_env < <(bun --no-install run scripts/census-arms.ts --env "$arm")'); + expect(emit[0]!.run).toContain('env "${arm_env[@]}" EVALS_TIER=gate bun run scripts/test-paid-shards.ts'); + const execute = executor.steps.filter(step => step.run?.includes('--plan ')); + expect(execute).toHaveLength(1); + expect(execute[0]!.run).toContain('--plan /tmp/gate-census-plan/${{ matrix.arm }}/manifest.json --slice ${{ matrix.slice }}'); + // The executor runs with its arm's host and model, and downloads every arm's plan. + expect(executor.steps.find(step => step.name === "Select the census arm's host and model")!.run).toBe('bun --no-install run scripts/census-arms.ts --env "$ARM" >> "$GITHUB_ENV"'); + expect(executor.steps.some(step => step.with?.path === '/tmp/gate-census-plan')).toBe(true); + const cliArgs = (run: string) => { + const command = /\bbun(?: --no-install)? run scripts\/test-paid-shards\.ts (.*)$/m.exec(run)!; + return command[1]!.replace(/\$\{\{\s*matrix\.slice\s*\}\}/g, '1').replace(/\$\{\{\s*matrix\.arm\s*\}\}/g, 'claude-opus').trim().split(/\s+/); + }; + const planned = parseCliOptions(cliArgs(emit[0]!.run!), { ...workflow.env, ...emit[0]!.env, EVALS_ALL: '1', EVALS_PROFILE: 'full' }); + const active = parseCliOptions(cliArgs(execute[0]!.run!), { ...workflow.env, ...executor.env, ...execute[0]!.env, EVALS_PROFILE: 'full' }); + expect(planned.tier).toBe('gate'); + expect(active.tier).toBe('gate'); + expect(active.jobs).toBe(planned.jobs); + expect(execute[0]!.env?.EVALS_JOBS).toBe(String(planned.jobs)); + const planDir = fs.mkdtempSync(path.join(os.tmpdir(), 'census-arms-')); + try { + for (const arm of Object.keys(CENSUS_ARMS)) { + const env = { ...workflow.env, ...emit[0]!.env, EVALS_ALL: '1', EVALS_PROFILE: 'full', ...censusArmEnv(arm) }; + const manifest = buildRunManifest({ tier: 'gate', profile: 'full', sliceBudgetMs: planned.sliceBudgetMs!, jobs: planned.jobs, + evalsAll: true, env, rootDir: ROOT, skipJudges: planned.skipJudges }); + expect(manifest.host ?? 'claude').toBe(CENSUS_ARMS[arm]!.host); + const sliceFiles = Array.from({ length: manifest.sliceCount }, (_, i) => sliceExecutionOrder( + manifest.entries.filter(entry => entry.status === 'planned' && entry.slice === i + 1)).map(entry => entry.file)); + const ceilings = sliceFiles.map(files => Math.ceil(Math.max(2 * planned.sliceBudgetMs!, + ...files.map(file => resolvePaidShardTimeoutMs([file])), + files.some(isOverlayTestFile) ? sliceSupervisedWallMs(files, planned.jobs) : 0) / 60_000) + CI_SETUP_ALLOWANCE_MINUTES); + expect(manifest.plan!.sliceCiTimeoutMinutes).toEqual(ceilings); + expect(manifest.plan!.ciTimeoutMinutes).toBeLessThanOrEqual(360); + fs.mkdirSync(path.join(planDir, arm)); + fs.writeFileSync(path.join(planDir, arm, 'manifest.json'), JSON.stringify(manifest)); + expect(censusMatrix([arm], planDir)).toEqual(ceilings.map((timeout, i) => ({ arm, slice: i + 1, timeout }))); + } + for (const arms of [WEEKLY_CENSUS_ARMS, MONTHLY_CENSUS_ARMS]) { + const rows = censusMatrix(arms, planDir); + expect(new Set(rows.map(row => `${row.arm}/${row.slice}`)).size).toBe(rows.length); + expect(rows.length, `${arms.join('+')} rows exceed max-parallel`).toBeLessThanOrEqual(executor.strategy!['max-parallel']); + } + } finally { + fs.rmSync(planDir, { recursive: true, force: true }); + } + }); + test('planner/executor/report tier=periodic agree and plan with the ~7-minute budget (W5c)', () => { expect(periodicYml).toMatch(/EVALS_TIER=periodic bun --no-install run scripts\/test-paid-shards\.ts --tier periodic --emit-plan/); expect(periodicYml).toMatch(/EVALS_TIER=periodic bun run scripts\/test-paid-shards\.ts --tier periodic --plan .* --slice /); @@ -207,7 +267,7 @@ describe('evals-periodic.yml sliced-lane wiring', () => { // Periodic work and the full gate census have distinct immutable plans. expect(plannerSites(periodicYml)).toEqual([ { manifest: '/tmp/paid-plan/manifest.json', budgetSeconds: 420, jobs: 2 }, - { manifest: '/tmp/gate-census-plan/manifest.json', budgetSeconds: 420, jobs: 2 }, + { manifest: '"/tmp/gate-census-plan/$arm/manifest.json"', budgetSeconds: 420, jobs: 2 }, ]); }); }); @@ -228,7 +288,7 @@ describe('evals-periodic.yml host-run Codex slices', () => { expect(matrixStep.run).toContain('} >> "$GITHUB_OUTPUT"'); expect(planner.outputs!.periodic_codex_slices).toBe('${{ steps.periodic-matrix.outputs.codex_slices }}'); expect(codex.needs).toEqual(['build-image', 'plan-slices']); - expect(codex.if).toBe("${{ needs.plan-slices.outputs.periodic_codex_slices != '[]' }}"); + expect(codex.if).toBe("${{ needs.plan-slices.outputs.periodic_lane == 'true' && needs.plan-slices.outputs.periodic_codex_slices != '[]' }}"); expect(codex.strategy!.matrix.slice).toBe('${{ fromJSON(needs.plan-slices.outputs.periodic_codex_slices) }}'); expect(codex['timeout-minutes']).toBe('${{ fromJSON(needs.plan-slices.outputs.periodic_slice_timeouts)[matrix.slice] }}'); expect(codex.steps[0]!.run).toBe('echo "GSTACK_SLICE_JOB_STARTED_AT=$(date +%s)" >> "$GITHUB_ENV"'); @@ -350,7 +410,7 @@ describe('panel verdict surfaces (eval reliability policy)', () => { test('planners size the capacity preflight with their executor cap', () => { for (const [source, executor, manifest] of [[evalsYml, 'eval-slices', '/tmp/paid-plan/manifest.json'], - [periodicYml, 'eval-slices', '/tmp/paid-plan/manifest.json'], [periodicYml, 'gate-census', '/tmp/gate-census-plan/manifest.json']] as const) { + [periodicYml, 'eval-slices', '/tmp/paid-plan/manifest.json'], [periodicYml, 'gate-census', '"/tmp/gate-census-plan/$arm/manifest.json"']] as const) { const jobs = jobsOf(source); const emit = jobs['plan-slices']!.steps.find(step => step.run?.includes(`--emit-plan ${manifest} `))!; const cap = Number(/--max-parallel (\d+)/.exec(emit.run!)?.[1]); @@ -401,6 +461,10 @@ describe('panel verdict surfaces (eval reliability policy)', () => { expect(redispatch.permissions).toEqual({ actions: 'write' }); expect(redispatch.if).toBe("${{ !cancelled() && needs.report.outputs.redispatch == 'true' }}"); expect(redispatch.steps[0]!.run).toContain('-f redispatch_of="$GITHUB_RUN_ID"'); + // A census-only run re-dispatches with its own arms, never as a weekly run. + expect(redispatch.steps[0]!.run).toContain('${CENSUS_ARMS:+-f census_arms="$CENSUS_ARMS"}'); + expect(redispatch.steps[0]!.env?.CENSUS_ARMS).toBe('${{ needs.report.outputs.redispatch_census_arms }}'); + expect(report.outputs!.redispatch_census_arms).toBe("${{ needs.plan-slices.outputs.periodic_lane != 'true' && join(fromJSON(needs.plan-slices.outputs.census_arms), ',') || '' }}"); const classify = report.steps.find(step => step.id === 'verdict')!; expect(classify.run).toContain('.verdict.redispatchEligible == true'); expect(classify.run).toContain('[ -z "$REDISPATCH_OF" ]'); @@ -426,7 +490,13 @@ describe('scheduled paid lanes: concurrency and branch dispatch scope', () => { test('a branch dispatch of evals-periodic runs the periodic lane only unless it opts into the gate census', () => { const wf = parse(periodicYml); expect(wf.on.workflow_dispatch!.inputs!.include_gate_census).toMatchObject({ type: 'boolean', default: false }); - expect(wf.jobs['gate-census']!.if).toBe("${{ github.ref == 'refs/heads/main' || inputs.include_gate_census }}"); + // The planner selects the census arms (scripts/census-arms.ts); no arm, no census job. + expect(wf.jobs['gate-census']!.if).toBe("${{ needs.plan-slices.outputs.gate_matrix != '[]' }}"); + const select = wf.jobs['plan-slices']!.steps.find(step => step.id === 'census')!; + expect(select.env?.INCLUDE_GATE_CENSUS).toBe("${{ github.ref == 'refs/heads/main' || inputs.include_gate_census }}"); + expect(select.run).toBe('bun --no-install run scripts/census-arms.ts --select >> "$GITHUB_OUTPUT"'); + expect(selectCensus({ schedule: '', censusArms: '', includeGateCensus: false })).toEqual({ arms: [], periodicLane: true }); + expect(selectCensus({ schedule: WEEKLY_CENSUS_SCHEDULE, includeGateCensus: true })).toEqual({ arms: [...WEEKLY_CENSUS_ARMS], periodicLane: true }); const report = wf.jobs.report!.steps; const reconcile = report.find(step => step.name === 'Reconcile gate census against the manifest (fail-closed)')!; expect(reconcile.env?.GATE_CENSUS).toBe('${{ needs.gate-census.result }}'); @@ -436,3 +506,80 @@ describe('scheduled paid lanes: concurrency and branch dispatch scope', () => { expect(fail.if).toContain(`!contains(fromJSON('["success","skipped"]'), needs.gate-census.result)`); }); }); + +describe('evals-periodic.yml census arms: Monday two-arm census, monthly Fable/Astra census', () => { + type CensusStep = Step & { if?: string }; + type CensusWf = { on: { schedule: Array<{ cron: string }>; workflow_dispatch: { inputs: Record } }; + concurrency: { group: string }; jobs: Record; steps: CensusStep[] }> }; + const wf = Bun.YAML.parse(periodicYml) as CensusWf; + const planner = wf.jobs['plan-slices']!; + const census = wf.jobs['gate-census']!; + const report = wf.jobs.report!; + + test('each arm is one host and model: the capture default and gpt-6.1-sol weekly, claude-fable-5-1 and gpt-6-astra monthly', () => { + expect(WEEKLY_CENSUS_ARMS).toEqual(['claude-opus', 'codex-sol']); + expect(MONTHLY_CENSUS_ARMS).toEqual(['claude-fable', 'codex-astra']); + // claude-opus pins no model: the Claude runners' capture default (lib/eval-model.ts) applies. + expect(censusArmEnv('claude-opus')).toEqual({ EVALS_HOST: 'claude' }); + expect(censusArmEnv('codex-sol')).toEqual({ EVALS_HOST: 'codex', GSTACK_CODEX_MODEL: 'gpt-6.1-sol' }); + expect(censusArmEnv('claude-fable')).toEqual({ EVALS_HOST: 'claude', EVALS_MODEL: 'claude-fable-5-1' }); + expect(censusArmEnv('codex-astra')).toEqual({ EVALS_HOST: 'codex', GSTACK_CODEX_MODEL: 'gpt-6-astra' }); + expect(() => censusArmEnv('codex-luna')).toThrow('unknown census arm'); + // The report files slice artifacts by arm suffix: no arm id may end another one. + const ids = Object.keys(CENSUS_ARMS); + for (const a of ids) for (const b of ids) if (a !== b) expect(a.endsWith(`-${b}`)).toBe(false); + for (const id of ids) expect(id).toMatch(/^[a-z]+-[a-z]+$/); + }); + + test('the monthly cron runs only the census, under its own concurrency group; the Monday and daily canary schedules are unchanged', () => { + expect(wf.on.schedule.map(entry => entry.cron)).toEqual([WEEKLY_CENSUS_SCHEDULE, '30 7 * * *', MONTHLY_CENSUS_SCHEDULE]); + expect(MONTHLY_CENSUS_SCHEDULE).toBe('0 6 1 * *'); + expect(wf.concurrency.group.endsWith("${{ github.event.schedule == '0 6 1 * *' && '-monthly' || '' }}")).toBe(true); + expect(wf.concurrency.group).toContain("(github.event.schedule == '30 7 * * *' || inputs.schema_canary_only) && '-schema-canary'"); + expect(selectCensus({ schedule: MONTHLY_CENSUS_SCHEDULE, includeGateCensus: true })).toEqual({ arms: [...MONTHLY_CENSUS_ARMS], periodicLane: false }); + const select = planner.steps.find(step => step.id === 'census')!; + expect(select.env).toMatchObject({ SCHEDULE: '${{ github.event.schedule }}', CENSUS_ARMS: '${{ inputs.census_arms }}' }); + expect(planner.outputs).toMatchObject({ periodic_lane: '${{ steps.census.outputs.periodic_lane }}', census_arms: '${{ steps.census.outputs.arms }}' }); + // Without the periodic lane no periodic plan is emitted, executed or reconciled. + for (const step of planner.steps.filter(step => step.run?.includes('/tmp/paid-plan/manifest.json') || step.with?.name === 'paid-plan')) { + expect(step.if, step.name ?? step.with?.name).toBe("steps.census.outputs.periodic_lane == 'true'"); + } + const reconcile = report.steps.find(step => step.id === 'reconcile')!; + expect(reconcile.env?.PERIODIC_LANE).toBe('${{ needs.plan-slices.outputs.periodic_lane }}'); + expect(reconcile.run).toContain('if [ "$PERIODIC_LANE" != "true" ]; then'); + // A census-only green run never closes the weekly tracking issue. + expect(report.steps.find(step => step.name === 'Close the tracking issue on a green run')!.if) + .toContain("needs.plan-slices.outputs.periodic_lane == 'true' && needs.eval-slices.result == 'success'"); + }); + + test('a census_arms dispatch runs only the named arms and refuses unknown ones', () => { + expect(wf.on.workflow_dispatch.inputs.census_arms).toMatchObject({ type: 'string', default: '' }); + expect(selectCensus({ censusArms: 'codex-sol', includeGateCensus: true })).toEqual({ arms: ['codex-sol'], periodicLane: false }); + expect(selectCensus({ censusArms: 'claude-opus, codex-sol,codex-sol', includeGateCensus: false })).toEqual({ arms: ['claude-opus', 'codex-sol'], periodicLane: false }); + expect(() => selectCensus({ censusArms: 'codex-sol,gpt', includeGateCensus: false })).toThrow('unknown census arm(s): gpt'); + const cli = (args: string[], env: Record) => { + const proc = spawnSync(process.execPath, ['--no-install', 'run', 'scripts/census-arms.ts', ...args], + { cwd: ROOT, env: { PATH: process.env.PATH ?? '', ...env }, encoding: 'utf8', timeout: 15_000 }); + expect(proc.status, proc.stderr).toBe(0); + return proc.stdout; + }; + expect(cli(['--select'], { SCHEDULE: MONTHLY_CENSUS_SCHEDULE })).toBe('arms=["claude-fable","codex-astra"]\nperiodic_lane=false\n'); + expect(cli(['--select'], { CENSUS_ARMS: 'codex-sol', INCLUDE_GATE_CENSUS: 'true' })).toBe('arms=["codex-sol"]\nperiodic_lane=false\n'); + expect(cli(['--env', 'codex-sol'], {})).toBe('EVALS_HOST=codex\nGSTACK_CODEX_MODEL=gpt-6.1-sol\n'); + }); + + test('census artifacts and reconciliation are per arm', () => { + const upload = census.steps.find(step => step.name === 'Upload census results')!; + expect(upload.with!.name).toBe('gate-census-${{ matrix.slice }}-a${{ github.run_attempt }}-${{ matrix.arm }}'); + expect(census.steps.find(step => step.name?.startsWith('Run gate census slice'))!.name).toContain('(${{ matrix.arm }})'); + expect(report.steps.some(step => step.with?.pattern === 'gate-census-[0-9]*' && step.with?.path === '/tmp/gate-census-slices')).toBe(true); + const reconcile = report.steps.find(step => step.id === 'gate-reconcile')!; + expect(reconcile.env?.ARMS).toBe('${{ needs.plan-slices.outputs.census_arms }}'); + expect(reconcile.run).toContain('cp "/tmp/gate-census-plan/$arm/manifest.json" "$dir/manifest.json" || status=1'); + expect(reconcile.run).toContain('for slice in /tmp/gate-census-slices/gate-census-*-"$arm"; do'); + expect(reconcile.run).toContain('EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --report "$dir"'); + // Every arm's report counts: one red arm fails the census. + expect(reconcile.run).toContain('if [ "$code" -ne 0 ]; then status=$code; fi'); + expect(reconcile.run).toContain('echo "exit=$status" >> "$GITHUB_OUTPUT"'); + }); +}); diff --git a/test/fixtures/codex-host/exec-commands.jsonl b/test/fixtures/codex-host/exec-commands.jsonl new file mode 100644 index 000000000..94518ca86 --- /dev/null +++ b/test/fixtures/codex-host/exec-commands.jsonl @@ -0,0 +1,7 @@ +{"type":"thread.started","thread_id":"01a150b3-ae5a-7932-ba51-4c295c66adf4"} +{"type":"turn.started"} +{"type":"item.completed","item":{"id":"item_0","type":"agent_message","text":"I’ll write and overwrite `hello.txt` with `printf`, then read it with `cat`.\n"}} +{"type":"item.started","item":{"id":"item_1","type":"command_execution","command":"/bin/bash -lc \"printf 'hi' > hello.txt\nprintf 'hello' > hello.txt\ncat hello.txt\"","aggregated_output":"","exit_code":null,"status":"in_progress"}} +{"type":"item.completed","item":{"id":"item_1","type":"command_execution","command":"/bin/bash -lc \"printf 'hi' > hello.txt\nprintf 'hello' > hello.txt\ncat hello.txt\"","aggregated_output":"hello","exit_code":0,"status":"completed"}} +{"type":"item.completed","item":{"id":"item_2","type":"agent_message","text":"`cat hello.txt` output: `hello`"}} +{"type":"turn.completed","usage":{"input_tokens":25009,"cached_input_tokens":12437,"cache_write_input_tokens":12566,"output_tokens":86,"reasoning_output_tokens":0}} diff --git a/test/fixtures/codex-host/exec-file-change.jsonl b/test/fixtures/codex-host/exec-file-change.jsonl new file mode 100644 index 000000000..41647909f --- /dev/null +++ b/test/fixtures/codex-host/exec-file-change.jsonl @@ -0,0 +1,9 @@ +{"type":"thread.started","thread_id":"01a150b3-be24-7f22-aa81-7eb4f79018e0"} +{"type":"turn.started"} +{"type":"item.completed","item":{"id":"item_0","type":"agent_message","text":"I’ll create `notes.md` with `draft`, then change it to `final` using two patches.\n"}} +{"type":"item.started","item":{"id":"item_1","type":"file_change","changes":[{"path":"/tmp/codex-host-fixture/notes.md","kind":"add"}],"status":"in_progress"}} +{"type":"item.completed","item":{"id":"item_1","type":"file_change","changes":[{"path":"/tmp/codex-host-fixture/notes.md","kind":"add"}],"status":"completed"}} +{"type":"item.started","item":{"id":"item_2","type":"file_change","changes":[{"path":"/tmp/codex-host-fixture/notes.md","kind":"update"}],"status":"in_progress"}} +{"type":"item.completed","item":{"id":"item_2","type":"file_change","changes":[{"path":"/tmp/codex-host-fixture/notes.md","kind":"update"}],"status":"completed"}} +{"type":"item.completed","item":{"id":"item_3","type":"agent_message","text":"Created `notes.md` with `draft`, then changed it to `final` using two `apply_patch` calls."}} +{"type":"turn.completed","usage":{"input_tokens":25020,"cached_input_tokens":12440,"cache_write_input_tokens":12574,"output_tokens":137,"reasoning_output_tokens":0}} diff --git a/test/helpers/anthropic-preflight.ts b/test/helpers/anthropic-preflight.ts index 9ca9f1b76..522090c12 100644 --- a/test/helpers/anthropic-preflight.ts +++ b/test/helpers/anthropic-preflight.ts @@ -11,9 +11,14 @@ * * Lives in its own module (not e2e-helpers) so the runner can import it * without dragging in bun:test. + * + * On the Codex host (EVALS_HOST=codex) Codex is the agent and no session + * spawns `claude`, so the ping is skipped; Codex API failures surface per + * session as error_api. */ import { spawnSync } from 'child_process'; +import { resolveEvalHost } from './e2e-hosts'; export type PreflightResult = 'skipped' | 'ok'; @@ -21,7 +26,7 @@ export function preflightAnthropicApi( env: NodeJS.ProcessEnv = process.env, spawn: typeof spawnSync = spawnSync, ): PreflightResult { - if (env.EVALS_PREFLIGHT_OK === '1') return 'skipped'; + if (env.EVALS_PREFLIGHT_OK === '1' || resolveEvalHost(env) === 'codex') return 'skipped'; const check = spawn( 'sh', ['-c', 'echo "ping" | claude -p --max-turns 1 --output-format stream-json --verbose --dangerously-skip-permissions'], diff --git a/test/helpers/codex-eval.ts b/test/helpers/codex-eval.ts index 1a402da51..02dc35297 100644 --- a/test/helpers/codex-eval.ts +++ b/test/helpers/codex-eval.ts @@ -1,5 +1,5 @@ /** One terminal record per Codex eval attempt, using the assertions as the oracle. */ -import { CODEX_DRAIN_GRACE_MS, CodexHarnessError, type CodexResult } from './codex-session-runner'; +import { CODEX_DRAIN_GRACE_MS, CodexHarnessError, codexUsageCostUsd, type CodexResult } from './codex-session-runner'; import { EvalCollector, getProjectEvalDir, shardSlugOfEvalDir, type EvalTestEntry } from './eval-store'; import * as path from 'node:path'; import * as fs from 'node:fs'; @@ -198,10 +198,13 @@ export async function runRecordedCodexEval(opts: CodexEvalOptions): Promise Bash ({ command } unwrapped from Codex's + * `bash -lc`, its output); file_change -> Write (add), Edit (update) or + * Delete per path; web_search -> WebSearch; mcp_tool_call -> mcp__server__tool; + * the last agent message is the output; + * - transcript: the Claude stream-json shape the cases' observers read: one + * system/init event (cwd, the Codex thread id as session_id), assistant text + * and tool_use events, user tool_result events (is_error from the exit code) + * and a final result event; + * - sandbox: GSTACK_CODEX_SANDBOX, default danger-full-access (the counterpart + * of claude -p --dangerously-skip-permissions; cases write outside their + * working directory, start loopback servers and browsers); workspace-write + * gets network access; + * - effort: GSTACK_CODEX_EFFORT sets model_reasoning_effort; unset keeps the + * model's default; + * - model: a non-Claude options.model, else GSTACK_CODEX_MODEL, else + * CODEX_FRONTIER_MODEL; a case's Claude model pin does not apply here; + * - cost: real accounting from `turn.completed` usage at lib/pricing.ts rates, + * recorded on the result, the eval record and the session ledger. + * - nativeLifecycle: onSpawn gets the codex process-group owner right after + * spawn; onSettled runs after its exit and group kill, within the drain grace. + * Not translated: maxTurns (Codex has no turn cap), allowedTools/tools (Codex + * has its own tool surface) and startupGraceMs (the timeout bounds the session). + */ +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { spawnSync } from 'child_process'; +import { CODEX_DRAIN_GRACE_MS, CodexHarnessError, codexUsageCostUsd, execCodex, type CodexResult } from './codex-session-runner'; +import { getProjectEvalDir } from './eval-store'; +import { hermeticChildEnv } from './hermetic-env'; +import { appendSessionLedger, sessionKey, type SessionEnd } from './session-ledger'; +import { sanitizeTestName, scanBrowseErrors, type SkillTestResult } from './session-runner'; +import { CODEX_FRONTIER_MODEL } from '../../scripts/resolvers/constants'; + +const ROOT = path.resolve(import.meta.dir, '..', '..'); +const SETUP_TIMEOUT_MS = 300_000; +export const CODEX_SANDBOXES = ['read-only', 'workspace-write', 'danger-full-access'] as const; +const EFFORT = /^[a-z]+$/; + +/** The Codex model for a session: a Claude pin never reaches `codex --model`. */ +export function codexHostModel(explicit: string | undefined, env: NodeJS.ProcessEnv = process.env): string { + if (explicit && !/^claude-/.test(explicit)) return explicit; + return env.GSTACK_CODEX_MODEL || CODEX_FRONTIER_MODEL; +} + +/** `codex exec` arguments for one session; the prompt is the last argument. */ +export function codexHostExecArgs(model: string, prompt: string, env: NodeJS.ProcessEnv = process.env): string[] { + const sandbox = env.GSTACK_CODEX_SANDBOX || 'danger-full-access'; + if (!(CODEX_SANDBOXES as readonly string[]).includes(sandbox)) { + throw new Error(`GSTACK_CODEX_SANDBOX must be one of ${CODEX_SANDBOXES.join(', ')}; got "${sandbox}"`); + } + const effort = env.GSTACK_CODEX_EFFORT; + if (effort && !EFFORT.test(effort)) throw new Error(`GSTACK_CODEX_EFFORT must be a bare effort name; got "${effort}"`); + return ['exec', '--json', '-s', sandbox, '--skip-git-repo-check', '--model', model, + ...(sandbox === 'workspace-write' ? ['-c', 'sandbox_workspace_write.network_access=true'] : []), + ...(effort ? ['-c', `model_reasoning_effort="${effort}"`] : []), + prompt]; +} + +/** The command inside Codex's `/bin/bash -lc ` wrapper, else the command unchanged. */ +export function unwrapShellCommand(command: string): string { + const match = /^(?:\/(?:usr\/)?bin\/)?(?:ba|z)?sh -lc (.+)$/s.exec(command); + if (!match) return command; + const word = match[1]!; + let out = ''; + let i = 0; + while (i < word.length) { + const c = word[i]!; + if (c === "'") { + const end = word.indexOf("'", i + 1); + if (end < 0) return command; + out += word.slice(i + 1, end); + i = end + 1; + } else if (c === '"') { + i++; + let closed = false; + while (i < word.length) { + if (word[i] === '\\' && i + 1 < word.length && '"\\$`\n'.includes(word[i + 1]!)) { out += word[i + 1]; i += 2; } + else if (word[i] === '"') { i++; closed = true; break; } + else out += word[i++]; + } + if (!closed) return command; + } else if (c === '\\' && i + 1 < word.length) { + out += word[i + 1]; + i += 2; + } else if (/\s/.test(c)) { + return command; + } else { + out += c; + i++; + } + } + return out; +} + +export type HostToolCall = SkillTestResult['toolCalls'][number]; + +const FILE_CHANGE_TOOLS: Record = { add: 'Write', update: 'Edit', delete: 'Delete' }; + +/** Claude-shaped tool calls from completed Codex items, in order. */ +export function codexHostToolCalls(rawLines: readonly string[]): HostToolCall[] { + const calls: HostToolCall[] = []; + for (const line of rawLines) { + let event: any; + try { event = JSON.parse(line); } catch { continue; } + if (event?.type !== 'item.completed' || !event.item) continue; + const item = event.item; + if (item.type === 'command_execution') { + calls.push({ tool: 'Bash', input: { command: unwrapShellCommand(String(item.command ?? '')) }, output: String(item.aggregated_output ?? '') }); + } else if (item.type === 'file_change') { + for (const change of Array.isArray(item.changes) ? item.changes : []) { + calls.push({ tool: FILE_CHANGE_TOOLS[change?.kind] ?? 'Edit', input: { file_path: String(change?.path ?? ''), kind: change?.kind }, output: '' }); + } + } else if (item.type === 'web_search') { + calls.push({ tool: 'WebSearch', input: { query: item.query }, output: '' }); + } else if (item.type === 'mcp_tool_call') { + calls.push({ tool: `mcp__${item.server}__${item.tool}`, input: item.arguments ?? {}, output: typeof item.result === 'string' ? item.result : JSON.stringify(item.result ?? '') }); + } + } + return calls; +} + +/** + * Codex events as the Claude stream-json transcript the cases' observers read + * (system/init, assistant text and tool_use, user tool_result, result). + */ +export function codexHostTranscript(rawLines: readonly string[], cwd: string, output: string): any[] { + const events: any[] = []; + let session = ''; + const say = (role: 'assistant' | 'user', content: any[]) => events.push({ type: role, session_id: session, parent_tool_use_id: null, message: { role, content } }); + for (const line of rawLines) { + let event: any; + try { event = JSON.parse(line); } catch { continue; } + if (event?.type === 'thread.started') { + session = String(event.thread_id ?? ''); + events.push({ type: 'system', subtype: 'init', cwd, session_id: session, tools: [] }); + continue; + } + if (event?.type !== 'item.completed' || !event.item) continue; + const item = event.item; + const id = String(item.id ?? `item_${events.length}`); + if (item.type === 'agent_message' && item.text) say('assistant', [{ type: 'text', text: String(item.text) }]); + else if (item.type === 'command_execution') { + say('assistant', [{ type: 'tool_use', id, name: 'Bash', input: { command: unwrapShellCommand(String(item.command ?? '')) } }]); + say('user', [{ type: 'tool_result', tool_use_id: id, content: String(item.aggregated_output ?? ''), is_error: item.exit_code !== 0 }]); + } else if (item.type === 'file_change') { + (Array.isArray(item.changes) ? item.changes : []).forEach((change: any, index: number) => { + const useId = `${id}_${index}`; + say('assistant', [{ type: 'tool_use', id: useId, name: FILE_CHANGE_TOOLS[change?.kind] ?? 'Edit', input: { file_path: String(change?.path ?? '') } }]); + say('user', [{ type: 'tool_result', tool_use_id: useId, content: '', is_error: item.status !== 'completed' }]); + }); + } + } + events.push({ type: 'result', subtype: 'success', session_id: session, result: output }); + return events; +} + +let runtimeRoot: string | undefined; + +/** + * Install the full gstack Codex runtime once per process: `./setup --host codex` + * into a private HOME/CODEX_HOME, with the login home's auth.json copied in. + * Returns the root holding home/.codex and tmp/ (Codex will not create its + * helper aliases when CODEX_HOME sits under its own TMPDIR). A caller that + * already prepared one passes it as GSTACK_CODEX_HOST_ROOT. + */ +export function prepareCodexHostRuntime(env: NodeJS.ProcessEnv = process.env): string { + if (runtimeRoot) return runtimeRoot; + const preset = env.GSTACK_CODEX_HOST_ROOT; + if (preset) { + if (!fs.existsSync(path.join(preset, 'home', '.codex'))) throw new Error(`GSTACK_CODEX_HOST_ROOT ${preset} has no home/.codex`); + fs.mkdirSync(path.join(preset, 'tmp'), { recursive: true }); + return runtimeRoot = preset; + } + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'codex-host-')); + const home = path.join(root, 'home'); + const codexHome = path.join(home, '.codex'); + const tmp = path.join(root, 'tmp'); + for (const dir of [codexHome, tmp]) fs.mkdirSync(dir, { recursive: true }); + const auth = path.join(env.CODEX_HOME || path.join(os.homedir(), '.codex'), 'auth.json'); + if (fs.existsSync(auth)) fs.copyFileSync(auth, path.join(codexHome, 'auth.json')); + const setup = spawnSync(path.join(ROOT, 'setup'), ['--host', 'codex'], { + cwd: ROOT, encoding: 'utf8', timeout: SETUP_TIMEOUT_MS, + env: hermeticChildEnv({ HOME: home, CODEX_HOME: codexHome, TMPDIR: tmp }, { extraAllow: ['OPENAI_API_KEY', 'CODEX_*'] }), + }); + if (setup.status !== 0) { + fs.rmSync(root, { recursive: true, force: true }); + throw new Error(`./setup --host codex failed (exit ${setup.status}${setup.error ? `, ${setup.error.message}` : ''}): ${(setup.stderr || setup.stdout || '').slice(-2000)}`); + } + process.once('exit', () => { try { fs.rmSync(root, { recursive: true, force: true }); } catch { /* best effort */ } }); + runtimeRoot = root; + return root; +} + +type RunSkillTestOptions = Parameters[0]; + +function exitReasonOf(result: CodexResult, aborted: boolean): string { + if (result.exitCode === 124) return 'timeout'; + const failedTurn = result.rawLines.some(line => /"type":"(?:turn\.failed|error)"/.test(line)); + if (result.exitCode === 0) return failedTurn ? 'error_api' : 'success'; + if (aborted) return 'timeout'; + return failedTurn && !result.rawLines.some(line => line.includes('"type":"item.completed"')) ? 'error_api' : `exit_code_${result.exitCode}`; +} + +const SESSION_ENDS: Record = { success: 'completed', timeout: 'session_timeout', error_api: 'api_error' }; + +/** runSkillTest on the Codex host. `appendSystemPrompt` arrives with any completion notice already applied. */ +export async function runCodexHostSkillTest(options: RunSkillTestOptions): Promise { + const startTime = Date.now(); + const startMono = performance.now(); + const { workingDirectory, timeout = 120_000, testName, runId, signal } = options; + const deadline = startTime + timeout; + const model = codexHostModel(options.model); + const prompt = options.appendSystemPrompt ? `${options.appendSystemPrompt}\n\n${options.prompt}` : options.prompt; + const root = prepareCodexHostRuntime(); + const safeName = testName ? sanitizeTestName(testName) : null; + let runDir: string | null = null; + if (runId) { + try { + runDir = path.join(path.dirname(getProjectEvalDir()), 'e2e-runs', runId); + fs.mkdirSync(runDir, { recursive: true }); + } catch { runDir = null; } + } + let firstResponseMs = 0; + let lastItemMs = 0; + let maxInterTurnMs = 0; + const onLine = (line: string, elapsedMs: number) => { + if (!firstResponseMs) firstResponseMs = Math.max(1, elapsedMs); + if (runDir && safeName) { try { fs.appendFileSync(path.join(runDir, `${safeName}.ndjson`), line + '\n'); } catch { /* non-fatal */ } } + if (!line.includes('"type":"item.completed"') || !/"type":"(?:command_execution|file_change)"/.test(line)) return; + if (lastItemMs) maxInterTurnMs = Math.max(maxInterTurnMs, elapsedMs - lastItemMs); + lastItemMs = elapsedMs; + }; + + let result: CodexResult; + let harnessReason: string | undefined; + let started = true; + const lifecycle = options.nativeLifecycle; + try { + result = await execCodex(codexHostExecArgs(model, prompt), { + cwd: workingDirectory, startTime, deadline, signal, onLine, ...(lifecycle ? { onSpawn: (pid: number) => lifecycle.onSpawn(pid) } : {}), + env: hermeticChildEnv( + { GSTACK_HEADLESS: '1', ...options.env, CODEX_HOME: path.join(root, 'home', '.codex'), TMPDIR: options.env?.TMPDIR ?? path.join(root, 'tmp') }, + { extraAllow: ['OPENAI_API_KEY', 'CODEX_*'] }, + ), + }); + } catch (error) { + if (!(error instanceof CodexHarnessError) || !error.result) throw error; + result = error.result; + started = !error.message.startsWith('Could not start Codex'); + harnessReason = /drain/.test(error.message) ? 'error_output_drain' : 'error_output_stream'; + } + if (lifecycle) { + const settleDeadline = Date.now() + CODEX_DRAIN_GRACE_MS; + let hookTimer: ReturnType | undefined; + try { + await Promise.race([ + lifecycle.onSettled({ deadline: settleDeadline, exited: started }), + new Promise((_, reject) => { hookTimer = setTimeout(() => reject(new Error('native lifecycle settlement deadline exceeded')), CODEX_DRAIN_GRACE_MS); }), + ]); + } finally { clearTimeout(hookTimer); } + } + + const exitReason = harnessReason ?? exitReasonOf(result, !!signal?.aborted); + const toolCalls = codexHostToolCalls(result.rawLines); + const events = result.rawLines.flatMap(line => { try { return [JSON.parse(line)]; } catch { return []; } }); + const messages = events.filter(event => event?.type === 'item.completed' && event.item?.type === 'agent_message'); + const output = String(messages.at(-1)?.item?.text ?? ''); + const turnsUsed = events.filter(event => event?.type === 'item.completed' && event.item?.type !== 'reasoning').length; + const transcript = codexHostTranscript(result.rawLines, workingDirectory, output); + const cost = result.usage ? codexUsageCostUsd(result.usage, model) : null; + const browseErrors = scanBrowseErrors(toolCalls.filter(call => call.tool === 'Bash').map(call => call.output).join('\n') + '\n' + result.stderr); + const duration = Date.now() - startTime; + + appendSessionLedger({ key: sessionKey('codex', testName), ...(testName ? { test_name: testName } : {}), runner: 'codex', + started_at: new Date(startTime).toISOString(), budget_ms: timeout, elapsed_ms: performance.now() - startMono, + end: exitReason === 'timeout' && signal?.aborted ? 'aborted' : SESSION_ENDS[exitReason] ?? 'error', + ...(exitReason === 'timeout' ? { evidence: `armed ${timeout}ms session timeout fired` } : exitReason !== 'success' && result.stderr ? { evidence: result.stderr } : {}), + billed: cost !== null, model, ...(cost !== null ? { cost_usd: cost, tokens: result.usage! } : {}) }); + + if (exitReason !== 'success' || browseErrors.length > 0) { + try { + const failureDir = runDir || path.join(workingDirectory, '.gstack', 'test-transcripts'); + fs.mkdirSync(failureDir, { recursive: true }); + fs.writeFileSync(path.join(failureDir, safeName ? `${safeName}-failure.json` : `e2e-${new Date().toISOString().replace(/[:.]/g, '-')}.json`), + JSON.stringify({ host: 'codex', model, prompt: prompt.slice(0, 500), testName: testName || 'unknown', exitReason, browseErrors, duration, + stderr: result.stderr.slice(0, 2000), output: output.slice(0, 500) }, null, 2)); + } catch { /* non-fatal */ } + } + + return { + toolCalls, browseErrors, exitReason, duration, output, transcript, model, firstResponseMs, maxInterTurnMs, + costEstimate: { inputChars: prompt.length, outputChars: output.length, + estimatedTokens: (result.usage?.input ?? 0) + (result.usage?.output ?? 0), estimatedCost: cost ?? 0, turnsUsed }, + ...(result.usage ? { tokenUsage: result.usage } : {}), + ...(cost === null ? { costKnown: false } : {}), + }; +} diff --git a/test/helpers/codex-session-runner.ts b/test/helpers/codex-session-runner.ts index 4eea35931..c040ebbfa 100644 --- a/test/helpers/codex-session-runner.ts +++ b/test/helpers/codex-session-runner.ts @@ -21,6 +21,7 @@ import { hermeticChildEnv } from './hermetic-env'; import { extractSkillSections } from './skill-fixture'; import { killProcessGroup } from '../../scripts/test-strict-output'; import { CODEX_FRONTIER_MODEL } from '../../scripts/resolvers/constants'; +import { estimateCostUsd, hasPricing } from '../../lib/pricing'; // --- Interfaces --- @@ -34,6 +35,47 @@ export interface CodexResult { sessionId: string | null; // Thread ID for session continuity rawLines: string[]; // Raw JSONL lines for debugging stderr: string; // Stderr output (skill loading errors, auth failures) + usage?: CodexTokenUsage | null; // Summed turn.completed usage; null when no turn completed + model?: string; // The --model this session ran +} + +/** + * Token classes of Codex `turn.completed` usage. `cached_input` (cache reads) + * and `cache_write_input` are subsets of `input`; `reasoning_output` is a + * subset of `output`. + */ +export interface CodexTokenUsage { + input: number; + cached_input: number; + cache_write_input: number; + output: number; + reasoning_output: number; +} + +export const NO_CODEX_USAGE: CodexTokenUsage = Object.freeze({ input: 0, cached_input: 0, cache_write_input: 0, output: 0, reasoning_output: 0 }); + +const count = (value: unknown) => (typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : 0); + +/** Add one `turn.completed` usage object to a running total. */ +export function addCodexTurnUsage(total: CodexTokenUsage, usage: Record | undefined): CodexTokenUsage { + return { + input: total.input + count(usage?.input_tokens), + cached_input: total.cached_input + count(usage?.cached_input_tokens), + cache_write_input: total.cache_write_input + count(usage?.cache_write_input_tokens), + output: total.output + count(usage?.output_tokens), + reasoning_output: total.reasoning_output + count(usage?.reasoning_output_tokens), + }; +} + +/** + * A session's cost at lib/pricing.ts rates: cache reads at the cached-input + * rate, all other input (cache writes included) at the input rate, output at + * the output rate. Null when the model has no price row. + */ +export function codexUsageCostUsd(usage: CodexTokenUsage, model: string | undefined): number | null { + if (!hasPricing(model)) return null; + const cached = Math.min(usage.cached_input, usage.input); + return estimateCostUsd({ input: usage.input - cached, cached, output: usage.output }, model); } /** Existing pipe-drain allowance, separate from the model's work budget. */ @@ -54,6 +96,7 @@ export interface ParsedCodexJSONL { toolCalls: string[]; tokens: number; sessionId: string | null; + usage: CodexTokenUsage | null; } /** @@ -71,6 +114,7 @@ export function parseCodexJSONL(lines: string[]): ParsedCodexJSONL { const toolCalls: string[] = []; let tokens = 0; let sessionId: string | null = null; + let usage: CodexTokenUsage | null = null; for (const line of lines) { if (!line.trim()) continue; @@ -95,9 +139,9 @@ export function parseCodexJSONL(lines: string[]): ParsedCodexJSONL { if (cmd) toolCalls.push(cmd); } } else if (t === 'turn.completed') { - const usage = obj.usage || {}; - const turnTokens = (usage.input_tokens || 0) + (usage.output_tokens || 0); - tokens += turnTokens; + const turn = obj.usage || {}; + tokens += (turn.input_tokens || 0) + (turn.output_tokens || 0); + usage = addCodexTurnUsage(usage ?? NO_CODEX_USAGE, turn); } } catch { /* skip malformed lines */ } } @@ -108,6 +152,7 @@ export function parseCodexJSONL(lines: string[]): ParsedCodexJSONL { toolCalls, tokens, sessionId, + usage, }; } @@ -166,6 +211,192 @@ export function installSkillToTempHome( return home; } +// --- Spawn core --- + +/** + * Spawn `codex ` in its own process group and collect its JSONL until + * exit, the deadline or an abort, with the existing drain grace. Shared by + * runCodexSkill and the Codex host (codex-host-runner.ts). `onLine` sees each + * JSONL line as it arrives with the elapsed milliseconds since `startTime`. + */ +export async function execCodex(args: string[], opts: { + cwd: string; + env: Record; + startTime: number; + deadline: number; + signal?: AbortSignal; + onLine?: (line: string, elapsedMs: number) => void; + /** Called with the group-owning pid right after spawn; a throw kills the group. */ + onSpawn?: (pid: number) => void; +}): Promise { + const { cwd, env, startTime, deadline, signal, onLine } = opts; + const proc = spawn('codex', args, { + cwd, + stdio: ['ignore', 'pipe', 'pipe'], + detached: process.platform !== 'win32', + env, + }); + if (opts.onSpawn) { + try { + if (proc.pid === undefined) throw new Error('Codex did not start'); + opts.onSpawn(proc.pid); + } catch (error) { + killProcessGroup(proc, 'SIGKILL'); + proc.stdout!.destroy(); + proc.stderr!.destroy(); + throw error; + } + } + const collectedLines: string[] = []; + // work --exit--> drain (<=5s) --> result + // \--timeout/abort--> kill group + close pipes --> timeout result + // Exit status comes from 'exit'; a forced drain after exit 0 is an error. + let stdoutBuffer = ''; + let stderr = ''; + let exitCode: number | undefined; + let timedOut = false; + let drainExpired = false; + let streamError: { stream: 'stdout' | 'stderr'; error: Error } | undefined; + let spawnError: Error | undefined; + let stdoutDone = false; + let stderrDone = false; + let stdoutEnded = false; + let stderrEnded = false; + let finalized = false; + const stdoutDecoder = new StringDecoder('utf8'); + const stderrDecoder = new StringDecoder('utf8'); + let workTimer: ReturnType | undefined; + let drainTimer: ReturnType | undefined; + let finish!: () => void; + const finished = new Promise((resolve) => { finish = resolve; }); + + const maybeFinish = () => { + if (exitCode !== undefined && stdoutDone && stderrDone) finish(); + }; + const closePipes = () => { + // Destroy both streams: canceling stdout alone cannot release an + // inherited stderr pipe held by a descendant in another process group. + proc.stdout!.destroy(); + proc.stderr!.destroy(); + }; + const expireDrain = () => { + drainExpired = !stdoutDone || !stderrDone; + killProcessGroup(proc, 'SIGKILL'); + closePipes(); + // Also bound the rare case where a failed group kill produces no exit. + finish(); + }; + const stopRun = () => { + // A real auth/crash exit that preceded a pipe stall stays a real exit. + if (exitCode === undefined) timedOut = true; + else drainExpired ||= !stdoutDone || !stderrDone; + killProcessGroup(proc, 'SIGKILL'); + closePipes(); + clearTimeout(drainTimer); + drainTimer = setTimeout(expireDrain, CODEX_DRAIN_GRACE_MS); + maybeFinish(); + }; + const onExit = (code: number | null, exitSignal: NodeJS.Signals | null) => { + exitCode = code ?? (exitSignal ? 128 + (os.constants.signals[exitSignal] ?? 0) : 1); + clearTimeout(workTimer); + clearTimeout(drainTimer); + // 'exit' means the child is gone; 'close' also waits for descendant + // pipes. Start the drain allowance at actual exit, never at close. + drainTimer = setTimeout(expireDrain, CODEX_DRAIN_GRACE_MS); + maybeFinish(); + }; + const onSpawnError = (error: Error) => { + if (finalized) return; + spawnError = error; + exitCode = 1; + closePipes(); + finish(); + }; + // Closure releases lifecycle waits, but only 'end' proves all bytes were + // drained. A destroyed pipe can emit 'close' without either EOF or error. + const onStdoutDone = () => { if (!finalized) { stdoutDone = true; maybeFinish(); } }; + const onStderrDone = () => { if (!finalized) { stderrDone = true; maybeFinish(); } }; + const onStdoutEnd = () => { if (!finalized) { stdoutBuffer += stdoutDecoder.end(); stdoutEnded = true; onStdoutDone(); } }; + const onStderrEnd = () => { if (!finalized) { stderr += stderrDecoder.end(); stderrEnded = true; onStderrDone(); } }; + const onStreamError = (stream: 'stdout' | 'stderr', error: Error) => { + if (!finalized) streamError ??= { stream, error }; + }; + const onStdout = (chunk: Buffer) => { + if (finalized) return; + stdoutBuffer += stdoutDecoder.write(chunk); + const lines = stdoutBuffer.split('\n'); + stdoutBuffer = lines.pop() || ''; + for (const line of lines) { + if (!line.trim()) continue; + collectedLines.push(line); + onLine?.(line, Date.now() - startTime); + try { + const event = JSON.parse(line); + if (event.type === 'item.completed' && event.item) { + const item = event.item; + const elapsed = Math.round((Date.now() - startTime) / 1000); + if (item.type === 'command_execution' && item.command) { + process.stderr.write(` [codex ${elapsed}s] ran: ${item.command.slice(0, 100)}\n`); + } else if (item.type === 'agent_message' && item.text) { + process.stderr.write(` [codex ${elapsed}s] message: ${item.text.slice(0, 100)}\n`); + } + } + } catch { /* malformed JSONL is ignored by parseCodexJSONL too */ } + } + }; + const onStderr = (chunk: Buffer) => { if (!finalized) stderr += stderrDecoder.write(chunk); }; + + proc.on('exit', onExit); + proc.on('error', onSpawnError); + proc.stdout!.on('data', onStdout); + proc.stderr!.on('data', onStderr); + proc.stdout!.on('end', onStdoutEnd).on('close', onStdoutDone).on('error', error => onStreamError('stdout', error)); + proc.stderr!.on('end', onStderrEnd).on('close', onStderrDone).on('error', error => onStreamError('stderr', error)); + signal?.addEventListener('abort', stopRun, { once: true }); + workTimer = setTimeout(stopRun, Math.max(0, deadline - Date.now())); + if (signal?.aborted || Date.now() >= deadline) stopRun(); + + try { + await finished; + if (stdoutBuffer.trim()) collectedLines.push(stdoutBuffer); + const parsed = parseCodexJSONL(collectedLines); + const result: CodexResult = { + ...parsed, + exitCode: timedOut ? 124 : exitCode ?? 1, + durationMs: Date.now() - startTime, + rawLines: collectedLines, + stderr, + }; + if (stderr.trim()) process.stderr.write(` [codex stderr] ${stderr.trim().slice(0, 200)}\n`); + if (spawnError) throw new CodexHarnessError(`Could not start Codex: ${spawnError.message}`, result); + const incompleteStreams = [!stdoutEnded && 'stdout', !stderrEnded && 'stderr'].filter(Boolean).join(' and '); + if (result.exitCode === 0 && (drainExpired || streamError || incompleteStreams)) { + throw new CodexHarnessError( + drainExpired ? `Codex output drain exceeded ${CODEX_DRAIN_GRACE_MS}ms after exit 0` + : streamError ? `Codex ${streamError.stream} stream failed: ${streamError.error.message}` + : `Codex ${incompleteStreams} closed before EOF after exit 0`, + result, + ); + } + return result; + } finally { + finalized = true; + clearTimeout(workTimer); + clearTimeout(drainTimer); + signal?.removeEventListener('abort', stopRun); + // Child exit and closed pipes do not prove its descendants exited: + // background tools may redirect both streams. Reap the owned group on + // every result, including ordinary success and genuine process failure. + killProcessGroup(proc, 'SIGKILL'); + closePipes(); + proc.removeListener('exit', onExit); + // Retain the harmless error listener through stream teardown so a late + // OS spawn error cannot turn into an unhandled event after cleanup. + proc.stdout!.removeListener('data', onStdout); + proc.stderr!.removeListener('data', onStderr); + } +} + // --- Main runner --- /** @@ -209,7 +440,7 @@ export async function runCodexSkill(opts: { const emptyResult = (exitCode: number, output = ''): CodexResult => ({ output, reasoning: [], toolCalls: [], tokens: 0, exitCode, - durationMs: Date.now() - startTime, sessionId: null, rawLines: [], stderr: '', + durationMs: Date.now() - startTime, sessionId: null, rawLines: [], stderr: '', usage: null, }); if (signal?.aborted || Date.now() >= deadline) return emptyResult(124); @@ -257,7 +488,8 @@ export async function runCodexSkill(opts: { // exactly that. Empirically verified against codex on this machine. const args = ['exec', '--json', '-s', sandbox, '--skip-git-repo-check']; if (ignoreUserConfig) args.push('--ignore-user-config'); - args.push('--model', model ?? process.env.GSTACK_CODEX_MODEL ?? CODEX_FRONTIER_MODEL); + const sessionModel = model ?? process.env.GSTACK_CODEX_MODEL ?? CODEX_FRONTIER_MODEL; + args.push('--model', sessionModel); for (const override of configOverrides) args.push('-c', override); args.push(prompt); @@ -275,162 +507,17 @@ export async function runCodexSkill(opts: { // dir beside CODEX_HOME instead of the shard TMPDIR that holds tempHome. const tempTmpDir = path.join(tempHome, 'tmp'); fs.mkdirSync(tempTmpDir, { recursive: true }); - const proc = spawn('codex', args, { - cwd: cwd || skillDir, - stdio: ['ignore', 'pipe', 'pipe'], - detached: process.platform !== 'win32', + const result = await execCodex(args, { + cwd: cwd || skillDir, startTime, deadline, signal, env: hermeticChildEnv( { HOME: tempHome, CODEX_HOME: tempCodexDir, TMPDIR: tempTmpDir }, { extraAllow: ['OPENAI_API_KEY', 'CODEX_*'] }, ), + }).catch((error: unknown) => { + if (error instanceof CodexHarnessError && error.result) error.result.model = sessionModel; + throw error; }); - const collectedLines: string[] = []; - // work --exit--> drain (<=5s) --> result - // \--timeout/abort--> kill group + close pipes --> timeout result - // Exit status comes from 'exit'; a forced drain after exit 0 is an error. - let stdoutBuffer = ''; - let stderr = ''; - let exitCode: number | undefined; - let timedOut = false; - let drainExpired = false; - let streamError: { stream: 'stdout' | 'stderr'; error: Error } | undefined; - let spawnError: Error | undefined; - let stdoutDone = false; - let stderrDone = false; - let stdoutEnded = false; - let stderrEnded = false; - let finalized = false; - const stdoutDecoder = new StringDecoder('utf8'); - const stderrDecoder = new StringDecoder('utf8'); - let workTimer: ReturnType | undefined; - let drainTimer: ReturnType | undefined; - let finish!: () => void; - const finished = new Promise((resolve) => { finish = resolve; }); - - const maybeFinish = () => { - if (exitCode !== undefined && stdoutDone && stderrDone) finish(); - }; - const closePipes = () => { - // Destroy both streams: canceling stdout alone cannot release an - // inherited stderr pipe held by a descendant in another process group. - proc.stdout!.destroy(); - proc.stderr!.destroy(); - }; - const expireDrain = () => { - drainExpired = !stdoutDone || !stderrDone; - killProcessGroup(proc, 'SIGKILL'); - closePipes(); - // Also bound the rare case where a failed group kill produces no exit. - finish(); - }; - const stopRun = () => { - // A real auth/crash exit that preceded a pipe stall stays a real exit. - if (exitCode === undefined) timedOut = true; - else drainExpired ||= !stdoutDone || !stderrDone; - killProcessGroup(proc, 'SIGKILL'); - closePipes(); - clearTimeout(drainTimer); - drainTimer = setTimeout(expireDrain, CODEX_DRAIN_GRACE_MS); - maybeFinish(); - }; - const onExit = (code: number | null, exitSignal: NodeJS.Signals | null) => { - exitCode = code ?? (exitSignal ? 128 + (os.constants.signals[exitSignal] ?? 0) : 1); - clearTimeout(workTimer); - clearTimeout(drainTimer); - // 'exit' means the child is gone; 'close' also waits for descendant - // pipes. Start the drain allowance at actual exit, never at close. - drainTimer = setTimeout(expireDrain, CODEX_DRAIN_GRACE_MS); - maybeFinish(); - }; - const onSpawnError = (error: Error) => { - if (finalized) return; - spawnError = error; - exitCode = 1; - closePipes(); - finish(); - }; - // Closure releases lifecycle waits, but only 'end' proves all bytes were - // drained. A destroyed pipe can emit 'close' without either EOF or error. - const onStdoutDone = () => { if (!finalized) { stdoutDone = true; maybeFinish(); } }; - const onStderrDone = () => { if (!finalized) { stderrDone = true; maybeFinish(); } }; - const onStdoutEnd = () => { if (!finalized) { stdoutBuffer += stdoutDecoder.end(); stdoutEnded = true; onStdoutDone(); } }; - const onStderrEnd = () => { if (!finalized) { stderr += stderrDecoder.end(); stderrEnded = true; onStderrDone(); } }; - const onStreamError = (stream: 'stdout' | 'stderr', error: Error) => { - if (!finalized) streamError ??= { stream, error }; - }; - const onStdout = (chunk: Buffer) => { - if (finalized) return; - stdoutBuffer += stdoutDecoder.write(chunk); - const lines = stdoutBuffer.split('\n'); - stdoutBuffer = lines.pop() || ''; - for (const line of lines) { - if (!line.trim()) continue; - collectedLines.push(line); - try { - const event = JSON.parse(line); - if (event.type === 'item.completed' && event.item) { - const item = event.item; - const elapsed = Math.round((Date.now() - startTime) / 1000); - if (item.type === 'command_execution' && item.command) { - process.stderr.write(` [codex ${elapsed}s] ran: ${item.command.slice(0, 100)}\n`); - } else if (item.type === 'agent_message' && item.text) { - process.stderr.write(` [codex ${elapsed}s] message: ${item.text.slice(0, 100)}\n`); - } - } - } catch { /* malformed JSONL is ignored by parseCodexJSONL too */ } - } - }; - const onStderr = (chunk: Buffer) => { if (!finalized) stderr += stderrDecoder.write(chunk); }; - - proc.on('exit', onExit); - proc.on('error', onSpawnError); - proc.stdout!.on('data', onStdout); - proc.stderr!.on('data', onStderr); - proc.stdout!.on('end', onStdoutEnd).on('close', onStdoutDone).on('error', error => onStreamError('stdout', error)); - proc.stderr!.on('end', onStderrEnd).on('close', onStderrDone).on('error', error => onStreamError('stderr', error)); - signal?.addEventListener('abort', stopRun, { once: true }); - workTimer = setTimeout(stopRun, Math.max(0, deadline - Date.now())); - if (signal?.aborted || Date.now() >= deadline) stopRun(); - - try { - await finished; - if (stdoutBuffer.trim()) collectedLines.push(stdoutBuffer); - const parsed = parseCodexJSONL(collectedLines); - const result: CodexResult = { - ...parsed, - exitCode: timedOut ? 124 : exitCode ?? 1, - durationMs: Date.now() - startTime, - rawLines: collectedLines, - stderr, - }; - if (stderr.trim()) process.stderr.write(` [codex stderr] ${stderr.trim().slice(0, 200)}\n`); - if (spawnError) throw new CodexHarnessError(`Could not start Codex: ${spawnError.message}`, result); - const incompleteStreams = [!stdoutEnded && 'stdout', !stderrEnded && 'stderr'].filter(Boolean).join(' and '); - if (result.exitCode === 0 && (drainExpired || streamError || incompleteStreams)) { - throw new CodexHarnessError( - drainExpired ? `Codex output drain exceeded ${CODEX_DRAIN_GRACE_MS}ms after exit 0` - : streamError ? `Codex ${streamError.stream} stream failed: ${streamError.error.message}` - : `Codex ${incompleteStreams} closed before EOF after exit 0`, - result, - ); - } - return result; - } finally { - finalized = true; - clearTimeout(workTimer); - clearTimeout(drainTimer); - signal?.removeEventListener('abort', stopRun); - // Child exit and closed pipes do not prove its descendants exited: - // background tools may redirect both streams. Reap the owned group on - // every result, including ordinary success and genuine process failure. - killProcessGroup(proc, 'SIGKILL'); - closePipes(); - proc.removeListener('exit', onExit); - // Retain the harmless error listener through stream teardown so a late - // OS spawn error cannot turn into an unhandled event after cleanup. - proc.stdout!.removeListener('data', onStdout); - proc.stderr!.removeListener('data', onStderr); - } + return { ...result, model: sessionModel }; } finally { // Clean up temp HOME diff --git a/test/helpers/e2e-helpers.ts b/test/helpers/e2e-helpers.ts index 2d4a2a530..6ee6937e6 100644 --- a/test/helpers/e2e-helpers.ts +++ b/test/helpers/e2e-helpers.ts @@ -16,6 +16,7 @@ import { WorktreeManager } from '../../lib/worktree'; import type { HarvestResult } from '../../lib/worktree'; import { preflightAnthropicApi } from './anthropic-preflight'; import { isHermeticEnabled } from './hermetic-env'; +import { hostSkipReason, resolveEvalHost } from './e2e-hosts'; import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; @@ -133,6 +134,17 @@ if (evalsEnabled && process.env.EVALS_TIER) { process.stderr.write(`EVALS_TIER=${tier}: ${selectedTests.length} tests\n\n`); } +// EVALS_HOST=codex: Claude-only cases are skipped with their reason, never run +// (and never failed); the report deselects them (test/helpers/e2e-hosts.ts). +if (evalsEnabled && resolveEvalHost() !== 'claude') { + const host = resolveEvalHost(); + const candidates = selectedTests ?? Object.keys(E2E_TIERS); + const skipped = candidates.filter(id => hostSkipReason(id, host)); + selectedTests = candidates.filter(id => !hostSkipReason(id, host)); + process.stderr.write(`EVALS_HOST=${host}: ${selectedTests.length} tests; ${skipped.length} Claude-only skipped (${skipped.join(', ') || 'none'})\n\n`); + (await import('./codex-host-runner')).prepareCodexHostRuntime(); +} + export const describeE2E = evalsEnabled ? describe : describe.skip; /** @@ -261,6 +273,8 @@ export function recordE2E( ...(!passed && isPreTurnInfraFailure(result) ? { failure_class: 'infra' as const } : {}), duration_ms: result.duration, cost_usd: result.costEstimate.estimatedCost, + ...(result.costKnown === false ? { cost_known: false } : {}), + ...(result.tokenUsage ? { token_usage: result.tokenUsage } : {}), transcript: result.transcript, output: result.output?.slice(0, 2000), turns_used: result.costEstimate.turnsUsed, diff --git a/test/helpers/e2e-hosts.ts b/test/helpers/e2e-hosts.ts new file mode 100644 index 000000000..622c3756d --- /dev/null +++ b/test/helpers/e2e-hosts.ts @@ -0,0 +1,126 @@ +/** + * Agent host selection for paid E2E cases (GSTA-49 TM2). EVALS_HOST picks the + * agent every runSkillTest session runs on: `claude` (default) or `codex` + * (codex exec, test/helpers/codex-host-runner.ts). E2E_HOSTS is the per-case + * host metadata over every gate case (classification in the GSTA-49 TM2 + * scoping): `both` runs on either host with the shared adapter; every other + * value is the reason the case is Claude-only. On the Codex host a Claude-only + * case is SKIPPED with its reason, planned as excluded and deselected by the + * report with zero credit, never counted as failed. A case without metadata + * (every non-gate case) is Claude-only. + */ + +export type EvalHost = 'claude' | 'codex'; +export const EVAL_HOSTS: readonly EvalHost[] = ['claude', 'codex']; + +export function resolveEvalHost(env: NodeJS.ProcessEnv = process.env): EvalHost { + const raw = (env.EVALS_HOST ?? '').trim(); + if (!raw) return 'claude'; + if ((EVAL_HOSTS as readonly string[]).includes(raw)) return raw as EvalHost; + throw new Error(`EVALS_HOST must be one of ${EVAL_HOSTS.join(', ')}; got "${raw}"`); +} + +export const CLAUDE_ONLY_REASONS = { + pty: 'drives the interactive Claude Code TUI (plan mode, native AskUserQuestion) through the PTY harness', + sdk: 'uses the Claude Agent SDK (canUseTool answers and PreToolUse hooks)', + subagents: 'asserts Claude Agent/Task subagent events', + 'skill-tool': 'depends on the Claude Skill tool load and registration', + hermetic: 'a claude -p hermetic canary; the Codex host needs its own canary twin', + 'self-host': 'the Claude host calling /codex; on the Codex host /codex is the agent itself', + rework: 'parses Claude stream-json tool events or prescribes native Claude tools; not ported to the Codex host', + 'host-neutral': 'runs no model session, so a second host adds nothing', +} as const; + +export type ClaudeOnlyReason = keyof typeof CLAUDE_ONLY_REASONS; +export type HostSupport = 'both' | ClaudeOnlyReason; + +/** Adapter-only cases: final output, files, git state, Bash commands, server logs or browseErrors. */ +const ADAPTER_ONLY: readonly string[] = [ + 'benchmark-workflow', + 'browse-basic', + 'browse-snapshot', + 'canary-workflow', + 'context-recovery-artifacts', + 'context-restore-loads-latest', + 'context-save-writes-file', + 'cso-diff-mode', + 'design-review-detector-shim', + 'design-review-detector-shim-dom', + 'design-review-plugin-handoff', + 'diagram-triplet', + 'document-release', + 'gstack-upgrade-happy-path', + 'land-and-deploy-first-run', + 'land-and-deploy-review-gate', + 'land-and-deploy-workflow', + 'learnings-show', + 'operational-learning', + 'plan-design-review-no-ui-scope', + 'plan-eng-coverage-audit', + 'plan-review-report', + 'plan-tune-inspect', + 'qa-bootstrap', + 'qa-quick', + 'retro-base-branch', + 'review-army-delivery-audit', + 'review-army-json-findings', + 'review-army-migration-safety', + 'review-army-quality-score', + 'review-base-branch', + 'review-coverage-audit', + 'review-enum-completeness', + 'review-sql-injection', + 'review-test-value', + 'safety-pair-agent-block', + 'safety-ship-stale-evidence', + 'setup-deploy-workflow', + 'ship-base-branch', + 'ship-coverage-audit', + 'ship-coverage-value', + 'ship-local-workflow', + 'ship-triage', + 'skillmd-no-local-binary', + 'skillmd-outside-git', + 'skillmd-setup-discovery', + 'test-audit-report-only', + 'tpa-absent-darwin', + 'tpa-absent-linux', + 'tpa-apple-ban', + 'tpa-broken', + 'tpa-present' +]; + +const CLAUDE_ONLY: Readonly> = { + pty: ['autoplan-guard-pty', 'office-hours-auto-mode', 'plan-ceo-finding-floor', 'plan-ceo-review-plan-mode', + 'plan-design-with-ui-scope', 'plan-devex-finding-floor', 'plan-devex-review-plan-mode', 'plan-mode-no-op'], + sdk: ['auq-format-gate', 'investigate-owned-abort', 'investigate-owned-completion', 'investigate-owned-ending-error', + 'shared-libs-review-index-flags', 'shared-libs-review-lifecycle', 'shared-libs-review-path-eligibility', + 'shared-libs-review-prior-coverage', 'shared-libs-review-revalidation', 'ship-local-hook-preservation', + 'ship-managed-hook-refresh', 'ship-skipped-queued-finding', 'ship-unmanaged-hook-consent'], + subagents: ['docsync-spawned', 'review-army-perf-n-plus-one'], + 'skill-tool': ['skillify-approval-reject', 'skillify-happy-path', 'skillify-provenance-refusal'], + hermetic: ['hermetic-canary', 'hermetic-sentinel'], + 'self-host': ['safety-codex-consult-embed'], + rework: ['qa-functional-cli-fix', 'qa-functional-cli-report', 'qa-functional-webhook-fix', 'qa-functional-webhook-report', + 'qa-only-no-fix', 'review-dashboard-via', 'review-exploratory-small-cli', 'review-gate-integrity', + 'shared-libs-read-only', 'shared-libs-unsupported-git', 'ship-docsync-completion', 'ship-docsync-current', + 'ship-docsync-failure', 'ship-docsync-late-result', 'ship-docsync-launch-failure', 'ship-docsync-missing-asset', + 'ship-docsync-missing-marker', 'ship-docsync-recovery', 'ship-docsync-stale-after', 'ship-docsync-stale-before', + 'ship-docsync-store', 'ship-docsync-timeout-unsettled', 'ship-exploratory-late-input', 'ship-exploratory-plan-checks', + 'ship-exploratory-small-cli', 'ship-exploratory-unavailable'], + 'host-neutral': ['timeline-event-flow'], +}; + +export const E2E_HOSTS: Readonly> = Object.freeze(Object.fromEntries([ + ...ADAPTER_ONLY.map(id => [id, 'both'] as const), + ...(Object.entries(CLAUDE_ONLY) as Array<[ClaudeOnlyReason, readonly string[]]>).flatMap(([reason, ids]) => ids.map(id => [id, reason] as const)), +])); + +/** Why a case does not run on `host`, or null when it does. */ +export function hostSkipReason(id: string, host: EvalHost, hosts: Readonly> = E2E_HOSTS): string | null { + if (host === 'claude') return null; + const support = hosts[id]; + if (support === 'both') return null; + if (support === undefined) return `Claude-only on the ${host} host: no Codex host metadata (E2E_HOSTS covers the gate cases)`; + return `Claude-only on the ${host} host: ${CLAUDE_ONLY_REASONS[support]}`; +} diff --git a/test/helpers/eval-store.ts b/test/helpers/eval-store.ts index 609c6d099..ed8721f13 100644 --- a/test/helpers/eval-store.ts +++ b/test/helpers/eval-store.ts @@ -14,6 +14,7 @@ import * as os from 'os'; import { spawnSync } from 'child_process'; import { isManualReviewEntry } from './cookie-workflow-manual-review'; import type { ManualJudgeReview } from './cookie-workflow-manual-review'; +import type { CodexTokenUsage } from './codex-session-runner'; // v2: EvalTestEntry.harvest gains optional {insertions, deletions, net} and // may be explicitly null (arm-benchmark harvest-failure taxonomy). Readers @@ -67,6 +68,8 @@ export interface EvalTestEntry { cost_usd: number; /** False when the harness captured no billing, so cost_usd 0 means unknown. Absent means known. */ cost_known?: boolean; + /** Codex sessions: the token classes behind cost_usd. */ + token_usage?: CodexTokenUsage; /** Absent in older records means executed; reuse is never a new model run. */ execution?: 'executed' | 'reused'; reused_from?: { input_key: string; run_id: string; revision: string; completed_at: string }; diff --git a/test/helpers/free-fixtures-data.ts b/test/helpers/free-fixtures-data.ts index 7c724f1c6..317e26b5d 100644 --- a/test/helpers/free-fixtures-data.ts +++ b/test/helpers/free-fixtures-data.ts @@ -54,6 +54,7 @@ export const FREE_FIXTURES: Readonly> = { 'test/fixtures/claude-agent-payload-2.1.292.json': ['test/autoplan-publication-payload.test.ts', 'test/autoplan-schema-canary-compare.test.ts'], 'test/fixtures/claude-native-journal-roots-2.1.284.json': ['test/autoplan-publication-hook.test.ts', 'test/plan-count-transcript.test.ts'], 'test/fixtures/codex-boundary-commands.json': ['test/codex-boundary-evidence.test.ts'], + 'test/fixtures/codex-host/**': ['test/codex-host.test.ts'], 'test/fixtures/codex-review-sandbox-unavailable-37158847998.json': ['test/outside-voice-evidence.test.ts'], 'test/fixtures/codex-review/**': ['test/codex-hardening.test.ts', 'test/outside-review-severity.test.ts', 'test/outside-voice-evidence.test.ts', 'test/skill-validation.test.ts'], 'test/fixtures/codex-sandbox/**': ['test/codex-eval-recording.test.ts', 'test/codex-model-probe.test.ts', 'test/outside-review-result.test.ts', 'test/outside-voice-invocation.test.ts'], diff --git a/test/helpers/session-ledger.ts b/test/helpers/session-ledger.ts index 6e3c9e837..425323284 100644 --- a/test/helpers/session-ledger.ts +++ b/test/helpers/session-ledger.ts @@ -12,6 +12,7 @@ */ import * as fs from 'fs'; import * as path from 'path'; +import type { CodexTokenUsage } from './codex-session-runner'; export const SESSION_LEDGER_FILE = 'session-ledger.jsonl'; export const SESSION_EVIDENCE_MAX = 200; @@ -48,6 +49,10 @@ export interface SessionLedgerRow { liveness?: LivenessSummary; /** Whether the runner captured this session's billing (a terminal result with total_cost_usd). */ billed?: boolean; + /** Codex sessions: the model, its billed cost and token classes. */ + model?: string; + cost_usd?: number; + tokens?: CodexTokenUsage; } /** One line, control characters stripped, mentions neutralized, capped. */ diff --git a/test/helpers/session-runner.ts b/test/helpers/session-runner.ts index d40eb29e8..cc65a66ad 100644 --- a/test/helpers/session-runner.ts +++ b/test/helpers/session-runner.ts @@ -17,6 +17,8 @@ import { hermeticChildEnv, isHermeticEnabled } from './hermetic-env'; import { killProcessGroup } from '../../scripts/test-strict-output'; import { resolveEvalModel } from '../../lib/eval-model'; import { SessionObserver, appendSessionLedger, sessionKey, type SessionEnd } from './session-ledger'; +import { resolveEvalHost } from './e2e-hosts'; +import type { CodexTokenUsage } from './codex-session-runner'; const GSTACK_DEV_DIR = path.join(os.homedir(), '.gstack-dev'); const HEARTBEAT_PATH = path.join(GSTACK_DEV_DIR, 'e2e-live.json'); // heartbeat stays global @@ -55,6 +57,10 @@ export interface SkillTestResult { firstResponseMs: number; /** Peak latency between consecutive tool calls, in ms */ maxInterTurnMs: number; + /** Codex host: token classes behind costEstimate.estimatedCost. */ + tokenUsage?: CodexTokenUsage; + /** Codex host: false when the session completed no turn or its model has no price row. */ + costKnown?: boolean; } /** Local default startup grace: 90s covers observed API queue latency @@ -76,6 +82,14 @@ const BROWSE_ERROR_PATTERNS = [ /no such file or directory.*\bbrowse(?:\.exe)?(?=$|[\s'":),])/i, ]; +/** First match of each browse failure pattern in tool output plus stderr. */ +export function scanBrowseErrors(text: string): string[] { + return BROWSE_ERROR_PATTERNS.flatMap(pattern => { + const match = text.match(pattern); + return match ? [match[0].slice(0, 200)] : []; + }); +} + // --- Testable NDJSON parser --- export interface ParsedNDJSON { @@ -300,6 +314,7 @@ Setup, CLI startup and API queueing consume this same window; it never resets. Before source Reads, use Bash to run exactly \`date -u +%Y-%m-%dT%H:%M:%SZ\`. After each saved checkpoint, compare the latest evidence capture's printed completedAt with the times above; run that clock read again only when no capture has completed since your last clock read. When remaining time is at most ${reserve / 1000} seconds, prioritize the remaining required completion outputs and verification. No required content or gate may be skipped. If the clock read fails, report timing unavailable; do not invent remaining time or restart the deadline.`; systemPrompt = systemPrompt ? `${systemPrompt}\n\n${notice}` : notice; } + if (resolveEvalHost() === 'codex') return (await import('./codex-host-runner')).runCodexHostSkillTest({ ...options, appendSystemPrompt: systemPrompt, completionReserveMs: undefined }); // Set up per-run log directory if runId is provided let runDir: string | null = null; @@ -661,13 +676,7 @@ Before source Reads, use Bash to run exactly \`date -u +%Y-%m-%dT%H:%M:%SZ\`. Af const { transcript, resultLine, toolCalls } = parsed; const browseErrors: string[] = []; - const allText = toolCalls.filter(call => call.tool === 'Bash').map(call => call.output).join('\n') + '\n' + stderr; - for (const pattern of BROWSE_ERROR_PATTERNS) { - const match = allText.match(pattern); - if (match) { - browseErrors.push(match[0].slice(0, 200)); - } - } + browseErrors.push(...scanBrowseErrors(toolCalls.filter(call => call.tool === 'Bash').map(call => call.output).join('\n') + '\n' + stderr)); // Native Claude uses exit 1 for its structured max-turns result. Preserve // that existing semantic outcome only after a complete, uncancelled drain; diff --git a/test/paid-retry-supervision.test.ts b/test/paid-retry-supervision.test.ts index a132bd53a..95c5fd3b3 100644 --- a/test/paid-retry-supervision.test.ts +++ b/test/paid-retry-supervision.test.ts @@ -217,8 +217,9 @@ test('eval:bg runs the PR and release commands with their declared workers (cap: const cliOptions = (step: { run: string; env?: Record }) => { const prefix = /\bbun(?: --no-install)? run scripts\/test-paid-shards\.ts /.exec(step.run); expect(prefix).not.toBeNull(); - const args = step.run.slice(prefix!.index + prefix![0].length) - .replace(/\$\{\{\s*matrix\.slice\s*\}\}/g, '1').trim().split(/\s+/); + // One command line: a census arm loop continues past it. + const args = step.run.slice(prefix!.index + prefix![0].length).split('\n')[0]! + .replace(/\$\{\{\s*matrix\.slice\s*\}\}/g, '1').replace(/\$\{\{\s*matrix\.arm\s*\}\}/g, 'claude-opus').trim().split(/\s+/); return parseCliOptions(args, step.env ?? {}); }; @@ -242,10 +243,16 @@ test('both gate executors plan the complete census and supervise every planned s expect(planned.skipJudges).toBe(skipJudges); expect(execute[0].env.EVALS_CONCURRENCY).toBe('2'); expect(executor.strategy['fail-fast']).toBe(false); - expect(executor.strategy.matrix.slice).toBe(`\${{ fromJSON(needs.plan-slices.outputs.${prefix}slices) }}`); - // Per-slice ceilings (W2c/ENG-2), or the largest of them as one job cap. - expect([`\${{ fromJSON(needs.plan-slices.outputs.${prefix}slice_timeouts)[matrix.slice] }}`, - `\${{ fromJSON(needs.plan-slices.outputs.${prefix}timeout_minutes) }}`]).toContain(executor['timeout-minutes']); + if (jobName === 'gate-census') { + // One row per census arm and planned slice, each with its arm's planned slice ceiling (scripts/census-arms.ts). + expect(executor.strategy.matrix.include).toBe('${{ fromJSON(needs.plan-slices.outputs.gate_matrix) }}'); + expect(executor['timeout-minutes']).toBe('${{ matrix.timeout }}'); + } else { + expect(executor.strategy.matrix.slice).toBe(`\${{ fromJSON(needs.plan-slices.outputs.${prefix}slices) }}`); + // Per-slice ceilings (W2c/ENG-2), or the largest of them as one job cap. + expect([`\${{ fromJSON(needs.plan-slices.outputs.${prefix}slice_timeouts)[matrix.slice] }}`, + `\${{ fromJSON(needs.plan-slices.outputs.${prefix}timeout_minutes) }}`]).toContain(executor['timeout-minutes']); + } const manifest = buildRunManifest({ tier: 'gate', sliceBudgetMs: planned.sliceBudgetMs!, jobs: planned.jobs, evalsAll: true, env: { EVALS_ALL: '1' }, skipJudges }); const files = [...new Set(manifest.entries.filter(row => row.status === 'planned').map(row => shardFile(row.file)))]; expect(files).toContain('test/skill-e2e-ship-skip.test.ts'); @@ -262,8 +269,8 @@ test('both gate executors plan the complete census and supervise every planned s expect(manifest.sliceCount).toBeLessThanOrEqual(executor.strategy['max-parallel']); if (jobName === 'gate-census') { expect(emit[0].env.EVALS_ALL).toBe('1'); - expect(emit[0].run).toContain('--emit-plan /tmp/gate-census-plan/manifest.json'); - expect(execute[0].run).toContain('--plan /tmp/gate-census-plan/manifest.json --slice ${{ matrix.slice }}'); + expect(emit[0].run).toContain('--emit-plan "/tmp/gate-census-plan/$arm/manifest.json"'); + expect(execute[0].run).toContain('--plan /tmp/gate-census-plan/${{ matrix.arm }}/manifest.json --slice ${{ matrix.slice }}'); expect(executor.steps.some((step: any) => step.with?.name === 'gate-census-plan')).toBe(true); expect(executor.steps.filter((step: any) => step.run?.includes('--emit-plan '))).toHaveLength(0); } @@ -305,8 +312,10 @@ test('gate census requires all seven distinct slice results and its own reconcil const reconcile = report.steps.find((step: any) => step.id === 'gate-reconcile'); expect(report.needs).toContain('gate-census'); expect(reconcile.if).toBe('always()'); - expect(reconcile.run).toContain('EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --report /tmp/gate-census-report'); - expect(reconcile.run).toContain('exit=${PIPESTATUS[0]}'); + // One reconciliation per census arm, each against its own plan and slices. + expect(reconcile.run).toContain('dir="/tmp/gate-census-report/$arm"'); + expect(reconcile.run).toContain('EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --report "$dir"'); + expect(reconcile.run).toContain('code=${PIPESTATUS[0]}'); const failureGuards = report.steps.filter((step: any) => /Upsert tracking|Fail the workflow/.test(step.name ?? '')); expect(failureGuards).toHaveLength(2); for (const step of failureGuards) { diff --git a/test/paid-shards.test.ts b/test/paid-shards.test.ts index 78d3b00d9..08f8629ac 100644 --- a/test/paid-shards.test.ts +++ b/test/paid-shards.test.ts @@ -178,7 +178,7 @@ describe('tier lane skip (B5)', () => { } const workflow = fs.readFileSync(path.join(ROOT, '.github/workflows/evals-periodic.yml'), 'utf8'); expect(workflow.match(/--skip-judges/g)).toHaveLength(1); - expect(workflow).toMatch(/--tier gate --emit-plan \/tmp\/gate-census-plan\/manifest\.json --slice-budget 420 --jobs 2 --skip-judges/); + expect(workflow).toMatch(/--tier gate --emit-plan "\/tmp\/gate-census-plan\/\$arm\/manifest\.json" --slice-budget 420 --jobs 2 --skip-judges/); }); });