From 635535d3c1d08550992f89efdf32bf79bbac6610 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Mon, 19 Oct 2026 12:22:29 -0700 Subject: [PATCH] v1.91.150.0 test(evals): re-pin older paid models to the current generation; prosons and cookie-judge repairs (GSTA-49 TM3) (#3293) * feat(evals): Opus 5.5 is the paid gate's model under test; judge stays Fable (GSTA-49 TM1) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test: re-baseline old-model paid pins onto the current generation (TM3) Repair the prosons capture prompt that tripped the reasoning_extraction classifier, and restructure /setup-browser-cookies so the Aside probe, decisions and target order are unambiguous. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs: CHANGELOG entry for the TM3 re-baseline under [Unreleased] Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * chore: stamp v1.91.150.0 Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- BROWSER.md | 2 +- CHANGELOG.md | 13 +++ TODOS.md | 27 +++--- VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- docs/OVERLAY_BENCHMARK_CONTRACT.md | 8 +- package.json | 2 +- setup-browser-cookies/SKILL.md | 96 ++++++++------------- setup-browser-cookies/SKILL.md.tmpl | 54 ++++++++---- test/codex-e2e-sol-scope.test.ts | 4 +- test/cookie-workflow-judge-input.test.ts | 4 +- test/fixtures/overlay-nudges.ts | 12 +-- test/helpers/cookie-workflow-judge-input.ts | 2 +- test/helpers/manual-judge-review-fixture.ts | 21 ++++- test/office-hours-attempt.test.ts | 2 +- test/office-hours-phase4-caller.test.ts | 5 +- test/overlay-lifecycle.test.ts | 2 +- test/overlay-measurement.test.ts | 7 +- test/skill-e2e-design.test.ts | 4 +- test/skill-e2e-office-hours-phase4.test.ts | 3 +- test/skill-e2e-plan-prosons.test.ts | 19 ++-- test/skill-e2e-plan.test.ts | 8 +- test/skill-e2e-qa-bugs.test.ts | 3 +- test/skill-positional-literals.test.ts | 2 +- 24 files changed, 169 insertions(+), 135 deletions(-) diff --git a/BROWSER.md b/BROWSER.md index 75cd99b9e..396bb6e9b 100644 --- a/BROWSER.md +++ b/BROWSER.md @@ -584,7 +584,7 @@ SITE=app.example.com DOMAIN=example.com $B goto "https://$SITE" $B cookie-import-browser opera-gx --domain "$DOMAIN" --profile Default # or: opera -$B reload # confirm the intended account +$B reload # view the page; only --verify-auth proves sign-in ``` Run `$B cookie-import-browser` with no flags to see which browsers were detected. `--profile` can be omitted when only one profile has cookies for the domain. If the receipt reports App-Bound Encryption, run `$B handoff`, sign in to the intended account in the window that opens, then `$B resume` (needs a display). diff --git a/CHANGELOG.md b/CHANGELOG.md index 71b30aaaf..77b816346 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## [1.91.150.0] - 2026-10-19 + +**The paid evals that still pinned older models now run on the current generation, and two cases that went red on the move are repaired at their cause.** Garry's direction (2026-10-18) is to stop testing older frontier models, so these files are re-baselined as a fresh measured reference rather than held to a threshold change. + +### Changed +- **Old-model pins:** the `claude-opus-4-7` pins in `skill-e2e-plan`, `skill-e2e-plan-prosons`, `skill-e2e-design`, `skill-e2e-office-hours-phase4` and `skill-e2e-qa-bugs` now follow `resolveEvalModel('capture')` (Opus 5.5). The overlay benchmark fixtures pin `claude-opus-5-5` and `claude-sonnet-5-5`, the cookie workflow judge pins `claude-sonnet-5-5`, and `codex-sol-scope-termination` runs on `gpt-6.1-sol` while still rendering the `gpt-5.6-sol` overlay. +- **/setup-browser-cookies:** the workflow is now five ordered steps. It checks for Aside first and ends there on `READY`, since no import is needed. It then asks every decision (source and profile, domains and target, verification, storage reset, all cookies) before any `$B` command, prepares gstack's browser (verification variables before the daemon starts, the fallback, the `Mode: cdp` stop), imports, and reports on three lines: Imported, Storage reset, Sign-in. The `{{BROWSE_SETUP}}` block, which contradicted the Aside probe, is gone from this skill. +- **Gate integrity:** no threshold, trial count, case kind or assertion changed. Free-test edits follow the new cookie template: two phrase and order assertions, the approval fixture's reverse edits, and the positional-literals checksum list, which no longer includes this skill. + +### Fixed +- **plan-review-prosons-hardstop-neg:** on Opus 5.5 one trial in three stopped with a provider refusal (`stop_reason: refusal`, category `reasoning_extraction`) on the turn that writes the captured question. Replays of the refused request were stopped 7 of 16 times with the capture prompt's "verbatim" and "Do NOT paraphrase. This is a format-capture test." wording and 0 of 24 without it. The shared prosons capture prompt now asks for "the text of the single AskUserQuestion you would have made", with the same required parts. The decision content and every assertion are unchanged; all four prosons cases pass 3/3. +- **Cookie workflow judge:** on `claude-sonnet-5-5` it scored clarity 3, completeness 4, actionability 3 against the old template. The rewritten template scores 4/4/4 with unchanged thresholds. + ## [1.91.149.0] - 2026-10-19 **Paid eval cases can now run on the Codex host, and the Monday census runs two arms: Claude Opus 5.5 and Codex gpt-6.1-sol.** `EVALS_HOST=codex` runs the 52 adapter-only gate cases through a full Codex runtime per case. On its first pilot gpt-6.1-sol passed 48 of 52 with no retries, at $4.63 for the 52 cases, against $28.93 for the same cases on Fable (0.16×). Claude-only cases are skipped on Codex with a recorded reason and never count as failed. GSTA-49 TM2. diff --git a/TODOS.md b/TODOS.md index 25de1dcf9..8453a0b26 100644 --- a/TODOS.md +++ b/TODOS.md @@ -1645,19 +1645,24 @@ and `test/dx-selected-navigation-ap.test.ts`. One shared table run once against only after `engFirstReviewAUQ` checks native completion once at entry; today each branch gates it separately, so the change alters a paid verdict and needs its own paid run. -### P3: Re-pin the five remaining claude-opus-4-7 paid files +### P3: Follow-ups from re-baselining the old-model paid pins on the current generation -**What:** The 2026-09 audit moved seven paid evals to the default capture model (`resolveEvalModel('capture')`). -`skill-e2e-design`, `skill-e2e-office-hours-phase4`, `skill-e2e-plan-prosons` and `skill-e2e-plan` keep -`claude-opus-4-7` because six cases failed on the default model in one run (plan-design-review-plan-mode timeout, -office-hours-phase4-fork format, plan-review-prosons-neutral-neg missing output, plan-ceo-review-selective and -plan-eng-review 600 s timeouts, plan-ceo-review-expansion-energy posture score 3). plan-review-report, the one PR-gate -case in `skill-e2e-plan`, left the pin on 2026-10-18 and follows the capture default. `skill-e2e-qa-bugs` returned -to `claude-opus-4-7` after `qa-b6-static` timed out on the default model in two of three runs (census 36597762183 -and a targeted local rerun): each time the stream stopped mid-message, with no pending tool, right after the model -found the disabled submit button, and emitted nothing until the 300 s case deadline. They measure an old model. +**What:** GSTA-49 TM3 (2026-10-18) moved every remaining old-model paid pin to the current generation: the +`claude-opus-4-7` pins in `skill-e2e-{plan,plan-prosons,design,office-hours-phase4,qa-bugs}` follow +`resolveEvalModel('capture')`, the overlay fixtures pin `claude-opus-5-5` and `claude-sonnet-5-5`, the cookie workflow +judge pins `claude-sonnet-5-5`, and `codex-sol-scope-termination` runs on `gpt-6.1-sol` (still rendering the +`gpt-5.6-sol` overlay). Three things remain: -**Re-entry:** fix the prompt, budget or rubric so each case passes on the default model in one run, then drop the pin. +- The base `claude-dedicated-tools-vs-bash` fixture now duplicates `claude-dedicated-tools-vs-bash-opus-5-5`. +- The `opus-4-7-*` fixtures measure `model-overlays/opus-4-7.md` on Opus 5.5, which gets the generic `claude` + overlay in product. +- Opus 5.5 refuses (stop category `reasoning_extraction`) capture prompts that ask for the "verbatim text" of a + question "you would have made" with "Do NOT paraphrase"; that wording stopped 7 of 16 replays of a prosons + request and 0 of 24 without it. `skill-e2e-plan-prosons` dropped it; `skill-e2e-plan-format` and + `skill-e2e-office-hours-phase4` still use it. + +**Re-entry:** retire one dedicated-tools fixture, decide the opus-4-7 overlay's future, and drop the extraction +wording from the remaining capture prompts with a paid run each. ### P3: Retire the unused CEO payment seeder diff --git a/VERSION b/VERSION index bdda80070..d31a4e90c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.149.0 +1.91.150.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index f819c371f..8a1c9f422 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.149.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.150.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/docs/OVERLAY_BENCHMARK_CONTRACT.md b/docs/OVERLAY_BENCHMARK_CONTRACT.md index d6f3cc38e..84f87b4d6 100644 --- a/docs/OVERLAY_BENCHMARK_CONTRACT.md +++ b/docs/OVERLAY_BENCHMARK_CONTRACT.md @@ -14,8 +14,10 @@ Selection IDs for the six retained paid cases stay unchanged. ## Blocking checks Both arms use the real Claude Code preset; ON appends the resolved overlay and -OFF does not. Each retained fixture runs ten samples per arm, separately for -`claude-opus-4-7` and `claude-sonnet-4-6`. Each model keeps its own results. +OFF does not. Each retained fixture runs ten samples per arm on its pinned +model: `claude-opus-5-5`, plus `claude-sonnet-5-5` for the dedicated-tools `-sonnet` case. +Each model keeps its own results. The `opus-4-7-*` cases keep their overlay +(`model-overlays/opus-4-7.md`) but run on Opus 5.5 since GSTA-49. Every planned sample in both arms must execute successfully, produce exactly one successful native terminal result without an error flag, preserve workspace @@ -38,7 +40,7 @@ are not reinterpreted. | `opus-4-7-effort-match-trivial` | Read-only workspace; native final JSON contains the exact version string | Correct answer in every ON sample | Native `usage.output_tokens_details.thinking_tokens` | | `opus-4-7-literal-interpretation` | Only `src/auth.ts`, `src/billing.ts`, and `src/notifications.ts` may change; public tests stay frozen; independent behavior checks run outside the writable fixture | All three target behaviors pass in every ON sample | Correct target behaviors, 0..3 | -Each family also has a `-sonnet` case. Output validation checks the native final +The dedicated-tools family also has `-sonnet` and `-opus-5-5` cases. Output validation checks the native final answer, so a correct word mentioned earlier cannot conceal a wrong final answer. The read-only prompts explicitly request the JSON shape. These exact-answer prompts are new in v2: compare ON/OFF within the same contract and fixture, not diff --git a/package.json b/package.json index 0a91faf86..302f0c4c1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.149", + "version": "1.91.150", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/setup-browser-cookies/SKILL.md b/setup-browser-cookies/SKILL.md index 1aa2a0433..830f9c7c2 100644 --- a/setup-browser-cookies/SKILL.md +++ b/setup-browser-cookies/SKILL.md @@ -152,83 +152,59 @@ Skills that run plan reviews (`/plan-*-review`, `/codex review`) include the EXI # Setup Browser Cookies -## 1. Choose the browser +## 1. Check for Aside -Use this checkout as the gstack root if it contains `BROWSER.md` and `browse/SKILL.md`; otherwise use the installed root containing `bin/gstack-skill-start`, never a generated host stub. Read that root's `browse/SKILL.md` **BROWSER SETUP** section and run its probe first. On `READY`, stop importing: use Aside's sessions or ask the user to sign in there. Otherwise follow the probe's fallback handling, then continue below. +Use this checkout as the gstack root if it contains `BROWSER.md` and `browse/SKILL.md`; otherwise use the installed root containing `bin/gstack-skill-start`, never a generated host stub. Read that root's `browse/SKILL.md` **BROWSER SETUP** section and run its Aside probe. -## SETUP (run this check BEFORE any browse command) +- **Aside probe prints `READY`:** no import is needed, because Aside already has the user's signed-in sessions. Tell the user to use Aside, or to sign in there, and end this skill. +- **Any other result:** continue with gstack's own headless browser (`$B`). + +## 2. Get every decision from the user first + +Ask these together, before running any `$B` command. Never guess an answer or treat the default Comet browser as consent. + +1. **Source browser and profile.** `--profile` takes the profile's directory (`Default`, `Profile 2`), not its display name. A profile that cannot be read is unknown, not empty. +2. **Domains to import,** and the **target**: one page on a chosen domain that the session should open, by URL. +3. **Sign-in verification (`--verify-auth`)?** Default no. It reloads the target and reports **verified** only after a successful same-origin response with exactly one visible element, found by a CSS selector, whose whitespace-normalized text equals the expected identity. If yes, the user supplies the selector and the expected identity privately. +4. **Clear stale storage (`--clear-storage`)?** Default no; needs explicit approval. Chromium only. It clears localStorage for the target's origin (shared by that origin's tabs) and sessionStorage for the target tab, and nothing else; other browser engines reject the reset but still import. A failed reset may already have cleared some storage and is not rolled back. +5. **Every cookie in the profile (`--all`)?** Default no; needs explicit consent. It cannot be combined with a domain or with clearing storage. + +Use the picker when any of answers 1-2 is still open; it asks for them. Use direct import when the user has named the browser, the profile directory and the domains. + +## 3. Prepare gstack's browser + +1. If the user wants verification, have them set `GSTACK_COOKIE_AUTH_SELECTOR` and `GSTACK_COOKIE_AUTH_EXPECTED_IDENTITY` privately in the shell that runs `$B`. The browser daemon reads them only when it starts, so if a gstack daemon is already running, run `$B stop` first. Never invent the values. +2. Follow `browse/SKILL.md`'s **Browser fallback** section to find `$B`, building it if the check prints `NEEDS_SETUP`. +3. Run `$B status`. If it reports `Mode: cdp`, `$B` is attached to the user's real browser and already has its sessions: tell the user no import is needed and end this skill. +4. Open the target URL from answer 2 and leave that tab alone until the import finishes: ```bash -_ROOT=$(git rev-parse --show-toplevel 2>/dev/null) -B="" -[ -n "$_ROOT" ] && [ -x "$_ROOT/.claude/skills/gstack/browse/dist/browse" ] && B="$_ROOT/.claude/skills/gstack/browse/dist/browse" -[ -z "$B" ] && B="$HOME/.claude/skills/gstack/browse/dist/browse" -if [ -x "$B" ] && "${B%/browse/*}/bin/gstack-browser-ensure"; then - echo "READY: $B" -else - echo "NEEDS_SETUP" -fi +$B goto "" ``` -`gstack-browser-ensure` installs Chromium on first use when the install skipped it (lazy browser); it prints `BROWSER_OK` or `BROWSER_UNAVAILABLE ` first. +## 4. Import -If `NEEDS_SETUP`: -1. Tell the user: "gstack browse needs a one-time build (~10 seconds). OK to proceed?" Then STOP and wait. -2. Run: `cd && ./setup` -3. If `bun` is not installed: - ```bash - if ! command -v bun >/dev/null 2>&1; then - BUN_VERSION="1.4.2" - BUN_INSTALL_SHA="bab8acfb046aac8c72407bdcce903957665d655d7acaa3e11c7c4616beae68dd" - tmpfile=$(mktemp "${TMPDIR:-/tmp}/bun-install.XXXXXX") - curl -fsSL "https://bun.sh/install" -o "$tmpfile" - # shasum is macOS/perl; coreutils-only Linux ships sha256sum instead — - # resolve whichever exists so the verify never fails on a missing tool. - if command -v sha256sum >/dev/null 2>&1; then - actual_sha=$(sha256sum < "$tmpfile" | awk '{print $(1)}') - else - actual_sha=$(shasum -a 256 < "$tmpfile" | awk '{print $(1)}') - fi - if [ "$actual_sha" != "$BUN_INSTALL_SHA" ]; then - echo "ERROR: bun install script checksum mismatch" >&2 - echo " expected: $BUN_INSTALL_SHA" >&2 - echo " got: $actual_sha" >&2 - rm "$tmpfile"; exit 1 - fi - bash "$tmpfile" "bun-v$BUN_VERSION" - rm "$tmpfile" - fi - ``` - -```bash -$B status -``` -If status says `Mode: cdp`, stop: the real browser already has sessions. - -## 2. Confirm options before import - -Open the known target and keep its tab unchanged. Both options default off and require explicit request: - -- **`--verify-auth` / picker checkbox:** reloads the target. Have the user privately configure daemon `GSTACK_COOKIE_AUTH_SELECTOR` and `GSTACK_COOKIE_AUTH_EXPECTED_IDENTITY` **before startup**. Never invent values or assume CLI env reconfigures an existing daemon. Missing config rejects before mutation. Require a successful same-origin response and exactly one visible element whose whitespace-normalized text exactly matches the expected identity. -- **`--clear-storage`:** for suspected stale storage, obtain explicit approval. Chromium only: clears captured-origin localStorage (shared across same-origin context tabs) and target-tab sessionStorage. Other origins, other tabs' sessionStorage, IndexedDB, and service workers stay intact. It uses an isolated world/native deadline; other engines reject reset, not imports/auth checks. Never auto-approve or claim rollback after partial failure. - -## 3. Select source and scope +Picker (the user confirms the browser, profile and domains there, ticks verification if they asked for it, then tells you when done; the link works once, for five minutes, so rerun the command if it expires): ```bash $B cookie-import-browser ``` -Ask the user to choose browser, account/profile, and domains, then say when done. Never guess accounts or treat default Comet as consent. Unreadable profiles are unknown, not empty. Rerun for an expired five-minute one-use link. +Direct import: run `$B cookie-import-browser` with the chosen browser name, one `--domain` per run and `--profile` set to the profile directory (not its display name); add `--verify-auth` or `--clear-storage` only if the user said yes in step 2. -Direct import: pass the chosen browser and `--domain` after navigating to a matching target. `--profile` takes a directory, not a display name; omit only for an unambiguous relevant profile, otherwise use the picker. `--all` requires consent for all non-expired profile cookies; it cannot accompany `--domain` or `--clear-storage`. +If the import rejects because verification settings are missing, nothing changed: tell the user which variable is missing and return to step 3. Read that same root's `BROWSER.md`, **Choosing a source and checking sign-in**, for other errors and receipt failure reasons; it has the error tables, examples, profile labels, supported sources and platform setup. -Read that same root's `BROWSER.md`, **Choosing a source and checking sign-in**, for examples, profile labels, supported sources and platform setup. +## 5. Report honestly -## 4. Report honestly +Report from the import receipt, not from raw `$B cookies`, as three lines: -Report receipt/picker counts, partial/zero/error and reset outcomes, not raw `$B cookies`. Imports affect the context, not one tab. **Not checked** means no requested check; **not verified** means it failed; **verified** requires positive target evidence. Zero imports, counts, or HTTP 200 never prove login. +- `Imported: (, as the receipt says) from for ` +- `Storage reset: ` +- `Sign-in: ` -Never request/publish cookie values, passwords, identity/profile text, session details, or raw errors in public logs. +**Not checked** means verification was not requested; **not verified** means it ran and failed; **verified** needs the identity match. Cookie counts, zero imports and HTTP 200 never prove sign-in. Imported cookies apply to the whole browser context, not one tab. + +Never request or publish cookie values, passwords, identity or profile text, session details, or raw errors in public logs. ## Platform boundaries diff --git a/setup-browser-cookies/SKILL.md.tmpl b/setup-browser-cookies/SKILL.md.tmpl index f6ed3f61e..82313c668 100644 --- a/setup-browser-cookies/SKILL.md.tmpl +++ b/setup-browser-cookies/SKILL.md.tmpl @@ -21,41 +21,59 @@ allowed-tools: # Setup Browser Cookies -## 1. Choose the browser +## 1. Check for Aside -Use this checkout as the gstack root if it contains `BROWSER.md` and `browse/SKILL.md`; otherwise use the installed root containing `bin/gstack-skill-start`, never a generated host stub. Read that root's `browse/SKILL.md` **BROWSER SETUP** section and run its probe first. On `READY`, stop importing: use Aside's sessions or ask the user to sign in there. Otherwise follow the probe's fallback handling, then continue below. +Use this checkout as the gstack root if it contains `BROWSER.md` and `browse/SKILL.md`; otherwise use the installed root containing `bin/gstack-skill-start`, never a generated host stub. Read that root's `browse/SKILL.md` **BROWSER SETUP** section and run its Aside probe. -{{BROWSE_SETUP}} +- **Aside probe prints `READY`:** no import is needed, because Aside already has the user's signed-in sessions. Tell the user to use Aside, or to sign in there, and end this skill. +- **Any other result:** continue with gstack's own headless browser (`$B`). + +## 2. Get every decision from the user first + +Ask these together, before running any `$B` command. Never guess an answer or treat the default Comet browser as consent. + +1. **Source browser and profile.** `--profile` takes the profile's directory (`Default`, `Profile 2`), not its display name. A profile that cannot be read is unknown, not empty. +2. **Domains to import,** and the **target**: one page on a chosen domain that the session should open, by URL. +3. **Sign-in verification (`--verify-auth`)?** Default no. It reloads the target and reports **verified** only after a successful same-origin response with exactly one visible element, found by a CSS selector, whose whitespace-normalized text equals the expected identity. If yes, the user supplies the selector and the expected identity privately. +4. **Clear stale storage (`--clear-storage`)?** Default no; needs explicit approval. Chromium only. It clears localStorage for the target's origin (shared by that origin's tabs) and sessionStorage for the target tab, and nothing else; other browser engines reject the reset but still import. A failed reset may already have cleared some storage and is not rolled back. +5. **Every cookie in the profile (`--all`)?** Default no; needs explicit consent. It cannot be combined with a domain or with clearing storage. + +Use the picker when any of answers 1-2 is still open; it asks for them. Use direct import when the user has named the browser, the profile directory and the domains. + +## 3. Prepare gstack's browser + +1. If the user wants verification, have them set `GSTACK_COOKIE_AUTH_SELECTOR` and `GSTACK_COOKIE_AUTH_EXPECTED_IDENTITY` privately in the shell that runs `$B`. The browser daemon reads them only when it starts, so if a gstack daemon is already running, run `$B stop` first. Never invent the values. +2. Follow `browse/SKILL.md`'s **Browser fallback** section to find `$B`, building it if the check prints `NEEDS_SETUP`. +3. Run `$B status`. If it reports `Mode: cdp`, `$B` is attached to the user's real browser and already has its sessions: tell the user no import is needed and end this skill. +4. Open the target URL from answer 2 and leave that tab alone until the import finishes: ```bash -$B status +$B goto "" ``` -If status says `Mode: cdp`, stop: the real browser already has sessions. -## 2. Confirm options before import +## 4. Import -Open the known target and keep its tab unchanged. Both options default off and require explicit request: - -- **`--verify-auth` / picker checkbox:** reloads the target. Have the user privately configure daemon `GSTACK_COOKIE_AUTH_SELECTOR` and `GSTACK_COOKIE_AUTH_EXPECTED_IDENTITY` **before startup**. Never invent values or assume CLI env reconfigures an existing daemon. Missing config rejects before mutation. Require a successful same-origin response and exactly one visible element whose whitespace-normalized text exactly matches the expected identity. -- **`--clear-storage`:** for suspected stale storage, obtain explicit approval. Chromium only: clears captured-origin localStorage (shared across same-origin context tabs) and target-tab sessionStorage. Other origins, other tabs' sessionStorage, IndexedDB, and service workers stay intact. It uses an isolated world/native deadline; other engines reject reset, not imports/auth checks. Never auto-approve or claim rollback after partial failure. - -## 3. Select source and scope +Picker (the user confirms the browser, profile and domains there, ticks verification if they asked for it, then tells you when done; the link works once, for five minutes, so rerun the command if it expires): ```bash $B cookie-import-browser ``` -Ask the user to choose browser, account/profile, and domains, then say when done. Never guess accounts or treat default Comet as consent. Unreadable profiles are unknown, not empty. Rerun for an expired five-minute one-use link. +Direct import: run `$B cookie-import-browser` with the chosen browser name, one `--domain` per run and `--profile` set to the profile directory (not its display name); add `--verify-auth` or `--clear-storage` only if the user said yes in step 2. -Direct import: pass the chosen browser and `--domain` after navigating to a matching target. `--profile` takes a directory, not a display name; omit only for an unambiguous relevant profile, otherwise use the picker. `--all` requires consent for all non-expired profile cookies; it cannot accompany `--domain` or `--clear-storage`. +If the import rejects because verification settings are missing, nothing changed: tell the user which variable is missing and return to step 3. Read that same root's `BROWSER.md`, **Choosing a source and checking sign-in**, for other errors and receipt failure reasons; it has the error tables, examples, profile labels, supported sources and platform setup. -Read that same root's `BROWSER.md`, **Choosing a source and checking sign-in**, for examples, profile labels, supported sources and platform setup. +## 5. Report honestly -## 4. Report honestly +Report from the import receipt, not from raw `$B cookies`, as three lines: -Report receipt/picker counts, partial/zero/error and reset outcomes, not raw `$B cookies`. Imports affect the context, not one tab. **Not checked** means no requested check; **not verified** means it failed; **verified** requires positive target evidence. Zero imports, counts, or HTTP 200 never prove login. +- `Imported: (, as the receipt says) from for ` +- `Storage reset: ` +- `Sign-in: ` -Never request/publish cookie values, passwords, identity/profile text, session details, or raw errors in public logs. +**Not checked** means verification was not requested; **not verified** means it ran and failed; **verified** needs the identity match. Cookie counts, zero imports and HTTP 200 never prove sign-in. Imported cookies apply to the whole browser context, not one tab. + +Never request or publish cookie values, passwords, identity or profile text, session details, or raw errors in public logs. ## Platform boundaries diff --git a/test/codex-e2e-sol-scope.test.ts b/test/codex-e2e-sol-scope.test.ts index 862f7b8de..c3f59a619 100644 --- a/test/codex-e2e-sol-scope.test.ts +++ b/test/codex-e2e-sol-scope.test.ts @@ -180,14 +180,14 @@ TODO: consider migrating this example to a larger configuration framework. name: 'codex-sol-scope-termination', suite: 'codex-e2e-sol-scope', budgetMs: CAPTURE_MS, - model: 'gpt-5.6-sol', + model: 'gpt-6.1-sol', outputLimit: Infinity, run: (signal) => runCodexSkill({ skillDir, skillName: 'gstack-investigate', cwd: scratch, sandbox: 'workspace-write', - model: 'gpt-5.6-sol', + model: 'gpt-6.1-sol', configOverrides: ['model_reasoning_effort="high"'], ignoreUserConfig: true, timeoutMs: CODEX_TIMEOUT_MS, diff --git a/test/cookie-workflow-judge-input.test.ts b/test/cookie-workflow-judge-input.test.ts index c245e04ab..a8a268a3b 100644 --- a/test/cookie-workflow-judge-input.test.ts +++ b/test/cookie-workflow-judge-input.test.ts @@ -244,8 +244,8 @@ describe('cookie workflow judge input', () => { expect(body).toContain('Use this checkout as the gstack root if it contains `BROWSER.md` and `browse/SKILL.md`'); expect(body).toContain('the installed root containing `bin/gstack-skill-start`, never a generated host stub'); expect(body).toContain("Read that root's `browse/SKILL.md`"); - expect(body).toContain('On `READY`, stop importing'); - expect(body.indexOf('**BROWSER SETUP**')).toBeLessThan(body.indexOf('## SETUP')); + expect(body).toContain('**Aside probe prints `READY`:** no import is needed'); + expect(body.indexOf('**BROWSER SETUP**')).toBeLessThan(body.indexOf('**Browser fallback**')); expect(body).toContain("Read that same root's `BROWSER.md`, **Choosing a source and checking sign-in**"); expect(body).not.toContain('../BROWSER.md'); } diff --git a/test/fixtures/overlay-nudges.ts b/test/fixtures/overlay-nudges.ts index 0b499d785..5baac1656 100644 --- a/test/fixtures/overlay-nudges.ts +++ b/test/fixtures/overlay-nudges.ts @@ -202,7 +202,7 @@ export const OVERLAY_FIXTURES: OverlayFixture[] = [ { id: 'claude-dedicated-tools-vs-bash', overlayPath: 'model-overlays/claude.md', - model: 'claude-opus-4-7', + model: 'claude-opus-5-5', trials: 10, concurrency: 3, direction: 'lower_is_better', @@ -241,7 +241,7 @@ export const OVERLAY_FIXTURES: OverlayFixture[] = [ { id: 'opus-4-7-effort-match-trivial', overlayPath: 'model-overlays/opus-4-7.md', - model: 'claude-opus-4-7', + model: 'claude-opus-5-5', trials: 10, concurrency: 3, direction: 'lower_is_better', @@ -269,7 +269,7 @@ export const OVERLAY_FIXTURES: OverlayFixture[] = [ { id: 'opus-4-7-literal-interpretation', overlayPath: 'model-overlays/opus-4-7.md', - model: 'claude-opus-4-7', + model: 'claude-opus-5-5', trials: 10, concurrency: 3, direction: 'higher_is_better', @@ -290,16 +290,16 @@ export const OVERLAY_FIXTURES: OverlayFixture[] = [ }, // ========================================================================= - // Sonnet 4.6 variants of the Opus-4.7 fixtures. + // Sonnet 5.5 variant of the Opus 5.5 dedicated-tools fixture. // - // Same overlays, prompts, metrics and trial counts; different pinned model. + // Same overlay, prompt, metric and trial count; different pinned model. // Model effects remain separate instead of pooling their observations. // ========================================================================= { id: 'claude-dedicated-tools-vs-bash-sonnet', overlayPath: 'model-overlays/claude.md', - model: 'claude-sonnet-4-6', + model: 'claude-sonnet-5-5', trials: 10, concurrency: 3, direction: 'lower_is_better', diff --git a/test/helpers/cookie-workflow-judge-input.ts b/test/helpers/cookie-workflow-judge-input.ts index 5e595f32d..814d66892 100644 --- a/test/helpers/cookie-workflow-judge-input.ts +++ b/test/helpers/cookie-workflow-judge-input.ts @@ -4,7 +4,7 @@ import { join } from 'node:path'; import { buildWorkflowJudgePrompt, type WorkflowJudgeFile, type WorkflowJudgeInput } from './workflow-judge-input'; export const COOKIE_WORKFLOW_JUDGE = { - model: 'claude-sonnet-4-6', + model: 'claude-sonnet-5-5', judgeContext: 'a fallback-browser cookie import workflow', judgeGoal: 'how to select an authorized source browser, profile, and domain without guessing an account; configure optional authentication verification before mutation; obtain explicit consent for precisely scoped storage reset; distinguish copied cookies from positive sign-in evidence; and recover within the documented platform and privacy boundaries', thresholds: { clarity: 4, completeness: 3, actionability: 4 }, diff --git a/test/helpers/manual-judge-review-fixture.ts b/test/helpers/manual-judge-review-fixture.ts index ded8a3508..0acb4f5d5 100644 --- a/test/helpers/manual-judge-review-fixture.ts +++ b/test/helpers/manual-judge-review-fixture.ts @@ -46,14 +46,31 @@ const LATER_ENTRYPOINT_EDITS: Array<[string, string]> = [ ['bash "$tmpfile" "bun-v$BUN_VERSION"', 'BUN_VERSION="$BUN_VERSION" bash "$tmpfile"'], ]; +// The GSTA-49 cookie-workflow repair rewrote the entrypoint's steps 1-5 (split +// the two probe READY outcomes, dropped the duplicate SETUP block for the browse +// fallback, moved source selection before the target, asked about verification +// before the daemon starts, added a report shape) and one BROWSER.md comment. +// Reversing it first restores the pre-repair text, which the edits below then +// take back further. +const LATER_COOKIE_ORDER_EDITS: Array<[string, string]> = [ + ["## 1. Check for Aside\n\nUse this checkout as the gstack root if it contains `BROWSER.md` and `browse/SKILL.md`; otherwise use the installed root containing `bin/gstack-skill-start`, never a generated host stub. Read that root's `browse/SKILL.md` **BROWSER SETUP** section and run its Aside probe.\n\n- **Aside probe prints `READY`:** no import is needed, because Aside already has the user's signed-in sessions. Tell the user to use Aside, or to sign in there, and end this skill.\n- **Any other result:** continue with gstack's own headless browser (`$B`).\n\n## 2. Get every decision from the user first\n\nAsk these together, before running any `$B` command. Never guess an answer or treat the default Comet browser as consent.\n\n1. **Source browser and profile.** `--profile` takes the profile's directory (`Default`, `Profile 2`), not its display name. A profile that cannot be read is unknown, not empty.\n2. **Domains to import,** and the **target**: one page on a chosen domain that the session should open, by URL.\n3. **Sign-in verification (`--verify-auth`)?** Default no. It reloads the target and reports **verified** only after a successful same-origin response with exactly one visible element, found by a CSS selector, whose whitespace-normalized text equals the expected identity. If yes, the user supplies the selector and the expected identity privately.\n4. **Clear stale storage (`--clear-storage`)?** Default no; needs explicit approval. Chromium only. It clears localStorage for the target's origin (shared by that origin's tabs) and sessionStorage for the target tab, and nothing else; other browser engines reject the reset but still import. A failed reset may already have cleared some storage and is not rolled back.\n5. **Every cookie in the profile (`--all`)?** Default no; needs explicit consent. It cannot be combined with a domain or with clearing storage.\n\nUse the picker when any of answers 1-2 is still open; it asks for them. Use direct import when the user has named the browser, the profile directory and the domains.\n\n## 3. Prepare gstack's browser\n\n1. If the user wants verification, have them set `GSTACK_COOKIE_AUTH_SELECTOR` and `GSTACK_COOKIE_AUTH_EXPECTED_IDENTITY` privately in the shell that runs `$B`. The browser daemon reads them only when it starts, so if a gstack daemon is already running, run `$B stop` first. Never invent the values.\n2. Follow `browse/SKILL.md`'s **Browser fallback** section to find `$B`, building it if the check prints `NEEDS_SETUP`.\n3. Run `$B status`. If it reports `Mode: cdp`, `$B` is attached to the user's real browser and already has its sessions: tell the user no import is needed and end this skill.\n4. Open the target URL from answer 2 and leave that tab alone until the import finishes:\n\n```bash\n$B goto \"\"\n```\n\n## 4. Import\n\nPicker (the user confirms the browser, profile and domains there, ticks verification if they asked for it, then tells you when done; the link works once, for five minutes, so rerun the command if it expires):\n\n```bash\n$B cookie-import-browser\n```\n\nDirect import: run `$B cookie-import-browser` with the chosen browser name, one `--domain` per run and `--profile` set to the profile directory (not its display name); add `--verify-auth` or `--clear-storage` only if the user said yes in step 2.\n\nIf the import rejects because verification settings are missing, nothing changed: tell the user which variable is missing and return to step 3. Read that same root's `BROWSER.md`, **Choosing a source and checking sign-in**, for other errors and receipt failure reasons; it has the error tables, examples, profile labels, supported sources and platform setup.\n\n## 5. Report honestly\n\nReport from the import receipt, not from raw `$B cookies`, as three lines:\n\n- `Imported: (, as the receipt says) from for `\n- `Storage reset: `\n- `Sign-in: `\n\n**Not checked** means verification was not requested; **not verified** means it ran and failed; **verified** needs the identity match. Cookie counts, zero imports and HTTP 200 never prove sign-in. Imported cookies apply to the whole browser context, not one tab.\n\nNever request or publish cookie values, passwords, identity or profile text, session details, or raw errors in public logs.\n\n## Platform boundaries", "## 1. Choose the browser\n\nUse this checkout as the gstack root if it contains `BROWSER.md` and `browse/SKILL.md`; otherwise use the installed root containing `bin/gstack-skill-start`, never a generated host stub. Read that root's `browse/SKILL.md` **BROWSER SETUP** section and run its probe first. On `READY`, stop importing: use Aside's sessions or ask the user to sign in there. Otherwise follow the probe's fallback handling, then continue below.\n\n## SETUP (run this check BEFORE any browse command)\n\n```bash\n_ROOT=$(git rev-parse --show-toplevel 2>/dev/null)\nB=\"\"\n[ -n \"$_ROOT\" ] && [ -x \"$_ROOT/.claude/skills/gstack/browse/dist/browse\" ] && B=\"$_ROOT/.claude/skills/gstack/browse/dist/browse\"\n[ -z \"$B\" ] && B=\"$HOME/.claude/skills/gstack/browse/dist/browse\"\nif [ -x \"$B\" ] && \"${B%/browse/*}/bin/gstack-browser-ensure\"; then\n echo \"READY: $B\"\nelse\n echo \"NEEDS_SETUP\"\nfi\n```\n\n`gstack-browser-ensure` installs Chromium on first use when the install skipped it (lazy browser); it prints `BROWSER_OK` or `BROWSER_UNAVAILABLE ` first.\n\nIf `NEEDS_SETUP`:\n1. Tell the user: \"gstack browse needs a one-time build (~10 seconds). OK to proceed?\" Then STOP and wait.\n2. Run: `cd && ./setup`\n3. If `bun` is not installed:\n ```bash\n if ! command -v bun >/dev/null 2>&1; then\n BUN_VERSION=\"1.4.2\"\n BUN_INSTALL_SHA=\"bab8acfb046aac8c72407bdcce903957665d655d7acaa3e11c7c4616beae68dd\"\n tmpfile=$(mktemp \"${TMPDIR:-/tmp}/bun-install.XXXXXX\")\n curl -fsSL \"https://bun.sh/install\" -o \"$tmpfile\"\n # shasum is macOS/perl; coreutils-only Linux ships sha256sum instead —\n # resolve whichever exists so the verify never fails on a missing tool.\n if command -v sha256sum >/dev/null 2>&1; then\n actual_sha=$(sha256sum < \"$tmpfile\" | awk '{print $(1)}')\n else\n actual_sha=$(shasum -a 256 < \"$tmpfile\" | awk '{print $(1)}')\n fi\n if [ \"$actual_sha\" != \"$BUN_INSTALL_SHA\" ]; then\n echo \"ERROR: bun install script checksum mismatch\" >&2\n echo \" expected: $BUN_INSTALL_SHA\" >&2\n echo \" got: $actual_sha\" >&2\n rm \"$tmpfile\"; exit 1\n fi\n bash \"$tmpfile\" \"bun-v$BUN_VERSION\"\n rm \"$tmpfile\"\n fi\n ```\n\n```bash\n$B status\n```\nIf status says `Mode: cdp`, stop: the real browser already has sessions.\n\n## 2. Confirm options before import\n\nOpen the known target and keep its tab unchanged. Both options default off and require explicit request:\n\n- **`--verify-auth` / picker checkbox:** reloads the target. Have the user privately configure daemon `GSTACK_COOKIE_AUTH_SELECTOR` and `GSTACK_COOKIE_AUTH_EXPECTED_IDENTITY` **before startup**. Never invent values or assume CLI env reconfigures an existing daemon. Missing config rejects before mutation. Require a successful same-origin response and exactly one visible element whose whitespace-normalized text exactly matches the expected identity.\n- **`--clear-storage`:** for suspected stale storage, obtain explicit approval. Chromium only: clears captured-origin localStorage (shared across same-origin context tabs) and target-tab sessionStorage. Other origins, other tabs' sessionStorage, IndexedDB, and service workers stay intact. It uses an isolated world/native deadline; other engines reject reset, not imports/auth checks. Never auto-approve or claim rollback after partial failure.\n\n## 3. Select source and scope\n\n```bash\n$B cookie-import-browser\n```\n\nAsk the user to choose browser, account/profile, and domains, then say when done. Never guess accounts or treat default Comet as consent. Unreadable profiles are unknown, not empty. Rerun for an expired five-minute one-use link.\n\nDirect import: pass the chosen browser and `--domain` after navigating to a matching target. `--profile` takes a directory, not a display name; omit only for an unambiguous relevant profile, otherwise use the picker. `--all` requires consent for all non-expired profile cookies; it cannot accompany `--domain` or `--clear-storage`.\n\nRead that same root's `BROWSER.md`, **Choosing a source and checking sign-in**, for examples, profile labels, supported sources and platform setup.\n\n## 4. Report honestly\n\nReport receipt/picker counts, partial/zero/error and reset outcomes, not raw `$B cookies`. Imports affect the context, not one tab. **Not checked** means no requested check; **not verified** means it failed; **verified** requires positive target evidence. Zero imports, counts, or HTTP 200 never prove login.\n\nNever request/publish cookie values, passwords, identity/profile text, session details, or raw errors in public logs.\n\n## Platform boundaries"], + ["# view the page; only --verify-auth proves sign-in", "# confirm the intended account"], +]; + export function approvedCookieWorkflowSource(source: string): string { let removedLines = 0; let removedEntrypointLines = 0; - let historical = source + let reordered = source; + for (const [later, earlier] of LATER_COOKIE_ORDER_EDITS) { + if (!reordered.includes(later)) continue; + removedEntrypointLines += later.split('\n').length - earlier.split('\n').length; + reordered = reordered.replace(later, earlier); + } + let historical = reordered .replace('sha256sum < "$tmpfile" | awk \'{print $(1)}\'', 'sha256sum "$tmpfile" | awk \'{print $1}\'') .replace('shasum -a 256 < "$tmpfile" | awk \'{print $(1)}\'', 'shasum -a 256 "$tmpfile" | awk \'{print $1}\'') .replace(LATER_BROWSER_BLOCK, block => { removedLines = block.split('\n').length - 1; return ''; }) - .replace(LATER_ENTRYPOINT_BLOCK, block => { removedEntrypointLines = block.split('\n').length - 1; return ''; }); + .replace(LATER_ENTRYPOINT_BLOCK, block => { removedEntrypointLines += block.split('\n').length - 1; return ''; }); for (const [later, earlier] of [...LATER_BROWSER_EDITS, ...LATER_PREAMBLE_EDITS, ...LATER_WAVE_EDITS, ...LATER_ENTRYPOINT_EDITS]) historical = historical.replace(later, earlier); return historical .replace(/(--- BEGIN FILE "BROWSER\.md" \(lines \d+-)(\d+)(; section\) ---)/, (_, head, end, tail) => `${head}${Number(end) - removedLines}${tail}`) diff --git a/test/office-hours-attempt.test.ts b/test/office-hours-attempt.test.ts index a5345a0f8..29946a398 100644 --- a/test/office-hours-attempt.test.ts +++ b/test/office-hours-attempt.test.ts @@ -654,7 +654,7 @@ describe('Plan format actual capture and judge lifecycle', () => { const starts = events.filter(event => event.kind === 'start'); expect(starts.map(({ timeout, maxTurns, model }) => ({ timeout, maxTurns, model }))) .toEqual([1, 2].map(() => ({ timeout: 300, maxTurns: 10, - model: file.includes('plan-prosons') ? 'claude-opus-4-7' : resolveEvalModel('capture') }))); + model: resolveEvalModel('capture') }))); expect(events.filter(event => event.kind === 'ready').map(event => event.fixtureExists)).toEqual([true, true]); expect(entries).toHaveLength(2); expect(entries.map(entry => entry.attempt)).toEqual([1, 2]); diff --git a/test/office-hours-phase4-caller.test.ts b/test/office-hours-phase4-caller.test.ts index 43c00f767..199ff2476 100644 --- a/test/office-hours-phase4-caller.test.ts +++ b/test/office-hours-phase4-caller.test.ts @@ -2,6 +2,7 @@ import { expect, test } from 'bun:test'; import * as fs from 'node:fs'; import * as path from 'node:path'; import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets'; +import { resolveEvalModel } from '../lib/eval-model'; const source = fs.readFileSync(path.join(import.meta.dir, 'skill-e2e-office-hours-phase4.test.ts'), 'utf8'); const question = (word = 'architectural', options = 'A) Put retrieval on the server\nB) Put retrieval on the client') => `Where should retrieval live?\nThis ${word} choice decides which component owns the retrieval algorithm and the cross-host API contract.\n${options}\nRecommendation: A because all hosts need a consistent algorithm with one owner.\n`; @@ -19,7 +20,7 @@ async function runCaller(captured: string | undefined, scenario: 'success' | 'ti const result = { exitReason: scenario === 'timeout' ? 'timeout' : 'success', browseErrors: [], durationMs: 123, costUsd: 0.1, output: 'public completion' }; const args: Record = { expect, beforeAll: (fn: () => void) => setups.push(fn), afterAll: (fn: () => Promise) => finalizers.push(fn), - CAPTURE_MS, CAPTURE_LONG_MS, ROOT: sourceRoot, runId: 'synthetic-run', + CAPTURE_MS, CAPTURE_LONG_MS, resolveEvalModel, ROOT: sourceRoot, runId: 'synthetic-run', describeIfSelected: (_title: string, _names: string[], fn: () => void) => fn(), testConcurrentIfSelected: (name: string, fn: () => Promise, timeout: number) => { expect(name).toBe('office-hours-phase4-fork'); expect(timeout).toBe(CAPTURE_LONG_MS); callbacks.push(fn); }, createEvalCollector: () => collector, finalizeEvalCollector: async () => {}, logCost: () => {}, @@ -35,7 +36,7 @@ async function runCaller(captured: string | undefined, scenario: 'success' | 'ti fs: { mkdtempSync:(prefix:string)=>prefix+'owned', mkdirSync:()=>{}, rmSync:()=>{}, existsSync:(name:string)=>files.has(name), writeFileSync:(name:string,body:string)=>files.set(name,body), readFileSync:(name:string)=>name===fixturePath.join(sourceRoot,'office-hours','SKILL.md') ? '## AskUserQuestion Format\nformat\n## Phase 4: Alternatives Generation\nworkflow\n## Phase 4.5\nnext' : files.get(name) }, runSkillTest: async (opts: any) => { - calls.push(opts); expect(opts.timeout).toBe(CAPTURE_MS); expect(opts.maxTurns).toBe(12); expect(opts.model).toBe('claude-opus-4-7'); + calls.push(opts); expect(opts.timeout).toBe(CAPTURE_MS); expect(opts.maxTurns).toBe(12); expect(opts.model).toBe(resolveEvalModel('capture')); expect(opts.prompt).toContain('Do NOT call any tool to ask the user.'); if (captured !== undefined) files.set(fixturePath.join(opts.workingDirectory, 'phase4-capture.md'), captured); return result; diff --git a/test/overlay-lifecycle.test.ts b/test/overlay-lifecycle.test.ts index b27124611..66232ce59 100644 --- a/test/overlay-lifecycle.test.ts +++ b/test/overlay-lifecycle.test.ts @@ -28,7 +28,7 @@ test('every fixture owns one paid wrapper; public models/trials/concurrency/turn expect(isPaidTestFile('test/overlay-lifecycle.test.ts')).toBe(false); expect(OVERLAY_FIXTURES.every(f => f.trials === 10 && f.concurrency === 3)).toBe(true); expect(OVERLAY_FIXTURES.map(f => f.maxTurns ?? 5)).toEqual([15,8,15,15,15]); - expect(OVERLAY_FIXTURES.map(f => f.model)).toEqual([...Array(3).fill('claude-opus-4-7'), 'claude-sonnet-4-6', 'claude-opus-5-5']); + expect(OVERLAY_FIXTURES.map(f => f.model)).toEqual([...Array(3).fill('claude-opus-5-5'), 'claude-sonnet-5-5', 'claude-opus-5-5']); expect([OVERLAY_CASE_WORK_MS, OVERLAY_RECORD_GRACE_MS, OVERLAY_CASE_OUTER_MS, OVERLAY_MIN_FILE_WALL_MS]).toEqual([1_800_000, 5_000, 1_810_000, 1_830_000]); }); diff --git a/test/overlay-measurement.test.ts b/test/overlay-measurement.test.ts index 273647113..1f965442e 100644 --- a/test/overlay-measurement.test.ts +++ b/test/overlay-measurement.test.ts @@ -65,7 +65,7 @@ describe('SDK overlay measurements', () => { }); describe('correctness versus efficacy', () => { - describe.each(['claude-opus-4-7', 'claude-sonnet-4-6'])('dedicated-tool gate for %s', (model) => { + describe.each(['claude-opus-5-5', 'claude-sonnet-5-5'])('dedicated-tool gate for %s', (model) => { const f = { ...OVERLAY_FIXTURES.find(f => f.metricName === 'bash_tool_calls' && f.model === model)!, trials: 3 }; // Contract v4: a trial whose final JSON fails the fixture's verify is a valid // measurement with a wrong answer, not an invalid measurement. @@ -276,9 +276,8 @@ describe('literal fixture task correctness', () => { expect(OVERLAY_FIXTURES).toHaveLength(5); expect(OVERLAY_FIXTURES.some((f) => f.id.includes('fanout') || f.comparison?.unsupportedHypothesis)).toBe(false); expect(OVERLAY_FIXTURES.every((f) => f.trials === 10 && f.comparison)).toBe(true); - expect(OVERLAY_FIXTURES.filter((f) => f.model === 'claude-opus-4-7')).toHaveLength(3); - expect(OVERLAY_FIXTURES.filter((f) => f.model === 'claude-sonnet-4-6')).toHaveLength(1); - expect(OVERLAY_FIXTURES.filter((f) => f.model === 'claude-opus-5-5')).toHaveLength(1); + expect(OVERLAY_FIXTURES.filter((f) => f.model === 'claude-opus-5-5')).toHaveLength(4); + expect(OVERLAY_FIXTURES.filter((f) => f.model === 'claude-sonnet-5-5')).toHaveLength(1); }); }); diff --git a/test/skill-e2e-design.test.ts b/test/skill-e2e-design.test.ts index 10c0fe72c..8f078f9ce 100644 --- a/test/skill-e2e-design.test.ts +++ b/test/skill-e2e-design.test.ts @@ -124,7 +124,7 @@ Write DESIGN.md and CLAUDE.md (or update it) in the working directory.`, timeout: CAPTURE_LONG_MS, testName: 'design-consultation-core', runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }); logCost('/design-consultation core', result); @@ -298,7 +298,7 @@ Run Phase 0's DESIGN.md format check exactly as written (the gstack bin director timeout: CAPTURE_LONG_MS, testName: 'design-consultation-existing', runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }); logCost('/design-consultation existing', result); diff --git a/test/skill-e2e-office-hours-phase4.test.ts b/test/skill-e2e-office-hours-phase4.test.ts index 2043f4a55..eb7f35433 100644 --- a/test/skill-e2e-office-hours-phase4.test.ts +++ b/test/skill-e2e-office-hours-phase4.test.ts @@ -22,6 +22,7 @@ import { expect, beforeAll, afterAll } from 'bun:test'; import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; +import { resolveEvalModel } from '../lib/eval-model'; import type { EvalTestEntry } from './helpers/eval-store'; import { ROOT, runId, @@ -166,7 +167,7 @@ After writing the file with that ONE Phase 4 question, stop. Do not continue to timeout: CAPTURE_MS, testName: 'office-hours-phase4-fork', runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }); logCost('/office-hours Phase 4 fork', result); diff --git a/test/skill-e2e-plan-prosons.test.ts b/test/skill-e2e-plan-prosons.test.ts index ab41f9606..7100850d0 100644 --- a/test/skill-e2e-plan-prosons.test.ts +++ b/test/skill-e2e-plan-prosons.test.ts @@ -30,6 +30,7 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { CAPTURE_MS } from './helpers/eval-budgets'; import { NEUTRAL_POSTURE_RE } from './helpers/prosons-posture'; import { runSkillTest } from './helpers/session-runner'; +import { resolveEvalModel } from '../lib/eval-model'; import { runRecordedOfficeHoursAttempt, OFFICE_HOURS_BUN_GRACE_MS } from './helpers/office-hours-attempt'; import { ROOT, runId, @@ -134,7 +135,7 @@ function setupPlanDir(tmpPrefix: string, planContent: string, skillName: string) } function captureInstruction(outFile: string): string { - return `Write the verbatim text of the single AskUserQuestion you would have made to ${outFile} (full text including D header, ELI10, Stakes, Recommendation, Pros/cons, and Net line — the complete rich markdown body). Do NOT call any tool to ask the user. Do NOT paraphrase. This is a format-capture test.`; + return `Write the text of the single AskUserQuestion you would have made to ${outFile} (full text including D header, ELI10, Stakes, Recommendation, Pros/cons, and Net line — the complete rich markdown body). Do NOT call any tool to ask the user.`; } // --- Case 1: Format positive — all v1.7.0.0 tokens present --- @@ -155,7 +156,7 @@ describeIfSelected('Plan Prosons — Format Positive', ['plan-review-prosons-for testConcurrentIfSelected('plan-review-prosons-format', async () => { await runRecordedOfficeHoursAttempt({ collector: evalCollector, name: '/plan-review-prosons-format', suite: 'Plan Prosons — Format Positive', - model: 'claude-opus-4-7', budgetMs: CAPTURE_MS, + model: resolveEvalModel('capture'), budgetMs: CAPTURE_MS, run: signal => runSkillTest({ signal, prompt: `Read plan-ceo-review/SKILL.md for the review workflow. @@ -170,7 +171,7 @@ After writing the file, stop.`, timeout: CAPTURE_MS, testName: 'plan-review-prosons-format', runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }), validate: async (result, signal) => { logCost('/plan-review prosons format positive', result); @@ -217,7 +218,7 @@ describeIfSelected('Plan Prosons — Hard-stop Negative', ['plan-review-prosons- testConcurrentIfSelected('plan-review-prosons-hardstop-neg', async () => { await runRecordedOfficeHoursAttempt({ collector: evalCollector, name: '/plan-review-prosons-hardstop-neg', suite: 'Plan Prosons — Hard-stop Negative', - model: 'claude-opus-4-7', budgetMs: CAPTURE_MS, + model: resolveEvalModel('capture'), budgetMs: CAPTURE_MS, run: signal => runSkillTest({ signal, prompt: `Read plan-ceo-review/SKILL.md. @@ -232,7 +233,7 @@ After writing the file, stop.`, timeout: CAPTURE_MS, testName: 'plan-review-prosons-hardstop-neg', runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }), validate: async (result, signal) => { logCost('/plan-review prosons hard-stop negative', result); @@ -270,7 +271,7 @@ describeIfSelected('Plan Prosons — Neutral-posture Negative', ['plan-review-pr testConcurrentIfSelected('plan-review-prosons-neutral-neg', async () => { await runRecordedOfficeHoursAttempt({ collector: evalCollector, name: '/plan-review-prosons-neutral-neg', suite: 'Plan Prosons — Neutral Negative', - model: 'claude-opus-4-7', budgetMs: CAPTURE_MS, + model: resolveEvalModel('capture'), budgetMs: CAPTURE_MS, run: signal => runSkillTest({ signal, prompt: `Read plan-ceo-review/SKILL.md. @@ -285,7 +286,7 @@ After writing the file, stop.`, timeout: CAPTURE_MS, testName: 'plan-review-prosons-neutral-neg', runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }), validate: async (result, signal) => { logCost('/plan-review prosons neutral negative', result); @@ -324,7 +325,7 @@ describeIfSelected('Plan Prosons — Hard-stop Positive', ['plan-ceo-review-pros testConcurrentIfSelected('plan-ceo-review-prosons-cadence', async () => { await runRecordedOfficeHoursAttempt({ collector: evalCollector, name: '/plan-ceo-review-prosons-cadence', suite: 'Plan Prosons — Hard-stop Positive', - model: 'claude-opus-4-7', budgetMs: CAPTURE_MS, + model: resolveEvalModel('capture'), budgetMs: CAPTURE_MS, run: signal => runSkillTest({ signal, prompt: `Read plan-ceo-review/SKILL.md. @@ -339,7 +340,7 @@ After writing the file, stop.`, timeout: CAPTURE_MS, testName: 'plan-ceo-review-prosons-cadence', runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }), validate: async (result, signal) => { logCost('/plan-review prosons hard-stop positive', result); diff --git a/test/skill-e2e-plan.test.ts b/test/skill-e2e-plan.test.ts index 92b79175c..2f7a7eec8 100644 --- a/test/skill-e2e-plan.test.ts +++ b/test/skill-e2e-plan.test.ts @@ -91,7 +91,7 @@ Focus on reviewing the plan content: architecture, error handling, security, and timeout: 540_000, testName: 'plan-ceo-review', runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }); logCost('/plan-ceo-review', result); @@ -178,7 +178,7 @@ Focus on reviewing the plan content: architecture, error handling, security, and timeout: CAPTURE_LONG_MS, testName: 'plan-ceo-review-selective', runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }); logCost('/plan-ceo-review (SELECTIVE)', result); @@ -246,7 +246,7 @@ Write your expansion proposals to ${planDir}/proposals.md with ONLY the proposal timeout: CAPTURE_LONG_MS, testName: 'plan-ceo-review-expansion-energy', runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }); logCost('/plan-ceo-review (EXPANSION ENERGY)', result); @@ -355,7 +355,7 @@ Focus on architecture, code quality, tests, and performance sections.`, timeout: CAPTURE_LONG_MS, testName: 'plan-eng-review', runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }); logCost('/plan-eng-review', result); diff --git a/test/skill-e2e-qa-bugs.test.ts b/test/skill-e2e-qa-bugs.test.ts index 14db97bf6..1a59acf33 100644 --- a/test/skill-e2e-qa-bugs.test.ts +++ b/test/skill-e2e-qa-bugs.test.ts @@ -1,6 +1,7 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; +import { resolveEvalModel } from '../lib/eval-model'; import { outcomeJudge } from './helpers/llm-judge'; import { judgePassed } from './helpers/eval-store'; import { @@ -108,7 +109,7 @@ CRITICAL RULES: timeout: CAPTURE_MS, testName: `qa-${label}`, runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }); logCost(`/qa ${label}`, result); diff --git a/test/skill-positional-literals.test.ts b/test/skill-positional-literals.test.ts index e8280a397..e691697c4 100644 --- a/test/skill-positional-literals.test.ts +++ b/test/skill-positional-literals.test.ts @@ -75,7 +75,7 @@ for (const host of ['claude', 'codex'] as const) for (const args of [[], tenArgu const file = backslashPath ? join(root, 'checksum\\path', 'install-script') : join(root, 'install-script'); mkdirSync(dirname(file), { recursive: true }); writeFileSync(file, 'synthetic installer bytes\n'); - for (const name of ['open-gstack-browser', 'pair-agent', 'setup-browser-cookies']) { + for (const name of ['open-gstack-browser', 'pair-agent']) { const lines = apply(skill(host, name)).split('\n').filter(line => line.includes('actual_sha=$(')); expect(lines.map(line => line.trim())).toEqual([ literals.checksum,