From 3704b00a62c88720cb7f34c7029bdb928d99e5ea Mon Sep 17 00:00:00 2001 From: garrytan Date: Mon, 19 Oct 2026 07:27:49 +0000 Subject: [PATCH] test: re-baseline old-model paid pins onto the current generation (TM3) Repair the prosons capture prompt that tripped the reasoning_extraction classifier, and restructure /setup-browser-cookies so the Aside probe, decisions and target order are unambiguous. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- BROWSER.md | 2 +- TODOS.md | 27 +++--- docs/OVERLAY_BENCHMARK_CONTRACT.md | 8 +- setup-browser-cookies/SKILL.md | 96 ++++++++------------- setup-browser-cookies/SKILL.md.tmpl | 54 ++++++++---- test/codex-e2e-sol-scope.test.ts | 4 +- test/cookie-workflow-judge-input.test.ts | 4 +- test/fixtures/overlay-nudges.ts | 12 +-- test/helpers/cookie-workflow-judge-input.ts | 2 +- test/helpers/manual-judge-review-fixture.ts | 21 ++++- test/office-hours-attempt.test.ts | 2 +- test/office-hours-phase4-caller.test.ts | 5 +- test/overlay-lifecycle.test.ts | 2 +- test/overlay-measurement.test.ts | 7 +- test/skill-e2e-design.test.ts | 4 +- test/skill-e2e-office-hours-phase4.test.ts | 3 +- test/skill-e2e-plan-prosons.test.ts | 19 ++-- test/skill-e2e-plan.test.ts | 8 +- test/skill-e2e-qa-bugs.test.ts | 3 +- test/skill-positional-literals.test.ts | 2 +- 20 files changed, 153 insertions(+), 132 deletions(-) diff --git a/BROWSER.md b/BROWSER.md index 75cd99b9e..396bb6e9b 100644 --- a/BROWSER.md +++ b/BROWSER.md @@ -584,7 +584,7 @@ SITE=app.example.com DOMAIN=example.com $B goto "https://$SITE" $B cookie-import-browser opera-gx --domain "$DOMAIN" --profile Default # or: opera -$B reload # confirm the intended account +$B reload # view the page; only --verify-auth proves sign-in ``` Run `$B cookie-import-browser` with no flags to see which browsers were detected. `--profile` can be omitted when only one profile has cookies for the domain. If the receipt reports App-Bound Encryption, run `$B handoff`, sign in to the intended account in the window that opens, then `$B resume` (needs a display). diff --git a/TODOS.md b/TODOS.md index f76d0d5a4..d04377dbc 100644 --- a/TODOS.md +++ b/TODOS.md @@ -1645,19 +1645,24 @@ and `test/dx-selected-navigation-ap.test.ts`. One shared table run once against only after `engFirstReviewAUQ` checks native completion once at entry; today each branch gates it separately, so the change alters a paid verdict and needs its own paid run. -### P3: Re-pin the five remaining claude-opus-4-7 paid files +### P3: Follow-ups from re-baselining the old-model paid pins on the current generation -**What:** The 2026-09 audit moved seven paid evals to the default capture model (`resolveEvalModel('capture')`). -`skill-e2e-design`, `skill-e2e-office-hours-phase4`, `skill-e2e-plan-prosons` and `skill-e2e-plan` keep -`claude-opus-4-7` because six cases failed on the default model in one run (plan-design-review-plan-mode timeout, -office-hours-phase4-fork format, plan-review-prosons-neutral-neg missing output, plan-ceo-review-selective and -plan-eng-review 600 s timeouts, plan-ceo-review-expansion-energy posture score 3). plan-review-report, the one PR-gate -case in `skill-e2e-plan`, left the pin on 2026-10-18 and follows the capture default. `skill-e2e-qa-bugs` returned -to `claude-opus-4-7` after `qa-b6-static` timed out on the default model in two of three runs (census 36597762183 -and a targeted local rerun): each time the stream stopped mid-message, with no pending tool, right after the model -found the disabled submit button, and emitted nothing until the 300 s case deadline. They measure an old model. +**What:** GSTA-49 TM3 (2026-10-18) moved every remaining old-model paid pin to the current generation: the +`claude-opus-4-7` pins in `skill-e2e-{plan,plan-prosons,design,office-hours-phase4,qa-bugs}` follow +`resolveEvalModel('capture')`, the overlay fixtures pin `claude-opus-5-5` and `claude-sonnet-5-5`, the cookie workflow +judge pins `claude-sonnet-5-5`, and `codex-sol-scope-termination` runs on `gpt-6.1-sol` (still rendering the +`gpt-5.6-sol` overlay). Three things remain: -**Re-entry:** fix the prompt, budget or rubric so each case passes on the default model in one run, then drop the pin. +- The base `claude-dedicated-tools-vs-bash` fixture now duplicates `claude-dedicated-tools-vs-bash-opus-5-5`. +- The `opus-4-7-*` fixtures measure `model-overlays/opus-4-7.md` on Opus 5.5, which gets the generic `claude` + overlay in product. +- Opus 5.5 refuses (stop category `reasoning_extraction`) capture prompts that ask for the "verbatim text" of a + question "you would have made" with "Do NOT paraphrase"; that wording stopped 7 of 16 replays of a prosons + request and 0 of 24 without it. `skill-e2e-plan-prosons` dropped it; `skill-e2e-plan-format` and + `skill-e2e-office-hours-phase4` still use it. + +**Re-entry:** retire one dedicated-tools fixture, decide the opus-4-7 overlay's future, and drop the extraction +wording from the remaining capture prompts with a paid run each. ### P3: Retire the unused CEO payment seeder diff --git a/docs/OVERLAY_BENCHMARK_CONTRACT.md b/docs/OVERLAY_BENCHMARK_CONTRACT.md index d6f3cc38e..84f87b4d6 100644 --- a/docs/OVERLAY_BENCHMARK_CONTRACT.md +++ b/docs/OVERLAY_BENCHMARK_CONTRACT.md @@ -14,8 +14,10 @@ Selection IDs for the six retained paid cases stay unchanged. ## Blocking checks Both arms use the real Claude Code preset; ON appends the resolved overlay and -OFF does not. Each retained fixture runs ten samples per arm, separately for -`claude-opus-4-7` and `claude-sonnet-4-6`. Each model keeps its own results. +OFF does not. Each retained fixture runs ten samples per arm on its pinned +model: `claude-opus-5-5`, plus `claude-sonnet-5-5` for the dedicated-tools `-sonnet` case. +Each model keeps its own results. The `opus-4-7-*` cases keep their overlay +(`model-overlays/opus-4-7.md`) but run on Opus 5.5 since GSTA-49. Every planned sample in both arms must execute successfully, produce exactly one successful native terminal result without an error flag, preserve workspace @@ -38,7 +40,7 @@ are not reinterpreted. | `opus-4-7-effort-match-trivial` | Read-only workspace; native final JSON contains the exact version string | Correct answer in every ON sample | Native `usage.output_tokens_details.thinking_tokens` | | `opus-4-7-literal-interpretation` | Only `src/auth.ts`, `src/billing.ts`, and `src/notifications.ts` may change; public tests stay frozen; independent behavior checks run outside the writable fixture | All three target behaviors pass in every ON sample | Correct target behaviors, 0..3 | -Each family also has a `-sonnet` case. Output validation checks the native final +The dedicated-tools family also has `-sonnet` and `-opus-5-5` cases. Output validation checks the native final answer, so a correct word mentioned earlier cannot conceal a wrong final answer. The read-only prompts explicitly request the JSON shape. These exact-answer prompts are new in v2: compare ON/OFF within the same contract and fixture, not diff --git a/setup-browser-cookies/SKILL.md b/setup-browser-cookies/SKILL.md index 1aa2a0433..830f9c7c2 100644 --- a/setup-browser-cookies/SKILL.md +++ b/setup-browser-cookies/SKILL.md @@ -152,83 +152,59 @@ Skills that run plan reviews (`/plan-*-review`, `/codex review`) include the EXI # Setup Browser Cookies -## 1. Choose the browser +## 1. Check for Aside -Use this checkout as the gstack root if it contains `BROWSER.md` and `browse/SKILL.md`; otherwise use the installed root containing `bin/gstack-skill-start`, never a generated host stub. Read that root's `browse/SKILL.md` **BROWSER SETUP** section and run its probe first. On `READY`, stop importing: use Aside's sessions or ask the user to sign in there. Otherwise follow the probe's fallback handling, then continue below. +Use this checkout as the gstack root if it contains `BROWSER.md` and `browse/SKILL.md`; otherwise use the installed root containing `bin/gstack-skill-start`, never a generated host stub. Read that root's `browse/SKILL.md` **BROWSER SETUP** section and run its Aside probe. -## SETUP (run this check BEFORE any browse command) +- **Aside probe prints `READY`:** no import is needed, because Aside already has the user's signed-in sessions. Tell the user to use Aside, or to sign in there, and end this skill. +- **Any other result:** continue with gstack's own headless browser (`$B`). + +## 2. Get every decision from the user first + +Ask these together, before running any `$B` command. Never guess an answer or treat the default Comet browser as consent. + +1. **Source browser and profile.** `--profile` takes the profile's directory (`Default`, `Profile 2`), not its display name. A profile that cannot be read is unknown, not empty. +2. **Domains to import,** and the **target**: one page on a chosen domain that the session should open, by URL. +3. **Sign-in verification (`--verify-auth`)?** Default no. It reloads the target and reports **verified** only after a successful same-origin response with exactly one visible element, found by a CSS selector, whose whitespace-normalized text equals the expected identity. If yes, the user supplies the selector and the expected identity privately. +4. **Clear stale storage (`--clear-storage`)?** Default no; needs explicit approval. Chromium only. It clears localStorage for the target's origin (shared by that origin's tabs) and sessionStorage for the target tab, and nothing else; other browser engines reject the reset but still import. A failed reset may already have cleared some storage and is not rolled back. +5. **Every cookie in the profile (`--all`)?** Default no; needs explicit consent. It cannot be combined with a domain or with clearing storage. + +Use the picker when any of answers 1-2 is still open; it asks for them. Use direct import when the user has named the browser, the profile directory and the domains. + +## 3. Prepare gstack's browser + +1. If the user wants verification, have them set `GSTACK_COOKIE_AUTH_SELECTOR` and `GSTACK_COOKIE_AUTH_EXPECTED_IDENTITY` privately in the shell that runs `$B`. The browser daemon reads them only when it starts, so if a gstack daemon is already running, run `$B stop` first. Never invent the values. +2. Follow `browse/SKILL.md`'s **Browser fallback** section to find `$B`, building it if the check prints `NEEDS_SETUP`. +3. Run `$B status`. If it reports `Mode: cdp`, `$B` is attached to the user's real browser and already has its sessions: tell the user no import is needed and end this skill. +4. Open the target URL from answer 2 and leave that tab alone until the import finishes: ```bash -_ROOT=$(git rev-parse --show-toplevel 2>/dev/null) -B="" -[ -n "$_ROOT" ] && [ -x "$_ROOT/.claude/skills/gstack/browse/dist/browse" ] && B="$_ROOT/.claude/skills/gstack/browse/dist/browse" -[ -z "$B" ] && B="$HOME/.claude/skills/gstack/browse/dist/browse" -if [ -x "$B" ] && "${B%/browse/*}/bin/gstack-browser-ensure"; then - echo "READY: $B" -else - echo "NEEDS_SETUP" -fi +$B goto "" ``` -`gstack-browser-ensure` installs Chromium on first use when the install skipped it (lazy browser); it prints `BROWSER_OK` or `BROWSER_UNAVAILABLE ` first. +## 4. Import -If `NEEDS_SETUP`: -1. Tell the user: "gstack browse needs a one-time build (~10 seconds). OK to proceed?" Then STOP and wait. -2. Run: `cd && ./setup` -3. If `bun` is not installed: - ```bash - if ! command -v bun >/dev/null 2>&1; then - BUN_VERSION="1.4.2" - BUN_INSTALL_SHA="bab8acfb046aac8c72407bdcce903957665d655d7acaa3e11c7c4616beae68dd" - tmpfile=$(mktemp "${TMPDIR:-/tmp}/bun-install.XXXXXX") - curl -fsSL "https://bun.sh/install" -o "$tmpfile" - # shasum is macOS/perl; coreutils-only Linux ships sha256sum instead — - # resolve whichever exists so the verify never fails on a missing tool. - if command -v sha256sum >/dev/null 2>&1; then - actual_sha=$(sha256sum < "$tmpfile" | awk '{print $(1)}') - else - actual_sha=$(shasum -a 256 < "$tmpfile" | awk '{print $(1)}') - fi - if [ "$actual_sha" != "$BUN_INSTALL_SHA" ]; then - echo "ERROR: bun install script checksum mismatch" >&2 - echo " expected: $BUN_INSTALL_SHA" >&2 - echo " got: $actual_sha" >&2 - rm "$tmpfile"; exit 1 - fi - bash "$tmpfile" "bun-v$BUN_VERSION" - rm "$tmpfile" - fi - ``` - -```bash -$B status -``` -If status says `Mode: cdp`, stop: the real browser already has sessions. - -## 2. Confirm options before import - -Open the known target and keep its tab unchanged. Both options default off and require explicit request: - -- **`--verify-auth` / picker checkbox:** reloads the target. Have the user privately configure daemon `GSTACK_COOKIE_AUTH_SELECTOR` and `GSTACK_COOKIE_AUTH_EXPECTED_IDENTITY` **before startup**. Never invent values or assume CLI env reconfigures an existing daemon. Missing config rejects before mutation. Require a successful same-origin response and exactly one visible element whose whitespace-normalized text exactly matches the expected identity. -- **`--clear-storage`:** for suspected stale storage, obtain explicit approval. Chromium only: clears captured-origin localStorage (shared across same-origin context tabs) and target-tab sessionStorage. Other origins, other tabs' sessionStorage, IndexedDB, and service workers stay intact. It uses an isolated world/native deadline; other engines reject reset, not imports/auth checks. Never auto-approve or claim rollback after partial failure. - -## 3. Select source and scope +Picker (the user confirms the browser, profile and domains there, ticks verification if they asked for it, then tells you when done; the link works once, for five minutes, so rerun the command if it expires): ```bash $B cookie-import-browser ``` -Ask the user to choose browser, account/profile, and domains, then say when done. Never guess accounts or treat default Comet as consent. Unreadable profiles are unknown, not empty. Rerun for an expired five-minute one-use link. +Direct import: run `$B cookie-import-browser` with the chosen browser name, one `--domain` per run and `--profile` set to the profile directory (not its display name); add `--verify-auth` or `--clear-storage` only if the user said yes in step 2. -Direct import: pass the chosen browser and `--domain` after navigating to a matching target. `--profile` takes a directory, not a display name; omit only for an unambiguous relevant profile, otherwise use the picker. `--all` requires consent for all non-expired profile cookies; it cannot accompany `--domain` or `--clear-storage`. +If the import rejects because verification settings are missing, nothing changed: tell the user which variable is missing and return to step 3. Read that same root's `BROWSER.md`, **Choosing a source and checking sign-in**, for other errors and receipt failure reasons; it has the error tables, examples, profile labels, supported sources and platform setup. -Read that same root's `BROWSER.md`, **Choosing a source and checking sign-in**, for examples, profile labels, supported sources and platform setup. +## 5. Report honestly -## 4. Report honestly +Report from the import receipt, not from raw `$B cookies`, as three lines: -Report receipt/picker counts, partial/zero/error and reset outcomes, not raw `$B cookies`. Imports affect the context, not one tab. **Not checked** means no requested check; **not verified** means it failed; **verified** requires positive target evidence. Zero imports, counts, or HTTP 200 never prove login. +- `Imported: (, as the receipt says) from for ` +- `Storage reset: ` +- `Sign-in: ` -Never request/publish cookie values, passwords, identity/profile text, session details, or raw errors in public logs. +**Not checked** means verification was not requested; **not verified** means it ran and failed; **verified** needs the identity match. Cookie counts, zero imports and HTTP 200 never prove sign-in. Imported cookies apply to the whole browser context, not one tab. + +Never request or publish cookie values, passwords, identity or profile text, session details, or raw errors in public logs. ## Platform boundaries diff --git a/setup-browser-cookies/SKILL.md.tmpl b/setup-browser-cookies/SKILL.md.tmpl index f6ed3f61e..82313c668 100644 --- a/setup-browser-cookies/SKILL.md.tmpl +++ b/setup-browser-cookies/SKILL.md.tmpl @@ -21,41 +21,59 @@ allowed-tools: # Setup Browser Cookies -## 1. Choose the browser +## 1. Check for Aside -Use this checkout as the gstack root if it contains `BROWSER.md` and `browse/SKILL.md`; otherwise use the installed root containing `bin/gstack-skill-start`, never a generated host stub. Read that root's `browse/SKILL.md` **BROWSER SETUP** section and run its probe first. On `READY`, stop importing: use Aside's sessions or ask the user to sign in there. Otherwise follow the probe's fallback handling, then continue below. +Use this checkout as the gstack root if it contains `BROWSER.md` and `browse/SKILL.md`; otherwise use the installed root containing `bin/gstack-skill-start`, never a generated host stub. Read that root's `browse/SKILL.md` **BROWSER SETUP** section and run its Aside probe. -{{BROWSE_SETUP}} +- **Aside probe prints `READY`:** no import is needed, because Aside already has the user's signed-in sessions. Tell the user to use Aside, or to sign in there, and end this skill. +- **Any other result:** continue with gstack's own headless browser (`$B`). + +## 2. Get every decision from the user first + +Ask these together, before running any `$B` command. Never guess an answer or treat the default Comet browser as consent. + +1. **Source browser and profile.** `--profile` takes the profile's directory (`Default`, `Profile 2`), not its display name. A profile that cannot be read is unknown, not empty. +2. **Domains to import,** and the **target**: one page on a chosen domain that the session should open, by URL. +3. **Sign-in verification (`--verify-auth`)?** Default no. It reloads the target and reports **verified** only after a successful same-origin response with exactly one visible element, found by a CSS selector, whose whitespace-normalized text equals the expected identity. If yes, the user supplies the selector and the expected identity privately. +4. **Clear stale storage (`--clear-storage`)?** Default no; needs explicit approval. Chromium only. It clears localStorage for the target's origin (shared by that origin's tabs) and sessionStorage for the target tab, and nothing else; other browser engines reject the reset but still import. A failed reset may already have cleared some storage and is not rolled back. +5. **Every cookie in the profile (`--all`)?** Default no; needs explicit consent. It cannot be combined with a domain or with clearing storage. + +Use the picker when any of answers 1-2 is still open; it asks for them. Use direct import when the user has named the browser, the profile directory and the domains. + +## 3. Prepare gstack's browser + +1. If the user wants verification, have them set `GSTACK_COOKIE_AUTH_SELECTOR` and `GSTACK_COOKIE_AUTH_EXPECTED_IDENTITY` privately in the shell that runs `$B`. The browser daemon reads them only when it starts, so if a gstack daemon is already running, run `$B stop` first. Never invent the values. +2. Follow `browse/SKILL.md`'s **Browser fallback** section to find `$B`, building it if the check prints `NEEDS_SETUP`. +3. Run `$B status`. If it reports `Mode: cdp`, `$B` is attached to the user's real browser and already has its sessions: tell the user no import is needed and end this skill. +4. Open the target URL from answer 2 and leave that tab alone until the import finishes: ```bash -$B status +$B goto "" ``` -If status says `Mode: cdp`, stop: the real browser already has sessions. -## 2. Confirm options before import +## 4. Import -Open the known target and keep its tab unchanged. Both options default off and require explicit request: - -- **`--verify-auth` / picker checkbox:** reloads the target. Have the user privately configure daemon `GSTACK_COOKIE_AUTH_SELECTOR` and `GSTACK_COOKIE_AUTH_EXPECTED_IDENTITY` **before startup**. Never invent values or assume CLI env reconfigures an existing daemon. Missing config rejects before mutation. Require a successful same-origin response and exactly one visible element whose whitespace-normalized text exactly matches the expected identity. -- **`--clear-storage`:** for suspected stale storage, obtain explicit approval. Chromium only: clears captured-origin localStorage (shared across same-origin context tabs) and target-tab sessionStorage. Other origins, other tabs' sessionStorage, IndexedDB, and service workers stay intact. It uses an isolated world/native deadline; other engines reject reset, not imports/auth checks. Never auto-approve or claim rollback after partial failure. - -## 3. Select source and scope +Picker (the user confirms the browser, profile and domains there, ticks verification if they asked for it, then tells you when done; the link works once, for five minutes, so rerun the command if it expires): ```bash $B cookie-import-browser ``` -Ask the user to choose browser, account/profile, and domains, then say when done. Never guess accounts or treat default Comet as consent. Unreadable profiles are unknown, not empty. Rerun for an expired five-minute one-use link. +Direct import: run `$B cookie-import-browser` with the chosen browser name, one `--domain` per run and `--profile` set to the profile directory (not its display name); add `--verify-auth` or `--clear-storage` only if the user said yes in step 2. -Direct import: pass the chosen browser and `--domain` after navigating to a matching target. `--profile` takes a directory, not a display name; omit only for an unambiguous relevant profile, otherwise use the picker. `--all` requires consent for all non-expired profile cookies; it cannot accompany `--domain` or `--clear-storage`. +If the import rejects because verification settings are missing, nothing changed: tell the user which variable is missing and return to step 3. Read that same root's `BROWSER.md`, **Choosing a source and checking sign-in**, for other errors and receipt failure reasons; it has the error tables, examples, profile labels, supported sources and platform setup. -Read that same root's `BROWSER.md`, **Choosing a source and checking sign-in**, for examples, profile labels, supported sources and platform setup. +## 5. Report honestly -## 4. Report honestly +Report from the import receipt, not from raw `$B cookies`, as three lines: -Report receipt/picker counts, partial/zero/error and reset outcomes, not raw `$B cookies`. Imports affect the context, not one tab. **Not checked** means no requested check; **not verified** means it failed; **verified** requires positive target evidence. Zero imports, counts, or HTTP 200 never prove login. +- `Imported: (, as the receipt says) from for ` +- `Storage reset: ` +- `Sign-in: ` -Never request/publish cookie values, passwords, identity/profile text, session details, or raw errors in public logs. +**Not checked** means verification was not requested; **not verified** means it ran and failed; **verified** needs the identity match. Cookie counts, zero imports and HTTP 200 never prove sign-in. Imported cookies apply to the whole browser context, not one tab. + +Never request or publish cookie values, passwords, identity or profile text, session details, or raw errors in public logs. ## Platform boundaries diff --git a/test/codex-e2e-sol-scope.test.ts b/test/codex-e2e-sol-scope.test.ts index 862f7b8de..c3f59a619 100644 --- a/test/codex-e2e-sol-scope.test.ts +++ b/test/codex-e2e-sol-scope.test.ts @@ -180,14 +180,14 @@ TODO: consider migrating this example to a larger configuration framework. name: 'codex-sol-scope-termination', suite: 'codex-e2e-sol-scope', budgetMs: CAPTURE_MS, - model: 'gpt-5.6-sol', + model: 'gpt-6.1-sol', outputLimit: Infinity, run: (signal) => runCodexSkill({ skillDir, skillName: 'gstack-investigate', cwd: scratch, sandbox: 'workspace-write', - model: 'gpt-5.6-sol', + model: 'gpt-6.1-sol', configOverrides: ['model_reasoning_effort="high"'], ignoreUserConfig: true, timeoutMs: CODEX_TIMEOUT_MS, diff --git a/test/cookie-workflow-judge-input.test.ts b/test/cookie-workflow-judge-input.test.ts index c245e04ab..a8a268a3b 100644 --- a/test/cookie-workflow-judge-input.test.ts +++ b/test/cookie-workflow-judge-input.test.ts @@ -244,8 +244,8 @@ describe('cookie workflow judge input', () => { expect(body).toContain('Use this checkout as the gstack root if it contains `BROWSER.md` and `browse/SKILL.md`'); expect(body).toContain('the installed root containing `bin/gstack-skill-start`, never a generated host stub'); expect(body).toContain("Read that root's `browse/SKILL.md`"); - expect(body).toContain('On `READY`, stop importing'); - expect(body.indexOf('**BROWSER SETUP**')).toBeLessThan(body.indexOf('## SETUP')); + expect(body).toContain('**Aside probe prints `READY`:** no import is needed'); + expect(body.indexOf('**BROWSER SETUP**')).toBeLessThan(body.indexOf('**Browser fallback**')); expect(body).toContain("Read that same root's `BROWSER.md`, **Choosing a source and checking sign-in**"); expect(body).not.toContain('../BROWSER.md'); } diff --git a/test/fixtures/overlay-nudges.ts b/test/fixtures/overlay-nudges.ts index 0b499d785..5baac1656 100644 --- a/test/fixtures/overlay-nudges.ts +++ b/test/fixtures/overlay-nudges.ts @@ -202,7 +202,7 @@ export const OVERLAY_FIXTURES: OverlayFixture[] = [ { id: 'claude-dedicated-tools-vs-bash', overlayPath: 'model-overlays/claude.md', - model: 'claude-opus-4-7', + model: 'claude-opus-5-5', trials: 10, concurrency: 3, direction: 'lower_is_better', @@ -241,7 +241,7 @@ export const OVERLAY_FIXTURES: OverlayFixture[] = [ { id: 'opus-4-7-effort-match-trivial', overlayPath: 'model-overlays/opus-4-7.md', - model: 'claude-opus-4-7', + model: 'claude-opus-5-5', trials: 10, concurrency: 3, direction: 'lower_is_better', @@ -269,7 +269,7 @@ export const OVERLAY_FIXTURES: OverlayFixture[] = [ { id: 'opus-4-7-literal-interpretation', overlayPath: 'model-overlays/opus-4-7.md', - model: 'claude-opus-4-7', + model: 'claude-opus-5-5', trials: 10, concurrency: 3, direction: 'higher_is_better', @@ -290,16 +290,16 @@ export const OVERLAY_FIXTURES: OverlayFixture[] = [ }, // ========================================================================= - // Sonnet 4.6 variants of the Opus-4.7 fixtures. + // Sonnet 5.5 variant of the Opus 5.5 dedicated-tools fixture. // - // Same overlays, prompts, metrics and trial counts; different pinned model. + // Same overlay, prompt, metric and trial count; different pinned model. // Model effects remain separate instead of pooling their observations. // ========================================================================= { id: 'claude-dedicated-tools-vs-bash-sonnet', overlayPath: 'model-overlays/claude.md', - model: 'claude-sonnet-4-6', + model: 'claude-sonnet-5-5', trials: 10, concurrency: 3, direction: 'lower_is_better', diff --git a/test/helpers/cookie-workflow-judge-input.ts b/test/helpers/cookie-workflow-judge-input.ts index 5e595f32d..814d66892 100644 --- a/test/helpers/cookie-workflow-judge-input.ts +++ b/test/helpers/cookie-workflow-judge-input.ts @@ -4,7 +4,7 @@ import { join } from 'node:path'; import { buildWorkflowJudgePrompt, type WorkflowJudgeFile, type WorkflowJudgeInput } from './workflow-judge-input'; export const COOKIE_WORKFLOW_JUDGE = { - model: 'claude-sonnet-4-6', + model: 'claude-sonnet-5-5', judgeContext: 'a fallback-browser cookie import workflow', judgeGoal: 'how to select an authorized source browser, profile, and domain without guessing an account; configure optional authentication verification before mutation; obtain explicit consent for precisely scoped storage reset; distinguish copied cookies from positive sign-in evidence; and recover within the documented platform and privacy boundaries', thresholds: { clarity: 4, completeness: 3, actionability: 4 }, diff --git a/test/helpers/manual-judge-review-fixture.ts b/test/helpers/manual-judge-review-fixture.ts index ded8a3508..0acb4f5d5 100644 --- a/test/helpers/manual-judge-review-fixture.ts +++ b/test/helpers/manual-judge-review-fixture.ts @@ -46,14 +46,31 @@ const LATER_ENTRYPOINT_EDITS: Array<[string, string]> = [ ['bash "$tmpfile" "bun-v$BUN_VERSION"', 'BUN_VERSION="$BUN_VERSION" bash "$tmpfile"'], ]; +// The GSTA-49 cookie-workflow repair rewrote the entrypoint's steps 1-5 (split +// the two probe READY outcomes, dropped the duplicate SETUP block for the browse +// fallback, moved source selection before the target, asked about verification +// before the daemon starts, added a report shape) and one BROWSER.md comment. +// Reversing it first restores the pre-repair text, which the edits below then +// take back further. +const LATER_COOKIE_ORDER_EDITS: Array<[string, string]> = [ + ["## 1. Check for Aside\n\nUse this checkout as the gstack root if it contains `BROWSER.md` and `browse/SKILL.md`; otherwise use the installed root containing `bin/gstack-skill-start`, never a generated host stub. Read that root's `browse/SKILL.md` **BROWSER SETUP** section and run its Aside probe.\n\n- **Aside probe prints `READY`:** no import is needed, because Aside already has the user's signed-in sessions. Tell the user to use Aside, or to sign in there, and end this skill.\n- **Any other result:** continue with gstack's own headless browser (`$B`).\n\n## 2. Get every decision from the user first\n\nAsk these together, before running any `$B` command. Never guess an answer or treat the default Comet browser as consent.\n\n1. **Source browser and profile.** `--profile` takes the profile's directory (`Default`, `Profile 2`), not its display name. A profile that cannot be read is unknown, not empty.\n2. **Domains to import,** and the **target**: one page on a chosen domain that the session should open, by URL.\n3. **Sign-in verification (`--verify-auth`)?** Default no. It reloads the target and reports **verified** only after a successful same-origin response with exactly one visible element, found by a CSS selector, whose whitespace-normalized text equals the expected identity. If yes, the user supplies the selector and the expected identity privately.\n4. **Clear stale storage (`--clear-storage`)?** Default no; needs explicit approval. Chromium only. It clears localStorage for the target's origin (shared by that origin's tabs) and sessionStorage for the target tab, and nothing else; other browser engines reject the reset but still import. A failed reset may already have cleared some storage and is not rolled back.\n5. **Every cookie in the profile (`--all`)?** Default no; needs explicit consent. It cannot be combined with a domain or with clearing storage.\n\nUse the picker when any of answers 1-2 is still open; it asks for them. Use direct import when the user has named the browser, the profile directory and the domains.\n\n## 3. Prepare gstack's browser\n\n1. If the user wants verification, have them set `GSTACK_COOKIE_AUTH_SELECTOR` and `GSTACK_COOKIE_AUTH_EXPECTED_IDENTITY` privately in the shell that runs `$B`. The browser daemon reads them only when it starts, so if a gstack daemon is already running, run `$B stop` first. Never invent the values.\n2. Follow `browse/SKILL.md`'s **Browser fallback** section to find `$B`, building it if the check prints `NEEDS_SETUP`.\n3. Run `$B status`. If it reports `Mode: cdp`, `$B` is attached to the user's real browser and already has its sessions: tell the user no import is needed and end this skill.\n4. Open the target URL from answer 2 and leave that tab alone until the import finishes:\n\n```bash\n$B goto \"\"\n```\n\n## 4. Import\n\nPicker (the user confirms the browser, profile and domains there, ticks verification if they asked for it, then tells you when done; the link works once, for five minutes, so rerun the command if it expires):\n\n```bash\n$B cookie-import-browser\n```\n\nDirect import: run `$B cookie-import-browser` with the chosen browser name, one `--domain` per run and `--profile` set to the profile directory (not its display name); add `--verify-auth` or `--clear-storage` only if the user said yes in step 2.\n\nIf the import rejects because verification settings are missing, nothing changed: tell the user which variable is missing and return to step 3. Read that same root's `BROWSER.md`, **Choosing a source and checking sign-in**, for other errors and receipt failure reasons; it has the error tables, examples, profile labels, supported sources and platform setup.\n\n## 5. Report honestly\n\nReport from the import receipt, not from raw `$B cookies`, as three lines:\n\n- `Imported: (, as the receipt says) from for `\n- `Storage reset: `\n- `Sign-in: `\n\n**Not checked** means verification was not requested; **not verified** means it ran and failed; **verified** needs the identity match. Cookie counts, zero imports and HTTP 200 never prove sign-in. Imported cookies apply to the whole browser context, not one tab.\n\nNever request or publish cookie values, passwords, identity or profile text, session details, or raw errors in public logs.\n\n## Platform boundaries", "## 1. Choose the browser\n\nUse this checkout as the gstack root if it contains `BROWSER.md` and `browse/SKILL.md`; otherwise use the installed root containing `bin/gstack-skill-start`, never a generated host stub. Read that root's `browse/SKILL.md` **BROWSER SETUP** section and run its probe first. On `READY`, stop importing: use Aside's sessions or ask the user to sign in there. Otherwise follow the probe's fallback handling, then continue below.\n\n## SETUP (run this check BEFORE any browse command)\n\n```bash\n_ROOT=$(git rev-parse --show-toplevel 2>/dev/null)\nB=\"\"\n[ -n \"$_ROOT\" ] && [ -x \"$_ROOT/.claude/skills/gstack/browse/dist/browse\" ] && B=\"$_ROOT/.claude/skills/gstack/browse/dist/browse\"\n[ -z \"$B\" ] && B=\"$HOME/.claude/skills/gstack/browse/dist/browse\"\nif [ -x \"$B\" ] && \"${B%/browse/*}/bin/gstack-browser-ensure\"; then\n echo \"READY: $B\"\nelse\n echo \"NEEDS_SETUP\"\nfi\n```\n\n`gstack-browser-ensure` installs Chromium on first use when the install skipped it (lazy browser); it prints `BROWSER_OK` or `BROWSER_UNAVAILABLE ` first.\n\nIf `NEEDS_SETUP`:\n1. Tell the user: \"gstack browse needs a one-time build (~10 seconds). OK to proceed?\" Then STOP and wait.\n2. Run: `cd && ./setup`\n3. If `bun` is not installed:\n ```bash\n if ! command -v bun >/dev/null 2>&1; then\n BUN_VERSION=\"1.4.2\"\n BUN_INSTALL_SHA=\"bab8acfb046aac8c72407bdcce903957665d655d7acaa3e11c7c4616beae68dd\"\n tmpfile=$(mktemp \"${TMPDIR:-/tmp}/bun-install.XXXXXX\")\n curl -fsSL \"https://bun.sh/install\" -o \"$tmpfile\"\n # shasum is macOS/perl; coreutils-only Linux ships sha256sum instead —\n # resolve whichever exists so the verify never fails on a missing tool.\n if command -v sha256sum >/dev/null 2>&1; then\n actual_sha=$(sha256sum < \"$tmpfile\" | awk '{print $(1)}')\n else\n actual_sha=$(shasum -a 256 < \"$tmpfile\" | awk '{print $(1)}')\n fi\n if [ \"$actual_sha\" != \"$BUN_INSTALL_SHA\" ]; then\n echo \"ERROR: bun install script checksum mismatch\" >&2\n echo \" expected: $BUN_INSTALL_SHA\" >&2\n echo \" got: $actual_sha\" >&2\n rm \"$tmpfile\"; exit 1\n fi\n bash \"$tmpfile\" \"bun-v$BUN_VERSION\"\n rm \"$tmpfile\"\n fi\n ```\n\n```bash\n$B status\n```\nIf status says `Mode: cdp`, stop: the real browser already has sessions.\n\n## 2. Confirm options before import\n\nOpen the known target and keep its tab unchanged. Both options default off and require explicit request:\n\n- **`--verify-auth` / picker checkbox:** reloads the target. Have the user privately configure daemon `GSTACK_COOKIE_AUTH_SELECTOR` and `GSTACK_COOKIE_AUTH_EXPECTED_IDENTITY` **before startup**. Never invent values or assume CLI env reconfigures an existing daemon. Missing config rejects before mutation. Require a successful same-origin response and exactly one visible element whose whitespace-normalized text exactly matches the expected identity.\n- **`--clear-storage`:** for suspected stale storage, obtain explicit approval. Chromium only: clears captured-origin localStorage (shared across same-origin context tabs) and target-tab sessionStorage. Other origins, other tabs' sessionStorage, IndexedDB, and service workers stay intact. It uses an isolated world/native deadline; other engines reject reset, not imports/auth checks. Never auto-approve or claim rollback after partial failure.\n\n## 3. Select source and scope\n\n```bash\n$B cookie-import-browser\n```\n\nAsk the user to choose browser, account/profile, and domains, then say when done. Never guess accounts or treat default Comet as consent. Unreadable profiles are unknown, not empty. Rerun for an expired five-minute one-use link.\n\nDirect import: pass the chosen browser and `--domain` after navigating to a matching target. `--profile` takes a directory, not a display name; omit only for an unambiguous relevant profile, otherwise use the picker. `--all` requires consent for all non-expired profile cookies; it cannot accompany `--domain` or `--clear-storage`.\n\nRead that same root's `BROWSER.md`, **Choosing a source and checking sign-in**, for examples, profile labels, supported sources and platform setup.\n\n## 4. Report honestly\n\nReport receipt/picker counts, partial/zero/error and reset outcomes, not raw `$B cookies`. Imports affect the context, not one tab. **Not checked** means no requested check; **not verified** means it failed; **verified** requires positive target evidence. Zero imports, counts, or HTTP 200 never prove login.\n\nNever request/publish cookie values, passwords, identity/profile text, session details, or raw errors in public logs.\n\n## Platform boundaries"], + ["# view the page; only --verify-auth proves sign-in", "# confirm the intended account"], +]; + export function approvedCookieWorkflowSource(source: string): string { let removedLines = 0; let removedEntrypointLines = 0; - let historical = source + let reordered = source; + for (const [later, earlier] of LATER_COOKIE_ORDER_EDITS) { + if (!reordered.includes(later)) continue; + removedEntrypointLines += later.split('\n').length - earlier.split('\n').length; + reordered = reordered.replace(later, earlier); + } + let historical = reordered .replace('sha256sum < "$tmpfile" | awk \'{print $(1)}\'', 'sha256sum "$tmpfile" | awk \'{print $1}\'') .replace('shasum -a 256 < "$tmpfile" | awk \'{print $(1)}\'', 'shasum -a 256 "$tmpfile" | awk \'{print $1}\'') .replace(LATER_BROWSER_BLOCK, block => { removedLines = block.split('\n').length - 1; return ''; }) - .replace(LATER_ENTRYPOINT_BLOCK, block => { removedEntrypointLines = block.split('\n').length - 1; return ''; }); + .replace(LATER_ENTRYPOINT_BLOCK, block => { removedEntrypointLines += block.split('\n').length - 1; return ''; }); for (const [later, earlier] of [...LATER_BROWSER_EDITS, ...LATER_PREAMBLE_EDITS, ...LATER_WAVE_EDITS, ...LATER_ENTRYPOINT_EDITS]) historical = historical.replace(later, earlier); return historical .replace(/(--- BEGIN FILE "BROWSER\.md" \(lines \d+-)(\d+)(; section\) ---)/, (_, head, end, tail) => `${head}${Number(end) - removedLines}${tail}`) diff --git a/test/office-hours-attempt.test.ts b/test/office-hours-attempt.test.ts index a5345a0f8..29946a398 100644 --- a/test/office-hours-attempt.test.ts +++ b/test/office-hours-attempt.test.ts @@ -654,7 +654,7 @@ describe('Plan format actual capture and judge lifecycle', () => { const starts = events.filter(event => event.kind === 'start'); expect(starts.map(({ timeout, maxTurns, model }) => ({ timeout, maxTurns, model }))) .toEqual([1, 2].map(() => ({ timeout: 300, maxTurns: 10, - model: file.includes('plan-prosons') ? 'claude-opus-4-7' : resolveEvalModel('capture') }))); + model: resolveEvalModel('capture') }))); expect(events.filter(event => event.kind === 'ready').map(event => event.fixtureExists)).toEqual([true, true]); expect(entries).toHaveLength(2); expect(entries.map(entry => entry.attempt)).toEqual([1, 2]); diff --git a/test/office-hours-phase4-caller.test.ts b/test/office-hours-phase4-caller.test.ts index 43c00f767..199ff2476 100644 --- a/test/office-hours-phase4-caller.test.ts +++ b/test/office-hours-phase4-caller.test.ts @@ -2,6 +2,7 @@ import { expect, test } from 'bun:test'; import * as fs from 'node:fs'; import * as path from 'node:path'; import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets'; +import { resolveEvalModel } from '../lib/eval-model'; const source = fs.readFileSync(path.join(import.meta.dir, 'skill-e2e-office-hours-phase4.test.ts'), 'utf8'); const question = (word = 'architectural', options = 'A) Put retrieval on the server\nB) Put retrieval on the client') => `Where should retrieval live?\nThis ${word} choice decides which component owns the retrieval algorithm and the cross-host API contract.\n${options}\nRecommendation: A because all hosts need a consistent algorithm with one owner.\n`; @@ -19,7 +20,7 @@ async function runCaller(captured: string | undefined, scenario: 'success' | 'ti const result = { exitReason: scenario === 'timeout' ? 'timeout' : 'success', browseErrors: [], durationMs: 123, costUsd: 0.1, output: 'public completion' }; const args: Record = { expect, beforeAll: (fn: () => void) => setups.push(fn), afterAll: (fn: () => Promise) => finalizers.push(fn), - CAPTURE_MS, CAPTURE_LONG_MS, ROOT: sourceRoot, runId: 'synthetic-run', + CAPTURE_MS, CAPTURE_LONG_MS, resolveEvalModel, ROOT: sourceRoot, runId: 'synthetic-run', describeIfSelected: (_title: string, _names: string[], fn: () => void) => fn(), testConcurrentIfSelected: (name: string, fn: () => Promise, timeout: number) => { expect(name).toBe('office-hours-phase4-fork'); expect(timeout).toBe(CAPTURE_LONG_MS); callbacks.push(fn); }, createEvalCollector: () => collector, finalizeEvalCollector: async () => {}, logCost: () => {}, @@ -35,7 +36,7 @@ async function runCaller(captured: string | undefined, scenario: 'success' | 'ti fs: { mkdtempSync:(prefix:string)=>prefix+'owned', mkdirSync:()=>{}, rmSync:()=>{}, existsSync:(name:string)=>files.has(name), writeFileSync:(name:string,body:string)=>files.set(name,body), readFileSync:(name:string)=>name===fixturePath.join(sourceRoot,'office-hours','SKILL.md') ? '## AskUserQuestion Format\nformat\n## Phase 4: Alternatives Generation\nworkflow\n## Phase 4.5\nnext' : files.get(name) }, runSkillTest: async (opts: any) => { - calls.push(opts); expect(opts.timeout).toBe(CAPTURE_MS); expect(opts.maxTurns).toBe(12); expect(opts.model).toBe('claude-opus-4-7'); + calls.push(opts); expect(opts.timeout).toBe(CAPTURE_MS); expect(opts.maxTurns).toBe(12); expect(opts.model).toBe(resolveEvalModel('capture')); expect(opts.prompt).toContain('Do NOT call any tool to ask the user.'); if (captured !== undefined) files.set(fixturePath.join(opts.workingDirectory, 'phase4-capture.md'), captured); return result; diff --git a/test/overlay-lifecycle.test.ts b/test/overlay-lifecycle.test.ts index b27124611..66232ce59 100644 --- a/test/overlay-lifecycle.test.ts +++ b/test/overlay-lifecycle.test.ts @@ -28,7 +28,7 @@ test('every fixture owns one paid wrapper; public models/trials/concurrency/turn expect(isPaidTestFile('test/overlay-lifecycle.test.ts')).toBe(false); expect(OVERLAY_FIXTURES.every(f => f.trials === 10 && f.concurrency === 3)).toBe(true); expect(OVERLAY_FIXTURES.map(f => f.maxTurns ?? 5)).toEqual([15,8,15,15,15]); - expect(OVERLAY_FIXTURES.map(f => f.model)).toEqual([...Array(3).fill('claude-opus-4-7'), 'claude-sonnet-4-6', 'claude-opus-5-5']); + expect(OVERLAY_FIXTURES.map(f => f.model)).toEqual([...Array(3).fill('claude-opus-5-5'), 'claude-sonnet-5-5', 'claude-opus-5-5']); expect([OVERLAY_CASE_WORK_MS, OVERLAY_RECORD_GRACE_MS, OVERLAY_CASE_OUTER_MS, OVERLAY_MIN_FILE_WALL_MS]).toEqual([1_800_000, 5_000, 1_810_000, 1_830_000]); }); diff --git a/test/overlay-measurement.test.ts b/test/overlay-measurement.test.ts index 273647113..1f965442e 100644 --- a/test/overlay-measurement.test.ts +++ b/test/overlay-measurement.test.ts @@ -65,7 +65,7 @@ describe('SDK overlay measurements', () => { }); describe('correctness versus efficacy', () => { - describe.each(['claude-opus-4-7', 'claude-sonnet-4-6'])('dedicated-tool gate for %s', (model) => { + describe.each(['claude-opus-5-5', 'claude-sonnet-5-5'])('dedicated-tool gate for %s', (model) => { const f = { ...OVERLAY_FIXTURES.find(f => f.metricName === 'bash_tool_calls' && f.model === model)!, trials: 3 }; // Contract v4: a trial whose final JSON fails the fixture's verify is a valid // measurement with a wrong answer, not an invalid measurement. @@ -276,9 +276,8 @@ describe('literal fixture task correctness', () => { expect(OVERLAY_FIXTURES).toHaveLength(5); expect(OVERLAY_FIXTURES.some((f) => f.id.includes('fanout') || f.comparison?.unsupportedHypothesis)).toBe(false); expect(OVERLAY_FIXTURES.every((f) => f.trials === 10 && f.comparison)).toBe(true); - expect(OVERLAY_FIXTURES.filter((f) => f.model === 'claude-opus-4-7')).toHaveLength(3); - expect(OVERLAY_FIXTURES.filter((f) => f.model === 'claude-sonnet-4-6')).toHaveLength(1); - expect(OVERLAY_FIXTURES.filter((f) => f.model === 'claude-opus-5-5')).toHaveLength(1); + expect(OVERLAY_FIXTURES.filter((f) => f.model === 'claude-opus-5-5')).toHaveLength(4); + expect(OVERLAY_FIXTURES.filter((f) => f.model === 'claude-sonnet-5-5')).toHaveLength(1); }); }); diff --git a/test/skill-e2e-design.test.ts b/test/skill-e2e-design.test.ts index 10c0fe72c..8f078f9ce 100644 --- a/test/skill-e2e-design.test.ts +++ b/test/skill-e2e-design.test.ts @@ -124,7 +124,7 @@ Write DESIGN.md and CLAUDE.md (or update it) in the working directory.`, timeout: CAPTURE_LONG_MS, testName: 'design-consultation-core', runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }); logCost('/design-consultation core', result); @@ -298,7 +298,7 @@ Run Phase 0's DESIGN.md format check exactly as written (the gstack bin director timeout: CAPTURE_LONG_MS, testName: 'design-consultation-existing', runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }); logCost('/design-consultation existing', result); diff --git a/test/skill-e2e-office-hours-phase4.test.ts b/test/skill-e2e-office-hours-phase4.test.ts index 2043f4a55..eb7f35433 100644 --- a/test/skill-e2e-office-hours-phase4.test.ts +++ b/test/skill-e2e-office-hours-phase4.test.ts @@ -22,6 +22,7 @@ import { expect, beforeAll, afterAll } from 'bun:test'; import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; +import { resolveEvalModel } from '../lib/eval-model'; import type { EvalTestEntry } from './helpers/eval-store'; import { ROOT, runId, @@ -166,7 +167,7 @@ After writing the file with that ONE Phase 4 question, stop. Do not continue to timeout: CAPTURE_MS, testName: 'office-hours-phase4-fork', runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }); logCost('/office-hours Phase 4 fork', result); diff --git a/test/skill-e2e-plan-prosons.test.ts b/test/skill-e2e-plan-prosons.test.ts index ab41f9606..7100850d0 100644 --- a/test/skill-e2e-plan-prosons.test.ts +++ b/test/skill-e2e-plan-prosons.test.ts @@ -30,6 +30,7 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { CAPTURE_MS } from './helpers/eval-budgets'; import { NEUTRAL_POSTURE_RE } from './helpers/prosons-posture'; import { runSkillTest } from './helpers/session-runner'; +import { resolveEvalModel } from '../lib/eval-model'; import { runRecordedOfficeHoursAttempt, OFFICE_HOURS_BUN_GRACE_MS } from './helpers/office-hours-attempt'; import { ROOT, runId, @@ -134,7 +135,7 @@ function setupPlanDir(tmpPrefix: string, planContent: string, skillName: string) } function captureInstruction(outFile: string): string { - return `Write the verbatim text of the single AskUserQuestion you would have made to ${outFile} (full text including D header, ELI10, Stakes, Recommendation, Pros/cons, and Net line — the complete rich markdown body). Do NOT call any tool to ask the user. Do NOT paraphrase. This is a format-capture test.`; + return `Write the text of the single AskUserQuestion you would have made to ${outFile} (full text including D header, ELI10, Stakes, Recommendation, Pros/cons, and Net line — the complete rich markdown body). Do NOT call any tool to ask the user.`; } // --- Case 1: Format positive — all v1.7.0.0 tokens present --- @@ -155,7 +156,7 @@ describeIfSelected('Plan Prosons — Format Positive', ['plan-review-prosons-for testConcurrentIfSelected('plan-review-prosons-format', async () => { await runRecordedOfficeHoursAttempt({ collector: evalCollector, name: '/plan-review-prosons-format', suite: 'Plan Prosons — Format Positive', - model: 'claude-opus-4-7', budgetMs: CAPTURE_MS, + model: resolveEvalModel('capture'), budgetMs: CAPTURE_MS, run: signal => runSkillTest({ signal, prompt: `Read plan-ceo-review/SKILL.md for the review workflow. @@ -170,7 +171,7 @@ After writing the file, stop.`, timeout: CAPTURE_MS, testName: 'plan-review-prosons-format', runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }), validate: async (result, signal) => { logCost('/plan-review prosons format positive', result); @@ -217,7 +218,7 @@ describeIfSelected('Plan Prosons — Hard-stop Negative', ['plan-review-prosons- testConcurrentIfSelected('plan-review-prosons-hardstop-neg', async () => { await runRecordedOfficeHoursAttempt({ collector: evalCollector, name: '/plan-review-prosons-hardstop-neg', suite: 'Plan Prosons — Hard-stop Negative', - model: 'claude-opus-4-7', budgetMs: CAPTURE_MS, + model: resolveEvalModel('capture'), budgetMs: CAPTURE_MS, run: signal => runSkillTest({ signal, prompt: `Read plan-ceo-review/SKILL.md. @@ -232,7 +233,7 @@ After writing the file, stop.`, timeout: CAPTURE_MS, testName: 'plan-review-prosons-hardstop-neg', runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }), validate: async (result, signal) => { logCost('/plan-review prosons hard-stop negative', result); @@ -270,7 +271,7 @@ describeIfSelected('Plan Prosons — Neutral-posture Negative', ['plan-review-pr testConcurrentIfSelected('plan-review-prosons-neutral-neg', async () => { await runRecordedOfficeHoursAttempt({ collector: evalCollector, name: '/plan-review-prosons-neutral-neg', suite: 'Plan Prosons — Neutral Negative', - model: 'claude-opus-4-7', budgetMs: CAPTURE_MS, + model: resolveEvalModel('capture'), budgetMs: CAPTURE_MS, run: signal => runSkillTest({ signal, prompt: `Read plan-ceo-review/SKILL.md. @@ -285,7 +286,7 @@ After writing the file, stop.`, timeout: CAPTURE_MS, testName: 'plan-review-prosons-neutral-neg', runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }), validate: async (result, signal) => { logCost('/plan-review prosons neutral negative', result); @@ -324,7 +325,7 @@ describeIfSelected('Plan Prosons — Hard-stop Positive', ['plan-ceo-review-pros testConcurrentIfSelected('plan-ceo-review-prosons-cadence', async () => { await runRecordedOfficeHoursAttempt({ collector: evalCollector, name: '/plan-ceo-review-prosons-cadence', suite: 'Plan Prosons — Hard-stop Positive', - model: 'claude-opus-4-7', budgetMs: CAPTURE_MS, + model: resolveEvalModel('capture'), budgetMs: CAPTURE_MS, run: signal => runSkillTest({ signal, prompt: `Read plan-ceo-review/SKILL.md. @@ -339,7 +340,7 @@ After writing the file, stop.`, timeout: CAPTURE_MS, testName: 'plan-ceo-review-prosons-cadence', runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }), validate: async (result, signal) => { logCost('/plan-review prosons hard-stop positive', result); diff --git a/test/skill-e2e-plan.test.ts b/test/skill-e2e-plan.test.ts index 92b79175c..2f7a7eec8 100644 --- a/test/skill-e2e-plan.test.ts +++ b/test/skill-e2e-plan.test.ts @@ -91,7 +91,7 @@ Focus on reviewing the plan content: architecture, error handling, security, and timeout: 540_000, testName: 'plan-ceo-review', runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }); logCost('/plan-ceo-review', result); @@ -178,7 +178,7 @@ Focus on reviewing the plan content: architecture, error handling, security, and timeout: CAPTURE_LONG_MS, testName: 'plan-ceo-review-selective', runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }); logCost('/plan-ceo-review (SELECTIVE)', result); @@ -246,7 +246,7 @@ Write your expansion proposals to ${planDir}/proposals.md with ONLY the proposal timeout: CAPTURE_LONG_MS, testName: 'plan-ceo-review-expansion-energy', runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }); logCost('/plan-ceo-review (EXPANSION ENERGY)', result); @@ -355,7 +355,7 @@ Focus on architecture, code quality, tests, and performance sections.`, timeout: CAPTURE_LONG_MS, testName: 'plan-eng-review', runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }); logCost('/plan-eng-review', result); diff --git a/test/skill-e2e-qa-bugs.test.ts b/test/skill-e2e-qa-bugs.test.ts index 14db97bf6..1a59acf33 100644 --- a/test/skill-e2e-qa-bugs.test.ts +++ b/test/skill-e2e-qa-bugs.test.ts @@ -1,6 +1,7 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets'; import { runSkillTest } from './helpers/session-runner'; +import { resolveEvalModel } from '../lib/eval-model'; import { outcomeJudge } from './helpers/llm-judge'; import { judgePassed } from './helpers/eval-store'; import { @@ -108,7 +109,7 @@ CRITICAL RULES: timeout: CAPTURE_MS, testName: `qa-${label}`, runId, - model: 'claude-opus-4-7', + model: resolveEvalModel('capture'), }); logCost(`/qa ${label}`, result); diff --git a/test/skill-positional-literals.test.ts b/test/skill-positional-literals.test.ts index e8280a397..e691697c4 100644 --- a/test/skill-positional-literals.test.ts +++ b/test/skill-positional-literals.test.ts @@ -75,7 +75,7 @@ for (const host of ['claude', 'codex'] as const) for (const args of [[], tenArgu const file = backslashPath ? join(root, 'checksum\\path', 'install-script') : join(root, 'install-script'); mkdirSync(dirname(file), { recursive: true }); writeFileSync(file, 'synthetic installer bytes\n'); - for (const name of ['open-gstack-browser', 'pair-agent', 'setup-browser-cookies']) { + for (const name of ['open-gstack-browser', 'pair-agent']) { const lines = apply(skill(host, name)).split('\n').filter(line => line.includes('actual_sha=$(')); expect(lines.map(line => line.trim())).toEqual([ literals.checksum,