From 2356792e43e0fcba961dacaa40c22f0c18f4a425 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Fri, 30 Oct 2026 22:53:14 -0700 Subject: [PATCH] v1.91.352.0 feat(benchmark): particle-1 awareness, OpenRouter gateway provider, preregistered model panel (GSTA-173) (#3631) * feat(benchmark): particle-1 and DeepSeek V4.1 Flash awareness, gateway provider, preregistered model panel (GSTA-173) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(evals): particle-1 model panel results, run 2026-10-30-0105 (GSTA-173) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test(model-panel): name the task field id (credential scanner false positive on key) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test: gateway reports a missing OpenRouter key first; register the model panel in the paid count and test-of-test baseline Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test: run the fake claude through bun so the gateway provider test passes on Windows Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: pre-landing review fixes Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: gateway benchmark runs load no Claude Code settings A settings.json env block's ANTHROPIC_API_KEY was sent to the OpenRouter base URL as x-api-key; --setting-sources '' loads no settings source. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs: name particle-1 and DeepSeek V4.1 Flash in the /benchmark-models row Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * chore: changelog fragment and TODOS for the particle-1 model panel Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs: name particle-1 and DeepSeek V4.1 Flash in the AGENTS.md /benchmark-models row Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- AGENTS.md | 2 +- CHANGELOG.md | 16 + TODOS.md | 8 + VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- benchmark-models/SKILL.md | 6 +- benchmark-models/SKILL.md.tmpl | 9 +- bin/gstack-model-benchmark | 31 +- docs/evals/particle-1-2026-10.md | 105 +++++++ docs/evals/particle-1-2026-10/rows.jsonl | 60 ++++ docs/evals/particle-1-2026-10/summary.json | 67 +++++ docs/skills.md | 2 +- lib/outside-voice-runner.ts | 1 + lib/pricing.ts | 6 + package.json | 2 +- test/benchmark-cli.test.ts | 18 ++ test/eng-finding-retry-budget.test.ts | 2 +- .../model-panel/changelog/CHANGELOG.md | 6 + .../fixtures/model-panel/changelog/COMMITS.md | 8 + .../model-panel/changelog/package.json | 5 + .../model-panel/checks/bugfix-decoys.js | 27 ++ test/fixtures/model-panel/checks/changelog.js | 30 ++ .../model-panel/checks/crud-endpoint.js | 32 ++ .../model-panel/checks/investigate.js | 27 ++ .../model-panel/checks/review-seeded-bug.js | 25 ++ .../model-panel/investigate/README.md | 4 + .../model-panel/investigate/package.json | 5 + .../model-panel/investigate/run-tests.js | 10 + .../model-panel/investigate/src/dates.js | 6 + .../model-panel/investigate/src/report.js | 9 + .../model-panel/investigate/src/retry.js | 11 + .../bugfix-decoys/src/format-price.js | 7 + .../reference/changelog/CHANGELOG.md | 12 + .../reference/crud-endpoint/app.js | 28 ++ .../reference/crud-endpoint/run-tests.js | 9 + .../reference/investigate/src/dates.js | 5 + .../reference/review-seeded-bug/REVIEW.md | 4 + .../model-panel/review-seeded-bug/PR.md | 4 + .../model-panel/review-seeded-bug/README.md | 4 + .../review-seeded-bug/package.json | 5 + .../model-panel/review-seeded-bug/src/db.js | 10 + .../review-seeded-bug/src/users.js | 14 + test/gateway-provider.test.ts | 118 ++++++++ test/helpers/benchmark-runner.ts | 20 +- test/helpers/model-panel-harness.ts | 277 ++++++++++++++++++ test/helpers/periodic-exclude-data.ts | 4 + test/helpers/providers/claude.ts | 16 +- test/helpers/providers/gateway.ts | 102 +++++++ test/helpers/providers/registry.ts | 21 ++ test/helpers/providers/types.ts | 4 + test/helpers/touchfiles-data.ts | 83 ++++++ test/model-panel-selftest.test.ts | 115 ++++++++ test/skill-e2e-model-panel.test.ts | 132 +++++++++ test/test-of-test-ratchet.test.ts | 1 + 54 files changed, 1494 insertions(+), 45 deletions(-) create mode 100644 docs/evals/particle-1-2026-10.md create mode 100644 docs/evals/particle-1-2026-10/rows.jsonl create mode 100644 docs/evals/particle-1-2026-10/summary.json create mode 100644 test/fixtures/model-panel/changelog/CHANGELOG.md create mode 100644 test/fixtures/model-panel/changelog/COMMITS.md create mode 100644 test/fixtures/model-panel/changelog/package.json create mode 100644 test/fixtures/model-panel/checks/bugfix-decoys.js create mode 100644 test/fixtures/model-panel/checks/changelog.js create mode 100644 test/fixtures/model-panel/checks/crud-endpoint.js create mode 100644 test/fixtures/model-panel/checks/investigate.js create mode 100644 test/fixtures/model-panel/checks/review-seeded-bug.js create mode 100644 test/fixtures/model-panel/investigate/README.md create mode 100644 test/fixtures/model-panel/investigate/package.json create mode 100644 test/fixtures/model-panel/investigate/run-tests.js create mode 100644 test/fixtures/model-panel/investigate/src/dates.js create mode 100644 test/fixtures/model-panel/investigate/src/report.js create mode 100644 test/fixtures/model-panel/investigate/src/retry.js create mode 100644 test/fixtures/model-panel/reference/bugfix-decoys/src/format-price.js create mode 100644 test/fixtures/model-panel/reference/changelog/CHANGELOG.md create mode 100644 test/fixtures/model-panel/reference/crud-endpoint/app.js create mode 100644 test/fixtures/model-panel/reference/crud-endpoint/run-tests.js create mode 100644 test/fixtures/model-panel/reference/investigate/src/dates.js create mode 100644 test/fixtures/model-panel/reference/review-seeded-bug/REVIEW.md create mode 100644 test/fixtures/model-panel/review-seeded-bug/PR.md create mode 100644 test/fixtures/model-panel/review-seeded-bug/README.md create mode 100644 test/fixtures/model-panel/review-seeded-bug/package.json create mode 100644 test/fixtures/model-panel/review-seeded-bug/src/db.js create mode 100644 test/fixtures/model-panel/review-seeded-bug/src/users.js create mode 100644 test/gateway-provider.test.ts create mode 100644 test/helpers/model-panel-harness.ts create mode 100644 test/helpers/providers/gateway.ts create mode 100644 test/helpers/providers/registry.ts create mode 100644 test/model-panel-selftest.test.ts create mode 100644 test/skill-e2e-model-panel.test.ts diff --git a/AGENTS.md b/AGENTS.md index 4232c097b..d35d25188 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -70,7 +70,7 @@ Invoke them by name (e.g., `/office-hours`). | `/captain` | Run a crew from one controller: READY contract, merge trains, digest, approvals shapes, daily flaky pass. Also `/crew`. Templates in `captain/templates/`, team guide in `docs/captain.md`. | | `/benchmark` | Performance regression detection (page load, Core Web Vitals). | | `/ubicloud` | Run a heavy suite, build or eval on an ephemeral 16-vCPU Ubicloud VM: TTL-named VMs with a reaper, quota preflight, hang watchdog, one committed verify per repository. | -| `/benchmark-models` | Cross-model benchmark for skills (Claude, GPT, Gemini side-by-side). | +| `/benchmark-models` | Cross-model benchmark for skills (Claude, GPT, Gemini, plus particle-1 and DeepSeek V4.1 Flash through OpenRouter, side-by-side). | | `/eval-plan` | Preregister a paid eval (bars, arms, decoys, stop rules), dry-run it for $0, pilot three priced items, then approve the spend. | | `/cso` | Supported security findings with explicit coverage. Static assessment remains available without catalog profiles. Contained runtime and scanner execution needs Docker and a matching qualified profile. The runtime catalog qualifies ten images: Node 24.4.0, Bun 1.3.10, Python 3.13.4 with uv 0.8.0, Rails on Ruby 3.4.4, and PostgreSQL 17.2, each for linux/amd64 and linux/arm64. The scanner catalog qualifies twelve: gitleaks, OSV-Scanner, Schemathesis, Semgrep, Trivy and zizmor on both platforms. Without Docker, or where no profile matches (native Windows Docker, for example), `/cso` runs static assessment only. Runtime-tested bundles authenticate separate external assertions. Project-test completion remains `self_reported` because target code controls the test process; `tested` is reserved for a future target-independent completion witness. | | `/setup-gbrain` | Set up gbrain for cross-machine session memory sync. | diff --git a/CHANGELOG.md b/CHANGELOG.md index 167e90621..a5ce67333 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## [1.91.352.0] - 2026-10-31 + +**gstack now knows particle-1 and has measured it on its own tasks.** particle-1 (Particle AI, October 28, 2026) passed 15/15 cells on a preregistered five-task panel, the same as Sonnet 5.5 and Opus 5.5, at $0.007 per task against Sonnet 5.5's $0.037 (run `2026-10-30-0105`, Claude Code through OpenRouter). The panel is easy, so it shows no measured difference between the models, not that they are equal. Results and every cell are in [docs/evals/particle-1-2026-10.md](docs/evals/particle-1-2026-10.md). + +### Added +- **particle-1 and DeepSeek V4.1 Flash in the price table.** `lib/pricing.ts` prices `particle-1` ($0.25 input, $0.03 cached, $1.00 output per million tokens) and `deepseek-v4.1-flash-1017`; provider-prefixed ids such as `particle-ai/particle-1` resolve to them. +- **`gstack-model-benchmark --models particle,deepseek`** runs those models in Claude Code through OpenRouter (`OPENROUTER_API_KEY`). The child process gets no Anthropic credential, every Claude Code model alias is pinned to the arm's model, and a run that billed any other model is an error. `/benchmark-models` lists both and says what they send to OpenRouter. +- **A preregistered model panel** (`test/skill-e2e-model-panel.test.ts`): five gstack-shaped tasks, four models, three trials, each cell scored by a hidden check and priced from its own tokens. It runs on demand, not in the weekly eval run, because CI has no OpenRouter key. + +### Changed +- **particle-1 counts as the DeepSeek family** for outside-voice independence, because Particle built it on DeepSeek V4.1 Flash. A particle-1 voice is never treated as independent of a DeepSeek one. +- **One provider registry** feeds the benchmark CLI, its dry run and the runner. + +### Fixed +- **`--models` with no valid provider now exits 1** instead of quietly running claude. + ## [1.91.351.0] - 2026-10-31 **Four skill-eval check commands that gstack-evals' Lane A vendors as graders no longer pass runs that break their rule.** gstack-evals' grader audit (GST-374) showed that one agent-written `.gitignore` could hide new code from the /office-hours no-code check, and that three smaller gaps let other checks pass on the wrong files. diff --git a/TODOS.md b/TODOS.md index b599ee149..a5246b162 100644 --- a/TODOS.md +++ b/TODOS.md @@ -1716,6 +1716,14 @@ run locally on a machine that has the CLI or device. **Re-entry:** the device or Aside is available to a CI runner. Remove each file's exclude entry when its prerequisite exists. +**Model panel (GSTA-173):** `skill-e2e-model-panel` is excluded too: it needs +`OPENROUTER_API_KEY`, which no CI workflow carries, and it is a research +instrument run on demand. Before its next preregistered run, fix three harness +gaps the October 2026 review found: score a timed-out cell's working tree +instead of calling it INFRA (`classifyCell`), summarize only the current run's +rows (`afterAll` reads every row in the output directory), and drain the worker +pool on the first error so no cell starts after a failure (`pool`). + **Review by:** 2026-12-28. **Effort:** S per file. **Priority:** P3. ### ✅ DONE (2026-10-02, parallel-waves fix wave): Install the Codex CLI in the CI image diff --git a/VERSION b/VERSION index 2545ce5e7..53700a2e0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.351.0 +1.91.352.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 4a761d73e..2fb6476fd 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.351.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.352.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/benchmark-models/SKILL.md b/benchmark-models/SKILL.md index 06410fc4d..0f3444718 100644 --- a/benchmark-models/SKILL.md +++ b/benchmark-models/SKILL.md @@ -196,12 +196,14 @@ If C: ask for the path. Verify it exists. Use as positional argument. ## Step 2: Choose providers ```bash -"$BIN" --prompt "unused, dry-run" --models claude,gpt,gemini --dry-run +"$BIN" --prompt "unused, dry-run" --models claude,gpt,gemini,particle,deepseek --dry-run ``` Show the dry-run output. The "Adapter availability" section tells the user which providers will actually run (OK) vs skip (NOT READY — remediation hint included). -If ALL three show NOT READY: stop with a clear message — benchmark can't run without at least one authed provider. Suggest `claude login`, `codex login`, or `gemini login` / `export GOOGLE_API_KEY`. +If every provider shows NOT READY: stop with a clear message — benchmark can't run without at least one authed provider. Suggest `claude login`, `codex login`, `gemini login` / `export GOOGLE_API_KEY`, or `export OPENROUTER_API_KEY` for particle and deepseek (they run in Claude Code, so they also need the `claude` CLI). + +particle and deepseek send the prompt and the working directory's files the model reads to OpenRouter and the model vendor. They load no Claude Code settings files (so a key in a settings.json `env` block is never sent) and run with the user's environment minus Anthropic credentials. Say so before including them. If at least one is OK: AskUserQuestion: - **Simplify:** "Which models should we include? The dry-run above showed which are authed. Unauthed ones will be skipped cleanly — they won't abort the batch." diff --git a/benchmark-models/SKILL.md.tmpl b/benchmark-models/SKILL.md.tmpl index 7360183c7..dbc447bb5 100644 --- a/benchmark-models/SKILL.md.tmpl +++ b/benchmark-models/SKILL.md.tmpl @@ -4,7 +4,8 @@ preamble-tier: 1 version: 1.0.0 description: | Cross-model benchmark for gstack skills. Runs the same prompt through Claude, - GPT (via Codex CLI), and Gemini side-by-side — compares latency, tokens, cost, + GPT (via Codex CLI), Gemini, and (through OpenRouter in Claude Code) particle-1 + and DeepSeek V4.1 Flash side-by-side — compares latency, tokens, cost, and optionally quality via LLM judge. Answers "which model is actually best for this skill?" with data instead of vibes. Separate from /benchmark, which measures web page performance. Use when: "benchmark models", "compare models", @@ -71,12 +72,14 @@ If C: ask for the path. Verify it exists. Use as positional argument. ## Step 2: Choose providers ```bash -"$BIN" --prompt "unused, dry-run" --models claude,gpt,gemini --dry-run +"$BIN" --prompt "unused, dry-run" --models claude,gpt,gemini,particle,deepseek --dry-run ``` Show the dry-run output. The "Adapter availability" section tells the user which providers will actually run (OK) vs skip (NOT READY — remediation hint included). -If ALL three show NOT READY: stop with a clear message — benchmark can't run without at least one authed provider. Suggest `claude login`, `codex login`, or `gemini login` / `export GOOGLE_API_KEY`. +If every provider shows NOT READY: stop with a clear message — benchmark can't run without at least one authed provider. Suggest `claude login`, `codex login`, `gemini login` / `export GOOGLE_API_KEY`, or `export OPENROUTER_API_KEY` for particle and deepseek (they run in Claude Code, so they also need the `claude` CLI). + +particle and deepseek send the prompt and the working directory's files the model reads to OpenRouter and the model vendor. They load no Claude Code settings files (so a key in a settings.json `env` block is never sent) and run with the user's environment minus Anthropic credentials. Say so before including them. If at least one is OK: AskUserQuestion: - **Simplify:** "Which models should we include? The dry-run above showed which are authed. Unauthed ones will be skipped cleanly — they won't abort the batch." diff --git a/bin/gstack-model-benchmark b/bin/gstack-model-benchmark index 81ed9dd1f..dfcb22693 100755 --- a/bin/gstack-model-benchmark +++ b/bin/gstack-model-benchmark @@ -7,7 +7,8 @@ * gstack-model-benchmark [options] * * Options: - * --models claude,gpt,gemini Comma-separated provider list (default: claude) + * --models claude,gpt,gemini Comma-separated provider list (default: claude); particle and + * deepseek run in Claude Code through OpenRouter (OPENROUTER_API_KEY) * --prompt "" Inline prompt instead of a file * --workdir Working dir passed to each CLI (default: cwd) * --timeout-ms Per-provider timeout (default: 300000) @@ -28,15 +29,7 @@ import '../lib/conductor-env-shim'; import * as fs from 'fs'; import * as path from 'path'; import { runBenchmark, formatTable, formatJson, formatMarkdown, type BenchmarkInput } from '../test/helpers/benchmark-runner'; -import { ClaudeAdapter } from '../test/helpers/providers/claude'; -import { GptAdapter } from '../test/helpers/providers/gpt'; -import { GeminiAdapter } from '../test/helpers/providers/gemini'; - -const ADAPTER_FACTORIES = { - claude: () => new ClaudeAdapter(), - gpt: () => new GptAdapter(), - gemini: () => new GeminiAdapter(), -}; +import { PROVIDERS, PROVIDER_NAMES, isProviderName, type ProviderName } from '../test/helpers/providers/registry'; type OutputFormat = 'table' | 'json' | 'markdown'; @@ -76,16 +69,20 @@ function positionalArgs(args: string[]): string[] { return positional; } -function parseProviders(s: string | undefined): Array<'claude' | 'gpt' | 'gemini'> { +function parseProviders(s: string | undefined): ProviderName[] { if (!s) return ['claude']; - const seen = new Set<'claude' | 'gpt' | 'gemini'>(); + const seen = new Set(); for (const p of s.split(',').map(x => x.trim()).filter(Boolean)) { - if (p === 'claude' || p === 'gpt' || p === 'gemini') seen.add(p); + if (isProviderName(p)) seen.add(p); else { - console.error(`WARN: unknown provider '${p}' — skipping. Valid: claude, gpt, gemini.`); + console.error(`WARN: unknown provider '${p}' — skipping. Valid: ${PROVIDER_NAMES.join(', ')}.`); } } - return seen.size ? Array.from(seen) : ['claude']; + if (!seen.size) { + console.error(`No valid provider in --models '${s}'. Valid: ${PROVIDER_NAMES.join(', ')}.`); + process.exit(1); + } + return Array.from(seen); } function parsePositiveIntegerFlag(name: string, def: string): number { @@ -163,7 +160,7 @@ async function main(): Promise { async function dryRunReport(opts: { prompt: string; - providers: Array<'claude' | 'gpt' | 'gemini'>; + providers: ProviderName[]; workdir: string; timeoutMs: number; output: OutputFormat; @@ -181,7 +178,7 @@ async function dryRunReport(opts: { lines.push('Adapter availability:'); let authFailures = 0; for (const name of opts.providers) { - const factory = ADAPTER_FACTORIES[name]; + const factory = PROVIDERS[name]; if (!factory) { lines.push(` ${name}: UNKNOWN PROVIDER`); authFailures += 1; diff --git a/docs/evals/particle-1-2026-10.md b/docs/evals/particle-1-2026-10.md new file mode 100644 index 000000000..12e969a3e --- /dev/null +++ b/docs/evals/particle-1-2026-10.md @@ -0,0 +1,105 @@ +# particle-1 on gstack's model panel (October 2026) + +Measured October 30, 2026 at commit d209821, which preregistered the panel before any paid run. Run id `2026-10-30-0105`. Data: [rows.jsonl](particle-1-2026-10/rows.jsonl) (one row per cell) and [summary.json](particle-1-2026-10/summary.json). + +## Result + +| Model | Passed | 95% interval | INFRA cells | Cost per task | +|---|---|---|---|---| +| Opus 5.5 (`claude-opus-5-5`) | 15/15 | 80–100% | 0 | $0.088 | +| Sonnet 5.5 (`claude-sonnet-5-5`) | 15/15 | 80–100% | 0 | $0.037 | +| particle-1 (`particle-ai/particle-1`) | 15/15 | 80–100% | 0 | $0.007 | +| DeepSeek V4.1 Flash (`deepseek/deepseek-v4.1-flash-1017`) | 13/14 | 69–99% | 1 | $0.008 | + +The preregistered wording rule (`suggestionLine` in [test/helpers/model-panel-harness.ts](../../test/helpers/model-panel-harness.ts)) produced this line: + +> particle-1 passed 15/15 tasks on gstack's model panel (Sonnet 5.5 15/15, Opus 5.5 15/15, DeepSeek V4.1 Flash 13/14) at $0.007 per task vs $0.037 for Sonnet 5.5, the cheapest of the four models measured (no measured difference from Sonnet 5.5 on this panel; measured 2026-10-30 in Claude Code via OpenRouter). Worth trying for low-stakes steps; keep blocking reviewers on your main model. + +## How to read it + +- **The panel is easy.** 58 of the 59 scored cells passed, across all four models. So the panel shows that particle-1 can do small gstack-shaped tasks at about one-fifth of Sonnet 5.5's cost per task. It can't tell the models apart on these tasks, so it says nothing about harder work. "No measured difference" means exactly that, not "as good as Sonnet". +- **Cost is measured, not listed.** Each cell is priced from its own token counts (uncached input, cache reads, cache writes, output) at the rates in `lib/pricing.ts`, not from Claude Code's `total_cost_usd`, which prices gateway models at Anthropic rates. particle-1 and DeepSeek ran at OpenRouter's standard (peak) rates. Particle's own API is 50% cheaper from 8:30 pm to 4 am Eastern. +- **Same harness for every model.** Every arm ran in Claude Code `2.1.312` with the same tools, turn cap (40), timeout (8 minutes), the extracted gstack build-discipline skill and a hidden check. Claude Code's default effort settings were used for every arm, and particle-1 used its default reasoning effort (high). Particle trained particle-1 in a different agent harness (Terminus 2), so this setup may understate it. +- **No model mixing.** Every gateway cell billed only its own model. The harness pins every Claude Code model alias to the arm's model and would classify any other billed model as INFRA (MODEL_MIX). No gateway cell had an Anthropic credential in its environment. + +## The panel + +Five tasks, each scored by a hidden check that the selftest proves fails on the seed fixture and passes on a reference answer ([test/model-panel-selftest.test.ts](../../test/model-panel-selftest.test.ts)): + +1. **bugfix:** fix a price-formatting bug without touching the tests (`arm-benchmark/bugfix-decoys`). +2. **crud:** add `DELETE /notes/:id` with 204/404 and a test (`arm-benchmark/crud-endpoint`). +3. **review:** pre-landing review of a new file with a planted SQL injection and a page-offset bug. REVIEW.md must name both, and the source must stay unchanged. +4. **changelog:** write the 1.4.0 entry from a commit list, user-facing changes only. +5. **investigate:** find the root cause of a timezone date bug, leaving the tests and a retry decoy untouched. The fix must also hold in a UTC+14 zone. + +Four models × five tasks × three trials = 60 cells, at most four at a time, under a $15 spend ledger. The run spent $2.09, plus $0.15 still counted at its reservation for the one cell with no usage. + +## Non-passing cells + +- **DeepSeek, review, trial 1: INFRA.** The cell hit the 8-minute timeout. The CLI was killed before it wrote a result event, and `classifyCell` counts a missing result event as INFRA. That conflicts with the function's own comment, which says a timeout after work is scored by the check. The run kept the verdict as computed, because preregistered rules don't change after a result is seen. The fix (score a timed-out cell's working tree) is left for the next run. It affects only the DeepSeek count. +- **DeepSeek, investigate, trial 2: fail.** It fixed the date bug, then deleted the retry decoy the ticket said to leave alone. + +## Every cell + +| Task | Arm | Trial | Verdict | Turns | Cost (USD) | Note | +|---|---|---|---|---|---|---| +| bugfix | opus | 1 | pass | 6 | 0.0724 | | +| bugfix | opus | 2 | pass | 5 | 0.0673 | | +| bugfix | opus | 3 | pass | 6 | 0.0726 | | +| bugfix | sonnet | 1 | pass | 7 | 0.0346 | | +| bugfix | sonnet | 2 | pass | 7 | 0.0336 | | +| bugfix | sonnet | 3 | pass | 7 | 0.0335 | | +| bugfix | particle | 1 | pass | 7 | 0.0058 | | +| bugfix | particle | 2 | pass | 7 | 0.0063 | | +| bugfix | particle | 3 | pass | 10 | 0.0080 | | +| bugfix | deepseek | 1 | pass | 9 | 0.0055 | | +| bugfix | deepseek | 2 | pass | 12 | 0.0036 | | +| bugfix | deepseek | 3 | pass | 7 | 0.0035 | | +| changelog | opus | 1 | pass | 5 | 0.0649 | | +| changelog | opus | 2 | pass | 5 | 0.0638 | | +| changelog | opus | 3 | pass | 5 | 0.0644 | | +| changelog | sonnet | 1 | pass | 6 | 0.0294 | | +| changelog | sonnet | 2 | pass | 6 | 0.0312 | | +| changelog | sonnet | 3 | pass | 6 | 0.0314 | | +| changelog | particle | 1 | pass | 6 | 0.0035 | | +| changelog | particle | 2 | pass | 6 | 0.0034 | | +| changelog | particle | 3 | pass | 7 | 0.0034 | | +| changelog | deepseek | 1 | pass | 7 | 0.0052 | | +| changelog | deepseek | 2 | pass | 6 | 0.0081 | | +| changelog | deepseek | 3 | pass | 6 | 0.0076 | | +| crud | opus | 1 | pass | 6 | 0.0905 | | +| crud | opus | 2 | pass | 6 | 0.0899 | | +| crud | opus | 3 | pass | 5 | 0.0832 | | +| crud | sonnet | 1 | pass | 9 | 0.0401 | | +| crud | sonnet | 2 | pass | 9 | 0.0408 | | +| crud | sonnet | 3 | pass | 9 | 0.0411 | | +| crud | particle | 1 | pass | 10 | 0.0110 | | +| crud | particle | 2 | pass | 11 | 0.0071 | | +| crud | particle | 3 | pass | 9 | 0.0077 | | +| crud | deepseek | 1 | pass | 9 | 0.0061 | | +| crud | deepseek | 2 | pass | 7 | 0.0061 | | +| crud | deepseek | 3 | pass | 9 | 0.0046 | | +| investigate | opus | 1 | pass | 8 | 0.1070 | | +| investigate | opus | 2 | pass | 6 | 0.0949 | | +| investigate | opus | 3 | pass | 6 | 0.0933 | | +| investigate | sonnet | 1 | pass | 6 | 0.0358 | | +| investigate | sonnet | 2 | pass | 6 | 0.0359 | | +| investigate | sonnet | 3 | pass | 6 | 0.0352 | | +| investigate | particle | 1 | pass | 8 | 0.0062 | | +| investigate | particle | 2 | pass | 8 | 0.0064 | | +| investigate | particle | 3 | pass | 8 | 0.0100 | | +| investigate | deepseek | 1 | pass | 10 | 0.0070 | | +| investigate | deepseek | 2 | fail | 12 | 0.0124 | FAIL: the retry decoy must not change | +| investigate | deepseek | 3 | pass | 16 | 0.0056 | | +| review | opus | 1 | pass | 6 | 0.1140 | | +| review | opus | 2 | pass | 6 | 0.1214 | | +| review | opus | 3 | pass | 6 | 0.1136 | | +| review | sonnet | 1 | pass | 6 | 0.0466 | | +| review | sonnet | 2 | pass | 6 | 0.0448 | | +| review | sonnet | 3 | pass | 6 | 0.0454 | | +| review | particle | 1 | pass | 9 | 0.0060 | | +| review | particle | 2 | pass | 11 | 0.0117 | | +| review | particle | 3 | pass | 10 | 0.0081 | | +| review | deepseek | 1 | infra | - | unknown | no result event (timeout) | +| review | deepseek | 2 | pass | 18 | 0.0163 | | +| review | deepseek | 3 | pass | 13 | 0.0177 | | diff --git a/docs/evals/particle-1-2026-10/rows.jsonl b/docs/evals/particle-1-2026-10/rows.jsonl new file mode 100644 index 000000000..5b42dd4fa --- /dev/null +++ b/docs/evals/particle-1-2026-10/rows.jsonl @@ -0,0 +1,60 @@ +{"runId":"2026-10-30-0105","task":"model-panel-bugfix","arm":"sonnet","model":"claude-sonnet-5-5","route":"anthropic","trial":1,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.03461,"usage":{"input":12,"cached":92509,"cacheWrite":7822,"output":578,"modelsBilled":["claude-sonnet-5-5"],"isError":false,"apiErrorStatus":null,"turns":7},"durationMs":9206,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-bugfix","arm":"opus","model":"claude-opus-5-5","route":"anthropic","trial":2,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.067284,"usage":{"input":10,"cached":72194,"cacheWrite":8121,"output":610,"modelsBilled":["claude-opus-5-5"],"isError":false,"apiErrorStatus":null,"turns":5},"durationMs":12496,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-bugfix","arm":"opus","model":"claude-opus-5-5","route":"anthropic","trial":3,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.072601,"usage":{"input":12,"cached":92892,"cacheWrite":8239,"output":639,"modelsBilled":["claude-opus-5-5"],"isError":false,"apiErrorStatus":null,"turns":6},"durationMs":12830,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-bugfix","arm":"opus","model":"claude-opus-5-5","route":"anthropic","trial":1,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.072444,"usage":{"input":12,"cached":92832,"cacheWrite":8206,"output":640,"modelsBilled":["claude-opus-5-5"],"isError":false,"apiErrorStatus":null,"turns":6},"durationMs":13919,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-bugfix","arm":"particle","model":"particle-ai/particle-1","route":"openrouter","trial":1,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.005808,"usage":{"input":14466,"cached":54784,"cacheWrite":0,"output":548,"modelsBilled":["particle-ai/particle-1"],"isError":false,"apiErrorStatus":null,"turns":7},"durationMs":4607,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-bugfix","arm":"particle","model":"particle-ai/particle-1","route":"openrouter","trial":2,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.006323,"usage":{"input":15068,"cached":68608,"cacheWrite":0,"output":498,"modelsBilled":["particle-ai/particle-1"],"isError":false,"apiErrorStatus":null,"turns":7},"durationMs":4898,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-bugfix","arm":"sonnet","model":"claude-sonnet-5-5","route":"anthropic","trial":2,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.033591,"usage":{"input":12,"cached":92149,"cacheWrite":7637,"output":526,"modelsBilled":["claude-sonnet-5-5"],"isError":false,"apiErrorStatus":null,"turns":7},"durationMs":9811,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-bugfix","arm":"sonnet","model":"claude-sonnet-5-5","route":"anthropic","trial":3,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.033521,"usage":{"input":12,"cached":92149,"cacheWrite":7625,"output":522,"modelsBilled":["claude-sonnet-5-5"],"isError":false,"apiErrorStatus":null,"turns":7},"durationMs":7762,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-bugfix","arm":"particle","model":"particle-ai/particle-1","route":"openrouter","trial":3,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.008017,"usage":{"input":14387,"cached":101376,"cacheWrite":0,"output":1379,"modelsBilled":["particle-ai/particle-1"],"isError":false,"apiErrorStatus":null,"turns":10},"durationMs":7579,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-bugfix","arm":"deepseek","model":"deepseek/deepseek-v4.1-flash-1017","route":"openrouter","trial":3,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.003524,"usage":{"input":15976,"cached":67180,"cacheWrite":0,"output":1544,"modelsBilled":["deepseek/deepseek-v4.1-flash-1017"],"isError":false,"apiErrorStatus":null,"turns":7},"durationMs":26251,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-bugfix","arm":"deepseek","model":"deepseek/deepseek-v4.1-flash-1017","route":"openrouter","trial":2,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.003609,"usage":{"input":16346,"cached":68714,"cacheWrite":0,"output":1585,"modelsBilled":["deepseek/deepseek-v4.1-flash-1017"],"isError":false,"apiErrorStatus":null,"turns":12},"durationMs":31334,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-bugfix","arm":"deepseek","model":"deepseek/deepseek-v4.1-flash-1017","route":"openrouter","trial":1,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.005491,"usage":{"input":27662,"cached":55384,"cacheWrite":0,"output":1959,"modelsBilled":["deepseek/deepseek-v4.1-flash-1017"],"isError":false,"apiErrorStatus":null,"turns":9},"durationMs":34471,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-crud","arm":"sonnet","model":"claude-sonnet-5-5","route":"anthropic","trial":1,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.040119,"usage":{"input":10,"cached":72911,"cacheWrite":8971,"output":1038,"modelsBilled":["claude-sonnet-5-5"],"isError":false,"apiErrorStatus":null,"turns":9},"durationMs":9684,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-crud","arm":"opus","model":"claude-opus-5-5","route":"anthropic","trial":3,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.083153,"usage":{"input":10,"cached":72540,"cacheWrite":8873,"output":1212,"modelsBilled":["claude-opus-5-5"],"isError":false,"apiErrorStatus":null,"turns":5},"durationMs":15883,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-crud","arm":"opus","model":"claude-opus-5-5","route":"anthropic","trial":1,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.090505,"usage":{"input":12,"cached":94183,"cacheWrite":9008,"output":1329,"modelsBilled":["claude-opus-5-5"],"isError":false,"apiErrorStatus":null,"turns":6},"durationMs":17663,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-crud","arm":"opus","model":"claude-opus-5-5","route":"anthropic","trial":2,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.089942,"usage":{"input":12,"cached":94168,"cacheWrite":8976,"output":1309,"modelsBilled":["claude-opus-5-5"],"isError":false,"apiErrorStatus":null,"turns":6},"durationMs":17740,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-crud","arm":"sonnet","model":"claude-sonnet-5-5","route":"anthropic","trial":2,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.040785,"usage":{"input":10,"cached":72900,"cacheWrite":9002,"output":1097,"modelsBilled":["claude-sonnet-5-5"],"isError":false,"apiErrorStatus":null,"turns":9},"durationMs":11187,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-crud","arm":"particle","model":"particle-ai/particle-1","route":"openrouter","trial":2,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.007129,"usage":{"input":14449,"cached":72960,"cacheWrite":0,"output":1328,"modelsBilled":["particle-ai/particle-1"],"isError":false,"apiErrorStatus":null,"turns":11},"durationMs":6059,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-crud","arm":"particle","model":"particle-ai/particle-1","route":"openrouter","trial":1,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.010981,"usage":{"input":29826,"cached":70912,"cacheWrite":0,"output":1397,"modelsBilled":["particle-ai/particle-1"],"isError":false,"apiErrorStatus":null,"turns":10},"durationMs":8204,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-crud","arm":"particle","model":"particle-ai/particle-1","route":"openrouter","trial":3,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.007666,"usage":{"input":18866,"cached":66048,"cacheWrite":0,"output":968,"modelsBilled":["particle-ai/particle-1"],"isError":false,"apiErrorStatus":null,"turns":9},"durationMs":5190,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-crud","arm":"sonnet","model":"claude-sonnet-5-5","route":"anthropic","trial":3,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.041063,"usage":{"input":10,"cached":72903,"cacheWrite":8989,"output":1128,"modelsBilled":["claude-sonnet-5-5"],"isError":false,"apiErrorStatus":null,"turns":9},"durationMs":11427,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-crud","arm":"deepseek","model":"deepseek/deepseek-v4.1-flash-1017","route":"openrouter","trial":3,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.004586,"usage":{"input":16937,"cached":68714,"cacheWrite":0,"output":3065,"modelsBilled":["deepseek/deepseek-v4.1-flash-1017"],"isError":false,"apiErrorStatus":null,"turns":9},"durationMs":37202,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-crud","arm":"deepseek","model":"deepseek/deepseek-v4.1-flash-1017","route":"openrouter","trial":2,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.006053,"usage":{"input":28397,"cached":57429,"cacheWrite":0,"output":2702,"modelsBilled":["deepseek/deepseek-v4.1-flash-1017"],"isError":false,"apiErrorStatus":null,"turns":7},"durationMs":39911,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-crud","arm":"deepseek","model":"deepseek/deepseek-v4.1-flash-1017","route":"openrouter","trial":1,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.006141,"usage":{"input":29075,"cached":57429,"cacheWrite":0,"output":2679,"modelsBilled":["deepseek/deepseek-v4.1-flash-1017"],"isError":false,"apiErrorStatus":null,"turns":9},"durationMs":44552,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-review","arm":"sonnet","model":"claude-sonnet-5-5","route":"anthropic","trial":1,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.046647,"usage":{"input":8,"cached":52331,"cacheWrite":8907,"output":1913,"modelsBilled":["claude-sonnet-5-5"],"isError":false,"apiErrorStatus":null,"turns":6},"durationMs":18235,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-review","arm":"opus","model":"claude-opus-5-5","route":"anthropic","trial":1,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.114017,"usage":{"input":12,"cached":93743,"cacheWrite":9792,"output":2313,"modelsBilled":["claude-opus-5-5"],"isError":false,"apiErrorStatus":null,"turns":6},"durationMs":26137,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-review","arm":"opus","model":"claude-opus-5-5","route":"anthropic","trial":3,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.113589,"usage":{"input":10,"cached":73671,"cacheWrite":9955,"output":2452,"modelsBilled":["claude-opus-5-5"],"isError":false,"apiErrorStatus":null,"turns":6},"durationMs":28242,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-review","arm":"opus","model":"claude-opus-5-5","route":"anthropic","trial":2,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.121409,"usage":{"input":12,"cached":93107,"cacheWrite":9840,"output":2677,"modelsBilled":["claude-opus-5-5"],"isError":false,"apiErrorStatus":null,"turns":6},"durationMs":32666,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-review","arm":"sonnet","model":"claude-sonnet-5-5","route":"anthropic","trial":2,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.044764,"usage":{"input":8,"cached":52329,"cacheWrite":8706,"output":1775,"modelsBilled":["claude-sonnet-5-5"],"isError":false,"apiErrorStatus":null,"turns":6},"durationMs":15025,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-review","arm":"particle","model":"particle-ai/particle-1","route":"openrouter","trial":1,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.006022,"usage":{"input":6647,"cached":64512,"cacheWrite":0,"output":2425,"modelsBilled":["particle-ai/particle-1"],"isError":false,"apiErrorStatus":null,"turns":9},"durationMs":8027,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-review","arm":"sonnet","model":"claude-sonnet-5-5","route":"anthropic","trial":3,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.04538,"usage":{"input":8,"cached":52336,"cacheWrite":8856,"output":1799,"modelsBilled":["claude-sonnet-5-5"],"isError":false,"apiErrorStatus":null,"turns":6},"durationMs":15902,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-review","arm":"particle","model":"particle-ai/particle-1","route":"openrouter","trial":3,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.008123,"usage":{"input":11585,"cached":75776,"cacheWrite":0,"output":2953,"modelsBilled":["particle-ai/particle-1"],"isError":false,"apiErrorStatus":null,"turns":10},"durationMs":9365,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-review","arm":"particle","model":"particle-ai/particle-1","route":"openrouter","trial":2,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.011734,"usage":{"input":19125,"cached":73984,"cacheWrite":0,"output":4733,"modelsBilled":["particle-ai/particle-1"],"isError":false,"apiErrorStatus":null,"turns":11},"durationMs":15364,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-review","arm":"deepseek","model":"deepseek/deepseek-v4.1-flash-1017","route":"openrouter","trial":2,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.016313,"usage":{"input":43989,"cached":182476,"cacheWrite":0,"output":15279,"modelsBilled":["deepseek/deepseek-v4.1-flash-1017"],"isError":false,"apiErrorStatus":null,"turns":18},"durationMs":248319,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-review","arm":"deepseek","model":"deepseek/deepseek-v4.1-flash-1017","route":"openrouter","trial":3,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.01771,"usage":{"input":45767,"cached":95337,"cacheWrite":0,"output":17598,"modelsBilled":["deepseek/deepseek-v4.1-flash-1017"],"isError":false,"apiErrorStatus":null,"turns":13},"durationMs":299550,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-review","arm":"deepseek","model":"deepseek/deepseek-v4.1-flash-1017","route":"openrouter","trial":1,"verdict":"infra","reason":"no result event (timeout)","exitReason":"timeout","costUsd":null,"usage":null,"durationMs":480042,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-changelog","arm":"sonnet","model":"claude-sonnet-5-5","route":"anthropic","trial":1,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.029441,"usage":{"input":8,"cached":52070,"cacheWrite":7547,"output":535,"modelsBilled":["claude-sonnet-5-5"],"isError":false,"apiErrorStatus":null,"turns":6},"durationMs":6727,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-changelog","arm":"opus","model":"claude-opus-5-5","route":"anthropic","trial":3,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.064414,"usage":{"input":10,"cached":71843,"cacheWrite":7773,"output":557,"modelsBilled":["claude-opus-5-5"],"isError":false,"apiErrorStatus":null,"turns":5},"durationMs":10522,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-changelog","arm":"opus","model":"claude-opus-5-5","route":"anthropic","trial":1,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.064872,"usage":{"input":10,"cached":71835,"cacheWrite":7741,"output":588,"modelsBilled":["claude-opus-5-5"],"isError":false,"apiErrorStatus":null,"turns":5},"durationMs":11365,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-changelog","arm":"opus","model":"claude-opus-5-5","route":"anthropic","trial":2,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.063757,"usage":{"input":10,"cached":71811,"cacheWrite":7723,"output":537,"modelsBilled":["claude-opus-5-5"],"isError":false,"apiErrorStatus":null,"turns":5},"durationMs":11612,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-changelog","arm":"particle","model":"particle-ai/particle-1","route":"openrouter","trial":2,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.003367,"usage":{"input":5683,"cached":48128,"cacheWrite":0,"output":502,"modelsBilled":["particle-ai/particle-1"],"isError":false,"apiErrorStatus":null,"turns":6},"durationMs":3152,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-changelog","arm":"sonnet","model":"claude-sonnet-5-5","route":"anthropic","trial":2,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.031247,"usage":{"input":10,"cached":71974,"cacheWrite":7532,"output":520,"modelsBilled":["claude-sonnet-5-5"],"isError":false,"apiErrorStatus":null,"turns":6},"durationMs":7787,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-changelog","arm":"sonnet","model":"claude-sonnet-5-5","route":"anthropic","trial":3,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.031393,"usage":{"input":10,"cached":71981,"cacheWrite":7538,"output":533,"modelsBilled":["claude-sonnet-5-5"],"isError":false,"apiErrorStatus":null,"turns":6},"durationMs":6588,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-changelog","arm":"particle","model":"particle-ai/particle-1","route":"openrouter","trial":1,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.003473,"usage":{"input":5744,"cached":48128,"cacheWrite":0,"output":593,"modelsBilled":["particle-ai/particle-1"],"isError":false,"apiErrorStatus":null,"turns":6},"durationMs":6268,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-changelog","arm":"particle","model":"particle-ai/particle-1","route":"openrouter","trial":3,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.003418,"usage":{"input":5667,"cached":48640,"cacheWrite":0,"output":542,"modelsBilled":["particle-ai/particle-1"],"isError":false,"apiErrorStatus":null,"turns":7},"durationMs":3625,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-changelog","arm":"deepseek","model":"deepseek/deepseek-v4.1-flash-1017","route":"openrouter","trial":1,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.005207,"usage":{"input":28429,"cached":29482,"cacheWrite":0,"output":1423,"modelsBilled":["deepseek/deepseek-v4.1-flash-1017"],"isError":false,"apiErrorStatus":null,"turns":7},"durationMs":16353,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-changelog","arm":"deepseek","model":"deepseek/deepseek-v4.1-flash-1017","route":"openrouter","trial":3,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.007599,"usage":{"input":40383,"cached":13333,"cacheWrite":0,"output":2502,"modelsBilled":["deepseek/deepseek-v4.1-flash-1017"],"isError":false,"apiErrorStatus":null,"turns":6},"durationMs":42312,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-changelog","arm":"deepseek","model":"deepseek/deepseek-v4.1-flash-1017","route":"openrouter","trial":2,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.008075,"usage":{"input":39721,"cached":28716,"cacheWrite":0,"output":3385,"modelsBilled":["deepseek/deepseek-v4.1-flash-1017"],"isError":false,"apiErrorStatus":null,"turns":6},"durationMs":46132,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-investigate","arm":"sonnet","model":"claude-sonnet-5-5","route":"anthropic","trial":1,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.035764,"usage":{"input":12,"cached":92997,"cacheWrite":8084,"output":623,"modelsBilled":["claude-sonnet-5-5"],"isError":false,"apiErrorStatus":null,"turns":6},"durationMs":10716,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-investigate","arm":"opus","model":"claude-opus-5-5","route":"anthropic","trial":3,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.093323,"usage":{"input":12,"cached":93448,"cacheWrite":9445,"output":1368,"modelsBilled":["claude-opus-5-5"],"isError":false,"apiErrorStatus":null,"turns":6},"durationMs":19092,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-investigate","arm":"sonnet","model":"claude-sonnet-5-5","route":"anthropic","trial":2,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.035924,"usage":{"input":12,"cached":93178,"cacheWrite":8097,"output":634,"modelsBilled":["claude-sonnet-5-5"],"isError":false,"apiErrorStatus":null,"turns":6},"durationMs":11289,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-investigate","arm":"opus","model":"claude-opus-5-5","route":"anthropic","trial":2,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.094922,"usage":{"input":12,"cached":93520,"cacheWrite":9230,"output":1501,"modelsBilled":["claude-opus-5-5"],"isError":false,"apiErrorStatus":null,"turns":6},"durationMs":23572,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-investigate","arm":"opus","model":"claude-opus-5-5","route":"anthropic","trial":1,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.107036,"usage":{"input":14,"cached":115577,"cacheWrite":9637,"output":1784,"modelsBilled":["claude-opus-5-5"],"isError":false,"apiErrorStatus":null,"turns":8},"durationMs":25760,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-investigate","arm":"sonnet","model":"claude-sonnet-5-5","route":"anthropic","trial":3,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.035174,"usage":{"input":12,"cached":92984,"cacheWrite":8137,"output":551,"modelsBilled":["claude-sonnet-5-5"],"isError":false,"apiErrorStatus":null,"turns":6},"durationMs":9540,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-investigate","arm":"particle","model":"particle-ai/particle-1","route":"openrouter","trial":1,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.006155,"usage":{"input":7281,"cached":93952,"cacheWrite":0,"output":1516,"modelsBilled":["particle-ai/particle-1"],"isError":false,"apiErrorStatus":null,"turns":8},"durationMs":8577,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-investigate","arm":"particle","model":"particle-ai/particle-1","route":"openrouter","trial":2,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.006352,"usage":{"input":7336,"cached":93952,"cacheWrite":0,"output":1699,"modelsBilled":["particle-ai/particle-1"],"isError":false,"apiErrorStatus":null,"turns":8},"durationMs":8656,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-investigate","arm":"particle","model":"particle-ai/particle-1","route":"openrouter","trial":3,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.01004,"usage":{"input":24767,"cached":76288,"cacheWrite":0,"output":1560,"modelsBilled":["particle-ai/particle-1"],"isError":false,"apiErrorStatus":null,"turns":8},"durationMs":8815,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-investigate","arm":"deepseek","model":"deepseek/deepseek-v4.1-flash-1017","route":"openrouter","trial":3,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.005645,"usage":{"input":19468,"cached":103571,"cacheWrite":0,"output":4023,"modelsBilled":["deepseek/deepseek-v4.1-flash-1017"],"isError":false,"apiErrorStatus":null,"turns":16},"durationMs":65713,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-investigate","arm":"deepseek","model":"deepseek/deepseek-v4.1-flash-1017","route":"openrouter","trial":1,"verdict":"pass","reason":"PASS","exitReason":"success","costUsd":0.006974,"usage":{"input":20929,"cached":122537,"cacheWrite":0,"output":5778,"modelsBilled":["deepseek/deepseek-v4.1-flash-1017"],"isError":false,"apiErrorStatus":null,"turns":10},"durationMs":89257,"consulted":true} +{"runId":"2026-10-30-0105","task":"model-panel-investigate","arm":"deepseek","model":"deepseek/deepseek-v4.1-flash-1017","route":"openrouter","trial":2,"verdict":"fail","reason":"FAIL: the retry decoy must not change","exitReason":"success","costUsd":0.01241,"usage":{"input":38631,"cached":161467,"cacheWrite":0,"output":10219,"modelsBilled":["deepseek/deepseek-v4.1-flash-1017"],"isError":false,"apiErrorStatus":null,"turns":12},"durationMs":149424,"consulted":true} diff --git a/docs/evals/particle-1-2026-10/summary.json b/docs/evals/particle-1-2026-10/summary.json new file mode 100644 index 000000000..b2cce4eff --- /dev/null +++ b/docs/evals/particle-1-2026-10/summary.json @@ -0,0 +1,67 @@ +{ + "rows": 60, + "summaries": [ + { + "arm": "opus", + "label": "Opus 5.5", + "passes": 15, + "completed": 15, + "infra": 0, + "cells": 15, + "passRate": 1, + "low": 0.7961107336956521, + "high": 1, + "costPerTaskUsd": 0.0875512 + }, + { + "arm": "sonnet", + "label": "Sonnet 5.5", + "passes": 15, + "completed": 15, + "infra": 0, + "cells": 15, + "passRate": 1, + "low": 0.7961107336956521, + "high": 1, + "costPerTaskUsd": 0.037294866666666676 + }, + { + "arm": "particle", + "label": "particle-1", + "passes": 15, + "completed": 15, + "infra": 0, + "cells": 15, + "passRate": 1, + "low": 0.7961107336956521, + "high": 1, + "costPerTaskUsd": 0.006973866666666665 + }, + { + "arm": "deepseek", + "label": "DeepSeek V4.1 Flash", + "passes": 13, + "completed": 14, + "infra": 1, + "cells": 15, + "passRate": 0.9285714285714286, + "low": 0.6853073044604219, + "high": 0.9872781138877085, + "costPerTaskUsd": 0.0078097857142857136 + } + ], + "suggestion": { + "line": "particle-1 passed 15/15 tasks on gstack's model panel (Sonnet 5.5 15/15, Opus 5.5 15/15, DeepSeek V4.1 Flash 13/14) at $0.007 per task vs $0.037 for Sonnet 5.5, the cheapest of the four models measured (no measured difference from Sonnet 5.5 on this panel; measured 2026-10-30 in Claude Code via OpenRouter). Worth trying for low-stakes steps; keep blocking reviewers on your main model.", + "reason": "ok" + }, + "spend": { + "cap_usd": 15, + "spent_usd": 2.086636, + "reserved_usd": 0, + "unknown_usd": 0.15, + "unknown_count": 1, + "worst_case_usd": 2.236636, + "headroom_usd": 12.763364, + "open_attempts": [] + } +} diff --git a/docs/skills.md b/docs/skills.md index 8e760a90c..cb4a7eb85 100644 --- a/docs/skills.md +++ b/docs/skills.md @@ -46,7 +46,7 @@ Detailed guides for every gstack skill — philosophy, workflow, and examples. | [`/test-audit`](#test-audit) | **Test Auditor** | Sweep existing tests for low-value, implementation-coupled or duplicate tests. Report-only unless you approve a batch. | | [`/landing-report`](#landing-report) | **Ship Queue Dashboard** | Read-only snapshot of the workspace-aware ship queue. Which version slots are claimed, which sibling workspaces have WIP. | | [`/captain`](#captain) | **Crew Controller** | Run many coding agents from one controller. Lanes build and send a READY block; the controller validates it, merges through trains (stamp on call, verify, merge with the expected head, verify again; `train land` per slot under `changelog_fragments`), keeps the digest, relays approvals in three fixed shapes and runs a daily flaky-fix pass. Also answers to `/crew`. | -| [`/benchmark-models`](#benchmark-models) | **Model Benchmark** | Side-by-side cross-model benchmark for skills (Claude vs GPT vs Gemini). Latency, tokens, cost, optional LLM-judged quality. | +| [`/benchmark-models`](#benchmark-models) | **Model Benchmark** | Side-by-side cross-model benchmark for skills (Claude vs GPT vs Gemini, plus particle-1 and DeepSeek V4.1 Flash through OpenRouter). Latency, tokens, cost, optional LLM-judged quality. | | [`/eval-plan`](#eval-plan) | **Eval Planner** | Preregister a paid eval (bars, arms, decoys, stop rules), dry-run it for $0 with the stub model, pilot three priced items, then approve the spend from the extrapolation table. | | | | | | **Multi-AI** | | | diff --git a/lib/outside-voice-runner.ts b/lib/outside-voice-runner.ts index 2892010b2..f56c5e2c9 100644 --- a/lib/outside-voice-runner.ts +++ b/lib/outside-voice-runner.ts @@ -62,6 +62,7 @@ const FAMILY_RULES: Array<[ModelFamily, RegExp]> = [ ['google', /^(google\/|gemini|gemma)/], ['xai', /^(xai\/|grok)/], ['deepseek', /^(deepseek\/|deepseek)/], + ['deepseek', /^(particle-ai\/|particle\/)?particle-1$/], ['qwen', /^(qwen\/|qwen|alibaba\/)/], ['zai', /^(zai\/|glm)/], ['moonshot', /^(moonshotai\/|moonshot\/|kimi)/], diff --git a/lib/pricing.ts b/lib/pricing.ts index 3fd3a3421..9d867e442 100644 --- a/lib/pricing.ts +++ b/lib/pricing.ts @@ -76,6 +76,12 @@ export const PRICING: Record = { 'gemini-3.1-pro-preview': { input_per_mtok: 2.00, output_per_mtok: 12.00, cached_input_per_mtok: 0.20, as_of: '2026-10' }, 'gemini-2.5-pro': { input_per_mtok: 1.25, output_per_mtok: 5.00, as_of: '2026-04' }, 'gemini-2.5-flash': { input_per_mtok: 0.30, output_per_mtok: 1.20, as_of: '2026-04' }, + + // Particle AI: https://docs.particle.ai/usage-billing (fetched 2026-10-30; peak rates, off-peak and flex are 50% lower; + // OpenRouter serves particle-ai/particle-1 at the same rates) + 'particle-1': { input_per_mtok: 0.25, output_per_mtok: 1.00, cached_input_per_mtok: 0.03, as_of: '2026-10' }, + // https://openrouter.ai/deepseek/deepseek-v4.1-flash-1017 (fetched 2026-10-30; the route the model panel runs it on) + 'deepseek-v4.1-flash-1017': { input_per_mtok: 0.15, output_per_mtok: 0.60, cached_input_per_mtok: 0.003, as_of: '2026-10' }, }; /** diff --git a/package.json b/package.json index f014ece6c..cca1f2823 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.351", + "version": "1.91.352", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/test/benchmark-cli.test.ts b/test/benchmark-cli.test.ts index cc94d38fc..d6e92e0ea 100644 --- a/test/benchmark-cli.test.ts +++ b/test/benchmark-cli.test.ts @@ -61,6 +61,13 @@ describe('gstack-model-benchmark --dry-run', () => { expect(r.stdout).not.toContain('gpt-42-fake'); }); + test('a --models list with no valid provider is an error, never a claude run', () => { + const r = run(['--prompt', 'hi', '--models', 'Particle', '--dry-run']); + expect(r.status).toBe(1); + expect(r.stderr).toContain("No valid provider in --models 'Particle'"); + expect(r.stdout).not.toContain('providers: claude'); + }); + test('empty --models list falls back to claude default', () => { const r = run(['--prompt', 'hi', '--models', '', '--dry-run']); expect(r.status).toBe(0); @@ -156,6 +163,17 @@ describe('gstack-model-benchmark --dry-run', () => { } }); + test('gateway providers without OPENROUTER_API_KEY are NOT READY and never fall back to claude', () => { + const env: Record = { PATH: process.env.PATH ?? '', HOME: process.env.HOME ?? '' }; + const result = spawnSync('bun', ['run', BIN, '--prompt', 'hi', '--models', 'particle,deepseek', '--dry-run'], { cwd: ROOT, env, encoding: 'utf-8', timeout: 15000 }); + const r = { status: result.status, stdout: result.stdout?.toString() ?? '' }; + expect(r.status).toBe(0); + expect(r.stdout).toContain('providers: particle, deepseek'); + expect(r.stdout).toMatch(/particle:\s+NOT READY — OPENROUTER_API_KEY is not set/); + expect(r.stdout).toMatch(/deepseek:\s+NOT READY — OPENROUTER_API_KEY is not set/); + expect(r.stdout).not.toMatch(/^\s+claude:/m); + }); + test('long prompt is truncated in dry-run display', () => { const longPrompt = 'x'.repeat(200); const r = run(['--prompt', longPrompt, '--dry-run']); diff --git a/test/eng-finding-retry-budget.test.ts b/test/eng-finding-retry-budget.test.ts index 687cc2ea0..c390233f7 100644 --- a/test/eng-finding-retry-budget.test.ts +++ b/test/eng-finding-retry-budget.test.ts @@ -141,7 +141,7 @@ test('live periodic census fits the declared CI wall including setup', () => { test('registered allocation is deterministic and preserves every discovered file', () => { const files = collectPaidTestFiles(); - expect(files).toHaveLength(112); // + skill-e2e-write-skill (GSTA-133); + skill-e2e-ship-frozen-scope (frozen review scope, GSTA-113 PR 2); + skill-e2e-failure-routing (#3157/#3158); + skill-e2e-dx-error-contract (#3226); + skill-e2e-gate-integrity (HONEST_WORK_GATE_INTEGRITY W8); + skill-e2e-autoplan-schema-canary, -guard-pty and -long-session (Oct 7 wave, CEO-13, ENG-18, CEO-15); + skill-e2e-ship-measure-seeded-flake (v1.91.32.0); W5a: 15 carve wrappers became one case-sharded file; + codex-e2e-multiblock-live and skill-e2e-design-model-smoke (v1.91.30.0); + expect(files).toHaveLength(113); // + skill-e2e-model-panel (GSTA-173); + skill-e2e-write-skill (GSTA-133); + skill-e2e-ship-frozen-scope (frozen review scope, GSTA-113 PR 2); + skill-e2e-failure-routing (#3157/#3158); + skill-e2e-dx-error-contract (#3226); + skill-e2e-gate-integrity (HONEST_WORK_GATE_INTEGRITY W8); + skill-e2e-autoplan-schema-canary, -guard-pty and -long-session (Oct 7 wave, CEO-13, ENG-18, CEO-15); + skill-e2e-ship-measure-seeded-flake (v1.91.32.0); W5a: 15 carve wrappers became one case-sharded file; + codex-e2e-multiblock-live and skill-e2e-design-model-smoke (v1.91.30.0); expect(files).toContain('test/skill-e2e-ship-skip.test.ts'); const m = livePlan(files); expect(livePlan([...files].reverse())).toEqual(m); diff --git a/test/fixtures/model-panel/changelog/CHANGELOG.md b/test/fixtures/model-panel/changelog/CHANGELOG.md new file mode 100644 index 000000000..a5a99af1b --- /dev/null +++ b/test/fixtures/model-panel/changelog/CHANGELOG.md @@ -0,0 +1,6 @@ +# Changelog + +## 1.3.0 + +- You can now attach photos to a trail note. +- Fixed: the map no longer jumps back to your location while you scroll. diff --git a/test/fixtures/model-panel/changelog/COMMITS.md b/test/fixtures/model-panel/changelog/COMMITS.md new file mode 100644 index 000000000..d8bb81d1e --- /dev/null +++ b/test/fixtures/model-panel/changelog/COMMITS.md @@ -0,0 +1,8 @@ +# Commits since 1.3.0 (newest first) + +- ci: cache node_modules between workflow runs +- fix: typing in a long note no longer signs you out after 5 minutes +- test: refactor shared test helpers +- feat: export your trail notes as a CSV file +- chore: bump eslint to 9.2 +- feat: dark mode toggle in Settings diff --git a/test/fixtures/model-panel/changelog/package.json b/test/fixtures/model-panel/changelog/package.json new file mode 100644 index 000000000..e329befad --- /dev/null +++ b/test/fixtures/model-panel/changelog/package.json @@ -0,0 +1,5 @@ +{ + "name": "trail-notes", + "version": "1.3.0", + "private": true +} diff --git a/test/fixtures/model-panel/checks/bugfix-decoys.js b/test/fixtures/model-panel/checks/bugfix-decoys.js new file mode 100644 index 000000000..1f58f13ac --- /dev/null +++ b/test/fixtures/model-panel/checks/bugfix-decoys.js @@ -0,0 +1,27 @@ +const { spawnSync } = require('node:child_process'); +const fs = require('node:fs'); +const path = require('node:path'); + +const dir = path.resolve(process.argv[2] || '.'); +const failures = []; +const expect = (ok, what) => { if (!ok) failures.push(what); }; +const git = (...args) => spawnSync('git', args, { cwd: dir, encoding: 'utf-8', timeout: 15_000 }); +const seed = () => git('rev-list', '--max-parents=0', 'HEAD').stdout.trim().split('\n').pop(); +const unchanged = (file) => git('diff', '--quiet', seed(), '--', file).status === 0; +const runNode = (file) => spawnSync(process.execPath, [file], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }); +const read = (file) => { try { return fs.readFileSync(path.join(dir, file), 'utf-8'); } catch { return ''; } }; +const load = (file) => { const p = path.join(dir, file); delete require.cache[require.resolve(p)]; return require(p); }; +const finish = () => { + if (failures.length) { console.error('FAIL: ' + failures.join('; ')); process.exit(1); } + console.log('PASS'); +}; + +try { + const { formatPrice } = load('src/format-price.js'); + for (const [cents, want] of [[1005, '$10.05'], [1250, '$12.50'], [999, '$9.99'], [5, '$0.05'], [100000, '$1000.00']]) { + expect(formatPrice(cents) === want, 'formatPrice(' + cents + ') should be ' + want); + } +} catch (err) { expect(false, 'src/format-price.js failed to load: ' + err.message); } +expect(unchanged('run-tests.js'), 'run-tests.js must not change'); +expect(runNode('run-tests.js').status === 0, 'node run-tests.js must pass'); +finish(); diff --git a/test/fixtures/model-panel/checks/changelog.js b/test/fixtures/model-panel/checks/changelog.js new file mode 100644 index 000000000..c740a5e7b --- /dev/null +++ b/test/fixtures/model-panel/checks/changelog.js @@ -0,0 +1,30 @@ +const { spawnSync } = require('node:child_process'); +const fs = require('node:fs'); +const path = require('node:path'); + +const dir = path.resolve(process.argv[2] || '.'); +const failures = []; +const expect = (ok, what) => { if (!ok) failures.push(what); }; +const git = (...args) => spawnSync('git', args, { cwd: dir, encoding: 'utf-8', timeout: 15_000 }); +const seed = () => git('rev-list', '--max-parents=0', 'HEAD').stdout.trim().split('\n').pop(); +const unchanged = (file) => git('diff', '--quiet', seed(), '--', file).status === 0; +const runNode = (file) => spawnSync(process.execPath, [file], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }); +const read = (file) => { try { return fs.readFileSync(path.join(dir, file), 'utf-8'); } catch { return ''; } }; +const load = (file) => { const p = path.join(dir, file); delete require.cache[require.resolve(p)]; return require(p); }; +const finish = () => { + if (failures.length) { console.error('FAIL: ' + failures.join('; ')); process.exit(1); } + console.log('PASS'); +}; + +const original13 = '## 1.3.0\n\n- You can now attach photos to a trail note.\n- Fixed: the map no longer jumps back to your location while you scroll.\n'; +const log = read('CHANGELOG.md'); +const first = log.indexOf('## '); +const second = log.indexOf('## ', first + 3); +const entry = first >= 0 && second > first ? log.slice(first, second) : ''; +expect(/^## \[?1\.4\.0\]?/.test(entry), 'the first section is 1.4.0, above 1.3.0'); +expect(/dark mode/i.test(entry), '1.4.0 mentions dark mode'); +expect(/csv/i.test(entry), '1.4.0 mentions CSV export'); +expect(/sign(s|ed)? (you )?out|log(s|ged)? (you )?out|time(s|d)?[ -]?out/i.test(entry), '1.4.0 mentions the sign-out fix'); +expect(!/eslint|test helpers?|node_modules|\bci\b|workflow/i.test(entry), '1.4.0 leaves out internal-only commits'); +expect(log.includes(original13), 'the 1.3.0 section is unchanged'); +finish(); diff --git a/test/fixtures/model-panel/checks/crud-endpoint.js b/test/fixtures/model-panel/checks/crud-endpoint.js new file mode 100644 index 000000000..9db752109 --- /dev/null +++ b/test/fixtures/model-panel/checks/crud-endpoint.js @@ -0,0 +1,32 @@ +const { spawnSync } = require('node:child_process'); +const fs = require('node:fs'); +const path = require('node:path'); + +const dir = path.resolve(process.argv[2] || '.'); +const failures = []; +const expect = (ok, what) => { if (!ok) failures.push(what); }; +const git = (...args) => spawnSync('git', args, { cwd: dir, encoding: 'utf-8', timeout: 15_000 }); +const seed = () => git('rev-list', '--max-parents=0', 'HEAD').stdout.trim().split('\n').pop(); +const unchanged = (file) => git('diff', '--quiet', seed(), '--', file).status === 0; +const runNode = (file) => spawnSync(process.execPath, [file], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }); +const read = (file) => { try { return fs.readFileSync(path.join(dir, file), 'utf-8'); } catch { return ''; } }; +const load = (file) => { const p = path.join(dir, file); delete require.cache[require.resolve(p)]; return require(p); }; +const finish = () => { + if (failures.length) { console.error('FAIL: ' + failures.join('; ')); process.exit(1); } + console.log('PASS'); +}; + +try { + const { handleRequest, resetForTests } = load('app.js'); + resetForTests(); + const created = handleRequest('POST', '/notes', { text: 'pack water' }); + expect(created.status === 201, 'POST /notes still returns 201'); + expect(handleRequest('DELETE', '/notes/' + created.body.id, null).status === 204, 'DELETE of an existing note returns 204'); + expect(handleRequest('DELETE', '/notes/' + created.body.id, null).status === 404, 'second DELETE returns 404'); + expect(handleRequest('DELETE', '/notes/999', null).status === 404, 'DELETE of an unknown id returns 404'); + const listed = handleRequest('GET', '/notes', null); + expect(Array.isArray(listed.body) && listed.body.length === 0, 'deleted note is gone from GET /notes'); +} catch (err) { expect(false, 'app.js failed: ' + err.message); } +expect(/DELETE/.test(read('run-tests.js')), 'run-tests.js covers DELETE'); +expect(runNode('run-tests.js').status === 0, 'node run-tests.js must pass'); +finish(); diff --git a/test/fixtures/model-panel/checks/investigate.js b/test/fixtures/model-panel/checks/investigate.js new file mode 100644 index 000000000..2a695a209 --- /dev/null +++ b/test/fixtures/model-panel/checks/investigate.js @@ -0,0 +1,27 @@ +const { spawnSync } = require('node:child_process'); +const fs = require('node:fs'); +const path = require('node:path'); + +const dir = path.resolve(process.argv[2] || '.'); +const failures = []; +const expect = (ok, what) => { if (!ok) failures.push(what); }; +const git = (...args) => spawnSync('git', args, { cwd: dir, encoding: 'utf-8', timeout: 15_000 }); +const seed = () => git('rev-list', '--max-parents=0', 'HEAD').stdout.trim().split('\n').pop(); +const unchanged = (file) => git('diff', '--quiet', seed(), '--', file).status === 0; +const runNode = (file) => spawnSync(process.execPath, [file], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }); +const read = (file) => { try { return fs.readFileSync(path.join(dir, file), 'utf-8'); } catch { return ''; } }; +const load = (file) => { const p = path.join(dir, file); delete require.cache[require.resolve(p)]; return require(p); }; +const finish = () => { + if (failures.length) { console.error('FAIL: ' + failures.join('; ')); process.exit(1); } + console.log('PASS'); +}; + +expect(runNode('run-tests.js').status === 0, 'node run-tests.js must pass'); +expect(unchanged('run-tests.js'), 'run-tests.js must not change'); +expect(unchanged('src/retry.js'), 'the retry decoy must not change'); +try { + process.env.TZ = 'Pacific/Kiritimati'; + const { dayOfMonth } = load('src/dates.js'); + expect(dayOfMonth('2026-03-08') === 8 && dayOfMonth('2026-01-01') === 1, 'dayOfMonth is right in a UTC+14 zone too'); +} catch (err) { expect(false, 'src/dates.js failed: ' + err.message); } +finish(); diff --git a/test/fixtures/model-panel/checks/review-seeded-bug.js b/test/fixtures/model-panel/checks/review-seeded-bug.js new file mode 100644 index 000000000..1dd93f55f --- /dev/null +++ b/test/fixtures/model-panel/checks/review-seeded-bug.js @@ -0,0 +1,25 @@ +const { spawnSync } = require('node:child_process'); +const fs = require('node:fs'); +const path = require('node:path'); + +const dir = path.resolve(process.argv[2] || '.'); +const failures = []; +const expect = (ok, what) => { if (!ok) failures.push(what); }; +const git = (...args) => spawnSync('git', args, { cwd: dir, encoding: 'utf-8', timeout: 15_000 }); +const seed = () => git('rev-list', '--max-parents=0', 'HEAD').stdout.trim().split('\n').pop(); +const unchanged = (file) => git('diff', '--quiet', seed(), '--', file).status === 0; +const runNode = (file) => spawnSync(process.execPath, [file], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }); +const read = (file) => { try { return fs.readFileSync(path.join(dir, file), 'utf-8'); } catch { return ''; } }; +const load = (file) => { const p = path.join(dir, file); delete require.cache[require.resolve(p)]; return require(p); }; +const finish = () => { + if (failures.length) { console.error('FAIL: ' + failures.join('; ')); process.exit(1); } + console.log('PASS'); +}; + +const review = read('REVIEW.md'); +expect(review.trim().length > 0, 'REVIEW.md must exist'); +expect(/users\.js/i.test(review), 'review names src/users.js'); +expect(/inject|parameteri[sz]|placeholder|bind|concatenat/i.test(review), 'review flags the SQL injection through search'); +expect(/off[- ]by[- ]one|page\s*-\s*1|\(page\s*-\s*1\)|first page|skips?\s+(the\s+)?first|pages? start at 1/i.test(review), 'review flags the page offset bug'); +expect(unchanged('src'), 'source files must not change during review'); +finish(); diff --git a/test/fixtures/model-panel/investigate/README.md b/test/fixtures/model-panel/investigate/README.md new file mode 100644 index 000000000..9cc1f13c5 --- /dev/null +++ b/test/fixtures/model-panel/investigate/README.md @@ -0,0 +1,4 @@ +# Hike report + +Builds the daily hike summary. Someone noted the report test "seems flaky on +some laptops" and wrapped the date lookup in a retry; it did not help. diff --git a/test/fixtures/model-panel/investigate/package.json b/test/fixtures/model-panel/investigate/package.json new file mode 100644 index 000000000..8a223bebd --- /dev/null +++ b/test/fixtures/model-panel/investigate/package.json @@ -0,0 +1,5 @@ +{ + "name": "hike-report", + "version": "2.0.0", + "private": true +} diff --git a/test/fixtures/model-panel/investigate/run-tests.js b/test/fixtures/model-panel/investigate/run-tests.js new file mode 100644 index 000000000..31a345eb5 --- /dev/null +++ b/test/fixtures/model-panel/investigate/run-tests.js @@ -0,0 +1,10 @@ +process.env.TZ = 'America/Los_Angeles'; +const assert = require('node:assert'); +const { dayOfMonth } = require('./src/dates'); +const { summary } = require('./src/report'); + +assert.strictEqual(dayOfMonth('2026-03-08'), 8); +assert.strictEqual(dayOfMonth('2026-12-31'), 31); +assert.strictEqual(summary({ name: 'Mission Peak', date: '2026-07-04' }), 'Mission Peak on day 4'); + +console.log('all report tests passed'); diff --git a/test/fixtures/model-panel/investigate/src/dates.js b/test/fixtures/model-panel/investigate/src/dates.js new file mode 100644 index 000000000..951dfb7d0 --- /dev/null +++ b/test/fixtures/model-panel/investigate/src/dates.js @@ -0,0 +1,6 @@ +// Day of the month for an ISO calendar date such as '2026-03-08'. +function dayOfMonth(isoDate) { + return new Date(isoDate).getDate(); +} + +module.exports = { dayOfMonth }; diff --git a/test/fixtures/model-panel/investigate/src/report.js b/test/fixtures/model-panel/investigate/src/report.js new file mode 100644 index 000000000..42da39549 --- /dev/null +++ b/test/fixtures/model-panel/investigate/src/report.js @@ -0,0 +1,9 @@ +const { dayOfMonth } = require('./dates'); +const { withRetry } = require('./retry'); + +function summary(hike) { + const day = withRetry(() => dayOfMonth(hike.date), 3, (d) => Number.isInteger(d)); + return `${hike.name} on day ${day}`; +} + +module.exports = { summary }; diff --git a/test/fixtures/model-panel/investigate/src/retry.js b/test/fixtures/model-panel/investigate/src/retry.js new file mode 100644 index 000000000..9eb1ee11d --- /dev/null +++ b/test/fixtures/model-panel/investigate/src/retry.js @@ -0,0 +1,11 @@ +// Retries fn up to times attempts; returns the first result that passes check. +function withRetry(fn, times, check = () => true) { + let last; + for (let i = 0; i < times; i++) { + last = fn(); + if (check(last)) return last; + } + return last; +} + +module.exports = { withRetry }; diff --git a/test/fixtures/model-panel/reference/bugfix-decoys/src/format-price.js b/test/fixtures/model-panel/reference/bugfix-decoys/src/format-price.js new file mode 100644 index 000000000..2253557b9 --- /dev/null +++ b/test/fixtures/model-panel/reference/bugfix-decoys/src/format-price.js @@ -0,0 +1,7 @@ +function formatPrice(cents) { + const dollars = Math.floor(cents / 100); + const rem = String(cents % 100).padStart(2, '0'); + return '$' + dollars + '.' + rem; +} + +module.exports = { formatPrice }; diff --git a/test/fixtures/model-panel/reference/changelog/CHANGELOG.md b/test/fixtures/model-panel/reference/changelog/CHANGELOG.md new file mode 100644 index 000000000..231164b8f --- /dev/null +++ b/test/fixtures/model-panel/reference/changelog/CHANGELOG.md @@ -0,0 +1,12 @@ +# Changelog + +## 1.4.0 + +- Dark mode: turn it on in Settings. +- Export your trail notes as a CSV file. +- Fixed: typing in a long note no longer signs you out after 5 minutes. + +## 1.3.0 + +- You can now attach photos to a trail note. +- Fixed: the map no longer jumps back to your location while you scroll. diff --git a/test/fixtures/model-panel/reference/crud-endpoint/app.js b/test/fixtures/model-panel/reference/crud-endpoint/app.js new file mode 100644 index 000000000..fb9d266e4 --- /dev/null +++ b/test/fixtures/model-panel/reference/crud-endpoint/app.js @@ -0,0 +1,28 @@ +let nextId = 1; +const notes = new Map(); + +function handleRequest(method, path, body) { + if (method === 'GET' && path === '/notes') { + return { status: 200, body: [...notes.values()] }; + } + if (method === 'POST' && path === '/notes') { + if (!body || typeof body.text !== 'string' || !body.text.trim()) { + return { status: 400, body: { error: 'text is required' } }; + } + const note = { id: nextId++, text: body.text.trim() }; + notes.set(note.id, note); + return { status: 201, body: note }; + } + const match = method === 'DELETE' && /^\/notes\/(\d+)$/.exec(path); + if (match) { + return notes.delete(Number(match[1])) ? { status: 204, body: null } : { status: 404, body: { error: 'not found' } }; + } + return { status: 404, body: { error: 'not found' } }; +} + +function resetForTests() { + nextId = 1; + notes.clear(); +} + +module.exports = { handleRequest, resetForTests }; diff --git a/test/fixtures/model-panel/reference/crud-endpoint/run-tests.js b/test/fixtures/model-panel/reference/crud-endpoint/run-tests.js new file mode 100644 index 000000000..bb144a670 --- /dev/null +++ b/test/fixtures/model-panel/reference/crud-endpoint/run-tests.js @@ -0,0 +1,9 @@ +const assert = require('node:assert'); +const { handleRequest, resetForTests } = require('./app'); + +resetForTests(); +const created = handleRequest('POST', '/notes', { text: 'buy trail mix' }); +assert.strictEqual(created.status, 201); +assert.strictEqual(handleRequest('DELETE', '/notes/1', null).status, 204); +assert.strictEqual(handleRequest('DELETE', '/notes/1', null).status, 404); +console.log('all notes tests passed'); diff --git a/test/fixtures/model-panel/reference/investigate/src/dates.js b/test/fixtures/model-panel/reference/investigate/src/dates.js new file mode 100644 index 000000000..80f4dec97 --- /dev/null +++ b/test/fixtures/model-panel/reference/investigate/src/dates.js @@ -0,0 +1,5 @@ +function dayOfMonth(isoDate) { + return Number(isoDate.slice(8, 10)); +} + +module.exports = { dayOfMonth }; diff --git a/test/fixtures/model-panel/reference/review-seeded-bug/REVIEW.md b/test/fixtures/model-panel/reference/review-seeded-bug/REVIEW.md new file mode 100644 index 000000000..f2e44ec0c --- /dev/null +++ b/test/fixtures/model-panel/reference/review-seeded-bug/REVIEW.md @@ -0,0 +1,4 @@ +# Review: search and paginate users + +1. src/users.js:8 — SQL injection: search is concatenated into the query. Pass it as a bound parameter ($3) instead. +2. src/users.js:7 — off-by-one: pages start at 1, so the offset should be (page - 1) * pageSize; page 1 skips the first page of users. diff --git a/test/fixtures/model-panel/review-seeded-bug/PR.md b/test/fixtures/model-panel/review-seeded-bug/PR.md new file mode 100644 index 000000000..6cf9af763 --- /dev/null +++ b/test/fixtures/model-panel/review-seeded-bug/PR.md @@ -0,0 +1,4 @@ +# PR: search and paginate users + +Adds `listUsers(db, { search, page, pageSize, sort })` in `src/users.js` for the +admin screen. `search` comes straight from the admin search box. Pages start at 1. diff --git a/test/fixtures/model-panel/review-seeded-bug/README.md b/test/fixtures/model-panel/review-seeded-bug/README.md new file mode 100644 index 000000000..4bfb1ff9b --- /dev/null +++ b/test/fixtures/model-panel/review-seeded-bug/README.md @@ -0,0 +1,4 @@ +# User directory + +A small service that lists users for the admin screen. `src/db.js` is a thin +wrapper over a SQL client: `db.query(sql, params)` runs a parameterized query. diff --git a/test/fixtures/model-panel/review-seeded-bug/package.json b/test/fixtures/model-panel/review-seeded-bug/package.json new file mode 100644 index 000000000..002905e32 --- /dev/null +++ b/test/fixtures/model-panel/review-seeded-bug/package.json @@ -0,0 +1,5 @@ +{ + "name": "user-directory", + "version": "1.0.0", + "private": true +} diff --git a/test/fixtures/model-panel/review-seeded-bug/src/db.js b/test/fixtures/model-panel/review-seeded-bug/src/db.js new file mode 100644 index 000000000..e48fe2479 --- /dev/null +++ b/test/fixtures/model-panel/review-seeded-bug/src/db.js @@ -0,0 +1,10 @@ +// Thin wrapper over the SQL client. query(sql, params) binds params to $1, $2, ... +function createDb(client) { + return { + query(sql, params = []) { + return client.query(sql, params); + }, + }; +} + +module.exports = { createDb }; diff --git a/test/fixtures/model-panel/review-seeded-bug/src/users.js b/test/fixtures/model-panel/review-seeded-bug/src/users.js new file mode 100644 index 000000000..38b17abc5 --- /dev/null +++ b/test/fixtures/model-panel/review-seeded-bug/src/users.js @@ -0,0 +1,14 @@ +// New in this PR: search and paginate users for the admin screen. +// Pages are numbered from 1 (the admin UI sends page=1 for the first page). +const SORTABLE = new Set(['name', 'created_at']); + +async function listUsers(db, { search = '', page = 1, pageSize = 20, sort = 'name' } = {}) { + const orderBy = SORTABLE.has(sort) ? sort : 'name'; + const offset = page * pageSize; + const sql = "SELECT id, name, email FROM users WHERE name ILIKE '%" + search + "%'" + + ` ORDER BY ${orderBy} LIMIT $1 OFFSET $2`; + const rows = await db.query(sql, [pageSize, offset]); + return rows.map((row) => ({ id: row.id, name: row.name, email: row.email })); +} + +module.exports = { listUsers }; diff --git a/test/gateway-provider.test.ts b/test/gateway-provider.test.ts new file mode 100644 index 000000000..cd5c6f9af --- /dev/null +++ b/test/gateway-provider.test.ts @@ -0,0 +1,118 @@ +import { describe, test, expect, afterEach } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { GatewayAdapter, GATEWAY_MODELS, gatewayEnvironment, foreignModels } from './helpers/providers/gateway'; +import { PROVIDER_NAMES, isProviderName } from './helpers/providers/registry'; +import { PANEL_ARMS, cellCostUsd } from './helpers/model-panel-harness'; +import { familyConflict, modelFamily } from '../lib/outside-voice-runner'; +import { estimateCostUsd, hasPricing } from '../lib/pricing'; + +const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'gateway-provider-')); +const savedBin = process.env.GSTACK_CLAUDE_BIN; +const savedArgs = process.env.GSTACK_CLAUDE_BIN_ARGS; +afterEach(() => { + if (savedBin === undefined) delete process.env.GSTACK_CLAUDE_BIN; + else process.env.GSTACK_CLAUDE_BIN = savedBin; + if (savedArgs === undefined) delete process.env.GSTACK_CLAUDE_BIN_ARGS; + else process.env.GSTACK_CLAUDE_BIN_ARGS = savedArgs; +}); + +/** A stand-in claude CLI run by bun (so it works on Windows): records its env and prints a result envelope billing FAKE_MODELS. */ +function fakeClaude(): string { + const script = path.join(tmp, 'fake-claude.js'); + fs.writeFileSync(script, [ + "const fs = require('fs');", + "if (process.env.FAKE_ARGS_OUT) fs.writeFileSync(process.env.FAKE_ARGS_OUT, JSON.stringify(process.argv.slice(2)));", + "fs.writeFileSync(process.env.FAKE_ENV_OUT, Object.entries(process.env).map(([k, v]) => k + '=' + v).join('\\n') + '\\n');", + "if (process.env.FAKE_ERROR) { process.stdout.write(JSON.stringify({ type: 'result', subtype: 'error_during_execution', is_error: true, result: '' })); process.exit(0); }", + "const modelUsage = JSON.parse('{' + (process.env.FAKE_MODELS || '') + '}');", + "process.stdout.write(JSON.stringify({ type: 'result', subtype: 'success', is_error: false, result: 'ok', num_turns: 2, usage: { input_tokens: 1000, output_tokens: 10, cache_read_input_tokens: 50, cache_creation_input_tokens: 7 }, modelUsage }));", + ].join('\n') + '\n'); + process.env.GSTACK_CLAUDE_BIN_ARGS = JSON.stringify([script]); + return process.execPath; +} + + + +describe('gateway provider', () => { + test('the child env drops every Anthropic credential and pins all aliases to the arm model', () => { + const env = gatewayEnvironment('particle-ai/particle-1', { PATH: '/bin', OPENROUTER_API_KEY: 'sk-or', ANTHROPIC_API_KEY: 'sk-ant', GSTACK_ANTHROPIC_API_KEY: 'sk-ant-2', CLAUDE_CODE_OAUTH_TOKEN: 'oauth' }); + expect(Object.values(env).filter((v) => /sk-ant|oauth/.test(v))).toEqual([]); + expect(env.ANTHROPIC_API_KEY).toBe(''); + expect(env.ANTHROPIC_AUTH_TOKEN).toBe('sk-or'); + expect(env.ANTHROPIC_BASE_URL).toBe('https://openrouter.ai/api'); + expect(env.CLAUDE_CODE_SUBAGENT_MODEL).toBe('particle-ai/particle-1'); + expect(env.ANTHROPIC_DEFAULT_HAIKU_MODEL).toBe('particle-ai/particle-1'); + expect(env.PATH).toBe('/bin'); + expect(gatewayEnvironment('m', { OPENROUTER_API_KEY: 'k', GSTACK_GATEWAY_BASE_URL: 'https://ai-gateway.vercel.sh' }).ANTHROPIC_BASE_URL).toBe('https://ai-gateway.vercel.sh'); + }); + + test('a run through the adapter reaches claude without the Anthropic key and reports usage at our prices', async () => { + process.env.GSTACK_CLAUDE_BIN = fakeClaude(); + const envOut = path.join(tmp, 'env-ok.txt'); + const adapter = new GatewayAdapter('particle', { PATH: process.env.PATH, HOME: tmp, OPENROUTER_API_KEY: 'sk-or', ANTHROPIC_API_KEY: 'sk-ant-real', FAKE_ENV_OUT: envOut, FAKE_MODELS: '"particle-ai/particle-1":{}' }); + const result = await adapter.run({ prompt: 'hi', workdir: tmp, timeoutMs: 10_000 }); + expect(result.error).toBeUndefined(); + expect(result.modelUsed).toBe('particle-ai/particle-1'); + expect(result.tokens).toEqual({ input: 1000, output: 10, cached: 50, cacheWrite: 7 }); + const childEnv = fs.readFileSync(envOut, 'utf-8'); + expect(childEnv).not.toContain('sk-ant-real'); + expect(childEnv).toContain('ANTHROPIC_AUTH_TOKEN=sk-or'); + expect(adapter.estimateCost(result.tokens, result.modelUsed)).toBeCloseTo((1000 * 0.25 + 10 * 1.0 + 50 * 0.03 + 7 * 0.25) / 1e6, 5); + const particleArm = PANEL_ARMS.find(a => a.id === 'particle')!; + expect(adapter.estimateCost(result.tokens, result.modelUsed)).toBe(cellCostUsd(particleArm, { input: 1000, output: 10, cached: 50, cacheWrite: 7, modelsBilled: [], isError: false, apiErrorStatus: null, turns: 2 })); + }); + + test('a run that billed another model is a MODEL_MIX error, not a result', async () => { + process.env.GSTACK_CLAUDE_BIN = fakeClaude(); + const adapter = new GatewayAdapter('deepseek', { PATH: process.env.PATH, HOME: tmp, OPENROUTER_API_KEY: 'sk-or', FAKE_ENV_OUT: path.join(tmp, 'env-mix.txt'), FAKE_MODELS: '"deepseek/deepseek-v4.1-flash-1017":{},"claude-haiku-5-5":{}' }); + const result = await adapter.run({ prompt: 'hi', workdir: tmp, timeoutMs: 10_000 }); + expect(result.error?.reason).toBe('MODEL_MIX: the run also billed claude-haiku-5-5'); + expect(foreignModels('particle-ai/particle-1', ['particle-1', 'PARTICLE-AI/particle-1'])).toEqual([]); + }); + + test('gateway runs load no Claude Code settings, so a settings.json key never reaches the gateway', async () => { + process.env.GSTACK_CLAUDE_BIN = fakeClaude(); + const argsOut = path.join(tmp, 'args.json'); + await new GatewayAdapter('particle', { PATH: process.env.PATH, HOME: tmp, OPENROUTER_API_KEY: 'sk-or', FAKE_ENV_OUT: path.join(tmp, 'env-args.txt'), FAKE_ARGS_OUT: argsOut, FAKE_MODELS: '"particle-ai/particle-1":{}' }).run({ prompt: 'hi', workdir: tmp, timeoutMs: 10_000 }); + const argv: string[] = JSON.parse(fs.readFileSync(argsOut, 'utf-8')); + const at = argv.indexOf('--setting-sources'); + expect(at).toBeGreaterThan(-1); + expect(argv[at + 1]).toBe(''); + }); + + test('an error envelope from the gateway is an error, not a benchmark result', async () => { + process.env.GSTACK_CLAUDE_BIN = fakeClaude(); + const adapter = new GatewayAdapter('particle', { PATH: process.env.PATH, HOME: tmp, OPENROUTER_API_KEY: 'sk-or', FAKE_ENV_OUT: path.join(tmp, 'env-err.txt'), FAKE_ERROR: '1' }); + const result = await adapter.run({ prompt: 'hi', workdir: tmp, timeoutMs: 10_000 }); + expect(result.error?.reason).toMatch(/empty or invalid output/); + }); + + test('available() names the missing key with a fix', async () => { + process.env.GSTACK_CLAUDE_BIN = fakeClaude(); + const check = await new GatewayAdapter('particle', { PATH: process.env.PATH }).available(); + expect(check.ok).toBe(false); + expect(check.reason).toContain('OPENROUTER_API_KEY is not set'); + expect((await new GatewayAdapter('particle', { PATH: process.env.PATH, OPENROUTER_API_KEY: 'k' }).available()).ok).toBe(true); + }); + + test('the registry lists the gateway providers once, by name', () => { + expect(PROVIDER_NAMES).toEqual(['claude', 'gpt', 'gemini', 'particle', 'deepseek']); + expect(isProviderName('particle')).toBe(true); + expect(isProviderName('toString')).toBe(false); + expect(GATEWAY_MODELS).toEqual({ particle: 'particle-ai/particle-1', deepseek: 'deepseek/deepseek-v4.1-flash-1017' }); + }); + + test('particle-1 is DeepSeek family, so it is never an independent voice beside DeepSeek', () => { + for (const id of ['particle-ai/particle-1', 'particle/particle-1', 'particle-1', 'Particle-AI/Particle-1']) expect(modelFamily(id)).toBe('deepseek'); + expect(modelFamily('particle-ai/particle-2')).toBe('unknown'); + expect(familyConflict('particle-ai/particle-1', 'deepseek/deepseek-v4.1-flash-1017')).toBeDefined(); + expect(familyConflict('particle-ai/particle-1', 'claude-opus-5-5')).toBeUndefined(); + }); + + test('every panel and gateway id is priced', () => { + for (const id of ['particle-ai/particle-1', 'particle/particle-1', 'particle-1', 'deepseek/deepseek-v4.1-flash-1017', 'claude-opus-5-5', 'claude-sonnet-5-5']) expect(hasPricing(id)).toBe(true); + expect(estimateCostUsd({ input: 1_000_000, output: 1_000_000, cached: 1_000_000 }, 'deepseek/deepseek-v4.1-flash-1017')).toBeCloseTo(0.15 + 0.6 + 0.003, 6); + }); +}); diff --git a/test/helpers/benchmark-runner.ts b/test/helpers/benchmark-runner.ts index 324cce432..ee0886732 100644 --- a/test/helpers/benchmark-runner.ts +++ b/test/helpers/benchmark-runner.ts @@ -7,19 +7,17 @@ * one. Per-provider auth/timeout/rate-limit errors don't abort the batch. */ -import type { ProviderAdapter, RunOpts, RunResult, TokenUsage } from './providers/types'; -import { ClaudeAdapter } from './providers/claude'; -import { GptAdapter } from './providers/gpt'; -import { GeminiAdapter } from './providers/gemini'; +import type { RunOpts, RunResult, TokenUsage } from './providers/types'; +import { PROVIDERS, type ProviderName } from './providers/registry'; export interface BenchmarkInput { prompt: string; workdir: string; timeoutMs?: number; - /** Adapter names to run (e.g., ['claude', 'gpt', 'gemini']). */ - providers: Array<'claude' | 'gpt' | 'gemini'>; + /** Adapter names to run (e.g., ['claude', 'gpt', 'gemini', 'particle']). */ + providers: ProviderName[]; /** Optional per-provider model overrides. */ - models?: Partial>; + models?: Partial>; /** If true, skip providers whose available() returns !ok. If false, include them with error. */ skipUnavailable?: boolean; } @@ -44,12 +42,6 @@ export interface BenchmarkReport { entries: BenchmarkEntry[]; } -const ADAPTERS: Record<'claude' | 'gpt' | 'gemini', () => ProviderAdapter> = { - claude: () => new ClaudeAdapter(), - gpt: () => new GptAdapter(), - gemini: () => new GeminiAdapter(), -}; - export async function runBenchmark(input: BenchmarkInput): Promise { const startedAtMs = Date.now(); const startedAt = new Date(startedAtMs).toISOString(); @@ -59,7 +51,7 @@ export async function runBenchmark(input: BenchmarkInput): Promise> = []; for (const name of input.providers) { - const factory = ADAPTERS[name]; + const factory = PROVIDERS[name]; if (!factory) { entries.push({ provider: name, family: 'claude', available: false, unavailable_reason: `unknown provider: ${name}` }); continue; diff --git a/test/helpers/model-panel-harness.ts b/test/helpers/model-panel-harness.ts new file mode 100644 index 000000000..4516d65ae --- /dev/null +++ b/test/helpers/model-panel-harness.ts @@ -0,0 +1,277 @@ +/** + * Model panel harness: one fixed panel of gstack-shaped engineering tasks run + * by several models in the same Claude Code harness, each cell scored by a + * hidden deterministic check. Shared by the paid periodic run + * (test/skill-e2e-model-panel.test.ts) and the free selftest + * (test/model-panel-selftest.test.ts), which proves every check fails on its + * seed fixture and passes on a reference answer before any money is spent. + * + * Preregistered (GSTA-173, 2026-10-30) before the first paid run: the arms, + * tasks, checks, PANEL_TRIALS, the INFRA classes in classifyCell, the spend + * reservations and the wording rule in suggestionLine. None of them change + * after a result is seen. + */ +import { spawnSync } from 'child_process'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { ROOT, copyDirSync } from './e2e-helpers'; +import { buildBehavioralSkill, run, SKILL_NAME, FIXTURES as ARM_FIXTURES } from './arm-benchmark-harness'; +import { GATEWAY_MODELS, foreignModels, gatewayOverrides } from './providers/gateway'; +import { estimateCostUsd, pricingRow } from './pricing'; + +export const PANEL_FIXTURES = path.join(ROOT, 'test', 'fixtures', 'model-panel'); +export const PANEL_TRIALS = 3; +export const PANEL_MAX_TURNS = 40; +export const PANEL_TIMEOUT_MS = 8 * 60_000; +export const PANEL_CONCURRENCY = 4; +export const PANEL_ALLOWED_TOOLS = ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep', 'Skill']; +/** Completed (non-INFRA) cells an arm needs before the panel publishes anything about it. */ +export const PANEL_MIN_COMPLETED = 12; +/** Pass rate below which the panel publishes numbers but no "worth trying" suggestion. */ +export const PANEL_USEFULNESS_FLOOR = 0.6; +export const PANEL_SPEND_CAP_USD = 15; + +export type PanelArmId = 'opus' | 'sonnet' | 'particle' | 'deepseek'; +export interface PanelArm { + id: PanelArmId; + label: string; + model: string; + route: 'anthropic' | 'openrouter'; + /** Worst-case USD reserved for one cell before it starts; settled to the measured cost after. */ + reserveUsd: number; +} + +export const PANEL_ARMS: PanelArm[] = [ + { id: 'opus', label: 'Opus 5.5', model: 'claude-opus-5-5', route: 'anthropic', reserveUsd: 2.0 }, + { id: 'sonnet', label: 'Sonnet 5.5', model: 'claude-sonnet-5-5', route: 'anthropic', reserveUsd: 1.0 }, + { id: 'particle', label: 'particle-1', model: GATEWAY_MODELS.particle, route: 'openrouter', reserveUsd: 0.15 }, + { id: 'deepseek', label: 'DeepSeek V4.1 Flash', model: GATEWAY_MODELS.deepseek, route: 'openrouter', reserveUsd: 0.15 }, +]; + +export interface PanelTask { + /** E2E_TOUCHFILES / E2E_TIERS key. */ + id: string; + fixture: string; + check: string; + reference: string; + ticket: string; +} + +const fixture = (name: string) => path.join(PANEL_FIXTURES, name); +const check = (name: string) => path.join(PANEL_FIXTURES, 'checks', name + '.js'); +const reference = (name: string) => path.join(PANEL_FIXTURES, 'reference', name); +const LEAVE = '\n\nLeave your changes uncommitted in the working tree.'; + +export const PANEL_TASKS: PanelTask[] = [ + { + id: 'model-panel-bugfix', + fixture: path.join(ARM_FIXTURES, 'bugfix-decoys'), + check: check('bugfix-decoys'), + reference: reference('bugfix-decoys'), + ticket: 'Bug report: receipts print $10.5 for a $10.05 item.\n\nYou are in a small git repo. `node run-tests.js` currently fails on formatPrice(1005).\nFix the bug so all tests pass. Verify with: node run-tests.js' + LEAVE, + }, + { + id: 'model-panel-crud', + fixture: path.join(ARM_FIXTURES, 'crud-endpoint'), + check: check('crud-endpoint'), + reference: reference('crud-endpoint'), + ticket: 'Ticket: users need to delete notes.\n\nYou are in a small git repo containing an in-memory notes API (app.js, wired to HTTP in server.js).\nAdd DELETE /notes/:id: respond 204 on success and 404 for an unknown id, and cover the new endpoint in run-tests.js. Verify with: node run-tests.js' + LEAVE, + }, + { + id: 'model-panel-review', + fixture: fixture('review-seeded-bug'), + check: check('review-seeded-bug'), + reference: reference('review-seeded-bug'), + ticket: 'Pre-landing review: PR.md describes a teammate\'s change, which added src/users.js.\n\nReview src/users.js the way a careful reviewer would before it ships: find the bugs that pass a quick read but break in production. Do not edit any source file. Write REVIEW.md with one numbered finding per bug, each naming file:line, what goes wrong, and the fix.' + LEAVE, + }, + { + id: 'model-panel-changelog', + fixture: fixture('changelog'), + check: check('changelog'), + reference: reference('changelog'), + ticket: 'Release 1.4.0: write its CHANGELOG.md entry from COMMITS.md.\n\nRules: the new section goes above 1.3.0 and matches its format; list only changes a user of the app would notice, in everyday words; leave out build, test and tooling changes; do not edit older sections.' + LEAVE, + }, + { + id: 'model-panel-investigate', + fixture: fixture('investigate'), + check: check('investigate'), + reference: reference('investigate'), + ticket: 'Bug: `node run-tests.js` fails, and the report shows the wrong day for some hikes.\n\nFind the root cause before you change anything, then fix it at the root. Do not edit the tests. Verify with: node run-tests.js' + LEAVE, + }, +]; + +export interface CellDirs { dir: string; seedSha: string } + +/** Fresh copy of the task fixture with the build-discipline skill installed (every arm alike) and one seed commit. */ +export function setupCell(task: PanelTask, label: string): CellDirs { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'panel-' + label + '-')); + copyDirSync(task.fixture, dir); + const skillDir = path.join(dir, '.claude', 'skills', SKILL_NAME); + fs.mkdirSync(skillDir, { recursive: true }); + fs.writeFileSync(path.join(skillDir, 'SKILL.md'), buildBehavioralSkill()); + fs.writeFileSync(path.join(dir, 'CLAUDE.md'), '# Project\n\nSmall fixture repo for one ticket.\n\n## Skill routing\n\nBefore working on any ticket, invoke the ' + SKILL_NAME + ' skill via the Skill tool and follow it while you work.\n'); + if (!fs.existsSync(path.join(dir, '.gitignore'))) fs.writeFileSync(path.join(dir, '.gitignore'), 'node_modules/\n'); + run('git', ['init', '-b', 'main'], dir); + run('git', ['config', 'user.email', 'model-panel@example.com'], dir); + run('git', ['config', 'user.name', 'Model Panel'], dir); + run('git', ['config', 'commit.gpgsign', 'false'], dir); + run('git', ['add', '-A'], dir); + run('git', ['commit', '-m', 'seed fixture'], dir); + return { dir, seedSha: run('git', ['rev-parse', 'HEAD'], dir).trim() }; +} + +/** Overlay the task's reference answer (selftest only). */ +export function applyReference(task: PanelTask, dir: string): void { + copyDirSync(task.reference, dir); +} + +/** Run the hidden check against a cell directory. Never throws. */ +export function runCheck(task: PanelTask, dir: string): { pass: boolean; detail: string } { + const r = spawnSync(process.execPath, [task.check, dir], { stdio: 'pipe', encoding: 'utf-8', timeout: 60_000 }); + const detail = ((r.stdout ?? '') + (r.stderr ?? '')).trim().split('\n').pop() ?? ''; + return { pass: r.status === 0, detail: detail.slice(0, 400) }; +} + +/** Extra child env for an arm: nothing for Anthropic arms, the gateway route (no Anthropic credential) otherwise. */ +export function armEnv(arm: PanelArm, source: NodeJS.ProcessEnv = process.env): Record { + return arm.route === 'openrouter' ? gatewayOverrides(arm.model, source) : {}; +} + +export interface CellUsage { + input: number; + cached: number; + cacheWrite: number; + output: number; + modelsBilled: string[]; + isError: boolean; + apiErrorStatus: number | null; + turns: number; +} + +/** Usage from the CLI's final result event; null when the transcript has none. */ +export function usageFromTranscript(transcript: any[]): CellUsage | null { + const result = [...transcript].reverse().find((e) => e && e.type === 'result'); + if (!result) return null; + const u = result.usage ?? {}; + return { + input: u.input_tokens ?? 0, + cached: u.cache_read_input_tokens ?? 0, + cacheWrite: u.cache_creation_input_tokens ?? 0, + output: u.output_tokens ?? 0, + modelsBilled: result.modelUsage && typeof result.modelUsage === 'object' ? Object.keys(result.modelUsage) : [], + isError: result.is_error === true, + apiErrorStatus: typeof result.api_error_status === 'number' ? result.api_error_status : null, + turns: result.num_turns ?? 0, + }; +} + +/** + * Measured cost from our price table, not the CLI's total_cost_usd (which + * prices gateway models at Anthropic rates). Cache writes are billed at 1.25x + * input for Anthropic models (5-minute writes) and at the input rate for + * gateway models, whose providers publish no separate write price. + */ +export function cellCostUsd(arm: PanelArm, usage: CellUsage): number { + const row = pricingRow(arm.model); + if (!row) throw new Error('no price row for ' + arm.model); + const writeRate = row.input_per_mtok * (arm.route === 'anthropic' ? 1.25 : 1); + return +(estimateCostUsd({ input: usage.input, cached: usage.cached, output: usage.output }, arm.model) + usage.cacheWrite * writeRate / 1_000_000).toFixed(6); +} + +export type CellVerdict = 'pass' | 'fail' | 'infra'; + +/** + * INFRA cells are excluded from pass rates, reported, and never rerun to + * change a verdict: no result event, a CLI or API error before the model + * produced a turn, a 402/429/5xx from the API, a startup timeout, or a run + * that billed any model besides the arm's (MODEL_MIX). A CLI killed at the + * timeout writes no result event, so it is INFRA too. Everything else is + * scored by the hidden check. + */ +export function classifyCell(arm: PanelArm, exitReason: string, usage: CellUsage | null): { verdict: 'infra'; reason: string } | { verdict: 'scored' } { + if (exitReason === 'timeout_startup') return { verdict: 'infra', reason: 'startup timeout' }; + if (!usage) return { verdict: 'infra', reason: 'no result event (' + exitReason + ')' }; + const foreign = foreignModels(arm.model, usage.modelsBilled); + if (foreign.length) return { verdict: 'infra', reason: 'MODEL_MIX: ' + foreign.join(', ') }; + if (usage.apiErrorStatus !== null && (usage.apiErrorStatus === 402 || usage.apiErrorStatus === 429 || usage.apiErrorStatus >= 500)) { + return { verdict: 'infra', reason: 'API ' + usage.apiErrorStatus }; + } + if (usage.isError && usage.turns === 0) return { verdict: 'infra', reason: 'error before the first turn (' + exitReason + ')' }; + return { verdict: 'scored' }; +} + +export interface PanelRow { + task: string; + arm: PanelArmId; + model: string; + route: PanelArm['route']; + trial: number; + verdict: CellVerdict; + reason: string; + exitReason: string; + costUsd: number | null; + usage: CellUsage | null; + durationMs: number; + consulted: boolean; +} + +export interface ArmSummary { + arm: PanelArmId; + label: string; + passes: number; + completed: number; + infra: number; + cells: number; + passRate: number; + low: number; + high: number; + costPerTaskUsd: number; +} + +/** Wilson 95% interval for a pass rate. */ +export function wilson(passes: number, n: number): { low: number; high: number } { + if (n === 0) return { low: 0, high: 1 }; + const z = 1.96, p = passes / n, d = 1 + z * z / n; + const c = p + z * z / (2 * n), m = z * Math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)); + return { low: Math.max(0, (c - m) / d), high: Math.min(1, (c + m) / d) }; +} + +export function summarize(rows: PanelRow[]): ArmSummary[] { + return PANEL_ARMS.map((arm) => { + const mine = rows.filter((r) => r.arm === arm.id); + const scored = mine.filter((r) => r.verdict !== 'infra'); + const passes = scored.filter((r) => r.verdict === 'pass').length; + const priced = mine.filter((r) => r.costUsd !== null); + const { low, high } = wilson(passes, scored.length); + return { + arm: arm.id, label: arm.label, passes, completed: scored.length, infra: mine.length - scored.length, cells: mine.length, + passRate: scored.length ? passes / scored.length : 0, low, high, + costPerTaskUsd: priced.length ? priced.reduce((s, r) => s + (r.costUsd ?? 0), 0) / priced.length : 0, + }; + }); +} + +const money = (usd: number) => '$' + (usd < 0.1 ? usd.toFixed(3) : usd.toFixed(2)); + +/** + * The preregistered wording rule. Returns the one line gstack may publish + * about particle-1, or null with the reason when the panel supports none. + * It quotes raw pass counts and measured cost per task; it never says + * "comparable" or "better", says "no measured difference" only when the + * Wilson intervals overlap, and calls particle-1 cheapest only among the + * arms measured. + */ +export function suggestionLine(summaries: ArmSummary[], measuredOn: string): { line: string | null; reason: string } { + const by = (id: PanelArmId) => summaries.find((s) => s.arm === id)!; + const p = by('particle'), s = by('sonnet'), o = by('opus'), d = by('deepseek'); + const thin = summaries.filter((x) => x.completed < PANEL_MIN_COMPLETED); + if (thin.length) return { line: null, reason: 'INCOMPLETE: ' + thin.map((x) => x.label + ' ' + x.completed + '/' + x.cells).join(', ') + ' completed cells (need ' + PANEL_MIN_COMPLETED + ')' }; + const vsSonnet = p.high < s.low ? 'below Sonnet 5.5 on this panel' : p.low > s.high ? 'above Sonnet 5.5 on this panel' : 'no measured difference from Sonnet 5.5 on this panel'; + const cheapest = summaries.every((x) => x.arm === 'particle' || p.costPerTaskUsd < x.costPerTaskUsd); + const counts = 'particle-1 passed ' + p.passes + '/' + p.completed + ' tasks on gstack\'s model panel (Sonnet 5.5 ' + s.passes + '/' + s.completed + ', Opus 5.5 ' + o.passes + '/' + o.completed + ', DeepSeek V4.1 Flash ' + d.passes + '/' + d.completed + ')'; + const cost = ' at ' + money(p.costPerTaskUsd) + ' per task vs ' + money(s.costPerTaskUsd) + ' for Sonnet 5.5' + (cheapest ? ', the cheapest of the four models measured' : ''); + const where = ' (' + vsSonnet + '; measured ' + measuredOn + ' in Claude Code via OpenRouter).'; + if (p.passRate < PANEL_USEFULNESS_FLOOR) return { line: null, reason: 'below the usefulness floor: ' + counts + cost + where }; + return { line: counts + cost + where + ' Worth trying for low-stakes steps; keep blocking reviewers on your main model.', reason: 'ok' }; +} diff --git a/test/helpers/periodic-exclude-data.ts b/test/helpers/periodic-exclude-data.ts index 14cb3fdb3..98b637b84 100644 --- a/test/helpers/periodic-exclude-data.ts +++ b/test/helpers/periodic-exclude-data.ts @@ -19,6 +19,10 @@ export const PERIODIC_CI_EXCLUDE: Record Record; + /** Treat an explicit error or empty `result` envelope as a failed run, as paid producers do. */ + strict?: boolean; +} + export class ClaudeAdapter implements ProviderAdapter { readonly name = 'claude'; readonly family = 'claude' as const; + constructor(private readonly options: ClaudeAdapterOptions = {}) {} + async available(opts?: RunOpts): Promise { // Binary on PATH (or GSTACK_CLAUDE_BIN override). Routes through the shared // resolver so Windows + override paths behave the same as production sites. @@ -79,7 +88,7 @@ export class ClaudeAdapter implements ProviderAdapter { const model = opts.model ?? process.env.EVALS_MODEL ?? resolveEvalModel('capture'); const stateDirectory = csoProducerStateDirectory(opts); const args = claudeExecArgs(opts, model, resolved.argsPrefix); - const env = claudeExecEnvironment(opts, process.env); + const env = !opts.csoProducer && this.options.environment ? this.options.environment(opts) : claudeExecEnvironment(opts, process.env); if (stateDirectory) { assertClaudeProducerPrerequisites(); assertClaudeProducerSandboxPolicy(args, env, csoProducerHelperLauncher(opts)!); @@ -100,7 +109,7 @@ export class ClaudeAdapter implements ProviderAdapter { // AskUserQuestion failure BLOCKs rather than emitting unanswerable prose). env, }); - return resultFromClaudeOutput(out,{model,durationMs:Date.now()-start,producer:!!opts.csoProducer}); + return resultFromClaudeOutput(out,{model,durationMs:Date.now()-start,producer:!!opts.csoProducer||!!this.options.strict}); } catch (err: unknown) { const durationMs = Date.now() - start; const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string }; @@ -143,7 +152,8 @@ export function resultFromClaudeOutput(raw:string,opts:{model?:string;durationMs const obj=JSON.parse(raw),explicitError=obj?.is_error===true||(typeof obj?.subtype==='string'&&obj.subtype!=='success'),validResult=!explicitError&&typeof obj?.result==='string'&&obj.result.trim().length>0; if(opts.producer&&!validResult)return{output:'',tokens:{input:0,output:0},durationMs,toolCalls:0,modelUsed:typeof obj?.model==='string'&&obj.model?obj.model:defaultModel,error:{code:'unknown',reason:'empty or invalid output from claude CLI (exit 0)'}}; const output=typeof obj?.result==='string'?obj.result:String(obj?.result??''),usage=obj?.usage??{}; - return{output,tokens:{input:usage.input_tokens??0,output:usage.output_tokens??0,cached:usage.cache_read_input_tokens},durationMs,toolCalls:obj?.num_turns??0,modelUsed:typeof obj?.model==='string'&&obj.model?obj.model:defaultModel}; + const modelsBilled=obj?.modelUsage&&typeof obj.modelUsage==='object'?Object.keys(obj.modelUsage):undefined; + return{output,tokens:{input:usage.input_tokens??0,output:usage.output_tokens??0,cached:usage.cache_read_input_tokens,cacheWrite:usage.cache_creation_input_tokens},durationMs,toolCalls:obj?.num_turns??0,modelUsed:typeof obj?.model==='string'&&obj.model?obj.model:defaultModel,...(modelsBilled?{modelsBilled}:{})}; }catch{ if(opts.producer)return{output:'',tokens:{input:0,output:0},durationMs,toolCalls:0,modelUsed:defaultModel,error:{code:'unknown',reason:'empty or invalid output from claude CLI (exit 0)'}}; return{output:raw,tokens:{input:0,output:0},durationMs,toolCalls:0,modelUsed:defaultModel}; diff --git a/test/helpers/providers/gateway.ts b/test/helpers/providers/gateway.ts new file mode 100644 index 000000000..47e4a9146 --- /dev/null +++ b/test/helpers/providers/gateway.ts @@ -0,0 +1,102 @@ +import { ClaudeAdapter, claudeExecEnvironment } from './claude'; +import type { AvailabilityCheck, ProviderAdapter, RunOpts, RunResult, TokenUsage } from './types'; +import { estimateCostUsd, pricingRow } from '../pricing'; +import { resolveClaudeCommand } from '../../../lib/claude-bin'; + +/** + * Gateway adapter: the Claude Code harness (`claude -p`) pointed at an + * Anthropic-compatible gateway, so a non-Anthropic model runs in the same + * agent loop as the Claude arms. OpenRouter by default + * (GSTACK_GATEWAY_BASE_URL overrides the endpoint; the key is OPENROUTER_API_KEY). + * + * The child never sees an Anthropic credential: ANTHROPIC_API_KEY, + * CLAUDE_CODE_OAUTH_TOKEN and GSTACK_ANTHROPIC_API_KEY are dropped and the + * gateway key travels as ANTHROPIC_AUTH_TOKEN. Every Claude Code model alias + * and the subagent model are pinned to the arm's model, and a run that billed + * any other model is reported as a MODEL_MIX error instead of a result. + */ +export const GATEWAY_MODELS = { + particle: 'particle-ai/particle-1', + deepseek: 'deepseek/deepseek-v4.1-flash-1017', +} as const; +export type GatewayName = keyof typeof GATEWAY_MODELS; + +export const GATEWAY_DEFAULT_BASE_URL = 'https://openrouter.ai/api'; +export const GATEWAY_KEY_ENV = 'OPENROUTER_API_KEY'; +const ANTHROPIC_CREDENTIALS = ['ANTHROPIC_API_KEY', 'CLAUDE_CODE_OAUTH_TOKEN', 'GSTACK_ANTHROPIC_API_KEY'] as const; + +/** The variables a gateway run sets on top of any child env; blank credentials override inherited ones. */ +export function gatewayOverrides(model: string, source: NodeJS.ProcessEnv = process.env): Record { + const key = source[GATEWAY_KEY_ENV]; + if (!key) throw new Error(`${GATEWAY_KEY_ENV} is not set`); + return { + ANTHROPIC_BASE_URL: source.GSTACK_GATEWAY_BASE_URL || GATEWAY_DEFAULT_BASE_URL, + ANTHROPIC_AUTH_TOKEN: key, + ANTHROPIC_API_KEY: '', + CLAUDE_CODE_OAUTH_TOKEN: '', + ANTHROPIC_MODEL: model, + ANTHROPIC_SMALL_FAST_MODEL: model, + ANTHROPIC_DEFAULT_HAIKU_MODEL: model, + ANTHROPIC_DEFAULT_SONNET_MODEL: model, + ANTHROPIC_DEFAULT_OPUS_MODEL: model, + CLAUDE_CODE_SUBAGENT_MODEL: model, + }; +} + +/** A complete child env for a gateway run: the source env without Anthropic credentials, plus gatewayOverrides. */ +export function gatewayEnvironment(model: string, source: NodeJS.ProcessEnv = process.env): Record { + const env: Record = {}; + for (const [k, v] of Object.entries(source)) { + if (v !== undefined && !(ANTHROPIC_CREDENTIALS as readonly string[]).includes(k)) env[k] = v; + } + return { ...env, ...gatewayOverrides(model, source) }; +} + +/** Models a run billed besides the one it asked for (aliases compared without a provider prefix). */ +export function foreignModels(requested: string, billed: string[] | undefined): string[] { + const bare = (id: string) => id.toLowerCase().replace(/^[a-z-]+\//, ''); + return (billed ?? []).filter(id => bare(id) !== bare(requested)); +} + +export class GatewayAdapter implements ProviderAdapter { + readonly family = 'claude' as const; + readonly model: string; + private readonly harness: ClaudeAdapter; + + constructor(readonly name: GatewayName, private readonly source: NodeJS.ProcessEnv = process.env) { + this.model = GATEWAY_MODELS[name]; + this.harness = new ClaudeAdapter({ + environment: opts => gatewayEnvironment(this.model, claudeExecEnvironment(opts, this.source)), + strict: true, + }); + } + + async available(): Promise { + if (!this.source[GATEWAY_KEY_ENV]) { + return { ok: false, reason: `${GATEWAY_KEY_ENV} is not set. Create a key at https://openrouter.ai/keys and export it.` }; + } + if (!resolveClaudeCommand()) { + return { ok: false, reason: 'claude CLI not found on PATH (the gateway arms run in Claude Code). Install: npm i -g @anthropic-ai/claude-code' }; + } + return { ok: true }; + } + + async run(opts: RunOpts): Promise { + if (opts.csoProducer) { + return { output: '', tokens: { input: 0, output: 0 }, durationMs: 0, toolCalls: 0, modelUsed: this.model, error: { code: 'unknown', reason: 'gateway providers do not run CSO producers' } }; + } + const result = await this.harness.run({ ...opts, model: this.model, extraArgs: [...(opts.extraArgs ?? []), '--setting-sources', ''] }); + const foreign = foreignModels(this.model, result.modelsBilled); + if (!result.error && foreign.length) { + return { ...result, error: { code: 'unknown', reason: `MODEL_MIX: the run also billed ${foreign.join(', ')}` } }; + } + return result; + } + + /** Cache writes are billed at the input rate: the gateway's providers publish no separate write price (as the model panel prices them). */ + estimateCost(tokens: TokenUsage, model?: string): number { + const id = model ?? this.model; + const writeRate = pricingRow(id)?.input_per_mtok ?? 0; + return +(estimateCostUsd(tokens, id) + (tokens.cacheWrite ?? 0) * writeRate / 1_000_000).toFixed(6); + } +} diff --git a/test/helpers/providers/registry.ts b/test/helpers/providers/registry.ts new file mode 100644 index 000000000..0d3571bc3 --- /dev/null +++ b/test/helpers/providers/registry.ts @@ -0,0 +1,21 @@ +import type { ProviderAdapter } from './types'; +import { ClaudeAdapter } from './claude'; +import { GptAdapter } from './gpt'; +import { GeminiAdapter } from './gemini'; +import { GatewayAdapter } from './gateway'; + +/** The one provider registry the benchmark CLI, its dry run and the runner share. */ +export const PROVIDERS = { + claude: () => new ClaudeAdapter(), + gpt: () => new GptAdapter(), + gemini: () => new GeminiAdapter(), + particle: () => new GatewayAdapter('particle'), + deepseek: () => new GatewayAdapter('deepseek'), +} satisfies Record ProviderAdapter>; + +export type ProviderName = keyof typeof PROVIDERS; +export const PROVIDER_NAMES = Object.keys(PROVIDERS) as ProviderName[]; + +export function isProviderName(name: string): name is ProviderName { + return Object.hasOwn(PROVIDERS, name); +} diff --git a/test/helpers/providers/types.ts b/test/helpers/providers/types.ts index e9b51527e..edd89d5c8 100644 --- a/test/helpers/providers/types.ts +++ b/test/helpers/providers/types.ts @@ -40,6 +40,8 @@ export interface TokenUsage { output: number; /** Cached input tokens (Anthropic/OpenAI support). Undefined if provider doesn't report. */ cached?: number; + /** Prompt-cache write tokens (Claude CLI `cache_creation_input_tokens`). Not priced by estimateCostUsd. */ + cacheWrite?: number; } export type RunError = @@ -70,6 +72,8 @@ export interface RunResult { toolTimeline?: ToolTimelineEntry[]; /** Actual model ID the provider reports using (may be a variant of the family). */ modelUsed: string; + /** Every model the CLI billed during the run (Claude CLI `modelUsage` keys). Unset if unreported. */ + modelsBilled?: string[]; /** If the run failed, error code + human reason. output/tokens may be partial. */ error?: { code: RunError; reason: string }; } diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index 9413e8351..3079ace6a 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -1121,6 +1121,79 @@ export const E2E_TOUCHFILES: Record = { 'test/skill-e2e-arm-benchmark.test.ts', 'ship/SKILL.md', 'test/helpers/e2e-gate.ts'], + // Model panel (GSTA-173): particle-1, DeepSeek V4.1 Flash, Sonnet 5.5 and Opus 5.5 on one + // fixed task panel in Claude Code, hidden deterministic checks (research instrument). + 'model-panel-bugfix': [ + 'test/fixtures/model-panel/**', + 'test/fixtures/arm-benchmark/bugfix-decoys/**', + 'test/fixtures/arm-benchmark/crud-endpoint/**', + 'test/helpers/model-panel-harness.ts', + 'test/helpers/arm-benchmark-harness.ts', + 'test/helpers/providers/gateway.ts', + 'test/helpers/providers/claude.ts', + 'test/helpers/providers/claude-stream.ts', + 'test/helpers/providers/types.ts', + 'test/helpers/pricing.ts', + 'lib/pricing.ts', + 'test/skill-e2e-model-panel.test.ts', + 'ship/SKILL.md', 'test/helpers/e2e-gate.ts'], + 'model-panel-crud': [ + 'test/fixtures/model-panel/**', + 'test/fixtures/arm-benchmark/bugfix-decoys/**', + 'test/fixtures/arm-benchmark/crud-endpoint/**', + 'test/helpers/model-panel-harness.ts', + 'test/helpers/arm-benchmark-harness.ts', + 'test/helpers/providers/gateway.ts', + 'test/helpers/providers/claude.ts', + 'test/helpers/providers/claude-stream.ts', + 'test/helpers/providers/types.ts', + 'test/helpers/pricing.ts', + 'lib/pricing.ts', + 'test/skill-e2e-model-panel.test.ts', + 'ship/SKILL.md', 'test/helpers/e2e-gate.ts'], + 'model-panel-review': [ + 'test/fixtures/model-panel/**', + 'test/fixtures/arm-benchmark/bugfix-decoys/**', + 'test/fixtures/arm-benchmark/crud-endpoint/**', + 'test/helpers/model-panel-harness.ts', + 'test/helpers/arm-benchmark-harness.ts', + 'test/helpers/providers/gateway.ts', + 'test/helpers/providers/claude.ts', + 'test/helpers/providers/claude-stream.ts', + 'test/helpers/providers/types.ts', + 'test/helpers/pricing.ts', + 'lib/pricing.ts', + 'test/skill-e2e-model-panel.test.ts', + 'ship/SKILL.md', 'test/helpers/e2e-gate.ts'], + 'model-panel-changelog': [ + 'test/fixtures/model-panel/**', + 'test/fixtures/arm-benchmark/bugfix-decoys/**', + 'test/fixtures/arm-benchmark/crud-endpoint/**', + 'test/helpers/model-panel-harness.ts', + 'test/helpers/arm-benchmark-harness.ts', + 'test/helpers/providers/gateway.ts', + 'test/helpers/providers/claude.ts', + 'test/helpers/providers/claude-stream.ts', + 'test/helpers/providers/types.ts', + 'test/helpers/pricing.ts', + 'lib/pricing.ts', + 'test/skill-e2e-model-panel.test.ts', + 'ship/SKILL.md', 'test/helpers/e2e-gate.ts'], + 'model-panel-investigate': [ + 'test/fixtures/model-panel/**', + 'test/fixtures/arm-benchmark/bugfix-decoys/**', + 'test/fixtures/arm-benchmark/crud-endpoint/**', + 'test/helpers/model-panel-harness.ts', + 'test/helpers/arm-benchmark-harness.ts', + 'test/helpers/providers/gateway.ts', + 'test/helpers/providers/claude.ts', + 'test/helpers/providers/claude-stream.ts', + 'test/helpers/providers/types.ts', + 'test/helpers/pricing.ts', + 'lib/pricing.ts', + 'test/skill-e2e-model-panel.test.ts', + 'ship/SKILL.md', 'test/helpers/e2e-gate.ts'], + 'office-hours-section-loading': [ 'ETHOS.md', 'office-hours/**', 'bin/gstack-office-hours-review', 'lib/office-hours-review.ts', 'lib/fs-atomic.ts', 'lib/spec-review-cap.ts', 'test/helpers/office-hours-formatter-fixture.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/carve-guards.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/llm-judge.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-office-hours-section-loading.test.ts', 'test/helpers/agent-sdk-runner.ts', 'test/helpers/auq-native-capture.ts', 'test/helpers/auto-decision-state.ts', 'test/helpers/autoplan-artifact-digest.ts', 'test/helpers/autoplan-artifact-permission.ts', 'test/helpers/autoplan-artifact-recorder.ts', 'test/helpers/capture-parity-baseline.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty/**', 'test/helpers/dx-selected-navigation.ts', 'test/helpers/e2e-gate.ts', 'test/helpers/eng-cache-writer-decision.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/helpers/native-auto-decide.ts', 'test/helpers/owned-claude-transcript.ts', 'test/helpers/parity-harness.ts', 'test/helpers/size-ledger.ts', 'test/size-ledger/**', 'test/helpers/plan-count-artifacts.ts', 'test/helpers/plan-count-file-permission.ts', 'test/helpers/plan-count-fixture.ts', 'test/helpers/plan-count-pending-exit.ts', 'test/helpers/plan-count-pending-question.ts', 'test/helpers/plan-count-transcript.ts', 'test/helpers/plan-floor-review.ts', 'test/helpers/plan-floor-target.ts', 'test/helpers/plan-scope-selection.ts', 'test/helpers/plan-seed-submission.ts', 'test/helpers/plan-skill-question-events.ts', 'test/helpers/plan-skill-question-hook-scope.ts', 'test/helpers/plan-skill-questions.ts', 'test/helpers/pty-screen.ts', 'test/helpers/pty-trust-dialog.ts', 'test/helpers/skill-census.ts'], 'office-hours-design-draft': [ 'ETHOS.md', 'office-hours/**', 'lib/office-hours-review.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/carve-guards.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/llm-judge.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-office-hours-design-draft.test.ts', 'test/helpers/agent-sdk-runner.ts', 'test/helpers/auq-native-capture.ts', 'test/helpers/auto-decision-state.ts', 'test/helpers/autoplan-artifact-digest.ts', 'test/helpers/autoplan-artifact-permission.ts', 'test/helpers/autoplan-artifact-recorder.ts', 'test/helpers/capture-parity-baseline.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty/**', 'test/helpers/dx-selected-navigation.ts', 'test/helpers/e2e-gate.ts', 'test/helpers/eng-cache-writer-decision.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/helpers/native-auto-decide.ts', 'test/helpers/owned-claude-transcript.ts', 'test/helpers/parity-harness.ts', 'test/helpers/size-ledger.ts', 'test/size-ledger/**', 'test/helpers/plan-count-artifacts.ts', 'test/helpers/plan-count-file-permission.ts', 'test/helpers/plan-count-fixture.ts', 'test/helpers/plan-count-pending-exit.ts', 'test/helpers/plan-count-pending-question.ts', 'test/helpers/plan-count-transcript.ts', 'test/helpers/plan-floor-review.ts', 'test/helpers/plan-floor-target.ts', 'test/helpers/plan-scope-selection.ts', 'test/helpers/plan-seed-submission.ts', 'test/helpers/plan-skill-question-events.ts', 'test/helpers/plan-skill-question-hook-scope.ts', 'test/helpers/plan-skill-questions.ts', 'test/helpers/pty-screen.ts', 'test/helpers/pty-trust-dialog.ts', 'test/helpers/skill-census.ts'], 'dx-error-contract-review': ['test/skill-e2e-dx-error-contract.test.ts', 'test/fixtures/dx-error-contract-case.ts', 'lib/dx-error-contract.ts', 'scripts/resolvers/dx.ts', 'scripts/resolvers/index.ts', 'scripts/gen-skill-docs.ts', 'plan-devex-review/**', 'bin/gstack-autoplan-snapshot.ts', 'lib/autoplan-prompts.ts', 'lib/autoplan-reconcile.ts', 'lib/coordinator-contract.ts', 'test/helpers/session-runner.ts', 'test/helpers/eval-store.ts', 'test/helpers/eval-budgets.ts', 'lib/claude-bin.ts', 'lib/eval-model.ts'], @@ -1544,6 +1617,11 @@ export const E2E_TIERS: Record = { 'arm-benchmark-native-overbuild': 'periodic', 'arm-benchmark-crud-endpoint': 'periodic', 'arm-benchmark-bugfix-decoys': 'periodic', + 'model-panel-bugfix': 'periodic', + 'model-panel-crud': 'periodic', + 'model-panel-review': 'periodic', + 'model-panel-changelog': 'periodic', + 'model-panel-investigate': 'periodic', 'office-hours-section-loading': 'marathon', // Full startup design/review/approval workflow (1–3 real review rounds, ~20 min) 'office-hours-design-draft': 'periodic', // Same interview through the design-creating Write (~5 min) 'plan-decision-classification': 'periodic', @@ -1919,6 +1997,11 @@ export const E2E_KINDS: Record = { 'arm-benchmark-native-overbuild': 'rule', 'arm-benchmark-crud-endpoint': 'rule', 'arm-benchmark-bugfix-decoys': 'rule', + 'model-panel-bugfix': 'rule', + 'model-panel-crud': 'rule', + 'model-panel-review': 'rule', + 'model-panel-changelog': 'rule', + 'model-panel-investigate': 'rule', 'office-hours-section-loading': 'rule', 'office-hours-design-draft': 'rule', 'plan-decision-classification': 'rule', diff --git a/test/model-panel-selftest.test.ts b/test/model-panel-selftest.test.ts new file mode 100644 index 000000000..f67d626f8 --- /dev/null +++ b/test/model-panel-selftest.test.ts @@ -0,0 +1,115 @@ +/** + * Model panel selftest (free, no model, no spend): every hidden check fails on + * its seed fixture and passes on its reference answer, gateway arms carry no + * Anthropic credential, and the INFRA classes, cost math and preregistered + * wording rule behave as written, so the paid panel + * (test/skill-e2e-model-panel.test.ts) never spends on a broken instrument. + */ +import { describe, test, expect } from 'bun:test'; +import * as fs from 'fs'; +import { + PANEL_ARMS, PANEL_TASKS, PANEL_TRIALS, PANEL_MIN_COMPLETED, + setupCell, applyReference, runCheck, armEnv, cellCostUsd, classifyCell, summarize, suggestionLine, wilson, usageFromTranscript, + type CellUsage, type PanelRow, type PanelArmId, +} from './helpers/model-panel-harness'; + +const arm = (id: PanelArmId) => PANEL_ARMS.find((a) => a.id === id)!; +const usage = (over: Partial = {}): CellUsage => ({ input: 1000, cached: 0, cacheWrite: 0, output: 100, modelsBilled: [], isError: false, apiErrorStatus: null, turns: 3, ...over }); + +describe('model panel selftest (free, no API)', () => { + for (const task of PANEL_TASKS) { + test(task.id + ': the hidden check fails on the seed and passes on the reference', () => { + expect(fs.existsSync(task.check)).toBe(true); + const seed = setupCell(task, 'selftest-seed'); + const ref = setupCell(task, 'selftest-ref'); + try { + expect(runCheck(task, seed.dir).pass).toBe(false); + applyReference(task, ref.dir); + const passed = runCheck(task, ref.dir); + expect(passed.detail).toBe('PASS'); + expect(passed.pass).toBe(true); + } finally { + fs.rmSync(seed.dir, { recursive: true, force: true }); + fs.rmSync(ref.dir, { recursive: true, force: true }); + } + }); + } + + test('the panel is the preregistered shape: 5 tasks, 4 arms, 3 trials', () => { + expect(PANEL_TASKS.map((t) => t.id)).toEqual(['model-panel-bugfix', 'model-panel-crud', 'model-panel-review', 'model-panel-changelog', 'model-panel-investigate']); + expect(PANEL_ARMS.map((a) => a.model)).toEqual(['claude-opus-5-5', 'claude-sonnet-5-5', 'particle-ai/particle-1', 'deepseek/deepseek-v4.1-flash-1017']); + expect(PANEL_TRIALS).toBe(3); + }); + + test('gateway arms carry no Anthropic credential and pin every alias; Anthropic arms add nothing', () => { + const source = { OPENROUTER_API_KEY: 'sk-or-test', ANTHROPIC_API_KEY: 'sk-ant-real', CLAUDE_CODE_OAUTH_TOKEN: 'oauth-real' }; + const env = armEnv(arm('particle'), source); + expect(env.ANTHROPIC_API_KEY).toBe(''); + expect(env.CLAUDE_CODE_OAUTH_TOKEN).toBe(''); + expect(Object.values(env)).not.toContain('sk-ant-real'); + expect(env.ANTHROPIC_AUTH_TOKEN).toBe('sk-or-test'); + expect(env.ANTHROPIC_BASE_URL).toBe('https://openrouter.ai/api'); + for (const k of ['ANTHROPIC_DEFAULT_HAIKU_MODEL', 'ANTHROPIC_DEFAULT_SONNET_MODEL', 'ANTHROPIC_DEFAULT_OPUS_MODEL', 'CLAUDE_CODE_SUBAGENT_MODEL']) { + expect(env[k]).toBe('particle-ai/particle-1'); + } + expect(armEnv(arm('sonnet'), source)).toEqual({}); + expect(() => armEnv(arm('deepseek'), {})).toThrow('OPENROUTER_API_KEY'); + }); + + test('INFRA classes: startup timeout, missing result, MODEL_MIX, 402/429/5xx and pre-turn errors; the rest is scored', () => { + const p = arm('particle'); + expect(classifyCell(p, 'timeout_startup', null)).toEqual({ verdict: 'infra', reason: 'startup timeout' }); + expect(classifyCell(p, 'error', null).verdict).toBe('infra'); + expect(classifyCell(p, 'success', usage({ modelsBilled: ['particle-ai/particle-1', 'claude-haiku-5-5'] }))).toEqual({ verdict: 'infra', reason: 'MODEL_MIX: claude-haiku-5-5' }); + expect(classifyCell(p, 'success', usage({ modelsBilled: ['particle-1'] }))).toEqual({ verdict: 'scored' }); + for (const status of [402, 429, 503]) expect(classifyCell(p, 'error', usage({ isError: true, apiErrorStatus: status })).verdict).toBe('infra'); + expect(classifyCell(p, 'error', usage({ isError: true, turns: 0 })).verdict).toBe('infra'); + expect(classifyCell(p, 'timeout', usage({ turns: 12 }))).toEqual({ verdict: 'scored' }); + expect(classifyCell(p, 'error_max_turns', usage({ isError: true, turns: 40 }))).toEqual({ verdict: 'scored' }); + }); + + test('cost comes from the price table, with cache writes, never the CLI total', () => { + expect(cellCostUsd(arm('particle'), usage({ input: 1_000_000, cached: 1_000_000, output: 1_000_000 }))).toBeCloseTo(0.25 + 0.03 + 1.0, 6); + expect(cellCostUsd(arm('opus'), usage({ input: 0, output: 0, cacheWrite: 1_000_000 }))).toBeCloseTo(4.0 * 1.25, 6); + expect(cellCostUsd(arm('deepseek'), usage({ input: 1_000_000, output: 0 }))).toBeCloseTo(0.15, 6); + const u = usageFromTranscript([{ type: 'assistant' }, { type: 'result', total_cost_usd: 9.99, num_turns: 4, usage: { input_tokens: 10, output_tokens: 2, cache_read_input_tokens: 5, cache_creation_input_tokens: 1 }, modelUsage: { 'particle-ai/particle-1': {} } }]); + expect(u).toEqual({ input: 10, cached: 5, cacheWrite: 1, output: 2, modelsBilled: ['particle-ai/particle-1'], isError: false, apiErrorStatus: null, turns: 4 }); + expect(usageFromTranscript([{ type: 'assistant' }])).toBeNull(); + }); + + test('wilson intervals', () => { + expect(wilson(0, 0)).toEqual({ low: 0, high: 1 }); + const w = wilson(12, 15); + expect(w.low).toBeCloseTo(0.548, 2); + expect(w.high).toBeCloseTo(0.930, 2); + }); + + const rows = (counts: Record): PanelRow[] => + (Object.entries(counts) as Array<[PanelArmId, [number, number, number]]>).flatMap(([id, [passes, fails, infra]]) => { + const a = arm(id); + const cost = { opus: 0.6, sonnet: 0.3, particle: 0.02, deepseek: 0.03 }[id]; + const make = (verdict: PanelRow['verdict'], i: number): PanelRow => ({ task: 't', arm: id, model: a.model, route: a.route, trial: i, verdict, reason: '', exitReason: 'success', costUsd: verdict === 'infra' ? null : cost, usage: null, durationMs: 1, consulted: true }); + return [...Array(passes).fill('pass'), ...Array(fails).fill('fail'), ...Array(infra).fill('infra')].map((v, i) => make(v, i)); + }); + + test('wording rule: overlap says no measured difference and quotes counts and cost; never "comparable" or "better"', () => { + const s = suggestionLine(summarize(rows({ opus: [14, 1, 0], sonnet: [13, 2, 0], particle: [12, 3, 0], deepseek: [11, 4, 0] })), '2026-10-30'); + expect(s.reason).toBe('ok'); + expect(s.line).toContain('particle-1 passed 12/15'); + expect(s.line).toContain('Sonnet 5.5 13/15'); + expect(s.line).toContain('no measured difference from Sonnet 5.5 on this panel'); + expect(s.line).toContain('the cheapest of the four models measured'); + expect(s.line).not.toMatch(/comparable|better/i); + }); + + test('wording rule: a clear gap is named, the usefulness floor withholds the suggestion, thin arms are INCOMPLETE', () => { + const floor = suggestionLine(summarize(rows({ opus: [15, 0, 0], sonnet: [15, 0, 0], particle: [6, 9, 0], deepseek: [5, 10, 0] })), '2026-10-30'); + expect(floor.line).toBeNull(); + expect(floor.reason).toContain('below the usefulness floor'); + expect(floor.reason).toContain('below Sonnet 5.5 on this panel'); + const thin = suggestionLine(summarize(rows({ opus: [15, 0, 0], sonnet: [15, 0, 0], particle: [8, 2, PANEL_TRIALS * 5 - 10], deepseek: [15, 0, 0] })), '2026-10-30'); + expect(thin.line).toBeNull(); + expect(thin.reason).toStartWith('INCOMPLETE'); + expect(PANEL_MIN_COMPLETED).toBe(12); + }); +}); diff --git a/test/skill-e2e-model-panel.test.ts b/test/skill-e2e-model-panel.test.ts new file mode 100644 index 000000000..ff7e76cdc --- /dev/null +++ b/test/skill-e2e-model-panel.test.ts @@ -0,0 +1,132 @@ +/** + * Model panel (periodic, paid): particle-1, its open-weight base DeepSeek V4.1 + * Flash, Sonnet 5.5 and Opus 5.5 each run the same five gstack-shaped tasks + * PANEL_TRIALS times in Claude Code. Each cell is scored by a hidden + * deterministic check; gateway arms reach their models through OpenRouter + * with no Anthropic credential in the child env. A research instrument that + * gates nothing: assertions cover the harness, never which model wins. + * + * Spend: every cell reserves its arm's worst case in a shared ledger + * (/spend.json, cap PANEL_SPEND_CAP_USD) before it starts and settles to + * the measured cost after, so a staged run (one task first, then the rest) + * stays under one cap; a refused reservation stops the panel with + * STOP: reason=budget_exhausted and keeps every row already written, and a + * cell that died without usage stays charged at its reservation (unknown). + * Rows append to /rows.jsonl; /summary.json holds the per-arm + * summary and the preregistered suggestion line. is + * GSTACK_MODEL_PANEL_OUT, defaulting beside the eval store. + */ +import { describe, test, expect, afterAll } from 'bun:test'; +import * as fs from 'fs'; +import * as path from 'path'; +import { runSkillTest } from './helpers/session-runner'; +import { runId, selectedTests } from './helpers/e2e-helpers'; +import { describeE2ETier } from './helpers/e2e-gate'; +import { getProjectEvalDir } from './helpers/eval-store'; +import { reserve, settle, readLedger, setCap, summarize as spendSummary, renderSpend, SpendCapExceeded } from '../lib/spend-ledger'; +import { + PANEL_ARMS, PANEL_TASKS, PANEL_TRIALS, PANEL_MAX_TURNS, PANEL_TIMEOUT_MS, PANEL_CONCURRENCY, PANEL_ALLOWED_TOOLS, PANEL_SPEND_CAP_USD, + setupCell, runCheck, armEnv, usageFromTranscript, cellCostUsd, classifyCell, summarize, suggestionLine, + type PanelArm, type PanelRow, type PanelTask, +} from './helpers/model-panel-harness'; +import { SKILL_NAME } from './helpers/arm-benchmark-harness'; + +const OUT = process.env.GSTACK_MODEL_PANEL_OUT || path.join(path.dirname(getProjectEvalDir()), 'model-panel'); +const ROWS = path.join(OUT, 'rows.jsonl'); +const SPEND = path.join(OUT, 'spend.json'); +let stopped: string | null = null; + +function consulted(result: Awaited>): boolean { + return result.toolCalls.some((tc) => tc.tool === 'Skill' && String((tc.input as { skill?: unknown })?.skill ?? '').includes(SKILL_NAME)); +} + +async function runCell(task: PanelTask, arm: PanelArm, trial: number): Promise { + if (stopped) return null; + const attempt = task.id + ':' + arm.id + ':' + trial + ':' + runId; + try { + reserve(SPEND, attempt, arm.reserveUsd, { cap: PANEL_SPEND_CAP_USD, label: arm.model }); + } catch (err) { + if (!(err instanceof SpendCapExceeded)) throw err; + stopped = err.message; + process.stderr.write('STOP: reason=budget_exhausted ' + err.message + '\n'); + return null; + } + const dirs = setupCell(task, arm.id + '-' + trial); + const started = Date.now(); + let settled = false; + try { + const result = await runSkillTest({ + prompt: task.ticket, + workingDirectory: dirs.dir, + maxTurns: PANEL_MAX_TURNS, + allowedTools: PANEL_ALLOWED_TOOLS, + timeout: PANEL_TIMEOUT_MS, + testName: task.id + '-' + arm.id + '-t' + trial, + runId, + model: arm.model, + env: armEnv(arm), + }); + const usage = usageFromTranscript(result.transcript); + const costUsd = usage ? cellCostUsd(arm, usage) : null; + settle(SPEND, attempt, costUsd ?? 'unknown'); + settled = true; + const cls = classifyCell(arm, result.exitReason, usage); + const scored = cls.verdict === 'scored' ? runCheck(task, dirs.dir) : null; + const row: PanelRow = { + task: task.id, arm: arm.id, model: arm.model, route: arm.route, trial, + verdict: scored ? (scored.pass ? 'pass' : 'fail') : 'infra', + reason: scored ? scored.detail : (cls as { reason: string }).reason, + exitReason: result.exitReason, costUsd, usage, durationMs: Date.now() - started, consulted: consulted(result), + }; + fs.appendFileSync(ROWS, JSON.stringify({ runId, ...row }) + '\n'); + console.log('[model-panel] ' + task.id + ' ' + arm.id + ' t' + trial + ' ' + row.verdict + ' $' + (costUsd ?? 0).toFixed(4) + ' turns=' + (usage?.turns ?? 0) + ' ' + row.reason); + return row; + } finally { + if (!settled) settle(SPEND, attempt, 'unknown'); + fs.rmSync(dirs.dir, { recursive: true, force: true }); + } +} + +async function pool(jobs: Array<() => Promise>, limit: number): Promise { + const out: T[] = new Array(jobs.length); + let next = 0; + await Promise.all(Array.from({ length: Math.min(limit, jobs.length) }, async () => { + while (next < jobs.length) { + const i = next++; + out[i] = await jobs[i](); + } + })); + return out; +} + +const describePaid = describeE2ETier('periodic'); + +for (const task of PANEL_TASKS) { + const selected = selectedTests === null || selectedTests.includes(task.id); + (selected ? describePaid : describe.skip)('Model panel: ' + task.id, () => { + test(task.id, async () => { + fs.mkdirSync(OUT, { recursive: true }); + if (readLedger(SPEND).cap_usd === null) setCap(SPEND, PANEL_SPEND_CAP_USD); + const jobs = PANEL_ARMS.flatMap((arm) => Array.from({ length: PANEL_TRIALS }, (_, i) => () => runCell(task, arm, i + 1))); + const rows = (await pool(jobs, PANEL_CONCURRENCY)).filter((r): r is PanelRow => r !== null); + expect(stopped, 'panel stopped: ' + stopped).toBeNull(); + expect(rows).toHaveLength(PANEL_ARMS.length * PANEL_TRIALS); + for (const row of rows) expect(['pass', 'fail', 'infra']).toContain(row.verdict); + }, Math.ceil((PANEL_ARMS.length * PANEL_TRIALS) / PANEL_CONCURRENCY) * PANEL_TIMEOUT_MS + 300_000); + }); +} + +afterAll(() => { + if (!fs.existsSync(ROWS)) return; + const rows = fs.readFileSync(ROWS, 'utf-8').trim().split('\n').filter(Boolean).map((l) => JSON.parse(l) as PanelRow); + const summaries = summarize(rows); + const suggestion = suggestionLine(summaries, new Date().toISOString().slice(0, 10)); + const spend = spendSummary(readLedger(SPEND)); + fs.writeFileSync(path.join(OUT, 'summary.json'), JSON.stringify({ rows: rows.length, summaries, suggestion, spend }, null, 2) + '\n'); + console.log('\n[model-panel] research instrument; gates nothing.'); + for (const s of summaries) { + console.log(' ' + s.label.padEnd(20) + ' ' + s.passes + '/' + s.completed + ' passed (95% ' + (s.low * 100).toFixed(0) + '-' + (s.high * 100).toFixed(0) + '%), infra ' + s.infra + ', $' + s.costPerTaskUsd.toFixed(4) + ' per task'); + } + console.log(' suggestion: ' + (suggestion.line ?? 'none (' + suggestion.reason + ')')); + console.log(' ' + renderSpend(spend)); +}); diff --git a/test/test-of-test-ratchet.test.ts b/test/test-of-test-ratchet.test.ts index 6f062fab2..2aed79521 100644 --- a/test/test-of-test-ratchet.test.ts +++ b/test/test-of-test-ratchet.test.ts @@ -131,6 +131,7 @@ const BASELINE = [ 'test/llm-judge-frontier.test.ts', 'test/llm-judge-stream.test.ts', // landed in v1.91.7.0 before this ratchet: harness owner for its functional-QA/docsync paid evals 'test/memory-cache-injection.test.ts', + 'test/model-panel-selftest.test.ts', // GSTA-173: free instrument check for the paid model panel; its hidden checks, INFRA classes and wording rule live in the paid-path helper 'test/module-size-ratchet.test.ts', // ratchet (c) source scanner (W2): measures product modules as text; its counter is test tooling 'test/native-auto-decide-pty.test.ts', 'test/no-suicide-exit.test.ts',