From d2098217c04c2c186d7698280543463c16079238 Mon Sep 17 00:00:00 2001 From: garrytan Date: Fri, 30 Oct 2026 01:04:59 +0000 Subject: [PATCH] feat(benchmark): particle-1 and DeepSeek V4.1 Flash awareness, gateway provider, preregistered model panel (GSTA-173) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- benchmark-models/SKILL.md | 4 +- benchmark-models/SKILL.md.tmpl | 7 +- bin/gstack-model-benchmark | 25 +- lib/outside-voice-runner.ts | 1 + lib/pricing.ts | 6 + test/benchmark-cli.test.ts | 11 + .../model-panel/changelog/CHANGELOG.md | 6 + .../fixtures/model-panel/changelog/COMMITS.md | 8 + .../model-panel/changelog/package.json | 5 + .../model-panel/checks/bugfix-decoys.js | 27 ++ test/fixtures/model-panel/checks/changelog.js | 30 ++ .../model-panel/checks/crud-endpoint.js | 32 ++ .../model-panel/checks/investigate.js | 27 ++ .../model-panel/checks/review-seeded-bug.js | 25 ++ .../model-panel/investigate/README.md | 4 + .../model-panel/investigate/package.json | 5 + .../model-panel/investigate/run-tests.js | 10 + .../model-panel/investigate/src/dates.js | 6 + .../model-panel/investigate/src/report.js | 9 + .../model-panel/investigate/src/retry.js | 11 + .../bugfix-decoys/src/format-price.js | 7 + .../reference/changelog/CHANGELOG.md | 12 + .../reference/crud-endpoint/app.js | 28 ++ .../reference/crud-endpoint/run-tests.js | 9 + .../reference/investigate/src/dates.js | 5 + .../reference/review-seeded-bug/REVIEW.md | 4 + .../model-panel/review-seeded-bug/PR.md | 4 + .../model-panel/review-seeded-bug/README.md | 4 + .../review-seeded-bug/package.json | 5 + .../model-panel/review-seeded-bug/src/db.js | 10 + .../review-seeded-bug/src/users.js | 14 + test/gateway-provider.test.ts | 90 ++++++ test/helpers/benchmark-runner.ts | 20 +- test/helpers/model-panel-harness.ts | 276 ++++++++++++++++++ test/helpers/providers/claude.ts | 16 +- test/helpers/providers/gateway.ts | 99 +++++++ test/helpers/providers/registry.ts | 21 ++ test/helpers/providers/types.ts | 4 + test/helpers/touchfiles-data.ts | 83 ++++++ test/model-panel-selftest.test.ts | 115 ++++++++ test/skill-e2e-model-panel.test.ts | 132 +++++++++ 41 files changed, 1179 insertions(+), 38 deletions(-) create mode 100644 test/fixtures/model-panel/changelog/CHANGELOG.md create mode 100644 test/fixtures/model-panel/changelog/COMMITS.md create mode 100644 test/fixtures/model-panel/changelog/package.json create mode 100644 test/fixtures/model-panel/checks/bugfix-decoys.js create mode 100644 test/fixtures/model-panel/checks/changelog.js create mode 100644 test/fixtures/model-panel/checks/crud-endpoint.js create mode 100644 test/fixtures/model-panel/checks/investigate.js create mode 100644 test/fixtures/model-panel/checks/review-seeded-bug.js create mode 100644 test/fixtures/model-panel/investigate/README.md create mode 100644 test/fixtures/model-panel/investigate/package.json create mode 100644 test/fixtures/model-panel/investigate/run-tests.js create mode 100644 test/fixtures/model-panel/investigate/src/dates.js create mode 100644 test/fixtures/model-panel/investigate/src/report.js create mode 100644 test/fixtures/model-panel/investigate/src/retry.js create mode 100644 test/fixtures/model-panel/reference/bugfix-decoys/src/format-price.js create mode 100644 test/fixtures/model-panel/reference/changelog/CHANGELOG.md create mode 100644 test/fixtures/model-panel/reference/crud-endpoint/app.js create mode 100644 test/fixtures/model-panel/reference/crud-endpoint/run-tests.js create mode 100644 test/fixtures/model-panel/reference/investigate/src/dates.js create mode 100644 test/fixtures/model-panel/reference/review-seeded-bug/REVIEW.md create mode 100644 test/fixtures/model-panel/review-seeded-bug/PR.md create mode 100644 test/fixtures/model-panel/review-seeded-bug/README.md create mode 100644 test/fixtures/model-panel/review-seeded-bug/package.json create mode 100644 test/fixtures/model-panel/review-seeded-bug/src/db.js create mode 100644 test/fixtures/model-panel/review-seeded-bug/src/users.js create mode 100644 test/gateway-provider.test.ts create mode 100644 test/helpers/model-panel-harness.ts create mode 100644 test/helpers/providers/gateway.ts create mode 100644 test/helpers/providers/registry.ts create mode 100644 test/model-panel-selftest.test.ts create mode 100644 test/skill-e2e-model-panel.test.ts diff --git a/benchmark-models/SKILL.md b/benchmark-models/SKILL.md index 06410fc4d..b8435d7ea 100644 --- a/benchmark-models/SKILL.md +++ b/benchmark-models/SKILL.md @@ -196,12 +196,12 @@ If C: ask for the path. Verify it exists. Use as positional argument. ## Step 2: Choose providers ```bash -"$BIN" --prompt "unused, dry-run" --models claude,gpt,gemini --dry-run +"$BIN" --prompt "unused, dry-run" --models claude,gpt,gemini,particle,deepseek --dry-run ``` Show the dry-run output. The "Adapter availability" section tells the user which providers will actually run (OK) vs skip (NOT READY — remediation hint included). -If ALL three show NOT READY: stop with a clear message — benchmark can't run without at least one authed provider. Suggest `claude login`, `codex login`, or `gemini login` / `export GOOGLE_API_KEY`. +If every provider shows NOT READY: stop with a clear message — benchmark can't run without at least one authed provider. Suggest `claude login`, `codex login`, `gemini login` / `export GOOGLE_API_KEY`, or `export OPENROUTER_API_KEY` for particle and deepseek (they run in Claude Code, so they also need the `claude` CLI). If at least one is OK: AskUserQuestion: - **Simplify:** "Which models should we include? The dry-run above showed which are authed. Unauthed ones will be skipped cleanly — they won't abort the batch." diff --git a/benchmark-models/SKILL.md.tmpl b/benchmark-models/SKILL.md.tmpl index 7360183c7..9cfda3934 100644 --- a/benchmark-models/SKILL.md.tmpl +++ b/benchmark-models/SKILL.md.tmpl @@ -4,7 +4,8 @@ preamble-tier: 1 version: 1.0.0 description: | Cross-model benchmark for gstack skills. Runs the same prompt through Claude, - GPT (via Codex CLI), and Gemini side-by-side — compares latency, tokens, cost, + GPT (via Codex CLI), Gemini, and (through OpenRouter in Claude Code) particle-1 + and DeepSeek V4.1 Flash side-by-side — compares latency, tokens, cost, and optionally quality via LLM judge. Answers "which model is actually best for this skill?" with data instead of vibes. Separate from /benchmark, which measures web page performance. Use when: "benchmark models", "compare models", @@ -71,12 +72,12 @@ If C: ask for the path. Verify it exists. Use as positional argument. ## Step 2: Choose providers ```bash -"$BIN" --prompt "unused, dry-run" --models claude,gpt,gemini --dry-run +"$BIN" --prompt "unused, dry-run" --models claude,gpt,gemini,particle,deepseek --dry-run ``` Show the dry-run output. The "Adapter availability" section tells the user which providers will actually run (OK) vs skip (NOT READY — remediation hint included). -If ALL three show NOT READY: stop with a clear message — benchmark can't run without at least one authed provider. Suggest `claude login`, `codex login`, or `gemini login` / `export GOOGLE_API_KEY`. +If every provider shows NOT READY: stop with a clear message — benchmark can't run without at least one authed provider. Suggest `claude login`, `codex login`, `gemini login` / `export GOOGLE_API_KEY`, or `export OPENROUTER_API_KEY` for particle and deepseek (they run in Claude Code, so they also need the `claude` CLI). If at least one is OK: AskUserQuestion: - **Simplify:** "Which models should we include? The dry-run above showed which are authed. Unauthed ones will be skipped cleanly — they won't abort the batch." diff --git a/bin/gstack-model-benchmark b/bin/gstack-model-benchmark index 81ed9dd1f..2f849b0d6 100755 --- a/bin/gstack-model-benchmark +++ b/bin/gstack-model-benchmark @@ -7,7 +7,8 @@ * gstack-model-benchmark [options] * * Options: - * --models claude,gpt,gemini Comma-separated provider list (default: claude) + * --models claude,gpt,gemini Comma-separated provider list (default: claude); particle and + * deepseek run in Claude Code through OpenRouter (OPENROUTER_API_KEY) * --prompt "" Inline prompt instead of a file * --workdir Working dir passed to each CLI (default: cwd) * --timeout-ms Per-provider timeout (default: 300000) @@ -28,15 +29,7 @@ import '../lib/conductor-env-shim'; import * as fs from 'fs'; import * as path from 'path'; import { runBenchmark, formatTable, formatJson, formatMarkdown, type BenchmarkInput } from '../test/helpers/benchmark-runner'; -import { ClaudeAdapter } from '../test/helpers/providers/claude'; -import { GptAdapter } from '../test/helpers/providers/gpt'; -import { GeminiAdapter } from '../test/helpers/providers/gemini'; - -const ADAPTER_FACTORIES = { - claude: () => new ClaudeAdapter(), - gpt: () => new GptAdapter(), - gemini: () => new GeminiAdapter(), -}; +import { PROVIDERS, PROVIDER_NAMES, isProviderName, type ProviderName } from '../test/helpers/providers/registry'; type OutputFormat = 'table' | 'json' | 'markdown'; @@ -76,13 +69,13 @@ function positionalArgs(args: string[]): string[] { return positional; } -function parseProviders(s: string | undefined): Array<'claude' | 'gpt' | 'gemini'> { +function parseProviders(s: string | undefined): ProviderName[] { if (!s) return ['claude']; - const seen = new Set<'claude' | 'gpt' | 'gemini'>(); + const seen = new Set(); for (const p of s.split(',').map(x => x.trim()).filter(Boolean)) { - if (p === 'claude' || p === 'gpt' || p === 'gemini') seen.add(p); + if (isProviderName(p)) seen.add(p); else { - console.error(`WARN: unknown provider '${p}' — skipping. Valid: claude, gpt, gemini.`); + console.error(`WARN: unknown provider '${p}' — skipping. Valid: ${PROVIDER_NAMES.join(', ')}.`); } } return seen.size ? Array.from(seen) : ['claude']; @@ -163,7 +156,7 @@ async function main(): Promise { async function dryRunReport(opts: { prompt: string; - providers: Array<'claude' | 'gpt' | 'gemini'>; + providers: ProviderName[]; workdir: string; timeoutMs: number; output: OutputFormat; @@ -181,7 +174,7 @@ async function dryRunReport(opts: { lines.push('Adapter availability:'); let authFailures = 0; for (const name of opts.providers) { - const factory = ADAPTER_FACTORIES[name]; + const factory = PROVIDERS[name]; if (!factory) { lines.push(` ${name}: UNKNOWN PROVIDER`); authFailures += 1; diff --git a/lib/outside-voice-runner.ts b/lib/outside-voice-runner.ts index 2892010b2..f56c5e2c9 100644 --- a/lib/outside-voice-runner.ts +++ b/lib/outside-voice-runner.ts @@ -62,6 +62,7 @@ const FAMILY_RULES: Array<[ModelFamily, RegExp]> = [ ['google', /^(google\/|gemini|gemma)/], ['xai', /^(xai\/|grok)/], ['deepseek', /^(deepseek\/|deepseek)/], + ['deepseek', /^(particle-ai\/|particle\/)?particle-1$/], ['qwen', /^(qwen\/|qwen|alibaba\/)/], ['zai', /^(zai\/|glm)/], ['moonshot', /^(moonshotai\/|moonshot\/|kimi)/], diff --git a/lib/pricing.ts b/lib/pricing.ts index 3fd3a3421..9d867e442 100644 --- a/lib/pricing.ts +++ b/lib/pricing.ts @@ -76,6 +76,12 @@ export const PRICING: Record = { 'gemini-3.1-pro-preview': { input_per_mtok: 2.00, output_per_mtok: 12.00, cached_input_per_mtok: 0.20, as_of: '2026-10' }, 'gemini-2.5-pro': { input_per_mtok: 1.25, output_per_mtok: 5.00, as_of: '2026-04' }, 'gemini-2.5-flash': { input_per_mtok: 0.30, output_per_mtok: 1.20, as_of: '2026-04' }, + + // Particle AI: https://docs.particle.ai/usage-billing (fetched 2026-10-30; peak rates, off-peak and flex are 50% lower; + // OpenRouter serves particle-ai/particle-1 at the same rates) + 'particle-1': { input_per_mtok: 0.25, output_per_mtok: 1.00, cached_input_per_mtok: 0.03, as_of: '2026-10' }, + // https://openrouter.ai/deepseek/deepseek-v4.1-flash-1017 (fetched 2026-10-30; the route the model panel runs it on) + 'deepseek-v4.1-flash-1017': { input_per_mtok: 0.15, output_per_mtok: 0.60, cached_input_per_mtok: 0.003, as_of: '2026-10' }, }; /** diff --git a/test/benchmark-cli.test.ts b/test/benchmark-cli.test.ts index cc94d38fc..ca50b2062 100644 --- a/test/benchmark-cli.test.ts +++ b/test/benchmark-cli.test.ts @@ -156,6 +156,17 @@ describe('gstack-model-benchmark --dry-run', () => { } }); + test('gateway providers without OPENROUTER_API_KEY are NOT READY and never fall back to claude', () => { + const env: Record = { PATH: process.env.PATH ?? '', HOME: process.env.HOME ?? '' }; + const result = spawnSync('bun', ['run', BIN, '--prompt', 'hi', '--models', 'particle,deepseek', '--dry-run'], { cwd: ROOT, env, encoding: 'utf-8', timeout: 15000 }); + const r = { status: result.status, stdout: result.stdout?.toString() ?? '' }; + expect(r.status).toBe(0); + expect(r.stdout).toContain('providers: particle, deepseek'); + expect(r.stdout).toMatch(/particle:\s+NOT READY — OPENROUTER_API_KEY is not set/); + expect(r.stdout).toMatch(/deepseek:\s+NOT READY — OPENROUTER_API_KEY is not set/); + expect(r.stdout).not.toMatch(/^\s+claude:/m); + }); + test('long prompt is truncated in dry-run display', () => { const longPrompt = 'x'.repeat(200); const r = run(['--prompt', longPrompt, '--dry-run']); diff --git a/test/fixtures/model-panel/changelog/CHANGELOG.md b/test/fixtures/model-panel/changelog/CHANGELOG.md new file mode 100644 index 000000000..a5a99af1b --- /dev/null +++ b/test/fixtures/model-panel/changelog/CHANGELOG.md @@ -0,0 +1,6 @@ +# Changelog + +## 1.3.0 + +- You can now attach photos to a trail note. +- Fixed: the map no longer jumps back to your location while you scroll. diff --git a/test/fixtures/model-panel/changelog/COMMITS.md b/test/fixtures/model-panel/changelog/COMMITS.md new file mode 100644 index 000000000..d8bb81d1e --- /dev/null +++ b/test/fixtures/model-panel/changelog/COMMITS.md @@ -0,0 +1,8 @@ +# Commits since 1.3.0 (newest first) + +- ci: cache node_modules between workflow runs +- fix: typing in a long note no longer signs you out after 5 minutes +- test: refactor shared test helpers +- feat: export your trail notes as a CSV file +- chore: bump eslint to 9.2 +- feat: dark mode toggle in Settings diff --git a/test/fixtures/model-panel/changelog/package.json b/test/fixtures/model-panel/changelog/package.json new file mode 100644 index 000000000..e329befad --- /dev/null +++ b/test/fixtures/model-panel/changelog/package.json @@ -0,0 +1,5 @@ +{ + "name": "trail-notes", + "version": "1.3.0", + "private": true +} diff --git a/test/fixtures/model-panel/checks/bugfix-decoys.js b/test/fixtures/model-panel/checks/bugfix-decoys.js new file mode 100644 index 000000000..1f58f13ac --- /dev/null +++ b/test/fixtures/model-panel/checks/bugfix-decoys.js @@ -0,0 +1,27 @@ +const { spawnSync } = require('node:child_process'); +const fs = require('node:fs'); +const path = require('node:path'); + +const dir = path.resolve(process.argv[2] || '.'); +const failures = []; +const expect = (ok, what) => { if (!ok) failures.push(what); }; +const git = (...args) => spawnSync('git', args, { cwd: dir, encoding: 'utf-8', timeout: 15_000 }); +const seed = () => git('rev-list', '--max-parents=0', 'HEAD').stdout.trim().split('\n').pop(); +const unchanged = (file) => git('diff', '--quiet', seed(), '--', file).status === 0; +const runNode = (file) => spawnSync(process.execPath, [file], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }); +const read = (file) => { try { return fs.readFileSync(path.join(dir, file), 'utf-8'); } catch { return ''; } }; +const load = (file) => { const p = path.join(dir, file); delete require.cache[require.resolve(p)]; return require(p); }; +const finish = () => { + if (failures.length) { console.error('FAIL: ' + failures.join('; ')); process.exit(1); } + console.log('PASS'); +}; + +try { + const { formatPrice } = load('src/format-price.js'); + for (const [cents, want] of [[1005, '$10.05'], [1250, '$12.50'], [999, '$9.99'], [5, '$0.05'], [100000, '$1000.00']]) { + expect(formatPrice(cents) === want, 'formatPrice(' + cents + ') should be ' + want); + } +} catch (err) { expect(false, 'src/format-price.js failed to load: ' + err.message); } +expect(unchanged('run-tests.js'), 'run-tests.js must not change'); +expect(runNode('run-tests.js').status === 0, 'node run-tests.js must pass'); +finish(); diff --git a/test/fixtures/model-panel/checks/changelog.js b/test/fixtures/model-panel/checks/changelog.js new file mode 100644 index 000000000..c740a5e7b --- /dev/null +++ b/test/fixtures/model-panel/checks/changelog.js @@ -0,0 +1,30 @@ +const { spawnSync } = require('node:child_process'); +const fs = require('node:fs'); +const path = require('node:path'); + +const dir = path.resolve(process.argv[2] || '.'); +const failures = []; +const expect = (ok, what) => { if (!ok) failures.push(what); }; +const git = (...args) => spawnSync('git', args, { cwd: dir, encoding: 'utf-8', timeout: 15_000 }); +const seed = () => git('rev-list', '--max-parents=0', 'HEAD').stdout.trim().split('\n').pop(); +const unchanged = (file) => git('diff', '--quiet', seed(), '--', file).status === 0; +const runNode = (file) => spawnSync(process.execPath, [file], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }); +const read = (file) => { try { return fs.readFileSync(path.join(dir, file), 'utf-8'); } catch { return ''; } }; +const load = (file) => { const p = path.join(dir, file); delete require.cache[require.resolve(p)]; return require(p); }; +const finish = () => { + if (failures.length) { console.error('FAIL: ' + failures.join('; ')); process.exit(1); } + console.log('PASS'); +}; + +const original13 = '## 1.3.0\n\n- You can now attach photos to a trail note.\n- Fixed: the map no longer jumps back to your location while you scroll.\n'; +const log = read('CHANGELOG.md'); +const first = log.indexOf('## '); +const second = log.indexOf('## ', first + 3); +const entry = first >= 0 && second > first ? log.slice(first, second) : ''; +expect(/^## \[?1\.4\.0\]?/.test(entry), 'the first section is 1.4.0, above 1.3.0'); +expect(/dark mode/i.test(entry), '1.4.0 mentions dark mode'); +expect(/csv/i.test(entry), '1.4.0 mentions CSV export'); +expect(/sign(s|ed)? (you )?out|log(s|ged)? (you )?out|time(s|d)?[ -]?out/i.test(entry), '1.4.0 mentions the sign-out fix'); +expect(!/eslint|test helpers?|node_modules|\bci\b|workflow/i.test(entry), '1.4.0 leaves out internal-only commits'); +expect(log.includes(original13), 'the 1.3.0 section is unchanged'); +finish(); diff --git a/test/fixtures/model-panel/checks/crud-endpoint.js b/test/fixtures/model-panel/checks/crud-endpoint.js new file mode 100644 index 000000000..9db752109 --- /dev/null +++ b/test/fixtures/model-panel/checks/crud-endpoint.js @@ -0,0 +1,32 @@ +const { spawnSync } = require('node:child_process'); +const fs = require('node:fs'); +const path = require('node:path'); + +const dir = path.resolve(process.argv[2] || '.'); +const failures = []; +const expect = (ok, what) => { if (!ok) failures.push(what); }; +const git = (...args) => spawnSync('git', args, { cwd: dir, encoding: 'utf-8', timeout: 15_000 }); +const seed = () => git('rev-list', '--max-parents=0', 'HEAD').stdout.trim().split('\n').pop(); +const unchanged = (file) => git('diff', '--quiet', seed(), '--', file).status === 0; +const runNode = (file) => spawnSync(process.execPath, [file], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }); +const read = (file) => { try { return fs.readFileSync(path.join(dir, file), 'utf-8'); } catch { return ''; } }; +const load = (file) => { const p = path.join(dir, file); delete require.cache[require.resolve(p)]; return require(p); }; +const finish = () => { + if (failures.length) { console.error('FAIL: ' + failures.join('; ')); process.exit(1); } + console.log('PASS'); +}; + +try { + const { handleRequest, resetForTests } = load('app.js'); + resetForTests(); + const created = handleRequest('POST', '/notes', { text: 'pack water' }); + expect(created.status === 201, 'POST /notes still returns 201'); + expect(handleRequest('DELETE', '/notes/' + created.body.id, null).status === 204, 'DELETE of an existing note returns 204'); + expect(handleRequest('DELETE', '/notes/' + created.body.id, null).status === 404, 'second DELETE returns 404'); + expect(handleRequest('DELETE', '/notes/999', null).status === 404, 'DELETE of an unknown id returns 404'); + const listed = handleRequest('GET', '/notes', null); + expect(Array.isArray(listed.body) && listed.body.length === 0, 'deleted note is gone from GET /notes'); +} catch (err) { expect(false, 'app.js failed: ' + err.message); } +expect(/DELETE/.test(read('run-tests.js')), 'run-tests.js covers DELETE'); +expect(runNode('run-tests.js').status === 0, 'node run-tests.js must pass'); +finish(); diff --git a/test/fixtures/model-panel/checks/investigate.js b/test/fixtures/model-panel/checks/investigate.js new file mode 100644 index 000000000..2a695a209 --- /dev/null +++ b/test/fixtures/model-panel/checks/investigate.js @@ -0,0 +1,27 @@ +const { spawnSync } = require('node:child_process'); +const fs = require('node:fs'); +const path = require('node:path'); + +const dir = path.resolve(process.argv[2] || '.'); +const failures = []; +const expect = (ok, what) => { if (!ok) failures.push(what); }; +const git = (...args) => spawnSync('git', args, { cwd: dir, encoding: 'utf-8', timeout: 15_000 }); +const seed = () => git('rev-list', '--max-parents=0', 'HEAD').stdout.trim().split('\n').pop(); +const unchanged = (file) => git('diff', '--quiet', seed(), '--', file).status === 0; +const runNode = (file) => spawnSync(process.execPath, [file], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }); +const read = (file) => { try { return fs.readFileSync(path.join(dir, file), 'utf-8'); } catch { return ''; } }; +const load = (file) => { const p = path.join(dir, file); delete require.cache[require.resolve(p)]; return require(p); }; +const finish = () => { + if (failures.length) { console.error('FAIL: ' + failures.join('; ')); process.exit(1); } + console.log('PASS'); +}; + +expect(runNode('run-tests.js').status === 0, 'node run-tests.js must pass'); +expect(unchanged('run-tests.js'), 'run-tests.js must not change'); +expect(unchanged('src/retry.js'), 'the retry decoy must not change'); +try { + process.env.TZ = 'Pacific/Kiritimati'; + const { dayOfMonth } = load('src/dates.js'); + expect(dayOfMonth('2026-03-08') === 8 && dayOfMonth('2026-01-01') === 1, 'dayOfMonth is right in a UTC+14 zone too'); +} catch (err) { expect(false, 'src/dates.js failed: ' + err.message); } +finish(); diff --git a/test/fixtures/model-panel/checks/review-seeded-bug.js b/test/fixtures/model-panel/checks/review-seeded-bug.js new file mode 100644 index 000000000..1dd93f55f --- /dev/null +++ b/test/fixtures/model-panel/checks/review-seeded-bug.js @@ -0,0 +1,25 @@ +const { spawnSync } = require('node:child_process'); +const fs = require('node:fs'); +const path = require('node:path'); + +const dir = path.resolve(process.argv[2] || '.'); +const failures = []; +const expect = (ok, what) => { if (!ok) failures.push(what); }; +const git = (...args) => spawnSync('git', args, { cwd: dir, encoding: 'utf-8', timeout: 15_000 }); +const seed = () => git('rev-list', '--max-parents=0', 'HEAD').stdout.trim().split('\n').pop(); +const unchanged = (file) => git('diff', '--quiet', seed(), '--', file).status === 0; +const runNode = (file) => spawnSync(process.execPath, [file], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }); +const read = (file) => { try { return fs.readFileSync(path.join(dir, file), 'utf-8'); } catch { return ''; } }; +const load = (file) => { const p = path.join(dir, file); delete require.cache[require.resolve(p)]; return require(p); }; +const finish = () => { + if (failures.length) { console.error('FAIL: ' + failures.join('; ')); process.exit(1); } + console.log('PASS'); +}; + +const review = read('REVIEW.md'); +expect(review.trim().length > 0, 'REVIEW.md must exist'); +expect(/users\.js/i.test(review), 'review names src/users.js'); +expect(/inject|parameteri[sz]|placeholder|bind|concatenat/i.test(review), 'review flags the SQL injection through search'); +expect(/off[- ]by[- ]one|page\s*-\s*1|\(page\s*-\s*1\)|first page|skips?\s+(the\s+)?first|pages? start at 1/i.test(review), 'review flags the page offset bug'); +expect(unchanged('src'), 'source files must not change during review'); +finish(); diff --git a/test/fixtures/model-panel/investigate/README.md b/test/fixtures/model-panel/investigate/README.md new file mode 100644 index 000000000..9cc1f13c5 --- /dev/null +++ b/test/fixtures/model-panel/investigate/README.md @@ -0,0 +1,4 @@ +# Hike report + +Builds the daily hike summary. Someone noted the report test "seems flaky on +some laptops" and wrapped the date lookup in a retry; it did not help. diff --git a/test/fixtures/model-panel/investigate/package.json b/test/fixtures/model-panel/investigate/package.json new file mode 100644 index 000000000..8a223bebd --- /dev/null +++ b/test/fixtures/model-panel/investigate/package.json @@ -0,0 +1,5 @@ +{ + "name": "hike-report", + "version": "2.0.0", + "private": true +} diff --git a/test/fixtures/model-panel/investigate/run-tests.js b/test/fixtures/model-panel/investigate/run-tests.js new file mode 100644 index 000000000..31a345eb5 --- /dev/null +++ b/test/fixtures/model-panel/investigate/run-tests.js @@ -0,0 +1,10 @@ +process.env.TZ = 'America/Los_Angeles'; +const assert = require('node:assert'); +const { dayOfMonth } = require('./src/dates'); +const { summary } = require('./src/report'); + +assert.strictEqual(dayOfMonth('2026-03-08'), 8); +assert.strictEqual(dayOfMonth('2026-12-31'), 31); +assert.strictEqual(summary({ name: 'Mission Peak', date: '2026-07-04' }), 'Mission Peak on day 4'); + +console.log('all report tests passed'); diff --git a/test/fixtures/model-panel/investigate/src/dates.js b/test/fixtures/model-panel/investigate/src/dates.js new file mode 100644 index 000000000..951dfb7d0 --- /dev/null +++ b/test/fixtures/model-panel/investigate/src/dates.js @@ -0,0 +1,6 @@ +// Day of the month for an ISO calendar date such as '2026-03-08'. +function dayOfMonth(isoDate) { + return new Date(isoDate).getDate(); +} + +module.exports = { dayOfMonth }; diff --git a/test/fixtures/model-panel/investigate/src/report.js b/test/fixtures/model-panel/investigate/src/report.js new file mode 100644 index 000000000..42da39549 --- /dev/null +++ b/test/fixtures/model-panel/investigate/src/report.js @@ -0,0 +1,9 @@ +const { dayOfMonth } = require('./dates'); +const { withRetry } = require('./retry'); + +function summary(hike) { + const day = withRetry(() => dayOfMonth(hike.date), 3, (d) => Number.isInteger(d)); + return `${hike.name} on day ${day}`; +} + +module.exports = { summary }; diff --git a/test/fixtures/model-panel/investigate/src/retry.js b/test/fixtures/model-panel/investigate/src/retry.js new file mode 100644 index 000000000..9eb1ee11d --- /dev/null +++ b/test/fixtures/model-panel/investigate/src/retry.js @@ -0,0 +1,11 @@ +// Retries fn up to times attempts; returns the first result that passes check. +function withRetry(fn, times, check = () => true) { + let last; + for (let i = 0; i < times; i++) { + last = fn(); + if (check(last)) return last; + } + return last; +} + +module.exports = { withRetry }; diff --git a/test/fixtures/model-panel/reference/bugfix-decoys/src/format-price.js b/test/fixtures/model-panel/reference/bugfix-decoys/src/format-price.js new file mode 100644 index 000000000..2253557b9 --- /dev/null +++ b/test/fixtures/model-panel/reference/bugfix-decoys/src/format-price.js @@ -0,0 +1,7 @@ +function formatPrice(cents) { + const dollars = Math.floor(cents / 100); + const rem = String(cents % 100).padStart(2, '0'); + return '$' + dollars + '.' + rem; +} + +module.exports = { formatPrice }; diff --git a/test/fixtures/model-panel/reference/changelog/CHANGELOG.md b/test/fixtures/model-panel/reference/changelog/CHANGELOG.md new file mode 100644 index 000000000..231164b8f --- /dev/null +++ b/test/fixtures/model-panel/reference/changelog/CHANGELOG.md @@ -0,0 +1,12 @@ +# Changelog + +## 1.4.0 + +- Dark mode: turn it on in Settings. +- Export your trail notes as a CSV file. +- Fixed: typing in a long note no longer signs you out after 5 minutes. + +## 1.3.0 + +- You can now attach photos to a trail note. +- Fixed: the map no longer jumps back to your location while you scroll. diff --git a/test/fixtures/model-panel/reference/crud-endpoint/app.js b/test/fixtures/model-panel/reference/crud-endpoint/app.js new file mode 100644 index 000000000..fb9d266e4 --- /dev/null +++ b/test/fixtures/model-panel/reference/crud-endpoint/app.js @@ -0,0 +1,28 @@ +let nextId = 1; +const notes = new Map(); + +function handleRequest(method, path, body) { + if (method === 'GET' && path === '/notes') { + return { status: 200, body: [...notes.values()] }; + } + if (method === 'POST' && path === '/notes') { + if (!body || typeof body.text !== 'string' || !body.text.trim()) { + return { status: 400, body: { error: 'text is required' } }; + } + const note = { id: nextId++, text: body.text.trim() }; + notes.set(note.id, note); + return { status: 201, body: note }; + } + const match = method === 'DELETE' && /^\/notes\/(\d+)$/.exec(path); + if (match) { + return notes.delete(Number(match[1])) ? { status: 204, body: null } : { status: 404, body: { error: 'not found' } }; + } + return { status: 404, body: { error: 'not found' } }; +} + +function resetForTests() { + nextId = 1; + notes.clear(); +} + +module.exports = { handleRequest, resetForTests }; diff --git a/test/fixtures/model-panel/reference/crud-endpoint/run-tests.js b/test/fixtures/model-panel/reference/crud-endpoint/run-tests.js new file mode 100644 index 000000000..bb144a670 --- /dev/null +++ b/test/fixtures/model-panel/reference/crud-endpoint/run-tests.js @@ -0,0 +1,9 @@ +const assert = require('node:assert'); +const { handleRequest, resetForTests } = require('./app'); + +resetForTests(); +const created = handleRequest('POST', '/notes', { text: 'buy trail mix' }); +assert.strictEqual(created.status, 201); +assert.strictEqual(handleRequest('DELETE', '/notes/1', null).status, 204); +assert.strictEqual(handleRequest('DELETE', '/notes/1', null).status, 404); +console.log('all notes tests passed'); diff --git a/test/fixtures/model-panel/reference/investigate/src/dates.js b/test/fixtures/model-panel/reference/investigate/src/dates.js new file mode 100644 index 000000000..80f4dec97 --- /dev/null +++ b/test/fixtures/model-panel/reference/investigate/src/dates.js @@ -0,0 +1,5 @@ +function dayOfMonth(isoDate) { + return Number(isoDate.slice(8, 10)); +} + +module.exports = { dayOfMonth }; diff --git a/test/fixtures/model-panel/reference/review-seeded-bug/REVIEW.md b/test/fixtures/model-panel/reference/review-seeded-bug/REVIEW.md new file mode 100644 index 000000000..f2e44ec0c --- /dev/null +++ b/test/fixtures/model-panel/reference/review-seeded-bug/REVIEW.md @@ -0,0 +1,4 @@ +# Review: search and paginate users + +1. src/users.js:8 — SQL injection: search is concatenated into the query. Pass it as a bound parameter ($3) instead. +2. src/users.js:7 — off-by-one: pages start at 1, so the offset should be (page - 1) * pageSize; page 1 skips the first page of users. diff --git a/test/fixtures/model-panel/review-seeded-bug/PR.md b/test/fixtures/model-panel/review-seeded-bug/PR.md new file mode 100644 index 000000000..6cf9af763 --- /dev/null +++ b/test/fixtures/model-panel/review-seeded-bug/PR.md @@ -0,0 +1,4 @@ +# PR: search and paginate users + +Adds `listUsers(db, { search, page, pageSize, sort })` in `src/users.js` for the +admin screen. `search` comes straight from the admin search box. Pages start at 1. diff --git a/test/fixtures/model-panel/review-seeded-bug/README.md b/test/fixtures/model-panel/review-seeded-bug/README.md new file mode 100644 index 000000000..4bfb1ff9b --- /dev/null +++ b/test/fixtures/model-panel/review-seeded-bug/README.md @@ -0,0 +1,4 @@ +# User directory + +A small service that lists users for the admin screen. `src/db.js` is a thin +wrapper over a SQL client: `db.query(sql, params)` runs a parameterized query. diff --git a/test/fixtures/model-panel/review-seeded-bug/package.json b/test/fixtures/model-panel/review-seeded-bug/package.json new file mode 100644 index 000000000..002905e32 --- /dev/null +++ b/test/fixtures/model-panel/review-seeded-bug/package.json @@ -0,0 +1,5 @@ +{ + "name": "user-directory", + "version": "1.0.0", + "private": true +} diff --git a/test/fixtures/model-panel/review-seeded-bug/src/db.js b/test/fixtures/model-panel/review-seeded-bug/src/db.js new file mode 100644 index 000000000..e48fe2479 --- /dev/null +++ b/test/fixtures/model-panel/review-seeded-bug/src/db.js @@ -0,0 +1,10 @@ +// Thin wrapper over the SQL client. query(sql, params) binds params to $1, $2, ... +function createDb(client) { + return { + query(sql, params = []) { + return client.query(sql, params); + }, + }; +} + +module.exports = { createDb }; diff --git a/test/fixtures/model-panel/review-seeded-bug/src/users.js b/test/fixtures/model-panel/review-seeded-bug/src/users.js new file mode 100644 index 000000000..38b17abc5 --- /dev/null +++ b/test/fixtures/model-panel/review-seeded-bug/src/users.js @@ -0,0 +1,14 @@ +// New in this PR: search and paginate users for the admin screen. +// Pages are numbered from 1 (the admin UI sends page=1 for the first page). +const SORTABLE = new Set(['name', 'created_at']); + +async function listUsers(db, { search = '', page = 1, pageSize = 20, sort = 'name' } = {}) { + const orderBy = SORTABLE.has(sort) ? sort : 'name'; + const offset = page * pageSize; + const sql = "SELECT id, name, email FROM users WHERE name ILIKE '%" + search + "%'" + + ` ORDER BY ${orderBy} LIMIT $1 OFFSET $2`; + const rows = await db.query(sql, [pageSize, offset]); + return rows.map((row) => ({ id: row.id, name: row.name, email: row.email })); +} + +module.exports = { listUsers }; diff --git a/test/gateway-provider.test.ts b/test/gateway-provider.test.ts new file mode 100644 index 000000000..29a21fd40 --- /dev/null +++ b/test/gateway-provider.test.ts @@ -0,0 +1,90 @@ +import { describe, test, expect, afterEach } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { GatewayAdapter, GATEWAY_MODELS, gatewayEnvironment, foreignModels } from './helpers/providers/gateway'; +import { PROVIDER_NAMES, isProviderName } from './helpers/providers/registry'; +import { familyConflict, modelFamily } from '../lib/outside-voice-runner'; +import { estimateCostUsd, hasPricing } from '../lib/pricing'; + +const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'gateway-provider-')); +const savedBin = process.env.GSTACK_CLAUDE_BIN; +afterEach(() => { + if (savedBin === undefined) delete process.env.GSTACK_CLAUDE_BIN; + else process.env.GSTACK_CLAUDE_BIN = savedBin; +}); + +/** A stand-in claude CLI: records its env and prints a result envelope billing FAKE_MODELS. */ +function fakeClaude(): string { + const bin = path.join(tmp, 'claude'); + fs.writeFileSync(bin, [ + '#!/bin/sh', + 'cat > /dev/null', + 'env > "$FAKE_ENV_OUT"', + 'printf \'{"type":"result","subtype":"success","is_error":false,"result":"ok","num_turns":2,"usage":{"input_tokens":1000,"output_tokens":10,"cache_read_input_tokens":50,"cache_creation_input_tokens":7},"modelUsage":{%s}}\' "$FAKE_MODELS"', + ].join('\n') + '\n', { mode: 0o755 }); + return bin; +} + +describe('gateway provider', () => { + test('the child env drops every Anthropic credential and pins all aliases to the arm model', () => { + const env = gatewayEnvironment('particle-ai/particle-1', { PATH: '/bin', OPENROUTER_API_KEY: 'sk-or', ANTHROPIC_API_KEY: 'sk-ant', GSTACK_ANTHROPIC_API_KEY: 'sk-ant-2', CLAUDE_CODE_OAUTH_TOKEN: 'oauth' }); + expect(Object.values(env).filter((v) => /sk-ant|oauth/.test(v))).toEqual([]); + expect(env.ANTHROPIC_API_KEY).toBe(''); + expect(env.ANTHROPIC_AUTH_TOKEN).toBe('sk-or'); + expect(env.ANTHROPIC_BASE_URL).toBe('https://openrouter.ai/api'); + expect(env.CLAUDE_CODE_SUBAGENT_MODEL).toBe('particle-ai/particle-1'); + expect(env.ANTHROPIC_DEFAULT_HAIKU_MODEL).toBe('particle-ai/particle-1'); + expect(env.PATH).toBe('/bin'); + expect(gatewayEnvironment('m', { OPENROUTER_API_KEY: 'k', GSTACK_GATEWAY_BASE_URL: 'https://ai-gateway.vercel.sh' }).ANTHROPIC_BASE_URL).toBe('https://ai-gateway.vercel.sh'); + }); + + test('a run through the adapter reaches claude without the Anthropic key and reports usage at our prices', async () => { + process.env.GSTACK_CLAUDE_BIN = fakeClaude(); + const envOut = path.join(tmp, 'env-ok.txt'); + const adapter = new GatewayAdapter('particle', { PATH: process.env.PATH, HOME: tmp, OPENROUTER_API_KEY: 'sk-or', ANTHROPIC_API_KEY: 'sk-ant-real', FAKE_ENV_OUT: envOut, FAKE_MODELS: '"particle-ai/particle-1":{}' }); + const result = await adapter.run({ prompt: 'hi', workdir: tmp, timeoutMs: 10_000 }); + expect(result.error).toBeUndefined(); + expect(result.modelUsed).toBe('particle-ai/particle-1'); + expect(result.tokens).toEqual({ input: 1000, output: 10, cached: 50, cacheWrite: 7 }); + const childEnv = fs.readFileSync(envOut, 'utf-8'); + expect(childEnv).not.toContain('sk-ant-real'); + expect(childEnv).toContain('ANTHROPIC_AUTH_TOKEN=sk-or'); + expect(adapter.estimateCost(result.tokens, result.modelUsed)).toBe(+((1000 * 0.25 + 10 * 1.0 + 50 * 0.03) / 1e6).toFixed(6)); + }); + + test('a run that billed another model is a MODEL_MIX error, not a result', async () => { + process.env.GSTACK_CLAUDE_BIN = fakeClaude(); + const adapter = new GatewayAdapter('deepseek', { PATH: process.env.PATH, HOME: tmp, OPENROUTER_API_KEY: 'sk-or', FAKE_ENV_OUT: path.join(tmp, 'env-mix.txt'), FAKE_MODELS: '"deepseek/deepseek-v4.1-flash-1017":{},"claude-haiku-5-5":{}' }); + const result = await adapter.run({ prompt: 'hi', workdir: tmp, timeoutMs: 10_000 }); + expect(result.error?.reason).toBe('MODEL_MIX: the run also billed claude-haiku-5-5'); + expect(foreignModels('particle-ai/particle-1', ['particle-1', 'PARTICLE-AI/particle-1'])).toEqual([]); + }); + + test('available() names the missing key with a fix', async () => { + process.env.GSTACK_CLAUDE_BIN = fakeClaude(); + const check = await new GatewayAdapter('particle', { PATH: process.env.PATH }).available(); + expect(check.ok).toBe(false); + expect(check.reason).toContain('OPENROUTER_API_KEY is not set'); + expect((await new GatewayAdapter('particle', { PATH: process.env.PATH, OPENROUTER_API_KEY: 'k' }).available()).ok).toBe(true); + }); + + test('the registry lists the gateway providers once, by name', () => { + expect(PROVIDER_NAMES).toEqual(['claude', 'gpt', 'gemini', 'particle', 'deepseek']); + expect(isProviderName('particle')).toBe(true); + expect(isProviderName('toString')).toBe(false); + expect(GATEWAY_MODELS).toEqual({ particle: 'particle-ai/particle-1', deepseek: 'deepseek/deepseek-v4.1-flash-1017' }); + }); + + test('particle-1 is DeepSeek family, so it is never an independent voice beside DeepSeek', () => { + for (const id of ['particle-ai/particle-1', 'particle/particle-1', 'particle-1', 'Particle-AI/Particle-1']) expect(modelFamily(id)).toBe('deepseek'); + expect(modelFamily('particle-ai/particle-2')).toBe('unknown'); + expect(familyConflict('particle-ai/particle-1', 'deepseek/deepseek-v4.1-flash-1017')).toBeDefined(); + expect(familyConflict('particle-ai/particle-1', 'claude-opus-5-5')).toBeUndefined(); + }); + + test('every panel and gateway id is priced', () => { + for (const id of ['particle-ai/particle-1', 'particle/particle-1', 'particle-1', 'deepseek/deepseek-v4.1-flash-1017', 'claude-opus-5-5', 'claude-sonnet-5-5']) expect(hasPricing(id)).toBe(true); + expect(estimateCostUsd({ input: 1_000_000, output: 1_000_000, cached: 1_000_000 }, 'deepseek/deepseek-v4.1-flash-1017')).toBeCloseTo(0.15 + 0.6 + 0.003, 6); + }); +}); diff --git a/test/helpers/benchmark-runner.ts b/test/helpers/benchmark-runner.ts index 324cce432..ee0886732 100644 --- a/test/helpers/benchmark-runner.ts +++ b/test/helpers/benchmark-runner.ts @@ -7,19 +7,17 @@ * one. Per-provider auth/timeout/rate-limit errors don't abort the batch. */ -import type { ProviderAdapter, RunOpts, RunResult, TokenUsage } from './providers/types'; -import { ClaudeAdapter } from './providers/claude'; -import { GptAdapter } from './providers/gpt'; -import { GeminiAdapter } from './providers/gemini'; +import type { RunOpts, RunResult, TokenUsage } from './providers/types'; +import { PROVIDERS, type ProviderName } from './providers/registry'; export interface BenchmarkInput { prompt: string; workdir: string; timeoutMs?: number; - /** Adapter names to run (e.g., ['claude', 'gpt', 'gemini']). */ - providers: Array<'claude' | 'gpt' | 'gemini'>; + /** Adapter names to run (e.g., ['claude', 'gpt', 'gemini', 'particle']). */ + providers: ProviderName[]; /** Optional per-provider model overrides. */ - models?: Partial>; + models?: Partial>; /** If true, skip providers whose available() returns !ok. If false, include them with error. */ skipUnavailable?: boolean; } @@ -44,12 +42,6 @@ export interface BenchmarkReport { entries: BenchmarkEntry[]; } -const ADAPTERS: Record<'claude' | 'gpt' | 'gemini', () => ProviderAdapter> = { - claude: () => new ClaudeAdapter(), - gpt: () => new GptAdapter(), - gemini: () => new GeminiAdapter(), -}; - export async function runBenchmark(input: BenchmarkInput): Promise { const startedAtMs = Date.now(); const startedAt = new Date(startedAtMs).toISOString(); @@ -59,7 +51,7 @@ export async function runBenchmark(input: BenchmarkInput): Promise> = []; for (const name of input.providers) { - const factory = ADAPTERS[name]; + const factory = PROVIDERS[name]; if (!factory) { entries.push({ provider: name, family: 'claude', available: false, unavailable_reason: `unknown provider: ${name}` }); continue; diff --git a/test/helpers/model-panel-harness.ts b/test/helpers/model-panel-harness.ts new file mode 100644 index 000000000..390f7cec0 --- /dev/null +++ b/test/helpers/model-panel-harness.ts @@ -0,0 +1,276 @@ +/** + * Model panel harness: one fixed panel of gstack-shaped engineering tasks run + * by several models in the same Claude Code harness, each cell scored by a + * hidden deterministic check. Shared by the paid periodic run + * (test/skill-e2e-model-panel.test.ts) and the free selftest + * (test/model-panel-selftest.test.ts), which proves every check fails on its + * seed fixture and passes on a reference answer before any money is spent. + * + * Preregistered (GSTA-173, 2026-10-30) before the first paid run: the arms, + * tasks, checks, PANEL_TRIALS, the INFRA classes in classifyCell, the spend + * reservations and the wording rule in suggestionLine. None of them change + * after a result is seen. + */ +import { spawnSync } from 'child_process'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { ROOT, copyDirSync } from './e2e-helpers'; +import { buildBehavioralSkill, run, SKILL_NAME, FIXTURES as ARM_FIXTURES } from './arm-benchmark-harness'; +import { GATEWAY_MODELS, foreignModels, gatewayOverrides } from './providers/gateway'; +import { estimateCostUsd, pricingRow } from './pricing'; + +export const PANEL_FIXTURES = path.join(ROOT, 'test', 'fixtures', 'model-panel'); +export const PANEL_TRIALS = 3; +export const PANEL_MAX_TURNS = 40; +export const PANEL_TIMEOUT_MS = 8 * 60_000; +export const PANEL_CONCURRENCY = 4; +export const PANEL_ALLOWED_TOOLS = ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep', 'Skill']; +/** Completed (non-INFRA) cells an arm needs before the panel publishes anything about it. */ +export const PANEL_MIN_COMPLETED = 12; +/** Pass rate below which the panel publishes numbers but no "worth trying" suggestion. */ +export const PANEL_USEFULNESS_FLOOR = 0.6; +export const PANEL_SPEND_CAP_USD = 15; + +export type PanelArmId = 'opus' | 'sonnet' | 'particle' | 'deepseek'; +export interface PanelArm { + id: PanelArmId; + label: string; + model: string; + route: 'anthropic' | 'openrouter'; + /** Worst-case USD reserved for one cell before it starts; settled to the measured cost after. */ + reserveUsd: number; +} + +export const PANEL_ARMS: PanelArm[] = [ + { id: 'opus', label: 'Opus 5.5', model: 'claude-opus-5-5', route: 'anthropic', reserveUsd: 2.0 }, + { id: 'sonnet', label: 'Sonnet 5.5', model: 'claude-sonnet-5-5', route: 'anthropic', reserveUsd: 1.0 }, + { id: 'particle', label: 'particle-1', model: GATEWAY_MODELS.particle, route: 'openrouter', reserveUsd: 0.15 }, + { id: 'deepseek', label: 'DeepSeek V4.1 Flash', model: GATEWAY_MODELS.deepseek, route: 'openrouter', reserveUsd: 0.15 }, +]; + +export interface PanelTask { + /** E2E_TOUCHFILES / E2E_TIERS key. */ + key: string; + fixture: string; + check: string; + reference: string; + ticket: string; +} + +const fixture = (name: string) => path.join(PANEL_FIXTURES, name); +const check = (name: string) => path.join(PANEL_FIXTURES, 'checks', name + '.js'); +const reference = (name: string) => path.join(PANEL_FIXTURES, 'reference', name); +const LEAVE = '\n\nLeave your changes uncommitted in the working tree.'; + +export const PANEL_TASKS: PanelTask[] = [ + { + key: 'model-panel-bugfix', + fixture: path.join(ARM_FIXTURES, 'bugfix-decoys'), + check: check('bugfix-decoys'), + reference: reference('bugfix-decoys'), + ticket: 'Bug report: receipts print $10.5 for a $10.05 item.\n\nYou are in a small git repo. `node run-tests.js` currently fails on formatPrice(1005).\nFix the bug so all tests pass. Verify with: node run-tests.js' + LEAVE, + }, + { + key: 'model-panel-crud', + fixture: path.join(ARM_FIXTURES, 'crud-endpoint'), + check: check('crud-endpoint'), + reference: reference('crud-endpoint'), + ticket: 'Ticket: users need to delete notes.\n\nYou are in a small git repo containing an in-memory notes API (app.js, wired to HTTP in server.js).\nAdd DELETE /notes/:id: respond 204 on success and 404 for an unknown id, and cover the new endpoint in run-tests.js. Verify with: node run-tests.js' + LEAVE, + }, + { + key: 'model-panel-review', + fixture: fixture('review-seeded-bug'), + check: check('review-seeded-bug'), + reference: reference('review-seeded-bug'), + ticket: 'Pre-landing review: PR.md describes a teammate\'s change, which added src/users.js.\n\nReview src/users.js the way a careful reviewer would before it ships: find the bugs that pass a quick read but break in production. Do not edit any source file. Write REVIEW.md with one numbered finding per bug, each naming file:line, what goes wrong, and the fix.' + LEAVE, + }, + { + key: 'model-panel-changelog', + fixture: fixture('changelog'), + check: check('changelog'), + reference: reference('changelog'), + ticket: 'Release 1.4.0: write its CHANGELOG.md entry from COMMITS.md.\n\nRules: the new section goes above 1.3.0 and matches its format; list only changes a user of the app would notice, in everyday words; leave out build, test and tooling changes; do not edit older sections.' + LEAVE, + }, + { + key: 'model-panel-investigate', + fixture: fixture('investigate'), + check: check('investigate'), + reference: reference('investigate'), + ticket: 'Bug: `node run-tests.js` fails, and the report shows the wrong day for some hikes.\n\nFind the root cause before you change anything, then fix it at the root. Do not edit the tests. Verify with: node run-tests.js' + LEAVE, + }, +]; + +export interface CellDirs { dir: string; seedSha: string } + +/** Fresh copy of the task fixture with the build-discipline skill installed (every arm alike) and one seed commit. */ +export function setupCell(task: PanelTask, label: string): CellDirs { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'panel-' + label + '-')); + copyDirSync(task.fixture, dir); + const skillDir = path.join(dir, '.claude', 'skills', SKILL_NAME); + fs.mkdirSync(skillDir, { recursive: true }); + fs.writeFileSync(path.join(skillDir, 'SKILL.md'), buildBehavioralSkill()); + fs.writeFileSync(path.join(dir, 'CLAUDE.md'), '# Project\n\nSmall fixture repo for one ticket.\n\n## Skill routing\n\nBefore working on any ticket, invoke the ' + SKILL_NAME + ' skill via the Skill tool and follow it while you work.\n'); + if (!fs.existsSync(path.join(dir, '.gitignore'))) fs.writeFileSync(path.join(dir, '.gitignore'), 'node_modules/\n'); + run('git', ['init', '-b', 'main'], dir); + run('git', ['config', 'user.email', 'model-panel@example.com'], dir); + run('git', ['config', 'user.name', 'Model Panel'], dir); + run('git', ['config', 'commit.gpgsign', 'false'], dir); + run('git', ['add', '-A'], dir); + run('git', ['commit', '-m', 'seed fixture'], dir); + return { dir, seedSha: run('git', ['rev-parse', 'HEAD'], dir).trim() }; +} + +/** Overlay the task's reference answer (selftest only). */ +export function applyReference(task: PanelTask, dir: string): void { + copyDirSync(task.reference, dir); +} + +/** Run the hidden check against a cell directory. Never throws. */ +export function runCheck(task: PanelTask, dir: string): { pass: boolean; detail: string } { + const r = spawnSync(process.execPath, [task.check, dir], { stdio: 'pipe', encoding: 'utf-8', timeout: 60_000 }); + const detail = ((r.stdout ?? '') + (r.stderr ?? '')).trim().split('\n').pop() ?? ''; + return { pass: r.status === 0, detail: detail.slice(0, 400) }; +} + +/** Extra child env for an arm: nothing for Anthropic arms, the gateway route (no Anthropic credential) otherwise. */ +export function armEnv(arm: PanelArm, source: NodeJS.ProcessEnv = process.env): Record { + return arm.route === 'openrouter' ? gatewayOverrides(arm.model, source) : {}; +} + +export interface CellUsage { + input: number; + cached: number; + cacheWrite: number; + output: number; + modelsBilled: string[]; + isError: boolean; + apiErrorStatus: number | null; + turns: number; +} + +/** Usage from the CLI's final result event; null when the transcript has none. */ +export function usageFromTranscript(transcript: any[]): CellUsage | null { + const result = [...transcript].reverse().find((e) => e && e.type === 'result'); + if (!result) return null; + const u = result.usage ?? {}; + return { + input: u.input_tokens ?? 0, + cached: u.cache_read_input_tokens ?? 0, + cacheWrite: u.cache_creation_input_tokens ?? 0, + output: u.output_tokens ?? 0, + modelsBilled: result.modelUsage && typeof result.modelUsage === 'object' ? Object.keys(result.modelUsage) : [], + isError: result.is_error === true, + apiErrorStatus: typeof result.api_error_status === 'number' ? result.api_error_status : null, + turns: result.num_turns ?? 0, + }; +} + +/** + * Measured cost from our price table, not the CLI's total_cost_usd (which + * prices gateway models at Anthropic rates). Cache writes are billed at 1.25x + * input for Anthropic models (5-minute writes) and at the input rate for + * gateway models, whose providers publish no separate write price. + */ +export function cellCostUsd(arm: PanelArm, usage: CellUsage): number { + const row = pricingRow(arm.model); + if (!row) throw new Error('no price row for ' + arm.model); + const writeRate = row.input_per_mtok * (arm.route === 'anthropic' ? 1.25 : 1); + return +(estimateCostUsd({ input: usage.input, cached: usage.cached, output: usage.output }, arm.model) + usage.cacheWrite * writeRate / 1_000_000).toFixed(6); +} + +export type CellVerdict = 'pass' | 'fail' | 'infra'; + +/** + * INFRA cells are excluded from pass rates, reported, and never rerun to + * change a verdict: no result event, a CLI or API error before the model + * produced a turn, a 402/429/5xx from the API, a startup timeout, or a run + * that billed any model besides the arm's (MODEL_MIX). Everything else is + * scored by the hidden check, including a run that timed out after working. + */ +export function classifyCell(arm: PanelArm, exitReason: string, usage: CellUsage | null): { verdict: 'infra'; reason: string } | { verdict: 'scored' } { + if (exitReason === 'timeout_startup') return { verdict: 'infra', reason: 'startup timeout' }; + if (!usage) return { verdict: 'infra', reason: 'no result event (' + exitReason + ')' }; + const foreign = foreignModels(arm.model, usage.modelsBilled); + if (foreign.length) return { verdict: 'infra', reason: 'MODEL_MIX: ' + foreign.join(', ') }; + if (usage.apiErrorStatus !== null && (usage.apiErrorStatus === 402 || usage.apiErrorStatus === 429 || usage.apiErrorStatus >= 500)) { + return { verdict: 'infra', reason: 'API ' + usage.apiErrorStatus }; + } + if (usage.isError && usage.turns === 0) return { verdict: 'infra', reason: 'error before the first turn (' + exitReason + ')' }; + return { verdict: 'scored' }; +} + +export interface PanelRow { + task: string; + arm: PanelArmId; + model: string; + route: PanelArm['route']; + trial: number; + verdict: CellVerdict; + reason: string; + exitReason: string; + costUsd: number | null; + usage: CellUsage | null; + durationMs: number; + consulted: boolean; +} + +export interface ArmSummary { + arm: PanelArmId; + label: string; + passes: number; + completed: number; + infra: number; + cells: number; + passRate: number; + low: number; + high: number; + costPerTaskUsd: number; +} + +/** Wilson 95% interval for a pass rate. */ +export function wilson(passes: number, n: number): { low: number; high: number } { + if (n === 0) return { low: 0, high: 1 }; + const z = 1.96, p = passes / n, d = 1 + z * z / n; + const c = p + z * z / (2 * n), m = z * Math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)); + return { low: Math.max(0, (c - m) / d), high: Math.min(1, (c + m) / d) }; +} + +export function summarize(rows: PanelRow[]): ArmSummary[] { + return PANEL_ARMS.map((arm) => { + const mine = rows.filter((r) => r.arm === arm.id); + const scored = mine.filter((r) => r.verdict !== 'infra'); + const passes = scored.filter((r) => r.verdict === 'pass').length; + const priced = mine.filter((r) => r.costUsd !== null); + const { low, high } = wilson(passes, scored.length); + return { + arm: arm.id, label: arm.label, passes, completed: scored.length, infra: mine.length - scored.length, cells: mine.length, + passRate: scored.length ? passes / scored.length : 0, low, high, + costPerTaskUsd: priced.length ? priced.reduce((s, r) => s + (r.costUsd ?? 0), 0) / priced.length : 0, + }; + }); +} + +const money = (usd: number) => '$' + (usd < 0.1 ? usd.toFixed(3) : usd.toFixed(2)); + +/** + * The preregistered wording rule. Returns the one line gstack may publish + * about particle-1, or null with the reason when the panel supports none. + * It quotes raw pass counts and measured cost per task; it never says + * "comparable" or "better", says "no measured difference" only when the + * Wilson intervals overlap, and calls particle-1 cheapest only among the + * arms measured. + */ +export function suggestionLine(summaries: ArmSummary[], measuredOn: string): { line: string | null; reason: string } { + const by = (id: PanelArmId) => summaries.find((s) => s.arm === id)!; + const p = by('particle'), s = by('sonnet'), o = by('opus'), d = by('deepseek'); + const thin = summaries.filter((x) => x.completed < PANEL_MIN_COMPLETED); + if (thin.length) return { line: null, reason: 'INCOMPLETE: ' + thin.map((x) => x.label + ' ' + x.completed + '/' + x.cells).join(', ') + ' completed cells (need ' + PANEL_MIN_COMPLETED + ')' }; + const vsSonnet = p.high < s.low ? 'below Sonnet 5.5 on this panel' : p.low > s.high ? 'above Sonnet 5.5 on this panel' : 'no measured difference from Sonnet 5.5 on this panel'; + const cheapest = summaries.every((x) => x.arm === 'particle' || p.costPerTaskUsd < x.costPerTaskUsd); + const counts = 'particle-1 passed ' + p.passes + '/' + p.completed + ' tasks on gstack\'s model panel (Sonnet 5.5 ' + s.passes + '/' + s.completed + ', Opus 5.5 ' + o.passes + '/' + o.completed + ', DeepSeek V4.1 Flash ' + d.passes + '/' + d.completed + ')'; + const cost = ' at ' + money(p.costPerTaskUsd) + ' per task vs ' + money(s.costPerTaskUsd) + ' for Sonnet 5.5' + (cheapest ? ', the cheapest of the four models measured' : ''); + const where = ' (' + vsSonnet + '; measured ' + measuredOn + ' in Claude Code via OpenRouter).'; + if (p.passRate < PANEL_USEFULNESS_FLOOR) return { line: null, reason: 'below the usefulness floor: ' + counts + cost + where }; + return { line: counts + cost + where + ' Worth trying for low-stakes steps; keep blocking reviewers on your main model.', reason: 'ok' }; +} diff --git a/test/helpers/providers/claude.ts b/test/helpers/providers/claude.ts index 945c82393..7c6d10461 100644 --- a/test/helpers/providers/claude.ts +++ b/test/helpers/providers/claude.ts @@ -26,10 +26,19 @@ import { runClaudeProducerStream } from './claude-stream'; * format. CSO producers stream events instead (claude-stream.ts) so a timed-out * run still reports its usage. */ +export interface ClaudeAdapterOptions { + /** Child environment for a non-producer run; defaults to claudeExecEnvironment over process.env. */ + environment?: (opts: RunOpts) => Record; + /** Treat an explicit error or empty `result` envelope as a failed run, as paid producers do. */ + strict?: boolean; +} + export class ClaudeAdapter implements ProviderAdapter { readonly name = 'claude'; readonly family = 'claude' as const; + constructor(private readonly options: ClaudeAdapterOptions = {}) {} + async available(opts?: RunOpts): Promise { // Binary on PATH (or GSTACK_CLAUDE_BIN override). Routes through the shared // resolver so Windows + override paths behave the same as production sites. @@ -79,7 +88,7 @@ export class ClaudeAdapter implements ProviderAdapter { const model = opts.model ?? process.env.EVALS_MODEL ?? resolveEvalModel('capture'); const stateDirectory = csoProducerStateDirectory(opts); const args = claudeExecArgs(opts, model, resolved.argsPrefix); - const env = claudeExecEnvironment(opts, process.env); + const env = !opts.csoProducer && this.options.environment ? this.options.environment(opts) : claudeExecEnvironment(opts, process.env); if (stateDirectory) { assertClaudeProducerPrerequisites(); assertClaudeProducerSandboxPolicy(args, env, csoProducerHelperLauncher(opts)!); @@ -100,7 +109,7 @@ export class ClaudeAdapter implements ProviderAdapter { // AskUserQuestion failure BLOCKs rather than emitting unanswerable prose). env, }); - return resultFromClaudeOutput(out,{model,durationMs:Date.now()-start,producer:!!opts.csoProducer}); + return resultFromClaudeOutput(out,{model,durationMs:Date.now()-start,producer:!!opts.csoProducer||!!this.options.strict}); } catch (err: unknown) { const durationMs = Date.now() - start; const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string }; @@ -143,7 +152,8 @@ export function resultFromClaudeOutput(raw:string,opts:{model?:string;durationMs const obj=JSON.parse(raw),explicitError=obj?.is_error===true||(typeof obj?.subtype==='string'&&obj.subtype!=='success'),validResult=!explicitError&&typeof obj?.result==='string'&&obj.result.trim().length>0; if(opts.producer&&!validResult)return{output:'',tokens:{input:0,output:0},durationMs,toolCalls:0,modelUsed:typeof obj?.model==='string'&&obj.model?obj.model:defaultModel,error:{code:'unknown',reason:'empty or invalid output from claude CLI (exit 0)'}}; const output=typeof obj?.result==='string'?obj.result:String(obj?.result??''),usage=obj?.usage??{}; - return{output,tokens:{input:usage.input_tokens??0,output:usage.output_tokens??0,cached:usage.cache_read_input_tokens},durationMs,toolCalls:obj?.num_turns??0,modelUsed:typeof obj?.model==='string'&&obj.model?obj.model:defaultModel}; + const modelsBilled=obj?.modelUsage&&typeof obj.modelUsage==='object'?Object.keys(obj.modelUsage):undefined; + return{output,tokens:{input:usage.input_tokens??0,output:usage.output_tokens??0,cached:usage.cache_read_input_tokens,cacheWrite:usage.cache_creation_input_tokens},durationMs,toolCalls:obj?.num_turns??0,modelUsed:typeof obj?.model==='string'&&obj.model?obj.model:defaultModel,...(modelsBilled?{modelsBilled}:{})}; }catch{ if(opts.producer)return{output:'',tokens:{input:0,output:0},durationMs,toolCalls:0,modelUsed:defaultModel,error:{code:'unknown',reason:'empty or invalid output from claude CLI (exit 0)'}}; return{output:raw,tokens:{input:0,output:0},durationMs,toolCalls:0,modelUsed:defaultModel}; diff --git a/test/helpers/providers/gateway.ts b/test/helpers/providers/gateway.ts new file mode 100644 index 000000000..88efa1fa8 --- /dev/null +++ b/test/helpers/providers/gateway.ts @@ -0,0 +1,99 @@ +import { ClaudeAdapter, claudeExecEnvironment } from './claude'; +import type { AvailabilityCheck, ProviderAdapter, RunOpts, RunResult } from './types'; +import { estimateCostUsd } from '../pricing'; +import { resolveClaudeCommand } from '../../../lib/claude-bin'; + +/** + * Gateway adapter: the Claude Code harness (`claude -p`) pointed at an + * Anthropic-compatible gateway, so a non-Anthropic model runs in the same + * agent loop as the Claude arms. OpenRouter by default + * (GSTACK_GATEWAY_BASE_URL overrides the endpoint; the key is OPENROUTER_API_KEY). + * + * The child never sees an Anthropic credential: ANTHROPIC_API_KEY, + * CLAUDE_CODE_OAUTH_TOKEN and GSTACK_ANTHROPIC_API_KEY are dropped and the + * gateway key travels as ANTHROPIC_AUTH_TOKEN. Every Claude Code model alias + * and the subagent model are pinned to the arm's model, and a run that billed + * any other model is reported as a MODEL_MIX error instead of a result. + */ +export const GATEWAY_MODELS = { + particle: 'particle-ai/particle-1', + deepseek: 'deepseek/deepseek-v4.1-flash-1017', +} as const; +export type GatewayName = keyof typeof GATEWAY_MODELS; + +export const GATEWAY_DEFAULT_BASE_URL = 'https://openrouter.ai/api'; +export const GATEWAY_KEY_ENV = 'OPENROUTER_API_KEY'; +const ANTHROPIC_CREDENTIALS = ['ANTHROPIC_API_KEY', 'CLAUDE_CODE_OAUTH_TOKEN', 'GSTACK_ANTHROPIC_API_KEY'] as const; + +/** The variables a gateway run sets on top of any child env; blank credentials override inherited ones. */ +export function gatewayOverrides(model: string, source: NodeJS.ProcessEnv = process.env): Record { + const key = source[GATEWAY_KEY_ENV]; + if (!key) throw new Error(`${GATEWAY_KEY_ENV} is not set`); + return { + ANTHROPIC_BASE_URL: source.GSTACK_GATEWAY_BASE_URL || GATEWAY_DEFAULT_BASE_URL, + ANTHROPIC_AUTH_TOKEN: key, + ANTHROPIC_API_KEY: '', + CLAUDE_CODE_OAUTH_TOKEN: '', + ANTHROPIC_MODEL: model, + ANTHROPIC_SMALL_FAST_MODEL: model, + ANTHROPIC_DEFAULT_HAIKU_MODEL: model, + ANTHROPIC_DEFAULT_SONNET_MODEL: model, + ANTHROPIC_DEFAULT_OPUS_MODEL: model, + CLAUDE_CODE_SUBAGENT_MODEL: model, + }; +} + +/** A complete child env for a gateway run: the source env without Anthropic credentials, plus gatewayOverrides. */ +export function gatewayEnvironment(model: string, source: NodeJS.ProcessEnv = process.env): Record { + const env: Record = {}; + for (const [k, v] of Object.entries(source)) { + if (v !== undefined && !(ANTHROPIC_CREDENTIALS as readonly string[]).includes(k)) env[k] = v; + } + return { ...env, ...gatewayOverrides(model, source) }; +} + +/** Models a run billed besides the one it asked for (aliases compared without a provider prefix). */ +export function foreignModels(requested: string, billed: string[] | undefined): string[] { + const bare = (id: string) => id.toLowerCase().replace(/^[a-z-]+\//, ''); + return (billed ?? []).filter(id => bare(id) !== bare(requested)); +} + +export class GatewayAdapter implements ProviderAdapter { + readonly family = 'claude' as const; + readonly model: string; + private readonly harness: ClaudeAdapter; + + constructor(readonly name: GatewayName, private readonly source: NodeJS.ProcessEnv = process.env) { + this.model = GATEWAY_MODELS[name]; + this.harness = new ClaudeAdapter({ + environment: opts => gatewayEnvironment(this.model, claudeExecEnvironment(opts, this.source)), + strict: true, + }); + } + + async available(): Promise { + if (!resolveClaudeCommand()) { + return { ok: false, reason: 'claude CLI not found on PATH (the gateway arms run in Claude Code). Install: npm i -g @anthropic-ai/claude-code' }; + } + if (!this.source[GATEWAY_KEY_ENV]) { + return { ok: false, reason: `${GATEWAY_KEY_ENV} is not set. Create a key at https://openrouter.ai/keys and export it.` }; + } + return { ok: true }; + } + + async run(opts: RunOpts): Promise { + if (opts.csoProducer) { + return { output: '', tokens: { input: 0, output: 0 }, durationMs: 0, toolCalls: 0, modelUsed: this.model, error: { code: 'unknown', reason: 'gateway providers do not run CSO producers' } }; + } + const result = await this.harness.run({ ...opts, model: this.model }); + const foreign = foreignModels(this.model, result.modelsBilled); + if (!result.error && foreign.length) { + return { ...result, error: { code: 'unknown', reason: `MODEL_MIX: the run also billed ${foreign.join(', ')}` } }; + } + return result; + } + + estimateCost(tokens: { input: number; output: number; cached?: number }, model?: string): number { + return estimateCostUsd(tokens, model ?? this.model); + } +} diff --git a/test/helpers/providers/registry.ts b/test/helpers/providers/registry.ts new file mode 100644 index 000000000..0d3571bc3 --- /dev/null +++ b/test/helpers/providers/registry.ts @@ -0,0 +1,21 @@ +import type { ProviderAdapter } from './types'; +import { ClaudeAdapter } from './claude'; +import { GptAdapter } from './gpt'; +import { GeminiAdapter } from './gemini'; +import { GatewayAdapter } from './gateway'; + +/** The one provider registry the benchmark CLI, its dry run and the runner share. */ +export const PROVIDERS = { + claude: () => new ClaudeAdapter(), + gpt: () => new GptAdapter(), + gemini: () => new GeminiAdapter(), + particle: () => new GatewayAdapter('particle'), + deepseek: () => new GatewayAdapter('deepseek'), +} satisfies Record ProviderAdapter>; + +export type ProviderName = keyof typeof PROVIDERS; +export const PROVIDER_NAMES = Object.keys(PROVIDERS) as ProviderName[]; + +export function isProviderName(name: string): name is ProviderName { + return Object.hasOwn(PROVIDERS, name); +} diff --git a/test/helpers/providers/types.ts b/test/helpers/providers/types.ts index e9b51527e..edd89d5c8 100644 --- a/test/helpers/providers/types.ts +++ b/test/helpers/providers/types.ts @@ -40,6 +40,8 @@ export interface TokenUsage { output: number; /** Cached input tokens (Anthropic/OpenAI support). Undefined if provider doesn't report. */ cached?: number; + /** Prompt-cache write tokens (Claude CLI `cache_creation_input_tokens`). Not priced by estimateCostUsd. */ + cacheWrite?: number; } export type RunError = @@ -70,6 +72,8 @@ export interface RunResult { toolTimeline?: ToolTimelineEntry[]; /** Actual model ID the provider reports using (may be a variant of the family). */ modelUsed: string; + /** Every model the CLI billed during the run (Claude CLI `modelUsage` keys). Unset if unreported. */ + modelsBilled?: string[]; /** If the run failed, error code + human reason. output/tokens may be partial. */ error?: { code: RunError; reason: string }; } diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index 9413e8351..3079ace6a 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -1121,6 +1121,79 @@ export const E2E_TOUCHFILES: Record = { 'test/skill-e2e-arm-benchmark.test.ts', 'ship/SKILL.md', 'test/helpers/e2e-gate.ts'], + // Model panel (GSTA-173): particle-1, DeepSeek V4.1 Flash, Sonnet 5.5 and Opus 5.5 on one + // fixed task panel in Claude Code, hidden deterministic checks (research instrument). + 'model-panel-bugfix': [ + 'test/fixtures/model-panel/**', + 'test/fixtures/arm-benchmark/bugfix-decoys/**', + 'test/fixtures/arm-benchmark/crud-endpoint/**', + 'test/helpers/model-panel-harness.ts', + 'test/helpers/arm-benchmark-harness.ts', + 'test/helpers/providers/gateway.ts', + 'test/helpers/providers/claude.ts', + 'test/helpers/providers/claude-stream.ts', + 'test/helpers/providers/types.ts', + 'test/helpers/pricing.ts', + 'lib/pricing.ts', + 'test/skill-e2e-model-panel.test.ts', + 'ship/SKILL.md', 'test/helpers/e2e-gate.ts'], + 'model-panel-crud': [ + 'test/fixtures/model-panel/**', + 'test/fixtures/arm-benchmark/bugfix-decoys/**', + 'test/fixtures/arm-benchmark/crud-endpoint/**', + 'test/helpers/model-panel-harness.ts', + 'test/helpers/arm-benchmark-harness.ts', + 'test/helpers/providers/gateway.ts', + 'test/helpers/providers/claude.ts', + 'test/helpers/providers/claude-stream.ts', + 'test/helpers/providers/types.ts', + 'test/helpers/pricing.ts', + 'lib/pricing.ts', + 'test/skill-e2e-model-panel.test.ts', + 'ship/SKILL.md', 'test/helpers/e2e-gate.ts'], + 'model-panel-review': [ + 'test/fixtures/model-panel/**', + 'test/fixtures/arm-benchmark/bugfix-decoys/**', + 'test/fixtures/arm-benchmark/crud-endpoint/**', + 'test/helpers/model-panel-harness.ts', + 'test/helpers/arm-benchmark-harness.ts', + 'test/helpers/providers/gateway.ts', + 'test/helpers/providers/claude.ts', + 'test/helpers/providers/claude-stream.ts', + 'test/helpers/providers/types.ts', + 'test/helpers/pricing.ts', + 'lib/pricing.ts', + 'test/skill-e2e-model-panel.test.ts', + 'ship/SKILL.md', 'test/helpers/e2e-gate.ts'], + 'model-panel-changelog': [ + 'test/fixtures/model-panel/**', + 'test/fixtures/arm-benchmark/bugfix-decoys/**', + 'test/fixtures/arm-benchmark/crud-endpoint/**', + 'test/helpers/model-panel-harness.ts', + 'test/helpers/arm-benchmark-harness.ts', + 'test/helpers/providers/gateway.ts', + 'test/helpers/providers/claude.ts', + 'test/helpers/providers/claude-stream.ts', + 'test/helpers/providers/types.ts', + 'test/helpers/pricing.ts', + 'lib/pricing.ts', + 'test/skill-e2e-model-panel.test.ts', + 'ship/SKILL.md', 'test/helpers/e2e-gate.ts'], + 'model-panel-investigate': [ + 'test/fixtures/model-panel/**', + 'test/fixtures/arm-benchmark/bugfix-decoys/**', + 'test/fixtures/arm-benchmark/crud-endpoint/**', + 'test/helpers/model-panel-harness.ts', + 'test/helpers/arm-benchmark-harness.ts', + 'test/helpers/providers/gateway.ts', + 'test/helpers/providers/claude.ts', + 'test/helpers/providers/claude-stream.ts', + 'test/helpers/providers/types.ts', + 'test/helpers/pricing.ts', + 'lib/pricing.ts', + 'test/skill-e2e-model-panel.test.ts', + 'ship/SKILL.md', 'test/helpers/e2e-gate.ts'], + 'office-hours-section-loading': [ 'ETHOS.md', 'office-hours/**', 'bin/gstack-office-hours-review', 'lib/office-hours-review.ts', 'lib/fs-atomic.ts', 'lib/spec-review-cap.ts', 'test/helpers/office-hours-formatter-fixture.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/carve-guards.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/llm-judge.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-office-hours-section-loading.test.ts', 'test/helpers/agent-sdk-runner.ts', 'test/helpers/auq-native-capture.ts', 'test/helpers/auto-decision-state.ts', 'test/helpers/autoplan-artifact-digest.ts', 'test/helpers/autoplan-artifact-permission.ts', 'test/helpers/autoplan-artifact-recorder.ts', 'test/helpers/capture-parity-baseline.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty/**', 'test/helpers/dx-selected-navigation.ts', 'test/helpers/e2e-gate.ts', 'test/helpers/eng-cache-writer-decision.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/helpers/native-auto-decide.ts', 'test/helpers/owned-claude-transcript.ts', 'test/helpers/parity-harness.ts', 'test/helpers/size-ledger.ts', 'test/size-ledger/**', 'test/helpers/plan-count-artifacts.ts', 'test/helpers/plan-count-file-permission.ts', 'test/helpers/plan-count-fixture.ts', 'test/helpers/plan-count-pending-exit.ts', 'test/helpers/plan-count-pending-question.ts', 'test/helpers/plan-count-transcript.ts', 'test/helpers/plan-floor-review.ts', 'test/helpers/plan-floor-target.ts', 'test/helpers/plan-scope-selection.ts', 'test/helpers/plan-seed-submission.ts', 'test/helpers/plan-skill-question-events.ts', 'test/helpers/plan-skill-question-hook-scope.ts', 'test/helpers/plan-skill-questions.ts', 'test/helpers/pty-screen.ts', 'test/helpers/pty-trust-dialog.ts', 'test/helpers/skill-census.ts'], 'office-hours-design-draft': [ 'ETHOS.md', 'office-hours/**', 'lib/office-hours-review.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/carve-guards.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/llm-judge.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-office-hours-design-draft.test.ts', 'test/helpers/agent-sdk-runner.ts', 'test/helpers/auq-native-capture.ts', 'test/helpers/auto-decision-state.ts', 'test/helpers/autoplan-artifact-digest.ts', 'test/helpers/autoplan-artifact-permission.ts', 'test/helpers/autoplan-artifact-recorder.ts', 'test/helpers/capture-parity-baseline.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty/**', 'test/helpers/dx-selected-navigation.ts', 'test/helpers/e2e-gate.ts', 'test/helpers/eng-cache-writer-decision.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/helpers/native-auto-decide.ts', 'test/helpers/owned-claude-transcript.ts', 'test/helpers/parity-harness.ts', 'test/helpers/size-ledger.ts', 'test/size-ledger/**', 'test/helpers/plan-count-artifacts.ts', 'test/helpers/plan-count-file-permission.ts', 'test/helpers/plan-count-fixture.ts', 'test/helpers/plan-count-pending-exit.ts', 'test/helpers/plan-count-pending-question.ts', 'test/helpers/plan-count-transcript.ts', 'test/helpers/plan-floor-review.ts', 'test/helpers/plan-floor-target.ts', 'test/helpers/plan-scope-selection.ts', 'test/helpers/plan-seed-submission.ts', 'test/helpers/plan-skill-question-events.ts', 'test/helpers/plan-skill-question-hook-scope.ts', 'test/helpers/plan-skill-questions.ts', 'test/helpers/pty-screen.ts', 'test/helpers/pty-trust-dialog.ts', 'test/helpers/skill-census.ts'], 'dx-error-contract-review': ['test/skill-e2e-dx-error-contract.test.ts', 'test/fixtures/dx-error-contract-case.ts', 'lib/dx-error-contract.ts', 'scripts/resolvers/dx.ts', 'scripts/resolvers/index.ts', 'scripts/gen-skill-docs.ts', 'plan-devex-review/**', 'bin/gstack-autoplan-snapshot.ts', 'lib/autoplan-prompts.ts', 'lib/autoplan-reconcile.ts', 'lib/coordinator-contract.ts', 'test/helpers/session-runner.ts', 'test/helpers/eval-store.ts', 'test/helpers/eval-budgets.ts', 'lib/claude-bin.ts', 'lib/eval-model.ts'], @@ -1544,6 +1617,11 @@ export const E2E_TIERS: Record = { 'arm-benchmark-native-overbuild': 'periodic', 'arm-benchmark-crud-endpoint': 'periodic', 'arm-benchmark-bugfix-decoys': 'periodic', + 'model-panel-bugfix': 'periodic', + 'model-panel-crud': 'periodic', + 'model-panel-review': 'periodic', + 'model-panel-changelog': 'periodic', + 'model-panel-investigate': 'periodic', 'office-hours-section-loading': 'marathon', // Full startup design/review/approval workflow (1–3 real review rounds, ~20 min) 'office-hours-design-draft': 'periodic', // Same interview through the design-creating Write (~5 min) 'plan-decision-classification': 'periodic', @@ -1919,6 +1997,11 @@ export const E2E_KINDS: Record = { 'arm-benchmark-native-overbuild': 'rule', 'arm-benchmark-crud-endpoint': 'rule', 'arm-benchmark-bugfix-decoys': 'rule', + 'model-panel-bugfix': 'rule', + 'model-panel-crud': 'rule', + 'model-panel-review': 'rule', + 'model-panel-changelog': 'rule', + 'model-panel-investigate': 'rule', 'office-hours-section-loading': 'rule', 'office-hours-design-draft': 'rule', 'plan-decision-classification': 'rule', diff --git a/test/model-panel-selftest.test.ts b/test/model-panel-selftest.test.ts new file mode 100644 index 000000000..1464448c5 --- /dev/null +++ b/test/model-panel-selftest.test.ts @@ -0,0 +1,115 @@ +/** + * Model panel selftest (free, no model, no spend): every hidden check fails on + * its seed fixture and passes on its reference answer, gateway arms carry no + * Anthropic credential, and the INFRA classes, cost math and preregistered + * wording rule behave as written, so the paid panel + * (test/skill-e2e-model-panel.test.ts) never spends on a broken instrument. + */ +import { describe, test, expect } from 'bun:test'; +import * as fs from 'fs'; +import { + PANEL_ARMS, PANEL_TASKS, PANEL_TRIALS, PANEL_MIN_COMPLETED, + setupCell, applyReference, runCheck, armEnv, cellCostUsd, classifyCell, summarize, suggestionLine, wilson, usageFromTranscript, + type CellUsage, type PanelRow, type PanelArmId, +} from './helpers/model-panel-harness'; + +const arm = (id: PanelArmId) => PANEL_ARMS.find((a) => a.id === id)!; +const usage = (over: Partial = {}): CellUsage => ({ input: 1000, cached: 0, cacheWrite: 0, output: 100, modelsBilled: [], isError: false, apiErrorStatus: null, turns: 3, ...over }); + +describe('model panel selftest (free, no API)', () => { + for (const task of PANEL_TASKS) { + test(task.key + ': the hidden check fails on the seed and passes on the reference', () => { + expect(fs.existsSync(task.check)).toBe(true); + const seed = setupCell(task, 'selftest-seed'); + const ref = setupCell(task, 'selftest-ref'); + try { + expect(runCheck(task, seed.dir).pass).toBe(false); + applyReference(task, ref.dir); + const passed = runCheck(task, ref.dir); + expect(passed.detail).toBe('PASS'); + expect(passed.pass).toBe(true); + } finally { + fs.rmSync(seed.dir, { recursive: true, force: true }); + fs.rmSync(ref.dir, { recursive: true, force: true }); + } + }); + } + + test('the panel is the preregistered shape: 5 tasks, 4 arms, 3 trials', () => { + expect(PANEL_TASKS.map((t) => t.key)).toEqual(['model-panel-bugfix', 'model-panel-crud', 'model-panel-review', 'model-panel-changelog', 'model-panel-investigate']); + expect(PANEL_ARMS.map((a) => a.model)).toEqual(['claude-opus-5-5', 'claude-sonnet-5-5', 'particle-ai/particle-1', 'deepseek/deepseek-v4.1-flash-1017']); + expect(PANEL_TRIALS).toBe(3); + }); + + test('gateway arms carry no Anthropic credential and pin every alias; Anthropic arms add nothing', () => { + const source = { OPENROUTER_API_KEY: 'sk-or-test', ANTHROPIC_API_KEY: 'sk-ant-real', CLAUDE_CODE_OAUTH_TOKEN: 'oauth-real' }; + const env = armEnv(arm('particle'), source); + expect(env.ANTHROPIC_API_KEY).toBe(''); + expect(env.CLAUDE_CODE_OAUTH_TOKEN).toBe(''); + expect(Object.values(env)).not.toContain('sk-ant-real'); + expect(env.ANTHROPIC_AUTH_TOKEN).toBe('sk-or-test'); + expect(env.ANTHROPIC_BASE_URL).toBe('https://openrouter.ai/api'); + for (const k of ['ANTHROPIC_DEFAULT_HAIKU_MODEL', 'ANTHROPIC_DEFAULT_SONNET_MODEL', 'ANTHROPIC_DEFAULT_OPUS_MODEL', 'CLAUDE_CODE_SUBAGENT_MODEL']) { + expect(env[k]).toBe('particle-ai/particle-1'); + } + expect(armEnv(arm('sonnet'), source)).toEqual({}); + expect(() => armEnv(arm('deepseek'), {})).toThrow('OPENROUTER_API_KEY'); + }); + + test('INFRA classes: startup timeout, missing result, MODEL_MIX, 402/429/5xx and pre-turn errors; the rest is scored', () => { + const p = arm('particle'); + expect(classifyCell(p, 'timeout_startup', null)).toEqual({ verdict: 'infra', reason: 'startup timeout' }); + expect(classifyCell(p, 'error', null).verdict).toBe('infra'); + expect(classifyCell(p, 'success', usage({ modelsBilled: ['particle-ai/particle-1', 'claude-haiku-5-5'] }))).toEqual({ verdict: 'infra', reason: 'MODEL_MIX: claude-haiku-5-5' }); + expect(classifyCell(p, 'success', usage({ modelsBilled: ['particle-1'] }))).toEqual({ verdict: 'scored' }); + for (const status of [402, 429, 503]) expect(classifyCell(p, 'error', usage({ isError: true, apiErrorStatus: status })).verdict).toBe('infra'); + expect(classifyCell(p, 'error', usage({ isError: true, turns: 0 })).verdict).toBe('infra'); + expect(classifyCell(p, 'timeout', usage({ turns: 12 }))).toEqual({ verdict: 'scored' }); + expect(classifyCell(p, 'error_max_turns', usage({ isError: true, turns: 40 }))).toEqual({ verdict: 'scored' }); + }); + + test('cost comes from the price table, with cache writes, never the CLI total', () => { + expect(cellCostUsd(arm('particle'), usage({ input: 1_000_000, cached: 1_000_000, output: 1_000_000 }))).toBeCloseTo(0.25 + 0.03 + 1.0, 6); + expect(cellCostUsd(arm('opus'), usage({ input: 0, output: 0, cacheWrite: 1_000_000 }))).toBeCloseTo(4.0 * 1.25, 6); + expect(cellCostUsd(arm('deepseek'), usage({ input: 1_000_000, output: 0 }))).toBeCloseTo(0.15, 6); + const u = usageFromTranscript([{ type: 'assistant' }, { type: 'result', total_cost_usd: 9.99, num_turns: 4, usage: { input_tokens: 10, output_tokens: 2, cache_read_input_tokens: 5, cache_creation_input_tokens: 1 }, modelUsage: { 'particle-ai/particle-1': {} } }]); + expect(u).toEqual({ input: 10, cached: 5, cacheWrite: 1, output: 2, modelsBilled: ['particle-ai/particle-1'], isError: false, apiErrorStatus: null, turns: 4 }); + expect(usageFromTranscript([{ type: 'assistant' }])).toBeNull(); + }); + + test('wilson intervals', () => { + expect(wilson(0, 0)).toEqual({ low: 0, high: 1 }); + const w = wilson(12, 15); + expect(w.low).toBeCloseTo(0.548, 2); + expect(w.high).toBeCloseTo(0.930, 2); + }); + + const rows = (counts: Record): PanelRow[] => + (Object.entries(counts) as Array<[PanelArmId, [number, number, number]]>).flatMap(([id, [passes, fails, infra]]) => { + const a = arm(id); + const cost = { opus: 0.6, sonnet: 0.3, particle: 0.02, deepseek: 0.03 }[id]; + const make = (verdict: PanelRow['verdict'], i: number): PanelRow => ({ task: 't', arm: id, model: a.model, route: a.route, trial: i, verdict, reason: '', exitReason: 'success', costUsd: verdict === 'infra' ? null : cost, usage: null, durationMs: 1, consulted: true }); + return [...Array(passes).fill('pass'), ...Array(fails).fill('fail'), ...Array(infra).fill('infra')].map((v, i) => make(v, i)); + }); + + test('wording rule: overlap says no measured difference and quotes counts and cost; never "comparable" or "better"', () => { + const s = suggestionLine(summarize(rows({ opus: [14, 1, 0], sonnet: [13, 2, 0], particle: [12, 3, 0], deepseek: [11, 4, 0] })), '2026-10-30'); + expect(s.reason).toBe('ok'); + expect(s.line).toContain('particle-1 passed 12/15'); + expect(s.line).toContain('Sonnet 5.5 13/15'); + expect(s.line).toContain('no measured difference from Sonnet 5.5 on this panel'); + expect(s.line).toContain('the cheapest of the four models measured'); + expect(s.line).not.toMatch(/comparable|better/i); + }); + + test('wording rule: a clear gap is named, the usefulness floor withholds the suggestion, thin arms are INCOMPLETE', () => { + const floor = suggestionLine(summarize(rows({ opus: [15, 0, 0], sonnet: [15, 0, 0], particle: [6, 9, 0], deepseek: [5, 10, 0] })), '2026-10-30'); + expect(floor.line).toBeNull(); + expect(floor.reason).toContain('below the usefulness floor'); + expect(floor.reason).toContain('below Sonnet 5.5 on this panel'); + const thin = suggestionLine(summarize(rows({ opus: [15, 0, 0], sonnet: [15, 0, 0], particle: [8, 2, PANEL_TRIALS * 5 - 10], deepseek: [15, 0, 0] })), '2026-10-30'); + expect(thin.line).toBeNull(); + expect(thin.reason).toStartWith('INCOMPLETE'); + expect(PANEL_MIN_COMPLETED).toBe(12); + }); +}); diff --git a/test/skill-e2e-model-panel.test.ts b/test/skill-e2e-model-panel.test.ts new file mode 100644 index 000000000..560f607e1 --- /dev/null +++ b/test/skill-e2e-model-panel.test.ts @@ -0,0 +1,132 @@ +/** + * Model panel (periodic, paid): particle-1, its open-weight base DeepSeek V4.1 + * Flash, Sonnet 5.5 and Opus 5.5 each run the same five gstack-shaped tasks + * PANEL_TRIALS times in Claude Code. Each cell is scored by a hidden + * deterministic check; gateway arms reach their models through OpenRouter + * with no Anthropic credential in the child env. A research instrument that + * gates nothing: assertions cover the harness, never which model wins. + * + * Spend: every cell reserves its arm's worst case in a shared ledger + * (/spend.json, cap PANEL_SPEND_CAP_USD) before it starts and settles to + * the measured cost after, so a staged run (one task first, then the rest) + * stays under one cap; a refused reservation stops the panel with + * STOP: reason=budget_exhausted and keeps every row already written, and a + * cell that died without usage stays charged at its reservation (unknown). + * Rows append to /rows.jsonl; /summary.json holds the per-arm + * summary and the preregistered suggestion line. is + * GSTACK_MODEL_PANEL_OUT, defaulting beside the eval store. + */ +import { describe, test, expect, afterAll } from 'bun:test'; +import * as fs from 'fs'; +import * as path from 'path'; +import { runSkillTest } from './helpers/session-runner'; +import { runId, selectedTests } from './helpers/e2e-helpers'; +import { describeE2ETier } from './helpers/e2e-gate'; +import { getProjectEvalDir } from './helpers/eval-store'; +import { reserve, settle, readLedger, setCap, summarize as spendSummary, renderSpend, SpendCapExceeded } from '../lib/spend-ledger'; +import { + PANEL_ARMS, PANEL_TASKS, PANEL_TRIALS, PANEL_MAX_TURNS, PANEL_TIMEOUT_MS, PANEL_CONCURRENCY, PANEL_ALLOWED_TOOLS, PANEL_SPEND_CAP_USD, + setupCell, runCheck, armEnv, usageFromTranscript, cellCostUsd, classifyCell, summarize, suggestionLine, + type PanelArm, type PanelRow, type PanelTask, +} from './helpers/model-panel-harness'; +import { SKILL_NAME } from './helpers/arm-benchmark-harness'; + +const OUT = process.env.GSTACK_MODEL_PANEL_OUT || path.join(path.dirname(getProjectEvalDir()), 'model-panel'); +const ROWS = path.join(OUT, 'rows.jsonl'); +const SPEND = path.join(OUT, 'spend.json'); +let stopped: string | null = null; + +function consulted(result: Awaited>): boolean { + return result.toolCalls.some((tc) => tc.tool === 'Skill' && String((tc.input as { skill?: unknown })?.skill ?? '').includes(SKILL_NAME)); +} + +async function runCell(task: PanelTask, arm: PanelArm, trial: number): Promise { + if (stopped) return null; + const attempt = task.key + ':' + arm.id + ':' + trial + ':' + runId; + try { + reserve(SPEND, attempt, arm.reserveUsd, { cap: PANEL_SPEND_CAP_USD, label: arm.model }); + } catch (err) { + if (!(err instanceof SpendCapExceeded)) throw err; + stopped = err.message; + process.stderr.write('STOP: reason=budget_exhausted ' + err.message + '\n'); + return null; + } + const dirs = setupCell(task, arm.id + '-' + trial); + const started = Date.now(); + let settled = false; + try { + const result = await runSkillTest({ + prompt: task.ticket, + workingDirectory: dirs.dir, + maxTurns: PANEL_MAX_TURNS, + allowedTools: PANEL_ALLOWED_TOOLS, + timeout: PANEL_TIMEOUT_MS, + testName: task.key + '-' + arm.id + '-t' + trial, + runId, + model: arm.model, + env: armEnv(arm), + }); + const usage = usageFromTranscript(result.transcript); + const costUsd = usage ? cellCostUsd(arm, usage) : null; + settle(SPEND, attempt, costUsd ?? 'unknown'); + settled = true; + const cls = classifyCell(arm, result.exitReason, usage); + const scored = cls.verdict === 'scored' ? runCheck(task, dirs.dir) : null; + const row: PanelRow = { + task: task.key, arm: arm.id, model: arm.model, route: arm.route, trial, + verdict: scored ? (scored.pass ? 'pass' : 'fail') : 'infra', + reason: scored ? scored.detail : (cls as { reason: string }).reason, + exitReason: result.exitReason, costUsd, usage, durationMs: Date.now() - started, consulted: consulted(result), + }; + fs.appendFileSync(ROWS, JSON.stringify({ runId, ...row }) + '\n'); + console.log('[model-panel] ' + task.key + ' ' + arm.id + ' t' + trial + ' ' + row.verdict + ' $' + (costUsd ?? 0).toFixed(4) + ' turns=' + (usage?.turns ?? 0) + ' ' + row.reason); + return row; + } finally { + if (!settled) settle(SPEND, attempt, 'unknown'); + fs.rmSync(dirs.dir, { recursive: true, force: true }); + } +} + +async function pool(jobs: Array<() => Promise>, limit: number): Promise { + const out: T[] = new Array(jobs.length); + let next = 0; + await Promise.all(Array.from({ length: Math.min(limit, jobs.length) }, async () => { + while (next < jobs.length) { + const i = next++; + out[i] = await jobs[i](); + } + })); + return out; +} + +const describePaid = describeE2ETier('periodic'); + +for (const task of PANEL_TASKS) { + const selected = selectedTests === null || selectedTests.includes(task.key); + (selected ? describePaid : describe.skip)('Model panel: ' + task.key, () => { + test(task.key, async () => { + fs.mkdirSync(OUT, { recursive: true }); + if (readLedger(SPEND).cap_usd === null) setCap(SPEND, PANEL_SPEND_CAP_USD); + const jobs = PANEL_ARMS.flatMap((arm) => Array.from({ length: PANEL_TRIALS }, (_, i) => () => runCell(task, arm, i + 1))); + const rows = (await pool(jobs, PANEL_CONCURRENCY)).filter((r): r is PanelRow => r !== null); + expect(stopped, 'panel stopped: ' + stopped).toBeNull(); + expect(rows).toHaveLength(PANEL_ARMS.length * PANEL_TRIALS); + for (const row of rows) expect(['pass', 'fail', 'infra']).toContain(row.verdict); + }, Math.ceil((PANEL_ARMS.length * PANEL_TRIALS) / PANEL_CONCURRENCY) * PANEL_TIMEOUT_MS + 300_000); + }); +} + +afterAll(() => { + if (!fs.existsSync(ROWS)) return; + const rows = fs.readFileSync(ROWS, 'utf-8').trim().split('\n').filter(Boolean).map((l) => JSON.parse(l) as PanelRow); + const summaries = summarize(rows); + const suggestion = suggestionLine(summaries, new Date().toISOString().slice(0, 10)); + const spend = spendSummary(readLedger(SPEND)); + fs.writeFileSync(path.join(OUT, 'summary.json'), JSON.stringify({ rows: rows.length, summaries, suggestion, spend }, null, 2) + '\n'); + console.log('\n[model-panel] research instrument; gates nothing.'); + for (const s of summaries) { + console.log(' ' + s.label.padEnd(20) + ' ' + s.passes + '/' + s.completed + ' passed (95% ' + (s.low * 100).toFixed(0) + '-' + (s.high * 100).toFixed(0) + '%), infra ' + s.infra + ', $' + s.costPerTaskUsd.toFixed(4) + ' per task'); + } + console.log(' suggestion: ' + (suggestion.line ?? 'none (' + suggestion.reason + ')')); + console.log(' ' + renderSpend(spend)); +});