CalvinBackup/gstack
1
0
Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-09-14 00:49:00 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix: default cross-model workflows to frontier models

Browse Source
This commit is contained in:
Garry Tan
2026-09-09 02:48:34 +00:00
parent 0530392821
commit 939da0c203
69 changed files with 603 additions and 255 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/helpers/agent-sdk-runner.ts
+4 -4
View File
@@ -36,6 +36,7 @@ import {
import * as fs from 'fs';
import * as path from 'path';
import { resolveClaudeBinary as resolveClaudeBinaryShared } from '../../lib/claude-bin';
import { resolveEvalModel } from '../../lib/eval-model';
import { hermeticChildEnv } from './hermetic-env';
import type { SkillTestResult } from './session-runner';
@@ -299,10 +300,9 @@ export async function runAgentSdkTest(
const sem = getApiSemaphore();
const maxRetries = opts.maxRetries ?? 3;
const queryImpl: QueryProvider = opts.queryProvider ?? query;
// Default matches session-runner's Sonnet (D1a, 2026-08): the old Opus
// default was an inconsistency between the two runners, not a choice —
// tests that need Opus pin it via opts.model (30+ already do).
const model = opts.model ?? 'claude-sonnet-4-6';
// Default matches session-runner's frontier eval fallback. Tests that need a
// cheaper or historical model pin it via opts.model or EVALS_MODEL.
const model = opts.model ?? process.env.EVALS_MODEL ?? resolveEvalModel('capture');
// NOTE on env: the SDK child gets the COMPLETE hermetic env (allowlist
// scrub + ANTHROPIC_API_KEY + hermetic CLAUDE_CONFIG_DIR/GSTACK_HOME), with
test/helpers/benchmark-judge.ts
+3 -2
View File
@@ -1,7 +1,7 @@
/**
* Benchmark quality judge — wraps llm-judge.ts for multi-provider scoring.
*
* The judge is always Anthropic SDK (claude-sonnet-4-6) for stability. It sees
* The judge uses the shared frontier Claude eval default. It sees
* the prompt + N provider outputs and scores each on: correctness, completeness,
* code quality, edge case handling. 0-10 per dimension; overall = average.
*
@@ -9,6 +9,7 @@
*/
import type { BenchmarkReport, BenchmarkEntry } from './benchmark-runner';
import { resolveEvalModel } from '../../lib/eval-model';
export async function judgeEntries(report: BenchmarkReport): Promise<void> {
if (!process.env.ANTHROPIC_API_KEY) {
@@ -26,7 +27,7 @@ export async function judgeEntries(report: BenchmarkReport): Promise<void> {
const judgePrompt = buildJudgePrompt(report.prompt, successful);
const msg = await client.messages.create({
model: 'claude-sonnet-4-6',
model: resolveEvalModel('judge'),
max_tokens: 2048,
messages: [{ role: 'user', content: judgePrompt }],
});
test/helpers/claude-pty-runner.ts
+5 -6
View File
@@ -80,9 +80,8 @@ export interface ClaudePtyOptions {
/**
* Model for the spawned interactive `claude`. Without an explicit --model the
* child inherits the operator's ~/.claude/settings.json model (e.g.
* claude-fable-5[1m]), which can spend 5+ min in extended thinking on an empty
* plan-mode context and blow every smoke budget. Resolution mirrors
* session-runner.ts:144 exactly: opts.model ?? EVALS_MODEL ?? 'claude-sonnet-4-6'.
* the operator's own settings. Resolution mirrors session-runner.ts exactly:
* opts.model ?? EVALS_MODEL ?? resolveEvalModel('capture').
* Pushed BEFORE extraArgs so a test-supplied --model still wins (last flag wins).
*/
model?: string;
@@ -1306,10 +1305,10 @@ export async function launchClaudePty(
const args: string[] = [];
// Pin the model so smokes don't inherit the operator's settings.json model
// (see ClaudePtyOptions.model). Chain mirrors session-runner.ts:144 so PTY and
// (see ClaudePtyOptions.model). Chain mirrors session-runner.ts so PTY and
// `claude -p` evals always agree. Pushed before extraArgs => a test-supplied
// --model wins (last flag wins).
const model = opts.model ?? process.env.EVALS_MODEL ?? 'claude-sonnet-4-6';
const model = opts.model ?? process.env.EVALS_MODEL ?? resolveEvalModel('capture');
args.push('--model', model);
// Permission mode: 'plan' default, null => omit flag entirely.
const permissionMode = opts.permissionMode === undefined ? 'plan' : opts.permissionMode;
@@ -1699,7 +1698,7 @@ export async function runPlanSkillObservation(opts: {
*/
initialPlanContent?: string;
/** Override the spawned model. Defaults via launchClaudePty's chain
* (opts.model ?? EVALS_MODEL ?? 'claude-sonnet-4-6'). */
* (opts.model ?? EVALS_MODEL ?? resolveEvalModel('capture')). */
model?: string;
/** Literal tokens to track as high-water marks over the CUMULATIVE visible
* buffer (case-sensitive). Results land in obs.tokensObserved. Use for
test/helpers/claude-pty-runner.unit.test.ts
+2 -2
View File
@@ -783,9 +783,9 @@ describe('launchClaudePty model pin (static tripwire)', () => {
test('spawn args push --model from the EVALS_MODEL fallback chain', () => {
expect(src).toContain("args.push('--model', model)");
// opts.model -> EVALS_MODEL -> 'claude-sonnet-4-6' (mirrors session-runner.ts:144)
// opts.model -> EVALS_MODEL -> resolveEvalModel('capture') (mirrors session-runner.ts)
expect(src).toMatch(
/opts\.model\s*\?\?\s*process\.env\.EVALS_MODEL\s*\?\?\s*'claude-sonnet-4-6'/,
/opts\.model\s*\?\?\s*process\.env\.EVALS_MODEL\s*\?\?\s*resolveEvalModel\('capture'\)/,
);
});
test/helpers/codex-session-runner.ts
+2 -1
View File
@@ -20,6 +20,7 @@ import { Readable } from 'node:stream';
import { hermeticChildEnv } from './hermetic-env';
import { extractSkillSections } from './skill-fixture';
import { killProcessGroup } from '../../scripts/test-strict-output';
import { CODEX_FRONTIER_MODEL } from '../../scripts/resolvers/constants';
// --- Interfaces ---
@@ -227,7 +228,7 @@ export async function runCodexSkill(opts: {
// exactly that. Empirically verified against codex on this machine.
const args = ['exec', '--json', '-s', sandbox, '--skip-git-repo-check'];
if (ignoreUserConfig) args.push('--ignore-user-config');
if (model) args.push('--model', model);
args.push('--model', model ?? process.env.GSTACK_CODEX_MODEL ?? CODEX_FRONTIER_MODEL);
for (const override of configOverrides) args.push('-c', override);
args.push(prompt);
test/helpers/llm-judge.ts
+13 -16
View File
@@ -11,7 +11,7 @@
import Anthropic from '@anthropic-ai/sdk';
import { resolveEvalModel } from '../../lib/eval-model';
import { CLAUDE_FRONTIER_EVAL_MODEL, resolveEvalModel } from '../../lib/eval-model';
export interface JudgeScore {
clarity: number; // 1-5
@@ -55,27 +55,21 @@ export interface RecommendationScore {
/**
* Call an Anthropic model with a prompt, extract JSON response.
* Jittered exponential backoff over three 429 retries. Model resolves via
* lib/eval-model's `judge` kind (Sonnet default); pass a model id
* lib/eval-model's `judge` kind (frontier Claude default); pass a model id
* (e.g. claude-haiku-4-5-20251001) for cheaper bounded judgments like
* judgeRecommendation.
*/
// Default judge model: Sonnet. D1a tried Haiku 4.5 here and the first live
// run regressed the doc-rubric family — a controlled A/B on the identical
// health-rubric prompt scored 2/2/2 under Haiku vs 4/3/4 under Sonnet (both
// with coherent reasoning; Haiku is simply a harsher grader on long-document
// rubrics, and every >=4 threshold in skill-llm-eval was calibrated against
// months of Sonnet baselines). Per D1a's pin-on-regressors protocol the
// default stays Sonnet; recalibrating the 25 rubrics for Haiku is separately
// scoped work. Override per run with GSTACK_EVAL_MODEL_JUDGE; Haiku remains
// the right default for classifier-grade duties (pty hung/working, warmup,
// distill — see lib/eval-model.ts).
// Default judge model: the current frontier Claude eval model. Override per run
// with GSTACK_EVAL_MODEL_JUDGE; Haiku remains the right default for
// classifier-grade duties (pty hung/working, warmup, distill — see
// lib/eval-model.ts).
export async function callJudge<T>(
prompt: string,
model?: string,
opts?: { temperature?: number; max_tokens?: number },
): Promise<T> {
// Routed through the documented single resolution point: explicit arg >
// GSTACK_EVAL_MODEL_JUDGE > GSTACK_EVAL_MODEL > sonnet default. The old
// GSTACK_EVAL_MODEL_JUDGE > GSTACK_EVAL_MODEL > frontier default. The old
// inline `GSTACK_EVAL_MODEL_JUDGE || sonnet` silently ignored the global
// GSTACK_EVAL_MODEL override that every other eval call site honors.
// opts (temperature/max_tokens) exist for bounded judgments like armJudge;
@@ -110,7 +104,10 @@ export async function callJudge<T>(
}
}
const text = response.content[0].type === 'text' ? response.content[0].text : '';
const text = response.content
.filter(block => block.type === 'text')
.map(block => block.text)
.join('\n');
const jsonMatch = text.match(/\{[\s\S]*\}/);
if (!jsonMatch) throw new Error(`Judge returned non-JSON: ${text.slice(0, 200)}`);
return JSON.parse(jsonMatch[0]) as T;
@@ -369,7 +366,7 @@ export interface ArmJudgeScore {
* point of a research instrument; a per-run judge swap silently moves the
* ruler.
*/
export const ARM_JUDGE_MODEL = 'claude-sonnet-4-6';
export const ARM_JUDGE_MODEL = CLAUDE_FRONTIER_EVAL_MODEL;
/** Bounded retry-on-malformed loop: total attempts, not extra retries. */
export const ARM_JUDGE_ATTEMPTS = 2;
@@ -468,7 +465,7 @@ export async function armJudge(
let lastError: unknown;
for (let attempt = 1; attempt <= ARM_JUDGE_ATTEMPTS; attempt++) {
try {
const raw = await call<Record<string, unknown>>(prompt, ARM_JUDGE_MODEL, { temperature: 0 });
const raw = await call<Record<string, unknown>>(prompt, ARM_JUDGE_MODEL);
return parseArmJudgeResponse(raw);
} catch (err) {
lastError = err;
test/helpers/providers/claude.ts
+10 -8
View File
@@ -5,6 +5,7 @@ import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import { resolveClaudeCommand } from '../../../lib/claude-bin';
import { resolveEvalModel } from '../../../lib/eval-model';
/**
* Claude adapter — wraps the `claude` CLI via claude -p.
@@ -60,8 +61,9 @@ export class ClaudeAdapter implements ProviderAdapter {
if (!resolved) {
throw new Error('claude CLI not resolvable (set GSTACK_CLAUDE_BIN or install)');
}
const model = opts.model ?? process.env.EVALS_MODEL ?? resolveEvalModel('capture');
const args = [...resolved.argsPrefix, '-p', '--output-format', 'json'];
if (opts.model) args.push('--model', opts.model);
args.push('--model', model);
if (opts.extraArgs) args.push(...opts.extraArgs);
try {
@@ -81,27 +83,27 @@ export class ClaudeAdapter implements ProviderAdapter {
tokens: parsed.tokens,
durationMs: Date.now() - start,
toolCalls: parsed.toolCalls,
modelUsed: parsed.modelUsed || opts.model || 'claude-opus-4-7',
modelUsed: parsed.modelUsed || model,
};
} catch (err: unknown) {
const durationMs = Date.now() - start;
const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
const stderr = e.stderr?.toString() ?? '';
if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') {
return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model);
return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, model);
}
if (/unauthorized|auth|login/i.test(stderr)) {
return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model);
return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, model);
}
if (/rate[- ]?limit|429/i.test(stderr)) {
return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model);
return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, model);
}
return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model);
return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, model);
}
}
estimateCost(tokens: { input: number; output: number; cached?: number }, model?: string): number {
return estimateCostUsd(tokens, model ?? 'claude-opus-4-7');
return estimateCostUsd(tokens, model ?? resolveEvalModel('capture'));
}
/**
@@ -137,7 +139,7 @@ export class ClaudeAdapter implements ProviderAdapter {
tokens: { input: 0, output: 0 },
durationMs,
toolCalls: 0,
modelUsed: model ?? 'claude-opus-4-7',
modelUsed: model ?? resolveEvalModel('capture'),
error,
};
}
test/helpers/providers/gpt.ts
+10 -9
View File
@@ -4,6 +4,7 @@ import { execFileSync, spawnSync } from 'child_process';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import { CODEX_FRONTIER_MODEL } from '../../../scripts/resolvers/constants';
/**
* GPT adapter — wraps the OpenAI `codex` CLI (codex exec with --json output).
@@ -36,8 +37,8 @@ export class GptAdapter implements ProviderAdapter {
// often run in temp dirs / non-git paths), so the read-only sandbox is now
// the only boundary preventing codex from mutating the workdir. If you ever
// remove `-s read-only`, drop `--skip-git-repo-check` too.
const args = ['exec', opts.prompt, '-C', opts.workdir, '-s', 'read-only', '--skip-git-repo-check', '--json'];
if (opts.model) args.push('-m', opts.model);
const model = opts.model ?? process.env.GSTACK_CODEX_MODEL ?? CODEX_FRONTIER_MODEL;
const args = ['exec', opts.prompt, '-C', opts.workdir, '-s', 'read-only', '--skip-git-repo-check', '--json', '-m', model];
if (opts.extraArgs) args.push(...opts.extraArgs);
try {
@@ -53,27 +54,27 @@ export class GptAdapter implements ProviderAdapter {
tokens: parsed.tokens,
durationMs: Date.now() - start,
toolCalls: parsed.toolCalls,
modelUsed: parsed.modelUsed || opts.model || 'gpt-5.4',
modelUsed: parsed.modelUsed || model,
};
} catch (err: unknown) {
const durationMs = Date.now() - start;
const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
const stderr = e.stderr?.toString() ?? '';
if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') {
return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model);
return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, model);
}
if (/unauthorized|auth|login/i.test(stderr)) {
return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model);
return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, model);
}
if (/rate[- ]?limit|429/i.test(stderr)) {
return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model);
return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, model);
}
return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model);
return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, model);
}
}
estimateCost(tokens: { input: number; output: number; cached?: number }, model?: string): number {
return estimateCostUsd(tokens, model ?? 'gpt-5.4');
return estimateCostUsd(tokens, model ?? CODEX_FRONTIER_MODEL);
}
/**
@@ -120,7 +121,7 @@ export class GptAdapter implements ProviderAdapter {
tokens: { input: 0, output: 0 },
durationMs,
toolCalls: 0,
modelUsed: model ?? 'gpt-5.4',
modelUsed: model ?? CODEX_FRONTIER_MODEL,
error,
};
}
test/helpers/session-runner.ts
+3 -2
View File
@@ -14,6 +14,7 @@ import { Readable } from 'node:stream';
import { getProjectEvalDir } from './eval-store';
import { hermeticChildEnv, isHermeticEnabled } from './hermetic-env';
import { killProcessGroup } from '../../scripts/test-strict-output';
import { resolveEvalModel } from '../../lib/eval-model';
const GSTACK_DEV_DIR = path.join(os.homedir(), '.gstack-dev');
const HEARTBEAT_PATH = path.join(GSTACK_DEV_DIR, 'e2e-live.json'); // heartbeat stays global
@@ -136,7 +137,7 @@ export async function runSkillTest(options: {
timeout?: number;
testName?: string;
runId?: string;
/** Model to use. Defaults to claude-sonnet-4-6 (overridable via EVALS_MODEL env). */
/** Model to use. Defaults to the frontier eval model (overridable via EVALS_MODEL env). */
model?: string;
/** Extra env vars merged into the spawned claude -p process. Useful for
* per-test GSTACK_HOME overrides so the test doesn't have to spell out
@@ -171,7 +172,7 @@ export async function runSkillTest(options: {
process.env.CI ? Math.max(requestedGrace, STARTUP_GRACE_CI_FLOOR_MS) : requestedGrace,
timeout,
);
const model = options.model ?? process.env.EVALS_MODEL ?? 'claude-sonnet-4-6';
const model = options.model ?? process.env.EVALS_MODEL ?? resolveEvalModel('capture');
const startTime = Date.now();
const startedAt = new Date().toISOString();