CalvinBackup/gstack
1
0
Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-09-23 13:20:48 +02:00
Code Issues Packages Projects Releases Wiki Activity

test(pty): pin spawned claude to EVALS model chain (default claude-sonnet-4-6)

Browse Source 
launchClaudePty spawned the interactive `claude` TUI with no --model flag, so
the child inherited the operator's ~/.claude/settings.json model. On a
slow-thinking model that meant 5+ min of extended thinking on empty plan-mode
context, timing out the plan-mode smoke tests regardless of contention. Pin the
model via opts.model ?? EVALS_MODEL ?? 'claude-sonnet-4-6' — byte-identical to
session-runner.ts:144, so PTY and `claude -p` evals always agree.

Pushed before extraArgs (last flag wins, so a per-test --model still overrides).
Placement leaves the spawn region byte-stable for a clean merge with the
in-flight hermetic-env branch. Plumbed model through the three plan-skill
wrappers. Static-grep tripwires guard the pin, its fallback chain, the
before-extraArgs ordering, and all three wrapper forwards.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-06-14 11:49:23 -07:00
co-authored by Claude Opus 4.8
parent 51e9351ed7
commit 6216c3e326
2 changed files with 66 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/helpers/claude-pty-runner.unit.test.ts
+40
View File
@@ -23,6 +23,7 @@
*/
import { describe, test, expect } from 'bun:test';
import { readFileSync } from 'node:fs';
import {
isPermissionDialogVisible,
isNumberedOptionListVisible,
@@ -552,6 +553,45 @@ describe('runPlanSkillObservation env passthrough surface', () => {
});
});
describe('launchClaudePty model pin (static tripwire)', () => {
// Why static-grep, not a behavioral assert: the spawn fires immediately
// inside launchClaudePty, so asserting the built args array would require
// extracting an arg-builder seam — which rewrites the exact region kyoto-v5's
// hermetic --strict-mcp-config insertion edits, reintroducing a merge
// conflict the placement deliberately avoids. The end-to-end behavioral proof
// is the live PTY smoke (skill-e2e-plan-*-plan-mode.test.ts) running under the
// pinned model. These grep-level guards stop a refactor from silently
// dropping the pin or reordering it past extraArgs.
const src = readFileSync(new URL('./claude-pty-runner.ts', import.meta.url), 'utf-8');
test('ClaudePtyOptions exposes model?: string', () => {
const opts: ClaudePtyOptions = { model: 'claude-sonnet-4-6' };
expect(opts.model).toBe('claude-sonnet-4-6');
});
test('spawn args push --model from the EVALS_MODEL fallback chain', () => {
expect(src).toContain("args.push('--model', model)");
// opts.model -> EVALS_MODEL -> 'claude-sonnet-4-6' (mirrors session-runner.ts:144)
expect(src).toMatch(
/opts\.model\s*\?\?\s*process\.env\.EVALS_MODEL\s*\?\?\s*'claude-sonnet-4-6'/,
);
});
test('--model is pushed BEFORE extraArgs so a per-test --model override wins', () => {
const modelPush = src.indexOf("args.push('--model', model)");
const extraArgsPush = src.indexOf('if (opts.extraArgs) args.push(...opts.extraArgs)');
expect(modelPush).toBeGreaterThan(-1);
expect(extraArgsPush).toBeGreaterThan(-1);
expect(modelPush).toBeLessThan(extraArgsPush);
});
test('all three plan-skill wrappers forward model to launchClaudePty', () => {
// Count must match the number of wrappers (observation, counting, floor).
const forwards = src.match(/^\s*model: opts\.model,$/gm) ?? [];
expect(forwards.length).toBe(3);
});
});
// ────────────────────────────────────────────────────────────────────────────
// Per-finding count primitives — Section 3 unit tests #1–#5, #7, #12.
// ────────────────────────────────────────────────────────────────────────────