import { afterEach, describe, expect, test } from 'bun:test'; import { spawnSync } from 'node:child_process'; import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { modelCatalogSha256 } from '../lib/model-catalog'; import { parseModelPolicyConfig, validatePolicyValue, type PolicyConfigKey } from '../lib/model-policy'; import { CLAUDE_CODE_RUNTIME_FILES } from '../lib/claude-code-migration'; const ROOT = path.resolve(import.meta.dir, '..'); const MODELS = path.join(ROOT, 'bin', 'gstack-models'); const CONFIG = path.join(ROOT, 'bin', 'gstack-config'); const roots: string[] = []; afterEach(() => { for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); }); function fixture(config?: string) { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack models cli ')); roots.push(root); const state = path.join(root, 'state'); const home = path.join(root, 'home'); fs.mkdirSync(state); fs.mkdirSync(home); if (config !== undefined) fs.writeFileSync(path.join(state, 'config.yaml'), config); return { root, state, home, env: { PATH: process.env.PATH ?? '', HOME: home, GSTACK_STATE_ROOT: state, BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0' } as Record }; } function models(args: string[], env: Record, cwd = path.dirname(env.HOME)) { const result = spawnSync(process.execPath, [MODELS, ...args], { encoding: 'utf-8', env, cwd, timeout: 30_000 }); return { code: result.status, stdout: result.stdout ?? '', stderr: result.stderr ?? '' }; } function config(args: string[], env: Record) { const result = spawnSync('bash', [CONFIG, ...args], { encoding: 'utf-8', env, timeout: 30_000 }); return { code: result.status, stdout: result.stdout ?? '', stderr: result.stderr ?? '' }; } function json(args: string[], env: Record, cwd?: string) { const result = models([...args, '--json'], env, cwd); expect(result.stderr).toBe(''); return { code: result.code, doc: JSON.parse(result.stdout) }; } function tree(dir: string): string[] { return fs.readdirSync(dir, { recursive: true }).map(String).sort(); } describe('gstack-models inspector', () => { test('--help explains roles, values, offline behavior, delegation and exits', () => { const result = models(['--help'], fixture().env); expect(result.code).toBe(0); for (const text of ['plan-review', 'implementation', 'frontier | smart | host', 'frontier | smart ', 'offline', 'no paid model call', 'host-controlled', 'Exit: 0', '2 invalid usage', 'schemaVersion 1']) { expect(result.stdout).toContain(text); } }); test('list is the default and emits one JSON document with both roles and tier entries', () => { const { env, state } = fixture(); const human = models([], env); expect(human.code).toBe(0); expect(human.stdout).toContain('plan-review via openai: gpt-6-astra'); expect(human.stdout).toContain('implementation via anthropic: claude-opus-5-5'); expect(human.stdout).toContain('gstack-config set plan_review_tier smart'); expect(human.stderr).toBe(''); expect(models(['list'], env).stdout).toBe(human.stdout); const { code, doc } = json([], env); expect(code).toBe(0); expect(doc).toMatchObject({ schemaVersion: 1, command: 'list', ok: true, offline: true, configPath: path.join(state, 'config.yaml'), errors: [] }); expect(doc.catalog).toMatchObject({ version: 1, sha256: modelCatalogSha256() }); expect(doc.selections.map((s: { role: string; provider: string }) => `${s.role}/${s.provider}`)).toEqual(['plan-review/anthropic', 'plan-review/openai', 'implementation/anthropic', 'implementation/openai']); expect(doc.tiers).toHaveLength(4); expect(doc.settings.plan_review_tier).toEqual({ value: null, effective: 'frontier', origin: 'default' }); expect(doc.settings.model_smart_openai).toEqual({ value: null, effective: 'gpt-6.1-sol', origin: 'default' }); }); test('resolve plan-review returns one tagged selection with its source', () => { const { env } = fixture('model_frontier_openai: gpt-pinned\n'); const { code, doc } = json(['resolve', '--role', 'plan-review', '--provider', 'openai'], { ...env, GSTACK_CLAUDE_MODEL: 'claude-ignored' }); expect(code).toBe(0); expect(doc.selections).toHaveLength(1); expect(doc.selections[0]).toMatchObject({ status: 'selected', role: 'plan-review', provider: 'openai', tier: 'frontier', requestedModel: 'gpt-pinned', source: { kind: 'config', key: 'model_frontier_openai' } }); const envWins = json(['resolve', '--role=plan-review', '--provider=openai'], { ...env, GSTACK_CODEX_MODEL: 'gpt-env' }); expect(envWins.doc.selections[0]).toMatchObject({ requestedModel: 'gpt-env', source: { kind: 'env', name: 'GSTACK_CODEX_MODEL' } }); }); test('every tier entry includes executable pin and reset commands', () => { const { env, root } = fixture(); const install = path.join(root, "installed gstack's runtime"); for (const file of CLAUDE_CODE_RUNTIME_FILES) { const target = path.join(install, file); fs.mkdirSync(path.dirname(target), { recursive: true }); fs.copyFileSync(path.join(ROOT, file), target); } const listed = spawnSync(process.execPath, [path.join(install, 'bin/gstack-models'), 'list'], { cwd: root, env, encoding: 'utf-8', timeout: 30_000 }); expect(listed.status).toBe(0); const lines = listed.stdout.split('\n').map(line => line.trim()); const { doc } = json([], env); for (const entry of doc.tiers) { const pin = lines.find(line => line.includes(` set ${entry.overrideKey} `)); const reset = lines.find(line => line.endsWith(` unset ${entry.overrideKey}`)); expect(pin).toBeDefined(); expect(reset).toBeDefined(); for (const command of [pin!, reset!]) { const result = spawnSync('bash', ['-c', command], { cwd: root, env, encoding: 'utf-8', timeout: 30_000 }); expect(result.status).toBe(0); const setting = json([], env).doc.settings[entry.overrideKey]; expect(setting.origin).toBe(command === pin ? 'config' : 'default'); expect(setting.effective).toBe(entry.catalogModel); } } }); test('resolve implementation returns both providers unless one is named', () => { const { env } = fixture('implementation_tier: frontier\n'); const both = json(['resolve', '--role', 'implementation'], env); expect(both.code).toBe(0); expect(both.doc.selections.map((s: { provider: string; requestedModel: string }) => [s.provider, s.requestedModel])).toEqual([['anthropic', 'claude-fable-5-1'], ['openai', 'gpt-6-astra']]); const one = json(['resolve', '--role', 'implementation', '--provider', 'openai'], env); expect(one.doc.selections).toHaveLength(1); expect(one.doc.selections[0].provider).toBe('openai'); }); test('host mode is a valid delegated resolution with a null model', () => { const { env } = fixture('plan_review_tier: host\n'); const { code, doc } = json(['resolve', '--role', 'plan-review', '--provider', 'anthropic'], env); expect(code).toBe(0); expect(doc.selections[0]).toMatchObject({ status: 'delegated-host', requestedModel: null, source: { kind: 'host', reason: 'host-mode' } }); const human = models(['resolve', '--role', 'plan-review', '--provider', 'anthropic'], env); expect(human.code).toBe(0); expect(human.stdout).toContain('host-controlled'); }); test.each([ [['bogus'], "unknown command 'bogus'"], [['resolve'], 'resolve needs --role'], [['resolve', '--role', 'coding'], "unknown role 'coding'"], [['resolve', '--role', 'plan-review'], 'needs --provider'], [['resolve', '--role', 'plan-review', '--provider', 'claude'], "unknown provider 'claude'"], [['resolve', '--role', 'implementation', '--provider', 'OpenAI'], "unknown provider 'OpenAI'"], [['resolve', '--role', 'plan-review', '--role', 'implementation', '--provider', 'openai'], '--role given twice'], [['resolve', '--role'], '--role needs a value'], [['list', '--provider', 'openai'], 'unknown argument'], ])('usage error %p exits 2', (args, message) => { const { env } = fixture(); const human = models(args, env); expect(human.code).toBe(2); expect(human.stdout).toBe(''); expect(human.stderr).toContain(message); const { code, doc } = json(args, env); expect(code).toBe(2); expect(doc).toMatchObject({ schemaVersion: 1, ok: false, selections: [] }); expect(doc.errors[0]).toMatchObject({ reason: 'usage' }); expect(doc.errors[0].problem).toContain(message); }); test('invalid config exits 1 with structured, source-aware repair', () => { const { env } = fixture('plan_review_tier: fronteir\n'); const { code, doc } = json([], env); expect(code).toBe(1); expect(doc).toMatchObject({ ok: false, selections: [], settings: null }); expect(doc.errors[0]).toMatchObject({ reason: 'invalid_plan_review_tier', key: 'plan_review_tier' }); expect(doc.errors[0].repair).toContain('gstack-config unset plan_review_tier'); expect(doc.errors[0].docs).toContain('#model-policy-config'); const human = models(['resolve', '--role', 'implementation'], env); expect(human.code).toBe(1); expect(human.stdout).toBe(''); expect(human.stderr).toContain('fix: gstack-config unset plan_review_tier'); }); test('an unusable environment override points at the env var', () => { const { env } = fixture(); const { code, doc } = json(['resolve', '--role', 'plan-review', '--provider', 'openai'], { ...env, GSTACK_CODEX_MODEL: 'gpt 6' }); expect(code).toBe(1); expect(doc.errors[0]).toMatchObject({ reason: 'invalid_model_id', key: 'GSTACK_CODEX_MODEL', role: 'plan-review', provider: 'openai' }); expect(doc.errors[0].repair[0]).toBe('unset GSTACK_CODEX_MODEL'); }); test('custom routing fails only the affected selections', () => { const { env } = fixture(); const { code, doc } = json([], { ...env, CLAUDE_CODE_USE_BEDROCK: '1' }); expect(code).toBe(1); expect(doc.selections.map((s: { role: string; provider: string }) => `${s.role}/${s.provider}`)).toEqual(['plan-review/openai', 'implementation/openai']); expect(doc.errors.map((e: { reason: string; role: string }) => `${e.reason}/${e.role}`)).toEqual(['custom_provider_requires_model/plan-review', 'custom_provider_requires_model/implementation']); }); test('project settings in the working directory are custom routing; detection limits are published', () => { const { env, root } = fixture(); const project = path.join(root, 'project'); fs.mkdirSync(path.join(project, '.claude'), { recursive: true }); fs.writeFileSync(path.join(project, '.claude', 'settings.local.json'), JSON.stringify({ env: { ANTHROPIC_BASE_URL: 'https://proxy.example.test' } })); const { code, doc } = json(['resolve', '--role', 'plan-review', '--provider', 'anthropic'], env, project); expect(code).toBe(1); expect(doc.errors[0]).toMatchObject({ reason: 'custom_provider_requires_model', provider: 'anthropic' }); expect(doc.providerDetection.unscanned.anthropic.join(' ')).toContain('server-managed'); expect(json(['resolve', '--role', 'plan-review', '--provider', 'anthropic'], env).code).toBe(0); expect(models(['--help'], env).stdout).toContain('cannot see server-managed or MDM policy'); }); test.each(['[]', 'null', '"unsupported"', 'true', '42'])('unsupported native settings root %s fails closed without writes', (contents) => { const { root, home, env } = fixture(); const directory = path.join(home, '.claude'); fs.mkdirSync(directory); const settings = path.join(directory, 'settings.json'); fs.writeFileSync(settings, '{}'); const args = ['resolve', '--role', 'plan-review', '--provider', 'anthropic']; expect(json(args, env).code).toBe(0); fs.writeFileSync(settings, contents); const before = tree(root); const result = json(args, env); expect(result.code).toBe(1); expect(result.doc.selections).toEqual([]); expect(result.doc.errors[0]).toMatchObject({ reason: 'native_provider_unresolved', key: settings }); expect(result.doc.errors[0].repair).toContain(`fix the settings object in ${settings}`); expect(tree(root)).toEqual(before); expect(json(args, { ...env, GSTACK_CLAUDE_MODEL: 'claude-explicit' }).doc.selections[0].requestedModel).toBe('claude-explicit'); }); test('inspection never writes to the state root or home', () => { const { root, env } = fixture('plan_review_tier: smart\n'); const before = tree(root); models([], env); models(['--json'], env); models(['resolve', '--role', 'plan-review', '--provider', 'anthropic', '--json'], env); models(['resolve', '--role', 'implementation'], env); expect(tree(root)).toEqual(before); }); }); describe('gstack-config model policy keys', () => { test('the annotated header has valid uncommentable tier settings and no empty pin examples', () => { const { env, state } = fixture(); expect(config(['set', 'telemetry', 'off'], env).code).toBe(0); const text = fs.readFileSync(path.join(state, 'config.yaml'), 'utf8'); const tiers = text.split('\n').filter(line => /^# (plan_review_tier|implementation_tier):/.test(line)).map(line => line.slice(2)); expect(tiers).toHaveLength(2); const parsed = parseModelPolicyConfig(tiers.join('\n'), '/example/config.yaml'); expect(parsed.planReviewTier.value).toBe('frontier'); expect(parsed.implementationTier.value).toBe('smart'); expect(text).not.toMatch(/^# model_(frontier|smart)_(claude|openai):\s*(?:#.*)?$/m); }); test('Anthropic pins preserve the 512-character contract with a POSIX-bounded regex engine', () => { const { root, env, state } = fixture(); const bin = path.join(root, 'portable-bin'); fs.mkdirSync(bin); fs.writeFileSync(path.join(bin, 'grep'), `#!/usr/bin/env bun import { spawnSync } from 'node:child_process'; const args = process.argv.slice(2); if (args.some(arg => [...arg.matchAll(/\\{\\d+,(\\d+)\\}/g)].some(match => Number(match[1]) > 255))) { console.error('REG_BADMAX: repetition exceeds the BSD RE_DUP_MAX of 255'); process.exit(2); } const result = spawnSync(${JSON.stringify(Bun.which('grep'))}, args, { stdio: 'inherit', timeout: 5000 }); process.exit(result.status ?? 1); `, { mode: 0o755 }); const portable = { ...env, PATH: `${bin}${path.delimiter}${env.PATH}` }; for (const key of ['model_frontier_claude', 'model_smart_claude']) { for (const value of ['claude-fable-5-1', 'a'.repeat(512)]) { expect(config(['set', key, value], portable).code).toBe(0); expect(config(['get', key], portable).stdout).toBe(value); } const before = fs.readFileSync(path.join(state, 'config.yaml'), 'utf-8'); expect(config(['set', key, 'a'.repeat(513)], portable).code).toBe(1); expect(fs.readFileSync(path.join(state, 'config.yaml'), 'utf-8')).toBe(before); } }); test('defaults, set, resolve again and unset restore', () => { const { env } = fixture(); expect(config(['get', 'plan_review_tier'], env)).toMatchObject({ code: 0, stdout: 'frontier' }); expect(config(['get', 'implementation_tier'], env)).toMatchObject({ code: 0, stdout: 'smart' }); expect(config(['get', 'model_frontier_claude'], env)).toMatchObject({ code: 0, stdout: '' }); expect(config(['defaults'], env).stdout).toContain('plan_review_tier:'); expect(config(['set', 'plan_review_tier', 'smart'], env).code).toBe(0); expect(config(['set', 'model_smart_openai', ' gpt-custom '], env).code).toBe(0); expect(config(['get', 'model_smart_openai'], env).stdout).toBe('gpt-custom'); expect(json(['resolve', '--role', 'plan-review', '--provider', 'openai'], env).doc.selections[0]).toMatchObject({ tier: 'smart', requestedModel: 'gpt-custom', source: { kind: 'config' } }); expect(config(['list'], env).stdout).toContain('model_smart_openai: gpt-custom (set)'); expect(config(['unset', 'plan_review_tier'], env).code).toBe(0); expect(config(['unset', 'model_smart_openai'], env).code).toBe(0); expect(json(['resolve', '--role', 'plan-review', '--provider', 'openai'], env).doc.selections[0]).toMatchObject({ tier: 'frontier', requestedModel: 'gpt-6-astra', source: { kind: 'catalog' } }); }); test.each([ ['plan_review_tier', 'fronteir'], ['plan_review_tier', ' '], ['implementation_tier', 'host'], ['model_frontier_openai', 'gpt 6'], ['model_frontier_claude', 'a'.repeat(513)], ])('invalid %s %p is rejected and the old value kept', (key, value) => { const { env, state } = fixture(); const good = key.startsWith('model_') ? 'model-1' : 'frontier'; expect(config(['set', key, good], env).code).toBe(0); const before = fs.readFileSync(path.join(state, 'config.yaml'), 'utf-8'); const result = config(['set', key, value], env); expect(result.code).toBe(1); expect(result.stderr).toContain('Existing value left unchanged'); expect(result.stderr).toContain(`gstack-config unset ${key}`); expect(result.stderr).toContain('#model-policy-config'); expect(fs.readFileSync(path.join(state, 'config.yaml'), 'utf-8')).toBe(before); }); test('set and resolve share one validation rule', () => { const samples: Array<[PolicyConfigKey, string]> = [ ['plan_review_tier', 'frontier'], ['plan_review_tier', 'smart'], ['plan_review_tier', 'host'], ['plan_review_tier', 'Smart'], ['implementation_tier', 'frontier'], ['implementation_tier', 'smart'], ['implementation_tier', 'host'], ['model_frontier_claude', 'claude-opus-4-6[1m]'], ['model_frontier_claude', 'vendor+claude@2026'], ['model_frontier_claude', 'a'.repeat(512)], ['model_frontier_claude', 'a'.repeat(513)], ['model_smart_claude', 'claude;x'], ['model_smart_claude', 'claude x'], ['model_smart_claude', '$(id)'], ['model_frontier_openai', 'gpt-6.1-sol'], ['model_frontier_openai', 'org/model:v1'], ['model_frontier_openai', 'a'.repeat(100)], ['model_frontier_openai', 'a'.repeat(101)], ['model_smart_openai', 'gpt+6'], ['model_smart_openai', 'gpt@6'], ]; const { env } = fixture(); for (const [key, value] of samples) { const accepted = config(['set', key, value], env).code === 0; expect({ key, value: value.slice(0, 20), accepted }).toEqual({ key, value: value.slice(0, 20), accepted: validatePolicyValue(key, value) === null }); } }); });