mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-11 03:00:26 +01:00
feat: add configurable model tiers and advisory freshness
This commit is contained in:
garrytan committed
1 parent
db745675bd
commit
5ffc050659
78 files changed
+5106
-325
No files matched your search
@@ -2,7 +2,8 @@ import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { migrateClaudeCodeSkills } from '../lib/claude-code-migration';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { CLAUDE_CODE_RUNTIME_FILES as runtimeFiles, migrateClaudeCodeSkills } from '../lib/claude-code-migration';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const banner = '<!-- AUTO-GENERATED from SKILL.md.tmpl — do not edit directly -->\n<!-- Regenerate: bun run gen:skill-docs -->';
|
||||
@@ -15,7 +16,7 @@ function fixture() {
|
||||
const root = path.join(dir, 'checkout'); const home = path.join(dir, 'home');
|
||||
const codex = path.join(home, 'custom-codex', 'skills');
|
||||
const kiro = path.join(home, '.kiro', 'skills');
|
||||
for (const rel of ['bin/gstack-claude-code', 'lib/claude-code.ts', 'lib/claude-code-windows-job.ts', 'lib/claude-bin.ts', 'lib/outside-review-result.ts']) put(path.join(root, rel), rel);
|
||||
for (const rel of runtimeFiles) put(path.join(root, rel), rel);
|
||||
const oldRender = path.join(root, '.agents', 'skills', 'gstack-claude');
|
||||
put(path.join(oldRender, 'SKILL.md'), skill('gstack-claude'));
|
||||
fs.mkdirSync(codex, { recursive: true }); fs.mkdirSync(kiro, { recursive: true });
|
||||
@@ -40,6 +41,44 @@ function fixture() {
|
||||
}
|
||||
|
||||
describe('Claude wrapper installed-name migration', () => {
|
||||
test('copied policy tools and wrapper load without the source checkout', () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
for (const rel of runtimeFiles) fs.copyFileSync(path.join(ROOT, rel), path.join(f.root, rel));
|
||||
put(path.join(f.codex, 'gstack-claude', 'SKILL.md'), skill('gstack-claude'));
|
||||
expect(f.run({ copy: true })).toEqual({ migrated: 1, pending: [] });
|
||||
fs.renameSync(f.root, `${f.root}-unavailable`);
|
||||
const runtime = path.join(f.codex, 'gstack');
|
||||
const env = {
|
||||
...process.env, HOME: f.home, USERPROFILE: f.home,
|
||||
GSTACK_STATE_ROOT: path.join(f.home, 'state'), CODEX_HOME: path.dirname(f.codex),
|
||||
CLAUDE_CONFIG_DIR: path.join(f.home, '.claude'),
|
||||
GSTACK_CLAUDE_MODEL: '', GSTACK_CODEX_MODEL: '', ANTHROPIC_BASE_URL: '', OPENAI_BASE_URL: '',
|
||||
CLAUDE_CODE_USE_BEDROCK: '', CLAUDE_CODE_USE_VERTEX: '', CLAUDE_CODE_USE_FOUNDRY: '',
|
||||
BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0',
|
||||
};
|
||||
const set = spawnSync('bash', [path.join(runtime, 'bin/gstack-config'), 'set', 'plan_review_tier', 'smart'], {
|
||||
cwd: f.dir, env, encoding: 'utf8', timeout: 30_000,
|
||||
});
|
||||
expect(set.status).toBe(0);
|
||||
const inspect = spawnSync(process.execPath, [path.join(runtime, 'bin/gstack-models'), 'resolve', '--role', 'plan-review', '--provider', 'anthropic', '--json'], {
|
||||
cwd: f.dir, env, encoding: 'utf8', timeout: 30_000,
|
||||
});
|
||||
expect(inspect.status).toBe(0);
|
||||
expect(JSON.parse(inspect.stdout).selections[0].requestedModel).toBe('claude-opus-5-5');
|
||||
const wrapper = spawnSync(process.execPath, [path.join(runtime, 'bin/gstack-claude-code'), '--help'], {
|
||||
cwd: f.dir, env, encoding: 'utf8', timeout: 30_000,
|
||||
});
|
||||
expect(wrapper.status).toBe(1);
|
||||
expect(JSON.parse(wrapper.stdout).error.code).toBe('arguments');
|
||||
const gate = spawnSync(process.execPath, ['-e', 'const {classifyOutsideReview} = await import(process.argv[1]); console.log(classifyOutsideReview({text:"",gate:"review",exit:1}).verdict)', path.join(runtime, 'lib/outside-review-result.ts')], {
|
||||
cwd: f.dir, env, encoding: 'utf8', timeout: 30_000,
|
||||
});
|
||||
expect(gate.status).toBe(0);
|
||||
expect(gate.stdout.trim()).toBe('unavailable');
|
||||
} finally { fs.rmSync(f.dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('migrates existing Codex and Kiro installs independently of the selected setup host', () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
|
||||
@@ -937,15 +937,17 @@ describe('codex broken-install detection (#2742)', () => {
|
||||
// hand-maintained copy that isn't resolver-generated — it must capture the
|
||||
// probe's exit code and route 2 to its own broken-install arm, or /autoplan
|
||||
// prints the wrong remedy for a broken binary.
|
||||
test('autoplan preflight (tmpl + rendered) captures the probe exit and routes 2 to broken-install', () => {
|
||||
for (const rel of ['autoplan/SKILL.md.tmpl', 'autoplan/SKILL.md']) {
|
||||
const raw = fs.readFileSync(path.join(ROOT, rel), 'utf-8');
|
||||
const src = rel.endsWith('.tmpl') ? raw.replace('{{OUTSIDE_PREFLIGHT:autoplan}}', RESOLVERS.OUTSIDE_PREFLIGHT({ host: 'claude', paths: HOST_PATHS.claude, skillName: 'autoplan', tmplPath: rel }, ['autoplan'])) : raw;
|
||||
expect(src).toContain('_gstack_codex_model_probe; _CODEX_MP=$?');
|
||||
expect(src).toMatch(/_CODEX_MP" -eq 2/);
|
||||
expect(src).toContain('binary cannot run');
|
||||
expect(src).not.toContain('elif ! _gstack_codex_model_probe');
|
||||
test('autoplan (tmpl + rendered): availability pays for no probe; the invocation probe exit, 2 included, ends it', () => {
|
||||
const ctx = { host: 'claude' as const, paths: HOST_PATHS.claude, skillName: 'autoplan', tmplPath: 'autoplan/SKILL.md.tmpl' };
|
||||
const rendered = ['autoplan/SKILL.md', ...['ceo', 'design', 'dx', 'eng'].map(phase => `autoplan/sections/${phase}-phase.md`)]
|
||||
.map(rel => fs.readFileSync(path.join(ROOT, rel), 'utf-8')).join('\n');
|
||||
for (const src of [RESOLVERS.OUTSIDE_PREFLIGHT(ctx, ['autoplan']) + RESOLVERS.OUTSIDE_INVOCATION(ctx, ['autoplan']), rendered]) {
|
||||
expect(src).toContain('_gstack_codex_role_ready exec "$_REPO_ROOT" || exit $?');
|
||||
expect(src).not.toMatch(/_gstack_codex_model_probe|_CODEX_MP/);
|
||||
}
|
||||
const role = fs.readFileSync(PROBE, 'utf-8').match(/_gstack_codex_role_ready\(\) \{[\s\S]*?\n\}/)![0];
|
||||
expect(role).toContain('_gstack_codex_model_probe "$1"');
|
||||
expect(role.trim().split('\n').at(-2)!.trim()).toBe('_gstack_codex_model_probe "$1"');
|
||||
});
|
||||
|
||||
test('the preflight resolver routes exit 2 to broken_install', () => {
|
||||
|
||||
@@ -0,0 +1,258 @@
|
||||
/**
|
||||
* Bounded multi-signature Codex probe cache (stubbed codex, no API spend):
|
||||
* 16 entries, atomic publication, same-signature serialization with a
|
||||
* bounded lock, abandoned-owner recovery, and unverified (never a duplicate
|
||||
* paid probe) on lock timeout or unreadable cache.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { spawn, spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const PROBE = path.join(ROOT, 'bin', 'gstack-codex-probe');
|
||||
const STUB = `#!/usr/bin/env bash
|
||||
[ "$1" = sandbox ] && exit 0
|
||||
printf '%s\\n' "$*" >> "$STUB_LOG"
|
||||
[ -n "\${STUB_SLEEP:-}" ] && sleep "$STUB_SLEEP"
|
||||
case "\${STUB_MODE:-ok}" in
|
||||
ok) echo OK; exit 0 ;;
|
||||
model400) echo 'ERROR: {"type":"error","status":400,"error":{"message":"The model is not supported when using Codex with a ChatGPT account."}}' >&2; exit 1 ;;
|
||||
quota) echo "ERROR: You've hit your usage limit. Try again at 3 AM." >&2; exit 1 ;;
|
||||
esac
|
||||
`;
|
||||
|
||||
interface Fixture { home: string; codexHome: string; state: string; log: string; cache: string; locks: string; env: Record<string, string> }
|
||||
|
||||
function fixture(): Fixture {
|
||||
const home = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-probe-cache-'));
|
||||
const bin = path.join(home, 'bin');
|
||||
const codexHome = path.join(home, '.codex');
|
||||
const state = path.join(home, 'state');
|
||||
for (const dir of [bin, codexHome, state]) fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(path.join(bin, 'codex'), STUB, { mode: 0o755 });
|
||||
fs.writeFileSync(path.join(codexHome, 'config.toml'), 'model = "gpt-5.4"\n');
|
||||
fs.writeFileSync(path.join(codexHome, 'auth.json'), '{}');
|
||||
const log = path.join(home, 'stub.log');
|
||||
return {
|
||||
home, codexHome, state, log,
|
||||
cache: path.join(state, '.codex-model-probe'),
|
||||
locks: path.join(state, '.codex-model-probe.locks'),
|
||||
env: { PATH: `${bin}:${process.env.PATH ?? ''}`, HOME: home, CODEX_HOME: codexHome, GSTACK_HOME: state, STUB_LOG: log, _TEL: 'off' },
|
||||
};
|
||||
}
|
||||
|
||||
function probe(f: Fixture, env: Record<string, string> = {}, call = '_gstack_codex_model_probe; echo "rc=$? state=${_GSTACK_CODEX_PROBE_STATE:-}"') {
|
||||
const r = spawnSync('bash', ['-c', `source "${PROBE}"\n${call}`], { env: { ...f.env, ...env }, encoding: 'utf8', timeout: 20000 });
|
||||
return { stdout: r.stdout ?? '', stderr: r.stderr ?? '', status: r.status };
|
||||
}
|
||||
|
||||
function probeAsync(f: Fixture, env: Record<string, string>): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn('bash', ['-c', `source "${PROBE}"\n_gstack_codex_model_probe; echo "rc=$?"`], { env: { ...f.env, ...env } });
|
||||
let out = '';
|
||||
child.stdout.on('data', chunk => { out += chunk; });
|
||||
const timer = setTimeout(() => { child.kill('SIGKILL'); reject(new Error('probe hung')); }, 30000);
|
||||
child.on('close', () => { clearTimeout(timer); resolve(out); });
|
||||
});
|
||||
}
|
||||
|
||||
const calls = (f: Fixture) => (fs.existsSync(f.log) ? fs.readFileSync(f.log, 'utf8').split('\n').filter(Boolean).length : 0);
|
||||
const entries = (f: Fixture) => fs.readFileSync(f.cache, 'utf8').split('\n').filter(line => /^MODEL_\S+ \d+ \d+$/.test(line));
|
||||
const model = (name: string) => ({ GSTACK_CODEX_MODEL: name });
|
||||
const withFixture = (body: (f: Fixture) => void | Promise<void>) => async () => {
|
||||
const f = fixture();
|
||||
try { await body(f); } finally { fs.rmSync(f.home, { recursive: true, force: true }); }
|
||||
};
|
||||
|
||||
describe('bounded multi-signature probe cache', () => {
|
||||
test('A -> B -> A makes exactly two paid probes', withFixture((f) => {
|
||||
expect(probe(f, model('gpt-a')).stdout).toContain('MODEL_OK');
|
||||
expect(probe(f, model('gpt-b')).stdout).toContain('MODEL_OK');
|
||||
const again = probe(f, model('gpt-a'));
|
||||
expect(again.stdout).toContain('MODEL_OK (cached)');
|
||||
expect(calls(f)).toBe(2);
|
||||
expect(entries(f)).toHaveLength(2);
|
||||
expect(fs.existsSync(f.locks) ? fs.readdirSync(f.locks) : []).toEqual([]);
|
||||
}));
|
||||
|
||||
test('concurrent same-signature misses share one probe', withFixture(async (f) => {
|
||||
const outputs = await Promise.all([0, 1, 2, 3].map(() => probeAsync(f, { ...model('gpt-same'), STUB_SLEEP: '1' })));
|
||||
expect(calls(f)).toBe(1);
|
||||
for (const out of outputs) expect(out).toMatch(/MODEL_OK(?: \(cached\))?\nrc=0/);
|
||||
expect(outputs.filter(out => out.includes('(cached)'))).toHaveLength(3);
|
||||
expect(entries(f)).toHaveLength(1);
|
||||
}));
|
||||
|
||||
test('the 17th signature evicts the oldest entry; the file never exceeds 16', withFixture((f) => {
|
||||
for (let i = 1; i <= 17; i++) expect(probe(f, model(`gpt-m${i}`)).stdout).toContain('MODEL_OK');
|
||||
expect(entries(f)).toHaveLength(16);
|
||||
expect(probe(f, model('gpt-m17')).stdout).toContain('MODEL_OK (cached)');
|
||||
expect(probe(f, model('gpt-m2')).stdout).toContain('MODEL_OK (cached)');
|
||||
expect(calls(f)).toBe(17);
|
||||
expect(probe(f, model('gpt-m1')).stdout).not.toContain('(cached)');
|
||||
expect(calls(f)).toBe(18);
|
||||
expect(entries(f)).toHaveLength(16);
|
||||
}));
|
||||
|
||||
test('expired entries are pruned and never hit; TTLs and negative classifications are unchanged', withFixture((f) => {
|
||||
probe(f, { ...model('gpt-bad'), STUB_MODE: 'model400' });
|
||||
probe(f, model('gpt-good'));
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
const lines = fs.readFileSync(f.cache, 'utf8').split('\n').filter(Boolean)
|
||||
.map(line => line.replace(/^(MODEL_UNUSABLE) (\d+)/, (_, s) => `${s} ${now - 901}`));
|
||||
fs.writeFileSync(f.cache, `${lines.join('\n')}\n`);
|
||||
expect(probe(f, { ...model('gpt-good') }).stdout).toContain('MODEL_OK (cached)');
|
||||
const bad = probe(f, { ...model('gpt-bad'), STUB_MODE: 'model400' });
|
||||
expect(bad.stdout).toContain('MODEL_UNUSABLE\n');
|
||||
expect(bad.stdout).toContain('rc=1');
|
||||
expect(calls(f)).toBe(3);
|
||||
expect(probe(f, { ...model('gpt-bad'), STUB_MODE: 'ok' }).stdout).toContain('MODEL_UNUSABLE (cached)');
|
||||
expect(calls(f)).toBe(3);
|
||||
const future = fs.readFileSync(f.cache, 'utf8').replace(/^MODEL_OK \d+/m, `MODEL_OK ${now + 999}`);
|
||||
fs.writeFileSync(f.cache, future);
|
||||
expect(probe(f, model('gpt-good')).stdout).not.toContain('(cached)');
|
||||
}));
|
||||
|
||||
test('quota detail lines survive among other entries and are relayed on a hit', withFixture((f) => {
|
||||
probe(f, model('gpt-x'));
|
||||
expect(probe(f, { ...model('gpt-q'), STUB_MODE: 'quota' }).stdout).toContain('rc=4');
|
||||
probe(f, model('gpt-y'));
|
||||
const hit = probe(f, model('gpt-q'));
|
||||
expect(hit.stdout).toContain('MODEL_QUOTA_EXHAUSTED (cached)');
|
||||
expect(hit.stdout).toContain("ERROR: You've hit your usage limit. Try again at 3 AM.");
|
||||
expect(hit.stdout).toContain('rc=4');
|
||||
expect(probe(f, { ...model('gpt-q'), GSTACK_CODEX_PROBE_RETRY: '1' }).stdout).toContain('MODEL_OK\n');
|
||||
expect(calls(f)).toBe(4);
|
||||
expect(entries(f)).toHaveLength(3);
|
||||
}));
|
||||
|
||||
test('a pre-v2 one-entry cache migrates by TTL: fresh honored, expired re-probed, stray lines ignored', withFixture((f) => {
|
||||
probe(f);
|
||||
const [status, ts, sig] = entries(f)[0].split(' ');
|
||||
expect(status).toBe('MODEL_OK');
|
||||
fs.writeFileSync(f.cache, `MODEL_OK ${ts} ${sig}\nan unprefixed legacy line\n`);
|
||||
expect(probe(f).stdout).toContain('MODEL_OK (cached)');
|
||||
fs.writeFileSync(f.cache, `MODEL_OK ${Number(ts) - 3700} ${sig}\n`);
|
||||
expect(probe(f).stdout).not.toContain('(cached)');
|
||||
expect(calls(f)).toBe(2);
|
||||
}));
|
||||
|
||||
test('an interrupted writer leaves a truncated line and temp file that never count as a hit', withFixture((f) => {
|
||||
probe(f, model('gpt-a'));
|
||||
fs.appendFileSync(f.cache, 'MODEL_OK 17');
|
||||
fs.writeFileSync(`${f.cache}.tmp.99999`, 'MODEL_OK 1 1\n');
|
||||
expect(probe(f, model('gpt-a')).stdout).toContain('MODEL_OK (cached)');
|
||||
expect(probe(f, model('gpt-b')).stdout).toContain('MODEL_OK\n');
|
||||
expect(fs.readFileSync(f.cache, 'utf8')).not.toContain('MODEL_OK 17\n');
|
||||
expect(entries(f)).toHaveLength(2);
|
||||
}));
|
||||
|
||||
test('config, auth and binary changes still invalidate a cached entry', withFixture((f) => {
|
||||
probe(f);
|
||||
const future = Date.now() / 1000 + 10;
|
||||
fs.utimesSync(path.join(f.codexHome, 'config.toml'), future, future);
|
||||
expect(probe(f).stdout).not.toContain('(cached)');
|
||||
fs.utimesSync(path.join(f.codexHome, 'auth.json'), future + 5, future + 5);
|
||||
expect(probe(f).stdout).not.toContain('(cached)');
|
||||
expect(probe(f, { CODEX_API_KEY: 'k-test' }).stdout).not.toContain('(cached)');
|
||||
expect(calls(f)).toBe(4);
|
||||
}));
|
||||
});
|
||||
|
||||
describe('probe lock recovery and failure', () => {
|
||||
const sigOf = (f: Fixture) => {
|
||||
probe(f);
|
||||
const sig = entries(f)[0].split(' ')[2];
|
||||
fs.rmSync(f.cache);
|
||||
fs.rmSync(f.log);
|
||||
return sig;
|
||||
};
|
||||
|
||||
test('a dead owner on this host is recovered at once, without a duplicate probe', withFixture((f) => {
|
||||
const sig = sigOf(f);
|
||||
const lock = path.join(f.locks, sig);
|
||||
fs.mkdirSync(lock, { recursive: true });
|
||||
const dead = spawnSync('bash', ['-c', 'echo $$'], { timeout: 5000 }).stdout.toString().trim();
|
||||
fs.writeFileSync(path.join(lock, 'owner'), `${dead} ${os.hostname()} ${Math.floor(Date.now() / 1000)}\n`);
|
||||
const r = probe(f, { _GSTACK_CODEX_LOCK_WAIT: '3' });
|
||||
expect(r.stdout).toContain('MODEL_OK\n');
|
||||
expect(calls(f)).toBe(1);
|
||||
expect(fs.existsSync(lock)).toBe(false);
|
||||
}));
|
||||
|
||||
test('a live owner past the stale bound is abandoned; within it the waiter is unverified without probing', withFixture((f) => {
|
||||
const sig = sigOf(f);
|
||||
const lock = path.join(f.locks, sig);
|
||||
fs.mkdirSync(lock, { recursive: true });
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
fs.writeFileSync(path.join(lock, 'owner'), `${process.pid} ${os.hostname()} ${now}\n`);
|
||||
const busy = probe(f, { _GSTACK_CODEX_LOCK_WAIT: '1' });
|
||||
expect(busy.stdout).toContain('MODEL_PROBE_INCONCLUSIVE (another probe of this model still holds its lock)');
|
||||
expect(busy.stdout).toContain('rc=0 state=inconclusive');
|
||||
expect(calls(f)).toBe(0);
|
||||
fs.writeFileSync(path.join(lock, 'owner'), `${process.pid} ${os.hostname()} ${now - 120}\n`);
|
||||
expect(probe(f, { _GSTACK_CODEX_LOCK_WAIT: '2' }).stdout).toContain('MODEL_OK\n');
|
||||
expect(calls(f)).toBe(1);
|
||||
}));
|
||||
|
||||
test('a lock without its owner file is the creation window until it is 5 seconds old', withFixture((f) => {
|
||||
const sig = sigOf(f);
|
||||
const lock = path.join(f.locks, sig);
|
||||
fs.mkdirSync(lock, { recursive: true });
|
||||
expect(probe(f, { _GSTACK_CODEX_LOCK_WAIT: '1' }).stdout).toContain('MODEL_PROBE_INCONCLUSIVE');
|
||||
const old = Date.now() / 1000 - 30;
|
||||
fs.utimesSync(lock, old, old);
|
||||
expect(probe(f, { _GSTACK_CODEX_LOCK_WAIT: '2' }).stdout).toContain('MODEL_OK\n');
|
||||
expect(calls(f)).toBe(1);
|
||||
}));
|
||||
|
||||
test('an unreadable cache or uncreatable lock is unverified and launches no paid probe', withFixture((f) => {
|
||||
fs.mkdirSync(f.cache);
|
||||
const dir = probe(f);
|
||||
expect(dir.stdout).toContain('MODEL_PROBE_INCONCLUSIVE (probe cache');
|
||||
expect(dir.stdout).toContain('rc=0 state=inconclusive');
|
||||
fs.rmdirSync(f.cache);
|
||||
fs.writeFileSync(f.cache, '');
|
||||
fs.chmodSync(f.cache, 0o000);
|
||||
expect(probe(f).stdout).toContain('MODEL_PROBE_INCONCLUSIVE (probe cache');
|
||||
fs.chmodSync(f.cache, 0o644);
|
||||
fs.rmSync(f.cache);
|
||||
fs.writeFileSync(f.locks, 'not a directory');
|
||||
expect(probe(f).stdout).toContain('MODEL_PROBE_INCONCLUSIVE (cannot create the probe lock');
|
||||
expect(calls(f)).toBe(0);
|
||||
}));
|
||||
|
||||
test('an owner record that cannot be written is lock I/O failure: unverified, no paid probe, no stray lock', withFixture((f) => {
|
||||
fs.mkdirSync(f.locks, { recursive: true });
|
||||
const r = probe(f, {}, '_gstack_codex_select_model exec 2>/dev/null; umask 0777; _gstack_codex_model_probe; echo "rc=$? state=${_GSTACK_CODEX_PROBE_STATE:-}"');
|
||||
expect(r.stdout).toContain('MODEL_PROBE_INCONCLUSIVE (cannot record ownership of the probe lock');
|
||||
expect(r.stdout).toContain('rc=0 state=inconclusive');
|
||||
expect(calls(f)).toBe(0);
|
||||
expect(fs.readdirSync(f.locks)).toEqual([]);
|
||||
expect(fs.existsSync(f.cache)).toBe(false);
|
||||
expect(probe(f).stdout).toContain('MODEL_OK\n');
|
||||
expect(calls(f)).toBe(1);
|
||||
}));
|
||||
|
||||
test('a cache write failure keeps the real probe result', withFixture((f) => {
|
||||
fs.chmodSync(f.state, 0o500);
|
||||
try {
|
||||
const r = probe(f, { STUB_MODE: 'model400' });
|
||||
expect(r.stdout).toContain('MODEL_PROBE_INCONCLUSIVE (cannot create the probe lock');
|
||||
fs.chmodSync(f.state, 0o700);
|
||||
fs.mkdirSync(f.locks);
|
||||
fs.writeFileSync(path.join(f.locks, 'write'), 'blocks the write lock');
|
||||
const real = probe(f, { STUB_MODE: 'model400' });
|
||||
expect(real.stdout).toContain('MODEL_UNUSABLE\n');
|
||||
expect(real.stdout).toContain('rc=1');
|
||||
expect(fs.existsSync(f.cache)).toBe(false);
|
||||
} finally { fs.chmodSync(f.state, 0o700); }
|
||||
}));
|
||||
|
||||
test('sourcing the probe still leaves the caller shell untouched', () => {
|
||||
const r = spawnSync('bash', ['-c', `B=$(set -o; trap -p; echo "$IFS"); source "${PROBE}"; A=$(set -o; trap -p; echo "$IFS"); [ "$A" = "$B" ] && echo CLEAN`], { encoding: 'utf8', timeout: 5000 });
|
||||
expect(r.stdout.trim()).toBe('CLEAN');
|
||||
});
|
||||
});
|
||||
@@ -90,7 +90,7 @@ describe('under-codex wiring renders (#2519)', () => {
|
||||
test('all three codexPreflight consumers render the probe', () => {
|
||||
for (const file of [
|
||||
path.join(ROOT, 'ship', 'sections', 'adversarial.md'),
|
||||
path.join(ROOT, 'plan-ceo-review', 'sections', 'review-sections.md'),
|
||||
path.join(ROOT, 'review', 'sections', 'adversarial.md'),
|
||||
path.join(ROOT, 'document-release', 'sections', 'release-body.md'),
|
||||
]) {
|
||||
const rendered = fs.readFileSync(file, 'utf-8');
|
||||
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { FRESHNESS_SOURCES, parseOfficialSource, sha256, type SourceId, type SourceResult } from '../../lib/model-policy-freshness-sources';
|
||||
import type { CatalogIdentity, FreshnessObservation } from '../../lib/model-policy-freshness';
|
||||
|
||||
export const freshnessCatalog: CatalogIdentity = {
|
||||
sha256: sha256('catalog-v1'), sourceSha256: sha256('catalog-source-v1'), version: 1,
|
||||
models: [
|
||||
{ provider: 'anthropic', tier: 'frontier', modelId: 'claude-fable-5-1' },
|
||||
{ provider: 'anthropic', tier: 'smart', modelId: 'claude-opus-5-5' },
|
||||
{ provider: 'openai', tier: 'frontier', modelId: 'gpt-6-astra' },
|
||||
{ provider: 'openai', tier: 'smart', modelId: 'gpt-6.1-sol' },
|
||||
],
|
||||
};
|
||||
|
||||
export function freshnessFixture(id: SourceId): string {
|
||||
return readFileSync(new URL(`./model-policy-freshness/${id}.md`, import.meta.url), 'utf8');
|
||||
}
|
||||
|
||||
export function freshnessObservation(id = '100', at = '2026-10-07T12:00:00.000Z', overrides: Partial<Record<SourceId, string | null>> = {}): FreshnessObservation {
|
||||
const sources: SourceResult[] = FRESHNESS_SOURCES.map(source => {
|
||||
const text = overrides[source.id] === undefined ? freshnessFixture(source.id) : overrides[source.id];
|
||||
return text === null ? { id: source.id, error: 'http-503' } : {
|
||||
id: source.id, receipt: { id: source.id, url: source.url, finalUrl: source.url, sha256: sha256(text!), bytes: Buffer.byteLength(text!), parsed: parseOfficialSource(source.id, text!) },
|
||||
};
|
||||
});
|
||||
return { catalog: structuredClone(freshnessCatalog), checkedAt: at, run: { id, attempt: 1, startedAt: new Date(Date.parse(at) - 1000).toISOString(), headSha: 'a'.repeat(40) }, sources };
|
||||
}
|
||||
|
||||
export function retirementFixture(): string {
|
||||
return freshnessFixture('anthropic-lifecycle').replace('claude-fable-5-1 | Active | N/A | Not sooner than September 1, 2027', 'claude-fable-5-1 | Deprecated | October 1, 2026 | November 1, 2026');
|
||||
}
|
||||
|
||||
export function freshnessReplacementObservation(): FreshnessObservation {
|
||||
const observation = freshnessObservation('101', '2026-10-08T12:00:00.000Z', {
|
||||
'anthropic-models': freshnessFixture('anthropic-models').replaceAll('fable-5-1', 'fable-5-2'),
|
||||
'anthropic-lifecycle': freshnessFixture('anthropic-lifecycle').replaceAll('fable-5-1', 'fable-5-2'),
|
||||
});
|
||||
observation.catalog.models[0].modelId = 'claude-fable-5-2';
|
||||
observation.catalog.sha256 = 'b'.repeat(64);
|
||||
observation.catalog.sourceSha256 = 'c'.repeat(64);
|
||||
return observation;
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
# Model deprecations
|
||||
|
||||
Partner-operated platforms (Amazon Bedrock and Google Cloud) set their own retirement schedules.
|
||||
|
||||
## Model status
|
||||
|
||||
| API model name | Current state | Deprecated | Tentative retirement date |
|
||||
| --- | --- | --- | --- |
|
||||
| claude-fable-5-1 | Active | N/A | Not sooner than September 1, 2027 |
|
||||
| claude-opus-5-5 | Active | N/A | Not sooner than September 22, 2027 |
|
||||
| claude-mythos-preview | Deprecated | June 9, 2026 | To be announced |
|
||||
| claude-opus-4-1-20250805 | Retired | June 5, 2026 | August 5, 2026 |
|
||||
|
||||
## Deprecation history
|
||||
|
||||
Amazon Bedrock partner discussion mentions `claude-fable-5-1` retirement; this is not a public API lifecycle row.
|
||||
@@ -0,0 +1,17 @@
|
||||
# Models overview
|
||||
|
||||
## Compare models
|
||||
|
||||
If you're unsure which model to use, start with [Claude Opus 5.5](https://platform.claude.com/docs/en/models/opus-5-5/overview) for most workloads. Use [Claude Fable 5.1](https://platform.claude.com/docs/en/models/fable-5-1/overview) for demanding reasoning and long-horizon agentic work.
|
||||
|
||||
| Feature | Claude Fable 5.1 | Claude Opus 5.5 | Claude Sonnet 5.5 | Claude Haiku 5.5 |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Model page | [Claude Fable 5.1](https://platform.claude.com/docs/en/models/fable-5-1/overview) | [Claude Opus 5.5](https://platform.claude.com/docs/en/models/opus-5-5/overview) | [Claude Sonnet 5.5](https://platform.claude.com/docs/en/models/sonnet-5-5/overview) | [Claude Haiku 5.5](https://platform.claude.com/docs/en/models/haiku-5-5/overview) |
|
||||
| Claude API ID | `claude-fable-5-1` | `claude-opus-5-5` | `claude-sonnet-5-5` | `claude-haiku-5-5` |
|
||||
| Amazon Bedrock ID | `anthropic.claude-fable-5-1` | `anthropic.claude-opus-5-5` | `anthropic.claude-sonnet-5-5` | `anthropic.claude-haiku-5-5` |
|
||||
|
||||
* **Pricing:** prompt cache reads cost 10% of the base input price (2.5% on Claude Fable 5.1 and Claude Mythos 5.1, 5% on Claude Opus 5.5). See Pricing for per-platform pricing. Incidental ID: `claude-mythos-5-1`.
|
||||
|
||||
## Using the Models API
|
||||
|
||||
Legacy `claude-fable-5` and unrelated `claude-mythos-5-1` are not recommendations.
|
||||
@@ -0,0 +1,24 @@
|
||||
# Deprecations
|
||||
|
||||
This page lists all API deprecations, along with recommended replacements.
|
||||
|
||||
## Upcoming deprecations
|
||||
|
||||
### 2026-10-01: GPT-5.3-Codex
|
||||
|
||||
| Shutdown date | Model / system | Recommended replacement |
|
||||
| --- | --- | --- |
|
||||
| Apr 1, 2027 | `gpt-5.3-codex` | `gpt-6-sol` |
|
||||
|
||||
## Past deprecations
|
||||
|
||||
| Shutdown date | Model snapshot | Substitute model |
|
||||
| --- | --- | --- |
|
||||
| Aug 5, 2026 | `gpt-old` | `gpt-6-astra` |
|
||||
| 2026‑03‑26 | `gpt-unicode-date` | `gpt-6-astra` |
|
||||
| 2026-02-12 | `gpt-iso-date` | `gpt-6-astra` |
|
||||
| at earliest 2024-06-13 | `gpt-date-floor` | `gpt-6-astra` |
|
||||
| 2026-09-24 | Videos API | --- |
|
||||
| 2024-01-04 | `/v1/edits` | `/v1/chat/completions` |
|
||||
|
||||
Incidental replacement text `gpt-6-astra` is not a deprecated model row.
|
||||
@@ -0,0 +1,13 @@
|
||||
# Models
|
||||
|
||||
## Featured models
|
||||
|
||||
- [GPT-6 Astra](/api/docs/models/gpt-6-astra.md): Start here for complex reasoning and coding.
|
||||
- [GPT-6.1 Sol](/api/docs/models/gpt-6.1-sol.md): Balance intelligence and cost.
|
||||
- [GPT-6 Luna](/api/docs/models/gpt-6-luna.md): Optimize cost-sensitive, high-volume workloads.
|
||||
|
||||
## Browse our full catalog of models
|
||||
|
||||
- [GPT-6 Astra](/api/docs/models/gpt-6-astra.md): Our most capable model for the most demanding work.
|
||||
- [GPT-6.1 Sol](/api/docs/models/gpt-6.1-sol.md): Near-Astra performance for complex work at a lower cost.
|
||||
- [GPT-9 Preview](/api/docs/models/gpt-9-preview.md): Incidental full-catalog listing, not a featured recommendation.
|
||||
+115
@@ -0,0 +1,115 @@
|
||||
{
|
||||
"source": "ship-exploratory-late-input, mtb1007a, 2026-10-07; public synthetic fixture events only",
|
||||
"events": [
|
||||
{
|
||||
"type": "assistant",
|
||||
"parent_tool_use_id": null,
|
||||
"session_id": "00ad7471-736b-4592-af5e-eace35c1d20d",
|
||||
"message": {
|
||||
"id": "msg_011CfoeQHondqjCMbvsEwUxZ",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "toolu_01PYaVcvVGnQUhMN7ZSxsQ5q",
|
||||
"name": "Read",
|
||||
"input": {
|
||||
"file_path": "/tmp/gstack-paid-shard-I5IeEv/tmp/qc-nJNpah/host/runtime/qa/sections/exploratory.md"
|
||||
},
|
||||
"caller": {
|
||||
"type": "direct"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "user",
|
||||
"parent_tool_use_id": null,
|
||||
"session_id": "00ad7471-736b-4592-af5e-eace35c1d20d",
|
||||
"message": {
|
||||
"content": [
|
||||
{
|
||||
"tool_use_id": "toolu_01PYaVcvVGnQUhMN7ZSxsQ5q",
|
||||
"type": "tool_result",
|
||||
"content": "1\t<!-- AUTO-GENERATED from exploratory.md.tmpl \u2014 do not edit directly -->\n2\t<!-- Regenerate: bun run gen:skill-docs -->\n3\t# Shared exploratory QA\n4\t\n5\tThe **caller** (/qa, /qa-only, /review or /ship) owns decisions, tests, fixes and publication. Discovery writes only reports/evidence\n6\tand owned fixture state; no workflows, framework installs or publication.\n7\t\n8\tComplete these Reads in order before writing charters or probing. Await their results before the first probe, never in the same response. Do not repeat a Read already completed in this invocation.\n9\t1. Read `sections/scope.md` relative to the installed `qa`/`gstack-qa` SKILL.md directory in full and select the surfaces.\n10\t2. Read the selected surface methods below in full.\n11\t\n12\t**Functional surfaces:**\n13\tRead `sections/system-functional.md` in full.\n14\t\n15\t**Browser surfaces only:**\n16\tRead `sections/qa-patterns.md` in full.\n17\t\n18\tMissing or unreadable assets, prerequisites or permission block affected probes, not independent safe checks. Report QA setup blockers.\n19\t\n20\t## 1. Charter and preflight\n21\t\n22\tReuse resolved REPORT_DIR; otherwise own a fresh `.gstack/qa-reports` subdirectory.\n23\tWrite a **charter** per behavior: contract, risk, entrypoint, isolation, exit condition, source, commands and inputs. Save charters as Markdown in the report.\n24\t\n25\tFor /review and /ship, no plan/server is required.\n26\tStop after 5 minutes or 12 probes, whichever comes first (SECONDS=300 across surfaces).\n27\tExplicit plan checks and revalidation remain required beyond this smoke budget.\n28\tFor /qa and /qa-only:\n29\t- Browser Quick: SECONDS=180. Browser Full/Regression: SECONDS=900.\n30\t- Functional Full, Quick and Regression have no default total timer.\n31\tSet SECONDS to the shorter mode/caller limit; an unlimited mode uses the caller's bound.\n32\tWithout a total time limit, do not create DEADLINE_FILE; announce finite command timeouts.\n33\tStop when scoped contracts are tested or blocked.\n34\tClocks/checkpoints use REPORT_DIR; mixed standalone runs use REPORT_DIR/browser and REPORT_DIR/functional, with one final report at REPORT_DIR. Caller paths win.\n35\tNames used below (quote every path):\n36\t- PROBE_DIR: this surface's owned probe directory per the line above. DEADLINE_FILE: `PROBE_DIR/deadline.json`, which exists only for a bounded run.\n37\t- SECONDS: the total probe budget set above. NNN: a fresh three-digit ID (001, 002, ...) for each capture or checkpoint; never reuse one.\n38\t- DEADLINE_TOOL = `/tmp/gstack-paid-shard-I5IeEv/tmp/qc-nJNpah/host/runtime/bin/gstack-qa-deadline`, the deadline guard: `start` creates DEADLINE_FILE, `status` prints `remainingMs` and `expired`, and `run` executes one command, stopping it at the deadline.\n39\t- EVIDENCE_TOOL = `/tmp/gstack-paid-shard-I5IeEv/tmp/qc-nJNpah/host/runtime/bin/gstack-qa-evidence`, the functional evidence recorder: `capture` runs one command, stores its exit code, stdout and stderr under `PROBE_DIR/.qa-evidence/NNN/`, and prints status `complete`, `incomplete` or `sensitive`.\n40\t\n41\tStart once before baseline: `bun DEADLINE_TOOL start DEADLINE_FILE SECONDS [EARLIER_UTC]` if bounded.\n42\tEARLIER_UTC = caller's absolute deadline, if set.\n43\tFunctional: `bun EVIDENCE_TOOL capture PROBE_DIR NNN [--public] --deadline DEADLINE_FILE -- COMMAND ARGS`.\n44\tUnbounded: use `--timeout-ms MS` instead of `--deadline DEADLINE_FILE`.\n45\t--public requires approved public/synthetic output; EVIDENCE_TOOL screens credentials. For complete private captures, await a safe Read of `PROBE_DIR/.qa-evidence/NNN/observation.json` (the decoded stdout). Sensitive/incomplete captures cannot anchor checkpoints.\n46\tBounded browsers: `bun DEADLINE_TOOL run DEADLINE_FILE -- COMMAND ARGS`. No detached probes.\n47\tNever reset DEADLINE_FILE/bypass DEADLINE_TOOL. Expiry or an invalid/missing DEADLINE_FILE stops probes; report unfinished coverage. QA_DEADLINE receipts are not observations.\n48\t\n49\t## 2. Probe loop\n50\t\n51\tEach probe is one native command/interaction plus checks, excluding bookkeeping.\n52\tNever batch probes.\n53\t\n54\t1. First demonstrate success: output AND durable effects. Guard if bounded; await completion.\n55\t2. **Decide whether another probe is needed.** If bounded, run `bun DEADLINE_TOOL status DEADLINE_FILE`.\n56\t If expired or no safe next probe remains, STOP exploration; write the report (\u00a74), not a checkpoint.\n57\t **Publish before probing.** Create (browser) or compose (functional) `exploration-NNN.json` in the probe directory, beside its deadline if bounded, with exactly four top-level fields:\n58\t observationCommand: last completed probe's full outer command, including guard.\n59\t observed: its exact decoded child JSON (no wrapper/extra keys), or its full non-JSON text.\n60\t hypothesis: why nextCommand. nextCommand: exact command/request, guarded if bounded.\n61\t PrLine truncated
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "assistant",
|
||||
"parent_tool_use_id": null,
|
||||
"session_id": "00ad7471-736b-4592-af5e-eace35c1d20d",
|
||||
"message": {
|
||||
"id": "msg_011CfoeQHondqjCMbvsEwUxZ",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "toolu_01YbU4FzEXBhHBftjs4URWf5",
|
||||
"name": "Bash",
|
||||
"input": {
|
||||
"command": "bun /tmp/gstack-paid-shard-I5IeEv/tmp/qc-nJNpah/host/runtime/bin/gstack-qa-evidence capture /tmp/gstack-paid-shard-I5IeEv/tmp/qc-nJNpah/product/reports 001 --public --deadline /tmp/gstack-paid-shard-I5IeEv/tmp/qc-nJNpah/product/reports/deadline.json -- bun scripts/probe.ts 3",
|
||||
"description": "Capture happy-path probe for input 3"
|
||||
},
|
||||
"caller": {
|
||||
"type": "direct"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "user",
|
||||
"parent_tool_use_id": null,
|
||||
"session_id": "00ad7471-736b-4592-af5e-eace35c1d20d",
|
||||
"message": {
|
||||
"content": [
|
||||
{
|
||||
"tool_use_id": "toolu_01YbU4FzEXBhHBftjs4URWf5",
|
||||
"type": "tool_result",
|
||||
"content": "{\"id\":\"probe-ba7e52d4-ad96-4df9-97d0-6dd79e52f184\",\"charter\":\"happy\",\"input\":\"3\",\"snapshot\":\"936ff5b3b13172fb2357a120804c1cf5cac68d911c82cae5bce630f04dd62b57\",\"status\":\"pass\",\"stdout\":\"6\\n\",\"stderr\":\"\",\"exit\":0}\n\nQA_EVIDENCE {\"producer\":\"gstack-qa-evidence\",\"version\":1,\"action\":\"capture\",\"id\":\"001\",\"status\":\"complete\",\"sha256\":\"135ce1a8f804e3c6d953e0f284ab4e1e7fe5ded56f3a35706ea73f630876d0be\",\"exitCode\":0,\"signal\":null,\"publicOutput\":true,\"startedAt\":\"2026-10-07T22:13:30.620Z\",\"completedAt\":\"2026-10-07T22:13:30.675Z\",\"durationMs\":55,\"remainingMs\":182686,\"next\":\"Another probe requires a checkpoint anchored on capture 001: add --after 001 --hypothesis 'TEXT' before --. To stop exploring, run none.\"}",
|
||||
"is_error": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"tool_use_result": {
|
||||
"interrupted": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "assistant",
|
||||
"parent_tool_use_id": null,
|
||||
"session_id": "00ad7471-736b-4592-af5e-eace35c1d20d",
|
||||
"message": {
|
||||
"id": "msg_011CfoeRZyA2j8av6NchLTmY",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "toolu_01CPUvNXrNHnLrEUa4n5PdWC",
|
||||
"name": "Read",
|
||||
"input": {
|
||||
"file_path": "/tmp/gstack-paid-shard-I5IeEv/tmp/qc-nJNpah/host/runtime/qa/sections/system-functional.md"
|
||||
},
|
||||
"caller": {
|
||||
"type": "direct"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "user",
|
||||
"parent_tool_use_id": null,
|
||||
"session_id": "00ad7471-736b-4592-af5e-eace35c1d20d",
|
||||
"message": {
|
||||
"content": [
|
||||
{
|
||||
"tool_use_id": "toolu_01CPUvNXrNHnLrEUa4n5PdWC",
|
||||
"type": "tool_result",
|
||||
"content": "1\t<!-- AUTO-GENERATED from system-functional.md.tmpl \u2014 do not edit directly -->\n2\t<!-- Regenerate: bun run gen:skill-docs -->\n3\t# Functional QA with repository-native tools\n4\t\n5\tUse documented repository commands, CLI/API clients and job/queue tools, not a new\n6\tharness or browser substitution.\n7\t\n8\t## Functional modes\n9\t\n10\tFor /qa and /qa-only, within the selected scope:\n11\t- **Full** (default): cover every applicable documented contract below.\n12\t- **Quick** (`--quick`): check success and the highest-risk changed edge; mark other\n13\t contracts not run.\n14\t- **Regression** (`--regression <previous-report>`): before probes, read the supplied\n15\t functional report and linked replay evidence. A missing, unreadable or wrong-target\n16\t baseline blocks regression mode. A browser-only `baseline.json` is not a functional\n17\t baseline. Re-establish owned setup; replay prior failed probes against the documented\n18\t expectation, never recorded buggy output, then check changed adjacent behavior.\n19\t Preserve the prior report; report fixed, still failing and new findings separately.\n20\t Missing safe replay inputs block affected probes, never count as passes.\n21\t\n22\tMixed runs apply each surface's mode separately. /review and /ship retain their caller's\n23\tbounded smoke and explicit plan checks, not Full exploration.\n24\t\n25\t## Contract map\n26\t\n27\tRecord each contract/source, isolated setup, exact probe, expectation and outcome:\n28\tpass/fail/blocked/not run/inconclusive/not applicable (reason).\n29\t\n30\t| Contract | Observe |\n31\t|---|---|\n32\t| Successful execution | Expected return/output and final business effect, not just launch/acceptance |\n33\t| Invalid/missing input | Declared rejection, correct status and no forbidden state change |\n34\t| Authentication/authorization | Valid identity, missing/invalid identity, wrong owner/role and durable no-effect boundary |\n35\t| CLI process contract | Exact exit code, stdout and stderr separately; resulting file/state changes |\n36\t| State transitions | Initial, intermediate and completed/failed states and their permitted transitions |\n37\t| Timeout/cancellation | Deadline, partial state, termination of owned work and recovery |\n38\t| Retry | Attempts/backoff/terminal state promised by the repository; no unbounded retry |\n39\t| Duplicates/idempotency | Repeated request/event and number of durable effects under the documented guarantee |\n40\t| Concurrency/order | Controlled competing operations in both relevant completion orders; final invariant |\n41\t| Partial-failure recovery | Interrupt after an effect, restart/replay, inspect completion/dead-letter state and duplicates |\n42\t\n43\tDo not impose universal exactly-once delivery. Separate acceptance, enqueue, processing,\n44\tretry/dead-letter and final effect; 2xx is not completion. Expected rejection/injected\n45\tfailure may pass; a missing service preventing execution blocks coverage.\n46\t\n47\t## Execute and retain evidence\n48\t\n49\t1. Apply the shared isolation/permission preflight. Verify cwd, command, environment\n50\t NAMES and safe reset; use synthetic data/credentials.\n51\t2. Follow the shared exploratory loop's order and written checkpoints.\n52\t For every probe, inspect initial/final durable state and retain exit/status and\n53\t stdout/stderr separately without masking failure.\n54\t3. On timeout, retain partial output/state and stop only owned work. Record setup errors\n55\t and untested contracts; never patch product code to hide missing prerequisites.\n56\t4. Record exact command or method/path/headers/body, setup/reset, expected contract/source,\n57\t observed output/state, revision/runtime, evidence paths and limits. Secrets are referenced\n58\t only by environment name. Disclose replay limits caused by redaction.\n59\t5. Use `templates/functional-report-template.md` relative to the installed QA SKILL.md.\n60\t Preserve evidence before owned cleanup and disclose leftovers. Return to the caller\n61\t without expanding discovery authority.\n62\t"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -123,6 +123,8 @@ export const FREE_FIXTURES: Readonly<Record<string, readonly string[]>> = {
|
||||
'test/fixtures/judge-panel-replay-2026-10-05.json': ['test/judge-panel-median.test.ts'],
|
||||
'test/fixtures/module-size-ratchet.json': ['test/helpers/module-size.ts', 'test/module-size-ratchet.test.ts'],
|
||||
'test/fixtures/module-size/**': ['test/helpers/module-size.ts', 'test/module-size-ratchet.test.ts'],
|
||||
'test/fixtures/model-policy-freshness.ts': ['test/model-policy-freshness.test.ts', 'test/model-policy-workflow.test.ts'],
|
||||
'test/fixtures/model-policy-freshness/**': ['test/fixtures/model-policy-freshness.ts'],
|
||||
'test/fixtures/native-auq-boxed-full-body-sep21.json': ['test/auq-native-capture.test.ts'],
|
||||
'test/fixtures/native-auq-clipped-elided-sep21.json': ['test/auq-native-capture.test.ts'],
|
||||
'test/fixtures/native-auq-devex-clipped-elided-sep21.json': ['test/auq-native-capture.test.ts'],
|
||||
@@ -138,6 +140,7 @@ export const FREE_FIXTURES: Readonly<Record<string, readonly string[]>> = {
|
||||
'test/fixtures/plan-create-cropped-title-batching.json': ['test/plan-create-combined-permission.test.ts'],
|
||||
'test/fixtures/plan-floor-edit-wrap-37176835584.json': ['test/plan-floor-edit-wrap.test.ts'],
|
||||
'test/fixtures/plan-format-kind-notes.json': ['test/plan-format-kind-note.test.ts'],
|
||||
'test/fixtures/qa-caller-read-probe-mtb1007a.json': ['test/qa-exploratory-callers.test.ts'],
|
||||
'test/fixtures/plans/autoplan-dashboard.md': ['test/autoplan-init.test.ts', 'test/autoplan-preconfigured-onboarding-ar.test.ts', 'test/autoplan-snapshot.test.ts'],
|
||||
'test/fixtures/plans/autoplan-focus-appearance-design.md': [],
|
||||
'test/fixtures/plans/autoplan-focus-appearance.md': [],
|
||||
|
||||
@@ -110,6 +110,11 @@ export function callerTools(transcript: unknown[]): CallerTool[] {
|
||||
return tools;
|
||||
}
|
||||
|
||||
export function callerProbeReadErrors(probe: CallerTool, resources: readonly (readonly [string, CallerTool | undefined])[]): string[] {
|
||||
const premature = (read: CallerTool) => probe.index <= read.resultIndex || !!probe.messageId && probe.messageId === read.messageId;
|
||||
return resources.flatMap(([name, read]) => read && premature(read) ? [`probe preceded resource read: ${name}`] : []);
|
||||
}
|
||||
|
||||
export interface CallerProbe {
|
||||
id: string;
|
||||
charter: string;
|
||||
@@ -358,10 +363,7 @@ export function validateCallerEvidence(input: {
|
||||
else {
|
||||
checkpointProbes.push({ command: String(tool.input.command), observed: probe, index: tool.index });
|
||||
if (parentRead && (tool.index <= parentRead.resultIndex || tool.messageId && tool.messageId === parentRead.messageId)) errors.push(`probe preceded parent entrypoint: ${probe.id}`);
|
||||
for (const id of ['exploratory', 'system-functional']) {
|
||||
const resource = readOf(`/qa/sections/${id}.md`);
|
||||
if (resource && (tool.index <= resource.resultIndex || tool.messageId && tool.messageId === resource.messageId)) errors.push(`probe preceded resource read: ${id}`);
|
||||
}
|
||||
errors.push(...callerProbeReadErrors(tool, ['exploratory', 'system-functional'].map(id => [id, readOf(`/qa/sections/${id}.md`)] as const)));
|
||||
if (input.requireGuardedSmoke) {
|
||||
const command = String(tool.input.command);
|
||||
const guarded = callerDeadlineCommand(command, deadline);
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
import { afterEach, describe, expect, test } from 'bun:test';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { modelCatalogSha256 } from '../lib/model-catalog';
|
||||
import { validatePolicyValue, type PolicyConfigKey } from '../lib/model-policy';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const MODELS = path.join(ROOT, 'bin', 'gstack-models');
|
||||
const CONFIG = path.join(ROOT, 'bin', 'gstack-config');
|
||||
const roots: string[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function fixture(config?: string) {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack models cli '));
|
||||
roots.push(root);
|
||||
const state = path.join(root, 'state');
|
||||
const home = path.join(root, 'home');
|
||||
fs.mkdirSync(state);
|
||||
fs.mkdirSync(home);
|
||||
if (config !== undefined) fs.writeFileSync(path.join(state, 'config.yaml'), config);
|
||||
return { root, state, home, env: { PATH: process.env.PATH ?? '', HOME: home, GSTACK_STATE_ROOT: state, BUN_RUNTIME_TRANSPILER_CACHE_PATH: '0' } as Record<string, string> };
|
||||
}
|
||||
|
||||
function models(args: string[], env: Record<string, string>, cwd = path.dirname(env.HOME)) {
|
||||
const result = spawnSync(process.execPath, [MODELS, ...args], { encoding: 'utf-8', env, cwd, timeout: 30_000 });
|
||||
return { code: result.status, stdout: result.stdout ?? '', stderr: result.stderr ?? '' };
|
||||
}
|
||||
|
||||
function config(args: string[], env: Record<string, string>) {
|
||||
const result = spawnSync('bash', [CONFIG, ...args], { encoding: 'utf-8', env, timeout: 30_000 });
|
||||
return { code: result.status, stdout: result.stdout ?? '', stderr: result.stderr ?? '' };
|
||||
}
|
||||
|
||||
function json(args: string[], env: Record<string, string>, cwd?: string) {
|
||||
const result = models([...args, '--json'], env, cwd);
|
||||
expect(result.stderr).toBe('');
|
||||
return { code: result.code, doc: JSON.parse(result.stdout) };
|
||||
}
|
||||
|
||||
function tree(dir: string): string[] {
|
||||
return fs.readdirSync(dir, { recursive: true }).map(String).sort();
|
||||
}
|
||||
|
||||
describe('gstack-models inspector', () => {
|
||||
test('--help explains roles, values, offline behavior, delegation and exits', () => {
|
||||
const result = models(['--help'], fixture().env);
|
||||
expect(result.code).toBe(0);
|
||||
for (const text of ['plan-review', 'implementation', 'frontier | smart | host', 'frontier | smart ', 'offline', 'no paid model call', 'host-controlled', 'Exit: 0', '2 invalid usage', 'schemaVersion 1']) {
|
||||
expect(result.stdout).toContain(text);
|
||||
}
|
||||
});
|
||||
|
||||
test('list is the default and emits one JSON document with both roles and tier entries', () => {
|
||||
const { env, state } = fixture();
|
||||
const human = models([], env);
|
||||
expect(human.code).toBe(0);
|
||||
expect(human.stdout).toContain('plan-review via openai: gpt-6-astra');
|
||||
expect(human.stdout).toContain('implementation via anthropic: claude-opus-5-5');
|
||||
expect(human.stdout).toContain('gstack-config set plan_review_tier smart');
|
||||
expect(human.stderr).toBe('');
|
||||
expect(models(['list'], env).stdout).toBe(human.stdout);
|
||||
|
||||
const { code, doc } = json([], env);
|
||||
expect(code).toBe(0);
|
||||
expect(doc).toMatchObject({ schemaVersion: 1, command: 'list', ok: true, offline: true, configPath: path.join(state, 'config.yaml'), errors: [] });
|
||||
expect(doc.catalog).toMatchObject({ version: 1, sha256: modelCatalogSha256() });
|
||||
expect(doc.selections.map((s: { role: string; provider: string }) => `${s.role}/${s.provider}`)).toEqual(['plan-review/anthropic', 'plan-review/openai', 'implementation/anthropic', 'implementation/openai']);
|
||||
expect(doc.tiers).toHaveLength(4);
|
||||
expect(doc.settings.plan_review_tier).toEqual({ value: null, effective: 'frontier', origin: 'default' });
|
||||
expect(doc.settings.model_smart_openai).toEqual({ value: null, effective: 'gpt-6.1-sol', origin: 'default' });
|
||||
});
|
||||
|
||||
test('resolve plan-review returns one tagged selection with its source', () => {
|
||||
const { env } = fixture('model_frontier_openai: gpt-pinned\n');
|
||||
const { code, doc } = json(['resolve', '--role', 'plan-review', '--provider', 'openai'], { ...env, GSTACK_CLAUDE_MODEL: 'claude-ignored' });
|
||||
expect(code).toBe(0);
|
||||
expect(doc.selections).toHaveLength(1);
|
||||
expect(doc.selections[0]).toMatchObject({ status: 'selected', role: 'plan-review', provider: 'openai', tier: 'frontier', requestedModel: 'gpt-pinned', source: { kind: 'config', key: 'model_frontier_openai' } });
|
||||
const envWins = json(['resolve', '--role=plan-review', '--provider=openai'], { ...env, GSTACK_CODEX_MODEL: 'gpt-env' });
|
||||
expect(envWins.doc.selections[0]).toMatchObject({ requestedModel: 'gpt-env', source: { kind: 'env', name: 'GSTACK_CODEX_MODEL' } });
|
||||
});
|
||||
|
||||
test('resolve implementation returns both providers unless one is named', () => {
|
||||
const { env } = fixture('implementation_tier: frontier\n');
|
||||
const both = json(['resolve', '--role', 'implementation'], env);
|
||||
expect(both.code).toBe(0);
|
||||
expect(both.doc.selections.map((s: { provider: string; requestedModel: string }) => [s.provider, s.requestedModel])).toEqual([['anthropic', 'claude-fable-5-1'], ['openai', 'gpt-6-astra']]);
|
||||
const one = json(['resolve', '--role', 'implementation', '--provider', 'openai'], env);
|
||||
expect(one.doc.selections).toHaveLength(1);
|
||||
expect(one.doc.selections[0].provider).toBe('openai');
|
||||
});
|
||||
|
||||
test('host mode is a valid delegated resolution with a null model', () => {
|
||||
const { env } = fixture('plan_review_tier: host\n');
|
||||
const { code, doc } = json(['resolve', '--role', 'plan-review', '--provider', 'anthropic'], env);
|
||||
expect(code).toBe(0);
|
||||
expect(doc.selections[0]).toMatchObject({ status: 'delegated-host', requestedModel: null, source: { kind: 'host', reason: 'host-mode' } });
|
||||
const human = models(['resolve', '--role', 'plan-review', '--provider', 'anthropic'], env);
|
||||
expect(human.code).toBe(0);
|
||||
expect(human.stdout).toContain('host-controlled');
|
||||
});
|
||||
|
||||
test.each([
|
||||
[['bogus'], "unknown command 'bogus'"],
|
||||
[['resolve'], 'resolve needs --role'],
|
||||
[['resolve', '--role', 'coding'], "unknown role 'coding'"],
|
||||
[['resolve', '--role', 'plan-review'], 'needs --provider'],
|
||||
[['resolve', '--role', 'plan-review', '--provider', 'claude'], "unknown provider 'claude'"],
|
||||
[['resolve', '--role', 'implementation', '--provider', 'OpenAI'], "unknown provider 'OpenAI'"],
|
||||
[['resolve', '--role', 'plan-review', '--role', 'implementation', '--provider', 'openai'], '--role given twice'],
|
||||
[['resolve', '--role'], '--role needs a value'],
|
||||
[['list', '--provider', 'openai'], 'unknown argument'],
|
||||
])('usage error %p exits 2', (args, message) => {
|
||||
const { env } = fixture();
|
||||
const human = models(args, env);
|
||||
expect(human.code).toBe(2);
|
||||
expect(human.stdout).toBe('');
|
||||
expect(human.stderr).toContain(message);
|
||||
const { code, doc } = json(args, env);
|
||||
expect(code).toBe(2);
|
||||
expect(doc).toMatchObject({ schemaVersion: 1, ok: false, selections: [] });
|
||||
expect(doc.errors[0]).toMatchObject({ reason: 'usage' });
|
||||
expect(doc.errors[0].problem).toContain(message);
|
||||
});
|
||||
|
||||
test('invalid config exits 1 with structured, source-aware repair', () => {
|
||||
const { env } = fixture('plan_review_tier: fronteir\n');
|
||||
const { code, doc } = json([], env);
|
||||
expect(code).toBe(1);
|
||||
expect(doc).toMatchObject({ ok: false, selections: [], settings: null });
|
||||
expect(doc.errors[0]).toMatchObject({ reason: 'invalid_plan_review_tier', key: 'plan_review_tier' });
|
||||
expect(doc.errors[0].repair).toContain('gstack-config unset plan_review_tier');
|
||||
expect(doc.errors[0].docs).toContain('#model-policy-config');
|
||||
const human = models(['resolve', '--role', 'implementation'], env);
|
||||
expect(human.code).toBe(1);
|
||||
expect(human.stdout).toBe('');
|
||||
expect(human.stderr).toContain('fix: gstack-config unset plan_review_tier');
|
||||
});
|
||||
|
||||
test('an unusable environment override points at the env var', () => {
|
||||
const { env } = fixture();
|
||||
const { code, doc } = json(['resolve', '--role', 'plan-review', '--provider', 'openai'], { ...env, GSTACK_CODEX_MODEL: 'gpt 6' });
|
||||
expect(code).toBe(1);
|
||||
expect(doc.errors[0]).toMatchObject({ reason: 'invalid_model_id', key: 'GSTACK_CODEX_MODEL', role: 'plan-review', provider: 'openai' });
|
||||
expect(doc.errors[0].repair[0]).toBe('unset GSTACK_CODEX_MODEL');
|
||||
});
|
||||
|
||||
test('custom routing fails only the affected selections', () => {
|
||||
const { env } = fixture();
|
||||
const { code, doc } = json([], { ...env, CLAUDE_CODE_USE_BEDROCK: '1' });
|
||||
expect(code).toBe(1);
|
||||
expect(doc.selections.map((s: { role: string; provider: string }) => `${s.role}/${s.provider}`)).toEqual(['plan-review/openai', 'implementation/openai']);
|
||||
expect(doc.errors.map((e: { reason: string; role: string }) => `${e.reason}/${e.role}`)).toEqual(['custom_provider_requires_model/plan-review', 'custom_provider_requires_model/implementation']);
|
||||
});
|
||||
|
||||
test('project settings in the working directory are custom routing; detection limits are published', () => {
|
||||
const { env, root } = fixture();
|
||||
const project = path.join(root, 'project');
|
||||
fs.mkdirSync(path.join(project, '.claude'), { recursive: true });
|
||||
fs.writeFileSync(path.join(project, '.claude', 'settings.local.json'), JSON.stringify({ env: { ANTHROPIC_BASE_URL: 'https://proxy.example.test' } }));
|
||||
const { code, doc } = json(['resolve', '--role', 'plan-review', '--provider', 'anthropic'], env, project);
|
||||
expect(code).toBe(1);
|
||||
expect(doc.errors[0]).toMatchObject({ reason: 'custom_provider_requires_model', provider: 'anthropic' });
|
||||
expect(doc.providerDetection.unscanned.anthropic.join(' ')).toContain('server-managed');
|
||||
expect(json(['resolve', '--role', 'plan-review', '--provider', 'anthropic'], env).code).toBe(0);
|
||||
expect(models(['--help'], env).stdout).toContain('cannot see server-managed or MDM policy');
|
||||
});
|
||||
|
||||
test('inspection never writes to the state root or home', () => {
|
||||
const { root, env } = fixture('plan_review_tier: smart\n');
|
||||
const before = tree(root);
|
||||
models([], env);
|
||||
models(['--json'], env);
|
||||
models(['resolve', '--role', 'plan-review', '--provider', 'anthropic', '--json'], env);
|
||||
models(['resolve', '--role', 'implementation'], env);
|
||||
expect(tree(root)).toEqual(before);
|
||||
});
|
||||
});
|
||||
|
||||
describe('gstack-config model policy keys', () => {
|
||||
test('defaults, set, resolve again and unset restore', () => {
|
||||
const { env } = fixture();
|
||||
expect(config(['get', 'plan_review_tier'], env)).toMatchObject({ code: 0, stdout: 'frontier' });
|
||||
expect(config(['get', 'implementation_tier'], env)).toMatchObject({ code: 0, stdout: 'smart' });
|
||||
expect(config(['get', 'model_frontier_claude'], env)).toMatchObject({ code: 0, stdout: '' });
|
||||
expect(config(['defaults'], env).stdout).toContain('plan_review_tier:');
|
||||
|
||||
expect(config(['set', 'plan_review_tier', 'smart'], env).code).toBe(0);
|
||||
expect(config(['set', 'model_smart_openai', ' gpt-custom '], env).code).toBe(0);
|
||||
expect(config(['get', 'model_smart_openai'], env).stdout).toBe('gpt-custom');
|
||||
expect(json(['resolve', '--role', 'plan-review', '--provider', 'openai'], env).doc.selections[0]).toMatchObject({ tier: 'smart', requestedModel: 'gpt-custom', source: { kind: 'config' } });
|
||||
expect(config(['list'], env).stdout).toContain('model_smart_openai: gpt-custom (set)');
|
||||
|
||||
expect(config(['unset', 'plan_review_tier'], env).code).toBe(0);
|
||||
expect(config(['unset', 'model_smart_openai'], env).code).toBe(0);
|
||||
expect(json(['resolve', '--role', 'plan-review', '--provider', 'openai'], env).doc.selections[0]).toMatchObject({ tier: 'frontier', requestedModel: 'gpt-6-astra', source: { kind: 'catalog' } });
|
||||
});
|
||||
|
||||
test.each([
|
||||
['plan_review_tier', 'fronteir'],
|
||||
['plan_review_tier', ' '],
|
||||
['implementation_tier', 'host'],
|
||||
['model_frontier_openai', 'gpt 6'],
|
||||
['model_frontier_claude', 'a'.repeat(513)],
|
||||
])('invalid %s %p is rejected and the old value kept', (key, value) => {
|
||||
const { env, state } = fixture();
|
||||
const good = key.startsWith('model_') ? 'model-1' : 'frontier';
|
||||
expect(config(['set', key, good], env).code).toBe(0);
|
||||
const before = fs.readFileSync(path.join(state, 'config.yaml'), 'utf-8');
|
||||
const result = config(['set', key, value], env);
|
||||
expect(result.code).toBe(1);
|
||||
expect(result.stderr).toContain('Existing value left unchanged');
|
||||
expect(result.stderr).toContain(`gstack-config unset ${key}`);
|
||||
expect(result.stderr).toContain('#model-policy-config');
|
||||
expect(fs.readFileSync(path.join(state, 'config.yaml'), 'utf-8')).toBe(before);
|
||||
});
|
||||
|
||||
test('set and resolve share one validation rule', () => {
|
||||
const samples: Array<[PolicyConfigKey, string]> = [
|
||||
['plan_review_tier', 'frontier'], ['plan_review_tier', 'smart'], ['plan_review_tier', 'host'], ['plan_review_tier', 'Smart'],
|
||||
['implementation_tier', 'frontier'], ['implementation_tier', 'smart'], ['implementation_tier', 'host'],
|
||||
['model_frontier_claude', 'claude-opus-4-6[1m]'], ['model_frontier_claude', 'vendor+claude@2026'], ['model_frontier_claude', 'a'.repeat(512)],
|
||||
['model_frontier_claude', 'a'.repeat(513)], ['model_smart_claude', 'claude;x'], ['model_smart_claude', 'claude x'], ['model_smart_claude', '$(id)'],
|
||||
['model_frontier_openai', 'gpt-6.1-sol'], ['model_frontier_openai', 'org/model:v1'], ['model_frontier_openai', 'a'.repeat(100)],
|
||||
['model_frontier_openai', 'a'.repeat(101)], ['model_smart_openai', 'gpt+6'], ['model_smart_openai', 'gpt@6'],
|
||||
];
|
||||
const { env } = fixture();
|
||||
for (const [key, value] of samples) {
|
||||
const accepted = config(['set', key, value], env).code === 0;
|
||||
expect({ key, value: value.slice(0, 20), accepted }).toEqual({ key, value: value.slice(0, 20), accepted: validatePolicyValue(key, value) === null });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,245 @@
|
||||
import { describe, expect, test, spyOn } from 'bun:test';
|
||||
import { collectFreshnessSources, fetchOfficialSource, FRESHNESS_BOUNDS, FRESHNESS_SOURCES, lifecycleDate, parseOfficialSource, parseOfficialSourceBounded, withinDeadline } from '../lib/model-policy-freshness-sources';
|
||||
import { assessFreshness, renderFreshnessReport, storedEvidenceStatus, validEvidence, validLifecycle, validState } from '../lib/model-policy-freshness';
|
||||
import { freshnessFixture, freshnessObservation, freshnessReplacementObservation, retirementFixture } from './fixtures/model-policy-freshness.ts';
|
||||
|
||||
describe('official recommendation and lifecycle structures', () => {
|
||||
test('four official fixtures parse, with Mythos and replacement mentions excluded', () => {
|
||||
const observation = freshnessObservation();
|
||||
expect(assessFreshness(observation).status).toBe('current');
|
||||
const anthropic = parseOfficialSource('anthropic-models', freshnessFixture('anthropic-models'));
|
||||
expect(anthropic.recommendations).toEqual({ frontier: 'claude-fable-5-1', smart: 'claude-opus-5-5' });
|
||||
expect(anthropic.lineup).not.toContain('claude-mythos-5-1');
|
||||
const openai = parseOfficialSource('openai-lifecycle', freshnessFixture('openai-lifecycle'));
|
||||
expect(openai.lifecycle?.map(row => row.modelId)).toEqual(['gpt-5.3-codex', 'gpt-old', 'gpt-unicode-date', 'gpt-iso-date', 'gpt-date-floor']);
|
||||
expect(openai.lifecycle?.find(row => row.modelId === 'gpt-date-floor')).toMatchObject({ deadline: null, supportFloor: '2024-06-13' });
|
||||
});
|
||||
|
||||
test('known-family and actual unknown-family lineup replacements become candidates', () => {
|
||||
const known = freshnessFixture('anthropic-models').replaceAll('fable-5-1', 'fable-5-2').replaceAll('Fable 5.1', 'Fable 5.2');
|
||||
const knownReport = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-models': known }));
|
||||
expect(knownReport.status).toBe('update-candidate');
|
||||
expect(knownReport.candidates[0].reason).toBe('changed-monitored-family-recommendation');
|
||||
const unknown = freshnessFixture('anthropic-models').replaceAll('fable-5-1', 'mythos-5-1').replaceAll('Fable 5.1', 'Mythos 5.1');
|
||||
const report = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-models': unknown }));
|
||||
expect(report.status).toBe('update-candidate');
|
||||
expect(report.candidates[0]).toMatchObject({ proposed: 'claude-mythos-5-1', reason: 'unknown-replacement-family-needs-triage' });
|
||||
expect(report.complete).toBe(true);
|
||||
});
|
||||
|
||||
test('OpenAI full-catalog newer IDs alone are not recommendations', () => {
|
||||
const original = freshnessFixture('openai-models');
|
||||
expect(assessFreshness(freshnessObservation()).candidates).toEqual([]);
|
||||
const changed = original.replaceAll('gpt-6.1-sol', 'gpt-6.2-sol').replaceAll('GPT-6.1 Sol', 'GPT-6.2 Sol');
|
||||
expect(assessFreshness(freshnessObservation('100', undefined, { 'openai-models': changed })).candidates[0]).toMatchObject({ proposed: 'gpt-6.2-sol' });
|
||||
});
|
||||
|
||||
test('active support floors are never retirement deadlines, even after the floor', () => {
|
||||
const parsed = parseOfficialSource('anthropic-lifecycle', freshnessFixture('anthropic-lifecycle'));
|
||||
expect(parsed.lifecycle?.[0]).toMatchObject({ state: 'active', deadline: null, supportFloor: '2027-09-01' });
|
||||
expect(assessFreshness(freshnessObservation('100', '2028-10-07T12:00:00.000Z')).state.lifecycle).toEqual([]);
|
||||
});
|
||||
|
||||
test('actual API retirement and removal are urgent, incidental partner prose is ignored', () => {
|
||||
const report = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
|
||||
expect(report.status).toBe('update-candidate');
|
||||
expect(report.state.lifecycle[0]).toMatchObject({ state: 'deprecated', deadline: '2026-11-01' });
|
||||
expect(renderFreshnessReport(report)).toContain('URGENT');
|
||||
expect(renderFreshnessReport(report)).toContain('Bedrock/Vertex/custom');
|
||||
const removed = assessFreshness(freshnessObservation('100', '2026-11-02T12:00:00.000Z', { 'anthropic-lifecycle': retirementFixture() }));
|
||||
expect(removed.state.lifecycle[0].state).toBe('removed');
|
||||
});
|
||||
|
||||
test('OpenAI shutdown rows apply to the deprecated column, not the replacement', () => {
|
||||
const text = freshnessFixture('openai-lifecycle').replace('`gpt-5.3-codex`', '`gpt-6-astra`');
|
||||
const report = assessFreshness(freshnessObservation('100', undefined, { 'openai-lifecycle': text }));
|
||||
expect(report.state.lifecycle).toHaveLength(1);
|
||||
expect(report.state.lifecycle[0]).toMatchObject({ modelId: 'gpt-6-astra', deadline: '2027-04-01' });
|
||||
});
|
||||
|
||||
test('missing required structures and semantic drift fail closed', () => {
|
||||
expect(() => parseOfficialSource('anthropic-models', freshnessFixture('anthropic-models').replace('Claude API ID', 'Google Cloud ID'))).toThrow('missing-api-id-row');
|
||||
expect(() => parseOfficialSource('anthropic-models', freshnessFixture('anthropic-models').replace('for demanding reasoning', 'for unexplained goals'))).toThrow('unrecognized-recommendation');
|
||||
expect(() => parseOfficialSource('openai-models', freshnessFixture('openai-models').replace('Balance intelligence and cost.', 'Best new model.'))).toThrow('unrecognized-recommendation');
|
||||
expect(() => parseOfficialSource('openai-lifecycle', '# New deprecations\nNo model tables')).toThrow('missing-or-ambiguous-section');
|
||||
expect(() => parseOfficialSource('openai-lifecycle', freshnessFixture('openai-lifecycle').replace('`gpt-5.3-codex`', 'gpt-5.3-codex'))).toThrow('unrecognized-deprecated-model-cell');
|
||||
expect(() => parseOfficialSource('anthropic-lifecycle', freshnessFixture('anthropic-lifecycle').replace('Active', 'Probably active'))).toThrow('unrecognized-lifecycle-state');
|
||||
expect(() => lifecycleDate('February 30, 2027')).toThrow('malformed-lifecycle-date');
|
||||
expect(() => lifecycleDate('tomorrow')).toThrow('malformed-lifecycle-date');
|
||||
expect(() => parseOfficialSource('anthropic-models', `${freshnessFixture('anthropic-models')}\n## Compare models\nDuplicate`)).toThrow('ambiguous-section');
|
||||
expect(() => parseOfficialSource('anthropic-models', freshnessFixture('anthropic-models').replace('## Using the Models API', 'Use [Other](https://platform.claude.com/docs/en/models/opus-5-5/overview) for demanding reasoning.\n\n## Using the Models API'))).toThrow('unrecognized-recommendation');
|
||||
});
|
||||
});
|
||||
|
||||
describe('bounded fixed-source fetching', () => {
|
||||
test('exact byte limit passes and a streamed byte beyond it fails without trusting content-length', async () => {
|
||||
const atCap = 'x'.repeat(FRESHNESS_BOUNDS.responseBytes);
|
||||
expect((await fetchOfficialSource('openai-models', async () => new Response(atCap))).bytes).toBe(2_097_152);
|
||||
await expect(fetchOfficialSource('openai-models', async () => new Response(`${atCap}x`))).rejects.toThrow('response-too-large');
|
||||
await expect(fetchOfficialSource('openai-models', async () => new Response('x', { headers: { 'content-length': '2097153' } }))).rejects.toThrow('response-too-large');
|
||||
expect(() => parseOfficialSource('openai-models', `${atCap}x`)).toThrow('response-too-large');
|
||||
});
|
||||
|
||||
test('all redirect destinations are checked; three redirects pass, four fail', async () => {
|
||||
const credentialUrl = new URL(FRESHNESS_SOURCES[2].url);
|
||||
credentialUrl.username = 'user';
|
||||
credentialUrl.password = 'pass';
|
||||
for (const location of ['https://evil.example/models.md', 'http://developers.openai.com/api/docs/models.md', credentialUrl.href, 'https://developers.openai.com/api/docs/models.md?untrusted=1']) {
|
||||
let calls = 0;
|
||||
await expect(fetchOfficialSource('openai-models', async () => { calls++; return new Response(null, { status: 302, headers: { location } }); })).rejects.toThrow('redirect-not-allowlisted');
|
||||
expect(calls).toBe(1);
|
||||
}
|
||||
let calls = 0;
|
||||
expect((await fetchOfficialSource('openai-models', async () => ++calls <= 3 ? new Response(null, { status: 302, headers: { location: FRESHNESS_SOURCES[2].url } }) : new Response('data'))).text).toBe('data');
|
||||
calls = 0;
|
||||
await expect(fetchOfficialSource('openai-models', async () => { calls++; return new Response(null, { status: 302, headers: { location: FRESHNESS_SOURCES[2].url } }); })).rejects.toThrow('redirect-limit');
|
||||
expect(calls).toBe(4);
|
||||
});
|
||||
|
||||
test('per-response and whole-check clocks are installed at the exact approved bounds', async () => {
|
||||
const installed: number[] = [];
|
||||
const real = globalThis.setTimeout;
|
||||
const timers = spyOn(globalThis, 'setTimeout').mockImplementation(((callback: () => void, milliseconds: number) => {
|
||||
installed.push(milliseconds);
|
||||
return real(callback, milliseconds === 30_000 ? 5 : milliseconds);
|
||||
}) as typeof setTimeout);
|
||||
try {
|
||||
const result = await collectFreshnessSources(async () => new Promise<Response>(() => {}));
|
||||
expect(result.every(source => source.error === 'deadline-exceeded')).toBe(true);
|
||||
expect(installed.filter(ms => ms === 30_000)).toHaveLength(4);
|
||||
expect(installed).toContain(180_000);
|
||||
} finally { timers.mockRestore(); }
|
||||
const controller = new AbortController();
|
||||
await expect(withinDeadline(new Promise(() => {}), 5, controller)).rejects.toThrow('deadline-exceeded');
|
||||
expect(controller.signal.aborted).toBe(true);
|
||||
});
|
||||
|
||||
test('only four fixed sources are fetched and failures never become parseable success', async () => {
|
||||
const urls: string[] = [];
|
||||
const result = await collectFreshnessSources(async (url, init) => {
|
||||
urls.push(url);
|
||||
expect(init.redirect).toBe('manual');
|
||||
const id = FRESHNESS_SOURCES.find(source => source.url === url)!.id;
|
||||
return id === 'openai-lifecycle' ? new Response('authentication needed', { status: 403 }) : new Response(freshnessFixture(id));
|
||||
});
|
||||
expect(urls.sort()).toEqual(FRESHNESS_SOURCES.map(source => source.url).sort());
|
||||
expect(result.find(source => source.id === 'openai-lifecycle')?.error).toBe('http-403');
|
||||
const observation = freshnessObservation();
|
||||
observation.sources = result;
|
||||
expect(assessFreshness(observation).status).toBe('unknown/source-unavailable');
|
||||
});
|
||||
|
||||
test('CPU-bound parsing is terminable and the complete-check deadline stops all blocked parsers', async () => {
|
||||
let terminations = 0;
|
||||
const createWorker = () => ({ postMessage() {}, terminate() { terminations++; }, onmessage: null, onerror: null }) as unknown as Worker;
|
||||
const controller = new AbortController();
|
||||
const blocked = parseOfficialSourceBounded('openai-models', freshnessFixture('openai-models'), controller.signal, createWorker);
|
||||
controller.abort();
|
||||
await expect(blocked).rejects.toThrow('deadline-exceeded');
|
||||
expect(terminations).toBe(1);
|
||||
const real = globalThis.setTimeout;
|
||||
const timers = spyOn(globalThis, 'setTimeout').mockImplementation(((callback: () => void, ms: number) => real(callback, ms === 180_000 ? 5 : ms)) as typeof setTimeout);
|
||||
try {
|
||||
const result = await collectFreshnessSources(async url => new Response(freshnessFixture(FRESHNESS_SOURCES.find(source => source.url === url)!.id)), createWorker);
|
||||
expect(result.every(source => source.error && !source.receipt)).toBe(true);
|
||||
} finally { timers.mockRestore(); }
|
||||
expect(terminations).toBe(5);
|
||||
});
|
||||
});
|
||||
|
||||
describe('trustworthy evidence and deterministic statuses', () => {
|
||||
test('source-check evidence is separate from catalog qualification and survives artifact expiry', () => {
|
||||
const report = assessFreshness(freshnessObservation());
|
||||
expect(validState(report.state)).toBe(true);
|
||||
expect(validEvidence(report.state.lastSuccess)).toBe(true);
|
||||
expect(report.state.lastSuccess?.nextCheckDueBy).toBe('2026-10-14T12:00:00.000Z');
|
||||
expect(report.state.lastSuccess?.sources).toHaveLength(4);
|
||||
expect(report.state.lastSuccess?.catalog).toEqual(freshnessObservation().catalog);
|
||||
expect(storedEvidenceStatus(freshnessObservation().catalog, JSON.parse(JSON.stringify(report.state.lastSuccess)), '2026-10-08T12:00:00.000Z')).toBe('current');
|
||||
expect(renderFreshnessReport(report)).toContain('not automatic liveness monitoring');
|
||||
});
|
||||
|
||||
test('missing/corrupt/mismatched/future evidence is unknown; thirty-day staleness has an exact boundary', () => {
|
||||
const observation = freshnessObservation();
|
||||
const evidence = assessFreshness(observation).state.lastSuccess!;
|
||||
expect(storedEvidenceStatus(observation.catalog, null, observation.checkedAt)).toBe('unknown/source-unavailable');
|
||||
expect(storedEvidenceStatus({ ...observation.catalog, sha256: 'b'.repeat(64) }, evidence, observation.checkedAt)).toBe('unknown/source-unavailable');
|
||||
expect(storedEvidenceStatus(observation.catalog, { ...evidence, parserVersion: 99 }, observation.checkedAt)).toBe('unknown/source-unavailable');
|
||||
expect(storedEvidenceStatus(observation.catalog, evidence, '2026-10-06T12:00:00.000Z')).toBe('unknown/source-unavailable');
|
||||
expect(storedEvidenceStatus(observation.catalog, evidence, '2026-11-06T12:00:00.000Z')).toBe('current');
|
||||
expect(storedEvidenceStatus(observation.catalog, evidence, '2026-11-06T12:00:00.001Z')).toBe('stale');
|
||||
});
|
||||
|
||||
test('partial recovery cannot refresh evidence or erase an unresolved retirement', () => {
|
||||
const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
|
||||
const next = assessFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z', { 'openai-lifecycle': null }), initial.state);
|
||||
expect(next.status).toBe('unknown/source-unavailable');
|
||||
expect(next.state.lastSuccess).toEqual(initial.state.lastSuccess);
|
||||
expect(next.state.lifecycle).toEqual(initial.state.lifecycle);
|
||||
const old = assessFreshness(freshnessObservation('102', '2026-11-08T12:00:00.000Z', { 'openai-lifecycle': null }), next.state);
|
||||
expect(old.evidenceStatus).toBe('stale');
|
||||
expect(old.status).toBe('unknown/source-unavailable');
|
||||
const recovered = assessFreshness(freshnessObservation('103', '2026-11-09T12:00:00.000Z'), old.state);
|
||||
expect(recovered.status).toBe('update-candidate');
|
||||
expect(recovered.state.lifecycle).toEqual(old.state.lifecycle);
|
||||
});
|
||||
|
||||
test('corrupt duplicate lifecycle keys and source receipts cannot certify current', () => {
|
||||
const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
|
||||
expect(validLifecycle([initial.state.lifecycle[0], initial.state.lifecycle[0]])).toBe(false);
|
||||
const observation = freshnessObservation();
|
||||
observation.sources[0].receipt!.finalUrl = 'https://evil.example/docs';
|
||||
expect(assessFreshness(observation).status).toBe('unknown/source-unavailable');
|
||||
const evidence = structuredClone(initial.state.lastSuccess!);
|
||||
evidence.sources[0] = evidence.sources[1];
|
||||
expect(validEvidence(evidence)).toBe(false);
|
||||
expect(validState({ ...initial.state, lastSuccess: evidence })).toBe(false);
|
||||
});
|
||||
|
||||
test('a partial run cannot downgrade an unresolved retirement to legacy or extend its deadline', () => {
|
||||
const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
|
||||
const legacy = freshnessFixture('anthropic-lifecycle').replace('claude-fable-5-1 | Active', 'claude-fable-5-1 | Legacy');
|
||||
const report = assessFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z', { 'anthropic-lifecycle': legacy, 'openai-lifecycle': null }), initial.state);
|
||||
expect(report.state.lifecycle[0]).toMatchObject({ state: 'deprecated', deadline: '2026-11-01' });
|
||||
expect(report.state.lastSuccess).toEqual(initial.state.lastSuccess);
|
||||
});
|
||||
|
||||
test('only a complete check of a reviewed replacement catalog resolves removed identities', () => {
|
||||
const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
|
||||
const replacement = freshnessObservation('101', '2026-10-08T12:00:00.000Z');
|
||||
replacement.catalog.models[0].modelId = 'claude-fable-5-2';
|
||||
replacement.catalog.sha256 = 'b'.repeat(64);
|
||||
replacement.sources = freshnessObservation('101', replacement.checkedAt, {
|
||||
'anthropic-models': freshnessFixture('anthropic-models').replaceAll('fable-5-1', 'fable-5-2'),
|
||||
'anthropic-lifecycle': freshnessFixture('anthropic-lifecycle').replaceAll('fable-5-1', 'fable-5-2'),
|
||||
}).sources;
|
||||
const incomplete = structuredClone(replacement);
|
||||
incomplete.sources[3] = { id: 'openai-lifecycle', error: 'http-503' };
|
||||
expect(assessFreshness(incomplete, initial.state).state.lifecycle[0].resolvedByCatalog).toBeUndefined();
|
||||
const complete = assessFreshness(replacement, initial.state);
|
||||
expect(complete.status).toBe('current');
|
||||
expect(complete.state.lifecycle[0].resolvedByCatalog).toBe('b'.repeat(64));
|
||||
});
|
||||
|
||||
test('complete catalog rollback reopens the historical retirement even when the current source row is active', () => {
|
||||
const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
|
||||
const replacement = assessFreshness(freshnessReplacementObservation(), initial.state);
|
||||
expect(replacement.status).toBe('current');
|
||||
expect(replacement.state.lifecycle[0].resolvedByCatalog).toBe('b'.repeat(64));
|
||||
const rollback = assessFreshness(freshnessObservation('102', '2026-10-09T12:00:00.000Z'), replacement.state);
|
||||
expect(rollback.status).toBe('update-candidate');
|
||||
expect(rollback.state.lifecycle[0]).toMatchObject({ state: 'deprecated', deadline: '2026-11-01', firstObservedAt: initial.state.lifecycle[0].firstObservedAt });
|
||||
expect(rollback.state.lifecycle[0].resolvedByCatalog).toBeUndefined();
|
||||
expect(replacement.state.lifecycle[0].resolvedByCatalog).toBe('b'.repeat(64));
|
||||
});
|
||||
|
||||
test('source-unavailable catalog rollback reopens history without refreshing prior replacement evidence', () => {
|
||||
const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
|
||||
const replacement = assessFreshness(freshnessReplacementObservation(), initial.state);
|
||||
const rollback = assessFreshness(freshnessObservation('102', '2026-10-09T12:00:00.000Z', { 'anthropic-lifecycle': null, 'openai-lifecycle': null }), replacement.state);
|
||||
expect(rollback.status).toBe('unknown/source-unavailable');
|
||||
expect(rollback.state.lastSuccess).toEqual(replacement.state.lastSuccess);
|
||||
expect(rollback.state.lifecycle[0]).toMatchObject({ state: 'deprecated', deadline: '2026-11-01' });
|
||||
expect(rollback.state.lifecycle[0].resolvedByCatalog).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,405 @@
|
||||
import { describe, expect, test, spyOn } from 'bun:test';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { MODEL_CATALOG, modelCatalogSha256 } from '../lib/model-catalog';
|
||||
import { assessFreshness, freshnessCatalogIdentity } from '../lib/model-policy-freshness';
|
||||
import { FRESHNESS_SOURCES, FRESHNESS_BOUNDS, sha256 } from '../lib/model-policy-freshness-sources';
|
||||
import { FRESHNESS_ISSUE_AUTHOR, FRESHNESS_ISSUE_MARKER, FRESHNESS_REGION_BEGIN, FRESHNESS_REGION_END, publishFreshness, readFreshnessIssue, writeFreshnessIssue, type FreshnessGitHub, type TrackingIssue } from '../lib/model-policy-freshness-publication';
|
||||
import { freshnessGitHub } from '../lib/model-policy-freshness-github';
|
||||
import { runModelPolicyFreshness } from '../scripts/model-policy-freshness';
|
||||
import { freshnessCatalog, freshnessFixture, freshnessObservation, freshnessReplacementObservation, retirementFixture } from './fixtures/model-policy-freshness.ts';
|
||||
|
||||
function mockGitHub(initial: (Omit<TrackingIssue, 'author'> & { author?: string })[] = []) {
|
||||
const issues: TrackingIssue[] = structuredClone(initial).map(issue => ({ author: FRESHNESS_ISSUE_AUTHOR, ...issue }));
|
||||
const calls: string[] = [];
|
||||
let identity = { defaultBranch: 'main', sourceSha256: freshnessCatalog.sourceSha256 };
|
||||
const github: FreshnessGitHub = {
|
||||
async defaultCatalog() { calls.push('catalog'); return identity; },
|
||||
async listIssues() { calls.push('list'); return structuredClone(issues); },
|
||||
async readIssue(number) { calls.push('read'); return structuredClone(issues.find(issue => issue.number === number)!); },
|
||||
async createIssue(body) { calls.push('create'); const issue: TrackingIssue = { number: issues.length + 1, body, state: 'open', author: FRESHNESS_ISSUE_AUTHOR }; issues.push(issue); return structuredClone(issue); },
|
||||
async updateIssue(number, update) { calls.push('update'); Object.assign(issues.find(issue => issue.number === number)!, update); },
|
||||
};
|
||||
return { github, issues, calls, setIdentity(value: typeof identity) { identity = value; } };
|
||||
}
|
||||
|
||||
describe('stable-marker issue ownership and evidence lifecycle', () => {
|
||||
test('healthy bootstrap creates one closed issue, retains evidence when closed, and repeat publication is idempotent', async () => {
|
||||
const api = mockGitHub();
|
||||
const observation = freshnessObservation();
|
||||
const created = await publishFreshness(observation, api.github, 'refs/heads/main');
|
||||
expect(created.action).toBe('created');
|
||||
expect(api.issues).toHaveLength(1);
|
||||
expect(api.issues[0].state).toBe('closed');
|
||||
expect(api.issues[0].body).toContain(FRESHNESS_ISSUE_MARKER);
|
||||
expect(readFreshnessIssue(api.issues[0].body).state?.lastSuccess?.checkedAt).toBe(observation.checkedAt);
|
||||
const body = api.issues[0].body;
|
||||
expect((await publishFreshness(observation, api.github, 'refs/heads/main')).action).toBe('rejected-older-observation');
|
||||
expect(api.issues[0].body).toBe(body);
|
||||
const next = freshnessObservation('101', '2026-10-08T12:00:00.000Z');
|
||||
expect((await publishFreshness(next, api.github, 'refs/heads/main')).action).toBe('updated');
|
||||
expect(api.issues).toHaveLength(1);
|
||||
expect(api.issues[0].state).toBe('closed');
|
||||
});
|
||||
|
||||
test('marker owns the issue, title collisions and human body/disposition text are untouched', async () => {
|
||||
const initial = writeFreshnessIssue('', assessFreshness(freshnessObservation()));
|
||||
const humanPrefix = 'Human notes: retain this default until qualification.\n';
|
||||
const humanSuffix = '\nHuman disposition: mitigation pending; do not delete.\n';
|
||||
const api = mockGitHub([
|
||||
{ number: 40, body: 'Model-policy freshness: advisory maintenance', state: 'open' },
|
||||
{ number: 41, body: humanPrefix + initial + humanSuffix, state: 'closed' },
|
||||
]);
|
||||
await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z', { 'anthropic-lifecycle': retirementFixture() }), api.github, 'refs/heads/main');
|
||||
expect(api.issues[0].body).toBe('Model-policy freshness: advisory maintenance');
|
||||
expect(api.issues[1].body.startsWith(humanPrefix)).toBe(true);
|
||||
expect(api.issues[1].body.endsWith(humanSuffix)).toBe(true);
|
||||
expect(api.issues[1].state).toBe('open');
|
||||
expect(readFreshnessIssue(api.issues[1].body).lifecycle[0].state).toBe('deprecated');
|
||||
});
|
||||
|
||||
test('retirement survives source failure and partial recovery; issue never closes on either', async () => {
|
||||
const api = mockGitHub();
|
||||
await publishFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }), api.github, 'refs/heads/main');
|
||||
const evidence = readFreshnessIssue(api.issues[0].body).state!.lastSuccess;
|
||||
const failure = await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z', { 'anthropic-lifecycle': null, 'openai-lifecycle': null }), api.github, 'refs/heads/main');
|
||||
expect(failure.report?.status).toBe('unknown/source-unavailable');
|
||||
expect(api.issues[0].state).toBe('open');
|
||||
expect(readFreshnessIssue(api.issues[0].body).state?.lastSuccess).toEqual(evidence);
|
||||
await publishFreshness(freshnessObservation('102', '2026-10-09T12:00:00.000Z', { 'openai-lifecycle': null }), api.github, 'refs/heads/main');
|
||||
expect(readFreshnessIssue(api.issues[0].body).lifecycle[0].deadline).toBe('2026-11-01');
|
||||
expect(api.issues[0].state).toBe('open');
|
||||
});
|
||||
|
||||
test('both complete and source-unavailable catalog rollbacks reopen the closed replacement issue', async () => {
|
||||
for (const unavailable of [false, true]) {
|
||||
const api = mockGitHub();
|
||||
await publishFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }), api.github, 'refs/heads/main');
|
||||
const replacement = freshnessReplacementObservation();
|
||||
api.setIdentity({ defaultBranch: 'main', sourceSha256: replacement.catalog.sourceSha256 });
|
||||
await publishFreshness(replacement, api.github, 'refs/heads/main');
|
||||
expect(api.issues[0].state).toBe('closed');
|
||||
expect(readFreshnessIssue(api.issues[0].body).lifecycle[0].resolvedByCatalog).toBe(replacement.catalog.sha256);
|
||||
api.setIdentity({ defaultBranch: 'main', sourceSha256: freshnessCatalog.sourceSha256 });
|
||||
const rollback = freshnessObservation('102', '2026-10-09T12:00:00.000Z', unavailable ? { 'anthropic-lifecycle': null, 'openai-lifecycle': null } : {});
|
||||
const result = await publishFreshness(rollback, api.github, 'refs/heads/main');
|
||||
expect(result.report?.status).toBe(unavailable ? 'unknown/source-unavailable' : 'update-candidate');
|
||||
expect(api.issues[0].state).toBe('open');
|
||||
expect(api.issues).toHaveLength(1);
|
||||
const stored = readFreshnessIssue(api.issues[0].body);
|
||||
expect(stored.lifecycle[0]).toMatchObject({ state: 'deprecated', deadline: '2026-11-01' });
|
||||
expect(stored.lifecycle[0].resolvedByCatalog).toBeUndefined();
|
||||
if (unavailable) expect(stored.state?.lastSuccess?.catalog.sha256).toBe(replacement.catalog.sha256);
|
||||
}
|
||||
});
|
||||
|
||||
test('corrupt state salvages independent lifecycle history and requires explicit manual repair', async () => {
|
||||
const initial = writeFreshnessIssue('', assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() })));
|
||||
const broken = initial.replace(/(<!-- gstack:model-policy-freshness:state:begin -->\n)[^\n]+/, '$1{broken');
|
||||
const api = mockGitHub([{ number: 7, body: `Human retain decision\n${broken}\nHuman footer`, state: 'closed' }]);
|
||||
expect(readFreshnessIssue(api.issues[0].body).state).toBeNull();
|
||||
expect(readFreshnessIssue(api.issues[0].body).lifecycle).toHaveLength(1);
|
||||
const result = await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z'), api.github, 'refs/heads/main');
|
||||
expect(result.report?.status).toBe('unknown/source-unavailable');
|
||||
expect(result.report?.state.lastSuccess).toBeNull();
|
||||
expect(result.report?.state.lifecycle).toHaveLength(1);
|
||||
expect(api.issues[0].state).toBe('open');
|
||||
expect(api.issues[0].body).toContain('Human retain decision');
|
||||
expect(api.issues[0].body).toContain('Human footer');
|
||||
expect(api.issues[0].body).toContain(Buffer.from(broken.slice(broken.indexOf(FRESHNESS_REGION_BEGIN) + FRESHNESS_REGION_BEGIN.length, broken.indexOf(FRESHNESS_REGION_END)).trim(), 'utf8').toString('base64'));
|
||||
const next = await publishFreshness(freshnessObservation('102', '2026-10-09T12:00:00.000Z'), api.github, 'refs/heads/main');
|
||||
expect(next.report?.state.recoveryRequired).toBe(true);
|
||||
});
|
||||
|
||||
test('missing machine state is recovery, malformed owned boundary is not overwritten, and output is bounded', async () => {
|
||||
const api = mockGitHub([{ number: 9, body: `${FRESHNESS_ISSUE_MARKER}\nMaintainer notes`, state: 'closed' }]);
|
||||
const result = await publishFreshness(freshnessObservation(), api.github, 'refs/heads/main');
|
||||
expect(result.report?.status).toBe('unknown/source-unavailable');
|
||||
expect(api.issues[0].state).toBe('open');
|
||||
expect(api.issues[0].body).toContain('Maintainer notes');
|
||||
expect(() => writeFreshnessIssue(`${FRESHNESS_ISSUE_MARKER}\n${FRESHNESS_REGION_BEGIN}\nbroken`, assessFreshness(freshnessObservation()))).toThrow('corrupt-owned-region');
|
||||
expect(() => writeFreshnessIssue('x'.repeat(FRESHNESS_BOUNDS.issueBytes), assessFreshness(freshnessObservation()))).toThrow('tracking-issue-size-limit');
|
||||
expect(() => writeFreshnessIssue(`${FRESHNESS_REGION_END}\n${FRESHNESS_REGION_BEGIN}`, assessFreshness(freshnessObservation()))).toThrow('corrupt-owned-region');
|
||||
});
|
||||
|
||||
test('duplicate markers fail visibly rather than selecting an arbitrary issue', async () => {
|
||||
const api = mockGitHub([{ number: 1, body: FRESHNESS_ISSUE_MARKER, state: 'open' }, { number: 2, body: FRESHNESS_ISSUE_MARKER, state: 'closed' }]);
|
||||
await expect(publishFreshness(freshnessObservation(), api.github, 'refs/heads/main')).rejects.toThrow('duplicate-tracking-marker');
|
||||
expect(api.calls).not.toContain('update');
|
||||
expect(api.calls).not.toContain('create');
|
||||
});
|
||||
|
||||
test('user-created marker copies never become authority or trigger replacement bootstrap', async () => {
|
||||
const body = writeFreshnessIssue('', assessFreshness(freshnessObservation()));
|
||||
for (const includeOwned of [false, true]) {
|
||||
const api = mockGitHub([{ number: 1, body, state: 'closed', author: 'untrusted-user' }, ...(includeOwned ? [{ number: 2, body, state: 'closed' as const }] : [])]);
|
||||
await expect(publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z'), api.github, 'refs/heads/main')).rejects.toThrow('untrusted-tracking-marker-owner');
|
||||
expect(api.calls).not.toContain('read');
|
||||
expect(api.calls).not.toContain('update');
|
||||
expect(api.calls).not.toContain('create');
|
||||
}
|
||||
});
|
||||
|
||||
test('CRLF and CR-only marker lines retain retirement history and publish to the same issue on source failure', async () => {
|
||||
const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
|
||||
for (const newline of ['\r\n', '\r']) {
|
||||
const prefix = `Human prefix remains byte-identical.${newline}${newline}`;
|
||||
const suffix = `${newline}${newline}Human suffix remains byte-identical.${newline}`;
|
||||
const body = prefix + writeFreshnessIssue('', initial).replaceAll('\n', newline) + suffix;
|
||||
const api = mockGitHub([{ number: 17, body, state: 'open' }]);
|
||||
const result = await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z', { 'anthropic-lifecycle': null, 'openai-lifecycle': null }), api.github, 'refs/heads/main');
|
||||
expect(result.action).toBe('updated');
|
||||
expect(result.issueNumber).toBe(17);
|
||||
expect(api.issues).toHaveLength(1);
|
||||
expect(api.calls).not.toContain('create');
|
||||
expect(api.issues[0].body.startsWith(prefix)).toBe(true);
|
||||
expect(api.issues[0].body.endsWith(suffix)).toBe(true);
|
||||
const stored = readFreshnessIssue(api.issues[0].body);
|
||||
expect(stored.lifecycle).toEqual(initial.state.lifecycle);
|
||||
expect(stored.state?.lastSuccess).toEqual(initial.state.lastSuccess);
|
||||
expect(result.report?.status).toBe('unknown/source-unavailable');
|
||||
}
|
||||
});
|
||||
|
||||
test('inline and quoted marker mentions preserve human prefix and suffix outside the real region', async () => {
|
||||
const initial = assessFreshness(freshnessObservation());
|
||||
const prefix = `Human note mentioning ${FRESHNESS_REGION_BEGIN} inline; preserve this entire sentence.\r\n> ${FRESHNESS_REGION_END}\r\nQuoted '${FRESHNESS_REGION_BEGIN}' and '${FRESHNESS_REGION_END}' are human prose.\r\n\r\n`;
|
||||
const suffix = `\r\nHuman suffix mentions ${FRESHNESS_REGION_END} inline.\r\n> ${FRESHNESS_REGION_BEGIN}\r\n`;
|
||||
const body = prefix + writeFreshnessIssue('', initial) + suffix;
|
||||
const report = assessFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z'), initial.state);
|
||||
const rewritten = writeFreshnessIssue(body, report);
|
||||
expect(rewritten.startsWith(prefix)).toBe(true);
|
||||
expect(rewritten.endsWith(suffix)).toBe(true);
|
||||
expect(readFreshnessIssue(rewritten).state).toEqual(report.state);
|
||||
const api = mockGitHub([{ number: 19, body, state: 'closed' }]);
|
||||
const result = await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z'), api.github, 'refs/heads/main');
|
||||
expect(result.action).toBe('updated');
|
||||
expect(api.issues).toHaveLength(1);
|
||||
expect(api.issues[0].body.startsWith(prefix)).toBe(true);
|
||||
expect(api.issues[0].body.endsWith(suffix)).toBe(true);
|
||||
});
|
||||
|
||||
test('malformed or missing top markers with owned evidence never become a healthy bootstrap', async () => {
|
||||
const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
|
||||
for (const marker of ['', `${FRESHNESS_ISSUE_MARKER} `, FRESHNESS_ISSUE_MARKER.replace(':v1', ':v2'), FRESHNESS_ISSUE_MARKER.replace('<!-- ', '<!--'), FRESHNESS_ISSUE_MARKER.slice(0, -1)]) {
|
||||
const body = writeFreshnessIssue('', initial).replace(FRESHNESS_ISSUE_MARKER, marker);
|
||||
const api = mockGitHub([{ number: 23, body, state: 'open' }]);
|
||||
await expect(publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z'), api.github, 'refs/heads/main')).rejects.toThrow('corrupt-tracking-marker');
|
||||
expect(api.issues).toHaveLength(1);
|
||||
expect(api.issues[0].body).toBe(body);
|
||||
expect(api.calls).not.toContain('create');
|
||||
expect(api.calls).not.toContain('update');
|
||||
expect(readFreshnessIssue(body).lifecycle).toEqual(initial.state.lifecycle);
|
||||
expect(readFreshnessIssue(body).recoveryRequired).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
test('standalone malformed top markers without readable regions block healthy bootstrap', async () => {
|
||||
for (const marker of [` ${FRESHNESS_ISSUE_MARKER}`, `${FRESHNESS_ISSUE_MARKER}\t`, FRESHNESS_ISSUE_MARKER.replace(':v1', ':v2'), FRESHNESS_ISSUE_MARKER.replace(':v1', '-v1')]) {
|
||||
const body = `Human text\r\n${marker}\r\nRetirement history needs manual recovery.`;
|
||||
const api = mockGitHub([{ number: 29, body, state: 'open' }]);
|
||||
await expect(publishFreshness(freshnessObservation(), api.github, 'refs/heads/main')).rejects.toThrow('corrupt-tracking-marker');
|
||||
expect(api.issues[0].body).toBe(body);
|
||||
expect(api.calls).not.toContain('create');
|
||||
expect(api.calls).not.toContain('update');
|
||||
}
|
||||
});
|
||||
|
||||
test('malformed owned boundaries salvage available lifecycle history but never certify a healthy publication', async () => {
|
||||
const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
|
||||
const body = writeFreshnessIssue('', initial).replace(FRESHNESS_REGION_BEGIN, FRESHNESS_REGION_BEGIN.replace(':begin', ':begn')).replace(FRESHNESS_REGION_END, FRESHNESS_REGION_END.replace(':end', ':en'));
|
||||
const stored = readFreshnessIssue(body);
|
||||
expect(stored.lifecycle).toEqual(initial.state.lifecycle);
|
||||
expect(stored.recoveryRequired).toBe(true);
|
||||
const api = mockGitHub([{ number: 31, body, state: 'open' }]);
|
||||
await expect(publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z'), api.github, 'refs/heads/main')).rejects.toThrow('corrupt-tracking-marker');
|
||||
expect(api.issues[0].body).toBe(body);
|
||||
expect(api.calls).not.toContain('create');
|
||||
expect(api.calls).not.toContain('update');
|
||||
});
|
||||
});
|
||||
|
||||
describe('publication ordering and default-branch guards', () => {
|
||||
test('both completion orders retain the newer unresolved retirement; old rerun is rejected even with a new observation date', async () => {
|
||||
for (const newerFirst of [false, true]) {
|
||||
const api = mockGitHub();
|
||||
const older = freshnessObservation('100');
|
||||
const newer = freshnessObservation('101', '2026-10-08T12:00:00.000Z', { 'anthropic-lifecycle': retirementFixture() });
|
||||
for (const observation of newerFirst ? [newer, older] : [older, newer]) await publishFreshness(observation, api.github, 'refs/heads/main');
|
||||
expect(api.issues[0].state).toBe('open');
|
||||
expect(readFreshnessIssue(api.issues[0].body).state?.latest.run.id).toBe('101');
|
||||
expect(readFreshnessIssue(api.issues[0].body).lifecycle[0].modelId).toBe('claude-fable-5-1');
|
||||
const rerun = freshnessObservation('100', '2026-10-09T12:00:00.000Z');
|
||||
rerun.run.attempt = 2;
|
||||
expect((await publishFreshness(rerun, api.github, 'refs/heads/main')).action).toBe('rejected-older-observation');
|
||||
}
|
||||
});
|
||||
|
||||
test('higher run ID cannot publish an older check timestamp, but a newer attempt can advance the same run', async () => {
|
||||
const api = mockGitHub();
|
||||
await publishFreshness(freshnessObservation('100', '2026-10-08T12:00:00.000Z'), api.github, 'refs/heads/main');
|
||||
expect((await publishFreshness(freshnessObservation('101', '2026-10-07T12:00:00.000Z'), api.github, 'refs/heads/main')).action).toBe('rejected-older-observation');
|
||||
const rerun = freshnessObservation('100', '2026-10-09T12:00:00.000Z');
|
||||
rerun.run.attempt = 2;
|
||||
expect((await publishFreshness(rerun, api.github, 'refs/heads/main')).action).toBe('updated');
|
||||
});
|
||||
|
||||
test('non-main is report-only with zero GitHub operations; a non-main default disables writes', async () => {
|
||||
const api = mockGitHub();
|
||||
expect((await publishFreshness(freshnessObservation(), api.github, 'refs/heads/feature')).action).toBe('report-only');
|
||||
expect(api.calls).toEqual([]);
|
||||
api.setIdentity({ defaultBranch: 'develop', sourceSha256: freshnessCatalog.sourceSha256 });
|
||||
await expect(publishFreshness(freshnessObservation(), api.github, 'refs/heads/main')).rejects.toThrow('default-branch-is-not-main');
|
||||
expect(api.calls).not.toContain('create');
|
||||
});
|
||||
|
||||
test('obsolete local catalog and catalog change immediately before write are rejected', async () => {
|
||||
const api = mockGitHub();
|
||||
api.setIdentity({ defaultBranch: 'main', sourceSha256: 'b'.repeat(64) });
|
||||
expect((await publishFreshness(freshnessObservation(), api.github, 'refs/heads/main')).action).toBe('rejected-obsolete-catalog');
|
||||
expect(api.calls).not.toContain('list');
|
||||
api.setIdentity({ defaultBranch: 'main', sourceSha256: freshnessCatalog.sourceSha256 });
|
||||
api.github.listIssues = async () => { api.setIdentity({ defaultBranch: 'main', sourceSha256: 'c'.repeat(64) }); return []; };
|
||||
expect((await publishFreshness(freshnessObservation(), api.github, 'refs/heads/main')).action).toBe('rejected-obsolete-catalog');
|
||||
expect(api.issues).toHaveLength(0);
|
||||
});
|
||||
|
||||
test('final issue read preserves human text added during the authoritative catalog check', async () => {
|
||||
const body = writeFreshnessIssue('', assessFreshness(freshnessObservation()));
|
||||
const api = mockGitHub([{ number: 1, body, state: 'closed' }]);
|
||||
const authority = api.github.defaultCatalog;
|
||||
api.github.defaultCatalog = async signal => {
|
||||
const identity = await authority(signal);
|
||||
if (api.calls.includes('list')) api.issues[0].body += '\nLate maintainer note';
|
||||
return identity;
|
||||
};
|
||||
await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z'), api.github, 'refs/heads/main');
|
||||
expect(api.issues[0].body.endsWith('\nLate maintainer note')).toBe(true);
|
||||
});
|
||||
|
||||
test('final issue read rejects an older observation when newer authoritative history arrives during the catalog check', async () => {
|
||||
const api = mockGitHub([{ number: 1, body: writeFreshnessIssue('', assessFreshness(freshnessObservation())), state: 'closed' }]);
|
||||
const authority = api.github.defaultCatalog;
|
||||
api.github.defaultCatalog = async signal => {
|
||||
const identity = await authority(signal);
|
||||
if (api.calls.includes('list')) api.issues[0] = { ...api.issues[0], state: 'open', body: writeFreshnessIssue(api.issues[0].body, assessFreshness(freshnessObservation('102', '2026-10-09T12:00:00.000Z', { 'anthropic-lifecycle': retirementFixture() }))) };
|
||||
return identity;
|
||||
};
|
||||
expect((await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z'), api.github, 'refs/heads/main')).action).toBe('rejected-older-observation');
|
||||
expect(api.calls).not.toContain('update');
|
||||
expect(readFreshnessIssue(api.issues[0].body).state?.latest.run.id).toBe('102');
|
||||
expect(readFreshnessIssue(api.issues[0].body).lifecycle[0].state).toBe('deprecated');
|
||||
});
|
||||
|
||||
test('publication has a separate exact sixty-second deadline and aborts the blocked API', async () => {
|
||||
const real = globalThis.setTimeout;
|
||||
const installed: number[] = [];
|
||||
const timers = spyOn(globalThis, 'setTimeout').mockImplementation(((callback: () => void, ms: number) => { installed.push(ms); return real(callback, 5); }) as typeof setTimeout);
|
||||
const api = mockGitHub();
|
||||
let signal: AbortSignal | undefined;
|
||||
api.github.defaultCatalog = async input => { signal = input; return new Promise(() => {}); };
|
||||
try { await expect(publishFreshness(freshnessObservation(), api.github, 'refs/heads/main')).rejects.toThrow('deadline-exceeded'); }
|
||||
finally { timers.mockRestore(); }
|
||||
expect(installed).toEqual([60_000]);
|
||||
expect(signal?.aborted).toBe(true);
|
||||
expect(api.issues).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('network adapter, real catalog and CLI seams', () => {
|
||||
test('real catalog export and canonical hash bind the four identities and separate source-file hash', () => {
|
||||
const source = readFileSync(new URL('../lib/model-catalog.ts', import.meta.url), 'utf8');
|
||||
const identity = freshnessCatalogIdentity(source);
|
||||
expect(identity.sha256).toBe(modelCatalogSha256());
|
||||
expect(identity.sourceSha256).toBe(sha256(source));
|
||||
expect(identity.models).toEqual(MODEL_CATALOG.map(entry => ({ provider: entry.provider, tier: entry.tier, modelId: entry.model })));
|
||||
});
|
||||
|
||||
test('GitHub reads all-state paginated issues, ignores PRs, and compares remote bytes without executing them', async () => {
|
||||
const calls: { url: string; method?: string }[] = [];
|
||||
const text = 'untrusted TS is data, never import or eval';
|
||||
const api = freshnessGitHub('owner/repo', 'fixture', async (url, init) => {
|
||||
calls.push({ url, method: init.method });
|
||||
expect(init.redirect).toBe('error');
|
||||
if (url.endsWith('/repos/owner/repo')) return Response.json({ default_branch: 'main' });
|
||||
if (url.includes('/contents/')) return Response.json({ encoding: 'base64', content: Buffer.from(text).toString('base64'), size: Buffer.byteLength(text) });
|
||||
if (new URL(url).searchParams.get('page') === '1') return Response.json(Array.from({ length: 100 }, (_, i) => ({ number: i + 1, body: '', state: 'closed', user: { login: FRESHNESS_ISSUE_AUTHOR }, ...(i === 0 ? {} : { pull_request: {} }) })));
|
||||
if (new URL(url).searchParams.get('page') === '2') return Response.json([{ number: 101, body: FRESHNESS_ISSUE_MARKER, state: 'closed', user: { login: FRESHNESS_ISSUE_AUTHOR } }]);
|
||||
return Response.json({ number: 101, body: FRESHNESS_ISSUE_MARKER, state: 'closed', user: { login: FRESHNESS_ISSUE_AUTHOR } });
|
||||
});
|
||||
const signal = new AbortController().signal;
|
||||
expect(await api.defaultCatalog(signal)).toEqual({ defaultBranch: 'main', sourceSha256: sha256(text) });
|
||||
expect((await api.listIssues(signal)).map(issue => issue.number)).toEqual([1, 101]);
|
||||
await api.updateIssue(101, { body: 'machine-owned data', state: 'open' }, signal);
|
||||
expect(calls.at(-1)?.method).toBe('PATCH');
|
||||
expect(calls.filter(call => call.url.includes('/issues?')).every(call => call.url.includes('state=all'))).toBe(true);
|
||||
expect(() => freshnessGitHub('owner/repo; shell', 'fixture')).toThrow('prerequisites-missing');
|
||||
await expect(freshnessGitHub('owner/repo', 'fixture', async () => new Response(null, { status: 403 })).listIssues(signal)).rejects.toThrow('github-http-403');
|
||||
});
|
||||
|
||||
test('creator-filtered discovery stays one request with more than one hundred unrelated issues and PRs', async () => {
|
||||
const unrelated = Array.from({ length: 301 }, (_, i) => ({ number: i + 1, body: 'Unrelated public issue body', state: 'open', user: { login: 'ordinary-user' }, ...(i % 2 ? { pull_request: {} } : {}) }));
|
||||
const items = [...unrelated, { number: 9000, body: FRESHNESS_ISSUE_MARKER, state: 'closed', user: { login: FRESHNESS_ISSUE_AUTHOR } }];
|
||||
const requests: URL[] = [];
|
||||
const api = freshnessGitHub('owner/repo', 'fixture', async url => {
|
||||
const request = new URL(url);
|
||||
requests.push(request);
|
||||
const creator = request.searchParams.get('creator');
|
||||
const selected = creator ? items.filter(issue => issue.user.login === creator) : items;
|
||||
const page = Number(request.searchParams.get('page'));
|
||||
return Response.json(selected.slice((page - 1) * 100, page * 100));
|
||||
});
|
||||
const found = await api.listIssues(new AbortController().signal);
|
||||
expect(requests).toHaveLength(1);
|
||||
expect(requests[0].searchParams.get('creator')).toBe(FRESHNESS_ISSUE_AUTHOR);
|
||||
expect(requests[0].searchParams.get('state')).toBe('all');
|
||||
expect(found.map(issue => issue.number)).toEqual([9000]);
|
||||
});
|
||||
|
||||
test('creator filter never replaces returned-owner validation', async () => {
|
||||
const api = freshnessGitHub('owner/repo', 'fixture', async () => Response.json([{ number: 7, body: FRESHNESS_ISSUE_MARKER, state: 'open', user: { login: 'untrusted-user' } }]));
|
||||
await expect(api.listIssues(new AbortController().signal)).rejects.toThrow('github-issue-owner-filter-mismatch');
|
||||
});
|
||||
|
||||
test('CLI non-main --publish still has no GitHub calls; main publication failure preserves source receipts and fails honestly', async () => {
|
||||
const api = mockGitHub();
|
||||
const fetcher = async (url: string) => new Response(freshnessFixture(FRESHNESS_SOURCES.find(source => source.url === url)!.id));
|
||||
const env = { GITHUB_REF: 'refs/heads/feature', GITHUB_RUN_ID: '100', GITHUB_SHA: 'a'.repeat(40) };
|
||||
const result = await runModelPolicyFreshness({ env, fetcher, github: api.github, publish: true, now: () => '2026-10-07T12:00:00.000Z' });
|
||||
expect(result.publication?.action).toBe('report-only');
|
||||
expect(api.calls).toEqual([]);
|
||||
api.github.defaultCatalog = async () => { throw new Error('github-http-403'); };
|
||||
const failed = await runModelPolicyFreshness({ env: { ...env, GITHUB_REF: 'refs/heads/main', GITHUB_REPOSITORY: 'owner/repo', GH_TOKEN: 'fixture' }, fetcher, github: api.github, publish: true, now: () => '2026-10-07T12:00:00.000Z' });
|
||||
expect(failed.publication?.action).toBe('failed');
|
||||
expect(failed.report.status).toBe('unknown/source-unavailable');
|
||||
expect(failed.observation.sources.filter(source => source.receipt)).toHaveLength(4);
|
||||
expect(api.issues).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('weekly/manual workflow contract', () => {
|
||||
test('one repository-wide serialized group, advisory schedule, main writes and non-main reports', () => {
|
||||
const text = readFileSync(new URL('../.github/workflows/model-policy-freshness.yml', import.meta.url), 'utf8');
|
||||
const workflow = Bun.YAML.parse(text) as { concurrency: { group: string; 'cancel-in-progress': boolean }; on: { schedule: { cron: string }[]; workflow_dispatch: unknown }; jobs: Record<string, { steps: { name?: string; if?: string; run?: string }[] }> };
|
||||
expect(workflow.concurrency).toEqual({ group: 'model-policy-freshness', 'cancel-in-progress': false });
|
||||
expect(workflow.on.schedule).toEqual([{ cron: '0 11 * * 1' }]);
|
||||
expect(Object.keys(workflow.jobs)).toEqual(['advisory']);
|
||||
expect(text).toContain('name: Model-policy freshness');
|
||||
expect(text).toContain('workflow_dispatch:');
|
||||
expect(text).toContain("cron: '0 11 * * 1'");
|
||||
expect(text.match(/group:/g)).toHaveLength(1);
|
||||
expect(text).toContain('group: model-policy-freshness\n cancel-in-progress: false');
|
||||
expect(text).not.toContain('model-policy-freshness-${{ github.ref }}');
|
||||
expect(text).toContain("if: github.ref == 'refs/heads/main'");
|
||||
expect(text).toContain("if: github.ref != 'refs/heads/main'");
|
||||
expect(text).toContain('run: bun scripts/model-policy-freshness.ts --publish');
|
||||
expect(text).toContain('run: bun scripts/model-policy-freshness.ts\n');
|
||||
expect(text).toContain('if-no-files-found: error');
|
||||
expect(text).toContain('if: always()');
|
||||
expect(text).toContain('bun-version: 1.4.2');
|
||||
expect(text).toContain('persist-credentials: false');
|
||||
expect(text).toContain('actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a');
|
||||
expect(text.match(/secrets\.[A-Z_]+/g)).toEqual(['secrets.GITHUB_TOKEN']);
|
||||
expect(text).not.toMatch(/pull_request|ANTHROPIC_API_KEY|OPENAI_API_KEY|codex exec|claude -p|gh issue|eval:bg|gstack-config set/);
|
||||
expect(text.match(/uses: [^\n]+/g)?.every(line => /@[a-f0-9]{40}$/.test(line))).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,524 @@
|
||||
import { afterAll, afterEach, describe, expect, test } from 'bun:test';
|
||||
import { createHash } from 'node:crypto';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { catalogEntry, MODEL_CATALOG, modelCatalogCanonicalJson, modelCatalogSha256 } from '../lib/model-catalog';
|
||||
import {
|
||||
ANTHROPIC_MODEL_ID_PATTERN,
|
||||
CUSTOM_PROVIDER_UNSCANNED,
|
||||
describeSelection,
|
||||
detectCustomProvider as detectWith,
|
||||
isValidModelId,
|
||||
ModelPolicyError,
|
||||
OPENAI_MODEL_ID_PATTERN,
|
||||
parseModelPolicyConfig,
|
||||
readModelPolicyConfig,
|
||||
resolveImplementationModels as implementationWith,
|
||||
resolvePlanReviewModel as planReviewWith,
|
||||
selectionRepair,
|
||||
validatePolicyValue,
|
||||
type ModelSelection,
|
||||
} from '../lib/model-policy';
|
||||
import { CLAUDE_FRONTIER_EVAL_MODEL } from '../lib/eval-model';
|
||||
import { CODEX_RUNTIME_MODEL_PATTERN } from '../scripts/resolve-codex-generation-model';
|
||||
import { CODEX_FRONTIER_MODEL } from '../scripts/resolvers/constants';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const roots: string[] = [];
|
||||
const ISOLATED_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-model-policy-native-'));
|
||||
const ISOLATED = { cwd: ISOLATED_DIR, claudeManagedDir: path.join(ISOLATED_DIR, 'managed'), codexSystemConfig: null };
|
||||
const resolvePlanReviewModel = (opts: Parameters<typeof planReviewWith>[0]) => planReviewWith({ ...ISOLATED, ...opts });
|
||||
const resolveImplementationModels = (opts: Parameters<typeof implementationWith>[0]) => implementationWith({ ...ISOLATED, ...opts });
|
||||
const detectCustomProvider = (provider: Parameters<typeof detectWith>[0], opts: Parameters<typeof detectWith>[1] = {}) => detectWith(provider, { ...ISOLATED, ...opts });
|
||||
|
||||
afterAll(() => fs.rmSync(ISOLATED_DIR, { recursive: true, force: true }));
|
||||
|
||||
afterEach(() => {
|
||||
for (const root of roots.splice(0)) {
|
||||
fs.chmodSync(root, 0o700);
|
||||
fs.rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
function fixture(config?: string) {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-model-policy-'));
|
||||
roots.push(root);
|
||||
const state = path.join(root, 'state');
|
||||
const home = path.join(root, 'home');
|
||||
fs.mkdirSync(state);
|
||||
fs.mkdirSync(home);
|
||||
if (config !== undefined) fs.writeFileSync(path.join(state, 'config.yaml'), config);
|
||||
const env: Record<string, string | undefined> = { HOME: home, GSTACK_STATE_ROOT: state };
|
||||
return { root, state, home, env };
|
||||
}
|
||||
|
||||
function policyError(run: () => unknown): ModelPolicyError {
|
||||
try {
|
||||
run();
|
||||
} catch (error) {
|
||||
if (error instanceof ModelPolicyError) return error;
|
||||
throw error;
|
||||
}
|
||||
throw new Error('expected a ModelPolicyError');
|
||||
}
|
||||
|
||||
function selected(selection: ModelSelection): string {
|
||||
if (selection.status !== 'selected') throw new Error(`expected selected, got ${selection.status}`);
|
||||
return selection.requestedModel;
|
||||
}
|
||||
|
||||
function tree(dir: string): string[] {
|
||||
return fs.readdirSync(dir, { recursive: true }).map(String).sort();
|
||||
}
|
||||
|
||||
describe('model catalog', () => {
|
||||
test('ships exactly the four approved tier IDs with provenance', () => {
|
||||
expect(MODEL_CATALOG.map(e => [e.tier, e.provider, e.model])).toEqual([
|
||||
['frontier', 'anthropic', 'claude-fable-5-1'],
|
||||
['frontier', 'openai', 'gpt-6-astra'],
|
||||
['smart', 'anthropic', 'claude-opus-5-5'],
|
||||
['smart', 'openai', 'gpt-6.1-sol'],
|
||||
]);
|
||||
for (const entry of MODEL_CATALOG) {
|
||||
expect(entry.sourceUrl).toMatch(/^https:\/\/(platform\.claude\.com|developers\.openai\.com)\//);
|
||||
expect(entry.verifiedAt).toMatch(/^\d{4}-\d{2}-\d{2}$/);
|
||||
expect(Object.isFrozen(entry)).toBe(true);
|
||||
}
|
||||
expect(Object.isFrozen(MODEL_CATALOG)).toBe(true);
|
||||
expect(catalogEntry('openai', 'smart').model).toBe('gpt-6.1-sol');
|
||||
});
|
||||
|
||||
test('identity is deterministic and independent of user config', () => {
|
||||
const json = modelCatalogCanonicalJson();
|
||||
expect(JSON.parse(json)).toEqual({ version: 1, entries: MODEL_CATALOG.map(e => ({ ...e })) });
|
||||
expect(modelCatalogSha256()).toBe(createHash('sha256').update(json).digest('hex'));
|
||||
expect(modelCatalogCanonicalJson()).toBe(json);
|
||||
const source = fs.readFileSync(path.join(ROOT, 'lib/model-catalog.ts'), 'utf-8');
|
||||
expect(source).not.toMatch(/process\.env|state-root|readFileSync|config\.yaml/);
|
||||
});
|
||||
|
||||
test('eval rulers and the no-role Codex pattern stay independent of the catalog', () => {
|
||||
expect(CLAUDE_FRONTIER_EVAL_MODEL).toBe('claude-fable-5-1');
|
||||
expect(CODEX_FRONTIER_MODEL).toBe('gpt-6-astra');
|
||||
for (const file of ['lib/eval-model.ts', 'scripts/resolvers/constants.ts']) {
|
||||
expect(fs.readFileSync(path.join(ROOT, file), 'utf-8')).not.toMatch(/from\s+['"][^'"]*model-(catalog|policy)/);
|
||||
}
|
||||
expect(OPENAI_MODEL_ID_PATTERN.source).toBe(CODEX_RUNTIME_MODEL_PATTERN.source);
|
||||
});
|
||||
});
|
||||
|
||||
describe('model ID validation', () => {
|
||||
test.each([
|
||||
['claude-opus-5-5', true],
|
||||
['us.anthropic.claude-opus-5-5-v1:0', true],
|
||||
['arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.anthropic.claude', true],
|
||||
['claude-opus-4@20250514', true],
|
||||
['claude-opus-4-6[1m]', true],
|
||||
['vendor+claude', true],
|
||||
['a'.repeat(512), true],
|
||||
['a'.repeat(513), false],
|
||||
['claude opus', false],
|
||||
['claude\topus', false],
|
||||
['claude\u0007', false],
|
||||
['$(touch x)', false],
|
||||
['claude;rm', false],
|
||||
['', false],
|
||||
])('anthropic %p => %p', (value, ok) => {
|
||||
expect(isValidModelId('anthropic', value)).toBe(ok);
|
||||
expect(ANTHROPIC_MODEL_ID_PATTERN.test(value)).toBe(ok);
|
||||
});
|
||||
|
||||
test.each([
|
||||
['gpt-6.1-sol', true],
|
||||
['org/model:v1', true],
|
||||
['a'.repeat(100), true],
|
||||
['a'.repeat(101), false],
|
||||
['gpt+6', false],
|
||||
['gpt 6', false],
|
||||
['gpt"6', false],
|
||||
])('openai %p => %p', (value, ok) => {
|
||||
expect(isValidModelId('openai', value)).toBe(ok);
|
||||
});
|
||||
|
||||
test('policy values are trimmed and empty values fail', () => {
|
||||
expect(validatePolicyValue('model_smart_claude', ' claude-x ')).toBeNull();
|
||||
expect(validatePolicyValue('plan_review_tier', ' host ')).toBeNull();
|
||||
expect(validatePolicyValue('model_smart_claude', ' ')?.reason).toBe('empty_value');
|
||||
expect(validatePolicyValue('implementation_tier', 'host')?.reason).toBe('invalid_implementation_tier');
|
||||
expect(validatePolicyValue('plan_review_tier', 'fronteir')?.reason).toBe('invalid_plan_review_tier');
|
||||
expect(validatePolicyValue('model_frontier_openai', 'gpt+6')?.reason).toBe('invalid_model_id');
|
||||
});
|
||||
});
|
||||
|
||||
describe('strict policy config snapshot', () => {
|
||||
test('a missing file or absent keys give the documented defaults', () => {
|
||||
const { env } = fixture();
|
||||
const config = readModelPolicyConfig({ env });
|
||||
expect(config.exists).toBe(false);
|
||||
expect(config.planReviewTier).toMatchObject({ value: 'frontier', origin: 'default' });
|
||||
expect(config.implementationTier).toMatchObject({ value: 'smart', origin: 'default' });
|
||||
expect(config.overrides).toEqual({ anthropic: { frontier: null, smart: null }, openai: { frontier: null, smart: null } });
|
||||
expect(readModelPolicyConfig({ env: fixture('telemetry: off\n').env }).exists).toBe(true);
|
||||
});
|
||||
|
||||
test('last value wins, CRLF and comments are tolerated, values are trimmed', () => {
|
||||
const config = parseModelPolicyConfig('# plan_review_tier: host\r\nplan_review_tier: host\r\nplan_review_tier: smart \r\nmodel_smart_openai: gpt-x\r\n', '/c.yaml');
|
||||
expect(config.planReviewTier).toMatchObject({ value: 'smart', origin: 'config', path: '/c.yaml' });
|
||||
expect(config.overrides.openai.smart).toBe('gpt-x');
|
||||
});
|
||||
|
||||
test.each([
|
||||
['plan_review_tier:\n', 'empty_value', 'plan_review_tier'],
|
||||
['plan_review_tier: fronteir\n', 'invalid_plan_review_tier', 'plan_review_tier'],
|
||||
['implementation_tier: host\n', 'invalid_implementation_tier', 'implementation_tier'],
|
||||
['model_frontier_openai: gpt 6\n', 'invalid_model_id', 'model_frontier_openai'],
|
||||
[' plan_review_tier: smart\n', 'config_malformed', 'plan_review_tier'],
|
||||
['model_smart_claude : claude-x\n', 'config_malformed', 'model_smart_claude'],
|
||||
])('%p fails with %p', (text, reason, key) => {
|
||||
const error = policyError(() => parseModelPolicyConfig(text, '/c.yaml'));
|
||||
expect(error.reason).toBe(reason as ModelPolicyError['reason']);
|
||||
expect(error.key).toBe(key);
|
||||
expect(error.repair.join('\n')).toContain(`gstack-config unset ${key}`);
|
||||
expect(error.docs).toContain('troubleshooting.md#model-policy-config');
|
||||
for (const step of error.repair) expect(step).not.toMatch(/[|]/);
|
||||
});
|
||||
|
||||
test('an invalid unused override still fails closed', () => {
|
||||
const { env } = fixture('plan_review_tier: smart\nmodel_frontier_claude: bad value\n');
|
||||
expect(policyError(() => resolvePlanReviewModel({ provider: 'openai', env })).reason).toBe('invalid_model_id');
|
||||
});
|
||||
|
||||
test('a directory, an unreadable file or a file state root is an error, never the default', () => {
|
||||
const dir = fixture();
|
||||
fs.mkdirSync(path.join(dir.state, 'config.yaml'));
|
||||
const isDir = policyError(() => readModelPolicyConfig({ env: dir.env }));
|
||||
expect(isDir).toMatchObject({ reason: 'config_unreadable', cause: 'config.yaml is a directory' });
|
||||
|
||||
const notDir = fixture();
|
||||
const fileRoot = path.join(notDir.root, 'file-root');
|
||||
fs.writeFileSync(fileRoot, '');
|
||||
expect(policyError(() => readModelPolicyConfig({ env: { ...notDir.env, GSTACK_STATE_ROOT: fileRoot } })).reason).toBe('config_unreadable');
|
||||
|
||||
if (process.platform !== 'win32' && process.getuid?.() !== 0) {
|
||||
const denied = fixture('plan_review_tier: smart\n');
|
||||
fs.chmodSync(path.join(denied.state, 'config.yaml'), 0o000);
|
||||
expect(policyError(() => resolvePlanReviewModel({ provider: 'openai', env: denied.env }))).toMatchObject({ reason: 'config_unreadable', cause: 'permission denied' });
|
||||
}
|
||||
});
|
||||
|
||||
test('a bound snapshot is used for the whole resolution even if the file is replaced', () => {
|
||||
const { state, env } = fixture('model_frontier_openai: gpt-old\n');
|
||||
const config = readModelPolicyConfig({ env });
|
||||
const next = path.join(state, 'config.yaml.next');
|
||||
fs.writeFileSync(next, 'plan_review_tier: smart\nmodel_frontier_openai: gpt-new\n');
|
||||
fs.renameSync(next, path.join(state, 'config.yaml'));
|
||||
expect(selected(resolvePlanReviewModel({ provider: 'openai', env, config }))).toBe('gpt-old');
|
||||
const fresh = resolvePlanReviewModel({ provider: 'openai', env });
|
||||
expect(fresh.tier).toBe('smart');
|
||||
expect(selected(fresh)).toBe('gpt-6.1-sol');
|
||||
});
|
||||
});
|
||||
|
||||
describe('plan-review precedence', () => {
|
||||
test.each(['anthropic', 'openai'] as const)('%s: request > env > per-tier config > catalog', provider => {
|
||||
const envName = provider === 'anthropic' ? 'GSTACK_CLAUDE_MODEL' : 'GSTACK_CODEX_MODEL';
|
||||
const key = provider === 'anthropic' ? 'model_frontier_claude' : 'model_frontier_openai';
|
||||
const { env, state } = fixture(`${key}: tier-model\n`);
|
||||
const withEnv = { ...env, [envName]: ' env-model ' };
|
||||
|
||||
const request = resolvePlanReviewModel({ provider, requestedModel: 'request-model', env: withEnv });
|
||||
expect(request).toMatchObject({ status: 'selected', role: 'plan-review', provider, tier: 'frontier', requestedModel: 'request-model', source: { kind: 'request' } });
|
||||
|
||||
const fromEnv = resolvePlanReviewModel({ provider, env: withEnv });
|
||||
expect(fromEnv).toMatchObject({ requestedModel: 'env-model', source: { kind: 'env', name: envName } });
|
||||
|
||||
const fromConfig = resolvePlanReviewModel({ provider, env });
|
||||
expect(fromConfig).toMatchObject({ requestedModel: 'tier-model', source: { kind: 'config', key, path: path.join(state, 'config.yaml') } });
|
||||
|
||||
const fromCatalog = resolvePlanReviewModel({ provider, env: fixture().env });
|
||||
expect(fromCatalog).toMatchObject({ requestedModel: catalogEntry(provider, 'frontier').model, source: { kind: 'catalog', catalogVersion: 1, verifiedAt: '2026-10-07' } });
|
||||
expect(fromCatalog.tierFrom).toMatchObject({ key: 'plan_review_tier', origin: 'default' });
|
||||
});
|
||||
|
||||
test('smart tier reads the smart override, not the frontier one', () => {
|
||||
const { env } = fixture('plan_review_tier: smart\nmodel_frontier_openai: gpt-frontier-pin\n');
|
||||
const selection = resolvePlanReviewModel({ provider: 'openai', env });
|
||||
expect(selection).toMatchObject({ tier: 'smart', requestedModel: 'gpt-6.1-sol', tierFrom: { origin: 'config' } });
|
||||
});
|
||||
|
||||
test('host mode delegates with a null model; request and env still win; tier overrides do not apply', () => {
|
||||
const { env } = fixture('plan_review_tier: host\nmodel_frontier_claude: claude-pin\nmodel_smart_claude: claude-pin\n');
|
||||
const delegated = resolvePlanReviewModel({ provider: 'anthropic', env });
|
||||
expect(delegated).toMatchObject({ status: 'delegated-host', requestedModel: null, tier: 'host', catalog: null, source: { kind: 'host', reason: 'host-mode', key: 'plan_review_tier' } });
|
||||
expect(describeSelection(delegated)).toContain('host-controlled');
|
||||
expect(selected(resolvePlanReviewModel({ provider: 'anthropic', env: { ...env, GSTACK_CLAUDE_MODEL: 'claude-env' } }))).toBe('claude-env');
|
||||
expect(selected(resolvePlanReviewModel({ provider: 'openai', requestedModel: 'gpt-req', env }))).toBe('gpt-req');
|
||||
});
|
||||
|
||||
test('an invalid environment override names the env var and its repair, not a tier', () => {
|
||||
const { env } = fixture();
|
||||
const error = policyError(() => resolvePlanReviewModel({ provider: 'openai', env: { ...env, GSTACK_CODEX_MODEL: 'gpt"; rm -rf /' } }));
|
||||
expect(error).toMatchObject({ reason: 'invalid_model_id', key: 'GSTACK_CODEX_MODEL', source: 'env' });
|
||||
expect(error.repair[0]).toBe('unset GSTACK_CODEX_MODEL');
|
||||
expect(error.docs).toContain('#model-policy-selection');
|
||||
expect(policyError(() => resolvePlanReviewModel({ provider: 'anthropic', requestedModel: 'two words', env })).key).toBe('request');
|
||||
});
|
||||
|
||||
test('an empty env override is unset, matching the legacy readers', () => {
|
||||
const { env } = fixture();
|
||||
expect(resolvePlanReviewModel({ provider: 'openai', env: { ...env, GSTACK_CODEX_MODEL: '' } }).source.kind).toBe('catalog');
|
||||
});
|
||||
|
||||
test('selectionRepair follows the winning source', () => {
|
||||
const { env } = fixture();
|
||||
expect(selectionRepair(resolvePlanReviewModel({ provider: 'openai', env: { ...env, GSTACK_CODEX_MODEL: 'gpt-x' } }))).toEqual(['unset GSTACK_CODEX_MODEL', 'export GSTACK_CODEX_MODEL=<model-id>']);
|
||||
const catalogRepair = selectionRepair(resolvePlanReviewModel({ provider: 'anthropic', env }));
|
||||
expect(catalogRepair).toContain('gstack-config set model_frontier_claude <model-id>');
|
||||
expect(catalogRepair).toContain('gstack-config set plan_review_tier host');
|
||||
});
|
||||
});
|
||||
|
||||
describe('custom-provider guard', () => {
|
||||
test.each(['CLAUDE_CODE_USE_BEDROCK', 'CLAUDE_CODE_USE_VERTEX', 'CLAUDE_CODE_USE_FOUNDRY'])('%s blocks the public default but not explicit choices', flag => {
|
||||
const { env } = fixture();
|
||||
for (const value of ['1', 'true', 'YES', 'on']) {
|
||||
const error = policyError(() => resolvePlanReviewModel({ provider: 'anthropic', env: { ...env, [flag]: value } }));
|
||||
expect(error).toMatchObject({ reason: 'custom_provider_requires_model', source: 'native' });
|
||||
expect(error.cause).toContain(flag);
|
||||
expect(error.repair).toEqual(['gstack-config set model_frontier_claude <model-id-your-endpoint-serves>', 'export GSTACK_CLAUDE_MODEL=<model-id-your-endpoint-serves>', 'gstack-config set plan_review_tier host']);
|
||||
expect(error.docs).toContain('#model-policy-provider');
|
||||
}
|
||||
for (const value of ['0', 'false', '', 'no']) {
|
||||
expect(resolvePlanReviewModel({ provider: 'anthropic', env: { ...env, [flag]: value } }).source.kind).toBe('catalog');
|
||||
}
|
||||
const custom = { ...env, [flag]: '1' };
|
||||
expect(selected(resolvePlanReviewModel({ provider: 'anthropic', env: { ...custom, GSTACK_CLAUDE_MODEL: 'us.anthropic.claude-x-v1:0' } }))).toBe('us.anthropic.claude-x-v1:0');
|
||||
expect(resolvePlanReviewModel({ provider: 'anthropic', env: { ...fixture('plan_review_tier: host\n').env, [flag]: '1' } }).status).toBe('delegated-host');
|
||||
expect(selected(resolvePlanReviewModel({ provider: 'anthropic', env: { ...fixture('model_frontier_claude: vertex-claude@1\n').env, [flag]: '1' } }))).toBe('vertex-claude@1');
|
||||
expect(resolvePlanReviewModel({ provider: 'openai', env: custom }).source.kind).toBe('catalog');
|
||||
});
|
||||
|
||||
test('ANTHROPIC_BASE_URL counts only when non-default and is never printed', () => {
|
||||
const { env } = fixture();
|
||||
expect(detectCustomProvider('anthropic', { env: { ...env, ANTHROPIC_BASE_URL: 'https://api.anthropic.com/' } }).custom).toBe(false);
|
||||
const endpoint = new URL('https://proxy.example/v1');
|
||||
endpoint.username = 'fixture-user';
|
||||
endpoint.password = 'fixture-password';
|
||||
const error = policyError(() => resolvePlanReviewModel({ provider: 'anthropic', env: { ...env, ANTHROPIC_BASE_URL: endpoint.href } }));
|
||||
expect(error.reason).toBe('custom_provider_requires_model');
|
||||
expect(JSON.stringify(error.toJSON())).not.toContain(endpoint.password);
|
||||
expect(JSON.stringify(error.toJSON())).not.toContain(endpoint.username);
|
||||
});
|
||||
|
||||
test('Claude settings.json env signals count; malformed settings are unresolved', () => {
|
||||
const { env, home } = fixture();
|
||||
fs.mkdirSync(path.join(home, '.claude'));
|
||||
const settings = path.join(home, '.claude', 'settings.json');
|
||||
fs.writeFileSync(settings, JSON.stringify({ env: { CLAUDE_CODE_USE_BEDROCK: '1' } }));
|
||||
expect(detectCustomProvider('anthropic', { env }).signals).toEqual([`CLAUDE_CODE_USE_BEDROCK (${settings})`]);
|
||||
fs.writeFileSync(settings, JSON.stringify({ env: { CLAUDE_CODE_USE_BEDROCK: '0' }, model: 'opus' }));
|
||||
expect(detectCustomProvider('anthropic', { env }).custom).toBe(false);
|
||||
fs.writeFileSync(settings, '{not json');
|
||||
expect(policyError(() => resolvePlanReviewModel({ provider: 'anthropic', env }))).toMatchObject({ reason: 'native_provider_unresolved', key: settings });
|
||||
expect(selected(resolvePlanReviewModel({ provider: 'anthropic', env: { ...env, GSTACK_CLAUDE_MODEL: 'claude-x' } }))).toBe('claude-x');
|
||||
const other = path.join(home, 'alt');
|
||||
fs.mkdirSync(other);
|
||||
fs.writeFileSync(path.join(other, 'settings.json'), JSON.stringify({ env: { CLAUDE_CODE_USE_VERTEX: 'true' } }));
|
||||
expect(detectCustomProvider('anthropic', { env: { ...env, CLAUDE_CONFIG_DIR: other } }).custom).toBe(true);
|
||||
});
|
||||
|
||||
test('Codex model_provider, profile inheritance and base URLs', () => {
|
||||
const { env, home } = fixture();
|
||||
const codex = path.join(home, '.codex');
|
||||
fs.mkdirSync(codex);
|
||||
const write = (toml: string) => fs.writeFileSync(path.join(codex, 'config.toml'), toml);
|
||||
const custom = () => detectCustomProvider('openai', { env }).custom;
|
||||
|
||||
expect(custom()).toBe(false);
|
||||
write('model = "gpt-5.5"\nmodel_provider = "openai"\n');
|
||||
expect(custom()).toBe(false);
|
||||
write('model_provider = "azure"\n');
|
||||
expect(policyError(() => resolvePlanReviewModel({ provider: 'openai', env })).reason).toBe('custom_provider_requires_model');
|
||||
write('model_provider = "azure"\nprofile = "work"\n[profiles.work]\nmodel_provider = "openai"\n');
|
||||
expect(custom()).toBe(false);
|
||||
write('profile = "work"\n[profiles.work]\nmodel_provider = "ollama"\n');
|
||||
expect(custom()).toBe(true);
|
||||
write('profile = "missing"\n');
|
||||
expect(policyError(() => detectCustomProvider('openai', { env })).reason).toBe('native_provider_unresolved');
|
||||
write('model_provider = 7\n');
|
||||
expect(policyError(() => detectCustomProvider('openai', { env })).reason).toBe('native_provider_unresolved');
|
||||
write('model_provider = [\n');
|
||||
expect(policyError(() => resolvePlanReviewModel({ provider: 'openai', env })).reason).toBe('native_provider_unresolved');
|
||||
expect(selected(resolvePlanReviewModel({ provider: 'openai', env: { ...env, GSTACK_CODEX_MODEL: 'gpt-x' } }))).toBe('gpt-x');
|
||||
write('openai_base_url = "https://gateway.example/v1"\n');
|
||||
expect(custom()).toBe(true);
|
||||
write('openai_base_url = "https://api.openai.com/v1/"\n');
|
||||
expect(custom()).toBe(false);
|
||||
expect(detectCustomProvider('openai', { env: { ...env, OPENAI_BASE_URL: 'https://gateway.example/v1' } }).custom).toBe(true);
|
||||
expect(policyError(() => detectCustomProvider('openai', { env: { ...env, CODEX_HOME: 'relative/codex' } })).reason).toBe('native_provider_unresolved');
|
||||
const alt = path.join(home, 'codex home with spaces');
|
||||
fs.mkdirSync(alt);
|
||||
fs.writeFileSync(path.join(alt, 'config.toml'), 'model_provider = "azure"\n');
|
||||
expect(detectCustomProvider('openai', { env: { ...env, CODEX_HOME: alt } }).custom).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('custom-provider settings scopes', () => {
|
||||
function project(settings: Record<string, string>) {
|
||||
const fx = fixture();
|
||||
for (const [rel, body] of Object.entries(settings)) {
|
||||
fs.mkdirSync(path.dirname(path.join(fx.root, rel)), { recursive: true });
|
||||
fs.writeFileSync(path.join(fx.root, rel), body);
|
||||
}
|
||||
return fx;
|
||||
}
|
||||
const proxy = JSON.stringify({ env: { ANTHROPIC_BASE_URL: 'https://proxy.example.test' } });
|
||||
const bedrock = JSON.stringify({ env: { CLAUDE_CODE_USE_BEDROCK: '1' } });
|
||||
|
||||
test.each([
|
||||
['shared project settings', 'app/.claude/settings.json', 'app'],
|
||||
['project local settings', 'app/.claude/settings.local.json', 'app'],
|
||||
['repository-root local settings from a subdirectory', 'repo/.claude/settings.local.json', 'repo/pkg'],
|
||||
])('%s block the public default', (_name, file, cwd) => {
|
||||
const fx = project({ [file]: proxy, 'repo/.git/HEAD': 'ref: refs/heads/main\n' });
|
||||
fs.mkdirSync(path.join(fx.root, cwd), { recursive: true });
|
||||
const scope = { env: fx.env, cwd: path.join(fx.root, cwd) };
|
||||
const error = policyError(() => resolvePlanReviewModel({ provider: 'anthropic', ...scope }));
|
||||
expect(error.reason).toBe('custom_provider_requires_model');
|
||||
expect(error.cause).toContain(path.join(fx.root, file));
|
||||
expect(error.cause).not.toContain('proxy.example.test');
|
||||
expect(policyError(() => resolveImplementationModels({ host: 'claude-code', ...scope })).reason).toBe('custom_provider_requires_model');
|
||||
expect(selected(resolvePlanReviewModel({ provider: 'anthropic', ...scope, env: { ...fx.env, GSTACK_CLAUDE_MODEL: 'proxy-claude' } }))).toBe('proxy-claude');
|
||||
expect(resolvePlanReviewModel({ provider: 'anthropic', ...scope, env: { ...fx.env, GSTACK_STATE_ROOT: project({ 'state/config.yaml': 'plan_review_tier: host\n' }).state } }).status).toBe('delegated-host');
|
||||
expect(resolvePlanReviewModel({ provider: 'openai', ...scope }).source.kind).toBe('catalog');
|
||||
});
|
||||
|
||||
test('a worktree also reads the main checkout local settings', () => {
|
||||
const fx = project({
|
||||
'main/.git/worktrees/wt/commondir': '../..\n',
|
||||
'main/.claude/settings.local.json': bedrock,
|
||||
});
|
||||
fs.mkdirSync(path.join(fx.root, 'wt'));
|
||||
fs.writeFileSync(path.join(fx.root, 'wt', '.git'), `gitdir: ${path.join(fx.root, 'main/.git/worktrees/wt')}\n`);
|
||||
const detection = detectCustomProvider('anthropic', { env: fx.env, cwd: path.join(fx.root, 'wt') });
|
||||
expect(detection.signals).toEqual([`CLAUDE_CODE_USE_BEDROCK (${path.join(fx.root, 'main/.claude/settings.local.json')})`]);
|
||||
});
|
||||
|
||||
test('shared settings are read from the working directory, as Claude does, not the repository root', () => {
|
||||
const fx = project({ 'repo/.git/HEAD': '', 'repo/.claude/settings.json': proxy });
|
||||
fs.mkdirSync(path.join(fx.root, 'repo/pkg'));
|
||||
expect(detectCustomProvider('anthropic', { env: fx.env, cwd: path.join(fx.root, 'repo/pkg') }).custom).toBe(false);
|
||||
expect(detectCustomProvider('anthropic', { env: fx.env, cwd: path.join(fx.root, 'repo') }).custom).toBe(true);
|
||||
});
|
||||
|
||||
test('managed settings file and drop-ins count; a malformed one is unresolved', () => {
|
||||
const fx = project({ 'managed/managed-settings.d/20-route.json': bedrock, 'managed/managed-settings.d/.hidden.json': '{', 'managed/managed-settings.d/notes.txt': '{' });
|
||||
const managed = path.join(fx.root, 'managed');
|
||||
expect(detectCustomProvider('anthropic', { env: fx.env, claudeManagedDir: managed }).signals).toEqual([`CLAUDE_CODE_USE_BEDROCK (${path.join(managed, 'managed-settings.d/20-route.json')})`]);
|
||||
fs.writeFileSync(path.join(managed, 'managed-settings.json'), '{broken');
|
||||
expect(policyError(() => resolvePlanReviewModel({ provider: 'anthropic', env: fx.env, claudeManagedDir: managed }))).toMatchObject({ reason: 'native_provider_unresolved', key: path.join(managed, 'managed-settings.json') });
|
||||
});
|
||||
|
||||
test('a requested cwd is honored instead of process.cwd()', () => {
|
||||
const fx = project({ 'elsewhere/.claude/settings.local.json': proxy });
|
||||
const cwd = path.join(fx.root, 'elsewhere');
|
||||
expect(process.cwd()).not.toBe(cwd);
|
||||
const managed = { claudeManagedDir: path.join(fx.root, 'none') };
|
||||
expect(policyError(() => planReviewWith({ provider: 'anthropic', env: fx.env, cwd, ...managed })).reason).toBe('custom_provider_requires_model');
|
||||
expect(policyError(() => planReviewWith({ provider: 'anthropic', env: fx.env, cwd: path.relative(process.cwd(), cwd), ...managed })).reason).toBe('custom_provider_requires_model');
|
||||
const original = process.cwd();
|
||||
process.chdir(fx.home);
|
||||
try {
|
||||
expect(planReviewWith({ provider: 'anthropic', env: fx.env, ...managed }).source.kind).toBe('catalog');
|
||||
process.chdir(cwd);
|
||||
expect(policyError(() => planReviewWith({ provider: 'anthropic', env: fx.env, ...managed })).reason).toBe('custom_provider_requires_model');
|
||||
} finally {
|
||||
process.chdir(original);
|
||||
}
|
||||
});
|
||||
|
||||
test('Codex system config is a lower layer than user config', () => {
|
||||
const fx = project({ 'etc/codex.toml': 'model_provider = "azure"\n' });
|
||||
const scope = { env: fx.env, codexSystemConfig: path.join(fx.root, 'etc/codex.toml') };
|
||||
expect(detectCustomProvider('openai', scope).signals).toEqual([`model_provider "azure" (${scope.codexSystemConfig})`]);
|
||||
fs.mkdirSync(path.join(fx.home, '.codex'));
|
||||
fs.writeFileSync(path.join(fx.home, '.codex', 'config.toml'), 'model_provider = "openai"\n');
|
||||
expect(detectCustomProvider('openai', scope).custom).toBe(false);
|
||||
});
|
||||
|
||||
test('a malformed native config reached from smart plan-review or implementation names no tier override', () => {
|
||||
const fx = project({ 'state/config.yaml': 'plan_review_tier: smart\n', 'home/.codex/config.toml': 'model_provider = [\n' });
|
||||
for (const run of [
|
||||
() => resolvePlanReviewModel({ provider: 'openai', env: fx.env }),
|
||||
() => resolveImplementationModels({ provider: 'openai', env: fx.env }),
|
||||
]) {
|
||||
const error = policyError(run);
|
||||
expect(error.reason).toBe('native_provider_unresolved');
|
||||
expect(error.repair).toEqual([`fix the TOML in ${path.join(fx.home, '.codex', 'config.toml')}`, 'or name the model explicitly for this request']);
|
||||
}
|
||||
});
|
||||
|
||||
test('detection reports what it scanned and what it cannot see', () => {
|
||||
const fx = fixture();
|
||||
const detection = detectCustomProvider('anthropic', { env: fx.env });
|
||||
expect(detection.scanned).toContain(path.join(ISOLATED_DIR, '.claude', 'settings.local.json'));
|
||||
expect(detection.scanned).toContain(path.join(ISOLATED.claudeManagedDir, 'managed-settings.json'));
|
||||
expect(detection.unscanned).toEqual([...CUSTOM_PROVIDER_UNSCANNED.anthropic]);
|
||||
expect(detectCustomProvider('openai', { env: fx.env }).unscanned.join(' ')).toContain('--profile');
|
||||
});
|
||||
});
|
||||
|
||||
describe('implementation recommendation', () => {
|
||||
test('known hosts recommend their own provider; unknown hosts get both', () => {
|
||||
const { env } = fixture();
|
||||
expect(resolveImplementationModels({ host: 'claude-code', env }).map(s => [s.provider, selected(s)])).toEqual([['anthropic', 'claude-opus-5-5']]);
|
||||
expect(resolveImplementationModels({ host: 'codex', env }).map(s => [s.provider, selected(s)])).toEqual([['openai', 'gpt-6.1-sol']]);
|
||||
for (const host of ['unknown', undefined] as const) {
|
||||
expect(resolveImplementationModels({ host, env }).map(s => s.provider)).toEqual(['anthropic', 'openai']);
|
||||
}
|
||||
expect(resolveImplementationModels({ host: 'codex', provider: 'anthropic', env }).map(s => s.provider)).toEqual(['anthropic']);
|
||||
});
|
||||
|
||||
test('frontier is honored, overrides apply, env overrides do not', () => {
|
||||
const { env } = fixture('implementation_tier: frontier\nmodel_frontier_openai: gpt-pin\n');
|
||||
const envWithOverrides = { ...env, GSTACK_CLAUDE_MODEL: 'claude-env', GSTACK_CODEX_MODEL: 'gpt-env' };
|
||||
const [anthropic, openai] = resolveImplementationModels({ env: envWithOverrides });
|
||||
expect(anthropic).toMatchObject({ role: 'implementation', tier: 'frontier', requestedModel: 'claude-fable-5-1', source: { kind: 'catalog' }, tierFrom: { key: 'implementation_tier', origin: 'config' } });
|
||||
expect(openai).toMatchObject({ requestedModel: 'gpt-pin', source: { kind: 'config', key: 'model_frontier_openai' } });
|
||||
expect(resolveImplementationModels({ provider: 'openai', requestedModel: 'gpt-req', env })[0]).toMatchObject({ requestedModel: 'gpt-req', source: { kind: 'request' } });
|
||||
});
|
||||
|
||||
test('an explicit model without a single provider is rejected', () => {
|
||||
const { env } = fixture();
|
||||
expect(policyError(() => resolveImplementationModels({ requestedModel: 'gpt-x', env })).reason).toBe('provider_required');
|
||||
});
|
||||
|
||||
test('custom routing needs an explicit implementation model; it is never delegated', () => {
|
||||
const { env } = fixture();
|
||||
const custom = { ...env, CLAUDE_CODE_USE_BEDROCK: '1' };
|
||||
const error = policyError(() => resolveImplementationModels({ host: 'claude-code', env: custom }));
|
||||
expect(error).toMatchObject({ reason: 'custom_provider_requires_model' });
|
||||
expect(error.repair[0]).toBe('gstack-config set model_smart_claude <model-id-your-endpoint-serves>');
|
||||
expect(error.repair.join(' ')).not.toContain('GSTACK_CLAUDE_MODEL');
|
||||
expect(error.repair.join(' ')).not.toContain('host');
|
||||
expect(selected(resolveImplementationModels({ host: 'claude-code', env: { ...fixture('model_smart_claude: us.anthropic.claude-x\n').env, CLAUDE_CODE_USE_BEDROCK: '1' } })[0])).toBe('us.anthropic.claude-x');
|
||||
});
|
||||
});
|
||||
|
||||
describe('purity', () => {
|
||||
test('resolution and inspection never write state', () => {
|
||||
const { root, env } = fixture('plan_review_tier: smart\n');
|
||||
const before = tree(root);
|
||||
resolvePlanReviewModel({ provider: 'openai', env });
|
||||
resolvePlanReviewModel({ provider: 'anthropic', env });
|
||||
resolveImplementationModels({ env });
|
||||
expect(tree(root)).toEqual(before);
|
||||
});
|
||||
|
||||
test('errors carry every structured field', () => {
|
||||
const error = policyError(() => parseModelPolicyConfig('plan_review_tier: nope\n', '/c.yaml'));
|
||||
expect(Object.keys(error.toJSON()).sort()).toEqual(['cause', 'docs', 'key', 'problem', 'reason', 'repair', 'source']);
|
||||
expect(error.repair.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,485 @@
|
||||
/**
|
||||
* Role-bearing outside invocations: exact plan-review membership, one bound
|
||||
* selection through probe and dispatch, the invocation-boundary notice, and
|
||||
* no-role negative controls. Fake CLIs only; no paid calls.
|
||||
*/
|
||||
import { afterAll, describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { outsideVoiceCommand, outsideVoicePreflight } from '../scripts/resolvers/outside-voice';
|
||||
import { RESOLVERS } from '../scripts/resolvers/index';
|
||||
import { generateAdversarialStep, generateCodexDocReview, generateCodexPlanReview, generateCodexSecondOpinion } from '../scripts/resolvers/outside-voice-steps';
|
||||
import { generateDesignOutsideVoices } from '../scripts/resolvers/design';
|
||||
import { generateImplementationModelHandoff } from '../scripts/resolvers/plan-review';
|
||||
import { HOST_PATHS, type TemplateContext } from '../scripts/resolvers/types';
|
||||
import { claudeCodeArgs } from '../lib/claude-code';
|
||||
import { resolvePlanReviewModel } from '../lib/model-policy';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const TMP = fs.mkdtempSync(path.join(os.tmpdir(), 'role-invocation-'));
|
||||
const REPO = path.join(TMP, 'repo');
|
||||
const BIN = path.join(TMP, 'bin');
|
||||
const CALLS = path.join(TMP, 'calls.jsonl');
|
||||
const PROMPT = path.join(TMP, 'prompt.txt');
|
||||
const FAKE = path.join(TMP, 'fake.ts');
|
||||
for (const dir of [REPO, BIN]) fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(PROMPT, 'Review this plan.\n');
|
||||
fs.writeFileSync(FAKE, `
|
||||
import { appendFileSync, writeFileSync } from 'node:fs';
|
||||
const args = process.argv.slice(2);
|
||||
if (args[0] === 'sandbox') process.exit(0);
|
||||
if (args[0] === '--version') { console.log('codex-cli ' + (process.env.FAKE_CODEX_VERSION ?? '0.160.0')); process.exit(0); }
|
||||
const claude = process.env.FAKE_AS === 'claude';
|
||||
const probe = !claude && args.includes('reply OK');
|
||||
appendFileSync(process.env.CALLS!, JSON.stringify({ args, probe }) + '\\n');
|
||||
if (probe) {
|
||||
if (process.env.REPLACE_CONFIG) writeFileSync(process.env.GSTACK_HOME + '/config.yaml', process.env.REPLACE_CONFIG);
|
||||
if (process.env.FAKE_PROBE === 'model400') { console.error('ERROR: {"status":400,"error":{"message":"The model is not supported."}}'); process.exit(1); }
|
||||
if (process.env.FAKE_PROBE === 'broken') { console.error('Error: spawn codex-vendor ENOENT'); process.exit(1); }
|
||||
console.log('OK');
|
||||
process.exit(0);
|
||||
}
|
||||
if (!claude || args.includes('-')) await Bun.stdin.text();
|
||||
const response = 'Recommendation: split the migration because the plan couples two rollouts.';
|
||||
if (claude) console.log(JSON.stringify({ result: response, session_id: 's', modelUsage: { 'model-actual': { inputTokens: 1 } } }));
|
||||
else writeFileSync(args[args.indexOf('-o') + 1], response);
|
||||
`);
|
||||
fs.writeFileSync(path.join(BIN, 'codex'), `#!/usr/bin/env bash\nexec bun "${FAKE}" "$@"\n`, { mode: 0o755 });
|
||||
|
||||
const git = (args: string[]) => {
|
||||
const r = spawnSync('git', args, { cwd: REPO, encoding: 'utf8', timeout: 5000, env: { ...process.env, GIT_AUTHOR_NAME: 'T', GIT_AUTHOR_EMAIL: 't@example.invalid', GIT_COMMITTER_NAME: 'T', GIT_COMMITTER_EMAIL: 't@example.invalid' } });
|
||||
if (r.status !== 0) throw new Error(r.stderr);
|
||||
};
|
||||
git(['init', '-q', '-b', 'main']);
|
||||
fs.writeFileSync(path.join(REPO, 'plan.md'), 'plan\n');
|
||||
git(['add', 'plan.md']);
|
||||
git(['commit', '-qm', 'init']);
|
||||
afterAll(() => fs.rmSync(TMP, { recursive: true, force: true }));
|
||||
|
||||
type Host = 'claude' | 'codex';
|
||||
const ctxFor = (skillName: string, host: Host, paths = HOST_PATHS.codex): TemplateContext =>
|
||||
({ skillName, tmplPath: `${skillName}/SKILL.md.tmpl`, host, paths });
|
||||
|
||||
let counter = 0;
|
||||
interface Run { stdout: string; stderr: string; status: number | null; calls: Array<{ args: string[]; probe: boolean }>; state: string }
|
||||
|
||||
function home(config = '', codexToml = 'model = "gpt-5.6-terra"\n') {
|
||||
const dir = path.join(TMP, `h${++counter}`);
|
||||
const state = path.join(dir, 'state');
|
||||
fs.mkdirSync(path.join(dir, '.codex'), { recursive: true });
|
||||
fs.mkdirSync(state, { recursive: true });
|
||||
fs.writeFileSync(path.join(dir, '.codex', 'config.toml'), codexToml);
|
||||
if (config) fs.writeFileSync(path.join(state, 'config.yaml'), config);
|
||||
return { dir, state };
|
||||
}
|
||||
|
||||
function env(h: { dir: string; state: string }, host: Host, extra: Record<string, string> = {}): Record<string, string> {
|
||||
return {
|
||||
PATH: `${BIN}${path.delimiter}${process.env.PATH ?? ''}`, HOME: h.dir, CODEX_HOME: path.join(h.dir, '.codex'),
|
||||
GSTACK_HOME: h.state, GSTACK_ROOT: ROOT, GSTACK_BIN: path.join(ROOT, 'bin'), CALLS, CODEX_API_KEY: 'test-key',
|
||||
GSTACK_CLAUDE_BIN: process.execPath, GSTACK_CLAUDE_BIN_ARGS: JSON.stringify([FAKE]), FAKE_AS: host === 'codex' ? 'claude' : 'codex',
|
||||
...(host === 'codex' ? { CODEX_THREAD_ID: 'fixture', GSTACK_ACTIVE_HOST: 'codex' } : { CLAUDECODE: '1', GSTACK_ACTIVE_HOST: 'claude' }),
|
||||
...extra,
|
||||
};
|
||||
}
|
||||
|
||||
function run(host: Host, h: { dir: string; state: string }, extra: Record<string, string> = {}, role: 'plan-review' | null = 'plan-review'): Run {
|
||||
fs.rmSync(CALLS, { force: true });
|
||||
const command = outsideVoiceCommand(ctxFor('plan-eng-review', host), { promptFile: PROMPT, timeoutMs: 8000, ...(role ? { role } : {}) });
|
||||
const r = spawnSync('bash', ['-c', command], { cwd: REPO, env: env(h, host, extra), encoding: 'utf8', timeout: 30000 });
|
||||
const calls = fs.existsSync(CALLS) ? fs.readFileSync(CALLS, 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line)) : [];
|
||||
return { stdout: r.stdout ?? '', stderr: r.stderr ?? '', status: r.status, calls, state: h.state };
|
||||
}
|
||||
|
||||
const codexModel = (call: { args: string[] }) => call.args.find(arg => arg.startsWith('model='));
|
||||
const claudeModels = (call: { args: string[] }) => call.args.flatMap((arg, i) => (arg === '--model' ? [call.args[i + 1]] : arg.startsWith('--model=') ? [arg.slice(8)] : []));
|
||||
const NOTICE = 'NOTICE: gstack plan reviews now use an independent plan-review model';
|
||||
const marker = (state: string) => path.join(state, '.model-policy-notice-v1');
|
||||
|
||||
describe('plan-review role membership in shared generators', () => {
|
||||
const roleBearing: Array<[string, (ctx: TemplateContext) => string]> = [
|
||||
['autoplan', ctx => RESOLVERS.OUTSIDE_INVOCATION(ctx, ['autoplan'])],
|
||||
['spec', ctx => RESOLVERS.OUTSIDE_INVOCATION(ctx, ['spec'])],
|
||||
['plan-ceo-review', generateCodexPlanReview],
|
||||
['plan-eng-review', generateCodexPlanReview],
|
||||
['plan-devex-review', generateCodexPlanReview],
|
||||
['plan-design-review', generateDesignOutsideVoices],
|
||||
];
|
||||
const noRole: Array<[string, (ctx: TemplateContext) => string]> = [
|
||||
['design-review', generateDesignOutsideVoices],
|
||||
['design-consultation', generateDesignOutsideVoices],
|
||||
['office-hours', generateCodexSecondOpinion],
|
||||
['office-hours', ctx => RESOLVERS.DESIGN_SKETCH(ctx)],
|
||||
['review', generateAdversarialStep],
|
||||
['ship', generateAdversarialStep],
|
||||
['ship', ctx => RESOLVERS.DESIGN_REVIEW_LITE(ctx)],
|
||||
['document-release', generateCodexDocReview],
|
||||
];
|
||||
for (const host of ['claude', 'codex'] as const) {
|
||||
const roleMarker = host === 'claude' ? '_gstack_codex_role_ready exec "$_REPO_ROOT" || exit $?' : '--role plan-review';
|
||||
for (const [skill, render] of roleBearing) {
|
||||
test(`${host}: ${skill} invokes with the plan-review role only`, () => {
|
||||
const text = render(ctxFor(skill, host, HOST_PATHS.claude));
|
||||
expect(text).toContain(roleMarker);
|
||||
expect(text).not.toMatch(/_gstack_codex_select_model (exec|review) \|\| exit 1/);
|
||||
expect(text).not.toContain('_gstack_codex_model_probe');
|
||||
expect(text).not.toContain('_CODEX_MP');
|
||||
expect(text).not.toContain('without overriding either');
|
||||
});
|
||||
}
|
||||
for (const [skill, render] of noRole) {
|
||||
test(`${host}: ${skill} stays no-role`, () => {
|
||||
const text = render(ctxFor(skill, host, HOST_PATHS.claude));
|
||||
expect(text).not.toContain('plan-review');
|
||||
expect(text).not.toContain('--role');
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
test('spec keeps medium effort, its 120s deadline and spec gate on both providers', () => {
|
||||
const codex = RESOLVERS.OUTSIDE_INVOCATION(ctxFor('spec', 'claude'), ['spec']);
|
||||
expect(codex).toContain(`model_reasoning_effort="medium"`);
|
||||
expect(codex).toContain('_gstack_codex_timeout_wrapper 120 codex exec');
|
||||
expect(codex).toMatch(/ spec "\$_OUTSIDE_TMP\/text"/);
|
||||
const claude = RESOLVERS.OUTSIDE_INVOCATION(ctxFor('spec', 'codex'), ['spec']);
|
||||
expect(claude).toContain('--timeout-ms 120000 --role plan-review');
|
||||
});
|
||||
|
||||
test('role-bearing availability checks pay for no model probe; no-role checks keep theirs', () => {
|
||||
for (const host of ['claude', 'codex'] as const) {
|
||||
for (const text of [
|
||||
RESOLVERS.OUTSIDE_PREFLIGHT(ctxFor('autoplan', host, HOST_PATHS.claude), ['autoplan']),
|
||||
RESOLVERS.OUTSIDE_PREFLIGHT(ctxFor('spec', host, HOST_PATHS.claude), ['opt-in', 'plan-review']),
|
||||
outsideVoicePreflight(ctxFor('plan-eng-review', host, HOST_PATHS.claude), { disabledBehavior: 'skip-all', role: 'plan-review' }),
|
||||
]) {
|
||||
expect(text).not.toMatch(/_gstack_codex_(model|auth)_probe|_CODEX_MP|without overriding either/);
|
||||
expect(text).toContain('the plan-review model (policy-selected, printed with its source) are checked by the actual invocation');
|
||||
}
|
||||
}
|
||||
expect(RESOLVERS.OUTSIDE_PREFLIGHT(ctxFor('spec', 'claude', HOST_PATHS.claude), ['opt-in'])).toContain('without overriding either');
|
||||
expect(generateCodexDocReview(ctxFor('document-release', 'claude', HOST_PATHS.claude))).toContain('{ _gstack_codex_model_probe; _CODEX_MP=$?; }');
|
||||
expect(generateAdversarialStep(ctxFor('review', 'claude', HOST_PATHS.claude))).toContain('_gstack_codex_model_probe review');
|
||||
expect(RESOLVERS.OUTSIDE_PREFLIGHT(ctxFor('spec', 'claude', HOST_PATHS.claude), ['opt-in'])).not.toContain('_gstack_codex_model_probe');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Codex plan-review invocation binds one selection', () => {
|
||||
test('a known-bad CLI version still warns without changing its advisory policy', () => {
|
||||
const result = run('claude', home(), { FAKE_CODEX_VERSION: '0.120.2' });
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stdout).toMatch(/WARN:.*0\.120\.2.*known stdin deadlock/);
|
||||
expect(result.calls.filter(call => call.probe)).toHaveLength(1);
|
||||
expect(result.calls.filter(call => !call.probe)).toHaveLength(1);
|
||||
expect(result.calls.every(call => codexModel(call) === 'model="gpt-6-astra"')).toBe(true);
|
||||
});
|
||||
|
||||
test('frontier catalog model wins over a native pin; probe and dispatch use it; notice once', () => {
|
||||
const h = home();
|
||||
const first = run('claude', h);
|
||||
expect(first.status).toBe(0);
|
||||
expect(first.stdout).toContain('OUTSIDE_STATUS: completed provider=codex');
|
||||
expect(first.calls.map(c => c.probe)).toEqual([true, false]);
|
||||
expect(first.calls.map(codexModel)).toEqual(['model="gpt-6-astra"', 'model="gpt-6-astra"']);
|
||||
expect(first.stderr).toContain('CODEX_MODEL: gpt-6-astra (exec; role: plan-review, tier: frontier; source: gstack catalog frontier/openai');
|
||||
expect(first.stderr).toContain(NOTICE);
|
||||
expect(first.stderr).toContain('gstack-config set plan_review_tier smart');
|
||||
expect(first.stderr.indexOf(NOTICE)).toBeLessThan(first.stderr.indexOf('CODEX_MODEL:'));
|
||||
expect(fs.existsSync(marker(h.state))).toBe(true);
|
||||
const second = run('claude', h);
|
||||
expect(second.status).toBe(0);
|
||||
expect(second.stderr).not.toContain('NOTICE: gstack plan reviews');
|
||||
expect(second.stdout).toContain('MODEL_OK (cached)');
|
||||
expect(second.calls.map(c => c.probe)).toEqual([false]);
|
||||
});
|
||||
|
||||
test('negative control: the same call without a role keeps native selection, no probe, no notice', () => {
|
||||
const h = home();
|
||||
const r = run('claude', h, {}, null);
|
||||
expect(r.status).toBe(0);
|
||||
expect(r.calls.map(c => c.probe)).toEqual([false]);
|
||||
expect(r.calls.map(codexModel)).toEqual(['model="gpt-5.6-terra"']);
|
||||
expect(r.stderr).not.toContain('NOTICE: gstack plan reviews');
|
||||
expect(fs.existsSync(marker(h.state))).toBe(false);
|
||||
});
|
||||
|
||||
test('a config replaced between probe and dispatch affects only the next invocation', () => {
|
||||
const h = home();
|
||||
const r = run('claude', h, { REPLACE_CONFIG: 'model_frontier_openai: gpt-replaced\n' });
|
||||
expect(r.status).toBe(0);
|
||||
expect(r.calls.map(codexModel)).toEqual(['model="gpt-6-astra"', 'model="gpt-6-astra"']);
|
||||
const next = run('claude', h);
|
||||
expect(next.calls.map(codexModel)).toEqual(['model="gpt-replaced"', 'model="gpt-replaced"']);
|
||||
expect(next.stderr).toContain('source: model_frontier_openai in');
|
||||
});
|
||||
|
||||
for (const [label, config, extra, model, fragment] of [
|
||||
['smart tier', 'plan_review_tier: smart\n', {}, 'gpt-6.1-sol', 'tier: smart'],
|
||||
['env override', '', { GSTACK_CODEX_MODEL: 'gpt-env-pick' }, 'gpt-env-pick', 'source: GSTACK_CODEX_MODEL'],
|
||||
['host mode', 'plan_review_tier: host\n', {}, 'gpt-5.6-terra', 'source: plan_review_tier=host'],
|
||||
] as const) {
|
||||
test(`${label} reaches both probe and dispatch with its real source`, () => {
|
||||
const r = run('claude', home(config), extra);
|
||||
expect(r.status).toBe(0);
|
||||
expect(r.calls.map(codexModel)).toEqual([`model="${model}"`, `model="${model}"`]);
|
||||
expect(r.stderr).toContain(fragment);
|
||||
});
|
||||
}
|
||||
|
||||
test('invalid policy config and unrouted custom providers stop before any Codex process', () => {
|
||||
const invalid = run('claude', home('plan_review_tier: turbo\n'));
|
||||
expect(invalid.status).toBe(1);
|
||||
expect(invalid.calls).toEqual([]);
|
||||
expect(invalid.stderr).toContain('gstack-config unset plan_review_tier');
|
||||
expect(invalid.stderr).toContain('outside review unavailable; missing coverage');
|
||||
const custom = run('claude', home('', 'model_provider = "azure"\n'));
|
||||
expect(custom.status).toBe(1);
|
||||
expect(custom.calls).toEqual([]);
|
||||
expect(custom.stderr).toContain('model_frontier_openai');
|
||||
expect(custom.stderr).not.toContain('NOTICE: gstack plan reviews');
|
||||
});
|
||||
|
||||
test('a rejected role model is unusable with a source-aware repair and never dispatches', () => {
|
||||
const r = run('claude', home(), { FAKE_PROBE: 'model400' });
|
||||
expect(r.status).toBe(1);
|
||||
expect(r.calls.map(c => c.probe)).toEqual([true]);
|
||||
expect(r.stdout).toContain('MODEL_UNUSABLE');
|
||||
expect(r.stdout).toContain('source: gstack catalog frontier/openai');
|
||||
expect(r.stdout).toContain('gstack-config set model_frontier_openai <model-id>');
|
||||
expect(r.stdout).not.toContain('set model in');
|
||||
});
|
||||
|
||||
test('a broken CLI found by the role probe exits 2 and never dispatches', () => {
|
||||
const r = run('claude', home(), { FAKE_PROBE: 'broken' });
|
||||
expect(r.status).toBe(2);
|
||||
expect(r.calls.map(c => c.probe)).toEqual([true]);
|
||||
expect(r.stdout).toContain('MODEL_UNUSABLE_INSTALL');
|
||||
expect(r.stdout).toContain('npm install -g @openai/codex');
|
||||
expect(r.stdout).not.toContain('OUTSIDE_STATUS: completed');
|
||||
});
|
||||
|
||||
test('missing authentication makes no Codex call', () => {
|
||||
const r = run('claude', home(), { CODEX_API_KEY: '' });
|
||||
expect(r.status).toBe(1);
|
||||
expect(r.calls).toEqual([]);
|
||||
expect(r.stdout).toContain('AUTH_FAILED');
|
||||
});
|
||||
|
||||
test('an explicit request outranks GSTACK_CODEX_MODEL in the probe helper', () => {
|
||||
const h = home();
|
||||
const r = spawnSync('bash', ['-c', `source "${ROOT}/bin/gstack-codex-probe" && _gstack_codex_select_model exec gpt-request plan-review && echo "SEL=$_GSTACK_CODEX_SEL"`],
|
||||
{ env: env(h, 'claude', { GSTACK_CODEX_MODEL: 'gpt-env' }), encoding: 'utf8', timeout: 15000 });
|
||||
expect(r.stdout).toContain('SEL=gpt-request');
|
||||
expect(r.stderr).toContain('source: explicit request');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Claude plan-review invocation emits one selected model or none', () => {
|
||||
const result = (r: Run) => JSON.parse(r.stdout.split('\n').find(line => line.startsWith('{"status"'))!);
|
||||
|
||||
test('catalog frontier model: exactly one --model, selection reported apart from modelUsage, notice once', () => {
|
||||
const h = home();
|
||||
const r = run('codex', h, { GSTACK_CLAUDE_MODEL: '' });
|
||||
expect(r.status).toBe(0);
|
||||
expect(r.calls).toHaveLength(1);
|
||||
expect(claudeModels(r.calls[0])).toEqual(['claude-fable-5-1']);
|
||||
expect(result(r).selection).toMatchObject({ role: 'plan-review', tier: 'frontier', status: 'selected', requested_model: 'claude-fable-5-1' });
|
||||
expect(result(r).modelUsage).toHaveProperty('model-actual');
|
||||
expect(r.stderr).toContain('CLAUDE_MODEL: plan-review via anthropic: claude-fable-5-1');
|
||||
expect(r.stderr).toContain(NOTICE);
|
||||
expect(run('codex', h).stderr).not.toContain('NOTICE: gstack plan reviews');
|
||||
});
|
||||
|
||||
test('GSTACK_CLAUDE_MODEL ranks inside the record and appears once', () => {
|
||||
const r = run('codex', home(), { GSTACK_CLAUDE_MODEL: 'claude-env-pick' });
|
||||
expect(claudeModels(r.calls[0])).toEqual(['claude-env-pick']);
|
||||
expect(result(r).selection.source).toContain('GSTACK_CLAUDE_MODEL');
|
||||
});
|
||||
|
||||
test('host mode delegates: zero --model flags and an unknown requested model', () => {
|
||||
const r = run('codex', home('plan_review_tier: host\n'));
|
||||
expect(r.status).toBe(0);
|
||||
expect(claudeModels(r.calls[0])).toEqual([]);
|
||||
expect(result(r).selection).toMatchObject({ status: 'delegated-host', requested_model: null });
|
||||
});
|
||||
|
||||
test('custom Anthropic routing without an explicit model never spawns Claude', () => {
|
||||
const r = run('codex', home(), { ANTHROPIC_BASE_URL: 'https://proxy.example.invalid' });
|
||||
expect(r.status).not.toBe(0);
|
||||
expect(r.calls).toEqual([]);
|
||||
expect(r.stdout).toContain('custom_provider_requires_model');
|
||||
expect(r.stdout).not.toContain('OUTSIDE_STATUS: completed');
|
||||
});
|
||||
|
||||
test('target-repo project settings with a custom endpoint stop a catalog default, whatever the parent cwd', () => {
|
||||
const target = path.join(TMP, 'custom-target');
|
||||
fs.mkdirSync(path.join(target, '.claude'), { recursive: true });
|
||||
fs.writeFileSync(path.join(target, '.claude', 'settings.json'), JSON.stringify({ env: { ANTHROPIC_BASE_URL: 'https://gateway.example.invalid' } }));
|
||||
const sub = path.join(target, 'pkg');
|
||||
fs.mkdirSync(sub, { recursive: true });
|
||||
spawnSync('git', ['init', '-q', '-b', 'main'], { cwd: target, timeout: 5000 });
|
||||
const h = home();
|
||||
fs.rmSync(CALLS, { force: true });
|
||||
const direct = spawnSync(path.join(ROOT, 'bin', 'gstack-claude-code'), ['--cwd', target, '--access', 'none', '--timeout-ms', '5000', '--role', 'plan-review'],
|
||||
{ cwd: TMP, input: 'Review this plan.', env: env(h, 'codex'), encoding: 'utf8', timeout: 20000 });
|
||||
expect(direct.status).toBe(1);
|
||||
expect(JSON.parse(direct.stdout).error.code).toBe('custom_provider_requires_model');
|
||||
expect(fs.existsSync(CALLS)).toBe(false);
|
||||
const command = outsideVoiceCommand(ctxFor('plan-eng-review', 'codex'), { promptFile: PROMPT, timeoutMs: 5000, role: 'plan-review' });
|
||||
const generated = spawnSync('bash', ['-c', command], { cwd: sub, env: env(h, 'codex'), encoding: 'utf8', timeout: 20000 });
|
||||
expect(generated.status).not.toBe(0);
|
||||
expect(generated.stdout).toContain('custom_provider_requires_model');
|
||||
expect(fs.existsSync(CALLS)).toBe(false);
|
||||
const explicit = spawnSync(path.join(ROOT, 'bin', 'gstack-claude-code'), ['--cwd', target, '--access', 'none', '--timeout-ms', '5000', '--role', 'plan-review'],
|
||||
{ cwd: TMP, input: 'Review this plan.', env: env(h, 'codex', { GSTACK_CLAUDE_MODEL: 'gateway-model' }), encoding: 'utf8', timeout: 20000 });
|
||||
expect(explicit.status).toBe(0);
|
||||
expect(JSON.parse(explicit.stdout).selection.requested_model).toBe('gateway-model');
|
||||
});
|
||||
|
||||
test('negative control: no role keeps zero --model without an override and shows no notice', () => {
|
||||
const h = home();
|
||||
const r = run('codex', h, {}, null);
|
||||
expect(r.status).toBe(0);
|
||||
expect(claudeModels(r.calls[0])).toEqual([]);
|
||||
expect(r.stderr).not.toContain('NOTICE: gstack plan reviews');
|
||||
expect(fs.existsSync(marker(h.state))).toBe(false);
|
||||
});
|
||||
|
||||
test('a selected record is never re-resolved from the environment', () => {
|
||||
const cmd = { command: 'claude', argsPrefix: [] };
|
||||
const h = home();
|
||||
const selected = resolvePlanReviewModel({ provider: 'anthropic', env: { GSTACK_HOME: h.state, HOME: h.dir } });
|
||||
if (selected.status !== 'selected') throw new Error('expected a selected record');
|
||||
const args = claudeCodeArgs({ access: 'none', selection: selected }, cmd, { GSTACK_CLAUDE_MODEL: 'claude-late-change' });
|
||||
expect(claudeModels({ args })).toEqual(['claude-fable-5-1']);
|
||||
const dashed = claudeCodeArgs({ access: 'none', selection: { ...selected, requestedModel: '-p' } }, cmd, {});
|
||||
expect(dashed.filter(arg => arg === '-p')).toHaveLength(1);
|
||||
expect(dashed).toContain('--model=-p');
|
||||
fs.writeFileSync(path.join(h.state, 'config.yaml'), 'plan_review_tier: host\n');
|
||||
const delegated = resolvePlanReviewModel({ provider: 'anthropic', env: { GSTACK_HOME: h.state, HOME: h.dir } });
|
||||
expect(claudeModels({ args: claudeCodeArgs({ access: 'none', selection: delegated }, cmd, { GSTACK_CLAUDE_MODEL: 'claude-late-change' }) })).toEqual([]);
|
||||
expect(claudeCodeArgs({ access: 'none' }, cmd, { GSTACK_CLAUDE_MODEL: 'legacy pick' }).slice(-2)).toEqual(['--model', 'legacy pick']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('manual /codex entry: explicit role only, paid probe only for the dispatched model', () => {
|
||||
const skill = fs.readFileSync(path.join(ROOT, 'codex', 'SKILL.md.tmpl'), 'utf8');
|
||||
const preflight = skill.match(/```bash\n(_TEL=[^\n]*\n_CODEX_ROLE=''[\s\S]*?)\n```/)![1]!
|
||||
.replace('{{OUTSIDE_SELF_GUARD:codex}}', RESOLVERS.OUTSIDE_SELF_GUARD(ctxFor('codex', 'claude', HOST_PATHS.claude), ['codex']));
|
||||
const withRole = (text: string, role: string) => text.replaceAll("_CODEX_ROLE=''", `_CODEX_ROLE='${role}'`);
|
||||
const shell = (h: { dir: string; state: string }, script: string) => {
|
||||
fs.mkdirSync(path.join(h.dir, '.claude', 'skills'), { recursive: true });
|
||||
if (!fs.existsSync(path.join(h.dir, '.claude', 'skills', 'gstack'))) fs.symlinkSync(ROOT, path.join(h.dir, '.claude', 'skills', 'gstack'));
|
||||
fs.rmSync(CALLS, { force: true });
|
||||
const r = spawnSync('bash', ['-c', script], { cwd: REPO, env: env(h, 'claude', { CLAUDECODE: '', GSTACK_ACTIVE_HOST: '' }), encoding: 'utf8', timeout: 30000 });
|
||||
const calls = fs.existsSync(CALLS) ? fs.readFileSync(CALLS, 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line)) : [];
|
||||
return { ...r, calls: calls.filter((c: { args: string[] }) => c.args[0] === 'exec') };
|
||||
};
|
||||
|
||||
test('Step 0.5 probes the native model without a role and makes no model call with it', () => {
|
||||
const plain = shell(home(), preflight);
|
||||
expect(plain.status).toBe(0);
|
||||
expect(plain.stdout).toContain('MODEL_OK');
|
||||
expect(plain.calls.map(codexModel)).toEqual(['model="gpt-5.6-terra"']);
|
||||
const role = shell(home(), withRole(preflight, 'plan-review'));
|
||||
expect(role.status).toBe(0);
|
||||
expect(role.stdout).not.toMatch(/MODEL_OK|MODEL_UNUSABLE|AUTH_FAILED/);
|
||||
expect(role.calls).toEqual([]);
|
||||
expect(role.stderr).not.toContain('NOTICE: gstack plan reviews');
|
||||
});
|
||||
|
||||
for (const file of ['challenge-mode', 'consult-mode', 'review-mode']) {
|
||||
const template = fs.readFileSync(path.join(ROOT, 'codex', 'sections', `${file}.md.tmpl`), 'utf8');
|
||||
const selections = [...template.matchAll(/_CODEX_ROLE=''\n(_gstack_codex_select_model (exec|review) '' "\$_CODEX_ROLE" "\$_REPO_ROOT" \|\| exit 1\n\[ -z "\$_CODEX_ROLE" \] \|\| _gstack_codex_model_probe \2 \|\| exit \$\?)\n/g)];
|
||||
test(`${file}: every dispatch block selects then, with the role only, probes that same model`, () => {
|
||||
expect(selections.length).toBe(file === 'review-mode' ? 2 : 1);
|
||||
expect(template.match(/_gstack_codex_select_model/g)!.length).toBe(selections.length + (file === 'review-mode' ? 1 : 0));
|
||||
for (const [block, , kind] of selections) {
|
||||
const script = (role: string) => `_REPO_ROOT='${REPO}'\nsource "${ROOT}/bin/gstack-codex-probe" || exit 1\n${withRole(block, role)}\necho "DISPATCH=$_GSTACK_CODEX_SEL"`;
|
||||
const plain = shell(home(), script(''));
|
||||
expect(plain.status).toBe(0);
|
||||
expect(plain.stdout).toContain('DISPATCH=gpt-5.6-terra');
|
||||
expect(plain.calls).toEqual([]);
|
||||
const h = home();
|
||||
const role = shell(h, script('plan-review'));
|
||||
expect(role.status).toBe(0);
|
||||
expect(role.stdout).toContain('DISPATCH=gpt-6-astra');
|
||||
expect(role.calls.map(codexModel)).toEqual(['model="gpt-6-astra"']);
|
||||
expect(role.stderr).toContain(`CODEX_MODEL: gpt-6-astra (${kind}; role: plan-review, tier: frontier`);
|
||||
expect(role.stderr).toContain(NOTICE);
|
||||
const rejected = shell(home(), `export FAKE_PROBE=model400\n${script('plan-review')}`);
|
||||
expect(rejected.status).toBe(1);
|
||||
expect(rejected.stdout).not.toContain('DISPATCH=');
|
||||
expect(rejected.stdout).toContain('gstack-config set model_frontier_openai <model-id>');
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
describe('invocation-boundary migration notice', () => {
|
||||
test('pure inspection writes nothing and does not suppress the first real notice', () => {
|
||||
const h = home();
|
||||
const inspect = spawnSync(path.join(ROOT, 'bin', 'gstack-models'), ['resolve', '--role', 'plan-review', '--provider', 'openai', '--json'],
|
||||
{ env: env(h, 'claude'), encoding: 'utf8', timeout: 15000 });
|
||||
expect(inspect.status).toBe(0);
|
||||
expect(fs.readdirSync(h.state)).toEqual([]);
|
||||
expect(run('claude', h).stderr).toContain(NOTICE);
|
||||
});
|
||||
|
||||
for (const host of ['codex', 'claude'] as const) {
|
||||
test(`${host === 'codex' ? 'Claude' : 'Codex'}: an unwritable marker repeats the notice instead of hiding it`, () => {
|
||||
const h = home();
|
||||
fs.chmodSync(h.state, 0o500);
|
||||
try {
|
||||
for (let i = 0; i < 2; i++) {
|
||||
const r = run(host, h);
|
||||
expect(r.status).toBe(0);
|
||||
expect(r.stderr).toContain(NOTICE);
|
||||
}
|
||||
expect(fs.existsSync(marker(h.state))).toBe(false);
|
||||
} finally { fs.chmodSync(h.state, 0o700); }
|
||||
});
|
||||
}
|
||||
|
||||
test('disabled reviews stay disabled: no probe, no notice, no Codex call', () => {
|
||||
const block = outsideVoicePreflight(ctxFor('plan-eng-review', 'claude', HOST_PATHS.claude), { disabledBehavior: 'skip-all', role: 'plan-review' })
|
||||
.match(/```bash\n([\s\S]*?)\n```/)![1]!;
|
||||
for (const [config, mode] of [['codex_reviews: disabled\n', 'CODEX_MODE: disabled'], ['', 'CODEX_MODE: ready']] as const) {
|
||||
const h = home(config);
|
||||
fs.mkdirSync(path.join(h.dir, '.claude', 'skills'), { recursive: true });
|
||||
fs.symlinkSync(ROOT, path.join(h.dir, '.claude', 'skills', 'gstack'));
|
||||
fs.rmSync(CALLS, { force: true });
|
||||
const r = spawnSync('bash', ['-c', block], { cwd: REPO, env: env(h, 'claude', { CLAUDECODE: '', GSTACK_ACTIVE_HOST: '' }), encoding: 'utf8', timeout: 20000 });
|
||||
expect(r.stdout).toContain(mode);
|
||||
const calls = fs.existsSync(CALLS) ? fs.readFileSync(CALLS, 'utf8') : '';
|
||||
expect(calls).not.toContain('"exec"');
|
||||
if (config) expect(calls).toBe('');
|
||||
expect(r.stderr).not.toContain('NOTICE: gstack plan reviews');
|
||||
expect(fs.existsSync(marker(h.state))).toBe(false);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('implementation handoff recommendation', () => {
|
||||
test('known harnesses pass their native provider; other hosts get both; nothing switches the session', () => {
|
||||
expect(generateImplementationModelHandoff(ctxFor('autoplan', 'claude', HOST_PATHS.claude))).toContain('gstack-models" resolve --role implementation --provider anthropic`');
|
||||
expect(generateImplementationModelHandoff(ctxFor('spec', 'codex'))).toContain('resolve --role implementation --provider openai`');
|
||||
const other = generateImplementationModelHandoff(ctxFor('plan-eng-review', 'kiro' as Host));
|
||||
expect(other).toContain('resolve --role implementation`');
|
||||
expect(other).toContain('one per provider');
|
||||
for (const text of [other]) {
|
||||
expect(text).toContain('cannot change this session');
|
||||
expect(text).not.toMatch(/claude-|gpt-/);
|
||||
}
|
||||
});
|
||||
|
||||
test('the recommended command resolves offline from current settings without writes', () => {
|
||||
const h = home('implementation_tier: frontier\n');
|
||||
const r = spawnSync(path.join(ROOT, 'bin', 'gstack-models'), ['resolve', '--role', 'implementation', '--provider', 'anthropic', '--json'],
|
||||
{ env: env(h, 'claude'), encoding: 'utf8', timeout: 15000 });
|
||||
expect(r.status).toBe(0);
|
||||
expect(JSON.parse(r.stdout).selections[0]).toMatchObject({ role: 'implementation', requestedModel: 'claude-fable-5-1' });
|
||||
expect(fs.readdirSync(h.state)).toEqual(['config.yaml']);
|
||||
});
|
||||
});
|
||||
@@ -114,7 +114,7 @@ describe('own-harness review fallback instructions', () => {
|
||||
test(`${host.name}: ${name} fallback names only the mode its preflight emits`, () => {
|
||||
const ctx: TemplateContext = { host: host.name, skillName: 'review', tmplPath: 'review/SKILL.md.tmpl', paths: HOST_PATHS[host.name] };
|
||||
const text = render(ctx);
|
||||
const mode = host.name === 'codex' ? 'under_current_harness' : 'under_codex';
|
||||
const mode = name === 'plan' || host.name === 'codex' ? 'under_current_harness' : 'under_codex';
|
||||
const preflight = text.match(/```bash\n([\s\S]*?)\n```/)![1];
|
||||
expect(preflight).toContain(mode);
|
||||
expect([...new Set(text.match(/under_codex|under_current_harness/g))]).toEqual([mode]);
|
||||
|
||||
@@ -138,8 +138,7 @@ describe('CI workflow clarity regressions', () => {
|
||||
test('CEO fallback names the current host mode and needs completed findings before the later report exists', () => {
|
||||
for (const host of ALL_HOST_CONFIGS) {
|
||||
const source = generateCodexPlanReview({ skillName: 'plan-ceo-review', host: host.name, paths: HOST_PATHS[host.name] } as TemplateContext);
|
||||
const mode = host.name === 'codex' ? 'under_current_harness' : 'under_codex';
|
||||
expect([...new Set(source.match(/under_codex|under_current_harness/g))]).toEqual([mode]);
|
||||
expect([...new Set(source.match(/under_codex|under_current_harness/g))]).toEqual(['under_current_harness']);
|
||||
expect(source).toMatch(/SOURCE=in-host, OUTSIDE_STATUS=unavailable, and STATUS=clean or issues_found/);
|
||||
expect(source).not.toContain('Sections 1-10/11 and current report');
|
||||
}
|
||||
@@ -646,7 +645,7 @@ describe('outside-voice commitment queue', () => {
|
||||
for (const host of ALL_HOST_CONFIGS) {
|
||||
const eng = generateCodexPlanReview({ host: host.name, paths: HOST_PATHS[host.name]!, skillName: 'plan-eng-review' } as TemplateContext);
|
||||
const provider = host.name === 'codex' ? 'Claude Code' : 'Codex';
|
||||
const mismatch = host.name === 'codex' ? 'under_current_harness' : 'under_codex';
|
||||
const mismatch = 'under_current_harness';
|
||||
// B1: the heading also admits `unverified`.
|
||||
expect(eng).toContain('**If `CODEX_MODE: ready`');
|
||||
expect(eng).toContain(`— run ${provider}:**`);
|
||||
@@ -665,7 +664,7 @@ describe('outside-voice commitment queue', () => {
|
||||
expect(eng).not.toContain('codex exec');
|
||||
} else {
|
||||
expect(eng).toContain('codex exec');
|
||||
expect(eng).toMatch(/then follow \*\*Native fallback\*\*/i);
|
||||
expect(eng).toContain('_gstack_codex_role_ready exec');
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
@@ -4,7 +4,7 @@ import * as path from 'node:path';
|
||||
import * as os from 'node:os';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import {
|
||||
callerExcerpt, callerReviewRecordTemplate, callerSnapshot, callerTools, createQaCallerFixture, qaCallerInstructions,
|
||||
callerExcerpt, callerProbeReadErrors, callerReviewRecordTemplate, callerSnapshot, callerTools, createQaCallerFixture, qaCallerInstructions,
|
||||
QA_CALLER_CASES, QA_CALLER_TEST_MS,
|
||||
qaCallerSessionOptions, qaCallerCommandAllowed, readCallerReceipt, retainQaCallerEvidence, runQaCaller, unloadedHelperCommand, validateCallerEvidence,
|
||||
type CallerProbe, type CallerReceipt, type QaCallerFixture,
|
||||
@@ -87,6 +87,27 @@ function rebuildCheckpoints(observed: ReturnType<typeof evidence>) {
|
||||
}
|
||||
|
||||
describe('caller native-event observer controls', () => {
|
||||
test('captured instruction Read and probe in one provider response is not consumed instruction evidence', () => {
|
||||
const captured = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/qa-caller-read-probe-mtb1007a.json'), 'utf8'));
|
||||
const tools = callerTools(captured.events);
|
||||
const probe = tools.find(tool => tool.name === 'Bash')!;
|
||||
const resources = ['exploratory', 'system-functional'].map(id => [id, tools.find(tool => String(tool.input.file_path).endsWith(`/qa/sections/${id}.md`))] as const);
|
||||
expect(resources[0][1]!.resultIndex).toBeLessThan(probe.index);
|
||||
expect(resources[0][1]!.messageId).toBe(probe.messageId);
|
||||
expect(callerProbeReadErrors(probe, resources)).toEqual([
|
||||
'probe preceded resource read: exploratory', 'probe preceded resource read: system-functional',
|
||||
]);
|
||||
expect(callerProbeReadErrors({ ...probe, messageId: 'separate-response-control' }, resources))
|
||||
.toEqual(['probe preceded resource read: system-functional']);
|
||||
const ordered = structuredClone(captured.events);
|
||||
const command = ordered.splice(2, 2);
|
||||
command[0].message.id = 'separate-response-control';
|
||||
ordered.push(...command);
|
||||
const ready = callerTools(ordered);
|
||||
expect(callerProbeReadErrors(ready.find(tool => tool.name === 'Bash')!,
|
||||
['exploratory', 'system-functional'].map(id => [id, ready.find(tool => String(tool.input.file_path).endsWith(`/qa/sections/${id}.md`))] as const))).toEqual([]);
|
||||
});
|
||||
|
||||
test.each(['pending-result', 'same-event', 'same-message-id'])('method Read %s cannot authorize a dependent probe', ordering => {
|
||||
const observed = evidence();
|
||||
const events = observed.result.transcript as any[];
|
||||
|
||||
@@ -59,6 +59,14 @@ function assertPlanExecution(text: string, shared = generateQAExploratory({ host
|
||||
}
|
||||
|
||||
describe('QA probe entry and checkpoint gates', () => {
|
||||
test('the caller exposes the response boundary before the first QA instruction read', () => {
|
||||
for (const skillName of ['review', 'ship']) {
|
||||
const text = generateQAReviewPreflight({ host: 'claude', skillName, tmplPath: '', paths: HOST_PATHS.claude });
|
||||
const beforeRead = text.slice(0, text.indexOf('{{QA_RESOURCE:exploratory}}'));
|
||||
expectMentions(beforeRead, [['reads', 'earlier', 'responses'], ['capture', 'probe'], ['never', 'batch', 'read']], 'caller entry gate');
|
||||
}
|
||||
});
|
||||
|
||||
test('the native CI preparation excerpt lacks the required acknowledged-read gate', () => {
|
||||
const captured = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/qa-functional-ci-36505065023.json'), 'utf8'));
|
||||
const output = captured.omittedReadEvents.flatMap(event => event.message.content)
|
||||
@@ -88,7 +96,9 @@ describe('QA probe entry and checkpoint gates', () => {
|
||||
const ctx = { host: host.name, skillName, tmplPath: '', paths: HOST_PATHS[host.name] };
|
||||
const preflight = generateQAReviewPreflight(ctx).replace(/\s+/g, ' ');
|
||||
const body = generateQAReview(ctx).replace(/\s+/g, ' ');
|
||||
expect(preflight).toMatch(/`sections\/\.\.\.` and `templates\/\.\.\.` paths from that installed QA SKILL\.md directory, not the caller or product directory/i);
|
||||
expect(preflight).toContain('`sections/...`');
|
||||
expect(preflight).toContain('`templates/...`');
|
||||
expectMentions(preflight, [['paths', 'installed qa skill.md', 'not', 'caller', 'product']], 'QA asset resolution');
|
||||
expect(body).toContain('Read QA\'s `sections/browser-setup.md`');
|
||||
expect(body).toContain('Read QA\'s `templates/functional-report-template.md`');
|
||||
if (skillName === 'review') {
|
||||
|
||||
@@ -366,7 +366,14 @@ describe('setup --team / --no-team / -q', () => {
|
||||
fs.writeFileSync(file, content, { mode: 0o755 });
|
||||
};
|
||||
try {
|
||||
for (const rel of ['setup', 'VERSION', 'SKILL.md', 'qa/SKILL.md', 'bin/gstack-config', 'bin/gstack-patch-names', 'bin/gstack-state-root.sh', 'bin/gstack-bun-version.sh', 'bin/gstack-install-registry.sh', 'bin/gstack-render-claude.sh', 'scripts/resolve-codex-generation-model.ts', 'scripts/models.ts', 'scripts/preflight-codex-overlap.ts', 'scripts/discover-skills.ts', 'scripts/external-skill-names.ts', 'scripts/host-config.ts']) {
|
||||
for (const rel of [
|
||||
'setup', 'VERSION', 'SKILL.md', 'qa/SKILL.md', 'bin/gstack-config', 'bin/gstack-patch-names',
|
||||
'bin/gstack-state-root.sh', 'bin/gstack-bun-version.sh', 'bin/gstack-install-registry.sh', 'bin/gstack-render-claude.sh',
|
||||
'scripts/resolve-codex-generation-model.ts', 'scripts/models.ts', 'scripts/preflight-codex-overlap.ts',
|
||||
'scripts/discover-skills.ts', 'scripts/external-skill-names.ts', 'scripts/host-config.ts',
|
||||
'lib/claude-code-migration.ts', 'lib/model-policy.ts', 'lib/model-catalog.ts',
|
||||
'lib/model-policy-notice.ts', 'lib/state-root.ts',
|
||||
]) {
|
||||
const dest = path.join(cwd, rel);
|
||||
fs.mkdirSync(path.dirname(dest), { recursive: true });
|
||||
fs.copyFileSync(path.join(ROOT, rel), dest);
|
||||
|
||||
Reference in new issue
Block a user