mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-10 02:30:39 +01:00
* Add failing tests: outside reviews read clean only from a last-line verdict
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* Count an outside review as clean only from its last VERDICT line
Every outside-review prompt now asks the reviewer to end with exactly
VERDICT: CLEAN or VERDICT: FINDINGS, and the parser reads a pass only from
that line. Prose no-findings sentences are no longer evidence for clean.
Codex's structured review and /codex review run through codex exec, since
codex review --base takes no prompt and could never give the line.
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* Repin the outside reviewer briefs and fix the tests the verdict line broke (GST-1821)
The outside brief, both adversarial sections and the review-army prompt-file line now carry the last-line VERDICT request; their pins move under the preregistered no-loss replay GST-1821. Test fakes that wrote a clean reply now end it with VERDICT: CLEAN, the review carve marker follows the codex exec sentence, size pins drop where skills shrank, and the plan-review prompt-file line is shortened so /plan-eng-review stays at its pin.
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* Give the plan-review invocation test's fake reviewer the verdict line it now needs
CI's free-suite shard 5 failed at 1de52b0 with 18 cases in test/model-role-invocation.test.ts: the fake Codex reply had no VERDICT line, so it read unverified (exit 4), and the review-mode template now has one codex exec dispatch block instead of two. The fake reply now ends with VERDICT: FINDINGS and the block count expects one.
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* Stop the changelog correction from claiming the verdict line closes every false-clean case
Review-Finding: p1/red-team#2 reader=red-team
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* Split the changelog correction's last sentence for readability
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* Close the review fail-opens the fix-check found (GST-1821)
A /codex review whose git diff fails now stops before Codex runs instead of sending an empty diff. The verdict is read from the raw last line, so inline code after VERDICT: CLEAN no longer reads clean, and VERDICT: CLEAN beside numbered items with no severity reads unverified. Codex structured review gets the base-branch diff appended to its prompt, the way the Claude path does, so the scope no longer depends on the prompt the host writes.
Fix round granted by the Captain (a-aec3882c9d48). Review-section pins move under replay GST-1821: the free 681-input replay and the 54 scored probe replies are unchanged through the fixed parser.
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* Say in the changelog which numbered-list replies now read unverified, and that /codex review stops without its diff
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* Say in the skills guide that /codex review now sends the diff through codex exec
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
* Check that the structured outside review appends the base-branch diff to its prompt
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
---------
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
741 lines
44 KiB
TypeScript
741 lines
44 KiB
TypeScript
/**
|
|
* Role-bearing outside invocations: exact plan-review membership, one bound
|
|
* selection through probe and dispatch, the invocation-boundary notice, and
|
|
* no-role negative controls. Fake CLIs only; no paid calls.
|
|
*/
|
|
import { afterAll, describe, expect, test } from 'bun:test';
|
|
import * as fs from 'node:fs';
|
|
import * as os from 'node:os';
|
|
import * as path from 'node:path';
|
|
import { spawnSync } from 'node:child_process';
|
|
import { outsideVoiceCommand, outsideVoicePreflight } from '../scripts/resolvers/outside-voice';
|
|
import { RESOLVERS } from '../scripts/resolvers/index';
|
|
import { generateAdversarialStep, generateCodexDocReview, generateCodexPlanReview, generateCodexSecondOpinion } from '../scripts/resolvers/outside-voice-steps';
|
|
import { generateDesignOutsideVoices } from '../scripts/resolvers/design';
|
|
import { generateImplementationModelHandoff } from '../scripts/resolvers/plan-review';
|
|
import { HOST_PATHS, type TemplateContext } from '../scripts/resolvers/types';
|
|
import { claudeCodeArgs } from '../lib/claude-code';
|
|
import { resolvePlanReviewModel } from '../lib/model-policy';
|
|
|
|
const ROOT = path.resolve(import.meta.dir, '..');
|
|
const TMP = fs.mkdtempSync(path.join(os.tmpdir(), 'role-invocation-'));
|
|
const REPO = path.join(TMP, 'repo');
|
|
const BIN = path.join(TMP, 'bin');
|
|
const CALLS = path.join(TMP, 'calls.jsonl');
|
|
const PROMPT = path.join(TMP, 'prompt.txt');
|
|
const FAKE = path.join(TMP, 'fake.ts');
|
|
for (const dir of [REPO, BIN]) fs.mkdirSync(dir, { recursive: true });
|
|
fs.writeFileSync(PROMPT, 'Review this plan.\n');
|
|
fs.writeFileSync(FAKE, `
|
|
import { appendFileSync, writeFileSync } from 'node:fs';
|
|
const args = process.argv.slice(2);
|
|
if (args[0] === 'sandbox') process.exit(0);
|
|
if (args[0] === '--version') { if (process.env.FAKE_VERSION_IGNORE_TERM) process.on('SIGTERM', () => {}); if (process.env.FAKE_VERSION_SLEEP) Bun.sleepSync(Number(process.env.FAKE_VERSION_SLEEP) * 1000); console.log('codex-cli ' + (process.env.FAKE_CODEX_VERSION ?? '0.160.0')); process.exit(0); }
|
|
const claude = process.env.FAKE_AS === 'claude';
|
|
const probe = !claude && args.includes('reply OK');
|
|
appendFileSync(process.env.CALLS!, JSON.stringify({ args, probe }) + '\\n');
|
|
if (probe) {
|
|
if (process.env.FAKE_PROBE_SLEEP) Bun.sleepSync(Number(process.env.FAKE_PROBE_SLEEP) * 1000);
|
|
if (process.env.REPLACE_CONFIG) writeFileSync(process.env.GSTACK_HOME + '/config.yaml', process.env.REPLACE_CONFIG);
|
|
if (process.env.FAKE_PROBE === 'model400') { console.error('ERROR: {"status":400,"error":{"message":"The model is not supported."}}'); process.exit(1); }
|
|
if (process.env.FAKE_PROBE === 'broken') { console.error('Error: spawn codex-vendor ENOENT'); process.exit(1); }
|
|
console.log('OK');
|
|
process.exit(0);
|
|
}
|
|
if (!claude || args.includes('-')) await Bun.stdin.text();
|
|
if (!claude && process.env.FAKE_DISPATCH_MODEL_ERROR) { console.error('ERROR: The model is not supported by this account.'); process.exit(1); }
|
|
const response = process.env.FAKE_RESPONSE || 'Medium: the plan couples two rollouts.\\nRecommendation: split the migration because the plan couples two rollouts.\\nVERDICT: FINDINGS';
|
|
if (claude && process.env.FAKE_PROVIDER_ERROR) { console.log(JSON.stringify({ is_error: true, result: process.env.FAKE_PROVIDER_ERROR })); process.exit(Number(process.env.FAKE_ERROR_EXIT ?? '1')); }
|
|
if (claude) console.log(JSON.stringify({ result: response, session_id: 's', modelUsage: { 'model-actual': { inputTokens: 1 } } }));
|
|
else writeFileSync(args[args.indexOf('-o') + 1], response);
|
|
`);
|
|
fs.writeFileSync(path.join(BIN, 'codex'), `#!/usr/bin/env bash\nexec bun "${FAKE}" "$@"\n`, { mode: 0o755 });
|
|
// Records when each supervised command starts and the deadline it was given.
|
|
const TIMEOUT_LOG = path.join(TMP, 'timeout.log');
|
|
const TIMEOUT_FAKE = path.join(TMP, 'timeout.ts');
|
|
fs.writeFileSync(TIMEOUT_FAKE, `
|
|
import { appendFileSync } from 'node:fs';
|
|
const original = process.argv.slice(2);
|
|
if (process.env.TIMEOUT_LOG) appendFileSync(process.env.TIMEOUT_LOG, (Date.now()/1000) + ' ' + original.join(' ') + '\\n');
|
|
const args = [...original];
|
|
const grace = args[0] === '-k' ? (args.shift(), Number(args.shift())) : 10;
|
|
const duration = Number(args.shift());
|
|
const child = Bun.spawn(args, { stdin: 'inherit', stdout: 'inherit', stderr: 'inherit' });
|
|
let expired = false;
|
|
let killTimer;
|
|
const timer = setTimeout(() => { expired = true; child.kill('SIGTERM'); killTimer = setTimeout(() => child.kill('SIGKILL'), grace*1000); }, duration*1000);
|
|
const code = await child.exited;
|
|
clearTimeout(timer); clearTimeout(killTimer);
|
|
process.exit(expired ? 124 : code);
|
|
`);
|
|
for (const name of ['gtimeout', 'timeout']) fs.writeFileSync(path.join(BIN, name), `#!/usr/bin/env bash\nexec bun "${TIMEOUT_FAKE}" "$@"\n`, { mode: 0o755 });
|
|
|
|
const git = (args: string[]) => {
|
|
const r = spawnSync('git', args, { cwd: REPO, encoding: 'utf8', timeout: 5000, env: { ...process.env, GIT_AUTHOR_NAME: 'T', GIT_AUTHOR_EMAIL: 't@example.invalid', GIT_COMMITTER_NAME: 'T', GIT_COMMITTER_EMAIL: 't@example.invalid' } });
|
|
if (r.status !== 0) throw new Error(r.stderr);
|
|
};
|
|
git(['init', '-q', '-b', 'main']);
|
|
fs.writeFileSync(path.join(REPO, 'plan.md'), 'plan\n');
|
|
git(['add', 'plan.md']);
|
|
git(['commit', '-qm', 'init']);
|
|
afterAll(() => fs.rmSync(TMP, { recursive: true, force: true }));
|
|
|
|
type Host = 'claude' | 'codex';
|
|
const ctxFor = (skillName: string, host: Host, paths = HOST_PATHS.codex): TemplateContext =>
|
|
({ skillName, tmplPath: `${skillName}/SKILL.md.tmpl`, host, paths });
|
|
|
|
let counter = 0;
|
|
interface Run { stdout: string; stderr: string; status: number | null; calls: Array<{ args: string[]; probe: boolean }>; state: string }
|
|
|
|
function home(config = '', codexToml = 'model = "gpt-5.6-terra"\n') {
|
|
const dir = path.join(TMP, `h${++counter}`);
|
|
const state = path.join(dir, 'state');
|
|
fs.mkdirSync(path.join(dir, '.codex'), { recursive: true });
|
|
fs.mkdirSync(state, { recursive: true });
|
|
fs.writeFileSync(path.join(dir, '.codex', 'config.toml'), codexToml);
|
|
if (config) fs.writeFileSync(path.join(state, 'config.yaml'), config);
|
|
return { dir, state };
|
|
}
|
|
|
|
function env(h: { dir: string; state: string }, host: Host, extra: Record<string, string> = {}): Record<string, string> {
|
|
return {
|
|
PATH: `${BIN}${path.delimiter}${process.env.PATH ?? ''}`, HOME: h.dir, CODEX_HOME: path.join(h.dir, '.codex'),
|
|
GSTACK_HOME: h.state, GSTACK_ROOT: ROOT, GSTACK_BIN: path.join(ROOT, 'bin'), CALLS, CODEX_API_KEY: 'test-key',
|
|
GSTACK_CLAUDE_BIN: process.execPath, GSTACK_CLAUDE_BIN_ARGS: JSON.stringify([FAKE]), FAKE_AS: host === 'codex' ? 'claude' : 'codex',
|
|
...(host === 'codex' ? { CODEX_THREAD_ID: 'fixture', GSTACK_ACTIVE_HOST: 'codex' } : { CLAUDECODE: '1', GSTACK_ACTIVE_HOST: 'claude' }),
|
|
...extra,
|
|
};
|
|
}
|
|
|
|
function run(host: Host, h: { dir: string; state: string }, extra: Record<string, string> = {}, role: 'plan-review' | null = 'plan-review'): Run {
|
|
fs.rmSync(CALLS, { force: true });
|
|
const command = outsideVoiceCommand(ctxFor('plan-eng-review', host), { promptFile: PROMPT, timeoutMs: 8000, ...(role ? { role } : {}) });
|
|
const r = spawnSync('bash', ['-c', command], { cwd: REPO, env: env(h, host, extra), encoding: 'utf8', timeout: 30000 });
|
|
const calls = fs.existsSync(CALLS) ? fs.readFileSync(CALLS, 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line)) : [];
|
|
return { stdout: r.stdout ?? '', stderr: r.stderr ?? '', status: r.status, calls, state: h.state };
|
|
}
|
|
|
|
const codexModel = (call: { args: string[] }) => call.args.find(arg => arg.startsWith('model='));
|
|
const claudeModels = (call: { args: string[] }) => call.args.flatMap((arg, i) => (arg === '--model' ? [call.args[i + 1]] : arg.startsWith('--model=') ? [arg.slice(8)] : []));
|
|
const NOTICE = 'NOTICE: gstack plan reviews now use an independent plan-review model';
|
|
const marker = (state: string) => path.join(state, '.model-policy-notice-v1');
|
|
|
|
describe('plan-review role membership in shared generators', () => {
|
|
test('role failures relay source-aware repairs while legacy native-setting advice stays no-role', () => {
|
|
const role = outsideVoicePreflight(ctxFor('autoplan', 'claude', HOST_PATHS.claude), { disabledBehavior: 'codex-only', role: 'plan-review' });
|
|
expect(role).toContain('source-specific Repair/HINT lines');
|
|
expect(role).toContain('policy, auth or model selection');
|
|
expect(role).not.toContain('or config.toml `model`');
|
|
const legacy = outsideVoicePreflight(ctxFor('review', 'claude', HOST_PATHS.claude), { disabledBehavior: 'codex-only' });
|
|
expect(legacy).toContain('or config.toml `model`');
|
|
});
|
|
|
|
test('readiness guidance never assigns quota meaning to the whole review exit status', () => {
|
|
const role = outsideVoicePreflight(ctxFor('autoplan', 'claude', HOST_PATHS.claude), { disabledBehavior: 'codex-only', role: 'plan-review' });
|
|
expect(role).not.toContain('its exit 1, 2 or 4 is');
|
|
expect(role).toContain("never infer quota from the review block's exit status");
|
|
const result = run('claude', home(), { FAKE_RESPONSE: 'Recommendation: proceed because this is a synthetic incomplete-format response.' });
|
|
expect(result.status).toBe(4);
|
|
expect(result.stdout).toContain('OUTSIDE_STATUS: unverified');
|
|
expect(result.stderr).not.toContain('MODEL_QUOTA_EXHAUSTED');
|
|
expect(result.calls.map((call: { probe: boolean }) => call.probe)).toEqual([true, false]);
|
|
});
|
|
|
|
const roleBearing: Array<[string, (ctx: TemplateContext) => string]> = [
|
|
['autoplan', ctx => RESOLVERS.OUTSIDE_INVOCATION(ctx, ['autoplan'])],
|
|
['spec', ctx => RESOLVERS.OUTSIDE_INVOCATION(ctx, ['spec'])],
|
|
['plan-ceo-review', generateCodexPlanReview],
|
|
['plan-eng-review', generateCodexPlanReview],
|
|
['plan-devex-review', generateCodexPlanReview],
|
|
['plan-design-review', generateDesignOutsideVoices],
|
|
];
|
|
const noRole: Array<[string, (ctx: TemplateContext) => string]> = [
|
|
['design-review', generateDesignOutsideVoices],
|
|
['design-consultation', generateDesignOutsideVoices],
|
|
['office-hours', generateCodexSecondOpinion],
|
|
['office-hours', ctx => RESOLVERS.DESIGN_SKETCH(ctx)],
|
|
['review', generateAdversarialStep],
|
|
['ship', generateAdversarialStep],
|
|
['ship', ctx => RESOLVERS.DESIGN_REVIEW_LITE(ctx)],
|
|
['document-release', generateCodexDocReview],
|
|
];
|
|
for (const host of ['claude', 'codex'] as const) {
|
|
const roleMarker = host === 'claude' ? '_CODEX_OUT=$("$_CODEX_PROBE" role-ready exec) || exit $?' : '--role plan-review';
|
|
for (const [skill, render] of roleBearing) {
|
|
test(`${host}: ${skill} invokes with the plan-review role only`, () => {
|
|
const text = render(ctxFor(skill, host, HOST_PATHS.claude));
|
|
expect(text).toContain(roleMarker);
|
|
expect(text).not.toMatch(/select-model (exec|review)|probe-model|_gstack_codex_/);
|
|
expect(text).not.toContain('without overriding either');
|
|
});
|
|
}
|
|
for (const [skill, render] of noRole) {
|
|
test(`${host}: ${skill} stays no-role`, () => {
|
|
const text = render(ctxFor(skill, host, HOST_PATHS.claude));
|
|
expect(text).not.toContain('plan-review');
|
|
expect(text).not.toContain('--role');
|
|
});
|
|
}
|
|
}
|
|
|
|
test('spec keeps medium effort, its 120s deadline and spec gate on both providers', () => {
|
|
const codex = RESOLVERS.OUTSIDE_INVOCATION(ctxFor('spec', 'claude'), ['spec']);
|
|
expect(codex).toContain(`model_reasoning_effort="medium"`);
|
|
expect(codex).toContain('export _CODEX_DEADLINE=$(($(date +%s)+120));');
|
|
expect(codex).toContain('run-with-timeout 120 codex exec');
|
|
expect(codex).toMatch(/ spec "\$_OUTSIDE_TMP\/text"/);
|
|
const claude = RESOLVERS.OUTSIDE_INVOCATION(ctxFor('spec', 'codex'), ['spec']);
|
|
expect(claude).toContain('--timeout-ms 120000 --role plan-review');
|
|
});
|
|
|
|
test('role-bearing availability checks pay for no model probe; no-role checks keep theirs', () => {
|
|
for (const host of ['claude', 'codex'] as const) {
|
|
for (const text of [
|
|
RESOLVERS.OUTSIDE_PREFLIGHT(ctxFor('autoplan', host, HOST_PATHS.claude), ['autoplan']),
|
|
RESOLVERS.OUTSIDE_PREFLIGHT(ctxFor('spec', host, HOST_PATHS.claude), ['opt-in', 'plan-review']),
|
|
outsideVoicePreflight(ctxFor('plan-eng-review', host, HOST_PATHS.claude), { disabledBehavior: 'skip-all', role: 'plan-review' }),
|
|
]) {
|
|
expect(text).not.toMatch(/probe-model|without overriding either/);
|
|
expect(text).toContain('[policy](https://github.com/garrytan/gstack/blob/main/docs/model-policy.md)');
|
|
}
|
|
}
|
|
const noRoleSpec = RESOLVERS.OUTSIDE_PREFLIGHT(ctxFor('spec', 'claude', HOST_PATHS.claude), ['opt-in']);
|
|
expect(noRoleSpec).toContain('without overriding either');
|
|
expect(noRoleSpec).not.toContain('probe-model');
|
|
expect(noRoleSpec).not.toContain('model-policy.md');
|
|
const probeLoop = '_CODEX_PO=$("$_CODEX_PROBE" probe-model $_CODEX_KIND); _CODEX_MP=$?';
|
|
expect(generateCodexDocReview(ctxFor('document-release', 'claude', HOST_PATHS.claude))).toContain(probeLoop);
|
|
const adversarial = generateAdversarialStep(ctxFor('review', 'claude', HOST_PATHS.claude));
|
|
expect(adversarial).toContain('for _CODEX_KIND in exec review; do');
|
|
expect(adversarial).toContain(probeLoop);
|
|
});
|
|
});
|
|
|
|
describe('Codex plan-review invocation binds one selection', () => {
|
|
test('a known-bad CLI version still warns without changing its advisory policy', () => {
|
|
const result = run('claude', home(), { FAKE_CODEX_VERSION: '0.120.2' });
|
|
expect(result.status).toBe(0);
|
|
expect(result.stderr).toMatch(/WARN:.*0\.120\.2.*known stdin deadlock/);
|
|
expect(result.calls.filter(call => call.probe)).toHaveLength(1);
|
|
expect(result.calls.filter(call => !call.probe)).toHaveLength(1);
|
|
expect(result.calls.every(call => codexModel(call) === 'model="gpt-6-astra"')).toBe(true);
|
|
});
|
|
|
|
test('frontier catalog model wins over a native pin; probe and dispatch use it; notice once', () => {
|
|
const h = home();
|
|
const first = run('claude', h);
|
|
expect(first.status).toBe(0);
|
|
expect(first.stdout).toContain('OUTSIDE_STATUS: completed provider=codex');
|
|
expect(first.calls.map(c => c.probe)).toEqual([true, false]);
|
|
expect(first.calls.map(codexModel)).toEqual(['model="gpt-6-astra"', 'model="gpt-6-astra"']);
|
|
expect(first.stderr).toContain('CODEX_MODEL: gpt-6-astra (exec; role: plan-review, tier: frontier; source: gstack catalog frontier/openai');
|
|
expect(first.stderr).toContain(NOTICE);
|
|
expect(first.stderr).toContain('gstack-config set plan_review_tier smart');
|
|
expect(first.stderr.indexOf(NOTICE)).toBeLessThan(first.stderr.indexOf('CODEX_MODEL:'));
|
|
expect(fs.existsSync(marker(h.state))).toBe(true);
|
|
const second = run('claude', h);
|
|
expect(second.status).toBe(0);
|
|
expect(second.stderr).not.toContain('NOTICE: gstack plan reviews');
|
|
expect(second.stderr).toContain('MODEL_OK (cached)');
|
|
expect(second.calls.map(c => c.probe)).toEqual([false]);
|
|
});
|
|
|
|
test('negative control: the same call without a role keeps native selection, no probe, no notice', () => {
|
|
const h = home();
|
|
const r = run('claude', h, {}, null);
|
|
expect(r.status).toBe(0);
|
|
expect(r.calls.map(c => c.probe)).toEqual([false]);
|
|
expect(r.calls.map(codexModel)).toEqual(['model="gpt-5.6-terra"']);
|
|
expect(r.stderr).not.toContain('NOTICE: gstack plan reviews');
|
|
expect(fs.existsSync(marker(h.state))).toBe(false);
|
|
});
|
|
|
|
test('a config replaced between probe and dispatch affects only the next invocation', () => {
|
|
const h = home();
|
|
const r = run('claude', h, { REPLACE_CONFIG: 'model_frontier_openai: gpt-replaced\n' });
|
|
expect(r.status).toBe(0);
|
|
expect(r.calls.map(codexModel)).toEqual(['model="gpt-6-astra"', 'model="gpt-6-astra"']);
|
|
const next = run('claude', h);
|
|
expect(next.calls.map(codexModel)).toEqual(['model="gpt-replaced"', 'model="gpt-replaced"']);
|
|
expect(next.stderr).toContain('source: model_frontier_openai in');
|
|
});
|
|
|
|
for (const [label, config, extra, model, fragment] of [
|
|
['smart tier', 'plan_review_tier: smart\n', {}, 'gpt-6.1-sol', 'tier: smart'],
|
|
['env override', '', { GSTACK_CODEX_MODEL: 'gpt-env-pick' }, 'gpt-env-pick', 'source: GSTACK_CODEX_MODEL'],
|
|
['host mode', 'plan_review_tier: host\n', {}, 'gpt-5.6-terra', 'source: plan_review_tier=host'],
|
|
] as const) {
|
|
test(`${label} reaches both probe and dispatch with its real source`, () => {
|
|
const r = run('claude', home(config), extra);
|
|
expect(r.status).toBe(0);
|
|
expect(r.calls.map(codexModel)).toEqual([`model="${model}"`, `model="${model}"`]);
|
|
expect(r.stderr).toContain(fragment);
|
|
});
|
|
}
|
|
|
|
test('invalid policy config and unrouted custom providers stop before any Codex process', () => {
|
|
const invalid = run('claude', home('plan_review_tier: turbo\n'));
|
|
expect(invalid.status).toBe(1);
|
|
expect(invalid.calls).toEqual([]);
|
|
expect(invalid.stderr).toContain('gstack-config unset plan_review_tier');
|
|
expect(invalid.stderr).toContain('outside review unavailable; missing coverage');
|
|
const custom = run('claude', home('', 'model_provider = "azure"\n'));
|
|
expect(custom.status).toBe(1);
|
|
expect(custom.calls).toEqual([]);
|
|
expect(custom.stderr).toContain('model_frontier_openai');
|
|
expect(custom.stderr).not.toContain('NOTICE: gstack plan reviews');
|
|
});
|
|
|
|
test('malformed known policy records stop both provider invocations before any CLI dispatch', () => {
|
|
for (const host of ['claude', 'codex'] as const) {
|
|
const r = run(host, home('plan_review_tier = smart\n'));
|
|
expect(r.status).not.toBe(0);
|
|
expect(r.calls).toEqual([]);
|
|
expect(r.stdout + r.stderr).toContain('remove or correct the malformed plan_review_tier line');
|
|
expect(r.stderr).not.toContain(NOTICE);
|
|
}
|
|
});
|
|
|
|
test('a rejected role model is unusable with a source-aware repair and never dispatches', () => {
|
|
const r = run('claude', home(), { FAKE_PROBE: 'model400' });
|
|
expect(r.status).toBe(1);
|
|
expect(r.calls.map(c => c.probe)).toEqual([true]);
|
|
expect(r.stderr).toContain('MODEL_UNUSABLE');
|
|
expect(r.stderr).toContain('source: gstack catalog frontier/openai');
|
|
expect(r.stderr).toContain('gstack-config set model_frontier_openai <model-id>');
|
|
expect(r.stderr).not.toContain('set model in');
|
|
expect(r.stdout).not.toContain('OUTSIDE_STATUS');
|
|
});
|
|
|
|
test('a dispatch-only Codex model rejection retains the selected-source repair after unverified readiness', () => {
|
|
const h = home();
|
|
fs.writeFileSync(path.join(h.state, '.codex-model-probe.locks'), 'not a directory');
|
|
const r = run('claude', h, { GSTACK_CODEX_MODEL: 'gpt-env', FAKE_DISPATCH_MODEL_ERROR: '1' });
|
|
expect(r.status).toBe(1);
|
|
expect(r.calls.map((call: { probe: boolean }) => call.probe)).toEqual([false]);
|
|
expect(r.stderr).toContain('MODEL_PROBE_INCONCLUSIVE');
|
|
expect(r.stderr).toContain('If the review rejects the selected model');
|
|
expect(r.stderr).toContain('unset GSTACK_CODEX_MODEL');
|
|
expect(r.stderr).toContain('model is not supported');
|
|
});
|
|
|
|
test('a broken CLI found by the role probe exits 2 and never dispatches', () => {
|
|
const r = run('claude', home(), { FAKE_PROBE: 'broken' });
|
|
expect(r.status).toBe(2);
|
|
expect(r.calls.map(c => c.probe)).toEqual([true]);
|
|
expect(r.stderr).toContain('MODEL_UNUSABLE_INSTALL');
|
|
expect(r.stderr).toContain('npm install -g @openai/codex');
|
|
expect(r.stdout).not.toContain('OUTSIDE_STATUS: completed');
|
|
});
|
|
|
|
test('missing authentication makes no Codex call', () => {
|
|
const r = run('claude', home(), { CODEX_API_KEY: '' });
|
|
expect(r.status).toBe(1);
|
|
expect(r.calls).toEqual([]);
|
|
expect(r.stderr).toContain('AUTH_FAILED');
|
|
});
|
|
|
|
test('role-ready: a named model outranks GSTACK_CODEX_MODEL and is the model it probes; KEY lines only on stdout', () => {
|
|
const h = home();
|
|
fs.rmSync(CALLS, { force: true });
|
|
const r = spawnSync(path.join(ROOT, 'bin', 'gstack-codex-probe'), ['role-ready', 'exec', '--model', 'gpt-request', '--cwd', REPO],
|
|
{ env: env(h, 'claude', { GSTACK_CODEX_MODEL: 'gpt-env' }), encoding: 'utf8', timeout: 15000 });
|
|
expect(r.status).toBe(0);
|
|
expect(r.stdout).toBe('CODEX_SEL: gpt-request\nCODEX_SEL_KIND: exec\nCODEX_SANDBOX: read-only\nCODEX_PROBE_STATE: ok\n');
|
|
expect(r.stderr).toContain('CODEX_MODEL: gpt-request (exec; role: plan-review');
|
|
expect(r.stderr).toContain('source: explicit request');
|
|
expect(fs.readFileSync(CALLS, 'utf8').trim().split('\n').map(line => codexModel(JSON.parse(line)))).toEqual(['model="gpt-request"']);
|
|
for (const bad of [['role-ready'], ['role-ready', 'plan'], ['role-ready', 'exec', '--model'], ['role-ready', 'exec', '--role', 'x']]) {
|
|
const u = spawnSync(path.join(ROOT, 'bin', 'gstack-codex-probe'), bad, { env: env(h, 'claude'), encoding: 'utf8', timeout: 15000 });
|
|
expect({ bad, status: u.status, stdout: u.stdout }).toEqual({ bad, status: 64, stdout: '' });
|
|
}
|
|
});
|
|
});
|
|
|
|
describe('one deadline bounds Codex role readiness and dispatch (native F3)', () => {
|
|
const T = 10;
|
|
interface Supervised { at: number; duration: number; command: string }
|
|
function timed(h: { dir: string; state: string }, extra: Record<string, string> = {}) {
|
|
fs.rmSync(TIMEOUT_LOG, { force: true });
|
|
fs.rmSync(CALLS, { force: true });
|
|
const command = outsideVoiceCommand(ctxFor('plan-eng-review', 'claude'), { promptFile: PROMPT, timeoutMs: T * 1000, role: 'plan-review' });
|
|
const t0 = Date.now() / 1000;
|
|
const r = spawnSync('bash', ['-c', command], { cwd: REPO, env: env(h, 'claude', { TIMEOUT_LOG, ...extra }), encoding: 'utf8', timeout: 60000 });
|
|
const supervised: Supervised[] = (fs.existsSync(TIMEOUT_LOG) ? fs.readFileSync(TIMEOUT_LOG, 'utf8').split('\n').filter(Boolean) : []).map(line => {
|
|
const [at, , , duration, ...command] = line.split(' ');
|
|
return { at: Number(at) - t0, duration: Number(duration), command: command.join(' ') };
|
|
});
|
|
const calls = fs.existsSync(CALLS) ? fs.readFileSync(CALLS, 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line)) : [];
|
|
return { ...r, out: `${r.stdout}${r.stderr}`, calls, supervised, dispatch: supervised.find(s => / exec - /.test(s.command)) };
|
|
}
|
|
// Readiness time already spent plus the dispatch's own limit stays within the
|
|
// provider budget (1.5s covers whole-second clock rounding and process start).
|
|
const withinBudget = (d: Supervised) => expect(d.at + d.duration).toBeLessThanOrEqual(T + 1.5);
|
|
|
|
test('cache hit: the dispatch keeps nearly the whole budget', () => {
|
|
const h = home();
|
|
expect(timed(h).status).toBe(0);
|
|
const r = timed(h);
|
|
expect(r.status).toBe(0);
|
|
expect(r.calls.filter((c: { probe: boolean }) => c.probe)).toHaveLength(0);
|
|
withinBudget(r.dispatch!);
|
|
expect(r.dispatch!.duration).toBeGreaterThanOrEqual(T - 2);
|
|
}, 60000);
|
|
|
|
test('cold cache: the probe elapsed time is charged to the dispatch', () => {
|
|
const r = timed(home(), { FAKE_PROBE_SLEEP: '3' });
|
|
expect(r.status).toBe(0);
|
|
expect(r.calls.map((c: { probe: boolean }) => c.probe)).toEqual([true, false]);
|
|
expect(r.calls.map(codexModel)).toEqual(['model="gpt-6-astra"', 'model="gpt-6-astra"']);
|
|
withinBudget(r.dispatch!);
|
|
}, 60000);
|
|
|
|
test('contention: a waiter behind a live probe owner gives up within half the budget, unverified, without a probe', () => {
|
|
const h = home();
|
|
expect(timed(h).status).toBe(0);
|
|
const cache = path.join(h.state, '.codex-model-probe');
|
|
const sig = fs.readFileSync(cache, 'utf8').split('\n')[0]!.split(' ')[2]!;
|
|
fs.rmSync(cache);
|
|
const lock = path.join(h.state, '.codex-model-probe.locks', sig);
|
|
fs.mkdirSync(lock, { recursive: true });
|
|
fs.writeFileSync(path.join(lock, 'owner'), `${process.pid} ${os.hostname()} ${Math.floor(Date.now() / 1000)}\n`);
|
|
const r = timed(h, { _GSTACK_CODEX_LOCK_WAIT: '4' });
|
|
expect(r.status).toBe(0);
|
|
expect(r.out).toContain('MODEL_PROBE_INCONCLUSIVE (another probe of this model still holds its lock)');
|
|
expect(r.calls.map((c: { probe: boolean }) => c.probe)).toEqual([false]);
|
|
expect(r.dispatch!.at).toBeLessThanOrEqual(T / 2 + 1.5);
|
|
withinBudget(r.dispatch!);
|
|
}, 60000);
|
|
|
|
test('transient: a probe still running at half the budget ends unverified and the dispatch gets only what remains', () => {
|
|
const h = home();
|
|
const r = timed(h, { FAKE_PROBE_SLEEP: '8' });
|
|
expect(r.status).toBe(0);
|
|
expect(r.out).toContain('MODEL_PROBE_INCONCLUSIVE');
|
|
expect(r.calls.map(codexModel)).toEqual(['model="gpt-6-astra"', 'model="gpt-6-astra"']);
|
|
withinBudget(r.dispatch!);
|
|
expect(fs.existsSync(path.join(h.state, '.codex-model-probe')) ? fs.readFileSync(path.join(h.state, '.codex-model-probe'), 'utf8') : '').not.toContain('MODEL_OK');
|
|
}, 60000);
|
|
|
|
test('exhausted: readiness that outlives the whole budget dispatches nothing and is missing coverage', () => {
|
|
const r = timed(home(), { FAKE_VERSION_SLEEP: String(T + 1), FAKE_VERSION_IGNORE_TERM: '1' });
|
|
expect(r.status).toBe(124);
|
|
expect(r.calls).toEqual([]);
|
|
expect(r.dispatch).toBeUndefined();
|
|
expect(r.out).not.toContain('OUTSIDE_STATUS: completed');
|
|
}, 60000);
|
|
|
|
test('a hanging version command is supervised before model readiness and dispatch', () => {
|
|
const r = timed(home(), { FAKE_VERSION_SLEEP: '8' });
|
|
expect(r.status).toBe(0);
|
|
const version = r.supervised.find(command => command.command === 'codex --version');
|
|
expect(version).toBeDefined();
|
|
expect(version!.duration).toBeLessThanOrEqual(5);
|
|
expect(r.out).toContain('codex --version` timed out');
|
|
withinBudget(r.dispatch!);
|
|
}, 60000);
|
|
});
|
|
|
|
describe('Claude plan-review invocation emits one selected model or none', () => {
|
|
const result = (r: Run) => JSON.parse(r.stdout.split('\n').find(line => line.startsWith('{"status"'))!);
|
|
|
|
test('provider model rejections identify the winning selection and its actionable repair without substitution', () => {
|
|
const rejection = 'API Error: 404 {"error":{"type":"not_found_error","message":"model: requested-model"}}';
|
|
for (const [config, extra, repair] of [
|
|
['', {}, 'gstack-config set plan_review_tier smart'],
|
|
['model_frontier_claude: claude-pin\n', {}, 'gstack-config unset model_frontier_claude'],
|
|
['', { GSTACK_CLAUDE_MODEL: 'claude-env' }, 'unset GSTACK_CLAUDE_MODEL'],
|
|
['plan_review_tier: host\n', {}, 'choose the model in Claude Code settings'],
|
|
] as const) {
|
|
for (const exit of ['0', '1']) {
|
|
const r = run('codex', home(config), { ...extra, FAKE_PROVIDER_ERROR: rejection, FAKE_ERROR_EXIT: exit });
|
|
expect(r.status).not.toBe(0);
|
|
expect(r.calls).toHaveLength(1);
|
|
const failed = result(r);
|
|
expect(failed.error.message).toContain(failed.selection.source);
|
|
expect(failed.error.message).toContain(repair);
|
|
expect(failed.error.message).toContain('#model-policy-selection');
|
|
expect(failed.error.message).toContain('No outside review completed');
|
|
if ('GSTACK_CLAUDE_MODEL' in extra) expect(failed.error.message).not.toContain('gstack-config set');
|
|
}
|
|
}
|
|
for (const message of ['authentication_error: invalid API key for model', 'API Error: 429 insufficient_quota for model', 'API Error: 503 server_error']) {
|
|
const r = run('codex', home(), { FAKE_PROVIDER_ERROR: message });
|
|
expect(result(r).error.message).not.toContain('gstack-config set plan_review_tier');
|
|
}
|
|
});
|
|
|
|
test('catalog frontier model: exactly one --model, selection reported apart from modelUsage, notice once', () => {
|
|
const h = home();
|
|
const r = run('codex', h, { GSTACK_CLAUDE_MODEL: '' });
|
|
expect(r.status).toBe(0);
|
|
expect(r.calls).toHaveLength(1);
|
|
expect(claudeModels(r.calls[0])).toEqual(['claude-fable-5-1']);
|
|
expect(result(r).selection).toMatchObject({ role: 'plan-review', tier: 'frontier', status: 'selected', requested_model: 'claude-fable-5-1' });
|
|
expect(result(r).modelUsage).toHaveProperty('model-actual');
|
|
expect(r.stderr).toContain('CLAUDE_MODEL: plan-review via anthropic: claude-fable-5-1');
|
|
expect(r.stderr).toContain(NOTICE);
|
|
expect(run('codex', h).stderr).not.toContain('NOTICE: gstack plan reviews');
|
|
});
|
|
|
|
test('GSTACK_CLAUDE_MODEL ranks inside the record and appears once', () => {
|
|
const r = run('codex', home(), { GSTACK_CLAUDE_MODEL: 'claude-env-pick' });
|
|
expect(claudeModels(r.calls[0])).toEqual(['claude-env-pick']);
|
|
expect(result(r).selection.source).toContain('GSTACK_CLAUDE_MODEL');
|
|
});
|
|
|
|
test('host mode delegates: zero --model flags and an unknown requested model', () => {
|
|
const r = run('codex', home('plan_review_tier: host\n'));
|
|
expect(r.status).toBe(0);
|
|
expect(claudeModels(r.calls[0])).toEqual([]);
|
|
expect(result(r).selection).toMatchObject({ status: 'delegated-host', requested_model: null });
|
|
});
|
|
|
|
test('custom Anthropic routing without an explicit model never spawns Claude', () => {
|
|
const r = run('codex', home(), { ANTHROPIC_BASE_URL: 'https://proxy.example.invalid' });
|
|
expect(r.status).not.toBe(0);
|
|
expect(r.calls).toEqual([]);
|
|
expect(r.stdout).toContain('custom_provider_requires_model');
|
|
expect(r.stdout).not.toContain('OUTSIDE_STATUS: completed');
|
|
});
|
|
|
|
test('target-repo project settings with a custom endpoint stop a catalog default, whatever the parent cwd', () => {
|
|
const target = path.join(TMP, 'custom-target');
|
|
fs.mkdirSync(path.join(target, '.claude'), { recursive: true });
|
|
fs.writeFileSync(path.join(target, '.claude', 'settings.json'), JSON.stringify({ env: { ANTHROPIC_BASE_URL: 'https://gateway.example.invalid' } }));
|
|
const sub = path.join(target, 'pkg');
|
|
fs.mkdirSync(sub, { recursive: true });
|
|
spawnSync('git', ['init', '-q', '-b', 'main'], { cwd: target, timeout: 5000 });
|
|
const h = home();
|
|
fs.rmSync(CALLS, { force: true });
|
|
const direct = spawnSync(path.join(ROOT, 'bin', 'gstack-claude-code'), ['--cwd', target, '--access', 'none', '--timeout-ms', '5000', '--role', 'plan-review'],
|
|
{ cwd: TMP, input: 'Review this plan.', env: env(h, 'codex'), encoding: 'utf8', timeout: 20000 });
|
|
expect(direct.status).toBe(1);
|
|
expect(JSON.parse(direct.stdout).error.code).toBe('custom_provider_requires_model');
|
|
expect(fs.existsSync(CALLS)).toBe(false);
|
|
const command = outsideVoiceCommand(ctxFor('plan-eng-review', 'codex'), { promptFile: PROMPT, timeoutMs: 5000, role: 'plan-review' });
|
|
const generated = spawnSync('bash', ['-c', command], { cwd: sub, env: env(h, 'codex'), encoding: 'utf8', timeout: 20000 });
|
|
expect(generated.status).not.toBe(0);
|
|
expect(generated.stdout).toContain('custom_provider_requires_model');
|
|
expect(fs.existsSync(CALLS)).toBe(false);
|
|
const explicit = spawnSync(path.join(ROOT, 'bin', 'gstack-claude-code'), ['--cwd', target, '--access', 'none', '--timeout-ms', '5000', '--role', 'plan-review'],
|
|
{ cwd: TMP, input: 'Review this plan.', env: env(h, 'codex', { GSTACK_CLAUDE_MODEL: 'gateway-model' }), encoding: 'utf8', timeout: 20000 });
|
|
expect(explicit.status).toBe(0);
|
|
expect(JSON.parse(explicit.stdout).selection.requested_model).toBe('gateway-model');
|
|
});
|
|
|
|
test('negative control: no role keeps zero --model without an override and shows no notice', () => {
|
|
const h = home();
|
|
const r = run('codex', h, {}, null);
|
|
expect(r.status).toBe(0);
|
|
expect(claudeModels(r.calls[0])).toEqual([]);
|
|
expect(r.stderr).not.toContain('NOTICE: gstack plan reviews');
|
|
expect(fs.existsSync(marker(h.state))).toBe(false);
|
|
});
|
|
|
|
test('a selected record is never re-resolved from the environment', () => {
|
|
const cmd = { command: 'claude', argsPrefix: [] };
|
|
const h = home();
|
|
const selected = resolvePlanReviewModel({ provider: 'anthropic', env: { GSTACK_HOME: h.state, HOME: h.dir } });
|
|
if (selected.status !== 'selected') throw new Error('expected a selected record');
|
|
const args = claudeCodeArgs({ access: 'none', selection: selected }, cmd, { GSTACK_CLAUDE_MODEL: 'claude-late-change' });
|
|
expect(claudeModels({ args })).toEqual(['claude-fable-5-1']);
|
|
const dashed = claudeCodeArgs({ access: 'none', selection: { ...selected, requestedModel: '-p' } }, cmd, {});
|
|
expect(dashed.filter(arg => arg === '-p')).toHaveLength(1);
|
|
expect(dashed).toContain('--model=-p');
|
|
fs.writeFileSync(path.join(h.state, 'config.yaml'), 'plan_review_tier: host\n');
|
|
const delegated = resolvePlanReviewModel({ provider: 'anthropic', env: { GSTACK_HOME: h.state, HOME: h.dir } });
|
|
expect(claudeModels({ args: claudeCodeArgs({ access: 'none', selection: delegated }, cmd, { GSTACK_CLAUDE_MODEL: 'claude-late-change' }) })).toEqual([]);
|
|
expect(claudeCodeArgs({ access: 'none' }, cmd, { GSTACK_CLAUDE_MODEL: 'legacy pick' }).slice(-2)).toEqual(['--model', 'legacy pick']);
|
|
});
|
|
});
|
|
|
|
describe('manual /codex entry: explicit role only, paid probe only for the dispatched model', () => {
|
|
const skill = fs.readFileSync(path.join(ROOT, 'codex', 'SKILL.md.tmpl'), 'utf8');
|
|
test('manual model recovery distinguishes role pins from native no-role settings', () => {
|
|
const recovery = skill.slice(skill.indexOf('Recovery, in order:'), skill.indexOf('- **`VERDICT: unavailable`:'));
|
|
expect(recovery).toContain('With `--role plan-review`');
|
|
expect(recovery).toContain('tier pins outrank native settings');
|
|
expect(recovery).toContain('only in `host` mode');
|
|
expect(recovery).toContain('Without a role');
|
|
expect(recovery).toContain('do not silently substitute');
|
|
});
|
|
const preflight = skill.match(/```bash\n(_CODEX_PROBE=[^\n]*\n_CODEX_ROLE=''[\s\S]*?)\n```/)![1]!
|
|
.replace('{{OUTSIDE_SELF_GUARD:codex}}', RESOLVERS.OUTSIDE_SELF_GUARD(ctxFor('codex', 'claude', HOST_PATHS.claude), ['codex']));
|
|
const withRole = (text: string, role: string) => text.replaceAll("_CODEX_ROLE=''", `_CODEX_ROLE='${role}'`);
|
|
const withEffort = (text: string, level: string) => text.replaceAll(`'<effort-level>'`, `'${level}'`);
|
|
const shell = (h: { dir: string; state: string }, script: string, sh = 'bash') => {
|
|
fs.mkdirSync(path.join(h.dir, '.claude', 'skills'), { recursive: true });
|
|
if (!fs.existsSync(path.join(h.dir, '.claude', 'skills', 'gstack'))) fs.symlinkSync(ROOT, path.join(h.dir, '.claude', 'skills', 'gstack'));
|
|
fs.rmSync(CALLS, { force: true });
|
|
const r = spawnSync(sh, sh === 'zsh' ? ['-f', '-c', script] : ['-c', script], { cwd: REPO, env: env(h, 'claude', { CLAUDECODE: '', GSTACK_ACTIVE_HOST: '' }), encoding: 'utf8', timeout: 30000 });
|
|
const calls = fs.existsSync(CALLS) ? fs.readFileSync(CALLS, 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line)) : [];
|
|
return { ...r, calls: calls.filter((c: { args: string[] }) => c.args[0] === 'exec') };
|
|
};
|
|
|
|
test('Step 0.5 probes the native model without a role and makes no model call with it', () => {
|
|
expect(skill).toContain('[Policy setup](https://github.com/garrytan/gstack/blob/main/docs/model-policy.md)');
|
|
const plain = shell(home(), withEffort(preflight, ''));
|
|
expect(plain.status).toBe(0);
|
|
expect(plain.stdout).toContain('MODEL_OK');
|
|
expect(plain.calls.map(codexModel)).toEqual(['model="gpt-5.6-terra"']);
|
|
const role = shell(home(), withEffort(withRole(preflight, 'plan-review'), ''));
|
|
expect(role.status).toBe(0);
|
|
expect(role.stdout).not.toMatch(/MODEL_OK|MODEL_UNUSABLE|AUTH_FAILED/);
|
|
expect(role.calls).toEqual([]);
|
|
expect(role.stderr).not.toContain('NOTICE: gstack plan reviews');
|
|
});
|
|
|
|
test('a valid /codex --effort level passes Step 0.5, shows its source once and probes once', () => {
|
|
const r = shell(home(), withEffort(preflight, 'medium'));
|
|
expect(r.status).toBe(0);
|
|
expect(r.stdout).not.toContain('CODEX_EFFORT');
|
|
expect(r.stderr).toContain('CODEX_EFFORT: medium (source: /codex --effort)');
|
|
expect(r.calls.map(codexModel)).toEqual(['model="gpt-5.6-terra"']);
|
|
});
|
|
|
|
test('an invalid /codex --effort level stops Step 0.5 before the paid model probe', () => {
|
|
const r = shell(home(), withEffort(preflight, 'turbo'));
|
|
expect(r.status).toBe(1);
|
|
expect(r.stderr).toContain("'turbo' from /codex --effort");
|
|
expect(r.stdout).not.toMatch(/MODEL_OK|MODEL_UNUSABLE|AUTH_FAILED/);
|
|
expect(r.calls).toEqual([]);
|
|
});
|
|
|
|
test('a missing manual challenge prompt stops before the paid readiness probe or notice', () => {
|
|
const h = home();
|
|
const template = fs.readFileSync(path.join(ROOT, 'codex/sections/challenge-mode.md.tmpl'), 'utf8');
|
|
const body = template.match(/```bash\n([\s\S]*?)\n```/)![1]!;
|
|
const prefix = body.slice(0, body.indexOf('"$_CODEX_PROBE" run-with-timeout'))
|
|
.replace('{{CODEX_SELECT:exec:540}}', RESOLVERS.CODEX_SELECT(ctxFor('codex', 'claude', HOST_PATHS.claude), ['exec', '540']))
|
|
.replaceAll('<prompt-file-name>', 'missing-plan-prompt');
|
|
const r = shell(h, `TMP_ROOT='${h.dir}'\n${withRole(prefix, 'plan-review')}`);
|
|
expect(r.status).toBe(1);
|
|
expect(r.stderr).toContain('prompt was never written');
|
|
expect(r.calls).toEqual([]);
|
|
expect(fs.existsSync(marker(h.state))).toBe(false);
|
|
});
|
|
|
|
for (const file of ['challenge-mode', 'consult-mode', 'review-mode']) {
|
|
const template = fs.readFileSync(path.join(ROOT, 'codex', 'sections', `${file}.md.tmpl`), 'utf8');
|
|
const selections = [...template.matchAll(/\{\{CODEX_SELECT:(exec|review):(\d+)\}\}/g)];
|
|
test(`${file}: every dispatch block selects then, with the role only, probes that same model under the block's own deadline`, () => {
|
|
expect(selections.length).toBe(1);
|
|
expect(template.match(/\{\{CODEX_SELECT/g)!.length).toBe(selections.length);
|
|
for (const block of template.split(/\{\{CODEX_SELECT:/).slice(1)) {
|
|
const [kind, budget] = block.split('}}')[0]!.split(':');
|
|
const wrappers = [...block.split('```')[0]!.matchAll(/run-with-timeout (?:--user-invoked )?(?:--effort \w+ (?:--request-effort '[^']*' )?)?(\d+) codex/g)].map(m => m[1]);
|
|
expect(wrappers.length).toBeGreaterThan(0);
|
|
expect(new Set(wrappers)).toEqual(new Set([budget]));
|
|
expect(kind).toMatch(/^(exec|review)$/);
|
|
}
|
|
for (const [, kind, budget] of selections) {
|
|
const rendered = RESOLVERS.CODEX_SELECT(ctxFor('codex', 'claude', HOST_PATHS.claude), [kind!, budget!]);
|
|
const script = (role: string, edit = (t: string) => t) => `_REPO_ROOT='${REPO}'\n${edit(withRole(rendered, role))}\necho "DISPATCH=$_CODEX_SEL DEADLINE=\${_CODEX_DEADLINE:-none}"`;
|
|
const plain = shell(home(), script(''));
|
|
expect(plain.status).toBe(0);
|
|
expect(plain.stdout).toContain('DISPATCH=gpt-5.6-terra DEADLINE=none');
|
|
expect(plain.calls).toEqual([]);
|
|
const h = home();
|
|
const before = Math.floor(Date.now() / 1000);
|
|
const role = shell(h, script('plan-review'));
|
|
expect(role.status).toBe(0);
|
|
expect(role.stdout).toContain('DISPATCH=gpt-6-astra');
|
|
const deadline = Number(role.stdout.match(/DEADLINE=(\d+)/)![1]);
|
|
expect(deadline).toBeGreaterThanOrEqual(before + Number(budget));
|
|
expect(deadline).toBeLessThanOrEqual(Math.ceil(Date.now() / 1000) + Number(budget));
|
|
expect(role.calls.map(codexModel)).toEqual(['model="gpt-6-astra"']);
|
|
expect(role.stderr).toContain(`CODEX_MODEL: gpt-6-astra (${kind}; role: plan-review, tier: frontier`);
|
|
expect(role.stderr).toContain(NOTICE);
|
|
const named = shell(home(), script('plan-review', t => t.replace(`role-ready ${kind}`, `role-ready ${kind} --model 'gpt-named'`)));
|
|
expect(named.status).toBe(0);
|
|
expect(named.stdout).toContain('DISPATCH=gpt-named');
|
|
expect(named.stderr).toContain('source: explicit request');
|
|
expect(named.calls.map(codexModel)).toEqual(['model="gpt-named"']);
|
|
const rejected = shell(home(), `export FAKE_PROBE=model400\n${script('plan-review')}`);
|
|
expect(rejected.status).toBe(1);
|
|
expect(rejected.stdout).not.toContain('DISPATCH=');
|
|
expect(rejected.stderr).toContain('gstack-config set model_frontier_openai <model-id>');
|
|
}
|
|
}, 60000);
|
|
}
|
|
|
|
test.skipIf(!Bun.which('zsh'))('zsh runs the same executed role selection and its deadline-capped dispatch', () => {
|
|
const rendered = RESOLVERS.CODEX_SELECT(ctxFor('codex', 'claude', HOST_PATHS.claude), ['exec', '540']);
|
|
const script = `_REPO_ROOT='${REPO}'\n${withRole(rendered, 'plan-review')}\n_CODEX_DEADLINE=$(( $(date +%s) + 1 ))\n"$_CODEX_PROBE" run-with-timeout 540 sleep 5; echo "RC=$? SEL=$_CODEX_SEL MODE=$_CODEX_SANDBOX_MODE"`;
|
|
const started = Date.now();
|
|
const r = shell(home(), script, 'zsh');
|
|
expect(r.status).toBe(0);
|
|
expect(r.stdout).toContain('RC=124 SEL=gpt-6-astra MODE=read-only');
|
|
expect(Date.now() - started).toBeLessThan(15000);
|
|
expect(r.calls.map(codexModel)).toEqual(['model="gpt-6-astra"']);
|
|
}, 30000);
|
|
});
|
|
|
|
describe('invocation-boundary migration notice', () => {
|
|
test('pure inspection writes nothing and does not suppress the first real notice', () => {
|
|
const h = home();
|
|
const inspect = spawnSync(path.join(ROOT, 'bin', 'gstack-models'), ['resolve', '--role', 'plan-review', '--provider', 'openai', '--json'],
|
|
{ env: env(h, 'claude'), encoding: 'utf8', timeout: 15000 });
|
|
expect(inspect.status).toBe(0);
|
|
expect(fs.readdirSync(h.state)).toEqual([]);
|
|
expect(run('claude', h).stderr).toContain(NOTICE);
|
|
});
|
|
|
|
for (const host of ['codex', 'claude'] as const) {
|
|
test(`${host === 'codex' ? 'Claude' : 'Codex'}: an unwritable marker repeats the notice instead of hiding it`, () => {
|
|
const h = home();
|
|
fs.chmodSync(h.state, 0o500);
|
|
try {
|
|
for (let i = 0; i < 2; i++) {
|
|
const r = run(host, h);
|
|
expect(r.status).toBe(0);
|
|
expect(r.stderr).toContain(NOTICE);
|
|
}
|
|
expect(fs.existsSync(marker(h.state))).toBe(false);
|
|
} finally { fs.chmodSync(h.state, 0o700); }
|
|
});
|
|
}
|
|
|
|
test('disabled reviews stay disabled: no probe, no notice, no Codex call', () => {
|
|
const block = outsideVoicePreflight(ctxFor('plan-eng-review', 'claude', HOST_PATHS.claude), { disabledBehavior: 'skip-all', role: 'plan-review' })
|
|
.match(/```bash\n([\s\S]*?)\n```/)![1]!;
|
|
for (const [config, mode] of [['codex_reviews: disabled\n', 'CODEX_MODE: disabled'], ['', 'CODEX_MODE: ready']] as const) {
|
|
const h = home(config);
|
|
fs.mkdirSync(path.join(h.dir, '.claude', 'skills'), { recursive: true });
|
|
fs.symlinkSync(ROOT, path.join(h.dir, '.claude', 'skills', 'gstack'));
|
|
fs.rmSync(CALLS, { force: true });
|
|
const r = spawnSync('bash', ['-c', block], { cwd: REPO, env: env(h, 'claude', { CLAUDECODE: '', GSTACK_ACTIVE_HOST: '' }), encoding: 'utf8', timeout: 20000 });
|
|
expect(r.stdout).toContain(mode);
|
|
const calls = fs.existsSync(CALLS) ? fs.readFileSync(CALLS, 'utf8') : '';
|
|
expect(calls).not.toContain('"exec"');
|
|
if (config) expect(calls).toBe('');
|
|
expect(r.stderr).not.toContain('NOTICE: gstack plan reviews');
|
|
expect(fs.existsSync(marker(h.state))).toBe(false);
|
|
}
|
|
});
|
|
});
|
|
|
|
describe('implementation handoff recommendation', () => {
|
|
test('known harnesses pass their native provider; other hosts get both; nothing switches the session', () => {
|
|
expect(generateImplementationModelHandoff(ctxFor('autoplan', 'claude', HOST_PATHS.claude))).toContain('gstack-models" resolve --role implementation --provider anthropic`');
|
|
expect(generateImplementationModelHandoff(ctxFor('spec', 'codex'))).toContain('resolve --role implementation --provider openai`');
|
|
const other = generateImplementationModelHandoff(ctxFor('plan-eng-review', 'kiro' as Host));
|
|
expect(other).toContain('resolve --role implementation`');
|
|
expect(other).toContain('one per provider');
|
|
for (const text of [other]) {
|
|
expect(text).toContain('cannot change this session');
|
|
expect(text).not.toMatch(/claude-|gpt-/);
|
|
}
|
|
});
|
|
|
|
test('the recommended command resolves offline from current settings without writes', () => {
|
|
const h = home('implementation_tier: frontier\n');
|
|
const r = spawnSync(path.join(ROOT, 'bin', 'gstack-models'), ['resolve', '--role', 'implementation', '--provider', 'anthropic', '--json'],
|
|
{ env: env(h, 'claude'), encoding: 'utf8', timeout: 15000 });
|
|
expect(r.status).toBe(0);
|
|
expect(JSON.parse(r.stdout).selections[0]).toMatchObject({ role: 'implementation', requestedModel: 'claude-fable-5-1' });
|
|
expect(fs.readdirSync(h.state)).toEqual(['config.yaml']);
|
|
});
|
|
});
|