mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-18 22:24:17 +01:00
The Codex readiness check and the second-model review pick the same model for a custom Codex provider (GST-3089) (#4491)
With a custom Codex provider, `gstack-codex-probe role-ready` and `gstack-outside-voice run --runner codex-cli` now resolve the plan-review model the same way, so a session no longer skips a review that would have run. Why: the check refused a `config.toml` naming `gpt-6-astra` while the runner ran it, and a day-one eval session dropped the second-model review in every phase. Nothing to do: a custom provider naming the frontier catalog ID just works; any other name now says how to opt in. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
b6ca54ec09
commit
45f879b534
8 files changed
+215
-20
No files matched your search
+18
-9
@@ -5,7 +5,9 @@
|
||||
# went before, and sets nothing in the caller's shell.
|
||||
#
|
||||
# Subcommands (each wraps one function below):
|
||||
# select-model <exec|review> [--model <id>] CODEX_SEL:, CODEX_SEL_KIND:, CODEX_SANDBOX:
|
||||
# select-model <exec|review> [--model <id>] [--role plan-review]
|
||||
# CODEX_SEL:, CODEX_SEL_KIND:, CODEX_SANDBOX:
|
||||
# (--role: role-ready's selection, without its checks)
|
||||
# check-effort <default-effort> [--request <level>]
|
||||
# CODEX_EFFORT:; exit 0, or 1 on an invalid setting
|
||||
# check-auth exit 0 authed, 1 not
|
||||
@@ -892,7 +894,9 @@ _gstack_codex_effort_check() {
|
||||
_gstack_codex_usage() {
|
||||
printf '%s\n' \
|
||||
'Usage: gstack-codex-probe <subcommand> [args]' \
|
||||
' select-model <exec|review> [--model <id>] print CODEX_SEL:, CODEX_SEL_KIND: and CODEX_SANDBOX:; exit 0 or 1' \
|
||||
' select-model <exec|review> [--model <id>] [--role plan-review]' \
|
||||
' print CODEX_SEL:, CODEX_SEL_KIND: and CODEX_SANDBOX:; exit 0 or 1;' \
|
||||
' --role selects as role-ready does, without its checks' \
|
||||
' check-effort <default-effort> [--request <level>]' \
|
||||
' the request level, else GSTACK_CODEX_EFFORT, else codex_effort,' \
|
||||
' else the default; print CODEX_EFFORT:; exit 0, 1 on an invalid setting' \
|
||||
@@ -930,17 +934,22 @@ _gstack_codex_kind_args() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Parses "<exec|review> [--model <id>] [--cwd <dir>]" for role-ready.
|
||||
# Parses "<exec|review> [--model <id>] [--cwd <dir>]" for role-ready, and
|
||||
# "<exec|review> [--model <id>] [--role plan-review]" for select-model ($1 = select).
|
||||
_gstack_codex_role_args() {
|
||||
local _for="$1"
|
||||
shift
|
||||
_GSTACK_CODEX_ARG_KIND="${1:-}"
|
||||
_GSTACK_CODEX_ARG_MODEL=""
|
||||
_GSTACK_CODEX_ARG_CWD=""
|
||||
_GSTACK_CODEX_ARG_ROLE=""
|
||||
case "$_GSTACK_CODEX_ARG_KIND" in exec|review) shift ;; *) return 64 ;; esac
|
||||
while [ "$#" -gt 0 ]; do
|
||||
{ [ "$#" -ge 2 ] && [ -n "$2" ]; } || return 64
|
||||
case "$1" in
|
||||
--model) _GSTACK_CODEX_ARG_MODEL="$2" ;;
|
||||
--cwd) _GSTACK_CODEX_ARG_CWD="$2" ;;
|
||||
case "$_for:$1" in
|
||||
*:--model) _GSTACK_CODEX_ARG_MODEL="$2" ;;
|
||||
ready:--cwd) _GSTACK_CODEX_ARG_CWD="$2" ;;
|
||||
select:--role) [ "$2" = plan-review ] || return 64; _GSTACK_CODEX_ARG_ROLE="$2" ;;
|
||||
*) return 64 ;;
|
||||
esac
|
||||
shift 2
|
||||
@@ -958,8 +967,8 @@ _gstack_codex_main() {
|
||||
case "$_cmd" in
|
||||
help|-h|--help) _gstack_codex_usage ;;
|
||||
select-model)
|
||||
_gstack_codex_kind_args "$@" || { _gstack_codex_usage >&2; return 64; }
|
||||
_gstack_codex_select_model "$_GSTACK_CODEX_ARG_KIND" "$_GSTACK_CODEX_ARG_MODEL" || return 1
|
||||
_gstack_codex_role_args select "$@" || { _gstack_codex_usage >&2; return 64; }
|
||||
_gstack_codex_select_model "$_GSTACK_CODEX_ARG_KIND" "$_GSTACK_CODEX_ARG_MODEL" "$_GSTACK_CODEX_ARG_ROLE" || return 1
|
||||
printf 'CODEX_SEL: %s\nCODEX_SEL_KIND: %s\nCODEX_SANDBOX: %s\n' "$_GSTACK_CODEX_SEL" "$_GSTACK_CODEX_SEL_KIND" "$_GSTACK_CODEX_SANDBOX" ;;
|
||||
check-effort)
|
||||
{ [ "$#" -eq 1 ] || { [ "$#" -eq 3 ] && [ "$2" = "--request" ]; }; } || { _gstack_codex_usage >&2; return 64; }
|
||||
@@ -992,7 +1001,7 @@ _gstack_codex_main() {
|
||||
esac
|
||||
return "$_rc" ;;
|
||||
role-ready)
|
||||
_gstack_codex_role_args "$@" || { _gstack_codex_usage >&2; return 64; }
|
||||
_gstack_codex_role_args ready "$@" || { _gstack_codex_usage >&2; return 64; }
|
||||
_gstack_codex_role_ready "$_GSTACK_CODEX_ARG_KIND" "$_GSTACK_CODEX_ARG_CWD" "$_GSTACK_CODEX_ARG_MODEL" >&2
|
||||
_rc=$?
|
||||
[ "$_rc" -eq 0 ] || return "$_rc"
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
- **The Codex readiness check and the second-model review now pick the same model for a custom Codex provider (GST-3089).** `gstack-codex-probe role-ready` refused a custom provider even when its `~/.codex/config.toml` named `gpt-6-astra`, while `gstack-outside-voice run --runner codex-cli` ran whatever that file named, so a session that trusted the check skipped a review that would have run. Both now use one selection. A custom provider's own `model` counts for plan review when it is exactly the selected tier's catalog ID, any other name is refused as `custom_provider_model_not_tier`, and no name stays `custom_provider_requires_model`. Each of these policy refusals prints a `CODEX_MODEL_POLICY: <reason>` line. For the `autoplan` and `spec` gates the runner now also follows the plan-review tier with the stock OpenAI provider, instead of `config.toml`'s model.
|
||||
+10
-1
@@ -106,7 +106,16 @@ an unknown or multi-provider host can show both choices.
|
||||
|
||||
Custom endpoints and partner platforms need an explicitly compatible model or
|
||||
`host` mode. Gstack does not guess that public API IDs work with Bedrock, Vertex,
|
||||
Foundry or a custom Codex provider. A model identifier passing syntax validation
|
||||
Foundry or a custom Codex provider. One case counts as explicit: when a custom
|
||||
Codex provider's own `config.toml` sets `model` to exactly the selected tier's
|
||||
catalog ID (the catalog's OpenAI frontier model, by default), plan reviews use it,
|
||||
because you already route that ID to your endpoint. Any other name there, such
|
||||
as a smart-tier model or a deployment alias, is refused
|
||||
(`custom_provider_model_not_tier`): it could be any model, so name it with the
|
||||
tier setting or the environment override if you want it as your plan reviewer.
|
||||
`gstack-codex-probe role-ready` and `gstack-outside-voice run --runner codex-cli`
|
||||
(for the `autoplan` and `spec` gates) share this selection, so the readiness
|
||||
check and the review never pick different models. A model identifier passing syntax validation
|
||||
is not proof of account availability. See [provider recovery](troubleshooting.md#model-policy-provider).
|
||||
|
||||
Detection is conservative, not a replacement for the CLI's settings engine.
|
||||
|
||||
@@ -68,6 +68,12 @@ custom endpoint or partner platform. A public API catalog identifier is not
|
||||
necessarily a valid deployment identifier there. Malformed native provider
|
||||
configuration also prevents gstack from assuming the public endpoint.
|
||||
|
||||
A custom Codex provider whose `config.toml` `model` is exactly the selected
|
||||
tier's catalog ID is used as is. The refusal prints
|
||||
`CODEX_MODEL_POLICY: custom_provider_requires_model` when `config.toml` names
|
||||
no model, and `CODEX_MODEL_POLICY: custom_provider_model_not_tier` when it names
|
||||
a different one.
|
||||
|
||||
**Fix.** Supply a compatible model for that request, set the provider's gstack
|
||||
environment override or tier-model setting, or choose
|
||||
`gstack-config set plan_review_tier host`. Keep credentials in the provider's
|
||||
|
||||
@@ -56,6 +56,7 @@ export type ModelPolicyReason =
|
||||
| 'invalid_implementation_tier'
|
||||
| 'invalid_model_id'
|
||||
| 'custom_provider_requires_model'
|
||||
| 'custom_provider_model_not_tier'
|
||||
| 'native_provider_unresolved'
|
||||
| 'provider_required';
|
||||
|
||||
@@ -64,6 +65,7 @@ export type SelectionSource =
|
||||
| { kind: 'env'; name: (typeof PROVIDER_ENV_OVERRIDES)[ModelProvider]; label: string }
|
||||
| { kind: 'config'; key: PolicyConfigKey; path: string; label: string }
|
||||
| { kind: 'catalog'; catalogVersion: typeof MODEL_CATALOG_VERSION; verifiedAt: string; sourceUrl: string; label: string }
|
||||
| { kind: 'native'; label: string }
|
||||
| { kind: 'host'; reason: 'host-mode'; key: 'plan_review_tier'; label: string };
|
||||
|
||||
export interface ModelPolicyErrorJson {
|
||||
@@ -528,6 +530,8 @@ export function resolvePlanReviewModel(opts: {
|
||||
claudeManagedDir?: string;
|
||||
codexSystemConfig?: string | null;
|
||||
config?: ModelPolicyConfig;
|
||||
/** The model the provider's own settings name (Codex config.toml), read only once custom routing is detected. */
|
||||
nativeModel?: () => { model: string; label: string } | null;
|
||||
}): ModelSelection {
|
||||
const env = opts.env ?? process.env;
|
||||
const platform = opts.platform ?? process.platform;
|
||||
@@ -562,6 +566,21 @@ export function resolvePlanReviewModel(opts: {
|
||||
const entry = catalogEntry(provider, tier);
|
||||
const custom = detectCustomProvider(provider, { ...opts, env, platform });
|
||||
if (custom.custom) {
|
||||
const native = opts.nativeModel?.() ?? null;
|
||||
if (native?.model === entry.model) {
|
||||
return { ...base, status: 'selected', requestedModel: native.model, source: { kind: 'native', label: native.label } };
|
||||
}
|
||||
if (native) {
|
||||
throw new ModelPolicyError({
|
||||
reason: 'custom_provider_model_not_tier',
|
||||
problem: `your custom ${PROVIDER_LABEL[provider]} endpoint is set to '${sanitize(native.model)}' (${sanitize(native.label)}), which is not gstack's ${tier} plan-review model ${entry.model}`,
|
||||
cause: `custom routing is active (${custom.signals.join(', ')}); gstack takes the endpoint's own model for plan review only when it is the ${tier} catalog ID, because any other name could be any model`,
|
||||
key: native.label,
|
||||
source: 'native',
|
||||
repair: [`gstack-config set ${overrideKey} <model-id-your-endpoint-serves>`, `export ${envName}=<model-id-your-endpoint-serves>`, 'gstack-config set plan_review_tier host'],
|
||||
docs: DOCS_PROVIDER,
|
||||
});
|
||||
}
|
||||
throw new ModelPolicyError({
|
||||
reason: 'custom_provider_requires_model',
|
||||
problem: `no ${tier} plan-review model is set for your custom ${PROVIDER_LABEL[provider]} endpoint`,
|
||||
@@ -654,6 +673,7 @@ export function selectionRepair(selection: ModelSelection): string[] {
|
||||
if (source.kind === 'request') return ['name a different model for this request, or omit it to use the configured policy'];
|
||||
if (source.kind === 'env') return [`unset ${source.name}`, `export ${source.name}=<model-id>`];
|
||||
if (source.kind === 'config') return [`gstack-config set ${source.key} <model-id>`, `gstack-config unset ${source.key}`];
|
||||
if (source.kind === 'native') return [`change ${source.label}`, `gstack-config set ${MODEL_OVERRIDE_KEYS[provider][selection.tier as ModelTier]} <model-id>`, 'gstack-config set plan_review_tier host'];
|
||||
const tierKey = selection.role === 'plan-review' ? 'plan_review_tier' : 'implementation_tier';
|
||||
if (source.kind === 'host') {
|
||||
const native = provider === 'anthropic' ? 'choose the model in Claude Code settings' : 'set model in ${CODEX_HOME:-~/.codex}/config.toml';
|
||||
|
||||
@@ -132,7 +132,8 @@ class CodexCliRunner implements OutsideRunner {
|
||||
if (!avail.ok) return finish({ ...base, status: 'unavailable', exit: 1, reason: 'cli_missing', detail: avail.reason }, started);
|
||||
const probe = path.join(req.binDir ?? DEFAULT_BIN, 'gstack-codex-probe');
|
||||
if (req.model) env.GSTACK_CODEX_MODEL = req.model;
|
||||
const select = Bun.spawnSync([probe, 'select-model', 'exec', ...(req.model ? ['--model', req.model] : [])], { cwd: req.cwd, env, stdout: 'pipe', stderr: 'pipe', timeout: 60_000 });
|
||||
const role = req.gate === 'autoplan' || req.gate === 'spec' ? ['--role', 'plan-review'] : [];
|
||||
const select = Bun.spawnSync([probe, 'select-model', 'exec', ...(req.model ? ['--model', req.model] : []), ...role], { cwd: req.cwd, env, stdout: 'pipe', stderr: 'pipe', timeout: 60_000 });
|
||||
const selOut = new TextDecoder().decode(select.stdout);
|
||||
const selected = /^CODEX_SEL: (.+)$/m.exec(selOut)?.[1]?.trim();
|
||||
const sandbox = /^CODEX_SANDBOX: (.+)$/m.exec(selOut)?.[1]?.trim();
|
||||
|
||||
@@ -145,9 +145,25 @@ export function resolveCodexRuntimeModel(opts: {
|
||||
};
|
||||
if (opts.explicit) return chosen('explicit request', opts.explicit);
|
||||
if (env.GSTACK_CODEX_MODEL) return chosen('GSTACK_CODEX_MODEL', env.GSTACK_CODEX_MODEL);
|
||||
const named = codexConfigModel({ ...opts, env });
|
||||
if (named.value === null) return { kind: opts.kind, model: CODEX_DEFAULT_MODEL, source: `gstack default (${named.source})` };
|
||||
return chosen(named.source, named.value);
|
||||
}
|
||||
|
||||
const home = opts.home ?? env.HOME ?? os.homedir();
|
||||
const codexHome = opts.codexHome ?? env.CODEX_HOME ?? path.join(home, '.codex');
|
||||
/**
|
||||
* The model Codex's own config.toml names for this invocation kind
|
||||
* (`review_model` first for native review, then the legacy active profile's
|
||||
* model, then `model`). `value` is null when it names none, and `source`
|
||||
* then says what was missing. Shared by the runtime and plan-review records.
|
||||
*/
|
||||
function codexConfigModel(opts: {
|
||||
kind: CodexInvocationKind;
|
||||
env: Record<string, string | undefined>;
|
||||
codexHome?: string;
|
||||
home?: string;
|
||||
}): { value: string | null; source: string } {
|
||||
const home = opts.home ?? opts.env.HOME ?? os.homedir();
|
||||
const codexHome = opts.codexHome ?? opts.env.CODEX_HOME ?? path.join(home, '.codex');
|
||||
if (!path.isAbsolute(codexHome)) {
|
||||
throw new Error(`CODEX_HOME '${sanitize(codexHome)}' is not an absolute path, so its config.toml model cannot be read. ${REPAIR}`);
|
||||
}
|
||||
@@ -156,9 +172,7 @@ export function resolveCodexRuntimeModel(opts: {
|
||||
try {
|
||||
raw = fs.readFileSync(configPath, 'utf8');
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
|
||||
return { kind: opts.kind, model: CODEX_DEFAULT_MODEL, source: `gstack default (no ${stripControl(configPath)})` };
|
||||
}
|
||||
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return { value: null, source: `no ${stripControl(configPath)}` };
|
||||
throw new Error(`Could not read ${sanitize(configPath)}. ${REPAIR}`);
|
||||
}
|
||||
let parsed: Record<string, unknown>;
|
||||
@@ -178,9 +192,9 @@ export function resolveCodexRuntimeModel(opts: {
|
||||
for (const [source, value] of candidates) {
|
||||
if (value === undefined) continue;
|
||||
if (typeof value !== 'string') throw new Error(`${sanitize(source)} is not a string. ${REPAIR}`);
|
||||
return chosen(source, value);
|
||||
return { value, source };
|
||||
}
|
||||
return { kind: opts.kind, model: CODEX_DEFAULT_MODEL, source: `gstack default (no model in ${stripControl(configPath)})` };
|
||||
return { value: null, source: `no model in ${stripControl(configPath)}` };
|
||||
}
|
||||
|
||||
export interface CodexPlanReviewSelection extends CodexRuntimeModelSelection {
|
||||
@@ -204,7 +218,11 @@ export function resolveCodexPlanReviewModel(opts: {
|
||||
cwd?: string;
|
||||
}): CodexPlanReviewSelection {
|
||||
const env = opts.env ?? process.env;
|
||||
const selection = resolvePlanReviewModel({ provider: 'openai', requestedModel: opts.explicit || undefined, env, cwd: opts.cwd });
|
||||
const nativeModel = () => {
|
||||
const named = codexConfigModel({ ...opts, env });
|
||||
return named.value === null ? null : { model: named.value.trim(), label: stripControl(named.source) };
|
||||
};
|
||||
const selection = resolvePlanReviewModel({ provider: 'openai', requestedModel: opts.explicit || undefined, env, cwd: opts.cwd, nativeModel });
|
||||
const repair = stripControl(selectionRepair(selection).join('; '));
|
||||
if (selection.status === 'selected') {
|
||||
const model = selection.requestedModel;
|
||||
@@ -219,7 +237,7 @@ export function resolveCodexPlanReviewModel(opts: {
|
||||
|
||||
function policyFailure(error: unknown): string {
|
||||
if (!(error instanceof ModelPolicyError)) return (error as Error).message;
|
||||
return [`${error.problem} ${error.cause}`.trim(), ...error.repair.map(step => `Repair: ${step}`), `Docs: ${error.docs}`].join('\n');
|
||||
return [`${error.problem} ${error.cause}`.trim(), ...error.repair.map(step => `Repair: ${step}`), `Docs: ${error.docs}`, `CODEX_MODEL_POLICY: ${error.reason}`].join('\n');
|
||||
}
|
||||
|
||||
function readArg(name: string): string | undefined {
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
/**
|
||||
* GST-3089: `gstack-codex-probe role-ready` and `gstack-outside-voice run
|
||||
* --runner codex-cli` resolve a custom Codex provider's plan-review model
|
||||
* through one resolver, so a session that trusts the probe never drops a
|
||||
* review the runner would have run, and the runner never runs a model the
|
||||
* probe refused. Synthetic CODEX_HOME/config.toml, fake codex CLI, no paid calls.
|
||||
*/
|
||||
import { afterAll, describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { RESULT_FORMAT_HEADER } from '../lib/autoplan-reconcile';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const PROBE = path.join(ROOT, 'bin', 'gstack-codex-probe');
|
||||
const OUTSIDE = path.join(ROOT, 'bin', 'gstack-outside-voice');
|
||||
const TMP = fs.mkdtempSync(path.join(os.tmpdir(), 'codex-custom-agree-'));
|
||||
const FAKE_BIN = path.join(TMP, 'bin');
|
||||
const FAKE = path.join(TMP, 'fake-codex.ts');
|
||||
const PROMPT = path.join(TMP, 'prompt.md');
|
||||
fs.mkdirSync(FAKE_BIN, { recursive: true });
|
||||
fs.writeFileSync(PROMPT, `You are the outside CEO reviewer.\n${RESULT_FORMAT_HEADER}\n`);
|
||||
fs.writeFileSync(FAKE, `
|
||||
import { appendFileSync, writeFileSync } from 'node:fs';
|
||||
const args = process.argv.slice(2);
|
||||
if (args[0] === 'sandbox') process.exit(0);
|
||||
if (args[0] === '--version') { console.log('codex-cli 0.160.0'); process.exit(0); }
|
||||
const probe = args.includes('reply OK');
|
||||
appendFileSync(process.env.CALLS!, JSON.stringify({ probe, model: args.find(a => a.startsWith('model=')) ?? null }) + '\\n');
|
||||
if (probe) { console.log('OK'); process.exit(0); }
|
||||
await Bun.stdin.text();
|
||||
writeFileSync(args[args.indexOf('-o') + 1], 'Recommendation: proceed because the plan is sound.\\nVERDICT: CLEAN\\n');
|
||||
`);
|
||||
fs.writeFileSync(path.join(FAKE_BIN, 'codex'), `#!/usr/bin/env bash\nexec bun "${FAKE}" "$@"\n`, { mode: 0o755 });
|
||||
afterAll(() => fs.rmSync(TMP, { recursive: true, force: true }));
|
||||
|
||||
const CUSTOM = (model: string | null) => [
|
||||
...(model ? [`model = "${model}"`] : []),
|
||||
'model_provider = "proxy"',
|
||||
'',
|
||||
'[model_providers.proxy]\nname = "Team proxy"\nbase_url = "https://proxy.example.invalid/v1"\nenv_key = "PROXY_API_KEY"\n',
|
||||
].join('\n');
|
||||
|
||||
let counter = 0;
|
||||
function fixture(configToml: string) {
|
||||
const dir = path.join(TMP, `h${++counter}`);
|
||||
for (const sub of ['.codex', 'state', 'repo']) fs.mkdirSync(path.join(dir, sub), { recursive: true });
|
||||
fs.writeFileSync(path.join(dir, '.codex', 'config.toml'), configToml);
|
||||
const calls = path.join(dir, 'calls.jsonl');
|
||||
const env: Record<string, string> = {};
|
||||
for (const [k, v] of Object.entries(process.env)) {
|
||||
if (v !== undefined && !/^(OPENAI|CODEX|GSTACK)_/.test(k)) env[k] = v;
|
||||
}
|
||||
Object.assign(env, {
|
||||
PATH: `${FAKE_BIN}${path.delimiter}${process.env.PATH ?? ''}`, HOME: dir, CODEX_HOME: path.join(dir, '.codex'),
|
||||
GSTACK_STATE_ROOT: path.join(dir, 'state'), PROXY_API_KEY: 'test-key', CODEX_API_KEY: 'test-key', CALLS: calls, _TEL: 'off',
|
||||
});
|
||||
const readCalls = () => fs.existsSync(calls) ? fs.readFileSync(calls, 'utf8').split('\n').filter(Boolean).map(l => JSON.parse(l) as { probe: boolean; model: string | null }) : [];
|
||||
return { dir, env, repo: path.join(dir, 'repo'), calls, readCalls };
|
||||
}
|
||||
|
||||
const POLICY = /^CODEX_MODEL_POLICY: (\S+)$/m;
|
||||
|
||||
function both(configToml: string) {
|
||||
const f = fixture(configToml);
|
||||
const ready = spawnSync(PROBE, ['role-ready', 'exec', '--cwd', f.repo], { env: f.env, encoding: 'utf8', timeout: 30_000 });
|
||||
const readyCalls = f.readCalls();
|
||||
fs.rmSync(f.calls, { force: true });
|
||||
const out = path.join(f.dir, 'out', 'outside.md');
|
||||
const voice = spawnSync('bun', [OUTSIDE, 'run', '--runner', 'codex-cli', '--prompt', PROMPT, '--out', out, '--cwd', f.repo, '--timeout-ms', '20000'],
|
||||
{ cwd: f.repo, env: f.env, encoding: 'utf8', timeout: 60_000 });
|
||||
const meta = fs.existsSync(`${out}.meta.json`) ? JSON.parse(fs.readFileSync(`${out}.meta.json`, 'utf8')) : null;
|
||||
return {
|
||||
ready: { status: ready.status, model: /^CODEX_SEL: (.+)$/m.exec(ready.stdout)?.[1] ?? null, policy: POLICY.exec(ready.stderr)?.[1] ?? null, stderr: ready.stderr, calls: readyCalls },
|
||||
voice: { status: voice.status, meta, policy: POLICY.exec(meta?.detail ?? '')?.[1] ?? null, stderr: voice.stderr, calls: f.readCalls() },
|
||||
};
|
||||
}
|
||||
|
||||
describe('role-ready and the codex-cli runner agree on a custom provider (GST-3089)', () => {
|
||||
test('config.toml names the frontier catalog model: both accept it and run it', () => {
|
||||
const r = both(CUSTOM('gpt-6-astra'));
|
||||
expect({ status: r.ready.status, model: r.ready.model }).toEqual({ status: 0, model: 'gpt-6-astra' });
|
||||
expect(r.ready.calls.filter(c => c.probe).map(c => c.model)).toEqual(['model="gpt-6-astra"']);
|
||||
expect(r.voice.meta).toMatchObject({ runner: 'codex-cli', model: 'gpt-6-astra' });
|
||||
expect(r.voice.meta.reason).not.toBe('model_unselected');
|
||||
expect(r.voice.calls.filter(c => !c.probe).map(c => c.model)).toEqual(['model="gpt-6-astra"']);
|
||||
});
|
||||
|
||||
test('config.toml names a non-frontier model: both refuse with the same reason code and send nothing', () => {
|
||||
const r = both(CUSTOM('gpt-6.1-sol'));
|
||||
expect(r.ready.status).toBe(1);
|
||||
expect(r.ready.model).toBeNull();
|
||||
expect(r.ready.policy).toBe('custom_provider_model_not_tier');
|
||||
expect(r.ready.stderr).toContain("'gpt-6.1-sol'");
|
||||
expect(r.ready.calls).toEqual([]);
|
||||
expect(r.voice.meta).toMatchObject({ status: 'unavailable', reason: 'model_unselected' });
|
||||
expect(r.voice.policy).toBe(r.ready.policy);
|
||||
expect(r.voice.calls).toEqual([]);
|
||||
});
|
||||
|
||||
test('config.toml names no model: both refuse with the same reason code and send nothing', () => {
|
||||
const r = both(CUSTOM(null));
|
||||
expect(r.ready.status).toBe(1);
|
||||
expect(r.ready.policy).toBe('custom_provider_requires_model');
|
||||
expect(r.ready.calls).toEqual([]);
|
||||
expect(r.voice.meta).toMatchObject({ status: 'unavailable', reason: 'model_unselected' });
|
||||
expect(r.voice.policy).toBe(r.ready.policy);
|
||||
expect(r.voice.calls).toEqual([]);
|
||||
});
|
||||
|
||||
test('the tier decides which catalog ID counts: plan_review_tier smart accepts the smart ID from config.toml', () => {
|
||||
const f = fixture(CUSTOM('gpt-6.1-sol'));
|
||||
fs.writeFileSync(path.join(f.dir, 'state', 'config.yaml'), 'plan_review_tier: smart\n');
|
||||
const r = spawnSync(PROBE, ['select-model', 'exec', '--role', 'plan-review'], { env: f.env, encoding: 'utf8', timeout: 30_000 });
|
||||
expect(r.status).toBe(0);
|
||||
expect(r.stdout.split('\n')[0]).toBe('CODEX_SEL: gpt-6.1-sol');
|
||||
expect(r.stderr).toContain('tier: smart; source: ');
|
||||
for (const bad of [['select-model', 'exec', '--role'], ['select-model', 'exec', '--role', 'implementation'], ['select-model', 'exec', '--cwd', f.repo]]) {
|
||||
const u = spawnSync(PROBE, bad, { env: f.env, encoding: 'utf8', timeout: 30_000 });
|
||||
expect({ bad, status: u.status, stdout: u.stdout }).toEqual({ bad, status: 64, stdout: '' });
|
||||
}
|
||||
});
|
||||
|
||||
test('negative control: the stock OpenAI provider keeps the catalog frontier model in both, whatever config.toml names', () => {
|
||||
const r = both('model = "gpt-6.1-sol"\n');
|
||||
expect({ status: r.ready.status, model: r.ready.model }).toEqual({ status: 0, model: 'gpt-6-astra' });
|
||||
expect(r.voice.meta).toMatchObject({ model: 'gpt-6-astra' });
|
||||
expect(r.voice.calls.filter(c => !c.probe).map(c => c.model)).toEqual(['model="gpt-6-astra"']);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user