mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-12 03:30:28 +01:00
feat(codex): run gstack-codex-probe as a command, one subcommand per probe
Generated skills now execute bin/gstack-codex-probe (select-model, check-auth, show-sandbox, check-sandbox, probe-model, check-version, show-first-use-notice, run-with-timeout, log-event, log-hang, help) and capture its KEY: value lines, so no helper state crosses into the calling shell and the preflight works from bash, zsh or sh. select-model keeps --model > GSTACK_CODEX_MODEL > config.toml precedence; run-with-timeout supervises with the existing gtimeout/timeout/ watchdog chain and returns 124 on a deadline; an unknown subcommand exits 64. Sourcing keeps working for skills rendered before the upgrade and prints one deprecation line naming /gstack-upgrade; it and the zsh self-locate block are removed in a release on or after 2026-10-21 (DX-18). Parity caps raised with measured notes for the longer capture lines. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed
1 parent
7fab2a7139
commit
64e8f90f15
47 files changed
+937
-399
No files matched your search
@@ -263,8 +263,8 @@ test.each(['assignment-only','set-config','other-config','foreign-reader','or-co
|
||||
if(kind==='inside-body')input.command=input.command.replace(guards+'\n','').replace('_OUTSIDE_EXIT=0','_OUTSIDE_EXIT=0\n'+guards);
|
||||
if(kind==='before-cd')input.command=guards+'\ncd '+f.dir+'\n'+f.options.commands.outside.replace("'<prepared-prompt-file>'","'"+f.file+"'");
|
||||
if(kind==='changed-harness')input.command=input.command.replace('exit 78','exit 0');
|
||||
if(kind==='changed-timeout')input.command=input.command.replace('_gstack_codex_timeout_wrapper 540','_gstack_codex_timeout_wrapper 1');
|
||||
if(kind==='changed-sandbox')input.command=input.command.replace('-s "${_GSTACK_CODEX_SANDBOX:?}"','-s danger-full-access');
|
||||
if(kind==='changed-timeout')input.command=input.command.replace('run-with-timeout 540','run-with-timeout 1');
|
||||
if(kind==='changed-sandbox')input.command=input.command.replace('-s "${_CODEX_SANDBOX_MODE:?}"','-s danger-full-access');
|
||||
if(kind==='changed-prompt')input.command=input.command.replace('codex exec - ','codex exec "Different plan" ');
|
||||
if(kind==='skipped-validator')input.command=input.command.replace(/^bun .*outside-review-result.*\n/m,'');
|
||||
if(kind==='suffix')input.command+='\ntrue';
|
||||
|
||||
@@ -120,7 +120,7 @@ describe('content-binding template drift', () => {
|
||||
const codex = rendered('codex/sections/review-mode.md');
|
||||
const starts = [...codex.matchAll(/gstack-review-log --start codex-review/g)];
|
||||
expect(starts).toHaveLength(2);
|
||||
expect(starts[0].index).toBeLessThan(codex.indexOf('_gstack_codex_timeout_wrapper 330 codex review'));
|
||||
expect(starts[0].index).toBeLessThan(codex.indexOf('run-with-timeout 330 codex review'));
|
||||
expect(starts[1].index).toBeLessThan(codex.indexOf('git diff "<base>...HEAD"'));
|
||||
expect(codex).toContain('--finish CODEX_REVIEW_START');
|
||||
expect(codex).toContain('"completed":COMPLETED,"converged":CONVERGED');
|
||||
|
||||
@@ -645,20 +645,20 @@ describe('codex timeout wrapper: /review + /ship diff passes', () => {
|
||||
|
||||
test(`${relPath}: both diff-review Codex calls run under the wrapper`, () => {
|
||||
const wrapped =
|
||||
read().match(/_gstack_codex_timeout_wrapper\s+\d+\s+codex\s+(exec|review)\b/g) ?? [];
|
||||
read().match(/run-with-timeout\s+\d+\s+codex\s+(exec|review)\b/g) ?? [];
|
||||
// Adversarial pass + structured review pass.
|
||||
expect(wrapped.length).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
|
||||
test(`${relPath}: does not claim \`timeout\` is unavailable on macOS`, () => {
|
||||
// _gstack_codex_timeout_wrapper resolves gtimeout -> timeout -> unwrapped,
|
||||
// run-with-timeout resolves gtimeout -> timeout -> a bash watchdog,
|
||||
// so the coreutils-less case is already handled. The old claim is what
|
||||
// steered these call sites away from the wrapper in the first place.
|
||||
expect(read()).not.toMatch(/doesn't exist on macOS/);
|
||||
});
|
||||
|
||||
test(`${relPath}: wrapper budget stays under the outer Bash gate`, () => {
|
||||
const budgets = [...read().matchAll(/_gstack_codex_timeout_wrapper\s+(\d+)\s+codex\b/g)].map(
|
||||
const budgets = [...read().matchAll(/run-with-timeout\s+(\d+)\s+codex\b/g)].map(
|
||||
(m) => Number(m[1]) * 1000,
|
||||
);
|
||||
expect(budgets.length).toBeGreaterThan(0);
|
||||
@@ -710,13 +710,13 @@ describe('codex skeleton+sections union: review sandbox + fail-closed gate + tim
|
||||
test(`${relPath}: (a) every scoped codex review invocation pins sandbox_mode from the selected sandbox`, () => {
|
||||
const invocations = read()
|
||||
.split('\n')
|
||||
.filter((l) => /_gstack_codex_timeout_wrapper\s+\d+\s+codex\s+review\b/.test(l));
|
||||
.filter((l) => /run-with-timeout\s+\d+\s+codex\s+review\b/.test(l));
|
||||
expect(invocations.length).toBeGreaterThanOrEqual(1);
|
||||
for (const line of invocations) {
|
||||
// _gstack_codex_select_model sets read-only (full access only for
|
||||
// select-model reports read-only (full access only for
|
||||
// GSTACK_CODEX_NO_SANDBOX=1, test/codex-model-probe.test.ts); :? stops
|
||||
// an unselected command instead of inheriting config.toml's default.
|
||||
expect(line).toContain('sandbox_mode=\\"${_GSTACK_CODEX_SANDBOX:?}\\"');
|
||||
expect(line).toContain('sandbox_mode=\\"${_CODEX_SANDBOX_MODE:?}\\"');
|
||||
// `codex review` has no -s/--sandbox flag (verified 0.147.0) — the
|
||||
// config override is the only lever. `-s read-only` here would fail
|
||||
// at argv parsing, which check (b) would then read as a gate FAIL.
|
||||
@@ -766,13 +766,13 @@ describe('codex skeleton+sections union: review sandbox + fail-closed gate + tim
|
||||
test(`${relPath}: (c) every Bash gate sits strictly above its section's wrapper budgets`, () => {
|
||||
// Split on `## ` headings; within any section that declares BOTH a Bash
|
||||
// tool gate (`timeout: N` in ms) and a wrapper budget
|
||||
// (`_gstack_codex_timeout_wrapper S codex`), every gate must be strictly
|
||||
// (`run-with-timeout S codex`), every gate must be strictly
|
||||
// greater than every wrapper budget so the wrapper fires first.
|
||||
const sections = read().split(/\n## /);
|
||||
const inspected: string[] = [];
|
||||
for (const section of sections) {
|
||||
const gates = [...section.matchAll(/timeout:\s*(\d{4,})/g)].map((m) => Number(m[1]));
|
||||
const wrappers = [...section.matchAll(/_gstack_codex_timeout_wrapper\s+(\d+)\s+codex\b/g)].map(
|
||||
const wrappers = [...section.matchAll(/run-with-timeout\s+(\d+)\s+codex\b/g)].map(
|
||||
(m) => Number(m[1]) * 1000,
|
||||
);
|
||||
if (gates.length === 0 || wrappers.length === 0) continue;
|
||||
@@ -941,17 +941,17 @@ describe('codex broken-install detection (#2742)', () => {
|
||||
for (const rel of ['autoplan/SKILL.md.tmpl', 'autoplan/SKILL.md']) {
|
||||
const raw = fs.readFileSync(path.join(ROOT, rel), 'utf-8');
|
||||
const src = rel.endsWith('.tmpl') ? raw.replace('{{OUTSIDE_PREFLIGHT:autoplan}}', RESOLVERS.OUTSIDE_PREFLIGHT({ host: 'claude', paths: HOST_PATHS.claude, skillName: 'autoplan', tmplPath: rel }, ['autoplan'])) : raw;
|
||||
expect(src).toContain('_gstack_codex_model_probe; _CODEX_MP=$?');
|
||||
expect(src).toContain('_CODEX_PO=$("$_CODEX_PROBE" probe-model $_CODEX_KIND); _CODEX_MP=$?');
|
||||
expect(src).toMatch(/_CODEX_MP" -eq 2/);
|
||||
expect(src).toContain('binary cannot run');
|
||||
expect(src).not.toContain('elif ! _gstack_codex_model_probe');
|
||||
expect(src).not.toContain('elif ! "$_CODEX_PROBE" probe-model');
|
||||
}
|
||||
});
|
||||
|
||||
test('the preflight resolver routes exit 2 to broken_install', () => {
|
||||
const src = fs.readFileSync(path.join(ROOT, 'scripts/resolvers/constants.ts'), 'utf8');
|
||||
expect(src).toContain('broken_install');
|
||||
// The chain must capture the probe's code; `elif ! _gstack_codex_model_probe`
|
||||
// The chain must capture the probe's code; `elif ! gstack-codex-probe probe-model`
|
||||
// collapses 1 and 2 into one branch and loses the distinction.
|
||||
expect(src).toContain('_CODEX_MP=$?');
|
||||
});
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
* B1 static coverage: every rendered fence that runs `codex exec` or
|
||||
* `codex review` as a command routes the result through the shared validator
|
||||
* (lib/outside-review-result.ts) later in the same fence, and takes its
|
||||
* sandbox from `_GSTACK_CODEX_SANDBOX` (read-only unless GSTACK_CODEX_NO_SANDBOX=1);
|
||||
* sandbox from `_CODEX_SANDBOX_MODE` (read-only unless GSTACK_CODEX_NO_SANDBOX=1);
|
||||
* every `codex exec` reads its prompt on stdin (E5).
|
||||
* Prose mentions of the commands are ignored: only command positions count.
|
||||
*/
|
||||
@@ -16,8 +16,8 @@ const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const TMP = fs.mkdtempSync(path.join(os.tmpdir(), 'codex-sites-'));
|
||||
afterAll(() => fs.rmSync(TMP, { recursive: true, force: true }));
|
||||
|
||||
/** `codex exec|review` at the start of a shell command, optionally behind the timeout wrapper. */
|
||||
const COMMAND = /(?:^|[;&|]\s*|\$\(\s*)(?:_gstack_codex_timeout_wrapper\s+\d+\s+)?codex\s+(?:exec|review)\b/;
|
||||
/** `codex exec|review` at the start of a shell command, optionally behind the probe's run-with-timeout. */
|
||||
const COMMAND = /(?:^|[;&|]\s*|\$\(\s*)(?:"\$_CODEX_PROBE"\s+run-with-timeout\s+\d+\s+)?codex\s+(?:exec|review)\b/;
|
||||
const VALIDATOR = /outside-review-result\.ts\b/;
|
||||
|
||||
interface Site { file: string; line: string; fence: string; after: string }
|
||||
@@ -74,8 +74,8 @@ describe('every rendered Codex invocation is classified by the shared validator'
|
||||
expect(argv).toEqual([]);
|
||||
});
|
||||
|
||||
test(`${host}: each codex exec/review command takes its sandbox from _GSTACK_CODEX_SANDBOX`, () => {
|
||||
const literal = rendered[host]!.filter(site => !site.line.includes('${_GSTACK_CODEX_SANDBOX:?}')).map(s => `${s.file}: ${s.line}`);
|
||||
test(`${host}: each codex exec/review command takes its sandbox from _CODEX_SANDBOX_MODE`, () => {
|
||||
const literal = rendered[host]!.filter(site => !site.line.includes('${_CODEX_SANDBOX_MODE:?}')).map(s => `${s.file}: ${s.line}`);
|
||||
expect(literal).toEqual([]);
|
||||
});
|
||||
}
|
||||
@@ -86,13 +86,13 @@ describe('every rendered Codex invocation is classified by the shared validator'
|
||||
'```bash',
|
||||
'# codex exec in a comment',
|
||||
'echo "use codex exec later"',
|
||||
'_gstack_codex_timeout_wrapper 30 codex exec "x" -s read-only',
|
||||
'"$_CODEX_PROBE" run-with-timeout 30 codex exec "x" -s read-only',
|
||||
'```',
|
||||
].join('\n');
|
||||
const sites = codexSites(doc, 'planted.md');
|
||||
expect(sites.map(s => s.line)).toEqual(['_gstack_codex_timeout_wrapper 30 codex exec "x" -s read-only']);
|
||||
expect(sites.map(s => s.line)).toEqual(['"$_CODEX_PROBE" run-with-timeout 30 codex exec "x" -s read-only']);
|
||||
expect(VALIDATOR.test(sites[0]!.after)).toBe(false);
|
||||
expect(sites[0]!.line.includes('${_GSTACK_CODEX_SANDBOX:?}')).toBe(false);
|
||||
expect(sites[0]!.line.includes('${_CODEX_SANDBOX_MODE:?}')).toBe(false);
|
||||
expect(codexSites('```bash\nX=$(codex exec "y")\n```', 'p.md')).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
@@ -34,6 +34,7 @@ import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { codexPreflight } from '../scripts/resolvers/constants';
|
||||
import { withoutDeprecation } from './helpers/codex-probe-sourcing';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const PROBE = path.join(ROOT, 'bin', 'gstack-codex-probe');
|
||||
@@ -129,7 +130,7 @@ function runProbe(f: Fixture, stubMode: string, extraEnv: Record<string, string>
|
||||
timeout: 10000,
|
||||
},
|
||||
);
|
||||
return { stdout: (result.stdout ?? '').toString(), stderr: (result.stderr ?? '').toString(), status: result.status ?? -1 };
|
||||
return { stdout: (result.stdout ?? '').toString(), stderr: withoutDeprecation((result.stderr ?? '').toString()), status: result.status ?? -1 };
|
||||
}
|
||||
|
||||
function lastArgs(f: Fixture): string {
|
||||
|
||||
@@ -0,0 +1,280 @@
|
||||
/**
|
||||
* bin/gstack-codex-probe is an executed command: one subcommand per probe,
|
||||
* KEY: value lines on stdout, human lines where they went before, nothing set
|
||||
* in the caller's shell. Generated skills run it as a command, so the calling
|
||||
* shell (bash, zsh, sh) no longer matters. Sourcing still works during the
|
||||
* compatibility window and says it is deprecated; executing never does.
|
||||
*/
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { PROBE_SOURCING_DEPRECATION } from './helpers/codex-probe-sourcing';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const PROBE = path.join(ROOT, 'bin', 'gstack-codex-probe');
|
||||
const HAS_ZSH = Boolean(Bun.which('zsh'));
|
||||
const ZSH_REQUIRED = Boolean(process.env.CI) && process.platform !== 'win32';
|
||||
const SHELLS = ['bash', 'zsh'] as const;
|
||||
const SUBCOMMANDS = ['select-model', 'check-auth', 'show-sandbox', 'check-sandbox', 'probe-model', 'check-version',
|
||||
'show-first-use-notice', 'run-with-timeout', 'log-event', 'log-hang', 'help'];
|
||||
|
||||
const STUB = `#!/usr/bin/env bash
|
||||
if [ "$1" = sandbox ]; then
|
||||
[ "\${STUB_SANDBOX:-ok}" = ok ] && exit 0
|
||||
echo 'bwrap: No permissions to create new namespace' >&2; exit 1
|
||||
fi
|
||||
case "$1" in --version) echo "codex-cli \${STUB_VERSION:-0.160.1}"; exit 0 ;; esac
|
||||
printf '%s\\n' "$*" >> "$STUB_ARGS_LOG"
|
||||
case "\${STUB_MODE:-ok}" in
|
||||
ok) echo OK; exit 0 ;;
|
||||
model400) echo 'ERROR: {"type":"error","status":400,"error":{"message":"The model is not supported when using Codex with a ChatGPT account."}}' >&2; exit 1 ;;
|
||||
broken) echo 'Error: spawn /usr/lib/codex/vendor/codex ENOENT' >&2; exit 1 ;;
|
||||
quota) echo "ERROR: You've hit your usage limit. Try again at Oct 10th, 2026 2:55 AM." >&2; exit 1 ;;
|
||||
ratelimit) echo 'ERROR: {"type":"error","status":429,"error":{"type":"rate_limit_exceeded"}}' >&2; exit 1 ;;
|
||||
transient) echo 'stream error: network unreachable' >&2; exit 7 ;;
|
||||
esac
|
||||
`;
|
||||
|
||||
interface Fixture { home: string; env: Record<string, string>; argsLog: string; state: string }
|
||||
|
||||
function fixture(config = 'model = "gpt-test-model"\n', auth = true): Fixture {
|
||||
const home = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'codex-probe-sub-')));
|
||||
const stubDir = path.join(home, 'stub-bin');
|
||||
const codexHome = path.join(home, '.codex');
|
||||
const state = path.join(home, '.gstack');
|
||||
fs.mkdirSync(stubDir);
|
||||
fs.mkdirSync(codexHome);
|
||||
fs.mkdirSync(state);
|
||||
fs.writeFileSync(path.join(stubDir, 'codex'), STUB, { mode: 0o755 });
|
||||
fs.writeFileSync(path.join(codexHome, 'config.toml'), config);
|
||||
if (auth) fs.writeFileSync(path.join(codexHome, 'auth.json'), '{}');
|
||||
const argsLog = path.join(home, 'args.log');
|
||||
const env = {
|
||||
PATH: [stubDir, path.dirname(process.execPath), process.env.PATH ?? ''].join(path.delimiter),
|
||||
HOME: home,
|
||||
CODEX_HOME: codexHome,
|
||||
GSTACK_HOME: state,
|
||||
STUB_ARGS_LOG: argsLog,
|
||||
};
|
||||
return { home, env, argsLog, state };
|
||||
}
|
||||
|
||||
function sh(shell: string, script: string, env: Record<string, string>, input?: string) {
|
||||
const args = shell === 'zsh' ? ['-f', '-c', script] : ['--noprofile', '--norc', '-c', script];
|
||||
const r = spawnSync(shell, args, { env, input, encoding: 'utf8', timeout: 20_000 });
|
||||
return { code: r.status, stdout: r.stdout ?? '', stderr: r.stderr ?? '' };
|
||||
}
|
||||
|
||||
function probe(f: Fixture, args: string, extra: Record<string, string> = {}, shell = 'bash', input?: string) {
|
||||
return sh(shell, `'${PROBE}' ${args}`, { ...f.env, ...extra }, input);
|
||||
}
|
||||
|
||||
const calls = (f: Fixture) => (fs.existsSync(f.argsLog) ? fs.readFileSync(f.argsLog, 'utf8').trim().split('\n') : []);
|
||||
const skipZsh = !HAS_ZSH && !ZSH_REQUIRED;
|
||||
|
||||
describe('gstack-codex-probe subcommands', () => {
|
||||
test('help lists every subcommand; unknown or missing subcommands print usage on stderr and exit 64', () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
const help = probe(f, 'help');
|
||||
expect(help.code).toBe(0);
|
||||
for (const name of SUBCOMMANDS) expect(help.stdout).toContain(` ${name}`);
|
||||
for (const bad of ['', 'select_model', 'auth', 'select-model bogus', 'select-model exec --model', 'probe-model exec extra',
|
||||
'run-with-timeout', 'run-with-timeout 5', 'run-with-timeout soon echo hi', 'show-sandbox extra']) {
|
||||
const r = probe(f, bad);
|
||||
expect({ bad, code: r.code, stdout: r.stdout, usage: r.stderr.startsWith('Usage: gstack-codex-probe') })
|
||||
.toEqual({ bad, code: 64, stdout: '', usage: true });
|
||||
}
|
||||
} finally { fs.rmSync(f.home, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
for (const shell of SHELLS) {
|
||||
test.skipIf(shell === 'zsh' && skipZsh)(`each subcommand runs once as a command from ${shell}, with its KEY: value contract and no deprecation line`, () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
const results: Record<string, ReturnType<typeof probe>> = {
|
||||
'select-model': probe(f, 'select-model exec', {}, shell),
|
||||
'check-auth': probe(f, 'check-auth', {}, shell),
|
||||
'show-sandbox': probe(f, 'show-sandbox', {}, shell),
|
||||
'check-sandbox': probe(f, 'check-sandbox', {}, shell),
|
||||
'probe-model': probe(f, 'probe-model exec', {}, shell),
|
||||
'check-version': probe(f, 'check-version', {}, shell),
|
||||
'show-first-use-notice': probe(f, 'show-first-use-notice', {}, shell),
|
||||
'run-with-timeout': probe(f, 'run-with-timeout 5 echo ran', {}, shell),
|
||||
'log-event': probe(f, 'log-event codex_timeout 540', {}, shell),
|
||||
'log-hang': probe(f, 'log-hang review 0', {}, shell),
|
||||
help: probe(f, 'help', {}, shell),
|
||||
};
|
||||
expect(Object.keys(results).sort()).toEqual([...SUBCOMMANDS].sort());
|
||||
for (const [name, r] of Object.entries(results)) {
|
||||
expect({ name, code: r.code, deprecated: r.stderr.includes('deprecated') }).toEqual({ name, code: 0, deprecated: false });
|
||||
}
|
||||
expect(results['select-model']!.stdout).toBe('CODEX_SEL: gpt-test-model\nCODEX_SEL_KIND: exec\nCODEX_SANDBOX: read-only\n');
|
||||
expect(results['select-model']!.stderr).toContain('CODEX_MODEL: gpt-test-model (exec; source: ');
|
||||
expect(results['check-auth']!.stdout).toBe('AUTH_OK\n');
|
||||
expect(results['show-sandbox']!.stdout).toBe('CODEX_SANDBOX: read-only\n');
|
||||
expect(results['check-sandbox']!.stdout).toBe('');
|
||||
expect(results['probe-model']!.stdout).toBe('MODEL_OK\nCODEX_PROBE_STATE: ok\n');
|
||||
expect(results['check-version']!.stdout).toBe('');
|
||||
expect(results['show-first-use-notice']!.stdout).toBe('');
|
||||
expect(results['show-first-use-notice']!.stderr).toContain('NOTICE: gstack outside reviews send the review prompt and code to Codex');
|
||||
expect(results['run-with-timeout']!.stdout).toBe('ran\n');
|
||||
expect(results['log-event']!.stdout + results['log-event']!.stderr).toBe('');
|
||||
expect(results['log-hang']!.stdout + results['log-hang']!.stderr).toBe('');
|
||||
expect(calls(f)).toHaveLength(1);
|
||||
} finally { fs.rmSync(f.home, { recursive: true, force: true }); }
|
||||
});
|
||||
}
|
||||
|
||||
test('nothing crosses into the caller: the selection lives only in the printed lines', () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
const r = sh('bash', `set -u; '${PROBE}' select-model exec >/dev/null 2>&1; echo "sel=\${_GSTACK_CODEX_SEL-unset} sandbox=\${_GSTACK_CODEX_SANDBOX-unset} bin=\${_GSTACK_CODEX_BIN-unset}"`, f.env);
|
||||
expect(r.stdout).toBe('sel=unset sandbox=unset bin=unset\n');
|
||||
} finally { fs.rmSync(f.home, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('select-model keeps the precedence: --model, then GSTACK_CODEX_MODEL, then config.toml (review_model for review)', () => {
|
||||
const f = fixture('model = "gpt-5.6-terra"\nreview_model = "gpt-5.6-luna"\n');
|
||||
try {
|
||||
const sel = (args: string, extra: Record<string, string> = {}) => probe(f, `select-model ${args}`, extra).stdout.split('\n')[0];
|
||||
expect(sel('exec')).toBe('CODEX_SEL: gpt-5.6-terra');
|
||||
expect(sel('review')).toBe('CODEX_SEL: gpt-5.6-luna');
|
||||
expect(sel('review', { GSTACK_CODEX_MODEL: 'gpt-6-sol' })).toBe('CODEX_SEL: gpt-6-sol');
|
||||
const explicit = probe(f, "select-model exec --model 'gpt-6-luna'", { GSTACK_CODEX_MODEL: 'gpt-6-sol' });
|
||||
expect(explicit.stdout.split('\n')[0]).toBe('CODEX_SEL: gpt-6-luna');
|
||||
expect(explicit.stderr).toContain('CODEX_MODEL: gpt-6-luna (exec; source: explicit request)');
|
||||
const invalid = probe(f, "select-model exec --model 'not a model'");
|
||||
expect([invalid.code, invalid.stdout]).toEqual([1, '']);
|
||||
expect(invalid.stderr).toContain('CODEX_MODEL: invalid');
|
||||
expect(calls(f)).toEqual([]);
|
||||
} finally { fs.rmSync(f.home, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('probe-model reports CODEX_PROBE_STATE with the old exit codes, probing exactly the requested model', () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
const cases: Array<[string, number, string]> = [
|
||||
['model400', 1, 'unusable'], ['broken', 2, 'broken_install'], ['quota', 4, 'quota_exhausted'],
|
||||
['ratelimit', 0, 'rate_limited'], ['transient', 0, 'inconclusive'], ['ok', 0, 'ok'],
|
||||
];
|
||||
for (const [mode, code, state] of cases) {
|
||||
fs.rmSync(path.join(f.state, '.codex-model-probe'), { force: true });
|
||||
const r = probe(f, 'probe-model exec', { STUB_MODE: mode });
|
||||
expect({ mode, code: r.code, state: r.stdout.match(/^CODEX_PROBE_STATE: (.*)$/m)?.[1] }).toEqual({ mode, code, state });
|
||||
}
|
||||
fs.rmSync(path.join(f.state, '.codex-model-probe'), { force: true });
|
||||
fs.rmSync(f.argsLog, { force: true });
|
||||
const named = probe(f, "probe-model review --model 'gpt-6-luna'");
|
||||
expect(named.code).toBe(0);
|
||||
expect(calls(f)).toHaveLength(1);
|
||||
expect(calls(f)[0]).toContain('model="gpt-6-luna"');
|
||||
const invalid = probe(f, "probe-model exec --model 'not a model'");
|
||||
expect(invalid.code).toBe(1);
|
||||
expect(invalid.stdout).toContain('MODEL_UNUSABLE\n');
|
||||
expect(invalid.stdout).toContain('CODEX_PROBE_STATE: unusable\n');
|
||||
expect(calls(f)).toHaveLength(1);
|
||||
} finally { fs.rmSync(f.home, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('check-sandbox and check-auth report by exit code, reasons on stderr', () => {
|
||||
const f = fixture('model = "gpt-test-model"\n', false);
|
||||
try {
|
||||
if (process.platform === 'linux') {
|
||||
const sandbox = probe(f, 'check-sandbox', { STUB_SANDBOX: 'userns' });
|
||||
expect([sandbox.code, sandbox.stdout]).toEqual([3, '']);
|
||||
expect(sandbox.stderr).toContain("Codex outside review unavailable: Codex's sandbox could not start here (bwrap: No permissions");
|
||||
}
|
||||
const auth = probe(f, 'check-auth');
|
||||
expect([auth.code, auth.stdout]).toEqual([1, 'AUTH_FAILED\n']);
|
||||
expect(probe(f, 'check-auth', { CODEX_API_KEY: 'present' }).code).toBe(0);
|
||||
const version = probe(f, 'check-version', { STUB_VERSION: '0.120.1' });
|
||||
expect([version.code, version.stdout]).toEqual([0, '']);
|
||||
expect(version.stderr).toContain('WARN: Codex CLI codex-cli 0.120.1 has known stdin deadlock bugs');
|
||||
} finally { fs.rmSync(f.home, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
for (const native of [false, true]) {
|
||||
test(`run-with-timeout supervises with ${native ? 'the bash watchdog' : 'timeout(1)'}: stdin passes through, a deadline exits 124`, () => {
|
||||
const f = fixture();
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codex-probe-sub-path-'));
|
||||
try {
|
||||
for (const tool of native ? ['bash', 'sleep', 'cat'] : []) {
|
||||
const target = spawnSync('bash', ['-c', `command -v ${tool}`], { encoding: 'utf8', timeout: 5000 }).stdout.trim();
|
||||
if (target.startsWith('/')) fs.symlinkSync(target, path.join(dir, tool));
|
||||
}
|
||||
const bash = spawnSync('bash', ['-c', 'command -v bash'], { encoding: 'utf8', timeout: 5000 }).stdout.trim();
|
||||
const env = native ? { ...f.env, PATH: dir } : f.env;
|
||||
const run = (args: string[], input?: string) => {
|
||||
const r = spawnSync(bash, [PROBE, 'run-with-timeout', ...args], { env, input, encoding: 'utf8', timeout: 20_000 });
|
||||
return { code: r.status, stdout: r.stdout ?? '' };
|
||||
};
|
||||
expect(run(['5', 'cat'], 'prompt on stdin')).toEqual({ code: 0, stdout: 'prompt on stdin' });
|
||||
const started = Date.now();
|
||||
expect(run(['1', 'sleep', '30']).code).toBe(124);
|
||||
expect(Date.now() - started).toBeLessThan(15_000);
|
||||
expect(run(['5', 'bash', '-c', 'exit 7']).code).toBe(7);
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
fs.rmSync(f.home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
test('log-event follows telemetry: the exported _TEL first, else gstack-config; log-hang never prints', () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
const usage = path.join(f.state, 'analytics', 'skill-usage.jsonl');
|
||||
const lines = () => (fs.existsSync(usage) ? fs.readFileSync(usage, 'utf8').trim().split('\n') : []);
|
||||
expect(probe(f, 'log-event codex_timeout 540').code).toBe(0);
|
||||
expect(lines()).toEqual([]);
|
||||
fs.writeFileSync(path.join(f.state, 'config.yaml'), 'telemetry: community\n');
|
||||
expect(probe(f, 'log-event codex_timeout 540').code).toBe(0);
|
||||
expect(lines()).toHaveLength(1);
|
||||
expect(JSON.parse(lines()[0]!)).toMatchObject({ skill: 'codex', event: 'codex_timeout', duration_s: '540' });
|
||||
expect(probe(f, 'log-event codex_auth_failed', { _TEL: 'off' }).code).toBe(0);
|
||||
expect(lines()).toHaveLength(1);
|
||||
} finally { fs.rmSync(f.home, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
for (const shell of SHELLS) {
|
||||
test.skipIf(shell === 'zsh' && skipZsh)(`sourcing from ${shell} still defines the functions and prints the deprecation line once`, () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
const r = sh(shell, `. '${PROBE}' && _gstack_codex_select_model exec && echo "sel=$_GSTACK_CODEX_SEL"`, f.env);
|
||||
expect(r.code).toBe(0);
|
||||
expect(r.stdout).toBe('sel=gpt-test-model\n');
|
||||
expect(r.stderr.startsWith(PROBE_SOURCING_DEPRECATION)).toBe(true);
|
||||
expect(r.stderr.split('deprecated').length - 1).toBe(1);
|
||||
expect(PROBE_SOURCING_DEPRECATION).toContain('/gstack-upgrade');
|
||||
} finally { fs.rmSync(f.home, { recursive: true, force: true }); }
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
describe('generated skills run the probe as a command', () => {
|
||||
function walk(dir: string, out: string[] = []): string[] {
|
||||
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||
if (entry.name === 'node_modules' || entry.name.startsWith('.') || entry.name === 'test') continue;
|
||||
const full = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) walk(full, out);
|
||||
else if (/(?:SKILL\.md|\.md\.tmpl|\.md)$/.test(entry.name) || (dir.endsWith(path.join('scripts', 'resolvers')) && entry.name.endsWith('.ts'))) out.push(full);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
test('no template, resolver or generated skill sources the probe or uses its sourced-only names', () => {
|
||||
const offenders: string[] = [];
|
||||
const files = [...walk(path.join(ROOT, 'scripts', 'resolvers')), ...fs.readdirSync(ROOT, { withFileTypes: true })
|
||||
.filter(e => e.isDirectory() && !e.name.startsWith('.') && !['node_modules', 'test', 'docs', 'scripts'].includes(e.name))
|
||||
.flatMap(e => walk(path.join(ROOT, e.name)))];
|
||||
expect(files.length).toBeGreaterThan(100);
|
||||
for (const file of files) {
|
||||
const text = fs.readFileSync(file, 'utf8');
|
||||
if (/(?:^|[;&|({]|then\s)[\t ]*(?:source|\.)\s+"?[^"\s;)]*gstack-codex-probe|_gstack_codex_|_GSTACK_CODEX_/m.test(text)) offenders.push(path.relative(ROOT, file));
|
||||
}
|
||||
expect(offenders).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -43,7 +43,7 @@ describe('Codex skill isolation (#2847)', () => {
|
||||
const calls: string[] = [];
|
||||
for (const file of renderedSkillFiles()) {
|
||||
for (const line of fs.readFileSync(file, 'utf8').split('\n')) {
|
||||
if (/_gstack_codex_timeout_wrapper\s+\d+\s+codex\s+(exec|review)\b/.test(line)) calls.push(`${path.relative(ROOT, file)}: ${line}`);
|
||||
if (/run-with-timeout\s+\d+\s+codex\s+(exec|review)\b/.test(line)) calls.push(`${path.relative(ROOT, file)}: ${line}`);
|
||||
}
|
||||
}
|
||||
expect(calls.length).toBeGreaterThan(10);
|
||||
|
||||
@@ -15,7 +15,7 @@ import { execFileSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'node:os';
|
||||
import { CODEX_MODEL_CONFIG_FLAG, CODEX_REVIEW_MODEL_CONFIG_FLAG, CODEX_WEB_SEARCH_FLAG } from '../scripts/resolvers/constants';
|
||||
import { CODEX_MODEL_CONFIG_FLAG, CODEX_REVIEW_MODEL_CONFIG_FLAG, CODEX_WEB_SEARCH_FLAG, codexSelect } from '../scripts/resolvers/constants';
|
||||
|
||||
const ROOT = path.join(import.meta.dir, '..');
|
||||
const DEPRECATED = '--enable web_search_cached';
|
||||
@@ -92,11 +92,12 @@ describe('deprecated codex web-search flag is gone (#2525)', () => {
|
||||
|
||||
describe('codex frontier model flag is present', () => {
|
||||
// #2914: the flag carries the runtime selection, never a hard-coded default.
|
||||
const flagArgv = (flag: string, select: string, env: Record<string, string>, config?: string) => {
|
||||
// The generated selection block, run exactly as skills run it: the executed probe, no sourcing.
|
||||
const flagArgv = (flag: string, kind: 'exec' | 'review', env: Record<string, string>, config?: string) => {
|
||||
const codexHome = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-flag-model-'));
|
||||
try {
|
||||
if (config !== undefined) fs.writeFileSync(path.join(codexHome, 'config.toml'), config);
|
||||
return execFileSync('bash', ['-c', `source "${path.join(ROOT, 'bin', 'gstack-codex-probe')}" && ${select} 2>/dev/null && printf '%s\\n' ${flag}`], {
|
||||
return execFileSync('bash', ['-c', `exec 2>/dev/null\n${codexSelect(kind, `'${path.join(ROOT, 'bin', 'gstack-codex-probe')}'`)}\nprintf '%s\\n' ${flag}`], {
|
||||
env: { ...process.env, GSTACK_CODEX_MODEL: '', CODEX_HOME: codexHome, ...env }, encoding: 'utf8', timeout: 10000,
|
||||
}).trim().split('\n');
|
||||
} finally {
|
||||
@@ -105,7 +106,7 @@ describe('codex frontier model flag is present', () => {
|
||||
};
|
||||
|
||||
test('the model flag passes the selected model: GSTACK_CODEX_MODEL, then config.toml, then gpt-6-astra', () => {
|
||||
const select = '_gstack_codex_select_model exec';
|
||||
const select = 'exec' as const;
|
||||
const isolation = ['-c', 'skills.include_instructions=false'];
|
||||
expect(flagArgv(CODEX_MODEL_CONFIG_FLAG, select, {})).toEqual(['-c', 'model="gpt-6-astra"', ...isolation]);
|
||||
expect(flagArgv(CODEX_MODEL_CONFIG_FLAG, select, {}, 'model = "gpt-5.6-terra"\n')).toEqual(['-c', 'model="gpt-5.6-terra"', ...isolation]);
|
||||
@@ -120,7 +121,7 @@ describe('codex frontier model flag is present', () => {
|
||||
[{ GSTACK_CODEX_MODEL: 'custom-codex' }, 'review_model = "gpt-5.6-luna"\n', 'custom-codex'],
|
||||
[{}, 'model = "gpt-5.6-terra"\nreview_model = "gpt-5.6-luna"\n', 'gpt-5.6-luna'],
|
||||
] as const) {
|
||||
const argv = flagArgv(CODEX_REVIEW_MODEL_CONFIG_FLAG, '_gstack_codex_select_model review', { ...env }, config);
|
||||
const argv = flagArgv(CODEX_REVIEW_MODEL_CONFIG_FLAG, 'review', { ...env }, config);
|
||||
expect(argv).toEqual(['-c', `review_model="${expected}"`, '-c', `model="${expected}"`, '-c', 'skills.include_instructions=false']);
|
||||
}
|
||||
for (const file of ['codex/sections/review-mode.md', 'review/sections/adversarial.md', 'ship/sections/adversarial.md']) {
|
||||
|
||||
@@ -14,7 +14,9 @@ import * as path from 'node:path';
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const QUERY_SCRIPTS = ['bin/gstack-learnings-search', 'bin/gstack-timeline-read'];
|
||||
const GATE_CALLS = ['outside-review-result.ts', 'gstack-learnings-search', 'gstack-timeline-read',
|
||||
'_gstack_codex_auth_probe', '_gstack_codex_model_probe', '_gstack_codex_sandbox_preflight'];
|
||||
'_gstack_codex_auth_probe', '_gstack_codex_model_probe', '_gstack_codex_sandbox_preflight',
|
||||
'gstack-codex-probe check-auth', 'gstack-codex-probe probe-model', 'gstack-codex-probe check-sandbox',
|
||||
'"$_CODEX_PROBE" check-auth', '"$_CODEX_PROBE" probe-model', '"$_CODEX_PROBE" check-sandbox'];
|
||||
const SILENT_EXIT = /2>\/dev\/null \|\| exit 0\b/;
|
||||
|
||||
function walk(dir: string, keep: (file: string) => boolean, out: string[] = []): string[] {
|
||||
|
||||
@@ -34,7 +34,7 @@ function block(source: string, needle: string): string {
|
||||
}
|
||||
export function loadAutoplanDualCommandContract(root: string) {
|
||||
return { probe: block(readFileSync(join(root, 'autoplan/SKILL.md'), 'utf8'), 'echo "CODEX_MODE: $_CODEX_MODE"'),
|
||||
outside: block(readFileSync(join(root, 'autoplan/sections/ceo-phase.md'), 'utf8'), '_gstack_codex_timeout_wrapper 540 codex exec') };
|
||||
outside: block(readFileSync(join(root, 'autoplan/sections/ceo-phase.md'), 'utf8'), 'run-with-timeout 540 codex exec') };
|
||||
}
|
||||
export interface AutoplanDualEvidenceOptions {
|
||||
ownedRoots: string[];
|
||||
|
||||
@@ -166,7 +166,7 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
|
||||
// wave's headline capability) grows the union to 1.195x. Deliberate:
|
||||
// the section is on-demand (loads only for Apple store targets), so
|
||||
// per-invocation cost for non-iOS ships is one manifest line.
|
||||
maxSizeRatio: 1.4975, // Shared advisory identity/dedup + critical-severity validation: 248,065 union bytes / 187,706 baseline = 1.3216 (2026-09-17). + test value bar in the lazy Step 7 section (value cards, weak paths, gate table, base control, machine checks; ~13.6KB): measured 1.396 (2026-09-29). + W1 guarded state-root resolution (`eval gstack-paths; : "${GSTACK_STATE_ROOT:?…}"`) in the Context Recovery preamble, the eureka log and each state-writing bash block; measured 1.401 (2026-09-30). + the shared QA review step's plan-check timing rule (plan checks and their revalidation run on --timeout-ms after smoke expiry); measured 1.4022 (2026-09-30). + the shared TEST_BOOTSTRAP owned-change undo rules (s03 H1-H3, M1) and the value-bar coverage wording (s03 M2/M3); measured 1.4044 (2026-10-02). + severe fix wave ship-review lane: G1 NO_VERSION dispatch, title and CHANGELOG arms (versionless repos ship without inventing a version), F1 parent-side plan binding before the audit dispatch (binding, docs/designs candidates, exact not-run line) and D3's data-not-instructions line; +2,126 union bytes, measured 1.4135 (2026-10-03). + INV-1 consumer lines (outside review `unverified`/`unavailable` is missing coverage in the readiness note and PR body); measured 1.4148 (2026-10-03). + v1.91.19.0 severe fix wave: verdict-form outside-review callers with sandbox preflight and first-use notice (INV-1/B1/Q2), P0/P1 gate and unverified wording, bound-plan discovery (F1), visible LEARNINGS failures (B5), carved external pointers; measured 267,752 / 187,706 = 1.4264 (2026-10-03). + v1.91.18.0 (#3023) merged into the v1.91.19.0 wave; measured 267,874 / 187,706 = 1.4271 (2026-10-04). + Oct 6 fix wave: quota_exhausted preflight branch (#3051) and quoted-heredoc issue template (#3046); measured 268,199 / 187,706 = 1.4288 (2026-10-06). + Step 10's inline no-PR check before the Greptile dispatch (#3020); measured 268,655 / 187,706 = 1.4313 (2026-10-06). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); measured 275,422 / 187,706 = 1.4673 (2026-10-06). Combined Oct 6 wave head (B free-text files + D measure loop + A rework): measured 280,363 / 187,706 = 1.4936 (2026-10-06). + gate-once push rule (no second paid gate after a CI push); measured 280,573 / 187,706 = 1.4947 (2026-10-06). + the measurement bar's decision step (exit-code branches for MEETS, MEETS-qualified, needs-classify with the qualifying and fix-round classes, EXTEND once; the measure prose was tightened to offset most of it); measured 281,058 / 187,706 = 1.4973 (2026-10-06).
|
||||
maxSizeRatio: 1.4995, // Shared advisory identity/dedup + critical-severity validation: 248,065 union bytes / 187,706 baseline = 1.3216 (2026-09-17). + test value bar in the lazy Step 7 section (value cards, weak paths, gate table, base control, machine checks; ~13.6KB): measured 1.396 (2026-09-29). + W1 guarded state-root resolution (`eval gstack-paths; : "${GSTACK_STATE_ROOT:?…}"`) in the Context Recovery preamble, the eureka log and each state-writing bash block; measured 1.401 (2026-09-30). + the shared QA review step's plan-check timing rule (plan checks and their revalidation run on --timeout-ms after smoke expiry); measured 1.4022 (2026-09-30). + the shared TEST_BOOTSTRAP owned-change undo rules (s03 H1-H3, M1) and the value-bar coverage wording (s03 M2/M3); measured 1.4044 (2026-10-02). + severe fix wave ship-review lane: G1 NO_VERSION dispatch, title and CHANGELOG arms (versionless repos ship without inventing a version), F1 parent-side plan binding before the audit dispatch (binding, docs/designs candidates, exact not-run line) and D3's data-not-instructions line; +2,126 union bytes, measured 1.4135 (2026-10-03). + INV-1 consumer lines (outside review `unverified`/`unavailable` is missing coverage in the readiness note and PR body); measured 1.4148 (2026-10-03). + v1.91.19.0 severe fix wave: verdict-form outside-review callers with sandbox preflight and first-use notice (INV-1/B1/Q2), P0/P1 gate and unverified wording, bound-plan discovery (F1), visible LEARNINGS failures (B5), carved external pointers; measured 267,752 / 187,706 = 1.4264 (2026-10-03). + v1.91.18.0 (#3023) merged into the v1.91.19.0 wave; measured 267,874 / 187,706 = 1.4271 (2026-10-04). + Oct 6 fix wave: quota_exhausted preflight branch (#3051) and quoted-heredoc issue template (#3046); measured 268,199 / 187,706 = 1.4288 (2026-10-06). + Step 10's inline no-PR check before the Greptile dispatch (#3020); measured 268,655 / 187,706 = 1.4313 (2026-10-06). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); measured 275,422 / 187,706 = 1.4673 (2026-10-06). Combined Oct 6 wave head (B free-text files + D measure loop + A rework): measured 280,363 / 187,706 = 1.4936 (2026-10-06). + gate-once push rule (no second paid gate after a CI push); measured 280,573 / 187,706 = 1.4947 (2026-10-06). + the measurement bar's decision step (exit-code branches for MEETS, MEETS-qualified, needs-classify with the qualifying and fix-round classes, EXTEND once; the measure prose was tightened to offset most of it); measured 281,058 / 187,706 = 1.4973 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 281,438 / 187,706 = 1.4994 (2026-10-07).
|
||||
},
|
||||
'plan-ceo-review': {
|
||||
skill: 'plan-ceo-review',
|
||||
@@ -221,7 +221,7 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
|
||||
// 1.08 → 1.10: the scope-gate exceptions block (+ its adversarial-review
|
||||
// hardening: host-anchored mode signal, precedence, passing-mention
|
||||
// guards) and the plan-mode preamble reword land the union at 1.092.
|
||||
maxSizeRatio: 1.185, // + clarity rules for saved decisions/setup gates + the Aside probe's failure reason; measured 1.1504. + test value bar and Tests to Retire in the lazy Test review section (~2.6KB); measured 1.168 + W1 guarded state-root resolution (`eval gstack-paths; : "${GSTACK_STATE_ROOT:?…}"`) in the Context Recovery preamble, the eureka log and each state-writing bash block; measured 1.173 (2026-09-30). + v1.91.12.0 merge of #2999 (headless rule: a disallowed question tool never qualifies) with #3002; measured 1.1741 (2026-10-01). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); measured 146,905 / 124,597 = 1.1790 (2026-10-06). Combined Oct 6 wave head (A and B): measured 147,524 / 124,597 = 1.1840 (2026-10-06).
|
||||
maxSizeRatio: 1.1855, // + clarity rules for saved decisions/setup gates + the Aside probe's failure reason; measured 1.1504. + test value bar and Tests to Retire in the lazy Test review section (~2.6KB); measured 1.168 + W1 guarded state-root resolution (`eval gstack-paths; : "${GSTACK_STATE_ROOT:?…}"`) in the Context Recovery preamble, the eureka log and each state-writing bash block; measured 1.173 (2026-09-30). + v1.91.12.0 merge of #2999 (headless rule: a disallowed question tool never qualifies) with #3002; measured 1.1741 (2026-10-01). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); measured 146,905 / 124,597 = 1.1790 (2026-10-06). Combined Oct 6 wave head (A and B): measured 147,524 / 124,597 = 1.1840 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 147,693 / 124,597 = 1.1854 (2026-10-07).
|
||||
},
|
||||
'plan-design-review': {
|
||||
skill: 'plan-design-review',
|
||||
@@ -243,7 +243,7 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
|
||||
// tier-2+ skeleton (measured 89,184). Main's v1.64.0.0 adds ~340 B more
|
||||
// (telemetry --error-message/--failed-step preamble prose, PR #769).
|
||||
// Budget covers the sum of both waves.
|
||||
maxSkeletonBytes: 84_700, // Harness-aware outside voice: validated dispatch and provenance. + v1.91.19.0 wave: design --version probe (B4) and verdict-form outside voice (INV-1) on top of v1.91.17.0's round accounting (#3022); measured 80,443 (2026-10-03). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); skeleton measured 84,669 (2026-10-06).
|
||||
maxSkeletonBytes: 84_850, // Harness-aware outside voice: validated dispatch and provenance. + v1.91.19.0 wave: design --version probe (B4) and verdict-form outside voice (INV-1) on top of v1.91.17.0's round accounting (#3022); measured 80,443 (2026-10-03). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); skeleton measured 84,669 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; skeleton measured 84,804 (2026-10-07).
|
||||
minUnionBytes: 99_200, // token-reduction Phases 1-2 (v1.69.x branch); measured union 110,293
|
||||
mustContain: ['design', 'visual'],
|
||||
maxSizeRatio: 1.12, // D1 1.104 + main's ~0.008
|
||||
@@ -348,10 +348,10 @@ do not launch the downstream skill or open a browser.`,
|
||||
// the #538 opt-out + D1 evidence directive — ratio 1.104 measured.
|
||||
// #2499 project-scope MCP jq in the brain-sync block grew every tier-2+
|
||||
// skeleton ~1.5KB (entry resolution emitted once per SKILL.md).
|
||||
maxSkeletonBytes: 92_600, // Office-hours + sketch outside voices include host guards and completion checks. + v1.91.19.0 wave: Q<N> open-question prose form (#2729, approved Q3), DESIGN_READY --version probe (B4), Aside ~/.local/bin fallback (E7), verdict-form outside voice (INV-1), plus v1.91.17.0's round accounting (#3022) merged in; measured 89,684 (2026-10-03). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); skeleton measured 92,590 (2026-10-06).
|
||||
maxSkeletonBytes: 92_900, // Office-hours + sketch outside voices include host guards and completion checks. + v1.91.19.0 wave: Q<N> open-question prose form (#2729, approved Q3), DESIGN_READY --version probe (B4), Aside ~/.local/bin fallback (E7), verdict-form outside voice (INV-1), plus v1.91.17.0's round accounting (#3022) merged in; measured 89,684 (2026-10-03). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); skeleton measured 92,590 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; skeleton measured 92,869 (2026-10-07).
|
||||
minUnionBytes: 115_800, // Phase 4 wave 4; measured union 118,175
|
||||
mustContain: ['design doc', 'problem statement'],
|
||||
maxSizeRatio: 1.143, // + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); measured 149,758 / 131,110 = 1.1422 (2026-10-06).
|
||||
maxSizeRatio: 1.1445, // + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); measured 149,758 / 131,110 = 1.1422 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 150,037 / 131,110 = 1.1444 (2026-10-07).
|
||||
},
|
||||
'document-release': {
|
||||
skill: 'document-release',
|
||||
@@ -411,7 +411,7 @@ do not launch the downstream skill or open a browser.`,
|
||||
// eureka log, the office-hours lookup and the taste-profile read; measured
|
||||
// 1.0834 (2026-09-30). + named design-direction defaults to avoid in the native
|
||||
// subagent prompt (s02 H4a); measured 1.0911 (2026-10-02).
|
||||
maxSizeRatio: 1.144, // + v1.91.19.0 wave: design --version probe (B4), taste-profile load (F3), verdict-form outside voice (INV-1), plus v1.91.17.0's round accounting and board/approval blocks (#3022) merged in; measured 100,446 / 90,375 = 1.1114 (2026-10-03). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); measured 103,384 / 90,375 = 1.1439 (2026-10-06).
|
||||
maxSizeRatio: 1.1456, // + v1.91.19.0 wave: design --version probe (B4), taste-profile load (F3), verdict-form outside voice (INV-1), plus v1.91.17.0's round accounting and board/approval blocks (#3022) merged in; measured 100,446 / 90,375 = 1.1114 (2026-10-03). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); measured 103,384 / 90,375 = 1.1439 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 103,522 / 90,375 = 1.1455 (2026-10-07).
|
||||
},
|
||||
cso: {
|
||||
skill: 'cso',
|
||||
@@ -485,7 +485,7 @@ do not launch the downstream skill or open a browser.`,
|
||||
maxSkeletonBytes: 74_881, // Shared-code identity/skip/action rules + critical-severity validation; measured 74,493 (2026-09-17). + v1.91.12.0 merge of #2999 (review clarity repairs: await reads, research alongside dispatch, /review deadline and setup authority, findings sources) with #3002 (guarded state-root lines, plan-check checkpoints); each fit alone; measured 74,881 (2026-10-01).
|
||||
minUnionBytes: 89_000, // Phase 4 wave 1; measured union 93,357
|
||||
mustContain: ['confidence', 'P1', 'P2', 'Review Army', 'adversarial'],
|
||||
maxSizeRatio: 1.217, // Shared-code feature + critical-severity validation: 128,042 union bytes / 108,523 baseline = 1.1799; preserves content floors. + v1.91.12.0 merge of #2999 (above, plus plan-completion fallback intent and specialist checklist-by-path) with #3002; measured 1.1843 (2026-10-01). + v1.91.19.0 wave: verdict-form Codex callers with sandbox preflight (INV-1/B1), P0/P1 structured gate (B1b), bound-plan discovery and data-not-instructions guard (F1/D3), visible LEARNINGS failures (B5); measured 129,941 / 108,523 = 1.1974 (2026-10-03). + v1.91.18.0 (#3023) merged into the v1.91.19.0 wave; measured 130,025 / 108,523 = 1.1981 (2026-10-04). + Oct 6 fix wave: quota_exhausted preflight branch (#3051); measured 130,359 / 108,523 = 1.2012 (2026-10-06). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); measured 131,314 / 108,523 = 1.2100 (2026-10-06). Combined Oct 6 wave head (A and B): measured 131,966 / 108,523 = 1.2160 (2026-10-06).
|
||||
maxSizeRatio: 1.2183, // Shared-code feature + critical-severity validation: 128,042 union bytes / 108,523 baseline = 1.1799; preserves content floors. + v1.91.12.0 merge of #2999 (above, plus plan-completion fallback intent and specialist checklist-by-path) with #3002; measured 1.1843 (2026-10-01). + v1.91.19.0 wave: verdict-form Codex callers with sandbox preflight (INV-1/B1), P0/P1 structured gate (B1b), bound-plan discovery and data-not-instructions guard (F1/D3), visible LEARNINGS failures (B5); measured 129,941 / 108,523 = 1.1974 (2026-10-03). + v1.91.18.0 (#3023) merged into the v1.91.19.0 wave; measured 130,025 / 108,523 = 1.1981 (2026-10-04). + Oct 6 fix wave: quota_exhausted preflight branch (#3051); measured 130,359 / 108,523 = 1.2012 (2026-10-06). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); measured 131,314 / 108,523 = 1.2100 (2026-10-06). Combined Oct 6 wave head (A and B): measured 131,966 / 108,523 = 1.2160 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 132,199 / 108,523 = 1.2182 (2026-10-07).
|
||||
},
|
||||
codex: {
|
||||
skill: 'codex',
|
||||
@@ -575,7 +575,7 @@ do not launch the downstream skill or open a browser.`,
|
||||
maxSkeletonBytes: 70_000, // Phase-specific outside coverage, native fallback, and harness guard.
|
||||
minUnionBytes: 85_000, // measured union 86,926
|
||||
mustContain: ['6 Decision Principles', 'TASTE DECISION', 'USER CHALLENGE', 'consensus', 'Restore Point'],
|
||||
maxSizeRatio: 1.12, // Four validated outside invocations replace raw CLI calls; phases keep independent coverage.
|
||||
maxSizeRatio: 1.121, // Four validated outside invocations replace raw CLI calls; phases keep independent coverage. + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 114,297 / 101,979 = 1.1208 (2026-10-07).
|
||||
},
|
||||
spec: {
|
||||
skill: 'spec',
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
/**
|
||||
* Sourcing bin/gstack-codex-probe is the compatibility form for skills rendered
|
||||
* before the executed subcommands: it still works and prints exactly this line
|
||||
* on stderr, first. Function-level tests that source the probe strip it with
|
||||
* withoutDeprecation(), which fails when the line is missing or moved.
|
||||
*/
|
||||
export const PROBE_SOURCING_DEPRECATION =
|
||||
'gstack: sourcing gstack-codex-probe is deprecated and stops working in a release on or after 2026-10-21. Run /gstack-upgrade so your skills run it as a command (gstack-codex-probe help).\n';
|
||||
|
||||
export function withoutDeprecation(stderr: string): string {
|
||||
if (!stderr.startsWith(PROBE_SOURCING_DEPRECATION)) {
|
||||
throw new Error(`sourcing gstack-codex-probe did not print its deprecation line first; stderr was:\n${stderr}`);
|
||||
}
|
||||
return stderr.slice(PROBE_SOURCING_DEPRECATION.length);
|
||||
}
|
||||
@@ -332,13 +332,15 @@ describe('generated outside-review dispatch', () => {
|
||||
test('autoplan retains its Codex timeout event and hang record', () => {
|
||||
const events = path.join(TMP, 'autoplan-events');
|
||||
const probe = path.join(BIN, 'gstack-codex-probe');
|
||||
fs.writeFileSync(probe, `_gstack_codex_select_model() { _GSTACK_CODEX_SEL=gpt-6-astra; _GSTACK_CODEX_SANDBOX=read-only; }
|
||||
_gstack_codex_sandbox_preflight() { return 0; }
|
||||
_gstack_codex_first_use_notice() { :; }
|
||||
_gstack_codex_timeout_wrapper() { echo 'Partial finding'; return 124; }
|
||||
_gstack_codex_log_event() { printf '%s %s\\n' "$1" "$2" >> "$FAKE_EVENTS"; }
|
||||
_gstack_codex_log_hang() { printf '%s %s\\n' "$1" "$2" >> "$FAKE_EVENTS"; }
|
||||
`);
|
||||
fs.writeFileSync(probe, `#!/usr/bin/env bash
|
||||
case "$1" in
|
||||
select-model) printf 'CODEX_SEL: gpt-6-astra\\nCODEX_SEL_KIND: exec\\nCODEX_SANDBOX: read-only\\n' ;;
|
||||
check-sandbox|show-first-use-notice) ;;
|
||||
run-with-timeout) echo 'Partial finding'; exit 124 ;;
|
||||
log-event|log-hang) printf '%s %s\\n' "$2" "$3" >> "$FAKE_EVENTS" ;;
|
||||
*) exit 64 ;;
|
||||
esac
|
||||
`, { mode: 0o755 });
|
||||
const ctx: TemplateContext = { skillName: 'autoplan', tmplPath: 'autoplan/SKILL.md.tmpl', host: 'claude',
|
||||
paths: { ...HOST_PATHS.claude, binDir: BIN, skillRoot: ROOT } };
|
||||
const command = outsideVoiceCommand(ctx, { promptFile: PROMPT, timeoutMs: 600000 });
|
||||
@@ -416,7 +418,7 @@ echo 'Recommendation: approve because the late answer arrived.'
|
||||
const rendered = outsideVoiceInvocation(ctx, { timeoutMs: requested });
|
||||
const gates = [...rendered.matchAll(/timeout: (\d+)/g)].map(m => Number(m[1]));
|
||||
expect(gates).toEqual([provider + 60000]);
|
||||
expect(rendered).toContain(host === 'claude' ? `_gstack_codex_timeout_wrapper ${provider / 1000} codex` : `--timeout-ms ${provider}`);
|
||||
expect(rendered).toContain(host === 'claude' ? `run-with-timeout ${provider / 1000} codex` : `--timeout-ms ${provider}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
@@ -22,6 +22,7 @@ import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { codexPreflight } from '../scripts/resolvers/constants';
|
||||
import { PROBE_SOURCING_DEPRECATION, withoutDeprecation } from './helpers/codex-probe-sourcing';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const BIN = path.join(ROOT, 'bin');
|
||||
@@ -52,6 +53,9 @@ const code = (text: string) => text.split('\n').filter(line => !/^\s*#/.test(lin
|
||||
* Every bin/ helper a generated skill or resolver loads with `source` or `.`,
|
||||
* plus every bin/ file those helpers source in turn (for example
|
||||
* gstack-state-root.sh, sourced by the codex probe and the egress lib).
|
||||
* Current skills run gstack-codex-probe as a command, but skills rendered
|
||||
* before that change still source it until the compatibility window closes,
|
||||
* so it stays on the list until sourcing is removed.
|
||||
*/
|
||||
function sourcedHelpers(): string[] {
|
||||
const files = [
|
||||
@@ -60,7 +64,7 @@ function sourcedHelpers(): string[] {
|
||||
.filter(e => e.isDirectory() && !e.name.startsWith('.') && e.name !== 'node_modules' && e.name !== 'test')
|
||||
.flatMap(e => walk(path.join(ROOT, e.name), f => /(?:SKILL\.md|\.md\.tmpl)$/.test(f))),
|
||||
];
|
||||
const names = new Set<string>();
|
||||
const names = new Set<string>(['gstack-codex-probe']);
|
||||
const direct = /(?:^|[\s;&|({]|then\s)(?:source|\.)\s+"?[^"\s;)<]*\bbin\/([A-Za-z0-9._-]+)/g;
|
||||
const viaVar = /\/(gstack-[A-Za-z0-9._-]+\.sh)"; \[ -r "\$[A-Z_]+" \] && \. "\$[A-Z_]+"/g;
|
||||
const nested = /(?:^|[\s;&|({]|then\s)(?:source|\.)\s+"?\$\{?[A-Za-z_][A-Za-z0-9_]*\}?\/([A-Za-z0-9._-]+)/g;
|
||||
@@ -167,7 +171,8 @@ describe('#3024: sourced helpers work under bash and zsh', () => {
|
||||
for (const name of helpers) {
|
||||
const file = path.join(BIN, name);
|
||||
const r = run(shell, `${shell} -n '${file}' && . '${file}' && echo SOURCED`);
|
||||
expect({ name, out: r.out.trim().split('\n').pop() }).toEqual({ name, out: 'SOURCED' });
|
||||
const deprecation = name === 'gstack-codex-probe' ? PROBE_SOURCING_DEPRECATION : '';
|
||||
expect({ name, stdout: r.stdout, stderr: r.stderr }).toEqual({ name, stdout: 'SOURCED\n', stderr: deprecation });
|
||||
}
|
||||
});
|
||||
|
||||
@@ -219,7 +224,8 @@ describe('#3024: sourced helpers work under bash and zsh', () => {
|
||||
LOCAL_PROVIDER_KEY: 'present',
|
||||
});
|
||||
fs.rmSync(codexHome, { recursive: true, force: true });
|
||||
expect(r.out.trim()).toBe('AUTH_OK');
|
||||
expect(r.stdout.trim()).toBe('AUTH_OK');
|
||||
expect(withoutDeprecation(r.stderr)).toBe('');
|
||||
expect(r.code).toBe(0);
|
||||
});
|
||||
|
||||
@@ -259,17 +265,20 @@ describe('#3024: sourced helpers work under bash and zsh', () => {
|
||||
expect(r.out).toContain('rc=1 bin=unset');
|
||||
});
|
||||
|
||||
for (const [label, shell, prefix, expected] of [
|
||||
['dash', 'dash', '', `gstack: cannot locate gstack-codex-probe (shell: dash). Source it from bash or zsh, or export GSTACK_ROOT=<install dir>. ${ANCHOR}`],
|
||||
['an unknown shell', 'bash', 'unset BASH_VERSION\n', `gstack: cannot locate gstack-codex-probe (shell: bash). Source it from bash or zsh, or export GSTACK_ROOT=<install dir>. ${ANCHOR}`],
|
||||
// The preflight runs the probe as a command, so the calling shell no longer
|
||||
// matters: dash and a shell without BASH_VERSION reach the same verdict.
|
||||
for (const [label, shell, prefix] of [
|
||||
['dash', 'dash', ''],
|
||||
['an unknown shell', 'bash', 'unset BASH_VERSION\n'],
|
||||
] as const) {
|
||||
test.skipIf(shell === 'dash' && skipDash)(`the full generated preflight under ${label} reports helper_unavailable with the helper's line`, () => {
|
||||
test.skipIf(shell === 'dash' && skipDash)(`the full generated preflight under ${label} runs the probe as a command and reaches CODEX_MODE: ready`, () => {
|
||||
const { home, env } = preflightHome();
|
||||
try {
|
||||
const r = run(shell, `${prefix}${PREFLIGHT}`, env, home);
|
||||
expect(r.out).toContain(`${expected}\nCODEX_MODE: helper_unavailable\n`);
|
||||
expect(r.out).not.toContain('not_authed');
|
||||
expect(r.out).not.toMatch(/Syntax error|not found/);
|
||||
expect(r.out).toContain('CODEX_MODEL: gpt-test-model (exec;');
|
||||
expect(r.out).toContain('CODEX_MODE: ready\n');
|
||||
expect(r.out).not.toContain('cannot locate');
|
||||
expect(r.out).not.toMatch(/Syntax error|not found|deprecated/);
|
||||
} finally { fs.rmSync(home, { recursive: true, force: true }); }
|
||||
});
|
||||
}
|
||||
|
||||
@@ -258,8 +258,8 @@ describe('outside-voice dispatch contract', () => {
|
||||
const apply = rendered.indexOf('**4. Apply the answered row.**');
|
||||
expect(answer).toBeGreaterThan(0);
|
||||
expect(apply).toBeGreaterThan(answer);
|
||||
// B1: the sandbox comes from _gstack_codex_select_model (read-only unless GSTACK_CODEX_NO_SANDBOX=1).
|
||||
expect(rendered).toContain(`-s "\${_GSTACK_CODEX_SANDBOX:?}" ${CODEX_MODEL_CONFIG_FLAG} -c 'model_reasoning_effort="high"'`);
|
||||
// B1: the sandbox comes from the probe's select-model (read-only unless GSTACK_CODEX_NO_SANDBOX=1).
|
||||
expect(rendered).toContain(`-s "\${_CODEX_SANDBOX_MODE:?}" ${CODEX_MODEL_CONFIG_FLAG} -c 'model_reasoning_effort="high"'`);
|
||||
});
|
||||
|
||||
test('the generated-carrier exception rejects missing wait, cancellation or result guards', () => {
|
||||
|
||||
@@ -1517,8 +1517,8 @@ describe('Codex skill', () => {
|
||||
const resumeCommand = match![0];
|
||||
expect(resumeCommand).not.toContain(' -C ');
|
||||
expect(resumeCommand).not.toMatch(/ -s /);
|
||||
// resume takes the sandbox only as config; _gstack_codex_select_model sets it (read-only by default).
|
||||
expect(resumeCommand).toContain('-c "sandbox_mode=\\"${_GSTACK_CODEX_SANDBOX:?}\\""');
|
||||
// resume takes the sandbox only as config; the probe's select-model reports it (read-only by default).
|
||||
expect(resumeCommand).toContain('-c "sandbox_mode=\\"${_CODEX_SANDBOX_MODE:?}\\""');
|
||||
});
|
||||
|
||||
test('codex union contains cost tracking', () => {
|
||||
|
||||
Reference in new issue
Block a user