mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-22 08:01:32 +01:00
A failed second-model review names the cause that stopped it, and never deletes its own prompt (GST-3188) (#4519)
When a second-model review fails, gstack now reports the failure that actually ended it: Codex's final error decides, a stray "402" in a request id no longer reads as a used-up budget, and a Claude Code 402 or usage limit reads as a budget stop with no Codex advice. Why: these were the four follow-ups ruled below the blocking bar on #4515; each one could send someone to fix the wrong thing. Nothing to do. The prose rule that reads "auth", "login" or "API key" in stderr as a login failure is not changed here (see below). Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
e95cea8404
commit
f82b6f76f3
10 files changed
+170
-17
No files matched your search
@@ -19,7 +19,7 @@ import * as path from 'node:path';
|
||||
import { EXIT, exitTableHelp } from '../lib/headless-artifacts';
|
||||
import { RESULT_FORMAT_HEADER } from '../lib/autoplan-reconcile';
|
||||
import { findingsLabel, type OutsideGate } from '../lib/outside-review-result';
|
||||
import { RUNNER_IDS, createRunner, familyConflict, modelFamily, outsideStatusLine, runnerTable, selectRunner, type RunnerId } from '../lib/outside-voice-runner';
|
||||
import { RUNNER_IDS, createRunner, familyConflict, modelFamily, outsideStatusLine, promptOutputClash, runnerTable, selectRunner, type RunnerId } from '../lib/outside-voice-runner';
|
||||
import { outsideConsent } from '../lib/outside-consent';
|
||||
import { apiTimeoutMs } from '../lib/outside-voice-api';
|
||||
import { resultLine } from '../lib/result-codes';
|
||||
@@ -81,6 +81,8 @@ async function main(argv: string[]): Promise<number> {
|
||||
if (!isRunner(requested)) return usage(`--runner must be one of ${RUNNER_IDS.join(', ')}`);
|
||||
if (!promptFile || !outFile) return usage('run needs --prompt <file> --out <file>');
|
||||
if (!fs.existsSync(promptFile)) return usage(`prompt file not found: ${promptFile}`);
|
||||
const clash = promptOutputClash(promptFile, outFile);
|
||||
if (clash) return usage(clash);
|
||||
const gate = (flag(rest, '--gate') ?? (fs.readFileSync(promptFile, 'utf8').includes(RESULT_FORMAT_HEADER) ? 'autoplan' : 'review')) as OutsideGate;
|
||||
if (!['review', 'structured', 'spec', 'proposal', 'autoplan'].includes(gate)) return usage('--gate must be review, structured, spec, proposal or autoplan');
|
||||
const maxTokensText = flag(rest, '--max-tokens');
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
- **A failed second-model review reports the cause that actually stopped it, and the runner never deletes its own prompt.** When Codex logged a 402 on an early attempt and then failed for good with a 401, gstack called it a used-up budget; the final failure now decides. A request id or a duration that happens to contain "402" no longer reads as a budget stop. Claude Code exiting 0 with `subtype: error_during_execution` and a 402 is now a budget stop instead of a generic provider error, and a Claude Code usage limit is reported as `budget_exhausted` instead of with Codex's usage-limit advice. `gstack-outside-voice run` refuses a `--prompt` that is one of the files the run writes for `--out` (such as `<out>.last-message`) instead of deleting it. When the reviewer's own result file has no final verdict, or its first line names another plan snapshot, Codex's complete closing message is used instead.
|
||||
@@ -546,11 +546,11 @@ for the account or API key on the provider's dashboard.
|
||||
**Expected result.** `CODEX_MODE: ready`, and the review completes.
|
||||
|
||||
<a id="outside-review-budget-exhausted"></a>
|
||||
### `unavailable: the provider refused the review with HTTP 402 (payment required)`
|
||||
### `unavailable: the provider refused the review with HTTP 402 (payment required) or a Claude Code usage limit`
|
||||
|
||||
**Meaning.** The second-model review (Codex, or Claude Code from a Codex
|
||||
session) reached its provider, and the provider refused the call with HTTP 402
|
||||
Payment Required. The money behind the key is used up: a metering proxy or
|
||||
Payment Required, or Claude Code stopped on its usage limit. The money behind the key is used up: a metering proxy or
|
||||
gateway in front of the provider hit its allowance, or the account ran out of
|
||||
credits or quota. The login worked, so this is not an authentication failure,
|
||||
even when the provider's message mentions the API key. gstack reports outside
|
||||
|
||||
+5
-4
@@ -4,7 +4,7 @@ import { resolveClaudeCommand, type ClaudeCommand } from './claude-bin';
|
||||
import { initializeWindowsReviewJob, WindowsReviewSupervisionError } from './claude-code-windows-job';
|
||||
import { ModelPolicyError, describeSelection, resolvePlanReviewModel, selectionRepair, type ModelSelection } from './model-policy';
|
||||
import { emitModelPolicyNotice } from './model-policy-notice';
|
||||
import { BUDGET_FAILURE, QUOTA_FAILURE } from './outside-review-result';
|
||||
import { BUDGET_FAILURE, CLAUDE_CODE_BUDGET_STOP, QUOTA_FAILURE } from './outside-review-result';
|
||||
|
||||
export const CLAUDE_CODE_OUTPUT_LIMIT = 32 * 1024 * 1024;
|
||||
const DRAIN_TIMEOUT_MS = 500;
|
||||
@@ -113,13 +113,14 @@ export function parseClaudeCodeResult(stdout: string, stderr: string, exitCode:
|
||||
try { raw = JSON.parse(stdout); } catch { /* Classified below, after CLI failure. */ }
|
||||
const obj = isObject(raw) ? raw : undefined;
|
||||
const response = typeof obj?.result === 'string' ? obj.result : typeof obj?.response === 'string' ? obj.response : '';
|
||||
const actualFailure = exitCode !== 0 || !obj || Boolean(obj.is_error) || !response.trim();
|
||||
const errorSubtype = typeof obj?.subtype === 'string' && obj.subtype.startsWith('error');
|
||||
const actualFailure = exitCode !== 0 || !obj || Boolean(obj.is_error) || errorSubtype || !response.trim();
|
||||
const failureText = `${diagnostic}\n${response}\n${stdout.slice(0, 4096)}`;
|
||||
const budget = actualFailure ? `${diagnostic}\n${obj ? response : stdout.slice(0, 4096)}`.split(/\r?\n/).map(line => line.trim()).find(line => BUDGET_FAILURE.test(line) || QUOTA_FAILURE.test(line)) : undefined;
|
||||
let result: ClaudeCodeResult;
|
||||
if (budget) {
|
||||
const cause = BUDGET_FAILURE.test(budget) ? 'its budget is used up (HTTP 402, payment required)' : 'its usage limit or quota is used up';
|
||||
result = failure('budget', `Claude Code's provider refused the call because ${cause}: ${budget.slice(0, 240)}. Check the allowance on the proxy or gateway in front of the provider, or the account's quota and credits; logging in again will not fix it.`, true);
|
||||
result = failure('budget', `${CLAUDE_CODE_BUDGET_STOP} ${cause}: ${budget.slice(0, 240)}. Check the allowance on the proxy or gateway in front of the provider, or the account's quota and credits; logging in again will not fix it.`, true);
|
||||
} else if (actualFailure && /\b(?:authentication(?:[_ ](?:failed|error))?|unauthorized|not authenticated|invalid (?:x-)?api[- ]key|login required|please (?:run .*login|log in)|not logged in)\b/i.test(failureText)) {
|
||||
result = failure('authentication', 'Claude Code authentication failed. Run claude interactively in this execution context to authenticate.', true);
|
||||
} else if (exitCode !== 0) {
|
||||
@@ -128,7 +129,7 @@ export function parseClaudeCodeResult(stdout: string, stderr: string, exitCode:
|
||||
result = failure('invalid-json', 'Claude Code returned invalid JSON. Check the CLI installation and diagnostic output.');
|
||||
} else if (!obj) {
|
||||
result = failure('invalid-response', 'Claude Code returned JSON that is not an object.');
|
||||
} else if (obj.is_error || (typeof obj.subtype === 'string' && obj.subtype.startsWith('error'))) {
|
||||
} else if (obj.is_error || errorSubtype) {
|
||||
result = failure('provider-error', `Claude Code reported an error.${response ? ` ${response.slice(0, 4096)}` : ''}`);
|
||||
} else if (!response.trim()) {
|
||||
result = failure('empty-response', 'Claude Code returned no response text.');
|
||||
|
||||
@@ -95,7 +95,7 @@ export const GATE_OUTCOMES = {
|
||||
},
|
||||
budget_exhausted: {
|
||||
state: 'unavailable', anchor: 'outside-review-budget-exhausted',
|
||||
summary: 'the provider refused the review with HTTP 402 (payment required): the budget, allowance or quota behind this key is used up',
|
||||
summary: 'the provider refused the review with HTTP 402 (payment required) or a Claude Code usage limit: the budget, allowance or quota behind this key is used up',
|
||||
fix: "check the allowance on the proxy or gateway in front of the provider, or the account's quota and credits, then re-run the review; signing in again will not help (https://github.com/garrytan/gstack/blob/main/docs/troubleshooting.md#outside-review-budget-exhausted)",
|
||||
},
|
||||
rate_limited: {
|
||||
|
||||
@@ -66,8 +66,13 @@ function codexErrorLines(stderr: string): string[] {
|
||||
}
|
||||
/** The account's usage limit: Codex's line carries the reset time, so it is relayed verbatim. Same signature as the probe. */
|
||||
export const QUOTA_FAILURE = /usage limit|insufficient_quota|exceeded your current quota|quota exceeded/i;
|
||||
/** HTTP 402 Payment Required: a metering proxy's allowance or the account's budget, never an auth failure. */
|
||||
export const BUDGET_FAILURE = /(?:^|[^0-9])402(?:[^0-9]|$)|payment.?required/i;
|
||||
/**
|
||||
* HTTP 402 Payment Required: a metering proxy's allowance or the account's budget, never an auth failure. The 402 counts only
|
||||
* as a status (`status 402`, `HTTP 402`, `"code":402`, `Error: 402`, `402 Payment Required`), never inside a request id or a duration.
|
||||
*/
|
||||
export const BUDGET_FAILURE = /(?:http(?:\/\d(?:\.\d)?)?|status|code|error)\b[\s:="']{0,4}402\b|\b402\s+payment|payment[\s_-]?required/i;
|
||||
/** How lib/claude-code.ts opens its budget stop (a 402 or a usage limit): read as budget_exhausted, never as Codex's quota stop. */
|
||||
export const CLAUDE_CODE_BUDGET_STOP = "Claude Code's provider refused the call because";
|
||||
/** OpenAI's cyber-security refusal (R95): Capy relays it as `OpenAI key declined the request: cyber_policy: ...`; read from error lines only. */
|
||||
export const CYBER_POLICY_REFUSAL = /\bcyber_policy\b/;
|
||||
/** A plain HTTP 429 or rate limit: transient, so a separate reason from the quota. */
|
||||
@@ -121,17 +126,17 @@ function execution(input: OutsideReviewInput): OutsideReviewClassification['exec
|
||||
if (exit !== 0) {
|
||||
const detail = sandbox(stderr);
|
||||
if (detail) return { state: 'unavailable', reason: 'sandbox_unavailable', detail };
|
||||
const failures = exit !== 124 ? failureEventMessages(input.events ?? '') : [];
|
||||
const errors = exit !== 124 ? [...codexErrorLines(stderr), ...failures] : [];
|
||||
const final = exit !== 124 ? failureEventMessages(input.events ?? '').at(-1) : undefined;
|
||||
const errors = exit === 124 ? [] : final !== undefined ? [final] : codexErrorLines(stderr);
|
||||
const refusal = errors.find(line => CYBER_POLICY_REFUSAL.test(line));
|
||||
if (refusal) return { state: 'unavailable', reason: 'cyber_policy_refused', detail: refusal.slice(0, 240) };
|
||||
const budget = errors.find(line => BUDGET_FAILURE.test(line));
|
||||
const budget = errors.find(line => BUDGET_FAILURE.test(line) || line.includes(CLAUDE_CODE_BUDGET_STOP));
|
||||
if (budget) return { state: 'unavailable', reason: 'budget_exhausted', detail: budget.slice(0, 240) };
|
||||
const quota = errors.find(line => QUOTA_FAILURE.test(line));
|
||||
if (quota) return { state: 'unavailable', reason: 'quota_exhausted', detail: quota.slice(0, 240) };
|
||||
const rateLimit = errors.find(line => RATE_LIMIT_FAILURE.test(line));
|
||||
if (rateLimit) return { state: 'unavailable', reason: 'rate_limited', detail: rateLimit.slice(0, 240) };
|
||||
const head = failures[failures.length - 1]?.slice(0, 240) ?? stderrHead(stderr);
|
||||
const head = final?.slice(0, 240) ?? stderrHead(stderr);
|
||||
return { state: 'unavailable', reason: exit === 124 ? 'timeout' : 'execution_failed', detail: head ? `exit ${exit}: ${head}` : `exit ${exit}` };
|
||||
}
|
||||
const commands = commandEvidence(input.events ?? '');
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { parseReceipt, RESULT_FORMAT_HEADER } from './autoplan-reconcile';
|
||||
import { classifyOutsideReview, usageFromEvents, type OutsideGate, type OutsideReviewClassification, type TokenUsage } from './outside-review-result';
|
||||
|
||||
export { usageFromEvents };
|
||||
@@ -100,6 +101,22 @@ function finish(base: Omit<RunnerResult, 'ended_at' | 'wall_s' | 'output_sha256'
|
||||
return { schema_version: 1, ...base, output: outputExists ? outFile : null, output_sha256: outputExists ? sha256(fs.readFileSync(outFile)) : null, ended_at: new Date(ended).toISOString(), wall_s: Math.round((ended - startedMs) / 1000) };
|
||||
}
|
||||
|
||||
/**
|
||||
* Refusal text when --prompt is one of the files a run writes or deletes for --out (the result, Codex's events, stderr and closing
|
||||
* message, the meta record), compared after symlinks resolve: the run would destroy its own input.
|
||||
*/
|
||||
export function promptOutputClash(promptFile: string, outFile: string): string | undefined {
|
||||
const real = (file: string) => {
|
||||
const abs = path.resolve(file);
|
||||
try { return fs.realpathSync(abs); } catch { /* not written yet */ }
|
||||
try { return path.join(fs.realpathSync(path.dirname(abs)), path.basename(abs)); } catch { return abs; }
|
||||
};
|
||||
const prompt = real(promptFile);
|
||||
const out = path.resolve(outFile);
|
||||
const clash = [out, `${out}.events.jsonl`, `${out}.stderr`, `${out}.last-message`, `${out}.meta.json`].find(file => real(file) === prompt);
|
||||
return clash && `--prompt ${promptFile} is ${clash === out ? 'the --out file' : `the ${clash.slice(out.length)} file this run writes beside --out ${outFile}`}, so the run would delete or overwrite its own prompt; write the prompt to another path`;
|
||||
}
|
||||
|
||||
function classify(text: string, gate: OutsideGate, extra: { stderr?: string; exit?: number; events?: string }): { status: RunnerStatus; verdict: OutsideReviewClassification } {
|
||||
const verdict = classifyOutsideReview({ text, gate, ...extra });
|
||||
const status: RunnerStatus = verdict.verdict === 'unavailable' ? 'unavailable' : verdict.verdict === 'unverified' ? 'unverified' : 'completed';
|
||||
@@ -130,6 +147,8 @@ class CodexCliRunner implements OutsideRunner {
|
||||
const base = { runner: this.id, capabilities: this.capabilities, input_sha256: sha256(prompt), started_at: new Date(started).toISOString(), usage: { usd: 'unknown' } as RunnerUsage, model: req.model ?? null, family: modelFamily(req.model), output: null } as const;
|
||||
const avail = this.available(req);
|
||||
if (!avail.ok) return finish({ ...base, status: 'unavailable', exit: 1, reason: 'cli_missing', detail: avail.reason }, started);
|
||||
const clash = promptOutputClash(req.promptFile, req.outFile);
|
||||
if (clash) return finish({ ...base, status: 'failed', exit: 2, reason: 'prompt_is_output', detail: clash }, started);
|
||||
const probe = path.join(req.binDir ?? DEFAULT_BIN, 'gstack-codex-probe');
|
||||
if (req.model) env.GSTACK_CODEX_MODEL = req.model;
|
||||
const role = req.gate === 'autoplan' || req.gate === 'spec' ? ['--role', 'plan-review'] : [];
|
||||
@@ -155,8 +174,16 @@ class CodexCliRunner implements OutsideRunner {
|
||||
const timedOut = Date.now() - started >= req.timeoutMs && exit !== 0;
|
||||
const own = fs.existsSync(req.outFile) ? fs.readFileSync(req.outFile, 'utf8') : '';
|
||||
const last = fs.existsSync(lastMessageFile) ? fs.readFileSync(lastMessageFile, 'utf8') : '';
|
||||
const marked = (candidate: string) => candidate.trim() !== '' && !['missing_markers', 'empty_response'].includes(classifyOutsideReview({ text: candidate, gate: req.gate }).reason ?? '');
|
||||
const text = marked(own) ? own : marked(last) || !own.trim() ? last : own;
|
||||
const receipt = req.gate === 'autoplan' ? /INPUT: (\S+) ([0-9a-f]{64})/.exec(prompt.toString('utf8').split(RESULT_FORMAT_HEADER).at(-1) ?? '') : null;
|
||||
const forThisInput = (candidate: string) => candidate.trim() !== '' && (!receipt || parseReceipt(candidate, { phase: receipt[1]!, sha256: receipt[2]! }).ok);
|
||||
const ownRead = classifyOutsideReview({ text: own, gate: req.gate });
|
||||
const lastRead = classifyOutsideReview({ text: last, gate: req.gate });
|
||||
const ownMarked = forThisInput(own) && !['missing_markers', 'empty_response'].includes(ownRead.reason ?? '');
|
||||
const ownUndecided = !ownMarked || ownRead.reason === 'missing_verdict';
|
||||
const lastComplete = forThisInput(last) && ['clean', 'findings'].includes(lastRead.verdict);
|
||||
const ownBlocks = ownRead.findings.highest === 'P0' || ownRead.findings.highest === 'P1';
|
||||
const lastMarked = forThisInput(last) && !['missing_markers', 'empty_response'].includes(lastRead.reason ?? '');
|
||||
const text = ownUndecided && lastComplete && (lastRead.verdict === 'findings' || !ownBlocks) ? last : !ownMarked && (lastMarked || !forThisInput(own)) ? last : own;
|
||||
if (text !== own) fs.writeFileSync(req.outFile, text);
|
||||
const events = fs.existsSync(eventsFile) ? fs.readFileSync(eventsFile, 'utf8') : '';
|
||||
const stderr = fs.existsSync(stderrFile) ? fs.readFileSync(stderrFile, 'utf8') : '';
|
||||
|
||||
@@ -3,7 +3,9 @@ import { existsSync, lstatSync, mkdtempSync, readFileSync, realpathSync, rmSync,
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { runClaudeCode } from '../lib/claude-code';
|
||||
import { parseClaudeCodeResult, runClaudeCode } from '../lib/claude-code';
|
||||
import { classifyOutsideReview } from '../lib/outside-review-result';
|
||||
import { gateOutcomeLine } from '../lib/gate-outcomes';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const DIR = mkdtempSync(path.join(tmpdir(), 'claude-code-runner-'));
|
||||
@@ -416,3 +418,19 @@ describe('Claude Code restricted execution', () => {
|
||||
expect(parsed.result.length).toBe(2 * 1024 * 1024);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Claude Code budget stops (GST-3188)', () => {
|
||||
test('exit 0 with subtype error_during_execution and a 402 in result is a budget stop, not a generic provider error', () => {
|
||||
const stdout = JSON.stringify({ type: 'result', subtype: 'error_during_execution', result: 'API Error: 402 Payment Required {"error":{"message":"API key allowance exhausted"}}', session_id: 's-1' });
|
||||
const result = parseClaudeCodeResult(stdout, '', 0);
|
||||
expect([result.status, result.error?.code]).toEqual(['unavailable', 'budget']);
|
||||
expect(result.stderr).toContain('402 Payment Required');
|
||||
});
|
||||
|
||||
test('a Claude Code usage limit is reported as budget_exhausted, never with Codex\'s usage-limit message', () => {
|
||||
const failed = parseClaudeCodeResult(JSON.stringify({ type: 'result', is_error: true, result: 'Claude AI usage limit reached|1763600000' }), '', 1);
|
||||
const verdict = classifyOutsideReview({ text: '', gate: 'review', exit: 1, stderr: failed.stderr });
|
||||
expect([verdict.verdict, verdict.reason]).toEqual(['unavailable', 'budget_exhausted']);
|
||||
expect(gateOutcomeLine('Claude Code outside review', verdict.reason!, verdict.detail)).not.toMatch(/Codex|GSTACK_CODEX/);
|
||||
});
|
||||
});
|
||||
@@ -677,3 +677,29 @@ describe('the execution_failed detail names the terminal Codex failure', () => {
|
||||
expect(result.detail).toBe(`exit 1: ${fatal}`);
|
||||
});
|
||||
});
|
||||
|
||||
describe('the final Codex failure decides the cause, and only a 402 status is a budget stop (GST-3188)', () => {
|
||||
test('an earlier 402 error event followed by a final 401 turn.failed reads as the 401, not budget_exhausted', () => {
|
||||
const final = 'unexpected status 401 Unauthorized: Incorrect API key provided, url: http://127.0.0.1:45349/v1/responses';
|
||||
const events = [
|
||||
JSON.stringify({ type: 'error', message: 'unexpected status 402 Payment Required: allowance used up, url: http://127.0.0.1:45349/v1/responses' }),
|
||||
JSON.stringify({ type: 'turn.failed', error: { message: final } }),
|
||||
].join('\n');
|
||||
const result = classifyOutsideReview({ text: '', gate: 'review', exit: 1, stderr: 'Reading additional input from stdin...', events });
|
||||
expect([result.verdict, result.reason, result.detail]).toEqual(['unavailable', 'execution_failed', `exit 1: ${final}`]);
|
||||
});
|
||||
|
||||
test('a 402 inside a request id or a duration is not a budget stop', () => {
|
||||
for (const line of ['ERROR: unexpected status 500 Internal Server Error (request id req_9c3f402ab)', 'ERROR: unexpected status 401 Unauthorized after 402 ms', 'ERROR: unexpected status 500 Internal Server Error (request id: req-402-payment-abc)']) {
|
||||
const result = classifyOutsideReview({ text: '', stderr: `${line}\n`, exit: 1, gate: 'review' });
|
||||
expect({ line, reason: result.reason }).toEqual({ line, reason: 'execution_failed' });
|
||||
}
|
||||
});
|
||||
|
||||
test('a 402 written as an HTTP status, a status or code field, or an API error still is', () => {
|
||||
for (const line of ['ERROR: HTTP 402', 'ERROR: HTTP/1.1 402', 'ERROR: {"status_code": 402}', 'ERROR: statusCode: 402', 'ERROR: http_status=402', 'ERROR: {"status":402,"title":"E_ALLOWANCE"}', 'ERROR: API Error: 402 {"type":"error"}']) {
|
||||
const result = classifyOutsideReview({ text: '', stderr: `${line}\n`, exit: 1, gate: 'review' });
|
||||
expect({ line, reason: result.reason }).toEqual({ line, reason: 'budget_exhausted' });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -195,6 +195,79 @@ echo '{"type":"turn.completed","usage":{"input_tokens":1200,"output_tokens":340}
|
||||
expect(result.status).toBe('completed');
|
||||
});
|
||||
|
||||
test('a reviewer file with no final verdict loses to a closing message that has one (GST-3188)', async () => {
|
||||
const draft = path.join(TMP, 'own-draft.md');
|
||||
fs.writeFileSync(draft, 'No issues found.\nRecommendation: ship because the change is small.\n');
|
||||
const out = path.join(TMP, 'own-draft', 'review.md');
|
||||
const prompt = path.join(TMP, 'own-draft-prompt.md');
|
||||
fs.writeFileSync(prompt, `You are the outside reviewer.\n3. Write the whole result to this file: ${out}\n`);
|
||||
const result = await createRunner('codex-cli').run({ promptFile: prompt, outFile: out, timeoutMs: 30_000, cwd: ROOT, gate: 'review', env: { ...cleanEnv(fakeBin), FAKE_CODEX_REVIEW: draft, FAKE_CODEX_OWN: '1', FAKE_CODEX_LAST: REVIEW.trimEnd() }, binDir: gstackBin });
|
||||
expect(fs.readFileSync(out, 'utf8')).toBe(REVIEW);
|
||||
expect(result.status).toBe('completed');
|
||||
});
|
||||
|
||||
test('a reviewer file whose receipt names another snapshot loses to a closing message that names this one (GST-3188)', async () => {
|
||||
const stale = path.join(TMP, 'own-stale.md');
|
||||
fs.writeFileSync(stale, AUTOPLAN_REVIEW.replace(SHA, 'b'.repeat(64)));
|
||||
const out = path.join(TMP, 'own-stale', 'ceo-outside.md');
|
||||
const prompt = path.join(TMP, 'own-stale-prompt.md');
|
||||
fs.writeFileSync(prompt, `You are the outside CEO reviewer.\n${RESULT_FORMAT('ceo', SHA, out)}`);
|
||||
const result = await createRunner('codex-cli').run({ promptFile: prompt, outFile: out, timeoutMs: 30_000, cwd: ROOT, gate: 'autoplan', env: { ...cleanEnv(fakeBin), FAKE_CODEX_REVIEW: stale, FAKE_CODEX_OWN: '1', FAKE_CODEX_LAST: AUTOPLAN_REVIEW.trimEnd() }, binDir: gstackBin });
|
||||
expect(fs.readFileSync(out, 'utf8')).toBe(AUTOPLAN_REVIEW);
|
||||
expect(result.status).toBe('completed');
|
||||
});
|
||||
|
||||
test('a reviewer file whose own verdict says it is not ready is kept over a clean closing message (GST-3188 review)', async () => {
|
||||
const own = path.join(TMP, 'own-unready.md');
|
||||
const ownText = 'Verdict: ready, but I could not run the test suite.\nRecommendation: run the suite because nothing was executed.\n';
|
||||
fs.writeFileSync(own, ownText);
|
||||
const out = path.join(TMP, 'own-unready', 'review.md');
|
||||
const prompt = path.join(TMP, 'own-unready-prompt.md');
|
||||
fs.writeFileSync(prompt, `You are the outside reviewer.\n3. Write the whole result to this file: ${out}\n`);
|
||||
const clean = 'No issues found.\nRecommendation: ship because the change is small.\nVERDICT: CLEAN';
|
||||
const result = await createRunner('codex-cli').run({ promptFile: prompt, outFile: out, timeoutMs: 30_000, cwd: ROOT, gate: 'review', env: { ...cleanEnv(fakeBin), FAKE_CODEX_REVIEW: own, FAKE_CODEX_OWN: '1', FAKE_CODEX_LAST: clean }, binDir: gstackBin });
|
||||
expect(fs.readFileSync(out, 'utf8')).toBe(ownText);
|
||||
expect(result.verdict!.verdict).not.toBe('clean');
|
||||
});
|
||||
|
||||
test('a reviewer draft with a finding and no verdict is not replaced by a closing message with no verdict (GST-3188 review)', async () => {
|
||||
const own = path.join(TMP, 'own-finding.md');
|
||||
const ownText = '[P2] The rollback docs are missing.\nRecommendation: add them because operators need them.\n';
|
||||
fs.writeFileSync(own, ownText);
|
||||
const out = path.join(TMP, 'own-finding', 'review.md');
|
||||
const prompt = path.join(TMP, 'own-finding-prompt.md');
|
||||
fs.writeFileSync(prompt, `You are the outside reviewer.\n3. Write the whole result to this file: ${out}\n`);
|
||||
await createRunner('codex-cli').run({ promptFile: prompt, outFile: out, timeoutMs: 30_000, cwd: ROOT, gate: 'review', env: { ...cleanEnv(fakeBin), FAKE_CODEX_REVIEW: own, FAKE_CODEX_OWN: '1', FAKE_CODEX_LAST: 'Recommendation: ship because everything looks fine.' }, binDir: gstackBin });
|
||||
expect(fs.readFileSync(out, 'utf8')).toBe(ownText);
|
||||
});
|
||||
|
||||
test('a reviewer file for another snapshot is never reported clean when the closing message is not a result (GST-3188 review)', async () => {
|
||||
const stale = path.join(TMP, 'own-stale-only.md');
|
||||
fs.writeFileSync(stale, AUTOPLAN_REVIEW.replace(SHA, 'b'.repeat(64)).replace(/\{"id":"F1".*\}\n/, ''));
|
||||
const out = path.join(TMP, 'own-stale-only', 'ceo-outside.md');
|
||||
const prompt = path.join(TMP, 'own-stale-only-prompt.md');
|
||||
fs.writeFileSync(prompt, `You are the outside CEO reviewer.\n${RESULT_FORMAT('ceo', SHA, out)}`);
|
||||
const result = await createRunner('codex-cli').run({ promptFile: prompt, outFile: out, timeoutMs: 30_000, cwd: ROOT, gate: 'autoplan', env: { ...cleanEnv(fakeBin), FAKE_CODEX_REVIEW: stale, FAKE_CODEX_OWN: '1' }, binDir: gstackBin });
|
||||
expect(result.status).toBe('unavailable');
|
||||
});
|
||||
|
||||
test('a prompt file that is one of the run\'s own output files is refused and left as it was (GST-3188)', async () => {
|
||||
const review = path.join(TMP, 'clash-review.md');
|
||||
fs.writeFileSync(review, REVIEW);
|
||||
const out = path.join(TMP, 'clash', 'review.md');
|
||||
fs.mkdirSync(path.dirname(out), { recursive: true });
|
||||
const prompt = `${out}.last-message`;
|
||||
const original = 'You are the outside reviewer.\n';
|
||||
fs.writeFileSync(prompt, original);
|
||||
const result = await createRunner('codex-cli').run({ promptFile: prompt, outFile: out, timeoutMs: 30_000, cwd: ROOT, gate: 'review', env: { ...cleanEnv(fakeBin), FAKE_CODEX_REVIEW: review }, binDir: gstackBin });
|
||||
expect(fs.readFileSync(prompt, 'utf8')).toBe(original);
|
||||
expect([result.status, result.reason]).toEqual(['failed', 'prompt_is_output']);
|
||||
const cli = run(['run', '--runner', 'codex-cli', '--prompt', prompt, '--out', out], { ...cleanEnv(fakeBin), FAKE_CODEX_REVIEW: review });
|
||||
expect(cli.status).toBe(2);
|
||||
expect(cli.stderr).toContain(`--prompt ${prompt}`);
|
||||
expect(fs.readFileSync(prompt, 'utf8')).toBe(original);
|
||||
});
|
||||
|
||||
// Captured from codex-cli 0.170.0 against a local stub answering 402 (application/problem+json, a metering proxy's allowance) and 401.
|
||||
const REFUSAL_EVENTS = (status: string) => [
|
||||
'{"type":"thread.started","thread_id":"01a1f2ae-a04c-7583-8012-638cc20a388d"}',
|
||||
|
||||
Reference in new issue
Block a user