From 234f6be09b9c6f778297cea48e0e7903ddd275ec Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 31 Oct 2026 18:32:48 +0000 Subject: [PATCH] outside-review-result: one Codex usage parser and an OUTSIDE_USAGE line (#2948) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- changelog.d/gsta214-d.md | 7 +++++ lib/outside-review-result.ts | 42 ++++++++++++++++++++++++++++-- lib/outside-voice-runner.ts | 21 +++------------ test/outside-review-result.test.ts | 32 ++++++++++++++++++++++- 4 files changed, 82 insertions(+), 20 deletions(-) create mode 100644 changelog.d/gsta214-d.md diff --git a/changelog.d/gsta214-d.md b/changelog.d/gsta214-d.md new file mode 100644 index 000000000..3cd877673 --- /dev/null +++ b/changelog.d/gsta214-d.md @@ -0,0 +1,7 @@ +**Codex outside reviews report their token usage (#2948).** Codex already emits token counts in its `--json` events, and gstack used to throw them away. When `lib/outside-review-result.ts` is given an events file, it now prints `OUTSIDE_USAGE: input=… cached_input=… output=… reasoning_output=…` (only the counters Codex reported), or `OUTSIDE_USAGE: unknown`, on every verdict. + +### Changed +- **Every Codex outside voice that runs `codex exec --json`** (/review, /ship, /autoplan, the plan reviews) now prints one more stdout line, `OUTSIDE_USAGE:`, after `VERDICT:` and `FINDINGS:`. +- **One usage parser** (`usageFromEvents`, now in `lib/outside-review-result.ts` and re-exported by `lib/outside-voice-runner.ts`). A cumulative `total_token_usage` snapshot wins; without one, the latest `turn.completed` usage does, because Codex reports the session's running total there. It adds cached, cache-write, reasoning and total counters when the events carry them. + +Thanks to @omicron-soft for the report. Storing usage in review-log rows, the native `codex review` path and a token column in /ship come in a follow-up. diff --git a/lib/outside-review-result.ts b/lib/outside-review-result.ts index c09ffb9a5..397c94ba2 100644 --- a/lib/outside-review-result.ts +++ b/lib/outside-review-result.ts @@ -297,6 +297,43 @@ const LEGACY_REASONS: Partial> = { review_refused: 'review refused', }; +export interface TokenUsage { + input_tokens?: number; cached_input_tokens?: number; cache_write_input_tokens?: number; + output_tokens?: number; reasoning_output_tokens?: number; total_tokens?: number; usd: number | 'unknown'; +} +const USAGE_COUNTERS = ['input_tokens', 'cached_input_tokens', 'cache_write_input_tokens', 'output_tokens', 'reasoning_output_tokens', 'total_tokens'] as const; + +/** + * Token usage from `codex exec --json` events, read loosely. A cumulative + * `total_token_usage` snapshot (info, payload.info or msg.info) wins, latest first; + * without one, the latest `usage` object does: codex exec fills turn.completed.usage + * with the session's running total, so summing turns would overcount. Only the + * counters the events carry are reported. + */ +export function usageFromEvents(events: string): TokenUsage { + let cumulative: Record | undefined; + let latest: Record = {}; + for (const line of events.split('\n')) { + if (!line.trim()) continue; + let obj: any; + try { obj = JSON.parse(line); } catch { continue; } + const snapshot = obj?.info?.total_token_usage ?? obj?.payload?.info?.total_token_usage ?? obj?.msg?.info?.total_token_usage; + if (snapshot && typeof snapshot === 'object') { cumulative = snapshot; continue; } + if (!obj?.usage || typeof obj.usage !== 'object') continue; + latest = obj.usage; + } + const source = cumulative ?? latest; + const usage: TokenUsage = { usd: 'unknown' }; + for (const key of USAGE_COUNTERS) if (typeof source[key] === 'number') usage[key] = source[key] as number; + return usage; +} + +/** The OUTSIDE_USAGE line: counters the events carried, or `unknown` when they carried none. */ +export function formatUsageLine(usage: TokenUsage): string { + const parts = USAGE_COUNTERS.filter(key => usage[key] !== undefined).map(key => `${key.replace(/_tokens$/, '')}=${usage[key]}`); + return `OUTSIDE_USAGE: ${parts.length ? parts.join(' ') : 'unknown'}`; +} + /** Text-only compatibility shape for direct importers (claude-code/SKILL.md.tmpl) and the two-argument CLI. */ export function validateOutsideReview(text: string, gate: OutsideGate): { completed: boolean; reason?: string; score?: number; gate?: 'pass' | 'fail' } { const checked = classifyOutsideReview({ text, gate }); @@ -347,12 +384,13 @@ if (import.meta.main) { if (!result.completed) { console.error(`Outside review unavailable: ${result.reason}; missing coverage.`); process.exit(1); } process.exit(0); } - const checked = classifyOutsideReview({ text, gate: gate as OutsideGate, exit, - stderr: await readOptional(flags['--stderr']), events: await readOptional(flags['--events']) }); + const events = await readOptional(flags['--events']); + const checked = classifyOutsideReview({ text, gate: gate as OutsideGate, exit, stderr: await readOptional(flags['--stderr']), events }); const label = flags['--label'] ?? 'Outside review'; console.log(`VERDICT: ${checked.verdict}`); console.log(`FINDINGS: ${checked.findings.highest ?? (checked.verdict === 'findings' ? 'unspecified' : 'none')}`); if (checked.unparsed) console.log(`UNPARSED: ${checked.unparsed}`); + if (flags['--events'] !== undefined) console.log(formatUsageLine(usageFromEvents(events))); if (checked.reason) { console.log(`REASON: ${checked.reason}`); console.error(gateOutcomeLine(label, checked.reason, checked.detail)); diff --git a/lib/outside-voice-runner.ts b/lib/outside-voice-runner.ts index f56c5e2c9..9ce1c7c5c 100644 --- a/lib/outside-voice-runner.ts +++ b/lib/outside-voice-runner.ts @@ -18,7 +18,9 @@ import { createHash } from 'node:crypto'; import * as fs from 'node:fs'; import * as path from 'node:path'; -import { classifyOutsideReview, type OutsideGate, type OutsideReviewClassification } from './outside-review-result'; +import { classifyOutsideReview, usageFromEvents, type OutsideGate, type OutsideReviewClassification, type TokenUsage } from './outside-review-result'; + +export { usageFromEvents }; export type RunnerId = 'codex-cli' | 'api' | 'host-subagent'; export const RUNNER_IDS: readonly RunnerId[] = ['codex-cli', 'api', 'host-subagent']; @@ -41,7 +43,7 @@ export interface RunnerRequest { binDir?: string; } export type RunnerStatus = 'completed' | 'unverified' | 'unavailable' | 'failed' | 'timeout' | 'cancelled'; -export interface RunnerUsage { input_tokens?: number; output_tokens?: number; usd: number | 'unknown' } +export type RunnerUsage = TokenUsage; export interface RunnerResult { schema_version: 1; runner: RunnerId; status: RunnerStatus; model: string | null; family: ModelFamily; capabilities: RunnerCapabilities; input_sha256: string; output_sha256: string | null; output: string | null; @@ -102,21 +104,6 @@ function classify(text: string, gate: OutsideGate, extra: { stderr?: string; exi return { status, verdict }; } -/** Token usage from `codex exec --json` events, read loosely: any object carrying usage counters. */ -export function usageFromEvents(events: string): RunnerUsage { - let input: number | undefined; let output: number | undefined; - for (const line of events.split('\n')) { - if (!line.trim()) continue; - let obj: any; - try { obj = JSON.parse(line); } catch { continue; } - const usage = obj?.usage ?? obj?.info?.total_token_usage ?? obj?.payload?.info?.total_token_usage ?? obj?.msg?.info?.total_token_usage; - if (!usage || typeof usage !== 'object') continue; - if (typeof usage.input_tokens === 'number') input = usage.input_tokens; - if (typeof usage.output_tokens === 'number') output = usage.output_tokens; - } - return { ...(input !== undefined ? { input_tokens: input } : {}), ...(output !== undefined ? { output_tokens: output } : {}), usd: 'unknown' }; -} - // --------------------------------------------------------------------------- // codex-cli: today's path, as one process this module can cancel // --------------------------------------------------------------------------- diff --git a/test/outside-review-result.test.ts b/test/outside-review-result.test.ts index 572b90f30..e9550285b 100644 --- a/test/outside-review-result.test.ts +++ b/test/outside-review-result.test.ts @@ -8,7 +8,7 @@ import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { spawnSync } from 'node:child_process'; -import { classifyOutsideReview, validateOutsideReview, type OutsideGate } from '../lib/outside-review-result'; +import { classifyOutsideReview, usageFromEvents, validateOutsideReview, type OutsideGate } from '../lib/outside-review-result'; import { gateOutcomeLine, type GateReason } from '../lib/gate-outcomes'; const ROOT = path.resolve(import.meta.dir, '..'); @@ -262,6 +262,36 @@ describe('B1: a review whose sandbox could not start is unavailable, not clean', }); }); +describe('Codex token usage from --json events (#2948)', () => { + const turn = (usage: Record) => JSON.stringify({ type: 'turn.completed', usage }); + + test('one turn: every counter the event carries, nothing invented', () => { + expect(usageFromEvents(turn({ input_tokens: 12016, cached_input_tokens: 0, cache_write_input_tokens: 12013, output_tokens: 5, reasoning_output_tokens: 0 }))) + .toEqual({ input_tokens: 12016, cached_input_tokens: 0, cache_write_input_tokens: 12013, output_tokens: 5, reasoning_output_tokens: 0, usd: 'unknown' }); + }); + + test('the latest turn wins (codex reports the running session total); malformed and unrelated lines are skipped', () => { + const events = [turn({ input_tokens: 100, output_tokens: 10 }), 'not json', '{"type":"item.completed","item":{"type":"agent_message"}}', turn({ input_tokens: 150, output_tokens: 15, reasoning_output_tokens: 2 })].join('\n'); + expect(usageFromEvents(events)).toEqual({ input_tokens: 150, output_tokens: 15, reasoning_output_tokens: 2, usd: 'unknown' }); + }); + + test('a cumulative total_token_usage snapshot wins over per-turn usage, latest first', () => { + const snap = (n: number) => JSON.stringify({ msg: { type: 'token_count', info: { total_token_usage: { input_tokens: n, output_tokens: n / 10, total_tokens: n * 1.1 } } } }); + expect(usageFromEvents([turn({ input_tokens: 7 }), snap(100), snap(300)].join('\n'))).toEqual({ input_tokens: 300, output_tokens: 30, total_tokens: 330, usd: 'unknown' }); + }); + + test('the CLI prints OUTSIDE_USAGE only when given --events, on every verdict', () => { + const events = path.join(TMP, 'usage-events.jsonl'); + fs.writeFileSync(events, turn({ input_tokens: 1200, cached_input_tokens: 200, output_tokens: 340 }) + '\n'); + const withEvents = cli(['--events', events, 'review', file('No issues found.')]); + expect(withEvents.stdout).toContain('OUTSIDE_USAGE: input=1200 cached_input=200 output=340\n'); + const empty = path.join(TMP, 'no-usage.jsonl'); + fs.writeFileSync(empty, '{"type":"thread.started"}\n'); + expect(cli(['--events', empty, 'review', file('')]).stdout).toContain('OUTSIDE_USAGE: unknown\n'); + expect(cli(['--verdict', 'review', file('No issues found.')]).stdout).not.toContain('OUTSIDE_USAGE'); + }); +}); + describe('a mid-run Codex usage limit is unavailable (quota_exhausted); a mid-run 429 is unavailable (rate_limited)', () => { const LINE = "ERROR: You've hit your usage limit. Upgrade to Pro (https://chatgpt.com/explore/pro), visit https://chatgpt.com/codex/settings/usage to purchase more credits or try again at Oct 10th, 2026 2:55 AM."; const RATE_429 = 'ERROR: {"type":"error","status":429,"error":{"type":"rate_limit_exceeded","message":"Rate limit reached"}}';