diff --git a/CHANGELOG.md b/CHANGELOG.md index 88736099e..eb22d4629 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## [1.93.41.0] - 2026-11-03 + +**Codex outside reviews report their token usage (#2948).** Codex already emits token counts in its `--json` events, and gstack used to throw them away. When `lib/outside-review-result.ts` is given an events file, it now prints `OUTSIDE_USAGE: input=… cached_input=… output=… reasoning_output=…` (only the counters Codex reported), or `OUTSIDE_USAGE: unknown`, on every verdict. + +### Changed +- **Every Codex outside voice that runs `codex exec --json`** (/review, /ship, /autoplan, the plan reviews) now prints one more stdout line, `OUTSIDE_USAGE:`, after `VERDICT:` and `FINDINGS:`. +- **One usage parser** (`usageFromEvents`, now in `lib/outside-review-result.ts` and re-exported by `lib/outside-voice-runner.ts`). A cumulative `total_token_usage` snapshot wins; without one, the latest `turn.completed` usage does, because Codex reports the session's running total there. It adds cached, cache-write, reasoning and total counters when the events carry them. + +Thanks to @omicron-soft for the report. Storing usage in review-log rows, the native `codex review` path and a token column in /ship come in a follow-up. + ## [1.93.40.0] - 2026-11-03 ### Fixed diff --git a/VERSION b/VERSION index 5ebe98992..abacf7c40 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.93.40.0 +1.93.41.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 937952f0e..3bab33017 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.93.40.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.93.41.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 56add0826..5a9bf8fb5 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -146,7 +146,9 @@ record. Another successful fetch alone cannot certify corrupted history. Outside reviews send your diff, plan or question to a second AI provider. Their verdict comes from `lib/outside-review-result.ts`, which prints -`VERDICT: clean|findings|unverified|unavailable`. A `findings` verdict with a +`VERDICT: clean|findings|unverified|unavailable`. When given an events file, it +also prints `OUTSIDE_USAGE:` with the token counters Codex reported, or +`OUTSIDE_USAGE: unknown` when it reported none. A `findings` verdict with a P0 or P1 finding blocks exactly like a native P0/P1. `unverified` and `unavailable` are missing coverage: /ship and /review continue, show the gap in the readiness dashboard and the PR body, and never count it as a pass. diff --git a/lib/outside-review-result.ts b/lib/outside-review-result.ts index c09ffb9a5..397c94ba2 100644 --- a/lib/outside-review-result.ts +++ b/lib/outside-review-result.ts @@ -297,6 +297,43 @@ const LEGACY_REASONS: Partial> = { review_refused: 'review refused', }; +export interface TokenUsage { + input_tokens?: number; cached_input_tokens?: number; cache_write_input_tokens?: number; + output_tokens?: number; reasoning_output_tokens?: number; total_tokens?: number; usd: number | 'unknown'; +} +const USAGE_COUNTERS = ['input_tokens', 'cached_input_tokens', 'cache_write_input_tokens', 'output_tokens', 'reasoning_output_tokens', 'total_tokens'] as const; + +/** + * Token usage from `codex exec --json` events, read loosely. A cumulative + * `total_token_usage` snapshot (info, payload.info or msg.info) wins, latest first; + * without one, the latest `usage` object does: codex exec fills turn.completed.usage + * with the session's running total, so summing turns would overcount. Only the + * counters the events carry are reported. + */ +export function usageFromEvents(events: string): TokenUsage { + let cumulative: Record | undefined; + let latest: Record = {}; + for (const line of events.split('\n')) { + if (!line.trim()) continue; + let obj: any; + try { obj = JSON.parse(line); } catch { continue; } + const snapshot = obj?.info?.total_token_usage ?? obj?.payload?.info?.total_token_usage ?? obj?.msg?.info?.total_token_usage; + if (snapshot && typeof snapshot === 'object') { cumulative = snapshot; continue; } + if (!obj?.usage || typeof obj.usage !== 'object') continue; + latest = obj.usage; + } + const source = cumulative ?? latest; + const usage: TokenUsage = { usd: 'unknown' }; + for (const key of USAGE_COUNTERS) if (typeof source[key] === 'number') usage[key] = source[key] as number; + return usage; +} + +/** The OUTSIDE_USAGE line: counters the events carried, or `unknown` when they carried none. */ +export function formatUsageLine(usage: TokenUsage): string { + const parts = USAGE_COUNTERS.filter(key => usage[key] !== undefined).map(key => `${key.replace(/_tokens$/, '')}=${usage[key]}`); + return `OUTSIDE_USAGE: ${parts.length ? parts.join(' ') : 'unknown'}`; +} + /** Text-only compatibility shape for direct importers (claude-code/SKILL.md.tmpl) and the two-argument CLI. */ export function validateOutsideReview(text: string, gate: OutsideGate): { completed: boolean; reason?: string; score?: number; gate?: 'pass' | 'fail' } { const checked = classifyOutsideReview({ text, gate }); @@ -347,12 +384,13 @@ if (import.meta.main) { if (!result.completed) { console.error(`Outside review unavailable: ${result.reason}; missing coverage.`); process.exit(1); } process.exit(0); } - const checked = classifyOutsideReview({ text, gate: gate as OutsideGate, exit, - stderr: await readOptional(flags['--stderr']), events: await readOptional(flags['--events']) }); + const events = await readOptional(flags['--events']); + const checked = classifyOutsideReview({ text, gate: gate as OutsideGate, exit, stderr: await readOptional(flags['--stderr']), events }); const label = flags['--label'] ?? 'Outside review'; console.log(`VERDICT: ${checked.verdict}`); console.log(`FINDINGS: ${checked.findings.highest ?? (checked.verdict === 'findings' ? 'unspecified' : 'none')}`); if (checked.unparsed) console.log(`UNPARSED: ${checked.unparsed}`); + if (flags['--events'] !== undefined) console.log(formatUsageLine(usageFromEvents(events))); if (checked.reason) { console.log(`REASON: ${checked.reason}`); console.error(gateOutcomeLine(label, checked.reason, checked.detail)); diff --git a/lib/outside-voice-runner.ts b/lib/outside-voice-runner.ts index 1ac6d569d..728dd0ec7 100644 --- a/lib/outside-voice-runner.ts +++ b/lib/outside-voice-runner.ts @@ -18,7 +18,9 @@ import { createHash } from 'node:crypto'; import * as fs from 'node:fs'; import * as path from 'node:path'; -import { classifyOutsideReview, type OutsideGate, type OutsideReviewClassification } from './outside-review-result'; +import { classifyOutsideReview, usageFromEvents, type OutsideGate, type OutsideReviewClassification, type TokenUsage } from './outside-review-result'; + +export { usageFromEvents }; export type RunnerId = 'codex-cli' | 'api' | 'host-subagent'; export const RUNNER_IDS: readonly RunnerId[] = ['codex-cli', 'api', 'host-subagent']; @@ -43,7 +45,7 @@ export interface RunnerRequest { binDir?: string; } export type RunnerStatus = 'completed' | 'unverified' | 'unavailable' | 'failed' | 'timeout' | 'cancelled'; -export interface RunnerUsage { input_tokens?: number; output_tokens?: number; usd: number | 'unknown' } +export type RunnerUsage = TokenUsage; export interface RunnerResult { schema_version: 1; runner: RunnerId; status: RunnerStatus; model: string | null; family: ModelFamily; capabilities: RunnerCapabilities; input_sha256: string; output_sha256: string | null; output: string | null; @@ -104,21 +106,6 @@ function classify(text: string, gate: OutsideGate, extra: { stderr?: string; exi return { status, verdict }; } -/** Token usage from `codex exec --json` events, read loosely: any object carrying usage counters. */ -export function usageFromEvents(events: string): RunnerUsage { - let input: number | undefined; let output: number | undefined; - for (const line of events.split('\n')) { - if (!line.trim()) continue; - let obj: any; - try { obj = JSON.parse(line); } catch { continue; } - const usage = obj?.usage ?? obj?.info?.total_token_usage ?? obj?.payload?.info?.total_token_usage ?? obj?.msg?.info?.total_token_usage; - if (!usage || typeof usage !== 'object') continue; - if (typeof usage.input_tokens === 'number') input = usage.input_tokens; - if (typeof usage.output_tokens === 'number') output = usage.output_tokens; - } - return { ...(input !== undefined ? { input_tokens: input } : {}), ...(output !== undefined ? { output_tokens: output } : {}), usd: 'unknown' }; -} - // --------------------------------------------------------------------------- // codex-cli: today's path, as one process this module can cancel // --------------------------------------------------------------------------- diff --git a/package.json b/package.json index ba4291f7c..f27bcb93f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.93.40", + "version": "1.93.41", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/test/outside-review-result.test.ts b/test/outside-review-result.test.ts index 572b90f30..e9550285b 100644 --- a/test/outside-review-result.test.ts +++ b/test/outside-review-result.test.ts @@ -8,7 +8,7 @@ import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { spawnSync } from 'node:child_process'; -import { classifyOutsideReview, validateOutsideReview, type OutsideGate } from '../lib/outside-review-result'; +import { classifyOutsideReview, usageFromEvents, validateOutsideReview, type OutsideGate } from '../lib/outside-review-result'; import { gateOutcomeLine, type GateReason } from '../lib/gate-outcomes'; const ROOT = path.resolve(import.meta.dir, '..'); @@ -262,6 +262,36 @@ describe('B1: a review whose sandbox could not start is unavailable, not clean', }); }); +describe('Codex token usage from --json events (#2948)', () => { + const turn = (usage: Record) => JSON.stringify({ type: 'turn.completed', usage }); + + test('one turn: every counter the event carries, nothing invented', () => { + expect(usageFromEvents(turn({ input_tokens: 12016, cached_input_tokens: 0, cache_write_input_tokens: 12013, output_tokens: 5, reasoning_output_tokens: 0 }))) + .toEqual({ input_tokens: 12016, cached_input_tokens: 0, cache_write_input_tokens: 12013, output_tokens: 5, reasoning_output_tokens: 0, usd: 'unknown' }); + }); + + test('the latest turn wins (codex reports the running session total); malformed and unrelated lines are skipped', () => { + const events = [turn({ input_tokens: 100, output_tokens: 10 }), 'not json', '{"type":"item.completed","item":{"type":"agent_message"}}', turn({ input_tokens: 150, output_tokens: 15, reasoning_output_tokens: 2 })].join('\n'); + expect(usageFromEvents(events)).toEqual({ input_tokens: 150, output_tokens: 15, reasoning_output_tokens: 2, usd: 'unknown' }); + }); + + test('a cumulative total_token_usage snapshot wins over per-turn usage, latest first', () => { + const snap = (n: number) => JSON.stringify({ msg: { type: 'token_count', info: { total_token_usage: { input_tokens: n, output_tokens: n / 10, total_tokens: n * 1.1 } } } }); + expect(usageFromEvents([turn({ input_tokens: 7 }), snap(100), snap(300)].join('\n'))).toEqual({ input_tokens: 300, output_tokens: 30, total_tokens: 330, usd: 'unknown' }); + }); + + test('the CLI prints OUTSIDE_USAGE only when given --events, on every verdict', () => { + const events = path.join(TMP, 'usage-events.jsonl'); + fs.writeFileSync(events, turn({ input_tokens: 1200, cached_input_tokens: 200, output_tokens: 340 }) + '\n'); + const withEvents = cli(['--events', events, 'review', file('No issues found.')]); + expect(withEvents.stdout).toContain('OUTSIDE_USAGE: input=1200 cached_input=200 output=340\n'); + const empty = path.join(TMP, 'no-usage.jsonl'); + fs.writeFileSync(empty, '{"type":"thread.started"}\n'); + expect(cli(['--events', empty, 'review', file('')]).stdout).toContain('OUTSIDE_USAGE: unknown\n'); + expect(cli(['--verdict', 'review', file('No issues found.')]).stdout).not.toContain('OUTSIDE_USAGE'); + }); +}); + describe('a mid-run Codex usage limit is unavailable (quota_exhausted); a mid-run 429 is unavailable (rate_limited)', () => { const LINE = "ERROR: You've hit your usage limit. Upgrade to Pro (https://chatgpt.com/explore/pro), visit https://chatgpt.com/codex/settings/usage to purchase more credits or try again at Oct 10th, 2026 2:55 AM."; const RATE_429 = 'ERROR: {"type":"error","status":429,"error":{"type":"rate_limit_exceeded","message":"Rate limit reached"}}'; diff --git a/test/run-in-background-guidance.test.ts b/test/run-in-background-guidance.test.ts index 5851cb6ea..8475a2661 100644 --- a/test/run-in-background-guidance.test.ts +++ b/test/run-in-background-guidance.test.ts @@ -107,8 +107,8 @@ exec ${quote(Bun.which('cat')!)} "$@" const findings = (id: string) => `${id}: current findings\nNo issues found.\nRecommendation: fix ${id} because this is the current finding.\n`; // INV-1: the verdict-form validator prints its verdict before the completed status. - const unavailable = (id: string) => `${findings(id)}VERDICT: unavailable\nFINDINGS: none\nREASON: execution_failed\n`; - const completed = (id: string) => `${findings(id)}VERDICT: clean\nFINDINGS: none\nOUTSIDE_STATUS: completed provider=codex host=claude\n`; + const unavailable = (id: string) => `${findings(id)}VERDICT: unavailable\nFINDINGS: none\nOUTSIDE_USAGE: unknown\nREASON: execution_failed\n`; + const completed = (id: string) => `${findings(id)}VERDICT: clean\nFINDINGS: none\nOUTSIDE_USAGE: unknown\nOUTSIDE_STATUS: completed provider=codex host=claude\n`; test('fresh shells retain the current stderr and clean its exact temporary file', () => { const f = fixture();