diff --git a/autoplan/DRIVER.md b/autoplan/DRIVER.md index fad3a04f7..3d3fc50bc 100644 --- a/autoplan/DRIVER.md +++ b/autoplan/DRIVER.md @@ -59,6 +59,8 @@ line yourself; gstack never writes the plan. - `gstack-autoplan next --out --plan [scope flags] [--light] [--spend-cap ] [--max-rounds ] [--estimate-usd ] [--scaffold]` first call: init the run, print PHASE:, SCOPE:, DRIVER: and one ATTEMPT: per voice; --max-rounds caps review rounds per phase (default 2 unattended); an unattended run with --spend-cap (or any unattended Capy run) reserves 5.00 per voice for each attempt, so the cap binds; --estimate-usd sets that reservation and the run keeps it for later phases. +- `gstack-autoplan run --out --plan --native-model --outside-model --native-runner api --runner api [--max-tokens ] [--spend-cap ]` + hosts without subagents: one command calls both models directly (ANTHROPIC_API_KEY, OPENAI_API_KEY) and loops next/submit to the gate; a response cut off at the output limit is reported truncated, so raise --max-tokens (Anthropic default 16384). - `gstack-autoplan next --out [--redispatch ]` later calls: open the next phase or reprint open attempts; never redispatches on its own. - `gstack-autoplan submit --out --phase

--voice native|outside --record --model [--attempt ]` @@ -75,10 +77,8 @@ line yourself; gstack never writes the plan. change one recommendation before relay; then render and relay again (a `rec` needs the new `--rec-hash`). - `gstack-autoplan revise --out

--plan --incorporates ` bind the amended plan; reopens every phase whose reviewed plan changed, Eng last. -- `gstack-autoplan revise --out --settle ` - discharge a packaging or ordering change with no plan edit; reopens nothing. -- `gstack-autoplan revise --out --withdraw ` - drop a pending amendment instead of applying it. +- `gstack-autoplan revise --out --settle | --withdraw ` + --settle discharges a packaging or ordering change with no plan edit and reopens nothing; --withdraw drops a pending amendment instead of applying it. - `gstack-autoplan close --out --phase

--deferred "" --by human|agent:` close a phase accepted-with-findings: its open findings go to tasks.jsonl, and Critical, urgent or user-challenge findings become gate items; also the recovery from `STOP: reason=budget_exhausted`. - `gstack-autoplan budget --out

[--spend-cap ] [--max-rounds ] --by human|agent:` diff --git a/bin/gstack-autoplan b/bin/gstack-autoplan index 753e182c0..4134ec9ff 100755 --- a/bin/gstack-autoplan +++ b/bin/gstack-autoplan @@ -9,7 +9,7 @@ * gstack-autoplan submit --out --phase

--voice outside --unavailable | --failed [--json] * gstack-autoplan revise --out

--plan [--incorporates ] [--withdraw ] [--settle ] [--json] review a revised plan in this run * gstack-autoplan run --out --plan --native-model --outside-model [--runner codex-cli|api] [--native-runner api] - * [--deadline ] [--spend-cap ] [--estimate-usd ] [--json] next in a loop; needs runners for both voices + * [--deadline ] [--spend-cap ] [--estimate-usd ] [--max-tokens ] [--json] next in a loop; needs runners for both voices * gstack-autoplan answer --out --gate-rev --reply "" --by > [--owner-reply |-] [--rec-hash ] [--json] * gstack-autoplan recommend --out --gate-rev --item --option --basis "" [--by ] * gstack-autoplan resume --out [--from ] [--json] take ownership; intent without a terminal record becomes EXECUTION_UNKNOWN @@ -48,6 +48,7 @@ import { FILES, RunError, currentPhase, initRun, markOutside, nextStep, readStat import { driverFile, type Phase } from '../lib/autoplan-prompts'; import { EXIT, exitTableHelp } from '../lib/headless-artifacts'; import { createRunner, familyConflict, type RunnerId } from '../lib/outside-voice-runner'; +import { apiTimeoutMs } from '../lib/outside-voice-api'; import { resultLine } from '../lib/result-codes'; import { release } from '../lib/spend-ledger'; import { COORDINATOR_CONTRACT } from '../lib/coordinator-contract'; @@ -80,7 +81,9 @@ DRIVER: ${driverFile()} comes into scope, the edit applies without re-review (APPLIED:); --withdraw (alone or with --plan) drops one; --settle records a packaging/ordering change that needs no plan edit and reopens nothing (ids: decision id or d4) run --out --plan --native-model --outside-model [--runner codex-cli|api] [--native-runner api] ... - next in a loop, only when a runner executes both voices here; --deadline cancels the children it started + next in a loop, only when a runner executes both voices here (hosts without subagents: --native-runner api --runner api); + --deadline cancels the children it started; --max-tokens caps each api call's output (Anthropic default 16384), and a + response cut off at that limit is reported truncated; submit's result checks judge every completed response answer --out --gate-rev --reply "" --by > [--json] --by agent: needs --owner-reply "" (- reads stdin; --owner-reply-file ), stored verbatim; --rec-hash binds a rec to the render the owner saw (required once recommend ran); accepted changes leave amend_pending @@ -305,6 +308,7 @@ async function doRun(out: string, rest: string[], json: boolean): Promise { const s = readState(out); s.status = 'interrupted'; writeState(s); }); process.stdout.write(resultLineFor('interrupted', out) + '\n'); return EXIT.fail; } const timer = deadlineAt ? setTimeout(() => runner.cancel(), remaining) : undefined; - const res = await runner.run({ promptFile: open.prompt, outFile: `${open.result}.raw`, model: voice === 'native' ? nativeModel : outsideModel, timeoutMs: Math.min(remaining, 600_000), cwd: process.cwd(), gate: 'review' }); + const res = await runner.run({ promptFile: open.prompt, outFile: `${open.result}.raw`, model: voice === 'native' ? nativeModel : outsideModel, timeoutMs: Math.min(remaining, runnerId === 'api' ? apiTimeoutMs(maxTokens) : 600_000), cwd: process.cwd(), gate: 'execution', maxTokens }); if (timer) clearTimeout(timer); if (!json) process.stdout.write(`VOICE: ${phase} ${voice} ${res.status} runner=${res.runner} model=${res.model ?? 'unknown'}\n`); if (res.status === 'cancelled') { withRunLock(out, () => { const s = readState(out); s.status = 'interrupted'; writeState(s); }); process.stdout.write(resultLineFor('interrupted', out) + '\n'); return EXIT.fail; } if (res.status !== 'completed' && res.status !== 'unverified') { if (voice === 'outside') { withRunLock(out, () => markOutside(out, phase as Phase, 'unavailable', `${res.runner} ${res.status}${res.detail ? `: ${res.detail}` : ''}`)); continue; } - process.stderr.write(`gstack-autoplan run: native voice ${res.status}${res.detail ? `: ${res.detail}` : ''}\n`); + process.stderr.write(`gstack-autoplan run: native voice ${res.status}${res.reason ? `: ${res.reason}` : ''}${res.detail ? `: ${res.detail}` : ''}\n`); withRunLock(out, () => { const s = readState(out); s.status = 'interrupted'; writeState(s); }); process.stdout.write(resultLineFor('interrupted', out) + '\n'); return EXIT.fail; diff --git a/bin/gstack-outside-voice b/bin/gstack-outside-voice index 28a03e3be..d2b2c56c1 100755 --- a/bin/gstack-outside-voice +++ b/bin/gstack-outside-voice @@ -4,7 +4,7 @@ * * gstack-outside-voice run --runner codex-cli|api|host-subagent --prompt --out * [--model ] [--result ] [--native-model ] [--fallback ] - * [--gate review|structured|spec|proposal] [--timeout-ms N] [--cwd ] [--json] + * [--gate review|structured|spec|proposal] [--timeout-ms N] [--max-tokens N] [--cwd ] [--json] * gstack-outside-voice runners [--model ] [--json] the runner table: capabilities and availability here * gstack-outside-voice family the model's family (same-family rule) * @@ -20,18 +20,21 @@ import { EXIT, exitTableHelp } from '../lib/headless-artifacts'; import type { OutsideGate } from '../lib/outside-review-result'; import { RUNNER_IDS, createRunner, familyConflict, modelFamily, outsideStatusLine, runnerTable, selectRunner, type RunnerId } from '../lib/outside-voice-runner'; import { outsideConsent } from '../lib/outside-consent'; +import { apiTimeoutMs } from '../lib/outside-voice-api'; import { resultLine } from '../lib/result-codes'; const HELP = `gstack-outside-voice — one outside voice, three runners (docs/unattended.md) run --runner --prompt --out [--model ] [--result ] [--native-model ] - [--fallback ] [--gate review|structured|spec|proposal] [--timeout-ms N] [--cwd ] [--json] + [--fallback ] [--gate review|structured|spec|proposal] [--timeout-ms N] [--max-tokens N] [--cwd ] [--json] runners [--model ] [--json] the runner table: id, capabilities, available here family print the model family (anthropic, openai, google, ...) Runners: codex-cli (repository review via codex exec), api (supplied-input review via OPENAI_API_KEY/ANTHROPIC_API_KEY; needs --model), host-subagent (binds --result from a -cross-family subagent the host ran; needs --model). --native-model enforces the same-family +cross-family subagent the host ran; needs --model). --max-tokens caps the api runner's output +(Anthropic default 16384; its default time limit grows with it); a response cut off at the limit +is unavailable (truncated). --native-model enforces the same-family rule. GSTACK_OUTSIDE_RUNNER names the fallback when --fallback is absent. Prints OUTSIDE_STATUS: provider= model= family= and writes .meta.json. codex_reviews=disabled (or an unreadable config) refuses every runner before it starts: @@ -75,9 +78,12 @@ async function main(argv: string[]): Promise { if (!fs.existsSync(promptFile)) return usage(`prompt file not found: ${promptFile}`); const gate = (flag(rest, '--gate') ?? 'review') as OutsideGate; if (!['review', 'structured', 'spec', 'proposal'].includes(gate)) return usage('--gate must be review, structured, spec or proposal'); + const maxTokensText = flag(rest, '--max-tokens'); + const maxTokens = maxTokensText === undefined ? undefined : Number(maxTokensText); + if (maxTokens !== undefined && (!Number.isInteger(maxTokens) || maxTokens <= 0)) return usage('--max-tokens needs a positive integer'); const timeoutText = flag(rest, '--timeout-ms'); - const timeoutMs = timeoutText === undefined ? 300_000 : Number(timeoutText); - if (!Number.isInteger(timeoutMs) || timeoutMs <= 0) return usage('--timeout-ms needs a positive integer'); + const timeoutMs = timeoutText === undefined ? undefined : Number(timeoutText); + if (timeoutMs !== undefined && (!Number.isInteger(timeoutMs) || timeoutMs <= 0)) return usage('--timeout-ms needs a positive integer'); const fallbackText = flag(rest, '--fallback') ?? process.env.GSTACK_OUTSIDE_RUNNER; if (fallbackText !== undefined && !isRunner(fallbackText)) return usage(`--fallback must be one of ${RUNNER_IDS.join(', ')}`); const model = flag(rest, '--model'); @@ -110,7 +116,7 @@ async function main(argv: string[]): Promise { const stop = () => runner.cancel(); process.on('SIGTERM', stop); process.on('SIGINT', stop); - const result = await runner.run({ promptFile: path.resolve(promptFile), outFile: path.resolve(outFile), model, timeoutMs, cwd, gate, resultFile: resultFile ? path.resolve(resultFile) : undefined }); + const result = await runner.run({ promptFile: path.resolve(promptFile), outFile: path.resolve(outFile), model, timeoutMs: timeoutMs ?? (choice.runner === 'api' ? apiTimeoutMs(maxTokens) : 300_000), cwd, gate, maxTokens, resultFile: resultFile ? path.resolve(resultFile) : undefined }); fs.mkdirSync(path.dirname(path.resolve(outFile)), { recursive: true }); fs.writeFileSync(`${path.resolve(outFile)}.meta.json`, JSON.stringify(result, null, 2) + '\n'); if (json) process.stdout.write(JSON.stringify(result) + '\n'); diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 911d8e829..624827953 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -314,6 +314,18 @@ network. **Expected result.** `OUTSIDE_STATUS: completed`. + +### `... outside review unavailable: the reviewer hit its output token limit and the response is cut off (...)` + +**Meaning.** The api runner's provider stopped at the output token limit +(Anthropic `stop_reason: max_tokens`, OpenAI `finish_reason: length`), so the +review ends mid-sentence and its findings may be missing. + +**Fix.** Re-run with a larger `--max-tokens` (`gstack-outside-voice run` or +`gstack-autoplan run`; Anthropic defaults to 16384), or shorten the prompt. + +**Expected result.** `OUTSIDE_STATUS: completed`. + ### `... outside review unavailable: the response lacks the markers this gate requires (...)` diff --git a/lib/gate-outcomes.ts b/lib/gate-outcomes.ts index 0834d58f2..d29ade2be 100644 --- a/lib/gate-outcomes.ts +++ b/lib/gate-outcomes.ts @@ -48,6 +48,11 @@ export const GATE_OUTCOMES = { summary: 'the reviewer declined to review', fix: 're-run the review; if it declines again, review the change natively', }, + truncated: { + state: 'unavailable', anchor: 'outside-review-truncated', + summary: 'the reviewer hit its output token limit and the response is cut off', + fix: 're-run with a larger --max-tokens (or a smaller prompt); a cut-off response never counts as a review', + }, missing_markers: { state: 'unavailable', anchor: 'outside-review-missing-markers', summary: 'the response lacks the markers this gate requires', diff --git a/lib/outside-voice-api.ts b/lib/outside-voice-api.ts index f5c6d17e2..1ec8b0760 100644 --- a/lib/outside-voice-api.ts +++ b/lib/outside-voice-api.ts @@ -13,7 +13,13 @@ import { modelFamily, type ModelFamily } from './outside-voice-runner'; export interface ApiCallRequest { model: string; prompt: string; timeoutMs: number; env?: Record; signal?: AbortSignal; maxTokens?: number } export interface ApiUsage { input_tokens?: number; output_tokens?: number } -export interface ApiCallResult { text: string; model: string; usage: ApiUsage; provider: 'openai' | 'anthropic'; status: number } +/** `truncated`: the provider stopped at the output token limit (Anthropic `max_tokens`, OpenAI `length`), so the text is cut off. */ +export interface ApiCallResult { text: string; model: string; usage: ApiUsage; provider: 'openai' | 'anthropic'; status: number; truncated: boolean } + +/** Anthropic requires max_tokens; a full review needs more than 8192. OpenAI gets a limit only when one is asked for. */ +export const DEFAULT_MAX_TOKENS = 16_384; +/** The api runner's default time limit: five minutes, or about 40 ms per output token the call may write. */ +export const apiTimeoutMs = (maxTokens?: number): number => Math.max(300_000, (maxTokens ?? DEFAULT_MAX_TOKENS) * 40); export const API_FAMILIES: readonly ModelFamily[] = ['openai', 'anthropic']; @@ -57,17 +63,17 @@ export async function callModel(req: ApiCallRequest): Promise { const model = req.model.includes('/') ? req.model.slice(req.model.indexOf('/') + 1) : req.model; if (auth.provider === 'anthropic') { const base = (env.ANTHROPIC_BASE_URL ?? 'https://api.anthropic.com').replace(/\/$/, ''); - const body = JSON.stringify({ model, max_tokens: req.maxTokens ?? 8192, messages: [{ role: 'user', content: req.prompt }] }); + const body = JSON.stringify({ model, max_tokens: req.maxTokens ?? DEFAULT_MAX_TOKENS, messages: [{ role: 'user', content: req.prompt }] }); const { status, json } = await post(`${base}/v1/messages`, { 'x-api-key': auth.key, 'anthropic-version': '2023-06-01' }, body, req, 'anthropic'); if (status < 200 || status >= 300) throw new Error(`anthropic ${status}: ${json?.error?.message ?? json?.raw ?? JSON.stringify(json)}`); const text = Array.isArray(json?.content) ? json.content.filter((c: any) => c?.type === 'text').map((c: any) => c.text).join('\n') : ''; - return { text, model: typeof json?.model === 'string' ? json.model : model, usage: { input_tokens: json?.usage?.input_tokens, output_tokens: json?.usage?.output_tokens }, provider: 'anthropic', status }; + return { text, model: typeof json?.model === 'string' ? json.model : model, usage: { input_tokens: json?.usage?.input_tokens, output_tokens: json?.usage?.output_tokens }, provider: 'anthropic', status, truncated: json?.stop_reason === 'max_tokens' }; } const base = (env.OPENAI_BASE_URL ?? 'https://api.openai.com/v1').replace(/\/$/, ''); - const body = JSON.stringify({ model, messages: [{ role: 'user', content: req.prompt }] }); + const body = JSON.stringify({ model, messages: [{ role: 'user', content: req.prompt }], ...(req.maxTokens ? { max_completion_tokens: req.maxTokens } : {}) }); const { status, json } = await post(`${base}/chat/completions`, { authorization: `Bearer ${auth.key}` }, body, req, 'openai'); if (status < 200 || status >= 300) throw new Error(`openai ${status}: ${json?.error?.message ?? json?.raw ?? JSON.stringify(json)}`); const content = json?.choices?.[0]?.message?.content; const text = typeof content === 'string' ? content : Array.isArray(content) ? content.map((c: any) => c?.text ?? '').join('\n') : ''; - return { text, model: typeof json?.model === 'string' ? json.model : model, usage: { input_tokens: json?.usage?.prompt_tokens, output_tokens: json?.usage?.completion_tokens }, provider: 'openai', status }; + return { text, model: typeof json?.model === 'string' ? json.model : model, usage: { input_tokens: json?.usage?.prompt_tokens, output_tokens: json?.usage?.completion_tokens }, provider: 'openai', status, truncated: json?.choices?.[0]?.finish_reason === 'length' }; } diff --git a/lib/outside-voice-runner.ts b/lib/outside-voice-runner.ts index f56c5e2c9..1ac6d569d 100644 --- a/lib/outside-voice-runner.ts +++ b/lib/outside-voice-runner.ts @@ -36,6 +36,8 @@ export interface RunnerRequest { promptFile: string; outFile: string; model?: string; timeoutMs: number; cwd: string; gate: OutsideGate; /** host-subagent: the file the host's subagent wrote. */ resultFile?: string; + /** api: the output token limit (lib/outside-voice-api.ts DEFAULT_MAX_TOKENS for Anthropic when unset). */ + maxTokens?: number; env?: Record; /** Directory of the gstack bins (gstack-codex-probe); defaults to this module's sibling bin/. */ binDir?: string; @@ -199,10 +201,11 @@ class ApiRunner implements OutsideRunner { if (!avail.ok) return finish({ ...base, status: 'unavailable', exit: 1, reason: 'runner_unavailable', detail: avail.reason }, started); const { callModel } = await import('./outside-voice-api'); try { - const res = await callModel({ model: req.model!, prompt, timeoutMs: req.timeoutMs, env: req.env, signal: this.controller.signal }); + const res = await callModel({ model: req.model!, prompt, timeoutMs: req.timeoutMs, env: req.env, signal: this.controller.signal, maxTokens: req.maxTokens }); fs.mkdirSync(path.dirname(req.outFile), { recursive: true }); fs.writeFileSync(req.outFile, res.text); const usage: RunnerUsage = { ...res.usage, usd: 'unknown' }; + if (res.truncated) return finish({ ...base, model: res.model, family: modelFamily(res.model), status: 'unavailable', exit: 0, usage, reason: 'truncated', detail: `stopped at the output limit after ${res.usage.output_tokens ?? 'an unknown number of'} output tokens; raise --max-tokens` }, started, req.outFile); const { status, verdict } = classify(res.text, req.gate, { exit: 0 }); return finish({ ...base, model: res.model, family: modelFamily(res.model), status, exit: 0, usage, verdict, ...(verdict.reason ? { reason: verdict.reason } : {}), ...(verdict.detail ? { detail: verdict.detail } : {}) }, started, req.outFile); } catch (e: any) { diff --git a/scripts/resolvers/autoplan-driver.ts b/scripts/resolvers/autoplan-driver.ts index b7f5c5352..bdfd3e5c2 100644 --- a/scripts/resolvers/autoplan-driver.ts +++ b/scripts/resolvers/autoplan-driver.ts @@ -18,6 +18,7 @@ import type { ResolverFn } from './types'; /** One entry per command line the loop runs, as the field-wave plan specifies them. */ export const AUTOPLAN_DRIVER_COMMANDS: Array<{ cmd: string; does: string }> = [ { cmd: 'gstack-autoplan next --out --plan [scope flags] [--light] [--spend-cap ] [--max-rounds ] [--estimate-usd ] [--scaffold]', does: `first call: init the run, print PHASE:, SCOPE:, DRIVER: and one ATTEMPT: per voice; --max-rounds caps review rounds per phase (default ${DEFAULT_MAX_ROUNDS} unattended); an unattended run with --spend-cap (or any unattended Capy run) reserves ${DEFAULT_HOST_ESTIMATE_USD.toFixed(2)} per voice for each attempt, so the cap binds; --estimate-usd sets that reservation and the run keeps it for later phases` }, + { cmd: 'gstack-autoplan run --out --plan --native-model --outside-model --native-runner api --runner api [--max-tokens ] [--spend-cap ]', does: 'hosts without subagents: one command calls both models directly (ANTHROPIC_API_KEY, OPENAI_API_KEY) and loops next/submit to the gate; a response cut off at the output limit is reported truncated, so raise --max-tokens (Anthropic default 16384)' }, { cmd: 'gstack-autoplan next --out [--redispatch ]', does: 'later calls: open the next phase or reprint open attempts; never redispatches on its own' }, { cmd: 'gstack-autoplan submit --out --phase

--voice native|outside --record --model [--attempt ]', does: 'bind a free-form review record (canonical fence, one findings table or list, or `No findings.`)' }, { cmd: 'gstack-autoplan submit ... --record --attest "" --attempt ', does: 'bind a record with no INPUT: line as receipt: attested (a wrong INPUT: always refuses)' }, @@ -26,8 +27,7 @@ export const AUTOPLAN_DRIVER_COMMANDS: Array<{ cmd: string; does: string }> = [ { cmd: 'gstack-autoplan answer --out

--gate-rev --reply "" --by human|agent: [--owner-reply |-] [--rec-hash ]', does: 'answer the gate with the reply grammar below; an agent answer stores the owner\'s words verbatim' }, { cmd: 'gstack-autoplan recommend --out --gate-rev --item --option --basis ""', does: 'change one recommendation before relay; then render and relay again (a `rec` needs the new `--rec-hash`)' }, { cmd: 'gstack-autoplan revise --out --plan --incorporates ', does: 'bind the amended plan; reopens every phase whose reviewed plan changed, Eng last' }, - { cmd: 'gstack-autoplan revise --out --settle ', does: 'discharge a packaging or ordering change with no plan edit; reopens nothing' }, - { cmd: 'gstack-autoplan revise --out --withdraw ', does: 'drop a pending amendment instead of applying it' }, + { cmd: 'gstack-autoplan revise --out --settle | --withdraw ', does: '--settle discharges a packaging or ordering change with no plan edit and reopens nothing; --withdraw drops a pending amendment instead of applying it' }, { cmd: 'gstack-autoplan close --out --phase

--deferred "" --by human|agent:', does: 'close a phase accepted-with-findings: its open findings go to tasks.jsonl, and Critical, urgent or user-challenge findings become gate items; also the recovery from `STOP: reason=budget_exhausted`' }, { cmd: 'gstack-autoplan budget --out

[--spend-cap ] [--max-rounds ] --by human|agent:', does: 'the owner\'s journaled cap raise; after a budget stop, `resume` continues the run' }, { cmd: 'gstack-autoplan export --out ', does: 'write the artifact set; then `gstack-artifact validate /run.json`' }, diff --git a/test/autoplan-run-api.test.ts b/test/autoplan-run-api.test.ts new file mode 100644 index 000000000..984df907c --- /dev/null +++ b/test/autoplan-run-api.test.ts @@ -0,0 +1,66 @@ +/** + * `gstack-autoplan run` with the api runner for both voices (GBRA-223 claim 6): + * a complete autoplan result (INPUT receipt plus the findings fence) binds and + * the run reaches its gate; a response stopped at the output limit is named + * truncated and interrupts the run. The provider is a local server, so no key + * or network is used. + */ +import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { AUTOPLAN, ROOT, SMALL_PLAN, env } from './helpers/autoplan-run-fixture'; + +const TMP = fs.mkdtempSync(path.join(os.tmpdir(), 'autoplan-run-api-')); +let mode: 'ok' | 'cut' = 'ok'; +let server: ReturnType; +beforeAll(() => { + server = Bun.serve({ + port: 0, + async fetch(req) { + const body = await req.json(); + const prompt = String(body.messages[0].content); + const m = /INPUT: ([a-z]+) ([0-9a-f]{64})/.exec(prompt.slice(prompt.lastIndexOf('RESULT FORMAT')))!; + const text = mode === 'cut' + ? `INPUT: ${m[1]} ${m[2]}\n\nA long review that ran out of output tokens before the findings fen` + : `INPUT: ${m[1]} ${m[2]}\n\nComplete review.\n\n\`\`\`gstack-findings\n{"id":"F1","severity":"Medium","title":"Name the retry metric","fix":"add retries_total"}\n\`\`\`\n`; + if (new URL(req.url).pathname.endsWith('/messages')) return Response.json({ model: body.model, stop_reason: mode === 'cut' ? 'max_tokens' : 'end_turn', content: [{ type: 'text', text }], usage: { input_tokens: 10, output_tokens: 20 } }); + return Response.json({ model: body.model, choices: [{ finish_reason: mode === 'cut' ? 'length' : 'stop', message: { content: text } }], usage: { prompt_tokens: 10, completion_tokens: 20 } }); + }, + }); +}); +afterAll(() => { server.stop(true); fs.rmSync(TMP, { recursive: true, force: true }); }); + +async function run(name: string) { + const dir = path.join(TMP, name); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, 'plan.md'), SMALL_PLAN); + const out = path.join(dir, 'run'); + const base = `http://127.0.0.1:${server.port}`; + const proc = Bun.spawn(['bun', AUTOPLAN, 'run', '--out', out, '--plan', path.join(dir, 'plan.md'), '--no-ui', '--no-dx', '--native-model', 'anthropic/claude-opus-5-5', '--outside-model', 'openai/gpt-6-astra', '--runner', 'api', '--native-runner', 'api'], { + cwd: ROOT, stdout: 'pipe', stderr: 'pipe', + env: env({ ANTHROPIC_API_KEY: 'ak-test', OPENAI_API_KEY: 'sk-test', ANTHROPIC_BASE_URL: base, OPENAI_BASE_URL: `${base}/v1`, GSTACK_AUTOPLAN_CHECKPOINT_DIR: '', GSTACK_STATE_ROOT: path.join(TMP, 'state') }), + }); + const timer = setTimeout(() => proc.kill('SIGKILL'), 120_000); + const [status, stdout, stderr] = await Promise.all([proc.exited, new Response(proc.stdout).text(), new Response(proc.stderr).text()]); + clearTimeout(timer); + return { status, stdout, stderr, out, state: JSON.parse(fs.readFileSync(path.join(out, 'run.json'), 'utf8')) }; +} + +describe('gstack-autoplan run --native-runner api --runner api', () => { + test('a complete result binds both voices in every phase and the run reaches its gate', async () => { + mode = 'ok'; + const r = await run('ok'); + expect(r.status, r.stderr).toBe(0); + expect(r.stdout).toContain(`GSTACK_RESULT: skill=autoplan status=gate_pending run=${r.out}`); + for (const phase of ['ceo', 'eng']) for (const voice of ['native', 'outside']) expect(r.state.phases[phase][voice]).toMatchObject({ status: 'completed', findings: 1 }); + }, 150_000); + + test('a response stopped at the output limit is named truncated and interrupts the run', async () => { + mode = 'cut'; + const r = await run('cut'); + expect(r.status).toBe(1); + expect(r.stderr).toMatch(/native voice unavailable: truncated: .*output limit/); + expect(r.state.status).toBe('interrupted'); + }, 150_000); +}); diff --git a/test/outside-runner-contract.test.ts b/test/outside-runner-contract.test.ts index 2c847e3ae..062f70595 100644 --- a/test/outside-runner-contract.test.ts +++ b/test/outside-runner-contract.test.ts @@ -240,3 +240,46 @@ describe('api runner (supplied-input review)', () => { expect(nokey.stderr).toMatch(/OPENAI_API_KEY is not set.*\(OUTSIDE_RUNNER_UNAVAILABLE\)$/m); }); }); + +describe('api runner output limits (GBRA-223 claim 6)', () => { + let server: ReturnType; + const bodies: Array<{ url: string; body: any }> = []; + beforeAll(() => { + server = Bun.serve({ + port: 0, + async fetch(req) { + const url = new URL(req.url); + const body = await req.json(); + bodies.push({ url: url.pathname, body }); + const cut = String(body.model).includes('cut'); + if (url.pathname === '/v1/messages') return Response.json({ model: body.model, stop_reason: cut ? 'max_tokens' : 'end_turn', content: [{ type: 'text', text: cut ? 'INPUT: ceo abc\n\nA long review cut off mid-sent' : REVIEW }], usage: { input_tokens: 10, output_tokens: body.max_tokens } }); + return Response.json({ model: body.model, choices: [{ finish_reason: cut ? 'length' : 'stop', message: { content: cut ? 'INPUT: ceo abc\n\nA long review cut off' : REVIEW } }], usage: { prompt_tokens: 10, completion_tokens: 50 } }); + }, + }); + }); + afterAll(() => server.stop(true)); + const anthropic = () => ({ ...cleanEnv(emptyBin), ANTHROPIC_API_KEY: 'ak-test', ANTHROPIC_BASE_URL: `http://127.0.0.1:${server.port}` }); + const openai = () => ({ ...cleanEnv(emptyBin), OPENAI_API_KEY: 'sk-test', OPENAI_BASE_URL: `http://127.0.0.1:${server.port}/v1` }); + const out = (name: string) => path.join(TMP, 'limits', name); + + test('anthropic asks for 16384 output tokens by default and --max-tokens sets it; openai sends max_completion_tokens only when set', async () => { + expect((await runAsync(['run', '--runner', 'api', '--prompt', promptFile, '--out', out('a1.md'), '--model', 'claude-opus-5-5'], anthropic())).status).toBe(0); + expect((await runAsync(['run', '--runner', 'api', '--prompt', promptFile, '--out', out('a2.md'), '--model', 'claude-opus-5-5', '--max-tokens', '4000'], anthropic())).status).toBe(0); + expect(bodies.filter(b => b.url === '/v1/messages').map(b => b.body.max_tokens)).toEqual([16384, 4000]); + expect((await runAsync(['run', '--runner', 'api', '--prompt', promptFile, '--out', out('o1.md'), '--model', 'gpt-6-astra'], openai())).status).toBe(0); + expect((await runAsync(['run', '--runner', 'api', '--prompt', promptFile, '--out', out('o2.md'), '--model', 'gpt-6-astra', '--max-tokens', '4000'], openai())).status).toBe(0); + expect(bodies.filter(b => b.url === '/v1/chat/completions').map(b => b.body.max_completion_tokens ?? null)).toEqual([null, 4000]); + }); + + test('a response stopped at the output limit is truncated (exit 1), never a completed or missing-markers review', async () => { + for (const [model, env, tokens] of [['claude-cut', anthropic(), '16384'], ['gpt-cut', openai(), '50']] as const) { + const file = out(`${model}.md`); + const r = await runAsync(['run', '--runner', 'api', '--prompt', promptFile, '--out', file, '--model', model], env); + expect(r.status).toBe(1); + expect(r.stderr).toMatch(/api unavailable \(truncated\): .*output limit/); + const meta = JSON.parse(fs.readFileSync(`${file}.meta.json`, 'utf8')); + expect(meta).toMatchObject({ status: 'unavailable', reason: 'truncated' }); + expect(meta.detail).toContain(tokens); + } + }); +});