From 800a9269fdcd600d80b8d59b02f5c3f1dd0b3c84 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Tue, 17 Nov 2026 19:28:47 +0800 Subject: [PATCH] gstack-runaway prices each wake and marks it cold or warm (GST-2770) (#4385) * gstack-runaway prices each wake and marks it cold or warm (GST-2770) Each wake read is priced in warm model calls, with a cold call (after the 5-minute prompt cache lapsed) at 19.4. The scan line reports cold and empty wakes' share of spend, and scan --wakes lists each wake. A message sent to another thread now counts as action. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Document gstack-runaway wake pricing in the /runaway-stopper reference Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * gstack-runaway charges a lapsed cache to the call after a long tool and ignores a null message target A cold wake no longer counts its first call twice, a call issued more than 5 minutes after the previous model call is cold whether or not an inbound item was recorded, and a report to the parent with an explicit null target stays a report. Review-Finding: p1/outside-voice#1 reader=outside-voice Review-Finding: p1/outside-voice#3 reader=outside-voice Review-Finding: p1/red-team#1 reader=red-team Review-Finding: p1/red-team#2 reader=red-team Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Describe gstack-runaway's cold_extra_spend and empty_spend as shares of spend in docs/skills.md Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- changelog.d/price-runaway-wakes.md | 1 + docs/skills.md | 3 + lib/runaway.ts | 115 +++++++++++++++++++++-------- runaway-stopper/SKILL.md | 5 +- runaway-stopper/SKILL.md.tmpl | 5 +- test/fixtures/context-budget.json | 2 +- test/runaway.test.ts | 60 ++++++++++++++- 7 files changed, 157 insertions(+), 34 deletions(-) create mode 100644 changelog.d/price-runaway-wakes.md diff --git a/changelog.d/price-runaway-wakes.md b/changelog.d/price-runaway-wakes.md new file mode 100644 index 000000000..d1df30a85 --- /dev/null +++ b/changelog.d/price-runaway-wakes.md @@ -0,0 +1 @@ +- **`gstack-runaway` now says what a thread's cold and empty wakes cost, not just how many there were (GST-2770).** Each wake it reads is marked cold when it started more than 5 minutes after the thread's last activity, so the prompt cache had lapsed, and is priced in warm model calls, with a cold call at 19.4 (measured on Opus 5.5: $2.42 against $0.125 per million counted tokens; a call after a tool that ran past 5 minutes is cold too). The RUNAWAY line adds `wakes`, `cold_wakes`, `empty_wakes`, `cold_extra_spend` and `empty_spend`, the empty-wake warning gives the empty wakes' share of spend, and `scan --wakes` prints one WAKE line per wake. A message sent to another thread now counts as action even when its state is "working", and more than 5 minutes idle inside a turn (not counting a tool's own run time) starts a new wake. On four coordinating threads over 4.7 days the meter's cold and empty shares came within 1.3 points of a hand count; before this, sends were counted as empty and the empty share read 4 to 12 points high. /runaway-stopper's fetch program now keeps each message's target and each call's run time. diff --git a/docs/skills.md b/docs/skills.md index dfc68690c..29d280878 100644 --- a/docs/skills.md +++ b/docs/skills.md @@ -1029,6 +1029,9 @@ sets the defaults that keep it from coming back. thread keeps its PR until it merges or closes and checks in with its Captain once past 1B, and no thread holding an open PR rotates), a subagent past 100M (stop), empty wakes past 25% of the last 40 turns (warm-wait check-ins are left out), and review rounds past 2 per PR. Captains are reported, never rotated. + The `RUNAWAY:` line also prices the wakes it read: `cold_wakes` and `empty_wakes` are their shares by count, + `cold_extra_spend` the share of spend that cold model calls add over warm ones (a wake that starts after the + 5-minute prompt cache lapsed, or a call after a tool that ran that long), and `empty_spend` the share the empty wakes cost. `scan --wakes` prints one `WAKE:` line per wake with its cost in warm model calls. - **Stop.** A live subagent past 100M is stopped with its reason. A standing thread is rotated at a clean point through a handoff file and a successor brief (`runaway-stopper/templates/`), then archived by its own Captain once none of its PRs is open (R91), through `gstack-archive-check thread`. A thread diff --git a/lib/runaway.ts b/lib/runaway.ts index ad4e3db76..3f8391bf4 100644 --- a/lib/runaway.ts +++ b/lib/runaway.ts @@ -9,7 +9,7 @@ */ import * as fs from 'node:fs'; import * as path from 'node:path'; -import { isKeepalive, type TranscriptItem } from './ship-calls'; +import { CACHE_TTL_S, isKeepalive, type TranscriptItem } from './ship-calls'; export const LIMITS = { ownRotate: 500e6, @@ -25,6 +25,8 @@ export const LIMITS = { warmWindowS: 30 * 60, maxRounds: 2, } as const; +/** A cold model call's price in warm calls: Opus 5.5 after the 5-minute prompt cache lapsed, $2.42 against $0.125 per million counted tokens (measured 2026-11-08). */ +export const COLD_CALL_PRICE = 19.4; export interface Task { label?: string; path?: string; title?: string; status?: string; cost?: { tokens?: number; credits?: number } } export interface ThreadDoc { @@ -32,12 +34,15 @@ export interface ThreadDoc { cost?: { tokens?: number; credits?: number }; tasks?: Task[]; tasks_total?: number; captain_thread?: unknown; crew?: unknown; events?: Item[]; standing?: boolean; pull_requests?: Array<{ number?: number; state?: string }>; pull_requests_total?: number; } -export interface Item extends TranscriptItem { input?: { text?: string } | string; state?: string } +export interface Item extends TranscriptItem { + input?: { text?: string; chars?: number } | string; state?: string; to?: unknown; text?: string; ms?: number; calls?: Array<{ tool?: string; durationMs?: number }>; +} +export interface Wake { start: string; idleS: number | null; cold: boolean; calls: number; coldCalls: number; empty: boolean; cost: number } export interface Flag { action: 'ROTATE' | 'STOP' | 'LOOK' | 'WARN' | 'REPORT' | 'CHECK_IN'; text: string } export interface Verdict { code: string; verdict: 'OK' | 'LOOK' | 'WARN' | 'STOP' | 'ROTATE' | 'REPORT' | 'CHECK_IN' | 'DEFER'; captain: boolean; own: number; ownBound: 'exact' | 'upper'; total: number; credits: number; ageDays: number; active: boolean; - empty: { turns: number; empty: number } | null; rounds: ReturnType; + empty: { turns: number; empty: number } | null; wakes: Wake[]; rounds: ReturnType; flags: Flag[]; defer: string[]; held: string[]; } @@ -66,41 +71,79 @@ function waitArgs(item: Item): { ops: string; seconds: number | null } | null { } catch { return null; } } +/** A message_agent send to another thread (it names `to`), which is action whatever its state; a report to the parent names none. A send's text is its target, which also covers an input cut short before `to`. */ +function addressed(item: Item): boolean { + if (item.to !== undefined) return !!item.to; + const input = typeof item.input === 'string' ? { text: item.input } : item.input; + return /[{,]\s*"to"\s*:\s*"/.test(input?.text ?? '') || (!!item.text && (input?.chars ?? 0) > (input?.text ?? '').length); +} + +const ranMs = (i: Item) => i.ms ?? (i.calls ?? []).reduce((s, c) => s + (c.durationMs ?? 0), 0); + /** - * Turns from transcript items: an inbound message starts one; end_turn, wait or a final/needs_you report - * ends one. A turn is empty when it made no call beyond status reads, waits and "working" reports. A warm-wait - * check-in (crew/docs/captain.md, Warm waits; GST-1597) is not a turn: one whose only call re-issues a wait on the - * operations the previous turn waited on, or a 240-second timer naming those operations or none, within 30 minutes - * of the wait that started the run. Any turn that closes on such a wait keeps the run's start, so a status read - * between check-ins counts but never restarts the 30 minutes. + * Wakes (turns) from transcript items: an inbound message starts one; end_turn, wait or a final/needs_you report + * ends one, and so does more than CACHE_TTL_S idle before a call (its gap less its own run time): an unrecorded + * platform event woke the thread. A wake is empty when it made no call beyond status reads, waits and "working" + * reports to the parent; a send to another thread is action. A warm-wait check-in (crew/docs/captain.md, Warm waits; + * GST-1597) is not a wake: one whose only call re-issues a wait on the operations the previous wake waited on, or a + * 240-second timer naming those operations or none, within 30 minutes of the wait that started the run. Any wake that + * closes on such a wait keeps the run's start, so a status read between check-ins counts but never restarts the 30 minutes. + * Each wake is priced in warm model calls, one per tool call, issued at its item's time less its own run time. A wake is + * cold when it started more than CACHE_TTL_S after the thread's previous model call, so the prompt cache had lapsed; its + * first call then costs COLD_CALL_PRICE once, and so does any later call issued more than CACHE_TTL_S after the model + * call before it (the call after a long tool). The first wake read has no previous model call and is priced warm. + * The price ratio and the 5-minute cache are Opus 5.5's; other models differ. */ -export function emptyWakes(items: readonly Item[]): { turns: number; empty: number } { - const sorted = items.filter(i => i.createdAt && i.source !== 'assistant').sort((a, b) => a.createdAt!.localeCompare(b.createdAt!)); - const turns: boolean[] = []; - let acted = false, open = false; +export function wakes(items: readonly Item[]): Wake[] { + const sorted = items.filter(i => Number.isFinite(Date.parse(i.createdAt ?? ''))).sort((a, b) => a.createdAt!.localeCompare(b.createdAt!)); + const out: Wake[] = []; + let acted = false, coldCalls = 0; let calls: Item[] = []; + let start: { at: string; idleS: number | null } | null = null; + let prevAt: number | null = null, prevModel: number | null = null; let last: { ops: string; since: number } | null = null; const close = () => { - if (!open) return; + if (!calls.length || !start) return; const closing = calls.at(-1); const args = closing?.tool === 'wait' ? waitArgs(closing) : null; const at = Date.parse(closing?.createdAt ?? ''); const same = !!args && !!last && (args.ops ? args.ops === last.ops : args.seconds === LIMITS.warmWaitS); - if (!(same && calls.length === 1 && at - last!.since <= LIMITS.warmWindowS * 1000)) turns.push(!acted); + if (!(same && calls.length === 1 && at - last!.since <= LIMITS.warmWindowS * 1000)) { + const cold = start.idleS !== null && start.idleS > CACHE_TTL_S; + const n = coldCalls + (cold ? 1 : 0); + out.push({ start: start.at, idleS: start.idleS, cold, calls: calls.length, coldCalls: n, empty: !acted, cost: calls.length + n * (COLD_CALL_PRICE - 1) }); + } last = args ? (same ? last : { ops: args.ops, since: at }) : null; - acted = false; open = false; calls = []; + acted = false; coldCalls = 0; calls = []; start = null; }; for (const i of sorted) { - if (i.source !== 'tool') { close(); continue; } - open = true; + const t = Date.parse(i.createdAt!); + if (i.source === 'assistant') { prevAt = prevModel = t; continue; } + if (i.source !== 'tool') { close(); start ??= { at: i.createdAt!, idleS: prevModel === null ? null : (t - prevModel) / 1000 }; prevAt = t; continue; } + const issued = Math.max(t - ranMs(i), prevAt ?? -Infinity); + if (calls.length && prevAt !== null && (issued - prevAt) / 1000 > CACHE_TTL_S) close(); + const sinceModelS = prevModel === null ? null : (issued - prevModel) / 1000; + start ??= { at: i.createdAt!, idleS: sinceModelS }; + const startCold = start.idleS !== null && start.idleS > CACHE_TTL_S; + if (sinceModelS !== null && sinceModelS > CACHE_TTL_S && (calls.length || !startCold)) coldCalls++; + prevAt = t; prevModel = issued; calls.push(i); const state = i.tool === 'message_agent' || i.tool === 'message_user' ? reportState(i) : undefined; - if (i.tool !== 'end_turn' && state !== 'working' && !isKeepalive(i)) acted = true; + const report = state === 'working' && !(i.tool === 'message_agent' && addressed(i)); + if (i.tool !== 'end_turn' && !report && !isKeepalive(i)) acted = true; if (i.tool === 'end_turn' || i.tool === 'wait' || state === 'final' || state === 'needs_you') close(); } close(); - const window = turns.slice(-LIMITS.emptyWindow); - return { turns: window.length, empty: window.filter(Boolean).length }; + return out; +} + +/** Shares of all wakes read: cold-started and empty wakes by count, and by price, the cold calls' extra over warm and the empty wakes' whole cost. */ +export function wakeCost(ws: readonly Wake[]): { wakes: number; cold: number; empty: number; coldExtra: number; emptySpend: number } { + const total = ws.reduce((s, w) => s + w.cost, 0) || 1; + const n = ws.length || 1; + const empty = ws.filter(w => w.empty); + return { wakes: ws.length, cold: ws.filter(w => w.cold).length / n, empty: empty.length / n, + coldExtra: ws.reduce((s, w) => s + w.coldCalls, 0) * (COLD_CALL_PRICE - 1) / total, emptySpend: empty.reduce((s, w) => s + w.cost, 0) / total }; } /** @@ -175,9 +218,12 @@ export function assess(doc: ThreadDoc, opts: { items?: Item[]; subagents?: { tas const live = !TERMINAL.has(t.status ?? ''); flags.push({ action: live ? 'STOP' : 'LOOK', text: 'subagent ' + pathOf(t) + ' ' + quote(t.title) + ' ' + fmtTokens(tok(t)) + ' > ' + fmtTokens(LIMITS.subagentStop) + ' (' + (t.status ?? 'unknown') + ')' }); } - const empty = opts.items?.length ? emptyWakes(opts.items) : null; + const ws = opts.items?.length ? wakes(opts.items) : []; + const recent = ws.slice(-LIMITS.emptyWindow); + const empty = ws.length ? { turns: recent.length, empty: recent.filter(w => w.empty).length } : null; if (empty && empty.turns >= LIMITS.emptyMinTurns && empty.empty / empty.turns > LIMITS.emptyRate) - flags.push({ action: 'WARN', text: 'empty_wake ' + Math.round(100 * empty.empty / empty.turns) + '% of ' + empty.turns + ' turns > ' + LIMITS.emptyRate * 100 + '%' }); + flags.push({ action: 'WARN', text: 'empty_wake ' + Math.round(100 * empty.empty / empty.turns) + '% of ' + empty.turns + ' turns > ' + LIMITS.emptyRate * 100 + '%' + + ', ' + Math.round(100 * wakeCost(recent).emptySpend) + '% of their spend' }); const rounds = reviewRounds(listed); if (rounds && rounds.rounds > LIMITS.maxRounds) flags.push({ action: 'WARN', text: 'review_rounds ' + rounds.rounds + ' on #' + rounds.pr + ' > ' + LIMITS.maxRounds }); @@ -188,15 +234,21 @@ export function assess(doc: ThreadDoc, opts: { items?: Item[]; subagents?: { tas const worst = (['ROTATE', 'REPORT', 'STOP', 'WARN', 'CHECK_IN', 'LOOK'] as const).find(a => flags.some(f => f.action === a)); const holds = flags.some(f => f.action === 'ROTATE') || stopsShipping; const verdict: Verdict['verdict'] = holds && defer.length ? 'DEFER' : held.length && (!worst || worst === 'LOOK') ? 'DEFER' : (worst ?? 'OK'); - return { code, verdict, captain, own, ownBound, total, credits: doc.cost?.credits ?? 0, ageDays, active, empty, rounds, flags, defer, held }; + return { code, verdict, captain, own, ownBound, total, credits: doc.cost?.credits ?? 0, ageDays, active, empty, wakes: ws, rounds, flags, defer, held }; } -export function render(v: Verdict): string[] { +const pct = (x: number) => Math.round(100 * x) + '%'; + +export function render(v: Verdict, opts: { wakes?: boolean } = {}): string[] { const e = v.empty ? Math.round(100 * v.empty.empty / Math.max(1, v.empty.turns)) + '%/' + v.empty.turns : 'unmeasured'; + const c = wakeCost(v.wakes); const out = ['RUNAWAY: thread=' + v.code + ' verdict=' + v.verdict + (v.captain ? ' captain=yes' : '') + ' own=' + fmtTokens(v.own) + '(' + v.ownBound + ')' + ' total=' + fmtTokens(v.total) + ' credits=' + Math.round(v.credits) + ' age_days=' + v.ageDays.toFixed(1) + ' active=' + (v.active ? 'yes' : 'no') + ' empty_wake=' + e + ' max_rounds=' + (v.rounds?.pr ? v.rounds.rounds + '(#' + v.rounds.pr + ')' : '0') - + ' review_subagents=' + (v.rounds?.reviewers ?? 0) + '(unattributed ' + (v.rounds?.unattributed ?? 0) + ')']; + + ' review_subagents=' + (v.rounds?.reviewers ?? 0) + '(unattributed ' + (v.rounds?.unattributed ?? 0) + ')' + + ' wakes=' + (c.wakes ? c.wakes + ' cold_wakes=' + pct(c.cold) + ' empty_wakes=' + pct(c.empty) + ' cold_extra_spend=' + pct(c.coldExtra) + ' empty_spend=' + pct(c.emptySpend) : 'unmeasured')]; + if (opts.wakes) for (const w of v.wakes) out.push('WAKE: thread=' + v.code + ' start=' + w.start + ' idle_s=' + (w.idleS === null ? 'unknown' : Math.round(w.idleS)) + + ' cache=' + (w.cold ? 'cold' : 'warm') + ' calls=' + w.calls + ' cold_calls=' + w.coldCalls + ' empty=' + (w.empty ? 'yes' : 'no') + ' cost=' + w.cost.toFixed(1)); for (const f of v.flags) out.push('FLAG: ' + f.action + ' ' + f.text); if (v.verdict === 'DEFER') for (const d of v.defer) out.push('DEFER: ' + d); for (const h of v.held) out.push('DEFER: ' + h); @@ -230,7 +282,7 @@ function itemsOf(x: unknown): Item[] { const USAGE = [ 'gstack-runaway — find runaway Capy threads and subagents (read-only; /runaway-stopper)', - ' scan --thread [--events ] [--subagents ] [--now ]', + ' scan --thread [--events ] [--subagents ] [--now ] [--wakes]', ' scan --dir [--now ] reads every .thread.json with its .events.json and .subagents.json', ' and /crew.json (the thread.list codes, {"items": [...]}): an empty crew list is RUNAWAY_INPUT, never a clean SWEEP.', 'Lines: RUNAWAY (one per thread, largest own tokens first), FLAG ROTATE|STOP|LOOK|WARN|REPORT|CHECK_IN, DEFER, SWEEP (--dir).', @@ -240,12 +292,17 @@ const USAGE = [ 'pull_requests are missing or partial (DEFER, R91); any other thread is a crew thread that keeps its PR and gets one', 'CHECK_IN past 1B own; subagent > 100M stop (LOOK when it already ended); empty wakes > 25% of the last 40 turns', '(20+; warm-wait check-ins left out); review rounds > 2 per PR; Captains REPORT, never rotate.', + 'Each wake read is priced in warm model calls (one per tool call); it is cold when it starts more than ' + CACHE_TTL_S + ' s after the', + ' thread\'s previous item (the prompt cache lapsed), and its first call then costs ' + COLD_CALL_PRICE + ', as does a call after a tool that ran', + ' that long. cold_extra_spend is the cold calls\' extra over warm, empty_spend the empty wakes\' cost, both as shares of the', + ' wakes read; --wakes prints each wake with its cost in warm calls.', 'A thread file that is not valid JSON or lacks cost.tokens or created_at prints RUNAWAY_INPUT and is skipped.', 'Exit 0 after a clean scan, 1 when any input was unreadable, 2 on a usage error.', ]; export function runawayMain(argv: string[]): number { - const [cmd, ...rest] = argv; + const [cmd, ...args] = argv; + const rest = args.filter(a => a !== '--wakes'); if (cmd === '--help' || cmd === '-h') { console.log(USAGE.join('\n')); return 0; } const fail = (msg: string) => { console.error('RUNAWAY_USAGE: ' + msg + '\n' + USAGE.join('\n')); return 2; }; if (cmd !== 'scan') return fail(cmd ? 'unknown command ' + JSON.stringify(cmd) : 'no command'); @@ -302,7 +359,7 @@ export function runawayMain(argv: string[]): number { } } verdicts.sort((a, b) => b.own - a.own); - for (const v of verdicts) console.log(render(v).join('\n')); + for (const v of verdicts) console.log(render(v, { wakes: args.includes('--wakes') }).join('\n')); if (dir) { const n = (k: Verdict['verdict']) => verdicts.filter(v => v.verdict === k).length; console.log('SWEEP: threads=' + verdicts.length + ' rotate=' + n('ROTATE') + ' stop=' + n('STOP') + ' defer=' + n('DEFER') diff --git a/runaway-stopper/SKILL.md b/runaway-stopper/SKILL.md index 182513520..13b094500 100644 --- a/runaway-stopper/SKILL.md +++ b/runaway-stopper/SKILL.md @@ -179,6 +179,8 @@ const fetch = async code => { doc ??= rest; events.push(...transcript.items.map(i => ({ source: i.source, tool: i.tool, createdAt: i.createdAt, calls: i.calls?.map(c => ({ tool: c.tool })), state: String(i.input?.text ?? "").match(/"state\\?"\s*:\s*\\?"(\w+)/)?.[1], + ms: i.calls?.reduce((s, c) => s + (c.durationMs ?? 0), 0), + ...(i.tool === "message_agent" ? { to: /[{,]\s*"to"\s*:\s*"/.test(i.input?.text ?? "") || (!!i.text && i.input?.chars > (i.input?.text ?? "").length) } : {}), ...(i.tool === "wait" ? { input: i.input } : {}) }))); if (!(before = transcript.beforeCursor)) break; } @@ -199,7 +201,8 @@ automation's standing thread) rotates past 500M own tokens (total minus subagent waiting, or a turn in the last 24 hours), warn at 400M. A crew thread keeps its PR until it merges or closes, never rotated mid-PR on a token count; past 1B own tokens it checks in with its Captain once. Open or unknown PRs block rotation (R91). A subagent past 100M: stop and look. Empty wakes past 25% of the -last 40 turns (warm-wait check-ins left out). Review rounds past 2 per PR. Act on each line: +last 40 turns (warm-wait check-ins left out). Review rounds past 2 per PR. `cold_extra_spend` and `empty_spend` are the shares of spend that +cold starts (after the 5-minute cache lapsed) and empty wakes cost; `scan --wakes` prices each wake. Act on each line: | Line | Do | |---|---| diff --git a/runaway-stopper/SKILL.md.tmpl b/runaway-stopper/SKILL.md.tmpl index 40ce382ab..e85ca270f 100644 --- a/runaway-stopper/SKILL.md.tmpl +++ b/runaway-stopper/SKILL.md.tmpl @@ -50,6 +50,8 @@ const fetch = async code => { doc ??= rest; events.push(...transcript.items.map(i => ({ source: i.source, tool: i.tool, createdAt: i.createdAt, calls: i.calls?.map(c => ({ tool: c.tool })), state: String(i.input?.text ?? "").match(/"state\\?"\s*:\s*\\?"(\w+)/)?.[1], + ms: i.calls?.reduce((s, c) => s + (c.durationMs ?? 0), 0), + ...(i.tool === "message_agent" ? { to: /[{,]\s*"to"\s*:\s*"/.test(i.input?.text ?? "") || (!!i.text && i.input?.chars > (i.input?.text ?? "").length) } : {}), ...(i.tool === "wait" ? { input: i.input } : {}) }))); if (!(before = transcript.beforeCursor)) break; } @@ -70,7 +72,8 @@ automation's standing thread) rotates past 500M own tokens (total minus subagent waiting, or a turn in the last 24 hours), warn at 400M. A crew thread keeps its PR until it merges or closes, never rotated mid-PR on a token count; past 1B own tokens it checks in with its Captain once. Open or unknown PRs block rotation (R91). A subagent past 100M: stop and look. Empty wakes past 25% of the -last 40 turns (warm-wait check-ins left out). Review rounds past 2 per PR. Act on each line: +last 40 turns (warm-wait check-ins left out). Review rounds past 2 per PR. `cold_extra_spend` and `empty_spend` are the shares of spend that +cold starts (after the 5-minute cache lapsed) and empty wakes cost; `scan --wakes` prices each wake. Act on each line: | Line | Do | |---|---| diff --git a/test/fixtures/context-budget.json b/test/fixtures/context-budget.json index b8f6d0ef3..0ffb9e8ae 100644 --- a/test/fixtures/context-budget.json +++ b/test/fixtures/context-budget.json @@ -56,7 +56,7 @@ "qa-only": 5726, "retro": 13484, "review": 15417, - "runaway-stopper": 3987, + "runaway-stopper": 4489, "scrape": 7312, "setup-browser-cookies": 3507, "setup-deploy": 5738, diff --git a/test/runaway.test.ts b/test/runaway.test.ts index c76a7d846..429081c20 100644 --- a/test/runaway.test.ts +++ b/test/runaway.test.ts @@ -116,6 +116,43 @@ describe('gstack-runaway scan', () => { expect(r.out).toContain('FLAG: WARN empty_wake 65% of 31 turns > 25%'); }); + // Value: protects=each wake carries its price and cache state, so the report says what cold starts and empty wakes cost (GST-2770); + // fails_when=a wake after more than 300 s of quiet is priced warm, a send counts as empty, or silence inside a turn hides a cold + // start; seam=wakes() through gstack-runaway scan --wakes + test('wakes are priced in warm calls: a start after 300 s of quiet pays a cold first call, and so does a call after silence mid-turn', () => { + const t = (s: number) => new Date(Date.parse('2026-11-06T12:00:00Z') + s * 1000).toISOString(); + const items = [ + { source: 'user', createdAt: t(0) }, { source: 'tool', tool: 'execute', calls: [{ tool: 'bash' }, { tool: 'edit' }], createdAt: t(5) }, { source: 'tool', tool: 'end_turn', createdAt: t(10) }, + { source: 'user', createdAt: t(610) }, { source: 'tool', tool: 'message_agent', state: 'working', to: true, createdAt: t(615) }, { source: 'tool', tool: 'end_turn', createdAt: t(620) }, + { source: 'user', createdAt: t(680) }, { source: 'tool', tool: 'message_agent', state: 'working', createdAt: t(685) }, { source: 'tool', tool: 'end_turn', createdAt: t(690) }, + { source: 'tool', tool: 'execute', calls: [{ tool: 'bash' }], createdAt: t(1100) }, { source: 'tool', tool: 'end_turn', createdAt: t(1105) }, + { source: 'user', createdAt: t(1200) }, { source: 'tool', tool: 'execute', calls: [{ tool: 'edit' }], createdAt: t(1205) }, { source: 'tool', tool: 'end_turn', createdAt: t(1600) }, + ]; + const doc = { code: 'GSTA-996', status: 'idle', created_at: '2026-11-06T02:00:00Z', cost: { tokens: 1e6 }, tasks: [], events: items }; + const f = write('priced.thread.json', doc); + const out = run('scan', '--thread', f, '--now', NOW, '--wakes').out; + expect(out).toContain('wakes=6 cold_wakes=50% empty_wakes=33% cold_extra_spend=85% empty_spend=33%'); + expect(out).toContain('WAKE: thread=GSTA-996 start=' + t(0) + ' idle_s=unknown cache=warm calls=2 cold_calls=0 empty=no cost=2.0'); + expect(out).toContain('WAKE: thread=GSTA-996 start=' + t(610) + ' idle_s=600 cache=cold calls=2 cold_calls=1 empty=no cost=20.4'); + expect(out).toContain('WAKE: thread=GSTA-996 start=' + t(680) + ' idle_s=60 cache=warm calls=2 cold_calls=0 empty=yes cost=2.0'); + expect(out).toContain('WAKE: thread=GSTA-996 start=' + t(1100) + ' idle_s=410 cache=cold calls=2 cold_calls=1 empty=no cost=20.4'); + expect(out).toContain('WAKE: thread=GSTA-996 start=' + t(1600) + ' idle_s=395 cache=cold calls=1 cold_calls=1 empty=yes cost=19.4'); + expect(run('scan', '--thread', f, '--now', NOW).out).not.toContain('WAKE:'); + const long = [{ source: 'user', createdAt: t(0) }, { source: 'tool', tool: 'execute', calls: [{ tool: 'bash', durationMs: 5000 }], createdAt: t(10) }, + { source: 'tool', tool: 'execute', calls: [{ tool: 'subagent_list', durationMs: 590_000 }], createdAt: t(610) }, { source: 'tool', tool: 'end_turn', createdAt: t(620) }]; + const ran = run('scan', '--thread', write('long.thread.json', { ...doc, events: [] }), '--events', write('long.events.json', { transcript: { items: long } }), '--now', NOW, '--wakes').out; + expect(ran).toContain('wakes=1 cold_wakes=0% empty_wakes=0% cold_extra_spend=86% empty_spend=0%'); + expect(ran).toContain('idle_s=unknown cache=warm calls=3 cold_calls=1 empty=no cost=21.4'); + const queued = [{ source: 'tool', tool: 'end_turn', createdAt: t(0) }, + { source: 'user', createdAt: t(600) }, { source: 'tool', tool: 'execute', calls: [{ tool: 'bash' }], createdAt: t(1200) }, { source: 'tool', tool: 'end_turn', createdAt: t(1206) }, + { source: 'user', createdAt: t(1300) }, { source: 'tool', tool: 'execute', calls: [{ tool: 'bash' }], createdAt: t(1900) }, { source: 'tool', tool: 'end_turn', createdAt: t(1905) }, + { source: 'tool', tool: 'execute', calls: [{ tool: 'bash', durationMs: 600_000 }], createdAt: t(2505) }, { source: 'tool', tool: 'end_turn', createdAt: t(2506) }]; + const q = run('scan', '--thread', write('queued.thread.json', { ...doc, events: [] }), '--events', write('queued.events.json', { transcript: { items: queued } }), '--now', NOW, '--wakes').out; + expect(q).toContain('start=' + t(600) + ' idle_s=600 cache=cold calls=2 cold_calls=1 empty=no cost=20.4'); + expect(q).toContain('start=' + t(1300) + ' idle_s=94 cache=warm calls=2 cold_calls=1 empty=no cost=20.4'); + expect(q).toContain('start=' + t(2505) + ' idle_s=0 cache=warm calls=2 cold_calls=1 empty=no cost=20.4'); + }); + const at = (s: number) => new Date(Date.parse('2026-11-06T12:00:00Z') + s * 1000).toISOString(); const wait = (s: number, input: Record) => ({ source: 'tool', tool: 'wait', input: { text: JSON.stringify({ description: 'w', ...input }) }, createdAt: at(s) }); const work = (s: number) => ({ source: 'tool', tool: 'execute', calls: [{ tool: 'bash' }, { tool: 'edit' }], createdAt: at(s) }); @@ -127,7 +164,7 @@ describe('gstack-runaway scan', () => { // Value: protects=warm-wait check-ins (crew/docs/captain.md Warm waits, #4060) are not empty wakes: a turn whose only call re-issues the // wait on the same operation, or a 240 s timer, inside 30 minutes of the wait that began the run is left out; a re-wait past 30 // minutes, a different wait, or a turn that also reads status still counts; fails_when=/runaway-stopper flags a lane that follows the - // rule (GST-1597); seam=emptyWakes; no_claim=whether the lane should have warm-waited at all + // rule (GST-1597); seam=wakes; no_claim=whether the lane should have warm-waited at all test('warm-wait check-ins inside 30 minutes are not empty wakes; past the window they count again', () => { const lane: unknown[] = []; for (let k = 0; k < 20; k++) { lane.push(work(k * 2000)); lane.push(wait(k * 2000 + 1, { operation_ids: ['op' + k], duration_seconds: 240 })); for (let c = 1; c <= 7; c++) lane.push(wait(k * 2000 + 1 + c * 240, c % 2 ? { duration_seconds: 240 } : { operation_ids: ['op' + k] })); } @@ -149,7 +186,7 @@ describe('gstack-runaway scan', () => { // Value: protects=a 240 s timer chains only on the run's operations or none, and a counted turn that re-waits on the run's operation // keeps the run's start, so polling one operation for hours with a status read every 28 minutes still counts; fails_when=a timer // naming other operations is left out, or a status-read turn restarts the 30-minute window (GST-1597 /ship findings 2 and 3); - // seam=emptyWakes; no_claim=how often a lane should read status + // seam=wakes; no_claim=how often a lane should read status test('a 240 s timer on other operations counts; a status-read turn does not restart the warm window', () => { const timer = (ops: (k: number) => Record) => { const items: unknown[] = []; @@ -348,6 +385,25 @@ describe('/runaway-stopper skill', () => { expect(out).toContain('empty_wake=0%/20'); expect(out).not.toContain('FLAG: WARN empty_wake'); }); + // Value: protects=a Captain's sends to its crew are action, so its empty-wake share is not inflated (four Captains measured 46%, 46%, + // 53% and 35% empty with sends counted as reports, against a hand count of 34%, 36%, 43% and 31%); fails_when=the projection drops + // `to`, loses it when thread.get cuts a long input short, or the scan treats an addressed "working" send as a report; + // seam=the template's projection, then gstack-runaway scan + test('the fetch program keeps whether a message_agent names a target: a send is action, a working report to the parent is not', () => { + const src = skill.match(/transcript\.items\.map\((i => \(\{[\s\S]*?\}\))\)\);/)?.[1]; + const project = new Function('return ' + src)() as (i: unknown) => unknown; + const at = (s: number) => new Date(Date.parse('2026-11-06T12:00:00Z') + s * 1000).toISOString(); + const page = (input: Record) => Array.from({ length: 20 }, (_, k) => [ + { id: 'u' + k, source: 'user', text: 'report', createdAt: at(k * 100) }, + { id: 'm' + k, source: 'tool', tool: 'message_agent', text: 'X', createdAt: at(k * 100 + 5), input: { text: JSON.stringify(input), chars: JSON.stringify(input).length } }, + { id: 'e' + k, source: 'tool', tool: 'end_turn', text: 'done', createdAt: at(k * 100 + 9) }]).flat().map(project); + const doc = { code: 'GSTA-997', status: 'idle', created_at: '2026-11-06T02:00:00Z', cost: { tokens: 1e6 }, tasks: [] }; + expect(run('scan', '--thread', write('sends.thread.json', { ...doc, events: page({ to: 'GSTA-12', state: 'working', text: 'go' }) }), '--now', NOW).out).toContain('empty_wake=0%/20'); + expect(run('scan', '--thread', write('notes.thread.json', { ...doc, events: page({ state: 'working', text: 'note' }) }), '--now', NOW).out).toContain('empty_wake=100%/20'); + expect(run('scan', '--thread', write('nulls.thread.json', { ...doc, events: page({ state: 'working', text: 'note', to: null }) }), '--now', NOW).out).toContain('empty_wake=100%/20'); + const cut = (project({ id: 'm', source: 'tool', tool: 'message_agent', text: 'GSTA-12', createdAt: at(0), input: { text: '{"state":"working","text":"a long brief', chars: 9000 } }) as { to?: boolean }).to; + expect(cut).toBe(true); + }); test('the fetch program marks standing threads, and only a standing thread with no open PR is rotated', () => { expect(skill).toContain('content: JSON.stringify({ ...doc, events, standing: standing.includes(doc.code) })'); expect(skill).toContain('| `FLAG: ROTATE` | A standing thread: Step 3. |');