From ff65fb0c482fa2e8a87da0639ac8d5e41d81dc6c Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Fri, 6 Nov 2026 05:40:49 -0800 Subject: [PATCH] runaway: warm-wait check-ins are not empty wakes (GST-1597) (#4069) * runaway: warm-wait check-ins are not empty wakes (GST-1597) lib/runaway.ts counted each 240-second warm-wait check-in (docs/captain.md, #4060) as an empty wake, so /runaway-stopper could flag a lane that follows the rule. A turn whose only call re-issues the wait on the same operation, or a 240-second timer, within 30 minutes of the wait that started the run is now left out of the rate. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: pre-landing review fixes Review-Finding: p1/specialist-batched#1 reader=red-team Review-Finding: p1/specialist-batched#2 reader=testing Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs: note the warm-wait exclusion in the /runaway-stopper summary; rewrap captain.md (GST-1597 docs audit) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * runaway: keep wait input in the skill collector; 240 s timers chain only on the run's operations; status reads keep the warm window (GST-1597) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- changelog.d/gst-1597-runaway-warm-wait.md | 1 + docs/captain.md | 11 ++-- docs/skills.md | 2 +- lib/runaway.ts | 33 +++++++++- runaway-stopper/SKILL.md | 5 +- runaway-stopper/SKILL.md.tmpl | 5 +- test/runaway.test.ts | 75 +++++++++++++++++++++++ 7 files changed, 120 insertions(+), 12 deletions(-) create mode 100644 changelog.d/gst-1597-runaway-warm-wait.md diff --git a/changelog.d/gst-1597-runaway-warm-wait.md b/changelog.d/gst-1597-runaway-warm-wait.md new file mode 100644 index 000000000..4e12dd0f3 --- /dev/null +++ b/changelog.d/gst-1597-runaway-warm-wait.md @@ -0,0 +1 @@ +- **`/runaway-stopper` no longer flags lanes that follow the warm-wait rule.** `gstack-runaway scan` counted each 240-second warm-wait check-in as an empty wake, so a compliant lane waiting on CI could cross the 25% empty-wake warning. A turn whose only call re-issues the wait on the same operation, or a 240-second timer on that operation or none, within 30 minutes of the wait that started the run is now left out of the rate, including when `/runaway-stopper` fetches the thread itself. A re-wait past 30 minutes, a wait on other operations, or a turn that also reads status still counts, and a status read does not restart the 30 minutes. (GST-1597) diff --git a/docs/captain.md b/docs/captain.md index 2c60a8aab..4bf03e654 100644 --- a/docs/captain.md +++ b/docs/captain.md @@ -152,10 +152,13 @@ cheap: operation with `duration_seconds: 240`, and on a timer wake that brings nothing new it re-issues the same wait silently, with no message. Once a warm wait has run about 30 minutes, or past the end the lane expected, the lane stops re-issuing the timer and - sleeps on the operation or pull request watch alone. A wait expected to run longer sleeps - on its operations alone. For short lane waits this overrides "waits go on operations, - never on polling wakes" ([unattended.md](unattended.md#report-length)). The controller - never warm-waits, and a lane still sleeps while /ship runs + sleeps on the operation or pull request watch alone. `gstack-runaway scan` leaves these + check-ins out of its empty-wake rate: a turn whose only call re-waits on the same + operation, or a 240-second timer on it or on nothing, within 30 minutes of the first wait. + A status read in between counts and does not restart the 30 minutes. A wait expected to run + longer sleeps on its operations alone. For short lane waits this overrides "waits go on + operations, never on polling wakes" ([unattended.md](unattended.md#report-length)). The + controller never warm-waits, and a lane still sleeps while /ship runs ([the lane's /ship sequence](#the-lanes-ship-sequence-before-ready)). - **Batched reads.** Independent reads (files, greps, pull request and thread state) go in one parallel program, not one call each. Each call returns 4k characters or less; long diff --git a/docs/skills.md b/docs/skills.md index e46831b02..11698b319 100644 --- a/docs/skills.md +++ b/docs/skills.md @@ -1020,7 +1020,7 @@ sets the defaults that keep it from coming back. - **Detect.** One program fetches each crew thread (`tools.thread.get`, with its transcript) to files, then `gstack-runaway scan --dir ` prints a `RUNAWAY:` line per thread and a `FLAG:` line per limit crossed: own tokens past 500M or active past 3 days (rotate), a subagent past 100M (stop), empty wakes past 25% of the - last 40 turns, and review rounds past 2 per PR. Captains are reported, never rotated. + last 40 turns (warm-wait check-ins are left out), and review rounds past 2 per PR. Captains are reported, never rotated. - **Stop.** A live subagent past 100M is stopped with its reason. A thread is rotated at a clean point through a handoff file and a successor brief (`runaway-stopper/templates/`), then archived by its own Captain. A thread mid-/ship or mid-merge gets `DEFER:` and nothing happens until that work finishes. diff --git a/lib/runaway.ts b/lib/runaway.ts index 116397009..52e3c9530 100644 --- a/lib/runaway.ts +++ b/lib/runaway.ts @@ -18,6 +18,8 @@ export const LIMITS = { emptyRate: 0.25, emptyWindow: 40, emptyMinTurns: 20, + warmWaitS: 240, + warmWindowS: 30 * 60, maxRounds: 2, } as const; @@ -51,18 +53,43 @@ function reportState(item: Item): string | undefined { return raw?.match(/\\?"state\\?"\s*:\s*\\?"(final|needs_you|working)/)?.[1]; } +/** A wait call's operations and timer, from its recorded input; null when the input is unreadable. */ +function waitArgs(item: Item): { ops: string; seconds: number | null } | null { + const raw = typeof item.input === 'string' ? item.input : item.input?.text; + try { + const v = JSON.parse(raw ?? '') as { operation_ids?: unknown; duration_seconds?: unknown }; + return { ops: Array.isArray(v.operation_ids) ? [...v.operation_ids].map(String).sort().join(',') : '', seconds: typeof v.duration_seconds === 'number' ? v.duration_seconds : null }; + } catch { return null; } +} + /** * Turns from transcript items: an inbound message starts one; end_turn, wait or a final/needs_you report - * ends one. A turn is empty when it made no call beyond status reads, waits and "working" reports. + * ends one. A turn is empty when it made no call beyond status reads, waits and "working" reports. A warm-wait + * check-in (docs/captain.md, Warm waits; GST-1597) is not a turn: one whose only call re-issues a wait on the + * operations the previous turn waited on, or a 240-second timer naming those operations or none, within 30 minutes + * of the wait that started the run. Any turn that closes on such a wait keeps the run's start, so a status read + * between check-ins counts but never restarts the 30 minutes. */ export function emptyWakes(items: readonly Item[]): { turns: number; empty: number } { const sorted = items.filter(i => i.createdAt && i.source !== 'assistant').sort((a, b) => a.createdAt!.localeCompare(b.createdAt!)); const turns: boolean[] = []; let acted = false, open = false; - const close = () => { if (open) turns.push(!acted); acted = false; open = false; }; + let calls: Item[] = []; + let last: { ops: string; since: number } | null = null; + const close = () => { + if (!open) return; + const closing = calls.at(-1); + const args = closing?.tool === 'wait' ? waitArgs(closing) : null; + const at = Date.parse(closing?.createdAt ?? ''); + const same = !!args && !!last && (args.ops ? args.ops === last.ops : args.seconds === LIMITS.warmWaitS); + if (!(same && calls.length === 1 && at - last!.since <= LIMITS.warmWindowS * 1000)) turns.push(!acted); + last = args ? (same ? last : { ops: args.ops, since: at }) : null; + acted = false; open = false; calls = []; + }; for (const i of sorted) { if (i.source !== 'tool') { close(); continue; } open = true; + calls.push(i); const state = i.tool === 'message_agent' || i.tool === 'message_user' ? reportState(i) : undefined; if (i.tool !== 'end_turn' && state !== 'working' && !isKeepalive(i)) acted = true; if (i.tool === 'end_turn' || i.tool === 'wait' || state === 'final' || state === 'needs_you') close(); @@ -184,7 +211,7 @@ const USAGE = [ 'Lines: RUNAWAY (one per thread, largest own tokens first), FLAG ROTATE|STOP|LOOK|WARN|REPORT, DEFER, SWEEP (--dir).', 'Review rounds count only subagent titles that name the PR (#N); the rest show as unattributed.', 'Limits (R70): own > 500M or active > 3d rotate (warn 400M); subagent > 100M stop (LOOK when it already ended);', - 'empty wakes > 25% of the last 40 turns (20+); review rounds > 2 per PR; Captains REPORT, never rotate.', + 'empty wakes > 25% of the last 40 turns (20+; warm-wait check-ins left out); review rounds > 2 per PR; Captains REPORT, never rotate.', 'A thread file that is not valid JSON or lacks cost.tokens or created_at prints RUNAWAY_INPUT and is skipped.', 'Exit 0 after a clean scan, 1 when any input was unreadable, 2 on a usage error.', ]; diff --git a/runaway-stopper/SKILL.md b/runaway-stopper/SKILL.md index fe137aedd..3ce6adbce 100644 --- a/runaway-stopper/SKILL.md +++ b/runaway-stopper/SKILL.md @@ -172,7 +172,8 @@ const fetch = async code => { const { transcript, recent, ...rest } = await tools.thread.get({ thread_id: code, transcript: { limit: 100, tools: true, before } }); doc ??= rest; events.push(...transcript.items.map(i => ({ source: i.source, tool: i.tool, createdAt: i.createdAt, - calls: i.calls?.map(c => ({ tool: c.tool })), state: String(i.input?.text ?? "").match(/"state\\?"\s*:\s*\\?"(\w+)/)?.[1] }))); + calls: i.calls?.map(c => ({ tool: c.tool })), state: String(i.input?.text ?? "").match(/"state\\?"\s*:\s*\\?"(\w+)/)?.[1], + ...(i.tool === "wait" ? { input: i.input } : {}) }))); if (!(before = transcript.beforeCursor)) break; } await tools.capy.write({ path: `${dir}/${doc.code}.thread.json`, content: JSON.stringify({ ...doc, events }) }); @@ -189,7 +190,7 @@ it lists the whole tree, where thread.get lists 200 rows at most (`own` is then Limits (org rule R70): own tokens (total minus subagents) past 500M, or active (working, waiting, or a turn in the last 24 hours) past 3 days: rotate, warn at 400M. A subagent past 100M: stop and look. Empty wakes past 25% of the -last 40 turns. Review rounds past 2 per PR. Act on each line: +last 40 turns (warm-wait check-ins left out). Review rounds past 2 per PR. Act on each line: | Line | Do | |---|---| diff --git a/runaway-stopper/SKILL.md.tmpl b/runaway-stopper/SKILL.md.tmpl index 64ade4525..40f40d7b7 100644 --- a/runaway-stopper/SKILL.md.tmpl +++ b/runaway-stopper/SKILL.md.tmpl @@ -48,7 +48,8 @@ const fetch = async code => { const { transcript, recent, ...rest } = await tools.thread.get({ thread_id: code, transcript: { limit: 100, tools: true, before } }); doc ??= rest; events.push(...transcript.items.map(i => ({ source: i.source, tool: i.tool, createdAt: i.createdAt, - calls: i.calls?.map(c => ({ tool: c.tool })), state: String(i.input?.text ?? "").match(/"state\\?"\s*:\s*\\?"(\w+)/)?.[1] }))); + calls: i.calls?.map(c => ({ tool: c.tool })), state: String(i.input?.text ?? "").match(/"state\\?"\s*:\s*\\?"(\w+)/)?.[1], + ...(i.tool === "wait" ? { input: i.input } : {}) }))); if (!(before = transcript.beforeCursor)) break; } await tools.capy.write({ path: `${dir}/${doc.code}.thread.json`, content: JSON.stringify({ ...doc, events }) }); @@ -65,7 +66,7 @@ it lists the whole tree, where thread.get lists 200 rows at most (`own` is then Limits (org rule R70): own tokens (total minus subagents) past 500M, or active (working, waiting, or a turn in the last 24 hours) past 3 days: rotate, warn at 400M. A subagent past 100M: stop and look. Empty wakes past 25% of the -last 40 turns. Review rounds past 2 per PR. Act on each line: +last 40 turns (warm-wait check-ins left out). Review rounds past 2 per PR. Act on each line: | Line | Do | |---|---| diff --git a/test/runaway.test.ts b/test/runaway.test.ts index fa774dc78..f637c920e 100644 --- a/test/runaway.test.ts +++ b/test/runaway.test.ts @@ -112,6 +112,57 @@ describe('gstack-runaway scan', () => { expect(r.out).toContain('FLAG: WARN empty_wake 65% of 31 turns > 25%'); }); + const at = (s: number) => new Date(Date.parse('2026-11-06T12:00:00Z') + s * 1000).toISOString(); + const wait = (s: number, input: Record) => ({ source: 'tool', tool: 'wait', input: { text: JSON.stringify({ description: 'w', ...input }) }, createdAt: at(s) }); + const work = (s: number) => ({ source: 'tool', tool: 'execute', calls: [{ tool: 'bash' }, { tool: 'edit' }], createdAt: at(s) }); + const scan = (items: unknown[], name: string) => { + const doc = { code: 'GSTA-999', status: 'idle', created_at: '2026-11-06T02:00:00Z', cost: { tokens: 1e6 }, tasks: [] }; + return run('scan', '--thread', write(name + '.json', doc), '--events', write(name + '.events.json', { items, cursor: null }), '--now', NOW).out; + }; + + // Value: protects=warm-wait check-ins (docs/captain.md Warm waits, #4060) are not empty wakes: a turn whose only call re-issues the + // wait on the same operation, or a 240 s timer, inside 30 minutes of the wait that began the run is left out; a re-wait past 30 + // minutes, a different wait, or a turn that also reads status still counts; fails_when=/runaway-stopper flags a lane that follows the + // rule (GST-1597); seam=emptyWakes; no_claim=whether the lane should have warm-waited at all + test('warm-wait check-ins inside 30 minutes are not empty wakes; past the window they count again', () => { + const lane: unknown[] = []; + for (let k = 0; k < 20; k++) { lane.push(work(k * 2000)); lane.push(wait(k * 2000 + 1, { operation_ids: ['op' + k], duration_seconds: 240 })); for (let c = 1; c <= 7; c++) lane.push(wait(k * 2000 + 1 + c * 240, c % 2 ? { duration_seconds: 240 } : { operation_ids: ['op' + k] })); } + expect(scan(lane, 'warm')).toContain('empty_wake=0%/20'); + const late: unknown[] = []; + for (let k = 0; k < 20; k++) { late.push(work(k * 4000)); late.push(wait(k * 4000 + 1, { operation_ids: ['op' + k] })); late.push(wait(k * 4000 + 1 + 1900, { operation_ids: ['op' + k] })); } + expect(scan(late, 'late')).toContain('empty_wake=50%/40'); + const other: unknown[] = []; + for (let k = 0; k < 20; k++) { other.push(work(k * 2000)); other.push(wait(k * 2000 + 1, { operation_ids: ['op' + k] })); other.push(wait(k * 2000 + 241, { operation_ids: ['other' + k], duration_seconds: 600 })); } + expect(scan(other, 'other')).toContain('empty_wake=50%/40'); + const reads: unknown[] = []; + for (let k = 0; k < 20; k++) { reads.push(work(k * 2000)); reads.push(wait(k * 2000 + 1, { operation_ids: ['op' + k] })); reads.push({ source: 'tool', tool: 'execute', calls: [{ tool: 'subagent_list' }], createdAt: at(k * 2000 + 241) }); reads.push(wait(k * 2000 + 242, { operation_ids: ['op' + k] })); } + expect(scan(reads, 'reads')).toContain('empty_wake=50%/40'); + const chain: unknown[] = [work(0), wait(1, { operation_ids: ['op'], duration_seconds: 240 })]; + for (let c = 1; c <= 60; c++) chain.push(wait(1 + c * 240, { operation_ids: ['op'], duration_seconds: 240 })); + expect(scan(chain, 'chain')).toContain('FLAG: WARN empty_wake 100% of 40 turns > 25%'); + }); + + // Value: protects=a 240 s timer chains only on the run's operations or none, and a counted turn that re-waits on the run's operation + // keeps the run's start, so polling one operation for hours with a status read every 28 minutes still counts; fails_when=a timer + // naming other operations is left out, or a status-read turn restarts the 30-minute window (GST-1597 /ship findings 2 and 3); + // seam=emptyWakes; no_claim=how often a lane should read status + test('a 240 s timer on other operations counts; a status-read turn does not restart the warm window', () => { + const timer = (ops: (k: number) => Record) => { + const items: unknown[] = []; + for (let k = 0; k < 20; k++) { items.push(work(k * 2000)); items.push(wait(k * 2000 + 1, { operation_ids: ['op' + k] })); items.push(wait(k * 2000 + 241, { ...ops(k), duration_seconds: 240 })); } + return items; + }; + expect(scan(timer(k => ({ operation_ids: ['other' + k] })), 'timer-other')).toContain('empty_wake=50%/40'); + expect(scan(timer(k => ({ operation_ids: ['op' + k] })), 'timer-same')).toContain('empty_wake=0%/20'); + expect(scan(timer(() => ({})), 'timer-none')).toContain('empty_wake=0%/20'); + const polled: unknown[] = [work(0), wait(1, { operation_ids: ['op'], duration_seconds: 240 })]; + for (let c = 1; c <= 60; c++) { + if (c % 7 === 0) polled.push({ source: 'tool', tool: 'execute', calls: [{ tool: 'subagent_list' }], createdAt: at(c * 240) }); + polled.push(wait(1 + c * 240, { operation_ids: ['op'], duration_seconds: 240 })); + } + expect(scan(polled, 'polled')).toContain('FLAG: WARN empty_wake 100% of 40 turns > 25%'); + }); + test('a subagent.list file replaces thread.get tasks: own sums top-level paths only, truncation makes it an upper bound', () => { const doc = { code: 'GSTA-266', status: 'working', created_at: '2026-11-03T20:00:00Z', cost: { tokens: 700e6 }, tasks: [] }; const subs = { total: 3, tasks: [ @@ -235,6 +286,30 @@ describe('/runaway-stopper skill', () => { expect(skill).toContain('rm -rf'); expect(skill).toContain('calls: i.calls?.map(c => ({ tool: c.tool }))'); }); + // Value: protects=the skill's fetch program keeps each wait's input, so the warm-wait exclusion runs through /runaway-stopper and not + // only on hand-built files; fails_when=the projection drops input and every warm check-in counts as an empty wake (GST-1597 /ship + // finding 1); seam=the template's projection run over items shaped like a tools.thread.get page (tools: true, read 2026-11-06), + // then gstack-runaway scan on the file it writes + test('the fetch program keeps wait input, so warm check-ins on a thread.get page are left out through the skill', () => { + const src = skill.match(/transcript\.items\.map\((i => \(\{[\s\S]*?\}\))\)\);/)?.[1]; + expect(src).toBeDefined(); + const project = new Function('return ' + src)() as (i: unknown) => unknown; + const at = (s: number) => new Date(Date.parse('2026-11-06T12:00:00Z') + s * 1000).toISOString(); + const text = (v: unknown) => ({ text: JSON.stringify(v), chars: JSON.stringify(v).length }); + const waitItem = (s: number, input: Record) => ({ id: 'toolu_w' + s, source: 'tool', tool: 'wait', text: 'Waiting for CI', createdAt: at(s), + input: text({ description: 'Waiting for CI', ...input }), result: { text: 'Waiting on 1 operation, resuming when any completes.', chars: 52, is_error: false } }); + const page: unknown[] = []; + for (let k = 0; k < 20; k++) { + page.push({ id: 'toolu_x' + k, source: 'tool', tool: 'execute', text: 'Fix and push', createdAt: at(k * 2000), + calls: [{ tool: 'bash', durationMs: 700, input: text({ command: 'git push' }) }, { tool: 'edit', durationMs: 20, input: text({ path: 'a.ts' }) }], input: text({ code: '...' }) }); + page.push(waitItem(k * 2000 + 1, { operation_ids: ['op' + k], duration_seconds: 240 })); + for (let c = 1; c <= 7; c++) page.push(waitItem(k * 2000 + 1 + c * 240, { operation_ids: ['op' + k], duration_seconds: 240 })); + } + const doc = { code: 'GSTA-998', status: 'idle', created_at: '2026-11-06T02:00:00Z', cost: { tokens: 1e6 }, tasks: [], events: page.map(project) }; + const out = run('scan', '--thread', write('skill-warm.thread.json', doc), '--now', NOW).out; + expect(out).toContain('empty_wake=0%/20'); + expect(out).not.toContain('FLAG: WARN empty_wake'); + }); test('rotation stops or lists live subagents, the successor subscribes to open PRs, and archiving waits for it', () => { expect(skill).toContain('stop your working and waiting subagents, or list each with its run'); expect(skill).toContain('subscribes to each open PR in the handoff');