mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-13 03:48:06 +01:00
runaway: warm-wait check-ins are not empty wakes (GST-1597) (#4069)
* runaway: warm-wait check-ins are not empty wakes (GST-1597) lib/runaway.ts counted each 240-second warm-wait check-in (docs/captain.md, #4060) as an empty wake, so /runaway-stopper could flag a lane that follows the rule. A turn whose only call re-issues the wait on the same operation, or a 240-second timer, within 30 minutes of the wait that started the run is now left out of the rate. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: pre-landing review fixes Review-Finding: p1/specialist-batched#1 reader=red-team Review-Finding: p1/specialist-batched#2 reader=testing Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs: note the warm-wait exclusion in the /runaway-stopper summary; rewrap captain.md (GST-1597 docs audit) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * runaway: keep wait input in the skill collector; 240 s timers chain only on the run's operations; status reads keep the warm window (GST-1597) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
bd36aa0b46
commit
ff65fb0c48
7 files changed
+120
-12
No files matched your search
@@ -0,0 +1 @@
|
||||
- **`/runaway-stopper` no longer flags lanes that follow the warm-wait rule.** `gstack-runaway scan` counted each 240-second warm-wait check-in as an empty wake, so a compliant lane waiting on CI could cross the 25% empty-wake warning. A turn whose only call re-issues the wait on the same operation, or a 240-second timer on that operation or none, within 30 minutes of the wait that started the run is now left out of the rate, including when `/runaway-stopper` fetches the thread itself. A re-wait past 30 minutes, a wait on other operations, or a turn that also reads status still counts, and a status read does not restart the 30 minutes. (GST-1597)
|
||||
+7
-4
@@ -152,10 +152,13 @@ cheap:
|
||||
operation with `duration_seconds: 240`, and on a timer wake that brings nothing new it
|
||||
re-issues the same wait silently, with no message. Once a warm wait has run about 30
|
||||
minutes, or past the end the lane expected, the lane stops re-issuing the timer and
|
||||
sleeps on the operation or pull request watch alone. A wait expected to run longer sleeps
|
||||
on its operations alone. For short lane waits this overrides "waits go on operations,
|
||||
never on polling wakes" ([unattended.md](unattended.md#report-length)). The controller
|
||||
never warm-waits, and a lane still sleeps while /ship runs
|
||||
sleeps on the operation or pull request watch alone. `gstack-runaway scan` leaves these
|
||||
check-ins out of its empty-wake rate: a turn whose only call re-waits on the same
|
||||
operation, or a 240-second timer on it or on nothing, within 30 minutes of the first wait.
|
||||
A status read in between counts and does not restart the 30 minutes. A wait expected to run
|
||||
longer sleeps on its operations alone. For short lane waits this overrides "waits go on
|
||||
operations, never on polling wakes" ([unattended.md](unattended.md#report-length)). The
|
||||
controller never warm-waits, and a lane still sleeps while /ship runs
|
||||
([the lane's /ship sequence](#the-lanes-ship-sequence-before-ready)).
|
||||
- **Batched reads.** Independent reads (files, greps, pull request and thread state) go in
|
||||
one parallel program, not one call each. Each call returns 4k characters or less; long
|
||||
|
||||
+1
-1
@@ -1020,7 +1020,7 @@ sets the defaults that keep it from coming back.
|
||||
- **Detect.** One program fetches each crew thread (`tools.thread.get`, with its transcript) to files, then
|
||||
`gstack-runaway scan --dir <dir>` prints a `RUNAWAY:` line per thread and a `FLAG:` line per limit crossed:
|
||||
own tokens past 500M or active past 3 days (rotate), a subagent past 100M (stop), empty wakes past 25% of the
|
||||
last 40 turns, and review rounds past 2 per PR. Captains are reported, never rotated.
|
||||
last 40 turns (warm-wait check-ins are left out), and review rounds past 2 per PR. Captains are reported, never rotated.
|
||||
- **Stop.** A live subagent past 100M is stopped with its reason. A thread is rotated at a clean point through
|
||||
a handoff file and a successor brief (`runaway-stopper/templates/`), then archived by its own Captain. A thread
|
||||
mid-/ship or mid-merge gets `DEFER:` and nothing happens until that work finishes.
|
||||
|
||||
+30
-3
@@ -18,6 +18,8 @@ export const LIMITS = {
|
||||
emptyRate: 0.25,
|
||||
emptyWindow: 40,
|
||||
emptyMinTurns: 20,
|
||||
warmWaitS: 240,
|
||||
warmWindowS: 30 * 60,
|
||||
maxRounds: 2,
|
||||
} as const;
|
||||
|
||||
@@ -51,18 +53,43 @@ function reportState(item: Item): string | undefined {
|
||||
return raw?.match(/\\?"state\\?"\s*:\s*\\?"(final|needs_you|working)/)?.[1];
|
||||
}
|
||||
|
||||
/** A wait call's operations and timer, from its recorded input; null when the input is unreadable. */
|
||||
function waitArgs(item: Item): { ops: string; seconds: number | null } | null {
|
||||
const raw = typeof item.input === 'string' ? item.input : item.input?.text;
|
||||
try {
|
||||
const v = JSON.parse(raw ?? '') as { operation_ids?: unknown; duration_seconds?: unknown };
|
||||
return { ops: Array.isArray(v.operation_ids) ? [...v.operation_ids].map(String).sort().join(',') : '', seconds: typeof v.duration_seconds === 'number' ? v.duration_seconds : null };
|
||||
} catch { return null; }
|
||||
}
|
||||
|
||||
/**
|
||||
* Turns from transcript items: an inbound message starts one; end_turn, wait or a final/needs_you report
|
||||
* ends one. A turn is empty when it made no call beyond status reads, waits and "working" reports.
|
||||
* ends one. A turn is empty when it made no call beyond status reads, waits and "working" reports. A warm-wait
|
||||
* check-in (docs/captain.md, Warm waits; GST-1597) is not a turn: one whose only call re-issues a wait on the
|
||||
* operations the previous turn waited on, or a 240-second timer naming those operations or none, within 30 minutes
|
||||
* of the wait that started the run. Any turn that closes on such a wait keeps the run's start, so a status read
|
||||
* between check-ins counts but never restarts the 30 minutes.
|
||||
*/
|
||||
export function emptyWakes(items: readonly Item[]): { turns: number; empty: number } {
|
||||
const sorted = items.filter(i => i.createdAt && i.source !== 'assistant').sort((a, b) => a.createdAt!.localeCompare(b.createdAt!));
|
||||
const turns: boolean[] = [];
|
||||
let acted = false, open = false;
|
||||
const close = () => { if (open) turns.push(!acted); acted = false; open = false; };
|
||||
let calls: Item[] = [];
|
||||
let last: { ops: string; since: number } | null = null;
|
||||
const close = () => {
|
||||
if (!open) return;
|
||||
const closing = calls.at(-1);
|
||||
const args = closing?.tool === 'wait' ? waitArgs(closing) : null;
|
||||
const at = Date.parse(closing?.createdAt ?? '');
|
||||
const same = !!args && !!last && (args.ops ? args.ops === last.ops : args.seconds === LIMITS.warmWaitS);
|
||||
if (!(same && calls.length === 1 && at - last!.since <= LIMITS.warmWindowS * 1000)) turns.push(!acted);
|
||||
last = args ? (same ? last : { ops: args.ops, since: at }) : null;
|
||||
acted = false; open = false; calls = [];
|
||||
};
|
||||
for (const i of sorted) {
|
||||
if (i.source !== 'tool') { close(); continue; }
|
||||
open = true;
|
||||
calls.push(i);
|
||||
const state = i.tool === 'message_agent' || i.tool === 'message_user' ? reportState(i) : undefined;
|
||||
if (i.tool !== 'end_turn' && state !== 'working' && !isKeepalive(i)) acted = true;
|
||||
if (i.tool === 'end_turn' || i.tool === 'wait' || state === 'final' || state === 'needs_you') close();
|
||||
@@ -184,7 +211,7 @@ const USAGE = [
|
||||
'Lines: RUNAWAY (one per thread, largest own tokens first), FLAG ROTATE|STOP|LOOK|WARN|REPORT, DEFER, SWEEP (--dir).',
|
||||
'Review rounds count only subagent titles that name the PR (#N); the rest show as unattributed.',
|
||||
'Limits (R70): own > 500M or active > 3d rotate (warn 400M); subagent > 100M stop (LOOK when it already ended);',
|
||||
'empty wakes > 25% of the last 40 turns (20+); review rounds > 2 per PR; Captains REPORT, never rotate.',
|
||||
'empty wakes > 25% of the last 40 turns (20+; warm-wait check-ins left out); review rounds > 2 per PR; Captains REPORT, never rotate.',
|
||||
'A thread file that is not valid JSON or lacks cost.tokens or created_at prints RUNAWAY_INPUT and is skipped.',
|
||||
'Exit 0 after a clean scan, 1 when any input was unreadable, 2 on a usage error.',
|
||||
];
|
||||
|
||||
@@ -172,7 +172,8 @@ const fetch = async code => {
|
||||
const { transcript, recent, ...rest } = await tools.thread.get({ thread_id: code, transcript: { limit: 100, tools: true, before } });
|
||||
doc ??= rest;
|
||||
events.push(...transcript.items.map(i => ({ source: i.source, tool: i.tool, createdAt: i.createdAt,
|
||||
calls: i.calls?.map(c => ({ tool: c.tool })), state: String(i.input?.text ?? "").match(/"state\\?"\s*:\s*\\?"(\w+)/)?.[1] })));
|
||||
calls: i.calls?.map(c => ({ tool: c.tool })), state: String(i.input?.text ?? "").match(/"state\\?"\s*:\s*\\?"(\w+)/)?.[1],
|
||||
...(i.tool === "wait" ? { input: i.input } : {}) })));
|
||||
if (!(before = transcript.beforeCursor)) break;
|
||||
}
|
||||
await tools.capy.write({ path: `${dir}/${doc.code}.thread.json`, content: JSON.stringify({ ...doc, events }) });
|
||||
@@ -189,7 +190,7 @@ it lists the whole tree, where thread.get lists 200 rows at most (`own` is then
|
||||
|
||||
Limits (org rule R70): own tokens (total minus subagents) past 500M, or active (working, waiting, or a turn in the
|
||||
last 24 hours) past 3 days: rotate, warn at 400M. A subagent past 100M: stop and look. Empty wakes past 25% of the
|
||||
last 40 turns. Review rounds past 2 per PR. Act on each line:
|
||||
last 40 turns (warm-wait check-ins left out). Review rounds past 2 per PR. Act on each line:
|
||||
|
||||
| Line | Do |
|
||||
|---|---|
|
||||
|
||||
@@ -48,7 +48,8 @@ const fetch = async code => {
|
||||
const { transcript, recent, ...rest } = await tools.thread.get({ thread_id: code, transcript: { limit: 100, tools: true, before } });
|
||||
doc ??= rest;
|
||||
events.push(...transcript.items.map(i => ({ source: i.source, tool: i.tool, createdAt: i.createdAt,
|
||||
calls: i.calls?.map(c => ({ tool: c.tool })), state: String(i.input?.text ?? "").match(/"state\\?"\s*:\s*\\?"(\w+)/)?.[1] })));
|
||||
calls: i.calls?.map(c => ({ tool: c.tool })), state: String(i.input?.text ?? "").match(/"state\\?"\s*:\s*\\?"(\w+)/)?.[1],
|
||||
...(i.tool === "wait" ? { input: i.input } : {}) })));
|
||||
if (!(before = transcript.beforeCursor)) break;
|
||||
}
|
||||
await tools.capy.write({ path: `${dir}/${doc.code}.thread.json`, content: JSON.stringify({ ...doc, events }) });
|
||||
@@ -65,7 +66,7 @@ it lists the whole tree, where thread.get lists 200 rows at most (`own` is then
|
||||
|
||||
Limits (org rule R70): own tokens (total minus subagents) past 500M, or active (working, waiting, or a turn in the
|
||||
last 24 hours) past 3 days: rotate, warn at 400M. A subagent past 100M: stop and look. Empty wakes past 25% of the
|
||||
last 40 turns. Review rounds past 2 per PR. Act on each line:
|
||||
last 40 turns (warm-wait check-ins left out). Review rounds past 2 per PR. Act on each line:
|
||||
|
||||
| Line | Do |
|
||||
|---|---|
|
||||
|
||||
@@ -112,6 +112,57 @@ describe('gstack-runaway scan', () => {
|
||||
expect(r.out).toContain('FLAG: WARN empty_wake 65% of 31 turns > 25%');
|
||||
});
|
||||
|
||||
const at = (s: number) => new Date(Date.parse('2026-11-06T12:00:00Z') + s * 1000).toISOString();
|
||||
const wait = (s: number, input: Record<string, unknown>) => ({ source: 'tool', tool: 'wait', input: { text: JSON.stringify({ description: 'w', ...input }) }, createdAt: at(s) });
|
||||
const work = (s: number) => ({ source: 'tool', tool: 'execute', calls: [{ tool: 'bash' }, { tool: 'edit' }], createdAt: at(s) });
|
||||
const scan = (items: unknown[], name: string) => {
|
||||
const doc = { code: 'GSTA-999', status: 'idle', created_at: '2026-11-06T02:00:00Z', cost: { tokens: 1e6 }, tasks: [] };
|
||||
return run('scan', '--thread', write(name + '.json', doc), '--events', write(name + '.events.json', { items, cursor: null }), '--now', NOW).out;
|
||||
};
|
||||
|
||||
// Value: protects=warm-wait check-ins (docs/captain.md Warm waits, #4060) are not empty wakes: a turn whose only call re-issues the
|
||||
// wait on the same operation, or a 240 s timer, inside 30 minutes of the wait that began the run is left out; a re-wait past 30
|
||||
// minutes, a different wait, or a turn that also reads status still counts; fails_when=/runaway-stopper flags a lane that follows the
|
||||
// rule (GST-1597); seam=emptyWakes; no_claim=whether the lane should have warm-waited at all
|
||||
test('warm-wait check-ins inside 30 minutes are not empty wakes; past the window they count again', () => {
|
||||
const lane: unknown[] = [];
|
||||
for (let k = 0; k < 20; k++) { lane.push(work(k * 2000)); lane.push(wait(k * 2000 + 1, { operation_ids: ['op' + k], duration_seconds: 240 })); for (let c = 1; c <= 7; c++) lane.push(wait(k * 2000 + 1 + c * 240, c % 2 ? { duration_seconds: 240 } : { operation_ids: ['op' + k] })); }
|
||||
expect(scan(lane, 'warm')).toContain('empty_wake=0%/20');
|
||||
const late: unknown[] = [];
|
||||
for (let k = 0; k < 20; k++) { late.push(work(k * 4000)); late.push(wait(k * 4000 + 1, { operation_ids: ['op' + k] })); late.push(wait(k * 4000 + 1 + 1900, { operation_ids: ['op' + k] })); }
|
||||
expect(scan(late, 'late')).toContain('empty_wake=50%/40');
|
||||
const other: unknown[] = [];
|
||||
for (let k = 0; k < 20; k++) { other.push(work(k * 2000)); other.push(wait(k * 2000 + 1, { operation_ids: ['op' + k] })); other.push(wait(k * 2000 + 241, { operation_ids: ['other' + k], duration_seconds: 600 })); }
|
||||
expect(scan(other, 'other')).toContain('empty_wake=50%/40');
|
||||
const reads: unknown[] = [];
|
||||
for (let k = 0; k < 20; k++) { reads.push(work(k * 2000)); reads.push(wait(k * 2000 + 1, { operation_ids: ['op' + k] })); reads.push({ source: 'tool', tool: 'execute', calls: [{ tool: 'subagent_list' }], createdAt: at(k * 2000 + 241) }); reads.push(wait(k * 2000 + 242, { operation_ids: ['op' + k] })); }
|
||||
expect(scan(reads, 'reads')).toContain('empty_wake=50%/40');
|
||||
const chain: unknown[] = [work(0), wait(1, { operation_ids: ['op'], duration_seconds: 240 })];
|
||||
for (let c = 1; c <= 60; c++) chain.push(wait(1 + c * 240, { operation_ids: ['op'], duration_seconds: 240 }));
|
||||
expect(scan(chain, 'chain')).toContain('FLAG: WARN empty_wake 100% of 40 turns > 25%');
|
||||
});
|
||||
|
||||
// Value: protects=a 240 s timer chains only on the run's operations or none, and a counted turn that re-waits on the run's operation
|
||||
// keeps the run's start, so polling one operation for hours with a status read every 28 minutes still counts; fails_when=a timer
|
||||
// naming other operations is left out, or a status-read turn restarts the 30-minute window (GST-1597 /ship findings 2 and 3);
|
||||
// seam=emptyWakes; no_claim=how often a lane should read status
|
||||
test('a 240 s timer on other operations counts; a status-read turn does not restart the warm window', () => {
|
||||
const timer = (ops: (k: number) => Record<string, unknown>) => {
|
||||
const items: unknown[] = [];
|
||||
for (let k = 0; k < 20; k++) { items.push(work(k * 2000)); items.push(wait(k * 2000 + 1, { operation_ids: ['op' + k] })); items.push(wait(k * 2000 + 241, { ...ops(k), duration_seconds: 240 })); }
|
||||
return items;
|
||||
};
|
||||
expect(scan(timer(k => ({ operation_ids: ['other' + k] })), 'timer-other')).toContain('empty_wake=50%/40');
|
||||
expect(scan(timer(k => ({ operation_ids: ['op' + k] })), 'timer-same')).toContain('empty_wake=0%/20');
|
||||
expect(scan(timer(() => ({})), 'timer-none')).toContain('empty_wake=0%/20');
|
||||
const polled: unknown[] = [work(0), wait(1, { operation_ids: ['op'], duration_seconds: 240 })];
|
||||
for (let c = 1; c <= 60; c++) {
|
||||
if (c % 7 === 0) polled.push({ source: 'tool', tool: 'execute', calls: [{ tool: 'subagent_list' }], createdAt: at(c * 240) });
|
||||
polled.push(wait(1 + c * 240, { operation_ids: ['op'], duration_seconds: 240 }));
|
||||
}
|
||||
expect(scan(polled, 'polled')).toContain('FLAG: WARN empty_wake 100% of 40 turns > 25%');
|
||||
});
|
||||
|
||||
test('a subagent.list file replaces thread.get tasks: own sums top-level paths only, truncation makes it an upper bound', () => {
|
||||
const doc = { code: 'GSTA-266', status: 'working', created_at: '2026-11-03T20:00:00Z', cost: { tokens: 700e6 }, tasks: [] };
|
||||
const subs = { total: 3, tasks: [
|
||||
@@ -235,6 +286,30 @@ describe('/runaway-stopper skill', () => {
|
||||
expect(skill).toContain('rm -rf');
|
||||
expect(skill).toContain('calls: i.calls?.map(c => ({ tool: c.tool }))');
|
||||
});
|
||||
// Value: protects=the skill's fetch program keeps each wait's input, so the warm-wait exclusion runs through /runaway-stopper and not
|
||||
// only on hand-built files; fails_when=the projection drops input and every warm check-in counts as an empty wake (GST-1597 /ship
|
||||
// finding 1); seam=the template's projection run over items shaped like a tools.thread.get page (tools: true, read 2026-11-06),
|
||||
// then gstack-runaway scan on the file it writes
|
||||
test('the fetch program keeps wait input, so warm check-ins on a thread.get page are left out through the skill', () => {
|
||||
const src = skill.match(/transcript\.items\.map\((i => \(\{[\s\S]*?\}\))\)\);/)?.[1];
|
||||
expect(src).toBeDefined();
|
||||
const project = new Function('return ' + src)() as (i: unknown) => unknown;
|
||||
const at = (s: number) => new Date(Date.parse('2026-11-06T12:00:00Z') + s * 1000).toISOString();
|
||||
const text = (v: unknown) => ({ text: JSON.stringify(v), chars: JSON.stringify(v).length });
|
||||
const waitItem = (s: number, input: Record<string, unknown>) => ({ id: 'toolu_w' + s, source: 'tool', tool: 'wait', text: 'Waiting for CI', createdAt: at(s),
|
||||
input: text({ description: 'Waiting for CI', ...input }), result: { text: 'Waiting on 1 operation, resuming when any completes.', chars: 52, is_error: false } });
|
||||
const page: unknown[] = [];
|
||||
for (let k = 0; k < 20; k++) {
|
||||
page.push({ id: 'toolu_x' + k, source: 'tool', tool: 'execute', text: 'Fix and push', createdAt: at(k * 2000),
|
||||
calls: [{ tool: 'bash', durationMs: 700, input: text({ command: 'git push' }) }, { tool: 'edit', durationMs: 20, input: text({ path: 'a.ts' }) }], input: text({ code: '...' }) });
|
||||
page.push(waitItem(k * 2000 + 1, { operation_ids: ['op' + k], duration_seconds: 240 }));
|
||||
for (let c = 1; c <= 7; c++) page.push(waitItem(k * 2000 + 1 + c * 240, { operation_ids: ['op' + k], duration_seconds: 240 }));
|
||||
}
|
||||
const doc = { code: 'GSTA-998', status: 'idle', created_at: '2026-11-06T02:00:00Z', cost: { tokens: 1e6 }, tasks: [], events: page.map(project) };
|
||||
const out = run('scan', '--thread', write('skill-warm.thread.json', doc), '--now', NOW).out;
|
||||
expect(out).toContain('empty_wake=0%/20');
|
||||
expect(out).not.toContain('FLAG: WARN empty_wake');
|
||||
});
|
||||
test('rotation stops or lists live subagents, the successor subscribes to open PRs, and archiving waits for it', () => {
|
||||
expect(skill).toContain('stop your working and waiting subagents, or list each with its run');
|
||||
expect(skill).toContain('subscribes to each open PR in the handoff');
|
||||
|
||||
Reference in new issue
Block a user