gstack/test/autoplan-token-efficiency.test.ts

/**
 * GSTA-131 token efficiency in the unattended /autoplan runner, on the
 * simulated Capy host: new unattended runs default to two review rounds (a
 * resumed run keeps what it persisted); `revise --incorporates` applies an
 * edit without re-review when every incorporated decision resolves below High
 * and nothing new comes into scope, and reopens on anything else; the gate at
 * the round cap prints ESCALATE while High findings stay open; a docs-only
 * plan runs CEO (native voice only) and Eng; export counts every dispatched
 * attempt and a relative input-token estimate. Every spawn has a timeout.
 */
import { afterAll, describe, expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { detectDocsOnly, docsClassPath } from '../lib/autoplan-scope';
import { ROOT, SMALL_PLAN } from './helpers/autoplan-run-fixture';
import { capyRunner, phaseVia, readJsonl, readRunJson, writePlan } from './helpers/autoplan-capy';

const r = capyRunner();
afterAll(() => r.cleanup());
let n = 0;
const NONE = { native: [], outside: [] };
const V2 = SMALL_PLAN.replace('cap 30 s', 'cap 30 s, jitter drawn from a seeded source');
const DX_V2 = SMALL_PLAN.replace('Tests cover', 'Add a CLI flag --max-retries and an error message for a bad value. Tests cover');
const DOCS_PLAN = '# Retry docs\n\n## Implementation plan\n\nRewrite README.md and docs/howto-retry.md so they explain the retry backoff, and add a CHANGELOG.md entry.\n\n## Review record\n';
const journal = (out: string) => readJsonl(path.join(out, 'attempts.jsonl'));
const voices = (stdout: string) => stdout.split('\n').filter(l => l.startsWith('ATTEMPT: ')).map(l => /voice=(\w+)/.exec(l)![1]);
function start(args: string[] = [], text = SMALL_PLAN) {
  const dir = path.join(r.host.base, `tok${++n}`);
  const plan = writePlan(dir, text);
  const out = path.join(dir, 'run');
  const first = r.ap(['next', '--out', out, '--plan', plan, '--no-ui', ...args]);
  if (first.status !== 0) throw new Error(`first next failed: ${first.stderr}\n${first.stdout}`);
  return { out, dir, plan, first };
}
const v2File = (dir: string, text = V2) => { const f = path.join(dir, `plan-v2-${++n}.md`); fs.writeFileSync(f, text); return f; };
const disagreement = (native: string, outside: string) => ({
  native: [{ id: 'F1', severity: native, title: 'Jitter source is unstated', file: 'lib/worker.ts', line: 19, fix: 'name the source' }],
  outside: [{ id: 'O1', severity: outside, title: 'Jitter source is unstated', file: 'lib/worker.ts', line: 19, fix: 'name the source' }],
});
/** A run gated on one ceo disagreement d1, answered with its plan-changing option: amend_pending on d1. */
function amended(native: string, outside: string, args: string[] = []) {
  const s = start(args);
  phaseVia(r, s.out, 'ceo', disagreement(native, outside));
  expect(phaseVia(r, s.out, 'eng', NONE).outside.stdout).toContain('status=gate_pending');
  const res = r.ap(['answer', '--by', 'human', '--out', s.out, '--gate-rev', '1', '--reply', 'all except d1b']);
  expect(res.status, res.stderr).toBe(0);
  expect(readRunJson(s.out).status).toBe('amend_pending');
  return { ...s, run: readRunJson(s.out).run as string };
}
function bindNative(out: string) {
  const run = readRunJson(out);
  expect(r.review(path.join(out, 'ceo-native-prompt.md'), { FIXTURE_FINDINGS: '[]' }).status).toBe(0);
  return r.ap(['submit', '--out', out, '--phase', 'ceo', '--voice', 'native', '--result', run.phases.ceo.native.attempts[0].result, '--model', 'anthropic/claude-opus-5-5', '--attempt', run.phases.ceo.native.attempt]);
}

describe('a quiet revise after a capped one', () => {
  // Value: protects=the owner still sees an earlier capped delta; fails_when=a below-High revise after a capped one leaves the
  // answered gate in place (gate_rev unchanged, the delta item never rendered); seam=none; no_claim=none
  test('regates so the capped delta reaches the owner', () => {
    const s = start(['--max-rounds', '1']);
    phaseVia(r, s.out, 'ceo', {
      native: [{ id: 'F1', severity: 'High', title: 'Jitter source is unstated', file: 'lib/worker.ts', line: 19, fix: 'name the source' },
               { id: 'F2', severity: 'Medium', title: 'Cap value is unexplained', file: 'lib/config.ts', line: 3, fix: 'explain the cap' }],
      outside: [{ id: 'O1', severity: 'Low', title: 'Jitter source is unstated', file: 'lib/worker.ts', line: 19, fix: 'name the source' },
                { id: 'O2', severity: 'Informational', title: 'Cap value is unexplained', file: 'lib/config.ts', line: 3, fix: 'explain the cap' }],
    });
    phaseVia(r, s.out, 'eng', NONE);
    expect(r.ap(['answer', '--by', 'human', '--out', s.out, '--gate-rev', '1', '--reply', 'all except d1b d2b']).status).toBe(0);
    const first = r.ap(['revise', '--out', s.out, '--plan', v2File(s.dir), '--incorporates', 'd1']);
    expect(first.stdout).toMatch(/capped=/);
    const second = r.ap(['revise', '--out', s.out, '--plan', v2File(s.dir, SMALL_PLAN.replace('cap 30 s', 'cap 30 s (explained), jitter drawn from a seeded source')), '--incorporates', 'd2']);
    expect(second.status, second.stderr).toBe(0);
    expect(second.stdout).toContain('without re-review (below High; plan +');
    const run = readRunJson(s.out);
    expect(run.status).toBe('gate_pending');
    expect(run.gate_rev).toBe(2);
    expect(JSON.parse(fs.readFileSync(path.join(s.out, 'gate.json'), 'utf8')).gate_rev).toBe(2);
  }, 30_000);
});

describe('round budget', () => {
  test('a new unattended run defaults to two rounds per phase', () => {
    const s = start();
    expect(s.first.stdout).toMatch(/^ROUNDS: ceo=1\/2 eng=1\/2 max_rounds=2 host_spend=unknown$/m);
    expect(readRunJson(s.out).max_rounds).toBe(2);
  });

  test('a resumed run keeps the max_rounds it persisted', () => {
    const s = start();
    fs.writeFileSync(path.join(s.out, 'run.json'), JSON.stringify({ ...readRunJson(s.out), max_rounds: 1 }, null, 2) + '\n');
    expect(r.ap(['resume', '--out', s.out]).status).toBe(0);
    expect(readRunJson(s.out).max_rounds).toBe(1);
    expect(r.ap(['status', '--out', s.out]).stdout).toMatch(/^ROUNDS: ceo=1\/1 eng=1\/1 max_rounds=1 /m);
  });
});

describe('revise by severity', () => {
  test('below-High decisions apply without re-review: no phase reopens, snapshots rebind, the run completes', () => {
    const g = amended('Medium', 'Informational');
    const before = readRunJson(g.out);
    const rev = r.ap(['revise', '--out', g.out, '--plan', v2File(g.dir), '--incorporates', 'd1']);
    expect(rev.status, rev.stderr).toBe(0);
    expect(rev.stdout).toMatch(/^REVISED: plan=[0-9a-f]{12} reopened=none$/m);
    expect(rev.stdout).toContain(`APPLIED: ${g.run}-d1 without re-review (below High; plan +`);
    expect(rev.stdout).toContain('GSTACK_RESULT: skill=autoplan status=complete');
    const run = readRunJson(g.out);
    const row = journal(g.out).find(e => e.event === 'applied_without_rereview');
    expect(row).toMatchObject({ decisions: [`${g.run}-d1`], severities: { [`${g.run}-d1`]: 'Medium' }, phases: ['ceo', 'eng'], old_plan_sha256: before.phases.ceo.snapshot });
    for (const p of ['ceo', 'eng']) {
      expect(run.phases[p].snapshot).toBe(row.new_plan_sha256);
      expect(run.phases[p].rebound).toEqual([expect.objectContaining({ from: before.phases[p].snapshot, to: row.new_plan_sha256, decisions: [`${g.run}-d1`] })]);
      expect(run.phases[p].closed_at).toBe(before.phases[p].closed_at);
    }
    expect(run.attempt_counter).toBe(before.attempt_counter);
    expect(journal(g.out).filter(e => e.event === 'reopened')).toEqual([]);
    expect(run.amendments[0]).toMatchObject({ status: 'incorporated' });
    const v = r.validate(g.out);
    expect(v.status, v.stdout + v.stderr).toBe(0);
  }, 30_000);

  test('a High decision reopens as before', () => {
    const g = amended('High', 'Low');
    const rev = r.ap(['revise', '--out', g.out, '--plan', v2File(g.dir), '--incorporates', 'd1']);
    expect(rev.stdout).toMatch(/^REVISED: plan=\S+ reopened=ceo,eng$/m);
    expect(rev.stdout).not.toContain('APPLIED:');
    expect(journal(g.out).some(e => e.event === 'applied_without_rereview')).toBe(false);
  }, 30_000);

  test('a decision of unknown severity reopens', () => {
    const g = amended('Medium', 'Informational');
    const file = path.join(g.out, 'decisions.jsonl');
    fs.writeFileSync(file, readJsonl(file).map(d => JSON.stringify(d.label === 'D1' ? { ...d, origin: undefined } : d)).join('\n') + '\n');
    const rev = r.ap(['revise', '--out', g.out, '--plan', v2File(g.dir), '--incorporates', 'd1']);
    expect(rev.status, rev.stderr).toBe(0);
    expect(rev.stdout).toMatch(/reopened=ceo,eng$/m);
    expect(rev.stdout).not.toContain('APPLIED:');
  }, 30_000);

  test('a below-High edit that brings a phase into scope reopens and adds it', () => {
    const g = amended('Medium', 'Informational');
    const rev = r.ap(['revise', '--out', g.out, '--plan', v2File(g.dir, DX_V2), '--incorporates', 'd1']);
    expect(rev.status, rev.stderr).toBe(0);
    expect(rev.stdout).toMatch(/^REVISED: plan=\S+ reopened=ceo,eng added=dx$/m);
    expect(rev.stdout).not.toContain('APPLIED:');
  }, 30_000);

  test('at the round cap a below-High edit applies without a capped delta; a High one keeps the unreviewed-delta gate item', () => {
    const low = amended('Medium', 'Informational', ['--max-rounds', '1']);
    const quiet = r.ap(['revise', '--out', low.out, '--plan', v2File(low.dir), '--incorporates', 'd1']);
    expect(quiet.stdout).toContain(`APPLIED: ${low.run}-d1 without re-review (below High; plan +`);
    expect(readRunJson(low.out).capped_deltas).toBeUndefined();
    const high = amended('High', 'Low', ['--max-rounds', '1']);
    const capped = r.ap(['revise', '--out', high.out, '--plan', v2File(high.dir), '--incorporates', 'd1']);
    expect(capped.stdout).toMatch(/^REVISED: plan=\S+ reopened=none capped=ceo,eng$/m);
    expect(capped.stdout).not.toContain('APPLIED:');
    expect(readRunJson(high.out).capped_deltas.map((d: any) => d.phase)).toEqual(['ceo', 'eng']);
  }, 30_000);
});

describe('ESCALATE at the round cap', () => {
  const HIGH = { native: [{ severity: 'High', title: 'Retries can double-charge', fix: 'idempotency key' }], outside: [] };
  test('the gate at the cap with a High finding open names the decision', () => {
    const s = start(['--max-rounds', '1']);
    phaseVia(r, s.out, 'ceo', HIGH);
    const eng = phaseVia(r, s.out, 'eng', NONE);
    expect(eng.outside.stdout).toMatch(/^GATE: gate_rev=1 /m);
    expect(eng.outside.stdout.split('\n').filter(l => l.startsWith('ESCALATE:'))).toEqual(['ESCALATE: evidence sufficient, decision needed (ceo: 1 High open)']);
  }, 30_000);
  test('control: below the cap, or at it with nothing High open, no ESCALATE', () => {
    const s = start();
    phaseVia(r, s.out, 'ceo', HIGH);
    expect(phaseVia(r, s.out, 'eng', NONE).outside.stdout).not.toContain('ESCALATE:');
    const m = start(['--max-rounds', '1']);
    phaseVia(r, m.out, 'ceo', { native: [{ severity: 'Medium', title: 'Jitter source is unstated' }], outside: [] });
    const gate = phaseVia(r, m.out, 'eng', NONE).outside.stdout;
    expect(gate).toMatch(/^GATE: /m);
    expect(gate).not.toContain('ESCALATE:');
  }, 30_000);
});

describe('docs-only', () => {
  test('docs-class paths', () => {
    for (const p of ['README.md', 'CHANGELOG.md', 'TODOS.md', 'docs/howto-retry.md', 'docs/designs/x/PLAN.md', 'notes.md', './README.md', 'docs/images/flow.png', 'docs/designs']) expect(docsClassPath(p, ROOT), p).toBe(true);
    for (const p of ['docs/review-policy.md', 'autoplan/SKILL.md.tmpl', 'autoplan/SKILL.md', 'review/sections/army.md', 'scripts/resolvers/autoplan-driver.ts', 'autoplan/DRIVER.md',
      'captain/templates/brief.md', 'autoplan/notes.md', 'AGENTS.md', 'CLAUDE.md', 'lib/worker.ts', 'bin/gstack-autoplan']) expect(docsClassPath(p, ROOT), p).toBe(false);
    expect(detectDocsOnly('Rewrite README.md and docs/howto.md.', ROOT)).toMatchObject({ docsOnly: true, code: [] });
    expect(detectDocsOnly('Rewrite README.md and docs/review-policy.md.', ROOT)).toMatchObject({ docsOnly: false, code: ['docs/review-policy.md'] });
    expect(detectDocsOnly('Make the retry docs clearer.', ROOT).docsOnly).toBe(false);
  });

  test('detected: CEO dispatches the native voice only, Eng both; SCOPE reports it; DISPATCHES counts what ran', () => {
    const s = start([], DOCS_PLAN);
    expect(s.first.stdout).toMatch(/^SCOPE: ui=no .* docs_only=yes \(detected\) source=ui:flag,dx:detected$/m);
    expect(s.first.stdout).toMatch(/^PHASE: ceo snapshot=[0-9a-f]{64} order=ceo,eng$/m);
    expect(s.first.stdout).toContain('OUTSIDE: ceo not_run reason=docs_only (native voice only)');
    expect(voices(s.first.stdout)).toEqual(['native']);
    expect(readRunJson(s.out).phases.ceo.outside).toMatchObject({ status: 'disabled', outcome: 'not_run', reason: 'docs_only' });
    expect(bindNative(s.out).stdout).toMatch(/^PHASE_CLOSED: ceo /m);
    const eng = phaseVia(r, s.out, 'eng', NONE);
    expect(voices(eng.next.stdout)).toEqual(['native', 'outside']);
    expect(eng.outside.stdout).toContain('status=gate_pending');
    const exp = r.ap(['export', '--out', s.out]);
    expect(exp.status, exp.stderr).toBe(0);
    const run = readRunJson(s.out);
    const bytes = journal(s.out).filter(e => e.event === 'dispatched').reduce((t, e) => t + e.prompt_bytes, 0);
    expect(run.dispatches).toEqual({ total: 3, by_phase: { ceo: { native: 1 }, eng: { native: 1, outside: 1 } } });
    expect(run.est_input_tokens).toBe(Math.round(bytes / 4));
    expect(exp.stdout).toContain(`DISPATCHES: 3 est_input_tokens=${run.est_input_tokens} (relative estimate)`);
    const v = r.validate(s.out);
    expect(v.status, v.stdout + v.stderr).toBe(0);
  }, 30_000);

  test('--docs-only on the first next forces CEO + Eng on a code plan; a phase-enabling flag outranks detection', () => {
    const flagged = start(['--docs-only', '--developer-tool']);
    expect(flagged.first.stdout).toMatch(/ docs_only=yes \(flag\) source=/m);
    expect(readRunJson(flagged.out).phase_order).toEqual(['ceo', 'eng']);
    const code = start();
    expect(code.first.stdout).toMatch(/ docs_only=no \(detected\) source=/m);
    expect(voices(code.first.stdout)).toEqual(['native', 'outside']);
    const dx = start(['--developer-tool'], DOCS_PLAN);
    expect(dx.first.stdout).toMatch(/ docs_only=no \(detected\) source=/m);
    expect(readRunJson(dx.out).phase_order).toEqual(['ceo', 'dx', 'eng']);
  });
});

describe('dispatch counts', () => {
  test('failed attempts count; a run.json without the fields still validates', () => {
    const s = start();
    expect(bindNative(s.out).status).toBe(0);
    expect(r.ap(['submit', '--out', s.out, '--phase', 'ceo', '--voice', 'outside', '--failed', 'runner crashed']).stdout).toMatch(/^PHASE_CLOSED: ceo /m);
    phaseVia(r, s.out, 'eng', NONE);
    const exp = r.ap(['export', '--out', s.out, '--json']);
    expect(JSON.parse(exp.stdout)).toMatchObject({ dispatches: { total: 4, by_phase: { ceo: { native: 1, outside: 1 }, eng: { native: 1, outside: 1 } } } });
    const run = readRunJson(s.out);
    expect(run.est_input_tokens).toBeGreaterThan(0);
    delete run.dispatches; delete run.est_input_tokens;
    fs.writeFileSync(path.join(s.out, 'run.json'), JSON.stringify(run, null, 2) + '\n');
    const v = r.validate(s.out);
    expect(v.status, v.stdout + v.stderr).toBe(0);
    expect(r.ap(['status', '--out', s.out]).status).toBe(0);
  }, 30_000);
});