gstack/test/autoplan-capy-e2e.test.ts

/**
 * End-to-end unattended runs on the simulated Capy host, driven only through
 * the CLI (plan B1a/B1b/B1c, the wave's mandatory free-tier acceptance): a
 * three-revision run validates with unique ids; phases bound from free-form
 * records with one outside voice unavailable close, answer, export and
 * validate; a run gated on <=1.91.94-style decision ids validates after the
 * load-time migration; run.json records gstack_version, a newer run refuses
 * and an older one migrates forward. One conformance run strings every
 * mechanism together.
 */
import { afterAll, describe, expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { RESULT_CODES } from '../lib/result-codes';
import { ROOT, SMALL_PLAN } from './helpers/autoplan-run-fixture';
import { capyRunner, phaseVia, readJsonl, readRunJson, writePlan, type CapyRunner } from './helpers/autoplan-capy';

const r = capyRunner();
afterAll(() => r.cleanup());
const VERSION = fs.readFileSync(path.join(ROOT, 'VERSION'), 'utf8').trim();
let n = 0;
function start(args: string[] = []): { out: string; dir: string; plan: string } {
  const dir = path.join(r.host.base, `e2e${++n}`);
  const plan = writePlan(dir);
  const out = path.join(dir, 'run');
  const res = r.ap(['next', '--out', out, '--plan', plan, '--no-ui', ...args]);
  if (res.status !== 0) throw new Error(`first next failed: ${res.stderr}\n${res.stdout}`);
  return { out, dir, plan };
}
function expectUniqueIds(out: string) {
  for (const f of ['decisions.jsonl', 'findings.jsonl', 'tasks.jsonl']) {
    const ids = readJsonl(path.join(out, f)).map(x => x.id);
    expect(new Set(ids).size, f).toBe(ids.length);
  }
  const attempts = readJsonl(path.join(out, 'attempts.jsonl')).filter(e => e.event === 'dispatched').map(e => e.attempt);
  expect(new Set(attempts).size).toBe(attempts.length);
}
/** Every completed voice reviewed its phase's snapshot (receipt or attestation), recorded as reviewed_sha256. */
function expectReviewedHashes(out: string) {
  const run = readRunJson(out);
  for (const p of run.phase_order) for (const voice of ['native', 'outside']) {
    const v = run.phases[p][voice];
    if (v.status !== 'completed') continue;
    expect(v.reviewed_sha256, `${p} ${voice}`).toBe(run.phases[p].snapshot);
    if (v.receipt !== 'attested') expect(fs.readFileSync(path.join(out, v.output), 'utf8')).toContain(`INPUT: ${p} ${run.phases[p].snapshot}`);
  }
}
const ok = (res: { status: number | null; stderr: string; stdout: string }) => { expect(res.status, res.stderr + res.stdout).toBe(0); return res; };

describe('unattended runs on the Capy host', () => {
  test('a three-revision run driven only through the CLI validates with unique ids and records gstack_version', () => {
    const s = start();
    expect(readRunJson(s.out).gstack_version).toBe(VERSION);
    for (const rev of [1, 2]) {
      phaseVia(r, s.out, 'ceo');
      expect(phaseVia(r, s.out, 'eng').outside.stdout).toContain('status=gate_pending');
      expectReviewedHashes(s.out);
      expect(ok(r.ap(['answer', '--out', s.out, '--gate-rev', String(rev), '--reply', 'all except p1d'])).stdout).toMatch(/^REOPENED: ceo,eng \(Eng last\) reason=revise$/m);
    }
    phaseVia(r, s.out, 'ceo');
    phaseVia(r, s.out, 'eng');
    expect(ok(r.ap(['answer', '--out', s.out, '--gate-rev', '3', '--reply', 'all'])).stdout).toContain('status=complete');
    ok(r.ap(['export', '--out', s.out]));
    const v = ok(r.validate(s.out));
    expect(v.stdout).toMatch(/^ARTIFACT_VALID: run=\S+ status=complete/m);
    expectUniqueIds(s.out);
    expectReviewedHashes(s.out);
    expect(readJsonl(path.join(s.out, 'decisions.jsonl')).filter(d => d.gate_rev === 3).every(d => d.id.includes('-r3-'))).toBe(true);
  });

  test('free-form records (table, list, No findings.) and one unavailable outside voice reach the gate, answer, export and validate', () => {
    const s = start();
    const run = readRunJson(s.out);
    const sha = run.phases.ceo.snapshot;
    const rec = (name: string, text: string) => { const f = path.join(s.dir, name); fs.writeFileSync(f, text); return f; };
    ok(r.ap(['submit', '--out', s.out, '--phase', 'ceo', '--voice', 'native', '--model', 'anthropic/claude-opus-5-5', '--attest', 'Capy subagent read the prompt file', '--attempt', run.phases.ceo.native.attempt,
      '--record', rec('ceo-n.md', '# CEO review\n\n| Severity | Title | File |\n|---|---|---|\n| High | Retry cap is unbounded on restart | lib/worker.ts:10 |\n| Low | Name the env var | |\n')]));
    const closed = ok(r.ap(['submit', '--out', s.out, '--phase', 'ceo', '--voice', 'outside', '--model', 'openai/gpt-6.1-sol', '--record', rec('ceo-o.md', `INPUT: ceo ${sha}\n\n- [High] Jitter bounds are untested\n- [Medium] Resume path skips the cap\n`)]));
    expect(closed.stdout).toMatch(/^PHASE_CLOSED: ceo .*coverage=both/m);
    const eng = ok(r.ap(['next', '--out', s.out]));
    const engSha = /^PHASE: eng snapshot=([0-9a-f]{64})/m.exec(eng.stdout)![1];
    ok(r.ap(['submit', '--out', s.out, '--phase', 'eng', '--voice', 'native', '--model', 'anthropic/claude-opus-5-5', '--record', rec('eng-n.md', `INPUT: eng ${engSha}\n\nRead the whole plan.\n\nNo findings.\n`)]));
    const gated = ok(r.ap(['submit', '--out', s.out, '--phase', 'eng', '--voice', 'outside', '--unavailable', 'no cross-family subagent on this host']));
    expect(gated.stdout).toMatch(/^PHASE_CLOSED: eng .*coverage=native_only/m);
    expect(gated.stdout).toContain('status=gate_pending');
    expect(ok(r.ap(['answer', '--out', s.out, '--gate-rev', '1', '--reply', 'all'])).stdout).toContain('status=complete');
    ok(r.ap(['export', '--out', s.out]));
    expect(ok(r.validate(s.out)).stdout).toMatch(/^ARTIFACT_VALID: run=\S+ status=complete attested=1$/m);
    expect(fs.readFileSync(path.join(s.out, 'review-record.md'), 'utf8')).toMatch(/attested/);
    expectReviewedHashes(s.out);
  });

  test('a run gated on <=1.91.94-style decision ids validates after the load-time migration', () => {
    const s = start();
    phaseVia(r, s.out, 'ceo');
    phaseVia(r, s.out, 'eng');
    ok(r.ap(['answer', '--out', s.out, '--gate-rev', '1', '--reply', 'all except p1d']));
    phaseVia(r, s.out, 'ceo');
    phaseVia(r, s.out, 'eng');
    const file = path.join(s.out, 'decisions.jsonl');
    fs.writeFileSync(file, readJsonl(file).map(d => JSON.stringify({ ...d, id: d.id.replace(/-r\d+-/, '-') })).join('\n') + '\n');
    const manifest = readRunJson(s.out);
    delete manifest.gstack_version;
    fs.writeFileSync(path.join(s.out, 'run.json'), JSON.stringify(manifest, null, 2) + '\n');
    const res = ok(r.ap(['answer', '--out', s.out, '--gate-rev', '2', '--reply', 'all']));
    expect(res.stdout).toMatch(/^GSTACK_VERSION_CHANGED: run unknown -> /m);
    expect(res.stdout).toContain('status=complete');
    ok(r.ap(['export', '--out', s.out]));
    const v = r.validate(s.out);
    expect(v.stderr).not.toContain('ARTIFACT_DUPLICATE_ID');
    expect(v.stderr).not.toContain('ARTIFACT_DANGLING_REF');
    expect(v.status, v.stderr).toBe(0);
    const migrated = readJsonl(path.join(s.out, 'attempts.jsonl')).filter(e => e.event === 'decision_ids_migrated');
    expect(migrated).toHaveLength(1);
    expect(migrated[0].n).toBeGreaterThan(0);
    expect(readRunJson(s.out).gstack_version).toBe(VERSION);
  });

  test('a run written by a newer gstack refuses; an older one prints GSTACK_VERSION_CHANGED once and moves forward', () => {
    const s = start();
    const set = (v: string) => { const m = readRunJson(s.out); m.gstack_version = v; fs.writeFileSync(path.join(s.out, 'run.json'), JSON.stringify(m, null, 2) + '\n'); };
    set('99.0.0.0');
    const newer = r.ap(['next', '--out', s.out]);
    expect(newer.status).toBe(3);
    expect(newer.stderr).toMatch(/99\.0\.0\.0.*\(GSTACK_VERSION_CHANGED\)$/m);
    set('1.91.22.0');
    const older = ok(r.ap(['next', '--out', s.out]));
    expect(older.stdout).toMatch(new RegExp(`^GSTACK_VERSION_CHANGED: run 1\\.91\\.22\\.0 -> ${VERSION.replace(/\./g, '\\.')}`, 'm'));
    expect(ok(r.ap(['next', '--out', s.out])).stdout).not.toMatch(/GSTACK_VERSION_CHANGED/);
    expect(readJsonl(path.join(s.out, 'attempts.jsonl')).filter(e => e.event === 'gstack_version_changed')).toHaveLength(1);
  });
});

describe('conformance: one run through every mechanism', () => {
  test('canonical and attested binds, an unavailable voice, partial then full replies, PLAN_AMEND_PENDING, revise --incorporates, an interruption, three gate revisions', () => {
    const c: CapyRunner = r;
    const s = start();
    const run = readRunJson(s.out).run;
    // ceo: native canonical (fixture), outside attested record
    ok(c.review(path.join(s.out, 'ceo-native-prompt.md')));
    ok(c.ap(['submit', '--out', s.out, '--phase', 'ceo', '--voice', 'native', '--result', path.join(s.out, 'ceo-native.md'), '--model', 'anthropic/claude-opus-5-5']));
    const hand = path.join(s.dir, 'ceo-outside-hand.md');
    fs.writeFileSync(hand, '| Severity | Title |\n|---|---|\n| Medium | Retry storage cost is unpriced |\n');
    ok(c.ap(['submit', '--out', s.out, '--phase', 'ceo', '--voice', 'outside', '--record', hand, '--model', 'openai/gpt-6.1-sol', '--attest', 'Capy subagent', '--attempt', readRunJson(s.out).phases.ceo.outside.attempt]));
    // eng: native challenge with changes, outside unavailable
    ok(c.ap(['next', '--out', s.out]));
    ok(c.review(path.join(s.out, 'eng-native-prompt.md'), { FIXTURE_FINDINGS: JSON.stringify([{ id: 'F1', severity: 'High', title: 'Gate the backoff behind a flag', user_challenge: true, changes: 'scope', fix: 'RETRY_BACKOFF=1' }]) }));
    ok(c.ap(['submit', '--out', s.out, '--phase', 'eng', '--voice', 'native', '--result', path.join(s.out, 'eng-native.md'), '--model', 'anthropic/claude-opus-5-5']));
    expect(ok(c.ap(['submit', '--out', s.out, '--phase', 'eng', '--voice', 'outside', '--unavailable', 'no outside runner'])).stdout).toContain('status=gate_pending');
    const accept = JSON.parse(fs.readFileSync(path.join(s.out, 'gate.json'), 'utf8')).items.find((i: any) => i.title.startsWith('Gate the backoff')).id;
    // partial, then full
    expect(ok(c.ap(['answer', '--out', s.out, '--gate-rev', '1', '--reply', `${accept}b`])).stdout).toMatch(/^GATE_PENDING: p1/m);
    const full = ok(c.ap(['answer', '--out', s.out, '--gate-rev', '1', '--reply', 'all']));
    expect(full.stdout).toMatch(new RegExp(`^APPLY: ${run}-${accept} `, 'm'));
    const pendingNext = c.ap(['next', '--out', s.out]).stderr;
    expect(pendingNext).toMatch(/\(PLAN_AMEND_PENDING\)$/m);
    expect(pendingNext).toContain(`; fix: ${RESULT_CODES.PLAN_AMEND_PENDING.fix} (PLAN_AMEND_PENDING)`);
    const v2 = path.join(s.dir, 'plan-v2.md');
    fs.writeFileSync(v2, SMALL_PLAN.replace('Tests cover', 'Ship behind RETRY_BACKOFF=1, default off. Tests cover'));
    expect(ok(c.ap(['revise', '--out', s.out, '--plan', v2, '--incorporates', `${run}-${accept}`])).stdout).toMatch(/^AMENDMENTS: pending=none$/m);
    // interruption: attempts dispatched, the driver dies holding the lock; resume reclaims and the parent redispatches
    ok(c.ap(['next', '--out', s.out]));
    fs.writeFileSync(path.join(s.out, 'run.lock'), JSON.stringify({ pid: 999_999_999, host: require('node:os').hostname(), at: new Date().toISOString() }));
    const resumed = ok(c.ap(['resume', '--out', s.out]));
    expect(resumed.stdout).toMatch(/execution_unknown=2/);
    const unknown = [...resumed.stdout.matchAll(/^EXECUTION_UNKNOWN: (\S+)$/gm)].map(m => m[1]!);
    ok(c.ap(['next', '--out', s.out, ...unknown.flatMap(a => ['--redispatch', a])]));
    for (const voice of ['native', 'outside']) ok(c.review(path.join(s.out, `ceo-${voice}-prompt.md`)));
    for (const voice of ['native', 'outside']) ok(c.ap(['submit', '--out', s.out, '--phase', 'ceo', '--voice', voice, '--result', path.join(s.out, `ceo-${voice}.md`), '--model', voice === 'native' ? 'anthropic/claude-opus-5-5' : 'openai/gpt-6.1-sol']));
    phaseVia(c, s.out, 'eng');
    expect(readRunJson(s.out).gate_rev).toBe(2);
    ok(c.ap(['answer', '--out', s.out, '--gate-rev', '2', '--reply', 'all except p1d']));
    phaseVia(c, s.out, 'ceo');
    phaseVia(c, s.out, 'eng');
    expect(ok(c.ap(['answer', '--out', s.out, '--gate-rev', '3', '--reply', 'all'])).stdout).toContain('status=complete');
    ok(c.ap(['export', '--out', s.out]));
    expect(ok(c.validate(s.out)).stdout).toMatch(/^ARTIFACT_VALID: run=\S+ status=complete/m);
    const final = readRunJson(s.out);
    expect(final.counts.decisions_pending).toBe(0);
    expect((final.amendments ?? []).filter((a: any) => a.status === 'pending')).toEqual([]);
    expect(final.gate_rev).toBe(3);
    expectUniqueIds(s.out);
    expectReviewedHashes(s.out);
    expect(fs.readFileSync(path.join(s.out, `ceo-native.md`), 'utf8')).toContain(final.phases.eng.snapshot);
  });
});