gstack/test/plan-count-timeout.test.ts

/** Free real-Bun/PTY regression: the counting body must settle before its equal outer timeout. */
import { expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { pathToFileURL } from 'node:url';
import { execFileSync } from 'node:child_process';

const ROOT = path.resolve(import.meta.dir, '..');
type Event = { event: string; at: number; invocation: number; pid?: number; start?: string; cwd?: string; data?: string; elapsed?: number; outcome?: string; fixtureGone?: boolean };

function ownedFake(event: Event, fake: string): boolean {
  if (!event.pid) return false;
  try {
    if (process.platform === 'linux') {
      const args = fs.readFileSync(`/proc/${event.pid}/cmdline`, 'utf8').split('\0');
      const start = fs.readFileSync(`/proc/${event.pid}/stat`, 'utf8').split(') ')[1]!.split(' ')[19];
      return args.includes(fake) && start === event.start;
    }
    return execFileSync('ps', ['-p', String(event.pid), '-o', 'command='], { encoding: 'utf8', timeout: 5000 }).includes(fake);
  } catch { return false; }
}

test.skipIf(process.platform === 'win32')('count timeout settles and cleans its real PTY before Bun retries the equal outer limit', async () => {
  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-timeout-'));
  const fake = path.join(dir, 'fake-claude');
  const worker = path.join(dir, 'worker.test.ts');
  const eventsPath = path.join(dir, 'events.jsonl');
  const fixtureTmp = path.join(dir, 'tmp');
  fs.mkdirSync(fixtureTmp);
  const helper = pathToFileURL(path.join(ROOT, 'test/helpers/claude-pty-runner.ts')).href;
  fs.writeFileSync(fake, `#!${process.execPath}\n` + String.raw`
import * as fs from 'node:fs';
const invocation = Number(process.env.TIMEOUT_INVOCATION);
const log = (event, extra = {}) => fs.appendFileSync(process.env.TIMEOUT_EVENTS, JSON.stringify({event, at:Date.now(), invocation, ...extra})+'\n');
const start = process.platform === 'linux' ? fs.readFileSync('/proc/self/stat','utf8').split(') ')[1].split(' ')[19] : '';
log('ready', {pid:process.pid, start, cwd:process.cwd()});
process.stdin.setRawMode(true);
process.stdin.resume();
process.stdin.on('data', bytes => log('input', {data:bytes.toString()}));
process.on('SIGINT', () => log('sigint')); // exercise the owned forced-exit fallback
process.stdout.write('COUNT_TIMEOUT_FIXTURE_READY\n');
setInterval(() => {}, 1000);
`, { mode: 0o755 });
  fs.writeFileSync(worker, `import {test} from 'bun:test';\nimport * as fs from 'node:fs';\nimport {runPlanSkillCounting} from ${JSON.stringify(helper)};\n` + String.raw`
let starts = 0;
const log = (event, invocation, extra={}) => fs.appendFileSync(process.env.TIMEOUT_EVENTS, JSON.stringify({event,at:Date.now(),invocation,...extra})+'\n');
test('owned counting timeout', async () => {
  const invocation = ++starts;
  const start = Date.now();
  log('body', invocation);
  try {
    const observation = await runPlanSkillCounting({skillName:'plan-design-review', slashCommand:'/plan-design-review',
      followUpPrompt:'# Timeout lifecycle fixture\nReview this plan.', isLastStep0AUQ:()=>false,
      reviewCountCeiling:8, timeoutMs:8000, startupReadyMarker:'COUNT_TIMEOUT_FIXTURE_READY',
      env:{TIMEOUT_INVOCATION:String(invocation),TIMEOUT_EVENTS:process.env.TIMEOUT_EVENTS}});
    const ready = fs.readFileSync(process.env.TIMEOUT_EVENTS,'utf8').trim().split('\n').map(line=>JSON.parse(line)).find(e=>e.event==='ready'&&e.invocation===invocation);
    log('returned', invocation, {elapsed:Date.now()-start,outcome:observation.outcome,fixtureGone:!fs.existsSync(ready.cwd)});
    throw new Error('HELPER_TIMEOUT_'+invocation);
  } finally { log('finally', invocation); }
}, 8000);
`);
  let rows: Event[] = [];
  let child: ReturnType<typeof Bun.spawn> | undefined;
  let output = '', error = '';
  let status: number | undefined;
  let watchdog: ReturnType<typeof setTimeout> | undefined;
  try {
    child = Bun.spawn([process.execPath, 'test', '--retry', '1', worker], {
      cwd: ROOT, env: { ...process.env, BROWSE_TERMINAL_BINARY: fake, EVALS_HERMETIC: '1', EVALS_RUN_ID: '',
        EVALS: '', GSTACK_EVAL_DIR: path.join(dir, 'evals'), TIMEOUT_EVENTS: eventsPath, TMPDIR: fixtureTmp },
      stdout: 'pipe', stderr: 'pipe',
    });
    watchdog = setTimeout(() => child?.kill('SIGKILL'), 42000);
    [status, output, error] = await Promise.all([child.exited, new Response(child.stdout).text(), new Response(child.stderr).text()]);
    rows = fs.readFileSync(eventsPath, 'utf8').trim().split('\n').map(line => JSON.parse(line));
    expect(status, output + error).toBe(1); // each retry retains its explicit timeout failure
    expect(error).toContain('HELPER_TIMEOUT_1');
    expect(error).toContain('HELPER_TIMEOUT_2');
    const starts = rows.filter(e => e.event === 'body');
    const ready = rows.filter(e => e.event === 'ready');
    const returned = rows.filter(e => e.event === 'returned');
    const finished = rows.filter(e => e.event === 'finally');
    expect(starts).toHaveLength(2);
    expect(ready).toHaveLength(2);
    expect(returned).toHaveLength(2);
    expect(finished).toHaveLength(2);
    expect(ready[0]!.pid).not.toBe(ready[1]!.pid);
    expect(ready[0]!.cwd).not.toBe(ready[1]!.cwd);
    expect(finished[0]!.at).toBeLessThanOrEqual(starts[1]!.at);
    for (const event of returned) {
      expect(event.outcome).toBe('timeout');
      expect(event.elapsed).toBeLessThan(8000);
      expect(event.fixtureGone).toBe(true);
    }
    for (const event of ready) {
      expect(ownedFake(event, fake)).toBe(false);
      const body = starts.find(e => e.invocation === event.invocation)!;
      const inputs = rows.filter(e => e.event === 'input' && e.invocation === event.invocation);
      expect(inputs.map(e => e.data).join('')).toBe('/plan-design-review\r');
      expect(inputs.every(e => e.at - body.at < 3000)).toBe(true);
    }
  } finally {
    if (watchdog) clearTimeout(watchdog);
    if (child && child.exitCode === null) { child.kill('SIGKILL'); await child.exited; }
    if (fs.existsSync(eventsPath)) rows = fs.readFileSync(eventsPath, 'utf8').trim().split('\n').filter(Boolean).map(line => JSON.parse(line));
    const cleanup: object[] = [];
    for (const event of rows.filter(e => e.event === 'ready')) {
      const owned = ownedFake(event, fake);
      if (owned) { process.kill(event.pid!, 'SIGKILL'); }
      cleanup.push({pid:event.pid, start:event.start, wasOwned:owned, signal:owned?'SIGKILL':null});
    }
    // Optional immutable owner proof; it never changes the test's assertions.
    if (process.env.PLAN_COUNT_TIMEOUT_PROOF_DIR) {
      const dest = process.env.PLAN_COUNT_TIMEOUT_PROOF_DIR;
      fs.mkdirSync(dest, { recursive: false });
      for (const name of ['events.jsonl','worker.test.ts','fake-claude']) if (fs.existsSync(path.join(dir,name))) fs.copyFileSync(path.join(dir,name),path.join(dest,name));
      fs.writeFileSync(path.join(dest,'worker.log'), output + error);
      fs.writeFileSync(path.join(dest,'result.json'), JSON.stringify({status,cleanup,rows},null,2)+'\n');
    }
    fs.rmSync(dir, { recursive: true, force: true });
  }
}, 48000);

test.skipIf(process.platform === 'win32')('deadline boundaries stop boot, late screen results and delayed Enter without abandoning cleanup', async () => {
  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-deadline-'));
  const fake = path.join(dir, 'fake-claude');
  const worker = path.join(dir, 'worker.ts');
  const helper = pathToFileURL(path.join(ROOT, 'test/helpers/claude-pty-runner.ts')).href;
  const screenModule = pathToFileURL(path.join(ROOT, 'test/helpers/pty-screen.ts')).href;
  fs.writeFileSync(fake, `#!${process.execPath}\n` + String.raw`
import * as fs from 'node:fs';
const log = (event, extra={}) => fs.appendFileSync(process.env.BOUNDARY_EVENTS,JSON.stringify({event,at:Date.now(),invocation:1,...extra})+'\n');
const start = process.platform === 'linux' ? fs.readFileSync('/proc/self/stat','utf8').split(') ')[1].split(' ')[19] : '';
log('ready',{pid:process.pid,start,cwd:process.cwd()});
process.stdin.setRawMode(true); process.stdin.resume();
process.stdin.on('data',data=>{
  log('input',{data:data.toString()});
  process.stdout.write('\x1b[2J\x1b[HWhich remedy should be used?\r\n❯1.First remedy\r\n2.Second remedy\r\n');
});
process.on('SIGINT',()=>{log('sigint');process.exit(0)});
process.stdout.write('COUNT_BOUNDARY_FIXTURE_READY\n');
setInterval(()=>{},1000);
`, { mode: 0o755 });
  fs.writeFileSync(worker, `import {mock} from 'bun:test';\nimport * as fs from 'node:fs';\n` +
    `const screenModule=${JSON.stringify(screenModule)};\nconst originalScreen=(await import(screenModule)).createPtyScreen;\n` + String.raw`
const mode=process.env.BOUNDARY_MODE;
let start=0, reads=0;
const log=(event,extra={})=>fs.appendFileSync(process.env.BOUNDARY_EVENTS,JSON.stringify({event,at:Date.now(),invocation:1,...extra})+'\n');
if(mode==='boot') {
  const sleep=Bun.sleep.bind(Bun);
  Bun.sleep=async ms=>{
    if(typeof ms==='number' && ms>250 && ms<1000) {
      log('early-clipped-wake',{requested:ms});
      return sleep(Math.max(0,ms-250));
    }
    return sleep(ms);
  };
}
if(mode==='screen') mock.module(screenModule,()=>({createPtyScreen:async(...args)=>{
  const screen=await originalScreen(...args);
  return {...screen,read:async()=>{reads++; await Bun.sleep(Math.max(0,start+3200-Date.now()));return screen.read();}};
}}));
` + `const {runPlanSkillCounting}=await import(${JSON.stringify(helper)});\n` + String.raw`
start=Date.now();log('body');
const observation=await runPlanSkillCounting({skillName:'plan-design-review',slashCommand:'/plan-design-review',
  followUpPrompt:'Review the deadline fixture.',isLastStep0AUQ:()=>false,reviewCountCeiling:8,timeoutMs:mode==='boot'?6000:8000,
  ...(mode==='boot'?{}:{startupReadyMarker:'COUNT_BOUNDARY_FIXTURE_READY'}),
  pickAUQ:(_routing,_active,context)=>{
    const ready=fs.readFileSync(process.env.BOUNDARY_EVENTS,'utf8').trim().split('\n').map(line=>JSON.parse(line)).find(event=>event.event==='ready');
    if(!Object.isFrozen(context)||context.cwd!==ready.cwd||!Number.isFinite(context.deadlineAt)||context.deadlineAt<=Date.now()||context.deadlineAt>start+8000)
      throw new Error('Picker did not receive its owned fixture and bounded deadline');
    log('picker');while(Date.now()-start<2800){};return 2;
  },
  env:{BOUNDARY_EVENTS:process.env.BOUNDARY_EVENTS}});
const events=fs.readFileSync(process.env.BOUNDARY_EVENTS,'utf8').trim().split('\n').map(line=>JSON.parse(line));
log('returned',{outcome:observation.outcome,elapsed:Date.now()-start,reads,fixtureGone:!fs.existsSync(events.find(e=>e.event==='ready').cwd)});
`);
  const children: ReturnType<typeof Bun.spawn>[] = [];
  const timers: ReturnType<typeof setTimeout>[] = [];
  try {
    const results = await Promise.allSettled(['boot','screen','enter'].map(async mode => {
      const eventPath=path.join(dir,mode+'.jsonl');
      const fixtureTmp=path.join(dir,mode); fs.mkdirSync(fixtureTmp);
      const child=Bun.spawn([process.execPath,worker], {cwd:ROOT,
        env:{...process.env,BROWSE_TERMINAL_BINARY:fake,EVALS_HERMETIC:'1',EVALS:'',EVALS_RUN_ID:'',
          GSTACK_EVAL_DIR:path.join(dir,'evals-'+mode),TMPDIR:fixtureTmp,BOUNDARY_MODE:mode,BOUNDARY_EVENTS:eventPath},stdout:'pipe',stderr:'pipe'});
      children.push(child);timers.push(setTimeout(()=>child.kill('SIGKILL'),25000));
      const [code,out,err]=await Promise.all([child.exited,new Response(child.stdout).text(),new Response(child.stderr).text()]);
      expect(code,out+err).toBe(0);
      const events=fs.readFileSync(eventPath,'utf8').trim().split('\n').map(line=>JSON.parse(line));
      const returned=events.find(e=>e.event==='returned');
      expect(returned.outcome).toBe('timeout'); expect(returned.fixtureGone).toBe(true);
      expect(returned.elapsed).toBeLessThan(mode==='boot'?6000:8000);
      const input=events.filter(e=>e.event==='input').map(e=>e.data).join('');
      expect(input).toBe(mode==='boot'?'':mode==='screen'?'/plan-design-review\r':'/plan-design-review\r2');
      if(mode==='boot') expect(events.some(e=>e.event==='early-clipped-wake')).toBe(true);
      if(mode==='screen') { expect(returned.reads).toBe(1);expect(events.some(e=>e.event==='picker')).toBe(false); }
      expect(ownedFake(events.find(e=>e.event==='ready'),fake)).toBe(false);
    }));
    for (const result of results) if (result.status === 'rejected') throw result.reason;
  } finally {
    timers.forEach(clearTimeout);
    for(const child of children) if(child.exitCode===null){child.kill('SIGKILL');await child.exited;}
    for(const mode of ['boot','screen','enter']) {
      const eventPath=path.join(dir,mode+'.jsonl');
      if(!fs.existsSync(eventPath))continue;
      for(const event of fs.readFileSync(eventPath,'utf8').trim().split('\n').filter(Boolean).map(line=>JSON.parse(line))) {
        if(event.event==='ready'&&ownedFake(event,fake))process.kill(event.pid,'SIGKILL');
      }
    }
    if(process.env.PLAN_COUNT_TIMEOUT_PROOF_DIR) {
      const dest=process.env.PLAN_COUNT_TIMEOUT_PROOF_DIR+'-boundaries';fs.mkdirSync(dest,{recursive:false});
      for(const name of ['worker.ts','fake-claude','boot.jsonl','screen.jsonl','enter.jsonl'])if(fs.existsSync(path.join(dir,name)))fs.copyFileSync(path.join(dir,name),path.join(dest,name));
    }
    fs.rmSync(dir,{recursive:true,force:true});
  }
},30000);