gstack/test/cso-watchdog.test.ts

import { afterEach, describe, expect, test } from 'bun:test';import * as fs from 'node:fs';import * as os from 'node:os';import * as path from 'node:path';import { spawn, spawnSync } from 'node:child_process';
import { supervisePreparedCall } from '../lib/cso/preparation-docker';
const dirs:string[]=[];const tmp=()=>{const p=fs.mkdtempSync(path.join(os.tmpdir(),'csowatchdog-'));dirs.push(p);return p;};afterEach(()=>{for(const p of dirs.splice(0))fs.rmSync(p,{recursive:true,force:true});});
function compile(dir:string){const out=path.join(dir,'watchdog'),r=spawnSync('/usr/bin/cc',['-std=c11','-D_POSIX_C_SOURCE=200809L','-O2','-Wall','-Wextra',path.resolve(import.meta.dir,'../lib/cso/watchdog.c'),'-o',out],{encoding:'utf8',timeout:30_000});expect(r.status).toBe(0);expect(r.stderr).toBe('');return out;}
function pinnedEndpoint(dir:string){const socket=path.join(dir,'daemon.sock'),made=spawnSync('/usr/bin/python3',['-c','import socket,sys; s=socket.socket(socket.AF_UNIX); s.bind(sys.argv[1]); s.close()',socket],{encoding:'utf8'});expect(made.status).toBe(0);const stat=fs.lstatSync(socket);expect(stat.isSocket()).toBe(true);return{uri:`unix://${socket}`,device:stat.dev,inode:stat.ino};}
describe('detached CSO watchdog',()=>{
  test('prepared-call guard acknowledges normal exact cleanup',async()=>{const run=tmp(),watchdog=compile(run),call=path.join(run,'preparation-execution','offline-call'),control=path.join(run,'supervision','prepared-call');fs.mkdirSync(call,{recursive:true});fs.mkdirSync(control,{recursive:true});fs.writeFileSync(path.join(call,'prepared-source'),'copy');const guard=await supervisePreparedCall({watchdogPath:watchdog,ownerPid:process.pid,deadline:Date.now()+10_000,runRoot:run,callRoot:call,controlRoot:control});await guard.dispose();expect(fs.existsSync(call)).toBe(false);expect(fs.existsSync(control)).toBe(false);});
  test('prepared-call guard removes retained output after owner death and deadline',async()=>{for(const mode of ['owner','deadline'] as const){const run=tmp(),watchdog=compile(run),call=path.join(run,'preparation-execution',mode),control=path.join(run,'supervision',mode),event=path.join(control,'attempt.event');fs.mkdirSync(call,{recursive:true});fs.mkdirSync(control,{recursive:true});fs.writeFileSync(path.join(call,'prepared-source'),'copy');const owner=mode==='owner'?spawn('/bin/sleep',['30'],{stdio:'ignore'}):undefined,guard=await supervisePreparedCall({watchdogPath:watchdog,ownerPid:owner?.pid??process.pid,deadline:Date.now()+(mode==='owner'?10_000:50),runRoot:run,callRoot:call,controlRoot:control});if(owner)owner.kill('SIGKILL');for(let attempt=0;attempt<40&&(!fs.existsSync(event)||fs.existsSync(call));attempt++)await Bun.sleep(100);expect(fs.existsSync(call)).toBe(false);expect(fs.readFileSync(event,'utf8')).toContain(`${mode==='owner'?'supervisor-death':'deadline'} execution-copy cleanup complete`);await guard.dispose();expect(fs.existsSync(control)).toBe(false);}});
  test('survives supervisor death and cleans journaled plus label-race resources',async()=>{const dir=tmp(),watchdog=compile(dir),endpoint=pinnedEndpoint(dir),log=path.join(dir,'docker.log'),docker=path.join(dir,'docker'),id='a'.repeat(64),race='e'.repeat(64),lease=path.join(dir,'lease'),token='c'.repeat(32);fs.mkdirSync(lease);fs.writeFileSync(path.join(lease,'lease.json'),'{}');fs.writeFileSync(path.join(lease,'lease.token'),token+'\n');fs.writeFileSync(docker,`#!/bin/sh\nif [ "$3" = ps ]; then if [ "$8" = "id=${id}" ]; then printf '%s\\n' '${id}'; else printf '%s\\n' '${race}'; fi; exit 0; fi\nprintf '%s\\n' "$*" >> '${log}'\n`,{mode:0o755});fs.writeFileSync(path.join(dir,'resources.journal'),`container:${id}\n`,{mode:0o600});const owner=spawn('/bin/sleep',['1'],{stdio:'ignore'}),child=spawn(watchdog,['--owner',String(owner.pid),'--deadline',String(Math.ceil((Date.now()+10_000)/1000)),'--run-dir',dir,'--docker',docker,'--endpoint',endpoint.uri,'--socket-device',String(endpoint.device),'--socket-inode',String(endpoint.inode),'--run-label','test-label','--lease-path',lease,'--lease-token',token],{stdio:'ignore'}),closed=new Promise<number|null>(resolve=>child.on('close',resolve));expect(await closed).toBe(0);const calls=fs.readFileSync(log,'utf8');expect(calls).toContain(`rm --force --volumes ${id}`);expect(calls).toContain(`rm --force --volumes ${race}`);expect(fs.readFileSync(path.join(dir,'watchdog.event'),'utf8')).toContain('supervisor-death cleanup complete');expect(fs.existsSync(lease)).toBe(false);});
  test('quarantines an authenticated lease before a concurrent TS recovery claim can strand release',async()=>{const dir=tmp(),watchdog=compile(dir),endpoint=pinnedEndpoint(dir),docker=path.join(dir,'docker'),lease=path.join(dir,'lease'),token='1'.repeat(32),claim='2'.repeat(32);fs.mkdirSync(lease,{mode:0o700});fs.writeFileSync(path.join(lease,'lease.json'),JSON.stringify({token}),{mode:0o600});fs.writeFileSync(path.join(lease,'lease.token'),token+'\n',{mode:0o600});fs.writeFileSync(path.join(lease,'.recovery'),JSON.stringify({pid:process.pid,processIdentity:null,token:claim,createdAt:Date.now()})+'\n',{mode:0o600});fs.writeFileSync(docker,'#!/bin/sh\nif [ "$3" = ps ]; then exit 0; fi\nexit 0\n',{mode:0o755});const owner=spawn('/bin/sleep',['0.05'],{stdio:'ignore'}),child=spawn(watchdog,['--owner',String(owner.pid),'--deadline',String(Math.ceil((Date.now()+10_000)/1000)),'--run-dir',dir,'--docker',docker,'--endpoint',endpoint.uri,'--socket-device',String(endpoint.device),'--socket-inode',String(endpoint.inode),'--run-label','release-race','--lease-path',lease,'--lease-token',token],{stdio:'ignore'}),closed=new Promise<number|null>(resolve=>child.on('close',resolve));const outcome=await Promise.race([closed,Bun.sleep(5000).then(()=>-999)]);if(outcome===-999)child.kill('SIGKILL');expect(outcome).toBe(0);expect(fs.existsSync(lease)).toBe(false);expect(fs.readdirSync(dir).some(name=>name.includes('.watchdog-release-'))).toBe(false);expect(fs.readFileSync(path.join(dir,'watchdog.event'),'utf8')).toContain('cleanup complete');});
  test('attempt mode removes a patched execution copy after supervisor death',async()=>{const run=tmp(),watchdog=compile(run),control=path.join(run,'supervision','attempt'),work=path.join(run,'verification','attempt');fs.mkdirSync(control,{recursive:true});fs.mkdirSync(work,{recursive:true});fs.writeFileSync(path.join(work,'patched-source'),'sensitive execution copy');const owner=spawn('/bin/sleep',['1'],{stdio:'ignore'}),child=spawn(watchdog,['--attempt-owner',String(owner.pid),'--deadline',String(Math.ceil((Date.now()+10_000)/1000)),'--control-dir',control,'--work-root',work,'--run-root',run],{stdio:'ignore'}),closed=new Promise<number|null>(resolve=>child.on('close',resolve));expect(await closed).toBe(0);expect(fs.existsSync(work)).toBe(false);expect(fs.readFileSync(path.join(control,'attempt.event'),'utf8')).toContain('execution-copy cleanup complete');});
  test('attempt cleanup cannot remove the concurrent Docker watchdog journal',async()=>{const run=tmp(),watchdog=compile(run),endpoint=pinnedEndpoint(run),attemptControl=path.join(run,'supervision','attempt'),dockerControl=path.join(attemptControl,'docker-groups','before'),work=path.join(run,'verification','attempt'),lease=path.join(run,'lease'),log=path.join(dockerControl,'docker.log'),docker=path.join(dockerControl,'docker'),id='a'.repeat(64),race='e'.repeat(64),token='c'.repeat(32);for(const dir of [attemptControl,dockerControl,work,lease])fs.mkdirSync(dir,{recursive:true});fs.writeFileSync(path.join(work,'patched-source'),'sensitive execution copy');fs.writeFileSync(path.join(lease,'lease.json'),'{}');fs.writeFileSync(path.join(lease,'lease.token'),token+'\n');fs.writeFileSync(path.join(dockerControl,'resources.journal'),`container:${id}\n`,{mode:0o600});fs.writeFileSync(docker,`#!/bin/sh\nif [ "$3" = ps ]; then if [ "$8" = "id=${id}" ]; then printf '%s\\n' '${id}'; else printf '%s\\n' '${race}'; fi; exit 0; fi\nprintf '%s\\n' "$*" >> '${log}'\n`,{mode:0o755});const owner=spawn('/bin/sleep',['1'],{stdio:'ignore'}),attempt=spawn(watchdog,['--attempt-owner',String(owner.pid),'--deadline',String(Math.ceil((Date.now()+10_000)/1000)),'--control-dir',attemptControl,'--work-root',work,'--run-root',run],{stdio:'ignore'}),containers=spawn(watchdog,['--owner',String(owner.pid),'--deadline',String(Math.ceil((Date.now()+10_000)/1000)),'--run-dir',dockerControl,'--docker',docker,'--endpoint',endpoint.uri,'--socket-device',String(endpoint.device),'--socket-inode',String(endpoint.inode),'--run-label','test-label','--lease-path',lease,'--lease-token',token],{stdio:'ignore'});const closed=(child:any)=>new Promise<number|null>(resolve=>child.on('close',resolve));expect(await Promise.all([closed(attempt),closed(containers)])).toEqual([0,0]);expect(fs.existsSync(work)).toBe(false);expect(fs.readFileSync(log,'utf8')).toContain(`rm --force --volumes ${id}`);expect(fs.existsSync(lease)).toBe(false);expect(fs.readFileSync(path.join(dockerControl,'watchdog.event'),'utf8')).toContain('cleanup complete');});
  test('a terminal marker prevents any cleanup call',async()=>{const dir=tmp(),watchdog=compile(dir),endpoint=pinnedEndpoint(dir),log=path.join(dir,'docker.log'),docker=path.join(dir,'docker'),lease=path.join(dir,'lease'),token='d'.repeat(32);fs.mkdirSync(lease);fs.writeFileSync(path.join(lease,'lease.json'),'{}');fs.writeFileSync(path.join(lease,'lease.token'),token+'\n');fs.writeFileSync(docker,`#!/bin/sh\ntouch '${log}'\n`,{mode:0o755});fs.writeFileSync(path.join(dir,'resources.journal'),`container:${'b'.repeat(64)}\n`);fs.writeFileSync(path.join(dir,'watchdog.terminal'),'done\n');const r=spawnSync(watchdog,['--owner',String(process.pid),'--deadline',String(Math.ceil((Date.now()+10_000)/1000)),'--run-dir',dir,'--docker',docker,'--endpoint',endpoint.uri,'--socket-device',String(endpoint.device),'--socket-inode',String(endpoint.inode),'--run-label','test-label','--lease-path',lease,'--lease-token',token],{timeout:30_000});expect(r.status).toBe(0);expect(fs.existsSync(log)).toBe(false);});
  test('acknowledges terminal cleanup well inside the caller deadline',async()=>{const dir=tmp(),watchdog=compile(dir),endpoint=pinnedEndpoint(dir),docker=path.join(dir,'docker'),lease=path.join(dir,'lease'),token='9'.repeat(32),ready=path.join(dir,'watchdog.ready'),stopped=path.join(dir,'watchdog.stopped');fs.mkdirSync(lease);fs.writeFileSync(path.join(lease,'lease.json'),'{}');fs.writeFileSync(path.join(lease,'lease.token'),token+'\n');fs.writeFileSync(docker,'#!/bin/sh\nexit 0\n',{mode:0o755});const child=spawn(watchdog,['--owner',String(process.pid),'--deadline',String(Math.ceil((Date.now()+10_000)/1000)),'--run-dir',dir,'--docker',docker,'--endpoint',endpoint.uri,'--socket-device',String(endpoint.device),'--socket-inode',String(endpoint.inode),'--run-label','ack-latency','--lease-path',lease,'--lease-token',token],{stdio:'ignore'}),closed=new Promise<number|null>(resolve=>child.once('close',resolve));for(let i=0;i<100&&!fs.existsSync(ready);i++)await Bun.sleep(10);expect(fs.existsSync(ready)).toBe(true);await Bun.sleep(50);const started=Date.now();fs.writeFileSync(path.join(dir,'watchdog.terminal'),'done\n');for(let i=0;i<100&&!fs.existsSync(stopped);i++)await Bun.sleep(10);expect(fs.existsSync(stopped)).toBe(true);expect(Date.now()-started).toBeLessThan(500);expect(await closed).toBe(0);});
  test('a torn final journal row cannot retain the machine lease after exact label sweeps',async()=>{const dir=tmp(),watchdog=compile(dir),endpoint=pinnedEndpoint(dir),docker=path.join(dir,'docker'),lease=path.join(dir,'lease'),token='e'.repeat(32),id='a'.repeat(64);fs.mkdirSync(lease);fs.writeFileSync(path.join(lease,'lease.json'),'{}');fs.writeFileSync(path.join(lease,'lease.token'),token+'\n');fs.writeFileSync(docker,'#!/bin/sh\nif [ "$3" = ps ]; then exit 0; fi\nexit 0\n',{mode:0o755});fs.writeFileSync(path.join(dir,'resources.journal'),`container:${id}\ncontainer:abcd`,{mode:0o600});const owner=spawn('/bin/sleep',['1'],{stdio:'ignore'}),child=spawn(watchdog,['--owner',String(owner.pid),'--deadline',String(Math.ceil((Date.now()+10_000)/1000)),'--run-dir',dir,'--docker',docker,'--endpoint',endpoint.uri,'--socket-device',String(endpoint.device),'--socket-inode',String(endpoint.inode),'--run-label','test-label','--lease-path',lease,'--lease-token',token],{stdio:'ignore'});expect(await new Promise<number|null>(resolve=>child.on('close',resolve))).toBe(0);expect(fs.existsSync(lease)).toBe(false);expect(fs.readFileSync(path.join(dir,'watchdog.event'),'utf8')).toContain('malformed journal ignored after two exact label sweeps');});
  test('a replaced Docker socket blocks cleanup and lease release',async()=>{const dir=tmp(),watchdog=compile(dir),endpoint=pinnedEndpoint(dir),docker=path.join(dir,'docker'),lease=path.join(dir,'lease'),token='f'.repeat(32),log=path.join(dir,'docker.log');fs.mkdirSync(lease);fs.writeFileSync(path.join(lease,'lease.json'),'{}');fs.writeFileSync(path.join(lease,'lease.token'),token+'\n');fs.writeFileSync(docker,`#!/bin/sh\ntouch '${log}'\nexit 0\n`,{mode:0o755});const owner=spawn('/bin/sleep',['30'],{stdio:'ignore'}),child=spawn(watchdog,['--owner',String(owner.pid),'--deadline',String(Math.ceil((Date.now()+30_000)/1000)),'--run-dir',dir,'--docker',docker,'--endpoint',endpoint.uri,'--socket-device',String(endpoint.device),'--socket-inode',String(endpoint.inode),'--run-label','test-label','--lease-path',lease,'--lease-token',token],{stdio:'ignore'});for(let i=0;i<100&&!fs.existsSync(path.join(dir,'watchdog.ready'));i++)await Bun.sleep(10);expect(fs.existsSync(path.join(dir,'watchdog.ready'))).toBe(true);const socketPath=endpoint.uri.slice('unix://'.length);fs.renameSync(socketPath,`${socketPath}.old`);const replacement=pinnedEndpoint(dir);expect(replacement.inode).not.toBe(endpoint.inode);owner.kill('SIGKILL');for(let i=0;i<50&&!fs.existsSync(path.join(dir,'watchdog.event'));i++)await Bun.sleep(100);expect(fs.readFileSync(path.join(dir,'watchdog.event'),'utf8')).toContain('socket identity changed');expect(fs.existsSync(lease)).toBe(true);expect(fs.existsSync(log)).toBe(false);child.kill('SIGKILL');await new Promise(resolve=>child.once('close',resolve));});
});