/* Independent CSO lease watchdog. It accepts only exact journaled container IDs. */ #ifdef __APPLE__ #define _DARWIN_C_SOURCE 1 #endif #define _XOPEN_SOURCE 700 #include #include #include #include #include #include #include #include #include #include #include #include static int safe_token(const char *s, size_t min, size_t max) { size_t n = s ? strlen(s) : 0; if (n < min || n > max) return 0; for (size_t i=0;i='a'&&s[i]<='z')||(s[i]>='A'&&s[i]<='Z')||(s[i]>='0'&&s[i]<='9')||strchr("._-/:@",s[i]))) return 0; return 1; } static int is_id(const char *s) { if (!s || strlen(s)!=64) return 0; for (size_t i=0;i<64;i++) if (!((s[i]>='a'&&s[i]<='f')||(s[i]>='0'&&s[i]<='9'))) return 0; return 1; } static int is_token(const char *s){ if(!s||strlen(s)!=32)return 0; for(size_t i=0;i<32;i++)if(!((s[i]>='a'&&s[i]<='f')||(s[i]>='0'&&s[i]<='9')))return 0; return 1; } static int safe_path(const char *s){ if(!s||s[0]!='/'||strstr(s,"/../")||strstr(s,"//"))return 0; size_t n=strlen(s);if(n>4095||(n>=3&&!strcmp(s+n-3,"/..")))return 0; for(size_t i=0;i 1 && (kill(pid,0)==0 || errno==EPERM); } static unsigned long long process_start(pid_t pid){ #ifdef __linux__ char path[64],line[4096];if(snprintf(path,sizeof path,"/proc/%ld/stat",(long)pid)>=(int)sizeof path)return 0;FILE *f=fopen(path,"r");if(!f)return 0;if(!fgets(line,sizeof line,f)){fclose(f);return 0;}fclose(f);char *cursor=strrchr(line,')');if(!cursor||cursor[1]!=' ')return 0;cursor+=2;char *save=NULL,*token=strtok_r(cursor," ",&save);for(int field=3;token;field++,token=strtok_r(NULL," ",&save))if(field==22){char *end=NULL;unsigned long long value=strtoull(token,&end,10);return end&&(*end=='\0'||*end=='\n')?value:0;}return 0; #else (void)pid;return 0; #endif } static int same_owner(pid_t pid,unsigned long long started){if(!owner_alive(pid))return 0;unsigned long long current=process_start(pid);return !started||!current||started==current;} static int wait_bounded(pid_t child,int *status,int seconds){ struct timespec delay={0,100000000}; for(int i=0;i=(int)sizeof output)return -1; int fd=open(output,O_WRONLY|O_CREAT|O_EXCL|O_NOFOLLOW,0600);if(fd<0)return -1; pid_t child=fork();if(child<0){close(fd);unlink(output);return -1;} if(child==0){ if(dup2(fd,STDOUT_FILENO)<0)_exit(127); close(fd); int nullfd=open("/dev/null",O_WRONLY);if(nullfd>=0){(void)dup2(nullfd,STDERR_FILENO);close(nullfd);} char filter[80];if(snprintf(filter,sizeof filter,"id=%s",id)>=(int)sizeof filter)_exit(127); char *const argv[]={(char*)docker,"--host",(char*)endpoint,"ps","--all","--quiet","--no-trunc","--filter",filter,NULL}; char *const envp[]={"PATH=/usr/local/bin:/usr/bin:/bin:/opt/homebrew/bin",NULL};execve(docker,argv,envp);_exit(127); } close(fd);int status=0;if(wait_bounded(child,&status,10)!=0){unlink(output);return -1;} if(!WIFEXITED(status)||WEXITSTATUS(status)!=0){unlink(output);return -1;} fd=open(output,O_RDONLY|O_NOFOLLOW);if(fd<0){unlink(output);return -1;}char value[128]={0};ssize_t n=read(fd,value,sizeof value-1);close(fd);unlink(output);if(n<0)return -1; value[strcspn(value,"\r\n")]=0;if(value[0]==0)return 0;return strcmp(value,id)==0?1:-1; } static int remove_container(const char *docker, const char *endpoint, const char *id) { pid_t child=fork(); if(child<0)return -1; if(child==0){ char *const argv[]={(char*)docker,"--host",(char*)endpoint,"rm","--force","--volumes",(char*)id,NULL}; char *const envp[]={"PATH=/usr/local/bin:/usr/bin:/bin:/opt/homebrew/bin",NULL}; execve(docker,argv,envp); _exit(127); } int status=0;if(wait_bounded(child,&status,10)!=0)return -1; return WIFEXITED(status)&&WEXITSTATUS(status)==0?0:-1; } static int cleanup_label(const char *run_dir,const char *docker,const char *endpoint,const char *label){ char output[4096],filter[160];if(snprintf(output,sizeof output,"%s/watchdog.labels.%ld",run_dir,(long)getpid())>=(int)sizeof output||snprintf(filter,sizeof filter,"label=com.gstack.cso.run=%s",label)>=(int)sizeof filter)return -1; int fd=open(output,O_WRONLY|O_CREAT|O_EXCL|O_NOFOLLOW,0600);if(fd<0)return -1;pid_t child=fork();if(child<0){close(fd);unlink(output);return -1;} if(child==0){if(dup2(fd,STDOUT_FILENO)<0)_exit(127);close(fd);int nullfd=open("/dev/null",O_WRONLY);if(nullfd>=0){(void)dup2(nullfd,STDERR_FILENO);close(nullfd);}char *const argv[]={(char*)docker,"--host",(char*)endpoint,"ps","--all","--quiet","--no-trunc","--filter",filter,NULL};char *const envp[]={"PATH=/usr/local/bin:/usr/bin:/bin:/opt/homebrew/bin",NULL};execve(docker,argv,envp);_exit(127);} close(fd);int status=0;if(wait_bounded(child,&status,10)!=0||!WIFEXITED(status)||WEXITSTATUS(status)!=0){unlink(output);return -1;} FILE *f=fopen(output,"r");if(!f){unlink(output);return -1;}char line[256];int failed=0;while(fgets(line,sizeof line,f)){line[strcspn(line,"\r\n")]=0;if(!is_id(line)||remove_container(docker,endpoint,line)!=0)failed=1;}fclose(f);unlink(output);return failed?-1:0; } static int cleanup(const char *run_dir,const char *docker,const char *endpoint,const char *label){ char journal[4096]; if(snprintf(journal,sizeof journal,"%s/resources.journal",run_dir)>=(int)sizeof journal)return -1; FILE *f=fopen(journal,"r");char line[256];int malformed=0; if(f){while(fgets(line,sizeof line,f)){size_t length=strlen(line);int complete=length>0&&(line[length-1]=='\n'||feof(f));line[strcspn(line,"\r\n")]=0;if(!complete||strncmp(line,"container:",10)!=0||!is_id(line+10)){malformed=1;continue;}int present=resource_present(run_dir,docker,endpoint,line+10);if(present>0)(void)remove_container(docker,endpoint,line+10);}if(ferror(f))malformed=1;fclose(f);}else if(errno!=ENOENT)malformed=1; /* The journal is an optimization and may end in a torn append. Two exact * label sweeps are the authoritative cleanup proof after owner death. */ if(cleanup_label(run_dir,docker,endpoint,label)!=0)return -1; if(cleanup_label(run_dir,docker,endpoint,label)!=0)return -1; return malformed?1:0; } static int remove_tree(const char *path); static int owned_regular(const char *path){struct stat st;return !lstat(path,&st)&&S_ISREG(st.st_mode)&&st.st_uid==getuid()&&st.st_nlink==1;} static int lease_token_matches(const char *dir,const char *token){ char path[4096];if(snprintf(path,sizeof path,"%s/lease.token",dir)>=(int)sizeof path)return -1; struct stat before;if(lstat(path,&before))return errno==ENOENT?0:-1;if(!S_ISREG(before.st_mode)||before.st_uid!=getuid()||before.st_nlink!=1||before.st_size<=0||before.st_size>64)return -1; int fd=open(path,O_RDONLY|O_NOFOLLOW);if(fd<0)return -1;struct stat opened;if(fstat(fd,&opened)||opened.st_dev!=before.st_dev||opened.st_ino!=before.st_ino||opened.st_nlink!=1){close(fd);return -1;}char value[64]={0};ssize_t n=read(fd,value,sizeof value-1);struct stat after;int bad=fstat(fd,&after)||after.st_dev!=opened.st_dev||after.st_ino!=opened.st_ino||after.st_nlink!=1;close(fd);if(n<0||bad)return -1;value[strcspn(value,"\r\n")]=0;return strcmp(value,token)==0?1:-1; } static int cleanup_release_tomb(const char *tomb,const char *token){ struct stat root;if(lstat(tomb,&root))return errno==ENOENT?0:-1;if(!S_ISDIR(root.st_mode)||root.st_uid!=getuid())return -1; int authenticated=lease_token_matches(tomb,token);if(authenticated<0)return -1; /* A deterministic token-qualified tomb is itself the recovery capability * after a crash or a concurrent authenticated TS release removed the token. * Delete the token last so ordinary interrupted cleanup remains verifiable. */ DIR *directory=opendir(tomb);if(!directory)return -1;struct dirent *entry;int failed=0; while((entry=readdir(directory))){const char *name=entry->d_name;if(!strcmp(name,".")||!strcmp(name,"..")||!strcmp(name,"lease.token"))continue;char child[4096];if(snprintf(child,sizeof child,"%s/%s",tomb,name)>=(int)sizeof child){failed=1;break;} if(!strcmp(name,".recovery")){struct stat st;if(lstat(child,&st)||st.st_uid!=getuid()){failed=1;break;}if(S_ISREG(st.st_mode)&&st.st_nlink==1){if(unlink(child)&&errno!=ENOENT){failed=1;break;}}else if(S_ISDIR(st.st_mode)){if(remove_tree(child)!=0){failed=1;break;}}else{failed=1;break;}continue;} if(strcmp(name,"lease.json")&&strncmp(name,".recovery.tmp.",14)&&strncmp(name,"lease.json.tmp.",15)){failed=1;break;} if(!owned_regular(child)||(unlink(child)&&errno!=ENOENT)){failed=1;break;} } if(closedir(directory))failed=1; if(failed)return -1; char token_path[4096];if(snprintf(token_path,sizeof token_path,"%s/lease.token",tomb)>=(int)sizeof token_path)return -1; if(authenticated>0&&unlink(token_path)&&errno!=ENOENT)return -1; if(rmdir(tomb)&&errno!=ENOENT)return -1; return 0; } static int release_lease(const char *path,const char *token){ if(!safe_path(path)||!is_token(token))return -1; char tomb[4096];if(snprintf(tomb,sizeof tomb,"%s.watchdog-release-%s",path,token)>=(int)sizeof tomb)return -1; struct stat tomb_st;if(!lstat(tomb,&tomb_st))return cleanup_release_tomb(tomb,token);if(errno!=ENOENT)return -1; struct stat observed;if(lstat(path,&observed))return errno==ENOENT?0:-1;if(!S_ISDIR(observed.st_mode)||observed.st_uid!=getuid())return -1; if(lease_token_matches(path,token)!=1)return -1; if(rename(path,tomb)){if(errno==ENOENT&&!lstat(tomb,&tomb_st))return cleanup_release_tomb(tomb,token);return -1;} struct stat moved;if(lstat(tomb,&moved)||!S_ISDIR(moved.st_mode)||moved.st_dev!=observed.st_dev||moved.st_ino!=observed.st_ino||moved.st_uid!=observed.st_uid)return -1; return cleanup_release_tomb(tomb,token); } static void marker(const char *run_dir,const char *name,const char *value){ char path[4096],tmp[4096]; if(snprintf(path,sizeof path,"%s/%s",run_dir,name)>=(int)sizeof path)return; if(snprintf(tmp,sizeof tmp,"%s/%s.tmp.%ld",run_dir,name,(long)getpid())>=(int)sizeof tmp)return; int fd=open(tmp,O_WRONLY|O_CREAT|O_EXCL|O_NOFOLLOW,0600);if(fd<0)return; size_t remaining=strlen(value);const char *cursor=value;int failed=0; while(remaining){ssize_t written=write(fd,cursor,remaining);if(written>0){cursor+=written;remaining-=(size_t)written;continue;}if(written<0&&errno==EINTR)continue;failed=1;break;} if(!failed&&fsync(fd))failed=1; if(close(fd))failed=1; if(!failed&&!rename(tmp,path))return; (void)unlink(tmp); } static int descendant(const char *root,const char *path){size_t n=strlen(root);return n>1&&!strncmp(root,path,n)&&path[n]=='/';} static int remove_tree(const char *path){ struct stat st;if(lstat(path,&st)){return errno==ENOENT?0:-1;}if(!S_ISDIR(st.st_mode)||st.st_uid!=getuid())return -1; pid_t child=fork();if(child<0)return -1;if(child==0){char *const args[]={"/bin/rm","-rf","--",(char*)path,NULL};char *const envp[]={"PATH=/usr/bin:/bin",NULL};execve("/bin/rm",args,envp);_exit(127);}int status=0;if(wait_bounded(child,&status,10)!=0)return -1;return WIFEXITED(status)&&WEXITSTATUS(status)==0?0:-1; } static int attempt_watchdog(int argc,char **argv){ if(argc!=11||strcmp(argv[1],"--attempt-owner")||strcmp(argv[3],"--deadline")||strcmp(argv[5],"--control-dir")||strcmp(argv[7],"--work-root")||strcmp(argv[9],"--run-root"))return 64; char *end=NULL;long owner_l=strtol(argv[2],&end,10);if(!end||*end||owner_l<=1)return 64;end=NULL;long long deadline=strtoll(argv[4],&end,10);if(!end||*end||deadline<=0)return 64; const char *control=argv[6],*work=argv[8],*run=argv[10];if(!safe_path(control)||!safe_path(work)||!safe_path(run)||!descendant(run,control)||!descendant(run,work))return 64; char real_run[4096],real_control[4096],real_work[4096];if(!realpath(run,real_run)||!realpath(control,real_control)||!realpath(work,real_work)||strcmp(real_run,run)||strcmp(real_control,control)||strcmp(real_work,work)||!descendant(real_run,real_control)||!descendant(real_run,real_work))return 64; struct stat st;if(lstat(control,&st)||!S_ISDIR(st.st_mode)||st.st_uid!=getuid()||lstat(work,&st)||!S_ISDIR(st.st_mode)||st.st_uid!=getuid())return 64; char terminal[4096];if(snprintf(terminal,sizeof terminal,"%s/attempt.terminal",control)>=(int)sizeof terminal)return 64;unsigned long long owner_start=process_start((pid_t)owner_l);marker(control,"attempt.ready","ready\n"); for(;;){if(access(terminal,F_OK)==0){marker(control,"attempt.stopped","normal cleanup acknowledged\n");return 0;}time_t now=time(NULL);int alive=same_owner((pid_t)owner_l,owner_start);if(!alive||(long long)now>=deadline){if(remove_tree(work)==0){marker(control,"attempt.event",!alive?"supervisor-death execution-copy cleanup complete\n":"deadline execution-copy cleanup complete\n");return 0;}marker(control,"attempt.event","execution-copy cleanup incomplete; retrying\n");}struct timespec delay={0,100000000};while(nanosleep(&delay,&delay)&&errno==EINTR){}delay.tv_sec=0;delay.tv_nsec=100000000;} } static int ephemeral_watchdog(int argc,char **argv){ if(argc!=11||strcmp(argv[1],"--ephemeral-owner")||strcmp(argv[3],"--deadline")||strcmp(argv[5],"--control-dir")||strcmp(argv[7],"--work-root")||strcmp(argv[9],"--run-root"))return 64; char *end=NULL;long owner_l=strtol(argv[2],&end,10);if(!end||*end||owner_l<=1)return 64;end=NULL;long long deadline=strtoll(argv[4],&end,10);if(!end||*end||deadline<=0)return 64; const char *control=argv[6],*work=argv[8],*run=argv[10];if(!safe_path(control)||!safe_path(work)||!safe_path(run)||!descendant(run,work)||!descendant(work,control))return 64; char real_run[4096],real_control[4096],real_work[4096];if(!realpath(run,real_run)||!realpath(control,real_control)||!realpath(work,real_work)||strcmp(real_run,run)||strcmp(real_control,control)||strcmp(real_work,work)||!descendant(real_run,real_work)||!descendant(real_work,real_control))return 64; struct stat control_st,work_st;if(lstat(control,&control_st)||!S_ISDIR(control_st.st_mode)||control_st.st_uid!=getuid()||lstat(work,&work_st)||!S_ISDIR(work_st.st_mode)||work_st.st_uid!=getuid())return 64; char terminal[4096];if(snprintf(terminal,sizeof terminal,"%s/attempt.terminal",control)>=(int)sizeof terminal)return 64;unsigned long long owner_start=process_start((pid_t)owner_l);marker(control,"attempt.ready","ready\n"); for(;;){ int terminal_requested=access(terminal,F_OK)==0;time_t now=time(NULL);int alive=same_owner((pid_t)owner_l,owner_start); if(terminal_requested||!alive||(long long)now>=deadline){ struct stat current;if(lstat(work,¤t)){if(errno==ENOENT)return 0;marker(control,"attempt.event","ephemeral cleanup incomplete; retrying\n");} else if(!S_ISDIR(current.st_mode)||current.st_uid!=getuid()||current.st_dev!=work_st.st_dev||current.st_ino!=work_st.st_ino){marker(control,"attempt.event","ephemeral cleanup identity changed; refusing removal\n");} else if(remove_tree(work)==0)return 0; else marker(control,"attempt.event","ephemeral cleanup incomplete; retrying\n"); } struct timespec delay={0,100000000};while(nanosleep(&delay,&delay)&&errno==EINTR){}delay.tv_sec=0;delay.tv_nsec=100000000; } } int main(int argc,char **argv){ if(argc>1&&!strcmp(argv[1],"--attempt-owner"))return attempt_watchdog(argc,argv); if(argc>1&&!strcmp(argv[1],"--ephemeral-owner"))return ephemeral_watchdog(argc,argv); if(argc!=21||strcmp(argv[1],"--owner")||strcmp(argv[3],"--deadline")||strcmp(argv[5],"--run-dir")||strcmp(argv[7],"--docker")||strcmp(argv[9],"--endpoint")||strcmp(argv[11],"--socket-device")||strcmp(argv[13],"--socket-inode")||strcmp(argv[15],"--run-label")||strcmp(argv[17],"--lease-path")||strcmp(argv[19],"--lease-token"))return 64; char *end=NULL;long owner_l=strtol(argv[2],&end,10);if(!end||*end||owner_l<=1)return 64; end=NULL;long long deadline=strtoll(argv[4],&end,10);if(!end||*end||deadline<=0)return 64; end=NULL;unsigned long long socket_device=strtoull(argv[12],&end,10);if(!end||*end)return 64; end=NULL;unsigned long long socket_inode=strtoull(argv[14],&end,10);if(!end||*end||!socket_inode)return 64; const char *run_dir=argv[6],*docker=argv[8],*endpoint=argv[10],*label=argv[16],*lease_path=argv[18],*lease_token=argv[20]; if(!safe_path(run_dir)||!safe_path(docker)||!safe_token(label,1,100)||!safe_path(lease_path)||!is_token(lease_token)||strncmp(endpoint,"unix:///",8)||!safe_path(endpoint+7))return 64; struct stat docker_st;if(lstat(docker,&docker_st)||!S_ISREG(docker_st.st_mode)||(docker_st.st_mode&0111)==0)return 64; struct stat st;if(lstat(run_dir,&st)||!S_ISDIR(st.st_mode)||st.st_uid!=getuid())return 64; const char *socket_path=endpoint+7;if(lstat(socket_path,&st)||!S_ISSOCK(st.st_mode)||(unsigned long long)st.st_dev!=socket_device||(unsigned long long)st.st_ino!=socket_inode)return 64; char terminal[4096];if(snprintf(terminal,sizeof terminal,"%s/watchdog.terminal",run_dir)>=(int)sizeof terminal)return 64; unsigned long long owner_start=process_start((pid_t)owner_l);marker(run_dir,"watchdog.ready","ready\n"); for(;;){ while(waitpid(-1,NULL,WNOHANG)>0){} if(access(terminal,F_OK)==0){marker(run_dir,"watchdog.stopped","normal cleanup acknowledged\n");return 0;} time_t now=time(NULL); int alive=same_owner((pid_t)owner_l,owner_start);if(!alive||(long long)now>=deadline){ if(lstat(socket_path,&st)||!S_ISSOCK(st.st_mode)||(unsigned long long)st.st_dev!=socket_device||(unsigned long long)st.st_ino!=socket_inode){marker(run_dir,"watchdog.event","Docker socket identity changed; cleanup and lease release blocked\n");continue;} int cleaned=cleanup(run_dir,docker,endpoint,label); if(cleaned>=0&&(!lstat(socket_path,&st)&&S_ISSOCK(st.st_mode)&&(unsigned long long)st.st_dev==socket_device&&(unsigned long long)st.st_ino==socket_inode)&&release_lease(lease_path,lease_token)==0){if(cleaned>0)marker(run_dir,"watchdog.event",!alive?"supervisor-death cleanup complete; malformed journal ignored after two exact label sweeps\n":"deadline cleanup complete; malformed journal ignored after two exact label sweeps\n");else marker(run_dir,"watchdog.event",!alive?"supervisor-death cleanup complete\n":"deadline cleanup complete\n");return 0;} marker(run_dir,"watchdog.event","cleanup incomplete; retrying exact journaled resources\n"); } struct timespec delay={0,100000000};while(nanosleep(&delay,&delay)&&errno==EINTR){} } }