v1.91.36.0 fix(cso): Python and Rails repairs replay, Rails verification fits its budget, killed attempts leave no scratch (#3074)

* fix(cso): Python and Rails repairs verify and replay; killed attempts leave no scratch

- Python offline preparation sets SOURCE_DATE_EPOCH so .pyc bytecode is reproducible.
- Prepared identity hashes skip RubyGems/mini_portile build diagnostics that name random build dirs.
- Rails verification attempts and replays get 900 s; the attempt watchdog, Docker
  verification executor and assertion witness share MAX_VERIFICATION_ATTEMPT_MS.
- resume and finish reclaim a killed attempt's scratch once its watchdog recorded cleanup.

v1.91.34.0

* Format lib/cso with prettier

* Take v1.91.36.0 (1.91.35.0 is claimed by #3073)

test/cso-cli-lifecycle.test.ts

@@ -45,12 +45,13 @@ afterEach(()=>fs.rmSync(root,{recursive:true,force:true}));
 describe('CSO interrupted-run and replay commands',()=>{
   test('resume reports watchdog recovery without replenishing the original policy or deadline',()=>{
     const run=JSON.parse(command(['start','--repo',repo,'--scope','auth','--offline','--budget','120']).stdout),dir=runDir(run),reportPath=path.join(dir,'report.json'),original=JSON.parse(fs.readFileSync(reportPath,'utf8'));
-    const recovery='supervisor-death execution-copy cleanup complete',dockerRecovery='deadline cleanup complete',control=path.join(dir,'supervision','repair-attempt'),preparationControl=path.join(dir,'preparation-execution','offline-attempt');fs.mkdirSync(control,{recursive:true});fs.mkdirSync(preparationControl,{recursive:true});fs.writeFileSync(path.join(control,'attempt.event'),`${recovery}\n`);fs.writeFileSync(path.join(preparationControl,'watchdog.event'),`${dockerRecovery}\n`);
+    const recovery='supervisor-death execution-copy cleanup complete',dockerRecovery='deadline cleanup complete',control=path.join(dir,'supervision','repair-attempt'),preparationControl=path.join(dir,'preparation-execution','offline-attempt');fs.mkdirSync(control,{recursive:true});fs.mkdirSync(preparationControl,{recursive:true});fs.writeFileSync(path.join(control,'attempt.event'),`${recovery}\n`);fs.writeFileSync(path.join(preparationControl,'watchdog.event'),`${dockerRecovery}\n`);const staging=path.join(dir,'archive-staging','acquire-partial');fs.mkdirSync(staging,{recursive:true});fs.writeFileSync(path.join(staging,'partial.tgz'),'partial');
     original.status='interrupted';saveReport(dir,original);
 
     const resumed=command(['resume',run.runId]);expect(resumed.status).toBe(0);const result=JSON.parse(resumed.stdout),active=JSON.parse(fs.readFileSync(reportPath,'utf8'));
     expect(result).toMatchObject({runId:run.runId,deadline:original.deadline,policy:original.policy,recovery:[recovery,dockerRecovery]});expect(active.status).toBe('running');expect(active.deadline).toBe(original.deadline);expect(active.policy).toEqual(original.policy);
     expect(active.events.filter((item:any)=>item.kind==='watchdog-recovery'&&[recovery,dockerRecovery].includes(item.message))).toHaveLength(2);expect(active.events.at(-1)).toMatchObject({kind:'resume',message:'Continued retained snapshot under original policy'});
+    for(const name of ['supervision','preparation-execution','archive-staging'])expect(fs.readdirSync(path.join(dir,name))).toEqual([]);
 
     active.status='interrupted';active.deadline=new Date(Date.now()-1_000).toISOString();const exhaustedDeadline=active.deadline;saveReport(dir,active);
     const expired=command(['resume',run.runId]);expect(expired.status).not.toBe(0);expect(expired.stderr).toContain('DEADLINE');expect(expired.stderr).toContain('Original run budget is exhausted');
@@ -82,3 +83,13 @@ describe('CSO interrupted-run and replay commands',()=>{
     fs.writeFileSync(path.join(repo,'app.js'),'module.exports = "vulnerable"\n');const supplied=command(['replay',verified.bundle.id,'--source',repo]);expect(supplied.status).not.toBe(0);expect(supplied.stderr).toContain('PREREQUISITE');expect(supplied.stderr).toContain('MISSING_QUALIFIED_RUNTIME');expect(supplied.stderr).not.toContain('does not match the bundle input hashes');
   },30_000);
 });
+
+describe('bounded verification attempts', () => {
+  test('a Rails attempt can prepare both phases; other stacks keep the five-minute bound', async () => {
+    const { verificationAttemptMs } = await import('../lib/cso/cli');
+    const { MAX_VERIFICATION_ATTEMPT_MS } = await import('../lib/cso/contracts');
+    for (const stack of ['node', 'bun', 'python']) expect(verificationAttemptMs(stack)).toBe(300_000);
+    expect(verificationAttemptMs('rails')).toBe(MAX_VERIFICATION_ATTEMPT_MS);
+    expect(MAX_VERIFICATION_ATTEMPT_MS).toBe(900_000);
+  });
+});

test/cso-preparation-executor.test.ts

@@ -10,8 +10,7 @@ import { inspectPreparation, type CsoStack } from '../lib/cso/preparation';
 import {
   admitPreparationRuntime, admitPreparationSidecar, PreparationExecutor,
   type AcquisitionReceipt, type OfflinePreparationReceipt, type PreparationAcquireRequest,
-  type PreparationSandboxRunner, type OfflinePreparationRequest,
-} from '../lib/cso/preparation-executor';
+  type PreparationSandboxRunner, type OfflinePreparationRequest, preparedIdentityEntry } from '../lib/cso/preparation-executor';
 import { CSO_HELPER_ABI } from '../lib/cso/runtime-catalog';
 import { DockerPreparationSandboxRunner, isBlockedRegistryAddress, RegistryEgressBroker } from '../lib/cso/preparation-docker';
 import { completeRuntimeCatalogFixture, qualifiedRuntimeFixture } from './helpers/cso-runtime-catalog';
@@ -191,6 +190,22 @@ describe('CSO constrained dependency preparation executor', () => {
     const before=await executor.prepareOffline({plan,admission,snapshot:source,closure,deadline});runner.mutatePrepared=root=>fs.writeFileSync(path.join(root,'node_modules','.cso-dependencies'),'changed toolchain bytes');const after=await executor.prepareOffline({plan,admission,snapshot:source,closure,deadline});expect(after.preparedDependencyHash).not.toBe(before.preparedDependencyHash);
   });
 
+  test('Rails build diagnostics with random build directories stay out of the prepared identity', () => {
+    for (const path of [
+      'vendor/bundle/ruby/3.4.0/extensions/x86_64-linux/3.4.0/date-3.5.1/gem_make.out',
+      'vendor/bundle/ruby/3.4.0/extensions/aarch64-linux/3.4.0/json-3.0.2/mkmf.log',
+      'vendor/bundle/ruby/3.4.0/gems/sqlite3-2.9.0/ext/sqlite3/tmp/x86_64-linux-gnu/ports/sqlite3/3.51.1/sqlite-autoconf-3510100/config.log',
+    ]) expect(preparedIdentityEntry('rails', path)).toBe(false);
+    for (const path of [
+      'vendor/bundle/ruby/3.4.0/extensions/x86_64-linux/3.4.0/date-3.5.1/date_core.so',
+      'vendor/bundle/ruby/3.4.0/extensions/x86_64-linux/3.4.0/date-3.5.1/gem.build_complete',
+      'vendor/bundle/ruby/3.4.0/gems/sqlite3-2.9.0/lib/sqlite3/sqlite3_native.so',
+      'vendor/bundle/ruby/3.4.0/gems/rails-8.1.2/lib/rails.rb',
+      'log/gem_make.out',
+    ]) expect(preparedIdentityEntry('rails', path)).toBe(true);
+    expect(preparedIdentityEntry('node', 'vendor/bundle/ruby/3.4.0/extensions/x86_64-linux/3.4.0/date-3.5.1/gem_make.out')).toBe(true);
+  });
+
   test('network-policy and exact-command receipt changes are rejected before promotion', async () => {
     const source = snapshot('node'), plan = inspectPreparation(source, 'node'), admission = admitPreparationRuntime({ plan, platform: 'linux/amd64', catalog: catalog('node') });
     const runner = new FakeRunner(), cache = cacheFixture(), executor = new PreparationExecutor({ cache, runner });

test/cso-preparation.test.ts

@@ -127,6 +127,12 @@ describe('CSO inert Python preparation', () => {
     expect(plan.acquisition[0].args).toContain('--only-binary=:all:'); expect(plan.acquisition[0].args).toContain('--require-hashes');
     expect(plan.metadata).toHaveLength(1); expect(plan.offline[1].args).toContain('--no-index');
   });
+  test('offline Python preparation writes reproducible checked-hash bytecode', () => {
+    const plan = inspectPreparation(fixture({ 'requirements.txt': `Flask==3.1.0 --hash=sha256:${hash}\n` }));
+    expect(plan.status).toBe('ready');
+    // venv/ensurepip and pip compile .pyc files; without a fixed epoch each preparation differs and replay cannot match.
+    for (const command of plan.offline) expect(command.env.SOURCE_DATE_EPOCH).toMatch(/^[0-9]+$/);
+  });
   for (const requirement of ['flask>=3', '-e .', 'flask @ https://evil.invalid/x.whl', '--index-url https://evil.invalid', '-r other.txt', 'flask==3.1.0']) test(`rejects unpinned or executable requirement ${requirement}`, () => {
     expect(inspectPreparation(fixture({ 'requirements.txt': requirement })).prerequisites[0].code).toBe('UNPINNED_REQUIREMENTS');
   });

test/cso-snapshot-state.test.ts

@@ -1,6 +1,6 @@
 import { afterAll, afterEach, describe, expect, spyOn, test } from 'bun:test';
 import * as fs from 'node:fs';import * as os from 'node:os';import * as path from 'node:path';import { spawn, spawnSync } from 'node:child_process';
-import { assertSnapshot, capture } from '../lib/cso/snapshot';import { CsoError, snapshotPathHandle } from '../lib/cso/contracts';import { runProcess, sanitizeForJson, sanitizeHelperForJson } from '../lib/cso/process';import { discardAtomicNoReplaceTemp, finalizeReplayTemporary, loadReport, newRun, privateRoot, readJson, recoverAtomicNoReplaceJson, retention, saveReport, secureDirectory, stateRoot, withLock, writeHelperJson, writeJson } from '../lib/cso/state';
+import { assertSnapshot, capture } from '../lib/cso/snapshot';import { CsoError, snapshotPathHandle } from '../lib/cso/contracts';import { runProcess, sanitizeForJson, sanitizeHelperForJson } from '../lib/cso/process';import { discardAtomicNoReplaceTemp, finalizeReplayTemporary,reclaimDeadAttemptScratch, loadReport, newRun, privateRoot, readJson, recoverAtomicNoReplaceJson, retention, saveReport, secureDirectory, stateRoot, withLock, writeHelperJson, writeJson } from '../lib/cso/state';
 const roots:string[]=[];const tmp=()=>{const p=fs.mkdtempSync(path.join(os.tmpdir(),'cso-snapshot-'));roots.push(p);return p;};afterEach(()=>{for(const p of roots.splice(0))fs.rmSync(p,{recursive:true,force:true});});
 const originalState=process.env.GSTACK_HOME,state=fs.mkdtempSync(path.join(os.tmpdir(),'cso-state-'));process.env.GSTACK_HOME=state;afterAll(()=>{if(originalState===undefined)delete process.env.GSTACK_HOME;else process.env.GSTACK_HOME=originalState;fs.rmSync(state,{recursive:true,force:true});});
 function git(repo:string,...args:string[]){const r=spawnSync('/usr/bin/git',['-C',repo,...args],{encoding:'utf8',env:{PATH:'/usr/bin:/bin',HOME:repo},timeout:30_000});if(r.status)throw new Error(r.stderr);return r.stdout;}
@@ -51,6 +51,7 @@ describe('CSO dirty snapshot boundary',()=>{
   test('bounded retention completes discovery before deleting any run',()=>{const now=Date.now(),repo='f'.repeat(24),run=`${now-31*86400_000}-${'9'.repeat(16)}`,dir=secureDirectory(path.join(privateRoot(),repo,run));fs.writeFileSync(path.join(dir,'report.json'),'expired private state\n',{mode:0o600});const result=retention(now,{deadlineMs:now+60_000,maxEntries:6});expect(result).toEqual({complete:false,visited:6});expect(fs.existsSync(dir)).toBe(true);retention(now);expect(fs.existsSync(dir)).toBe(false);});
   test('bounded retention resumes an interrupted tree deletion without one recursive walk',()=>{const previous=process.env.GSTACK_HOME,isolate=tmp();process.env.GSTACK_HOME=isolate;try{const now=Date.now(),repo='1'.repeat(24),run=`${now-31*86400_000}-${'a'.repeat(16)}`,repoDir=path.join(privateRoot(),repo),dir=secureDirectory(path.join(repoDir,run)),snapshot=secureDirectory(path.join(dir,'snapshot'));for(let index=0;index<20;index++)fs.writeFileSync(path.join(snapshot,`${index}.txt`),'private state\n',{mode:0o600});const result=retention(now,{deadlineMs:now+60_000,maxEntries:30});expect(result).toEqual({complete:false,visited:30});expect(fs.existsSync(dir)).toBe(false);expect(fs.readdirSync(repoDir).some(name=>name.startsWith(`.retired-${run}-`))).toBe(true);retention(now);expect(fs.readdirSync(repoDir).some(name=>name.startsWith(`.retired-${run}-`))).toBe(false);}finally{if(previous===undefined)delete process.env.GSTACK_HOME;else process.env.GSTACK_HOME=previous;}});
   test('repair bundles expire from their authenticated bundle time, independent of run age',()=>{const now=Date.now(),repo='c'.repeat(24),run=`${now-31*86400_000}-${'2'.repeat(16)}`,dir=secureDirectory(path.join(privateRoot(),repo,run)),bundles=secureDirectory(path.join(dir,'bundles')),expiredId='3'.repeat(32),retainedId='4'.repeat(32),record=(id:string,created:number)=>{const createdAt=new Date(created).toISOString();return{schemaVersion:3,id,runId:run,createdAt,expiresAt:new Date(created+30*86400_000).toISOString(),verification:{id,runId:run,createdAt}};};writeHelperJson(path.join(bundles,`${expiredId}.json`),record(expiredId,now-31*86400_000));writeHelperJson(path.join(bundles,`${retainedId}.json`),record(retainedId,now-86400_000));retention(now);expect(fs.existsSync(path.join(bundles,`${expiredId}.json`))).toBe(false);expect(fs.existsSync(path.join(bundles,`${retainedId}.json`))).toBe(true);expect(fs.existsSync(dir)).toBe(true);retention(now+30*86400_000);expect(fs.existsSync(dir)).toBe(false);});
+  test('a killed attempt\'s scratch is reclaimed only after its detached watchdog records cleanup',()=>{const now=Date.now(),repo='c'.repeat(24),run=`${now}-${'6'.repeat(16)}`,dir=secureDirectory(path.join(privateRoot(),repo,run)),call=secureDirectory(path.join(dir,'preparation-execution','acquire-attempt')),staging=secureDirectory(path.join(dir,'archive-staging','acquire-partial')),kept=path.join(dir,'report.json');fs.writeFileSync(kept,'{}',{mode:0o600});fs.writeFileSync(path.join(staging,'partial.tgz'),'partial',{mode:0o600});fs.writeFileSync(path.join(call,'watchdog.ready'),'ready\n',{mode:0o600});fs.writeFileSync(path.join(call,'watchdog.event'),'cleanup incomplete; retrying exact journaled resources\n',{mode:0o600});expect(reclaimDeadAttemptScratch(dir)).toBe(false);expect(fs.existsSync(call)).toBe(true);expect(fs.existsSync(staging)).toBe(true);fs.writeFileSync(path.join(call,'watchdog.event'),'supervisor-death cleanup complete\n',{mode:0o600});expect(reclaimDeadAttemptScratch(dir)).toBe(true);for(const name of ['preparation-execution','archive-staging'])expect(fs.readdirSync(path.join(dir,name))).toEqual([]);expect(fs.existsSync(kept)).toBe(true);fs.rmSync(path.join(dir,'archive-staging'),{recursive:true});fs.symlinkSync(os.tmpdir(),path.join(dir,'archive-staging'));expect(()=>reclaimDeadAttemptScratch(dir)).toThrow('Attempt scratch is not a private directory');});
   test('replay temporary state survives cleanup failure until every detached watchdog acknowledges cleanup',()=>{const now=Date.now(),repo='b'.repeat(24),run=`${now}-${'5'.repeat(16)}`,dir=secureDirectory(path.join(privateRoot(),repo,run)),control=secureDirectory(path.join(dir,'supervision','repair-attempt')),docker=secureDirectory(path.join(dir,'preparation-execution','offline-attempt'));fs.writeFileSync(path.join(control,'attempt.ready'),'ready\n',{mode:0o600});fs.writeFileSync(path.join(control,'attempt.terminal'),'normal cleanup complete\n',{mode:0o600});fs.writeFileSync(path.join(docker,'watchdog.ready'),'ready\n',{mode:0o600});fs.writeFileSync(path.join(docker,'watchdog.event'),'cleanup incomplete; retrying exact journaled resources\n',{mode:0o600});finalizeReplayTemporary(dir);expect(fs.existsSync(path.join(dir,'.ephemeral-replay.json'))).toBe(true);retention(now);expect(fs.existsSync(control)).toBe(true);expect(fs.existsSync(docker)).toBe(true);fs.writeFileSync(path.join(control,'attempt.stopped'),'normal cleanup acknowledged\n',{mode:0o600});retention(now);expect(fs.existsSync(dir)).toBe(true);fs.writeFileSync(path.join(docker,'watchdog.stopped'),'normal cleanup acknowledged\n',{mode:0o600});retention(now);expect(fs.existsSync(dir)).toBe(false);});
   test('retention recovers a hidden run retirement left by a dead helper',()=>{const now=Date.now(),repo='a'.repeat(24),run=`${now-31*86400_000}-${'6'.repeat(16)}`,retired=secureDirectory(path.join(privateRoot(),repo,`.retired-${run}-${'7'.repeat(32)}`));fs.writeFileSync(path.join(retired,'payload'),'private state\n',{mode:0o600});retention(now);expect(fs.existsSync(retired)).toBe(false);});
 });

test/cso-witness.test.ts

@@ -4,9 +4,9 @@ import * as os from 'node:os';
 import * as path from 'node:path';
 import { spawnSync } from 'node:child_process';
 import { generateKeyPairSync } from 'node:crypto';
-import { AssertionWitnessBinding, CsoError, VerificationObservation, canonical, sha256 } from '../lib/cso/contracts';
+import { AssertionWitnessBinding, CsoError, MAX_VERIFICATION_ATTEMPT_MS, VerificationObservation, canonical, sha256 } from '../lib/cso/contracts';
 import { canonicalStartPlan, canonicalTestPlan, patchHash, treeHash, validateRepairBundle, verifyRepair } from '../lib/cso/verification';
-import { AssertionWitnessSession, assertionWitnessChildCommand, assertionWitnessReplayHash, testExecutionPassed, validateStoredAssertionWitnessReceipt } from '../lib/cso/witness';
+import { AssertionWitnessSession, assertionWitnessChildCommand, assertionWitnessReplayHash, testExecutionPassed, validateAssertionWitnessBinding, validateStoredAssertionWitnessReceipt } from '../lib/cso/witness';
 
 const roots:string[]=[];
 const temporary=()=>{const root=fs.mkdtempSync(path.join(os.tmpdir(),'cso-witness-'));roots.push(root);return root;};
@@ -33,6 +33,14 @@ function stable(phase:'before'|'after'):Omit<AssertionWitnessBinding,'schemaVers
 }
 
 describe('CSO authenticated external assertion witness',()=>{
+  test('a challenge lives as long as its attempt, never past the longest attempt bound',()=>{
+    const now=Date.now(),lifetime=(deadline:number)=>{const binding=new AssertionWitnessSession(temporary(),deadline).handle(stable('before')).binding;return Date.parse(binding.expiresAt)-Date.parse(binding.issuedAt);};
+    expect(lifetime(now+300_000)).toBeLessThanOrEqual(300_000);
+    expect(lifetime(now+MAX_VERIFICATION_ATTEMPT_MS)).toBeGreaterThan(300_000);
+    expect(lifetime(now+2*MAX_VERIFICATION_ATTEMPT_MS)).toBeLessThanOrEqual(MAX_VERIFICATION_ATTEMPT_MS);
+    const binding=new AssertionWitnessSession(temporary(),now+60_000).handle(stable('before')).binding;
+    expect(()=>validateAssertionWitnessBinding({...binding,expiresAt:new Date(Date.parse(binding.issuedAt)+MAX_VERIFICATION_ATTEMPT_MS+1).toISOString()})).toThrow('lifetime exceeds');
+  });
   test('rejects forged, stale, and mismatched receipts after a valid out-of-process attestation',async()=>{
     const work=temporary(),session=new AssertionWitnessSession(work,Date.now()+60_000),handle=session.handle(stable('before')),observation:VerificationObservation={booted:true,legitimate:true,security:'intended_failure',existingTests:false,output:'external verifier passed',inputHash:''},command={executable:'/usr/local/bin/node',args:['--test','--test-reporter=tap','./app.test.js']};
     const receipt=await handle.attest(observation,[{command,code:0,output:tap,minimumPassingTests:1}]);