import { afterAll, expect, test } from 'bun:test'; import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { spawn, spawnSync } from 'node:child_process'; import { randomBytes } from 'node:crypto'; const CLI = path.resolve(import.meta.dir, '../bin/gstack-qa-evidence'); const ROOT = fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), 'qa-evidence-')); afterAll(() => fs.rmSync(ROOT, { recursive: true, force: true })); function fixture() { const root = fs.mkdtempSync(path.join(ROOT, 'case-')); const run = (...args: string[]) => { expect(JSON.stringify([process.execPath, CLI, ...args]).length, 'Fixture argv must stay short for Windows process launch').toBeLessThan(8192); const result = spawnSync(process.execPath, [CLI, ...args], { cwd: root, encoding: 'utf8', timeout: 10_000 }); expect(result.error, result.error?.message).toBeUndefined(); return result; }; const json = (name: string, value: unknown) => fs.writeFileSync(path.join(root, name), JSON.stringify(value), { mode: 0o600 }); const capture = (id: string, program: string, timeout = '4000') => run('capture', root, id, '--timeout-ms', timeout, '--', process.execPath, '-e', program); return { root, run, json, capture }; } function receipt(text: string) { expect(text.trim().startsWith('QA_EVIDENCE ')).toBe(true); return JSON.parse(text.trim().slice('QA_EVIDENCE '.length)); } test('native capture executes once, preserves exact JSON and stderr, and materializes without transcription', () => { const f = fixture(); const observed = { stateRoot: '/home/runner/.cache/owned', windows: 'C:\\owned\\a.json', text: '雪\n\t"\\', rows: ['abc'.repeat(20000)], value: 7, absent: null }; f.json('payload.json', observed); const result = f.capture('001', `const fs = require('node:fs'); fs.appendFileSync('effects', 'once'); process.stderr.write('diagnostic\\n'); console.log(fs.readFileSync('payload.json', 'utf8'));`); expect(result.status, result.stderr).toBe(0); const captured = receipt(result.stdout); expect(captured).toMatchObject({ action: 'capture', status: 'complete', id: '001', exitCode: 0 }); expect(result.stdout).not.toContain('stateRoot'); expect(result.stderr).toBe(''); expect(fs.readFileSync(path.join(f.root, 'effects'), 'utf8')).toBe('once'); expect(fs.readFileSync(path.join(f.root, '.qa-evidence/001/stdout'), 'utf8')).toBe(JSON.stringify(observed) + '\n'); expect(fs.readFileSync(path.join(f.root, '.qa-evidence/001/stderr'), 'utf8')).toBe('diagnostic\n'); f.json('intent.json', { capture: '001', observationCommand: 'first native command', hypothesis: 'The successful boundary suggests testing the rejected input next.', nextCommand: 'second native command' }); const checkpoint = f.run('checkpoint', f.root, '001', 'intent.json'); expect(checkpoint.status, checkpoint.stderr).toBe(0); expect(receipt(checkpoint.stdout)).toMatchObject({ action: 'checkpoint', status: 'complete', id: '001' }); expect(JSON.parse(fs.readFileSync(path.join(f.root, 'exploration-001.json'), 'utf8'))).toEqual({ observationCommand: 'first native command', observed, hypothesis: 'The successful boundary suggests testing the rejected input next.', nextCommand: 'second native command', }); f.json('annotations.json', { revision: 'revision', runtime: 'runtime', cwd: f.root, limits: ['Only the declared contract was checked.'], evidence: [{ capture: '001', command: 'first native command', contract: 'README.md', expected: 'Declared exact result', classification: 'pass' }], learning: ['001'] }); const report = f.run('materialize', f.root, 'annotations.json'); expect(report.status, report.stderr).toBe(0); const final = JSON.parse(fs.readFileSync(path.join(f.root, 'evidence.json'), 'utf8')); expect(final.evidence).toEqual([{ command: 'first native command', contract: 'README.md', expected: 'Declared exact result', classification: 'pass', observed }]); expect(final.learning).toEqual([{ observationCommand: 'first native command', hypothesis: 'The successful boundary suggests testing the rejected input next.', nextCommand: 'second native command' }]); expect(fs.readFileSync(path.join(f.root, 'effects'), 'utf8')).toBe('once'); expect(f.capture('001', `require('node:fs').appendFileSync('effects', 'twice')`).status).toBe(2); expect(fs.readFileSync(path.join(f.root, 'effects'), 'utf8')).toBe('once'); expect(f.run('checkpoint', f.root, '001', 'intent.json').status).toBe(2); expect(f.run('materialize', f.root, 'annotations.json').status).toBe(2); if (process.platform !== 'win32') { for (const name of ['exploration-001.json', 'evidence.json', '.qa-evidence/001/stdout', '.qa-evidence/001/stderr', '.qa-evidence/001/receipt.json']) expect(fs.statSync(path.join(f.root, name)).mode & 0o777).toBe(0o600); expect(fs.statSync(path.join(f.root, '.qa-evidence')).mode & 0o777).toBe(0o700); expect(fs.statSync(path.join(f.root, '.qa-evidence/001')).mode & 0o777).toBe(0o700); } }); test('complete nonzero commands retain their actual exit, and non-JSON observations retain all bytes', () => { const f = fixture(); const result = f.capture('001', `process.stdout.write(' exact\\ntext\\n'); process.stderr.write('separate warning\\n'); process.exitCode = 69;`); expect(result.status).toBe(69); expect(receipt(result.stdout)).toMatchObject({ status: 'complete', exitCode: 69 }); f.json('intent.json', { capture: '001', observationCommand: 'dependency check', hypothesis: 'The unavailable dependency leaves an independent valid contract to check.', nextCommand: 'adjacent check' }); expect(f.run('checkpoint', f.root, '001', 'intent.json').status).toBe(0); expect(JSON.parse(fs.readFileSync(path.join(f.root, 'exploration-001.json'), 'utf8')).observed).toBe(' exact\ntext\n'); }); test('expired and timed-out captures cannot produce checkpoint observations', () => { const f = fixture(); const result = f.capture('001', `console.log('{}'); setInterval(() => {}, 1000);`, '100'); expect(result.status).toBe(124); expect(receipt(result.stdout)).toMatchObject({ status: 'incomplete', exitCode: 124 }); f.json('intent.json', { capture: '001', observationCommand: 'timed-out check', hypothesis: 'An incomplete capture must never be promoted to passing evidence.', nextCommand: 'next check' }); expect(f.run('checkpoint', f.root, '001', 'intent.json').status).toBe(2); expect(fs.existsSync(path.join(f.root, 'exploration-001.json'))).toBe(false); }); test('capture shares a working-directory-relative deadline without resetting or relocating it', () => { const f = fixture(); fs.mkdirSync(path.join(f.root, 'reports')); const guard = spawnSync(process.execPath, [path.resolve(import.meta.dir, '../bin/gstack-qa-deadline'), 'start', 'reports/deadline.json', '5'], { cwd: f.root, encoding: 'utf8', timeout: 10000 }); expect(guard.status, guard.stderr).toBe(0); const before = fs.readFileSync(path.join(f.root, 'reports/deadline.json')); const result = f.run('capture', 'reports', '001', '--deadline', 'reports/deadline.json', '--', process.execPath, '-e', 'console.log("{}")'); expect(result.status, result.stderr).toBe(0); expect(receipt(result.stdout)).toMatchObject({ status: 'complete', exitCode: 0 }); expect(fs.readFileSync(path.join(f.root, 'reports/deadline.json'))).toEqual(before); expect(fs.existsSync(path.join(f.root, 'reports/.qa-evidence/001/deadline.json'))).toBe(false); }); test.each(['stdout', 'stderr', 'receipt.json'])('changed captured %s fails closed', name => { const f = fixture(); expect(f.capture('001', `console.log(JSON.stringify({ value: 7 }));`).status).toBe(0); fs.appendFileSync(path.join(f.root, '.qa-evidence/001', name), 'changed'); f.json('intent.json', { capture: '001', observationCommand: 'first command', hypothesis: 'The next command checks an adjacent boundary with retained evidence.', nextCommand: 'next command' }); expect(f.run('checkpoint', f.root, '001', 'intent.json').status).toBe(2); expect(fs.existsSync(path.join(f.root, 'exploration-001.json'))).toBe(false); }); test.each(['intent-link', 'capture-link', 'hard-link', 'outside-root', 'extra-observed', 'missing-capture'])('unowned or fabricated source fails closed: %s', mode => { const f = fixture(); expect(f.capture('001', `console.log('{}');`).status).toBe(0); const intent: Record = { capture: mode === 'missing-capture' ? '002' : '001', observationCommand: 'first command', hypothesis: 'The next command tests a different boundary without rewriting observations.', nextCommand: 'next command' }; if (mode === 'extra-observed') intent.observed = { invented: true }; f.json('intent.json', intent); let source = 'intent.json'; if (mode === 'intent-link') { fs.symlinkSync(path.join(f.root, source), path.join(f.root, 'linked.json')); source = 'linked.json'; } if (mode === 'hard-link') fs.linkSync(path.join(f.root, source), path.join(f.root, 'linked.json')); if (mode === 'capture-link') { fs.renameSync(path.join(f.root, '.qa-evidence/001/stdout'), path.join(f.root, 'original')); fs.symlinkSync(path.join(f.root, 'original'), path.join(f.root, '.qa-evidence/001/stdout')); } if (mode === 'outside-root') source = '../outside.json'; expect(f.run('checkpoint', f.root, '001', source).status).toBe(2); expect(fs.existsSync(path.join(f.root, 'exploration-001.json'))).toBe(false); }); test('explicit public capture returns exact safe observations and inline intent preserves the four-field checkpoint', () => { const f = fixture(); const observed = { stateRoot: '/owned/.cache/project', value: '雪\\path\n' }; const captured = f.run('capture', f.root, '001', '--public', '--timeout-ms', '4000', '--', process.execPath, '-e', `console.log(${JSON.stringify(JSON.stringify(observed))})`); expect(captured.status, captured.stderr).toBe(0); expect(JSON.parse(captured.stdout.split('\n')[0])).toEqual(observed); expect(receipt(captured.stdout.split('\n').find(line => line.startsWith('QA_EVIDENCE '))!)).toMatchObject({ publicOutput: true, status: 'complete' }); expect(f.run('checkpoint', f.root, '001', '001', 'observed command', 'The completed public result suggests checking another boundary.', 'next command').status).toBe(0); const note = JSON.parse(fs.readFileSync(path.join(f.root, 'exploration-001.json'), 'utf8')); expect(Object.keys(note).sort()).toEqual(['hypothesis', 'nextCommand', 'observationCommand', 'observed']); expect(note.observed).toEqual(observed); }); test('public permission never exposes detected credentials in output or receipts', () => { const f = fixture(); const credential = ['gh', 'p_'].join('') + randomBytes(18).toString('hex'); const result = f.run('capture', f.root, '001', '--public', '--timeout-ms', '4000', '--', process.execPath, '-e', `console.log(JSON.stringify({ credential: ${JSON.stringify(credential)} }));`); expect(result.status).toBe(2); expect(receipt(result.stdout)).toMatchObject({ status: 'sensitive', publicOutput: true }); expect(result.stdout + result.stderr).not.toContain(credential); expect(fs.existsSync(path.join(f.root, '.qa-evidence/001/observation.json'))).toBe(false); expect(f.run('checkpoint', f.root, '001', '001', 'sensitive command', 'This cannot be promoted into a safe observation or passing checkpoint.', 'next command').status).toBe(2); }); test.each(['resumed', 'blocked-forever'])('evidence receipts survive a genuinely %s stdout pipe without extending command execution', async mode => { const f = fixture(); const preload = path.join(f.root, 'full-pipe.ts'); fs.writeFileSync(preload, `import { write } from 'node:fs'; if (process.argv[1] === ${JSON.stringify(CLI)}) { write(1, Buffer.alloc(2 * 1024 * 1024, 32), () => {}); await Bun.sleep(100); }`); const child = spawn(process.execPath, ['--preload', preload, CLI, 'capture', f.root, '001', '--timeout-ms', '1000', '--', process.execPath, '-e', `require('node:fs').writeFileSync(${JSON.stringify(path.join(f.root, 'effect'))}, 'once'); process.exit(69);`], { cwd: f.root, stdio: ['ignore', 'pipe', 'pipe'] }); let stdout = '', stderr = ''; child.stdout!.on('data', bytes => { stdout += bytes; }); child.stderr!.on('data', bytes => { stderr += bytes; }); child.stdout!.pause(); const finished = new Promise(resolve => child.once('exit', resolve)); const closed = new Promise(resolve => child.once('close', () => resolve())); let timer: ReturnType | undefined; try { const file = path.join(f.root, '.qa-evidence/001/receipt.json'); for (let index = 0; index < 300 && !fs.existsSync(file); index++) await Bun.sleep(10); expect(fs.existsSync(file)).toBe(true); expect(JSON.parse(fs.readFileSync(file, 'utf8'))).toMatchObject({ status: 'complete', exitCode: 69 }); expect(fs.readFileSync(path.join(f.root, 'effect'), 'utf8')).toBe('once'); expect(child.exitCode).toBeNull(); if (mode === 'resumed') child.stdout!.resume(); const exit = await Promise.race([finished, new Promise(resolve => { timer = setTimeout(() => resolve(null), 7000); })]); expect(exit, stderr).toBe(mode === 'resumed' ? 69 : 2); child.stdout!.resume(); await closed; if (mode === 'resumed') expect(receipt(stdout.split('\n').find(line => line.startsWith('QA_EVIDENCE '))!)).toMatchObject({ status: 'complete', exitCode: 69 }); } finally { clearTimeout(timer); child.stdout!.resume(); child.kill('SIGKILL'); await closed; } }, 15_000); test.each(['early', 'timeout'])('capture closes owned descendants and their output handles after %s completion', async mode => { const f = fixture(); const leaf = path.join(f.root, 'leaf.ts'); const driver = path.join(f.root, 'driver.ts'); const pidFile = path.join(f.root, 'leaf.pid'); fs.writeFileSync(leaf, `require('node:fs').writeFileSync(process.argv[2], String(process.pid)); console.log('leaf ready'); setInterval(() => {}, 1000);`); fs.writeFileSync(driver, `import { spawn } from 'node:child_process'; import { existsSync } from 'node:fs'; const child = spawn(process.execPath, [${JSON.stringify(leaf)}, ${JSON.stringify(pidFile)}], { stdio: 'inherit' }); child.unref(); const until = Date.now() + 3000; while (!existsSync(${JSON.stringify(pidFile)})) { if (Date.now() > until) process.exit(11); await Bun.sleep(10); } ${mode === 'early' ? 'process.exit(7);' : 'setInterval(() => {}, 1000);'}`); let pid = 0; const alive = () => { try { process.kill(pid, 0); return true; } catch { return false; } }; try { const result = f.run('capture', f.root, '001', '--timeout-ms', '1000', '--', process.execPath, driver); expect(result.status, result.stderr).toBe(mode === 'early' ? 7 : 124); expect(receipt(result.stdout)).toMatchObject({ status: mode === 'early' ? 'complete' : 'incomplete' }); pid = Number(fs.readFileSync(pidFile, 'utf8')); for (let index = 0; index < 100 && alive(); index++) await Bun.sleep(20); expect(alive()).toBe(false); } finally { if (pid && alive()) process.kill(pid, 'SIGKILL'); } }); test('UTF-8 capture retains a BOM and control bytes rather than silently changing non-JSON output', () => { const f = fixture(); const value = '\ufeffnative\u0000text\r\n'; expect(f.capture('001', `process.stdout.write(${JSON.stringify(value)});`).status).toBe(0); expect(f.run('checkpoint', f.root, '001', '001', 'native command', 'The exact text suggests checking the next declared input boundary.', 'next command').status).toBe(0); expect(JSON.parse(fs.readFileSync(path.join(f.root, 'exploration-001.json'), 'utf8')).observed).toBe(value); }); test('invalid UTF-8 is retained privately as incomplete, never reconstructed as text', () => { const f = fixture(); const result = f.capture('001', 'process.stdout.write(Buffer.from([0xff, 0x80]));'); expect(result.status).toBe(2); expect(receipt(result.stdout)).toMatchObject({ status: 'incomplete', exitCode: 0 }); expect([...fs.readFileSync(path.join(f.root, '.qa-evidence/001/stdout'))]).toEqual([255, 128]); expect(f.run('checkpoint', f.root, '001', '001', 'native command', 'An incomplete text capture cannot qualify as an observed result.', 'next command').status).toBe(2); }); test.skipIf(process.platform === 'win32')('unavailable Windows containment refuses the command before dispatch', () => { const f = fixture(); const preload = path.join(f.root, 'unavailable.ts'); const marker = path.join(f.root, 'effect'); fs.writeFileSync(preload, `Object.defineProperty(process, 'platform', { value: 'win32' });`); const result = spawnSync(process.execPath, ['--preload', preload, CLI, 'capture', f.root, '001', '--timeout-ms', '1000', '--', process.execPath, '-e', `require('node:fs').writeFileSync(${JSON.stringify(marker)}, 'unexpected')`], { cwd: f.root, encoding: 'utf8', timeout: 10000 }); expect(result.status).toBe(2); expect(fs.existsSync(marker)).toBe(false); }); test.skipIf(process.platform !== 'win32')('abrupt Windows evidence-wrapper exit closes the inherited job', async () => { const f = fixture(); const marker = path.join(f.root, 'native.pid'); const child = spawn(process.execPath, [CLI, 'capture', f.root, '001', '--timeout-ms', '10000', '--', process.execPath, '-e', `require('node:fs').writeFileSync(${JSON.stringify(marker)}, String(process.pid)); setInterval(() => {}, 1000);`], { cwd: f.root, stdio: 'ignore' }); const closed = new Promise(resolve => child.once('close', () => resolve())); let pid = 0; const alive = () => { try { process.kill(pid, 0); return true; } catch { return false; } }; try { for (let index = 0; index < 300 && !fs.existsSync(marker); index++) await Bun.sleep(10); expect(fs.existsSync(marker)).toBe(true); pid = Number(fs.readFileSync(marker, 'utf8')); child.kill('SIGKILL'); await closed; for (let index = 0; index < 100 && alive(); index++) await Bun.sleep(20); expect(alive()).toBe(false); } finally { child.kill('SIGKILL'); if (pid && alive()) process.kill(pid, 'SIGKILL'); await closed; } });