mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-02 17:40:02 +02:00
garrytan
8dd3c44fc6
qa-functional-webhook-report failed in two of three censuses because the report linked .qa-evidence/NNN capture folders as "checkpoints" and never linked exploration-NNN.json. The checkpoint receipt now prints link: [checkpoint NNN](exploration-NNN.json), and the functional report template says capture folders are not checkpoints.
246 lines
18 KiB
TypeScript
246 lines
18 KiB
TypeScript
import { afterAll, expect, test } from 'bun:test';
|
|
import * as fs from 'node:fs';
|
|
import * as os from 'node:os';
|
|
import * as path from 'node:path';
|
|
import { spawn, spawnSync } from 'node:child_process';
|
|
import { randomBytes } from 'node:crypto';
|
|
|
|
const CLI = path.resolve(import.meta.dir, '../bin/gstack-qa-evidence');
|
|
const ROOT = fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), 'qa-evidence-'));
|
|
afterAll(() => fs.rmSync(ROOT, { recursive: true, force: true }));
|
|
|
|
function fixture() {
|
|
const root = fs.mkdtempSync(path.join(ROOT, 'case-'));
|
|
const run = (...args: string[]) => {
|
|
expect(JSON.stringify([process.execPath, CLI, ...args]).length, 'Fixture argv must stay short for Windows process launch').toBeLessThan(8192);
|
|
const result = spawnSync(process.execPath, [CLI, ...args], { cwd: root, encoding: 'utf8', timeout: 10_000 });
|
|
expect(result.error, result.error?.message).toBeUndefined();
|
|
return result;
|
|
};
|
|
const json = (name: string, value: unknown) => fs.writeFileSync(path.join(root, name), JSON.stringify(value), { mode: 0o600 });
|
|
const capture = (id: string, program: string, timeout = '4000') => run('capture', root, id, '--timeout-ms', timeout, '--', process.execPath, '-e', program);
|
|
return { root, run, json, capture };
|
|
}
|
|
|
|
function receipt(text: string) {
|
|
expect(text.trim().startsWith('QA_EVIDENCE ')).toBe(true);
|
|
return JSON.parse(text.trim().slice('QA_EVIDENCE '.length));
|
|
}
|
|
|
|
test('native capture executes once, preserves exact JSON and stderr, and materializes without transcription', () => {
|
|
const f = fixture();
|
|
const observed = { stateRoot: '/home/runner/.cache/owned', windows: 'C:\\owned\\a.json', text: '雪\n\t"\\', rows: ['abc'.repeat(20000)], value: 7, absent: null };
|
|
f.json('payload.json', observed);
|
|
const result = f.capture('001', `const fs = require('node:fs'); fs.appendFileSync('effects', 'once'); process.stderr.write('diagnostic\\n'); console.log(fs.readFileSync('payload.json', 'utf8'));`);
|
|
expect(result.status, result.stderr).toBe(0);
|
|
const captured = receipt(result.stdout);
|
|
expect(captured).toMatchObject({ action: 'capture', status: 'complete', id: '001', exitCode: 0 });
|
|
expect(result.stdout).not.toContain('stateRoot');
|
|
expect(result.stderr).toBe('');
|
|
expect(fs.readFileSync(path.join(f.root, 'effects'), 'utf8')).toBe('once');
|
|
expect(fs.readFileSync(path.join(f.root, '.qa-evidence/001/stdout'), 'utf8')).toBe(JSON.stringify(observed) + '\n');
|
|
expect(fs.readFileSync(path.join(f.root, '.qa-evidence/001/stderr'), 'utf8')).toBe('diagnostic\n');
|
|
f.json('intent.json', { capture: '001', observationCommand: 'first native command', hypothesis: 'The successful boundary suggests testing the rejected input next.', nextCommand: 'second native command' });
|
|
const checkpoint = f.run('checkpoint', f.root, '001', 'intent.json');
|
|
expect(checkpoint.status, checkpoint.stderr).toBe(0);
|
|
expect(receipt(checkpoint.stdout)).toMatchObject({ action: 'checkpoint', status: 'complete', id: '001', link: '[checkpoint 001](exploration-001.json)' });
|
|
expect(JSON.parse(fs.readFileSync(path.join(f.root, 'exploration-001.json'), 'utf8'))).toEqual({
|
|
observationCommand: 'first native command', observed, hypothesis: 'The successful boundary suggests testing the rejected input next.', nextCommand: 'second native command',
|
|
});
|
|
f.json('annotations.json', { revision: 'revision', runtime: 'runtime', cwd: f.root, limits: ['Only the declared contract was checked.'], evidence: [{ capture: '001', command: 'first native command', contract: 'README.md', expected: 'Declared exact result', classification: 'pass' }], learning: ['001'] });
|
|
const report = f.run('materialize', f.root, 'annotations.json');
|
|
expect(report.status, report.stderr).toBe(0);
|
|
const final = JSON.parse(fs.readFileSync(path.join(f.root, 'evidence.json'), 'utf8'));
|
|
expect(final.evidence).toEqual([{ command: 'first native command', contract: 'README.md', expected: 'Declared exact result', classification: 'pass', observed }]);
|
|
expect(final.learning).toEqual([{ observationCommand: 'first native command', hypothesis: 'The successful boundary suggests testing the rejected input next.', nextCommand: 'second native command' }]);
|
|
expect(fs.readFileSync(path.join(f.root, 'effects'), 'utf8')).toBe('once');
|
|
expect(f.capture('001', `require('node:fs').appendFileSync('effects', 'twice')`).status).toBe(2);
|
|
expect(fs.readFileSync(path.join(f.root, 'effects'), 'utf8')).toBe('once');
|
|
expect(f.run('checkpoint', f.root, '001', 'intent.json').status).toBe(2);
|
|
expect(f.run('materialize', f.root, 'annotations.json').status).toBe(2);
|
|
if (process.platform !== 'win32') {
|
|
for (const name of ['exploration-001.json', 'evidence.json', '.qa-evidence/001/stdout', '.qa-evidence/001/stderr', '.qa-evidence/001/receipt.json']) expect(fs.statSync(path.join(f.root, name)).mode & 0o777).toBe(0o600);
|
|
expect(fs.statSync(path.join(f.root, '.qa-evidence')).mode & 0o777).toBe(0o700);
|
|
expect(fs.statSync(path.join(f.root, '.qa-evidence/001')).mode & 0o777).toBe(0o700);
|
|
}
|
|
});
|
|
|
|
test('complete nonzero commands retain their actual exit, and non-JSON observations retain all bytes', () => {
|
|
const f = fixture();
|
|
const result = f.capture('001', `process.stdout.write(' exact\\ntext\\n'); process.stderr.write('separate warning\\n'); process.exitCode = 69;`);
|
|
expect(result.status).toBe(69);
|
|
expect(receipt(result.stdout)).toMatchObject({ status: 'complete', exitCode: 69 });
|
|
f.json('intent.json', { capture: '001', observationCommand: 'dependency check', hypothesis: 'The unavailable dependency leaves an independent valid contract to check.', nextCommand: 'adjacent check' });
|
|
expect(f.run('checkpoint', f.root, '001', 'intent.json').status).toBe(0);
|
|
expect(JSON.parse(fs.readFileSync(path.join(f.root, 'exploration-001.json'), 'utf8')).observed).toBe(' exact\ntext\n');
|
|
});
|
|
|
|
test('expired and timed-out captures cannot produce checkpoint observations', () => {
|
|
const f = fixture();
|
|
const result = f.capture('001', `console.log('{}'); setInterval(() => {}, 1000);`, '100');
|
|
expect(result.status).toBe(124);
|
|
expect(receipt(result.stdout)).toMatchObject({ status: 'incomplete', exitCode: 124 });
|
|
f.json('intent.json', { capture: '001', observationCommand: 'timed-out check', hypothesis: 'An incomplete capture must never be promoted to passing evidence.', nextCommand: 'next check' });
|
|
expect(f.run('checkpoint', f.root, '001', 'intent.json').status).toBe(2);
|
|
expect(fs.existsSync(path.join(f.root, 'exploration-001.json'))).toBe(false);
|
|
});
|
|
|
|
test('capture shares a working-directory-relative deadline without resetting or relocating it', () => {
|
|
const f = fixture();
|
|
fs.mkdirSync(path.join(f.root, 'reports'));
|
|
const guard = spawnSync(process.execPath, [path.resolve(import.meta.dir, '../bin/gstack-qa-deadline'), 'start', 'reports/deadline.json', '5'],
|
|
{ cwd: f.root, encoding: 'utf8', timeout: 10000 });
|
|
expect(guard.status, guard.stderr).toBe(0);
|
|
const before = fs.readFileSync(path.join(f.root, 'reports/deadline.json'));
|
|
const result = f.run('capture', 'reports', '001', '--deadline', 'reports/deadline.json', '--', process.execPath, '-e', 'console.log("{}")');
|
|
expect(result.status, result.stderr).toBe(0);
|
|
expect(receipt(result.stdout)).toMatchObject({ status: 'complete', exitCode: 0 });
|
|
expect(fs.readFileSync(path.join(f.root, 'reports/deadline.json'))).toEqual(before);
|
|
expect(fs.existsSync(path.join(f.root, 'reports/.qa-evidence/001/deadline.json'))).toBe(false);
|
|
});
|
|
|
|
test.each(['stdout', 'stderr', 'receipt.json'])('changed captured %s fails closed', name => {
|
|
const f = fixture();
|
|
expect(f.capture('001', `console.log(JSON.stringify({ value: 7 }));`).status).toBe(0);
|
|
fs.appendFileSync(path.join(f.root, '.qa-evidence/001', name), 'changed');
|
|
f.json('intent.json', { capture: '001', observationCommand: 'first command', hypothesis: 'The next command checks an adjacent boundary with retained evidence.', nextCommand: 'next command' });
|
|
expect(f.run('checkpoint', f.root, '001', 'intent.json').status).toBe(2);
|
|
expect(fs.existsSync(path.join(f.root, 'exploration-001.json'))).toBe(false);
|
|
});
|
|
|
|
test.each(['intent-link', 'capture-link', 'hard-link', 'outside-root', 'extra-observed', 'missing-capture'])('unowned or fabricated source fails closed: %s', mode => {
|
|
const f = fixture();
|
|
expect(f.capture('001', `console.log('{}');`).status).toBe(0);
|
|
const intent: Record<string, unknown> = { capture: mode === 'missing-capture' ? '002' : '001', observationCommand: 'first command', hypothesis: 'The next command tests a different boundary without rewriting observations.', nextCommand: 'next command' };
|
|
if (mode === 'extra-observed') intent.observed = { invented: true };
|
|
f.json('intent.json', intent);
|
|
let source = 'intent.json';
|
|
if (mode === 'intent-link') { fs.symlinkSync(path.join(f.root, source), path.join(f.root, 'linked.json')); source = 'linked.json'; }
|
|
if (mode === 'hard-link') fs.linkSync(path.join(f.root, source), path.join(f.root, 'linked.json'));
|
|
if (mode === 'capture-link') { fs.renameSync(path.join(f.root, '.qa-evidence/001/stdout'), path.join(f.root, 'original')); fs.symlinkSync(path.join(f.root, 'original'), path.join(f.root, '.qa-evidence/001/stdout')); }
|
|
if (mode === 'outside-root') source = '../outside.json';
|
|
expect(f.run('checkpoint', f.root, '001', source).status).toBe(2);
|
|
expect(fs.existsSync(path.join(f.root, 'exploration-001.json'))).toBe(false);
|
|
});
|
|
|
|
test('explicit public capture returns exact safe observations and inline intent preserves the four-field checkpoint', () => {
|
|
const f = fixture();
|
|
const observed = { stateRoot: '/owned/.cache/project', value: '雪\\path\n' };
|
|
const captured = f.run('capture', f.root, '001', '--public', '--timeout-ms', '4000', '--', process.execPath, '-e', `console.log(${JSON.stringify(JSON.stringify(observed))})`);
|
|
expect(captured.status, captured.stderr).toBe(0);
|
|
expect(JSON.parse(captured.stdout.split('\n')[0])).toEqual(observed);
|
|
expect(receipt(captured.stdout.split('\n').find(line => line.startsWith('QA_EVIDENCE '))!)).toMatchObject({ publicOutput: true, status: 'complete' });
|
|
expect(f.run('checkpoint', f.root, '001', '001', 'observed command', 'The completed public result suggests checking another boundary.', 'next command').status).toBe(0);
|
|
const note = JSON.parse(fs.readFileSync(path.join(f.root, 'exploration-001.json'), 'utf8'));
|
|
expect(Object.keys(note).sort()).toEqual(['hypothesis', 'nextCommand', 'observationCommand', 'observed']);
|
|
expect(note.observed).toEqual(observed);
|
|
});
|
|
|
|
test('public permission never exposes detected credentials in output or receipts', () => {
|
|
const f = fixture();
|
|
const credential = ['gh', 'p_'].join('') + randomBytes(18).toString('hex');
|
|
const result = f.run('capture', f.root, '001', '--public', '--timeout-ms', '4000', '--', process.execPath, '-e', `console.log(JSON.stringify({ credential: ${JSON.stringify(credential)} }));`);
|
|
expect(result.status).toBe(2);
|
|
expect(receipt(result.stdout)).toMatchObject({ status: 'sensitive', publicOutput: true });
|
|
expect(result.stdout + result.stderr).not.toContain(credential);
|
|
expect(fs.existsSync(path.join(f.root, '.qa-evidence/001/observation.json'))).toBe(false);
|
|
expect(f.run('checkpoint', f.root, '001', '001', 'sensitive command', 'This cannot be promoted into a safe observation or passing checkpoint.', 'next command').status).toBe(2);
|
|
});
|
|
|
|
test.each(['resumed', 'blocked-forever'])('evidence receipts survive a genuinely %s stdout pipe without extending command execution', async mode => {
|
|
const f = fixture();
|
|
const preload = path.join(f.root, 'full-pipe.ts');
|
|
fs.writeFileSync(preload, `import { write } from 'node:fs'; if (process.argv[1] === ${JSON.stringify(CLI)}) { write(1, Buffer.alloc(2 * 1024 * 1024, 32), () => {}); await Bun.sleep(100); }`);
|
|
const child = spawn(process.execPath, ['--preload', preload, CLI, 'capture', f.root, '001', '--timeout-ms', '1000', '--', process.execPath, '-e', `require('node:fs').writeFileSync(${JSON.stringify(path.join(f.root, 'effect'))}, 'once'); process.exit(69);`], { cwd: f.root, stdio: ['ignore', 'pipe', 'pipe'] });
|
|
let stdout = '', stderr = '';
|
|
child.stdout!.on('data', bytes => { stdout += bytes; });
|
|
child.stderr!.on('data', bytes => { stderr += bytes; });
|
|
child.stdout!.pause();
|
|
const finished = new Promise<number | null>(resolve => child.once('exit', resolve));
|
|
const closed = new Promise<void>(resolve => child.once('close', () => resolve()));
|
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
try {
|
|
const file = path.join(f.root, '.qa-evidence/001/receipt.json');
|
|
for (let index = 0; index < 300 && !fs.existsSync(file); index++) await Bun.sleep(10);
|
|
expect(fs.existsSync(file)).toBe(true);
|
|
expect(JSON.parse(fs.readFileSync(file, 'utf8'))).toMatchObject({ status: 'complete', exitCode: 69 });
|
|
expect(fs.readFileSync(path.join(f.root, 'effect'), 'utf8')).toBe('once');
|
|
expect(child.exitCode).toBeNull();
|
|
if (mode === 'resumed') child.stdout!.resume();
|
|
const exit = await Promise.race([finished, new Promise<null>(resolve => { timer = setTimeout(() => resolve(null), 7000); })]);
|
|
expect(exit, stderr).toBe(mode === 'resumed' ? 69 : 2);
|
|
child.stdout!.resume();
|
|
await closed;
|
|
if (mode === 'resumed') expect(receipt(stdout.split('\n').find(line => line.startsWith('QA_EVIDENCE '))!)).toMatchObject({ status: 'complete', exitCode: 69 });
|
|
} finally {
|
|
clearTimeout(timer);
|
|
child.stdout!.resume();
|
|
child.kill('SIGKILL');
|
|
await closed;
|
|
}
|
|
}, 15_000);
|
|
|
|
test.each(['early', 'timeout'])('capture closes owned descendants and their output handles after %s completion', async mode => {
|
|
const f = fixture();
|
|
const leaf = path.join(f.root, 'leaf.ts');
|
|
const driver = path.join(f.root, 'driver.ts');
|
|
const pidFile = path.join(f.root, 'leaf.pid');
|
|
fs.writeFileSync(leaf, `require('node:fs').writeFileSync(process.argv[2], String(process.pid)); console.log('leaf ready'); setInterval(() => {}, 1000);`);
|
|
fs.writeFileSync(driver, `import { spawn } from 'node:child_process'; import { existsSync } from 'node:fs'; const child = spawn(process.execPath, [${JSON.stringify(leaf)}, ${JSON.stringify(pidFile)}], { stdio: 'inherit' }); child.unref(); const until = Date.now() + 3000; while (!existsSync(${JSON.stringify(pidFile)})) { if (Date.now() > until) process.exit(11); await Bun.sleep(10); } ${mode === 'early' ? 'process.exit(7);' : 'setInterval(() => {}, 1000);'}`);
|
|
let pid = 0;
|
|
const alive = () => { try { process.kill(pid, 0); return true; } catch { return false; } };
|
|
try {
|
|
const result = f.run('capture', f.root, '001', '--timeout-ms', '1000', '--', process.execPath, driver);
|
|
expect(result.status, result.stderr).toBe(mode === 'early' ? 7 : 124);
|
|
expect(receipt(result.stdout)).toMatchObject({ status: mode === 'early' ? 'complete' : 'incomplete' });
|
|
pid = Number(fs.readFileSync(pidFile, 'utf8'));
|
|
for (let index = 0; index < 100 && alive(); index++) await Bun.sleep(20);
|
|
expect(alive()).toBe(false);
|
|
} finally { if (pid && alive()) process.kill(pid, 'SIGKILL'); }
|
|
});
|
|
|
|
test('UTF-8 capture retains a BOM and control bytes rather than silently changing non-JSON output', () => {
|
|
const f = fixture();
|
|
const value = '\ufeffnative\u0000text\r\n';
|
|
expect(f.capture('001', `process.stdout.write(${JSON.stringify(value)});`).status).toBe(0);
|
|
expect(f.run('checkpoint', f.root, '001', '001', 'native command', 'The exact text suggests checking the next declared input boundary.', 'next command').status).toBe(0);
|
|
expect(JSON.parse(fs.readFileSync(path.join(f.root, 'exploration-001.json'), 'utf8')).observed).toBe(value);
|
|
});
|
|
|
|
test('invalid UTF-8 is retained privately as incomplete, never reconstructed as text', () => {
|
|
const f = fixture();
|
|
const result = f.capture('001', 'process.stdout.write(Buffer.from([0xff, 0x80]));');
|
|
expect(result.status).toBe(2);
|
|
expect(receipt(result.stdout)).toMatchObject({ status: 'incomplete', exitCode: 0 });
|
|
expect([...fs.readFileSync(path.join(f.root, '.qa-evidence/001/stdout'))]).toEqual([255, 128]);
|
|
expect(f.run('checkpoint', f.root, '001', '001', 'native command', 'An incomplete text capture cannot qualify as an observed result.', 'next command').status).toBe(2);
|
|
});
|
|
|
|
test.skipIf(process.platform === 'win32')('unavailable Windows containment refuses the command before dispatch', () => {
|
|
const f = fixture();
|
|
const preload = path.join(f.root, 'unavailable.ts');
|
|
const marker = path.join(f.root, 'effect');
|
|
fs.writeFileSync(preload, `Object.defineProperty(process, 'platform', { value: 'win32' });`);
|
|
const result = spawnSync(process.execPath, ['--preload', preload, CLI, 'capture', f.root, '001', '--timeout-ms', '1000', '--', process.execPath, '-e', `require('node:fs').writeFileSync(${JSON.stringify(marker)}, 'unexpected')`], { cwd: f.root, encoding: 'utf8', timeout: 10000 });
|
|
expect(result.status).toBe(2);
|
|
expect(fs.existsSync(marker)).toBe(false);
|
|
});
|
|
|
|
test.skipIf(process.platform !== 'win32')('abrupt Windows evidence-wrapper exit closes the inherited job', async () => {
|
|
const f = fixture();
|
|
const marker = path.join(f.root, 'native.pid');
|
|
const child = spawn(process.execPath, [CLI, 'capture', f.root, '001', '--timeout-ms', '10000', '--', process.execPath, '-e', `require('node:fs').writeFileSync(${JSON.stringify(marker)}, String(process.pid)); setInterval(() => {}, 1000);`], { cwd: f.root, stdio: 'ignore' });
|
|
const closed = new Promise<void>(resolve => child.once('close', () => resolve()));
|
|
let pid = 0;
|
|
const alive = () => { try { process.kill(pid, 0); return true; } catch { return false; } };
|
|
try {
|
|
for (let index = 0; index < 300 && !fs.existsSync(marker); index++) await Bun.sleep(10);
|
|
expect(fs.existsSync(marker)).toBe(true);
|
|
pid = Number(fs.readFileSync(marker, 'utf8'));
|
|
child.kill('SIGKILL');
|
|
await closed;
|
|
for (let index = 0; index < 100 && alive(); index++) await Bun.sleep(20);
|
|
expect(alive()).toBe(false);
|
|
} finally { child.kill('SIGKILL'); if (pid && alive()) process.kill(pid, 'SIGKILL'); await closed; }
|
|
});
|