import { expect, test } from 'bun:test'; import * as fs from 'node:fs'; import * as path from 'node:path'; import { spawnSync } from 'node:child_process'; import { createQAFunctionalFixture, fixtureCommand } from './helpers/qa-functional-fixture'; import { nativeCalls, readQACheckpointFiles, validateQACheckpoints } from './helpers/qa-checkpoint-evidence'; import { qaEvidenceCommand, qaNativeCapture } from './helpers/qa-evidence-producer'; import { observeQAWrites, qaWriteVerdict } from './helpers/qa-functional-observer'; import { parseNDJSON } from './helpers/session-runner'; import { qaNativeProbes } from './helpers/qa-functional-evidence'; import { createQaCallerFixture, validateCallerEvidence } from './helpers/qa-callers-fixture'; function recordedFixture(publicOutput = false) { const fixture = createQAFunctionalFixture('webhook'); const reportRoot = path.join(fixture.root, 'qa-reports'); const context = { cwd: fixture.root, reportRoot, executable: path.join(fixture.root, 'bin/gstack-qa-evidence') }; const transcript: any[] = []; const record = (name: string, input: any, output: string, metadata?: any) => { const id = `native-${transcript.length}`; transcript.push({ type: 'assistant', message: { content: [{ type: 'tool_use', id, name, input }] } }); transcript.push({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: id, content: output }] }, ...(metadata ? { tool_use_result: metadata } : {}) }); }; const execute = (command: string) => { const result = spawnSync('bash', ['-c', command], { cwd: fixture.root, encoding: 'utf8', timeout: 10000, env: { ...process.env, QA_STATE_ROOT: path.join(fixture.root, '.qa-state'), GIT_OPTIONAL_LOCKS: '0' } }); expect(result.status, result.stderr).toBe(0); record('Bash', { command }, result.stdout + result.stderr); return result; }; const first = `bun bin/gstack-qa-evidence capture qa-reports 001 ${publicOutput ? '--public ' : ''}--timeout-ms 10000 -- bun run probe -- happy`; const second = `bun bin/gstack-qa-evidence capture qa-reports 002 ${publicOutput ? '--public ' : ''}--timeout-ms 10000 -- bun run probe -- partial`; execute(first); const hypothesis = 'One successful delivery suggests checking replay after an interrupted worker.'; if (publicOutput) execute(`bun bin/gstack-qa-evidence checkpoint qa-reports 001 001 '${first}' '${hypothesis}' '${second}'`); else { const view = path.join(reportRoot, '.qa-evidence/001/observation.json'); const content = fs.readFileSync(view, 'utf8'); record('Read', { file_path: view }, content.split('\n').map((line, index) => `${index + 1}\t${line}`).join('\n'), { type: 'text', file: { filePath: view, content, startLine: 1, numLines: content.split('\n').length, totalLines: content.split('\n').length } }); const intent = JSON.stringify({ capture: '001', observationCommand: first, hypothesis, nextCommand: second }); const intentPath = path.join(reportRoot, 'intent-001.json'); fs.writeFileSync(intentPath, intent, { mode: 0o600 }); record('Write', { file_path: intentPath, content: intent }, 'File created successfully'); execute('bun bin/gstack-qa-evidence checkpoint qa-reports 001 intent-001.json'); } execute(second); const probes = nativeCalls(transcript, []).flatMap(call => { const capture = qaNativeCapture(call, context); return capture ? [{ command: call.input.command, observed: capture.captured.observed }] : []; }); const input = { transcript, reportRoot, producer: context, probes, requiredProbes: probes.slice(1), files: readQACheckpointFiles(reportRoot), reportMarkdown: '[checkpoint 001](exploration-001.json)' }; return { fixture, context, input, execute }; } test('real production captures bind completed reads, causal intent, immutable publication and the actual next command', () => { const f = recordedFixture(); try { expect(f.input.probes).toHaveLength(2); expect(validateQACheckpoints(f.input)).toEqual([]); expect(fs.readdirSync(path.join(f.fixture.root, '.qa-state')).filter(name => name.startsWith('happy-'))).toHaveLength(1); expect(fs.readdirSync(path.join(f.fixture.root, '.qa-state')).filter(name => name.startsWith('partial-'))).toHaveLength(1); const annotations = { revision: f.fixture.revision, runtime: `bun ${Bun.version}`, cwd: f.fixture.root, evidence: f.input.probes.map((probe, index) => ({ capture: String(index + 1).padStart(3, '0'), command: probe.command, contract: 'README.md', expected: 'One durable effect', classification: index ? 'product-defect' : 'pass' })), learning: ['001'], limits: ['Only these two scenarios were executed.'] }; fs.writeFileSync(path.join(f.input.reportRoot, 'annotations.json'), JSON.stringify(annotations), { mode: 0o600 }); f.execute('bun bin/gstack-qa-evidence materialize qa-reports annotations.json'); const report = JSON.parse(fs.readFileSync(path.join(f.input.reportRoot, 'evidence.json'), 'utf8')); expect(report.evidence.map((row: any) => row.observed)).toEqual(f.input.probes.map(probe => probe.observed)); expect(report.evidence.map((row: any) => row.classification)).toEqual(['pass', 'product-defect']); } finally { f.fixture.cleanup(); } }); test('declared public observations and inline causal intent use the same native producer without extra model turns', () => { const f = recordedFixture(true); try { expect(f.input.transcript).toHaveLength(6); expect(validateQACheckpoints(f.input)).toEqual([]); const parsed = parseNDJSON(f.input.transcript.map(event => JSON.stringify(event))); expect(qaNativeProbes(parsed, f.fixture.root).map(probe => ({ command: probe.command, observed: probe.observed }))).toEqual(f.input.probes); for (const mode of ['altered-observation', 'missing-receipt', 'forged-helper', 'changed-intent', 'interrupted-result']) { const transcript = structuredClone(f.input.transcript); if (mode === 'altered-observation') transcript[1].message.content[0].content = transcript[1].message.content[0].content.replace('"scenario":"happy"', '"scenario":"invented"'); if (mode === 'missing-receipt') transcript[1].message.content[0].content = transcript[1].message.content[0].content.replace(/QA_EVIDENCE [^\n]*\n/, ''); if (mode === 'forged-helper') transcript[0].message.content[0].input.command = transcript[0].message.content[0].input.command.replace('bin/gstack-qa-evidence', '/tmp/forged/bin/gstack-qa-evidence'); if (mode === 'changed-intent') transcript[2].message.content[0].input.command = transcript[2].message.content[0].input.command.replace('One successful delivery', 'An invented successful delivery'); if (mode === 'interrupted-result') transcript[1].tool_use_result = { interrupted: true }; expect(validateQACheckpoints({ ...f.input, transcript }).length, mode).toBeGreaterThan(0); } } finally { f.fixture.cleanup(); } }); test.each(['missing-capture-result', 'failed-capture', 'forged-capture-receipt', 'changed-native-value', 'missing-read', 'failed-read', 'partial-read', 'unacknowledged-read', 'stale-intent', 'retrospective-intent', 'pending-publication', 'failed-publication', 'forged-publication', 'next-command', 'altered-artifact', 'malformed-artifact'])('native producer rejects %s without weakening direct-Write validation', mode => { const f = recordedFixture(); try { const events = f.input.transcript; if (mode === 'missing-capture-result') events[1].message.content = []; if (mode === 'failed-capture') events[1].message.content[0].is_error = true; if (mode === 'forged-capture-receipt') events[1].message.content[0].content = events[1].message.content[0].content.replace(/"sha256":"[a-f0-9]+"/, '"sha256":"' + '0'.repeat(64) + '"'); if (mode === 'changed-native-value') (f.input.probes[0].observed as any).stateRoot += '-invented'; if (mode === 'missing-read') events[2].message.content[0].input.file_path += '.other'; if (mode === 'failed-read') events[3].message.content[0].is_error = true; if (mode === 'partial-read') events[3].tool_use_result.file.totalLines++; if (mode === 'unacknowledged-read') events[3].message.content = []; if (mode === 'stale-intent') events.unshift(...events.splice(4, 2)); if (mode === 'retrospective-intent') events.push(...events.splice(4, 2)); if (mode === 'pending-publication') events[7].message.content = []; if (mode === 'failed-publication') events[7].message.content[0].is_error = true; if (mode === 'forged-publication') events[7].message.content[0].content = events[7].message.content[0].content.replace(/"intentSha256":"[a-f0-9]+"/, '"intentSha256":"' + '0'.repeat(64) + '"'); if (mode === 'next-command') events[8].message.content[0].input.command += ' extra'; if (mode === 'altered-artifact' || mode === 'malformed-artifact') { const file = path.join(f.input.reportRoot, 'exploration-001.json'); const value = JSON.parse(fs.readFileSync(file, 'utf8')); value.observed.stateRoot += '-invented'; fs.writeFileSync(file, mode === 'malformed-artifact' ? '{"observed":"\\.cache"}' : JSON.stringify(value)); f.input.files = readQACheckpointFiles(f.input.reportRoot); } expect(validateQACheckpoints(f.input).length).toBeGreaterThan(0); } finally { f.fixture.cleanup(); } }); test('the registered native permission callback admits only the owned helper and the existing probe grammar', () => { const fixture = createQAFunctionalFixture('webhook'); try { const hook = JSON.parse(fs.readFileSync(path.join(fixture.config, 'settings.json'), 'utf8')).hooks.PreToolUse[0].hooks[0].command; const permitted = 'bun bin/gstack-qa-evidence capture qa-reports 001 --timeout-ms 10000 -- bun run probe -- happy'; for (const [command, expected] of [ [permitted, 'allow'], ['bun bin/gstack-qa-evidence checkpoint qa-reports 001 intent-001.json', 'allow'], ['bun bin/gstack-qa-evidence materialize qa-reports annotations.json', 'allow'], [permitted.replace('bun run probe -- happy', 'bun -e evil'), 'deny'], [permitted.replace('qa-reports', '../outside'), 'deny'], [permitted.replace('bin/gstack-qa-evidence', '/tmp/forged/bin/gstack-qa-evidence'), 'deny'], [permitted + ' | cat', 'deny'], [permitted + ' > output', 'deny'], ['bun bin/gstack-qa-evidence checkpoint qa-reports 001 ../outside.json', 'deny'], ]) { const result = spawnSync(hook, { shell: true, cwd: fixture.root, encoding: 'utf8', timeout: 5000, input: JSON.stringify({ hook_event_name: 'PreToolUse', cwd: fixture.root, tool_name: 'Bash', tool_input: { command } }) }); expect(result.status, result.stderr).toBe(0); expect(JSON.parse(result.stdout).hookSpecificOutput.permissionDecision, command).toBe(expected); } expect(qaEvidenceCommand(permitted.replace(' -- bun ', ' -- bun; '), { cwd: fixture.root, reportRoot: path.join(fixture.root, 'qa-reports'), executable: path.join(fixture.root, 'bin/gstack-qa-evidence') })).toBeUndefined(); } finally { fixture.cleanup(); } }); test('native observer retains the same filesystem boundary while the production helper captures and publishes', async () => { const fixture = createQAFunctionalFixture('webhook'); const observer = await observeQAWrites(fixture.root, { evidenceProducer: true }); let stopped = false; try { const captured = fixtureCommand(fixture.root, ['bin/gstack-qa-evidence', 'capture', 'qa-reports', '001', '--timeout-ms', '10000', '--', 'bun', 'run', 'probe', '--', 'happy']); expect(captured.exit, captured.stderr).toBe(0); observer.drain(); fs.writeFileSync(path.join(fixture.root, 'qa-reports/intent.json'), JSON.stringify({ capture: '001', observationCommand: 'observed command', hypothesis: 'The next probe tests the adjacent failure boundary after success.', nextCommand: 'next command' })); const published = fixtureCommand(fixture.root, ['bin/gstack-qa-evidence', 'checkpoint', 'qa-reports', '001', 'intent.json']); expect(published.exit, published.stderr).toBe(0); const result = observer.stop(); stopped = true; expect(result.complete, result.failures.join('\n')).toBe(true); expect(qaWriteVerdict(result, 'qa-only')).toEqual([]); } finally { if (!stopped) observer.stop(); fixture.cleanup(); } }); for (const expired of [false, true]) test(`bounded parent uses the production capture and retains truthful ${expired ? 'expired' : 'nonzero'} evidence`, async () => { const fixture = createQaCallerFixture('review-exploratory-small-cli'); const transcript: any[] = []; const reportRoot = path.join(fixture.cwd, 'reports'); const record = (name: string, input: any, output: string, failed = false) => { const id = `native-${transcript.length}`; transcript.push({ type: 'assistant', message: { content: [{ type: 'tool_use', id, name, input }] } }); transcript.push({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: id, content: output, is_error: failed }] } }); }; const execute = (command: string) => { const result = spawnSync('bash', ['-c', command], { cwd: fixture.cwd, encoding: 'utf8', timeout: 10000, env: { ...process.env, GIT_OPTIONAL_LOCKS: '0' } }); expect(result.error).toBeUndefined(); record('Bash', { command }, (result.status !== 0 ? `Exit code ${result.status}\n` : '') + result.stdout + result.stderr, result.status !== 0); return result; }; try { await fixture.observe(); for (const file of [path.join(fixture.cwd, 'caller-review.md'), path.join(fixture.runtime, 'qa/sections/exploratory.md'), path.join(fixture.runtime, 'qa/sections/system-functional.md')]) { record('Read', { file_path: file }, fs.readFileSync(file, 'utf8')); } const deadline = path.join(reportRoot, 'deadline.json'); expect(execute(`bun ${fixture.runtime}/bin/gstack-qa-deadline start ${deadline} ${expired ? '1' : '30'}`).status).toBe(0); const first = `bun ${fixture.runtime}/bin/gstack-qa-evidence capture ${reportRoot} 001 --public --deadline ${deadline} -- bun scripts/probe.ts 3`; const next = `bun ${fixture.runtime}/bin/gstack-qa-evidence capture ${reportRoot} 002 --public --deadline ${deadline} -- bun scripts/probe.ts 0`; expect(execute(first).status).toBe(0); expect(execute(`bun ${fixture.runtime}/bin/gstack-qa-evidence checkpoint ${reportRoot} 001 001 '${first}' 'The positive input suggests checking the declared zero boundary next.' '${next}'`).status).toBe(0); if (expired) await Bun.sleep(Math.max(1, Date.parse(JSON.parse(fs.readFileSync(deadline, 'utf8')).deadlineAt) - Date.now() + 10)); expect(execute(next).status).toBe(expired ? 124 : 2); await fixture.close(); const probes = fixture.probes(); expect(probes).toHaveLength(expired ? 1 : 2); const input = { caller: fixture.caller, result: { transcript, exitReason: 'success' as const }, probes, receipt: { status: expired ? 'blocked' as const : 'fail' as const, probes: probes.map(probe => probe.id), remaining: ['The adverse boundary is unresolved.'] }, currentSnapshot: fixture.snapshot(), requiredCharters: ['happy', 'adverse'], mutations: fixture.mutationEvents, observerComplete: fixture.observation?.complete === true && fixture.observerErrors.length === 0, fixtureRoot: fixture.cwd, runtime: fixture.runtime, requireGuardedSmoke: true, requireCapturedEvidence: true, reportRoot, checkpointFiles: readQACheckpointFiles(reportRoot), reportMarkdown: '[checkpoint](exploration-001.json)' }; expect(validateCallerEvidence(input)).toEqual([]); const wrongExit = structuredClone(transcript); wrongExit.at(-1).message.content[0].content = wrongExit.at(-1).message.content[0].content.replace(/^Exit code \d+\n/, 'Exit code 127\n'); expect(validateCallerEvidence({ ...input, result: { ...input.result, transcript: wrongExit } }).length).toBeGreaterThan(0); expect(validateCallerEvidence({ ...input, receipt: { ...input.receipt, status: 'pass', remaining: [] } }).length).toBeGreaterThan(0); } finally { await fixture.close(); fs.rmSync(fixture.root, { recursive: true, force: true }); } });