diff --git a/lib/qa-evidence.ts b/lib/qa-evidence.ts index b1b6a9dc2..0bf6e9347 100644 --- a/lib/qa-evidence.ts +++ b/lib/qa-evidence.ts @@ -1,6 +1,7 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; import { atomicWriteSync } from './fs-atomic'; import { qaDeadlineStatus, readQaDeadline, runQaDeadlineCommand, runQaWindowsWorker, startQaDeadline, withQaReceiptOutput } from './qa-deadline'; import { scan } from './redact-engine'; @@ -9,6 +10,11 @@ const object = (value: unknown): value is Record => value !== null const hash = (value: string | Buffer) => createHash('sha256').update(value).digest('hex'); const exact = (value: unknown, keys: string[]) => object(value) && Object.keys(value).sort().join(',') === keys.sort().join(','); class QaEvidenceError extends Error {} +const currentRevision = () => { + const result = spawnSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8', timeout: 5000, env: { ...process.env, GIT_OPTIONAL_LOCKS: '0' } }); + if (result.status !== 0 || !/^[0-9a-f]{40,64}$/.test(result.stdout.trim())) throw new QaEvidenceError('Invalid report annotations: revision is required when git rev-parse HEAD is unavailable'); + return result.stdout.trim(); +}; function id(value: string): string { if (!/^\d{3}$/.test(value)) throw new QaEvidenceError('Capture and checkpoint IDs must be three digits'); @@ -108,9 +114,29 @@ export function readQaCapture(reportRoot: string, captureId: string, expectedHas return { receipt, sha256, stdout: out, stderr: err, observed, observationText }; } +const anchoredOn = (command: unknown, captureId: string) => typeof command === 'string' && new RegExp(`\\scapture\\s+\\S+\\s+${captureId}(?:\\s|$)`).test(command); +const nativeCommand = (command: string) => command.slice(command.indexOf(' -- ') + 4).trim(); + +function checkpointNotes(root: string): Record[] { + return fs.readdirSync(root).filter(name => /^exploration-\d{3}\.json$/.test(name)).sort() + .map(name => ({ name, ...JSON.parse(decode(read(root, name))) })); +} + +function latestCompleteCapture(root: string): string | undefined { + if (!fs.existsSync(path.join(root, '.qa-evidence'))) return undefined; + return fs.readdirSync(owned(root, '.qa-evidence')).filter(name => /^\d{3}$/.test(name) && fs.existsSync(path.join(root, '.qa-evidence', name, 'receipt.json'))) + .map(name => JSON.parse(decode(read(root, `.qa-evidence/${name}/receipt.json`)))) + .filter(receipt => receipt.status === 'complete') + .sort((a, b) => Date.parse(a.completedAt) - Date.parse(b.completedAt)).at(-1)?.id; +} + async function capture(root: string, captureId: string, publicOutput: boolean, option: string, budget: string, command: string, args: string[]) { id(captureId); if (!command || !['--deadline', '--timeout-ms'].includes(option)) throw new QaEvidenceError('Capture requires a deadline or finite command timeout'); + const previous = latestCompleteCapture(root); + if (previous && !checkpointNotes(root).some(note => anchoredOn(note.observationCommand, previous) && anchoredOn(note.nextCommand, captureId))) { + throw new QaEvidenceError(`Checkpoint required before capture ${captureId}: first publish a checkpoint whose observationCommand is capture ${previous}'s full command and whose nextCommand is this exact capture command (checkpoint ROOT NNN ${previous} 'capture ${previous} command' 'causal hypothesis' 'this command'), then rerun this command unchanged. To stop exploring instead, run no further probe.`); + } if (option === '--timeout-ms' && (!/^[1-9]\d*$/.test(budget) || !Number.isSafeInteger(Number(budget)) || Number(budget) > 2_147_483_647)) throw new QaEvidenceError('Invalid command timeout'); privateDirectory(root, '.qa-evidence'); const directory = privateDirectory(root, `.qa-evidence/${captureId}`, true); @@ -182,7 +208,8 @@ async function capture(root: string, captureId: string, publicOutput: boolean, o ...streams }; const sha256 = publish(root, `.qa-evidence/${captureId}/receipt.json`, receipt); return { action: 'capture', id: captureId, status, sha256, exitCode, signal: result.signal, publicOutput, - startedAt, completedAt, durationMs: Date.parse(completedAt) - Date.parse(startedAt), ...(remainingMs === undefined ? {} : { remainingMs }) }; + startedAt, completedAt, durationMs: Date.parse(completedAt) - Date.parse(startedAt), ...(remainingMs === undefined ? {} : { remainingMs }), + ...(status === 'complete' ? { next: `Another probe requires a checkpoint anchored on capture ${captureId} first; to stop exploring, publish none.` } : {}) }; } function checkpoint(root: string, checkpointId: string, source: string | Record) { @@ -204,11 +231,23 @@ function checkpoint(root: string, checkpointId: string, source: string | Record< function materialize(root: string, source: string) { const bytes = read(root, source); if (scan(decode(bytes)).findings.some(finding => finding.tier === 'HIGH')) throw new QaEvidenceError('Sensitive annotations cannot be published'); - const annotations = JSON.parse(decode(bytes)); + const supplied = JSON.parse(decode(bytes)); + if (!object(supplied)) throw new QaEvidenceError('Invalid report annotations: need a JSON object'); + const notes = checkpointNotes(root); + const annotations: Record = { + revision: supplied.revision ?? currentRevision(), + runtime: supplied.runtime ?? `bun ${Bun.version}`, + cwd: supplied.cwd ?? process.cwd(), + limits: supplied.limits, + evidence: supplied.evidence, + learning: supplied.learning ?? notes.filter(note => typeof note.observationCommand === 'string' && typeof note.nextCommand === 'string' + && nativeCommand(note.observationCommand) !== nativeCommand(note.nextCommand)).map(note => note.name.slice(12, 15)), + ...Object.fromEntries(Object.entries(supplied).filter(([key]) => !['revision', 'runtime', 'cwd', 'limits', 'evidence', 'learning'].includes(key))), + }; if (!exact(annotations, ['revision', 'runtime', 'cwd', 'limits', 'evidence', 'learning']) || !['revision', 'runtime', 'cwd'].every(key => typeof annotations[key] === 'string' && annotations[key].trim()) || !Array.isArray(annotations.limits) || !annotations.limits.length || !annotations.limits.every((limit: unknown) => typeof limit === 'string' && limit.trim()) - || !Array.isArray(annotations.evidence) || !Array.isArray(annotations.learning)) throw new QaEvidenceError('Invalid report annotations: need exactly revision, runtime and cwd (non-empty strings), limits (non-empty string array), evidence (row array) and learning (checkpoint ID array)'); + || !Array.isArray(annotations.evidence) || !Array.isArray(annotations.learning)) throw new QaEvidenceError('Invalid report annotations: need limits (non-empty string array) and evidence (row array), no other keys; revision, runtime and cwd (non-empty strings) and learning (checkpoint ID array) are filled in when omitted'); const captures = new Set(); const evidence = annotations.evidence.map((row: any) => { if (!exact(row, ['capture', 'command', 'contract', 'expected', 'classification']) @@ -224,7 +263,9 @@ function materialize(root: string, source: string) { return { observationCommand: note.observationCommand, hypothesis: note.hypothesis, nextCommand: note.nextCommand }; }); const sha256 = publish(root, 'evidence.json', { ...annotations, evidence, learning }); - return { action: 'materialize', status: 'complete', sha256, annotationsSha256: hash(bytes), exitCode: 0 }; + return { action: 'materialize', status: 'complete', sha256, annotationsSha256: hash(bytes), exitCode: 0, + reportLinks: notes.map(note => `[checkpoint ${note.name.slice(12, 15)}](${note.name})`), + next: 'Include every reportLinks entry in the Markdown report.' }; } export async function qaEvidenceMain(args: string[]): Promise { diff --git a/qa-only/sections/exploratory.md b/qa-only/sections/exploratory.md index 3a0a56619..ad03e1bd7 100644 --- a/qa-only/sections/exploratory.md +++ b/qa-only/sections/exploratory.md @@ -96,8 +96,8 @@ with their failing contract and expected assertion; never create tests or freeze ## 4. Final report Use the surface report template; link each checkpoint. Separate browser scores, functional outcomes and proposed/executed tests. -Write R/annotations.json: {revision, runtime, cwd, evidence: [{capture, command, contract, expected, classification}], learning: [checkpoint IDs], limits}. -Before Markdown, `bun Q materialize R annotations.json` builds evidence.json from it; Q fills observed/learning, not classifications. Retain all safe probes, including failures/replays; disclose withheld/incomplete evidence. +Write R/annotations.json: {evidence: [{capture, command, contract, expected, classification}], limits}. +Before Markdown, `bun Q materialize R annotations.json` builds evidence.json; Q fills observed, revision, runtime, cwd and learning and prints reportLinks to include; you classify. Retain all safe probes, including failures/replays; disclose withheld/incomplete evidence. Evidence is invocation-local. Missing prerequisites/expectations/observations, timeouts and refusal never pass. Pass requires all required current-input contracts to pass with no required remainder. diff --git a/qa/sections/exploratory.md b/qa/sections/exploratory.md index 779eb593a..760ee41c0 100644 --- a/qa/sections/exploratory.md +++ b/qa/sections/exploratory.md @@ -81,8 +81,8 @@ Never freeze buggy output, weaken tests or delete valid red tests. ## 4. Final report Use the surface report template; link each checkpoint. Separate browser scores, functional outcomes and proposed/executed tests. -Write R/annotations.json: {revision, runtime, cwd, evidence: [{capture, command, contract, expected, classification}], learning: [checkpoint IDs], limits}. -Before Markdown, `bun Q materialize R annotations.json` builds evidence.json from it; Q fills observed/learning, not classifications. Retain all safe probes, including failures/replays; disclose withheld/incomplete evidence. +Write R/annotations.json: {evidence: [{capture, command, contract, expected, classification}], limits}. +Before Markdown, `bun Q materialize R annotations.json` builds evidence.json; Q fills observed, revision, runtime, cwd and learning and prints reportLinks to include; you classify. Retain all safe probes, including failures/replays; disclose withheld/incomplete evidence. Evidence is invocation-local; /ship reruns once per invocation. Missing prerequisites/expectations/observations, timeouts and refusal never pass. Pass requires all required current-input contracts to pass with no required remainder. diff --git a/scripts/resolvers/qa.ts b/scripts/resolvers/qa.ts index 6a56bc113..bcdc40a3d 100644 --- a/scripts/resolvers/qa.ts +++ b/scripts/resolvers/qa.ts @@ -143,8 +143,8 @@ Never freeze buggy output, weaken tests or delete valid red tests.`} ## 4. Final report Use the surface report template; link each checkpoint. Separate browser scores, functional outcomes and proposed/executed tests. -Write R/annotations.json: {revision, runtime, cwd, evidence: [{capture, command, contract, expected, classification}], learning: [checkpoint IDs], limits}. -Before Markdown, \`bun Q materialize R annotations.json\` builds evidence.json from it; Q fills observed/learning, not classifications. Retain all safe probes, including failures/replays; disclose withheld/incomplete evidence. +Write R/annotations.json: {evidence: [{capture, command, contract, expected, classification}], limits}. +Before Markdown, \`bun Q materialize R annotations.json\` builds evidence.json; Q fills observed, revision, runtime, cwd and learning and prints reportLinks to include; you classify. Retain all safe probes, including failures/replays; disclose withheld/incomplete evidence. Evidence is invocation-local${reportOnly ? '.' : '; /ship reruns once per invocation.'} Missing prerequisites/expectations/observations, timeouts and refusal never pass. Pass requires all required current-input contracts to pass with no required remainder. diff --git a/test/qa-evidence.test.ts b/test/qa-evidence.test.ts index 406e25fa4..f1e860619 100644 --- a/test/qa-evidence.test.ts +++ b/test/qa-evidence.test.ts @@ -52,7 +52,7 @@ test('native capture executes once, preserves exact JSON and stderr, and materia f.json('annotations.json', { revision: 'revision', runtime: 'runtime', cwd: f.root, evidence: [], learning: [] }); const rejected = f.run('materialize', f.root, 'annotations.json'); expect(rejected.status).toBe(2); - expect(receipt(rejected.stderr).message).toContain('limits (non-empty string array)'); + expect(receipt(rejected.stderr).message).toContain('need limits (non-empty string array)'); f.json('annotations.json', { revision: 'revision', runtime: 'runtime', cwd: f.root, limits: ['Only the declared contract was checked.'], evidence: [{ capture: '001', command: 'first native command', contract: 'README.md', expected: 'Declared exact result', classification: 'pass' }], learning: ['001'] }); const report = f.run('materialize', f.root, 'annotations.json'); expect(report.status, report.stderr).toBe(0); @@ -252,3 +252,25 @@ test.skipIf(process.platform !== 'win32')('abrupt Windows evidence-wrapper exit expect(alive()).toBe(false); } finally { child.kill('SIGKILL'); if (pid && alive()) process.kill(pid, 'SIGKILL'); await closed; } }); + +test('a later capture requires a checkpoint anchored on the latest complete capture, and materialize fills metadata, learning and report links', () => { + const f = fixture(); + expect(f.capture('001', 'console.log(JSON.stringify({ step: 1 }))').status).toBe(0); + const second = (id: string) => `bun gstack-qa-evidence capture ${f.root} ${id} --timeout-ms 4000 -- probe two`; + const refused = f.capture('002', 'console.log(JSON.stringify({ step: 2 }))'); + expect(refused.status).toBe(2); + expect(receipt(refused.stderr).message).toContain('Checkpoint required before capture 002'); + expect(fs.existsSync(path.join(f.root, '.qa-evidence/002'))).toBe(false); + const first = `bun gstack-qa-evidence capture ${f.root} 001 --timeout-ms 4000 -- probe one`; + expect(f.run('checkpoint', f.root, '001', '001', first, 'The first observation makes the second input the riskiest next probe.', second('002')).status).toBe(0); + const allowed = f.capture('002', 'console.log(JSON.stringify({ step: 2 }))'); + expect(allowed.status, allowed.stderr).toBe(0); + expect(receipt(allowed.stdout).next).toContain('anchored on capture 002'); + f.json('annotations.json', { revision: 'fixture-revision', limits: ['Only two probes ran.'], evidence: [{ capture: '001', command: first, contract: 'README.md', expected: 'step 1', classification: 'pass' }] }); + const report = f.run('materialize', f.root, 'annotations.json'); + expect(report.status, report.stderr).toBe(0); + expect(receipt(report.stdout).reportLinks).toEqual(['[checkpoint 001](exploration-001.json)']); + const final = JSON.parse(fs.readFileSync(path.join(f.root, 'evidence.json'), 'utf8')); + expect(final).toMatchObject({ runtime: `bun ${Bun.version}`, cwd: f.root }); + expect(final.learning).toEqual([{ observationCommand: first, hypothesis: 'The first observation makes the second input the riskiest next probe.', nextCommand: second('002') }]); +});