diff --git a/lib/qa-evidence.ts b/lib/qa-evidence.ts index 0bf6e9347..22eaa8fec 100644 --- a/lib/qa-evidence.ts +++ b/lib/qa-evidence.ts @@ -12,8 +12,7 @@ const exact = (value: unknown, keys: string[]) => object(value) && Object.keys(v class QaEvidenceError extends Error {} const currentRevision = () => { const result = spawnSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8', timeout: 5000, env: { ...process.env, GIT_OPTIONAL_LOCKS: '0' } }); - if (result.status !== 0 || !/^[0-9a-f]{40,64}$/.test(result.stdout.trim())) throw new QaEvidenceError('Invalid report annotations: revision is required when git rev-parse HEAD is unavailable'); - return result.stdout.trim(); + return result.status === 0 && /^[0-9a-f]{40,64}$/.test(result.stdout.trim()) ? result.stdout.trim() : undefined; }; function id(value: string): string { @@ -234,10 +233,17 @@ function materialize(root: string, source: string) { const supplied = JSON.parse(decode(bytes)); if (!object(supplied)) throw new QaEvidenceError('Invalid report annotations: need a JSON object'); const notes = checkpointNotes(root); + const measured: Record = { revision: currentRevision(), runtime: `bun ${Bun.version}`, cwd: process.cwd() }; + for (const [key, value] of Object.entries(measured)) { + if (value !== undefined && supplied[key] !== undefined && supplied[key] !== value) { + throw new QaEvidenceError(`Invalid report annotations: ${key} must be ${JSON.stringify(value)}; omit it and Q fills it`); + } + } + if (!measured.revision && supplied.revision === undefined) throw new QaEvidenceError('Invalid report annotations: revision is required when git rev-parse HEAD is unavailable'); const annotations: Record = { - revision: supplied.revision ?? currentRevision(), - runtime: supplied.runtime ?? `bun ${Bun.version}`, - cwd: supplied.cwd ?? process.cwd(), + revision: measured.revision ?? supplied.revision, + runtime: measured.runtime, + cwd: measured.cwd, limits: supplied.limits, evidence: supplied.evidence, learning: supplied.learning ?? notes.filter(note => typeof note.observationCommand === 'string' && typeof note.nextCommand === 'string' @@ -260,6 +266,9 @@ function materialize(root: string, source: string) { if (typeof name !== 'string') throw new QaEvidenceError('Invalid checkpoint reference'); const note = JSON.parse(decode(read(root, `exploration-${id(name)}.json`))); if (!exact(note, ['observationCommand', 'observed', 'hypothesis', 'nextCommand'])) throw new QaEvidenceError('Invalid referenced checkpoint'); + if (nativeCommand(note.observationCommand) === nativeCommand(note.nextCommand)) { + throw new QaEvidenceError(`Invalid learning: checkpoint ${name} replays the same probe; name checkpoints whose next probe differs, or omit learning and Q selects them`); + } return { observationCommand: note.observationCommand, hypothesis: note.hypothesis, nextCommand: note.nextCommand }; }); const sha256 = publish(root, 'evidence.json', { ...annotations, evidence, learning }); diff --git a/test/helpers/docsync-observer.ts b/test/helpers/docsync-observer.ts index 417d5739f..d4bc1f4d6 100644 --- a/test/helpers/docsync-observer.ts +++ b/test/helpers/docsync-observer.ts @@ -8,7 +8,7 @@ import { DOC_PATH, type DocsScenario, type fixtureDocs } from './docsync-fixture import { sliceBetween } from './skill-fixture'; export async function observeDocsWrites(fixture: ReturnType) { - return observeQAWrites(fixture.repo); + return observeQAWrites(fixture.repo, { atomicTargets: [DOC_PATH] }); } type DocsWriteContext = { diff --git a/test/helpers/qa-functional-observer.ts b/test/helpers/qa-functional-observer.ts index 3230b9cba..18e86877d 100644 --- a/test/helpers/qa-functional-observer.ts +++ b/test/helpers/qa-functional-observer.ts @@ -72,7 +72,7 @@ export interface QAWriteObservation { limits: string[]; } -export async function observeQAWrites(root: string, options: { reportDirectory?: string; evidenceProducer?: boolean } = {}) { +export async function observeQAWrites(root: string, options: { reportDirectory?: string; evidenceProducer?: boolean; atomicTargets?: string[] } = {}) { if (process.platform !== 'linux') throw new Error('QA write observer unavailable: Linux inotify required'); if (fs.realpathSync(root) !== root) throw new Error('Observer root must be canonical'); let reportDirectory: string | undefined; @@ -82,7 +82,8 @@ export async function observeQAWrites(root: string, options: { reportDirectory?: reportDirectory = path.relative(root, directory); } const transientFile = (relative: string) => qaWriteAllowed(relative, 'qa-only') - || (reportDirectory !== undefined && relative.startsWith(reportDirectory + path.sep)); + || (reportDirectory !== undefined && relative.startsWith(reportDirectory + path.sep)) + || (options.atomicTargets ?? []).some(target => relative.startsWith(target + '.tmp.') && /^\.tmp\.[1-9]\d*\.[0-9a-f]{12}$/.test(relative.slice(target.length))); const before = qaTreeSnapshot(root); const { dlopen, FFIType, ptr } = await import('bun:ffi'); const libc = dlopen('libc.so.6', { diff --git a/test/qa-caller-report-observer.test.ts b/test/qa-caller-report-observer.test.ts index 5cf93f609..68004e864 100644 --- a/test/qa-caller-report-observer.test.ts +++ b/test/qa-caller-report-observer.test.ts @@ -14,11 +14,11 @@ function fixture() { return root; } -async function atomicPublication(directory: string, declared?: string) { +async function atomicPublication(directory: string, declared?: string, atomicTargets?: string[]) { const root = fixture(); const temporary = path.join(root, directory, 'exploration-004.json.tmp.2644.340bb6ad0afe'); const target = path.join(root, directory, 'exploration-004.json'); - const observer = await observeQAWrites(root, { reportDirectory: declared }); + const observer = await observeQAWrites(root, { reportDirectory: declared, atomicTargets }); const stat = fs.lstatSync; let renamedAtWatch = false; let stopped = false; @@ -75,6 +75,14 @@ async function atomicPublication(directory: string, declared?: string) { }); } + test('an authorized atomic target outside the report directory publishes without the per-file watch race', async () => { + const result = await atomicPublication('foreign/reports', undefined, ['foreign/reports/exploration-004.json']); + expect(result.renamedAtWatch).toBe(false); + expect(result.observation.failures).toEqual([]); + expect(result.observation.complete).toBe(true); + expect(result.observation.events).toContainEqual(expect.objectContaining({ path: result.target, mask: 0x80 })); + }); + test('supports an explicitly selected nested report directory, not an implicit reports name', async () => { const result = await atomicPublication('foreign/reports', 'foreign/reports'); expect(result.observation.complete).toBe(true); diff --git a/test/qa-evidence.test.ts b/test/qa-evidence.test.ts index f1e860619..e9c4c23f8 100644 --- a/test/qa-evidence.test.ts +++ b/test/qa-evidence.test.ts @@ -49,11 +49,11 @@ test('native capture executes once, preserves exact JSON and stderr, and materia expect(JSON.parse(fs.readFileSync(path.join(f.root, 'exploration-001.json'), 'utf8'))).toEqual({ observationCommand: 'first native command', observed, hypothesis: 'The successful boundary suggests testing the rejected input next.', nextCommand: 'second native command', }); - f.json('annotations.json', { revision: 'revision', runtime: 'runtime', cwd: f.root, evidence: [], learning: [] }); + f.json('annotations.json', { revision: 'revision', evidence: [], learning: [] }); const rejected = f.run('materialize', f.root, 'annotations.json'); expect(rejected.status).toBe(2); expect(receipt(rejected.stderr).message).toContain('need limits (non-empty string array)'); - f.json('annotations.json', { revision: 'revision', runtime: 'runtime', cwd: f.root, limits: ['Only the declared contract was checked.'], evidence: [{ capture: '001', command: 'first native command', contract: 'README.md', expected: 'Declared exact result', classification: 'pass' }], learning: ['001'] }); + f.json('annotations.json', { revision: 'revision', limits: ['Only the declared contract was checked.'], evidence: [{ capture: '001', command: 'first native command', contract: 'README.md', expected: 'Declared exact result', classification: 'pass' }], learning: ['001'] }); const report = f.run('materialize', f.root, 'annotations.json'); expect(report.status, report.stderr).toBe(0); const final = JSON.parse(fs.readFileSync(path.join(f.root, 'evidence.json'), 'utf8')); @@ -274,3 +274,23 @@ test('a later capture requires a checkpoint anchored on the latest complete capt expect(final).toMatchObject({ runtime: `bun ${Bun.version}`, cwd: f.root }); expect(final.learning).toEqual([{ observationCommand: first, hypothesis: 'The first observation makes the second input the riskiest next probe.', nextCommand: second('002') }]); }); + +test('materialize rejects placeholder metadata and same-probe learning with the fix in the message', () => { + const f = fixture(); + expect(f.capture('001', 'console.log(JSON.stringify({ step: 1 }))').status).toBe(0); + const command = (id: string) => `bun gstack-qa-evidence capture ${f.root} ${id} --timeout-ms 4000 -- probe same`; + expect(f.run('checkpoint', f.root, '001', '001', command('001'), 'Replaying the identical probe checks whether the first result is deterministic.', command('002')).status).toBe(0); + const row = { capture: '001', command: command('001'), contract: 'README.md', expected: 'step 1', classification: 'pass' }; + f.json('annotations.json', { revision: 'fixture-revision', runtime: 'bun', limits: ['One probe.'], evidence: [row] }); + const placeholder = f.run('materialize', f.root, 'annotations.json'); + expect(placeholder.status).toBe(2); + expect(receipt(placeholder.stderr).message).toContain(`runtime must be "bun ${Bun.version}"`); + f.json('annotations.json', { revision: 'fixture-revision', limits: ['One probe.'], evidence: [row], learning: ['001'] }); + const replay = f.run('materialize', f.root, 'annotations.json'); + expect(replay.status).toBe(2); + expect(receipt(replay.stderr).message).toContain('checkpoint 001 replays the same probe'); + f.json('annotations.json', { revision: 'fixture-revision', limits: ['One probe.'], evidence: [row] }); + const selected = f.run('materialize', f.root, 'annotations.json'); + expect(selected.status, selected.stderr).toBe(0); + expect(JSON.parse(fs.readFileSync(path.join(f.root, 'evidence.json'), 'utf8')).learning).toEqual([]); +});