From 1643cd94de4220712afdcc86bf32dbd895338aa0 Mon Sep 17 00:00:00 2001 From: garrytan Date: Wed, 30 Sep 2026 18:24:46 +0000 Subject: [PATCH] fix(qa-evidence,observer): reject placeholder metadata and replay-only learning; declare the docs atomic-write target - materialize measures revision, runtime and cwd itself and rejects supplied values that differ (CI run wrote revision "HEAD" and runtime "bun"), and refuses learning checkpoints that replay the same probe, naming the fix. - The docs write observer treats Claude Code's atomic temp for the authorized doc target as transient, so a temp renamed before its per-file watch no longer marks the observation incomplete (ship-docsync-completion flake). Per-file monitoring outside declared targets stays fail-closed. --- lib/qa-evidence.ts | 19 ++++++++++++++----- test/helpers/docsync-observer.ts | 2 +- test/helpers/qa-functional-observer.ts | 5 +++-- test/qa-caller-report-observer.test.ts | 12 ++++++++++-- test/qa-evidence.test.ts | 24 ++++++++++++++++++++++-- 5 files changed, 50 insertions(+), 12 deletions(-) diff --git a/lib/qa-evidence.ts b/lib/qa-evidence.ts index 0bf6e9347..22eaa8fec 100644 --- a/lib/qa-evidence.ts +++ b/lib/qa-evidence.ts @@ -12,8 +12,7 @@ const exact = (value: unknown, keys: string[]) => object(value) && Object.keys(v class QaEvidenceError extends Error {} const currentRevision = () => { const result = spawnSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8', timeout: 5000, env: { ...process.env, GIT_OPTIONAL_LOCKS: '0' } }); - if (result.status !== 0 || !/^[0-9a-f]{40,64}$/.test(result.stdout.trim())) throw new QaEvidenceError('Invalid report annotations: revision is required when git rev-parse HEAD is unavailable'); - return result.stdout.trim(); + return result.status === 0 && /^[0-9a-f]{40,64}$/.test(result.stdout.trim()) ? result.stdout.trim() : undefined; }; function id(value: string): string { @@ -234,10 +233,17 @@ function materialize(root: string, source: string) { const supplied = JSON.parse(decode(bytes)); if (!object(supplied)) throw new QaEvidenceError('Invalid report annotations: need a JSON object'); const notes = checkpointNotes(root); + const measured: Record = { revision: currentRevision(), runtime: `bun ${Bun.version}`, cwd: process.cwd() }; + for (const [key, value] of Object.entries(measured)) { + if (value !== undefined && supplied[key] !== undefined && supplied[key] !== value) { + throw new QaEvidenceError(`Invalid report annotations: ${key} must be ${JSON.stringify(value)}; omit it and Q fills it`); + } + } + if (!measured.revision && supplied.revision === undefined) throw new QaEvidenceError('Invalid report annotations: revision is required when git rev-parse HEAD is unavailable'); const annotations: Record = { - revision: supplied.revision ?? currentRevision(), - runtime: supplied.runtime ?? `bun ${Bun.version}`, - cwd: supplied.cwd ?? process.cwd(), + revision: measured.revision ?? supplied.revision, + runtime: measured.runtime, + cwd: measured.cwd, limits: supplied.limits, evidence: supplied.evidence, learning: supplied.learning ?? notes.filter(note => typeof note.observationCommand === 'string' && typeof note.nextCommand === 'string' @@ -260,6 +266,9 @@ function materialize(root: string, source: string) { if (typeof name !== 'string') throw new QaEvidenceError('Invalid checkpoint reference'); const note = JSON.parse(decode(read(root, `exploration-${id(name)}.json`))); if (!exact(note, ['observationCommand', 'observed', 'hypothesis', 'nextCommand'])) throw new QaEvidenceError('Invalid referenced checkpoint'); + if (nativeCommand(note.observationCommand) === nativeCommand(note.nextCommand)) { + throw new QaEvidenceError(`Invalid learning: checkpoint ${name} replays the same probe; name checkpoints whose next probe differs, or omit learning and Q selects them`); + } return { observationCommand: note.observationCommand, hypothesis: note.hypothesis, nextCommand: note.nextCommand }; }); const sha256 = publish(root, 'evidence.json', { ...annotations, evidence, learning }); diff --git a/test/helpers/docsync-observer.ts b/test/helpers/docsync-observer.ts index 417d5739f..d4bc1f4d6 100644 --- a/test/helpers/docsync-observer.ts +++ b/test/helpers/docsync-observer.ts @@ -8,7 +8,7 @@ import { DOC_PATH, type DocsScenario, type fixtureDocs } from './docsync-fixture import { sliceBetween } from './skill-fixture'; export async function observeDocsWrites(fixture: ReturnType) { - return observeQAWrites(fixture.repo); + return observeQAWrites(fixture.repo, { atomicTargets: [DOC_PATH] }); } type DocsWriteContext = { diff --git a/test/helpers/qa-functional-observer.ts b/test/helpers/qa-functional-observer.ts index 3230b9cba..18e86877d 100644 --- a/test/helpers/qa-functional-observer.ts +++ b/test/helpers/qa-functional-observer.ts @@ -72,7 +72,7 @@ export interface QAWriteObservation { limits: string[]; } -export async function observeQAWrites(root: string, options: { reportDirectory?: string; evidenceProducer?: boolean } = {}) { +export async function observeQAWrites(root: string, options: { reportDirectory?: string; evidenceProducer?: boolean; atomicTargets?: string[] } = {}) { if (process.platform !== 'linux') throw new Error('QA write observer unavailable: Linux inotify required'); if (fs.realpathSync(root) !== root) throw new Error('Observer root must be canonical'); let reportDirectory: string | undefined; @@ -82,7 +82,8 @@ export async function observeQAWrites(root: string, options: { reportDirectory?: reportDirectory = path.relative(root, directory); } const transientFile = (relative: string) => qaWriteAllowed(relative, 'qa-only') - || (reportDirectory !== undefined && relative.startsWith(reportDirectory + path.sep)); + || (reportDirectory !== undefined && relative.startsWith(reportDirectory + path.sep)) + || (options.atomicTargets ?? []).some(target => relative.startsWith(target + '.tmp.') && /^\.tmp\.[1-9]\d*\.[0-9a-f]{12}$/.test(relative.slice(target.length))); const before = qaTreeSnapshot(root); const { dlopen, FFIType, ptr } = await import('bun:ffi'); const libc = dlopen('libc.so.6', { diff --git a/test/qa-caller-report-observer.test.ts b/test/qa-caller-report-observer.test.ts index 5cf93f609..68004e864 100644 --- a/test/qa-caller-report-observer.test.ts +++ b/test/qa-caller-report-observer.test.ts @@ -14,11 +14,11 @@ function fixture() { return root; } -async function atomicPublication(directory: string, declared?: string) { +async function atomicPublication(directory: string, declared?: string, atomicTargets?: string[]) { const root = fixture(); const temporary = path.join(root, directory, 'exploration-004.json.tmp.2644.340bb6ad0afe'); const target = path.join(root, directory, 'exploration-004.json'); - const observer = await observeQAWrites(root, { reportDirectory: declared }); + const observer = await observeQAWrites(root, { reportDirectory: declared, atomicTargets }); const stat = fs.lstatSync; let renamedAtWatch = false; let stopped = false; @@ -75,6 +75,14 @@ async function atomicPublication(directory: string, declared?: string) { }); } + test('an authorized atomic target outside the report directory publishes without the per-file watch race', async () => { + const result = await atomicPublication('foreign/reports', undefined, ['foreign/reports/exploration-004.json']); + expect(result.renamedAtWatch).toBe(false); + expect(result.observation.failures).toEqual([]); + expect(result.observation.complete).toBe(true); + expect(result.observation.events).toContainEqual(expect.objectContaining({ path: result.target, mask: 0x80 })); + }); + test('supports an explicitly selected nested report directory, not an implicit reports name', async () => { const result = await atomicPublication('foreign/reports', 'foreign/reports'); expect(result.observation.complete).toBe(true); diff --git a/test/qa-evidence.test.ts b/test/qa-evidence.test.ts index f1e860619..e9c4c23f8 100644 --- a/test/qa-evidence.test.ts +++ b/test/qa-evidence.test.ts @@ -49,11 +49,11 @@ test('native capture executes once, preserves exact JSON and stderr, and materia expect(JSON.parse(fs.readFileSync(path.join(f.root, 'exploration-001.json'), 'utf8'))).toEqual({ observationCommand: 'first native command', observed, hypothesis: 'The successful boundary suggests testing the rejected input next.', nextCommand: 'second native command', }); - f.json('annotations.json', { revision: 'revision', runtime: 'runtime', cwd: f.root, evidence: [], learning: [] }); + f.json('annotations.json', { revision: 'revision', evidence: [], learning: [] }); const rejected = f.run('materialize', f.root, 'annotations.json'); expect(rejected.status).toBe(2); expect(receipt(rejected.stderr).message).toContain('need limits (non-empty string array)'); - f.json('annotations.json', { revision: 'revision', runtime: 'runtime', cwd: f.root, limits: ['Only the declared contract was checked.'], evidence: [{ capture: '001', command: 'first native command', contract: 'README.md', expected: 'Declared exact result', classification: 'pass' }], learning: ['001'] }); + f.json('annotations.json', { revision: 'revision', limits: ['Only the declared contract was checked.'], evidence: [{ capture: '001', command: 'first native command', contract: 'README.md', expected: 'Declared exact result', classification: 'pass' }], learning: ['001'] }); const report = f.run('materialize', f.root, 'annotations.json'); expect(report.status, report.stderr).toBe(0); const final = JSON.parse(fs.readFileSync(path.join(f.root, 'evidence.json'), 'utf8')); @@ -274,3 +274,23 @@ test('a later capture requires a checkpoint anchored on the latest complete capt expect(final).toMatchObject({ runtime: `bun ${Bun.version}`, cwd: f.root }); expect(final.learning).toEqual([{ observationCommand: first, hypothesis: 'The first observation makes the second input the riskiest next probe.', nextCommand: second('002') }]); }); + +test('materialize rejects placeholder metadata and same-probe learning with the fix in the message', () => { + const f = fixture(); + expect(f.capture('001', 'console.log(JSON.stringify({ step: 1 }))').status).toBe(0); + const command = (id: string) => `bun gstack-qa-evidence capture ${f.root} ${id} --timeout-ms 4000 -- probe same`; + expect(f.run('checkpoint', f.root, '001', '001', command('001'), 'Replaying the identical probe checks whether the first result is deterministic.', command('002')).status).toBe(0); + const row = { capture: '001', command: command('001'), contract: 'README.md', expected: 'step 1', classification: 'pass' }; + f.json('annotations.json', { revision: 'fixture-revision', runtime: 'bun', limits: ['One probe.'], evidence: [row] }); + const placeholder = f.run('materialize', f.root, 'annotations.json'); + expect(placeholder.status).toBe(2); + expect(receipt(placeholder.stderr).message).toContain(`runtime must be "bun ${Bun.version}"`); + f.json('annotations.json', { revision: 'fixture-revision', limits: ['One probe.'], evidence: [row], learning: ['001'] }); + const replay = f.run('materialize', f.root, 'annotations.json'); + expect(replay.status).toBe(2); + expect(receipt(replay.stderr).message).toContain('checkpoint 001 replays the same probe'); + f.json('annotations.json', { revision: 'fixture-revision', limits: ['One probe.'], evidence: [row] }); + const selected = f.run('materialize', f.root, 'annotations.json'); + expect(selected.status, selected.stderr).toBe(0); + expect(JSON.parse(fs.readFileSync(path.join(f.root, 'evidence.json'), 'utf8')).learning).toEqual([]); +});