fix(qa-evidence,observer): reject placeholder metadata and replay-only learning; declare the docs atomic-write target

- materialize measures revision, runtime and cwd itself and rejects supplied
  values that differ (CI run wrote revision "HEAD" and runtime "bun"), and
  refuses learning checkpoints that replay the same probe, naming the fix.
- The docs write observer treats Claude Code's atomic temp for the authorized
  doc target as transient, so a temp renamed before its per-file watch no
  longer marks the observation incomplete (ship-docsync-completion flake).
  Per-file monitoring outside declared targets stays fail-closed.
This commit is contained in:
garrytan committed 2026-09-30 18:24:46 +00:00
1 parent 131d43be0a
commit 1643cd94de
5 files changed
+50 -12

No files matched your search

+14 -5
View File
@@ -12,8 +12,7 @@ const exact = (value: unknown, keys: string[]) => object(value) && Object.keys(v
class QaEvidenceError extends Error {} class QaEvidenceError extends Error {}
const currentRevision = () => { const currentRevision = () => {
const result = spawnSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8', timeout: 5000, env: { ...process.env, GIT_OPTIONAL_LOCKS: '0' } }); const result = spawnSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8', timeout: 5000, env: { ...process.env, GIT_OPTIONAL_LOCKS: '0' } });
if (result.status !== 0 || !/^[0-9a-f]{40,64}$/.test(result.stdout.trim())) throw new QaEvidenceError('Invalid report annotations: revision is required when git rev-parse HEAD is unavailable'); return result.status === 0 && /^[0-9a-f]{40,64}$/.test(result.stdout.trim()) ? result.stdout.trim() : undefined;
return result.stdout.trim();
}; };
function id(value: string): string { function id(value: string): string {
@@ -234,10 +233,17 @@ function materialize(root: string, source: string) {
const supplied = JSON.parse(decode(bytes)); const supplied = JSON.parse(decode(bytes));
if (!object(supplied)) throw new QaEvidenceError('Invalid report annotations: need a JSON object'); if (!object(supplied)) throw new QaEvidenceError('Invalid report annotations: need a JSON object');
const notes = checkpointNotes(root); const notes = checkpointNotes(root);
const measured: Record<string, string | undefined> = { revision: currentRevision(), runtime: `bun ${Bun.version}`, cwd: process.cwd() };
for (const [key, value] of Object.entries(measured)) {
if (value !== undefined && supplied[key] !== undefined && supplied[key] !== value) {
throw new QaEvidenceError(`Invalid report annotations: ${key} must be ${JSON.stringify(value)}; omit it and Q fills it`);
}
}
if (!measured.revision && supplied.revision === undefined) throw new QaEvidenceError('Invalid report annotations: revision is required when git rev-parse HEAD is unavailable');
const annotations: Record<string, any> = { const annotations: Record<string, any> = {
revision: supplied.revision ?? currentRevision(), revision: measured.revision ?? supplied.revision,
runtime: supplied.runtime ?? `bun ${Bun.version}`, runtime: measured.runtime,
cwd: supplied.cwd ?? process.cwd(), cwd: measured.cwd,
limits: supplied.limits, limits: supplied.limits,
evidence: supplied.evidence, evidence: supplied.evidence,
learning: supplied.learning ?? notes.filter(note => typeof note.observationCommand === 'string' && typeof note.nextCommand === 'string' learning: supplied.learning ?? notes.filter(note => typeof note.observationCommand === 'string' && typeof note.nextCommand === 'string'
@@ -260,6 +266,9 @@ function materialize(root: string, source: string) {
if (typeof name !== 'string') throw new QaEvidenceError('Invalid checkpoint reference'); if (typeof name !== 'string') throw new QaEvidenceError('Invalid checkpoint reference');
const note = JSON.parse(decode(read(root, `exploration-${id(name)}.json`))); const note = JSON.parse(decode(read(root, `exploration-${id(name)}.json`)));
if (!exact(note, ['observationCommand', 'observed', 'hypothesis', 'nextCommand'])) throw new QaEvidenceError('Invalid referenced checkpoint'); if (!exact(note, ['observationCommand', 'observed', 'hypothesis', 'nextCommand'])) throw new QaEvidenceError('Invalid referenced checkpoint');
if (nativeCommand(note.observationCommand) === nativeCommand(note.nextCommand)) {
throw new QaEvidenceError(`Invalid learning: checkpoint ${name} replays the same probe; name checkpoints whose next probe differs, or omit learning and Q selects them`);
}
return { observationCommand: note.observationCommand, hypothesis: note.hypothesis, nextCommand: note.nextCommand }; return { observationCommand: note.observationCommand, hypothesis: note.hypothesis, nextCommand: note.nextCommand };
}); });
const sha256 = publish(root, 'evidence.json', { ...annotations, evidence, learning }); const sha256 = publish(root, 'evidence.json', { ...annotations, evidence, learning });
+1 -1
View File
@@ -8,7 +8,7 @@ import { DOC_PATH, type DocsScenario, type fixtureDocs } from './docsync-fixture
import { sliceBetween } from './skill-fixture'; import { sliceBetween } from './skill-fixture';
export async function observeDocsWrites(fixture: ReturnType<typeof fixtureDocs>) { export async function observeDocsWrites(fixture: ReturnType<typeof fixtureDocs>) {
return observeQAWrites(fixture.repo); return observeQAWrites(fixture.repo, { atomicTargets: [DOC_PATH] });
} }
type DocsWriteContext = { type DocsWriteContext = {
+3 -2
View File
@@ -72,7 +72,7 @@ export interface QAWriteObservation {
limits: string[]; limits: string[];
} }
export async function observeQAWrites(root: string, options: { reportDirectory?: string; evidenceProducer?: boolean } = {}) { export async function observeQAWrites(root: string, options: { reportDirectory?: string; evidenceProducer?: boolean; atomicTargets?: string[] } = {}) {
if (process.platform !== 'linux') throw new Error('QA write observer unavailable: Linux inotify required'); if (process.platform !== 'linux') throw new Error('QA write observer unavailable: Linux inotify required');
if (fs.realpathSync(root) !== root) throw new Error('Observer root must be canonical'); if (fs.realpathSync(root) !== root) throw new Error('Observer root must be canonical');
let reportDirectory: string | undefined; let reportDirectory: string | undefined;
@@ -82,7 +82,8 @@ export async function observeQAWrites(root: string, options: { reportDirectory?:
reportDirectory = path.relative(root, directory); reportDirectory = path.relative(root, directory);
} }
const transientFile = (relative: string) => qaWriteAllowed(relative, 'qa-only') const transientFile = (relative: string) => qaWriteAllowed(relative, 'qa-only')
|| (reportDirectory !== undefined && relative.startsWith(reportDirectory + path.sep)); || (reportDirectory !== undefined && relative.startsWith(reportDirectory + path.sep))
|| (options.atomicTargets ?? []).some(target => relative.startsWith(target + '.tmp.') && /^\.tmp\.[1-9]\d*\.[0-9a-f]{12}$/.test(relative.slice(target.length)));
const before = qaTreeSnapshot(root); const before = qaTreeSnapshot(root);
const { dlopen, FFIType, ptr } = await import('bun:ffi'); const { dlopen, FFIType, ptr } = await import('bun:ffi');
const libc = dlopen('libc.so.6', { const libc = dlopen('libc.so.6', {
+10 -2
View File
@@ -14,11 +14,11 @@ function fixture() {
return root; return root;
} }
async function atomicPublication(directory: string, declared?: string) { async function atomicPublication(directory: string, declared?: string, atomicTargets?: string[]) {
const root = fixture(); const root = fixture();
const temporary = path.join(root, directory, 'exploration-004.json.tmp.2644.340bb6ad0afe'); const temporary = path.join(root, directory, 'exploration-004.json.tmp.2644.340bb6ad0afe');
const target = path.join(root, directory, 'exploration-004.json'); const target = path.join(root, directory, 'exploration-004.json');
const observer = await observeQAWrites(root, { reportDirectory: declared }); const observer = await observeQAWrites(root, { reportDirectory: declared, atomicTargets });
const stat = fs.lstatSync; const stat = fs.lstatSync;
let renamedAtWatch = false; let renamedAtWatch = false;
let stopped = false; let stopped = false;
@@ -75,6 +75,14 @@ async function atomicPublication(directory: string, declared?: string) {
}); });
} }
test('an authorized atomic target outside the report directory publishes without the per-file watch race', async () => {
const result = await atomicPublication('foreign/reports', undefined, ['foreign/reports/exploration-004.json']);
expect(result.renamedAtWatch).toBe(false);
expect(result.observation.failures).toEqual([]);
expect(result.observation.complete).toBe(true);
expect(result.observation.events).toContainEqual(expect.objectContaining({ path: result.target, mask: 0x80 }));
});
test('supports an explicitly selected nested report directory, not an implicit reports name', async () => { test('supports an explicitly selected nested report directory, not an implicit reports name', async () => {
const result = await atomicPublication('foreign/reports', 'foreign/reports'); const result = await atomicPublication('foreign/reports', 'foreign/reports');
expect(result.observation.complete).toBe(true); expect(result.observation.complete).toBe(true);
+22 -2
View File
@@ -49,11 +49,11 @@ test('native capture executes once, preserves exact JSON and stderr, and materia
expect(JSON.parse(fs.readFileSync(path.join(f.root, 'exploration-001.json'), 'utf8'))).toEqual({ expect(JSON.parse(fs.readFileSync(path.join(f.root, 'exploration-001.json'), 'utf8'))).toEqual({
observationCommand: 'first native command', observed, hypothesis: 'The successful boundary suggests testing the rejected input next.', nextCommand: 'second native command', observationCommand: 'first native command', observed, hypothesis: 'The successful boundary suggests testing the rejected input next.', nextCommand: 'second native command',
}); });
f.json('annotations.json', { revision: 'revision', runtime: 'runtime', cwd: f.root, evidence: [], learning: [] }); f.json('annotations.json', { revision: 'revision', evidence: [], learning: [] });
const rejected = f.run('materialize', f.root, 'annotations.json'); const rejected = f.run('materialize', f.root, 'annotations.json');
expect(rejected.status).toBe(2); expect(rejected.status).toBe(2);
expect(receipt(rejected.stderr).message).toContain('need limits (non-empty string array)'); expect(receipt(rejected.stderr).message).toContain('need limits (non-empty string array)');
f.json('annotations.json', { revision: 'revision', runtime: 'runtime', cwd: f.root, limits: ['Only the declared contract was checked.'], evidence: [{ capture: '001', command: 'first native command', contract: 'README.md', expected: 'Declared exact result', classification: 'pass' }], learning: ['001'] }); f.json('annotations.json', { revision: 'revision', limits: ['Only the declared contract was checked.'], evidence: [{ capture: '001', command: 'first native command', contract: 'README.md', expected: 'Declared exact result', classification: 'pass' }], learning: ['001'] });
const report = f.run('materialize', f.root, 'annotations.json'); const report = f.run('materialize', f.root, 'annotations.json');
expect(report.status, report.stderr).toBe(0); expect(report.status, report.stderr).toBe(0);
const final = JSON.parse(fs.readFileSync(path.join(f.root, 'evidence.json'), 'utf8')); const final = JSON.parse(fs.readFileSync(path.join(f.root, 'evidence.json'), 'utf8'));
@@ -274,3 +274,23 @@ test('a later capture requires a checkpoint anchored on the latest complete capt
expect(final).toMatchObject({ runtime: `bun ${Bun.version}`, cwd: f.root }); expect(final).toMatchObject({ runtime: `bun ${Bun.version}`, cwd: f.root });
expect(final.learning).toEqual([{ observationCommand: first, hypothesis: 'The first observation makes the second input the riskiest next probe.', nextCommand: second('002') }]); expect(final.learning).toEqual([{ observationCommand: first, hypothesis: 'The first observation makes the second input the riskiest next probe.', nextCommand: second('002') }]);
}); });
test('materialize rejects placeholder metadata and same-probe learning with the fix in the message', () => {
const f = fixture();
expect(f.capture('001', 'console.log(JSON.stringify({ step: 1 }))').status).toBe(0);
const command = (id: string) => `bun gstack-qa-evidence capture ${f.root} ${id} --timeout-ms 4000 -- probe same`;
expect(f.run('checkpoint', f.root, '001', '001', command('001'), 'Replaying the identical probe checks whether the first result is deterministic.', command('002')).status).toBe(0);
const row = { capture: '001', command: command('001'), contract: 'README.md', expected: 'step 1', classification: 'pass' };
f.json('annotations.json', { revision: 'fixture-revision', runtime: 'bun', limits: ['One probe.'], evidence: [row] });
const placeholder = f.run('materialize', f.root, 'annotations.json');
expect(placeholder.status).toBe(2);
expect(receipt(placeholder.stderr).message).toContain(`runtime must be "bun ${Bun.version}"`);
f.json('annotations.json', { revision: 'fixture-revision', limits: ['One probe.'], evidence: [row], learning: ['001'] });
const replay = f.run('materialize', f.root, 'annotations.json');
expect(replay.status).toBe(2);
expect(receipt(replay.stderr).message).toContain('checkpoint 001 replays the same probe');
f.json('annotations.json', { revision: 'fixture-revision', limits: ['One probe.'], evidence: [row] });
const selected = f.run('materialize', f.root, 'annotations.json');
expect(selected.status, selected.stderr).toBe(0);
expect(JSON.parse(fs.readFileSync(path.join(f.root, 'evidence.json'), 'utf8')).learning).toEqual([]);
});