fix(qa-evidence,observer): reject placeholder metadata and replay-only learning; declare the docs atomic-write target

- materialize measures revision, runtime and cwd itself and rejects supplied
  values that differ (CI run wrote revision "HEAD" and runtime "bun"), and
  refuses learning checkpoints that replay the same probe, naming the fix.
- The docs write observer treats Claude Code's atomic temp for the authorized
  doc target as transient, so a temp renamed before its per-file watch no
  longer marks the observation incomplete (ship-docsync-completion flake).
  Per-file monitoring outside declared targets stays fail-closed.
This commit is contained in:
garrytan committed 2026-09-30 18:24:46 +00:00
1 parent 131d43be0a
commit 1643cd94de
5 files changed
+50 -12

No files matched your search

+14 -5
View File
@@ -12,8 +12,7 @@ const exact = (value: unknown, keys: string[]) => object(value) && Object.keys(v
class QaEvidenceError extends Error {}
const currentRevision = () => {
const result = spawnSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8', timeout: 5000, env: { ...process.env, GIT_OPTIONAL_LOCKS: '0' } });
if (result.status !== 0 || !/^[0-9a-f]{40,64}$/.test(result.stdout.trim())) throw new QaEvidenceError('Invalid report annotations: revision is required when git rev-parse HEAD is unavailable');
return result.stdout.trim();
return result.status === 0 && /^[0-9a-f]{40,64}$/.test(result.stdout.trim()) ? result.stdout.trim() : undefined;
};
function id(value: string): string {
@@ -234,10 +233,17 @@ function materialize(root: string, source: string) {
const supplied = JSON.parse(decode(bytes));
if (!object(supplied)) throw new QaEvidenceError('Invalid report annotations: need a JSON object');
const notes = checkpointNotes(root);
const measured: Record<string, string | undefined> = { revision: currentRevision(), runtime: `bun ${Bun.version}`, cwd: process.cwd() };
for (const [key, value] of Object.entries(measured)) {
if (value !== undefined && supplied[key] !== undefined && supplied[key] !== value) {
throw new QaEvidenceError(`Invalid report annotations: ${key} must be ${JSON.stringify(value)}; omit it and Q fills it`);
}
}
if (!measured.revision && supplied.revision === undefined) throw new QaEvidenceError('Invalid report annotations: revision is required when git rev-parse HEAD is unavailable');
const annotations: Record<string, any> = {
revision: supplied.revision ?? currentRevision(),
runtime: supplied.runtime ?? `bun ${Bun.version}`,
cwd: supplied.cwd ?? process.cwd(),
revision: measured.revision ?? supplied.revision,
runtime: measured.runtime,
cwd: measured.cwd,
limits: supplied.limits,
evidence: supplied.evidence,
learning: supplied.learning ?? notes.filter(note => typeof note.observationCommand === 'string' && typeof note.nextCommand === 'string'
@@ -260,6 +266,9 @@ function materialize(root: string, source: string) {
if (typeof name !== 'string') throw new QaEvidenceError('Invalid checkpoint reference');
const note = JSON.parse(decode(read(root, `exploration-${id(name)}.json`)));
if (!exact(note, ['observationCommand', 'observed', 'hypothesis', 'nextCommand'])) throw new QaEvidenceError('Invalid referenced checkpoint');
if (nativeCommand(note.observationCommand) === nativeCommand(note.nextCommand)) {
throw new QaEvidenceError(`Invalid learning: checkpoint ${name} replays the same probe; name checkpoints whose next probe differs, or omit learning and Q selects them`);
}
return { observationCommand: note.observationCommand, hypothesis: note.hypothesis, nextCommand: note.nextCommand };
});
const sha256 = publish(root, 'evidence.json', { ...annotations, evidence, learning });
+1 -1
View File
@@ -8,7 +8,7 @@ import { DOC_PATH, type DocsScenario, type fixtureDocs } from './docsync-fixture
import { sliceBetween } from './skill-fixture';
export async function observeDocsWrites(fixture: ReturnType<typeof fixtureDocs>) {
return observeQAWrites(fixture.repo);
return observeQAWrites(fixture.repo, { atomicTargets: [DOC_PATH] });
}
type DocsWriteContext = {
+3 -2
View File
@@ -72,7 +72,7 @@ export interface QAWriteObservation {
limits: string[];
}
export async function observeQAWrites(root: string, options: { reportDirectory?: string; evidenceProducer?: boolean } = {}) {
export async function observeQAWrites(root: string, options: { reportDirectory?: string; evidenceProducer?: boolean; atomicTargets?: string[] } = {}) {
if (process.platform !== 'linux') throw new Error('QA write observer unavailable: Linux inotify required');
if (fs.realpathSync(root) !== root) throw new Error('Observer root must be canonical');
let reportDirectory: string | undefined;
@@ -82,7 +82,8 @@ export async function observeQAWrites(root: string, options: { reportDirectory?:
reportDirectory = path.relative(root, directory);
}
const transientFile = (relative: string) => qaWriteAllowed(relative, 'qa-only')
|| (reportDirectory !== undefined && relative.startsWith(reportDirectory + path.sep));
|| (reportDirectory !== undefined && relative.startsWith(reportDirectory + path.sep))
|| (options.atomicTargets ?? []).some(target => relative.startsWith(target + '.tmp.') && /^\.tmp\.[1-9]\d*\.[0-9a-f]{12}$/.test(relative.slice(target.length)));
const before = qaTreeSnapshot(root);
const { dlopen, FFIType, ptr } = await import('bun:ffi');
const libc = dlopen('libc.so.6', {
+10 -2
View File
@@ -14,11 +14,11 @@ function fixture() {
return root;
}
async function atomicPublication(directory: string, declared?: string) {
async function atomicPublication(directory: string, declared?: string, atomicTargets?: string[]) {
const root = fixture();
const temporary = path.join(root, directory, 'exploration-004.json.tmp.2644.340bb6ad0afe');
const target = path.join(root, directory, 'exploration-004.json');
const observer = await observeQAWrites(root, { reportDirectory: declared });
const observer = await observeQAWrites(root, { reportDirectory: declared, atomicTargets });
const stat = fs.lstatSync;
let renamedAtWatch = false;
let stopped = false;
@@ -75,6 +75,14 @@ async function atomicPublication(directory: string, declared?: string) {
});
}
test('an authorized atomic target outside the report directory publishes without the per-file watch race', async () => {
const result = await atomicPublication('foreign/reports', undefined, ['foreign/reports/exploration-004.json']);
expect(result.renamedAtWatch).toBe(false);
expect(result.observation.failures).toEqual([]);
expect(result.observation.complete).toBe(true);
expect(result.observation.events).toContainEqual(expect.objectContaining({ path: result.target, mask: 0x80 }));
});
test('supports an explicitly selected nested report directory, not an implicit reports name', async () => {
const result = await atomicPublication('foreign/reports', 'foreign/reports');
expect(result.observation.complete).toBe(true);
+22 -2
View File
@@ -49,11 +49,11 @@ test('native capture executes once, preserves exact JSON and stderr, and materia
expect(JSON.parse(fs.readFileSync(path.join(f.root, 'exploration-001.json'), 'utf8'))).toEqual({
observationCommand: 'first native command', observed, hypothesis: 'The successful boundary suggests testing the rejected input next.', nextCommand: 'second native command',
});
f.json('annotations.json', { revision: 'revision', runtime: 'runtime', cwd: f.root, evidence: [], learning: [] });
f.json('annotations.json', { revision: 'revision', evidence: [], learning: [] });
const rejected = f.run('materialize', f.root, 'annotations.json');
expect(rejected.status).toBe(2);
expect(receipt(rejected.stderr).message).toContain('need limits (non-empty string array)');
f.json('annotations.json', { revision: 'revision', runtime: 'runtime', cwd: f.root, limits: ['Only the declared contract was checked.'], evidence: [{ capture: '001', command: 'first native command', contract: 'README.md', expected: 'Declared exact result', classification: 'pass' }], learning: ['001'] });
f.json('annotations.json', { revision: 'revision', limits: ['Only the declared contract was checked.'], evidence: [{ capture: '001', command: 'first native command', contract: 'README.md', expected: 'Declared exact result', classification: 'pass' }], learning: ['001'] });
const report = f.run('materialize', f.root, 'annotations.json');
expect(report.status, report.stderr).toBe(0);
const final = JSON.parse(fs.readFileSync(path.join(f.root, 'evidence.json'), 'utf8'));
@@ -274,3 +274,23 @@ test('a later capture requires a checkpoint anchored on the latest complete capt
expect(final).toMatchObject({ runtime: `bun ${Bun.version}`, cwd: f.root });
expect(final.learning).toEqual([{ observationCommand: first, hypothesis: 'The first observation makes the second input the riskiest next probe.', nextCommand: second('002') }]);
});
test('materialize rejects placeholder metadata and same-probe learning with the fix in the message', () => {
const f = fixture();
expect(f.capture('001', 'console.log(JSON.stringify({ step: 1 }))').status).toBe(0);
const command = (id: string) => `bun gstack-qa-evidence capture ${f.root} ${id} --timeout-ms 4000 -- probe same`;
expect(f.run('checkpoint', f.root, '001', '001', command('001'), 'Replaying the identical probe checks whether the first result is deterministic.', command('002')).status).toBe(0);
const row = { capture: '001', command: command('001'), contract: 'README.md', expected: 'step 1', classification: 'pass' };
f.json('annotations.json', { revision: 'fixture-revision', runtime: 'bun', limits: ['One probe.'], evidence: [row] });
const placeholder = f.run('materialize', f.root, 'annotations.json');
expect(placeholder.status).toBe(2);
expect(receipt(placeholder.stderr).message).toContain(`runtime must be "bun ${Bun.version}"`);
f.json('annotations.json', { revision: 'fixture-revision', limits: ['One probe.'], evidence: [row], learning: ['001'] });
const replay = f.run('materialize', f.root, 'annotations.json');
expect(replay.status).toBe(2);
expect(receipt(replay.stderr).message).toContain('checkpoint 001 replays the same probe');
f.json('annotations.json', { revision: 'fixture-revision', limits: ['One probe.'], evidence: [row] });
const selected = f.run('materialize', f.root, 'annotations.json');
expect(selected.status, selected.stderr).toBe(0);
expect(JSON.parse(fs.readFileSync(path.join(f.root, 'evidence.json'), 'utf8')).learning).toEqual([]);
});