fix(qa-evidence): materialize rejects an unrecognized classification before publishing, so the one-shot verdict cannot be locked inconclusive by a descriptive label

This commit is contained in:
garrytan committed 2026-10-01 20:33:07 +00:00
1 parent b4228cb1c4
commit 158a671191
2 files changed
+21 -1

No files matched your search

+5 -1
View File
@@ -258,6 +258,9 @@ function checkpoint(root: string, checkpointId: string, source: string | Record<
link: `[checkpoint ${checkpointId}](exploration-${checkpointId}.json)`, exitCode: 0 };
}
/** Labels the verdict reads; an unrecognized label is rejected before publication so it can be corrected. */
const QA_CLASSIFICATIONS = ['pass', 'superseded', 'product-defect', 'fail', 'setup-blocked', 'blocked', 'inconclusive'];
function materialize(root: string, source: string) {
const bytes = read(root, source);
if (scan(decode(bytes)).findings.some(finding => finding.tier === 'HIGH')) throw new QaEvidenceError('Sensitive annotations cannot be published');
@@ -289,6 +292,7 @@ function materialize(root: string, source: string) {
const evidence = annotations.evidence.map((row: any) => {
if (!exact(row, ['capture', 'command', 'contract', 'expected', 'classification'])
|| !Object.values(row).every(value => typeof value === 'string' && value.trim()) || captures.has(row.capture)) throw new QaEvidenceError('Invalid evidence annotation: each row needs exactly capture, command, contract, expected and classification as non-empty strings, with a unique capture');
if (!QA_CLASSIFICATIONS.includes(row.classification)) throw new QaEvidenceError(`Invalid evidence annotation: capture ${row.capture} classification must be one of ${QA_CLASSIFICATIONS.join(', ')}; put the reason in limits or Markdown, not the label`);
captures.add(row.capture);
const captured = readQaCapture(root, row.capture);
argv.push(JSON.stringify(captured.receipt.argv));
@@ -337,7 +341,7 @@ function materialize(root: string, source: string) {
next: `Include every reportLinks entry in the Markdown report, and report the overall status as ${verdict.status}${verdict.open.length ? ` (open: ${verdict.open.join('; ')})` : ''}; rerun what is open first if a pass is required.` };
}
const QA_EVIDENCE_USAGE = 'capture ROOT ID [--public] --deadline FILE|--timeout-ms MS [--after PREVIOUS_CAPTURE --hypothesis TEXT] -- COMMAND ARGS (--after publishes checkpoint ID linking PREVIOUS_CAPTURE to this probe; required after the first complete capture unless a checkpoint was published) | checkpoint ROOT ID CAPTURE OBSERVATION_COMMAND HYPOTHESIS NEXT_COMMAND | checkpoint ROOT ID INTENT_FILE | materialize ROOT ANNOTATIONS (annotations: {evidence: [{capture, command, contract, expected, classification}], limits: [..]}; revision, runtime, cwd and learning are filled in)';
const QA_EVIDENCE_USAGE = 'capture ROOT ID [--public] --deadline FILE|--timeout-ms MS [--after PREVIOUS_CAPTURE --hypothesis TEXT] -- COMMAND ARGS (--after publishes checkpoint ID linking PREVIOUS_CAPTURE to this probe; required after the first complete capture unless a checkpoint was published) | checkpoint ROOT ID CAPTURE OBSERVATION_COMMAND HYPOTHESIS NEXT_COMMAND | checkpoint ROOT ID INTENT_FILE | materialize ROOT ANNOTATIONS (annotations: {evidence: [{capture, command, contract, expected, classification: pass|superseded|product-defect|fail|setup-blocked|blocked|inconclusive}], limits: [..]}; revision, runtime, cwd and learning are filled in)';
export async function qaEvidenceMain(args: string[]): Promise<number> {
return withQaReceiptOutput(false, 'qa-evidence-receipt', value => value.event === 'observation'
+16
View File
@@ -432,6 +432,22 @@ test('a superseded row closes only when the same native probe was rerun on the c
expect(materialize(row('001', 'superseded'), row('002', 'pass'), row('003', 'pass'))).toEqual({ status: 'pass', open: [] });
});
test('a descriptive classification is rejected before publication, so the corrected label can still materialize', () => {
// gate-census-6 of run 36920606897: "pass (current snapshot)" left the one-shot verdict inconclusive.
const f = fixture();
expect(f.capture('001', 'console.log(JSON.stringify({ step: 1 }))').status).toBe(0);
const row = (classification: string) => ({ capture: '001', command: 'capture 001', contract: 'README.md', expected: 'declared', classification });
f.json('annotations.json', { revision: 'fixture-revision', limits: ['One probe ran.'], evidence: [row('pass (current snapshot)')] });
const rejected = f.run('materialize', f.root, 'annotations.json');
expect(rejected.status).toBe(2);
expect(receipt(rejected.stderr).message).toContain('classification must be one of pass, superseded');
expect(fs.existsSync(path.join(f.root, 'evidence.json'))).toBe(false);
f.json('annotations.json', { revision: 'fixture-revision', limits: ['One probe ran.'], evidence: [row('pass')] });
const accepted = f.run('materialize', f.root, 'annotations.json');
expect(accepted.status, accepted.stderr).toBe(0);
expect(receipt(accepted.stdout).verdict).toEqual({ status: 'pass', open: [] });
});
test('captures list declared-but-unrun required probes without judging them', () => {
const f = fixture();
const required = [`${process.execPath} -e console.log(JSON.stringify({step:1}))`, 'bun run probe -- reject'];