From 158a67119166acade1fe7490d8a583191a868157 Mon Sep 17 00:00:00 2001 From: garrytan Date: Thu, 1 Oct 2026 20:33:07 +0000 Subject: [PATCH] fix(qa-evidence): materialize rejects an unrecognized classification before publishing, so the one-shot verdict cannot be locked inconclusive by a descriptive label --- lib/qa-evidence.ts | 6 +++++- test/qa-evidence.test.ts | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/lib/qa-evidence.ts b/lib/qa-evidence.ts index 121d99aa9..2f84c8e0f 100644 --- a/lib/qa-evidence.ts +++ b/lib/qa-evidence.ts @@ -258,6 +258,9 @@ function checkpoint(root: string, checkpointId: string, source: string | Record< link: `[checkpoint ${checkpointId}](exploration-${checkpointId}.json)`, exitCode: 0 }; } +/** Labels the verdict reads; an unrecognized label is rejected before publication so it can be corrected. */ +const QA_CLASSIFICATIONS = ['pass', 'superseded', 'product-defect', 'fail', 'setup-blocked', 'blocked', 'inconclusive']; + function materialize(root: string, source: string) { const bytes = read(root, source); if (scan(decode(bytes)).findings.some(finding => finding.tier === 'HIGH')) throw new QaEvidenceError('Sensitive annotations cannot be published'); @@ -289,6 +292,7 @@ function materialize(root: string, source: string) { const evidence = annotations.evidence.map((row: any) => { if (!exact(row, ['capture', 'command', 'contract', 'expected', 'classification']) || !Object.values(row).every(value => typeof value === 'string' && value.trim()) || captures.has(row.capture)) throw new QaEvidenceError('Invalid evidence annotation: each row needs exactly capture, command, contract, expected and classification as non-empty strings, with a unique capture'); + if (!QA_CLASSIFICATIONS.includes(row.classification)) throw new QaEvidenceError(`Invalid evidence annotation: capture ${row.capture} classification must be one of ${QA_CLASSIFICATIONS.join(', ')}; put the reason in limits or Markdown, not the label`); captures.add(row.capture); const captured = readQaCapture(root, row.capture); argv.push(JSON.stringify(captured.receipt.argv)); @@ -337,7 +341,7 @@ function materialize(root: string, source: string) { next: `Include every reportLinks entry in the Markdown report, and report the overall status as ${verdict.status}${verdict.open.length ? ` (open: ${verdict.open.join('; ')})` : ''}; rerun what is open first if a pass is required.` }; } -const QA_EVIDENCE_USAGE = 'capture ROOT ID [--public] --deadline FILE|--timeout-ms MS [--after PREVIOUS_CAPTURE --hypothesis TEXT] -- COMMAND ARGS (--after publishes checkpoint ID linking PREVIOUS_CAPTURE to this probe; required after the first complete capture unless a checkpoint was published) | checkpoint ROOT ID CAPTURE OBSERVATION_COMMAND HYPOTHESIS NEXT_COMMAND | checkpoint ROOT ID INTENT_FILE | materialize ROOT ANNOTATIONS (annotations: {evidence: [{capture, command, contract, expected, classification}], limits: [..]}; revision, runtime, cwd and learning are filled in)'; +const QA_EVIDENCE_USAGE = 'capture ROOT ID [--public] --deadline FILE|--timeout-ms MS [--after PREVIOUS_CAPTURE --hypothesis TEXT] -- COMMAND ARGS (--after publishes checkpoint ID linking PREVIOUS_CAPTURE to this probe; required after the first complete capture unless a checkpoint was published) | checkpoint ROOT ID CAPTURE OBSERVATION_COMMAND HYPOTHESIS NEXT_COMMAND | checkpoint ROOT ID INTENT_FILE | materialize ROOT ANNOTATIONS (annotations: {evidence: [{capture, command, contract, expected, classification: pass|superseded|product-defect|fail|setup-blocked|blocked|inconclusive}], limits: [..]}; revision, runtime, cwd and learning are filled in)'; export async function qaEvidenceMain(args: string[]): Promise { return withQaReceiptOutput(false, 'qa-evidence-receipt', value => value.event === 'observation' diff --git a/test/qa-evidence.test.ts b/test/qa-evidence.test.ts index 4548dbfaf..cc8a22027 100644 --- a/test/qa-evidence.test.ts +++ b/test/qa-evidence.test.ts @@ -432,6 +432,22 @@ test('a superseded row closes only when the same native probe was rerun on the c expect(materialize(row('001', 'superseded'), row('002', 'pass'), row('003', 'pass'))).toEqual({ status: 'pass', open: [] }); }); +test('a descriptive classification is rejected before publication, so the corrected label can still materialize', () => { + // gate-census-6 of run 36920606897: "pass (current snapshot)" left the one-shot verdict inconclusive. + const f = fixture(); + expect(f.capture('001', 'console.log(JSON.stringify({ step: 1 }))').status).toBe(0); + const row = (classification: string) => ({ capture: '001', command: 'capture 001', contract: 'README.md', expected: 'declared', classification }); + f.json('annotations.json', { revision: 'fixture-revision', limits: ['One probe ran.'], evidence: [row('pass (current snapshot)')] }); + const rejected = f.run('materialize', f.root, 'annotations.json'); + expect(rejected.status).toBe(2); + expect(receipt(rejected.stderr).message).toContain('classification must be one of pass, superseded'); + expect(fs.existsSync(path.join(f.root, 'evidence.json'))).toBe(false); + f.json('annotations.json', { revision: 'fixture-revision', limits: ['One probe ran.'], evidence: [row('pass')] }); + const accepted = f.run('materialize', f.root, 'annotations.json'); + expect(accepted.status, accepted.stderr).toBe(0); + expect(receipt(accepted.stdout).verdict).toEqual({ status: 'pass', open: [] }); +}); + test('captures list declared-but-unrun required probes without judging them', () => { const f = fixture(); const required = [`${process.execPath} -e console.log(JSON.stringify({step:1}))`, 'bun run probe -- reject'];