diff --git a/lib/qa-evidence.ts b/lib/qa-evidence.ts index ab00504ed..80d8a2f9c 100644 --- a/lib/qa-evidence.ts +++ b/lib/qa-evidence.ts @@ -121,13 +121,15 @@ function checkpointNotes(root: string): Record[] { .map(name => ({ name, ...JSON.parse(decode(read(root, name))) })); } -function latestCompleteCapture(root: string): string | undefined { - if (!fs.existsSync(path.join(root, '.qa-evidence'))) return undefined; +function completeReceipts(root: string): Record[] { + if (!fs.existsSync(path.join(root, '.qa-evidence'))) return []; return fs.readdirSync(owned(root, '.qa-evidence')).filter(name => /^\d{3}$/.test(name) && fs.existsSync(path.join(root, '.qa-evidence', name, 'receipt.json'))) .map(name => JSON.parse(decode(read(root, `.qa-evidence/${name}/receipt.json`)))) .filter(receipt => receipt.status === 'complete') - .sort((a, b) => Date.parse(a.completedAt) - Date.parse(b.completedAt)).at(-1)?.id; + .sort((a, b) => Date.parse(a.completedAt) - Date.parse(b.completedAt)); } +const completeCaptures = (root: string): string[] => completeReceipts(root).map(receipt => receipt.id); +const latestCompleteCapture = (root: string): string | undefined => completeCaptures(root).at(-1); async function capture(root: string, captureId: string, publicOutput: boolean, option: string, budget: string, command: string, args: string[]) { id(captureId); @@ -262,6 +264,9 @@ function materialize(root: string, source: string) { const captured = readQaCapture(root, row.capture); return { command: row.command, contract: row.contract, expected: row.expected, classification: row.classification, observed: captured.observed }; }); + const missing = completeCaptures(root).filter(capture => !captures.has(capture) + && !annotations.limits.some((limit: string) => new RegExp(`\\b${capture}\\b`).test(limit))); + if (missing.length) throw new QaEvidenceError(`Invalid report annotations: add an evidence row for capture ${missing.join(', ')} (every complete capture needs one, or name it in limits with why it is withheld)`); const learning = annotations.learning.map((name: unknown) => { if (typeof name !== 'string') throw new QaEvidenceError('Invalid checkpoint reference'); const note = JSON.parse(decode(read(root, `exploration-${id(name)}.json`))); diff --git a/test/qa-evidence.test.ts b/test/qa-evidence.test.ts index f94de8ac4..b496383a1 100644 --- a/test/qa-evidence.test.ts +++ b/test/qa-evidence.test.ts @@ -269,7 +269,12 @@ test('a later capture requires a checkpoint anchored on the latest complete capt const allowed = f.capture('002', 'console.log(JSON.stringify({ step: 2 }))'); expect(allowed.status, allowed.stderr).toBe(0); expect(receipt(allowed.stdout).next).toContain('anchored on capture 002'); - f.json('annotations.json', { revision: 'fixture-revision', limits: ['Only two probes ran.'], evidence: [{ capture: '001', command: first, contract: 'README.md', expected: 'step 1', classification: 'pass' }] }); + const firstRow = { capture: '001', command: first, contract: 'README.md', expected: 'step 1', classification: 'pass' }; + f.json('annotations.json', { revision: 'fixture-revision', limits: ['Only two probes ran.'], evidence: [firstRow] }); + const omitted = f.run('materialize', f.root, 'annotations.json'); + expect(omitted.status).toBe(2); + expect(receipt(omitted.stderr).message).toContain('add an evidence row for capture 002'); + f.json('annotations.json', { revision: 'fixture-revision', limits: ['Only two probes ran.'], evidence: [firstRow, { capture: '002', command: second('002'), contract: 'README.md', expected: 'step 2', classification: 'pass' }] }); const report = f.run('materialize', f.root, 'annotations.json'); expect(report.status, report.stderr).toBe(0); expect(receipt(report.stdout).reportLinks).toEqual(['[checkpoint 001](exploration-001.json)']); diff --git a/test/skill-e2e-third-party-actions.test.ts b/test/skill-e2e-third-party-actions.test.ts index 799dc17f0..d9f15c9ea 100644 --- a/test/skill-e2e-third-party-actions.test.ts +++ b/test/skill-e2e-third-party-actions.test.ts @@ -303,7 +303,7 @@ describeIfSelected('third-party-actions consent gate', TPA_TESTS, () => { // passes; a rendered consent option offering a drive fails. expect(text).not.toMatch(/^\s*[A-D]\)[^\n]*(drive|browse|Aside)/im); expect(text).not.toMatch(/drive\s+account\.apple\.com/i); - expect(text).toMatch(/app-specific password/i); + expect(text).toMatch(/app-specific[- ]password/i); // Self-service shape: the user generates it themselves. expect(text).toMatch(/generate|any device|fastlane-credentials/i); } finally { cleanup(); }