diff --git a/lib/qa-evidence.ts b/lib/qa-evidence.ts index 2f84c8e0f..efabc5c83 100644 --- a/lib/qa-evidence.ts +++ b/lib/qa-evidence.ts @@ -147,6 +147,26 @@ function requiredRemaining(root: string): { requiredRemaining?: string[] } { return { requiredRemaining: required.filter(command => !run.has(command)) }; } +const snapshotOf = (observed: unknown) => object(observed) && typeof observed.snapshot === 'string' ? observed.snapshot : undefined; + +/** + * Commands whose complete captures declared an older input snapshot than the + * latest capture and were not rerun on the current snapshot. Materialize keeps + * each one open (the verdict cannot pass), and it publishes only once, so + * every capture names them while they can still be rerun. + */ +function staleCommands(root: string): string[][] { + const observed = completeReceipts(root).flatMap(receipt => { + try { return [{ argv: receipt.argv as string[], snapshot: snapshotOf(readQaCapture(root, receipt.id).observed) }]; } catch { return []; } + }); + const current = observed.at(-1)?.snapshot; + if (current === undefined) return []; + const rerun = new Set(observed.filter(row => row.snapshot === current).map(row => JSON.stringify(row.argv))); + const stale = new Map(observed.filter(row => row.snapshot !== undefined && !rerun.has(JSON.stringify(row.argv))) + .map(row => [JSON.stringify(row.argv), row.argv])); + return [...stale.values()]; +} + async function capture(root: string, captureId: string, publicOutput: boolean, option: string, budget: string, command: string, args: string[], after?: { capture: string; hypothesis: string }) { id(captureId); if (!command || !['--deadline', '--timeout-ms'].includes(option)) throw new QaEvidenceError('Capture requires a deadline or finite command timeout'); @@ -235,10 +255,13 @@ async function capture(root: string, captureId: string, publicOutput: boolean, o completedAt, exitCode, signal: result.signal, status, observation, publicOutput, ...streams }; const sha256 = publish(root, `.qa-evidence/${captureId}/receipt.json`, receipt); + const revalidate = status === 'complete' ? staleCommands(root) : []; return { action: 'capture', id: captureId, status, sha256, exitCode, signal: result.signal, publicOutput, startedAt, completedAt, durationMs: Date.parse(completedAt) - Date.parse(startedAt), ...(remainingMs === undefined ? {} : { remainingMs }), ...(checkpointSha256 ? { checkpoint: captureId, checkpointSha256, link: `[checkpoint ${captureId}](exploration-${captureId}.json)` } : {}), - ...(status === 'complete' ? { next: `Another probe requires a checkpoint anchored on capture ${captureId}: add --after ${captureId} --hypothesis 'TEXT' before --. To stop exploring, run none.` } : {}), + ...(status === 'complete' ? { next: `Another probe requires a checkpoint anchored on capture ${captureId}: add --after ${captureId} --hypothesis 'TEXT' before --. To stop exploring, run none.${revalidate.length + ? ` Inputs changed since these commands ran; materialize runs once and keeps each one open (the verdict cannot pass) until it is rerun on current inputs: ${revalidate.map(argv => argv.join(' ')).join('; ')}.` : ''}` } : {}), + ...(revalidate.length ? { revalidate: revalidate.map(argv => argv.join(' ')) } : {}), ...requiredRemaining(root) }; } @@ -298,7 +321,6 @@ function materialize(root: string, source: string) { argv.push(JSON.stringify(captured.receipt.argv)); return { command: row.command, contract: row.contract, expected: row.expected, classification: row.classification, observed: captured.observed }; }); - const snapshotOf = (observed: unknown) => object(observed) && typeof observed.snapshot === 'string' ? observed.snapshot : undefined; const latestCapture = latestCompleteCapture(root); const currentSnapshot = latestCapture ? snapshotOf(readQaCapture(root, latestCapture).observed) : undefined; const superseded = currentSnapshot === undefined ? [] : annotations.evidence.filter((row: any, index: number) => { @@ -338,7 +360,7 @@ function materialize(root: string, source: string) { const sha256 = publish(root, 'evidence.json', { ...annotations, evidence, learning, verdict }); return { action: 'materialize', status: 'complete', sha256, annotationsSha256: hash(bytes), exitCode: 0, verdict, reportLinks: notes.map(note => `[checkpoint ${note.name.slice(12, 15)}](${note.name})`), - next: `Include every reportLinks entry in the Markdown report, and report the overall status as ${verdict.status}${verdict.open.length ? ` (open: ${verdict.open.join('; ')})` : ''}; rerun what is open first if a pass is required.` }; + next: `Include every reportLinks entry in the Markdown report, and report the overall status as ${verdict.status}${verdict.open.length ? ` (open: ${verdict.open.join('; ')})` : ''}; this verdict is final for this report root.` }; } const QA_EVIDENCE_USAGE = 'capture ROOT ID [--public] --deadline FILE|--timeout-ms MS [--after PREVIOUS_CAPTURE --hypothesis TEXT] -- COMMAND ARGS (--after publishes checkpoint ID linking PREVIOUS_CAPTURE to this probe; required after the first complete capture unless a checkpoint was published) | checkpoint ROOT ID CAPTURE OBSERVATION_COMMAND HYPOTHESIS NEXT_COMMAND | checkpoint ROOT ID INTENT_FILE | materialize ROOT ANNOTATIONS (annotations: {evidence: [{capture, command, contract, expected, classification: pass|superseded|product-defect|fail|setup-blocked|blocked|inconclusive}], limits: [..]}; revision, runtime, cwd and learning are filled in)'; diff --git a/test/qa-evidence.test.ts b/test/qa-evidence.test.ts index 11cb8204b..77299ad42 100644 --- a/test/qa-evidence.test.ts +++ b/test/qa-evidence.test.ts @@ -436,6 +436,34 @@ test('a superseded row closes only when the same native probe was rerun on the c expect(materialize(row('001', 'superseded'), row('002', 'pass'), row('003', 'pass'))).toEqual({ status: 'pass', open: [] }); }); +test('every capture after an input change names the commands still to rerun, until each has run on current inputs', () => { + // gate-census-6 of run 37158847998: the fixture locale changed after captures 001 and 002; the agent reran + // only input 9, materialized once with capture 001 still open, and the published verdict stayed inconclusive. + const f = fixture(); + const hypothesis = 'The input snapshot changed, so the earlier probes must be rerun on current inputs next.'; + const program = (input: string) => `console.log(JSON.stringify({ snapshot: require('node:fs').readFileSync('snap', 'utf8'), input: '${input}' }))`; + const probe = (id: string, after: string, input: string) => receipt(f.run('capture', f.root, id, '--timeout-ms', '4000', '--after', after, '--hypothesis', hypothesis, '--', process.execPath, '-e', program(input)).stdout); + const command = (input: string) => [process.execPath, '-e', program(input)].join(' '); + fs.writeFileSync(path.join(f.root, 'snap'), 'C'); + expect(receipt(f.capture('001', program('3')).stdout).revalidate).toBeUndefined(); + expect(probe('002', '001', '9').revalidate).toBeUndefined(); + fs.writeFileSync(path.join(f.root, 'snap'), 'POSIX'); + const changed = probe('003', '002', '10'); + expect(changed.revalidate).toEqual([command('3'), command('9')]); + expect(changed.next).toContain('materialize runs once and keeps each one open (the verdict cannot pass) until it is rerun on current inputs'); + expect(probe('004', '003', '9').revalidate).toEqual([command('3')]); + const current = probe('005', '004', '3'); + expect(current.revalidate).toBeUndefined(); + expect(current.next).not.toContain('Inputs changed'); + const row = (capture: string, classification: string) => ({ capture, command: `capture ${capture}`, contract: 'README.md', expected: 'declared', classification }); + f.json('annotations.json', { revision: 'fixture-revision', limits: ['Locale changed after capture 002.'], + evidence: [row('001', 'superseded'), row('002', 'superseded'), row('003', 'pass'), row('004', 'pass'), row('005', 'pass')] }); + const materialized = f.run('materialize', f.root, 'annotations.json'); + expect(materialized.status, materialized.stderr).toBe(0); + expect(receipt(materialized.stdout).verdict).toEqual({ status: 'pass', open: [] }); + expect(receipt(materialized.stdout).next).toContain('this verdict is final for this report root'); +}); + test('a descriptive classification is rejected before publication, so the corrected label can still materialize', () => { // gate-census-6 of run 36920606897: "pass (current snapshot)" left the one-shot verdict inconclusive. const f = fixture();