From ba50f756f80846ba31ac6f50a652c8b2ed09e54e Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 23:29:45 +0000 Subject: [PATCH] I1: QA evidence names the probes to rerun after an input change, before the one-shot materialize ship-exploratory-late-input returned inconclusive in gate census 37158847998: the fixture locale changed after captures 001 and 002, the agent reran only input 9, and materialize (which publishes once) recorded capture 001 as open. Its next line then said to rerun what is open, which the published verdict no longer allowed. Every capture now lists the commands that observed an older input snapshot and have no rerun on current inputs (revalidate, plus the next line), and materialize's next line says the verdict is final. --- lib/qa-evidence.ts | 28 +++++++++++++++++++++++++--- test/qa-evidence.test.ts | 28 ++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/lib/qa-evidence.ts b/lib/qa-evidence.ts index 2f84c8e0f..efabc5c83 100644 --- a/lib/qa-evidence.ts +++ b/lib/qa-evidence.ts @@ -147,6 +147,26 @@ function requiredRemaining(root: string): { requiredRemaining?: string[] } { return { requiredRemaining: required.filter(command => !run.has(command)) }; } +const snapshotOf = (observed: unknown) => object(observed) && typeof observed.snapshot === 'string' ? observed.snapshot : undefined; + +/** + * Commands whose complete captures declared an older input snapshot than the + * latest capture and were not rerun on the current snapshot. Materialize keeps + * each one open (the verdict cannot pass), and it publishes only once, so + * every capture names them while they can still be rerun. + */ +function staleCommands(root: string): string[][] { + const observed = completeReceipts(root).flatMap(receipt => { + try { return [{ argv: receipt.argv as string[], snapshot: snapshotOf(readQaCapture(root, receipt.id).observed) }]; } catch { return []; } + }); + const current = observed.at(-1)?.snapshot; + if (current === undefined) return []; + const rerun = new Set(observed.filter(row => row.snapshot === current).map(row => JSON.stringify(row.argv))); + const stale = new Map(observed.filter(row => row.snapshot !== undefined && !rerun.has(JSON.stringify(row.argv))) + .map(row => [JSON.stringify(row.argv), row.argv])); + return [...stale.values()]; +} + async function capture(root: string, captureId: string, publicOutput: boolean, option: string, budget: string, command: string, args: string[], after?: { capture: string; hypothesis: string }) { id(captureId); if (!command || !['--deadline', '--timeout-ms'].includes(option)) throw new QaEvidenceError('Capture requires a deadline or finite command timeout'); @@ -235,10 +255,13 @@ async function capture(root: string, captureId: string, publicOutput: boolean, o completedAt, exitCode, signal: result.signal, status, observation, publicOutput, ...streams }; const sha256 = publish(root, `.qa-evidence/${captureId}/receipt.json`, receipt); + const revalidate = status === 'complete' ? staleCommands(root) : []; return { action: 'capture', id: captureId, status, sha256, exitCode, signal: result.signal, publicOutput, startedAt, completedAt, durationMs: Date.parse(completedAt) - Date.parse(startedAt), ...(remainingMs === undefined ? {} : { remainingMs }), ...(checkpointSha256 ? { checkpoint: captureId, checkpointSha256, link: `[checkpoint ${captureId}](exploration-${captureId}.json)` } : {}), - ...(status === 'complete' ? { next: `Another probe requires a checkpoint anchored on capture ${captureId}: add --after ${captureId} --hypothesis 'TEXT' before --. To stop exploring, run none.` } : {}), + ...(status === 'complete' ? { next: `Another probe requires a checkpoint anchored on capture ${captureId}: add --after ${captureId} --hypothesis 'TEXT' before --. To stop exploring, run none.${revalidate.length + ? ` Inputs changed since these commands ran; materialize runs once and keeps each one open (the verdict cannot pass) until it is rerun on current inputs: ${revalidate.map(argv => argv.join(' ')).join('; ')}.` : ''}` } : {}), + ...(revalidate.length ? { revalidate: revalidate.map(argv => argv.join(' ')) } : {}), ...requiredRemaining(root) }; } @@ -298,7 +321,6 @@ function materialize(root: string, source: string) { argv.push(JSON.stringify(captured.receipt.argv)); return { command: row.command, contract: row.contract, expected: row.expected, classification: row.classification, observed: captured.observed }; }); - const snapshotOf = (observed: unknown) => object(observed) && typeof observed.snapshot === 'string' ? observed.snapshot : undefined; const latestCapture = latestCompleteCapture(root); const currentSnapshot = latestCapture ? snapshotOf(readQaCapture(root, latestCapture).observed) : undefined; const superseded = currentSnapshot === undefined ? [] : annotations.evidence.filter((row: any, index: number) => { @@ -338,7 +360,7 @@ function materialize(root: string, source: string) { const sha256 = publish(root, 'evidence.json', { ...annotations, evidence, learning, verdict }); return { action: 'materialize', status: 'complete', sha256, annotationsSha256: hash(bytes), exitCode: 0, verdict, reportLinks: notes.map(note => `[checkpoint ${note.name.slice(12, 15)}](${note.name})`), - next: `Include every reportLinks entry in the Markdown report, and report the overall status as ${verdict.status}${verdict.open.length ? ` (open: ${verdict.open.join('; ')})` : ''}; rerun what is open first if a pass is required.` }; + next: `Include every reportLinks entry in the Markdown report, and report the overall status as ${verdict.status}${verdict.open.length ? ` (open: ${verdict.open.join('; ')})` : ''}; this verdict is final for this report root.` }; } const QA_EVIDENCE_USAGE = 'capture ROOT ID [--public] --deadline FILE|--timeout-ms MS [--after PREVIOUS_CAPTURE --hypothesis TEXT] -- COMMAND ARGS (--after publishes checkpoint ID linking PREVIOUS_CAPTURE to this probe; required after the first complete capture unless a checkpoint was published) | checkpoint ROOT ID CAPTURE OBSERVATION_COMMAND HYPOTHESIS NEXT_COMMAND | checkpoint ROOT ID INTENT_FILE | materialize ROOT ANNOTATIONS (annotations: {evidence: [{capture, command, contract, expected, classification: pass|superseded|product-defect|fail|setup-blocked|blocked|inconclusive}], limits: [..]}; revision, runtime, cwd and learning are filled in)'; diff --git a/test/qa-evidence.test.ts b/test/qa-evidence.test.ts index 11cb8204b..77299ad42 100644 --- a/test/qa-evidence.test.ts +++ b/test/qa-evidence.test.ts @@ -436,6 +436,34 @@ test('a superseded row closes only when the same native probe was rerun on the c expect(materialize(row('001', 'superseded'), row('002', 'pass'), row('003', 'pass'))).toEqual({ status: 'pass', open: [] }); }); +test('every capture after an input change names the commands still to rerun, until each has run on current inputs', () => { + // gate-census-6 of run 37158847998: the fixture locale changed after captures 001 and 002; the agent reran + // only input 9, materialized once with capture 001 still open, and the published verdict stayed inconclusive. + const f = fixture(); + const hypothesis = 'The input snapshot changed, so the earlier probes must be rerun on current inputs next.'; + const program = (input: string) => `console.log(JSON.stringify({ snapshot: require('node:fs').readFileSync('snap', 'utf8'), input: '${input}' }))`; + const probe = (id: string, after: string, input: string) => receipt(f.run('capture', f.root, id, '--timeout-ms', '4000', '--after', after, '--hypothesis', hypothesis, '--', process.execPath, '-e', program(input)).stdout); + const command = (input: string) => [process.execPath, '-e', program(input)].join(' '); + fs.writeFileSync(path.join(f.root, 'snap'), 'C'); + expect(receipt(f.capture('001', program('3')).stdout).revalidate).toBeUndefined(); + expect(probe('002', '001', '9').revalidate).toBeUndefined(); + fs.writeFileSync(path.join(f.root, 'snap'), 'POSIX'); + const changed = probe('003', '002', '10'); + expect(changed.revalidate).toEqual([command('3'), command('9')]); + expect(changed.next).toContain('materialize runs once and keeps each one open (the verdict cannot pass) until it is rerun on current inputs'); + expect(probe('004', '003', '9').revalidate).toEqual([command('3')]); + const current = probe('005', '004', '3'); + expect(current.revalidate).toBeUndefined(); + expect(current.next).not.toContain('Inputs changed'); + const row = (capture: string, classification: string) => ({ capture, command: `capture ${capture}`, contract: 'README.md', expected: 'declared', classification }); + f.json('annotations.json', { revision: 'fixture-revision', limits: ['Locale changed after capture 002.'], + evidence: [row('001', 'superseded'), row('002', 'superseded'), row('003', 'pass'), row('004', 'pass'), row('005', 'pass')] }); + const materialized = f.run('materialize', f.root, 'annotations.json'); + expect(materialized.status, materialized.stderr).toBe(0); + expect(receipt(materialized.stdout).verdict).toEqual({ status: 'pass', open: [] }); + expect(receipt(materialized.stdout).next).toContain('this verdict is final for this report root'); +}); + test('a descriptive classification is rejected before publication, so the corrected label can still materialize', () => { // gate-census-6 of run 36920606897: "pass (current snapshot)" left the one-shot verdict inconclusive. const f = fixture();