diff --git a/lib/qa-evidence.ts b/lib/qa-evidence.ts index 9e92142eb..bfede3a66 100644 --- a/lib/qa-evidence.ts +++ b/lib/qa-evidence.ts @@ -131,6 +131,15 @@ function completeReceipts(root: string): Record[] { const completeCaptures = (root: string): string[] => completeReceipts(root).map(receipt => receipt.id); const latestCompleteCapture = (root: string): string | undefined => completeCaptures(root).at(-1); +/** Required native probes the caller declared (GSTACK_QA_REQUIRED_PROBES, a JSON array of child commands) that no complete capture has run yet. Informational only. */ +function requiredRemaining(root: string): { requiredRemaining?: string[] } { + let required: unknown; + try { required = JSON.parse(process.env.GSTACK_QA_REQUIRED_PROBES ?? 'null'); } catch { return {}; } + if (!Array.isArray(required) || !required.every(item => typeof item === 'string')) return {}; + const run = new Set(completeReceipts(root).map(receipt => Array.isArray(receipt.argv) ? receipt.argv.join(' ') : '')); + return { requiredRemaining: required.filter(command => !run.has(command)) }; +} + async function capture(root: string, captureId: string, publicOutput: boolean, option: string, budget: string, command: string, args: string[]) { id(captureId); if (!command || !['--deadline', '--timeout-ms'].includes(option)) throw new QaEvidenceError('Capture requires a deadline or finite command timeout'); @@ -210,7 +219,8 @@ async function capture(root: string, captureId: string, publicOutput: boolean, o const sha256 = publish(root, `.qa-evidence/${captureId}/receipt.json`, receipt); return { action: 'capture', id: captureId, status, sha256, exitCode, signal: result.signal, publicOutput, startedAt, completedAt, durationMs: Date.parse(completedAt) - Date.parse(startedAt), ...(remainingMs === undefined ? {} : { remainingMs }), - ...(status === 'complete' ? { next: `Another probe requires a checkpoint anchored on capture ${captureId} first; to stop exploring, publish none.` } : {}) }; + ...(status === 'complete' ? { next: `Another probe requires a checkpoint anchored on capture ${captureId} first; to stop exploring, publish none.` } : {}), + ...requiredRemaining(root) }; } function checkpoint(root: string, checkpointId: string, source: string | Record) { diff --git a/test/helpers/qa-functional-eval.ts b/test/helpers/qa-functional-eval.ts index 319d3b898..fbe2398c9 100644 --- a/test/helpers/qa-functional-eval.ts +++ b/test/helpers/qa-functional-eval.ts @@ -10,7 +10,7 @@ import { runRecordedOfficeHoursAttempt, OFFICE_HOURS_BUN_GRACE_MS } from './offi import { resolveEvalModel } from '../../lib/eval-model'; import { createQAFunctionalFixture, fixtureGit, ownedPath, qaFixtureActor, QA_TOOLS, type QAFamily, type QAMode } from './qa-functional-fixture'; import { observeQAWrites, type QAWriteObservation } from './qa-functional-observer'; -import { qaFunctionalVerdict, verifyQANativeRegression, preserveQAArtifact, qaCaptureArtifacts } from './qa-functional-evidence'; +import { QA_WEBHOOK_REQUIRED_SCENARIOS, qaFunctionalVerdict, verifyQANativeRegression, preserveQAArtifact, qaCaptureArtifacts } from './qa-functional-evidence'; import { QA_EVIDENCE_RUNTIME, qaEvidenceCommand, qaProducerReceipt, qaEvidenceHash } from './qa-evidence-producer'; import { nativeCalls } from './qa-checkpoint-evidence'; @@ -115,7 +115,8 @@ export async function runQAFunctionalCase(entry: { id: string; family: QAFamily; workingDirectory: fixture.root, maxTurns: 40, allowedTools: QA_TOOLS, tools: QA_TOOLS, timeout, completionReserveMs: timeout / 4, testName: entry.id, runId, signal, env: { CLAUDE_CONFIG_DIR: fixture.config, - GIT_OPTIONAL_LOCKS: '0', QA_STATE_ROOT: path.join(fixture.root, '.qa-state') }, + GIT_OPTIONAL_LOCKS: '0', QA_STATE_ROOT: path.join(fixture.root, '.qa-state'), + ...(entry.family === 'webhook' ? { GSTACK_QA_REQUIRED_PROBES: JSON.stringify(QA_WEBHOOK_REQUIRED_SCENARIOS[entry.mode].map(scenario => `bun run probe -- ${scenario}`)) } : {}) }, }); return result; }, diff --git a/test/helpers/qa-functional-evidence.ts b/test/helpers/qa-functional-evidence.ts index 4ecc095b3..a1dbaae4a 100644 --- a/test/helpers/qa-functional-evidence.ts +++ b/test/helpers/qa-functional-evidence.ts @@ -7,6 +7,12 @@ import { readQACheckpointFiles, validateQACheckpoints } from './qa-checkpoint-ev import { nativeCalls } from './qa-checkpoint-evidence'; import { qaNativeCapture } from './qa-evidence-producer'; +/** Webhook scenarios a run must observe, per mode; the verdict and the fixture's capture nudge share this list. */ +export const QA_WEBHOOK_REQUIRED_SCENARIOS: Record = { + 'qa-only': ['happy', 'reject', 'duplicate', 'partial', 'concurrent-ab', 'concurrent-ba', 'cancel', 'dependency'], + qa: ['happy', 'cancel', 'dependency'], +}; + const canonical = (value: any): string => JSON.stringify(value && typeof value === 'object' ? Array.isArray(value) ? value.map(item => JSON.parse(canonical(item))) : Object.fromEntries(Object.keys(value).sort().map(key => [key, JSON.parse(canonical(value[key]))])) : value) ?? 'null'; @@ -126,7 +132,7 @@ export function qaFunctionalVerdict(fixture: QAFunctionalFixture, mode: QAMode, const cancellations = probes.filter(probe => fixture.family === 'cli' ? nativeCommand(probe) === 'bun cancel.ts' : probe.observed.scenario === 'cancel'); if (!cancellations.length) failures.push('missing cancellation observation'); if (fixture.family === 'webhook') { - for (const scenario of mode === 'qa-only' ? ['happy', 'reject', 'duplicate', 'partial', 'concurrent-ab', 'concurrent-ba'] : ['happy']) { + for (const scenario of QA_WEBHOOK_REQUIRED_SCENARIOS[mode].filter(scenario => !['cancel', 'dependency'].includes(scenario))) { if (!probes.some(probe => probe.observed.scenario === scenario)) failures.push(`missing native ${scenario} probe`); } } else if (!probes.some(probe => probe.observed.args?.[0] === 'apply' && qaProbeClassification(probe.observed) === 'pass')) failures.push('missing adjacent valid CLI apply'); diff --git a/test/qa-evidence.test.ts b/test/qa-evidence.test.ts index 0c8f1f1a0..9fa58d986 100644 --- a/test/qa-evidence.test.ts +++ b/test/qa-evidence.test.ts @@ -333,3 +333,12 @@ test('materialize refuses evidence whose declared input snapshot predates the la f.json('annotations.json', { revision: 'fixture-revision', limits: ['Adverse coverage predates the input change.'], evidence: [{ ...rows[0], classification: 'superseded' }, rows[1]] }); expect(f.run('materialize', f.root, 'annotations.json').status).toBe(0); }); + +test('captures list declared-but-unrun required probes without judging them', () => { + const f = fixture(); + const required = [`${process.execPath} -e console.log(JSON.stringify({step:1}))`, 'bun run probe -- reject']; + const result = spawnSync(process.execPath, [CLI, 'capture', f.root, '001', '--timeout-ms', '4000', '--', process.execPath, '-e', 'console.log(JSON.stringify({step:1}))'], + { cwd: f.root, encoding: 'utf8', timeout: 10_000, env: { ...process.env, GSTACK_QA_REQUIRED_PROBES: JSON.stringify(required) } }); + expect(result.status, result.stderr).toBe(0); + expect(receipt(result.stdout).requiredRemaining).toEqual(['bun run probe -- reject']); +});