feat(qa-evidence): captures list the caller's declared-but-unrun required probes

GSTACK_QA_REQUIRED_PROBES (a JSON array of native child commands) makes every
capture print requiredRemaining; it never judges pass or fail. The functional
eval passes the webhook list from QA_WEBHOOK_REQUIRED_SCENARIOS, which the
verdict now reads too, so the nudge and the verdict share one source (agreed
with #3002's owner). CI webhook-report kept stopping with scenarios unrun.
This commit is contained in:
garrytan committed 2026-09-30 20:42:29 +00:00
1 parent a615303213
commit 90203acda3
4 files changed
+30 -4

No files matched your search

+11 -1
View File
@@ -131,6 +131,15 @@ function completeReceipts(root: string): Record<string, any>[] {
const completeCaptures = (root: string): string[] => completeReceipts(root).map(receipt => receipt.id);
const latestCompleteCapture = (root: string): string | undefined => completeCaptures(root).at(-1);
/** Required native probes the caller declared (GSTACK_QA_REQUIRED_PROBES, a JSON array of child commands) that no complete capture has run yet. Informational only. */
function requiredRemaining(root: string): { requiredRemaining?: string[] } {
let required: unknown;
try { required = JSON.parse(process.env.GSTACK_QA_REQUIRED_PROBES ?? 'null'); } catch { return {}; }
if (!Array.isArray(required) || !required.every(item => typeof item === 'string')) return {};
const run = new Set(completeReceipts(root).map(receipt => Array.isArray(receipt.argv) ? receipt.argv.join(' ') : ''));
return { requiredRemaining: required.filter(command => !run.has(command)) };
}
async function capture(root: string, captureId: string, publicOutput: boolean, option: string, budget: string, command: string, args: string[]) {
id(captureId);
if (!command || !['--deadline', '--timeout-ms'].includes(option)) throw new QaEvidenceError('Capture requires a deadline or finite command timeout');
@@ -210,7 +219,8 @@ async function capture(root: string, captureId: string, publicOutput: boolean, o
const sha256 = publish(root, `.qa-evidence/${captureId}/receipt.json`, receipt);
return { action: 'capture', id: captureId, status, sha256, exitCode, signal: result.signal, publicOutput,
startedAt, completedAt, durationMs: Date.parse(completedAt) - Date.parse(startedAt), ...(remainingMs === undefined ? {} : { remainingMs }),
...(status === 'complete' ? { next: `Another probe requires a checkpoint anchored on capture ${captureId} first; to stop exploring, publish none.` } : {}) };
...(status === 'complete' ? { next: `Another probe requires a checkpoint anchored on capture ${captureId} first; to stop exploring, publish none.` } : {}),
...requiredRemaining(root) };
}
function checkpoint(root: string, checkpointId: string, source: string | Record<string, string>) {
+3 -2
View File
@@ -10,7 +10,7 @@ import { runRecordedOfficeHoursAttempt, OFFICE_HOURS_BUN_GRACE_MS } from './offi
import { resolveEvalModel } from '../../lib/eval-model';
import { createQAFunctionalFixture, fixtureGit, ownedPath, qaFixtureActor, QA_TOOLS, type QAFamily, type QAMode } from './qa-functional-fixture';
import { observeQAWrites, type QAWriteObservation } from './qa-functional-observer';
import { qaFunctionalVerdict, verifyQANativeRegression, preserveQAArtifact, qaCaptureArtifacts } from './qa-functional-evidence';
import { QA_WEBHOOK_REQUIRED_SCENARIOS, qaFunctionalVerdict, verifyQANativeRegression, preserveQAArtifact, qaCaptureArtifacts } from './qa-functional-evidence';
import { QA_EVIDENCE_RUNTIME, qaEvidenceCommand, qaProducerReceipt, qaEvidenceHash } from './qa-evidence-producer';
import { nativeCalls } from './qa-checkpoint-evidence';
@@ -115,7 +115,8 @@ export async function runQAFunctionalCase(entry: { id: string; family: QAFamily;
workingDirectory: fixture.root, maxTurns: 40, allowedTools: QA_TOOLS, tools: QA_TOOLS,
timeout, completionReserveMs: timeout / 4,
testName: entry.id, runId, signal, env: { CLAUDE_CONFIG_DIR: fixture.config,
GIT_OPTIONAL_LOCKS: '0', QA_STATE_ROOT: path.join(fixture.root, '.qa-state') },
GIT_OPTIONAL_LOCKS: '0', QA_STATE_ROOT: path.join(fixture.root, '.qa-state'),
...(entry.family === 'webhook' ? { GSTACK_QA_REQUIRED_PROBES: JSON.stringify(QA_WEBHOOK_REQUIRED_SCENARIOS[entry.mode].map(scenario => `bun run probe -- ${scenario}`)) } : {}) },
});
return result;
},
+7 -1
View File
@@ -7,6 +7,12 @@ import { readQACheckpointFiles, validateQACheckpoints } from './qa-checkpoint-ev
import { nativeCalls } from './qa-checkpoint-evidence';
import { qaNativeCapture } from './qa-evidence-producer';
/** Webhook scenarios a run must observe, per mode; the verdict and the fixture's capture nudge share this list. */
export const QA_WEBHOOK_REQUIRED_SCENARIOS: Record<QAMode, string[]> = {
'qa-only': ['happy', 'reject', 'duplicate', 'partial', 'concurrent-ab', 'concurrent-ba', 'cancel', 'dependency'],
qa: ['happy', 'cancel', 'dependency'],
};
const canonical = (value: any): string => JSON.stringify(value && typeof value === 'object'
? Array.isArray(value) ? value.map(item => JSON.parse(canonical(item)))
: Object.fromEntries(Object.keys(value).sort().map(key => [key, JSON.parse(canonical(value[key]))])) : value) ?? 'null';
@@ -126,7 +132,7 @@ export function qaFunctionalVerdict(fixture: QAFunctionalFixture, mode: QAMode,
const cancellations = probes.filter(probe => fixture.family === 'cli' ? nativeCommand(probe) === 'bun cancel.ts' : probe.observed.scenario === 'cancel');
if (!cancellations.length) failures.push('missing cancellation observation');
if (fixture.family === 'webhook') {
for (const scenario of mode === 'qa-only' ? ['happy', 'reject', 'duplicate', 'partial', 'concurrent-ab', 'concurrent-ba'] : ['happy']) {
for (const scenario of QA_WEBHOOK_REQUIRED_SCENARIOS[mode].filter(scenario => !['cancel', 'dependency'].includes(scenario))) {
if (!probes.some(probe => probe.observed.scenario === scenario)) failures.push(`missing native ${scenario} probe`);
}
} else if (!probes.some(probe => probe.observed.args?.[0] === 'apply' && qaProbeClassification(probe.observed) === 'pass')) failures.push('missing adjacent valid CLI apply');
+9
View File
@@ -333,3 +333,12 @@ test('materialize refuses evidence whose declared input snapshot predates the la
f.json('annotations.json', { revision: 'fixture-revision', limits: ['Adverse coverage predates the input change.'], evidence: [{ ...rows[0], classification: 'superseded' }, rows[1]] });
expect(f.run('materialize', f.root, 'annotations.json').status).toBe(0);
});
test('captures list declared-but-unrun required probes without judging them', () => {
const f = fixture();
const required = [`${process.execPath} -e console.log(JSON.stringify({step:1}))`, 'bun run probe -- reject'];
const result = spawnSync(process.execPath, [CLI, 'capture', f.root, '001', '--timeout-ms', '4000', '--', process.execPath, '-e', 'console.log(JSON.stringify({step:1}))'],
{ cwd: f.root, encoding: 'utf8', timeout: 10_000, env: { ...process.env, GSTACK_QA_REQUIRED_PROBES: JSON.stringify(required) } });
expect(result.status, result.stderr).toBe(0);
expect(receipt(result.stdout).requiredRemaining).toEqual(['bun run probe -- reject']);
});