mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-04 18:36:54 +02:00
feat(qa-evidence): captures list the caller's declared-but-unrun required probes
GSTACK_QA_REQUIRED_PROBES (a JSON array of native child commands) makes every capture print requiredRemaining; it never judges pass or fail. The functional eval passes the webhook list from QA_WEBHOOK_REQUIRED_SCENARIOS, which the verdict now reads too, so the nudge and the verdict share one source (agreed with #3002's owner). CI webhook-report kept stopping with scenarios unrun.
This commit is contained in:
1 parent
a615303213
commit
90203acda3
4 files changed
+30
-4
No files matched your search
@@ -7,6 +7,12 @@ import { readQACheckpointFiles, validateQACheckpoints } from './qa-checkpoint-ev
|
||||
import { nativeCalls } from './qa-checkpoint-evidence';
|
||||
import { qaNativeCapture } from './qa-evidence-producer';
|
||||
|
||||
/** Webhook scenarios a run must observe, per mode; the verdict and the fixture's capture nudge share this list. */
|
||||
export const QA_WEBHOOK_REQUIRED_SCENARIOS: Record<QAMode, string[]> = {
|
||||
'qa-only': ['happy', 'reject', 'duplicate', 'partial', 'concurrent-ab', 'concurrent-ba', 'cancel', 'dependency'],
|
||||
qa: ['happy', 'cancel', 'dependency'],
|
||||
};
|
||||
|
||||
const canonical = (value: any): string => JSON.stringify(value && typeof value === 'object'
|
||||
? Array.isArray(value) ? value.map(item => JSON.parse(canonical(item)))
|
||||
: Object.fromEntries(Object.keys(value).sort().map(key => [key, JSON.parse(canonical(value[key]))])) : value) ?? 'null';
|
||||
@@ -126,7 +132,7 @@ export function qaFunctionalVerdict(fixture: QAFunctionalFixture, mode: QAMode,
|
||||
const cancellations = probes.filter(probe => fixture.family === 'cli' ? nativeCommand(probe) === 'bun cancel.ts' : probe.observed.scenario === 'cancel');
|
||||
if (!cancellations.length) failures.push('missing cancellation observation');
|
||||
if (fixture.family === 'webhook') {
|
||||
for (const scenario of mode === 'qa-only' ? ['happy', 'reject', 'duplicate', 'partial', 'concurrent-ab', 'concurrent-ba'] : ['happy']) {
|
||||
for (const scenario of QA_WEBHOOK_REQUIRED_SCENARIOS[mode].filter(scenario => !['cancel', 'dependency'].includes(scenario))) {
|
||||
if (!probes.some(probe => probe.observed.scenario === scenario)) failures.push(`missing native ${scenario} probe`);
|
||||
}
|
||||
} else if (!probes.some(probe => probe.observed.args?.[0] === 'apply' && qaProbeClassification(probe.observed) === 'pass')) failures.push('missing adjacent valid CLI apply');
|
||||
|
||||
Reference in new issue
Block a user