test(qa callers): an accepted review-log record may cite checkpoints as finding evidence

This commit is contained in:
garrytan committed 2026-10-01 18:58:52 +00:00
1 parent 625debd7eb
commit afd57e4ca9
3 files changed
+12 -1

No files matched your search

+3
View File
@@ -416,6 +416,9 @@ export function validateCallerEvidence(input: {
probes: checkpointProbes, requiredProbes: checkpointProbes.slice(1),
additionalTargets: expiredTargets,
files: input.checkpointFiles, reportMarkdown: input.reportMarkdown,
// An installed review-log record cites checkpoints in its findings' evidence.
citesCheckpointsOnly: command => /^\S*\/gstack-review-log '/.test(command.trim())
&& qaCallerCommandAllowed(command, input.workflowCommands, deadline),
}));
const selected = input.receipt.probes.map(id => input.probes.find(probe => probe.id === id));
if (selected.some(probe => !probe)) errors.push('receipt references an unobserved probe');
+4 -1
View File
@@ -91,6 +91,8 @@ export function validateQACheckpoints(input: {
files: Record<string, string>;
reportMarkdown: string;
producer?: QaEvidenceContext;
/** Caller-authorized Bash that names checkpoints only as evidence citations, never as files to read or write. */
citesCheckpointsOnly?: (command: string) => boolean;
}): string[] {
const failures: string[] = [];
let disk: Record<string, string>;
@@ -181,7 +183,8 @@ export function validateQACheckpoints(input: {
attempted = path.join(input.reportRoot, name);
content = disk[name];
}
if (call.name === 'Bash' && producerCommand?.action !== 'checkpoint' && typeof call.input.command === 'string' && /exploration-\d+\.json/.test(call.input.command)) {
if (call.name === 'Bash' && producerCommand?.action !== 'checkpoint' && typeof call.input.command === 'string' && /exploration-\d+\.json/.test(call.input.command)
&& !input.citesCheckpointsOnly?.(call.input.command)) {
failures.push('Unsupported checkpoint Bash interaction');
}
if (typeof attempted !== 'string' || !path.basename(attempted).startsWith('exploration-')) continue;
+5
View File
@@ -130,6 +130,11 @@ describe('caller native-event observer controls', () => {
expect(errorsFor(record('blocked'))).toEqual(['review record status outside the review vocabulary: blocked']);
expect(errorsFor(record('issues_found'))).toEqual([]);
expect(errorsFor(record('unavailable'))).toEqual(['review record status outside the review vocabulary: unavailable']);
// ci-36907899270-1-eval-slices-6: an accepted record whose finding cites checkpoints as evidence.
const cited = record('issues_found').replace('"action":"ask-pending"}', '"action":"ask-pending","evidence":["reports/exploration-002.json","reports/exploration-003.json"]}');
expect(cited).not.toBe(record('issues_found'));
expect(errorsFor(cited)).toEqual([]);
expect(errorsFor(`bun ${cited}`)).toEqual(['command outside declared caller observation interface', 'QA checkpoint: Unsupported checkpoint Bash interaction']);
expect(errorsFor(record('unavailable'), 'ship').filter(error => error.includes('vocabulary'))).toEqual([]);
expect(errorsFor(generatedReviewRecord('/runtime/bin/gstack-review-log', 'native-token').replace('"status":"clean"', '"status":"blocked"'))).toEqual([]);
const prompt = (id: 'review-exploratory-small-cli' | 'ship-exploratory-small-cli') => {