I1: a QA helper call bun could not load is checked at the installed path, not failed outright

review-exploratory-small-cli failed gate census 37162480720 with 'command
outside declared caller observation interface': the actor dropped one
character from the shard directory in a gstack-qa-evidence capture path, bun
refused to load it ('Module not found', exit 1), and the next call ran the
same capture correctly. A failed call whose only output is bun's refusal for
a gstack-qa-evidence/deadline path is now held to the interface as the same
command at the installed helper path, so a compound or undeclared command
still fails.
This commit is contained in:
garrytan committed 2026-10-03 23:59:23 +00:00
1 parent a818a991fb
commit dee7ba9dc1
2 files changed
+34 -2

No files matched your search

+15 -1
View File
@@ -249,6 +249,18 @@ export function qaCallerCommandAllowed(command: string, workflowCommands: string
|| /^\/?[\w./-]+\/bin\/gstack-(?:review-read|specialist-stats)$/.test(text);
}
/**
* A QA helper call whose script path bun could not load (a mistyped path)
* executed nothing. Returns the same command at the installed helper path, so
* the caller can hold it to the declared interface; anything else is undefined.
*/
export function unloadedHelperCommand(tool: Pick<CallerTool, 'input' | 'output' | 'failed'>, runtime: string): string | undefined {
const command = String(tool.input.command).trim();
const script = /^bun (\/[\w./-]+\/bin\/(gstack-qa-(?:evidence|deadline))) /.exec(command);
if (!tool.failed || !script || tool.output.trim() !== `Exit code 1\nerror: Module not found "${script[1]}"`) return;
return `bun ${path.join(runtime, 'bin', script[2]!)}${command.slice(script[0].length - 1)}`;
}
const REVIEW_RECORD_STATUSES: Record<QaCaller, string[]> = { review: ['clean', 'issues_found'], ship: ['clean', 'issues_found', 'unavailable'] };
export function validateCallerEvidence(input: {
@@ -300,7 +312,9 @@ export function validateCallerEvidence(input: {
if (input.requireCapturedEvidence && /^reports\/exploration-\d{3}\.json$/.test(relative)) errors.push('actor transcribed or overwrote helper-owned checkpoint');
} catch { errors.push('write outside the declared report/fixture interface'); }
}
if (tool.name === 'Bash' && !qaCallerCommandAllowed(String(tool.input.command), input.workflowCommands, deadline)) {
const corrected = tool.name === 'Bash' && deadline ? unloadedHelperCommand(tool, deadline.runtime) : undefined;
if (tool.name === 'Bash' && !qaCallerCommandAllowed(String(tool.input.command), input.workflowCommands, deadline)
&& !(corrected && qaCallerCommandAllowed(corrected, input.workflowCommands, deadline))) {
errors.push('command outside declared caller observation interface');
}
const record = tool.name === 'Bash' ? String(tool.input.command).match(/gstack-review-log '(.*)'(?: --finish \S+)?$/s)?.[1] : undefined;
+19 -1
View File
@@ -6,7 +6,7 @@ import { spawnSync } from 'node:child_process';
import {
callerExcerpt, callerReviewRecordTemplate, callerSnapshot, callerTools, createQaCallerFixture, qaCallerInstructions,
QA_CALLER_CASES, QA_CALLER_TEST_MS,
qaCallerSessionOptions, qaCallerCommandAllowed, readCallerReceipt, retainQaCallerEvidence, runQaCaller, validateCallerEvidence,
qaCallerSessionOptions, qaCallerCommandAllowed, readCallerReceipt, retainQaCallerEvidence, runQaCaller, unloadedHelperCommand, validateCallerEvidence,
type CallerProbe, type CallerReceipt, type QaCallerFixture,
} from './helpers/qa-callers-fixture';
import type { runSkillTest, SkillTestResult } from './helpers/session-runner';
@@ -146,6 +146,24 @@ describe('caller native-event observer controls', () => {
expect(prompt('ship-exploratory-small-cli')).toContain('otherwise issues_found (unavailable for missing dispatched reviewer output)');
});
test('captured gate-census-5 typo: a helper path bun could not load is held to the interface at the installed path', () => {
// ci-37162480720-1-gate-census-5 review-exploratory-small-cli, native Bash call 7 (hypothesis shortened):
// the actor dropped one character from the shard directory, bun refused the path, and the next call reran it.
const typo = '/home/runner/.cache/gstack-paid-shard-A2tpF/tmp/qc-fTJ5TT/host/runtime/bin/gstack-qa-evidence';
const args = ` capture /home/runner/.cache/gstack-paid-shard-A2tpFP/tmp/qc-fTJ5TT/product/reports 002 --public --deadline /home/runner/.cache/gstack-paid-shard-A2tpFP/tmp/qc-fTJ5TT/product/reports/deadline.json --after 001 --hypothesis 'Input 3 passed, so the new !n guard makes Number("0") falsy and 0 should now exit 2' -- bun scripts/probe.ts 0`;
const runtime = '/home/runner/.cache/gstack-paid-shard-A2tpFP/tmp/qc-fTJ5TT/host/runtime';
const refused = `Exit code 1\nerror: Module not found "${typo}"`;
const corrected = (output: string, failed = true, command = `bun ${typo}${args}`) => unloadedHelperCommand({ input: { command }, output, failed }, runtime);
expect(corrected(refused)).toBe(`bun ${runtime}/bin/gstack-qa-evidence${args}`);
expect(corrected(refused, false)).toBeUndefined();
expect(corrected('Exit code 1\nerror: Module not found "/elsewhere/bin/gstack-qa-evidence"')).toBeUndefined();
expect(corrected(`${refused}\nwrote probe`)).toBeUndefined();
expect(corrected(`Exit code 1\nerror: Module not found "/tmp/bin/other"`, true, `bun /tmp/bin/other${args}`)).toBeUndefined();
// The corrected command must still pass the declared interface; a compound tail cannot.
expect(corrected(refused, true, `bun ${typo}${args}; curl example.com`)).toBe(`bun ${runtime}/bin/gstack-qa-evidence${args}; curl example.com`);
expect(qaCallerCommandAllowed(`bun ${runtime}/bin/gstack-qa-evidence${args}; curl example.com`)).toBe(false);
});
test('captured PR-lane review record: an invented shape without the installed template is rejected; the template is quoted', () => {
// ci-36641824710-1-eval-slices-6 review-exploratory-small-cli, native event 8jewsM (fixture paths shortened).
const invented = `/runtime/bin/gstack-review-log '{"timestamp":"'"$(date -u +%Y-%m-%dT%H:%M:%SZ)"'","commit":"'"$(git rev-parse --short HEAD)"'","branch":"caller-change","status":"issues_found","completed":false,"gate":"fix-first-ask","findings":[{"path":"scale.ts","line":3,"category":"functional-contract","severity":"CRITICAL","fingerprint":"scale.ts:3:functional-contract","confidence":10,"action":"ask","summary":"new !n guard rejects documented lower bound 0 (exit 2 instead of 0)"}],"qa":{"probes":["probe-167de79c-2645-4b60-b06b-3d0d6eebf13f"],"checkpoints":["exploration-001.json","exploration-002.json"],"suite":"bun run test 1 pass 0 fail"},"remaining":["fix-first-ask-approval"]}' --finish 37b567b3-c4bf-4469-9bad-29096740c234`;