diff --git a/test/helpers/qa-callers-fixture.ts b/test/helpers/qa-callers-fixture.ts index eb4828ccc..c8391e519 100644 --- a/test/helpers/qa-callers-fixture.ts +++ b/test/helpers/qa-callers-fixture.ts @@ -249,6 +249,18 @@ export function qaCallerCommandAllowed(command: string, workflowCommands: string || /^\/?[\w./-]+\/bin\/gstack-(?:review-read|specialist-stats)$/.test(text); } +/** + * A QA helper call whose script path bun could not load (a mistyped path) + * executed nothing. Returns the same command at the installed helper path, so + * the caller can hold it to the declared interface; anything else is undefined. + */ +export function unloadedHelperCommand(tool: Pick, runtime: string): string | undefined { + const command = String(tool.input.command).trim(); + const script = /^bun (\/[\w./-]+\/bin\/(gstack-qa-(?:evidence|deadline))) /.exec(command); + if (!tool.failed || !script || tool.output.trim() !== `Exit code 1\nerror: Module not found "${script[1]}"`) return; + return `bun ${path.join(runtime, 'bin', script[2]!)}${command.slice(script[0].length - 1)}`; +} + const REVIEW_RECORD_STATUSES: Record = { review: ['clean', 'issues_found'], ship: ['clean', 'issues_found', 'unavailable'] }; export function validateCallerEvidence(input: { @@ -300,7 +312,9 @@ export function validateCallerEvidence(input: { if (input.requireCapturedEvidence && /^reports\/exploration-\d{3}\.json$/.test(relative)) errors.push('actor transcribed or overwrote helper-owned checkpoint'); } catch { errors.push('write outside the declared report/fixture interface'); } } - if (tool.name === 'Bash' && !qaCallerCommandAllowed(String(tool.input.command), input.workflowCommands, deadline)) { + const corrected = tool.name === 'Bash' && deadline ? unloadedHelperCommand(tool, deadline.runtime) : undefined; + if (tool.name === 'Bash' && !qaCallerCommandAllowed(String(tool.input.command), input.workflowCommands, deadline) + && !(corrected && qaCallerCommandAllowed(corrected, input.workflowCommands, deadline))) { errors.push('command outside declared caller observation interface'); } const record = tool.name === 'Bash' ? String(tool.input.command).match(/gstack-review-log '(.*)'(?: --finish \S+)?$/s)?.[1] : undefined; diff --git a/test/qa-exploratory-callers.test.ts b/test/qa-exploratory-callers.test.ts index b1ed75eb3..7471b23e8 100644 --- a/test/qa-exploratory-callers.test.ts +++ b/test/qa-exploratory-callers.test.ts @@ -6,7 +6,7 @@ import { spawnSync } from 'node:child_process'; import { callerExcerpt, callerReviewRecordTemplate, callerSnapshot, callerTools, createQaCallerFixture, qaCallerInstructions, QA_CALLER_CASES, QA_CALLER_TEST_MS, - qaCallerSessionOptions, qaCallerCommandAllowed, readCallerReceipt, retainQaCallerEvidence, runQaCaller, validateCallerEvidence, + qaCallerSessionOptions, qaCallerCommandAllowed, readCallerReceipt, retainQaCallerEvidence, runQaCaller, unloadedHelperCommand, validateCallerEvidence, type CallerProbe, type CallerReceipt, type QaCallerFixture, } from './helpers/qa-callers-fixture'; import type { runSkillTest, SkillTestResult } from './helpers/session-runner'; @@ -146,6 +146,24 @@ describe('caller native-event observer controls', () => { expect(prompt('ship-exploratory-small-cli')).toContain('otherwise issues_found (unavailable for missing dispatched reviewer output)'); }); + test('captured gate-census-5 typo: a helper path bun could not load is held to the interface at the installed path', () => { + // ci-37162480720-1-gate-census-5 review-exploratory-small-cli, native Bash call 7 (hypothesis shortened): + // the actor dropped one character from the shard directory, bun refused the path, and the next call reran it. + const typo = '/home/runner/.cache/gstack-paid-shard-A2tpF/tmp/qc-fTJ5TT/host/runtime/bin/gstack-qa-evidence'; + const args = ` capture /home/runner/.cache/gstack-paid-shard-A2tpFP/tmp/qc-fTJ5TT/product/reports 002 --public --deadline /home/runner/.cache/gstack-paid-shard-A2tpFP/tmp/qc-fTJ5TT/product/reports/deadline.json --after 001 --hypothesis 'Input 3 passed, so the new !n guard makes Number("0") falsy and 0 should now exit 2' -- bun scripts/probe.ts 0`; + const runtime = '/home/runner/.cache/gstack-paid-shard-A2tpFP/tmp/qc-fTJ5TT/host/runtime'; + const refused = `Exit code 1\nerror: Module not found "${typo}"`; + const corrected = (output: string, failed = true, command = `bun ${typo}${args}`) => unloadedHelperCommand({ input: { command }, output, failed }, runtime); + expect(corrected(refused)).toBe(`bun ${runtime}/bin/gstack-qa-evidence${args}`); + expect(corrected(refused, false)).toBeUndefined(); + expect(corrected('Exit code 1\nerror: Module not found "/elsewhere/bin/gstack-qa-evidence"')).toBeUndefined(); + expect(corrected(`${refused}\nwrote probe`)).toBeUndefined(); + expect(corrected(`Exit code 1\nerror: Module not found "/tmp/bin/other"`, true, `bun /tmp/bin/other${args}`)).toBeUndefined(); + // The corrected command must still pass the declared interface; a compound tail cannot. + expect(corrected(refused, true, `bun ${typo}${args}; curl example.com`)).toBe(`bun ${runtime}/bin/gstack-qa-evidence${args}; curl example.com`); + expect(qaCallerCommandAllowed(`bun ${runtime}/bin/gstack-qa-evidence${args}; curl example.com`)).toBe(false); + }); + test('captured PR-lane review record: an invented shape without the installed template is rejected; the template is quoted', () => { // ci-36641824710-1-eval-slices-6 review-exploratory-small-cli, native event 8jewsM (fixture paths shortened). const invented = `/runtime/bin/gstack-review-log '{"timestamp":"'"$(date -u +%Y-%m-%dT%H:%M:%SZ)"'","commit":"'"$(git rev-parse --short HEAD)"'","branch":"caller-change","status":"issues_found","completed":false,"gate":"fix-first-ask","findings":[{"path":"scale.ts","line":3,"category":"functional-contract","severity":"CRITICAL","fingerprint":"scale.ts:3:functional-contract","confidence":10,"action":"ask","summary":"new !n guard rejects documented lower bound 0 (exit 2 instead of 0)"}],"qa":{"probes":["probe-167de79c-2645-4b60-b06b-3d0d6eebf13f"],"checkpoints":["exploration-001.json","exploration-002.json"],"suite":"bun run test 1 pass 0 fail"},"remaining":["fix-first-ask-approval"]}' --finish 37b567b3-c4bf-4469-9bad-29096740c234`;