From a09b3b85320ab405462fa38ed2270c153c7a4d94 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Mon, 19 Oct 2026 02:52:57 -0700 Subject: [PATCH] v1.91.145.0 fix(evals): qa-only opts in to the read-only grammar; stderr-to-null forms accepted (#3295) * fix(evals): qa-only opts in to the read-only grammar; stderr-to-null forms accepted Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(evals): qa-only checkpoint rule counts guarded "$B" stop as browser cleanup, like closetab PR #3272's Opus gate 39217387697 (slice 8) failed qa-only-no-fix when the agent stopped the browser inside the guard; the rule counted stop as a third probe that needed a checkpoint. The prompt now names both cleanup children. A free replay of the captured three guarded calls passes (and fails on the old rule); stop before a probe, a disguised probe, stop --all and stop outside the guard still fail. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test(evals): register the qa-only stop-cleanup replay fixture in qa-only-no-fix touchfiles Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(evals): docs-sync observer ignores an unexecuted candidate call that exited 127 at a missing path PR #3295's gate (run 39223031058, slice 2) failed ship-docsync-current after Opus called gstack-docs-candidate compare at a path missing the fixture-home segment, got exit 127 (no such file or directory), and reran it at the declared path. The observer now ignores such a call only when the absolute binary path does not exist, the recorded result is exactly the shell's exit-127 not-found message for that path, and a later call runs the same command at the declared path without failing. A free replay of all 43 captured tool calls passes and fails on the previous observer; negative controls cover an existing wrong-path binary, other exits and messages, a missing/earlier/failing/changed retry, and non-candidate or relative binaries. GStack Minder-approved. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * v1.91.145.0 chore(release): stamp qa-only read-only opt-in, stderr forms, stop cleanup and docs-sync exit-127 exception Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 11 + VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- package.json | 2 +- test/docsync-command-grammar.test.ts | 2 +- test/docsync-lifecycle-interface.test.ts | 85 ++++- .../docsync-candidate-not-found-public.json | 308 ++++++++++++++++ test/fixtures/docsync-observer-commands.json | 10 +- .../fixtures/qa-only-stop-cleanup-public.json | 340 ++++++++++++++++++ test/helpers/docsync-observer.ts | 23 +- test/helpers/qa-browser-deadline-evidence.ts | 20 +- test/helpers/read-only-commands.ts | 8 +- test/helpers/touchfiles-data.ts | 10 +- test/qa-browser-deadline-evidence.test.ts | 54 ++- test/read-only-command-grammar.test.ts | 91 ++++- 15 files changed, 940 insertions(+), 28 deletions(-) create mode 100644 test/fixtures/docsync-candidate-not-found-public.json create mode 100644 test/fixtures/qa-only-stop-cleanup-public.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 566320d96..ff628320e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## [1.91.145.0] - 2026-10-19 + +**The qa-only deadline fixture accepts pure local reads, so `qa-only-no-fix` no longer fails when the agent checks a skill file with `wc -l …; ls …`.** Opus 5.5's first Bash call in TM1's focused `qa-only-no-fix` run at e03bfeb was `wc -l qa-only/SKILL.md; ls qa-only qa-only/sections qa/sections qa/templates`. The fixture's narrow list rejected it with `QA deadline: unsupported outer shell composition`, though it only read inside the fixture. The fixture now opts into the shared read-only grammar from 1.91.143.0, as the docs-sync fixture did. A focused Opus run with the opt-in then failed on one more read, `ls; ls qa/templates; cat TODOS.md 2>/dev/null | head -20`, so the grammar now also lets a read discard or merge its stderr. PR #3272's Opus gate (run 39217387697) then failed the case after the agent stopped the browser inside the guard with `"$B" stop`, which the checkpoint rule counted as a third probe. This PR's own gate (run 39223031058) failed `ship-docsync-current` after Opus called the docs candidate helper at a path that does not exist, got exit 127, and reran the same command at the declared path. + +### Changed +- **`test/helpers/qa-browser-deadline-evidence.ts`:** the deadline check also accepts any command the shared grammar allows inside the fixture directory, and the fixture prompt declares the grammar instead of "no other ls, wc, cat or find Bash calls". Receipts are still trusted only from the guard: a grammar read whose output has a line starting with `QA_DEADLINE ` fails as `receipt outside trusted guard invocation`. A grammar read of skill prose that only mentions QA_DEADLINE receipts mid-line passes, just as the same file read with Read does; the second focused Opus run tripped on `cat qa-only/sections/exploratory.md`. The fixture's existing allowed commands keep the stricter any-occurrence check. +- **Gate integrity, widening:** pure reads inside the fixture root, such as listing `qa/sections` or `ls -laR` of the screenshots directory, now pass where they used to fail. A free replay runs the captured slip inside a real guarded attempt. Reads that leave the root (`…/../../..`, `/etc`), joins with a non-read tool, redirects, `| tee`, `awk` and `sed -i` still fail. No threshold, kind, trial or behavior assertion changes. The `qa-quick`, `qa-only-no-fix`, `qa-fix-loop` and `qa-bootstrap` touchfiles add `test/helpers/read-only-commands.ts`. +- **Gate integrity, widening (shared grammar, GStack Minder-approved):** a read may end with exactly `2>/dev/null` or `2>&1`. Both fixtures that use the grammar accept it: qa-only and docs-sync. Every other redirect still fails, each with its own negative control: `2>` to a file, `2> /dev/null` with a space, `>` and `>>` on stdout, `&>`, `1>/dev/null`, `3>/dev/null`, `>&` and `2>&` to a file, `2>&2`, `2>/dev/tcp/…`, a leading `2>/dev/null`, a quoted `'2'>`, and `| tee`. The main control `ls docs 2>/dev/null` became `ls docs 2>docs/err.txt`. Two existing docs-sync controls only failed because of their stderr redirect, and each got a replacement that still fails: `ls 2>&1 | head` became `ls 2>&1 | sh`. The stored miss `cat {other}/bash-placeholder 2>/dev/null; cat {home}/state/config.yaml` moved to known_good_posix, and its `2>` to a file and `1>/dev/null` variants stay known_bad. The docs-sync prompt carries the updated grammar text, so the next PR gate selects the docs-sync cases. +- **Gate integrity, widening (checkpoint rule):** a guarded child that is exactly `"$B" stop`, `$B stop` or ` stop` now counts as browser cleanup, the same as `closetab`. Cleanup needs no checkpoint and may come only after the last probe. The fixture prompt names both cleanup commands. A free replay of the three guarded calls captured in run 39217387697 (two probes with one checkpoint, then `stop`) passes, and fails on the previous rule with `follow-up lacks one acknowledged preceding checkpoint`. These still fail: `stop` before a probe (`browser cleanup preceded a probe`), a probe disguised as cleanup (`"$B" goto …; "$B" stop`), `stop --all`, and `stop` outside the guard (direct, or through a shell variable). The replay fixture `test/fixtures/qa-only-stop-cleanup-public.json` is registered in the `qa-only-no-fix` touchfiles, next to the existing observation fixture. +- **Gate integrity, widening (docs-sync observer, GStack Minder-approved):** only executed candidate calls count. The observer ignores a `gstack-docs-candidate` call outside the declared path only when all of these hold: the binary is an absolute literal path that does not exist, the recorded result is exactly `Exit code 127` with the shell's `no such file or directory: `, and a later call runs the same command at the declared path without failing. A free replay of all 43 tool calls captured in run 39223031058 passes, and fails on the previous observer. These still fail: a wrong-path binary that exists (whether it runs or claims 127), any other exit code or shell message, a 127 with no later declared-path run (or one that comes before, fails, or changes the arguments), and a non-candidate or relative binary. No threshold, kind, trial or behavior assertion changes. The replay fixture `test/fixtures/docsync-candidate-not-found-public.json` is registered in the `ship-docsync-current` touchfiles. + ## [1.91.144.0] - 2026-10-18 **The paid PR gate now tests Claude Opus 5.5 instead of Fable 5.1. Its first full Opus gate went 132/132 green and recorded $32.33, against Fable's $93.16 mean over ten full gates.** Garry's direction (2026-10-18) is to test each lab's best code model. Calibration run 39157502890 passed all 108 rule cases and 24 judges on one attempt each, with no reused receipts, on the same case set every Fable gate passed. Opus 5.5 sessions cost $30.86 (Fable: $90.49), judges $1.44 and Haiku side calls $0.08; with the unrecorded PTY sessions estimated, the full-gate scale is $37.13 against $106.17. Output tokens fell 37% (thinking 69%) and session time from 11,859 s to 6,159 s. The Claude CLI runs Opus 5.5 at medium effort by default and Fable at high. diff --git a/VERSION b/VERSION index 86cb1072d..3a1f6f11b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.144.0 +1.91.145.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 6456dbd20..597099b8f 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.144.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.145.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/package.json b/package.json index 59d46a149..002f7ece6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.144", + "version": "1.91.145", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/test/docsync-command-grammar.test.ts b/test/docsync-command-grammar.test.ts index 75c64dbb6..63a6d9942 100644 --- a/test/docsync-command-grammar.test.ts +++ b/test/docsync-command-grammar.test.ts @@ -122,7 +122,7 @@ test('harmless read wrappers do not fail a docs run', () => { 'ls 2>&1', 'git status 2>&1']) { expect(docsToolFailures(nativeResult(command, 'successful tool acknowledgment', 'docs-dispatch'), fixture, [], true)).toEqual([]); } - for (const command of ['git status 2>&1 > out.txt', 'ls 2>&1 | head', `bun ${path.join(fixture.home, 'publish.ts')} 2>&1`, 'ls 2>out.txt']) { + for (const command of ['git status 2>&1 > out.txt', 'ls 2>&1 | sh', `bun ${path.join(fixture.home, 'publish.ts')} 2>&1`, 'ls 2>out.txt']) { expect(docsCommandAllowed(command, fixture, [path.join(fixture.home, 'publish.ts')])).toBe(false); } }); diff --git a/test/docsync-lifecycle-interface.test.ts b/test/docsync-lifecycle-interface.test.ts index a8028988c..cf3dc1306 100644 --- a/test/docsync-lifecycle-interface.test.ts +++ b/test/docsync-lifecycle-interface.test.ts @@ -3,7 +3,7 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import { fixtureDocs } from './helpers/docsync-fixture'; import { docsCommandAllowed, docsNativeInterface, docsPreambleCommands, docsToolFailures } from './helpers/docsync-observer'; -import type { SkillTestResult } from './helpers/session-runner'; +import { parseNDJSON, type SkillTestResult } from './helpers/session-runner'; let fixture: ReturnType; beforeAll(() => { fixture = fixtureDocs('updated'); }); @@ -140,3 +140,86 @@ describe('stored docsync observer commands', () => { } }); }); + +describe.skipIf(process.platform === 'win32')('candidate call at a missing binary path (exit 127)', () => { + const captured = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/docsync-candidate-not-found-public.json'), 'utf8')) as + { calls: SkillTestResult['toolCalls'] }; + let current: ReturnType; + let publish: string; + let missing: string; + beforeAll(() => { + current = fixtureDocs('current'); + publish = path.join(current.home, 'fixture-publish.ts'); + missing = path.join(current.home, 'missing-shard'); + }); + afterAll(() => current?.clean()); + const fill = (text: string) => text.replaceAll('{shard}', missing).replaceAll('{home}', current.home); + const replay = (calls = captured.calls) => ({ toolCalls: calls.map(call => ({ tool: call.tool, output: fill(call.output), + input: Object.fromEntries(Object.entries(call.input).map(([key, value]) => [key, typeof value === 'string' ? fill(value) : value])) })) }) as SkillTestResult; + const wrong = () => captured.calls.findIndex(call => call.output.startsWith('Exit code 127')); + const retry = () => wrong() + 1; + const failures = (result: SkillTestResult) => docsToolFailures(result, current, [publish], true); + const pair = (wrongOutput: string, retryOutput = '{}', wrongCommand = captured.calls[wrong()].input.command) => replay([ + { tool: 'Bash', input: { command: wrongCommand }, output: wrongOutput }, + { tool: 'Bash', input: { command: captured.calls[retry()].input.command }, output: retryOutput }, + ]); + const notFound = (binary: string) => `Exit code 127\n(eval):1: no such file or directory: ${binary}`; + const wrongBinary = () => fill(captured.calls[wrong()].input.command.split(' ')[0]); + + test('captured run 39223031058 ship-docsync-current passes; its wrong-path call alone is outside the interface', () => { + const result = replay(); + expect(result.toolCalls[retry()].input.command).toBe(result.toolCalls[wrong()].input.command.replace(wrongBinary(), + `${current.skills}/bin/gstack-docs-candidate`)); + expect(docsCommandAllowed(result.toolCalls[wrong()].input.command, current, [publish])).toBe(false); + expect(failures(result)).toEqual([]); + }); + + test('the native exit-127 tool result reaches the observer as the recorded output', () => { + const command = fill(captured.calls[wrong()].input.command); + const parsed = parseNDJSON([ + { type: 'assistant', parent_tool_use_id: 'child', message: { content: [{ type: 'tool_use', id: 'wrong', name: 'Bash', input: { command } }] } }, + { type: 'user', parent_tool_use_id: 'child', message: { content: [{ type: 'tool_result', tool_use_id: 'wrong', content: notFound(wrongBinary()), is_error: true }] }, + tool_use_result: `Error: ${notFound(wrongBinary())}` }, + ].map(event => JSON.stringify(event))); + expect(parsed.toolCalls[0].output).toBe(notFound(wrongBinary())); + }); + + test('a 127 with no later run of the same command at the declared path still fails', () => { + const calls = captured.calls.filter((_, index) => index !== retry()); + expect(failures(replay(calls))).toContain('command outside declared docs observation interface'); + const before = [captured.calls[retry()], ...calls]; + expect(failures(replay(before))).toContain('command outside declared docs observation interface'); + expect(failures(pair(notFound(wrongBinary()), 'Exit code 1\nboom'))).toContain('command outside declared docs observation interface'); + const otherArgs = captured.calls[wrong()].input.command.replace('ship2-docs-a1.json', 'other.json'); + expect(failures(pair(notFound(wrongBinary()), '{}', otherArgs))).toContain('command outside declared docs observation interface'); + }); + + test('any other exit or shell message outside the declared path still fails', () => { + expect(failures(pair(notFound(wrongBinary())))).toEqual([]); + for (const output of [`Exit code 126\n(eval):1: permission denied: ${wrongBinary()}`, 'Exit code 1\nerror', '', + `Exit code 127\n(eval):1: command not found: ${wrongBinary()}`, notFound(wrongBinary()) + '\nextra', + notFound(path.join(missing, 'other/gstack-docs-candidate'))]) { + expect(failures(pair(output)), output).toContain('command outside declared docs observation interface'); + } + }); + + test('a wrong-path binary that exists still fails, whether it ran or claims 127', () => { + fs.mkdirSync(path.dirname(wrongBinary()), { recursive: true }); + fs.writeFileSync(wrongBinary(), 'candidate stand-in\n'); + try { + for (const output of ['{}', notFound(wrongBinary())]) { + expect(failures(pair(output)), output).toContain('command outside declared docs observation interface'); + } + } finally { fs.rmSync(missing, { recursive: true, force: true }); } + }); + + test('only a candidate binary at an absolute literal path qualifies', () => { + const declared = current.skills.split(path.sep).join('/'); + const skillEnd = `${missing}/.claude/skills/gstack/bin/gstack-skill-start --help`; + expect(failures(replay([{ tool: 'Bash', input: { command: skillEnd }, output: notFound(skillEnd.split(' ')[0]) }, + { tool: 'Bash', input: { command: `${declared}/bin/gstack-skill-start --help` }, output: 'usage' }]))).toContain('command outside declared docs observation interface'); + const relative = 'missing/gstack-docs-candidate --help'; + expect(failures(replay([{ tool: 'Bash', input: { command: relative }, output: notFound('missing/gstack-docs-candidate') }, + { tool: 'Bash', input: { command: `${declared}/bin/gstack-docs-candidate --help` }, output: 'usage' }]))).toContain('command outside declared docs observation interface'); + }); +}); diff --git a/test/fixtures/docsync-candidate-not-found-public.json b/test/fixtures/docsync-candidate-not-found-public.json new file mode 100644 index 000000000..be9ba39e7 --- /dev/null +++ b/test/fixtures/docsync-candidate-not-found-public.json @@ -0,0 +1,308 @@ +{ + "sourceRun": "39223031058 slice 2 ship-docsync-current (PR #3295 head 3918043)", + "sourceFileSha256": "7e4a144fcb47cfff0d54f51a839dd51451054cd8af9e9d36c73516b8b0ae6aef", + "note": "Every tool call of the captured parent and child in order. {home} is the fixture home and {shard} its shard root. Inputs keep only command or file_path; outputs are kept only for gstack-docs-candidate calls and failed calls.", + "calls": [ + { + "tool": "Read", + "input": { + "file_path": "{home}/phase.md" + }, + "output": "" + }, + { + "tool": "Read", + "input": { + "file_path": "{home}/fixture-interface.md" + }, + "output": "" + }, + { + "tool": "Read", + "input": { + "file_path": "{home}/.claude/skills/gstack/ship/sections/documentation.md" + }, + "output": "" + }, + { + "tool": "Bash", + "input": { + "command": "find {home}/.claude/skills/gstack/document-release {home}/.claude/skills/gstack/bin -maxdepth 3" + }, + "output": "" + }, + { + "tool": "Read", + "input": { + "file_path": "{home}/.claude/skills/gstack/document-release/SKILL.md" + }, + "output": "" + }, + { + "tool": "Read", + "input": { + "file_path": "{home}/.claude/skills/gstack/document-release/sections/audit-scope.md" + }, + "output": "" + }, + { + "tool": "Read", + "input": { + "file_path": "{home}/.claude/skills/gstack/document-release/sections/release-body.md" + }, + "output": "" + }, + { + "tool": "Bash", + "input": { + "command": "git status" + }, + "output": "" + }, + { + "tool": "Bash", + "input": { + "command": "git merge-base main HEAD" + }, + "output": "" + }, + { + "tool": "Bash", + "input": { + "command": "git ls-files --cached --others --exclude-standard" + }, + "output": "" + }, + { + "tool": "Bash", + "input": { + "command": "{home}/.claude/skills/gstack/bin/gstack-docs-candidate --help" + }, + "output": "usage: gstack-docs-candidate snapshot --out --audit-id --mode edit|read-only --base [--select ]... [--docs ]...\n gstack-docs-candidate compare " + }, + { + "tool": "Bash", + "input": { + "command": "git diff c37ee3a1a1c98972abc7cd26cb49a6f388c7c608 HEAD --stat" + }, + "output": "" + }, + { + "tool": "Bash", + "input": { + "command": "git diff c37ee3a1a1c98972abc7cd26cb49a6f388c7c608 HEAD" + }, + "output": "" + }, + { + "tool": "Bash", + "input": { + "command": "ls -la {home} {home}/repo" + }, + "output": "" + }, + { + "tool": "Bash", + "input": { + "command": "{home}/.claude/skills/gstack/bin/gstack-docs-candidate snapshot --out {home}/ship2-docs-a1.json --audit-id ship2-docs-20261019-a1 --mode edit --base c37ee3a1a1c98972abc7cd26cb49a6f388c7c608 --docs handbook --docs README.md --docs AGENTS.md" + }, + "output": "CANDIDATE: {home}/ship2-docs-a1.json\nbase=c37ee3a1a1c98972abc7cd26cb49a6f388c7c608 head=940ecb8e1008f52768d1f98ffed974d039a32251 branch=feature/docs mode=edit\ncommitted=1 staged=0 unstaged=0 untracked=1 selected=2 docs=3 hashed=4\ncommitted: \"handbook/reference/commands/widget.md.tmpl\"\nuntracked: \"personal-note.txt\"\ndocs: \"AGENTS.md\" \"README.md\" \"handbook/reference/commands/widget.md.tmpl\"" + }, + { + "tool": "Bash", + "input": { + "command": "{home}/.claude/skills/gstack/bin/gstack-docs-candidate snapshot --out {home}/ship2-docs-a1.json --audit-id ship2-docs-20261019-a1 --mode edit --base c37ee3a1a1c98972abc7cd26cb49a6f388c7c608 --select handbook/reference/commands/widget.md.tmpl --docs handbook --docs README.md --docs AGENTS.md" + }, + "output": "CANDIDATE: {home}/ship2-docs-a1.json\nbase=c37ee3a1a1c98972abc7cd26cb49a6f388c7c608 head=940ecb8e1008f52768d1f98ffed974d039a32251 branch=feature/docs mode=edit\ncommitted=1 staged=0 unstaged=0 untracked=1 selected=1 docs=3 hashed=4\ncommitted: \"handbook/reference/commands/widget.md.tmpl\"\nuntracked: \"personal-note.txt\"\ndocs: \"AGENTS.md\" \"README.md\" \"handbook/reference/commands/widget.md.tmpl\"" + }, + { + "tool": "Agent", + "input": { + "run_in_background": false + }, + "output": "" + }, + { + "tool": "Read", + "input": { + "file_path": "{home}/fixture-interface.md" + }, + "output": "" + }, + { + "tool": "Read", + "input": { + "file_path": "{home}/.claude/skills/gstack/document-release/SKILL.md" + }, + "output": "" + }, + { + "tool": "Read", + "input": { + "file_path": "{home}/.claude/skills/gstack/document-release/sections/audit-scope.md" + }, + "output": "" + }, + { + "tool": "Read", + "input": { + "file_path": "{home}/.claude/skills/gstack/document-release/sections/release-body.md" + }, + "output": "" + }, + { + "tool": "Bash", + "input": { + "command": "GSTACK_SESSION_KIND=spawned {home}/.claude/skills/gstack/bin/gstack-skill-start --skill document-release --model claude" + }, + "output": "" + }, + { + "tool": "Bash", + "input": { + "command": "cat {home}/ship2-docs-a1.json" + }, + "output": "" + }, + { + "tool": "Bash", + "input": { + "command": "git status" + }, + "output": "" + }, + { + "tool": "Bash", + "input": { + "command": "git ls-files -z --cached --others --exclude-standard" + }, + "output": "" + }, + { + "tool": "Bash", + "input": { + "command": "git diff c37ee3a1a1c98972abc7cd26cb49a6f388c7c608 HEAD" + }, + "output": "" + }, + { + "tool": "Bash", + "input": { + "command": "git diff --cached" + }, + "output": "" + }, + { + "tool": "Bash", + "input": { + "command": "git diff" + }, + "output": "" + }, + { + "tool": "Bash", + "input": { + "command": "cat AGENTS.md README.md handbook/reference/commands/widget.md.tmpl app.ts package.json" + }, + "output": "" + }, + { + "tool": "Bash", + "input": { + "command": "git log c37ee3a1a1c98972abc7cd26cb49a6f388c7c608..HEAD --oneline" + }, + "output": "" + }, + { + "tool": "Bash", + "input": { + "command": "cat CHANGELOG.md VERSION TODOS.md" + }, + "output": "" + }, + { + "tool": "Bash", + "input": { + "command": "{home}/.claude/skills/gstack/bin/gstack-skill-end --skill document-release --outcome success --session-id 842-1792399327-88f7a744 --tel-start 1792399327 --used-browse no" + }, + "output": "" + }, + { + "tool": "Bash", + "input": { + "command": "{shard}/.claude/skills/gstack/bin/gstack-docs-candidate compare {home}/ship2-docs-a1.json" + }, + "output": "Exit code 127\n(eval):1: no such file or directory: {shard}/.claude/skills/gstack/bin/gstack-docs-candidate" + }, + { + "tool": "Bash", + "input": { + "command": "{home}/.claude/skills/gstack/bin/gstack-docs-candidate compare {home}/ship2-docs-a1.json" + }, + "output": "{\"audit_id\":\"ship2-docs-20261019-a1\",\"head_changed\":false,\"index_changed\":false,\"content_changed\":[],\"newly_dirty\":[]}" + }, + { + "tool": "Bash", + "input": { + "command": "git status" + }, + "output": "" + }, + { + "tool": "Bash", + "input": { + "command": "git rev-parse HEAD" + }, + "output": "" + }, + { + "tool": "Bash", + "input": { + "command": "cat app.ts README.md AGENTS.md handbook/reference/commands/widget.md.tmpl CHANGELOG.md VERSION package.json TODOS.md" + }, + "output": "" + }, + { + "tool": "Bash", + "input": { + "command": "{home}/.claude/skills/gstack/bin/gstack-docs-candidate snapshot --out {home}/ship2-docs-20261019-a1-post.json --audit-id ship2-docs-20261019-a1 --mode edit --base c37ee3a1a1c98972abc7cd26cb49a6f388c7c608 --select handbook/reference/commands/widget.md.tmpl --docs handbook --docs README.md --docs AGENTS.md" + }, + "output": "CANDIDATE: {home}/ship2-docs-20261019-a1-post.json\nbase=c37ee3a1a1c98972abc7cd26cb49a6f388c7c608 head=940ecb8e1008f52768d1f98ffed974d039a32251 branch=feature/docs mode=edit\ncommitted=1 staged=0 unstaged=0 untracked=1 selected=1 docs=3 hashed=4\ncommitted: \"handbook/reference/commands/widget.md.tmpl\"\nuntracked: \"personal-note.txt\"\ndocs: \"AGENTS.md\" \"README.md\" \"handbook/reference/commands/widget.md.tmpl\"" + }, + { + "tool": "Write", + "input": { + "file_path": "{home}/ship2-docs-20261019-a1-documentation.md" + }, + "output": "" + }, + { + "tool": "Write", + "input": { + "file_path": "{home}/ship-report.md" + }, + "output": "" + }, + { + "tool": "Bash", + "input": { + "command": "cat {home}/ship2-docs-20261019-a1-documentation.md >> {home}/ship-report.md" + }, + "output": "" + }, + { + "tool": "Bash", + "input": { + "command": "bun {home}/fixture-publish.ts" + }, + "output": "" + }, + { + "tool": "Edit", + "input": { + "file_path": "{home}/ship-report.md" + }, + "output": "" + } + ] +} diff --git a/test/fixtures/docsync-observer-commands.json b/test/fixtures/docsync-observer-commands.json index 0ec117873..2899c2ad5 100644 --- a/test/fixtures/docsync-observer-commands.json +++ b/test/fixtures/docsync-observer-commands.json @@ -1,5 +1,5 @@ { - "_comment": "Bash commands from docsync E2E runs and their mutations. {repo} is the fixture repo, {other} a directory outside it, {home} the fixture home. known_good: read-only forms the observer must accept. known_bad: forms it must keep rejecting. known_bad also keeps genuine model misses from PR gate runs as negative controls: the composed cat; cat of ship-docsync-failure 38984977679 (bash#12). The Bash grep of ship-docsync-missing-asset 38983306684 (bash#13) is a pure read inside the fixture home, so the shared read-only grammar (GSTA-49) accepts it in known_good_posix (the grammar resolves POSIX paths only and accepts nothing on Windows), as it does the Opus slips wc -l (39173752116) and ls with a glob (effort study t10); its tee, outside-path, recursive-credential and denied-glob mutations stay in known_bad. The narrow pwd/ls/cat/stat/sha256sum forms only read inside the fixture home, never a Claude config file but settings.json, nor actor state (GSTA-49 tightening; the config dir itself stays listable): cat {home}/cc/.claude.json and cat /etc/hostname are known_bad. Fable 5.1 (PR #3288 E2E 39186278010 and a focused rerun): the grammar join applies only to its own reads, so joined Git reads stay known_bad; a trailing ; on a Git read is a harmless wrapper (known_good); a grep paged through sed -n with a line-range print script and an ls of the Claude config dir are known_good_posix; other sed scripts stay known_bad.", + "_comment": "Bash commands from docsync E2E runs and their mutations. {repo} is the fixture repo, {other} a directory outside it, {home} the fixture home. known_good: read-only forms the observer must accept. known_bad: forms it must keep rejecting. known_bad also keeps genuine model misses from PR gate runs as negative controls. The composed cat; cat of ship-docsync-failure 38984977679 (bash#12) reads only inside the fixture home with stderr discarded, so the grammar's 2>/dev/null widening (GSTA-49, Minder-approved) moves it to known_good_posix; its 2> to a file and 1>/dev/null mutations stay known_bad. The Bash grep of ship-docsync-missing-asset 38983306684 (bash#13) is a pure read inside the fixture home, so the shared read-only grammar (GSTA-49) accepts it in known_good_posix (the grammar resolves POSIX paths only and accepts nothing on Windows), as it does the Opus slips wc -l (39173752116) and ls with a glob (effort study t10); its tee, outside-path, recursive-credential and denied-glob mutations stay in known_bad. The narrow pwd/ls/cat/stat/sha256sum forms only read inside the fixture home, never a Claude config file but settings.json, nor actor state (GSTA-49 tightening; the config dir itself stays listable): cat {home}/cc/.claude.json and cat /etc/hostname are known_bad. Fable 5.1 (PR #3288 E2E 39186278010 and a focused rerun): the grammar join applies only to its own reads, so joined Git reads stay known_bad; a trailing ; on a Git read is a harmless wrapper (known_good); a grep paged through sed -n with a line-range print script and an ls of the Claude config dir are known_good_posix; other sed scripts stay known_bad.", "known_good": [ "git status", "git -C {repo} status", @@ -19,7 +19,8 @@ "wc -l {home}/.claude/skills/gstack/ship/SKILL.md", "ls -la {home}/.claude/skills/gstack/ship/sections {home}/.claude/skills/gstack/document-release {home}/.claude/skills/gstack/document-release/*", "grep -n \"Ship-owned\\|^## \\|^### \\|read-only\\|spawned\" {home}/.claude/skills/gstack/document-release/SKILL.md | sed -n 1,200p", - "cat {home}/candidate.json; echo; ls -la {home}/state {home}/cc {home}/.claude/skills/gstack/document-release {home}/.claude/skills/gstack/ship" + "cat {home}/candidate.json; echo; ls -la {home}/state {home}/cc {home}/.claude/skills/gstack/document-release {home}/.claude/skills/gstack/ship", + "cat {other}/bash-placeholder 2>/dev/null; cat {home}/state/config.yaml" ], "known_bad": [ "git -C {other} status", @@ -39,7 +40,6 @@ "bun arbitrary.ts 2>/dev/null", "git status $EXTRA", "git status\ntrue", - "cat {other}/bash-placeholder 2>/dev/null; cat {home}/state/config.yaml", "grep -n -i \"14.5\\|documentation\\|Step 15\\|Step 16\" {home}/.claude/skills/gstack/ship/SKILL.md | tee {home}/notes.md", "grep -n -i \"14.5\\|documentation\\|Step 15\\|Step 16\" /etc/passwd", "grep -rn apiKey {home}", @@ -49,6 +49,8 @@ "cat /etc/hostname", "git rev-parse HEAD; git status", "sed -n 1w {repo}/out.txt {repo}/README.md", - "sed -i s/a/b/ {repo}/README.md" + "sed -i s/a/b/ {repo}/README.md", + "cat {other}/bash-placeholder 2>{repo}/err.txt; cat {home}/state/config.yaml", + "cat {other}/bash-placeholder 1>/dev/null; cat {home}/state/config.yaml" ] } diff --git a/test/fixtures/qa-only-stop-cleanup-public.json b/test/fixtures/qa-only-stop-cleanup-public.json new file mode 100644 index 000000000..9bc493ea8 --- /dev/null +++ b/test/fixtures/qa-only-stop-cleanup-public.json @@ -0,0 +1,340 @@ +{ + "sourceArchiveSha256": "33b7cd6a929fe2a0ab827dd0dd77aea9261357491f15220c0ef1359287b9d8d3", + "sourceRun": "39217387697 slice 8 qa-only-no-fix (PR #3272 head 3986ac5)", + "sourceEventIndices": [ + 466, + 470, + 656, + 660, + 701, + 705, + 772, + 778 + ], + "events": [ + { + "type": "assistant", + "message": { + "model": "claude-opus-5-5", + "id": "msg_011CgBEbc3VPt2BCbX7eGKc6", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_01L7wBef1fN7eNy9mwQRni6M", + "name": "Bash", + "input": { + "command": "bun '__QA_GUARD__' run '__QA_FIXTURE__/qa-reports/deadline.json' -- bash -c 'bun \"__QA_FIXTURE__/fixture-browser-probe.ts\" \"__QA_BROWSE__\" \"http://127.0.0.1:39269/qa-only.html\" \"__QA_FIXTURE__/qa-reports/screenshots/initial.png\"'", + "description": "Run baseline browser probe under deadline guard" + }, + "caller": { + "type": "direct" + } + } + ], + "container": null, + "stop_reason": null, + "stop_sequence": null, + "stop_details": null, + "usage": { + "input_tokens": 2, + "cache_creation_input_tokens": 329, + "cache_read_input_tokens": 45732, + "cache_creation": { + "ephemeral_5m_input_tokens": 329, + "ephemeral_1h_input_tokens": 0 + }, + "output_tokens": 16, + "service_tier": "standard", + "inference_geo": "global", + "speed": "standard" + }, + "input_transformations": [], + "diagnostics": null, + "context_management": null + }, + "parent_tool_use_id": null, + "session_id": "58ff97a2-be5b-40db-a3ce-fb390aad7c88", + "uuid": "35b66938-677e-4421-b459-c0a4c376b84d", + "timestamp": "2026-10-19T07:54:11.131Z", + "request_id": "req_011CgBEbbm8oyfaUy4RiMpug", + "wire_tool_inputs": { + "toolu_01L7wBef1fN7eNy9mwQRni6M": { + "command": "bun '__QA_GUARD__' run '__QA_FIXTURE__/qa-reports/deadline.json' -- bash -c 'bun \"__QA_FIXTURE__/fixture-browser-probe.ts\" \"__QA_BROWSE__\" \"http://127.0.0.1:39269/qa-only.html\" \"__QA_FIXTURE__/qa-reports/screenshots/initial.png\"'", + "description": "Run baseline browser probe under deadline guard" + } + } + }, + { + "type": "user", + "message": { + "role": "user", + "content": [ + { + "tool_use_id": "toolu_01L7wBef1fN7eNy9mwQRni6M", + "type": "tool_result", + "content": "QA_DEADLINE {\"guard\":\"qa-deadline\",\"event\":\"started\",\"version\":1,\"startedAt\":\"2026-10-19T07:54:07.184Z\",\"deadlineAt\":\"2026-10-19T07:54:37.184Z\",\"budgetMs\":30000,\"observedAt\":\"2026-10-19T07:54:11.155Z\",\"remainingMs\":26029,\"expired\":false}\n{\"navigation\":{\"stdout\":\"Navigated to http://127.0.0.1:39269/qa-only.html (200)\\n\",\"stderr\":\"[browse] Starting server...\\n\",\"exitCode\":0},\"console\":{\"stdout\":\"--- BEGIN UNTRUSTED EXTERNAL CONTENT (source: http://127.0.0.1:39269/qa-only.html) ---\\n[2026-10-19T07:54:11.698Z] [error] TypeError: Cannot read properties of undefined (reading 'map')\\n--- END UNTRUSTED EXTERNAL CONTENT ---\\n\",\"stderr\":\"\",\"exitCode\":0},\"screenshot\":{\"stdout\":\"Screenshot saved: __QA_FIXTURE__/qa-reports/screenshots/initial.png\\n\",\"stderr\":\"\",\"exitCode\":0}}\n\nQA_DEADLINE {\"guard\":\"qa-deadline\",\"event\":\"finished\",\"observedAt\":\"2026-10-19T07:54:11.808Z\",\"deadlineAt\":\"2026-10-19T07:54:37.184Z\",\"timedOut\":false,\"exitCode\":0}", + "is_error": false + } + ] + }, + "parent_tool_use_id": null, + "session_id": "58ff97a2-be5b-40db-a3ce-fb390aad7c88", + "uuid": "a48d1ba7-2580-412a-9c36-a7e92de980d5", + "timestamp": "2026-10-19T07:54:11.818Z", + "tool_use_result": { + "stdout": "QA_DEADLINE {\"guard\":\"qa-deadline\",\"event\":\"started\",\"version\":1,\"startedAt\":\"2026-10-19T07:54:07.184Z\",\"deadlineAt\":\"2026-10-19T07:54:37.184Z\",\"budgetMs\":30000,\"observedAt\":\"2026-10-19T07:54:11.155Z\",\"remainingMs\":26029,\"expired\":false}\n{\"navigation\":{\"stdout\":\"Navigated to http://127.0.0.1:39269/qa-only.html (200)\\n\",\"stderr\":\"[browse] Starting server...\\n\",\"exitCode\":0},\"console\":{\"stdout\":\"--- BEGIN UNTRUSTED EXTERNAL CONTENT (source: http://127.0.0.1:39269/qa-only.html) ---\\n[2026-10-19T07:54:11.698Z] [error] TypeError: Cannot read properties of undefined (reading 'map')\\n--- END UNTRUSTED EXTERNAL CONTENT ---\\n\",\"stderr\":\"\",\"exitCode\":0},\"screenshot\":{\"stdout\":\"Screenshot saved: __QA_FIXTURE__/qa-reports/screenshots/initial.png\\n\",\"stderr\":\"\",\"exitCode\":0}}\n\nQA_DEADLINE {\"guard\":\"qa-deadline\",\"event\":\"finished\",\"observedAt\":\"2026-10-19T07:54:11.808Z\",\"deadlineAt\":\"2026-10-19T07:54:37.184Z\",\"timedOut\":false,\"exitCode\":0}", + "stderr": "", + "interrupted": false, + "isImage": false, + "noOutputExpected": false, + "bashEditDiff": { + "files": [], + "moreFiles": 1, + "changedFiles": [ + "__QA_FIXTURE__/qa-reports/screenshots/initial.png" + ] + } + } + }, + { + "type": "assistant", + "message": { + "model": "claude-opus-5-5", + "id": "msg_011CgBEc8zLFuDZaD1fV1X27", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_01B5gouuWiBq47RwAQzdjaW6", + "name": "Write", + "input": { + "file_path": "__QA_FIXTURE__/qa-reports/exploration-001.json", + "content": "{\n \"observationCommand\": \"bun '__QA_GUARD__' run '__QA_FIXTURE__/qa-reports/deadline.json' -- bash -c 'bun \\\"__QA_FIXTURE__/fixture-browser-probe.ts\\\" \\\"__QA_BROWSE__\\\" \\\"http://127.0.0.1:39269/qa-only.html\\\" \\\"__QA_FIXTURE__/qa-reports/screenshots/initial.png\\\"'\",\n \"observed\": {\"navigation\":{\"stdout\":\"Navigated to http://127.0.0.1:39269/qa-only.html (200)\\n\",\"stderr\":\"[browse] Starting server...\\n\",\"exitCode\":0},\"console\":{\"stdout\":\"--- BEGIN UNTRUSTED EXTERNAL CONTENT (source: http://127.0.0.1:39269/qa-only.html) ---\\n[2026-10-19T07:54:11.698Z] [error] TypeError: Cannot read properties of undefined (reading 'map')\\n--- END UNTRUSTED EXTERNAL CONTENT ---\\n\",\"stderr\":\"\",\"exitCode\":0},\"screenshot\":{\"stdout\":\"Screenshot saved: __QA_FIXTURE__/qa-reports/screenshots/initial.png\\n\",\"stderr\":\"\",\"exitCode\":0}},\n \"hypothesis\": \"Baseline shows one console TypeError on load; exact replay of the same probe with a fresh screenshot confirms it is reproducible.\",\n \"nextCommand\": \"bun '__QA_GUARD__' run '__QA_FIXTURE__/qa-reports/deadline.json' -- bash -c 'bun \\\"__QA_FIXTURE__/fixture-browser-probe.ts\\\" \\\"__QA_BROWSE__\\\" \\\"http://127.0.0.1:39269/qa-only.html\\\" \\\"__QA_FIXTURE__/qa-reports/screenshots/replay-001.png\\\"'\"\n}\n" + }, + "caller": { + "type": "direct" + } + } + ], + "container": null, + "stop_reason": null, + "stop_sequence": null, + "stop_details": null, + "usage": { + "input_tokens": 2, + "cache_creation_input_tokens": 354, + "cache_read_input_tokens": 46991, + "cache_creation": { + "ephemeral_5m_input_tokens": 354, + "ephemeral_1h_input_tokens": 0 + }, + "output_tokens": 8, + "service_tier": "standard", + "inference_geo": "global", + "speed": "standard" + }, + "input_transformations": [], + "diagnostics": null, + "context_management": null + }, + "parent_tool_use_id": null, + "session_id": "58ff97a2-be5b-40db-a3ce-fb390aad7c88", + "uuid": "542289f5-d545-463a-a8b3-4789658d86d1", + "timestamp": "2026-10-19T07:54:23.625Z", + "request_id": "req_011CgBEc8jht4KHH1idTVyEj", + "wire_tool_inputs": { + "toolu_01B5gouuWiBq47RwAQzdjaW6": { + "file_path": "__QA_FIXTURE__/qa-reports/exploration-001.json", + "content": "{\n \"observationCommand\": \"bun '__QA_GUARD__' run '__QA_FIXTURE__/qa-reports/deadline.json' -- bash -c 'bun \\\"__QA_FIXTURE__/fixture-browser-probe.ts\\\" \\\"__QA_BROWSE__\\\" \\\"http://127.0.0.1:39269/qa-only.html\\\" \\\"__QA_FIXTURE__/qa-reports/screenshots/initial.png\\\"'\",\n \"observed\": {\"navigation\":{\"stdout\":\"Navigated to http://127.0.0.1:39269/qa-only.html (200)\\n\",\"stderr\":\"[browse] Starting server...\\n\",\"exitCode\":0},\"console\":{\"stdout\":\"--- BEGIN UNTRUSTED EXTERNAL CONTENT (source: http://127.0.0.1:39269/qa-only.html) ---\\n[2026-10-19T07:54:11.698Z] [error] TypeError: Cannot read properties of undefined (reading 'map')\\n--- END UNTRUSTED EXTERNAL CONTENT ---\\n\",\"stderr\":\"\",\"exitCode\":0},\"screenshot\":{\"stdout\":\"Screenshot saved: __QA_FIXTURE__/qa-reports/screenshots/initial.png\\n\",\"stderr\":\"\",\"exitCode\":0}},\n \"hypothesis\": \"Baseline shows one console TypeError on load; exact replay of the same probe with a fresh screenshot confirms it is reproducible.\",\n \"nextCommand\": \"bun '__QA_GUARD__' run '__QA_FIXTURE__/qa-reports/deadline.json' -- bash -c 'bun \\\"__QA_FIXTURE__/fixture-browser-probe.ts\\\" \\\"__QA_BROWSE__\\\" \\\"http://127.0.0.1:39269/qa-only.html\\\" \\\"__QA_FIXTURE__/qa-reports/screenshots/replay-001.png\\\"'\"\n}\n" + } + } + }, + { + "type": "user", + "message": { + "role": "user", + "content": [ + { + "tool_use_id": "toolu_01B5gouuWiBq47RwAQzdjaW6", + "type": "tool_result", + "content": "File created successfully at: __QA_FIXTURE__/qa-reports/exploration-001.json (file state is current in your context \u2014 no need to Read it back)" + } + ] + }, + "parent_tool_use_id": null, + "session_id": "58ff97a2-be5b-40db-a3ce-fb390aad7c88", + "uuid": "f4b34e9f-8772-491e-96cb-5029c43001c9", + "timestamp": "2026-10-19T07:54:23.636Z", + "tool_use_result": { + "type": "create", + "filePath": "__QA_FIXTURE__/qa-reports/exploration-001.json", + "content": "{\n \"observationCommand\": \"bun '__QA_GUARD__' run '__QA_FIXTURE__/qa-reports/deadline.json' -- bash -c 'bun \\\"__QA_FIXTURE__/fixture-browser-probe.ts\\\" \\\"__QA_BROWSE__\\\" \\\"http://127.0.0.1:39269/qa-only.html\\\" \\\"__QA_FIXTURE__/qa-reports/screenshots/initial.png\\\"'\",\n \"observed\": {\"navigation\":{\"stdout\":\"Navigated to http://127.0.0.1:39269/qa-only.html (200)\\n\",\"stderr\":\"[browse] Starting server...\\n\",\"exitCode\":0},\"console\":{\"stdout\":\"--- BEGIN UNTRUSTED EXTERNAL CONTENT (source: http://127.0.0.1:39269/qa-only.html) ---\\n[2026-10-19T07:54:11.698Z] [error] TypeError: Cannot read properties of undefined (reading 'map')\\n--- END UNTRUSTED EXTERNAL CONTENT ---\\n\",\"stderr\":\"\",\"exitCode\":0},\"screenshot\":{\"stdout\":\"Screenshot saved: __QA_FIXTURE__/qa-reports/screenshots/initial.png\\n\",\"stderr\":\"\",\"exitCode\":0}},\n \"hypothesis\": \"Baseline shows one console TypeError on load; exact replay of the same probe with a fresh screenshot confirms it is reproducible.\",\n \"nextCommand\": \"bun '__QA_GUARD__' run '__QA_FIXTURE__/qa-reports/deadline.json' -- bash -c 'bun \\\"__QA_FIXTURE__/fixture-browser-probe.ts\\\" \\\"__QA_BROWSE__\\\" \\\"http://127.0.0.1:39269/qa-only.html\\\" \\\"__QA_FIXTURE__/qa-reports/screenshots/replay-001.png\\\"'\"\n}\n", + "structuredPatch": [], + "originalFile": null, + "userModified": false + } + }, + { + "type": "assistant", + "message": { + "model": "claude-opus-5-5", + "id": "msg_011CgBEc8zLFuDZaD1fV1X27", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_015WHowRypEs12DpL8sq2iBD", + "name": "Bash", + "input": { + "command": "bun '__QA_GUARD__' run '__QA_FIXTURE__/qa-reports/deadline.json' -- bash -c 'bun \"__QA_FIXTURE__/fixture-browser-probe.ts\" \"__QA_BROWSE__\" \"http://127.0.0.1:39269/qa-only.html\" \"__QA_FIXTURE__/qa-reports/screenshots/replay-001.png\"'", + "description": "Replay browser probe with fresh screenshot" + }, + "caller": { + "type": "direct" + } + } + ], + "container": null, + "stop_reason": null, + "stop_sequence": null, + "stop_details": null, + "usage": { + "input_tokens": 2, + "cache_creation_input_tokens": 354, + "cache_read_input_tokens": 46991, + "cache_creation": { + "ephemeral_5m_input_tokens": 354, + "ephemeral_1h_input_tokens": 0 + }, + "output_tokens": 8, + "service_tier": "standard", + "inference_geo": "global", + "speed": "standard" + }, + "input_transformations": [], + "diagnostics": null, + "context_management": null + }, + "parent_tool_use_id": null, + "session_id": "58ff97a2-be5b-40db-a3ce-fb390aad7c88", + "uuid": "7ca220bf-403d-483d-bef3-9ec8cde921a0", + "timestamp": "2026-10-19T07:54:25.902Z", + "request_id": "req_011CgBEc8jht4KHH1idTVyEj", + "wire_tool_inputs": { + "toolu_015WHowRypEs12DpL8sq2iBD": { + "command": "bun '__QA_GUARD__' run '__QA_FIXTURE__/qa-reports/deadline.json' -- bash -c 'bun \"__QA_FIXTURE__/fixture-browser-probe.ts\" \"__QA_BROWSE__\" \"http://127.0.0.1:39269/qa-only.html\" \"__QA_FIXTURE__/qa-reports/screenshots/replay-001.png\"'", + "description": "Replay browser probe with fresh screenshot" + } + } + }, + { + "type": "user", + "message": { + "role": "user", + "content": [ + { + "tool_use_id": "toolu_015WHowRypEs12DpL8sq2iBD", + "type": "tool_result", + "content": "QA_DEADLINE {\"guard\":\"qa-deadline\",\"event\":\"started\",\"version\":1,\"startedAt\":\"2026-10-19T07:54:07.184Z\",\"deadlineAt\":\"2026-10-19T07:54:37.184Z\",\"budgetMs\":30000,\"observedAt\":\"2026-10-19T07:54:25.941Z\",\"remainingMs\":11243,\"expired\":false}\n{\"navigation\":{\"stdout\":\"Navigated to http://127.0.0.1:39269/qa-only.html (200)\\n\",\"stderr\":\"\",\"exitCode\":0},\"console\":{\"stdout\":\"--- BEGIN UNTRUSTED EXTERNAL CONTENT (source: http://127.0.0.1:39269/qa-only.html) ---\\n[2026-10-19T07:54:11.698Z] [error] TypeError: Cannot read properties of undefined (reading 'map')\\n[2026-10-19T07:54:25.967Z] [error] TypeError: Cannot read properties of undefined (reading 'map')\\n--- END UNTRUSTED EXTERNAL CONTENT ---\\n\",\"stderr\":\"\",\"exitCode\":0},\"screenshot\":{\"stdout\":\"Screenshot saved: __QA_FIXTURE__/qa-reports/screenshots/replay-001.png\\n\",\"stderr\":\"\",\"exitCode\":0}}\n\nQA_DEADLINE {\"guard\":\"qa-deadline\",\"event\":\"finished\",\"observedAt\":\"2026-10-19T07:54:26.040Z\",\"deadlineAt\":\"2026-10-19T07:54:37.184Z\",\"timedOut\":false,\"exitCode\":0}", + "is_error": false + } + ] + }, + "parent_tool_use_id": null, + "session_id": "58ff97a2-be5b-40db-a3ce-fb390aad7c88", + "uuid": "4cc8ce35-f0d7-4076-991b-451564cf0144", + "timestamp": "2026-10-19T07:54:26.050Z", + "tool_use_result": { + "stdout": "QA_DEADLINE {\"guard\":\"qa-deadline\",\"event\":\"started\",\"version\":1,\"startedAt\":\"2026-10-19T07:54:07.184Z\",\"deadlineAt\":\"2026-10-19T07:54:37.184Z\",\"budgetMs\":30000,\"observedAt\":\"2026-10-19T07:54:25.941Z\",\"remainingMs\":11243,\"expired\":false}\n{\"navigation\":{\"stdout\":\"Navigated to http://127.0.0.1:39269/qa-only.html (200)\\n\",\"stderr\":\"\",\"exitCode\":0},\"console\":{\"stdout\":\"--- BEGIN UNTRUSTED EXTERNAL CONTENT (source: http://127.0.0.1:39269/qa-only.html) ---\\n[2026-10-19T07:54:11.698Z] [error] TypeError: Cannot read properties of undefined (reading 'map')\\n[2026-10-19T07:54:25.967Z] [error] TypeError: Cannot read properties of undefined (reading 'map')\\n--- END UNTRUSTED EXTERNAL CONTENT ---\\n\",\"stderr\":\"\",\"exitCode\":0},\"screenshot\":{\"stdout\":\"Screenshot saved: __QA_FIXTURE__/qa-reports/screenshots/replay-001.png\\n\",\"stderr\":\"\",\"exitCode\":0}}\n\nQA_DEADLINE {\"guard\":\"qa-deadline\",\"event\":\"finished\",\"observedAt\":\"2026-10-19T07:54:26.040Z\",\"deadlineAt\":\"2026-10-19T07:54:37.184Z\",\"timedOut\":false,\"exitCode\":0}", + "stderr": "", + "interrupted": false, + "isImage": false, + "noOutputExpected": false, + "bashEditDiff": { + "files": [], + "moreFiles": 1, + "changedFiles": [ + "__QA_FIXTURE__/qa-reports/screenshots/replay-001.png" + ] + } + } + }, + { + "type": "assistant", + "message": { + "model": "claude-opus-5-5", + "id": "msg_011CgBEdLuDaVg7evd7cJGEr", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_01Fy5esoJHRgnfH9ts4HSMbR", + "name": "Bash", + "input": { + "command": "bun '__QA_GUARD__' run '__QA_FIXTURE__/qa-reports/deadline.json' -- bash -c 'B=\"__QA_BROWSE__\"; \"$B\" stop'", + "description": "Stop owned headless browser daemon under guard" + }, + "caller": { + "type": "direct" + } + } + ], + "container": null, + "stop_reason": null, + "stop_sequence": null, + "stop_details": null, + "usage": { + "input_tokens": 2, + "cache_creation_input_tokens": 567, + "cache_read_input_tokens": 49621, + "cache_creation": { + "ephemeral_5m_input_tokens": 567, + "ephemeral_1h_input_tokens": 0 + }, + "output_tokens": 8, + "service_tier": "standard", + "inference_geo": "global", + "speed": "standard" + }, + "input_transformations": [], + "diagnostics": null, + "context_management": null + }, + "parent_tool_use_id": null, + "session_id": "58ff97a2-be5b-40db-a3ce-fb390aad7c88", + "uuid": "b83d8680-da53-458a-bc98-72e2b46ff137", + "timestamp": "2026-10-19T07:54:33.997Z", + "request_id": "req_011CgBEdLeavMxE7VgpzTQCw", + "wire_tool_inputs": { + "toolu_01Fy5esoJHRgnfH9ts4HSMbR": { + "command": "bun '__QA_GUARD__' run '__QA_FIXTURE__/qa-reports/deadline.json' -- bash -c 'B=\"__QA_BROWSE__\"; \"$B\" stop'", + "description": "Stop owned headless browser daemon under guard" + } + } + }, + { + "type": "user", + "message": { + "role": "user", + "content": [ + { + "type": "tool_result", + "content": "Exit code 124\n\nQA_DEADLINE {\"guard\":\"qa-deadline\",\"event\":\"started\",\"version\":1,\"startedAt\":\"2026-10-19T07:54:07.184Z\",\"deadlineAt\":\"2026-10-19T07:54:37.184Z\",\"budgetMs\":30000,\"observedAt\":\"2026-10-19T07:54:34.020Z\",\"remainingMs\":3164,\"expired\":false}\nServer stopped\n\nQA_DEADLINE {\"guard\":\"qa-deadline\",\"event\":\"finished\",\"observedAt\":\"2026-10-19T07:54:37.185Z\",\"deadlineAt\":\"2026-10-19T07:54:37.184Z\",\"timedOut\":true,\"exitCode\":124}", + "is_error": true, + "tool_use_id": "toolu_01Fy5esoJHRgnfH9ts4HSMbR" + } + ] + }, + "parent_tool_use_id": null, + "session_id": "58ff97a2-be5b-40db-a3ce-fb390aad7c88", + "uuid": "074875af-080b-4620-80fc-5430f1aaf123", + "timestamp": "2026-10-19T07:54:37.190Z", + "tool_use_result": "Error: Exit code 124\n\nQA_DEADLINE {\"guard\":\"qa-deadline\",\"event\":\"started\",\"version\":1,\"startedAt\":\"2026-10-19T07:54:07.184Z\",\"deadlineAt\":\"2026-10-19T07:54:37.184Z\",\"budgetMs\":30000,\"observedAt\":\"2026-10-19T07:54:34.020Z\",\"remainingMs\":3164,\"expired\":false}\nServer stopped\n\nQA_DEADLINE {\"guard\":\"qa-deadline\",\"event\":\"finished\",\"observedAt\":\"2026-10-19T07:54:37.185Z\",\"deadlineAt\":\"2026-10-19T07:54:37.184Z\",\"timedOut\":true,\"exitCode\":124}" + } + ] +} diff --git a/test/helpers/docsync-observer.ts b/test/helpers/docsync-observer.ts index bfdbddbd7..318e25540 100644 --- a/test/helpers/docsync-observer.ts +++ b/test/helpers/docsync-observer.ts @@ -367,12 +367,31 @@ function docsPrivateArtifact(file: string, fixture: ReturnType, scripts: string[]): boolean { + const command = String(calls[index].input?.command ?? '').trim(); + const binary = command.split(' ')[0]!; + const candidate = path.join(fixture.skills, 'bin/gstack-docs-candidate').split(path.sep).join('/'); + if (!/^\/[\w.\/-]+\/gstack-docs-candidate$/.test(binary) || binary === candidate) return false; + if (calls[index].output.trimEnd() !== `Exit code 127\n(eval):1: no such file or directory: ${binary}`) return false; + try { + fs.lstatSync(binary); + return false; + } catch (error) { + if (!['ENOENT', 'ENOTDIR'].includes((error as NodeJS.ErrnoException).code ?? '')) return false; + } + const retry = candidate + command.slice(binary.length); + return docsCommandAllowed(retry, fixture, scripts) && calls.slice(index + 1).some(call => + call.tool === 'Bash' && String(call.input?.command ?? '').trim() === retry && !call.output.startsWith('Exit code')); +} + export function docsToolFailures(result: SkillTestResult, fixture: ReturnType, scripts: string[] = [], readOnly = false): string[] { const failures: string[] = []; const repo = fs.realpathSync(fixture.repo); const authoredDoc = path.join(repo, DOC_PATH); - for (const call of result.toolCalls) { - if (call.tool === 'Bash' && !docsCommandAllowed(String(call.input?.command ?? ''), fixture, scripts)) failures.push('command outside declared docs observation interface'); + for (const [index, call] of result.toolCalls.entries()) { + if (call.tool === 'Bash' && !docsCommandAllowed(String(call.input?.command ?? ''), fixture, scripts) && + !candidateNotFound(result.toolCalls, index, fixture, scripts)) failures.push('command outside declared docs observation interface'); if (['Write', 'Edit'].includes(call.tool)) { const file = path.resolve(fixture.repo, call.input?.file_path ?? ''); const actual = actualWritePath(file); diff --git a/test/helpers/qa-browser-deadline-evidence.ts b/test/helpers/qa-browser-deadline-evidence.ts index 4b5dced79..5d2b628da 100644 --- a/test/helpers/qa-browser-deadline-evidence.ts +++ b/test/helpers/qa-browser-deadline-evidence.ts @@ -2,6 +2,7 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import { isDeepStrictEqual } from 'node:util'; import { nativeCalls, readQACheckpointFiles } from './qa-checkpoint-evidence'; +import { readOnlyCommandAllowed, readOnlyInterfaceText } from './read-only-commands'; type Call = { tool: string; input: any; output: string }; type Options = { directory: string; guard: string; browse: string; started: number; ended: number }; @@ -24,12 +25,13 @@ export function qaDeadlineShellPolicy(directory: string, guard: string, browse: const file = path.join(directory, 'qa-reports/deadline.json'); return { allowed, file, prompt: `This native fixture accepts a deliberately narrow executable outer form. Run the BROWSER SETUP Aside readiness bash block verbatim, as its own Bash call (no appended commands). The fallback binary is already supplied. Other than that block, these standalone setup/bookkeeping Bash calls are allowed: ${allowed.slice(1).join('\n')} -Use Read/Glob for every other local inspection (no other ls, wc, cat or find Bash calls) and Write only inside ${path.join(directory, 'qa-reports')}; the report and screenshots directories already exist. Memory files and learning stores outside that directory are not authorized. Write the initial charters and final report to the same caller-owned file ${path.join(directory, 'qa-reports/qa-only-report.md')}, not a separate charter file. +${readOnlyInterfaceText(directory)} +Use Read/Glob for any other local inspection and Write only inside ${path.join(directory, 'qa-reports')}; the report and screenshots directories already exist. Memory files and learning stores outside that directory are not authorized. Write the initial charters and final report to the same caller-owned file ${path.join(directory, 'qa-reports/qa-only-report.md')}, not a separate charter file. Every other Bash call must be exactly one invocation of bun ${quote(guard)}, with literal arguments, no outer assignments, substitutions, globs, pipes, redirects, prefixes, suffixes or shell operators. The exact runtime path ${quote(process.execPath)} may replace bun; no other launcher is accepted. Start exactly once, and only after you have read the workflow sections you need and your Write of the initial charter to ${path.join(directory, 'qa-reports/qa-only-report.md')} has completed; the 30-second deadline begins at start: bun ${quote(guard)} start ${quote(file)} 30 Then use bun ${quote(guard)} status ${quote(file)} or bun ${quote(guard)} run ${quote(file)} -- COMMAND ARGS. Use the absolute guard path directly, not $G. For browser scripts, put all assignments, pipelines and scripts INSIDE the child: bun ${quote(guard)} run ${quote(file)} -- bash -c 'B="${browse}"; "$B" goto URL; "$B" snapshot -i'. Literal argv and single/double quoted literal arguments are accepted; expansions are only allowed inside the single-quoted child script. -Do not write, reset, replace, chmod or remove deadline.json, invoke the guard recursively, or print QA_DEADLINE receipts yourself. Browser cleanup also goes inside the guard; after expiry only local report bookkeeping is allowed. Fallback screenshots must be saved directly in the owned screenshots directory; retain them and write the report after expiry without new browser calls. This fixture does not accept a separate post-expiry shell copy from Aside's session directory; mark that artifact unavailable rather than launching new browser work. +Do not write, reset, replace, chmod or remove deadline.json, invoke the guard recursively, or print QA_DEADLINE receipts yourself. Browser cleanup also goes inside the guard, as a whole child that is exactly "$B" closetab or "$B" stop (B set to the supplied browse binary); any other child counts as a probe; after expiry only local report bookkeeping is allowed. Fallback screenshots must be saved directly in the owned screenshots directory; retain them and write the report after expiry without new browser calls. This fixture does not accept a separate post-expiry shell copy from Aside's session directory; mark that artifact unavailable rather than launching new browser work. An expired refusal is not an executed probe or a pass. Report unfinished coverage honestly; do not try to finish every page after expiry.` }; } @@ -108,13 +110,13 @@ export function assertQaBrowserPreparation(transcript: unknown[], options: Pick< /** * The fixture prompt routes browser cleanup through the guard, and a probe - * excludes bookkeeping. Only closing the tab with the supplied browse binary, - * as the whole child, is cleanup; anything else in the child is a probe. + * excludes bookkeeping. Only closing the tab or stopping the supplied browse + * binary, as the whole child, is cleanup; anything else in the child is a probe. */ function browserCleanupChild(child: string[], browse: string): boolean { - if (child.length === 2) return child[0] === browse && child[1] === 'closetab'; - return child.length === 3 && child[0] === 'bash' && child[1] === '-c' - && [`B="${browse}"; "$B" closetab`, `"${browse}" closetab`].includes(child[2].trim()); + if (child.length === 2) return child[0] === browse && ['closetab', 'stop'].includes(child[1]); + return child.length === 3 && child[0] === 'bash' && child[1] === '-c' && ['closetab', 'stop'].some(verb => + [`B="${browse}"; "$B" ${verb}`, `B="${browse}"; $B ${verb}`, `"${browse}" ${verb}`].includes(child[2].trim())); } export function assertQaBrowserCheckpoints(transcript: unknown[], options: Pick) { @@ -213,6 +215,10 @@ export function assertQaBrowserDeadline(calls: Call[], options: Options & { expe if ((call.output ?? '').includes('QA_DEADLINE ')) fail('receipt outside trusted guard invocation'); continue; } + if (readOnlyCommandAllowed(command, { root: directory })) { + if ((call.output ?? '').split('\n').some(line => line.startsWith('QA_DEADLINE '))) fail('receipt outside trusted guard invocation'); + continue; + } const outer = literalArgv(command); if (!['bun', process.execPath].includes(outer[0])) fail('untrusted runtime or unguarded command'); const argv = outer.slice(1); diff --git a/test/helpers/read-only-commands.ts b/test/helpers/read-only-commands.ts index c95fe522c..5ec75897d 100644 --- a/test/helpers/read-only-commands.ts +++ b/test/helpers/read-only-commands.ts @@ -11,7 +11,8 @@ * inside the root. Every path operand, path-valued option and unquoted glob * match must resolve physically (symlinks and .. applied) inside the fixture * root and outside its denied paths. - * Rejected: every other tool, redirects, substitutions, variables, tilde and + * Stderr may be discarded with 2>/dev/null or merged with 2>&1 after a word. + * Rejected: every other tool, every other redirect, substitutions, variables, tilde and * brace expansion, backgrounding, multi-line input, and the write, exec, * symlink-following and output-file options of the listed tools. */ @@ -109,7 +110,7 @@ const FIND_PATHS = new Set(['-newer', '-anewer', '-cnewer', '-samefile']); /** Interface text an opted-in fixture declares to its actor. */ export function readOnlyInterfaceText(root: string): string { - return `Shared read-only grammar: Bash also accepts pure local reads whose every path, unquoted glob matches included, resolves inside ${root} with no .., symlink or absolute-path escape: ${Object.keys(READ_TOOLS).join(', ')} (find without -exec, -ok, -delete or -fprint*; sed only as -n with a line-range print script such as sed -n 1,200p; no output-file, symlink-following or external-program options; echo and printf with literal arguments only, no printf -v). Join only these reads with ;, && or ||, pipe only into ${[...STDIN_TOOLS].join(', ')}, and cd only into a directory inside that root. Redirects, tee, substitutions, variables, tilde or brace expansion, background jobs, multi-line commands and every other tool stay outside this grammar.`; + return `Shared read-only grammar: Bash also accepts pure local reads whose every path, unquoted glob matches included, resolves inside ${root} with no .., symlink or absolute-path escape: ${Object.keys(READ_TOOLS).join(', ')} (find without -exec, -ok, -delete or -fprint*; sed only as -n with a line-range print script such as sed -n 1,200p; no output-file, symlink-following or external-program options; echo and printf with literal arguments only, no printf -v). Join only these reads with ;, && or ||, pipe only into ${[...STDIN_TOOLS].join(', ')}, and cd only into a directory inside that root. Redirects other than 2>/dev/null or 2>&1 right after a read's arguments, tee, substitutions, variables, tilde or brace expansion, background jobs, multi-line commands and every other tool stay outside this grammar.`; } export function readOnlyCommandAllowed(command: string, scope: ReadOnlyScope): boolean { @@ -199,6 +200,9 @@ function tokenize(command: string): Token[] | string { continue; } if (c === '$' || c === '`') return 'substitution or variable'; + const discard = c === '>' && word?.text === '2' && !word.quoted && tokens.length > 0 && !('op' in tokens.at(-1)!) + ? /^>(?:\/dev\/null|&1)(?=[ \t;|&]|$)/.exec(command.slice(i)) : null; + if (discard) { word = undefined; i += discard[0].length - 1; continue; } if (c === '>' || c === '<') return 'redirect'; if ('(){}'.includes(c)) return 'subshell, grouping or brace expansion'; if (!word && (c === '~' || c === '#')) return 'tilde expansion or comment'; diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index 3be2e6b7b..4433c4fa0 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -106,15 +106,15 @@ export const E2E_TOUCHFILES: Record = { // QA (+ test-server dependency). /qa drives Aside first (the resolver) and // the browse binary as fallback (browse/src), so both are deps. - 'qa-quick': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'bin/gstack-qa-deadline', 'lib/qa-deadline.ts', 'lib/claude-code-windows-job.ts', 'qa/**', 'scripts/resolvers/browse.ts', 'scripts/resolvers/aside.ts', 'browse/src/**', 'browse/test/test-server.ts', 'test/skill-e2e-qa-workflow.test.ts', 'test/helpers/aside-available.ts', 'test/fixtures/qa-only-browser-probe.ts', 'test/helpers/bootstrap-retention.ts', 'test/helpers/qa-browser-deadline-evidence.ts', 'test/helpers/qa-checkpoint-evidence.ts', 'test/helpers/qa-evidence-producer.ts', 'test/helpers/qa-only-cleanup.ts'], + 'qa-quick': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'bin/gstack-qa-deadline', 'lib/qa-deadline.ts', 'lib/claude-code-windows-job.ts', 'qa/**', 'scripts/resolvers/browse.ts', 'scripts/resolvers/aside.ts', 'browse/src/**', 'browse/test/test-server.ts', 'test/skill-e2e-qa-workflow.test.ts', 'test/helpers/aside-available.ts', 'test/fixtures/qa-only-browser-probe.ts', 'test/helpers/bootstrap-retention.ts', 'test/helpers/qa-browser-deadline-evidence.ts', 'test/helpers/read-only-commands.ts', 'test/helpers/qa-checkpoint-evidence.ts', 'test/helpers/qa-evidence-producer.ts', 'test/helpers/qa-only-cleanup.ts'], 'qa-b6-static': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'qa/**', 'scripts/resolvers/aside.ts', 'browse/src/**', 'browse/test/test-server.ts', 'test/helpers/llm-judge.ts', 'browse/test/fixtures/qa-eval.html', 'test/fixtures/qa-eval-ground-truth.json', 'test/skill-e2e-qa-bugs.test.ts', 'test/helpers/aside-available.ts'], 'qa-b7-spa': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'qa/**', 'scripts/resolvers/aside.ts', 'browse/src/**', 'browse/test/test-server.ts', 'test/helpers/llm-judge.ts', 'browse/test/fixtures/qa-eval-spa.html', 'test/fixtures/qa-eval-spa-ground-truth.json', 'test/skill-e2e-qa-bugs.test.ts', 'test/helpers/aside-available.ts'], 'qa-b8-checkout': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'qa/**', 'scripts/resolvers/aside.ts', 'browse/src/**', 'browse/test/test-server.ts', 'test/helpers/llm-judge.ts', 'browse/test/fixtures/qa-eval-checkout.html', 'test/fixtures/qa-eval-checkout-ground-truth.json', 'test/skill-e2e-qa-bugs.test.ts', 'test/helpers/aside-available.ts'], - 'qa-only-no-fix': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'test/helpers/qa-only-cleanup.ts', 'test/helpers/qa-checkpoint-evidence.ts', 'test/fixtures/qa-only-observation-public.json', 'test/fixtures/qa-only-charter-public.json', 'test/fixtures/qa-only-browser-probe.ts', 'browse/test/fixtures/qa-only.html', 'test/helpers/qa-browser-deadline-evidence.ts', 'bin/gstack-qa-deadline', 'lib/qa-deadline.ts', 'lib/claude-code-windows-job.ts', 'qa-only/**', 'qa/sections/**', 'qa/templates/**', 'scripts/resolvers/aside.ts', 'browse/src/**', 'browse/test/test-server.ts', 'test/skill-e2e-qa-workflow.test.ts', 'test/helpers/aside-available.ts', 'test/helpers/bootstrap-retention.ts', 'test/helpers/qa-evidence-producer.ts'], + 'qa-only-no-fix': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'test/helpers/qa-only-cleanup.ts', 'test/helpers/qa-checkpoint-evidence.ts', 'test/fixtures/qa-only-observation-public.json', 'test/fixtures/qa-only-stop-cleanup-public.json', 'test/fixtures/qa-only-charter-public.json', 'test/fixtures/qa-only-browser-probe.ts', 'browse/test/fixtures/qa-only.html', 'test/helpers/qa-browser-deadline-evidence.ts', 'test/helpers/read-only-commands.ts', 'bin/gstack-qa-deadline', 'lib/qa-deadline.ts', 'lib/claude-code-windows-job.ts', 'qa-only/**', 'qa/sections/**', 'qa/templates/**', 'scripts/resolvers/aside.ts', 'browse/src/**', 'browse/test/test-server.ts', 'test/skill-e2e-qa-workflow.test.ts', 'test/helpers/aside-available.ts', 'test/helpers/bootstrap-retention.ts', 'test/helpers/qa-evidence-producer.ts'], 'qa-fix-loop': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'bin/gstack-qa-deadline', 'lib/qa-deadline.ts', 'lib/claude-code-windows-job.ts', 'qa/**', 'scripts/resolvers/aside.ts', 'browse/src/**', 'browse/test/test-server.ts', 'test/skill-e2e-qa-workflow.test.ts', - 'test/helpers/aside-available.ts', 'test/fixtures/qa-only-browser-probe.ts', 'test/helpers/bootstrap-retention.ts', 'test/helpers/qa-browser-deadline-evidence.ts', 'test/helpers/qa-checkpoint-evidence.ts', 'test/helpers/qa-evidence-producer.ts', 'test/helpers/qa-only-cleanup.ts'], + 'test/helpers/aside-available.ts', 'test/fixtures/qa-only-browser-probe.ts', 'test/helpers/bootstrap-retention.ts', 'test/helpers/qa-browser-deadline-evidence.ts', 'test/helpers/read-only-commands.ts', 'test/helpers/qa-checkpoint-evidence.ts', 'test/helpers/qa-evidence-producer.ts', 'test/helpers/qa-only-cleanup.ts'], 'qa-bootstrap': ['test/helpers/bootstrap-retention.ts', 'qa/**', 'ship/**', 'test/skill-e2e-qa-workflow.test.ts', - 'scripts/resolvers/testing.ts', 'test/helpers/aside-available.ts', 'test/fixtures/qa-only-browser-probe.ts', 'test/helpers/qa-browser-deadline-evidence.ts', 'test/helpers/qa-checkpoint-evidence.ts', 'test/helpers/qa-evidence-producer.ts', 'test/helpers/qa-only-cleanup.ts'], + 'scripts/resolvers/testing.ts', 'test/helpers/aside-available.ts', 'test/fixtures/qa-only-browser-probe.ts', 'test/helpers/qa-browser-deadline-evidence.ts', 'test/helpers/read-only-commands.ts', 'test/helpers/qa-checkpoint-evidence.ts', 'test/helpers/qa-evidence-producer.ts', 'test/helpers/qa-only-cleanup.ts'], // Review 'review-sql-injection': [ 'review/**', 'test/fixtures/review-eval-vuln.rb', 'test/skill-e2e-review.test.ts', @@ -815,7 +815,7 @@ export const E2E_TOUCHFILES: Record = { 'scripts/resolvers/testing.ts' ], 'ship-docsync-completion': ['bin/gstack-docs-candidate', 'bin/gstack-state-root.sh', 'lib/state-root.ts', 'ship/**', 'document-release/**', 'test/skill-e2e-ship-docsync.test.ts', 'test/helpers/docsync-*.ts', 'test/helpers/read-only-commands.ts', 'test/helpers/session-runner.ts', 'test/helpers/hermetic-env.ts', 'bin/gstack-skill-start', 'bin/gstack-session-kind', 'scripts/resolvers/sections.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/gen-skill-docs.ts', 'scripts/resolvers/testing.ts', 'test/helpers/qa-checkpoint-evidence.ts', 'test/helpers/qa-functional-observer.ts', 'test/helpers/e2e-gate.ts', 'test/helpers/qa-evidence-producer.ts', 'test/helpers/qa-functional-fixture.ts', 'bin/gstack-context-recovery'], - 'ship-docsync-current': ['bin/gstack-docs-candidate', 'bin/gstack-state-root.sh', 'lib/state-root.ts', 'ship/**', 'document-release/**', 'test/skill-e2e-ship-docsync.test.ts', 'test/helpers/docsync-*.ts', 'test/helpers/read-only-commands.ts', 'test/helpers/session-runner.ts', 'test/helpers/hermetic-env.ts', 'bin/gstack-skill-start', 'bin/gstack-session-kind', 'scripts/resolvers/sections.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/gen-skill-docs.ts', 'scripts/resolvers/testing.ts', 'test/helpers/qa-checkpoint-evidence.ts', 'test/helpers/qa-functional-observer.ts', 'test/helpers/e2e-gate.ts', 'test/helpers/qa-evidence-producer.ts', 'test/helpers/qa-functional-fixture.ts', 'bin/gstack-context-recovery'], + 'ship-docsync-current': ['bin/gstack-docs-candidate', 'bin/gstack-state-root.sh', 'lib/state-root.ts', 'ship/**', 'document-release/**', 'test/skill-e2e-ship-docsync.test.ts', 'test/helpers/docsync-*.ts', 'test/helpers/read-only-commands.ts', 'test/helpers/session-runner.ts', 'test/helpers/hermetic-env.ts', 'bin/gstack-skill-start', 'bin/gstack-session-kind', 'scripts/resolvers/sections.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/gen-skill-docs.ts', 'scripts/resolvers/testing.ts', 'test/helpers/qa-checkpoint-evidence.ts', 'test/helpers/qa-functional-observer.ts', 'test/helpers/e2e-gate.ts', 'test/helpers/qa-evidence-producer.ts', 'test/helpers/qa-functional-fixture.ts', 'bin/gstack-context-recovery', 'test/fixtures/docsync-candidate-not-found-public.json'], 'ship-docsync-failure': ['bin/gstack-docs-candidate', 'bin/gstack-state-root.sh', 'lib/state-root.ts', 'ship/**', 'document-release/**', 'test/skill-e2e-ship-docsync.test.ts', 'test/helpers/docsync-*.ts', 'test/helpers/read-only-commands.ts', 'test/helpers/session-runner.ts', 'test/helpers/hermetic-env.ts', 'bin/gstack-skill-start', 'bin/gstack-session-kind', 'scripts/resolvers/sections.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/gen-skill-docs.ts', 'scripts/resolvers/testing.ts', 'test/helpers/qa-checkpoint-evidence.ts', 'test/helpers/qa-functional-observer.ts', 'test/helpers/e2e-gate.ts', 'test/helpers/qa-evidence-producer.ts', 'test/helpers/qa-functional-fixture.ts', 'bin/gstack-context-recovery'], 'ship-docsync-store': ['bin/gstack-docs-candidate', 'bin/gstack-state-root.sh', 'lib/state-root.ts', 'ship/**', 'document-release/**', 'test/skill-e2e-ship-docsync.test.ts', 'test/helpers/docsync-*.ts', 'test/helpers/read-only-commands.ts', 'test/helpers/session-runner.ts', 'test/helpers/hermetic-env.ts', 'bin/gstack-skill-start', 'bin/gstack-session-kind', 'scripts/resolvers/sections.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/gen-skill-docs.ts', 'scripts/resolvers/testing.ts', 'test/helpers/qa-checkpoint-evidence.ts', 'test/helpers/qa-functional-observer.ts', 'test/helpers/e2e-gate.ts', 'test/helpers/qa-evidence-producer.ts', 'test/helpers/qa-functional-fixture.ts', 'bin/gstack-context-recovery'], // #2733 behavioral proof: the JSON contract survives a firing gate inside a diff --git a/test/qa-browser-deadline-evidence.test.ts b/test/qa-browser-deadline-evidence.test.ts index 43b73c851..87d3d36c2 100644 --- a/test/qa-browser-deadline-evidence.test.ts +++ b/test/qa-browser-deadline-evidence.test.ts @@ -6,6 +6,7 @@ import { spawnSync } from 'node:child_process'; import { assertQaBrowserDeadline, assertQaBrowserPreparation, assertQaBrowserCheckpoints, qaDeadlineShellPolicy } from './helpers/qa-browser-deadline-evidence'; import capturedPreparation from './fixtures/qa-only-charter-public.json'; import capturedObservation from './fixtures/qa-only-observation-public.json'; +import capturedStop from './fixtures/qa-only-stop-cleanup-public.json'; const ROOT = path.resolve(import.meta.dir, '..'); const guard = path.join(ROOT, 'bin/gstack-qa-deadline'); @@ -89,6 +90,50 @@ test.each(['cleanup-after-last-probe', 'cleanup-direct-argv', 'probe-after-clean else expect(check).toThrow(scenario === 'probe-after-cleanup' ? 'browser cleanup preceded a probe' : 'follow-up lacks one acknowledged preceding checkpoint'); }); +// PR #3272 Opus gate 39217387697 slice 8 (qa-only-no-fix, head 3986ac5): two guarded probes with one +// checkpoint, then the browser stopped inside the guard. stop is cleanup like closetab; a disguised probe is not. +test.each(['captured', 'unquoted-variable', 'direct-argv', 'closetab-then-stop', 'stop-before-probe', 'disguised-probe', 'other-verb']) + ('guarded "$B" stop after the last probe is cleanup: %s', scenario => { + const f = fixture(); + const events = JSON.parse(JSON.stringify(capturedStop.events) + .replaceAll('__QA_FIXTURE__', f.directory).replaceAll('__QA_GUARD__', guard).replaceAll('__QA_BROWSE__', browse)); + const write = events[2].message.content[0]; + fs.writeFileSync(write.input.file_path, write.input.content); + const stop = events[6].message.content[0]; + const prefix = `bun ${quote(guard)} run ${quote(path.join(f.directory, 'qa-reports/deadline.json'))} -- `; + expect(stop.input.command).toBe(prefix + `bash -c 'B="${browse}"; "$B" stop'`); + const child = ({ + 'unquoted-variable': `bash -c 'B="${browse}"; $B stop'`, + 'direct-argv': quote(browse) + ' stop', + 'disguised-probe': `bash -c 'B="${browse}"; "$B" goto http://127.0.0.1:1/; "$B" stop'`, + 'other-verb': `bash -c 'B="${browse}"; "$B" stop --all'`, + } as Record)[scenario]; + if (child) stop.input.command = prefix + child; + if (scenario === 'closetab-then-stop') { + const closetab = structuredClone(events.slice(6, 8)); + closetab[0].message.content[0].id = closetab[1].message.content[0].tool_use_id = 'toolu_closetab'; + closetab[0].message.content[0].input.command = prefix + `bash -c 'B="${browse}"; "$B" closetab'`; + events.splice(6, 0, ...closetab); + } + if (scenario === 'stop-before-probe') events.splice(4, 0, ...events.splice(6, 2)); + const check = () => assertQaBrowserCheckpoints(events, { directory: f.directory, guard, browse }); + if (['captured', 'unquoted-variable', 'direct-argv', 'closetab-then-stop'].includes(scenario)) expect(check).not.toThrow(); + else expect(check).toThrow(scenario === 'stop-before-probe' ? 'browser cleanup preceded a probe' : 'follow-up lacks one acknowledged preceding checkpoint'); + }); + +test.each([['direct', (b: string) => `${quote(b)} stop`], ['shell variable', (b: string) => `B=${quote(b)}; "$B" stop`]]) + ('browser stop outside the guard is not cleanup: %s', (_, command) => { + const f = fixture(); + f.run(); + f.calls.push({ tool: 'Bash', input: { command: command(browse) }, output: '' }); + expect(() => f.check()).toThrow('QA deadline: '); + }); + +test('the deadline prompt names both accepted cleanup children', () => { + const { policy } = fixture(); + expect(policy.prompt).toContain('exactly "$B" closetab or "$B" stop'); +}); + test('R70 retained public ordering has no completed report Write before its clock/baseline', () => { const f = fixture(); const events = JSON.parse(JSON.stringify(capturedPreparation.events) @@ -148,7 +193,10 @@ test('native literal argv, status, scripts, readiness and artifact bookkeeping a // PR gate run 38969924945 (qa-only-no-fix): after saving fallback screenshots into the owned // directory the prompt names, the agent listed it with this exact spelling. -test.each(['captured-screenshots-listing', 'parent-escape', 'other-subdirectory', 'operator', 'extra-option']) +// GSTA-49: qa-only-no-fix opts into the shared read-only grammar, so pure reads inside the fixture root +// (parent-escape stays inside it, other-subdirectory, extra-option) are accepted; leaving the root or +// composing with a non-read tool is not. +test.each(['captured-screenshots-listing', 'parent-escape', 'other-subdirectory', 'operator', 'extra-option', 'root-escape', 'absolute-outside']) ('listing the owned screenshots directory is bookkeeping; nothing wider is: %s', scenario => { const f = fixture(); f.run(); @@ -159,9 +207,11 @@ test.each(['captured-screenshots-listing', 'parent-escape', 'other-subdirectory' 'other-subdirectory': `ls -la ${quote(path.join(f.directory, 'qa/sections'))}`, operator: `ls -la ${quote(screenshots)}; ${quote(browse)} --version`, 'extra-option': `ls -laR ${quote(screenshots)}`, + 'root-escape': `ls -la ${quote(screenshots + '/../../..')}`, + 'absolute-outside': 'ls -la /etc', }; f.calls.push({ tool: 'Bash', input: { command: commands[scenario as keyof typeof commands] }, output: 'total 0' }); - if (scenario === 'captured-screenshots-listing') expect(f.check()).toEqual({ launchedRuns: 1, completedRuns: 1, refusedRuns: 0, timedOutRuns: 0 }); + if (['captured-screenshots-listing', 'parent-escape', 'other-subdirectory', 'extra-option'].includes(scenario)) expect(f.check()).toEqual({ launchedRuns: 1, completedRuns: 1, refusedRuns: 0, timedOutRuns: 0 }); else expect(() => f.check()).toThrow(scenario === 'operator' ? 'QA deadline: unsupported outer shell composition' : 'QA deadline: untrusted runtime or unguarded command'); }); diff --git a/test/read-only-command-grammar.test.ts b/test/read-only-command-grammar.test.ts index 301901a48..a2011c0b7 100644 --- a/test/read-only-command-grammar.test.ts +++ b/test/read-only-command-grammar.test.ts @@ -11,6 +11,7 @@ import { spawnSync } from 'node:child_process'; import { fixtureDocs } from './helpers/docsync-fixture'; import { docsCommandAllowed, docsCompletedRead, docsNativeInterface, docsToolFailures } from './helpers/docsync-observer'; import { readOnlyCommandAllowed, readOnlyCommandRejection, readOnlyInterfaceText, READ_TOOLS, STDIN_TOOLS } from './helpers/read-only-commands'; +import { assertQaBrowserDeadline, qaDeadlineShellPolicy } from './helpers/qa-browser-deadline-evidence'; import type { SkillTestResult } from './helpers/session-runner'; let base: string; @@ -70,6 +71,7 @@ const ACCEPTED = [ 'wc -l qa-only/SKILL.md; ls qa-only', 'ls docs && wc -l docs/guide.md', 'ls missing || ls docs', 'cd qa && ls sections', 'cd qa-only; wc -l SKILL.md', 'ls docs;', 'echo ----', 'echo -n done', "printf '%s\\n' done", 'printf -- -x', 'cat docs/guide.md; echo ----; cat qa-only/SKILL.md', 'echo done | wc -c', 'sed -n 1,2p docs/guide.md', 'grep -n step qa-only/SKILL.md | sed -n 1,5p', + 'ls docs 2>/dev/null', 'ls missing 2>/dev/null || ls docs', 'cat docs/guide.md 2>&1 | head -1', 'ls missing 2>&1 | wc -l', ]; describe.skipIf(process.platform === 'win32')('accepted pure local reads', () => { @@ -105,7 +107,22 @@ const REJECTED: Array<[string, string, RegExp]> = [ ['write', 'find . -fprintf list.txt %p', /unsupported find primary: -fprintf/], ['redirect', 'cat docs/guide.md > copy.md', /redirect/], ['redirect', 'cat docs/guide.md >> copy.md', /redirect/], - ['redirect', 'ls docs 2>/dev/null', /redirect/], + ['redirect', 'ls docs 2>docs/err.txt', /redirect/], + ['redirect', 'ls docs 2> /dev/null', /redirect/], + ['redirect', 'ls docs 1>/dev/null', /redirect/], + ['redirect', 'ls docs 3>/dev/null', /redirect/], + ['redirect', 'ls docs >/dev/null', /redirect/], + ['redirect', 'ls docs >>/dev/null', /redirect/], + ['redirect', 'ls docs &>/dev/null', /background job or unsupported operator/], + ['redirect', 'ls docs >&docs/out.txt', /redirect/], + ['redirect', 'ls docs 2>&docs/out.txt', /redirect/], + ['redirect', 'ls docs 2>&2', /redirect/], + ['redirect', 'ls docs 2>/dev/tcp/example.com/80', /redirect/], + ['redirect', 'ls docs 2>/dev/nullx', /redirect/], + ['redirect', '2>/dev/null ls docs', /redirect/], + ['redirect', "ls docs '2'>/dev/null", /redirect/], + ['redirect', 'ls docs 2>/dev/null > docs/out.txt', /redirect/], + ['writing pipe', 'ls docs 2>&1 | tee docs/out.txt', /unlisted tool: tee/], ['redirect', 'wc -l < docs/guide.md', /redirect/], ['writing pipe', 'cat docs/guide.md | tee copy.md', /unlisted tool: tee/], ['writing pipe', 'ls | xargs rm', /unlisted tool: xargs/], @@ -271,6 +288,13 @@ describe.skipIf(process.platform === 'win32')('free replays of the captured slip expect(docsCommandAllowed(command, fixture), command).toBe(true); }); + test('docsync accepts discarded stderr on grammar reads (2>&1 into a read filter)', () => { + for (const command of ['ls 2>&1 | head', 'ls -la ' + fixture.home + ' 2>/dev/null; cat README.md 2>&1 | head -5']) + expect(docsCommandAllowed(command, fixture), command).toBe(true); + for (const command of ['ls 2>&1 | sh', 'cat README.md 2>' + path.join(fixture.repo, 'err.txt'), 'cat README.md 1>/dev/null']) + expect(docsCommandAllowed(command, fixture), command).toBe(false); + }); + test('the docsync interface declares the grammar over the fixture home', () => { expect(docsNativeInterface(fixture)).toContain(readOnlyInterfaceText(fixture.home)); }); @@ -323,3 +347,68 @@ describe.skipIf(process.platform === 'win32')('free replays of the captured slip expect(readOnlyCommandRejection(QA_ONLY_SLIP.replaceAll('{dir}', root) + ' > ' + path.join(root, 'out.txt'), scope())).toMatch(/redirect/); }); }); + +// qa-only-no-fix opts in through its deadline fixture: replay the captured slip inside a real guarded attempt. +describe.skipIf(process.platform === 'win32')('qa-only deadline fixture opt-in', () => { + const guard = path.join(path.resolve(import.meta.dir, '..'), 'bin/gstack-qa-deadline'); + const browse = process.execPath; + const quote = (value: string) => "'" + value.replaceAll("'", "'\"'\"'") + "'"; + function attempt(slip: string, slipOutput?: string) { + const started = Date.now(); + const directory = fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), 'qa-only-ro-')); + for (const dir of ['qa/sections', 'qa/templates', 'qa-only/sections', 'qa-reports/screenshots']) fs.mkdirSync(path.join(directory, dir), { recursive: true }); + fs.copyFileSync(path.join(path.resolve(import.meta.dir, '..'), 'qa/sections/browser-setup.md'), path.join(directory, 'qa/sections/browser-setup.md')); + fs.writeFileSync(path.join(directory, 'qa-only/SKILL.md'), '# qa-only\n'); + const policy = qaDeadlineShellPolicy(directory, guard, browse); + const calls: Array<{ tool: string; input: any; output: string }> = []; + const bash = (command: string, output?: string) => { + const result = spawnSync('bash', ['-c', command], { cwd: directory, encoding: 'utf8', timeout: 10000 }); + expect(result.error).toBeUndefined(); + calls.push({ tool: 'Bash', input: { command }, output: output ?? result.stdout + result.stderr }); + }; + bash(['bun', guard, 'start', policy.file, '30'].map(quote).join(' ')); + bash(slip.replaceAll('{dir}', directory), slipOutput); + bash(['bun', guard, 'run', policy.file, '--', browse, '--version'].map(quote).join(' ')); + const check = () => assertQaBrowserDeadline(calls, { directory, guard, browse, started, ended: Date.now() }); + return { directory, policy, check, cleanup: () => fs.rmSync(directory, { recursive: true, force: true }) }; + } + + // Focused Opus 5.5 qa-only-no-fix at 3766eda + this opt-in: the only rejected call discarded stderr. + test('the captured ls; ls; cat 2>/dev/null | head slip passes the deadline assertion', () => { + const run = attempt('ls; ls qa/templates; cat TODOS.md 2>/dev/null | head -20'); + try { expect(run.check()).toMatchObject({ launchedRuns: 1 }); } finally { run.cleanup(); } + }); + + test('the captured wc -l; ls slip passes the deadline assertion and the prompt declares the grammar', () => { + const run = attempt(QA_ONLY_SLIP); + try { + expect(run.check()).toMatchObject({ launchedRuns: 1 }); + expect(run.policy.prompt).toContain(readOnlyInterfaceText(run.directory)); + } finally { run.cleanup(); } + }); + + test.each([ + ['redirect', QA_ONLY_SLIP + ' > {dir}/qa-reports/count.txt'], + ['stderr redirect to a file', 'ls; cat TODOS.md 2>{dir}/qa-reports/err.txt | head -20'], + ['writing pipe', 'wc -l {dir}/qa-only/SKILL.md | tee {dir}/qa-reports/count.txt'], + ['path escape', 'wc -l {dir}/../outside.txt'], + ['unlisted tool', 'awk 1 {dir}/qa-only/SKILL.md'], + ['sed write', 'sed -i s/a/b/ {dir}/qa-only/SKILL.md'], + ])('a %s variant stays outside the fixture interface', (_, command) => { + const run = attempt(command); + try { expect(run.check).toThrow('QA deadline: '); } finally { run.cleanup(); } + }); + + // Second focused Opus run: cat of the installed exploratory section, whose prose names QA_DEADLINE receipts. + test('a grammar read of skill prose that mentions QA_DEADLINE is not a receipt', () => { + const prose = fs.readFileSync(path.join(path.resolve(import.meta.dir, '..'), 'qa-only/sections/exploratory.md'), 'utf8'); + expect(prose).toContain('QA_DEADLINE '); + const run = attempt('cat qa-only/SKILL.md', prose); + try { expect(run.check()).toMatchObject({ launchedRuns: 1 }); } finally { run.cleanup(); } + }); + + test('a grammar read cannot carry a forged deadline receipt', () => { + const run = attempt('cat {dir}/qa-only/SKILL.md', 'QA_DEADLINE {"event":"finished"}\n'); + try { expect(run.check).toThrow('receipt outside trusted guard invocation'); } finally { run.cleanup(); } + }); +});