From 2699b45a868ce9f2226e9ef5bc9ceed29dc7e5f5 Mon Sep 17 00:00:00 2001 From: garrytan Date: Mon, 19 Oct 2026 07:59:07 +0000 Subject: [PATCH] fix(evals): qa-only opts in to the read-only grammar; stderr-to-null forms accepted Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 9 ++ test/docsync-command-grammar.test.ts | 2 +- test/fixtures/docsync-observer-commands.json | 10 ++- test/helpers/qa-browser-deadline-evidence.ts | 8 +- test/helpers/read-only-commands.ts | 8 +- test/helpers/touchfiles-data.ts | 8 +- test/qa-browser-deadline-evidence.test.ts | 9 +- test/read-only-command-grammar.test.ts | 91 +++++++++++++++++++- 8 files changed, 130 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 566320d96..a241f9463 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## [Unreleased] + +**The qa-only deadline fixture accepts pure local reads, so `qa-only-no-fix` no longer fails when the agent checks a skill file with `wc -l …; ls …`.** Opus 5.5's first Bash call in TM1's focused `qa-only-no-fix` run at e03bfeb was `wc -l qa-only/SKILL.md; ls qa-only qa-only/sections qa/sections qa/templates`. The fixture's narrow list rejected it with `QA deadline: unsupported outer shell composition`, though it only read inside the fixture. The fixture now opts into the shared read-only grammar from 1.91.143.0, as the docs-sync fixture did. A focused Opus run with the opt-in then failed on one more read, `ls; ls qa/templates; cat TODOS.md 2>/dev/null | head -20`, so the grammar now also lets a read discard or merge its stderr. + +### Changed +- **`test/helpers/qa-browser-deadline-evidence.ts`:** the deadline check also accepts any command the shared grammar allows inside the fixture directory, and the fixture prompt declares the grammar instead of "no other ls, wc, cat or find Bash calls". Receipts are still trusted only from the guard: a grammar read whose output has a line starting with `QA_DEADLINE ` fails as `receipt outside trusted guard invocation`. A grammar read of skill prose that only mentions QA_DEADLINE receipts mid-line passes, just as the same file read with Read does; the second focused Opus run tripped on `cat qa-only/sections/exploratory.md`. The fixture's existing allowed commands keep the stricter any-occurrence check. +- **Gate integrity, widening:** pure reads inside the fixture root, such as listing `qa/sections` or `ls -laR` of the screenshots directory, now pass where they used to fail. A free replay runs the captured slip inside a real guarded attempt. Reads that leave the root (`…/../../..`, `/etc`), joins with a non-read tool, redirects, `| tee`, `awk` and `sed -i` still fail. No threshold, kind, trial or behavior assertion changes. The `qa-quick`, `qa-only-no-fix`, `qa-fix-loop` and `qa-bootstrap` touchfiles add `test/helpers/read-only-commands.ts`. +- **Gate integrity, widening (shared grammar, GStack Minder-approved):** a read may end with exactly `2>/dev/null` or `2>&1`. Both fixtures that use the grammar accept it: qa-only and docs-sync. Every other redirect still fails, each with its own negative control: `2>` to a file, `2> /dev/null` with a space, `>` and `>>` on stdout, `&>`, `1>/dev/null`, `3>/dev/null`, `>&` and `2>&` to a file, `2>&2`, `2>/dev/tcp/…`, a leading `2>/dev/null`, a quoted `'2'>`, and `| tee`. The main control `ls docs 2>/dev/null` became `ls docs 2>docs/err.txt`. Two existing docs-sync controls only failed because of their stderr redirect, and each got a replacement that still fails: `ls 2>&1 | head` became `ls 2>&1 | sh`. The stored miss `cat {other}/bash-placeholder 2>/dev/null; cat {home}/state/config.yaml` moved to known_good_posix, and its `2>` to a file and `1>/dev/null` variants stay known_bad. The docs-sync prompt carries the updated grammar text, so the next PR gate selects the docs-sync cases. + ## [1.91.144.0] - 2026-10-18 **The paid PR gate now tests Claude Opus 5.5 instead of Fable 5.1. Its first full Opus gate went 132/132 green and recorded $32.33, against Fable's $93.16 mean over ten full gates.** Garry's direction (2026-10-18) is to test each lab's best code model. Calibration run 39157502890 passed all 108 rule cases and 24 judges on one attempt each, with no reused receipts, on the same case set every Fable gate passed. Opus 5.5 sessions cost $30.86 (Fable: $90.49), judges $1.44 and Haiku side calls $0.08; with the unrecorded PTY sessions estimated, the full-gate scale is $37.13 against $106.17. Output tokens fell 37% (thinking 69%) and session time from 11,859 s to 6,159 s. The Claude CLI runs Opus 5.5 at medium effort by default and Fable at high. diff --git a/test/docsync-command-grammar.test.ts b/test/docsync-command-grammar.test.ts index 75c64dbb6..63a6d9942 100644 --- a/test/docsync-command-grammar.test.ts +++ b/test/docsync-command-grammar.test.ts @@ -122,7 +122,7 @@ test('harmless read wrappers do not fail a docs run', () => { 'ls 2>&1', 'git status 2>&1']) { expect(docsToolFailures(nativeResult(command, 'successful tool acknowledgment', 'docs-dispatch'), fixture, [], true)).toEqual([]); } - for (const command of ['git status 2>&1 > out.txt', 'ls 2>&1 | head', `bun ${path.join(fixture.home, 'publish.ts')} 2>&1`, 'ls 2>out.txt']) { + for (const command of ['git status 2>&1 > out.txt', 'ls 2>&1 | sh', `bun ${path.join(fixture.home, 'publish.ts')} 2>&1`, 'ls 2>out.txt']) { expect(docsCommandAllowed(command, fixture, [path.join(fixture.home, 'publish.ts')])).toBe(false); } }); diff --git a/test/fixtures/docsync-observer-commands.json b/test/fixtures/docsync-observer-commands.json index 0ec117873..2899c2ad5 100644 --- a/test/fixtures/docsync-observer-commands.json +++ b/test/fixtures/docsync-observer-commands.json @@ -1,5 +1,5 @@ { - "_comment": "Bash commands from docsync E2E runs and their mutations. {repo} is the fixture repo, {other} a directory outside it, {home} the fixture home. known_good: read-only forms the observer must accept. known_bad: forms it must keep rejecting. known_bad also keeps genuine model misses from PR gate runs as negative controls: the composed cat; cat of ship-docsync-failure 38984977679 (bash#12). The Bash grep of ship-docsync-missing-asset 38983306684 (bash#13) is a pure read inside the fixture home, so the shared read-only grammar (GSTA-49) accepts it in known_good_posix (the grammar resolves POSIX paths only and accepts nothing on Windows), as it does the Opus slips wc -l (39173752116) and ls with a glob (effort study t10); its tee, outside-path, recursive-credential and denied-glob mutations stay in known_bad. The narrow pwd/ls/cat/stat/sha256sum forms only read inside the fixture home, never a Claude config file but settings.json, nor actor state (GSTA-49 tightening; the config dir itself stays listable): cat {home}/cc/.claude.json and cat /etc/hostname are known_bad. Fable 5.1 (PR #3288 E2E 39186278010 and a focused rerun): the grammar join applies only to its own reads, so joined Git reads stay known_bad; a trailing ; on a Git read is a harmless wrapper (known_good); a grep paged through sed -n with a line-range print script and an ls of the Claude config dir are known_good_posix; other sed scripts stay known_bad.", + "_comment": "Bash commands from docsync E2E runs and their mutations. {repo} is the fixture repo, {other} a directory outside it, {home} the fixture home. known_good: read-only forms the observer must accept. known_bad: forms it must keep rejecting. known_bad also keeps genuine model misses from PR gate runs as negative controls. The composed cat; cat of ship-docsync-failure 38984977679 (bash#12) reads only inside the fixture home with stderr discarded, so the grammar's 2>/dev/null widening (GSTA-49, Minder-approved) moves it to known_good_posix; its 2> to a file and 1>/dev/null mutations stay known_bad. The Bash grep of ship-docsync-missing-asset 38983306684 (bash#13) is a pure read inside the fixture home, so the shared read-only grammar (GSTA-49) accepts it in known_good_posix (the grammar resolves POSIX paths only and accepts nothing on Windows), as it does the Opus slips wc -l (39173752116) and ls with a glob (effort study t10); its tee, outside-path, recursive-credential and denied-glob mutations stay in known_bad. The narrow pwd/ls/cat/stat/sha256sum forms only read inside the fixture home, never a Claude config file but settings.json, nor actor state (GSTA-49 tightening; the config dir itself stays listable): cat {home}/cc/.claude.json and cat /etc/hostname are known_bad. Fable 5.1 (PR #3288 E2E 39186278010 and a focused rerun): the grammar join applies only to its own reads, so joined Git reads stay known_bad; a trailing ; on a Git read is a harmless wrapper (known_good); a grep paged through sed -n with a line-range print script and an ls of the Claude config dir are known_good_posix; other sed scripts stay known_bad.", "known_good": [ "git status", "git -C {repo} status", @@ -19,7 +19,8 @@ "wc -l {home}/.claude/skills/gstack/ship/SKILL.md", "ls -la {home}/.claude/skills/gstack/ship/sections {home}/.claude/skills/gstack/document-release {home}/.claude/skills/gstack/document-release/*", "grep -n \"Ship-owned\\|^## \\|^### \\|read-only\\|spawned\" {home}/.claude/skills/gstack/document-release/SKILL.md | sed -n 1,200p", - "cat {home}/candidate.json; echo; ls -la {home}/state {home}/cc {home}/.claude/skills/gstack/document-release {home}/.claude/skills/gstack/ship" + "cat {home}/candidate.json; echo; ls -la {home}/state {home}/cc {home}/.claude/skills/gstack/document-release {home}/.claude/skills/gstack/ship", + "cat {other}/bash-placeholder 2>/dev/null; cat {home}/state/config.yaml" ], "known_bad": [ "git -C {other} status", @@ -39,7 +40,6 @@ "bun arbitrary.ts 2>/dev/null", "git status $EXTRA", "git status\ntrue", - "cat {other}/bash-placeholder 2>/dev/null; cat {home}/state/config.yaml", "grep -n -i \"14.5\\|documentation\\|Step 15\\|Step 16\" {home}/.claude/skills/gstack/ship/SKILL.md | tee {home}/notes.md", "grep -n -i \"14.5\\|documentation\\|Step 15\\|Step 16\" /etc/passwd", "grep -rn apiKey {home}", @@ -49,6 +49,8 @@ "cat /etc/hostname", "git rev-parse HEAD; git status", "sed -n 1w {repo}/out.txt {repo}/README.md", - "sed -i s/a/b/ {repo}/README.md" + "sed -i s/a/b/ {repo}/README.md", + "cat {other}/bash-placeholder 2>{repo}/err.txt; cat {home}/state/config.yaml", + "cat {other}/bash-placeholder 1>/dev/null; cat {home}/state/config.yaml" ] } diff --git a/test/helpers/qa-browser-deadline-evidence.ts b/test/helpers/qa-browser-deadline-evidence.ts index 4b5dced79..46f8e91aa 100644 --- a/test/helpers/qa-browser-deadline-evidence.ts +++ b/test/helpers/qa-browser-deadline-evidence.ts @@ -2,6 +2,7 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import { isDeepStrictEqual } from 'node:util'; import { nativeCalls, readQACheckpointFiles } from './qa-checkpoint-evidence'; +import { readOnlyCommandAllowed, readOnlyInterfaceText } from './read-only-commands'; type Call = { tool: string; input: any; output: string }; type Options = { directory: string; guard: string; browse: string; started: number; ended: number }; @@ -24,7 +25,8 @@ export function qaDeadlineShellPolicy(directory: string, guard: string, browse: const file = path.join(directory, 'qa-reports/deadline.json'); return { allowed, file, prompt: `This native fixture accepts a deliberately narrow executable outer form. Run the BROWSER SETUP Aside readiness bash block verbatim, as its own Bash call (no appended commands). The fallback binary is already supplied. Other than that block, these standalone setup/bookkeeping Bash calls are allowed: ${allowed.slice(1).join('\n')} -Use Read/Glob for every other local inspection (no other ls, wc, cat or find Bash calls) and Write only inside ${path.join(directory, 'qa-reports')}; the report and screenshots directories already exist. Memory files and learning stores outside that directory are not authorized. Write the initial charters and final report to the same caller-owned file ${path.join(directory, 'qa-reports/qa-only-report.md')}, not a separate charter file. +${readOnlyInterfaceText(directory)} +Use Read/Glob for any other local inspection and Write only inside ${path.join(directory, 'qa-reports')}; the report and screenshots directories already exist. Memory files and learning stores outside that directory are not authorized. Write the initial charters and final report to the same caller-owned file ${path.join(directory, 'qa-reports/qa-only-report.md')}, not a separate charter file. Every other Bash call must be exactly one invocation of bun ${quote(guard)}, with literal arguments, no outer assignments, substitutions, globs, pipes, redirects, prefixes, suffixes or shell operators. The exact runtime path ${quote(process.execPath)} may replace bun; no other launcher is accepted. Start exactly once, and only after you have read the workflow sections you need and your Write of the initial charter to ${path.join(directory, 'qa-reports/qa-only-report.md')} has completed; the 30-second deadline begins at start: bun ${quote(guard)} start ${quote(file)} 30 Then use bun ${quote(guard)} status ${quote(file)} or bun ${quote(guard)} run ${quote(file)} -- COMMAND ARGS. Use the absolute guard path directly, not $G. @@ -213,6 +215,10 @@ export function assertQaBrowserDeadline(calls: Call[], options: Options & { expe if ((call.output ?? '').includes('QA_DEADLINE ')) fail('receipt outside trusted guard invocation'); continue; } + if (readOnlyCommandAllowed(command, { root: directory })) { + if ((call.output ?? '').split('\n').some(line => line.startsWith('QA_DEADLINE '))) fail('receipt outside trusted guard invocation'); + continue; + } const outer = literalArgv(command); if (!['bun', process.execPath].includes(outer[0])) fail('untrusted runtime or unguarded command'); const argv = outer.slice(1); diff --git a/test/helpers/read-only-commands.ts b/test/helpers/read-only-commands.ts index c95fe522c..5ec75897d 100644 --- a/test/helpers/read-only-commands.ts +++ b/test/helpers/read-only-commands.ts @@ -11,7 +11,8 @@ * inside the root. Every path operand, path-valued option and unquoted glob * match must resolve physically (symlinks and .. applied) inside the fixture * root and outside its denied paths. - * Rejected: every other tool, redirects, substitutions, variables, tilde and + * Stderr may be discarded with 2>/dev/null or merged with 2>&1 after a word. + * Rejected: every other tool, every other redirect, substitutions, variables, tilde and * brace expansion, backgrounding, multi-line input, and the write, exec, * symlink-following and output-file options of the listed tools. */ @@ -109,7 +110,7 @@ const FIND_PATHS = new Set(['-newer', '-anewer', '-cnewer', '-samefile']); /** Interface text an opted-in fixture declares to its actor. */ export function readOnlyInterfaceText(root: string): string { - return `Shared read-only grammar: Bash also accepts pure local reads whose every path, unquoted glob matches included, resolves inside ${root} with no .., symlink or absolute-path escape: ${Object.keys(READ_TOOLS).join(', ')} (find without -exec, -ok, -delete or -fprint*; sed only as -n with a line-range print script such as sed -n 1,200p; no output-file, symlink-following or external-program options; echo and printf with literal arguments only, no printf -v). Join only these reads with ;, && or ||, pipe only into ${[...STDIN_TOOLS].join(', ')}, and cd only into a directory inside that root. Redirects, tee, substitutions, variables, tilde or brace expansion, background jobs, multi-line commands and every other tool stay outside this grammar.`; + return `Shared read-only grammar: Bash also accepts pure local reads whose every path, unquoted glob matches included, resolves inside ${root} with no .., symlink or absolute-path escape: ${Object.keys(READ_TOOLS).join(', ')} (find without -exec, -ok, -delete or -fprint*; sed only as -n with a line-range print script such as sed -n 1,200p; no output-file, symlink-following or external-program options; echo and printf with literal arguments only, no printf -v). Join only these reads with ;, && or ||, pipe only into ${[...STDIN_TOOLS].join(', ')}, and cd only into a directory inside that root. Redirects other than 2>/dev/null or 2>&1 right after a read's arguments, tee, substitutions, variables, tilde or brace expansion, background jobs, multi-line commands and every other tool stay outside this grammar.`; } export function readOnlyCommandAllowed(command: string, scope: ReadOnlyScope): boolean { @@ -199,6 +200,9 @@ function tokenize(command: string): Token[] | string { continue; } if (c === '$' || c === '`') return 'substitution or variable'; + const discard = c === '>' && word?.text === '2' && !word.quoted && tokens.length > 0 && !('op' in tokens.at(-1)!) + ? /^>(?:\/dev\/null|&1)(?=[ \t;|&]|$)/.exec(command.slice(i)) : null; + if (discard) { word = undefined; i += discard[0].length - 1; continue; } if (c === '>' || c === '<') return 'redirect'; if ('(){}'.includes(c)) return 'subshell, grouping or brace expansion'; if (!word && (c === '~' || c === '#')) return 'tilde expansion or comment'; diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index 3be2e6b7b..4757dc891 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -106,15 +106,15 @@ export const E2E_TOUCHFILES: Record = { // QA (+ test-server dependency). /qa drives Aside first (the resolver) and // the browse binary as fallback (browse/src), so both are deps. - 'qa-quick': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'bin/gstack-qa-deadline', 'lib/qa-deadline.ts', 'lib/claude-code-windows-job.ts', 'qa/**', 'scripts/resolvers/browse.ts', 'scripts/resolvers/aside.ts', 'browse/src/**', 'browse/test/test-server.ts', 'test/skill-e2e-qa-workflow.test.ts', 'test/helpers/aside-available.ts', 'test/fixtures/qa-only-browser-probe.ts', 'test/helpers/bootstrap-retention.ts', 'test/helpers/qa-browser-deadline-evidence.ts', 'test/helpers/qa-checkpoint-evidence.ts', 'test/helpers/qa-evidence-producer.ts', 'test/helpers/qa-only-cleanup.ts'], + 'qa-quick': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'bin/gstack-qa-deadline', 'lib/qa-deadline.ts', 'lib/claude-code-windows-job.ts', 'qa/**', 'scripts/resolvers/browse.ts', 'scripts/resolvers/aside.ts', 'browse/src/**', 'browse/test/test-server.ts', 'test/skill-e2e-qa-workflow.test.ts', 'test/helpers/aside-available.ts', 'test/fixtures/qa-only-browser-probe.ts', 'test/helpers/bootstrap-retention.ts', 'test/helpers/qa-browser-deadline-evidence.ts', 'test/helpers/read-only-commands.ts', 'test/helpers/qa-checkpoint-evidence.ts', 'test/helpers/qa-evidence-producer.ts', 'test/helpers/qa-only-cleanup.ts'], 'qa-b6-static': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'qa/**', 'scripts/resolvers/aside.ts', 'browse/src/**', 'browse/test/test-server.ts', 'test/helpers/llm-judge.ts', 'browse/test/fixtures/qa-eval.html', 'test/fixtures/qa-eval-ground-truth.json', 'test/skill-e2e-qa-bugs.test.ts', 'test/helpers/aside-available.ts'], 'qa-b7-spa': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'qa/**', 'scripts/resolvers/aside.ts', 'browse/src/**', 'browse/test/test-server.ts', 'test/helpers/llm-judge.ts', 'browse/test/fixtures/qa-eval-spa.html', 'test/fixtures/qa-eval-spa-ground-truth.json', 'test/skill-e2e-qa-bugs.test.ts', 'test/helpers/aside-available.ts'], 'qa-b8-checkout': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'qa/**', 'scripts/resolvers/aside.ts', 'browse/src/**', 'browse/test/test-server.ts', 'test/helpers/llm-judge.ts', 'browse/test/fixtures/qa-eval-checkout.html', 'test/fixtures/qa-eval-checkout-ground-truth.json', 'test/skill-e2e-qa-bugs.test.ts', 'test/helpers/aside-available.ts'], - 'qa-only-no-fix': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'test/helpers/qa-only-cleanup.ts', 'test/helpers/qa-checkpoint-evidence.ts', 'test/fixtures/qa-only-observation-public.json', 'test/fixtures/qa-only-charter-public.json', 'test/fixtures/qa-only-browser-probe.ts', 'browse/test/fixtures/qa-only.html', 'test/helpers/qa-browser-deadline-evidence.ts', 'bin/gstack-qa-deadline', 'lib/qa-deadline.ts', 'lib/claude-code-windows-job.ts', 'qa-only/**', 'qa/sections/**', 'qa/templates/**', 'scripts/resolvers/aside.ts', 'browse/src/**', 'browse/test/test-server.ts', 'test/skill-e2e-qa-workflow.test.ts', 'test/helpers/aside-available.ts', 'test/helpers/bootstrap-retention.ts', 'test/helpers/qa-evidence-producer.ts'], + 'qa-only-no-fix': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'test/helpers/qa-only-cleanup.ts', 'test/helpers/qa-checkpoint-evidence.ts', 'test/fixtures/qa-only-observation-public.json', 'test/fixtures/qa-only-charter-public.json', 'test/fixtures/qa-only-browser-probe.ts', 'browse/test/fixtures/qa-only.html', 'test/helpers/qa-browser-deadline-evidence.ts', 'test/helpers/read-only-commands.ts', 'bin/gstack-qa-deadline', 'lib/qa-deadline.ts', 'lib/claude-code-windows-job.ts', 'qa-only/**', 'qa/sections/**', 'qa/templates/**', 'scripts/resolvers/aside.ts', 'browse/src/**', 'browse/test/test-server.ts', 'test/skill-e2e-qa-workflow.test.ts', 'test/helpers/aside-available.ts', 'test/helpers/bootstrap-retention.ts', 'test/helpers/qa-evidence-producer.ts'], 'qa-fix-loop': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'bin/gstack-qa-deadline', 'lib/qa-deadline.ts', 'lib/claude-code-windows-job.ts', 'qa/**', 'scripts/resolvers/aside.ts', 'browse/src/**', 'browse/test/test-server.ts', 'test/skill-e2e-qa-workflow.test.ts', - 'test/helpers/aside-available.ts', 'test/fixtures/qa-only-browser-probe.ts', 'test/helpers/bootstrap-retention.ts', 'test/helpers/qa-browser-deadline-evidence.ts', 'test/helpers/qa-checkpoint-evidence.ts', 'test/helpers/qa-evidence-producer.ts', 'test/helpers/qa-only-cleanup.ts'], + 'test/helpers/aside-available.ts', 'test/fixtures/qa-only-browser-probe.ts', 'test/helpers/bootstrap-retention.ts', 'test/helpers/qa-browser-deadline-evidence.ts', 'test/helpers/read-only-commands.ts', 'test/helpers/qa-checkpoint-evidence.ts', 'test/helpers/qa-evidence-producer.ts', 'test/helpers/qa-only-cleanup.ts'], 'qa-bootstrap': ['test/helpers/bootstrap-retention.ts', 'qa/**', 'ship/**', 'test/skill-e2e-qa-workflow.test.ts', - 'scripts/resolvers/testing.ts', 'test/helpers/aside-available.ts', 'test/fixtures/qa-only-browser-probe.ts', 'test/helpers/qa-browser-deadline-evidence.ts', 'test/helpers/qa-checkpoint-evidence.ts', 'test/helpers/qa-evidence-producer.ts', 'test/helpers/qa-only-cleanup.ts'], + 'scripts/resolvers/testing.ts', 'test/helpers/aside-available.ts', 'test/fixtures/qa-only-browser-probe.ts', 'test/helpers/qa-browser-deadline-evidence.ts', 'test/helpers/read-only-commands.ts', 'test/helpers/qa-checkpoint-evidence.ts', 'test/helpers/qa-evidence-producer.ts', 'test/helpers/qa-only-cleanup.ts'], // Review 'review-sql-injection': [ 'review/**', 'test/fixtures/review-eval-vuln.rb', 'test/skill-e2e-review.test.ts', diff --git a/test/qa-browser-deadline-evidence.test.ts b/test/qa-browser-deadline-evidence.test.ts index 43b73c851..f74246f67 100644 --- a/test/qa-browser-deadline-evidence.test.ts +++ b/test/qa-browser-deadline-evidence.test.ts @@ -148,7 +148,10 @@ test('native literal argv, status, scripts, readiness and artifact bookkeeping a // PR gate run 38969924945 (qa-only-no-fix): after saving fallback screenshots into the owned // directory the prompt names, the agent listed it with this exact spelling. -test.each(['captured-screenshots-listing', 'parent-escape', 'other-subdirectory', 'operator', 'extra-option']) +// GSTA-49: qa-only-no-fix opts into the shared read-only grammar, so pure reads inside the fixture root +// (parent-escape stays inside it, other-subdirectory, extra-option) are accepted; leaving the root or +// composing with a non-read tool is not. +test.each(['captured-screenshots-listing', 'parent-escape', 'other-subdirectory', 'operator', 'extra-option', 'root-escape', 'absolute-outside']) ('listing the owned screenshots directory is bookkeeping; nothing wider is: %s', scenario => { const f = fixture(); f.run(); @@ -159,9 +162,11 @@ test.each(['captured-screenshots-listing', 'parent-escape', 'other-subdirectory' 'other-subdirectory': `ls -la ${quote(path.join(f.directory, 'qa/sections'))}`, operator: `ls -la ${quote(screenshots)}; ${quote(browse)} --version`, 'extra-option': `ls -laR ${quote(screenshots)}`, + 'root-escape': `ls -la ${quote(screenshots + '/../../..')}`, + 'absolute-outside': 'ls -la /etc', }; f.calls.push({ tool: 'Bash', input: { command: commands[scenario as keyof typeof commands] }, output: 'total 0' }); - if (scenario === 'captured-screenshots-listing') expect(f.check()).toEqual({ launchedRuns: 1, completedRuns: 1, refusedRuns: 0, timedOutRuns: 0 }); + if (['captured-screenshots-listing', 'parent-escape', 'other-subdirectory', 'extra-option'].includes(scenario)) expect(f.check()).toEqual({ launchedRuns: 1, completedRuns: 1, refusedRuns: 0, timedOutRuns: 0 }); else expect(() => f.check()).toThrow(scenario === 'operator' ? 'QA deadline: unsupported outer shell composition' : 'QA deadline: untrusted runtime or unguarded command'); }); diff --git a/test/read-only-command-grammar.test.ts b/test/read-only-command-grammar.test.ts index 301901a48..a2011c0b7 100644 --- a/test/read-only-command-grammar.test.ts +++ b/test/read-only-command-grammar.test.ts @@ -11,6 +11,7 @@ import { spawnSync } from 'node:child_process'; import { fixtureDocs } from './helpers/docsync-fixture'; import { docsCommandAllowed, docsCompletedRead, docsNativeInterface, docsToolFailures } from './helpers/docsync-observer'; import { readOnlyCommandAllowed, readOnlyCommandRejection, readOnlyInterfaceText, READ_TOOLS, STDIN_TOOLS } from './helpers/read-only-commands'; +import { assertQaBrowserDeadline, qaDeadlineShellPolicy } from './helpers/qa-browser-deadline-evidence'; import type { SkillTestResult } from './helpers/session-runner'; let base: string; @@ -70,6 +71,7 @@ const ACCEPTED = [ 'wc -l qa-only/SKILL.md; ls qa-only', 'ls docs && wc -l docs/guide.md', 'ls missing || ls docs', 'cd qa && ls sections', 'cd qa-only; wc -l SKILL.md', 'ls docs;', 'echo ----', 'echo -n done', "printf '%s\\n' done", 'printf -- -x', 'cat docs/guide.md; echo ----; cat qa-only/SKILL.md', 'echo done | wc -c', 'sed -n 1,2p docs/guide.md', 'grep -n step qa-only/SKILL.md | sed -n 1,5p', + 'ls docs 2>/dev/null', 'ls missing 2>/dev/null || ls docs', 'cat docs/guide.md 2>&1 | head -1', 'ls missing 2>&1 | wc -l', ]; describe.skipIf(process.platform === 'win32')('accepted pure local reads', () => { @@ -105,7 +107,22 @@ const REJECTED: Array<[string, string, RegExp]> = [ ['write', 'find . -fprintf list.txt %p', /unsupported find primary: -fprintf/], ['redirect', 'cat docs/guide.md > copy.md', /redirect/], ['redirect', 'cat docs/guide.md >> copy.md', /redirect/], - ['redirect', 'ls docs 2>/dev/null', /redirect/], + ['redirect', 'ls docs 2>docs/err.txt', /redirect/], + ['redirect', 'ls docs 2> /dev/null', /redirect/], + ['redirect', 'ls docs 1>/dev/null', /redirect/], + ['redirect', 'ls docs 3>/dev/null', /redirect/], + ['redirect', 'ls docs >/dev/null', /redirect/], + ['redirect', 'ls docs >>/dev/null', /redirect/], + ['redirect', 'ls docs &>/dev/null', /background job or unsupported operator/], + ['redirect', 'ls docs >&docs/out.txt', /redirect/], + ['redirect', 'ls docs 2>&docs/out.txt', /redirect/], + ['redirect', 'ls docs 2>&2', /redirect/], + ['redirect', 'ls docs 2>/dev/tcp/example.com/80', /redirect/], + ['redirect', 'ls docs 2>/dev/nullx', /redirect/], + ['redirect', '2>/dev/null ls docs', /redirect/], + ['redirect', "ls docs '2'>/dev/null", /redirect/], + ['redirect', 'ls docs 2>/dev/null > docs/out.txt', /redirect/], + ['writing pipe', 'ls docs 2>&1 | tee docs/out.txt', /unlisted tool: tee/], ['redirect', 'wc -l < docs/guide.md', /redirect/], ['writing pipe', 'cat docs/guide.md | tee copy.md', /unlisted tool: tee/], ['writing pipe', 'ls | xargs rm', /unlisted tool: xargs/], @@ -271,6 +288,13 @@ describe.skipIf(process.platform === 'win32')('free replays of the captured slip expect(docsCommandAllowed(command, fixture), command).toBe(true); }); + test('docsync accepts discarded stderr on grammar reads (2>&1 into a read filter)', () => { + for (const command of ['ls 2>&1 | head', 'ls -la ' + fixture.home + ' 2>/dev/null; cat README.md 2>&1 | head -5']) + expect(docsCommandAllowed(command, fixture), command).toBe(true); + for (const command of ['ls 2>&1 | sh', 'cat README.md 2>' + path.join(fixture.repo, 'err.txt'), 'cat README.md 1>/dev/null']) + expect(docsCommandAllowed(command, fixture), command).toBe(false); + }); + test('the docsync interface declares the grammar over the fixture home', () => { expect(docsNativeInterface(fixture)).toContain(readOnlyInterfaceText(fixture.home)); }); @@ -323,3 +347,68 @@ describe.skipIf(process.platform === 'win32')('free replays of the captured slip expect(readOnlyCommandRejection(QA_ONLY_SLIP.replaceAll('{dir}', root) + ' > ' + path.join(root, 'out.txt'), scope())).toMatch(/redirect/); }); }); + +// qa-only-no-fix opts in through its deadline fixture: replay the captured slip inside a real guarded attempt. +describe.skipIf(process.platform === 'win32')('qa-only deadline fixture opt-in', () => { + const guard = path.join(path.resolve(import.meta.dir, '..'), 'bin/gstack-qa-deadline'); + const browse = process.execPath; + const quote = (value: string) => "'" + value.replaceAll("'", "'\"'\"'") + "'"; + function attempt(slip: string, slipOutput?: string) { + const started = Date.now(); + const directory = fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), 'qa-only-ro-')); + for (const dir of ['qa/sections', 'qa/templates', 'qa-only/sections', 'qa-reports/screenshots']) fs.mkdirSync(path.join(directory, dir), { recursive: true }); + fs.copyFileSync(path.join(path.resolve(import.meta.dir, '..'), 'qa/sections/browser-setup.md'), path.join(directory, 'qa/sections/browser-setup.md')); + fs.writeFileSync(path.join(directory, 'qa-only/SKILL.md'), '# qa-only\n'); + const policy = qaDeadlineShellPolicy(directory, guard, browse); + const calls: Array<{ tool: string; input: any; output: string }> = []; + const bash = (command: string, output?: string) => { + const result = spawnSync('bash', ['-c', command], { cwd: directory, encoding: 'utf8', timeout: 10000 }); + expect(result.error).toBeUndefined(); + calls.push({ tool: 'Bash', input: { command }, output: output ?? result.stdout + result.stderr }); + }; + bash(['bun', guard, 'start', policy.file, '30'].map(quote).join(' ')); + bash(slip.replaceAll('{dir}', directory), slipOutput); + bash(['bun', guard, 'run', policy.file, '--', browse, '--version'].map(quote).join(' ')); + const check = () => assertQaBrowserDeadline(calls, { directory, guard, browse, started, ended: Date.now() }); + return { directory, policy, check, cleanup: () => fs.rmSync(directory, { recursive: true, force: true }) }; + } + + // Focused Opus 5.5 qa-only-no-fix at 3766eda + this opt-in: the only rejected call discarded stderr. + test('the captured ls; ls; cat 2>/dev/null | head slip passes the deadline assertion', () => { + const run = attempt('ls; ls qa/templates; cat TODOS.md 2>/dev/null | head -20'); + try { expect(run.check()).toMatchObject({ launchedRuns: 1 }); } finally { run.cleanup(); } + }); + + test('the captured wc -l; ls slip passes the deadline assertion and the prompt declares the grammar', () => { + const run = attempt(QA_ONLY_SLIP); + try { + expect(run.check()).toMatchObject({ launchedRuns: 1 }); + expect(run.policy.prompt).toContain(readOnlyInterfaceText(run.directory)); + } finally { run.cleanup(); } + }); + + test.each([ + ['redirect', QA_ONLY_SLIP + ' > {dir}/qa-reports/count.txt'], + ['stderr redirect to a file', 'ls; cat TODOS.md 2>{dir}/qa-reports/err.txt | head -20'], + ['writing pipe', 'wc -l {dir}/qa-only/SKILL.md | tee {dir}/qa-reports/count.txt'], + ['path escape', 'wc -l {dir}/../outside.txt'], + ['unlisted tool', 'awk 1 {dir}/qa-only/SKILL.md'], + ['sed write', 'sed -i s/a/b/ {dir}/qa-only/SKILL.md'], + ])('a %s variant stays outside the fixture interface', (_, command) => { + const run = attempt(command); + try { expect(run.check).toThrow('QA deadline: '); } finally { run.cleanup(); } + }); + + // Second focused Opus run: cat of the installed exploratory section, whose prose names QA_DEADLINE receipts. + test('a grammar read of skill prose that mentions QA_DEADLINE is not a receipt', () => { + const prose = fs.readFileSync(path.join(path.resolve(import.meta.dir, '..'), 'qa-only/sections/exploratory.md'), 'utf8'); + expect(prose).toContain('QA_DEADLINE '); + const run = attempt('cat qa-only/SKILL.md', prose); + try { expect(run.check()).toMatchObject({ launchedRuns: 1 }); } finally { run.cleanup(); } + }); + + test('a grammar read cannot carry a forged deadline receipt', () => { + const run = attempt('cat {dir}/qa-only/SKILL.md', 'QA_DEADLINE {"event":"finished"}\n'); + try { expect(run.check).toThrow('receipt outside trusted guard invocation'); } finally { run.cleanup(); } + }); +});