test(evals): register the qa-only stop-cleanup replay fixture in qa-only-no-fix touchfiles

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed 2026-10-19 08:24:07 +00:00
1 parent c0c199fb35
commit 3918043a27
2 files changed
+2 -2

No files matched your search

+1 -1
View File
@@ -8,7 +8,7 @@
- **`test/helpers/qa-browser-deadline-evidence.ts`:** the deadline check also accepts any command the shared grammar allows inside the fixture directory, and the fixture prompt declares the grammar instead of "no other ls, wc, cat or find Bash calls". Receipts are still trusted only from the guard: a grammar read whose output has a line starting with `QA_DEADLINE ` fails as `receipt outside trusted guard invocation`. A grammar read of skill prose that only mentions QA_DEADLINE receipts mid-line passes, just as the same file read with Read does; the second focused Opus run tripped on `cat qa-only/sections/exploratory.md`. The fixture's existing allowed commands keep the stricter any-occurrence check.
- **Gate integrity, widening:** pure reads inside the fixture root, such as listing `qa/sections` or `ls -laR` of the screenshots directory, now pass where they used to fail. A free replay runs the captured slip inside a real guarded attempt. Reads that leave the root (`…/../../..`, `/etc`), joins with a non-read tool, redirects, `| tee`, `awk` and `sed -i` still fail. No threshold, kind, trial or behavior assertion changes. The `qa-quick`, `qa-only-no-fix`, `qa-fix-loop` and `qa-bootstrap` touchfiles add `test/helpers/read-only-commands.ts`.
- **Gate integrity, widening (shared grammar, GStack Minder-approved):** a read may end with exactly `2>/dev/null` or `2>&1`. Both fixtures that use the grammar accept it: qa-only and docs-sync. Every other redirect still fails, each with its own negative control: `2>` to a file, `2> /dev/null` with a space, `>` and `>>` on stdout, `&>`, `1>/dev/null`, `3>/dev/null`, `>&` and `2>&` to a file, `2>&2`, `2>/dev/tcp/…`, a leading `2>/dev/null`, a quoted `'2'>`, and `| tee`. The main control `ls docs 2>/dev/null` became `ls docs 2>docs/err.txt`. Two existing docs-sync controls only failed because of their stderr redirect, and each got a replacement that still fails: `ls 2>&1 | head` became `ls 2>&1 | sh`. The stored miss `cat {other}/bash-placeholder 2>/dev/null; cat {home}/state/config.yaml` moved to known_good_posix, and its `2>` to a file and `1>/dev/null` variants stay known_bad. The docs-sync prompt carries the updated grammar text, so the next PR gate selects the docs-sync cases.
- **Gate integrity, widening (checkpoint rule):** a guarded child that is exactly `"$B" stop`, `$B stop` or `<browse> stop` now counts as browser cleanup, the same as `closetab`. Cleanup needs no checkpoint and may come only after the last probe. The fixture prompt names both cleanup commands. A free replay of the three guarded calls captured in run 39217387697 (two probes with one checkpoint, then `stop`) passes, and fails on the previous rule with `follow-up lacks one acknowledged preceding checkpoint`. These still fail: `stop` before a probe (`browser cleanup preceded a probe`), a probe disguised as cleanup (`"$B" goto …; "$B" stop`), `stop --all`, and `stop` outside the guard (direct, or through a shell variable).
- **Gate integrity, widening (checkpoint rule):** a guarded child that is exactly `"$B" stop`, `$B stop` or `<browse> stop` now counts as browser cleanup, the same as `closetab`. Cleanup needs no checkpoint and may come only after the last probe. The fixture prompt names both cleanup commands. A free replay of the three guarded calls captured in run 39217387697 (two probes with one checkpoint, then `stop`) passes, and fails on the previous rule with `follow-up lacks one acknowledged preceding checkpoint`. These still fail: `stop` before a probe (`browser cleanup preceded a probe`), a probe disguised as cleanup (`"$B" goto …; "$B" stop`), `stop --all`, and `stop` outside the guard (direct, or through a shell variable). The replay fixture `test/fixtures/qa-only-stop-cleanup-public.json` is registered in the `qa-only-no-fix` touchfiles, next to the existing observation fixture.
## [1.91.144.0] - 2026-10-18
+1 -1
View File
@@ -110,7 +110,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
'qa-b6-static': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'qa/**', 'scripts/resolvers/aside.ts', 'browse/src/**', 'browse/test/test-server.ts', 'test/helpers/llm-judge.ts', 'browse/test/fixtures/qa-eval.html', 'test/fixtures/qa-eval-ground-truth.json', 'test/skill-e2e-qa-bugs.test.ts', 'test/helpers/aside-available.ts'],
'qa-b7-spa': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'qa/**', 'scripts/resolvers/aside.ts', 'browse/src/**', 'browse/test/test-server.ts', 'test/helpers/llm-judge.ts', 'browse/test/fixtures/qa-eval-spa.html', 'test/fixtures/qa-eval-spa-ground-truth.json', 'test/skill-e2e-qa-bugs.test.ts', 'test/helpers/aside-available.ts'],
'qa-b8-checkout': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'qa/**', 'scripts/resolvers/aside.ts', 'browse/src/**', 'browse/test/test-server.ts', 'test/helpers/llm-judge.ts', 'browse/test/fixtures/qa-eval-checkout.html', 'test/fixtures/qa-eval-checkout-ground-truth.json', 'test/skill-e2e-qa-bugs.test.ts', 'test/helpers/aside-available.ts'],
'qa-only-no-fix': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'test/helpers/qa-only-cleanup.ts', 'test/helpers/qa-checkpoint-evidence.ts', 'test/fixtures/qa-only-observation-public.json', 'test/fixtures/qa-only-charter-public.json', 'test/fixtures/qa-only-browser-probe.ts', 'browse/test/fixtures/qa-only.html', 'test/helpers/qa-browser-deadline-evidence.ts', 'test/helpers/read-only-commands.ts', 'bin/gstack-qa-deadline', 'lib/qa-deadline.ts', 'lib/claude-code-windows-job.ts', 'qa-only/**', 'qa/sections/**', 'qa/templates/**', 'scripts/resolvers/aside.ts', 'browse/src/**', 'browse/test/test-server.ts', 'test/skill-e2e-qa-workflow.test.ts', 'test/helpers/aside-available.ts', 'test/helpers/bootstrap-retention.ts', 'test/helpers/qa-evidence-producer.ts'],
'qa-only-no-fix': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'test/helpers/qa-only-cleanup.ts', 'test/helpers/qa-checkpoint-evidence.ts', 'test/fixtures/qa-only-observation-public.json', 'test/fixtures/qa-only-stop-cleanup-public.json', 'test/fixtures/qa-only-charter-public.json', 'test/fixtures/qa-only-browser-probe.ts', 'browse/test/fixtures/qa-only.html', 'test/helpers/qa-browser-deadline-evidence.ts', 'test/helpers/read-only-commands.ts', 'bin/gstack-qa-deadline', 'lib/qa-deadline.ts', 'lib/claude-code-windows-job.ts', 'qa-only/**', 'qa/sections/**', 'qa/templates/**', 'scripts/resolvers/aside.ts', 'browse/src/**', 'browse/test/test-server.ts', 'test/skill-e2e-qa-workflow.test.ts', 'test/helpers/aside-available.ts', 'test/helpers/bootstrap-retention.ts', 'test/helpers/qa-evidence-producer.ts'],
'qa-fix-loop': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'bin/gstack-qa-deadline', 'lib/qa-deadline.ts', 'lib/claude-code-windows-job.ts', 'qa/**', 'scripts/resolvers/aside.ts', 'browse/src/**', 'browse/test/test-server.ts', 'test/skill-e2e-qa-workflow.test.ts',
'test/helpers/aside-available.ts', 'test/fixtures/qa-only-browser-probe.ts', 'test/helpers/bootstrap-retention.ts', 'test/helpers/qa-browser-deadline-evidence.ts', 'test/helpers/read-only-commands.ts', 'test/helpers/qa-checkpoint-evidence.ts', 'test/helpers/qa-evidence-producer.ts', 'test/helpers/qa-only-cleanup.ts'],
'qa-bootstrap': ['test/helpers/bootstrap-retention.ts', 'qa/**', 'ship/**', 'test/skill-e2e-qa-workflow.test.ts',