From 3cccf074adb613b7fa950c5d7f8f55469be0ded7 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Thu, 19 Nov 2026 23:16:23 +0800 Subject: [PATCH] Free-suite CI survives a slow Ubuntu mirror, and four flaky tests and eval checks stop failing (GST-3206) (#4530) Free-suite CI stops failing before any test runs when archive.ubuntu.com is slow, and four tests that failed and then passed in the last day are fixed at their cause. Why: 27 setup failures and 5 flaky reds on 2026-11-19 cost reruns and blocked READY checks across PRs and main. Nothing to do: the first run on each runner image fills the new package cache. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- .github/workflows/free-tests.yml | 102 +++++++++++++++++- changelog.d/flaky-fix-2026-11-19.md | 1 + lib/first-use.ts | 2 +- test/coverage-audit-evidence.test.ts | 61 +++++++++++ test/docsync-command-grammar.test.ts | 26 +++++ test/first-use.test.ts | 14 ++- test/fixtures/coverage-audit-relative-cd.json | 38 +++++++ test/free-tests-workflow-wiring.test.ts | 41 ++++++- test/gstack-lease.test.ts | 8 +- test/helpers/coverage-audit-evidence.ts | 49 ++++++++- test/helpers/free-fixtures-data.ts | 1 + test/helpers/read-only-commands.ts | 3 + test/read-only-command-grammar.test.ts | 8 +- 13 files changed, 338 insertions(+), 16 deletions(-) create mode 100644 changelog.d/flaky-fix-2026-11-19.md create mode 100644 test/fixtures/coverage-audit-relative-cd.json diff --git a/.github/workflows/free-tests.yml b/.github/workflows/free-tests.yml index db8462f9f..364fa056c 100644 --- a/.github/workflows/free-tests.yml +++ b/.github/workflows/free-tests.yml @@ -472,6 +472,32 @@ jobs: with: path: ~/.cache/ms-playwright key: linux-playwright-chromium-${{ steps.playwright.outputs.version }} + # --with-deps fetches about 38 MB of system packages from archive.ubuntu.com + # on every run. On 2026-11-19 that mirror served some runners at 85 to + # 280 kB/s: 17 of 456 first attempts hit the 5-minute limit, and each + # retry then died in seconds on the dpkg lock that the killed attempt's + # apt-get still held (run 44485949701). apt now waits for that lock, and + # the packages one run downloads are cached under a key made from the + # runner's installed package list and the Playwright version, so a later + # run on the same image installs them with dpkg and never touches the mirror. + - name: Make apt wait for its lock and keep the packages it downloads + run: printf 'DPkg::Lock::Timeout "240";\nAPT::Keep-Downloaded-Packages "true";\nBinary::apt::APT::Keep-Downloaded-Packages "true";\n' | sudo tee /etc/apt/apt.conf.d/90gstack-ci > /dev/null + - name: Fingerprint the runner's installed system packages + id: apt_base + env: + PLAYWRIGHT_VERSION: ${{ steps.playwright.outputs.version }} + run: echo "cache_key=linux-apt-debs-$(dpkg-query -W -f='${binary:Package}=${Version}\n' | sha256sum | cut -c1-16)-playwright-$PLAYWRIGHT_VERSION" >> "$GITHUB_OUTPUT" + - name: Restore the system packages an earlier run downloaded + id: apt_cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: ~/.cache/gstack-apt-debs + key: ${{ steps.apt_base.outputs.cache_key }} + - name: Install the cached system packages without the network + id: apt_offline + if: steps.apt_cache.outputs.cache-hit == 'true' + continue-on-error: true + run: sudo dpkg -i "$HOME"/.cache/gstack-apt-debs/*.deb # The cache restores the browser; --with-deps still runs apt for system # libraries. On 2026-10-21 apt stalled on an archive.ubuntu.com fetch and # held free-suite (12) for the whole 20-minute job timeout (run @@ -482,11 +508,17 @@ jobs: id: playwright_install timeout-minutes: 5 continue-on-error: true - run: npx playwright install --with-deps chromium + env: + SYSTEM_PACKAGES: ${{ steps.apt_offline.outcome }} + run: | + if [ "$SYSTEM_PACKAGES" = success ]; then npx playwright install chromium; else npx playwright install --with-deps chromium; fi - name: Retry the Playwright Chromium install once if: steps.playwright_install.outcome == 'failure' timeout-minutes: 5 - run: npx playwright install --with-deps chromium + env: + SYSTEM_PACKAGES: ${{ steps.apt_offline.outcome }} + run: | + if [ "$SYSTEM_PACKAGES" = success ]; then npx playwright install chromium; else npx playwright install --with-deps chromium; fi - name: Configure the bundled Chromium sandbox helper run: | set -euo pipefail @@ -516,6 +548,21 @@ jobs: sudo apparmor_parser -r "$profile" - name: Install Xvfb + X11 utilities + gate tools run: sudo apt-get install -y --no-install-recommends xvfb x11-utils poppler-utils fonts-noto-color-emoji zsh + - name: Keep the system packages this run downloaded + id: apt_keep + if: steps.apt_cache.outputs.cache-hit != 'true' + run: | + mkdir -p ~/.cache/gstack-apt-debs + if compgen -G '/var/cache/apt/archives/*.deb' > /dev/null; then + cp /var/cache/apt/archives/*.deb ~/.cache/gstack-apt-debs/ + echo "save=true" >> "$GITHUB_OUTPUT" + fi + - name: Save the system packages for the next run on this image + if: steps.apt_keep.outputs.save == 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: ~/.cache/gstack-apt-debs + key: ${{ steps.apt_cache.outputs.cache-primary-key }} - name: Configure git identity (tests init temp repos) run: | git config --global user.email "free-tests-ci@gstack.test" @@ -595,6 +642,32 @@ jobs: path: ~/.cache/ms-playwright key: linux-playwright-chromium-${{ steps.playwright.outputs.version }} + # --with-deps fetches about 38 MB of system packages from archive.ubuntu.com + # on every run. On 2026-11-19 that mirror served some runners at 85 to + # 280 kB/s: 17 of 456 first attempts hit the 5-minute limit, and each + # retry then died in seconds on the dpkg lock that the killed attempt's + # apt-get still held (run 44485949701). apt now waits for that lock, and + # the packages one run downloads are cached under a key made from the + # runner's installed package list and the Playwright version, so a later + # run on the same image installs them with dpkg and never touches the mirror. + - name: Make apt wait for its lock and keep the packages it downloads + run: printf 'DPkg::Lock::Timeout "240";\nAPT::Keep-Downloaded-Packages "true";\nBinary::apt::APT::Keep-Downloaded-Packages "true";\n' | sudo tee /etc/apt/apt.conf.d/90gstack-ci > /dev/null + - name: Fingerprint the runner's installed system packages + id: apt_base + env: + PLAYWRIGHT_VERSION: ${{ steps.playwright.outputs.version }} + run: echo "cache_key=linux-apt-debs-$(dpkg-query -W -f='${binary:Package}=${Version}\n' | sha256sum | cut -c1-16)-playwright-$PLAYWRIGHT_VERSION" >> "$GITHUB_OUTPUT" + - name: Restore the system packages an earlier run downloaded + id: apt_cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: ~/.cache/gstack-apt-debs + key: ${{ steps.apt_base.outputs.cache_key }} + - name: Install the cached system packages without the network + id: apt_offline + if: steps.apt_cache.outputs.cache-hit == 'true' + continue-on-error: true + run: sudo dpkg -i "$HOME"/.cache/gstack-apt-debs/*.deb # The cache restores the browser; --with-deps still runs apt for system # libraries. On 2026-10-21 apt stalled on an archive.ubuntu.com fetch and # held free-suite (12) for the whole 20-minute job timeout (run @@ -605,12 +678,18 @@ jobs: id: playwright_install timeout-minutes: 5 continue-on-error: true - run: npx playwright install --with-deps chromium + env: + SYSTEM_PACKAGES: ${{ steps.apt_offline.outcome }} + run: | + if [ "$SYSTEM_PACKAGES" = success ]; then npx playwright install chromium; else npx playwright install --with-deps chromium; fi - name: Retry the Playwright Chromium install once if: steps.playwright_install.outcome == 'failure' timeout-minutes: 5 - run: npx playwright install --with-deps chromium + env: + SYSTEM_PACKAGES: ${{ steps.apt_offline.outcome }} + run: | + if [ "$SYSTEM_PACKAGES" = success ]; then npx playwright install chromium; else npx playwright install --with-deps chromium; fi - name: Configure the bundled Chromium sandbox helper run: | @@ -655,6 +734,21 @@ jobs: # skip instead of running. - name: Install Xvfb + X11 utilities + gate tools run: sudo apt-get install -y --no-install-recommends xvfb x11-utils poppler-utils fonts-noto-color-emoji zsh + - name: Keep the system packages this run downloaded + id: apt_keep + if: steps.apt_cache.outputs.cache-hit != 'true' + run: | + mkdir -p ~/.cache/gstack-apt-debs + if compgen -G '/var/cache/apt/archives/*.deb' > /dev/null; then + cp /var/cache/apt/archives/*.deb ~/.cache/gstack-apt-debs/ + echo "save=true" >> "$GITHUB_OUTPUT" + fi + - name: Save the system packages for the next run on this image + if: steps.apt_keep.outputs.save == 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: ~/.cache/gstack-apt-debs + key: ${{ steps.apt_cache.outputs.cache-primary-key }} - name: Configure git identity (tests init temp repos) run: | diff --git a/changelog.d/flaky-fix-2026-11-19.md b/changelog.d/flaky-fix-2026-11-19.md new file mode 100644 index 000000000..2620208bc --- /dev/null +++ b/changelog.d/flaky-fix-2026-11-19.md @@ -0,0 +1 @@ +- **Free-suite CI no longer fails before any test runs when Ubuntu's package mirror is slow.** Playwright's system packages and the gate tools are cached per runner image and Playwright version, so a run on a known image installs them with dpkg instead of downloading about 38 MB from archive.ubuntu.com, and apt waits for a timed-out attempt's lock instead of failing the retry in seconds. On 2026-11-19, 27 free-suite and quick-tier jobs failed this way. Two flaky free tests are fixed too: `first-use` now lists several user drives in sorted order on every filesystem, and the gstack-lease lock test waits for the owner line instead of reading a file that was created but not yet written. Two eval checkers stop refusing correct reads. The read-only command grammar accepts `\"` inside double quotes, as bash and zsh do, and the coverage audit follows a `cd` into a subfolder followed by `cd ..` in the next call. diff --git a/lib/first-use.ts b/lib/first-use.ts index 3f7ed6a7e..24d4811be 100644 --- a/lib/first-use.ts +++ b/lib/first-use.ts @@ -113,7 +113,7 @@ export function recordRoot(env: NodeJS.ProcessEnv = process.env): Result<{ root: if (env.GSTACK_CAPY !== '1') return { root: path.join(resolveStateRoot(env), 'first-use') }; const drive = path.join(env.HOME || os.homedir(), '.capy', 'drive'); let found: string[] = []; - try { found = fs.readdirSync(drive).filter(d => d.startsWith('user-') && fs.statSync(path.join(drive, d)).isDirectory()); } catch { /* no drive mounted */ } + try { found = fs.readdirSync(drive).filter(d => d.startsWith('user-') && fs.statSync(path.join(drive, d)).isDirectory()).sort(); } catch { /* no drive mounted */ } if (found.length !== 1) return fail('FIRST_USE_NO_PRIVATE_DRIVE', found.length ? `${found.length} user-level drive folders (${found.join(', ')})` : `no user-level drive folder under ${drive}`); return { root: path.join(drive, found[0]!, 'first-use') }; } diff --git a/test/coverage-audit-evidence.test.ts b/test/coverage-audit-evidence.test.ts index 0f9e47281..a27eab3d6 100644 --- a/test/coverage-audit-evidence.test.ts +++ b/test/coverage-audit-evidence.test.ts @@ -21,6 +21,7 @@ import captured_wc_anchor from './fixtures/coverage-audit-wc-anchor.json'; import captured_separator from './fixtures/coverage-audit-separator-collision.json'; import captured_two_file_grep from './fixtures/coverage-audit-two-file-grep.json'; import captured_plan_eng_wc_bytes from './fixtures/coverage-audit-plan-eng-wc-bytes.json'; +import captured_relative_cd from './fixtures/coverage-audit-relative-cd.json'; const clone = (v:T):T => structuredClone(v); const diagram = '```text\nsrc/billing.ts\n├── processPayment: happy path [TESTED]\n└── refundPayment [UNTESTED]\n```'; @@ -1399,3 +1400,63 @@ describe('coverage audit native evidence: zsh abandons the line at an echo ==== expect(reads(command, 'src/billing.ts was read' + error)).toEqual(neither); }); }); + +describe('coverage audit native evidence: cd .. back to the cwd after an earlier cd into review/ (census 44351657692 slice 15)', () => { + const capture = captured_relative_cd, [failed, listed, readBoth] = capture.calls; + const files = { cwd: capture.cwd, source: { path: capture.cwd + '/src/billing.ts', content: capture.files.source }, + tests: { path: capture.cwd + '/test/billing.test.ts', content: capture.files.tests } }; + const both = { sourceRead: true, testsRead: true }, neither = { sourceRead: false, testsRead: false }; + type Call = { toolUseId: string; command: string; isError: boolean; output: string }; + const event = (type: 'assistant' | 'user', content: unknown[], parent: string | null = null) => + ({ type, session_id: capture.sessionId, parent_tool_use_id: parent, message: { role: type, content } }); + const use = (call: Call) => ({ type: 'tool_use', id: call.toolUseId, name: 'Bash', input: { command: call.command } }); + const result = (call: Call) => ({ type: 'tool_result', tool_use_id: call.toolUseId, is_error: call.isError, content: call.output }); + const transcript = (calls: Call[]) => [{ type: 'system', subtype: 'init', session_id: capture.sessionId, cwd: capture.cwd }, + ...calls.flatMap(call => [event('assistant', [use(call)]), event('user', [result(call)])])]; + const reads = (calls: Call[]) => coverageAuditReadEvidence(transcript(calls), files); + const call = (id: string, command: string, output = '', isError = false): Call => ({ toolUseId: id, command, isError, output }); + const nonce = capture.files.source.match(/coverage-read-evidence: (\S+)/)![1]!; + + test('the recorded calls credit both reads and the recorded run passes', () => { + expect(capture.provenance.recordedPassed).toBe(false); + expect(capture.files.source).toBe(fixture.files.source + '\n// coverage-read-evidence: ' + nonce + '\n'); + expect([failed!.isError, listed!.isError, readBoth!.isError]).toEqual([true, false, false]); + expect(listed!.command).toBe(`cd ${capture.cwd}/review; ls -R; wc -c SKILL.md sections/* specialists/* 2>&1`); + expect(readBoth!.command).toBe('cd .. && cat -n src/billing.ts && echo ---- && cat -n test/billing.test.ts'); + expect(reads(capture.calls)).toEqual(both); + expect(coverageAuditVerdict({ exitReason: 'success', browseErrors: [], output: capture.finalOutput, transcript: transcript(capture.calls) } as any, files)) + .toEqual({ ...both, diagram: true, passed: true, failures: [] }); + const deeper = call('toolu_deeper', `cd ${capture.cwd}/review/sections; ls`, 'adversarial.md\n'); + expect(reads([failed!, listed!, deeper, { ...readBoth!, command: readBoth!.command.replace('cd ..', 'cd ../..') }])).toEqual(both); + }); + + test('cd .. gets no credit unless the earlier calls prove the shell sat one level below the cwd', () => { + expect(reads([readBoth!])).toEqual(neither); + expect(reads([failed!, readBoth!])).toEqual(neither); + expect(reads([{ ...listed!, isError: true }, readBoth!])).toEqual(neither); + expect(reads([{ ...listed!, output: `(eval):cd:1: no such file or directory: ${capture.cwd}/review\n` + listed!.output }, readBoth!])).toEqual(neither); + for (const command of [`cd /foreign/review; ls -R`, `cd ${capture.cwd}/../review; ls -R`, `cd review; ls -R`, `cd ${capture.cwd}; ls -R`, + `cd ${capture.cwd}/review; ls -R; cd sections`, `cd ${capture.cwd}/review; pushd sections`, `cd ${capture.cwd}/review; source x.sh`, + `cd ${capture.cwd}/review; ls & cd sections`, `cd ${capture.cwd}/review; ls $(cd sections)`, `cd ${capture.cwd}/review; ls; python3 x.py`, + `cd ${capture.cwd}/review || ls`, `cd "${capture.cwd}/review"/sections/..; ls`]) { + expect(reads([{ ...listed!, command }, readBoth!]), command).toEqual(neither); + } + const moved = call('toolu_moved', `cd ${capture.cwd}/review/sections; ls`, 'adversarial.md\n'); + expect(reads([failed!, listed!, moved, readBoth!])).toEqual(neither); + expect(reads([failed!, listed!, call('toolu_unknown', 'cd sections && ls', 'adversarial.md\n'), readBoth!])).toEqual(neither); + }); + + test('a concurrent or subagent Bash call voids the proved shell; the reads keep their own grammar', () => { + const parallel = [{ type: 'system', subtype: 'init', session_id: capture.sessionId, cwd: capture.cwd }, + event('assistant', [use(listed!)]), event('assistant', [use(readBoth!)]), event('user', [result(listed!)]), event('user', [result(readBoth!)])]; + expect(coverageAuditReadEvidence(parallel, files)).toEqual(neither); + const child = [...transcript([listed!]).slice(0, 3), event('assistant', [{ type: 'tool_use', id: 'toolu_child', name: 'Bash', input: { command: 'cd /tmp' } }], 'toolu_agent'), + ...transcript([readBoth!]).slice(1)]; + expect(coverageAuditReadEvidence(child, files)).toEqual(neither); + for (const command of ['cd .. ; cat -n src/billing.ts; echo ----; cat -n test/billing.test.ts', 'cd ../src && cat -n billing.ts && echo ---- && cat -n ../test/billing.test.ts', + 'cd ~/.. && cat -n src/billing.ts && echo ---- && cat -n test/billing.test.ts', 'cd .. && cat -n src/billing.ts && echo ---- && cat -n test/billing.test.ts; rm -f x']) { + expect(reads([listed!, { ...readBoth!, command }]), command).toEqual(neither); + } + expect(reads([listed!, { ...readBoth!, output: readBoth!.output.replace(nonce, nonce.slice(1) + '0') }])).toMatchObject({ sourceRead: false }); + }); +}); diff --git a/test/docsync-command-grammar.test.ts b/test/docsync-command-grammar.test.ts index ad9743cff..e56bb73a0 100644 --- a/test/docsync-command-grammar.test.ts +++ b/test/docsync-command-grammar.test.ts @@ -333,6 +333,32 @@ test.each(JOINED_READ_LINES)('captured joined read line from %s is a declared re } finally { owned.clean(); } }); +// Gate 44321344888 slice 7 @ 3ecd7168, ship-docsync-failure (claude-opus-5-5): after prepare for docs-a2 the parent checked +// the new prompt and candidate with one grep whose double-quoted pattern ends in an escaped quote, then a tail. The shared +// grammar rejected every escape inside double quotes, so this read failed the run as an undeclared command. +test('captured grep with an escaped double quote inside its pattern is a declared read; other escapes and joins stay refused', () => { + const owned = fixtureDocs('current'); + try { + const actor = path.join(import.meta.dir, 'helpers/docsync-fault-actor.ts'); + const prompt = path.join(owned.home, 'prompt-docs-a2.md'), candidate = path.join(owned.home, 'candidate-docs-a2.json'); + fs.writeFileSync(prompt, 'RETRY_NOTE: x\nAudit id docs-a2\n'); + fs.writeFileSync(candidate, '{"handbook/reference/commands/widget.md.tmpl": "b64635c"}\n'); + const captured = 'grep -n "RETRY_NOTE\\|Audit id\\|widget.md.tmpl\\"" {home}/prompt-docs-a2.md {home}/candidate-docs-a2.json; tail -n 3 {home}/prompt-docs-a2.md'; + const command = captured.replaceAll('{home}', owned.home); + const run = spawnSync('bash', ['-c', command], { cwd: owned.repo, encoding: 'utf8', timeout: 10000 }); + expect(run.status).toBe(0); + expect(run.stdout).toContain('widget.md.tmpl"'); + expect(docsCommandAllowed(command, owned, [actor])).toBe(true); + expect(docsToolFailures(nativeResult(command, run.stdout), owned, [actor])).toEqual([]); + for (const denied of [ + command.replace('widget.md.tmpl\\"', 'widget.md.tmpl\\$HOME'), command.replace('widget.md.tmpl\\"', 'widget.md.tmpl\\\\'), + command.replace('widget.md.tmpl\\"', 'widget.md.tmpl\\`id\\`'), `${command} > ${path.join(owned.home, 'out.md')}`, `${command}; rm -f ${prompt}`, + command.replace(candidate, path.join(owned.home, 'actor-state.json')), command.replace(candidate, '/etc/passwd'), + 'grep -n "widget.md.tmpl\\" README.md', 'grep -n "x\\"" README.md; git add app.ts', + ]) expect(docsCommandAllowed(denied, owned, [actor]), denied).toBe(false); + } finally { owned.clean(); } +}); + test('operand-free tag listings are reads, and every tag write stays refused', () => { for (const listing of ['git tag', 'git tag -l', 'git tag --list', 'git status; git tag; git log --oneline -3']) expect(docsCommandAllowed(listing, fixture), listing).toBe(true); diff --git a/test/first-use.test.ts b/test/first-use.test.ts index 96b2e543d..f8f6dce02 100644 --- a/test/first-use.test.ts +++ b/test/first-use.test.ts @@ -6,7 +6,7 @@ * than silently changing what the check freezes. Runs under a temporary HOME, drive and * record root, serves its links from a local server, and proves the real drive is untouched. */ -import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; +import { afterAll, beforeAll, describe, expect, spyOn, test } from 'bun:test'; import { spawnSync } from 'node:child_process'; import * as fs from 'node:fs'; import * as os from 'node:os'; @@ -152,6 +152,18 @@ describe('where records live', () => { expect(recordRoot({ HOME: home, GSTACK_CAPY: '1' })).toMatchObject({ error: { code: 'FIRST_USE_NO_PRIVATE_DRIVE', detail: '2 user-level drive folders (user-ada-example, user-bo-example)' } }); expect(recordRoot({ HOME: home, GSTACK_STATE_ROOT: path.join(home, 'state') })).toEqual({ root: path.join(home, 'state', 'first-use') }); }); + + test('several user drives are named in sorted order whatever order the filesystem lists them', () => { + const home = fresh('home'); + for (const d of ['user-ada-example', 'user-bo-example']) fs.mkdirSync(path.join(home, '.capy', 'drive', d), { recursive: true }); + const readdir = fs.readdirSync; + const spy = spyOn(fs, 'readdirSync').mockImplementation(((p: fs.PathLike, o?: unknown) => (readdir(p, o as undefined) as string[]).slice().sort().reverse()) as typeof fs.readdirSync); + try { + expect(recordRoot({ HOME: home, GSTACK_CAPY: '1' })).toMatchObject({ error: { code: 'FIRST_USE_NO_PRIVATE_DRIVE', detail: '2 user-level drive folders (user-ada-example, user-bo-example)' } }); + } finally { + spy.mockRestore(); + } + }); }); describe('the gate', () => { diff --git a/test/fixtures/coverage-audit-relative-cd.json b/test/fixtures/coverage-audit-relative-cd.json new file mode 100644 index 000000000..6e26bce78 --- /dev/null +++ b/test/fixtures/coverage-audit-relative-cd.json @@ -0,0 +1,38 @@ +{ + "provenance": { + "source": "https://github.com/garrytan/gstack/actions/runs/44351657692 eval slice 15 (main 2767d229, full census), review-coverage-audit (claude-opus-5-5); the three parent Bash calls and their results, byte-exact (the three Read calls between the second and third, of review skill files, are omitted)", + "recordedPassed": false, + "recordedFailures": [ + "missing successful source-file read", + "missing successful test-file read" + ], + "note": "The first call cd-ed into /review and failed at a zsh echo ====== separator; the second cd-ed there again and listed it. Claude Code keeps a call's final cwd, so the third call, cd .. && cat -n src/billing.ts && echo ---- && cat -n test/billing.test.ts, read the owned files from the recorded cwd. The checker only accepted a leading cd to the absolute recorded cwd, so it credited neither read." + }, + "cwd": "/home/runner/.cache/gstack-paid-shard-dT3Otw/tmp/skill-e2e-review-coverage-v7u3iN", + "sessionId": "ed8701ac-c025-448b-a772-b867d220639e", + "calls": [ + { + "toolUseId": "toolu_01Hzqn9sssjPsqahBCq2JxFM", + "command": "cd /home/runner/.cache/gstack-paid-shard-dT3Otw/tmp/skill-e2e-review-coverage-v7u3iN/review; cat SKILL.md; echo ======; cat sections/review-army.md; echo =====; cat specialists/testing.md", + "isError": true, + "output": "Exit code 1\n---\nname: review\npreamble-tier: 4\nversion: 1.0.0\ndescription: \"Use when you review a diff before it lands: local changes, a branch or a PR. Use for: review my changes, look over my diff, review this pr. (gstack)\"\nallowed-tools:\n - Bash\n - Read\n - Edit\n - Write\n - Grep\n - Glob\n - Agent\n - AskUserQuestion\n - WebSearch\ntriggers:\n - review my changes\n - look over my diff\n - review this pr\n - code review\n - check my diff\n - pre-landing review\ncontract:\n version: 1\n items:\n - id: specialists-dispatched\n kind: selected_specialists_dispatched\n marker: \"Dispatching N specialists:\"\n - id: adversarial-pass\n kind: agent_dispatch\n match: Think like an attacker and a chaos engineer\n - id: review-report\n kind: report_labels\n heading: Pre-Landing Review\n labels:\n - Gate edits\n---\n\n\n\n\n## Done means\n\nFinish only when each of these exists (contract v1):\n\n- `specialists-dispatched`: the specialist selection printed (`Dispatching N specialists:` ...), then one Agent dispatch per specialist it names (none when a small diff skips them).\n- `adversarial-pass`: an Agent dispatch for the adversarial pass, its prompt including \"Think like an attacker and a chaos engineer\".\n- `review-report`: a `Pre-Landing Review` with Gate edits filled in (`none: ` where the skill's own stop rule leaves one empty).\n\n`~/.claude/skills/gstack/bin/gstack-skill-contract check` reads them from the transcript (docs/skill-routing-and-contracts.md). For an item that truly does not apply, run `~/.claude/skills/gstack/bin/gstack-skill-contract waive --reason \"\"` (this session only; it counts as a miss).\n\n## Step 0: Detect platform and base branch\n\nFirst, detect the git hosting platform from the remote URL:\n\n```bash\ngit remote get-url origin 2>/dev/null\n```\n\n- If the URL contains \"github.com\" → platform is **GitHub**\n- If the URL contains \"gitlab\" → platform is **GitLab**\n- Otherwise, check CLI availability:\n - `gh auth status 2>/dev/null` succeeds → platform is **GitHub** (covers GitHub Enterprise)\n - `glab auth status 2>/dev/null` succeeds → platform is **GitLab** (covers self-hosted)\n - Neither → **unknown** (use git-native commands only)\n\nDetermine which branch this PR/MR targets, or the repo's default branch if no\nPR/MR exists. Use the result as \"the base branch\" in all subsequent steps.\n\n**If GitHub:**\n1. `gh pr view --json baseRefName -q .baseRefName` — if succeeds, use it\n2. `gh repo view --json defaultBranchRef -q .defaultBranchRef.name` — if succeeds, use it\n\n**If GitLab:**\n1. `glab mr view -F json 2>/dev/null` and extract the `target_branch` field — if succeeds, use it\n2. `glab repo view -F json 2>/dev/null` and extract the `default_branch` field — if succeeds, use it\n\n**Git-native fallback (if unknown platform, or CLI commands fail):**\n1. `git symbolic-ref refs/remotes/origin/HEAD 2>/dev/null | sed 's|refs/remotes/origin/||'`\n2. If that fails: `git rev-parse --verify origin/main 2>/dev/null` → use `main`\n3. If that fails: `git rev-parse --verify origin/master 2>/dev/null` → use `master`\n\nIf all fail, fall back to `main`.\n\nPrint the detected base branch name. In every subsequent `git diff`, `git log`,\n`git fetch`, `git merge`, and PR/MR creation command, substitute the detected\nbranch name wherever the instructions say \"the base branch\" or ``.\n\n---\n\n# Pre-Landing PR Review\n\nReview the branch diff against the base for structural issues tests miss.\n\n---\n\n## Section index — Read each section when its situation applies\n\nThis skill is a decision-tree skeleton. The steps below point to on-demand\nsections. Read a section in full before doing its step; do not work from memory.\n\n| When | Read this section |\n|------|-------------------|\n| leaving Step 1.5 | `sections/plan-completion.md` |\n| Select surfaces and read QA methods | Inline in [Step 4](#step-4-critical-pass-core-review); setup and probes run in Step 4.7 |\n| dispatching the Review Army specialists and merging their findings after the critical pass (Step 4.5) | `sections/review-army.md` |\n| running the always-on native adversarial review before fixes (Step 4.8) | `sections/adversarial.md` |\n| reusing explicitly skipped shared-code advice (Step 5.0) | `sections/shared-code-reuse.md` |\n\n---\n\n## Step 1: Check branch\n\n1. Run `git branch --show-current`.\n2. If on the base branch, output: **\"Nothing to review — you're on the base branch or have no changes against it.\"** and stop.\n3. Run `git fetch origin --quiet && echo \"BASE_REFRESH: fresh\" || echo \"BASE_REFRESH: stale $(git rev-parse --short origin/)\"`, then `DIFF_BASE=$(git merge-base origin/ HEAD) && git diff \"$DIFF_BASE\" --stat`. If no diff, output the same message and stop. `stale` is not an empty diff: continue; report `Base coverage: stale at `.\n4. Run `~/.claude/skills/gstack/bin/gstack-contributor-mode det\n\n... [19960 characters truncated] ...\n\n`.\n\n**Identity before merge or suppression:** Use installed `sharedLibsFingerprint`, never model-generated hashes. Send literal JSON on stdin (actual paths/symbol; keep the quoted delimiter), not interpolated shell code:\n\n```bash\nGSTACK_SHARED_LIB=~/.claude/skills/gstack/lib/review-evidence.ts\nbun --no-env-file --config=/dev/null -e 'const { sharedLibsFingerprint } = await import(process.argv[1]); const value = sharedLibsFingerprint(JSON.parse(await Bun.stdin.text())); if (!value) process.exit(1); console.log(value);' \"$GSTACK_SHARED_LIB\" <<'GSTACK_SHARED_LIBS_JSON'\n{\"evidence_paths\":[\"src/caller-a.ts\",\"src/caller-b.ts\"],\"helper_target\":{\"path\":\"src/shared.ts\",\"symbol\":\"sharedHelper\"}}\nGSTACK_SHARED_LIBS_JSON\n```\n\nUse the returned fingerprint; malformed/missing metadata requires revalidation. Real defects follow Fix-First independently: advice or a prior Skip cannot suppress, downgrade or replace them, even with a shared supplied fingerprint.\n\nCore findings use the confidence gates below; Step 4.6 applies its specialist gates.\nUse CRITICAL/INFORMATIONAL labels in the finding format.\nStep 5.8 combines these finding lines with the checklist's action groups.\n\n## Confidence Calibration\n\nVerify evidence first, then score every finding (1-10) and apply its display rule.\n\n### Pre-emit verification gate\n\n1. **Quote the specific code line:** file:line and verbatim text. For a missing field,\n quote its class definition; for a nullable value, its initialization; for a race, both sides.\n2. For framework-generated symbols, read and quote their generating metaclass,\n descriptor, ORM Meta block, migration, decorator or schema. Missing literal\n names in the class body or grep results do not prove absence.\n3. **If you cannot quote the motivating line(s), the finding is unverified.**\n Force its confidence to 4-5: use 4 for appendix-only reporting, or 5 only when\n the finding belongs in the main report with the medium-confidence caveat below.\n Never invent speculative confidence 7+.\n\n| Score | Meaning | Display rule |\n|-------|---------|-------------|\n| 9-10 | Specific code verifies a concrete bug or exploit. | Show normally |\n| 7-8 | High-confidence pattern match; very likely correct. | Show normally |\n| 5-6 | Moderate; could be a false positive. | Show with caveat: \"Medium confidence, verify this is actually an issue\" |\n| 3-4 | Suspicious but may be fine. | Suppress from main report. Include in appendix only. |\n| 1-2 | Speculation. | Only report a suspected release-blocking catastrophe (widespread data loss, total outage or system-wide compromise); label it CRITICAL and explicitly speculative. |\n\n**Finding format:**\n\n`[CRITICAL|INFORMATIONAL] (confidence: N/10) file:line — description`\n\nExample:\n`[CRITICAL] (confidence: 9/10) user.rb:42 — SQL injection via string interpolation`\n\n**Calibration learning:** If the user confirms a reported finding scored < 7 is\nreal, log it per Operational Self-Improvement.\n\n### TODOS cross-reference\n\nIf root `TODOS.md` exists, report closed items as \"This PR addresses TODO: \".\nFlag new TODOs as informational and cite related items; otherwise skip silently.\n\n### Documentation staleness check\n\nRead root `.md` files. When changed code affects a documented feature or workflow\nbut its doc was not updated, flag an INFORMATIONAL finding naming the file and\naffected behavior. Propose `/document-release` for the parent's decision, never a\ncritical finding or another writer during collection. Skip silently if no docs exist.\n\n---\n\n> **STOP.** Before dispatching the Review Army specialists and merging their findings after the critical pass (Step 4.5), Read `~/.claude/skills/gstack/review/sections/review-army.md` and execute it\n> in full. Do not work from memory — that section is the source of truth for this step.\n\n---\n\n### Step 4.7: Exploratory QA (before Fix-First)\n\nOnly the parent runs report-only discovery.\nNever overwrite another run's reports. Batch only independent Reads.\n\n**1. Set the charter and isolation.**\nReuse Step 4's surfaces and completed Reads. Finish any missing scope/method Reads before charters, setup or probes; do not repeat completed Reads.\nWrite the Charter and complete the shared isolation/permission preflight before setup.\n\n**2. Check readiness and list required checks.**\nFor browsers, Read QA's `sections/browser-setup.md` and follow its report-only rules.\nReuse setup only with verified tools/session/target/ownership; otherwise recheck.\nNever install, import cookies or bootstrap tests. Functional-only skips browser setup.\n- Smoke: 5 minutes/12 probes, one success and the riskiest changed failure/edge.\n Required even for small diffs or missing plans/servers.\n- Required: plan commands/assertions, listed separately. Other ideas are optional, untested.\n\n**3. Run smoke and plan checks.**\nFollow the shared Probe loop for smoke checks and replays until the smoke limit.\nThen run required plan checks and revalidation, even after smoke expires, using the same procedure bu" + }, + { + "toolUseId": "toolu_011cpmVj5bo6i2W6jDpYHHU3", + "command": "cd /home/runner/.cache/gstack-paid-shard-dT3Otw/tmp/skill-e2e-review-coverage-v7u3iN/review; ls -R; wc -c SKILL.md sections/* specialists/* 2>&1", + "isError": false, + "output": ".:\nSKILL.md\nSKILL.md.tmpl\nTODOS-format.md\nchecklist.md\ndesign-checklist.md\ngate-integrity.md\ngreptile-triage.md\nsections\nspecialists\n\n./sections:\nadversarial.md\nadversarial.md.tmpl\nmanifest.json\nplan-completion.md\nplan-completion.md.tmpl\npreamble.md\nreview-army.md\nreview-army.md.tmpl\nshared-code-reuse.md\nshared-code-reuse.md.tmpl\n\n./specialists:\napi-contract.md\ndata-migration.md\nmaintainability.md\nperformance.md\nred-team.md\nsecurity.md\nsimplification.md\nstate-machines.md\ntargeted.md\ntesting.md\n 52633 SKILL.md\n 23929 sections/adversarial.md\n 21 sections/adversarial.md.tmpl\n 1373 sections/manifest.json\n 17186 sections/plan-completion.md\n 299 sections/plan-completion.md.tmpl\n 23282 sections/preamble.md\n 18261 sections/review-army.md\n 16 sections/review-army.md.tmpl\n 2342 sections/shared-code-reuse.md\n 22 sections/shared-code-reuse.md.tmpl\n 2357 specialists/api-contract.md\n 2306 specialists/data-migration.md\n 2616 specialists/maintainability.md\n 2618 specialists/performance.md\n 2258 specialists/red-team.md\n 4060 specialists/security.md\n 4421 specialists/simplification.md\n 3748 specialists/state-machines.md\n 3309 specialists/targeted.md\n 9163 specialists/testing.md\n176220 total" + }, + { + "toolUseId": "toolu_015UemgddjS7qg4bZSbVVGvF", + "command": "cd .. && cat -n src/billing.ts && echo ---- && cat -n test/billing.test.ts", + "isError": false, + "output": " 1\t\n 2\texport function processPayment(amount: number, currency: string) {\n 3\t if (amount <= 0) throw new Error('Invalid amount');\n 4\t if (currency !== 'USD' && currency !== 'EUR') throw new Error('Unsupported currency');\n 5\t return { status: 'success', amount, currency };\n 6\t}\n 7\t\n 8\texport function refundPayment(paymentId: string, reason: string) {\n 9\t if (!paymentId) throw new Error('Payment ID required');\n 10\t if (!reason) throw new Error('Reason required');\n 11\t return { status: 'refunded', paymentId, reason };\n 12\t}\n 13\t\n 14\t// coverage-read-evidence: 1b7e293f-bb26-4289-9b75-958c7613d49b\n----\n 1\t\n 2\timport { describe, test, expect } from 'vitest';\n 3\timport { processPayment } from '../src/billing';\n 4\t\n 5\tdescribe('processPayment', () => {\n 6\t test('processes valid payment', () => {\n 7\t const result = processPayment(100, 'USD');\n 8\t expect(result.status).toBe('success');\n 9\t });\n 10\t // GAP: no test for invalid amount\n 11\t // GAP: no test for unsupported currency\n 12\t // GAP: refundPayment not tested at all\n 13\t});\n 14\t\n 15\t// coverage-read-evidence: 59099786-6c84-45d7-a782-2be271408a36" + } + ], + "finalOutput": "I applied only the testing specialist checklist to the two files. I didn't dispatch other specialists, run the adversarial pass, or change any files.\n\n```text\nsrc/billing.ts\nprocessPayment(amount, currency)\n├── valid USD happy path, processPayment(100, 'USD') returns status 'success' [OK]\n├── valid EUR path, currency === 'EUR' returns success (billing.ts:4) [GAP]\n├── amount <= 0 guard throws 'Invalid amount' (billing.ts:3) [GAP]\n│ └── boundary values amount = 0 and negative amounts [GAP]\n└── unsupported currency guard throws 'Unsupported currency' (billing.ts:4) [GAP]\nrefundPayment(paymentId, reason)\n├── refund success path returns status 'refunded' (billing.ts:11) [GAP]\n├── empty paymentId guard throws 'Payment ID required' (billing.ts:9) [GAP]\n└── empty reason guard throws 'Reason required' (billing.ts:10) [GAP]\n (refundPayment is not imported in test/billing.test.ts:3)\nLegend: [OK] tested [GAP] no test\n```\n\n{\"severity\":\"INFORMATIONAL\",\"confidence\":10,\"path\":\"src/billing.ts\",\"line\":8,\"category\":\"testing\",\"summary\":\"refundPayment is a public function with no tests and is not imported in test/billing.test.ts; its success path and both guards are uncovered\",\"fix\":\"Import refundPayment and test the success return plus both throw paths\",\"fingerprint\":\"src/billing.ts:8:testing\",\"specialist\":\"testing\",\"test_stub\":\"describe('refundPayment', () => { test('refunds valid payment', () => { expect(refundPayment('pay_1','dup').status).toBe('refunded'); }); test('rejects missing paymentId', () => { expect(() => refundPayment('','dup')).toThrow('Payment ID required'); }); test('rejects missing reason', () => { expect(() => refundPayment('pay_1','')).toThrow('Reason required'); }); });\"}\n{\"severity\":\"INFORMATIONAL\",\"confidence\":10,\"path\":\"src/billing.ts\",\"line\":3,\"category\":\"testing\",\"summary\":\"The guard `if (amount <= 0) throw new Error('Invalid amount')` has no negative-path test, and the boundary values 0 and negative amounts are untested\",\"fix\":\"Add toThrow tests for amou", + "files": { + "source": "\nexport function processPayment(amount: number, currency: string) {\n if (amount <= 0) throw new Error('Invalid amount');\n if (currency !== 'USD' && currency !== 'EUR') throw new Error('Unsupported currency');\n return { status: 'success', amount, currency };\n}\n\nexport function refundPayment(paymentId: string, reason: string) {\n if (!paymentId) throw new Error('Payment ID required');\n if (!reason) throw new Error('Reason required');\n return { status: 'refunded', paymentId, reason };\n}\n\n// coverage-read-evidence: 1b7e293f-bb26-4289-9b75-958c7613d49b\n", + "tests": "\nimport { describe, test, expect } from 'vitest';\nimport { processPayment } from '../src/billing';\n\ndescribe('processPayment', () => {\n test('processes valid payment', () => {\n const result = processPayment(100, 'USD');\n expect(result.status).toBe('success');\n });\n // GAP: no test for invalid amount\n // GAP: no test for unsupported currency\n // GAP: refundPayment not tested at all\n});\n\n// coverage-read-evidence: 59099786-6c84-45d7-a782-2be271408a36\n" + } +} diff --git a/test/free-tests-workflow-wiring.test.ts b/test/free-tests-workflow-wiring.test.ts index 59d970f7c..7d7a0eceb 100644 --- a/test/free-tests-workflow-wiring.test.ts +++ b/test/free-tests-workflow-wiring.test.ts @@ -234,14 +234,51 @@ describe('free-tests workflow wiring', () => { expect(retry, name).toBe(install + 1); for (const step of [steps[install], steps[retry]]) { expect(step['timeout-minutes'], name).toBe(5); - expect(step.run, name).toBe('npx playwright install --with-deps chromium'); + expect(step.run.trim(), name).toBe('if [ "$SYSTEM_PACKAGES" = success ]; then npx playwright install chromium; else npx playwright install --with-deps chromium; fi'); + expect(step.env, name).toEqual({ SYSTEM_PACKAGES: '${{ steps.apt_offline.outcome }}' }); } expect(steps[install]['continue-on-error'], name).toBe(true); - const failOpen = name === 'free-suite' ? [steps[install], steps.find((step: any) => step.name === 'Upload shard JUnit reports')] : [steps[install]]; + const offline = steps.find((step: any) => step.id === 'apt_offline'); + const failOpen = name === 'free-suite' ? [offline, steps[install], steps.find((step: any) => step.name === 'Upload shard JUnit reports')] : [offline, steps[install]]; expect(steps.filter((step: any) => step['continue-on-error']), name).toEqual(failOpen); } }); + test('system packages are cached per runner image and Playwright version, and apt waits for a killed attempt\'s lock (2026-11-19, run 44485949701)', () => { + const jobs = (Bun.YAML.parse(source) as any).jobs; + for (const name of ['free-suite', 'quick-bun-default']) { + const steps = jobs[name].steps; + const at = (pred: (step: any) => boolean) => steps.findIndex(pred); + const config = at((step: any) => step.run?.includes('/etc/apt/apt.conf.d/90gstack-ci')); + const base = at((step: any) => step.id === 'apt_base'); + const restore = at((step: any) => step.id === 'apt_cache'); + const offline = at((step: any) => step.id === 'apt_offline'); + const install = at((step: any) => step.id === 'playwright_install'); + const xvfb = at((step: any) => step.name === 'Install Xvfb + X11 utilities + gate tools'); + const keep = at((step: any) => step.id === 'apt_keep'); + const save = keep + 1; + expect(steps[config].run, name).toContain('DPkg::Lock::Timeout "240";'); + expect(steps[config].run, name).toContain('APT::Keep-Downloaded-Packages "true";'); + expect(steps[base].run, name).toContain("dpkg-query -W -f='${binary:Package}=${Version}\\n' | sha256sum"); + expect(steps[restore].uses, name).toStartWith('actions/cache/restore@'); + expect(steps[base].env, name).toEqual({ PLAYWRIGHT_VERSION: '${{ steps.playwright.outputs.version }}' }); + expect(steps[base].run, name).toContain('echo "cache_key=linux-apt-debs-$('); + expect(steps[base].run, name).toContain(')-playwright-$PLAYWRIGHT_VERSION" >> "$GITHUB_OUTPUT"'); + expect(steps[restore].with, name).toEqual({ path: '~/.cache/gstack-apt-debs', key: '${{ steps.apt_base.outputs.cache_key }}' }); + expect(steps[offline].if, name).toBe("steps.apt_cache.outputs.cache-hit == 'true'"); + expect(steps[offline].run, name).toBe('sudo dpkg -i "$HOME"/.cache/gstack-apt-debs/*.deb'); + expect(steps[keep].if, name).toBe("steps.apt_cache.outputs.cache-hit != 'true'"); + expect(steps[keep].run, name).toContain('cp /var/cache/apt/archives/*.deb ~/.cache/gstack-apt-debs/'); + expect(steps[save].uses, name).toStartWith('actions/cache/save@'); + expect(steps[save].if, name).toBe("steps.apt_keep.outputs.save == 'true'"); + expect(steps[save].with, name).toEqual({ path: '~/.cache/gstack-apt-debs', key: '${{ steps.apt_cache.outputs.cache-primary-key }}' }); + expect([config, base, restore, offline, install], name).toEqual([config, base, restore, offline, install].sort((a, b) => a - b)); + expect(config, name).toBeGreaterThan(-1); + expect(xvfb, name).toBeGreaterThan(install); + expect(keep, name).toBe(xvfb + 1); + } + }); + test('test jobs give PR and dispatch runs the local main that push runs get from checkout (#3237)', () => { const jobs = (Bun.YAML.parse(source) as any).jobs; for (const name of ['free-suite', 'quick-bun-default']) { diff --git a/test/gstack-lease.test.ts b/test/gstack-lease.test.ts index cf86ed56d..dd5ac972f 100644 --- a/test/gstack-lease.test.ts +++ b/test/gstack-lease.test.ts @@ -281,14 +281,16 @@ describe('gstack-lease', () => { const { dir, leases } = checkout(); const [first, second] = [sleeper(), sleeper()]; // The first touch holds the lock across its prune (find runs under the lock) until the test releases it. + // Its owner write also waits a moment after the O_EXCL create, so the owner file is briefly empty. const release = path.join(dir, 'release'); - const slow = bashEnv(dir, `find() { case "$*" in *'+120'*) until [ -e "${release}" ]; do sleep 0.05; done ;; esac; command find "$@"; }\n`); + const slow = bashEnv(dir, `find() { case "$*" in *'+120'*) until [ -e "${release}" ]; do sleep 0.05; done ;; esac; command find "$@"; }\nprintf() { [ -n "\${_gl_dir:-}" ] && sleep 0.3; builtin printf "$@"; }\n`); const holder = spawn('bash', [LEASE, 'touch', '--root', dir, '--pid', String(first)], { env: { ...baseEnv, BASH_ENV: slow }, stdio: 'ignore' }); children.push(holder); const exited = new Promise<number | null>((resolve) => holder.on('exit', resolve)); const owner = path.join(leases, '.lock', 'owner'); - for (let i = 0; i < 500 && !fs.existsSync(owner); i++) await Bun.sleep(20); - expect(fs.readFileSync(owner, 'utf8')).toMatch(new RegExp(`^${holder.pid} `)); + const ownerLine = () => { try { return fs.readFileSync(owner, 'utf8'); } catch { return ''; } }; + for (let i = 0; i < 500 && !ownerLine().endsWith('\n'); i++) await Bun.sleep(20); + expect(ownerLine()).toMatch(new RegExp(`^${holder.pid} `)); const blocked = run(['touch', '--root', dir, '--pid', String(second)], { GSTACK_LEASE_LOCK_TRIES: '3' }); fs.writeFileSync(release, ''); expect(blocked.status, blocked.stdout + blocked.stderr).toBe(3); diff --git a/test/helpers/coverage-audit-evidence.ts b/test/helpers/coverage-audit-evidence.ts index a813d98f0..4a8e5b34e 100644 --- a/test/helpers/coverage-audit-evidence.ts +++ b/test/helpers/coverage-audit-evidence.ts @@ -24,7 +24,8 @@ type Paths = typeof posix; interface Owned { path: string; content: string } /** One displayed stdout unit: a literal echo, a complete owned file or unknown neighboring output. */ type Piece = { kind: 'label'; text: string } | ({ kind: 'owned' } & Owned) | { kind: 'unknown'; min: number; max: number }; -interface ShellScope { cwd: string; path: Paths; owned: Owned[] } +/** shell: the session shell's working directory before this call, when the transcript proves it (sessionShell). */ +interface ShellScope { cwd: string; path: Paths; owned: Owned[]; shell?: string } /** Words and the && ; || | operators of a single-line command. Quotes stay in * their word; expansion, substitution, redirection (except discarded stderr), @@ -165,11 +166,17 @@ function commandPieces(command: unknown, scope: ShellScope): Piece[] | undefined // One leading assertion of the recorded cwd, followed by one uniform list. // `cd CWD;` runs the list whether or not cd succeeds, exactly like an all-; // list, so only it may precede an && chain. Other mixes (`cd CWD && A; B`) - // could run later relative reads elsewhere and stay unsupported. + // could run later relative reads elsewhere and stay unsupported. A bare `cd ..` (or ../..) also asserts the + // recorded cwd when the proved session shell sits that many levels below it (census 44351657692: `cd <cwd>/review` + // in an earlier call, then `cd .. && cat -n src/billing.ts && ...`); .. names skip CDPATH and tilde expansion, and + // only && keeps the reads from running elsewhere when that cd fails. if (lists[0]!.length === 1 && lists[0]![0]![0] === 'cd') { const [, target, ...rest] = lists[0]![0]!; const [assertion, ...chain] = separators, kinds = new Set(chain.filter(separator => separator !== '||')); - if (rest.length || target === undefined || literal(target) !== scope.cwd || lists.length < 2 || assertion === '||' || + const value = target === undefined ? undefined : literal(target); + const asserted = value === scope.cwd || (assertion === '&&' && scope.shell !== undefined && /^\.\.(?:\/\.\.)*\/?$/.test(value ?? '') && + scope.path.resolve(scope.shell, value!) === scope.cwd); + if (rest.length || !asserted || lists.length < 2 || assertion === '||' || kinds.size > 1 || (assertion !== ';' && kinds.size && !kinds.has(assertion!))) return undefined; lists.shift(); separators.shift(); } @@ -316,6 +323,26 @@ function pieceReads(pieces: Piece[] | undefined, output: unknown, whole: boolean return whole && (pending || !/^\n*$/.test(out.slice(cursor))) ? new Set() : credited; } +const SHELL_NEUTRAL = new Set(['ls', 'cat', 'wc', 'head', 'tail', 'grep', 'sed', 'pwd', 'echo', 'stat', 'file', 'du', 'sort', 'uniq', 'cut', 'nl', 'git']); +/** + * The session shell's cwd after one main-session Bash call, or undefined when the transcript cannot prove it. Claude + * Code keeps a Bash call's final working directory for the next call. Two forms are tracked: a line of plain read + * commands with no quote, expansion, grouping, redirect (but discarded or merged stderr) or background job leaves the + * cwd as it was; the same line after a leading `cd <absolute dir inside the recorded cwd>` then ; or && moves it + * there, only when the call succeeded and printed no cd error. Anything else makes the cwd unknown. + */ +function sessionShell(command: unknown, result: Record<string, any>, before: string | undefined, scope: Pick<ShellScope, 'cwd' | 'path'>): string | undefined { + if (typeof command !== 'string') return undefined; + const lead = /^cd[ \t]+(\S+)[ \t]*(?:;|&&)[ \t]*/.exec(command.trim()); + const rest = (lead ? command.trim().slice(lead[0].length) : command.trim()).replace(/[ \t]2>(?:&1|\/dev\/null)(?=[ \t;|]|$)/g, ''); + if (!rest.split(/;|&&|\|\|?/).every(part => /^[\w.\/*?[\]:=,+@%^ \t-]*$/.test(part) && SHELL_NEUTRAL.has(part.trim().split(/[ \t]+/)[0]!))) return undefined; + if (!lead) return before; + const target = literal(lead[1]!), relative = target === undefined ? '' : scope.path.relative(scope.cwd, target); + if (target === undefined || !scope.path.isAbsolute(target) || scope.path.resolve(target) !== target || !relative || + relative.startsWith('..') || scope.path.isAbsolute(relative)) return undefined; + return (result.is_error !== undefined && result.is_error !== false) || /\bcd:/.test(outputText(result.content)) ? undefined : target; +} + function delivered(output: unknown, expected: string): boolean { const text = outputText(output); const body = normalized(expected); @@ -341,7 +368,13 @@ export function coverageAuditReadEvidence(transcript: unknown[], files: Coverage const session = init[0].session_id, uses = new Map<string, { name: string; input: Record<string, any> } | null>(), results = new Set<string>(); + // The proved session shell cwd, lost (undefined) on any Bash call whose effect on it is unknown: a subagent's, or + // one issued while another Bash call was still outstanding, since their execution order is not recorded. + let shell: string | undefined = files.cwd; + const outstanding = new Set<string>(), concurrent = new Set<string>(); for (const e of transcript.slice(transcript.indexOf(init[0]) + 1)) { + if (object(e) && e.parent_tool_use_id !== null && e.parent_tool_use_id !== undefined && object(e.message) && Array.isArray(e.message.content) && + e.message.content.some(b => object(b) && b.type === 'tool_use' && b.name === 'Bash')) shell = undefined; if (!object(e) || e.session_id !== session || (e.parent_tool_use_id !== null && e.parent_tool_use_id !== undefined) || !object(e.message) || !Array.isArray(e.message.content)) continue; for (const b of e.message.content) { @@ -351,12 +384,20 @@ export function coverageAuditReadEvidence(transcript: unknown[], files: Coverage if (e.type === 'assistant' && b.type === 'tool_use' && typeof b.id === 'string') { if (uses.has(b.id)) return { sourceRead: false, testsRead: false }; uses.set(b.id, e.message.role === 'assistant' && object(b.input) ? { name: b.name, input: b.input } : null); + if (b.name === 'Bash') { + if (outstanding.size) [...outstanding, b.id].forEach(id => concurrent.add(id)); + outstanding.add(b.id); + } } else if (e.type === 'user' && b.type === 'tool_result' && typeof b.tool_use_id === 'string') { if (results.has(b.tool_use_id)) return { sourceRead: false, testsRead: false }; results.add(b.tool_use_id); const u = e.message.role === 'user' ? uses.get(b.tool_use_id) : undefined; + outstanding.delete(b.tool_use_id); + const before = concurrent.has(b.tool_use_id) ? undefined : shell; + if (!u) shell = undefined; + else if (u.name === 'Bash') shell = concurrent.has(b.tool_use_id) ? undefined : sessionShell(u.input.command, b, before, { cwd: files.cwd, path }); if (!u || (b.is_error !== undefined && b.is_error !== false)) continue; - const credited = u.name === 'Bash' ? shellReads(u.input.command, b.content, { cwd: files.cwd, path, owned: [files.source, files.tests] }) : undefined; + const credited = u.name === 'Bash' ? shellReads(u.input.command, b.content, { cwd: files.cwd, path, owned: [files.source, files.tests], shell: before }) : undefined; for (const [key, file] of [['sourceRead', files.source], ['testsRead', files.tests]] as const) { if (u.name === 'Read' ? typeof u.input.file_path === 'string' && path.resolve(files.cwd, u.input.file_path) === file.path && delivered(b.content, file.content) diff --git a/test/helpers/free-fixtures-data.ts b/test/helpers/free-fixtures-data.ts index d7a662186..b68163af5 100644 --- a/test/helpers/free-fixtures-data.ts +++ b/test/helpers/free-fixtures-data.ts @@ -69,6 +69,7 @@ export const FREE_FIXTURES: Readonly<Record<string, readonly string[]>> = { 'test/fixtures/coverage-audit-cd-chain.json': ['test/coverage-audit-evidence.test.ts'], 'test/fixtures/coverage-audit-parallel-column.json': ['test/coverage-audit-evidence.test.ts'], 'test/fixtures/coverage-audit-plan-eng-wc-bytes.json': ['test/coverage-audit-evidence.test.ts'], + 'test/fixtures/coverage-audit-relative-cd.json': ['test/coverage-audit-evidence.test.ts'], 'test/fixtures/coverage-audit-separator-collision.json': ['test/coverage-audit-evidence.test.ts'], 'test/fixtures/coverage-audit-summary.json': ['test/coverage-audit-evidence.test.ts'], 'test/fixtures/coverage-audit-two-file-grep.json': ['test/coverage-audit-evidence.test.ts'], diff --git a/test/helpers/read-only-commands.ts b/test/helpers/read-only-commands.ts index e31cf2c37..21d31d1cd 100644 --- a/test/helpers/read-only-commands.ts +++ b/test/helpers/read-only-commands.ts @@ -187,6 +187,9 @@ function tokenize(command: string): Token[] | string { let text = ''; for (i++; i < command.length && command[i] !== '"'; i++) { if (command[i] === '$' || command[i] === '`') return 'substitution or variable'; + // Both bash and zsh read \" inside double quotes as one literal quote character (gate 44321344888: the parent + // grepped for widget.md.tmpl" with "...widget.md.tmpl\""); every other escape there stays rejected. + if (command[i] === '\\' && command[i + 1] === '"') { text += '"'; i++; continue; } if (command[i] === '\\' && /[$`"\\]/.test(command[i + 1] ?? '"')) return 'escape inside double quotes'; text += command[i]; } diff --git a/test/read-only-command-grammar.test.ts b/test/read-only-command-grammar.test.ts index 29527d22e..2e1d90043 100644 --- a/test/read-only-command-grammar.test.ts +++ b/test/read-only-command-grammar.test.ts @@ -64,7 +64,7 @@ const ACCEPTED = [ 'tail -n +2 docs/guide.md', 'cat -n docs/guide.md', 'cat guide-link.md', 'stat docs/guide.md', 'stat -c %s docs/guide.md', 'file docs/guide.md', 'du -sh qa', 'du -d 1 .', 'find . -name "*.md" -type f', "find qa -maxdepth 2 -name '*.md' -print", 'find . -path ./private -prune -o -type f -print', 'grep -n step qa-only/SKILL.md', 'rg -n step qa-only', 'grep -rn template qa', - 'grep -n -i "step\\|report" qa-only/SKILL.md', 'grep -c -e step -e report qa-only/SKILL.md', 'sort docs/guide.md', + 'grep -n -i "step\\|report" qa-only/SKILL.md', 'grep -n "step\\|report\\"" qa-only/SKILL.md', 'grep -c -e step -e report qa-only/SKILL.md', 'sort docs/guide.md', 'sort -u docs/guide.md', 'uniq -c docs/guide.md', 'cut -c1-3 docs/guide.md', 'nl -ba docs/guide.md', 'sha256sum docs/guide.md', 'md5sum docs/guide.md qa-only/SKILL.md', 'sha1sum docs/guide.md', 'sha512sum --tag docs/guide.md', 'ls qa | sha256sum', 'readlink -f guide-link.md', 'realpath docs/guide.md', 'cat docs/guide.md | sort | uniq -c', 'ls qa | wc -l', 'grep -rn step qa-only | head -5', 'find qa -type f | sort', @@ -200,6 +200,12 @@ const REJECTED: Array<[string, string, RegExp]> = [ ['shell syntax', 'ls\nrm docs/guide.md', /multi-line/], ['shell syntax', 'cat docs/{guide,other}.md', /brace expansion/], ['shell syntax', '(ls)', /subshell/], + ['escape', 'grep -n "a\\$X" docs/guide.md', /escape inside double quotes/], + ['escape', 'grep -n "a\\\\" docs/guide.md', /escape inside double quotes/], + ['escape', 'grep -n "a\\' + '\x60' + 'id' + '\x60' + '" docs/guide.md', /escape inside double quotes/], + ['escape', 'grep -n "a\\" docs/guide.md', /unterminated quote/], + ['redirect', 'grep -n "a\\"" docs/guide.md > copy.md', /redirect/], + ['write', 'grep -n "a\\"" docs/guide.md; rm docs/guide.md', /unlisted tool: rm/], ['shell syntax', 'tail -f docs/guide.md', /unsupported tail option: -f/], ];