From c5c2f28994c7fb3815384bbc576febdd55b6f070 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sat, 17 Oct 2026 18:07:35 -0700 Subject: [PATCH 1/4] v1.91.128.0 ci(seed): weekly standing recorder for the Linux and Windows duration seeds (#3253) * ci(seed): weekly standing recorder for the Linux and Windows duration seeds (W4) free-durations-record.yml runs Mondays 04:00 UTC (before test-health) and on dispatch. Linux records on ubicloud-standard-8, the free-suite shard runner, with scripts/ubicloud/setup-free-suite.sh; Windows records through windows-free-tests.yml's record_durations path (now also workflow_call). When either seed's file set drifts from main, the propose job force-updates bot/free-duration-seeds, opens or refreshes one seed PR stating the seed ratchet result and the files that failed while recording, and dispatches the free lanes on it. A red file keeps its duration; an interrupted recording uploads no seed (this also stops the Windows dispatch from uploading a stale copy), and more than 5 red files on either OS proposes nothing. The bot never edits the allowlist. open_pr=false on dispatch records and reports only. The workflow is classified free-lane only in scripts/test-pr-profile.ts, with a non-behavioral HARNESS_VERSION decision. Intentional pin change: test/windows-native-workflows.test.ts now expects workflow_call and the complete-recording guard on the Windows record job. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * ci(seed): keep maintainer commits on the open seed PR, publish only from main, survive long ratchet reports /review on #3253 (gpt-6.1-sol): a refresh force-pushed over maintainer repairs on the seed PR, a dispatch from a feature branch would publish that branch's commits, and grep | head under pipefail could abort the step on a long ratchet report. The refresh now leaves an open seed PR alone when it holds any commit besides the bot's one refresh commit and pushes with an exact lease; only main publishes; the report is cut with sed. Each has a functional test that fails on the old workflow. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(changelog): unreleased entry for the weekly duration-seed recorder Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * chore: stamp v1.91.128.0 Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- .github/workflows/free-durations-record.yml | 172 +++++++++++++++++++ .github/workflows/windows-free-tests.yml | 25 ++- CHANGELOG.md | 11 ++ VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- docs/TESTING_INTERNALS.md | 26 ++- package.json | 2 +- scripts/harness-version.json | 2 +- scripts/test-pr-profile.ts | 1 + test/free-durations-record-workflow.test.ts | 177 ++++++++++++++++++++ test/windows-native-workflows.test.ts | 11 +- 11 files changed, 413 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/free-durations-record.yml create mode 100644 test/free-durations-record-workflow.test.ts diff --git a/.github/workflows/free-durations-record.yml b/.github/workflows/free-durations-record.yml new file mode 100644 index 000000000..848be5dfc --- /dev/null +++ b/.github/workflows/free-durations-record.yml @@ -0,0 +1,172 @@ +name: Free-suite duration seeds + +# Weekly standing recorder for the two free-suite duration seeds that the +# shard packers read: scripts/free-test-durations.json (Linux, recorded on the +# same ubicloud-standard-8 runner as the free-suite shards) and +# scripts/free-test-durations-windows.json (windows-latest, through the +# record_durations path of windows-free-tests.yml). When either recording adds +# or drops a test file, it force-updates one bot branch and opens or refreshes +# a seed PR. A file that fails while recording turns its job red but keeps its +# duration (a red file is still a real cost) and is named in the PR body; an +# interrupted recording, or more than 5 red files on either OS, proposes +# nothing (docs/TESTING_INTERNALS.md#free-suite-duration-seed). + +on: + schedule: + - cron: '0 4 * * 1' # Monday 04:00 UTC, before the 10:00 test-health run + workflow_dispatch: + inputs: + open_pr: + description: Push the bot branch and open or refresh the seed PR (false records and reports only) + type: boolean + default: true + +concurrency: + group: free-durations-record + cancel-in-progress: false + +permissions: + contents: read + +jobs: + linux: + runs-on: ubicloud-standard-8 + timeout-minutes: 45 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - name: Prepare the same inputs as a free-suite shard + run: | + bash scripts/ubicloud/setup-free-suite.sh + echo "$HOME/.bun/bin" >> "$GITHUB_PATH" + - name: Time every free file alone + id: record + env: + GSTACK_EXPECT_BINARIES: '1' + run: | + set -o pipefail + xvfb-run -a bun run test:free --record-durations 2>&1 | tee "$RUNNER_TEMP/record.log" + - name: Keep the recording only when every file was timed + id: seed + if: always() && steps.record.outcome != 'skipped' + run: | + grep -q 'wrote [0-9]* durations to' "$RUNNER_TEMP/record.log" || { echo "::error::The recording was interrupted; no Linux seed to upload."; exit 1; } + mkdir -p "$RUNNER_TEMP/seed" + cp scripts/free-test-durations.json "$RUNNER_TEMP/seed/free-test-durations.json" + sed -n '/failed while recording:/,$p' "$RUNNER_TEMP/record.log" | sed 's/\r$//' | { grep -E '^ ✗ [^ ]+$' || true; } | sed 's/^ ✗ //' > "$RUNNER_TEMP/seed/record-failures.txt" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + if: always() + with: + name: linux-record-durations-logs + path: .context/free-test-logs/gstack-free-test-*.log + include-hidden-files: true + if-no-files-found: ignore + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + if: always() && steps.seed.outcome == 'success' + with: + name: free-test-durations + path: ${{ runner.temp }}/seed/ + if-no-files-found: error + + windows: + uses: ./.github/workflows/windows-free-tests.yml + with: + record_durations: true + + propose: + needs: [linux, windows] + if: ${{ !cancelled() }} + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: write + pull-requests: write + actions: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + fetch-depth: 0 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.4.2 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + name: free-test-durations + path: ${{ runner.temp }}/linux + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + name: free-test-durations-windows + path: ${{ runner.temp }}/windows + - name: Open or refresh the seed PR when either seed's file set changed + env: + GH_TOKEN: ${{ github.token }} + OPEN_PR: ${{ github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || inputs.open_pr) }} + run: | + set -euo pipefail + linux_failed=$(wc -l < "$RUNNER_TEMP/linux/record-failures.txt") + windows_failed=$(wc -l < "$RUNNER_TEMP/windows/record-failures.txt") + if [ "$linux_failed" -gt 5 ] || [ "$windows_failed" -gt 5 ]; then + echo "::error::$linux_failed Linux and $windows_failed Windows files failed while recording; more than 5 on either OS means a broken tree, so no seed is proposed." + exit 1 + fi + failures() { if [ -s "$1" ]; then paste -sd, "$1" | sed 's/,/, /g'; else echo none; fi; } + drift() { # committed recorded -> " " + jq -n --slurpfile old "$1" --slurpfile new "$2" \ + '[($new[0].durations|keys) - ($old[0].durations|keys), ($old[0].durations|keys) - ($new[0].durations|keys)] | map(length) | join(" ")' + } + read -r linux_added linux_removed <<<"$(drift scripts/free-test-durations.json "$RUNNER_TEMP/linux/free-test-durations.json" | tr -d '"')" + read -r windows_added windows_removed <<<"$(drift scripts/free-test-durations-windows.json "$RUNNER_TEMP/windows/free-test-durations-windows.json" | tr -d '"')" + echo "Linux seed: +$linux_added -$linux_removed files; Windows seed: +$windows_added -$windows_removed files" | tee -a "$GITHUB_STEP_SUMMARY" + if [ "$((linux_added + linux_removed + windows_added + windows_removed))" -eq 0 ]; then + echo "Both seeds cover the current test files; no PR." | tee -a "$GITHUB_STEP_SUMMARY" + exit 0 + fi + cp "$RUNNER_TEMP/linux/free-test-durations.json" scripts/free-test-durations.json + cp "$RUNNER_TEMP/windows/free-test-durations-windows.json" scripts/free-test-durations-windows.json + if ratchet=$(GSTACK_FREE_SEED_BASE="$(git rev-parse HEAD)" bun test test/free-seed-ratchet.test.ts 2>&1); then + ratchet_line="passes" + else + ratchet_line="FAILS; a maintainer must split the named file or add an annotated allowlist entry on this branch before merge:"$'\n\n'"\`\`\`"$'\n'"$(printf '%s\n' "$ratchet" | sed -n -E '/Free seed ratchet|^- /p' | sed -n '1,20p')"$'\n'"\`\`\`" + fi + failed_line="Linux: $(failures "$RUNNER_TEMP/linux/record-failures.txt"); Windows: $(failures "$RUNNER_TEMP/windows/record-failures.txt")" + echo "Seed growth ratchet: $ratchet_line" >> "$GITHUB_STEP_SUMMARY" + echo "Failed while recording: $failed_line" | tee -a "$GITHUB_STEP_SUMMARY" + if [ "$OPEN_PR" != true ]; then + echo "Report only (open_pr=false, or a ref other than main): no branch pushed." | tee -a "$GITHUB_STEP_SUMMARY" + exit 0 + fi + branch=bot/free-duration-seeds + subject="test(seed): refresh the free-suite duration seeds from the weekly recording" + number=$(gh pr list --head "$branch" --state open --json number -q '.[0].number // empty') + remote_head= + if git fetch -q origin "+refs/heads/$branch:refs/remotes/origin/$branch" 2>/dev/null; then + remote_head=$(git rev-parse "origin/$branch") + fi + if [ -n "$number" ] && [ -n "$remote_head" ] && [ "$(git log --format=%s "HEAD..$remote_head")" != "$subject" ]; then + echo "::warning::Seed PR #$number has commits besides the bot's one refresh commit; this run leaves them in place. Merge or close that PR and the next run refreshes the seeds." | tee -a "$GITHUB_STEP_SUMMARY" + exit 0 + fi + git switch -C "$branch" + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add scripts/free-test-durations.json scripts/free-test-durations-windows.json + git commit -m "$subject" + git push --force-with-lease="refs/heads/$branch:$remote_head" origin "HEAD:refs/heads/$branch" + cat > pr-body.md <&1 | tee "$RUNNER_TEMP/record.log" - name: Name the recording as the Windows seed - if: always() + id: seed + if: always() && steps.record.outcome != 'skipped' shell: bash - run: cp scripts/free-test-durations.json "$RUNNER_TEMP/free-test-durations-windows.json" + run: | + grep -q 'wrote [0-9]* durations to' "$RUNNER_TEMP/record.log" || { echo "::error::The recording was interrupted; no Windows seed to upload."; exit 1; } + cp scripts/free-test-durations.json "$RUNNER_TEMP/free-test-durations-windows.json" + sed -n '/failed while recording:/,$p' "$RUNNER_TEMP/record.log" | sed 's/\r$//' | { grep -E '^ ✗ [^ ]+$' || true; } | sed 's/^ ✗ //' > "$RUNNER_TEMP/record-failures.txt" - name: Upload full recording logs if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 @@ -251,8 +264,10 @@ jobs: include-hidden-files: true if-no-files-found: ignore - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 - if: always() + if: always() && steps.seed.outcome == 'success' with: name: free-test-durations-windows - path: ${{ runner.temp }}/free-test-durations-windows.json + path: | + ${{ runner.temp }}/free-test-durations-windows.json + ${{ runner.temp }}/record-failures.txt if-no-files-found: error diff --git a/CHANGELOG.md b/CHANGELOG.md index e2057130d..2db7e8018 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## [1.91.128.0] - 2026-10-18 + +**The two free-suite duration seeds now refresh themselves weekly.** The shard packers read a Linux and a Windows duration seed, and both went stale as test files came and went. They were refreshed by hand twice this week. A new weekly workflow records both seeds and opens one seed-refresh PR when either seed's file set changes. That PR goes through the normal review. + +### Added +- **`free-durations-record.yml`** runs Mondays at 04:00 UTC, before test-health, and on dispatch. It records the Linux seed on `ubicloud-standard-8`, the free-suite shard runner, and the Windows seed through `windows-free-tests.yml`'s `record_durations` path (now also callable with `workflow_call`). When either seed adds or drops a file, it opens or refreshes one PR on `bot/free-duration-seeds` and runs the free lanes on it. The PR body states the seed-ratchet result and names every file that failed while recording. +- **Guards:** the bot never edits `scripts/free-test-seed-allowlist.json`. While the open seed PR holds any commit besides the bot's own refresh, later runs leave it alone. Only runs on `main` publish, and `open_pr=false` records and reports only. An interrupted recording, or more than 5 red files on either OS, proposes nothing. + +### Fixed +- **An interrupted manual Windows recording no longer uploads the unchanged committed seed** under the Windows seed name. + ## [1.91.127.0] - 2026-10-17 **The paid PR gate stops failing on its own fixtures.** A 132-verdict PR gate was fully green only about a quarter of the time. Every red verdict since #3231 was diagnosed from its transcript and replayed through the real detector. This release fixes the remaining causes where they start: fixture actors that broke the production protocol, detectors that misread a correct run, and skill prompts that left the agent guessing. No threshold, trial count, case kind or quarantine changed. diff --git a/VERSION b/VERSION index 78a0be812..d6ddc385a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.127.0 +1.91.128.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index bf1d005c4..2e080b314 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.127.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.128.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/docs/TESTING_INTERNALS.md b/docs/TESTING_INTERNALS.md index 6bf03294b..2befad2fe 100644 --- a/docs/TESTING_INTERNALS.md +++ b/docs/TESTING_INTERNALS.md @@ -1164,10 +1164,11 @@ changed. Cut or unstarted reruns are incomplete, not passes. ### Free suite duration seed `scripts/free-test-durations.json` drives shard packing; files missing from it -pack at the seed's 99th percentile. Acceptable seeds come only from -`bun run test:ubicloud --record-durations` (each file timed in its own child on a -VM with the CI lane's environment; it needs `UBICLOUD_API_KEY` and copies the -refreshed seed back into the checkout). A 4-core laptop or small-sandbox +pack at the seed's 99th percentile. Acceptable seeds come only from a recording +in the CI lane's environment, each file timed in its own child: the weekly +`free-durations-record.yml` run, or `bun run test:ubicloud --record-durations` +(it needs `UBICLOUD_API_KEY` and copies the refreshed seed back into the +checkout). A 4-core laptop or small-sandbox recording is not acceptable: browser and display tests skip or slow down there, so the seed misstates them. Contributors without `UBICLOUD_API_KEY` leave the refresh to a maintainer. @@ -1188,6 +1189,20 @@ refresh to a maintainer. `{file, reason, todo}` and the test prints it. A merge-base without the allowlist file counts as the first allowlist. Fix a failure by splitting the file, then rerun `bun run test:ubicloud --record-durations`. +- `free-durations-record.yml` (Mondays 04:00 UTC, before test-health, and on + dispatch) records the Linux seed on `ubicloud-standard-8` with + `scripts/ubicloud/setup-free-suite.sh` and the Windows seed through + `windows-free-tests.yml`'s `record_durations` path. When either recording adds + or drops a test file, it force-updates the `bot/free-duration-seeds` branch, + opens or refreshes one seed PR whose body states the ratchet result, and + dispatches the free lanes on it. A file that fails while recording turns its + job red but keeps its duration, and the PR body names it; the seed PR's own + free checks decide whether the tree is healthy. An interrupted recording + uploads no seed, and more than 5 red files on either OS proposes nothing. The + bot never edits the allowlist: a ratchet failure is fixed on the branch by a + maintainer before the PR merges, and while the open seed PR holds any commit + besides the bot's one refresh commit, later runs leave it alone. Only runs on + `main` push the branch; a dispatch from another ref reports only. ### Windows free lane @@ -1195,7 +1210,8 @@ refresh to a maintainer. runs one strict serial shard per `windows-latest` job (`bun run test:windows --ci-run`), and verifies every shard receipt on Linux (`--windows-only --ci-verify`). Packing uses Windows-measured durations from -`scripts/free-test-durations-windows.json`; refresh them with +`scripts/free-test-durations-windows.json`; the weekly +`free-durations-record.yml` run refreshes them, or by hand run `gh workflow run windows-free-tests.yml --ref -f record_durations=true`, then download the `free-test-durations-windows` artifact and commit it. diff --git a/package.json b/package.json index 87e40ee61..5bc904faf 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.127", + "version": "1.91.128", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/scripts/harness-version.json b/scripts/harness-version.json index b3c02f3a5..ee40de983 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -16,7 +16,7 @@ "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":5}, "scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":5}, "scripts/test-paid-shards.ts": {"blob":"8ddee5f11ed7856876135520dd8025ab989e2db9","since":5}, - "scripts/test-pr-profile.ts": {"blob":"ddc1e5991636c8c94722cd32472edce26489b26d","since":5}, + "scripts/test-pr-profile.ts": {"blob":"67d3e163dd443bfc5e381586e28f7e83ed59125a","since":5,"nonBehavioral":"Classifies the new free-durations-record.yml workflow as free-lane only; no paid case reads it, so no verdict can change"}, "scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":5}, "test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":5}, "test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":5}, diff --git a/scripts/test-pr-profile.ts b/scripts/test-pr-profile.ts index ddc1e5991..67d3e163d 100644 --- a/scripts/test-pr-profile.ts +++ b/scripts/test-pr-profile.ts @@ -230,6 +230,7 @@ export const FREE_ONLY_PR_FILES = [ '.github/workflows/cso-runtime-qualification.yml', '.github/workflows/cso-scanner-images.yml', '.github/workflows/dependency-review.yml', + '.github/workflows/free-durations-record.yml', '.github/workflows/free-tests.yml', '.github/workflows/make-pdf-gate.yml', '.github/workflows/model-policy-freshness.yml', diff --git a/test/free-durations-record-workflow.test.ts b/test/free-durations-record-workflow.test.ts new file mode 100644 index 000000000..7a61d7c6b --- /dev/null +++ b/test/free-durations-record-workflow.test.ts @@ -0,0 +1,177 @@ +/** + * Static pins and a functional check for the weekly duration-seed recorder + * (.github/workflows/free-durations-record.yml): Linux on the free-suite + * runner, Windows through the record_durations path, and one bot seed PR only + * when a seed's file set drifts. + */ +import { afterEach, expect, test } from 'bun:test'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; + +const root = path.resolve(import.meta.dir, '..'); +const load = (file: string) => Bun.YAML.parse(readFileSync(path.join(root, '.github/workflows', file), 'utf8')) as any; +const recorder = load('free-durations-record.yml'); +const freeTests = load('free-tests.yml'); +const proposeStep = recorder.jobs.propose.steps.find((step: any) => step.name === 'Open or refresh the seed PR when either seed\'s file set changed'); +const dirs: string[] = []; +afterEach(() => { for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); }); + +test('the recorder runs weekly before test-health and on dispatch, read-only except the propose job', () => { + expect(recorder.on.schedule).toEqual([{ cron: '0 4 * * 1' }]); + expect(Object.keys(recorder.on).sort()).toEqual(['schedule', 'workflow_dispatch']); + expect(Object.keys(recorder.on.workflow_dispatch.inputs)).toEqual(['open_pr']); + expect(load('test-health.yml').on.schedule[0].cron).toBe('0 10 * * 1'); + expect(recorder.permissions).toEqual({ contents: 'read' }); + expect(recorder.concurrency['cancel-in-progress']).toBe(false); + expect(recorder.jobs.linux.permissions).toBeUndefined(); + expect(recorder.jobs.propose.permissions).toEqual({ contents: 'write', 'pull-requests': 'write', actions: 'write' }); +}); + +test('Linux records on the free-suite runner, turns red on any red file, and uploads only a complete recording', () => { + const linux = recorder.jobs.linux; + expect(linux['runs-on']).toBe(freeTests.jobs['free-suite']['runs-on']); + const runs = linux.steps.map((step: any) => step.run).filter(Boolean).join('\n'); + expect(runs).toContain('bash scripts/ubicloud/setup-free-suite.sh'); + const record = linux.steps.find((step: any) => step.name === 'Time every free file alone'); + expect(record.run).toBe('set -o pipefail\nxvfb-run -a bun run test:free --record-durations 2>&1 | tee "$RUNNER_TEMP/record.log"\n'); + expect(record['continue-on-error']).toBeUndefined(); + const keep = linux.steps.find((step: any) => step.id === 'seed'); + expect(keep.if).toBe("always() && steps.record.outcome != 'skipped'"); + expect(keep.run).toContain('grep -q \'wrote [0-9]* durations to\' "$RUNNER_TEMP/record.log" || {'); + expect(linux.steps.at(-1).if).toBe("always() && steps.seed.outcome == 'success'"); + expect(linux.steps.at(-1).with).toMatchObject({ name: 'free-test-durations', path: '${{ runner.temp }}/seed/' }); +}); + +test('Windows records through the windows-free-tests record_durations path, and propose needs both complete recordings', () => { + expect(recorder.jobs.windows).toEqual({ uses: './.github/workflows/windows-free-tests.yml', with: { record_durations: true } }); + expect(recorder.jobs.propose.needs).toEqual(['linux', 'windows']); + expect(recorder.jobs.propose.if).toBe('${{ !cancelled() }}'); + const downloads = recorder.jobs.propose.steps.filter((step: any) => step.uses?.startsWith('actions/download-artifact@')); + expect(downloads.map((step: any) => step.with.name)).toEqual(['free-test-durations', 'free-test-durations-windows']); + expect(downloads.every((step: any) => step['continue-on-error'] === undefined)).toBe(true); +}); + +test('both recorders list exactly the files the recorder reported red, and more than 5 on either OS proposes nothing', () => { + const windowsRecord = load('windows-free-tests.yml').jobs['windows-record-durations'].steps.find((step: any) => step.id === 'seed').run as string; + const linuxRecord = recorder.jobs.linux.steps.find((step: any) => step.id === 'seed').run as string; + const extract = (run: string) => run.split('\n').find(line => line.startsWith("sed -n '/failed while recording:/"))!; + expect(extract(linuxRecord).replace('seed/record-failures.txt', 'record-failures.txt')).toBe(extract(windowsRecord)); + const dir = mkdtempSync(path.join(os.tmpdir(), 'seed-failures-')); + dirs.push(dir); + const list = (log: string) => { + writeFileSync(path.join(dir, 'record.log'), log); + const result = spawnSync('bash', ['-c', `set -euo pipefail\n${extract(windowsRecord)}\ncat "$RUNNER_TEMP/record-failures.txt"`], { encoding: 'utf8', timeout: 10_000, env: { ...process.env, RUNNER_TEMP: dir } }); + expect(result.status, result.stderr).toBe(0); + return result.stdout; + }; + const red = ' ✗ test/a.test.ts — suite > case\r\n[test:free] wrote 3 durations to scripts\\free-test-durations.json\r\n[test:free] WARNING: 2 file(s) failed while recording:\r\n ✗ test/a.test.ts\r\n ✗ browse/test/b.test.ts\r\n'; + expect(list(red)).toBe('test/a.test.ts\nbrowse/test/b.test.ts\n'); + expect(list(' ✗ test/a.test.ts — suite > flaky then fixed\n[test:free] wrote 3 durations to scripts/free-test-durations.json\n')).toBe(''); + const script: string = proposeStep.run; + expect(script).toContain('if [ "$linux_failed" -gt 5 ] || [ "$windows_failed" -gt 5 ]; then'); + expect(script.indexOf('-gt 5 ]')).toBeLessThan(script.indexOf('drift() {')); + expect(script).toContain('Files that failed while recording (durations kept; this PR\'s own free checks decide): $failed_line'); +}); + +test('the propose step touches only the two seeds, never the allowlist, on one bot branch', () => { + const script: string = proposeStep.run; + expect(script).toContain('git add scripts/free-test-durations.json scripts/free-test-durations-windows.json\n'); + expect(script).not.toContain('free-test-seed-allowlist.json'); + expect(script).toContain('branch=bot/free-duration-seeds'); + expect(script).toContain('bun test test/free-seed-ratchet.test.ts'); + expect(script).toContain('gh workflow run free-tests.yml'); + expect(proposeStep.env.OPEN_PR).toBe("${{ github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || inputs.open_pr) }}"); + expect(script.indexOf('if [ "$OPEN_PR" != true ]')).toBeLessThan(script.indexOf('git push')); + expect(recorder.on.workflow_dispatch.inputs.open_pr).toMatchObject({ type: 'boolean', default: true }); +}); + +test('a long ratchet report reaches the PR body instead of aborting the step under pipefail', () => { + const script: string = proposeStep.run; + const line = script.split('\n').find(entry => entry.trimStart().startsWith('ratchet_line="FAILS'))!.trim(); + const ratchet = ['error: Free seed ratchet:', ...Array.from({ length: 3000 }, (_, i) => `- test/f${i}.test.ts is seeded at 61s, over the 60s ceiling. Fix: split it, then rerun the recording.`)].join('\n'); + const dir = mkdtempSync(path.join(os.tmpdir(), 'seed-ratchet-')); + dirs.push(dir); + writeFileSync(path.join(dir, 'ratchet.txt'), ratchet); + const result = spawnSync('bash', ['-c', `set -euo pipefail\nratchet=$(cat "$1")\n${line}\nprintf '%s' "$ratchet_line"`, 'ratchet', path.join(dir, 'ratchet.txt')], { encoding: 'utf8', timeout: 10_000 }); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain('- test/f18.test.ts is seeded at 61s, over'); + expect(result.stdout).not.toContain('- test/f19.test.ts'); +}); + +test('a refresh keeps a maintainer\'s commits on the open seed PR and leases the bot branch it replaces', () => { + const script: string = proposeStep.run; + const publish = script.slice(script.indexOf('branch=bot/free-duration-seeds'), script.indexOf('cat > pr-body.md')); + const dir = mkdtempSync(path.join(os.tmpdir(), 'seed-publish-')); + dirs.push(dir); + const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.startsWith('GIT_'))); + const git = (cwd: string, ...args: string[]) => { + const result = spawnSync('git', args, { cwd, encoding: 'utf8', timeout: 10_000, env }); + expect(result.status, result.stderr).toBe(0); + return result.stdout.trim(); + }; + const origin = path.join(dir, 'origin.git'); + const work = path.join(dir, 'work'); + git(dir, 'init', '-q', '--bare', '-b', 'main', origin); + git(dir, 'clone', '-q', origin, work); + const commit = (email: string, file: string, message: string) => { + git(work, 'config', 'user.name', email); + git(work, 'config', 'user.email', email); + writeFileSync(path.join(work, file), message); + git(work, 'add', file); + git(work, 'commit', '-q', '-m', message); + }; + for (const file of ['free-test-durations.json', 'free-test-durations-windows.json']) { + writeFileSync(path.join(work, file), '{}'); + } + git(work, 'checkout', '-q', '-b', 'main'); + commit('dev@example.com', 'README', 'main'); + git(work, 'push', '-q', 'origin', 'main'); + const subject = 'test(seed): refresh the free-suite duration seeds from the weekly recording'; + git(work, 'checkout', '-q', '-b', 'bot/free-duration-seeds'); + commit('bot@example.com', 'seed.txt', subject); + git(work, 'push', '-q', 'origin', 'bot/free-duration-seeds'); + const stub = path.join(dir, 'bin'); + spawnSync('mkdir', [stub], { timeout: 10_000 }); + writeFileSync(path.join(stub, 'gh'), '#!/bin/sh\necho 7\n', { mode: 0o755 }); + const run = () => { + git(work, 'checkout', '-q', 'main'); + spawnSync('mkdir', ['-p', path.join(work, 'scripts')], { timeout: 10_000 }); + writeFileSync(path.join(work, 'scripts/free-test-durations.json'), `{"run":${Date.now()}}`); + writeFileSync(path.join(work, 'scripts/free-test-durations-windows.json'), '{}'); + const result = spawnSync('bash', ['-c', `set -euo pipefail\n${publish}`], { cwd: work, encoding: 'utf8', timeout: 20_000, + env: { ...env, PATH: `${stub}:${process.env.PATH}`, GITHUB_STEP_SUMMARY: path.join(dir, 'summary.md') } }); + expect(result.status, result.stderr).toBe(0); + return git(origin, 'rev-parse', 'refs/heads/bot/free-duration-seeds'); + }; + const replaced = run(); + expect(git(origin, 'log', '--format=%s', `main..${replaced}`)).toBe(subject); + expect(git(origin, 'show', `${replaced}:scripts/free-test-durations.json`)).toContain('"run"'); + git(work, 'checkout', '-q', '-B', 'bot/free-duration-seeds', replaced); + commit('maintainer@example.com', 'allowlist.json', 'maintainer repair'); + git(work, 'push', '-q', 'origin', 'HEAD:refs/heads/bot/free-duration-seeds'); + const repaired = git(origin, 'rev-parse', 'refs/heads/bot/free-duration-seeds'); + expect(run()).toBe(repaired); + expect(readFileSync(path.join(dir, 'summary.md'), 'utf8')).toContain("Seed PR #7 has commits besides the bot's one refresh commit"); +}); + +test('the drift check counts added and removed files, so an unchanged file set proposes nothing', () => { + const script: string = proposeStep.run; + const drift = script.slice(script.indexOf('drift() {'), script.indexOf('\n}', script.indexOf('drift() {')) + 2); + const dir = mkdtempSync(path.join(os.tmpdir(), 'seed-drift-')); + dirs.push(dir); + const seed = (name: string, files: Record) => { + const file = path.join(dir, name); + writeFileSync(file, JSON.stringify({ version: 1, recordedAt: '2026-10-17T00:00:00.000Z', durations: files })); + return file; + }; + const committed = seed('committed.json', { 'test/a.test.ts': 100, 'test/b.test.ts': 200 }); + const run = (recorded: string) => spawnSync('bash', ['-c', `${drift}\ndrift "$1" "$2"`, 'drift', committed, recorded], { encoding: 'utf8', timeout: 10_000 }); + const same = run(seed('same.json', { 'test/a.test.ts': 900, 'test/b.test.ts': 50 })); + expect(same.status, same.stderr).toBe(0); + expect(same.stdout.trim()).toBe('"0 0"'); + const drifted = run(seed('drifted.json', { 'test/a.test.ts': 100, 'test/c.test.ts': 1, 'test/d.test.ts': 2 })); + expect(drifted.status, drifted.stderr).toBe(0); + expect(drifted.stdout.trim()).toBe('"2 1"'); +}); diff --git a/test/windows-native-workflows.test.ts b/test/windows-native-workflows.test.ts index 838d7c528..013e89fed 100644 --- a/test/windows-native-workflows.test.ts +++ b/test/windows-native-workflows.test.ts @@ -23,8 +23,9 @@ const native = load('native-qualification.yml'); const WINDOWS_CURATED_FLOOR = 638; test('the Windows lane plans, runs one strict shard per job and verifies every result', () => { - expect(Object.keys(windows.on).sort()).toEqual(['pull_request', 'workflow_dispatch']); + expect(Object.keys(windows.on).sort()).toEqual(['pull_request', 'workflow_call', 'workflow_dispatch']); expect(Object.keys(windows.on.workflow_dispatch.inputs)).toEqual(['record_durations']); + expect(Object.keys(windows.on.workflow_call.inputs)).toEqual(['record_durations']); const plan = windows.jobs['windows-plan']; expect(plan['runs-on']).toBe('ubuntu-24.04'); const planStep = plan.steps.find((step: any) => step.id === 'plan'); @@ -62,15 +63,17 @@ test('a record_durations dispatch times each Windows-safe file alone and uploads expect(job.if).toBe('${{ inputs.record_durations }}'); expect(job['runs-on']).toBe('windows-latest'); const record = job.steps.find((step: any) => step.name === 'Time every Windows-safe file alone'); - expect(record.run).toBe('bun run test:windows --record-durations --wall-timeout 1200'); + expect(record.run).toBe('set -o pipefail\nbun run test:windows --record-durations --wall-timeout 1200 2>&1 | tee "$RUNNER_TEMP/record.log"\n'); expect(record.env.GSTACK_FREE_TEST_DURATIONS).toBeUndefined(); const rename = job.steps.find((step: any) => step.name === 'Name the recording as the Windows seed'); - expect(rename.if).toBe('always()'); - expect(rename.run).toBe('cp scripts/free-test-durations.json "$RUNNER_TEMP/free-test-durations-windows.json"'); + expect(rename.if).toBe("always() && steps.record.outcome != 'skipped'"); + expect(rename.run).toContain('grep -q \'wrote [0-9]* durations to\' "$RUNNER_TEMP/record.log" || {'); + expect(rename.run).toContain('cp scripts/free-test-durations.json "$RUNNER_TEMP/free-test-durations-windows.json"'); const logs = job.steps.find((step: any) => step.name === 'Upload full recording logs'); expect(logs.if).toBe('always()'); expect(logs.with['include-hidden-files']).toBe(true); expect(job.steps.at(-1).with.name).toBe('free-test-durations-windows'); + expect(job.steps.at(-1).if).toBe("always() && steps.seed.outcome == 'success'"); }); test('Windows shards use the pinned Node runtime and retain complete logs on every run', () => { From a669dafcb43b9291aea50dd9f47042bac7d5da5e Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sat, 17 Oct 2026 18:22:15 -0700 Subject: [PATCH 2/4] v1.91.129.0 fix(pr-selection): scan shebang and .ts imports, fail closed on scan errors, report cost per run (#3257) * fix(pr-selection): scan shebang and .ts imports, fail closed on scan errors, classify repo config safely A1-1: strip a leading #! before scanImports at every site, treat bun/node shebang scripts as code, pick the loader by extension (the TSX loader misread 15 .ts files), and restore the full gate when any other import scan fails. Broadens selection only: 0 (file, case) pairs lost vs main, 41 files move from no-consumer to consumed. A1-2: FREE_ONLY_PR_FILES gains six repository-config files, capy/ is derivable, and a full-gate file is labelled 'full-gate input' instead of 'real unknown dependency'. The replay MISS check honours the same audited free-only exemptions as the selector. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * feat(test-health): report gate cases and known cost per PR run, lead with cost per merged PR (A1-3) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(testing): correct the PR fallback section, document the lockfile decision (A1-3, A1-4) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * chore(harness): record the A1 test-pr-profile.ts change as non-behavioral (selection only) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * chore(release): v1.91.127.0 Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Revert "chore(release): v1.91.127.0" This reverts commit 79e3ef6c6330b3e21e0355f90a2f465d1ebb54cf. * chore(release): v1.91.129.0 Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 16 ++++++ VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- docs/TESTING_INTERNALS.md | 39 ++++++++++++--- package.json | 2 +- scripts/harness-version.json | 2 +- scripts/pr-dependencies.ts | 55 +++++++++++++++------ scripts/replay-pr-selection.ts | 8 +-- scripts/test-health-report.ts | 42 +++++++++++++--- scripts/test-pr-profile.ts | 19 ++++++- test/free-fixtures.test.ts | 4 +- test/paid-free-boundary.test.ts | 2 +- test/pr-dependencies.test.ts | 87 +++++++++++++++++++++++++++++++++ test/test-health-report.test.ts | 26 +++++++++- 14 files changed, 265 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2db7e8018..0ecfedfec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## [1.91.129.0] - 2026-10-18 + +**PR paid-lane selection reads every script it depends on, and fails closed when it can't.** Selection derives which paid cases a changed file reaches by scanning imports. It skipped extensionless `bun`/`node` scripts, read 15 `.ts` files with the TSX loader, and treated a scan error as "no consumers", which under-selects. Across all 2,223 code files in the repo, scans now finish with 0 errors, and no (file, case) pair that main selected is lost. + +### Fixed +- **Shebang scripts and `.ts` files are scanned correctly.** A leading `#!` line is stripped before scanning, extensionless `bun`/`node` scripts count as code, and the loader is chosen by file extension. 41 files that reached no paid case now reach the cases that use them. +- **A failed import scan restores the full gate** with the label `import scan failed` and the file to fix, instead of silently selecting nothing. +- **Repository config no longer looks like an unknown dependency.** `.gstack/ship-policy.example.json`, `.github/dependabot.yml`, `.osv-scanner.toml`, `setup.ps1`, `.gitlab-ci.yml` and `.github/ISSUE_TEMPLATE/**` are audited free-only files; `capy/` paths are derived like other source; a full-gate file is labelled `full-gate input`. + +### Added +- **`bun run test:health` reports cost per merged PR first**, plus gate cases per PR run and known cost per PR run, split by selection mode. + +### For contributors +- The selection replay (`scripts/replay-pr-selection.ts`) honours the same free-only exemptions as the selector when it counts misses. +- docs/TESTING_INTERNALS.md#pr-paid-lane-fallback describes the fallback classes as they now behave, and records why the lockfile stays a full-gate input. + ## [1.91.128.0] - 2026-10-18 **The two free-suite duration seeds now refresh themselves weekly.** The shard packers read a Linux and a Windows duration seed, and both went stale as test files came and went. They were refreshed by hand twice this week. A new weekly workflow records both seeds and opens one seed-refresh PR when either seed's file set changes. That PR goes through the normal review. diff --git a/VERSION b/VERSION index d6ddc385a..de898c0c6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.128.0 +1.91.129.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 2e080b314..fbec08ef6 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.128.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.129.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/docs/TESTING_INTERNALS.md b/docs/TESTING_INTERNALS.md index 2befad2fe..7b7e7956a 100644 --- a/docs/TESTING_INTERNALS.md +++ b/docs/TESTING_INTERNALS.md @@ -1009,17 +1009,44 @@ select nothing: docs, any non-paid `*.test.ts(x)`, `FREE_ONLY_PR_FILES` (the free and paid duration seeds, `scripts/ubicloud/**`, every workflow that runs no paid evals, and free-only tools), and the fixtures in `FREE_FIXTURES` (`test/helpers/free-fixtures-data.ts`, each fixture with its free consumers). -`PAID_WORKFLOW_FILES` (`evals.yml`, `evals-periodic.yml`, `evals-marathon.yml`) -and any other unknown file restore the full gate (`prCoverage.mode = -full-fallback`). The job summary and PR comment list each file that caused a -fallback with one of three labels: +Unmapped files under `DERIVABLE_PREFIXES` (bin/, lib/, scripts/, browse/, +design/, make-pdf/, hosts/, extension/, model-overlays/, agents/, test/helpers/, +capy/) are not unknown: `scripts/pr-dependencies.ts` derives the cases whose +reference closure reaches them (`prCoverage.mode = dependents`), or reports that +no paid case reaches them. The derivation scans the imports of every code file, +including `#!`-shebang `.ts` files and extensionless Bun or Node scripts; if any +import scan fails, every unmapped file restores the full gate (label **import +scan failed**). `PR_FULL_GATE_FILES`, `PAID_WORKFLOW_FILES` (`evals.yml`, +`evals-periodic.yml`, `evals-marathon.yml`) and any other unknown file restore +the full gate (`prCoverage.mode = full-fallback`). The job summary and PR comment +list each unmapped file that caused a fallback with one of these labels: - **needs touchfile entry**: a paid test, helper or fixture no case maps; add it to the owning case in `E2E_TOUCHFILES` (`test/helpers/touchfiles-data.ts`). - **add to FREE_ONLY_PR_FILES**: a file only free lanes read; add it to the list in `scripts/test-pr-profile.ts` (or a fixture to `FREE_FIXTURES`). -- **real unknown dependency**: shared code (bin/, lib/, resolvers, hosts, setup) - whose dependents are not mapped; the full gate is the correct result. +- **full-gate input**: a `PR_FULL_GATE_FILES` entry every case consumes; the + full gate is intended and no mapping narrows it. +- **deleted but still referenced** and **import scan failed**: fix the named + reference or parse error. +- **real unknown dependency**: a file outside the derivable directories that no + list classifies (for example `ETHOS.md`); map it, exempt it, or derive its + directory. + +The fallback rate is a label, not the bill: a dependents run whose derived +closure reaches every gate case costs about what a full-fallback run does. +`bun run test:health` leads with the cost per merged PR and reports +`pr-cases-per-run` (mean selected gate cases and judges per PR run, per mode) +and `pr-known-cost-per-run` (mean known trial cost per PR run, per mode) beside +`pr-fallback-rate`. + +`package.json` (beyond a version-only change) and `bun.lock` stay full-gate +inputs. Over the 225 PR runs from 2026-10-07 to 2026-10-17, no run fell back on +them alone, dependabot PRs never reach the paid lane (`evals.yml` skips +`github.actor == 'dependabot[bot]'`), and every runtime package's importer +(`test/helpers/agent-sdk-runner.ts`, `test/helpers/llm-judge.ts`, `browse/src/*`, +`make-pdf/src/*`) reaches every gate case or is in the global set, so narrowing +them would select the same cases. `test/free-fixtures.test.ts` holds the inventory: each `test/fixtures/**` path belongs to a paid touchfile or `FREE_FIXTURES`, and each workflow to the paid or diff --git a/package.json b/package.json index 5bc904faf..5bb38fc1d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.128", + "version": "1.91.129", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/scripts/harness-version.json b/scripts/harness-version.json index ee40de983..f37b3d4c3 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -16,7 +16,7 @@ "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":5}, "scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":5}, "scripts/test-paid-shards.ts": {"blob":"8ddee5f11ed7856876135520dd8025ab989e2db9","since":5}, - "scripts/test-pr-profile.ts": {"blob":"67d3e163dd443bfc5e381586e28f7e83ed59125a","since":5,"nonBehavioral":"Classifies the new free-durations-record.yml workflow as free-lane only; no paid case reads it, so no verdict can change"}, + "scripts/test-pr-profile.ts": {"blob":"64cd7acfc0f2a4f53472bbdc8a50216984230c5a","since":5,"nonBehavioral":"PR selection only (A1 merged with #3253's free-only entry): shebang and per-extension import scanning, fail-closed scan errors, free-only repository-config files, capy/ derivable, the full-gate fallback label; it changes which cases a PR runs, never how any case is judged"}, "scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":5}, "test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":5}, "test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":5}, diff --git a/scripts/pr-dependencies.ts b/scripts/pr-dependencies.ts index fbf1ef43d..a6dc94e3a 100644 --- a/scripts/pr-dependencies.ts +++ b/scripts/pr-dependencies.ts @@ -49,6 +49,12 @@ export interface DerivedDependencies { tracked: Set; /** The reference chain from an E2E case's declared inputs to `file` (for fix text and audits), else null. */ trace(caseId: string, file: string): string[] | null; + /** + * Files whose import scan failed for a reason other than a leading `#!` + * line (their imports are unknown). PR selection restores the full gate + * while this is non-empty; it fills as the derivation reads files. + */ + scanErrors: Map; /** * Tracked files whose reference text still names `missing`, a path absent * from the tree (deleted in the diff): its path or extensionless import @@ -60,8 +66,10 @@ export interface DerivedDependencies { } const CODE = /\.(?:[cm]?[jt]s|tsx|jsx)$/; +/** An extensionless script Bun or Node runs: its imports are code, not shell. */ +const CODE_SHEBANG = /^#![^\n]*\b(?:bun|node)\b/; const TEXT = /\.(?:[cm]?[jt]s|tsx|jsx|sh|md|tmpl|json|ya?ml|toml|txt|py)$/; -const REPO_PATH = /(?:^|[^\w.@-])((?:bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents|test\/helpers|test\/fixtures)\/[\w.@-]+(?:\/[\w.@-]+)*)/g; +const REPO_PATH = /(?:^|[^\w.@-])((?:bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents|capy|test\/helpers|test\/fixtures)\/[\w.@-]+(?:\/[\w.@-]+)*)/g; /** `path.join(ROOT, 'lib', 'x.ts')`: a top-level directory and its entry as separate string literals. */ const JOINED_PATH = /['"`](bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents)['"`]\s*,\s*['"`]([\w.@-]+)['"`]/g; const BIN_NAME = /\bgstack-[a-z0-9][a-z0-9-]*[a-z0-9]\b/g; @@ -85,7 +93,26 @@ export const SELECTION_DATA_MODULES: readonly string[] = [ ]; const DEFAULT_JUDGE_OWNER = 'test/skill-llm-eval.test.ts'; -const transpiler = new Bun.Transpiler({ loader: 'tsx' }); +const transpilers = { + ts: new Bun.Transpiler({ loader: 'ts' }), tsx: new Bun.Transpiler({ loader: 'tsx' }), + js: new Bun.Transpiler({ loader: 'js' }), jsx: new Bun.Transpiler({ loader: 'jsx' }), +}; + +/** Code by extension, or an extensionless file whose shebang runs Bun or Node. */ +export function isCodeFile(file: string, text: string): boolean { + return CODE.test(file) || (!/\.[^/]+$/.test(path.posix.basename(file)) && CODE_SHEBANG.test(text)); +} + +/** + * Relative import specifiers of a code file. A leading `#!` line is blanked + * first (`scanImports` rejects it), and each extension gets its own loader + * (the TSX loader misreads `.ts` generics and casts). Any other parse error + * throws, so callers fail closed. + */ +export function scanCodeImports(file: string, text: string): string[] { + const loader = /\.tsx$/.test(file) ? 'tsx' : /\.jsx$/.test(file) ? 'jsx' : /\.[cm]?js$/.test(file) ? 'js' : 'ts'; + return transpilers[loader].scanImports(text.replace(/^#![^\n]*/, '')).map(item => item.path); +} /** * Text a reference scan reads: whole-line comments are prose about other @@ -168,15 +195,17 @@ export function deriveDependencies(maps: DependencyMaps, root: string): DerivedD }; const dirs = new Set(tracked.flatMap(file => file.split('/').slice(0, -1).map((_, i, parts) => parts.slice(0, i + 1).join('/')))); + const scanErrors = new Map(); + const importsOf = (file: string): string[] => { + try { return scanCodeImports(file, read(file)); } catch (error) { scanErrors.set(file, String(error)); return []; } + }; const edges = new Map(); const refs = (file: string): string[] => { if (edges.has(file)) return edges.get(file)!; const raw = read(file); const text = (SELECTION_DATA_MODULES as readonly string[]).includes(file) ? '' : referenceText(file, raw); const out = new Set(); - if (CODE.test(file)) { - try { for (const item of transpiler.scanImports(raw)) { const hit = resolveSpecifier(root, item.path, file); if (hit) out.add(hit); } } catch { /* literal scan below still applies */ } - } + if (isCodeFile(file, raw)) for (const specifier of importsOf(file)) { const hit = resolveSpecifier(root, specifier, file); if (hit) out.add(hit); } for (const match of text.matchAll(REPO_PATH)) { const ref = match[1]!.replace(/[.]+$/, ''); if (trackedSet.has(ref)) out.add(ref); @@ -238,19 +267,17 @@ export function deriveDependencies(maps: DependencyMaps, root: string): DerivedD const file = pending.pop()!; if (global.has(file)) continue; global.add(file); - if (!CODE.test(file)) continue; - try { for (const item of transpiler.scanImports(read(file))) { const hit = resolveSpecifier(root, item.path, file); if (hit && !global.has(hit)) pending.push(hit); } } catch { /* unparseable: its own entry stays global */ } + if (!isCodeFile(file, read(file))) continue; + for (const specifier of importsOf(file)) { const hit = resolveSpecifier(root, specifier, file); if (hit && !global.has(hit)) pending.push(hit); } } const importTargets = new Map(); const targetsOf = (file: string): string[] => { if (!importTargets.has(file)) { const targets: string[] = []; - if (CODE.test(file) && !(SELECTION_DATA_MODULES as readonly string[]).includes(file)) { - try { - for (const item of transpiler.scanImports(read(file))) { - if (item.path.startsWith('.')) targets.push(path.posix.normalize(path.posix.join(path.posix.dirname(file), item.path))); - } - } catch { /* literal mentions below still apply */ } + if (isCodeFile(file, read(file)) && !(SELECTION_DATA_MODULES as readonly string[]).includes(file)) { + for (const specifier of importsOf(file)) { + if (specifier.startsWith('.')) targets.push(path.posix.normalize(path.posix.join(path.posix.dirname(file), specifier))); + } } importTargets.set(file, targets); } @@ -287,7 +314,7 @@ export function deriveDependencies(maps: DependencyMaps, root: string): DerivedD while (parents.has(chain[0]!)) chain.unshift(parents.get(chain[0]!)!); return chain; }; - return { e2e, judges: consumers(maps.judgeTouchfiles, DEFAULT_JUDGE_OWNER), everyCase, global, tracked: trackedSet, trace, referencers }; + return { e2e, judges: consumers(maps.judgeTouchfiles, DEFAULT_JUDGE_OWNER), everyCase, global, tracked: trackedSet, trace, referencers, scanErrors }; } const cache = new Map(); diff --git a/scripts/replay-pr-selection.ts b/scripts/replay-pr-selection.ts index 71e79d5de..3e6ed6903 100644 --- a/scripts/replay-pr-selection.ts +++ b/scripts/replay-pr-selection.ts @@ -26,7 +26,7 @@ import { packageChangeOnlyVersion } from './test-pr-profile'; import { E2E_TIERS, E2E_TOUCHFILES, GLOBAL_TOUCHFILES } from '../test/helpers/touchfiles-data'; import { matchGlob } from '../test/helpers/test-selection'; import { derivedDependencies } from './pr-dependencies'; -import { PR_PROFILE_MAPS } from './test-pr-profile'; +import { knownNonBehaviorFile, PR_PROFILE_MAPS } from './test-pr-profile'; export interface ReplayInput { runId: number; @@ -59,7 +59,9 @@ export interface ReplayRow { /** * Whether a case depends on any changed file under today's dependency graph: * its touchfiles, the globals, and the derived reference closure - * (scripts/pr-dependencies.ts, its global runner closure included). + * (scripts/pr-dependencies.ts, its global runner closure included). Like + * the selector, a touchfile match wins over the audited free-only exemptions, + * and an exempt file reaches no case through the derived closure. */ export function caseTouchesDiff(id: string, changedFiles: readonly string[]): boolean { const patterns = [...(E2E_TOUCHFILES[id] ?? []), ...GLOBAL_TOUCHFILES]; @@ -67,7 +69,7 @@ export function caseTouchesDiff(id: string, changedFiles: readonly string[]): bo // A path absent from today's tree reaches a case through the live files that still reference it. const via = (file: string) => derived.tracked.has(file) ? [file] : derived.referencers(file); return changedFiles.some(file => patterns.some(pattern => matchGlob(file, pattern)) - || via(file).some(source => derived.global.has(source) || !!derived.e2e.get(source)?.has(id))); + || (!knownNonBehaviorFile(file) && via(file).some(source => derived.global.has(source) || !!derived.e2e.get(source)?.has(id)))); } export function replayRun(input: ReplayInput, oldSelection: readonly string[]): ReplayRow { diff --git a/scripts/test-health-report.ts b/scripts/test-health-report.ts index e4dc5da46..cf11e4f3e 100644 --- a/scripts/test-health-report.ts +++ b/scripts/test-health-report.ts @@ -33,7 +33,7 @@ export const COMPLEXITY_PATHS = ['scripts', 'test/helpers', '.github/workflows'] export interface Metric { id: string; title: string; rule: string; value?: string; detail?: string[]; unavailable?: string } export interface Check { id: string; title: string; status: 'pass' | 'fail' | 'unavailable'; detail: string; fix: string } export type IssueAction = 'upsert' | 'keep-open' | 'close'; -export interface HealthReport { repo: string; sinceDays: number; generatedAt: string; metrics: Metric[]; checks: Check[]; issueAction: IssueAction } +export interface HealthReport { repo: string; sinceDays: number; generatedAt: string; headline?: string; metrics: Metric[]; checks: Check[]; issueAction: IssueAction } export interface HealthInputs { client: GhClient; @@ -188,14 +188,17 @@ export function collectHealth(inputs: HealthInputs): HealthReport { const need = (value: T | Error): T => { if (value instanceof Error) throw value; return value; }; const prRecords = attempt(() => need(prRuns).filter(run => run.conclusion !== 'cancelled') .map(run => ({ run, trials: records(artifactDirs(run, name => name.startsWith('trial-outcomes'))) }))); + const prManifests = attempt(() => need(prRuns).flatMap(run => { + const file = artifactDirs(run, name => name === 'paid-plan').map(dir => path.join(dir, 'manifest.json')).find(candidate => fs.existsSync(candidate)); + return file ? [{ run, manifest: readJson(file) }] : []; + })); const metrics: Metric[] = []; + let costPerMergedPr: string | undefined; - metrics.push(metric('pr-fallback-rate', 'PR full-fallback rate', 'share of evals.yml pull_request runs whose paid-plan manifest has prCoverage.mode = full-fallback', () => { + metrics.push(metric('pr-fallback-rate', 'PR full-fallback rate', 'share of evals.yml pull_request runs whose paid-plan manifest has prCoverage.mode = full-fallback (a label, not spend: dependents runs select about as much)', () => { const modes = new Map(); - for (const run of need(prRuns)) { - const manifest = artifactDirs(run, name => name === 'paid-plan').flatMap(dir => [path.join(dir, 'manifest.json')]).find(file => fs.existsSync(file)); - if (!manifest) continue; - const mode = String(readJson(manifest).prCoverage?.mode ?? 'none'); + for (const { manifest } of need(prManifests)) { + const mode = String(manifest.prCoverage?.mode ?? 'none'); modes.set(mode, (modes.get(mode) ?? 0) + 1); } const total = [...modes.values()].reduce((a, b) => a + b, 0); @@ -204,6 +207,27 @@ export function collectHealth(inputs: HealthInputs): HealthReport { detail: [...modes].map(([mode, count]) => `${mode}: ${count}`) }; })); + metrics.push(metric('pr-cases-per-run', 'Selected gate cases and judges per PR run', 'mean length of prCoverage.e2e (gate cases) and prCoverage.judges in the paid-plan manifests of evals.yml pull_request runs, overall and per mode', () => { + const planned = need(prManifests).filter(({ manifest }) => Array.isArray(manifest.prCoverage?.e2e) && Array.isArray(manifest.prCoverage?.judges)); + if (planned.length === 0) throw new Error('no paid-plan manifest with prCoverage.e2e and prCoverage.judges in the window'); + const mean = (entries: typeof planned) => `${(entries.reduce((sum, { manifest }) => sum + manifest.prCoverage.e2e.length, 0) / entries.length).toFixed(1)} gate cases, ${(entries.reduce((sum, { manifest }) => sum + manifest.prCoverage.judges.length, 0) / entries.length).toFixed(1)} judges`; + const modes = [...new Set(planned.map(({ manifest }) => String(manifest.prCoverage.mode)))].sort(); + return { value: `${mean(planned)} per run (${planned.length} runs)`, + detail: modes.map(mode => { const entries = planned.filter(({ manifest }) => String(manifest.prCoverage.mode) === mode); return `${mode}: ${mean(entries)} (${entries.length} runs)`; }) }; + })); + + metrics.push(metric('pr-known-cost-per-run', 'Known cost per PR run', 'mean over non-cancelled evals.yml pull_request runs of executed trial cost_usd with a known cost (a run with no trial records counts $0), overall and per manifest mode; unknown-cost trials (PTY) are counted, not priced', () => { + const entries = need(prRecords); + if (entries.length === 0) throw new Error('no non-cancelled PR runs in the window'); + const known = (trials: TrialRecord[]) => trials.filter(record => record.execution === 'executed' && record.cost_known !== false).reduce((sum, record) => sum + record.cost_usd, 0); + const unknown = entries.flatMap(entry => entry.trials).filter(record => record.execution === 'executed' && record.cost_known === false).length; + const modeOf = new Map((prManifests instanceof Error ? [] : prManifests).map(({ run, manifest }) => [run.id, String(manifest.prCoverage?.mode ?? 'none')])); + const mean = (list: typeof entries) => `$${(list.reduce((sum, entry) => sum + known(entry.trials), 0) / list.length).toFixed(2)}`; + const modes = [...new Set(entries.map(entry => modeOf.get(entry.run.id) ?? 'no manifest'))].sort(); + return { value: `${mean(entries)} per run (${entries.length} runs; ${unknown} executed trial(s) with unknown cost)`, + detail: modes.map(mode => { const list = entries.filter(entry => (modeOf.get(entry.run.id) ?? 'no manifest') === mode); return `${mode}: ${mean(list)} (${list.length} runs)`; }) }; + })); + metrics.push(metric('pr-reused-trials', 'Reused PR trial records', 'trial-outcomes records with execution = reused across non-cancelled PR runs', () => { const all = need(prRecords).flatMap(entry => entry.trials); const reused = all.filter(record => record.execution === 'reused').length; @@ -216,6 +240,7 @@ export function collectHealth(inputs: HealthInputs): HealthReport { .reduce((sum, record) => sum + record.cost_usd, 0); const revisions = new Set(entries.filter(entry => entry.run.conclusion === 'success' || entry.run.conclusion === 'failure').map(entry => entry.run.sha)).size; const merged = listMergedPulls(client, repo, since); + if (merged) costPerMergedPr = `$${(spend / merged).toFixed(2)} (${merged} merged PRs, $${spend.toFixed(2)} recorded)`; return { value: `$${spend.toFixed(2)} recorded`, detail: [ `per merged PR: ${merged ? `$${(spend / merged).toFixed(2)} (${merged} merged)` : 'n/a (0 merged)'}`, `per validated revision: ${revisions ? `$${(spend / revisions).toFixed(2)} (${revisions} revisions)` : 'n/a'}`] }; @@ -267,7 +292,8 @@ export function collectHealth(inputs: HealthInputs): HealthReport { })); const checks = [flakeCheck(flakes), unseededCheck(root)]; - return { repo, sinceDays, generatedAt: new Date(now).toISOString(), metrics, checks, issueAction: issueActionFor(checks) }; + const headline = `Cost per merged PR: ${costPerMergedPr ?? `unavailable (${metrics.find(entry => entry.id === 'pr-spend')?.unavailable ?? '0 merged PRs'})`}`; + return { repo, sinceDays, generatedAt: new Date(now).toISOString(), headline, metrics, checks, issueAction: issueActionFor(checks) }; } function listMergedPulls(client: GhClient, repo: string, since: string): number { @@ -340,7 +366,7 @@ function censusMetrics(inputs: HealthInputs, artifactDirs: (run: CiRun, match: ( // --- Rendering --- export function formatHealth(report: HealthReport): string { - const lines = [`test:health ${report.repo}, last ${report.sinceDays} day(s), ${report.generatedAt}`]; + const lines = [`test:health ${report.repo}, last ${report.sinceDays} day(s), ${report.generatedAt}`, ...(report.headline ? [report.headline] : [])]; for (const entry of report.metrics) { lines.push(`- ${entry.title}: ${entry.unavailable ? `unavailable: ${entry.unavailable}` : entry.value}`); lines.push(` rule: ${entry.rule}`); diff --git a/scripts/test-pr-profile.ts b/scripts/test-pr-profile.ts index 67d3e163d..64cd7acfc 100644 --- a/scripts/test-pr-profile.ts +++ b/scripts/test-pr-profile.ts @@ -216,6 +216,12 @@ export const FREE_ONLY_PR_FILES = [ '.gstack/generated.json', // gstack's own regen registry: gstack-regen/gstack-pregate read it from origin/ of the repo they run in; paid cases run in fixture repos. '.gstack/pregate.json', // gstack's own pre-gate policy: same trust boundary, same reason. '.gstack/ship-policy.json', // gstack's own ship policy: gstack-ship-policy/gstack-restamp read it from origin/ of the repo they run in; same reason. + '.gstack/ship-policy.example.json', // The example beside it: only .gitignore and docs/ship-policy.md name it. + '.github/dependabot.yml', // Dependabot's own config: read by GitHub, and by free test/compare-pr-version.test.ts; dependabot runs skip the paid lane. + '.osv-scanner.toml', // Read by .github/workflows/osv-scanner.yml and the free OSV tests (test/osv-*.test.ts). + 'setup.ps1', // The Windows installer: run only by .github/workflows/windows-setup-e2e.yml; the paid lane is Linux. + '.gitlab-ci.yml', // gstack's own GitLab config: skills probe the fixture repo's .gitlab-ci.yml, never this one. + '.github/ISSUE_TEMPLATE/**', // Issue templates: read by GitHub and free test/gstack-doctor.test.ts. 'scripts/retired-command.ts', // One-release stubs for retired package scripts; no paid case imports it. 'careful/bin/check-careful.sh', // /careful PreToolUse hook: covered by test/hook-scripts.test.ts; no paid fixture copies or runs it. // Reporting and launch tools: they read CI history or start a lane, never run inside a paid case. @@ -274,9 +280,9 @@ export const PR_FULL_GATE_FILES = [ export const PERIODIC_ONLY_PR_FILES = ['.github/workflows/evals-periodic.yml', '.github/workflows/evals-marathon.yml'] as const; /** Directories the reference derivation covers: an unmapped file there with no consumer is consumed by no paid case. */ -export const DERIVABLE_PREFIXES = ['bin/', 'lib/', 'scripts/', 'browse/', 'design/', 'make-pdf/', 'hosts/', 'extension/', 'model-overlays/', 'agents/', 'test/helpers/'] as const; +export const DERIVABLE_PREFIXES = ['bin/', 'lib/', 'scripts/', 'browse/', 'design/', 'make-pdf/', 'hosts/', 'extension/', 'model-overlays/', 'agents/', 'test/helpers/', 'capy/'] as const; -function knownNonBehaviorFile(file: string): boolean { +export function knownNonBehaviorFile(file: string): boolean { // A mapped dependency still wins over these exemptions. New helper, // runtime, dependency, or paid-workflow files are deliberately not exempted. return /^(?:docs\/|(?:README|CONTRIBUTING|ARCHITECTURE|CHANGELOG|TODOS)\.md$|VERSION$)/.test(file) @@ -294,6 +300,9 @@ export const FALLBACK_FIX_ANCHOR = 'docs/TESTING_INTERNALS.md#pr-paid-lane-fallb /** Why one changed file restored the full gate, and the edit that narrows it next time (DX-6). */ export function unknownFileLabel(file: string): { label: string; fix: string } { + if (matches(file, PR_FULL_GATE_FILES)) { + return { label: 'full-gate input', fix: `${file} is in PR_FULL_GATE_FILES (scripts/test-pr-profile.ts): every paid case consumes it, so the full gate is intended and no mapping narrows it` }; + } if (isPaidTestFile(file) || file.startsWith('test/helpers/')) { return { label: 'needs touchfile entry', fix: `register ${file} under the cases that consume it in test/helpers/touchfiles-data.ts` }; } @@ -381,6 +390,12 @@ export function selectPrProfile(options: { if (found.e2e.size + found.judges.size > 0) consumers.set(file, found); else noConsumerFiles.push(file); } + // A failed import scan leaves some closure unknown: every unmapped file restores the full gate (fail closed). + const scanFailed = derived && unmapped.length ? [...derived.scanErrors.keys()].sort() : []; + if (scanFailed.length) { + consumers.clear(); derivedGlobal.length = 0; noConsumerFiles.length = 0; + for (const file of unmapped) deletedLabels.set(file, { label: 'import scan failed', fix: `scripts/pr-dependencies.ts could not scan the imports of ${scanFailed.slice(0, 3).join(', ')}${scanFailed.length > 3 ? ` and ${scanFailed.length - 3} more` : ''}; fix the parse error so ${file}'s consumers can be derived` }); + } const derivedFiles = [...consumers.keys()]; const unknownFiles = unmapped.filter(file => !derivedGlobal.includes(file) && !consumers.has(file) && !noConsumerFiles.includes(file)); const sharedInputs = files.filter(file => depends(file, PR_FULL_GATE_FILES)); diff --git a/test/free-fixtures.test.ts b/test/free-fixtures.test.ts index c9cb1d933..4feeb8c37 100644 --- a/test/free-fixtures.test.ts +++ b/test/free-fixtures.test.ts @@ -85,7 +85,7 @@ describe('workflow inventory (CEO-20)', () => { test('the inventory rule also reads .yaml files', () => { expect(unknownFileLabel('.github/workflows/new-check.yaml').label).toBe('add to FREE_ONLY_PR_FILES'); - expect(unknownFileLabel('.github/workflows/evals.yml').label).toBe('real unknown dependency'); + expect(unknownFileLabel('.github/workflows/evals.yml').label).toBe('full-gate input'); }); }); @@ -111,7 +111,7 @@ describe('PR selection over the W1a single-file fallbacks', () => { const result = select(['.github/workflows/evals.yml']); expect(result.mode).toBe('full-fallback'); expect(result.unknownFileLabels).toEqual([{ file: '.github/workflows/evals.yml', ...unknownFileLabel('.github/workflows/evals.yml') }]); - expect(result.reasons[0]).toContain('.github/workflows/evals.yml (real unknown dependency)'); + expect(result.reasons[0]).toContain('.github/workflows/evals.yml (full-gate input)'); }); test('a mapped free fixture is exempt; a paid-registered fixture selects its cases; an unmapped one falls back', () => { diff --git a/test/paid-free-boundary.test.ts b/test/paid-free-boundary.test.ts index 402ac6917..dd4ea3eb9 100644 --- a/test/paid-free-boundary.test.ts +++ b/test/paid-free-boundary.test.ts @@ -114,7 +114,7 @@ describe('paid/free dependency boundary', () => { expect(result.selection).toEqual({ e2e: [], judges: [] }); } // Tracked files outside every mapping and outside the derivable directories restore the full gate. - for (const file of ['ETHOS.md', 'SECURITY.md', 'conductor.json', '.osv-scanner.toml']) { + for (const file of ['ETHOS.md', 'SECURITY.md', 'conductor.json', 'NOTICE.md']) { const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: [FREE_ONLY_PR_FILES[0], file] }); expect(result.coverage?.mode, file).toBe('full-fallback'); expect(result.coverage?.unknownFiles).toContain(file); diff --git a/test/pr-dependencies.test.ts b/test/pr-dependencies.test.ts index 2470405e9..a3ddcf73d 100644 --- a/test/pr-dependencies.test.ts +++ b/test/pr-dependencies.test.ts @@ -96,6 +96,93 @@ describe('reference derivation on a fixture tree', () => { }); }); +describe('shebang files and import scan failures (A1-1)', () => { + const tree = (extra: Record = {}) => fixtureRepo({ + 'test/skill-e2e-alpha.test.ts': "import '../lib/entry';\n", 'test/skill-e2e-beta.test.ts': '', + 'lib/entry.ts': '#!/usr/bin/env bun\nimport { y } from \'./typed\';\nexport const x = y;\n', + 'lib/typed.ts': 'export const y = (v: T) => v as unknown as T;\nimport \'./typed-dep\';\n', 'lib/typed-dep.ts': '', + 'alpha/SKILL.md.tmpl': 'Run `gstack-sb`.\n', 'bin/gstack-sb': '#!/usr/bin/env -S bun run\nimport { z } from \'../lib/sb-dep\';\n', 'lib/sb-dep.ts': '', + 'bin/gstack-sh': '#!/bin/sh\nimport ../lib/sh-only.ts\n', 'lib/sh-only.ts': '', + 'scripts/build.sh': '', 'scripts/resolvers/index.ts': '', 'package.json': '{}', ...extra, + }); + const maps = { e2eTouchfiles: { alpha: ['test/skill-e2e-alpha.test.ts', 'alpha/**'], beta: ['test/skill-e2e-beta.test.ts'] }, judgeTouchfiles: {}, globalTouchfiles: [] as string[] }; + const profileMaps = { ...maps, tiers: { alpha: 'gate', beta: 'gate' } as const }; + const pick = (derived: ReturnType, changedFiles: string[]) => + selectPrProfile({ maps: profileMaps, profile: [], selectedE2E: [], selectedJudges: [], changedFiles, derived }); + + test('a shebang .ts file, a bun-shebang extensionless bin and a .ts generic arrow are all import-scanned', () => { + const derived = deriveDependencies(maps, tree()); + expect([...derived.e2e.get('lib/typed.ts') ?? []]).toEqual(['alpha']); + expect([...derived.e2e.get('lib/typed-dep.ts') ?? []]).toEqual(['alpha']); + expect([...derived.e2e.get('lib/sb-dep.ts') ?? []]).toEqual(['alpha']); + expect(derived.e2e.get('lib/sh-only.ts')).toBeUndefined(); + expect(derived.scanErrors.size).toBe(0); + expect(pick(derived, ['lib/sb-dep.ts'])).toMatchObject({ mode: 'dependents', e2e: ['alpha'] }); + }); + + test('lib/fs-utils.ts selects the cases of bin/gstack-redact-prepush, which imports it', () => { + const bin = gate.filter(id => real.e2e.get('bin/gstack-redact-prepush')?.has(id)); + expect(bin.length).toBeGreaterThan(0); + const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: ['lib/fs-utils.ts'], packageVersionOnly: true }); + expect(result.coverage?.mode).toBe('dependents'); + for (const id of bin) expect(result.selection.e2e, id).toContain(id); + expect(real.scanErrors.size).toBe(0); + expect(real.global.has('test/helpers/bootstrap-retention.ts')).toBe(true); + }); + + test('any other scan error fails closed at every site: case closures, the global walk and deleted-file referencers', () => { + const broken = "import { from './nowhere';\n"; + const inClosure = deriveDependencies(maps, tree({ 'lib/typed-dep.ts': broken })); + expect([...inClosure.scanErrors.keys()]).toEqual(['lib/typed-dep.ts']); + const closed = pick(inClosure, ['lib/sb-dep.ts']); + expect(closed).toMatchObject({ mode: 'full-fallback', e2e: ['alpha', 'beta'], unknownFiles: ['lib/sb-dep.ts'] }); + expect(closed.unknownFileLabels[0]).toMatchObject({ label: 'import scan failed' }); + expect(closed.unknownFileLabels[0]!.fix).toContain('lib/typed-dep.ts'); + + const inGlobal = deriveDependencies({ ...maps, globalTouchfiles: ['lib/runner.ts'] }, tree({ 'lib/runner.ts': broken })); + expect([...inGlobal.scanErrors.keys()]).toEqual(['lib/runner.ts']); + expect(pick(inGlobal, ['lib/sb-dep.ts']).mode).toBe('full-fallback'); + + const unreached = deriveDependencies(maps, tree({ 'lib/orphan.ts': broken })); + expect(unreached.scanErrors.size).toBe(0); + expect(pick(unreached, ['lib/gone.ts'])).toMatchObject({ mode: 'full-fallback', unknownFiles: ['lib/gone.ts'] }); + expect([...unreached.scanErrors.keys()]).toEqual(['lib/orphan.ts']); + }); +}); + +describe('safe classification (A1-2)', () => { + test('capy/support.env derives to the bins that read it instead of restoring the full gate', () => { + for (const bin of ['bin/gstack-install-verdict.sh', 'bin/gstack-doctor-check.sh']) { + for (const id of real.e2e.get(bin) ?? []) expect(real.e2e.get('capy/support.env')?.has(id), `${bin} ${id}`).toBe(true); + } + const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: ['capy/support.env'], packageVersionOnly: true }); + expect(result.coverage?.unknownFiles).toEqual([]); + expect(result.coverage?.mode).toBe('dependents'); + expect(DERIVABLE_PREFIXES).toContain('capy/'); + }); + + test('repository config read only by free lanes or GitHub stays in the PR profile', () => { + const files = ['.github/dependabot.yml', '.osv-scanner.toml', 'setup.ps1', '.gitlab-ci.yml', '.github/ISSUE_TEMPLATE/bug_report.md', '.gstack/ship-policy.example.json']; + for (const file of files) expect(real.tracked.has(file), file).toBe(true); + const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: files, packageVersionOnly: true }); + expect(result.coverage?.mode).toBe('pr'); + expect(result.selection).toEqual({ e2e: [], judges: [] }); + }); + + test('a full-gate file is never labelled a real unknown dependency', () => { + for (const file of ['.github/workflows/evals.yml', 'test-setup.ts', '.gitattributes']) { + const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: [file], packageVersionOnly: true }); + expect(result.coverage?.mode, file).toBe('full-fallback'); + for (const label of result.coverage!.unknownFileLabels) { + expect(label.label, file).toBe('full-gate input'); + expect(label.fix).toContain('PR_FULL_GATE_FILES'); + } + expect(unknownFileLabel(file).label).toBe('full-gate input'); + } + expect(unknownFileLabel('ETHOS.md').label).toBe('real unknown dependency'); + }); +}); + describe('derived dependencies of this checkout', () => { test('a bin script a skill template invokes selects that skill\'s cases', () => { const consumers = real.e2e.get('bin/gstack-next-version') ?? new Set(); diff --git a/test/test-health-report.test.ts b/test/test-health-report.test.ts index 75970cef3..38a2e31f4 100644 --- a/test/test-health-report.test.ts +++ b/test/test-health-report.test.ts @@ -191,7 +191,7 @@ describe('collectHealth over recorded fixture data', () => { [/runs\/61\/artifacts/, () => ({ artifacts: [] })], [/workflows\/windows-free-tests\.yml\/runs/, () => ({ workflow_runs: windowsRuns })], ], { - 11: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'full-fallback' } }) }, + 11: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'full-fallback', e2e: ['gate-a', 'gate-b'], judges: ['judge-a'] } }) }, 12: { 'trial-outcomes.jsonl': prTrials }, 51: { 'trial-outcomes.jsonl': census }, 61: { 'flake-ledger.jsonl': ledger('test/a.test.ts') }, @@ -205,6 +205,10 @@ describe('collectHealth over recorded fixture data', () => { expect(value('net-complexity').unavailable).toContain('git grep HEAD failed'); expect(value('pr-reused-trials').value).toBe('1 of 2 trial records'); expect(value('pr-spend')).toMatchObject({ value: '$2.50 recorded', detail: ['per merged PR: $2.50 (1 merged)', 'per validated revision: $2.50 (1 revisions)'] }); + expect(value('pr-cases-per-run')).toMatchObject({ value: '2.0 gate cases, 1.0 judges per run (1 runs)', detail: ['full-fallback: 2.0 gate cases, 1.0 judges (1 runs)'] }); + expect(value('pr-known-cost-per-run')).toMatchObject({ value: '$2.50 per run (1 runs; 0 executed trial(s) with unknown cost)', detail: ['full-fallback: $2.50 (1 runs)'] }); + expect(report.headline).toBe('Cost per merged PR: $2.50 (1 merged PRs, $2.50 recorded)'); + expect(formatHealth(report).split('\n')[1]).toBe('Cost per merged PR: $2.50 (1 merged PRs, $2.50 recorded)'); expect(value('cancelled-slice-minutes').value).toBe('30 min/week (1 cancelled run(s))'); expect(value('push-to-verdict').value).toBe('p50 10.0 min, p95 10.0 min (1 runs)'); expect(value('census-periodic').value).toBe('1.00 expected failed trials/run (1/3 over 1 run(s)); P(red) 33.3% in force, 33.3% single-trial'); @@ -223,6 +227,26 @@ describe('collectHealth over recorded fixture data', () => { expect(formatHealth(report)).toContain('rule: share of evals.yml pull_request runs'); }); + test('known cost per PR run prices only known-cost executed trials and counts the unknown ones', () => { + const trials = formatTrialOutcomes([trialRecord('gate-a', 'passed', { tier: 'gate', lane: 'gate/pr', cost_usd: 4 }), + trialRecord('gate-pty', 'passed', { tier: 'gate', lane: 'gate/pr', cost_usd: 0, cost_known: false })]); + const client = fakeClient([ + [/workflows\/evals\.yml\/runs/, () => ({ workflow_runs: [run(1, { event: 'pull_request', head_branch: 'a' }), run(2, { event: 'pull_request', head_branch: 'b' })] })], + [/runs\/1\/artifacts/, () => ({ artifacts: [{ id: 11, name: 'paid-plan', size_in_bytes: 10 }, { id: 12, name: 'trial-outcomes-pr-a1', size_in_bytes: 10 }] })], + [/runs\/2\/artifacts/, () => ({ artifacts: [{ id: 21, name: 'paid-plan', size_in_bytes: 10 }] })], + [/pulls\?state=closed/, () => []], + ], { + 11: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'dependents', e2e: ['gate-a', 'gate-pty', 'gate-c'], judges: ['j'] } }) }, + 12: { 'trial-outcomes.jsonl': trials }, + 21: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'pr', e2e: [], judges: [] } }) }, + }); + const report = collectHealth({ client, repo: 'o/r', now: NOW, sinceDays: 7, cacheDir: tmp('health-cache-'), root: freeRoot(3, 0) }); + const value = (id: string) => report.metrics.find(entry => entry.id === id)!; + expect(value('pr-known-cost-per-run')).toMatchObject({ value: '$2.00 per run (2 runs; 1 executed trial(s) with unknown cost)', detail: ['dependents: $4.00 (1 runs)', 'pr: $0.00 (1 runs)'] }); + expect(value('pr-cases-per-run')).toMatchObject({ value: '1.5 gate cases, 0.5 judges per run (2 runs)', detail: ['dependents: 3.0 gate cases, 1.0 judges (1 runs)', 'pr: 0.0 gate cases, 0.0 judges (1 runs)'] }); + expect(report.headline).toBe('Cost per merged PR: unavailable (0 merged PRs)'); + }); + test('the Windows wall counts only successful pull_request runs and says so when there are none', () => { const wall = (windowsRuns: unknown[]) => collectHealth({ client: fixture(windowsRuns), repo: 'o/r', now: NOW, sinceDays: 7, cacheDir: tmp('health-cache-'), root: freeRoot(3, 0) }) .metrics.find(entry => entry.id === 'free-wall')!; From dbf4a3c67e0adc312b7773e82eb4fa315171ca9b Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sat, 17 Oct 2026 18:37:29 -0700 Subject: [PATCH 3/4] v1.91.130.0 feat(pr-selection): exercised-path consumer map in shadow mode, MAP_GAP and census refresh (#3259) * fix(pr-selection): scan shebang and .ts imports, fail closed on scan errors, classify repo config safely A1-1: strip a leading #! before scanImports at every site, treat bun/node shebang scripts as code, pick the loader by extension (the TSX loader misread 15 .ts files), and restore the full gate when any other import scan fails. Broadens selection only: 0 (file, case) pairs lost vs main, 41 files move from no-consumer to consumed. A1-2: FREE_ONLY_PR_FILES gains six repository-config files, capy/ is derivable, and a full-gate file is labelled 'full-gate input' instead of 'real unknown dependency'. The replay MISS check honours the same audited free-only exemptions as the selector. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * feat(test-health): report gate cases and known cost per PR run, lead with cost per merged PR (A1-3) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(testing): correct the PR fallback section, document the lockfile decision (A1-3, A1-4) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * chore(harness): record the A1 test-pr-profile.ts change as non-behavioral (selection only) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * feat(pr-selection): exercised-path consumers and a shadow selection block (A2-1, A2-2, A2-3) exercisedConsumers() unions touchfiles, the test-file import closure, executable skill shell and the bins recorded trials ran (test/helpers/exercised-commands.json). selectPrProfile reports what that map would select in prCoverage.shadow and never changes the real selection; a map-only diff selects no paid case. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * feat(exercised-map): union-only refresh with a fail-closed contract, census refresh, MAP_GAP and the replay receipt (A2-1, A2-4) bun run test:exercised-map refreshes from a census run or local artifacts and writes nothing on unavailable (exit 3) or partial (exit 4) input. The paid report prints non-blocking MAP_GAP lines; --replay pins 0 misses over the 47 recorded gate reds. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * feat(test-health): report shadow savings as a lower bound and split red drops into activation misses and flaky drops (A2-3) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(testing): document the exercised-path map, its refresh contract and shadow metrics Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * chore(harness): record the A2 test-pr-profile.ts and paid-report.ts changes as non-behavioral Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test(fixtures): register the exercised-map replay fixture as free (read by scripts/exercised-map.ts --replay) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * chore(release): v1.91.127.0 Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Revert "chore(release): v1.91.127.0" This reverts commit 79e3ef6c6330b3e21e0355f90a2f465d1ebb54cf. * chore(release): v1.91.130.0 Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- .github/workflows/evals-periodic.yml | 28 + CHANGELOG.md | 15 + VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- docs/TESTING_INTERNALS.md | 72 + package.json | 3 +- scripts/exercised-map.ts | 384 +++++ scripts/harness-version.json | 4 +- scripts/lib/paid-report.ts | 28 + scripts/pr-dependencies.ts | 294 ++++ scripts/test-health-report.ts | 24 + scripts/test-pr-profile.ts | 92 +- test/exercised-map.test.ts | 357 +++++ test/fixtures/exercised-map-replay.json | 44 + test/helpers/exercised-commands.json | 1905 +++++++++++++++++++++++ test/helpers/exercised-prototype.ts | 149 ++ test/helpers/free-fixtures-data.ts | 1 + test/test-health-report.test.ts | 36 +- 18 files changed, 3430 insertions(+), 10 deletions(-) create mode 100644 scripts/exercised-map.ts create mode 100644 test/exercised-map.test.ts create mode 100644 test/fixtures/exercised-map-replay.json create mode 100644 test/helpers/exercised-commands.json create mode 100644 test/helpers/exercised-prototype.ts diff --git a/.github/workflows/evals-periodic.yml b/.github/workflows/evals-periodic.yml index 45aeae1ce..da9f8dde6 100644 --- a/.github/workflows/evals-periodic.yml +++ b/.github/workflows/evals-periodic.yml @@ -723,6 +723,34 @@ jobs: if-no-files-found: ignore retention-days: 90 + # Exercised-path map (shadow, non-blocking; docs/TESTING_INTERNALS.md#exercised-path-map): + # union this census's own recorded commands into the map, report MAP_GAP lines and + # cases that recorded no commands, and print the refresh command. Never fails the run. + - name: Refresh the exercised-path map from this census + if: always() + continue-on-error: true + run: | + mkdir -p /tmp/exercised-map + dirs=() + for dir in /tmp/gate-census-report /tmp/paid-report; do [ -d "$dir" ] && dirs+=(--from-dir "$dir"); done + status=0 + bun --no-install run scripts/exercised-map.ts "${dirs[@]}" \ + --run-id "$GITHUB_RUN_ID" --out /tmp/exercised-map/exercised-commands.json --report /tmp/exercised-map/report.md || status=$? + if [ "$status" -ne 0 ]; then + echo "exercised-path map refresh failed with exit $status (non-blocking; no map uploaded)" | tee -a "$GITHUB_STEP_SUMMARY" + fi + if [ -f /tmp/exercised-map/report.md ]; then cat /tmp/exercised-map/report.md >> "$GITHUB_STEP_SUMMARY"; fi + + - name: Upload the refreshed exercised-path map + if: always() + continue-on-error: true + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: exercised-map-a${{ github.run_attempt }} + path: /tmp/exercised-map + if-no-files-found: ignore + retention-days: 90 + # Weekly pass-rate gate over the last 10 weekly runs (drift, rule cases # behaving like behavior, quarantine exit/expiry/cap). Fails closed when # history cannot be fetched. diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ecfedfec..796a13f7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog +## [1.91.130.0] - 2026-10-18 + +**Every PR plan now shows what a tighter, exercised-path case map would run, without changing what actually runs.** Today a change to a bin or lib file reaches almost every gate case, because selection follows every mention of the file. The exercised-path map counts only what a case really executes: its touchfiles, its test's imports, the shell its skill runs, and the bins its recorded trials invoked. It runs in shadow mode. Replayed over every PR run that had a gate red between 2026-10-07 and 2026-10-17 (37 runs, 47 failed gate cases), it keeps all 47: `bun run test:exercised-map --replay`. + +### Added +- **A shadow line in each PR plan's coverage summary.** It names the gate cases the map would select, drop and add, with per-file provenance and a predicted saving. The saving is a lower bound, because about 39% of observed case runs (the PTY trials) have no known cost. +- **`bun run test:health` reports the shadow saving and shadow misses per week.** A red on a case the shadow would drop counts as a miss unless the case passed later at the same head SHA, which lists it as a flaky drop. +- **MAP_GAP lines in the paid report.** A command a case ran outside its map is printed as `MAP_GAP `, marked non-blocking for this PR, and widens the map at the next refresh. It never affects a verdict. +- **`bun run test:exercised-map`** refreshes the committed map (`test/helpers/exercised-commands.json`) from a census run (`--from-run `) or from local artifacts (`--from-dir`). Refreshes only add commands. Unavailable or partial input writes nothing and exits with a stable code: 2 usage, 3 unavailable, 4 incomplete. + +### For contributors +- The weekly census unions its own recorded commands into the map and uploads it as `exercised-map-a`. +- A PR that changes only the map selects no paid case while the map is in shadow mode. +- docs/TESTING_INTERNALS.md#exercised-path-map documents the edge classes, the refresh contract and the activation bar. + ## [1.91.129.0] - 2026-10-18 **PR paid-lane selection reads every script it depends on, and fails closed when it can't.** Selection derives which paid cases a changed file reaches by scanning imports. It skipped extensionless `bun`/`node` scripts, read 15 `.ts` files with the TSX loader, and treated a scan error as "no consumers", which under-selects. Across all 2,223 code files in the repo, scans now finish with 0 errors, and no (file, case) pair that main selected is lost. diff --git a/VERSION b/VERSION index de898c0c6..97bde5585 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.129.0 +1.91.130.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index fbec08ef6..895f1b42e 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.129.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.130.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/docs/TESTING_INTERNALS.md b/docs/TESTING_INTERNALS.md index 7b7e7956a..6e8f0408a 100644 --- a/docs/TESTING_INTERNALS.md +++ b/docs/TESTING_INTERNALS.md @@ -1054,6 +1054,78 @@ free-only list. Editing a paid test file runs its own gate cases on the PR when their Bun test is named by the case id; the file's other cases stay deferred with a reason that says to name the test by its case id. +### Exercised-path map + +The derived closure (above) reaches almost every gate case from most bin/ and +lib/ files, because it follows prose, string and doc-comment mentions. The +exercised-path map is a tighter consumer set that runs in **shadow mode**: it +is computed and reported on every PR plan, and it never changes which cases a +PR runs. Activating it is a later decision that needs PTY command capture for +the cases that record no commands, two weeks of shadow data with zero shadow +misses, and Garry's call on the measured saving. + +A case's exercised consumers (`exercisedConsumers` in +`scripts/pr-dependencies.ts`) are the union of its touchfile matches, the +closure of its test file and helpers, the executable shell of its loaded skill +(template and generated SKILL.md) and of every bin, lib and script that shell +runs, and the bins its recorded trials invoked. Shell edges are command-position +`gstack-*` names and repo paths, `.`/`source`, `./setup`, quoted path +literals, `path.join` pairs, data files, `bun run` scripts and compiled outfiles +(`browse/dist/browse`, `$B`, `design/dist/design`, `$D`, `make-pdf/dist/pdf`). +Comments, echo/printf/die messages, case patterns, bare `VAR=gstack-x` strings +and slash-command skill names are not edges; command substitutions are kept, an +expanding heredoc keeps its substitutions and a heredoc an interpreter reads +stays whole. The closure stops at `PR_FULL_GATE_FILES` and the global set, which +stay full. `test/exercised-map.test.ts` pins that this function equals or is a +reviewed superset of the measured prototype (`test/helpers/exercised-prototype.ts`). + +The observed half is `test/helpers/exercised-commands.json` (generated; per case +the bins and compiled outfiles its recorded trials ran, observation counts, the +source run ids and a mean known cost per run). Refreshes are union-only: an +observation never removes a command and a missing or empty artifact never erases +one; removing a command is a reviewed edit. While the map is in shadow mode it +changes no selection, so a PR that changes only the map selects no paid case. + +Dollar figures are lower bounds. Known cost excludes PTY trials, whose cost is +unknown: about 39% of observed case runs in the committed map. The shadow block +and `test:health` both say so. + +- **Shadow block:** `prCoverage.shadow` in the paid-plan manifest lists the gate + cases the map would select, would drop and would add, a provenance line per + changed file ("selects N case(s): touchfile (k), import (k), skill bash via + bin/gstack-x (k), observed bin/gstack-y (k)"), the predicted saving and the + unknown-cost share. The coverage summary prints one line of it. +- **Shadow misses (`bun run test:health`, metric `pr-shadow-map`):** a red case + (more executed failed than passed trials in the run) that the shadow would + drop is an activation miss unless the case passed in a later run of the same + branch at the same head SHA. That later pass marks it a flaky drop, which is + listed but not counted. The D3 activation bar counts activation misses only. +- **MAP_GAP:** the paid report runs the same bin matcher over each executed + case's recorded commands and prints, and writes to `map-gaps.txt`, + `MAP_GAP : non-blocking for this PR; the next census refresh + widens the map (docs/TESTING_INTERNALS.md#exercised-path-map)` for a command + outside the case's static-plus-observed closure. It is its own result: never a + case verdict, a lane problem or an INFRA re-dispatch input. +- **Census refresh:** the weekly census unions its own recorded commands into the + map (no REST artifact downloads), uploads it as `exercised-map-a`, + reports MAP_GAP lines and the cases that recorded no commands, and prints the + refresh command. Run `bun run test:exercised-map --from-run `, or + `--from-dir ` on a locally downloaded `exercised-map-a` artifact + when the download hits HTTP 403, and open the change as an ordinary PR. + `bun run test:exercised-map --check` validates the committed file. +- **Refresh contract:** a refresh whose input is unavailable or partial writes + nothing, keeps the committed union and exits with a stable code that names + what was missing: `2` usage; `3` input unavailable (a missing directory, or a + failed or HTTP 403 download); `4` input incomplete (no `shards/`, no + `trial-outcomes.jsonl`, an executed case without a shard artifact, or a missing + or malformed `exercised-commands.json`). The census step reports a failed + refresh in its job summary and uploads no map. +- **Replay receipt:** `bun run test:exercised-map --replay` replays the recorded + red PR runs in `test/fixtures/exercised-map-replay.json` (every executed failed + gate trial in 225 `evals.yml` pull_request runs, 2026-10-07 to 2026-10-17) + through today's selector and its shadow, and exits 1 on any MISS: a failed + gate case the shadow would not run. `test/exercised-map.test.ts` pins zero. + ### PR receipts, recovery and debounce Passes and failures of single-trial shards are written as receipts at the diff --git a/package.json b/package.json index 5bb38fc1d..1a72f3fc1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.129", + "version": "1.91.130", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", @@ -50,6 +50,7 @@ "eval:bg:gate": "bun run scripts/eval-bg.ts gate", "eval:bg:periodic": "bun run scripts/eval-bg.ts periodic", "test:health": "bun run scripts/test-health-report.ts", + "test:exercised-map": "bun run scripts/exercised-map.ts", "eval:list": "bun run scripts/eval-list.ts", "eval:compare": "bun run scripts/eval-compare.ts", "eval:summary": "bun run scripts/eval-summary.ts", diff --git a/scripts/exercised-map.ts b/scripts/exercised-map.ts new file mode 100644 index 000000000..126fa8b03 --- /dev/null +++ b/scripts/exercised-map.ts @@ -0,0 +1,384 @@ +#!/usr/bin/env bun +/** + * test:exercised-map — maintain test/helpers/exercised-commands.json, the + * observed half of the exercised-path consumer map (scripts/pr-dependencies.ts + * exercisedConsumers; shadow mode, docs/TESTING_INTERNALS.md#exercised-path-map). + * + * Refreshes are union-only: an observation adds commands, never removes one. + * A refresh whose input is unavailable or partial writes nothing, keeps the + * committed union and exits non-zero with a stable code naming what was + * missing: 2 usage, 3 input unavailable (missing directory, failed or HTTP 403 + * download), 4 input incomplete (no shards, no trial outcomes, an executed case + * without a shard artifact, a missing or malformed exercised-commands.json). + * Removing a command is a reviewed edit of the JSON file. + * + * Usage: + * bun run test:exercised-map --from-run [--out ] + * download that census run's exercised-map artifact and union it into the committed map + * bun run test:exercised-map --from-dir [--out ] + * the same from a locally downloaded artifact (the fallback when the download hits HTTP 403) + * bun run test:exercised-map --from-dir [--from-dir ...] --run-id [--out ] [--report ] + * read paid-slice / gate-census report directories (shards//...: transcripts, junit.xml, + * PTY terminal.visible.log, trial-outcomes.jsonl) and union their commands into the map + * bun run test:exercised-map --check + * validate the committed map + * bun run test:exercised-map --replay [] + * replay recorded red PR runs (default test/fixtures/exercised-map-replay.json) through + * today's selector and its shadow; a MISS is a failed gate case the shadow would not run + */ +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { + checkoutExercisedConsumers, EXERCISED_MAP_PATH, parseExercisedMap, type ExercisedCase, type ExercisedConsumers, type ExercisedMap, +} from './pr-dependencies'; +import { caseOwnerFile, EXERCISED_MAP_ANCHOR, PR_FULL_GATE_FILES, PR_PROFILE_MAPS } from './test-pr-profile'; +import { CASE_TEST_NAMES } from './lib/paid-cases'; +import { computePaidCaseSelection } from './test-paid-shards'; + +const ROOT = path.resolve(import.meta.dir, '..'); +export const EXERCISED_MAP_ARTIFACT = 'exercised-map'; + +/** Commands one shard directory recorded: per Bun test name, and shard-level (no test name). */ +export interface ShardCommands { shard: string; executed: string[]; perTest: Record; shardLevel: string[] } + +function walk(dir: string): string[] { + return fs.readdirSync(dir, { withFileTypes: true }).flatMap(entry => entry.isDirectory() ? walk(path.join(dir, entry.name)) : [path.join(dir, entry.name)]); +} + +function toolCommands(node: unknown, out: string[]): void { + if (Array.isArray(node)) { for (const item of node) toolCommands(item, out); return; } + if (!node || typeof node !== 'object') return; + const record = node as Record; + if (record.type === 'tool_use' && typeof record.input?.command === 'string') out.push(record.input.command); + for (const value of Object.values(record)) if (value && typeof value === 'object') toolCommands(value, out); +} + +/** Every `shards//` directory under `dir`, with the Bash commands its transcripts, JSONL logs and PTY screens recorded. */ +export function readShardCommands(dir: string): ShardCommands[] { + if (!fs.existsSync(dir)) return []; + const shardDirs = walk(dir).map(file => path.relative(dir, file).split(path.sep)).flatMap(parts => { + const at = parts.lastIndexOf('shards'); + return at >= 0 && parts.length > at + 2 ? [path.join(dir, ...parts.slice(0, at + 2))] : []; + }); + return [...new Set(shardDirs)].sort().map(shardDir => { + const record: ShardCommands = { shard: path.basename(shardDir), executed: [], perTest: {}, shardLevel: [] }; + for (const file of walk(shardDir)) { + const base = path.basename(file); + try { + if (base === 'junit.xml') { + for (const match of fs.readFileSync(file, 'utf8').matchAll(/]*?)(\/>|>([\s\S]*?)<\/testcase>)/g)) { + const name = /\bname="([^"]*)"/.exec(match[1]!)?.[1]; + if (name && !/ test?.transcript)) { + for (const test of parsed.tests) { const out: string[] = []; toolCommands(test.transcript ?? [], out); if (out.length) (record.perTest[test.name] ??= []).push(...out); } + } else toolCommands(parsed, record.shardLevel); + } else if (base.endsWith('.jsonl')) { + for (const line of fs.readFileSync(file, 'utf8').split('\n')) if (line.includes('tool_use')) { try { toolCommands(JSON.parse(line), record.shardLevel); } catch { /* partial line */ } } + } else if (base === 'terminal.visible.log') { + for (const match of fs.readFileSync(file, 'utf8').matchAll(/Bash\(([^\r\n]{0,400})/g)) record.shardLevel.push(`PTY:${match[1]}`); + } + } catch { /* unreadable evidence file: its commands are simply not observed */ } + } + record.shardLevel = [...new Set(record.shardLevel)]; + for (const name of Object.keys(record.perTest)) record.perTest[name] = [...new Set(record.perTest[name])]; + return record; + }); +} + +const testNameCase = (name: string, ids: string[]): string | null => + ids.includes(name) ? name : Object.keys(CASE_TEST_NAMES).find(id => CASE_TEST_NAMES[id] === name && ids.includes(id)) ?? ids.find(id => name.includes(id)) ?? null; + +/** + * Raw commands per case. A case shard (`--`) is that case; a file + * shard's per-test commands go to the case its test name names, and its + * file-level commands to every case of the file that ran (by JUnit name, else + * all of them): conservative over-attribution, as in the measured prototype. + */ +export function attributeCommands(shards: ShardCommands[], maps = PR_PROFILE_MAPS): Map { + const ids = Object.keys(maps.e2eTouchfiles); + const byFile = new Map(); + for (const id of ids) { const owner = caseOwnerFile(id, maps); if (owner) byFile.set(owner, [...byFile.get(owner) ?? [], id]); } + const out = new Map(); + const add = (id: string, commands: string[]) => out.set(id, [...out.get(id) ?? [], ...commands]); + for (const shard of shards) { + const [stem, caseKey] = shard.shard.split('--'); + const caseId = caseKey?.replace(/\.t\d+$/, ''); + const owned = byFile.get(`test/${stem}.test.ts`) ?? []; + const ran = owned.filter(id => shard.executed.some(name => testNameCase(name, [id]) === id)); + const targets = caseId && ids.includes(caseId) ? [caseId] : ran.length ? ran : owned; + for (const id of targets) add(id, shard.shardLevel); + for (const [name, commands] of Object.entries(shard.perTest)) { + const id = testNameCase(name, targets) ?? testNameCase(name, ids); + for (const target of id ? [id] : targets) add(target, commands); + } + } + return out; +} + +/** Known trial cost per case summed over the trial-outcomes.jsonl files under `dir` (executed trials only). */ +export function readCaseCosts(dir: string): Map { + const out = new Map(); + if (!fs.existsSync(dir)) return out; + for (const file of walk(dir).filter(file => path.basename(file) === 'trial-outcomes.jsonl')) { + for (const line of fs.readFileSync(file, 'utf8').split('\n').filter(Boolean)) { + try { + const record = JSON.parse(line); + if (record.execution === 'executed' && record.cost_known !== false && typeof record.case === 'string') out.set(record.case, (out.get(record.case) ?? 0) + (Number(record.cost_usd) || 0)); + } catch { /* malformed line */ } + } + } + return out; +} + +/** One run's observations as a map: the bins each attributed case invoked, and its known cost. */ +export function observeRun(commandsByCase: Map, ex: Pick, run: { runId: string; observedAt: string; costs?: Map }): ExercisedMap { + const cases: Record = {}; + for (const [id, commands] of commandsByCase) { + cases[id] = { observations: 1, withCommands: commands.length ? 1 : 0, commands: [...new Set(commands.flatMap(command => ex.binsInCommand(command)))].sort() }; + } + for (const [id, cost] of run.costs ?? []) { + cases[id] ??= { observations: 1, withCommands: 0, commands: [] }; + Object.assign(cases[id], { costUsd: Math.round(cost * 10000) / 10000, costRuns: 1 }); + } + return { schema: 1, generatedAt: run.observedAt, observedThrough: run.observedAt, sourceRuns: [run.runId], cases }; +} + +/** + * Union-only merge: commands are unioned, counts add for runs the base has + * not seen, costs average by run count, and nothing the base holds is ever + * removed (an absent or empty case in `fresh` changes nothing). + */ +export function mergeExercisedMap(base: ExercisedMap, fresh: ExercisedMap, generatedAt = new Date().toISOString()): ExercisedMap { + const counts = fresh.sourceRuns.some(run => !base.sourceRuns.includes(run)); + const cases: Record = structuredClone(base.cases); + for (const [id, entry] of Object.entries(fresh.cases)) { + const old = cases[id]; + const merged: ExercisedCase = { + observations: (old?.observations ?? 0) + (counts ? entry.observations : 0), + withCommands: (old?.withCommands ?? 0) + (counts ? entry.withCommands : 0), + commands: [...new Set([...old?.commands ?? [], ...entry.commands])].sort(), + }; + const oldRuns = old?.costRuns ?? 0, freshRuns = counts && entry.costUsd !== undefined ? entry.costRuns ?? 1 : 0; + if (freshRuns) Object.assign(merged, { costUsd: Math.round(((old?.costUsd ?? 0) * oldRuns + entry.costUsd! * freshRuns) / (oldRuns + freshRuns) * 10000) / 10000, costRuns: oldRuns + freshRuns }); + else if (old?.costUsd !== undefined) Object.assign(merged, { costUsd: old.costUsd, costRuns: oldRuns }); + cases[id] = merged; + } + return { + schema: 1, generatedAt, + observedThrough: [base.observedThrough, fresh.observedThrough].sort().at(-1)!, + sourceRuns: [...new Set([...base.sourceRuns, ...fresh.sourceRuns])].sort(), + cases: Object.fromEntries(Object.entries(cases).sort(([a], [b]) => a.localeCompare(b))), + }; +} + +export function formatExercisedMap(map: ExercisedMap): string { + return `${JSON.stringify(map, null, 2)}\n`; +} + +/** `MAP_GAP `: a recorded command whose files all lie outside the case's static-plus-observed exercised closure. */ +export function mapGaps(commandsByCase: Map, ex: ExercisedConsumers): Array<{ id: string; command: string }> { + const gaps: Array<{ id: string; command: string }> = []; + for (const [id, commands] of [...commandsByCase].sort(([a], [b]) => a.localeCompare(b))) { + const closure = ex.closure(id); + for (const command of [...new Set(commands.flatMap(raw => ex.binsInCommand(raw)))].sort()) { + const files = ex.commandFiles(command); + if (files.length && !files.some(file => closure.has(file))) gaps.push({ id, command }); + } + } + return gaps; +} + +/** Executed cases that recorded no command although their loaded skill's shell runs bins: blind to gap detection. */ +export function blindCases(commandsByCase: Map, ex: ExercisedConsumers): string[] { + return [...commandsByCase].filter(([id, commands]) => commands.length === 0 + && [...ex.closure(id)].some(([file, label]) => file.startsWith('bin/') && label.startsWith('skill bash via'))).map(([id]) => id).sort(); +} + +/** Stable exit codes of test:exercised-map (docs/TESTING_INTERNALS.md#exercised-path-map). */ +export const REFRESH_EXIT = { usage: 2, unavailable: 3, incomplete: 4 } as const; + +/** A refresh that cannot be trusted: nothing is written and the committed union stays as it was. */ +export class RefreshError extends Error { + constructor(readonly exitCode: typeof REFRESH_EXIT.unavailable | typeof REFRESH_EXIT.incomplete, message: string) { super(message); } +} + +/** One MAP_GAP line, saying who it is for and what follows from it. */ +export function formatMapGap(gap: { id: string; command: string }): string { + return `MAP_GAP ${gap.id} ${gap.command}: non-blocking for this PR; the next census refresh widens the map (${EXERCISED_MAP_ANCHOR})`; +} + +function writeAtomically(file: string, text: string): void { + fs.mkdirSync(path.dirname(path.resolve(file)), { recursive: true }); + const temp = `${file}.${process.pid}.tmp`; + fs.writeFileSync(temp, text); + fs.renameSync(temp, file); +} + +/** + * Census refresh from local report directories (paid-slice artifacts under + * shards/, plus trial-outcomes.jsonl). Every executed case must have shard + * evidence; a missing directory, no shards, no trial outcomes or an executed + * case without a shard throws before anything is written. + */ +export function refreshFromReportDirs(dirs: string[], runId: string, base: ExercisedMap, ex: ExercisedConsumers, observedAt = new Date().toISOString(), maps = PR_PROFILE_MAPS) { + const missing = dirs.filter(dir => !fs.existsSync(dir)); + if (missing.length) throw new RefreshError(REFRESH_EXIT.unavailable, `report director${missing.length > 1 ? 'ies' : 'y'} not found: ${missing.join(', ')}`); + const shards = dirs.flatMap(readShardCommands); + if (shards.length === 0) throw new RefreshError(REFRESH_EXIT.incomplete, `no shards// directories under ${dirs.join(', ')}; download every paid-slice-* artifact of the run first`); + const outcomes = dirs.flatMap(dir => walk(dir).filter(file => path.basename(file) === 'trial-outcomes.jsonl')); + if (outcomes.length === 0) throw new RefreshError(REFRESH_EXIT.incomplete, `no trial-outcomes.jsonl under ${dirs.join(', ')}, so the executed cases cannot be checked against the shard evidence`); + const commands = attributeCommands(shards, maps); + const executed = new Set(outcomes.flatMap(file => fs.readFileSync(file, 'utf8').split('\n').flatMap(line => { + try { const record = JSON.parse(line); return record.execution === 'executed' && typeof record.case === 'string' ? [record.case as string] : []; } catch { return []; } + }))); + const uncovered = [...executed].filter(id => Object.hasOwn(maps.e2eTouchfiles, id) && !commands.has(id)).sort(); + if (uncovered.length) throw new RefreshError(REFRESH_EXIT.incomplete, `${uncovered.length} executed case(s) have no shard artifact: ${uncovered.slice(0, 10).join(', ')}${uncovered.length > 10 ? ' ...' : ''}`); + const costs = new Map(); + for (const dir of dirs) for (const [id, cost] of readCaseCosts(dir)) costs.set(id, (costs.get(id) ?? 0) + cost); + const merged = mergeExercisedMap(base, observeRun(commands, ex, { runId, observedAt, costs }), observedAt); + return { merged, commands }; +} + +function census(dirs: string[], runId: string, out: string, reportPath: string | undefined): void { + const ex = checkoutExercisedConsumers(PR_PROFILE_MAPS, ROOT, PR_FULL_GATE_FILES); + const base = ex.map!; + const { merged, commands } = refreshFromReportDirs(dirs, runId, base, ex); + writeAtomically(out, formatExercisedMap(merged)); + const added = Object.entries(merged.cases).flatMap(([id, entry]) => entry.commands.filter(command => !base.cases[id]?.commands.includes(command)).map(command => `${id} ${command}`)); + const gaps = mapGaps(commands, ex); + const blind = blindCases(commands, ex); + const lines = [ + '## Exercised-path map (shadow, non-blocking)', '', + `- ${commands.size} executed case(s), ${[...commands.values()].reduce((sum, list) => sum + list.length, 0)} recorded command(s); ${added.length} new observed edge(s).`, + `- Refresh: \`bun run test:exercised-map --from-run ${runId}\`, then open the change as an ordinary PR (shadow-only data: it selects no paid case).`, + `- MAP_GAP (a command outside its case's static-plus-observed closure): ${gaps.length}`, + ...gaps.map(gap => ` - ${formatMapGap(gap)}`), + `- Cases that recorded no commands although their skill runs bins (PTY capture is the activation prerequisite): ${blind.length ? blind.join(', ') : 'none'}`, + ...added.slice(0, 50).map(edge => ` - new: ${edge}`), + `- Docs: ${EXERCISED_MAP_ANCHOR}`, + ]; + console.log(lines.join('\n')); + if (reportPath) fs.writeFileSync(reportPath, `${lines.join('\n')}\n`); +} + +/** Union every exercised-commands.json of a downloaded exercised-map artifact into `base`; any missing or malformed file throws. */ +export function mergeArtifactDir(dir: string, base: ExercisedMap, generatedAt = new Date().toISOString()): ExercisedMap { + const files = fs.existsSync(dir) ? walk(dir).filter(file => path.basename(file) === 'exercised-commands.json') : []; + if (files.length === 0) throw new RefreshError(REFRESH_EXIT.incomplete, `no exercised-commands.json in ${dir} (the census refresh step failed or its ${EXERCISED_MAP_ARTIFACT}-a artifact is partial)`); + let map = base; + for (const file of files.sort()) { + let fresh: ExercisedMap; + try { fresh = parseExercisedMap(fs.readFileSync(file, 'utf8')); } + catch (error) { throw new RefreshError(REFRESH_EXIT.incomplete, `${path.relative(dir, file)} is not a complete exercised map: ${error instanceof Error ? error.message : String(error)}`); } + map = mergeExercisedMap(map, fresh, generatedAt); + } + return map; +} + +export type ArtifactDownload = (runId: string, dir: string) => { status: number | null; stderr: string }; + +const ghDownload: ArtifactDownload = (runId, dir) => { + const result = spawnSync('gh', ['run', 'download', runId, '--pattern', `${EXERCISED_MAP_ARTIFACT}-*`, '--dir', dir], { encoding: 'utf8', timeout: 120_000 }); + return { status: result.status, stderr: result.stderr || result.error?.message || '' }; +}; + +/** Download a census run's exercised-map artifact and union it into `target`; on any failure nothing is written. */ +export function refreshFromRun(runId: string, target: string, download: ArtifactDownload = ghDownload): ExercisedMap { + if (!/^\d+$/.test(runId)) throw new RefreshError(REFRESH_EXIT.unavailable, `--from-run needs a numeric GitHub run id, got ${runId}`); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'exercised-map-')); + try { + const result = download(runId, dir); + if (result.status !== 0) { + const reason = result.stderr.trim(); + const hint = /\b403\b/.test(reason) + ? `HTTP 403 (the installation's artifact download limit); download the ${EXERCISED_MAP_ARTIFACT}-a artifact from the run page and pass its directory with --from-dir` + : `check \`gh auth status\` and that run ${runId} uploaded an ${EXERCISED_MAP_ARTIFACT}-a artifact`; + throw new RefreshError(REFRESH_EXIT.unavailable, `gh run download ${runId} failed: ${reason || `exit ${result.status}`}; ${hint}`); + } + const map = mergeArtifactDir(dir, parseExercisedMap(fs.readFileSync(target, 'utf8'))); + writeAtomically(target, formatExercisedMap(map)); + return map; + } finally { fs.rmSync(dir, { recursive: true, force: true }); } +} + +export const REPLAY_FIXTURE = 'test/fixtures/exercised-map-replay.json'; + +export interface ReplayRun { runId: number; pr: number; packageVersionOnly: boolean; failedGateCases: string[]; changedFiles: string[] } + +/** One recorded red run under today's selection and its shadow: which failed gate cases each would run. */ +export interface ShadowReplayRow { runId: number; pr: number; mode: string; today: number; shadow: number; savingUsd: number; caughtToday: string[]; misses: string[]; retired: string[] } + +export function replayShadow(runs: readonly ReplayRun[]): ShadowReplayRow[] { + const gateIds = Object.keys(PR_PROFILE_MAPS.tiers).filter(id => PR_PROFILE_MAPS.tiers[id] === 'gate'); + return runs.map(run => { + const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: run.changedFiles, packageVersionOnly: run.packageVersionOnly }); + const today = new Set(result.selection.e2e ?? gateIds); + const shadow = result.coverage?.shadow; + if (shadow?.error) throw new Error(`run ${run.runId}: shadow unavailable (${shadow.error})`); + const shadowSet = new Set(shadow?.e2e ?? today); + const live = run.failedGateCases.filter(id => PR_PROFILE_MAPS.tiers[id] === 'gate'); + return { runId: run.runId, pr: run.pr, mode: result.coverage?.mode ?? 'full', today: today.size, shadow: shadowSet.size, savingUsd: shadow?.predictedSavingUsd ?? 0, + caughtToday: live.filter(id => today.has(id)), misses: live.filter(id => !shadowSet.has(id)), retired: run.failedGateCases.filter(id => !live.includes(id)) }; + }); +} + +export function formatShadowReplay(rows: readonly ShadowReplayRow[]): string[] { + const pairs = (list: readonly ShadowReplayRow[]) => list.reduce((sum, row) => sum + new Set([...row.caughtToday, ...row.misses]).size, 0); + const misses = rows.flatMap(row => row.misses.map(id => `${row.runId} #${row.pr} ${id}`)); + const modes = new Map(); + for (const row of rows) modes.set(row.mode, (modes.get(row.mode) ?? 0) + 1); + const mean = (pick: (row: ShadowReplayRow) => number) => rows.length ? (rows.reduce((sum, row) => sum + pick(row), 0) / rows.length).toFixed(1) : 'n/a'; + return [ + `Exercised-path shadow replay over ${rows.length} recorded red PR run(s) on this checkout`, + `- failed gate (run, case) pairs: ${pairs(rows)}, ${pairs(rows.filter(row => row.mode !== 'full-fallback'))} of them in runs that do not fall back to the full gate (${rows.reduce((sum, row) => sum + row.retired.length, 0)} more on cases no longer in the gate)`, + `- today's selection runs ${rows.reduce((sum, row) => sum + row.caughtToday.length, 0)} of them; modes ${[...modes].map(([mode, n]) => `${mode} ${n}`).join(', ')}`, + `- mean gate cases: today ${mean(row => row.today)}, shadow ${mean(row => row.shadow)}; predicted shadow saving over these runs $${rows.reduce((sum, row) => sum + row.savingUsd, 0).toFixed(2)}`, + `- MISSES (failed, not in the shadow selection): ${misses.length}`, + ...misses.map(miss => ` - ${miss}`), + ]; +} + +const USAGE = 'usage: bun run test:exercised-map --from-run [--out ] | --from-dir ... [--run-id ] [--out ] [--report ] | --check | --replay []'; + +if (import.meta.main) { + const argv = process.argv.slice(2); + const values = (flag: string) => argv.flatMap((arg, index) => arg === flag && argv[index + 1] ? [argv[index + 1]!] : []); + const out = values('--out')[0] ?? path.join(ROOT, EXERCISED_MAP_PATH); + try { + if (argv.includes('--check')) { + const map = parseExercisedMap(fs.readFileSync(path.join(ROOT, EXERCISED_MAP_PATH), 'utf8')); + console.log(`${EXERCISED_MAP_PATH}: ${Object.keys(map.cases).length} cases, ${map.sourceRuns.length} source runs, observed through ${map.observedThrough}`); + } else if (argv.includes('--replay')) { + const fixture = JSON.parse(fs.readFileSync(path.resolve(ROOT, values('--replay')[0] ?? REPLAY_FIXTURE), 'utf8')) as { runs: ReplayRun[] }; + const rows = replayShadow(fixture.runs); + console.log(formatShadowReplay(rows).join('\n')); + if (rows.some(row => row.misses.length)) process.exit(1); + } else if (values('--from-run').length) { + const runId = values('--from-run')[0]!; + const map = refreshFromRun(runId, out); + console.log(`union-merged run ${runId}'s ${EXERCISED_MAP_ARTIFACT} artifact into ${path.relative(ROOT, out)} (${Object.keys(map.cases).length} cases, observed through ${map.observedThrough}); review the diff and open it as a PR`); + } else if (values('--from-dir').length) { + const dirs = values('--from-dir'); + if (dirs.length === 1 && fs.existsSync(dirs[0]!) && readShardCommands(dirs[0]!).length === 0 && walk(dirs[0]!).some(file => path.basename(file) === 'exercised-commands.json')) { + const map = mergeArtifactDir(dirs[0]!, parseExercisedMap(fs.readFileSync(out, 'utf8'))); + writeAtomically(out, formatExercisedMap(map)); + console.log(`union-merged the ${EXERCISED_MAP_ARTIFACT} artifact in ${dirs[0]} into ${path.relative(ROOT, out)}; review the diff and open it as a PR`); + } else { + const runId = values('--run-id')[0]; + if (!runId) { console.error(`test:exercised-map: --from-dir with report directories needs --run-id (the run whose artifacts these are)\n${USAGE}`); process.exit(REFRESH_EXIT.usage); } + census(dirs, runId, out, values('--report')[0]); + } + } else { + console.error(USAGE); + process.exit(REFRESH_EXIT.usage); + } + } catch (error) { + console.error(`test:exercised-map: ${error instanceof Error ? error.message : String(error)}${error instanceof RefreshError ? '; nothing was written, the committed union is unchanged' : ''}`); + process.exit(error instanceof RefreshError ? error.exitCode : 1); + } +} diff --git a/scripts/harness-version.json b/scripts/harness-version.json index f37b3d4c3..a306a10bc 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -12,11 +12,11 @@ "files": { "scripts/lib/paid-cases.ts": {"blob":"9bacd52446192a7ecf3dfaae901ea443cf9c9ae2","since":5}, "scripts/lib/paid-plan.ts": {"blob":"45721ce128a112666a0caed6c13c66637b74ca54","since":5}, - "scripts/lib/paid-report.ts": {"blob":"3fda96a8b1f7f3d55797934847353f254cc4d054","since":5}, + "scripts/lib/paid-report.ts": {"blob":"b66c41237611ccecaaa7bd6a4d7b7aac8dc548cd","since":5,"nonBehavioral":"A2 exercised-path map: test-pr-profile.ts adds a shadow block and classifies the shadow-only map file as selecting no case (selection only); paid-report.ts writes non-blocking MAP_GAP lines after the verdict inputs, inside try/catch, never a verdict or lane problem"}, "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":5}, "scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":5}, "scripts/test-paid-shards.ts": {"blob":"8ddee5f11ed7856876135520dd8025ab989e2db9","since":5}, - "scripts/test-pr-profile.ts": {"blob":"64cd7acfc0f2a4f53472bbdc8a50216984230c5a","since":5,"nonBehavioral":"PR selection only (A1 merged with #3253's free-only entry): shebang and per-extension import scanning, fail-closed scan errors, free-only repository-config files, capy/ derivable, the full-gate fallback label; it changes which cases a PR runs, never how any case is judged"}, + "scripts/test-pr-profile.ts": {"blob":"2029a89e14f511866cc91bebfee00b4fe9e7cf63","since":5,"nonBehavioral":"A2 exercised-path map: test-pr-profile.ts adds a shadow block and classifies the shadow-only map file as selecting no case (selection only); paid-report.ts writes non-blocking MAP_GAP lines after the verdict inputs, inside try/catch, never a verdict or lane problem"}, "scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":5}, "test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":5}, "test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":5}, diff --git a/scripts/lib/paid-report.ts b/scripts/lib/paid-report.ts index 3fda96a8b..b66c41237 100644 --- a/scripts/lib/paid-report.ts +++ b/scripts/lib/paid-report.ts @@ -22,6 +22,9 @@ import { type ManifestEntry, PAID_TEST_DURATIONS_FILE, type PaidRunManifest, typ import { DEFAULT_JOBS, type PaidCaseSelection, type PaidTier, ROOT, type ShardTrialRecord } from './paid-types'; import { collectPaidTestFiles, isAllSkippedPass, shardSlug, paidSelectionEnv } from './paid-select'; import type { RunShardsOptions } from '../test-paid-shards'; +import { attributeCommands, formatMapGap, mapGaps, readShardCommands } from '../exercised-map'; +import { checkoutExercisedConsumers } from '../pr-dependencies'; +import { PR_FULL_GATE_FILES, PR_PROFILE_MAPS } from '../test-pr-profile'; // ─── Local diagnosis: one case through the CI panel runner (A9) ──────────── @@ -482,6 +485,30 @@ export function infraOnly(problems: readonly string[]): boolean { return problems.length > 0 && problems.every(problem => INFRA_PROBLEMS.some(re => re.test(problem))); } +/** + * Shadow gap detection (docs/TESTING_INTERNALS.md#exercised-path-map): a + * command an executed case ran outside its static-plus-observed exercised + * closure prints a MAP_GAP line (formatMapGap: non-blocking for this PR, the + * next census refresh widens the map) and lands in map-gaps.txt. It is its own + * result: never a case verdict, a lane problem or a re-dispatch input. + */ +export function reportMapGaps(reportDir: string, executedCases: ReadonlySet, rootDir = ROOT): string[] { + const out = path.join(reportDir, 'map-gaps.txt'); + try { + const commands = new Map([...attributeCommands(readShardCommands(reportDir))].filter(([id, list]) => executedCases.has(id) && list.length > 0)); + const lines = commands.size === 0 ? [] : mapGaps(commands, checkoutExercisedConsumers(PR_PROFILE_MAPS, rootDir, PR_FULL_GATE_FILES)).map(formatMapGap); + fs.writeFileSync(out, lines.length ? `${lines.join('\n')}\n` : ''); + console.log(`[test:paid] exercised-path map (shadow, non-blocking): ${lines.length} MAP_GAP line(s) over ${commands.size} executed case(s) with recorded commands`); + for (const line of lines) console.log(` ${line}`); + return lines; + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); + fs.writeFileSync(out, `MAP_GAP unavailable: ${reason}\n`); + console.log(`[test:paid] exercised-path map (shadow, non-blocking): MAP_GAP unavailable: ${reason}`); + return []; + } +} + /** * Report mode: reconcile slice artifacts against the manifest (fail-closed), * compute every panel verdict with panelVerdict(), write collector-outcomes @@ -743,6 +770,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo ...(t.exit_reason ? { exit_reason: t.exit_reason } : {}), ...(t.error ? { error: t.error } : {}) })) })), // evals.yml wraps these in a fixed fence for the PR comment. failures: failureLines.map(line => sanitizeFixedFenceLine(line)) }, null, 2) + '\n'); + reportMapGaps(reportDir, new Set(executed.map(record => record.case)), rootDir); if (verdict.problems.length) { console.error(`[test:paid] report: ${verdict.problems.length} problem(s):`); for (const problem of verdict.problems) console.error(` ✗ ${problem}`); diff --git a/scripts/pr-dependencies.ts b/scripts/pr-dependencies.ts index a6dc94e3a..077e01f71 100644 --- a/scripts/pr-dependencies.ts +++ b/scripts/pr-dependencies.ts @@ -90,6 +90,8 @@ export const SELECTION_DATA_MODULES: readonly string[] = [ 'scripts/build.sh', // The reason-code table: its `fix:` prose names the command a person reruns, not a file the module consumes. 'lib/result-codes.ts', 'lib/result-codes-release.ts', + // The exercised-path map: observed commands per case, read only by exercisedConsumers() (shadow data; a change selects no paid case). + 'test/helpers/exercised-commands.json', ]; const DEFAULT_JUDGE_OWNER = 'test/skill-llm-eval.test.ts'; @@ -326,3 +328,295 @@ export function derivedDependencies(maps: DependencyMaps, root: string): Derived return cache.get(key)!; } +// ─── Exercised-path consumer map (W2-A2, shadow mode) ────────────────────── + +/** The committed per-case observed commands (generated by scripts/exercised-map.ts; union-only refreshes). */ +export const EXERCISED_MAP_PATH = 'test/helpers/exercised-commands.json'; + +export interface ExercisedCase { + /** Recorded trials (run, case) the commands were taken from. */ + observations: number; + /** Of those, trials that recorded at least one command. */ + withCommands: number; + /** Bins and compiled outfiles the case invoked (binsInCommand over every recorded command). */ + commands: string[]; + /** Mean known trial cost per run, for the shadow cost estimate. */ + costUsd?: number; + costRuns?: number; +} + +export interface ExercisedMap { + schema: 1; + generatedAt: string; + observedThrough: string; + sourceRuns: string[]; + cases: Record; +} + +export function isExercisedMap(value: unknown): value is ExercisedMap { + const map = value as ExercisedMap; + return !!map && map.schema === 1 && typeof map.generatedAt === 'string' && typeof map.observedThrough === 'string' + && Array.isArray(map.sourceRuns) && map.sourceRuns.every(run => typeof run === 'string') + && !!map.cases && typeof map.cases === 'object' && Object.values(map.cases).every(entry => !!entry + && Number.isSafeInteger(entry.observations) && Number.isSafeInteger(entry.withCommands) + && Array.isArray(entry.commands) && entry.commands.every(command => typeof command === 'string')); +} + +/** Parse an exercised map's text; throws on malformed input so callers fail closed. */ +export function parseExercisedMap(text: string): ExercisedMap { + const parsed = JSON.parse(text); + if (!isExercisedMap(parsed)) throw new Error(`${EXERCISED_MAP_PATH}: not a schema-1 exercised map`); + return parsed; +} + +export interface ExercisedConsumers { + /** Repository file -> case ids whose exercised closure contains it. */ + e2e: Map>; + /** One case's exercised closure: file -> how it is reached (touchfile, import, skill bash via , observed , declared via , host). */ + closure(caseId: string): Map; + /** Bins (repo paths) and compiled outfiles one recorded command invokes. */ + binsInCommand(command: string): string[]; + /** The tracked files a bin or compiled outfile runs. */ + commandFiles(command: string): string[]; + /** The observed map the consumers were built with. */ + map: ExercisedMap | null; +} + +const SHELL_SHEBANG = /^#![^\n]*\b(?:ba|z|da)?sh\b/; +const MESSAGE_CALL = /^(?:echo|printf|say|warn|die|fail|err|log|info|note|_dc_row|row)\b/; +const CMD_START = /(?:^|[;|&({\x60]|\$\(|\b(?:then|do|else|exec|command|nohup|bash|sh|bun|node|timeout\s+\S+|env(?:\s+\w+=\S*)*|if|while|until|!|time))\s*"?$/; +const CMD_PREFIX = /(?:"?(?:\$\{?\w+\}?|\$\(dirname\s+"?\$0"?\)|~|\.)?(?:\/(?:[\w.@-]+\/)*|(?:[\w.@-]+\/)+))?"?$/; +const INTERPRETER = /\b(?:bash|sh|zsh|bun|node|eval|source|python3?)\b/; + +/** Text between `$(`…`)` and backticks in a span: substitutions run even inside a message. */ +function substitutions(span: string): string[] { + const out: string[] = []; + for (let i = 0; i < span.length; i++) { + if (span[i] === '\\') { i++; continue; } + if (span[i] === '\x60') { const end = span.indexOf('\x60', i + 1); if (end < 0) break; out.push(`$(${span.slice(i + 1, end)})`); i = end; continue; } + if (span[i] !== '$' || span[i + 1] !== '(') continue; + let depth = 0, j = i + 1; + for (; j < span.length; j++) { if (span[j] === '(') depth++; else if (span[j] === ')' && --depth === 0) break; } + out.push(span.slice(i, j + 1)); + i = j; + } + return out; +} + +/** Message calls (echo, printf, die …) are prose; their command substitutions still execute and are kept. */ +function stripMessages(text: string): string { + let out = '', i = 0; + while (i < text.length) { + const prev = i === 0 ? '\n' : text[i - 1]!; + const call = /[\s;|&(\x60{]/.test(prev) ? MESSAGE_CALL.exec(text.slice(i, i + 8)) : null; + if (!call) { out += text[i]; i++; continue; } + let j = i + call[0].length, quote: string | null = null, depth = 0; + while (j < text.length) { + const c = text[j]!; + if (c === '\\') { j += 2; continue; } + if (quote) { if (c === quote) quote = null; else if (quote === '"' && c === '$' && text[j + 1] === '(') { depth++; j++; } else if (c === ')' && depth) depth--; j++; continue; } + if (c === '"' || c === "'") { quote = c; j++; continue; } + if (c === '$' && text[j + 1] === '(') { depth++; j += 2; continue; } + if (c === ')' && depth) { depth--; j++; continue; } + if (c === '\n' || c === ';' || c === '|' || c === '&' || c === ')') break; + j++; + } + out += ` ${substitutions(text.slice(i, j)).join('; ')} `; + i = j; + } + return out; +} + +/** + * Executable shell text: comments, case patterns and message calls are + * prose. A heredoc an interpreter reads stays whole; an expanding (unquoted) + * heredoc keeps its command substitutions; a quoted one fed to anything else + * is dropped. + */ +export function executableShell(text: string): string { + return stripMessages(text + .replace(/^(.*?)<<-?\s*(['"]?)(\w+)\2([^\n]*)\n([\s\S]*?)\n\s*\3\s*$/gm, (whole, head: string, quote: string, _word: string, rest: string, body: string) => + INTERPRETER.test(head + rest) ? whole : quote ? head : [head, ...substitutions(body)].join('\n')) + .split('\n').filter(line => !/^\s*#(?!!)/.test(line)) + .map(line => line.replace(/(^\s*|\bin\s+|;;\s*)[\w*."'-]+(?:\s*\|\s*[\w*."'-]+)*\)(?=\s|$)/g, '$1').replace(/\s#\s.*$/, '')) + .join('\n')); +} + +function commandPosition(before: string): boolean { + const head = before.replace(CMD_PREFIX, ''); + return CMD_START.test(head) || (/\/$/.test(before) && /\b\w+="?$/.test(head)); +} + +/** + * Consumers by exercised paths: a case consumes its touchfile matches, the + * closure of its test file, the executable shell of its loaded skill + * (template and generated SKILL.md) and of every bin, lib and script that + * shell runs, and the bins its recorded trials invoked (`observed`). The + * closure stops at `stopAt` (the full-gate files) and the global set, + * which stay full. + */ +export function exercisedConsumers(maps: DependencyMaps, root: string, options: { observed: ExercisedMap | null; stopAt: readonly string[] }): ExercisedConsumers { + const base = derivedDependencies(maps, root); + const tracked = [...base.tracked]; + const texts = new Map(); + const read = (file: string): string => { + if (!texts.has(file)) { + let text = ''; + try { if (base.tracked.has(file) && fs.statSync(path.join(root, file)).size < 3_000_000) text = fs.readFileSync(path.join(root, file), 'utf8'); } catch { /* unreadable */ } + texts.set(file, text); + } + return texts.get(file)!; + }; + const isShell = (file: string) => !isCodeFile(file, read(file)) && (/\.sh$/.test(file) || file === 'setup' || SHELL_SHEBANG.test(read(file))); + const dirs = new Set(tracked.flatMap(file => file.split('/').slice(0, -1).map((_, i, parts) => parts.slice(0, i + 1).join('/')))); + const expandRef = (ref: string): string[] => { + const clean = ref.replace(/[.]+$/, ''); + if (base.tracked.has(clean)) return [clean]; + return dirs.has(clean) ? tracked.filter(file => file.startsWith(`${clean}/`)) : []; + }; + const expanded = new Map(); + const expand = (pattern: string): string[] => { + if (!pattern.includes('*')) return base.tracked.has(pattern) ? [pattern] : []; + if (!expanded.has(pattern)) expanded.set(pattern, tracked.filter(file => matchGlob(file, pattern))); + return expanded.get(pattern)!; + }; + const binByName = new Map(); + for (const file of tracked.filter(file => file.startsWith('bin/') || file.startsWith('browse/bin/'))) { + binByName.set(path.posix.basename(file), file); + binByName.set(path.posix.basename(file).replace(/\.(?:ts|sh|js)$/, ''), file); + } + const binaries = [...read('scripts/build.sh').matchAll(/--compile\b[^\n]*?\s([\w./-]+\.ts)\s+--outfile\s+([\w./-]+)/g)] + .map(match => ({ outfile: match[2]!, tree: match[1]!.startsWith('bin/') ? [match[1]!] : tracked.filter(file => matchGlob(file, `${path.posix.dirname(match[1]!)}/**`)) })); + const outfileHit = (outfile: string, text: string) => text.includes(outfile) + || (outfile === 'browse/dist/browse' && /\$B\b|\$\{B\}|find-browse/.test(text)) || (outfile === 'design/dist/design' && /\$D\b|\$\{D\}/.test(text)); + let scripts: Record = {}; + try { scripts = JSON.parse(read('package.json')).scripts ?? {}; } catch { /* no package scripts */ } + + const shellEdges = (text: string, from: string, out: Set) => { + for (const match of text.matchAll(/gstack-[a-z0-9][a-z0-9-]*[a-z0-9](?:\.(?:sh|ts))?\b/g)) { + const start = text.lastIndexOf('\n', match.index! - 1) + 1; + const bin = binByName.get(match[0]); + if (bin && commandPosition(text.slice(start, match.index))) out.add(bin); + } + for (const match of text.matchAll(/(?:bin|lib|scripts|browse|design|make-pdf)\/[\w.@-]+(?:\/[\w.@-]+)*/g)) { + const before = text.slice(text.lastIndexOf('\n', match.index! - 1) + 1, match.index); + if (/[\w.@-]$/.test(before)) continue; + if (commandPosition(before) || /\bbun\s+(?:run\s+)?"?(?:\$\{?\w+\}?\/)?$/.test(before)) for (const file of expandRef(match[0])) out.add(file); + } + for (const match of text.matchAll(/(?:bin|lib|scripts|browse|design|make-pdf|hosts|capy|test\/fixtures|test\/helpers)\/[\w.@-]+(?:\/[\w.@-]+)*\.(?:json|jsonl|env|txt|ya?ml|toml|csv|tsv)\b/g)) for (const file of expandRef(match[0])) out.add(file); + for (const match of text.matchAll(/(?:^|[\s;&|({])(?:\.|source)\s+([^;|&\n]+)/gm)) { + for (const ref of match[1]!.matchAll(/(?:bin|lib|scripts)\/[\w.@-]+(?:\/[\w.@-]+)*\.sh\b/g)) for (const file of expandRef(ref[0])) out.add(file); + const name = [...match[1]!.matchAll(/([\w.-]+\.sh|gstack-[\w.-]+)/g)].map(found => found[1]!).at(-1); + if (!name) continue; + const sibling = path.posix.join(path.posix.dirname(from), name); + if (base.tracked.has(sibling)) out.add(sibling); + else if (binByName.has(name)) out.add(binByName.get(name)!); + } + if (/(?:^|[\s;&|(])"?(?:\.|\$\{?\w+\}?)\/setup\b/m.test(text) && base.tracked.has('setup')) out.add('setup'); + for (const { outfile, tree } of binaries) if (outfileHit(outfile, text)) for (const file of tree) out.add(file); + for (const match of text.matchAll(PACKAGE_RUN)) { + const command = scripts[match[1]!]; + if (command) for (const ref of command.matchAll(REPO_PATH)) for (const file of expandRef(ref[1]!)) out.add(file); + } + }; + const edges = new Map(); + const execRefs = (file: string): string[] => { + if (edges.has(file)) return edges.get(file)!; + const raw = read(file); + const out = new Set(); + if (raw && !(SELECTION_DATA_MODULES as readonly string[]).includes(file)) { + if (isCodeFile(file, raw)) { + try { for (const specifier of scanCodeImports(file, raw)) { const hit = resolveSpecifier(root, specifier, file); if (hit) out.add(hit); } } + catch (error) { base.scanErrors.set(file, String(error)); } + const code = raw.replace(/^#![^\n]*/, '').replace(/\/\*[\s\S]*?\*\//g, '').replace(/^[ \t]*\/\/.*$/gm, ''); + for (const match of code.matchAll(/['"\x60](?:\$\{[^}]+\}\/)?((?:bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents|test\/helpers|test\/fixtures)\/[\w.@-]+(?:\/[\w.@-]+)*)['"\x60]/g)) for (const hit of expandRef(match[1]!)) out.add(hit); + for (const match of code.matchAll(JOINED_PATH)) for (const hit of expandRef(`${match[1]}/${match[2]}`)) out.add(hit); + if (/join\([^)\n]*['"]setup['"]\s*\)/.test(code) && base.tracked.has('setup')) out.add('setup'); + for (const { outfile, tree } of binaries) if (code.includes(`'${outfile}'`) || code.includes(`"${outfile}"`)) for (const hit of tree) out.add(hit); + } else if (isShell(file)) { + shellEdges(executableShell(raw), file, out); + } else if (/\.(?:md|tmpl)$/.test(file)) { + shellEdges(executableShell([...raw.matchAll(/\x60\x60\x60(?:bash|sh|shell)\n([\s\S]*?)\x60\x60\x60/g)].map(match => match[1]!).join('\n')), file, out); + } + } + out.delete(file); + edges.set(file, [...out]); + return edges.get(file)!; + }; + const stopped = new Map(); + const stops = (file: string): boolean => { + if (!stopped.has(file)) stopped.set(file, base.global.has(file) || options.stopAt.some(pattern => matchGlob(file, pattern))); + return stopped.get(file)!; + }; + const binsInCommand = (command: string): string[] => { + const out = new Set(); + const pty = command.startsWith('PTY:'); + for (const match of command.matchAll(/gstack-[a-z0-9-]*[a-z0-9](?:\.(?:sh|ts))?/g)) { + let token = match[0]; + if (pty) while (token.length > 7 && !binByName.has(token)) { const cut = token.lastIndexOf('-'); token = cut > 6 ? token.slice(0, cut) : ''; } + const at = match.index!; + if (command[at - 1] === '/' && at >= 2 && /[\s"'\x60(]/.test(command[at - 2]!)) continue; + const bin = binByName.get(token); + if (bin) out.add(bin); + } + for (const match of command.matchAll(/browse\/bin\/([a-z-]+)/g)) { const bin = binByName.get(match[1]!); if (bin) out.add(bin); } + for (const { outfile } of binaries) if (outfileHit(outfile, command) || (outfile === 'make-pdf/dist/pdf' && command.includes(outfile))) out.add(outfile); + return [...out]; + }; + const commandFiles = (command: string) => base.tracked.has(command) ? [command] : binaries.find(binary => binary.outfile === command)?.tree ?? []; + + const hostIds = tracked.filter(file => /^hosts\/[\w-]+\.ts$/.test(file) && file !== 'hosts/index.ts').map(file => path.posix.basename(file, '.ts')); + const closures = new Map>(); + const caseClosure = (id: string): Map => { + if (closures.has(id)) return closures.get(id)!; + const patterns = maps.e2eTouchfiles[id] ?? []; + const owners = patterns.filter(pattern => OWNER.test(pattern) && isPaidTestFile(pattern)); + const declared = patterns.flatMap(expand); + const generated = declared.filter(file => file.endsWith('.tmpl')).map(file => file.replace(/\.tmpl$/, '')).filter(file => base.tracked.has(file)); + const touchfiles = new Set([...declared, ...generated]); + const skillDoc = (file: string) => /\.(?:md|tmpl)$/.test(file); + const seeds: Array<[string, string]> = [ + ...owners.map(file => [file, 'import'] as [string, string]), + ...[...declared, ...generated].map(file => [file, skillDoc(file) ? 'skill' : `declared via ${file}`] as [string, string]), + ...(options.observed?.cases[id]?.commands ?? []).flatMap(command => commandFiles(command).map(file => [file, `observed ${command}`] as [string, string])), + ]; + const ownerText = owners.map(read).join('\n'); + for (const host of hostIds) { + if (new RegExp(`['"\x60]${host}['"\x60]|--host[ =]${host}\\b`).test(ownerText)) for (const file of [`hosts/${host}.ts`, ...expand(`hosts/${host}/**`)]) seeds.push([file, 'host']); + } + const reached = new Map(); + const queue = seeds.filter(([file]) => base.tracked.has(file)); + for (let head = 0; head < queue.length; head++) { + const [file, label] = queue[head]!; + if (reached.has(file)) continue; + reached.set(file, touchfiles.has(file) ? 'touchfile' : label); + if (stops(file)) continue; + for (const next of execRefs(file)) { + if (reached.has(next) || (/\.test\.tsx?$/.test(next) && isPaidTestFile(next))) continue; + queue.push([next, label === 'skill' ? `skill bash via ${next}` : label]); + } + } + closures.set(id, reached); + return reached; + }; + const e2e = new Map>(); + for (const id of Object.keys(maps.e2eTouchfiles)) { + for (const file of caseClosure(id).keys()) { + if (!e2e.has(file)) e2e.set(file, new Set()); + e2e.get(file)!.add(id); + } + } + return { e2e, closure: caseClosure, binsInCommand, commandFiles, map: options.observed }; +} + +const exercisedCache = new Map(); + +/** The exercised consumers of the checkout at `root` with its committed map, computed once per process; throws when the map is malformed. */ +export function checkoutExercisedConsumers(maps: DependencyMaps, root: string, stopAt: readonly string[]): ExercisedConsumers { + const key = `${root}\0${Object.keys(maps.e2eTouchfiles).length}`; + if (!exercisedCache.has(key)) { + const observed = parseExercisedMap(fs.readFileSync(path.join(root, EXERCISED_MAP_PATH), 'utf8')); + exercisedCache.set(key, exercisedConsumers(maps, root, { observed, stopAt })); + } + return exercisedCache.get(key)!; +} diff --git a/scripts/test-health-report.ts b/scripts/test-health-report.ts index cf11e4f3e..bccc11639 100644 --- a/scripts/test-health-report.ts +++ b/scripts/test-health-report.ts @@ -228,6 +228,30 @@ export function collectHealth(inputs: HealthInputs): HealthReport { detail: modes.map(mode => { const list = entries.filter(entry => (modeOf.get(entry.run.id) ?? 'no manifest') === mode); return `${mode}: ${mean(list)} (${list.length} runs)`; }) }; })); + metrics.push(metric('pr-shadow-map', 'Exercised-path map shadow (not active)', 'over evals.yml pull_request runs whose paid-plan manifest carries prCoverage.shadow: the predicted saving (map per-case known cost of dropped minus added cases; a lower bound, since known cost excludes PTY trials with unknown cost) and shadow drops of red cases (more executed failed than passed trials in the run, on a case the shadow would drop). A drop counts toward the D3 activation bar as a miss unless the case passed in a later run of the same branch at the same head SHA, which marks it a flaky drop. Scaled to 7 days; docs/TESTING_INTERNALS.md#exercised-path-map', () => { + const shadowed = need(prManifests).filter(({ manifest }) => manifest.prCoverage?.shadow && !manifest.prCoverage.shadow.error); + if (shadowed.length === 0) throw new Error('no paid-plan manifest with a prCoverage.shadow block in the window'); + const records = prRecords instanceof Error ? [] : prRecords; + const executed = (trials: TrialRecord[], id: string, outcome: string) => trials.filter(record => record.case === id && record.execution === 'executed' && record.outcome === outcome).length; + const saving = shadowed.reduce((sum, { manifest }) => sum + (Number(manifest.prCoverage.shadow.predictedSavingUsd) || 0), 0); + const shares = shadowed.map(({ manifest }) => Number(manifest.prCoverage.shadow.unknownCostShare)).filter(share => Number.isFinite(share) && share > 0); + const misses: string[] = []; + const flaky: string[] = []; + for (const { run, manifest } of shadowed) { + const trials = records.find(entry => entry.run.id === run.id)?.trials ?? []; + for (const id of (manifest.prCoverage.shadow.wouldDrop ?? []) as string[]) { + if (executed(trials, id, 'failed') <= executed(trials, id, 'passed')) continue; + const laterPass = records.find(entry => entry.run.branch === run.branch && entry.run.sha === run.sha + && (entry.run.createdAt > run.createdAt || (entry.run.id === run.id && entry.run.attempt > run.attempt)) && executed(entry.trials, id, 'passed') > 0); + if (laterPass) flaky.push(`${run.id}:${id} (passed in run ${laterPass.run.id})`); else misses.push(`${run.id}:${id}`); + } + } + const week = 7 / sinceDays; + const bound = shares.length ? `a lower bound: known cost excludes the ${Math.round(100 * Math.max(...shares))}% of observed case runs with unknown cost (PTY)` : 'a lower bound: known cost excludes PTY trials with unknown cost'; + return { value: `$${(saving * week).toFixed(2)} predicted saving/week (${bound}); ${(misses.length * week).toFixed(1)} activation miss(es)/week, ${flaky.length} flaky drop(s) not counted (${shadowed.length} runs)`, + detail: [...misses.slice(0, 20).map(miss => `shadow miss: ${miss}`), ...flaky.slice(0, 20).map(drop => `flaky drop: ${drop}`)] }; + })); + metrics.push(metric('pr-reused-trials', 'Reused PR trial records', 'trial-outcomes records with execution = reused across non-cancelled PR runs', () => { const all = need(prRecords).flatMap(entry => entry.trials); const reused = all.filter(record => record.execution === 'reused').length; diff --git a/scripts/test-pr-profile.ts b/scripts/test-pr-profile.ts index 64cd7acfc..2029a89e1 100644 --- a/scripts/test-pr-profile.ts +++ b/scripts/test-pr-profile.ts @@ -5,7 +5,9 @@ import { E2E_TOUCHFILES, E2E_TIERS, GLOBAL_TOUCHFILES, LLM_JUDGE_TOUCHFILES } fr import { matchGlob, TOUCHFILES_DATA_PATH } from '../test/helpers/test-selection'; import { isPaidTestFile } from '../test/helpers/paid-test-set'; import { FREE_FIXTURES } from '../test/helpers/free-fixtures-data'; -import { derivedDependencies, type DerivedDependencies } from './pr-dependencies'; +import { + checkoutExercisedConsumers, derivedDependencies, EXERCISED_MAP_PATH, type DerivedDependencies, type ExercisedConsumers, +} from './pr-dependencies'; /** Existing short behavioral probes; intersect with changed-input selection. */ export const PR_PROFILE_CASE_IDS = [ @@ -157,6 +159,27 @@ export interface PrProfileSelection { missingCoverage: string[]; needsFullValidation: boolean; reasons: string[]; + /** What the exercised-path map would select (shadow mode: never changes e2e or judges). */ + shadow?: ShadowSelection; +} + +/** The exercised-path map's selection beside the real one (docs/TESTING_INTERNALS.md#exercised-path-map). */ +export interface ShadowSelection { + /** Gate cases the map would select. */ + e2e: string[]; + /** Selected gate cases the map would not select. */ + wouldDrop: string[]; + /** Gate cases the map would select that today's selection does not. */ + wouldAdd: string[]; + /** Per changed file with exercised consumers: how many gate cases it reaches, and through which edge. */ + provenance: Array<{ file: string; summary: string }>; + /** Mean known cost of the dropped cases minus the added ones (the map's per-case costUsd; an unpriced case counts $0): a lower bound. */ + predictedSavingUsd: number; + unpricedCases: number; + /** Share of observed case runs whose trials carry no known cost (PTY trials); their spend is missing from every dollar figure. */ + unknownCostShare: number; + /** Why no shadow could be computed (the real selection is unaffected). */ + error?: string; } /** Reject stale profile registrations and deferred cases with no scheduled home. */ @@ -297,6 +320,7 @@ export function knownNonBehaviorFile(file: string): boolean { } export const FALLBACK_FIX_ANCHOR = 'docs/TESTING_INTERNALS.md#pr-paid-lane-fallback'; +export const EXERCISED_MAP_ANCHOR = 'docs/TESTING_INTERNALS.md#exercised-path-map'; /** Why one changed file restored the full gate, and the edit that narrows it next time (DX-6). */ export function unknownFileLabel(file: string): { label: string; fix: string } { @@ -348,6 +372,8 @@ export function selectPrProfile(options: { readSource?: (file: string) => string; /** Derived references; default: derived from the checkout for the real maps, none for fixture maps. */ derived?: DerivedDependencies | null; + /** Exercised-path consumers for the shadow block; default: the checkout's committed map for the real maps, none for fixture maps. */ + exercised?: ExercisedConsumers | null; }): PrProfileSelection { const maps = options.maps ?? PR_PROFILE_MAPS; const profile = options.profile ?? PR_PROFILE_CASE_IDS; @@ -361,7 +387,7 @@ export function selectPrProfile(options: { ...Object.values(maps.e2eTouchfiles).flat(), ...Object.values(maps.judgeTouchfiles).flat(), ...maps.globalTouchfiles, ])]; - const unmapped = files.filter(file => file !== TOUCHFILES_DATA_PATH + const unmapped = files.filter(file => file !== TOUCHFILES_DATA_PATH && file !== EXERCISED_MAP_PATH && !depends(file, dependencyPatterns) && !knownNonBehaviorFile(file)); // Unmapped files are placed by derived references where the derivation covers them; the rest restore the full gate. const derived = options.derived !== undefined ? options.derived : maps === PR_PROFILE_MAPS ? derivedDependencies(maps, ROOT) : null; @@ -438,25 +464,80 @@ export function selectPrProfile(options: { if (sharedInputs.length) reasons.push(`Shared runtime/build inputs restore every gate case and judge: ${sharedInputs.join(', ')}`); if (dependentsMode) reasons.push(`Derived references select every dependent gate case: ${derivedFiles.map(file => `${file} (${consumers.get(file)!.e2e.size} cases)`).join(', ')}`); else if (!fallback) reasons.push('Changed-input selection intersected with the fast PR profile; selected judges retained'); + if (files.includes(EXERCISED_MAP_PATH)) reasons.push(`${EXERCISED_MAP_PATH} is shadow-only data (it changes no selection yet), so it selects no paid case (${EXERCISED_MAP_ANCHOR})`); if (derivedGlobal.length) reasons.push(`Paid-runner inputs (global touchfile import closure) select every case: ${derivedGlobal.join(', ')}`); if (noConsumerFiles.length) reasons.push(`No paid case's reference closure reaches (or, deleted, still references): ${noConsumerFiles.join(', ')}`); if (directCases.length) reasons.push(`Edited paid test files select their own gate cases: ${directCases.join(', ')}`); if (deferred.length) reasons.push(`${deferred.length} selected behaviors remain scheduled/release coverage, not PR passes`); if (deferredPromptFiles.length) reasons.push(`No quick live coverage; known broad prompt checks deferred: ${deferredPromptFiles.join(', ')}`); if (missingCoverage.length) reasons.push(`Full validation required for prompts without a relevant PR check: ${missingCoverage.join(', ')}`); + const exercised = options.exercised !== undefined ? options.exercised : maps === PR_PROFILE_MAPS ? checkoutExercised() : null; + const shadow = exercised instanceof Error ? emptyShadow(e2e, exercised.message) + : exercised ? shadowSelection({ ...options, maps, profile }, { e2e, fallback, derived, exercised, files }) : undefined; return { mode: fallback ? 'full-fallback' : dependentsMode ? 'dependents' : 'pr', e2e, judges, deferred, unknownFiles, unknownFileLabels: unknownFiles.map(file => ({ file, ...(deletedLabels.get(file) ?? unknownFileLabel(file)) })), directCases, derivedFiles: derivedFiles.map(file => ({ file, e2e: consumers.get(file)!.e2e.size, judges: consumers.get(file)!.judges.size })), noConsumerFiles, - deferredPromptFiles, missingCoverage, needsFullValidation: missingCoverage.length > 0, reasons, + deferredPromptFiles, missingCoverage, needsFullValidation: missingCoverage.length > 0, reasons, ...(shadow ? { shadow } : {}), }; } +function checkoutExercised(): ExercisedConsumers | Error { + try { return checkoutExercisedConsumers(PR_PROFILE_MAPS, ROOT, PR_FULL_GATE_FILES); } + catch (error) { return error instanceof Error ? error : new Error(String(error)); } +} + +function emptyShadow(e2e: string[], error: string): ShadowSelection { + return { e2e, wouldDrop: [], wouldAdd: [], provenance: [], predictedSavingUsd: 0, unpricedCases: 0, unknownCostShare: 0, error }; +} + +/** + * Shadow selection (variant cu of the measured prototype): the same policy + * with the exercised map in place of the derived closure, unioned with the + * exercised consumers of every changed file that is not global or full-gate. + * A full-gate fallback stays full. + */ +function shadowSelection(options: Parameters[0] & { maps: PrProfileMaps }, + today: { e2e: string[]; fallback: boolean; derived: DerivedDependencies | null; exercised: ExercisedConsumers; files: string[] }): ShadowSelection { + const { maps } = options; + const { exercised } = today; + const observedCases = Object.values(exercised.map?.cases ?? {}); + const observations = observedCases.reduce((sum, entry) => sum + entry.observations, 0); + const unknownCostShare = observations ? Math.round(1000 * (1 - observedCases.reduce((sum, entry) => sum + (entry.costRuns ?? 0), 0) / observations)) / 1000 : 0; + if (today.fallback) return { e2e: today.e2e, wouldDrop: [], wouldAdd: [], provenance: [], predictedSavingUsd: 0, unpricedCases: 0, unknownCostShare }; + const gate = (ids: Iterable) => [...ids].filter(id => maps.tiers[id] === 'gate'); + const reach = new Map(); + const inner = selectPrProfile({ ...options, derived: today.derived ? { ...today.derived, e2e: exercised.e2e } : null, exercised: null }); + const sources = [...new Set([...today.files, ...Object.values(options.sourceAliases ?? {})])]; + const shadowSet = new Set(inner.e2e); + for (const file of sources) { + if (today.derived?.global.has(file) || matches(file, PR_FULL_GATE_FILES) || knownNonBehaviorFile(file)) continue; + const via = !today.derived || today.derived.tracked.has(file) ? [file] : today.derived.referencers(file); + const ids = gate(new Set(via.flatMap(source => [...exercised.e2e.get(source) ?? []]))); + if (ids.length) reach.set(file, ids); + for (const id of ids) shadowSet.add(id); + } + const e2e = [...shadowSet].sort(); + const wouldDrop = today.e2e.filter(id => !shadowSet.has(id)); + const wouldAdd = e2e.filter(id => !today.e2e.includes(id)); + const cost = (ids: string[]) => ids.reduce((sum, id) => sum + (exercised.map?.cases[id]?.costUsd ?? 0), 0); + const provenance = [...reach].map(([file, ids]) => { + const groups = new Map(); + for (const id of ids) { + const label = exercised.closure(id).get(file) ?? 'referencer'; + groups.set(label, (groups.get(label) ?? 0) + 1); + } + return { file, summary: `selects ${ids.length} case(s): ${[...groups].sort((a, b) => b[1] - a[1]).slice(0, 6).map(([label, n]) => `${label} (${n})`).join(', ')}` }; + }); + return { e2e, wouldDrop, wouldAdd, provenance, predictedSavingUsd: Math.round((cost(wouldDrop) - cost(wouldAdd)) * 100) / 100, + unpricedCases: [...wouldDrop, ...wouldAdd].filter(id => exercised.map?.cases[id]?.costUsd === undefined).length, unknownCostShare }; +} + /** The manifest fields the coverage summary reads (a PaidRunManifest subset). */ export interface CoverageSummaryInput { profile?: string; selection?: { e2e: string[] | null; judges: string[] | null }; - prCoverage?: Pick & Partial>; + prCoverage?: Pick & Partial>; } /** @@ -477,6 +558,9 @@ export function formatPrCoverageSummary(manifest: CoverageSummaryInput, totals?: lines.push(`- Derived dependents (scripts/pr-dependencies.ts) of ${coverage.derivedFiles.length} unmapped file(s):`); for (const item of coverage.derivedFiles.slice(0, 40)) lines.push(` - \`${item.file}\`: ${item.e2e} case(s), ${item.judges} judge(s)`); } + const shadow = coverage?.shadow; + if (shadow) lines.push(shadow.error ? `- Shadow exercised-path map: unavailable (${shadow.error}); selection unaffected` + : `- Shadow exercised-path map (not active): would select ${shadow.e2e.length} gate case(s), dropping ${shadow.wouldDrop.length} and adding ${shadow.wouldAdd.length}; predicted saving $${shadow.predictedSavingUsd.toFixed(2)}${shadow.unpricedCases ? ` (${shadow.unpricedCases} unpriced)` : ''}, a lower bound: known cost excludes the ${Math.round(100 * (shadow.unknownCostShare ?? 0))}% of observed case runs with unknown cost (PTY) (${EXERCISED_MAP_ANCHOR})`); if (coverage?.noConsumerFiles?.length) lines.push(`- No paid case reaches: ${coverage.noConsumerFiles.slice(0, 40).map(file => `\`${file}\``).join(', ')}`); if (coverage?.mode === 'full-fallback' && coverage.unknownFiles.length) { lines.push(`- Full gate restored by ${coverage.unknownFiles.length} file(s) (fix: ${FALLBACK_FIX_ANCHOR}):`); diff --git a/test/exercised-map.test.ts b/test/exercised-map.test.ts new file mode 100644 index 000000000..9a3c5165a --- /dev/null +++ b/test/exercised-map.test.ts @@ -0,0 +1,357 @@ +/** + * Exercised-path consumer map (W2-A2, shadow mode): edge classes on a + * synthetic tree, the five variant-c miss shapes as negative controls, + * union-only refreshes, the refresh contract on unavailable or partial input, + * shadow-only map data selecting nothing, shadow output that never changes the + * real selection, MAP_GAP detection, and parity with the + * measured prototype (docs/TESTING_INTERNALS.md#exercised-path-map). + */ +import { describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { + deriveDependencies, EXERCISED_MAP_PATH, exercisedConsumers, parseExercisedMap, type ExercisedMap, +} from '../scripts/pr-dependencies'; +import { + formatPrCoverageSummary, PR_FULL_GATE_FILES, PR_PROFILE_MAPS, selectPrProfile, +} from '../scripts/test-pr-profile'; +import { + attributeCommands, blindCases, formatExercisedMap, formatMapGap, formatShadowReplay, mapGaps, mergeExercisedMap, observeRun, readCaseCosts, readShardCommands, + REFRESH_EXIT, RefreshError, refreshFromReportDirs, refreshFromRun, REPLAY_FIXTURE, replayShadow, type ArtifactDownload, type ReplayRun, +} from '../scripts/exercised-map'; +import { reportMapGaps } from '../scripts/lib/paid-report'; +import { computePaidCaseSelection } from '../scripts/test-paid-shards'; +import { prototypeConsumers } from './helpers/exercised-prototype'; + +const ROOT = path.resolve(import.meta.dir, '..'); + +function fixtureRepo(files: Record): string { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'exercised-map-')); + for (const [file, text] of Object.entries(files)) { + fs.mkdirSync(path.dirname(path.join(root, file)), { recursive: true }); + fs.writeFileSync(path.join(root, file), text); + } + for (const args of [['init', '-q'], ['add', '-A']]) { + const result = spawnSync('git', args, { cwd: root, encoding: 'utf8', timeout: 10_000 }); + if (result.status !== 0) throw new Error(`git ${args[0]} failed: ${result.stderr}`); + } + return root; +} + +const map = (cases: ExercisedMap['cases'], sourceRuns = ['1'], observedThrough = '2026-10-01T00:00:00Z'): ExercisedMap => + ({ schema: 1, generatedAt: observedThrough, observedThrough, sourceRuns, cases }); + +const FENCE = '```'; +const skill = [ + '# alpha', '', 'Prose naming gstack-prose and /gstack-slash is not run.', '', FENCE + 'bash', + 'gstack-start --quiet', + 'echo "see gstack-echoed for details"', + 'echo "now $(gstack-subst --print)"', + '# gstack-commented', + 'case "$x" in gstack-casepat) true ;; esac', + 'VAR=gstack-bare', + '"$ROOT/bin/gstack-pathed" run', + '. "$ROOT/lib/sourced.sh"', + "cat <<'EOF'", 'gstack-quoted-heredoc', 'EOF', + 'cat < [`bin/gstack-${name}`, '#!/bin/sh\n'])), + 'test/skill-e2e-beta.test.ts': '', 'test/skill-e2e-gamma.test.ts': '', 'test/skill-e2e-delta.test.ts': '', + 'gamma/SKILL.md.tmpl': `# gamma\n\n${FENCE}bash\ngstack-config get proactive\n${FENCE}\n`, + 'scripts/build.sh': 'bun build --compile browse/src/cli.ts --outfile browse/dist/browse\n', 'browse/src/cli.ts': "import './page';\n", 'browse/src/page.ts': '', + 'scripts/thing.ts': '', 'scripts/resolvers/index.ts': '', 'setup': '#!/usr/bin/env bash\n', + 'package.json': JSON.stringify({ scripts: { 'build-thing': 'bun run scripts/thing.ts' } }), +}; +const maps = { + e2eTouchfiles: { + alpha: ['test/skill-e2e-alpha.test.ts', 'alpha/**'], beta: ['test/skill-e2e-beta.test.ts'], + gamma: ['test/skill-e2e-gamma.test.ts', 'gamma/**'], delta: ['test/skill-e2e-delta.test.ts', 'bin/gstack-config', 'lib/deep.ts'], + }, + judgeTouchfiles: {}, globalTouchfiles: [] as string[], +}; +const profileMaps = { ...maps, tiers: { alpha: 'gate', beta: 'gate', gamma: 'gate', delta: 'gate' } as const }; +const observed = map({ beta: { observations: 3, withCommands: 2, commands: ['bin/gstack-observed'], costUsd: 2, costRuns: 3 }, + alpha: { observations: 4, withCommands: 4, commands: [], costUsd: 5, costRuns: 4 }, gamma: { observations: 2, withCommands: 0, commands: [] } }); + +describe('exercised consumers on a synthetic tree', () => { + const root = fixtureRepo(tree); + const ex = exercisedConsumers(maps, root, { observed, stopAt: ['setup'] }); + const alpha = ex.closure('alpha'); + + test('each kept edge class reaches its file, with its provenance', () => { + expect(alpha.get('alpha/SKILL.md.tmpl')).toBe('touchfile'); + expect(alpha.get('bin/gstack-start')).toBe('skill bash via bin/gstack-start'); + expect(alpha.get('lib/deep.ts')).toBe('skill bash via bin/gstack-start'); + expect(alpha.get('lib/joined.ts')).toBe('import'); + expect(alpha.get('lib/quoted.ts')).toBe('import'); + for (const file of ['bin/gstack-subst', 'bin/gstack-pathed', 'lib/sourced.sh', 'bin/gstack-expanding', 'bin/gstack-interp', 'browse/src/page.ts', 'scripts/thing.ts', 'setup']) { + expect(alpha.has(file), file).toBe(true); + } + expect(ex.closure('beta').get('bin/gstack-observed')).toBe('observed bin/gstack-observed'); + }); + + test('messages, comments, case patterns, bare assignments, quoted and expanded heredoc prose, prose and slash commands are not edges', () => { + for (const name of ['echoed', 'commented', 'casepat', 'bare', 'quoted-heredoc', 'heredoc-prose', 'prose', 'slash']) { + expect(alpha.has(`bin/gstack-${name}`), name).toBe(false); + } + expect(ex.e2e.get('lib/unreached.ts')).toBeUndefined(); + }); + + test('the closure stops at a full-gate file and at the global set', () => { + const stopped = exercisedConsumers({ ...maps, globalTouchfiles: ['bin/gstack-start'] }, fixtureRepo(tree), { observed, stopAt: ['setup'] }); + expect(stopped.closure('alpha').has('bin/gstack-start')).toBe(true); + expect(stopped.closure('alpha').has('lib/deep.ts')).toBe(false); + }); + + test('bins in recorded commands: paths, bare names, PTY lines and compiled outfiles', () => { + expect(ex.binsInCommand('gstack-start --x && "$B" goto y')).toEqual(['bin/gstack-start', 'browse/dist/browse']); + expect(ex.binsInCommand('PTY:gstack-config-get proactive')).toEqual(['bin/gstack-config']); + expect(ex.binsInCommand('"$ROOT/bin/gstack-start" --x')).toEqual(['bin/gstack-start']); + expect(ex.binsInCommand('run /gstack-start next')).toEqual([]); + expect(ex.commandFiles('browse/dist/browse').sort()).toEqual(['browse/src/cli.ts', 'browse/src/page.ts']); + }); + + test('the variant-c miss shapes stay selected in the shadow (negative controls)', () => { + const shadow = (changedFiles: string[], selectedE2E: string[]) => selectPrProfile({ + maps: profileMaps, profile: ['alpha', 'beta', 'gamma', 'delta'], selectedE2E, selectedJudges: [], changedFiles, + derived: deriveDependencies(maps, root), exercised: ex, + }).shadow!; + // A preamble hub a touchfile maps only for another case (skill-start/end x3; gstack-config for the PTY case with no observations). + expect(shadow(['bin/gstack-config'], ['delta']).e2e).toEqual(['delta', 'gamma']); + expect(observed.cases.gamma!.withCommands).toBe(0); + // A lib reached only through a bin the skill runs (redact-patterns via gstack-redact). + expect(shadow(['lib/deep.ts'], ['delta']).e2e).toEqual(['alpha', 'delta']); + expect(shadow(['lib/deep.ts'], ['delta']).provenance).toEqual([{ file: 'lib/deep.ts', summary: 'selects 2 case(s): skill bash via bin/gstack-start (1), touchfile (1)' }]); + }); +}); + +describe('the five recorded variant-c misses on this checkout', () => { + const shadowOf = (file: string) => computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: [file], packageVersionOnly: true }).coverage!.shadow!; + + test('skill-start reds stay in the shadow of a gstack-skill-start change', () => { + const shadow = shadowOf('bin/gstack-skill-start'); + expect(shadow.error).toBeUndefined(); + for (const id of ['qa-functional-webhook-fix', 'qa-functional-cli-report', 'ship-exploratory-plan-checks']) expect(shadow.e2e, id).toContain(id); + }, 60_000); + + test('gstack-config reaches the PTY case that records no commands; redact-patterns reaches ship-docsync-store', () => { + expect(shadowOf('bin/gstack-config').e2e).toContain('plan-mode-no-op'); + const committed = parseExercisedMap(fs.readFileSync(path.join(ROOT, EXERCISED_MAP_PATH), 'utf8')); + const ex = exercisedConsumers(PR_PROFILE_MAPS, ROOT, { observed: committed, stopAt: PR_FULL_GATE_FILES }); + expect(ex.closure('plan-mode-no-op').get('bin/gstack-config')).toStartWith('skill bash via'); + expect(ex.closure('ship-docsync-store').has('lib/redact-patterns.ts')).toBe(true); + }, 60_000); +}); + +describe('shadow output never changes the real selection', () => { + test('mode, cases, judges and reasons equal the selection without a shadow', () => { + for (const files of [['bin/gstack-config'], ['design/src/evolve.ts'], ['review/SKILL.md.tmpl'], ['lib/fs-utils.ts', 'docs/x.md'], ['ETHOS.md']]) { + const withShadow = selectPrProfile({ selectedE2E: [], selectedJudges: [], changedFiles: files }); + const without = selectPrProfile({ selectedE2E: [], selectedJudges: [], changedFiles: files, exercised: null }); + expect(withShadow.shadow, files.join()).toBeDefined(); + const { shadow, ...rest } = withShadow; + expect(rest, files.join()).toEqual(without); + if (withShadow.mode === 'full-fallback') expect(shadow!.e2e).toEqual(withShadow.e2e); + } + }, 60_000); + + test('the coverage summary prints one shadow line; a broken map is reported, not fatal', () => { + const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: ['design/src/evolve.ts'], packageVersionOnly: true }); + const lines = formatPrCoverageSummary({ profile: 'pr', selection: result.selection, prCoverage: result.coverage }); + expect(lines.filter(line => line.includes('Shadow exercised-path map (not active)'))).toHaveLength(1); + const broken = selectPrProfile({ selectedE2E: [], selectedJudges: [], changedFiles: ['design/src/evolve.ts'], exercised: null }); + expect(formatPrCoverageSummary({ prCoverage: { ...broken, shadow: { e2e: broken.e2e, wouldDrop: [], wouldAdd: [], provenance: [], predictedSavingUsd: 0, unpricedCases: 0, unknownCostShare: 0, error: 'bad map' } } }) + .some(line => line.includes('unavailable (bad map); selection unaffected'))).toBe(true); + }); +}); + +describe('union-only refreshes', () => { + const base = map({ a: { observations: 2, withCommands: 1, commands: ['bin/x'], costUsd: 4, costRuns: 2 } }, ['1']); + + test('disjoint branches merge to the same union in either order, and nothing is removed', () => { + const left = map({ a: { observations: 1, withCommands: 1, commands: ['bin/y'] } }, ['2'], '2026-10-02T00:00:00Z'); + const right = map({ b: { observations: 1, withCommands: 1, commands: ['bin/z'], costUsd: 1, costRuns: 1 } }, ['3'], '2026-10-03T00:00:00Z'); + const one = mergeExercisedMap(mergeExercisedMap(base, left, 't'), right, 't'); + const two = mergeExercisedMap(mergeExercisedMap(base, right, 't'), left, 't'); + expect(one).toEqual(two); + expect(one.cases.a!.commands).toEqual(['bin/x', 'bin/y']); + expect(one.cases.b!.commands).toEqual(['bin/z']); + expect(one.sourceRuns).toEqual(['1', '2', '3']); + expect(one.observedThrough).toBe('2026-10-03T00:00:00Z'); + expect(one.cases.a!.costUsd).toBe(4); + }); + + test('empty or missing artifacts never erase consumers, and a run already merged is not counted twice', () => { + expect(mergeExercisedMap(base, map({}, ['9']), 't').cases).toEqual(base.cases); + expect(mergeExercisedMap(base, map({ a: { observations: 1, withCommands: 0, commands: [] } }, ['9']), 't').cases.a!.commands).toEqual(['bin/x']); + expect(mergeExercisedMap(base, map({ a: { observations: 5, withCommands: 5, commands: ['bin/x'], costUsd: 9, costRuns: 1 } }, ['1']), 't').cases.a) + .toEqual(base.cases.a!); + const costed = mergeExercisedMap(base, map({ a: { observations: 1, withCommands: 1, commands: [], costUsd: 7, costRuns: 1 } }, ['4']), 't'); + expect(costed.cases.a).toMatchObject({ observations: 3, costUsd: 5, costRuns: 3 }); + expect(parseExercisedMap(formatExercisedMap(costed))).toEqual(costed); + }); +}); + +describe('shadow-only map data', () => { + test('a map-only diff selects no paid case and says why', () => { + const selection = selectPrProfile({ selectedE2E: [], selectedJudges: [], changedFiles: [EXERCISED_MAP_PATH] }); + expect(selection).toMatchObject({ mode: 'pr', e2e: [], judges: [], unknownFiles: [], derivedFiles: [], noConsumerFiles: [] }); + expect(selection.shadow!.e2e).toEqual([]); + expect(selection.reasons).toContain(`${EXERCISED_MAP_PATH} is shadow-only data (it changes no selection yet), so it selects no paid case (docs/TESTING_INTERNALS.md#exercised-path-map)`); + const plan = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: [EXERCISED_MAP_PATH, 'package.json'], packageVersionOnly: true }); + expect(plan.selection.e2e).toEqual([]); + expect(plan.selection.judges).toEqual([]); + }, 60_000); +}); + +describe('refresh contract: unavailable or partial input writes nothing', () => { + const root = fixtureRepo(tree); + const ex = exercisedConsumers(maps, root, { observed, stopAt: ['setup'] }); + const report = (files: Record) => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'exercised-refresh-')); + for (const [file, text] of Object.entries(files)) { fs.mkdirSync(path.dirname(path.join(dir, file)), { recursive: true }); fs.writeFileSync(path.join(dir, file), text); } + return dir; + }; + const outcome = (id: string) => JSON.stringify({ case: id, execution: 'executed', cost_usd: 1 }); + const shard = { 'paid-slice-1-a1/shards/skill-e2e-beta/junit.xml': '' }; + const code = (run: () => unknown) => { try { run(); return 0; } catch (error) { return error instanceof RefreshError ? error.exitCode : -1; } }; + + test('report directories: missing is unavailable; no shards, no trial outcomes or an executed case without a shard is incomplete', () => { + expect(code(() => refreshFromReportDirs([path.join(os.tmpdir(), 'no-such-census-dir')], '1', observed, ex, 't', profileMaps))).toBe(REFRESH_EXIT.unavailable); + expect(code(() => refreshFromReportDirs([report({ 'trial-outcomes.jsonl': outcome('beta') })], '1', observed, ex, 't', profileMaps))).toBe(REFRESH_EXIT.incomplete); + expect(code(() => refreshFromReportDirs([report(shard)], '1', observed, ex, 't', profileMaps))).toBe(REFRESH_EXIT.incomplete); + expect(() => refreshFromReportDirs([report({ ...shard, 'trial-outcomes.jsonl': `${outcome('beta')}\n${outcome('alpha')}` })], '1', observed, ex, 't', profileMaps)) + .toThrow('1 executed case(s) have no shard artifact: alpha'); + const { merged } = refreshFromReportDirs([report({ ...shard, 'trial-outcomes.jsonl': outcome('beta') })], '9', observed, ex, 't', profileMaps); + expect(merged.sourceRuns).toEqual(['1', '9']); + expect(merged.cases.beta!.observations).toBe(4); + }); + + test('--from-run: a 403, a missing map or a malformed map leaves the target byte-identical; a complete artifact is unioned', () => { + const target = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'exercised-target-')), 'exercised-commands.json'); + fs.writeFileSync(target, formatExercisedMap(observed)); + const before = fs.readFileSync(target, 'utf8'); + const artifact = (files: Record): ArtifactDownload => (_runId, dir) => { + for (const [file, text] of Object.entries(files)) { fs.mkdirSync(path.dirname(path.join(dir, file)), { recursive: true }); fs.writeFileSync(path.join(dir, file), text); } + return { status: 0, stderr: '' }; + }; + let forbidden: unknown; + try { refreshFromRun('42', target, () => ({ status: 1, stderr: 'HTTP 403: Resource not accessible by integration' })); } catch (error) { forbidden = error; } + expect(forbidden).toBeInstanceOf(RefreshError); + expect((forbidden as RefreshError).exitCode).toBe(REFRESH_EXIT.unavailable); + expect((forbidden as Error).message).toContain('pass its directory with --from-dir'); + expect(code(() => refreshFromRun('42', target, artifact({ 'exercised-map-a1/report.md': 'x' })))).toBe(REFRESH_EXIT.incomplete); + expect(code(() => refreshFromRun('42', target, artifact({ 'exercised-map-a1/exercised-commands.json': '{"schema":1,"cases":' })))).toBe(REFRESH_EXIT.incomplete); + expect(code(() => refreshFromRun('42', target, artifact({ 'exercised-map-a1/exercised-commands.json': formatExercisedMap(map({}, ['5'])), 'exercised-map-a2/exercised-commands.json': '{}' })))).toBe(REFRESH_EXIT.incomplete); + expect(code(() => refreshFromRun('not-a-run', target))).toBe(REFRESH_EXIT.unavailable); + expect(fs.readFileSync(target, 'utf8')).toBe(before); + const fresh = map({ beta: { observations: 1, withCommands: 1, commands: ['bin/gstack-new'] } }, ['5'], '2026-10-09T00:00:00Z'); + refreshFromRun('5', target, artifact({ 'exercised-map-a1/exercised-commands.json': formatExercisedMap(fresh) })); + const after = parseExercisedMap(fs.readFileSync(target, 'utf8')); + expect(after.cases.beta!.commands).toEqual(['bin/gstack-new', 'bin/gstack-observed']); + expect(after.sourceRuns).toEqual(['1', '5']); + }); + + test('the CLI exits 2 on usage and 3 on a missing report directory, writing nothing', () => { + const out = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'exercised-cli-')), 'map.json'); + const cli = (args: string[]) => spawnSync('bun', ['run', path.join(ROOT, 'scripts/exercised-map.ts'), ...args], { cwd: ROOT, encoding: 'utf8', timeout: 60_000 }); + expect(cli([]).status).toBe(REFRESH_EXIT.usage); + const missing = cli(['--from-dir', path.join(os.tmpdir(), 'no-such-census-dir'), '--run-id', '1', '--out', out]); + expect(missing.status).toBe(REFRESH_EXIT.unavailable); + expect(missing.stderr).toContain('nothing was written, the committed union is unchanged'); + expect(fs.existsSync(out)).toBe(false); + }, 90_000); +}); + +describe('recorded commands and MAP_GAP', () => { + const root = fixtureRepo(tree); + const ex = exercisedConsumers(maps, root, { observed, stopAt: ['setup'] }); + const report = fs.mkdtempSync(path.join(os.tmpdir(), 'exercised-report-')); + const write = (file: string, text: string) => { fs.mkdirSync(path.dirname(path.join(report, file)), { recursive: true }); fs.writeFileSync(path.join(report, file), text); }; + write('paid-slice-1-a1/shards/skill-e2e-alpha--alpha.t1/result.json', JSON.stringify({ tests: [{ name: 'alpha', transcript: [{ type: 'tool_use', input: { command: 'gstack-start && $B goto x' } }] }] })); + write('paid-slice-1-a1/shards/skill-e2e-beta/junit.xml', ''); + write('paid-slice-1-a1/shards/skill-e2e-beta/session.jsonl', `${JSON.stringify({ type: 'tool_use', input: { command: 'bin/gstack-gapper --x' } })}\n`); + write('paid-slice-1-a1/shards/skill-e2e-gamma/junit.xml', ''); + write('paid-slice-1-a1/trial-outcomes.jsonl', [{ case: 'alpha', execution: 'executed', cost_usd: 1.5 }, { case: 'alpha', execution: 'executed', cost_usd: 0, cost_known: false }, { case: 'beta', execution: 'reused', cost_usd: 3 }].map(record => JSON.stringify(record)).join('\n')); + + test('commands are read per shard and attributed to their cases', () => { + const commands = attributeCommands(readShardCommands(report), profileMaps); + expect(commands.get('alpha')).toEqual(['gstack-start && $B goto x']); + expect(commands.get('beta')).toEqual(['bin/gstack-gapper --x']); + expect(commands.get('gamma')).toEqual([]); + expect(observeRun(commands, ex, { runId: '7', observedAt: 't', costs: readCaseCosts(report) }).cases.alpha) + .toEqual({ observations: 1, withCommands: 1, commands: ['bin/gstack-start', 'browse/dist/browse'], costUsd: 1.5, costRuns: 1 }); + }); + + test('a command outside the static-plus-observed closure is a MAP_GAP; a skill-running case with no commands is blind', () => { + const commands = attributeCommands(readShardCommands(report), profileMaps); + expect(mapGaps(commands, ex)).toEqual([{ id: 'beta', command: 'bin/gstack-gapper' }]); + expect(blindCases(commands, ex)).toEqual(['gamma']); + }); + + test('the paid report writes MAP_GAP lines without failing, and an unreadable checkout degrades to unavailable', () => { + const live = fs.mkdtempSync(path.join(os.tmpdir(), 'exercised-live-')); + const committed = parseExercisedMap(fs.readFileSync(path.join(ROOT, EXERCISED_MAP_PATH), 'utf8')); + const ex = exercisedConsumers(PR_PROFILE_MAPS, ROOT, { observed: committed, stopAt: PR_FULL_GATE_FILES }); + const outside = fs.readdirSync(path.join(ROOT, 'bin')).map(name => `bin/${name}`).find(file => /^bin\/gstack-[a-z-]+$/.test(file) && !ex.closure('ship-docsync-store').has(file))!; + fs.mkdirSync(path.join(live, 'paid-slice-2-a1/shards/skill-e2e-ship-docsync--ship-docsync-store.t1'), { recursive: true }); + fs.writeFileSync(path.join(live, 'paid-slice-2-a1/shards/skill-e2e-ship-docsync--ship-docsync-store.t1/r.json'), + JSON.stringify({ tests: [{ name: 'ship-docsync-store', transcript: [{ type: 'tool_use', input: { command: `${path.basename(outside)} --help` } }] }] })); + const line = `MAP_GAP ship-docsync-store ${outside}: non-blocking for this PR; the next census refresh widens the map (docs/TESTING_INTERNALS.md#exercised-path-map)`; + expect(formatMapGap({ id: 'ship-docsync-store', command: outside })).toBe(line); + expect(reportMapGaps(live, new Set(['ship-docsync-store']), ROOT)).toEqual([line]); + expect(fs.readFileSync(path.join(live, 'map-gaps.txt'), 'utf8')).toBe(`${line}\n`); + expect(reportMapGaps(live, new Set(['ship-docsync-store']), path.join(live, 'no-checkout'))).toEqual([]); + expect(fs.readFileSync(path.join(live, 'map-gaps.txt'), 'utf8')).toStartWith('MAP_GAP unavailable:'); + }, 60_000); +}); + +describe('parity with the measured prototype', () => { + /** Files the shipped function reaches beyond the prototype, each with the reviewed edge class behind it. */ + const REVIEWED_SUPERSET: Record = { + setup: './setup, $ROOT/setup and join(..., "setup") exec edges (plan A2-2); setup is a full-gate file, so the closure stops there', + }; + + test('the shipped map equals or is a reviewed superset of the prototype map on this checkout', () => { + const committed = parseExercisedMap(fs.readFileSync(path.join(ROOT, EXERCISED_MAP_PATH), 'utf8')); + const prototype = prototypeConsumers(PR_PROFILE_MAPS, ROOT, committed, PR_FULL_GATE_FILES); + const shipped = exercisedConsumers(PR_PROFILE_MAPS, ROOT, { observed: committed, stopAt: PR_FULL_GATE_FILES }); + const missing = [...prototype].flatMap(([file, ids]) => [...ids].filter(id => !shipped.e2e.get(file)?.has(id)).map(id => `${file} ${id}`)); + expect(missing).toEqual([]); + const unreviewed = [...new Set([...shipped.e2e].filter(([file, ids]) => [...ids].some(id => !prototype.get(file)?.has(id))).map(([file]) => file))] + .filter(file => !Object.hasOwn(REVIEWED_SUPERSET, file)); + expect(unreviewed.map(file => `${file}: reached beyond the prototype; review the edge class and add it to REVIEWED_SUPERSET with its reason`)).toEqual([]); + }, 60_000); +}); + +describe('replay of the recorded red PR runs', () => { + test('every recorded failed gate case stays in the shadow selection (the PR receipt: bun run test:exercised-map --replay)', () => { + const fixture = JSON.parse(fs.readFileSync(path.join(ROOT, REPLAY_FIXTURE), 'utf8')) as { failedGateCases: number; runs: ReplayRun[] }; + expect(fixture.runs.reduce((sum, run) => sum + run.failedGateCases.length, 0)).toBe(fixture.failedGateCases); + const rows = replayShadow(fixture.runs); + expect(rows.flatMap(row => row.misses.map(id => `${row.runId}:${id}`))).toEqual([]); + expect(formatShadowReplay(rows).at(-1)).toBe('- MISSES (failed, not in the shadow selection): 0'); + }, 120_000); + + test('a red the shadow drops is reported as a miss', () => { + const [row] = replayShadow([{ runId: 1, pr: 1, packageVersionOnly: true, failedGateCases: ['plan-mode-no-op'], changedFiles: ['docs/designs/x.md'] }]); + expect(row!.misses).toEqual(['plan-mode-no-op']); + expect(formatShadowReplay([row!]).slice(-2)).toEqual(['- MISSES (failed, not in the shadow selection): 1', ' - 1 #1 plan-mode-no-op']); + }); +}); diff --git a/test/fixtures/exercised-map-replay.json b/test/fixtures/exercised-map-replay.json new file mode 100644 index 000000000..0023f6b3d --- /dev/null +++ b/test/fixtures/exercised-map-replay.json @@ -0,0 +1,44 @@ +{ + "schema": 1, + "source": "evals.yml pull_request runs 2026-10-07..2026-10-17 (225 runs): every run with an executed failed gate trial, deduplicated per (run, case); changed files are the three-dot compare of the merge-base with the run head", + "failedGateCases": 47, + "runs": [ + {"runId":37560420246,"pr":3059,"headSha":"d6250a058af971dd697036acd6e05f7740e8e210","packageVersionOnly":true,"failedGateCases":["plan-mode-no-op"],"changedFiles":[".github/actionlint.yaml",".github/workflows/eval-sweep.yml","CHANGELOG.md","CONTRIBUTING.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-config","docs/TESTING_INTERNALS.md","package.json","scripts/eval-flake-rank.ts","scripts/harness-version.json","scripts/lib/measure-bar.ts","scripts/ship-measure-sweep.ts","scripts/ship-measure.ts","scripts/test-pr-profile.ts","ship/sections/measure.md","ship/sections/measure.md.tmpl","test/ci-image-tag-binding.test.ts","test/eval-sweep-workflow.test.ts","test/fixtures/module-size-ratchet.json","test/helpers/carve-guards.ts","test/helpers/touchfiles-data.ts","test/ship-measure-bar.test.ts","test/ship-measure-loop-skill.test.ts","test/ship-measure-seeded-fixture.test.ts","test/ship-measure-sweep.test.ts","test/ship-measure.test.ts"]}, + {"runId":37564096977,"pr":3059,"headSha":"36c0cf1579a2f823783c2ecee5c509e872b0c20a","packageVersionOnly":true,"failedGateCases":["qa-functional-cli-report"],"changedFiles":[".github/actionlint.yaml",".github/workflows/eval-sweep.yml","CHANGELOG.md","CONTRIBUTING.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-config","docs/TESTING_INTERNALS.md","package.json","scripts/eval-flake-rank.ts","scripts/harness-version.json","scripts/lib/measure-bar.ts","scripts/ship-measure-sweep.ts","scripts/ship-measure.ts","scripts/test-pr-profile.ts","ship/sections/measure.md","ship/sections/measure.md.tmpl","test/ci-image-tag-binding.test.ts","test/eval-sweep-workflow.test.ts","test/fixtures/module-size-ratchet.json","test/helpers/carve-guards.ts","test/helpers/touchfiles-data.ts","test/ship-measure-bar.test.ts","test/ship-measure-loop-skill.test.ts","test/ship-measure-seeded-fixture.test.ts","test/ship-measure-sweep.test.ts","test/ship-measure.test.ts","test/skill-e2e-plan-mode-no-op.test.ts"]}, + {"runId":37691665506,"pr":3069,"headSha":"068a3099baf660e995c693b5625b4b3e4546b7a8","packageVersionOnly":true,"failedGateCases":["shared-libs-read-only"],"changedFiles":[".github/ISSUE_TEMPLATE/bug_report.md",".github/docker/Dockerfile.ci",".github/workflows/evals-periodic.yml",".github/workflows/free-tests.yml",".github/workflows/measure-journal-read.yml","CHANGELOG.md","README.md","SKILL.md","TODOS.md","VERSION","agents-digest/gstack-AGENTS.md","autoplan/SKILL.md","autoplan/bin/guard-journal.ts","autoplan/bin/guard-log.ts","autoplan/bin/guard-reasons.ts","autoplan/bin/phase-publication-hook.ts","autoplan/sections/ceo-phase.md","autoplan/sections/ceo-phase.md.tmpl","autoplan/sections/design-phase.md","autoplan/sections/design-phase.md.tmpl","autoplan/sections/dx-phase.md","autoplan/sections/dx-phase.md.tmpl","autoplan/sections/eng-phase.md","autoplan/sections/eng-phase.md.tmpl","autoplan/sections/phase-close.md","autoplan/sections/phase-close.md.tmpl","benchmark-models/SKILL.md","benchmark/SKILL.md","bin/gstack-artifacts-init","bin/gstack-brain-lock.sh","bin/gstack-brain-restore","bin/gstack-brain-sync","bin/gstack-codex-probe","bin/gstack-codex-status.sh","bin/gstack-config","bin/gstack-doctor","bin/gstack-gbrain-source-wireup","bin/gstack-greptile-early","bin/gstack-post","bin/gstack-pr-title-rewrite.sh","bin/gstack-question-preference","bin/gstack-redact-prepush","bin/gstack-skill-start","bin/gstack-slug","browse/SKILL.md","browse/test/extension-sender-auth.test.ts","canary/SKILL.md","codex/SKILL.md","codex/SKILL.md.tmpl","codex/sections/challenge-mode.md","codex/sections/challenge-mode.md.tmpl","codex/sections/consult-mode.md","codex/sections/consult-mode.md.tmpl","codex/sections/review-mode.md","codex/sections/review-mode.md.tmpl","context-restore/SKILL.md","context-restore/SKILL.md.tmpl","context-save/SKILL.md","context-save/SKILL.md.tmpl","design-consultation/SKILL.md","design-consultation/sections/proposal-and-preview.md","design-html/SKILL.md","design-review/SKILL.md","design-shotgun/SKILL.md","devex-review/SKILL.md","diagram/SKILL.md","docs/autoplan-guard-troubleshooting.md","docs/designs/FOLLOWUP_WAVE_2026_10_07.md","docs/gbrain-sync-errors.md","docs/gbrain-sync.md","docs/gbrain-write-surfaces.md","docs/troubleshooting.md","document-generate/SKILL.md","document-release/SKILL.md","document-release/sections/release-body.md","health/SKILL.md","investigate/SKILL.md","ios-clean/SKILL.md","ios-design-review/SKILL.md","ios-fix/SKILL.md","ios-qa/SKILL.md","ios-sync/SKILL.md","land-and-deploy/SKILL.md","landing-report/SKILL.md","learn/SKILL.md","lib/claude-journal-records.ts","lib/claude-owned-journal.ts","lib/claude-public-transcript.ts","lib/greptile-early.ts","lib/gstack-post.ts","lib/outside-review-result.ts","make-pdf/SKILL.md","office-hours/SKILL.md","office-hours/sections/design-and-handoff.md","open-gstack-browser/SKILL.md","package.json","pair-agent/SKILL.md","plan-ceo-review/SKILL.md","plan-ceo-review/sections/review-sections.md","plan-design-review/SKILL.md","plan-devex-review/SKILL.md","plan-devex-review/sections/review-sections.md","plan-eng-review/SKILL.md","plan-eng-review/sections/review-sections.md","plan-tune/SKILL.md","plan-tune/SKILL.md.tmpl","qa-only/SKILL.md","qa/SKILL.md","qa/sections/browser-setup.md","retro/SKILL.md","review/SKILL.md","review/greptile-triage.md","review/sections/adversarial.md","review/sections/review-army.md","scrape/SKILL.md","scripts/harness-version.json","scripts/lib/windows-curation.ts","scripts/measure-journal-read.ts","scripts/resolvers/aside.ts","scripts/resolvers/constants.ts","scripts/resolvers/design.ts","scripts/resolvers/gbrain.ts","scripts/resolvers/index.ts","scripts/resolvers/outside-voice-steps.ts","scripts/resolvers/outside-voice.ts","scripts/resolvers/preamble/generate-brain-sync-block.ts","scripts/resolvers/question-tuning.ts","scripts/resolvers/redact-doc.ts","scripts/resolvers/review-army.ts","scripts/resolvers/spec-review.ts","scripts/test-pr-profile.ts","scripts/typecheck-test-baseline.json","setup","setup-browser-cookies/SKILL.md","setup-deploy/SKILL.md","setup-gbrain/SKILL.md","ship/SKILL.md","ship/SKILL.md.tmpl","ship/sections/adversarial.md","ship/sections/documentation.md","ship/sections/greptile-early.md","ship/sections/greptile-early.md.tmpl","ship/sections/greptile.md","ship/sections/greptile.md.tmpl","ship/sections/manifest.json","ship/sections/pr-body.md","ship/sections/pr-body.md.tmpl","ship/sections/review-army.md","skillify/SKILL.md","spec/SKILL.md","spec/sections/gate-and-file.md","spec/sections/gate-and-file.md.tmpl","sync-gbrain/SKILL.md","test-audit/SKILL.md","test/aside-driver.test.ts","test/autoplan-bounded-journal.test.ts","test/autoplan-dual-voice-evidence.test.ts","test/autoplan-guard-log.test.ts","test/autoplan-guard-reasons.test.ts","test/autoplan-long-session-observer.test.ts","test/autoplan-publication-guard.test.ts","test/autoplan-publication-hook.test.ts","test/autoplan-publication-payload.test.ts","test/autoplan-schema-canary-compare.test.ts","test/binding-template-drift.test.ts","test/brain-sync.test.ts","test/ceo-mode-preference-al.test.ts","test/codex-hardening.test.ts","test/codex-invocation-sites.test.ts","test/codex-model-probe.test.ts","test/codex-probe-subcommands.test.ts","test/codex-skill-isolation.test.ts","test/codex-web-search-flag.test.ts","test/context-identity.test.ts","test/design-consultation-contract.test.ts","test/eng-finding-retry-budget.test.ts","test/fixtures/claude-agent-payload-2.1.292.json","test/fixtures/golden/claude-ship-SKILL.md","test/fixtures/golden/codex-ship-SKILL.md","test/fixtures/golden/factory-ship-SKILL.md","test/fixtures/module-size-ratchet.json","test/fixtures/outside-review-severity/ceo-codex-shape.txt","test/fixtures/outside-review-severity/dx-codex-shape.txt","test/fixtures/outside-review-severity/eng-codex-tagged.txt","test/fixtures/outside-review-severity/prose-negative.txt","test/fixtures/outside-review-severity/prose-unlabeled.txt","test/fixtures/outside-review-severity/table-shape.txt","test/gate-tail-lint.test.ts","test/gen-skill-docs.test.ts","test/generated-bash-placeholders.test.ts","test/greptile-early-cli.test.ts","test/greptile-early.test.ts","test/gstack-doctor.test.ts","test/gstack-post-cli.test.ts","test/gstack-post.test.ts","test/gstack-question-preference.test.ts","test/gstack-skill-start.test.ts","test/helpers/autoplan-dual-voice-evidence.ts","test/helpers/autoplan-guard-fixture.ts","test/helpers/autoplan-guard-pty.ts","test/helpers/carve-guards.ts","test/helpers/codex-probe-sourcing.ts","test/helpers/free-fixtures-data.ts","test/helpers/journal-padding.ts","test/helpers/native-auto-decide.ts","test/helpers/placeholder-allowlist.ts","test/helpers/schema-canary.ts","test/helpers/touchfiles-data.ts","test/outside-review-result.test.ts","test/outside-review-severity.test.ts","test/outside-voice-invocation.test.ts","test/outside-voice-preflight.test.ts","test/outside-voice-receipt.test.ts","test/pr-title-rewrite.test.ts","test/redact-doc-resolver.test.ts","test/redact-prepush-emails.test.ts","test/regression-issue3024-zsh-sourced-helpers.test.ts","test/regression-pr1169-mktemp-fallbacks.test.ts","test/resolvers-gbrain-save-results.test.ts","test/run-in-background-guidance.test.ts","test/ship-control-flow.test.ts","test/ship-greptile-early.test.ts","test/ship-publication-gates.test.ts","test/ship-reentry-gates.test.ts","test/ship-template-redaction.test.ts","test/ship-version-sync.test.ts","test/ship-workflow-clarity.test.ts","test/skill-e2e-autoplan-guard-pty.test.ts","test/skill-e2e-autoplan-long-session.test.ts","test/skill-e2e-autoplan-schema-canary.test.ts","test/skill-validation.test.ts","test/spec-template-invariants.test.ts"]}, + {"runId":37694473528,"pr":3069,"headSha":"c2d1aa1976a1403a0a6f7846a748080f61c8c726","packageVersionOnly":true,"failedGateCases":["qa-quick","shared-libs-read-only","shared-libs-unsupported-git"],"changedFiles":[".github/ISSUE_TEMPLATE/bug_report.md",".github/docker/Dockerfile.ci",".github/workflows/evals-periodic.yml",".github/workflows/free-tests.yml",".github/workflows/measure-journal-read.yml","CHANGELOG.md","README.md","SKILL.md","TODOS.md","VERSION","agents-digest/gstack-AGENTS.md","autoplan/SKILL.md","autoplan/bin/guard-journal.ts","autoplan/bin/guard-log.ts","autoplan/bin/guard-reasons.ts","autoplan/bin/phase-publication-hook.ts","autoplan/sections/ceo-phase.md","autoplan/sections/ceo-phase.md.tmpl","autoplan/sections/design-phase.md","autoplan/sections/design-phase.md.tmpl","autoplan/sections/dx-phase.md","autoplan/sections/dx-phase.md.tmpl","autoplan/sections/eng-phase.md","autoplan/sections/eng-phase.md.tmpl","autoplan/sections/phase-close.md","autoplan/sections/phase-close.md.tmpl","benchmark-models/SKILL.md","benchmark/SKILL.md","bin/gstack-artifacts-init","bin/gstack-brain-lock.sh","bin/gstack-brain-restore","bin/gstack-brain-sync","bin/gstack-codex-probe","bin/gstack-codex-status.sh","bin/gstack-config","bin/gstack-doctor","bin/gstack-gbrain-source-wireup","bin/gstack-greptile-early","bin/gstack-post","bin/gstack-pr-title-rewrite.sh","bin/gstack-question-preference","bin/gstack-redact-prepush","bin/gstack-skill-start","bin/gstack-slug","browse/SKILL.md","browse/test/extension-sender-auth.test.ts","canary/SKILL.md","codex/SKILL.md","codex/SKILL.md.tmpl","codex/sections/challenge-mode.md","codex/sections/challenge-mode.md.tmpl","codex/sections/consult-mode.md","codex/sections/consult-mode.md.tmpl","codex/sections/review-mode.md","codex/sections/review-mode.md.tmpl","context-restore/SKILL.md","context-restore/SKILL.md.tmpl","context-save/SKILL.md","context-save/SKILL.md.tmpl","design-consultation/SKILL.md","design-consultation/sections/proposal-and-preview.md","design-html/SKILL.md","design-review/SKILL.md","design-shotgun/SKILL.md","devex-review/SKILL.md","diagram/SKILL.md","docs/autoplan-guard-troubleshooting.md","docs/designs/FOLLOWUP_WAVE_2026_10_07.md","docs/gbrain-sync-errors.md","docs/gbrain-sync.md","docs/gbrain-write-surfaces.md","docs/troubleshooting.md","document-generate/SKILL.md","document-release/SKILL.md","document-release/sections/release-body.md","health/SKILL.md","investigate/SKILL.md","ios-clean/SKILL.md","ios-design-review/SKILL.md","ios-fix/SKILL.md","ios-qa/SKILL.md","ios-sync/SKILL.md","land-and-deploy/SKILL.md","landing-report/SKILL.md","learn/SKILL.md","lib/claude-journal-records.ts","lib/claude-owned-journal.ts","lib/claude-public-transcript.ts","lib/greptile-early.ts","lib/gstack-post.ts","lib/outside-review-result.ts","make-pdf/SKILL.md","office-hours/SKILL.md","office-hours/sections/design-and-handoff.md","open-gstack-browser/SKILL.md","package.json","pair-agent/SKILL.md","plan-ceo-review/SKILL.md","plan-ceo-review/sections/review-sections.md","plan-design-review/SKILL.md","plan-devex-review/SKILL.md","plan-devex-review/sections/review-sections.md","plan-eng-review/SKILL.md","plan-eng-review/sections/review-sections.md","plan-tune/SKILL.md","plan-tune/SKILL.md.tmpl","qa-only/SKILL.md","qa/SKILL.md","qa/sections/browser-setup.md","retro/SKILL.md","review/SKILL.md","review/greptile-triage.md","review/sections/adversarial.md","review/sections/review-army.md","scrape/SKILL.md","scripts/harness-version.json","scripts/lib/windows-curation.ts","scripts/measure-journal-read.ts","scripts/resolvers/aside.ts","scripts/resolvers/constants.ts","scripts/resolvers/design.ts","scripts/resolvers/gbrain.ts","scripts/resolvers/index.ts","scripts/resolvers/outside-voice-steps.ts","scripts/resolvers/outside-voice.ts","scripts/resolvers/preamble/generate-brain-sync-block.ts","scripts/resolvers/question-tuning.ts","scripts/resolvers/redact-doc.ts","scripts/resolvers/review-army.ts","scripts/resolvers/spec-review.ts","scripts/test-pr-profile.ts","scripts/typecheck-test-baseline.json","setup","setup-browser-cookies/SKILL.md","setup-deploy/SKILL.md","setup-gbrain/SKILL.md","ship/SKILL.md","ship/SKILL.md.tmpl","ship/sections/adversarial.md","ship/sections/documentation.md","ship/sections/greptile-early.md","ship/sections/greptile-early.md.tmpl","ship/sections/greptile.md","ship/sections/greptile.md.tmpl","ship/sections/manifest.json","ship/sections/pr-body.md","ship/sections/pr-body.md.tmpl","ship/sections/review-army.md","skillify/SKILL.md","spec/SKILL.md","spec/sections/gate-and-file.md","spec/sections/gate-and-file.md.tmpl","sync-gbrain/SKILL.md","test-audit/SKILL.md","test/aside-driver.test.ts","test/autoplan-bounded-journal.test.ts","test/autoplan-dual-voice-evidence.test.ts","test/autoplan-guard-log.test.ts","test/autoplan-guard-reasons.test.ts","test/autoplan-long-session-observer.test.ts","test/autoplan-publication-guard.test.ts","test/autoplan-publication-hook.test.ts","test/autoplan-publication-payload.test.ts","test/autoplan-schema-canary-compare.test.ts","test/binding-template-drift.test.ts","test/brain-sync.test.ts","test/ceo-mode-preference-al.test.ts","test/codex-hardening.test.ts","test/codex-invocation-sites.test.ts","test/codex-model-probe.test.ts","test/codex-probe-subcommands.test.ts","test/codex-skill-isolation.test.ts","test/codex-web-search-flag.test.ts","test/context-identity.test.ts","test/design-consultation-contract.test.ts","test/eng-finding-retry-budget.test.ts","test/fixtures/claude-agent-payload-2.1.292.json","test/fixtures/golden/claude-ship-SKILL.md","test/fixtures/golden/codex-ship-SKILL.md","test/fixtures/golden/factory-ship-SKILL.md","test/fixtures/module-size-ratchet.json","test/fixtures/outside-review-severity/ceo-codex-shape.txt","test/fixtures/outside-review-severity/dx-codex-shape.txt","test/fixtures/outside-review-severity/eng-codex-tagged.txt","test/fixtures/outside-review-severity/prose-negative.txt","test/fixtures/outside-review-severity/prose-unlabeled.txt","test/fixtures/outside-review-severity/table-shape.txt","test/fixtures/workflow-required-steps.json","test/gate-tail-lint.test.ts","test/gen-skill-docs.test.ts","test/generated-bash-placeholders.test.ts","test/greptile-early-cli.test.ts","test/greptile-early.test.ts","test/gstack-doctor.test.ts","test/gstack-post-cli.test.ts","test/gstack-post.test.ts","test/gstack-question-preference.test.ts","test/gstack-skill-start.test.ts","test/helpers/autoplan-dual-voice-evidence.ts","test/helpers/autoplan-guard-fixture.ts","test/helpers/autoplan-guard-pty.ts","test/helpers/carve-guards.ts","test/helpers/codex-probe-sourcing.ts","test/helpers/free-fixtures-data.ts","test/helpers/journal-padding.ts","test/helpers/manual-judge-review-fixture.ts","test/helpers/native-auto-decide.ts","test/helpers/placeholder-allowlist.ts","test/helpers/schema-canary.ts","test/helpers/touchfiles-data.ts","test/outside-review-result.test.ts","test/outside-review-severity.test.ts","test/outside-voice-invocation.test.ts","test/outside-voice-preflight.test.ts","test/outside-voice-receipt.test.ts","test/pr-title-rewrite.test.ts","test/qa-supervision-selection.test.ts","test/redact-doc-resolver.test.ts","test/redact-prepush-emails.test.ts","test/regression-issue3024-zsh-sourced-helpers.test.ts","test/regression-pr1169-mktemp-fallbacks.test.ts","test/resolvers-gbrain-save-results.test.ts","test/run-in-background-guidance.test.ts","test/ship-control-flow.test.ts","test/ship-document-release-dispatch.test.ts","test/ship-greptile-early.test.ts","test/ship-publication-gates.test.ts","test/ship-reentry-gates.test.ts","test/ship-template-redaction.test.ts","test/ship-version-sync.test.ts","test/ship-workflow-clarity.test.ts","test/skill-e2e-autoplan-guard-pty.test.ts","test/skill-e2e-autoplan-long-session.test.ts","test/skill-e2e-autoplan-schema-canary.test.ts","test/skill-validation.test.ts","test/spec-template-invariants.test.ts","test/workflow-excerpt.test.ts"]}, + {"runId":37731666423,"pr":3073,"headSha":"872e1542e42d40b3a95356d5284c6e4a3553c31f","packageVersionOnly":true,"failedGateCases":["plan-ceo-finding-floor","ship-docsync-missing-asset"],"changedFiles":[".github/workflows/model-policy-freshness.yml","CHANGELOG.md","CONTRIBUTING.md","README.md","VERSION","agents-digest/gstack-AGENTS.md","autoplan/SKILL.md","autoplan/SKILL.md.tmpl","autoplan/sections/ceo-phase.md","autoplan/sections/design-phase.md","autoplan/sections/dx-phase.md","autoplan/sections/eng-phase.md","bin/gstack-codex-probe","bin/gstack-config","bin/gstack-models","claude-code/SKILL.md.tmpl","codex/SKILL.md","codex/SKILL.md.tmpl","codex/sections/challenge-mode.md","codex/sections/challenge-mode.md.tmpl","codex/sections/consult-mode.md","codex/sections/consult-mode.md.tmpl","codex/sections/review-mode.md","codex/sections/review-mode.md.tmpl","docs/model-policy.md","docs/skills.md","docs/troubleshooting.md","lib/claude-code-migration.ts","lib/claude-code.ts","lib/model-catalog.ts","lib/model-policy-cli.ts","lib/model-policy-freshness-github.ts","lib/model-policy-freshness-parser-worker.ts","lib/model-policy-freshness-publication.ts","lib/model-policy-freshness-sources.ts","lib/model-policy-freshness.ts","lib/model-policy-notice.ts","lib/model-policy.ts","package.json","plan-ceo-review/sections/review-sections.md","plan-design-review/SKILL.md","plan-devex-review/sections/review-sections.md","plan-eng-review/sections/review-sections.md","plan-eng-review/sections/review-sections.md.tmpl","review/SKILL.md","scripts/harness-version.json","scripts/model-policy-freshness.ts","scripts/preflight-codex-overlap.ts","scripts/resolve-codex-generation-model.ts","scripts/resolvers/constants.ts","scripts/resolvers/design.ts","scripts/resolvers/index.ts","scripts/resolvers/outside-voice-steps.ts","scripts/resolvers/outside-voice.ts","scripts/resolvers/plan-review.ts","scripts/resolvers/qa.ts","scripts/test-pr-profile.ts","ship/sections/review-army.md","spec/SKILL.md","spec/SKILL.md.tmpl","spec/sections/gate-and-file.md","spec/sections/gate-and-file.md.tmpl","test/claude-code-migration.test.ts","test/claude-code-skill.test.ts","test/codex-hardening.test.ts","test/codex-probe-cache.test.ts","test/codex-under-codex-detection.test.ts","test/fixtures/model-policy-freshness.ts","test/fixtures/model-policy-freshness/anthropic-lifecycle.md","test/fixtures/model-policy-freshness/anthropic-models.md","test/fixtures/model-policy-freshness/openai-lifecycle.md","test/fixtures/model-policy-freshness/openai-models.md","test/fixtures/qa-caller-read-probe-mtb1007a.json","test/helpers/free-fixtures-data.ts","test/helpers/pty/launch.ts","test/helpers/qa-callers-fixture.ts","test/model-catalog-isolation.test.ts","test/model-policy-cli.test.ts","test/model-policy-freshness.test.ts","test/model-policy-workflow.test.ts","test/model-policy.test.ts","test/model-role-invocation.test.ts","test/outside-voice-invocation.test.ts","test/outside-voice-preflight.test.ts","test/plan-review-cases.test.ts","test/pty-output-wake.test.ts","test/qa-exploratory-callers.test.ts","test/qa-probe-gates.test.ts","test/run-in-background-guidance.test.ts","test/setup-claude-code-migration.test.ts","test/team-mode.test.ts"]}, + {"runId":37739845694,"pr":3077,"headSha":"e2e725ad3ddefac67b57f91c2a000848de56b752","packageVersionOnly":true,"failedGateCases":["ship-docsync-missing-asset"],"changedFiles":["CHANGELOG.md","TODOS.md","VERSION","agents-digest/gstack-AGENTS.md","careful/SKILL.md","careful/SKILL.md.tmpl","careful/bin/check-careful.sh","careful/bin/hook-extract.sh","docs/TESTING_INTERNALS.md","docs/skills.md","freeze/SKILL.md","freeze/SKILL.md.tmpl","freeze/bin/check-freeze.sh","guard/SKILL.md","guard/SKILL.md.tmpl","investigate/SKILL.md","investigate/SKILL.md.tmpl","package.json","scripts/harness-version.json","scripts/lib/windows-curation.ts","scripts/test-pr-profile.ts","test/fixtures/context-budget.json","test/hook-scripts.test.ts"]}, + {"runId":37743622252,"pr":3079,"headSha":"4637fae03baad83603ab7d554f4137c4702b4bd4","packageVersionOnly":true,"failedGateCases":["qa-quick"],"changedFiles":[".github/workflows/free-tests.yml",".github/workflows/make-pdf-gate.yml",".github/workflows/windows-setup-e2e.yml",".gitignore","CHANGELOG.md","README.md","TODOS.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-doctor","bin/gstack-global-discover.ts","bin/gstack-launch-probe.sh","browse/src/cli.ts","browse/src/find-browse.ts","browse/test/sidebar-ux.test.ts","docs/troubleshooting.md","extension/sidepanel-terminal.js","extension/sidepanel.css","make-pdf/src/cli.ts","package.json","scripts/build-cso.sh","scripts/build.sh","setup","test/binary-version-probe.test.ts","test/build-cso-nonfatal.test.ts","test/build-cso-result-record.test.ts","test/cso-windows-launcher.test.ts","test/gstack-doctor.test.ts","test/gstack-launch-probe.test.ts","test/setup-cso-degrade.test.ts","test/setup-cso-windows-probe.test.ts","test/setup-windows-launch-hint.test.ts"]}, + {"runId":37762901988,"pr":3079,"headSha":"b29f02250967df02f0e02dc4c8310155771cdf41","packageVersionOnly":true,"failedGateCases":["ship-exploratory-unavailable"],"changedFiles":[".github/workflows/free-tests.yml",".github/workflows/make-pdf-gate.yml",".github/workflows/windows-setup-e2e.yml",".gitignore","CHANGELOG.md","README.md","TODOS.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-doctor","bin/gstack-global-discover.ts","bin/gstack-launch-probe.sh","browse/src/cli.ts","browse/src/find-browse.ts","browse/test/sidebar-ux.test.ts","docs/troubleshooting.md","extension/sidepanel-terminal.js","extension/sidepanel.css","make-pdf/src/cli.ts","package.json","scripts/build-cso.sh","scripts/build.sh","setup","test/binary-version-probe.test.ts","test/build-cso-nonfatal.test.ts","test/build-cso-result-record.test.ts","test/cso-windows-launcher.test.ts","test/gstack-doctor.test.ts","test/gstack-launch-probe.test.ts","test/setup-cso-degrade.test.ts","test/setup-cso-windows-probe.test.ts","test/setup-windows-launch-hint.test.ts"]}, + {"runId":38087752336,"pr":3114,"headSha":"3e51fb06ba5c9c8943e7211ba32fb93fcccfe402","packageVersionOnly":false,"failedGateCases":["qa-functional-cli-report"],"changedFiles":["AGENTS.md","ETHOS.md","README.md","SKILL.md","autoplan/SKILL.md","benchmark-models/SKILL.md","benchmark/SKILL.md","bin/gstack-evidence","bin/gstack-gate-diff","bin/gstack-review-log","browse/SKILL.md","canary/SKILL.md","codex/SKILL.md","context-restore/SKILL.md","context-save/SKILL.md","design-consultation/SKILL.md","design-html/SKILL.md","design-review/SKILL.md","design-shotgun/SKILL.md","devex-review/SKILL.md","diagram/SKILL.md","docs/designs/HONEST_WORK_GATE_INTEGRITY.md","docs/designs/gate-diff-calibration.md","document-generate/SKILL.md","document-release/SKILL.md","health/SKILL.md","investigate/SKILL.md","ios-clean/SKILL.md","ios-design-review/SKILL.md","ios-fix/SKILL.md","ios-qa/SKILL.md","ios-sync/SKILL.md","land-and-deploy/SKILL.md","landing-report/SKILL.md","learn/SKILL.md","lib/gate-diff/classify.ts","lib/gate-diff/git.ts","lib/gate-diff/index.ts","lib/gate-diff/render.ts","lib/gate-diff/tags.ts","lib/gate-diff/types.ts","lib/review-evidence.ts","make-pdf/SKILL.md","office-hours/SKILL.md","open-gstack-browser/SKILL.md","pair-agent/SKILL.md","plan-ceo-review/SKILL.md","plan-ceo-review/sections/review-sections.md","plan-ceo-review/sections/review-sections.md.tmpl","plan-design-review/SKILL.md","plan-devex-review/SKILL.md","plan-eng-review/SKILL.md","plan-eng-review/sections/review-sections.md","plan-tune/SKILL.md","qa-only/SKILL.md","qa/SKILL.md","retro/SKILL.md","review/SKILL.md","review/SKILL.md.tmpl","review/checklist.md","review/gate-integrity.md","review/specialists/testing.md","scrape/SKILL.md","scripts/gate-diff-calibrate.ts","scripts/harness-version.json","scripts/question-registry.ts","scripts/resolvers/dispatch-rules.ts","scripts/resolvers/gate-scan.ts","scripts/resolvers/index.ts","scripts/resolvers/plan-gates.ts","scripts/resolvers/preamble/generate-completion-status.ts","scripts/resolvers/preamble/generate-evidence-directive.ts","scripts/resolvers/test-value.ts","scripts/resolvers/testing.ts","scripts/test-pr-profile.ts","setup-browser-cookies/SKILL.md","setup-deploy/SKILL.md","setup-gbrain/SKILL.md","ship/SKILL.md","ship/SKILL.md.tmpl","ship/sections/plan-completion.md","ship/sections/plan-completion.md.tmpl","ship/sections/pr-body.md","ship/sections/pr-body.md.tmpl","ship/sections/review-army.md","ship/sections/review-army.md.tmpl","ship/sections/test-coverage.md","ship/sections/test-coverage.md.tmpl","ship/sections/tests.md","ship/sections/tests.md.tmpl","skillify/SKILL.md","spec/SKILL.md","spec/SKILL.md.tmpl","spec/sections/gate-and-file.md","spec/sections/gate-and-file.md.tmpl","sync-gbrain/SKILL.md","test-audit/SKILL.md","test/dispatch-rules.test.ts","test/eng-finding-retry-budget.test.ts","test/env-host-runtime-assets.test.ts","test/evidence.test.ts","test/fixtures/context-budget.json","test/fixtures/gate-diff/adv-forged-measured.base.patch","test/fixtures/gate-diff/adv-forged-measured.patch","test/fixtures/gate-diff/adv-golden-beside-unrelated.base.patch","test/fixtures/gate-diff/adv-golden-beside-unrelated.patch","test/fixtures/gate-diff/adv-kotlin-ignore.base.patch","test/fixtures/gate-diff/adv-kotlin-ignore.patch","test/fixtures/gate-diff/control-added-test.base.patch","test/fixtures/gate-diff/control-added-test.patch","test/fixtures/gate-diff/control-rename.base.patch","test/fixtures/gate-diff/control-rename.patch","test/fixtures/gate-diff/control-snapshot-with-owner.base.patch","test/fixtures/gate-diff/control-snapshot-with-owner.patch","test/fixtures/gate-diff/control-spec-skill-dir.base.patch","test/fixtures/gate-diff/control-spec-skill-dir.patch","test/fixtures/gate-diff/control-test-deleted-with-source.base.patch","test/fixtures/gate-diff/control-test-deleted-with-source.patch","test/fixtures/gate-diff/package-json-scripts.base.patch","test/fixtures/gate-diff/package-json-scripts.patch","test/fixtures/gate-diff/rh1-removed.base.patch","test/fixtures/gate-diff/rh1-removed.patch","test/fixtures/gate-diff/rh1-skip.base.patch","test/fixtures/gate-diff/rh1-skip.patch","test/fixtures/gate-diff/rh12-env.base.patch","test/fixtures/gate-diff/rh12-env.patch","test/fixtures/gate-diff/rh13-pragma.base.patch","test/fixtures/gate-diff/rh13-pragma.patch","test/fixtures/gate-diff/rh14-bypass.base.patch","test/fixtures/gate-diff/rh14-bypass.patch","test/fixtures/gate-diff/rh15-bound.base.patch","test/fixtures/gate-diff/rh15-bound.patch","test/fixtures/gate-diff/rh15-timeout.base.patch","test/fixtures/gate-diff/rh15-timeout.patch","test/fixtures/gate-diff/rh16-swallow.base.patch","test/fixtures/gate-diff/rh16-swallow.patch","test/fixtures/gate-diff/rh3-golden.base.patch","test/fixtures/gate-diff/rh3-golden.patch","test/fixtures/gate-diff/rh3-snapshot-no-owner.base.patch","test/fixtures/gate-diff/rh3-snapshot-no-owner.patch","test/fixtures/gate-diff/rh4-placeholder.base.patch","test/fixtures/gate-diff/rh4-placeholder.patch","test/fixtures/golden/agy-ship-SKILL.md","test/fixtures/golden/claude-ship-SKILL.md","test/fixtures/golden/codex-ship-SKILL.md","test/fixtures/golden/factory-ship-SKILL.md","test/fixtures/module-size-ratchet.json","test/fixtures/workflow-required-steps.json","test/gate-diff-cli.test.ts","test/gate-diff.test.ts","test/gate-integrity-templates.test.ts","test/helpers/carve-guards.ts","test/helpers/gate-diff-repo.ts","test/helpers/placeholder-allowlist.ts","test/helpers/touchfiles-data.ts","test/honesty-kernel.test.ts","test/outside-voice-routing.test.ts","test/plan-tune.test.ts","test/review-start-evidence.test.ts","test/review-workflow-clarity.test.ts","test/ship-greptile-early.test.ts","test/ship-plan-completion-invariants.test.ts","test/skill-e2e-gate-integrity.test.ts","test/test-value-bar.test.ts"]}, + {"runId":38095294863,"pr":3116,"headSha":"82828943447b168adbbdc173c2095fcf33ba6f53","packageVersionOnly":true,"failedGateCases":["ship-exploratory-plan-checks"],"changedFiles":["CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","benchmark/SKILL.md","bin/gstack-browser-ensure","bin/gstack-bun-version.sh","bin/gstack-capy-install","bin/gstack-doctor","bin/gstack-doctor-check.sh","bin/gstack-doctor-components.sh","bin/gstack-runtime-pins.ts","browse/SKILL.md","canary/SKILL.md","design-consultation/SKILL.md","design-review/SKILL.md","devex-review/SKILL.md","docs/capy-install.md","docs/state-root.md","docs/troubleshooting.md","land-and-deploy/SKILL.md","lib/doctor-components.ts","lib/result-codes.ts","lib/runtime-pins.ts","open-gstack-browser/SKILL.md","package.json","pair-agent/SKILL.md","qa/sections/browser-setup.md","scrape/SKILL.md","scripts/gen-doctor-components.ts","scripts/gen-skill-docs.ts","scripts/resolvers/browse.ts","scripts/typecheck-test-baseline.json","setup","setup-browser-cookies/SKILL.md","test/bun-version-drift.test.ts","test/doctor-components.test.ts","test/egress-receipt-wiring.test.ts","test/fixtures/module-size-ratchet.json","test/gen-skill-docs-checks.test.ts","test/gen-skill-docs-prune-stale.test.ts","test/gstack-browser-ensure.test.ts","test/gstack-capy-install.test.ts","test/gstack-doctor.test.ts","test/helpers/manual-judge-review-fixture.ts","test/host-conformance.test.ts","test/runtime-pins.test.ts","test/troubleshooting-anchors.test.ts"]}, + {"runId":38097941308,"pr":3117,"headSha":"d0febdf79b1608f3de47f3c1ec099f7829f709f5","packageVersionOnly":true,"failedGateCases":["qa-functional-cli-report"],"changedFiles":["CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","autoplan/SKILL.md","autoplan/SKILL.md.tmpl","autoplan/sections/manifest.json","autoplan/sections/unattended.md","autoplan/sections/unattended.md.tmpl","bin/gstack-autoplan","bin/gstack-codex-login","bin/gstack-doctor-components.sh","bin/gstack-office-hours-review","bin/gstack-outside-voice","docs/troubleshooting.md","docs/unattended.md","lib/autoplan-export.ts","lib/autoplan-gate.ts","lib/autoplan-prompts.ts","lib/autoplan-reconcile.ts","lib/autoplan-run.ts","lib/autoplan-state.ts","lib/headless-artifacts.ts","lib/office-hours-review.ts","lib/outside-voice-api.ts","lib/outside-voice-runner.ts","lib/result-codes.ts","lib/spec-review-cap.ts","lib/spend-ledger.ts","office-hours/sections/design-and-handoff.md","package.json","plan-ceo-review/SKILL.md","plan-ceo-review/sections/checklist.md","plan-ceo-review/sections/checklist.md.tmpl","plan-ceo-review/sections/manifest.json","plan-design-review/sections/checklist.md","plan-design-review/sections/checklist.md.tmpl","plan-design-review/sections/manifest.json","plan-devex-review/sections/checklist.md","plan-devex-review/sections/checklist.md.tmpl","plan-devex-review/sections/manifest.json","plan-eng-review/sections/checklist.md","plan-eng-review/sections/checklist.md.tmpl","plan-eng-review/sections/manifest.json","scripts/resolvers/sections.ts","scripts/resolvers/spec-review.ts","test/autoplan-answer-invalidation.test.ts","test/autoplan-run-concurrency.test.ts","test/autoplan-run-recovery.test.ts","test/egress-receipt-wiring.test.ts","test/fixtures/module-size-ratchet.json","test/fixtures/multi-agent-wave/fixture-reviewer.ts","test/helpers/autoplan-run-fixture.ts","test/helpers/free-fixtures-data.ts","test/helpers/office-hours-completion.ts","test/helpers/workflow-judge-input.ts","test/office-hours-completion.test.ts","test/office-hours-review.test.ts","test/outside-runner-contract.test.ts","test/review-checklists.test.ts","test/spend-ledger.test.ts","test/workflow-judge-input.test.ts"]}, + {"runId":38101502677,"pr":3119,"headSha":"c9a1b4c273a2479dd62d10ba1477d0fe5c6c5824","packageVersionOnly":true,"failedGateCases":["land-and-deploy-review-gate","qa-functional-cli-fix"],"changedFiles":["CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","autoplan/SKILL.md","autoplan/SKILL.md.tmpl","autoplan/sections/ceo-phase.md","autoplan/sections/ceo-phase.md.tmpl","autoplan/sections/eng-phase.md","autoplan/sections/eng-phase.md.tmpl","benchmark/SKILL.md","benchmark/SKILL.md.tmpl","bin/gstack-doctor-components.sh","bin/gstack-plan-reality","cso/SKILL.md","cso/SKILL.md.tmpl","docs/troubleshooting.md","investigate/SKILL.md","investigate/SKILL.md.tmpl","investigate/sections/backlog.md","investigate/sections/backlog.md.tmpl","investigate/sections/eval-bisect.md","investigate/sections/eval-bisect.md.tmpl","investigate/sections/flake.md","investigate/sections/flake.md.tmpl","investigate/sections/manifest.json","investigate/sections/polluter.md","investigate/sections/polluter.md.tmpl","investigate/sections/race.md","investigate/sections/race.md.tmpl","lib/plan-reality.ts","lib/result-codes.ts","package.json","plan-ceo-review/SKILL.md","plan-ceo-review/SKILL.md.tmpl","plan-ceo-review/sections/checklist.md","plan-ceo-review/sections/checklist.md.tmpl","plan-ceo-review/sections/review-sections.md","plan-ceo-review/sections/review-sections.md.tmpl","plan-eng-review/sections/checklist.md","plan-eng-review/sections/checklist.md.tmpl","plan-eng-review/sections/review-sections.md","plan-eng-review/sections/review-sections.md.tmpl","review/SKILL.md","review/SKILL.md.tmpl","review/checklist.md","review/sections/plan-completion.md","scripts/resolvers/index.ts","scripts/resolvers/plan-gates.ts","scripts/resolvers/plan-reality.ts","scripts/resolvers/sideways.ts","test/carve-section-loading.test.ts","test/fixtures/module-size-ratchet.json","test/fixtures/multi-agent-wave/INCIDENTS.md","test/fixtures/multi-agent-wave/reality-rows.json","test/helpers/carve-guards.ts","test/helpers/free-fixtures-data.ts","test/helpers/parity-harness.ts","test/helpers/touchfiles-data.ts","test/investigate-recipes.test.ts","test/plan-reality.test.ts","test/qa-supervision-selection.test.ts","test/sideways-review.test.ts","test/skill-fixture.test.ts"]}, + {"runId":38101616162,"pr":3121,"headSha":"819ef8e083162096a5259625b6cc74465d3889df","packageVersionOnly":true,"failedGateCases":["design-review-plugin-handoff"],"changedFiles":[".gitignore",".gstack/generated.json",".gstack/pregate.json","CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-doctor-components.sh","bin/gstack-issue-links","bin/gstack-pregate","bin/gstack-regen","bin/gstack-ship-receipt","docs/pregate.md","docs/troubleshooting.md","lib/headless-artifacts.ts","lib/issue-links.ts","lib/pregate/checks/lanes.ts","lib/pregate/checks/literals.ts","lib/pregate/checks/regen.ts","lib/pregate/checks/secrets.ts","lib/pregate/checks/strays.ts","lib/pregate/config.ts","lib/pregate/context.ts","lib/pregate/import-graph.ts","lib/pregate/index.ts","lib/pregate/lane-selection.ts","lib/pregate/types.ts","lib/pregate/workflows.ts","lib/regen.ts","lib/result-codes.ts","lib/runtime-pins.ts","lib/ship-receipt.ts","package.json","scripts/gen-ship-goldens.ts","ship/sections/pr-body.md","ship/sections/pr-body.md.tmpl","ship/sections/tests.md","ship/sections/tests.md.tmpl","test/fixtures/module-size-ratchet.json","test/fixtures/multi-agent-wave/INCIDENTS.md","test/gstack-regen.test.ts","test/helpers/pregate-fixture.ts","test/pregate-invalidation.test.ts","test/pregate-selection.test.ts","test/ship-issue-intent.test.ts"]}, + {"runId":38336400566,"pr":3166,"headSha":"8bf348ddee3a31bed6e914f0cd95e49bf12f3d8e","packageVersionOnly":true,"failedGateCases":["ship-docsync-launch-failure"],"changedFiles":["CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-doctor-components.sh","bin/gstack-eval-plan","bin/gstack-pregate","bin/gstack-redact-prepush","docs/designs/GSTACK_FIELD_WAVE_2026_10_12.md","docs/eval-calls.md","docs/pregate.md","docs/ship-policy.md","docs/troubleshooting.md","docs/unattended.md","eval-plan/SKILL.md","eval-plan/SKILL.md.tmpl","lib/coordinator-contract.ts","lib/eval-plan.ts","lib/pregate/checks/guards.ts","lib/pregate/checks/lanes.ts","lib/pregate/config.ts","lib/pregate/index.ts","lib/pregate/regate.ts","lib/pregate/runtime.ts","lib/pricing.ts","lib/restamp.ts","lib/result-codes.ts","lib/sealed-guard.ts","lib/ship-policy.ts","lib/tree-receipt.ts","package.json","scripts/lib/windows-curation.ts","scripts/pr-dependencies.ts","spec/sections/gate-and-file.md","test/benchmark-runner.test.ts","test/fixtures/module-size-ratchet.json","test/helpers/capy-host.ts","test/helpers/placeholder-allowlist.ts","test/lane-coordination.test.ts","test/pr-dependencies.test.ts","test/pregate-guards.test.ts","test/pregate-invalidation.test.ts","test/pregate-regate.test.ts","test/pregate-selection.test.ts","test/pricing-eval-plan.test.ts","test/restamp-command-boundary.test.ts","test/sealed-guard.test.ts","test/stamp-templates.test.ts"]}, + {"runId":38337974933,"pr":3164,"headSha":"f6997e51f4351963fc198bf389872a92ee5b95c1","packageVersionOnly":true,"failedGateCases":["tpa-apple-ban"],"changedFiles":["CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-bun-version.sh","bin/gstack-capy-install","bin/gstack-capy-session","bin/gstack-doctor","bin/gstack-doctor-check.sh","bin/gstack-doctor-components.sh","bin/gstack-install-verdict.sh","bin/gstack-skill-end","bin/gstack-skill-start","bin/gstack-upgrade","capy/skill/gstack/SKILL.md","capy/skill/gstack/scripts/install.sh","capy/support.env","docs/CHANGELOG_STYLE.md","docs/capy-install.md","docs/designs/GSTACK_FIELD_WAVE_2026_10_12.md","docs/troubleshooting.md","docs/unattended.md","lib/doctor-components.ts","lib/install-verdict.ts","lib/result-codes.ts","package.json","scripts/harness-version.json","scripts/test-pr-profile.ts","test/capy-skill-wrapper.test.ts","test/fixtures/context-budget.json","test/gstack-capy-install.test.ts","test/gstack-doctor.test.ts","test/gstack-skill-start.test.ts","test/gstack-upgrade-bin.test.ts","test/helpers/capy-host.ts","test/install-verdict.test.ts","test/test-pr-profile.test.ts"]}, + {"runId":38345572852,"pr":3164,"headSha":"2dc7afe70762ddefd69c5ebbf15eea4cc3669fa4","packageVersionOnly":true,"failedGateCases":["qa-functional-webhook-report"],"changedFiles":["CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-bun-version.sh","bin/gstack-capy-install","bin/gstack-capy-session","bin/gstack-doctor","bin/gstack-doctor-check.sh","bin/gstack-doctor-components.sh","bin/gstack-install-verdict.sh","bin/gstack-skill-end","bin/gstack-skill-start","bin/gstack-upgrade","capy/skill/gstack/SKILL.md","capy/skill/gstack/scripts/install.sh","capy/support.env","docs/CHANGELOG_STYLE.md","docs/capy-install.md","docs/designs/GSTACK_FIELD_WAVE_2026_10_12.md","docs/troubleshooting.md","docs/unattended.md","lib/doctor-components.ts","lib/install-verdict.ts","lib/result-codes.ts","package.json","scripts/harness-version.json","scripts/test-pr-profile.ts","test/capy-skill-wrapper.test.ts","test/fixtures/context-budget.json","test/gstack-capy-install.test.ts","test/gstack-doctor.test.ts","test/gstack-skill-start.test.ts","test/gstack-upgrade-bin.test.ts","test/helpers/capy-host.ts","test/install-verdict.test.ts","test/test-pr-profile.test.ts"]}, + {"runId":38418037093,"pr":3170,"headSha":"44d380c494816ea915db718c7e3e98889e8f3256","packageVersionOnly":true,"failedGateCases":["shared-libs-read-only"],"changedFiles":["CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","autoplan/DRIVER.md","autoplan/DRIVER.md.tmpl","autoplan/SKILL.md","autoplan/SKILL.md.tmpl","bin/gstack-artifact","bin/gstack-autoplan","bin/gstack-autoplan-snapshot.ts","bin/gstack-doctor-components.sh","bin/gstack-plan-reality","docs/troubleshooting.md","docs/unattended.md","lib/autoplan-admission.ts","lib/autoplan-gate.ts","lib/autoplan-migrate.ts","lib/autoplan-plan-text.ts","lib/autoplan-prompts.ts","lib/autoplan-reconcile.ts","lib/autoplan-revise.ts","lib/autoplan-run.ts","lib/autoplan-scope.ts","lib/autoplan-state.ts","lib/gate-list.ts","lib/headless-artifacts.ts","lib/plan-reality.ts","lib/result-codes.ts","lib/source-root.ts","package.json","scripts/gen-skill-docs.ts","scripts/resolvers/autoplan-driver.ts","scripts/resolvers/index.ts","test/autoplan-admission.test.ts","test/autoplan-answer-invalidation.test.ts","test/autoplan-capy-e2e.test.ts","test/autoplan-driver.test.ts","test/autoplan-gate-judgment.test.ts","test/autoplan-record.test.ts","test/autoplan-revise.test.ts","test/autoplan-run-concurrency.test.ts","test/autoplan-scope.test.ts","test/fixtures/module-size-ratchet.json","test/gen-skill-docs-checks.test.ts","test/helpers/autoplan-capy.ts","test/plan-reality.test.ts","test/review-checklists.test.ts"]}, + {"runId":38428252213,"pr":3171,"headSha":"e2b08d308cd72f6d9bfbfcf514f0ece0891da153","packageVersionOnly":true,"failedGateCases":["land-and-deploy-workflow","ship-docsync-stale-before","ship-exploratory-plan-checks"],"changedFiles":["CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-doctor-check.sh","capy/skill/gstack/SKILL.md","docs/troubleshooting.md","lib/result-codes.ts","package.json","test/gstack-doctor.test.ts"]}, + {"runId":38478947158,"pr":3173,"headSha":"b6389e0631c99449be8f6ac8e9466f26b3daa611","packageVersionOnly":true,"failedGateCases":["ship-exploratory-plan-checks"],"changedFiles":["CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","autoplan/DRIVER.md","autoplan/DRIVER.md.tmpl","bin/gstack-autoplan","bin/gstack-capy-session","bin/gstack-doctor-components.sh","bin/gstack-gbrain-sync.ts","bin/gstack-harness-pid.sh","bin/gstack-lease","bin/gstack-skill-start","bin/gstack-upgrade","capy/skill/gstack/SKILL.md","capy/support.env","docs/capy-install.md","docs/designs/FIX_WAVE_2026_10_12.md","docs/troubleshooting.md","docs/unattended.md","lib/autoplan-admission.ts","lib/autoplan-export.ts","lib/autoplan-gate.ts","lib/autoplan-prompts.ts","lib/autoplan-run.ts","lib/gate-list.ts","lib/headless-artifacts.ts","lib/owner-brief.ts","lib/restamp.ts","lib/result-codes.ts","package.json","scripts/harness-version.json","test-setup.ts","test/auto-decide-fixture.test.ts","test/autoplan-answer-invalidation.test.ts","test/autoplan-capy-e2e.test.ts","test/autoplan-gate-judgment.test.ts","test/autoplan-preconfigured-onboarding-ar.test.ts","test/autoplan-residuals.test.ts","test/autoplan-revise.test.ts","test/capy-skill-wrapper.test.ts","test/ceo-finding-fixture.test.ts","test/gstack-lease.test.ts","test/gstack-skill-start.test.ts","test/gstack-upgrade-bin.test.ts","test/helpers/capy-host.ts","test/helpers/hermetic-env.ts"]}, + {"runId":38829994639,"pr":3209,"headSha":"2961edad7f56c982d78c97a85c87015bab29b155","packageVersionOnly":false,"failedGateCases":["qa-functional-cli-report"],"changedFiles":[".github/dependabot.yml",".github/workflows/arm-setup-smoke.yml",".github/workflows/cso-runtime-images.yml",".github/workflows/cso-runtime-promote.yml",".github/workflows/cso-runtime-qualification.yml",".github/workflows/cso-scanner-images.yml",".github/workflows/eval-sweep.yml",".github/workflows/evals-marathon.yml",".github/workflows/evals-periodic.yml",".github/workflows/evals.yml",".github/workflows/model-policy-freshness.yml",".github/workflows/native-qualification.yml",".github/workflows/scorecard.yml",".github/workflows/version-gate.yml",".github/workflows/windows-free-tests.yml",".github/workflows/windows-pregate-e2e.yml",".github/workflows/windows-setup-e2e.yml",".osv-scanner.toml","CHANGELOG.md","README.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-browser-ensure","bin/gstack-bun-version.sh","bin/gstack-ceo-plan-find","bin/gstack-claude-root.sh","bin/gstack-config","bin/gstack-doctor-components.sh","bin/gstack-evidence","bin/gstack-ff-update","bin/gstack-gate-diff","bin/gstack-gbrain-sync.ts","bin/gstack-memorable","bin/gstack-sanitize-echo.sh","bin/gstack-session-update","bin/gstack-skill-end","bin/gstack-timeline-log","browse/src/browser-manager.ts","browse/src/meta-commands.ts","browse/src/server.ts","browse/src/terminal-agent.ts","browse/test/tab-external-close-dispatch.test.ts","browse/test/tab-external-close.test.ts","browse/test/terminal-agent-signal-cleanup.test.ts","bun.lock","docs/autoplan-guard-troubleshooting.md","docs/designs/FIX_WAVE_2026_10_15.md","docs/howto-ios-testing-with-gstack.md","docs/troubleshooting.md","gstack-upgrade/SKILL.md","gstack-upgrade/SKILL.md.tmpl","hosts/claude/hooks/spawn-bin.ts","ios-qa/daemon/src/devicectl.ts","ios-qa/daemon/src/tunnel-bootstrap.ts","ios-qa/daemon/test/devicectl-launch-env.test.ts","lib/claude-journal-records.ts","lib/cso/docker.ts","lib/cso/process.ts","lib/cso/scanner-executor.ts","lib/cso/watchdog.c","lib/gate-diff/git.ts","lib/gbrain-exec.ts","lib/pregate/checks/secrets.ts","lib/pregate/import-graph.ts","lib/pregate/lane-selection.ts","lib/redact-engine.ts","lib/regen.ts","lib/restamp.ts","lib/result-codes.ts","lib/sanitize-echo.ts","lib/script-invocation.ts","package.json","plan-ceo-review/sections/review-sections.md","plan-devex-review/sections/review-sections.md","plan-eng-review/SKILL.md","plan-eng-review/SKILL.md.tmpl","plan-eng-review/sections/review-sections.md","scripts/harness-version.json","scripts/resolvers/outside-voice-steps.ts","scripts/test-pr-profile.ts","setup","setup.ps1","test/autoplan-compaction-preserved-segment.test.ts","test/bun-version-shells.test.ts","test/ceo-plan-find-cross-branch.test.ts","test/cso-docker-argv0.test.ts","test/cso-json-redaction.test.ts","test/cso-scanner-release.test.ts","test/dependency-security.test.ts","test/design-doc-data-guard.test.ts","test/fixtures/module-size-ratchet.json","test/fixtures/session-update-heartbeat.sh","test/fixtures/windows-bash-resolution.json","test/gbrain-source-gitignore.test.ts","test/gbrain-spawn-windows-shell.test.ts","test/gbrain-sync-no-bash-message.test.ts","test/gstack-browser-ensure.test.ts","test/gstack-ceo-plan-find.test.ts","test/gstack-ff-update.test.ts","test/gstack-memorable.test.ts","test/gstack-upgrade-fresh-shell.test.ts","test/held-install-recovery.test.ts","test/helpers/auq-sdk-capture.ts","test/helpers/carve-guards.ts","test/helpers/free-fixtures-data.ts","test/helpers/touchfiles-data.ts","test/model-policy-workflow.test.ts","test/osv-ignore-expiry.test.ts","test/outside-voice-native-deadline.test.ts","test/plan-tune-cathedral.test.ts","test/pregate-import-graph-paths.test.ts","test/pregate-manifest-paths.test.ts","test/readme-skill-lists.test.ts","test/redact-prepush-rebase-force-push.test.ts","test/redact-prepush-scan-range.test.ts","test/run-in-background-guidance.test.ts","test/sanitize-echo.test.ts","test/script-invocation-tripwire.test.ts","test/script-invocation.test.ts","test/section-capture-native-tools.test.ts","test/session-update-autostash.test.ts","test/session-update-heartbeat.test.ts","test/session-update-held-banner.test.ts","test/session-update-hook-hold.test.ts","test/session-update-stages.test.ts","test/setup-claude-code-migration.test.ts","test/setup-cursor-legacy-skills.test.ts","test/setup-hook-canonical-paths.test.ts","test/setup-hook-syntax-gate.test.ts","test/setup-playwright-best-effort.test.ts","test/setup-ps1.test.ts","test/ship-measure-sweep.test.ts","test/ship-measure.test.ts","test/skill-end-completion-durability.test.ts","test/state-root-config-merge.test.ts","test/team-mode.test.ts","test/test-value-bar.test.ts","test/timeline-stop-hook-slug.test.ts","test/upgrade-setup-recovery.test.ts","test/upgrade-template-pins.test.ts","test/workflow-action-pins.test.ts","test/wtree-consumers.test.ts"]}, + {"runId":38969924945,"pr":3225,"headSha":"09fe44fe435e7ccf6291f688d4883bcdcb5464da","packageVersionOnly":false,"failedGateCases":["qa-only-no-fix","ship-docsync-store"],"changedFiles":["benchmark/SKILL.md","bin/gstack-upgrade","browse/SKILL.md","canary/SKILL.md","design-consultation/SKILL.md","design-review/SKILL.md","devex-review/SKILL.md","land-and-deploy/SKILL.md","lib/redact-patterns.ts","plan-design-review/sections/review-sections.md","plan-design-review/sections/review-sections.md.tmpl","qa/sections/browser-setup.md","scrape/SKILL.md","scripts/resolvers/aside.ts","test/aside-driver.test.ts","test/cso-git-hardening.test.ts","test/gstack-upgrade-path-spelling.test.ts","test/redact-engine.test.ts","test/review-entry-and-design-clarity-au.test.ts"]}, + {"runId":38973329985,"pr":3225,"headSha":"c130a85ad516ff288a2d0840dfb3180b07b83eca","packageVersionOnly":false,"failedGateCases":["ship-exploratory-small-cli"],"changedFiles":["benchmark/SKILL.md","bin/gstack-upgrade","browse/SKILL.md","canary/SKILL.md","design-consultation/SKILL.md","design-review/SKILL.md","devex-review/SKILL.md","land-and-deploy/SKILL.md","lib/pregate/checks/secrets.ts","lib/redact-patterns.ts","plan-design-review/sections/review-sections.md","plan-design-review/sections/review-sections.md.tmpl","plan-devex-review/sections/review-sections.md","plan-devex-review/sections/review-sections.md.tmpl","qa/sections/browser-setup.md","scrape/SKILL.md","scripts/resolvers/aside.ts","test/aside-driver.test.ts","test/cso-git-hardening.test.ts","test/gstack-upgrade-path-spelling.test.ts","test/pregate-secrets-source-context.test.ts","test/redact-engine.test.ts","test/review-entry-and-design-clarity-au.test.ts"]}, + {"runId":38974669950,"pr":3190,"headSha":"03968f0a7c3f54d4181a20c37ee77594de26974f","packageVersionOnly":true,"failedGateCases":["qa-functional-webhook-fix"],"changedFiles":[".github/workflows/free-tests.yml","CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","autoplan/DRIVER.md","autoplan/DRIVER.md.tmpl","autoplan/sections/ceo-phase.md","autoplan/sections/eng-phase.md","bin/gstack-autoplan","bin/gstack-bun-version.sh","bin/gstack-capy-install","bin/gstack-capy-session","bin/gstack-doctor-check.sh","bin/gstack-doctor-components.sh","bin/gstack-gate","bin/gstack-install-verdict.sh","bin/gstack-issue-links","bin/gstack-plan-reality","bin/gstack-pregate","bin/gstack-ship-receipt","bin/gstack-upgrade","capy/skill/gstack/SKILL.md","capy/skill/gstack/scripts/install.sh","docs/capy-install.md","docs/pregate.md","docs/troubleshooting.md","docs/unattended.md","lib/autoplan-checkpoint.ts","lib/autoplan-export.ts","lib/autoplan-gate.ts","lib/autoplan-plan-text.ts","lib/autoplan-prompts.ts","lib/autoplan-reconcile.ts","lib/autoplan-revise.ts","lib/autoplan-run.ts","lib/autoplan-state.ts","lib/doctor-components.ts","lib/gate-list.ts","lib/headless-artifacts.ts","lib/install-verdict.ts","lib/issue-links.ts","lib/plan-reality.ts","lib/pregate/checks/guards.ts","lib/pregate/checks/lanes.ts","lib/pregate/config.ts","lib/pregate/context.ts","lib/pregate/index.ts","lib/pregate/runtime.ts","lib/pregate/supervise.ts","lib/pregate/types.ts","lib/result-codes.ts","lib/runtime-pins.ts","lib/ship-receipt.ts","package.json","scripts/lib/windows-curation.ts","scripts/resolvers/autoplan-driver.ts","scripts/resolvers/plan-reality.ts","scripts/ubicloud/ubi-runner.sh","setup","ship/SKILL.md","ship/SKILL.md.tmpl","ship/sections/handoff.md","ship/sections/handoff.md.tmpl","ship/sections/pr-body.md","ship/sections/pr-body.md.tmpl","ship/sections/tests.md","ship/sections/tests.md.tmpl","test/autoplan-answer-invalidation.test.ts","test/autoplan-capy-e2e.test.ts","test/autoplan-checkpoint.test.ts","test/autoplan-driver.test.ts","test/autoplan-gate-artifacts.test.ts","test/autoplan-gate-judgment.test.ts","test/autoplan-pairing.test.ts","test/autoplan-record.test.ts","test/autoplan-residuals.test.ts","test/autoplan-revise.test.ts","test/autoplan-run-concurrency.test.ts","test/autoplan-run-recovery.test.ts","test/bun-resolve.test.ts","test/bun-version-drift.test.ts","test/capy-skill-wrapper.test.ts","test/cso-distribution.test.ts","test/fixtures/autoplan-pairing/run-20261014-pairs.json","test/fixtures/context-budget.json","test/fixtures/golden/agy-ship-SKILL.md","test/fixtures/golden/claude-ship-SKILL.md","test/fixtures/golden/codex-ship-SKILL.md","test/fixtures/golden/factory-ship-SKILL.md","test/gate-list.test.ts","test/gstack-capy-install.test.ts","test/gstack-doctor.test.ts","test/gstack-upgrade-bin.test.ts","test/helpers/autoplan-capy.ts","test/helpers/autoplan-run-fixture.ts","test/helpers/carve-guards.ts","test/helpers/free-fixtures-data.ts","test/helpers/pregate-fixture.ts","test/host-conformance.test.ts","test/install-verdict.test.ts","test/plan-reality.test.ts","test/pregate-guards.test.ts","test/pregate-remote.test.ts","test/pregate-supervise.test.ts","test/redact-prepush-hook.test.ts","test/runtime-pins.test.ts","test/script-invocation-tripwire.test.ts","test/setup-hook-syntax-gate.test.ts","test/setup-playwright-best-effort.test.ts","test/ship-issue-intent.test.ts","test/ubicloud-runner.test.ts"]}, + {"runId":38976253774,"pr":3190,"headSha":"7d68dedae91ffadf494f4cd9a1ab5c253f4b73d6","packageVersionOnly":true,"failedGateCases":["autoplan-guard-pty"],"changedFiles":[".github/workflows/free-tests.yml","CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","autoplan/DRIVER.md","autoplan/DRIVER.md.tmpl","autoplan/sections/ceo-phase.md","autoplan/sections/eng-phase.md","bin/gstack-autoplan","bin/gstack-bun-version.sh","bin/gstack-capy-install","bin/gstack-capy-session","bin/gstack-doctor-check.sh","bin/gstack-doctor-components.sh","bin/gstack-gate","bin/gstack-install-verdict.sh","bin/gstack-issue-links","bin/gstack-plan-reality","bin/gstack-pregate","bin/gstack-ship-receipt","bin/gstack-upgrade","capy/skill/gstack/SKILL.md","capy/skill/gstack/scripts/install.sh","docs/capy-install.md","docs/pregate.md","docs/troubleshooting.md","docs/unattended.md","lib/autoplan-checkpoint.ts","lib/autoplan-export.ts","lib/autoplan-gate.ts","lib/autoplan-plan-text.ts","lib/autoplan-prompts.ts","lib/autoplan-reconcile.ts","lib/autoplan-revise.ts","lib/autoplan-run.ts","lib/autoplan-state.ts","lib/doctor-components.ts","lib/gate-list.ts","lib/headless-artifacts.ts","lib/install-verdict.ts","lib/issue-links.ts","lib/plan-reality.ts","lib/pregate/checks/guards.ts","lib/pregate/checks/lanes.ts","lib/pregate/config.ts","lib/pregate/context.ts","lib/pregate/index.ts","lib/pregate/runtime.ts","lib/pregate/supervise.ts","lib/pregate/types.ts","lib/result-codes.ts","lib/runtime-pins.ts","lib/ship-receipt.ts","package.json","scripts/lib/windows-curation.ts","scripts/resolvers/autoplan-driver.ts","scripts/resolvers/plan-reality.ts","scripts/ubicloud/ubi-runner.sh","setup","ship/SKILL.md","ship/SKILL.md.tmpl","ship/sections/handoff.md","ship/sections/handoff.md.tmpl","ship/sections/pr-body.md","ship/sections/pr-body.md.tmpl","ship/sections/tests.md","ship/sections/tests.md.tmpl","test/autoplan-answer-invalidation.test.ts","test/autoplan-capy-e2e.test.ts","test/autoplan-checkpoint.test.ts","test/autoplan-driver.test.ts","test/autoplan-gate-artifacts.test.ts","test/autoplan-gate-judgment.test.ts","test/autoplan-pairing.test.ts","test/autoplan-record.test.ts","test/autoplan-residuals.test.ts","test/autoplan-revise.test.ts","test/autoplan-run-concurrency.test.ts","test/autoplan-run-recovery.test.ts","test/bun-resolve.test.ts","test/bun-version-drift.test.ts","test/capy-skill-wrapper.test.ts","test/cso-distribution.test.ts","test/fixtures/autoplan-pairing/run-20261014-pairs.json","test/fixtures/context-budget.json","test/fixtures/golden/agy-ship-SKILL.md","test/fixtures/golden/claude-ship-SKILL.md","test/fixtures/golden/codex-ship-SKILL.md","test/fixtures/golden/factory-ship-SKILL.md","test/gate-list.test.ts","test/gstack-capy-install.test.ts","test/gstack-doctor.test.ts","test/gstack-upgrade-bin.test.ts","test/helpers/autoplan-capy.ts","test/helpers/autoplan-run-fixture.ts","test/helpers/carve-guards.ts","test/helpers/free-fixtures-data.ts","test/helpers/pregate-fixture.ts","test/helpers/qa-functional-fixture.ts","test/host-conformance.test.ts","test/install-verdict.test.ts","test/plan-reality.test.ts","test/pregate-guards.test.ts","test/pregate-remote.test.ts","test/pregate-supervise.test.ts","test/redact-prepush-hook.test.ts","test/runtime-pins.test.ts","test/script-invocation-tripwire.test.ts","test/setup-hook-syntax-gate.test.ts","test/setup-playwright-best-effort.test.ts","test/ship-issue-intent.test.ts","test/ubicloud-runner.test.ts"]}, + {"runId":38980005793,"pr":3225,"headSha":"3f4a98ecac3fc434f8aeef61bc9def17080718d9","packageVersionOnly":false,"failedGateCases":["design-review-detector-shim-dom","qa-quick"],"changedFiles":["benchmark/SKILL.md","bin/gstack-upgrade","browse/SKILL.md","canary/SKILL.md","design-consultation/SKILL.md","design-review/SKILL.md","devex-review/SKILL.md","land-and-deploy/SKILL.md","lib/pregate/checks/secrets.ts","lib/redact-patterns.ts","plan-design-review/sections/review-sections.md","plan-design-review/sections/review-sections.md.tmpl","plan-devex-review/sections/review-sections.md","plan-devex-review/sections/review-sections.md.tmpl","qa/sections/browser-setup.md","scrape/SKILL.md","scripts/resolvers/aside.ts","test/aside-driver.test.ts","test/cso-git-hardening.test.ts","test/gstack-upgrade-path-spelling.test.ts","test/pregate-secrets-source-context.test.ts","test/redact-engine.test.ts","test/review-entry-and-design-clarity-au.test.ts"]}, + {"runId":38981523623,"pr":3231,"headSha":"3ba8b460645b972f91042bb73e6a7a07eb4b05ab","packageVersionOnly":false,"failedGateCases":["ship-exploratory-plan-checks"],"changedFiles":["bin/gstack-skill-end","bin/gstack-skill-start","test/detector-corpus-ship-docsync-launch-failure.test.ts","test/docsync-command-grammar.test.ts","test/fixtures/detector-corpora/ship-docsync-launch-failure/38336400566-t1-fail-derived-current-claim.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38336400566-t1-fail-derived-prose-claim.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38336400566-t1-fail.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38829994639-t1-pass.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38972920715-t1-pass.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/inventory.json","test/fixtures/docsync-store-38969924945-bash.json","test/gstack-skill-start.test.ts","test/helpers/docsync-fault-eval.ts","test/helpers/docsync-observer.ts","test/helpers/free-fixtures-data.ts","test/helpers/qa-browser-deadline-evidence.ts","test/qa-browser-deadline-evidence.test.ts","test/test-of-test-ratchet.test.ts"]}, + {"runId":38983306684,"pr":3231,"headSha":"84c29e9ee3325c19626edff677dc1ff2ff4f21d9","packageVersionOnly":false,"failedGateCases":["qa-functional-cli-report","ship-docsync-missing-asset"],"changedFiles":["bin/gstack-skill-end","bin/gstack-skill-start","review/sections/review-army.md","scripts/resolvers/review-army.ts","ship/sections/review-army.md","test/detector-corpus-ship-docsync-launch-failure.test.ts","test/docsync-command-grammar.test.ts","test/fixtures/detector-corpora/ship-docsync-launch-failure/38336400566-t1-fail-derived-current-claim.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38336400566-t1-fail-derived-prose-claim.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38336400566-t1-fail.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38829994639-t1-pass.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38972920715-t1-pass.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/inventory.json","test/fixtures/docsync-store-38969924945-bash.json","test/gstack-skill-start.test.ts","test/helpers/docsync-fault-eval.ts","test/helpers/docsync-observer.ts","test/helpers/free-fixtures-data.ts","test/helpers/qa-browser-deadline-evidence.ts","test/qa-browser-deadline-evidence.test.ts","test/review-workflow-clarity.test.ts","test/test-of-test-ratchet.test.ts"]}, + {"runId":38984977679,"pr":3226,"headSha":"122638084111bcea20fc3f62e4a8cd286cbaa437","packageVersionOnly":false,"failedGateCases":["ship-docsync-failure"],"changedFiles":["README.md","autoplan/SKILL.md","autoplan/SKILL.md.tmpl","autoplan/sections/dx-phase.md","autoplan/sections/dx-phase.md.tmpl","bin/gstack-autoplan-snapshot.ts","bin/gstack-doctor-components.sh","bin/gstack-error-capture","devex-review/SKILL.md","devex-review/SKILL.md.tmpl","docs/agent-ready-errors.md","docs/skills.md","docs/troubleshooting.md","lib/autoplan-prompts.ts","lib/dx-error-capture.ts","lib/dx-error-contract.ts","lib/qa-deadline.ts","lib/result-codes.ts","plan-devex-review/SKILL.md","plan-devex-review/SKILL.md.tmpl","plan-devex-review/dx-hall-of-fame.md","plan-devex-review/sections/checklist.md","plan-devex-review/sections/checklist.md.tmpl","plan-devex-review/sections/review-sections.md","plan-devex-review/sections/review-sections.md.tmpl","scripts/resolvers/dx.ts","scripts/resolvers/index.ts","test/dx-error-capture.test.ts","test/dx-error-contract-grader.test.ts","test/dx-error-contract.test.ts","test/eng-finding-retry-budget.test.ts","test/fixtures/dx-error-contract-case.ts","test/helpers/touchfiles-data.ts","test/skill-e2e-dx-error-contract.test.ts"]}, + {"runId":38986120559,"pr":3231,"headSha":"6fd20bef0fe9387cf13dee88cb144e7b3914575b","packageVersionOnly":true,"failedGateCases":["qa-functional-webhook-fix"],"changedFiles":["CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-skill-end","bin/gstack-skill-start","package.json","review/sections/review-army.md","scripts/resolvers/review-army.ts","ship/sections/review-army.md","test/detector-corpus-ship-docsync-launch-failure.test.ts","test/docsync-command-grammar.test.ts","test/fixtures/detector-corpora/ship-docsync-launch-failure/38336400566-t1-fail-derived-current-claim.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38336400566-t1-fail-derived-prose-claim.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38336400566-t1-fail.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38829994639-t1-pass.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38972920715-t1-pass.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/inventory.json","test/fixtures/docsync-store-38969924945-bash.json","test/gstack-skill-start.test.ts","test/helpers/docsync-fault-eval.ts","test/helpers/docsync-observer.ts","test/helpers/free-fixtures-data.ts","test/helpers/qa-browser-deadline-evidence.ts","test/qa-browser-deadline-evidence.test.ts","test/review-workflow-clarity.test.ts","test/test-of-test-ratchet.test.ts"]}, + {"runId":38988927989,"pr":3232,"headSha":"641916c650257b7fb5719ec45bef70b00066874e","packageVersionOnly":false,"failedGateCases":["tpa-absent-linux"],"changedFiles":["bin/gstack-capy-install","bin/gstack-capy-session","bin/gstack-lease","bin/gstack-skill-start","capy/skill/gstack/SKILL.md","docs/capy-install.md","docs/troubleshooting.md","test/capy-skill-wrapper.test.ts","test/gstack-capy-install.test.ts","test/gstack-lease.test.ts","test/gstack-upgrade-bin.test.ts"]}, + {"runId":38990111230,"pr":3226,"headSha":"6c07d85c82d17bda81860cd0d78616de3dc14cc2","packageVersionOnly":false,"failedGateCases":["ship-docsync-stale-after"],"changedFiles":["README.md","autoplan/SKILL.md","autoplan/SKILL.md.tmpl","autoplan/sections/dx-phase.md","autoplan/sections/dx-phase.md.tmpl","bin/gstack-autoplan-snapshot.ts","bin/gstack-doctor-components.sh","bin/gstack-error-capture","devex-review/SKILL.md","devex-review/SKILL.md.tmpl","docs/agent-ready-errors.md","docs/skills.md","docs/troubleshooting.md","lib/autoplan-prompts.ts","lib/dx-error-capture.ts","lib/dx-error-contract.ts","lib/qa-deadline.ts","lib/result-codes.ts","plan-devex-review/SKILL.md","plan-devex-review/SKILL.md.tmpl","plan-devex-review/dx-hall-of-fame.md","plan-devex-review/sections/checklist.md","plan-devex-review/sections/checklist.md.tmpl","plan-devex-review/sections/review-sections.md","plan-devex-review/sections/review-sections.md.tmpl","scripts/resolvers/dx.ts","scripts/resolvers/index.ts","test/dx-error-capture.test.ts","test/dx-error-contract-grader.test.ts","test/dx-error-contract.test.ts","test/eng-finding-retry-budget.test.ts","test/fixtures/dx-error-contract-case.ts","test/helpers/touchfiles-data.ts","test/skill-e2e-dx-error-contract.test.ts"]}, + {"runId":38993076767,"pr":3234,"headSha":"b40d8418f9435804a4803981dcb01c7288d87e73","packageVersionOnly":false,"failedGateCases":["ship-docsync-store","ship-exploratory-small-cli"],"changedFiles":["autoplan/DRIVER.md","autoplan/DRIVER.md.tmpl","bin/gstack-artifact","bin/gstack-autoplan","bin/gstack-capy-install","bin/gstack-doctor-check.sh","bin/gstack-doctor-components.sh","bin/gstack-eval-plan","bin/gstack-install-verdict.sh","capy/skill/gstack/SKILL.md","capy/skill/gstack/scripts/install.sh","docs/capy-install.md","docs/designs/field-list-2-2026-10-16/PLAN.md","docs/designs/field-list-2-2026-10-16/ROUND2-OBLIGATIONS.md","docs/eval-models.md","docs/troubleshooting.md","docs/unattended.md","eval-plan/SKILL.md","eval-plan/SKILL.md.tmpl","lib/autoplan-budget.ts","lib/autoplan-checkpoint.ts","lib/autoplan-confirm.ts","lib/autoplan-export.ts","lib/autoplan-gate.ts","lib/autoplan-prompts.ts","lib/autoplan-public.ts","lib/autoplan-reconcile.ts","lib/autoplan-revise.ts","lib/autoplan-run.ts","lib/autoplan-state.ts","lib/coordinator-contract.ts","lib/doctor-components.ts","lib/eval-models.ts","lib/eval-plan.ts","lib/gate-list.ts","lib/headless-artifacts.ts","lib/install-verdict.ts","lib/result-codes.ts","lib/spend-ledger.ts","scripts/resolvers/autoplan-driver.ts","ship/sections/handoff.md","ship/sections/handoff.md.tmpl","spec/sections/gate-and-file.md","test/autoplan-budget.test.ts","test/autoplan-checkpoint.test.ts","test/autoplan-durability.test.ts","test/autoplan-public-export.test.ts","test/capy-review-voice.test.ts","test/capy-skill-wrapper.test.ts","test/doctor-components.test.ts","test/eval-models.test.ts","test/eval-plan-prereg-diagnostics.test.ts","test/fixtures/context-budget.json","test/fixtures/module-size-ratchet.json","test/gstack-capy-install.test.ts","test/gstack-doctor.test.ts"]}, + {"runId":39001107574,"pr":3236,"headSha":"79554ccaba38b47a06d55f43dc8fdbb922b738b9","packageVersionOnly":true,"failedGateCases":["qa-functional-cli-report"],"changedFiles":[".github/workflows/windows-pregate-e2e.yml","CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-doctor","bin/gstack-doctor-components.sh","bin/gstack-pregate","bin/gstack-windows-bash.ts","docs/ADDING_A_HOST.md","docs/troubleshooting.md","lib/pregate/checks/lanes.ts","lib/pregate/context.ts","lib/pregate/import-graph.ts","lib/pregate/index.ts","lib/regen.ts","lib/result-codes-release.ts","lib/script-invocation.ts","package.json","scripts/lib/windows-curation.ts","scripts/resolvers/preamble.ts","scripts/resolvers/preamble/generate-completion-status.ts","test/fixtures/golden/agy-ship-SKILL.md","test/fixtures/golden/codex-ship-SKILL.md","test/fixtures/golden/factory-ship-SKILL.md","test/fixtures/skill-md-size-allowlist.json","test/gen-skill-docs.test.ts","test/gstack-doctor.test.ts","test/helpers/free-fixtures-data.ts","test/helpers/pregate-fixture.ts","test/helpers/restamp-fixture.ts","test/pregate-bats-nested.test.ts","test/pregate-wtree-refusal.test.ts","test/review-entry-and-design-clarity-au.test.ts","test/script-invocation-tripwire.test.ts","test/script-invocation.test.ts","test/skill-md-size-guard.test.ts"]}, + {"runId":39002269281,"pr":3239,"headSha":"8e878d9ee9f06a196ff04ccfb0a4d836c13cb1aa","packageVersionOnly":false,"failedGateCases":["ship-exploratory-plan-checks"],"changedFiles":[".github/dependabot.yml",".github/workflows/version-gate.yml",".osv-scanner.toml","TODOS.md","bun.lock","lib/diagram-render/bun.lock","lib/diagram-render/package.json","make-pdf/src/image-size.ts","make-pdf/src/orchestrator.ts","make-pdf/test/docx-image-size.test.ts","make-pdf/test/fixtures/docx-images/doc.md","make-pdf/test/fixtures/docx-images/document.golden.xml","make-pdf/test/fixtures/docx-images/square.gif","make-pdf/test/fixtures/docx-images/tall.jpg","make-pdf/test/fixtures/docx-images/wide.png","package.json","scripts/compare-pr-version.ts","scripts/typecheck-test-baseline.json","test/ceo-finding-fixture.test.ts","test/compare-pr-version.test.ts","test/osv-config-wiring.test.ts","test/osv-ignore-expiry.test.ts","test/workflow-action-pins.test.ts"]}, + {"runId":39004466422,"pr":3239,"headSha":"65ca746bda533a348a66032a6541647b43339416","packageVersionOnly":false,"failedGateCases":["plan-mode-no-op"],"changedFiles":[".github/dependabot.yml",".github/workflows/version-gate.yml",".osv-scanner.toml","TODOS.md","bun.lock","lib/diagram-render/bun.lock","lib/diagram-render/package.json","make-pdf/src/docx-images.ts","make-pdf/src/html-to-vdom.d.ts","make-pdf/src/orchestrator.ts","make-pdf/test/docx-image-size.test.ts","make-pdf/test/fixtures/docx-images/doc.md","make-pdf/test/fixtures/docx-images/document.golden.xml","make-pdf/test/fixtures/docx-images/square.gif","make-pdf/test/fixtures/docx-images/tall.jpg","make-pdf/test/fixtures/docx-images/wide.png","package.json","scripts/compare-pr-version.ts","scripts/typecheck-test-baseline.json","test/ceo-finding-fixture.test.ts","test/compare-pr-version.test.ts","test/osv-config-wiring.test.ts","test/osv-ignore-expiry.test.ts","test/workflow-action-pins.test.ts","tsconfig.test.json"]}, + {"runId":39006896632,"pr":3239,"headSha":"3449a9cd5fd1ed51fd9242bde3a5636ae8d97b77","packageVersionOnly":false,"failedGateCases":["qa-functional-webhook-report"],"changedFiles":[".github/dependabot.yml",".github/workflows/version-gate.yml",".osv-scanner.toml","CHANGELOG.md","TODOS.md","VERSION","agents-digest/gstack-AGENTS.md","bun.lock","lib/diagram-render/bun.lock","lib/diagram-render/package.json","make-pdf/src/docx-images.ts","make-pdf/src/html-to-vdom.d.ts","make-pdf/src/orchestrator.ts","make-pdf/test/docx-image-size.test.ts","make-pdf/test/fixtures/docx-images/doc.md","make-pdf/test/fixtures/docx-images/document.golden.xml","make-pdf/test/fixtures/docx-images/square.gif","make-pdf/test/fixtures/docx-images/tall.jpg","make-pdf/test/fixtures/docx-images/wide.png","package.json","scripts/compare-pr-version.ts","scripts/typecheck-test-baseline.json","test/ceo-finding-fixture.test.ts","test/compare-pr-version.test.ts","test/osv-config-wiring.test.ts","test/osv-ignore-expiry.test.ts","test/workflow-action-pins.test.ts","tsconfig.test.json"]}, + {"runId":39011101113,"pr":3239,"headSha":"599d3f044952ad9ca5f92e6e4f78d342278b44e9","packageVersionOnly":false,"failedGateCases":["tpa-absent-linux"],"changedFiles":[".github/dependabot.yml",".github/workflows/version-gate.yml",".osv-scanner.toml","CHANGELOG.md","TODOS.md","VERSION","agents-digest/gstack-AGENTS.md","bun.lock","lib/diagram-render/bun.lock","lib/diagram-render/package.json","make-pdf/src/docx-images.ts","make-pdf/src/html-to-vdom.d.ts","make-pdf/src/orchestrator.ts","make-pdf/test/docx-image-size.test.ts","make-pdf/test/fixtures/docx-images/doc.md","make-pdf/test/fixtures/docx-images/document.golden.xml","make-pdf/test/fixtures/docx-images/square.gif","make-pdf/test/fixtures/docx-images/tall.jpg","make-pdf/test/fixtures/docx-images/wide.png","package.json","scripts/compare-pr-version.ts","scripts/typecheck-test-baseline.json","test/ceo-finding-fixture.test.ts","test/compare-pr-version.test.ts","test/osv-config-wiring.test.ts","test/osv-ignore-expiry.test.ts","test/workflow-action-pins.test.ts","tsconfig.test.json"]} + ] +} diff --git a/test/helpers/exercised-commands.json b/test/helpers/exercised-commands.json new file mode 100644 index 000000000..e2647ee8a --- /dev/null +++ b/test/helpers/exercised-commands.json @@ -0,0 +1,1905 @@ +{ + "schema": 1, + "generatedAt": "2026-10-17T19:29:23.672Z", + "observedThrough": "2026-10-08T06:50:33Z", + "sourceRuns": [ + "36205327552", + "36220963292", + "36278514291", + "36302678692", + "36315459968", + "36318630639", + "36391800575", + "36465804057", + "36468274063", + "36481155145", + "36481295477", + "36485436963", + "36491692039", + "36493452542", + "36493616568", + "36497566037", + "36505065023", + "36516246523", + "36521598255", + "36526785991", + "36537184166", + "36539603295", + "36551562564", + "36579289566", + "36589496678", + "36615456309", + "36620926989", + "36636909400", + "36637107473", + "36637661593", + "36638638110", + "36640582052", + "36640919692", + "36641031019", + "36641547412", + "36641801658", + "36641824710", + "36670216491", + "36682017584", + "36683045674", + "36685469932", + "36709485593", + "36711269409", + "36711836168", + "36713227509", + "36713549527", + "36714493404", + "36717552105", + "36717834406", + "36718311034", + "36718368006", + "36718436969", + "36718514685", + "36719119326", + "36720872161", + "36722276798", + "36725680777", + "36730053861", + "36730220242", + "36740979126", + "36741522675", + "36743390793", + "36745049965", + "36753939836", + "36755432181", + "36758397400", + "36760073893", + "36761671799", + "36762284181", + "36764370405", + "36768263333", + "36768400454", + "36770061511", + "36774149885", + "36775002132", + "36775932413", + "36776096754", + "36777335875", + "36781170544", + "36783214499", + "36783749239", + "36786871824", + "36787264898", + "36790850100", + "36793107556", + "36794197606", + "36794871032", + "36797095286", + "36797191931", + "36797673165", + "36798542003", + "36823310593", + "36834605576", + "36897642796", + "36898539571", + "36899642230", + "36903605989", + "36905008393", + "36907899270", + "36914699033", + "36914734297", + "36917041210", + "36920579887", + "36922422698", + "36965719520", + "37024904147", + "37025330680", + "37026118550", + "37033755821", + "37060128357", + "37060776291", + "37061202449", + "37062583804", + "37063836999", + "37064151421", + "37066055701", + "37070126890", + "37073429415", + "37077373640", + "37080729942", + "37082906717", + "37087797888", + "37090604134", + "37090925543", + "37091093956", + "37091743395", + "37092497208", + "37092677055", + "37093937027", + "37094035231", + "37097119454", + "37097590162", + "37098585955", + "37099423845", + "37100121821", + "37111582659", + "37116024111", + "37143137677", + "37146354490", + "37152219461", + "37153488058", + "37157517739", + "37158193827", + "37158662432", + "37160557715", + "37161546382", + "37162342436", + "37162847235", + "37163124276", + "37163226077", + "37163388183", + "37163438154", + "37163862654", + "37163935292", + "37164023690", + "37164663577", + "37164967731", + "37165022930", + "37166243626", + "37166586458", + "37168384066", + "37168819307", + "37170258467", + "37170610789", + "37172266374", + "37172907071", + "37173609560", + "37176237665", + "37176835584", + "37177847549", + "37178146922", + "37178720668", + "37179168735", + "37181717177", + "37182208158", + "37184467123", + "37186091144", + "37186918046", + "37188014112", + "37188668421", + "37189974078", + "37192833320", + "37194511147", + "37196784022", + "37196811165", + "37196900729", + "37197708438", + "37199579960", + "37199589136", + "37228201907", + "37228564701", + "37230295724", + "37230639146", + "37230870756", + "37232374839", + "37233114987", + "37234681021", + "37234978002", + "37235771700", + "37237194905", + "37238469830", + "37239016858", + "37241493377", + "37242691913", + "37246213438", + "37246668414", + "37248168494", + "37248388404", + "37248650879", + "37250221023", + "37250545914", + "37250645386", + "37251015146", + "37251229631", + "37251358164", + "37251850439", + "37252022510", + "37252831019", + "37253679118", + "37254021038", + "37256076805", + "37256265663", + "37265273012", + "37266174921", + "37266371726", + "37278830943", + "37296674175", + "37324177668", + "37324423488", + "37327038121", + "37346036236", + "37346541110", + "37347305215", + "37349332448", + "37374409928", + "37375445479", + "37375586954", + "37376593071", + "37389946212", + "37394378126", + "37395241262", + "37397309034", + "37398521095", + "37413464119", + "37415996926", + "37416279318", + "37494809494", + "37504870257", + "37506300031", + "37507007764", + "37510319705", + "37512150975", + "37523820462", + "37540020229", + "37540144327", + "37541010145", + "37542732748", + "37543978473", + "37545094812", + "37545716546", + "37546366036", + "37559982334", + "37560420246", + "37564096977", + "37565786884", + "37640916778", + "37691104551", + "37691665506", + "37693401496", + "37694473528", + "37701124596", + "37707903246", + "37718098351", + "37728544941", + "37731666423", + "37734578981", + "37734678850", + "37734795948", + "37738679055", + "37739140797", + "37739365674", + "37739711591", + "37739845694", + "37739864688", + "37739923087", + "37740005194", + "37741231927", + "37741305367", + "37741434771", + "37742517444", + "37742917771", + "37743060710", + "37743622252", + "37744583272", + "37745729841", + "37747272797", + "37747282656", + "37749419367", + "37761125544", + "37762901988", + "37762951968", + "37766639989", + "37799081635", + "37801128025", + "37803230995", + "37803446927", + "37805183773", + "37810177756", + "37811045289", + "37812094929", + "37813893652", + "37816281826", + "37834873799", + "37887208552", + "37887677058", + "37888258971", + "37892285758", + "37892504905", + "37897476192", + "38009577313", + "38009751458", + "38077369614", + "38087167729", + "38087206509", + "38087752336", + "38089176598", + "38093707846", + "38093940617", + "38094956965", + "38095294863", + "38096225501", + "38097202777", + "38097643686", + "38097811247", + "38097941308", + "38098277242", + "38098894928", + "38099588461", + "38100216011", + "38100768977", + "38101113898", + "38101502677", + "38101616162", + "38102454452", + "38102657574", + "38103692513", + "38105628107", + "38106695235", + "38114482325", + "38114743865", + "38114753174", + "38115320223", + "38115388121", + "38115472535", + "38116231959", + "38130211171", + "38146252958", + "38149926651", + "38150794867", + "38151498909", + "38151698313", + "38152603302", + "38152625837", + "38152692645", + "38152760329", + "38153103642", + "38153201884", + "38153296624", + "38153330406", + "38153440113", + "38153468775", + "38153478324", + "38153524782", + "38153545779", + "38154139862", + "38154283169", + "38154299190", + "38154371839", + "38154587215", + "38155175678", + "38155221280", + "38156016740", + "38159532295", + "38159643806", + "38160879452", + "38161561304", + "38162103937", + "38165705267", + "38168912246", + "38186078591", + "38189716570", + "38190450074", + "38191930614", + "38210650474", + "38254994238", + "38292011452", + "38295240201", + "38304815896", + "38320765570", + "38335637302", + "38335683298", + "38336135754", + "38336400566", + "38336529808", + "38337974933", + "38339027571", + "38344574992", + "38345572852", + "38416346427", + "38416389781", + "38418037093", + "38421455413", + "38428252213", + "38439240685", + "38478095918", + "38478947158", + "38508804072", + "38512515424", + "38514180564", + "38518769031", + "38531754196", + "38604265113", + "38605448921", + "38624056266", + "38635247772", + "38637567300", + "38669096043", + "38669488941", + "38671593723", + "38673080757", + "38674274388", + "38725836286", + "38785477428", + "38800593092", + "38815098576", + "38817374062", + "38824819744", + "38827690901", + "38829062754", + "38829515637", + "38829994639", + "38832498886", + "38833122675", + "38835263541", + "38842124064", + "38864846669", + "38964584946", + "38964801793", + "38964995514", + "38965078377", + "38965221701", + "38965260163", + "38969798938", + "38969924945", + "38972920715", + "38973329985", + "38973338025", + "38973420671", + "38974669950", + "38975010169", + "38975560167", + "38975789267", + "38976253774", + "38976351570", + "38978531340", + "38978533724", + "38978745945", + "38979194128", + "38979925666", + "38980005793", + "38980008914", + "38980089002", + "38980452190", + "38980914399", + "38981388043", + "38981492840", + "38981523623", + "38981831429", + "38982033023", + "38982079721", + "38982609798", + "38982836813", + "38983306684", + "38983355048", + "38983422226", + "38983471330", + "38983723250", + "38983767702", + "38984904318", + "38984910364", + "38984977679", + "38985082997", + "38985111066", + "38985644097", + "38985800954", + "38986120559", + "38986222182", + "38986505512", + "38987526600", + "38987672759", + "38987773038", + "38987895421", + "38988823602", + "38988898707", + "38988927989", + "38989956807", + "38990111230", + "38990141270", + "38991706282", + "38991832658", + "38992787872", + "38992860354", + "38993076767", + "38994816769", + "38996100960", + "38996605130", + "38996899290", + "38997410828", + "38998005661", + "38999746202", + "39000951833", + "39001049596", + "39001107574", + "39001765726", + "39001816434", + "39002175316", + "39002269281", + "39003503204", + "39003571667", + "39004466422", + "39004497896", + "39005025976", + "39005664700", + "39006896632", + "39009803881", + "39010388987", + "39010883961", + "39011101113", + "39017702380", + "39044530662", + "39051148306" + ], + "cases": { + "auq-format-gate": { + "observations": 162, + "withCommands": 0, + "commands": [], + "costUsd": 0, + "costRuns": 110 + }, + "autoplan-guard-pty": { + "observations": 7, + "withCommands": 0, + "commands": [], + "costUsd": 0, + "costRuns": 89 + }, + "benchmark-workflow": { + "observations": 179, + "withCommands": 179, + "commands": [ + "bin/gstack-ci-gate", + "bin/gstack-context-recovery", + "bin/gstack-decision-log", + "bin/gstack-decision-search", + "bin/gstack-diff-scope", + "bin/gstack-evidence", + "bin/gstack-issue-guard", + "bin/gstack-learnings-log", + "bin/gstack-next-version", + "bin/gstack-paths", + "bin/gstack-question-log", + "bin/gstack-question-preference", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-skill-end", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-wtree", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 91 + }, + "browse-basic": { + "observations": 158, + "withCommands": 158, + "commands": [ + "bin/gstack-learnings-log", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 106 + }, + "browse-snapshot": { + "observations": 158, + "withCommands": 158, + "commands": [ + "bin/gstack-learnings-log", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 106 + }, + "canary-workflow": { + "observations": 182, + "withCommands": 182, + "commands": [ + "bin/gstack-ci-gate", + "bin/gstack-context-recovery", + "bin/gstack-decision-log", + "bin/gstack-decision-search", + "bin/gstack-diff-scope", + "bin/gstack-evidence", + "bin/gstack-issue-guard", + "bin/gstack-learnings-log", + "bin/gstack-next-version", + "bin/gstack-paths", + "bin/gstack-question-log", + "bin/gstack-question-preference", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-skill-end", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-wtree", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 91 + }, + "context-recovery-artifacts": { + "observations": 166, + "withCommands": 161, + "commands": [ + "bin/gstack-context-recovery", + "bin/gstack-decision-search", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-paths", + "bin/gstack-remote-identity.sh", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-state-root.sh", + "bin/gstack-timeline-log", + "bin/gstack-timeline-read", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 91 + }, + "context-restore-loads-latest": { + "observations": 167, + "withCommands": 162, + "commands": [ + "bin/gstack-context-recovery", + "bin/gstack-decision-search", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-paths", + "bin/gstack-remote-identity.sh", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-state-root.sh", + "bin/gstack-timeline-log", + "bin/gstack-timeline-read", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 105 + }, + "context-save-writes-file": { + "observations": 166, + "withCommands": 161, + "commands": [ + "bin/gstack-context-recovery", + "bin/gstack-decision-search", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-paths", + "bin/gstack-remote-identity.sh", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-state-root.sh", + "bin/gstack-timeline-log", + "bin/gstack-timeline-read", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 91 + }, + "cso-diff-mode": { + "observations": 157, + "withCommands": 157, + "commands": [ + "design/dist/design" + ], + "costUsd": 1.5371, + "costRuns": 91 + }, + "design-review-detector-shim": { + "observations": 174, + "withCommands": 174, + "commands": [ + "bin/gstack-design-detect.ts", + "bin/gstack-redact", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 92 + }, + "design-review-detector-shim-dom": { + "observations": 170, + "withCommands": 170, + "commands": [ + "bin/gstack-design-detect.ts", + "bin/gstack-redact", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 91 + }, + "design-review-plugin-handoff": { + "observations": 162, + "withCommands": 162, + "commands": [ + "bin/gstack-config", + "bin/gstack-design-detect.ts", + "bin/gstack-learnings-search", + "bin/gstack-redact", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 91 + }, + "diagram-triplet": { + "observations": 141, + "withCommands": 0, + "commands": [], + "costUsd": 0, + "costRuns": 106 + }, + "docsync-spawned": { + "observations": 181, + "withCommands": 180, + "commands": [ + "bin/gstack-docs-candidate", + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 1.2246, + "costRuns": 107 + }, + "document-release": { + "observations": 155, + "withCommands": 155, + "commands": [ + "bin/gstack-bun-version.sh", + "bin/gstack-codex-probe", + "bin/gstack-config", + "bin/gstack-context-recovery", + "bin/gstack-decision-log", + "bin/gstack-decision-search", + "bin/gstack-learnings-log", + "bin/gstack-next-version", + "bin/gstack-paths", + "bin/gstack-review-log", + "bin/gstack-skill-end", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-update-check", + "bin/gstack-upgrade", + "bin/gstack-version-bump", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 91 + }, + "gstack-upgrade-happy-path": { + "observations": 157, + "withCommands": 156, + "commands": [ + "bin/gstack-bun-version.sh", + "bin/gstack-config", + "bin/gstack-paths", + "bin/gstack-update-check", + "bin/gstack-upgrade" + ], + "costUsd": 0, + "costRuns": 117 + }, + "hermetic-canary": { + "observations": 167, + "withCommands": 167, + "commands": [], + "costUsd": 0.0127, + "costRuns": 101 + }, + "hermetic-sentinel": { + "observations": 167, + "withCommands": 167, + "commands": [], + "costUsd": 0.013, + "costRuns": 101 + }, + "investigate-owned-abort": { + "observations": 178, + "withCommands": 178, + "commands": [], + "costUsd": 0.2212, + "costRuns": 105 + }, + "investigate-owned-completion": { + "observations": 164, + "withCommands": 164, + "commands": [], + "costUsd": 0.2184, + "costRuns": 105 + }, + "investigate-owned-ending-error": { + "observations": 178, + "withCommands": 178, + "commands": [], + "costUsd": 0.2694, + "costRuns": 105 + }, + "land-and-deploy-first-run": { + "observations": 179, + "withCommands": 179, + "commands": [ + "bin/gstack-ci-gate", + "bin/gstack-config", + "bin/gstack-context-recovery", + "bin/gstack-decision-log", + "bin/gstack-decision-search", + "bin/gstack-diff-scope", + "bin/gstack-evidence", + "bin/gstack-issue-guard", + "bin/gstack-learnings-log", + "bin/gstack-next-version", + "bin/gstack-paths", + "bin/gstack-question-log", + "bin/gstack-question-preference", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-skill-end", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-wtree", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 91 + }, + "land-and-deploy-review-gate": { + "observations": 175, + "withCommands": 175, + "commands": [ + "bin/gstack-ci-gate", + "bin/gstack-config", + "bin/gstack-context-recovery", + "bin/gstack-decision-log", + "bin/gstack-decision-search", + "bin/gstack-diff-scope", + "bin/gstack-evidence", + "bin/gstack-issue-guard", + "bin/gstack-learnings-log", + "bin/gstack-next-version", + "bin/gstack-paths", + "bin/gstack-question-log", + "bin/gstack-question-preference", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-skill-end", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-wtree", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 92 + }, + "land-and-deploy-workflow": { + "observations": 180, + "withCommands": 180, + "commands": [ + "bin/gstack-ci-gate", + "bin/gstack-config", + "bin/gstack-context-recovery", + "bin/gstack-decision-log", + "bin/gstack-decision-search", + "bin/gstack-diff-scope", + "bin/gstack-evidence", + "bin/gstack-issue-guard", + "bin/gstack-learnings-log", + "bin/gstack-next-version", + "bin/gstack-paths", + "bin/gstack-question-log", + "bin/gstack-question-preference", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-skill-end", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-wtree", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 92 + }, + "learnings-show": { + "observations": 168, + "withCommands": 168, + "commands": [ + "bin/gstack-context-recovery", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-skill-end", + "bin/gstack-skill-start", + "bin/gstack-slug" + ], + "costUsd": 0, + "costRuns": 105 + }, + "office-hours-auto-mode": { + "observations": 181, + "withCommands": 0, + "commands": [], + "costUsd": 0, + "costRuns": 108 + }, + "operational-learning": { + "observations": 155, + "withCommands": 155, + "commands": [ + "bin/gstack-learnings-log", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 91 + }, + "plan-ceo-finding-floor": { + "observations": 199, + "withCommands": 197, + "commands": [ + "bin/gstack-ceo-mode-handoff", + "bin/gstack-config", + "bin/gstack-context-recovery", + "bin/gstack-learnings-search", + "bin/gstack-paths", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-skill-start", + "bin/gstack-slug", + "browse/bin/remote-slug" + ], + "costUsd": 0, + "costRuns": 92 + }, + "plan-ceo-review": { + "observations": 59, + "withCommands": 59, + "commands": [] + }, + "plan-ceo-review-plan-mode": { + "observations": 166, + "withCommands": 166, + "commands": [ + "bin/gstack-context-recovery", + "bin/gstack-skill-start", + "bin/gstack-slug", + "browse/bin/remote-slug" + ], + "costUsd": 0, + "costRuns": 109 + }, + "plan-design-review-no-ui-scope": { + "observations": 166, + "withCommands": 166, + "commands": [ + "bin/gstack-design-detect.ts", + "bin/gstack-learnings-log", + "bin/gstack-redact", + "bin/gstack-skill-end", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 106 + }, + "plan-design-with-ui-scope": { + "observations": 183, + "withCommands": 183, + "commands": [ + "bin/gstack-config", + "bin/gstack-context-recovery", + "bin/gstack-decision-log", + "bin/gstack-learnings-search", + "bin/gstack-paths", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-skill-start", + "bin/gstack-slug", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 91 + }, + "plan-devex-finding-floor": { + "observations": 180, + "withCommands": 178, + "commands": [ + "bin/gstack-config", + "bin/gstack-context-recovery", + "bin/gstack-question-log", + "bin/gstack-review-log", + "bin/gstack-skill-start", + "bin/gstack-slug", + "browse/bin/remote-slug", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 91 + }, + "plan-devex-review-plan-mode": { + "observations": 161, + "withCommands": 0, + "commands": [], + "costUsd": 0, + "costRuns": 90 + }, + "plan-eng-coverage-audit": { + "observations": 179, + "withCommands": 178, + "commands": [], + "costUsd": 0.4321, + "costRuns": 111 + }, + "plan-mode-no-op": { + "observations": 180, + "withCommands": 0, + "commands": [], + "costUsd": 0, + "costRuns": 93 + }, + "plan-review-report": { + "observations": 183, + "withCommands": 100, + "commands": [ + "bin/gstack-codex-probe", + "bin/gstack-config", + "bin/gstack-office-hours-review", + "bin/gstack-review-log" + ], + "costUsd": 0, + "costRuns": 111 + }, + "plan-tune-inspect": { + "observations": 168, + "withCommands": 168, + "commands": [ + "bin/gstack-config", + "bin/gstack-developer-profile", + "bin/gstack-paths", + "bin/gstack-question-log", + "bin/gstack-question-preference", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-state-root.sh" + ], + "costUsd": 0, + "costRuns": 105 + }, + "qa-bootstrap": { + "observations": 174, + "withCommands": 173, + "commands": [ + "bin/gstack-config", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-paths", + "bin/gstack-qa-deadline", + "bin/gstack-qa-evidence", + "bin/gstack-slug", + "browse/bin/find-browse", + "browse/bin/remote-slug", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 108 + }, + "qa-functional-cli-fix": { + "observations": 159, + "withCommands": 158, + "commands": [ + "bin/gstack-qa-evidence" + ], + "costUsd": 1.3402, + "costRuns": 106 + }, + "qa-functional-cli-report": { + "observations": 168, + "withCommands": 168, + "commands": [ + "bin/gstack-qa-evidence" + ], + "costUsd": 1.1367, + "costRuns": 106 + }, + "qa-functional-webhook-fix": { + "observations": 159, + "withCommands": 158, + "commands": [ + "bin/gstack-qa-evidence" + ], + "costUsd": 1.4296, + "costRuns": 106 + }, + "qa-functional-webhook-report": { + "observations": 168, + "withCommands": 168, + "commands": [ + "bin/gstack-qa-evidence" + ], + "costUsd": 1.3103, + "costRuns": 106 + }, + "qa-only-no-fix": { + "observations": 174, + "withCommands": 174, + "commands": [ + "bin/gstack-config", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-paths", + "bin/gstack-qa-deadline", + "bin/gstack-qa-evidence", + "bin/gstack-slug", + "browse/bin/find-browse", + "browse/bin/remote-slug", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 92 + }, + "qa-quick": { + "observations": 174, + "withCommands": 174, + "commands": [ + "bin/gstack-config", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-paths", + "bin/gstack-qa-deadline", + "bin/gstack-qa-evidence", + "bin/gstack-slug", + "browse/bin/find-browse", + "browse/bin/remote-slug", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 92 + }, + "retro": { + "observations": 134, + "withCommands": 134, + "commands": [ + "bin/gstack-config", + "bin/gstack-decision-search", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-paths", + "bin/gstack-retro-metrics", + "bin/gstack-slug", + "bin/gstack-state-root.sh" + ] + }, + "retro-base-branch": { + "observations": 159, + "withCommands": 25, + "commands": [ + "bin/gstack-config", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-paths", + "bin/gstack-retro-metrics", + "bin/gstack-slug" + ], + "costUsd": 0, + "costRuns": 94 + }, + "review-army-delivery-audit": { + "observations": 177, + "withCommands": 177, + "commands": [ + "bin/gstack-diff-scope", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-review-log", + "bin/gstack-specialist-stats" + ], + "costUsd": 0, + "costRuns": 95 + }, + "review-army-json-findings": { + "observations": 160, + "withCommands": 160, + "commands": [ + "bin/gstack-diff-scope", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-review-log", + "bin/gstack-specialist-stats" + ], + "costUsd": 0, + "costRuns": 95 + }, + "review-army-migration-safety": { + "observations": 164, + "withCommands": 164, + "commands": [ + "bin/gstack-diff-scope", + "bin/gstack-egress-lib.sh", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-review-log", + "bin/gstack-specialist-stats", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 95 + }, + "review-army-perf-n-plus-one": { + "observations": 165, + "withCommands": 165, + "commands": [ + "bin/gstack-decision-search", + "bin/gstack-diff-scope", + "bin/gstack-egress-lib.sh", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-review-log", + "bin/gstack-specialist-stats", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 95 + }, + "review-army-quality-score": { + "observations": 160, + "withCommands": 160, + "commands": [ + "bin/gstack-diff-scope", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-review-log", + "bin/gstack-specialist-stats" + ], + "costUsd": 0.3515, + "costRuns": 95 + }, + "review-base-branch": { + "observations": 168, + "withCommands": 168, + "commands": [ + "bin/gstack-diff-scope", + "bin/gstack-review-read" + ], + "costUsd": 0, + "costRuns": 91 + }, + "review-coverage-audit": { + "observations": 178, + "withCommands": 167, + "commands": [], + "costUsd": 0.4079, + "costRuns": 105 + }, + "review-dashboard-via": { + "observations": 168, + "withCommands": 168, + "commands": [ + "bin/gstack-diff-scope", + "bin/gstack-review-read" + ], + "costUsd": 0, + "costRuns": 93 + }, + "review-enum-completeness": { + "observations": 166, + "withCommands": 166, + "commands": [ + "bin/gstack-decision-search", + "bin/gstack-review-log" + ], + "costUsd": 0, + "costRuns": 91 + }, + "review-exploratory-small-cli": { + "observations": 170, + "withCommands": 169, + "commands": [ + "bin/gstack-codex-probe", + "bin/gstack-config", + "bin/gstack-diff-scope", + "bin/gstack-learnings-search", + "bin/gstack-qa-deadline", + "bin/gstack-qa-evidence", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-specialist-stats" + ], + "costUsd": 1.9457, + "costRuns": 115 + }, + "review-gate-integrity": { + "observations": 0, + "withCommands": 0, + "commands": [], + "costUsd": 0, + "costRuns": 81 + }, + "review-sql-injection": { + "observations": 168, + "withCommands": 168, + "commands": [ + "bin/gstack-decision-search", + "bin/gstack-review-log" + ], + "costUsd": 0, + "costRuns": 105 + }, + "review-test-value": { + "observations": 169, + "withCommands": 169, + "commands": [], + "costUsd": 0.6768, + "costRuns": 105 + }, + "safety-codex-consult-embed": { + "observations": 68, + "withCommands": 68, + "commands": [ + "bin/gstack-codex-probe", + "bin/gstack-paths" + ], + "costUsd": 0.9964, + "costRuns": 91 + }, + "safety-pair-agent-block": { + "observations": 68, + "withCommands": 68, + "commands": [ + "bin/gstack-config", + "browse/dist/browse" + ], + "costUsd": 0.3055, + "costRuns": 91 + }, + "safety-ship-stale-evidence": { + "observations": 67, + "withCommands": 67, + "commands": [ + "bin/gstack-ci-gate", + "bin/gstack-evidence", + "bin/gstack-slug", + "bin/gstack-wtree" + ], + "costUsd": 0.5915, + "costRuns": 91 + }, + "setup-deploy-workflow": { + "observations": 181, + "withCommands": 181, + "commands": [ + "bin/gstack-ci-gate", + "bin/gstack-context-recovery", + "bin/gstack-decision-log", + "bin/gstack-decision-search", + "bin/gstack-diff-scope", + "bin/gstack-evidence", + "bin/gstack-issue-guard", + "bin/gstack-learnings-log", + "bin/gstack-next-version", + "bin/gstack-paths", + "bin/gstack-question-log", + "bin/gstack-question-preference", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-skill-end", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-wtree", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 106 + }, + "shared-libs-read-only": { + "observations": 166, + "withCommands": 166, + "commands": [ + "bin/gstack-safe-git", + "browse/dist/browse" + ], + "costUsd": 0.786, + "costRuns": 92 + }, + "shared-libs-review-index-flags": { + "observations": 177, + "withCommands": 176, + "commands": [ + "bin/gstack-review-log", + "bin/gstack-review-read" + ], + "costUsd": 2.4926, + "costRuns": 91 + }, + "shared-libs-review-lifecycle": { + "observations": 174, + "withCommands": 173, + "commands": [ + "bin/gstack-egress-lib.sh", + "bin/gstack-review-log", + "bin/gstack-review-read" + ], + "costUsd": 2.4643, + "costRuns": 92 + }, + "shared-libs-review-path-eligibility": { + "observations": 171, + "withCommands": 170, + "commands": [ + "bin/gstack-review-log", + "bin/gstack-review-read" + ], + "costUsd": 3.9227, + "costRuns": 91 + }, + "shared-libs-review-prior-coverage": { + "observations": 164, + "withCommands": 164, + "commands": [ + "bin/gstack-review-log", + "bin/gstack-review-read" + ], + "costUsd": 2.517, + "costRuns": 91 + }, + "shared-libs-review-revalidation": { + "observations": 174, + "withCommands": 171, + "commands": [ + "bin/gstack-review-log", + "bin/gstack-review-read" + ], + "costUsd": 4.8033, + "costRuns": 92 + }, + "shared-libs-unsupported-git": { + "observations": 159, + "withCommands": 159, + "commands": [ + "bin/gstack-safe-git", + "browse/dist/browse" + ], + "costUsd": 0.5855, + "costRuns": 92 + }, + "ship-base-branch": { + "observations": 168, + "withCommands": 168, + "commands": [ + "bin/gstack-diff-scope", + "bin/gstack-review-read" + ], + "costUsd": 0, + "costRuns": 92 + }, + "ship-coverage-audit": { + "observations": 157, + "withCommands": 156, + "commands": [ + "bin/gstack-bun-version.sh", + "bin/gstack-config", + "bin/gstack-paths", + "bin/gstack-slug", + "bin/gstack-update-check", + "bin/gstack-upgrade" + ], + "costUsd": 0.7746, + "costRuns": 107 + }, + "ship-coverage-value": { + "observations": 169, + "withCommands": 168, + "commands": [ + "bin/gstack-paths", + "bin/gstack-slug" + ], + "costUsd": 0.9375, + "costRuns": 107 + }, + "ship-docsync-completion": { + "observations": 152, + "withCommands": 151, + "commands": [ + "bin/gstack-docs-candidate", + "bin/gstack-session-kind", + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 3.5392, + "costRuns": 110 + }, + "ship-docsync-current": { + "observations": 153, + "withCommands": 152, + "commands": [ + "bin/gstack-docs-candidate", + "bin/gstack-session-kind", + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 3.2887, + "costRuns": 110 + }, + "ship-docsync-failure": { + "observations": 156, + "withCommands": 155, + "commands": [ + "bin/gstack-docs-candidate", + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 2.028, + "costRuns": 110 + }, + "ship-docsync-late-result": { + "observations": 170, + "withCommands": 168, + "commands": [ + "bin/gstack-docs-candidate", + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 0.9185, + "costRuns": 109 + }, + "ship-docsync-launch-failure": { + "observations": 154, + "withCommands": 153, + "commands": [ + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 0.4608, + "costRuns": 110 + }, + "ship-docsync-missing-asset": { + "observations": 148, + "withCommands": 147, + "commands": [ + "bin/gstack-docs-candidate", + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 1.3017, + "costRuns": 109 + }, + "ship-docsync-missing-marker": { + "observations": 175, + "withCommands": 174, + "commands": [ + "bin/gstack-docs-candidate", + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 0.5063, + "costRuns": 110 + }, + "ship-docsync-recovery": { + "observations": 160, + "withCommands": 159, + "commands": [ + "bin/gstack-docs-candidate", + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 1.0055, + "costRuns": 110 + }, + "ship-docsync-stale-after": { + "observations": 172, + "withCommands": 171, + "commands": [ + "bin/gstack-docs-candidate", + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 1.1196, + "costRuns": 110 + }, + "ship-docsync-stale-before": { + "observations": 165, + "withCommands": 164, + "commands": [ + "bin/gstack-docs-candidate", + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 1.1591, + "costRuns": 110 + }, + "ship-docsync-store": { + "observations": 154, + "withCommands": 153, + "commands": [ + "bin/gstack-docs-candidate", + "bin/gstack-session-kind", + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 3.5563, + "costRuns": 110 + }, + "ship-docsync-timeout-unsettled": { + "observations": 167, + "withCommands": 166, + "commands": [ + "bin/gstack-skill-end", + "bin/gstack-skill-start" + ], + "costUsd": 0.5324, + "costRuns": 110 + }, + "ship-exploratory-late-input": { + "observations": 168, + "withCommands": 167, + "commands": [ + "bin/gstack-config", + "bin/gstack-diff-scope", + "bin/gstack-learnings-search", + "bin/gstack-qa-deadline", + "bin/gstack-qa-evidence", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-specialist-stats" + ], + "costUsd": 2.1399, + "costRuns": 115 + }, + "ship-exploratory-plan-checks": { + "observations": 167, + "withCommands": 166, + "commands": [ + "bin/gstack-config", + "bin/gstack-diff-scope", + "bin/gstack-learnings-search", + "bin/gstack-qa-deadline", + "bin/gstack-qa-evidence", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-specialist-stats" + ], + "costUsd": 1.8717, + "costRuns": 115 + }, + "ship-exploratory-small-cli": { + "observations": 171, + "withCommands": 170, + "commands": [ + "bin/gstack-config", + "bin/gstack-diff-scope", + "bin/gstack-learnings-search", + "bin/gstack-qa-deadline", + "bin/gstack-qa-evidence", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-specialist-stats" + ], + "costUsd": 1.8332, + "costRuns": 115 + }, + "ship-exploratory-unavailable": { + "observations": 172, + "withCommands": 171, + "commands": [ + "bin/gstack-config", + "bin/gstack-diff-scope", + "bin/gstack-learnings-search", + "bin/gstack-qa-deadline", + "bin/gstack-qa-evidence", + "bin/gstack-review-log", + "bin/gstack-review-read", + "bin/gstack-specialist-stats" + ], + "costUsd": 1.837, + "costRuns": 115 + }, + "ship-local-hook-preservation": { + "observations": 161, + "withCommands": 161, + "commands": [ + "bin/gstack-config", + "bin/gstack-paths", + "bin/gstack-redact" + ], + "costUsd": 0.307, + "costRuns": 107 + }, + "ship-local-workflow": { + "observations": 157, + "withCommands": 156, + "commands": [ + "bin/gstack-bun-version.sh", + "bin/gstack-config", + "bin/gstack-paths", + "bin/gstack-update-check", + "bin/gstack-upgrade" + ], + "costUsd": 0, + "costRuns": 107 + }, + "ship-managed-hook-refresh": { + "observations": 160, + "withCommands": 160, + "commands": [ + "bin/gstack-config", + "bin/gstack-paths", + "bin/gstack-redact" + ], + "costUsd": 0.2737, + "costRuns": 107 + }, + "ship-skipped-queued-finding": { + "observations": 154, + "withCommands": 153, + "commands": [], + "costUsd": 0.6029, + "costRuns": 92 + }, + "ship-triage": { + "observations": 180, + "withCommands": 180, + "commands": [], + "costUsd": 0, + "costRuns": 92 + }, + "ship-unmanaged-hook-consent": { + "observations": 161, + "withCommands": 161, + "commands": [ + "bin/gstack-config", + "bin/gstack-paths", + "bin/gstack-redact" + ], + "costUsd": 0.3538, + "costRuns": 107 + }, + "skillify-approval-reject": { + "observations": 186, + "withCommands": 175, + "commands": [ + "bin/gstack-learnings-log", + "bin/gstack-state-root.sh", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 91 + }, + "skillify-happy-path": { + "observations": 186, + "withCommands": 175, + "commands": [ + "bin/gstack-learnings-log", + "bin/gstack-state-root.sh", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 91 + }, + "skillify-provenance-refusal": { + "observations": 190, + "withCommands": 175, + "commands": [ + "bin/gstack-learnings-log", + "bin/gstack-state-root.sh", + "browse/dist/browse", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 108 + }, + "skillmd-no-local-binary": { + "observations": 155, + "withCommands": 155, + "commands": [ + "bin/gstack-learnings-log", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 91 + }, + "skillmd-outside-git": { + "observations": 155, + "withCommands": 155, + "commands": [ + "bin/gstack-learnings-log", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 91 + }, + "skillmd-setup-discovery": { + "observations": 156, + "withCommands": 156, + "commands": [ + "bin/gstack-learnings-log", + "browse/dist/browse" + ], + "costUsd": 0, + "costRuns": 117 + }, + "test-audit-report-only": { + "observations": 169, + "withCommands": 154, + "commands": [], + "costUsd": 0.9871, + "costRuns": 106 + }, + "timeline-event-flow": { + "observations": 166, + "withCommands": 161, + "commands": [ + "bin/gstack-context-recovery", + "bin/gstack-decision-search", + "bin/gstack-learnings-log", + "bin/gstack-learnings-search", + "bin/gstack-paths", + "bin/gstack-remote-identity.sh", + "bin/gstack-skill-start", + "bin/gstack-slug", + "bin/gstack-state-root.sh", + "bin/gstack-timeline-log", + "bin/gstack-timeline-read", + "design/dist/design" + ], + "costUsd": 0, + "costRuns": 90 + }, + "tpa-absent-darwin": { + "observations": 184, + "withCommands": 180, + "commands": [], + "costUsd": 0.2257, + "costRuns": 91 + }, + "tpa-absent-linux": { + "observations": 185, + "withCommands": 185, + "commands": [], + "costUsd": 0.2243, + "costRuns": 106 + }, + "tpa-apple-ban": { + "observations": 184, + "withCommands": 149, + "commands": [], + "costUsd": 0.3062, + "costRuns": 91 + }, + "tpa-broken": { + "observations": 184, + "withCommands": 181, + "commands": [], + "costUsd": 0.2265, + "costRuns": 91 + }, + "tpa-present": { + "observations": 185, + "withCommands": 185, + "commands": [], + "costUsd": 0.2325, + "costRuns": 106 + } + } +} diff --git a/test/helpers/exercised-prototype.ts b/test/helpers/exercised-prototype.ts new file mode 100644 index 000000000..0d7dd801a --- /dev/null +++ b/test/helpers/exercised-prototype.ts @@ -0,0 +1,149 @@ +/** + * Frozen parity oracle for the exercised-path map (W2-A2): the measured + * prototype (GSTA-45 investigation, exercised.ts at 521fd42, REPORT-2) with + * only its module-level root and base derivation turned into parameters, its + * path-prefix regex rewritten to an equivalent non-backtracking form, its + * import scan switched to A1's scanCodeImports (the prototype ran on the + * shebang-fixed base; A1 also fixed the TSX loader misreading .ts generics), + * and the per-case build of its replay (variant cu) appended. Do not edit its + * edge rules: test/exercised-map.test.ts pins that the shipped + * exercisedConsumers() equals or is a reviewed superset of this map. + */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { derivedDependencies, scanCodeImports, SELECTION_DATA_MODULES, type DependencyMaps, type ExercisedMap } from '../../scripts/pr-dependencies'; +import { matchGlob } from './test-selection'; +import { isPaidTestFile } from './paid-test-set'; + +export function prototypeConsumers(maps: DependencyMaps, ROOT: string, observedMap: ExercisedMap | null, PR_FULL_GATE_FILES: readonly string[]): Map> { + const base = derivedDependencies(maps, ROOT); + const tracked: string[] = [...base.tracked]; + const trackedSet = base.tracked as Set; + const cache = new Map(); + const read = (f: string) => { if (!cache.has(f)) { let t = ''; try { if (trackedSet.has(f) && fs.statSync(path.join(ROOT, f)).size < 3_000_000) t = fs.readFileSync(path.join(ROOT, f), 'utf8'); } catch {} cache.set(f, t); } return cache.get(f)!; }; + const CODE = /\.(?:[cm]?[jt]s|tsx|jsx)$/; + const isBunShebang = (f: string) => /^#!.*\b(?:bun|node)\b/.test(read(f).slice(0, 200)); + const isCode = (f: string) => CODE.test(f) || isBunShebang(f); + const isShell = (f: string) => !isCode(f) && (/\.sh$/.test(f) || f === 'setup' || /^#!.*\b(?:ba|z|da)?sh\b/.test(read(f).slice(0, 200))); + const resolveSpec = (spec: string, from: string) => { + if (!spec.startsWith('.')) return null; + const b = path.posix.normalize(path.posix.join(path.posix.dirname(from), spec)); + for (const c of [b, b + '.ts', b + '.tsx', b + '.js', b + '.mjs', b + '/index.ts', b + '/index.js']) if (trackedSet.has(c)) return c; + return null; + }; + const dirs = new Set(tracked.flatMap(f => f.split('/').slice(0, -1).map((_, i, p) => p.slice(0, i + 1).join('/')))); + const REPO_PATH = /(?:^|[^\w.@-])((?:bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents|test\/helpers|test\/fixtures|capy)\/[\w.@-]+(?:\/[\w.@-]+)*)/g; + const JOINED = /['"\x60](bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents)['"\x60]\s*,\s*['"\x60]([\w.@-]+)['"\x60]/g; + const binByName = new Map(); + for (const f of tracked) if (f.startsWith('bin/') || f.startsWith('browse/bin/')) { const b = path.posix.basename(f); binByName.set(b, f); binByName.set(b.replace(/\.(ts|sh|js)$/, ''), f); } + const BINARIES = [...read('scripts/build.sh').matchAll(/--compile\b[^\n]*?\s([\w./-]+\.ts)\s+--outfile\s+([\w./-]+)/g)].map(m => ({ outfile: m[2]!, entry: m[1]!, tree: m[1]!.startsWith('bin/') ? [m[1]!] : tracked.filter(f => matchGlob(f, path.posix.dirname(m[1]!) + '/**')) })); + let pkgScripts: Record = {}; try { pkgScripts = JSON.parse(read('package.json')).scripts ?? {}; } catch {} + const expandRef = (ref: string) => { ref = ref.replace(/[.]+$/, ''); if (trackedSet.has(ref)) return [ref]; if (dirs.has(ref)) return tracked.filter(f => f.startsWith(ref + '/')); return []; }; + const stripCode = (t: string) => t.replace(/\/\*[\s\S]*?\*\//g, '').replace(/^[ \t]*\/\/.*$/gm, ''); + const stripMessages = (t: string) => { + let out = '', i = 0; + while (i < t.length) { + const m = /^(?:echo|printf|say|warn|die|fail|err|log|info|note|_dc_row|row)\b/.exec(t.slice(i, i + 8)); + const prev = i === 0 ? '\n' : t[i - 1]!; + if (m && /[\s;|&(\x60{]/.test(prev)) { + let j = i + m[0].length, q: string | null = null; + while (j < t.length) { const c = t[j]!; if (q) { if (c === '\\') { j += 2; continue; } if (c === q) q = null; j++; continue; } if (c === '"' || c === "'") { q = c; j++; continue; } if (c === '\n' || c === ';' || c === '|' || c === '&' || c === ')') break; j++; } + out += ' '; i = j; continue; + } + out += t[i]; i++; + } + return out; + }; + const stripShell = (t: string) => stripMessages(t + .replace(/<<-?\s*['"]?(\w+)['"]?[^\n]*\n[\s\S]*?\n\s*\1\s*$/gm, '') + .split('\n').filter(l => !/^\s*#(?!!)/.test(l)) + .map(l => l.replace(/(^\s*|\bin\s+|;;\s*)[\w*."'-]+(?:\s*\|\s*[\w*."'-]+)*\)(?=\s|$)/g, '$1').replace(/\s#\s.*$/, '')) + .join('\n')); + const CMD_START = /(?:^|[;|&({\x60]|\$\(|\b(?:then|do|else|exec|command|nohup|bash|sh|bun|node|timeout\s+\S+|env(?:\s+\w+=\S*)*|if|while|until|!|time))\s*"?$/; + // Changed from the prototype: its `(?:\/?[\w.@-]+)*\/` path prefix backtracks exponentially; this is the same language (words separated by single slashes, ending in a slash) in an unambiguous form. + const PREFIX = /(?:"?(?:\$\{?\w+\}?|\$\(dirname\s+"?\$0"?\)|~|\.)?(?:\/(?:[\w.@-]+\/)*|(?:[\w.@-]+\/)+))?"?$/; + const cmdOk = (before: string) => { const b = before.replace(PREFIX, ''); return CMD_START.test(b) || (/\/$/.test(before) && /\b\w+="?$/.test(b)); }; + const shellExec = (text: string, from: string, out: Set) => { + for (const m of text.matchAll(/gstack-[a-z0-9][a-z0-9-]*[a-z0-9](?:\.(?:sh|ts))?\b/g)) { + const i = m.index!; const ls = text.lastIndexOf('\n', i - 1) + 1; + if (!cmdOk(text.slice(ls, i))) continue; + const b = binByName.get(m[0]); if (b) out.add(b); + } + for (const m of text.matchAll(/(?:bin|lib|scripts|browse|design|make-pdf)\/[\w.@-]+(?:\/[\w.@-]+)*/g)) { + const i = m.index!; const ls = text.lastIndexOf('\n', i - 1) + 1; const before = text.slice(ls, i); + if (/[\w.@-]$/.test(before)) continue; + if (!cmdOk(before) && !/\bbun\s+(?:run\s+)?"?(?:\$\{?\w+\}?\/)?$/.test(before)) continue; + for (const f of expandRef(m[0])) out.add(f); + } + for (const m of text.matchAll(/(?:bin|lib|scripts|browse|design|make-pdf|hosts|capy|test\/fixtures|test\/helpers)\/[\w.@-]+(?:\/[\w.@-]+)*\.(?:json|jsonl|env|txt|ya?ml|toml|csv|tsv)\b/g)) for (const f of expandRef(m[0])) out.add(f); + for (const m of text.matchAll(/(?:^|[\s;&|({])(?:\.|source)\s+([^;|&\n]+)/gm)) { const names = [...m[1]!.matchAll(/([\w.-]+\.sh|gstack-[\w.-]+)/g)].map(x => x[1]!); if (!names.length) continue; const b = names[names.length - 1]!; const sib = path.posix.join(path.posix.dirname(from), b); if (trackedSet.has(sib)) out.add(sib); else if (binByName.get(b)) out.add(binByName.get(b)!); } + for (const bin of BINARIES) if (text.includes(bin.outfile) || (bin.outfile === 'browse/dist/browse' && /\$B\b|find-browse/.test(text)) || (bin.outfile === 'design/dist/design' && /\$D\b/.test(text))) for (const f of bin.tree) out.add(f); + for (const m of text.matchAll(/\b(?:bun|npm) run ([a-z][\w:.-]*)/g)) { const c = pkgScripts[m[1]!]; if (c) for (const r of c.matchAll(REPO_PATH)) for (const f of expandRef(r[1]!)) out.add(f); } + }; + const bashBlocks = (md: string) => [...md.matchAll(/\x60\x60\x60(?:bash|sh|shell)\n([\s\S]*?)\x60\x60\x60/g)].map(m => m[1]!).join('\n'); + const edges = new Map(); + const execRefs = (f: string): string[] => { + if (edges.has(f)) return edges.get(f)!; + const out = new Set(); + const raw = read(f); + if ((SELECTION_DATA_MODULES as readonly string[]).includes(f) || !raw) { edges.set(f, []); return []; } + if (isCode(f)) { + try { for (const spec of scanCodeImports(f, raw)) { const h = resolveSpec(spec, f); if (h) out.add(h); } } catch {} + const code = stripCode(raw.replace(/^#!.*/, '')); + for (const m of code.matchAll(/['"\x60](?:\$\{[^}]+\}\/)?((?:bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents|test\/helpers|test\/fixtures)\/[\w.@-]+(?:\/[\w.@-]+)*)['"\x60]/g)) for (const x of expandRef(m[1]!)) out.add(x); + for (const m of code.matchAll(JOINED)) for (const x of expandRef(m[1] + '/' + m[2])) out.add(x); + for (const bin of BINARIES) if (code.includes("'" + bin.outfile + "'") || code.includes('"' + bin.outfile + '"')) for (const x of bin.tree) out.add(x); + } else if (isShell(f)) { + shellExec(stripShell(raw), f, out); + } else if (/\.(md|tmpl)$/.test(f)) { + shellExec(stripShell(bashBlocks(raw)), f, out); + } + out.delete(f); + const list = [...out]; edges.set(f, list); return list; + }; + const isFullGate = (f: string) => PR_FULL_GATE_FILES.some((p: string) => matchGlob(f, p)); + const closure = (entries: Iterable) => { + const seen = new Set(); const q = [...entries].filter(f => trackedSet.has(f)); + while (q.length) { const f = q.pop()!; if (seen.has(f)) continue; seen.add(f); if (base.global.has(f) || isFullGate(f)) continue; for (const n of execRefs(f)) if (!seen.has(n) && !(/\.test\.tsx?$/.test(n) && isPaidTestFile(n))) q.push(n); } + return seen; + }; + const expand = (p: string) => p.includes('*') ? tracked.filter(f => matchGlob(f, p)) : trackedSet.has(p) ? [p] : []; + const OWNER = /^test\/[^/]+\.test\.ts$/; + function caseEntries(patterns: string[], defaultOwner: string | null) { + const owners = patterns.filter(p => OWNER.test(p) && isPaidTestFile(p)); + const files = [...(owners.length ? owners : defaultOwner ? [defaultOwner] : []), ...patterns.flatMap(expand)]; + const gen = files.filter(f => f.endsWith('.tmpl')).map(f => f.replace(/\.tmpl$/, '')).filter(f => trackedSet.has(f)); + return [...files, ...gen]; + } + function binsInCommand(cmd: string): string[] { + const out = new Set(); + const pty = cmd.startsWith('PTY:'); + for (const m of cmd.matchAll(/gstack-[a-z0-9-]*[a-z0-9](?:\.(?:sh|ts))?/g)) { + let tok = m[0]!; + if (pty) { while (tok.length > 7 && !binByName.has(tok)) { const k = tok.lastIndexOf('-'); tok = k > 6 ? tok.slice(0, k) : ''; } } + const i = m.index!; if (cmd[i - 1] === '/' && i >= 2 && /[\s"'\x60(]/.test(cmd[i - 2]!)) continue; + const b = binByName.get(tok); if (b) out.add(b); + } + for (const m of cmd.matchAll(/browse\/bin\/([a-z-]+)/g)) { const b = binByName.get(m[1]!); if (b) out.add(b); } + for (const bin of BINARIES) { + const hit = cmd.includes(bin.outfile) || (bin.outfile === 'browse/dist/browse' && /\$B\b|\$\{B\}|find-browse/.test(cmd)) || (bin.outfile === 'design/dist/design' && /\$D\b|\$\{D\}/.test(cmd)) || (bin.outfile === 'make-pdf/dist/pdf' && /make-pdf\/dist\/pdf/.test(cmd)); + if (hit) out.add(bin.outfile); + } + return [...out]; + } + const binaryTree = (outfile: string) => BINARIES.find(b => b.outfile === outfile)?.tree ?? []; + + const observed = observedMap?.cases ?? {}; + const hostIds = tracked.filter((f: string) => /^hosts\/[\w-]+\.ts$/.test(f) && f !== 'hosts/index.ts').map((f: string) => path.basename(f, '.ts')); + const byFile = new Map>(); + for (const [id, patterns] of Object.entries(maps.e2eTouchfiles)) { + const entries = caseEntries(patterns, null); + for (const b of observed[id]?.commands ?? []) { if (trackedSet.has(b)) entries.push(b); else entries.push(...binaryTree(b)); } + const files = closure(entries); + const ownerText = entries.filter((f: string) => /^test\/[^/]+\.test\.ts$/.test(f)).map(read).join('\n'); + for (const h of hostIds) if (new RegExp('[\'"\x60]' + h + '[\'"\x60]|--host[ =]' + h + '\\b').test(ownerText)) for (const f of closure(['hosts/' + h + '.ts', ...expand('hosts/' + h + '/**')])) files.add(f); + for (const f of files) { if (!byFile.has(f)) byFile.set(f, new Set()); byFile.get(f)!.add(id); } + } + void binsInCommand; void stripShell; void execRefs; + return byFile; +} diff --git a/test/helpers/free-fixtures-data.ts b/test/helpers/free-fixtures-data.ts index e76d37780..3cb515285 100644 --- a/test/helpers/free-fixtures-data.ts +++ b/test/helpers/free-fixtures-data.ts @@ -116,6 +116,7 @@ export const FREE_FIXTURES: Readonly> = { 'test/fixtures/eng-test-plan-edit-cli.js': ['test/eng-test-plan-edit-approval.test.ts'], 'test/fixtures/eng-test-plan-edit-dacc.json': ['test/eng-test-plan-edit-approval.test.ts'], 'test/fixtures/eval-history/wave-reds-8-censuses.json': ['test/eval-history.test.ts'], + 'test/fixtures/exercised-map-replay.json': ['scripts/exercised-map.ts'], 'test/fixtures/exit-propagation/**': ['test/exit-propagation.test.ts'], 'test/fixtures/free-shard-capture.ts': ['test/test-free-shards-capture.test.ts'], 'test/fixtures/golden/**': ['test/host-config.test.ts', 'test/ship-review-loop.test.ts'], diff --git a/test/test-health-report.test.ts b/test/test-health-report.test.ts index 38a2e31f4..0d654dba8 100644 --- a/test/test-health-report.test.ts +++ b/test/test-health-report.test.ts @@ -191,7 +191,8 @@ describe('collectHealth over recorded fixture data', () => { [/runs\/61\/artifacts/, () => ({ artifacts: [] })], [/workflows\/windows-free-tests\.yml\/runs/, () => ({ workflow_runs: windowsRuns })], ], { - 11: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'full-fallback', e2e: ['gate-a', 'gate-b'], judges: ['judge-a'] } }) }, + 11: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'full-fallback', e2e: ['gate-a', 'gate-b'], judges: ['judge-a'], + shadow: { e2e: ['gate-a', 'gate-b'], wouldDrop: [], wouldAdd: [], provenance: [], predictedSavingUsd: 0, unpricedCases: 0 } } }) }, 12: { 'trial-outcomes.jsonl': prTrials }, 51: { 'trial-outcomes.jsonl': census }, 61: { 'flake-ledger.jsonl': ledger('test/a.test.ts') }, @@ -207,6 +208,7 @@ describe('collectHealth over recorded fixture data', () => { expect(value('pr-spend')).toMatchObject({ value: '$2.50 recorded', detail: ['per merged PR: $2.50 (1 merged)', 'per validated revision: $2.50 (1 revisions)'] }); expect(value('pr-cases-per-run')).toMatchObject({ value: '2.0 gate cases, 1.0 judges per run (1 runs)', detail: ['full-fallback: 2.0 gate cases, 1.0 judges (1 runs)'] }); expect(value('pr-known-cost-per-run')).toMatchObject({ value: '$2.50 per run (1 runs; 0 executed trial(s) with unknown cost)', detail: ['full-fallback: $2.50 (1 runs)'] }); + expect(value('pr-shadow-map')).toMatchObject({ value: '$0.00 predicted saving/week (a lower bound: known cost excludes PTY trials with unknown cost); 0.0 activation miss(es)/week, 0 flaky drop(s) not counted (1 runs)', detail: [] }); expect(report.headline).toBe('Cost per merged PR: $2.50 (1 merged PRs, $2.50 recorded)'); expect(formatHealth(report).split('\n')[1]).toBe('Cost per merged PR: $2.50 (1 merged PRs, $2.50 recorded)'); expect(value('cancelled-slice-minutes').value).toBe('30 min/week (1 cancelled run(s))'); @@ -247,6 +249,38 @@ describe('collectHealth over recorded fixture data', () => { expect(report.headline).toBe('Cost per merged PR: unavailable (0 merged PRs)'); }); + test('the shadow map metric counts red drops as activation misses unless the case passed later at the same head SHA, and labels dollars a lower bound', () => { + const gate = (caseId: string, outcome: 'passed' | 'failed', extra: Partial = {}) => trialRecord(caseId, outcome, { tier: 'gate', lane: 'gate/pr', ...extra }); + const first = formatTrialOutcomes([gate('gate-a', 'failed', { cost_usd: 3 }), gate('gate-b', 'failed'), gate('gate-c', 'failed'), + gate('gate-d', 'failed', { execution: 'reused', cost_usd: 0 }), gate('gate-e', 'passed')]); + const rerun = formatTrialOutcomes([gate('gate-b', 'passed'), gate('gate-a', 'failed')]); + const shadow = (wouldDrop: string[], predictedSavingUsd: number, unknownCostShare = 0.391) => + ({ e2e: [], wouldDrop, wouldAdd: [], provenance: [], predictedSavingUsd, unpricedCases: 0, unknownCostShare }); + const client = fakeClient([ + [/workflows\/evals\.yml\/runs/, () => ({ workflow_runs: [ + run(1, { event: 'pull_request', head_branch: 'a', head_sha: 'same', created_at: day(3) }), + run(2, { event: 'pull_request', head_branch: 'a', head_sha: 'same', created_at: day(2) }), + run(3, { event: 'pull_request', head_branch: 'c' }), + run(4, { event: 'pull_request', head_branch: 'd' })] })], + [/runs\/1\/artifacts/, () => ({ artifacts: [{ id: 11, name: 'paid-plan', size_in_bytes: 10 }, { id: 12, name: 'trial-outcomes-pr-a1', size_in_bytes: 10 }] })], + [/runs\/2\/artifacts/, () => ({ artifacts: [{ id: 21, name: 'paid-plan', size_in_bytes: 10 }, { id: 22, name: 'trial-outcomes-pr-a1', size_in_bytes: 10 }] })], + [/runs\/3\/artifacts/, () => ({ artifacts: [{ id: 31, name: 'paid-plan', size_in_bytes: 10 }] })], + [/runs\/4\/artifacts/, () => ({ artifacts: [{ id: 41, name: 'paid-plan', size_in_bytes: 10 }] })], + [/pulls\?state=closed/, () => []], + ], { + 11: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'dependents', e2e: [], judges: [], shadow: shadow(['gate-a', 'gate-b', 'gate-d', 'gate-e'], 4.5) } }) }, + 12: { 'trial-outcomes.jsonl': first }, + 21: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'pr', e2e: [], judges: [], shadow: shadow([], -1.5) } }) }, + 22: { 'trial-outcomes.jsonl': rerun }, + 31: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'pr', e2e: [], judges: [], shadow: { ...shadow([], 99), error: 'bad map' } } }) }, + 41: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'pr', e2e: [], judges: [] } }) }, + }); + const report = collectHealth({ client, repo: 'o/r', now: NOW, sinceDays: 7, cacheDir: tmp('health-cache-'), root: freeRoot(3, 0) }); + expect(report.metrics.find(entry => entry.id === 'pr-shadow-map')).toMatchObject({ + value: '$3.00 predicted saving/week (a lower bound: known cost excludes the 39% of observed case runs with unknown cost (PTY)); 1.0 activation miss(es)/week, 1 flaky drop(s) not counted (2 runs)', + detail: ['shadow miss: 1:gate-a', 'flaky drop: 1:gate-b (passed in run 2)'] }); + }); + test('the Windows wall counts only successful pull_request runs and says so when there are none', () => { const wall = (windowsRuns: unknown[]) => collectHealth({ client: fixture(windowsRuns), repo: 'o/r', now: NOW, sinceDays: 7, cacheDir: tmp('health-cache-'), root: freeRoot(3, 0) }) .metrics.find(entry => entry.id === 'free-wall')!; From 8f103baa713c2c7cdf34aeb152f022db569c7b11 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sat, 17 Oct 2026 18:54:14 -0700 Subject: [PATCH 4/4] v1.91.131.0 feat(pregate): Python import lanes (#3201) and a missing-dependencies regen diagnostic (#3252) * feat(pregate): lanes follows Python imports and mock.patch targets The import graph only parsed JS/TS imports, so in a Python repo every touched module failed `lanes` with PREGATE_NO_LANE even though SOURCE_EXT, TEST_FILE_RE and the pytest runner/summary already understood Python. Python sources now contribute `import a.b`, `from a.b import c` (c may be a submodule), parenthesised multi-line lists and relative `from ..pkg import x` edges, resolved to `a/b.py` or `a/b/__init__.py` from the repository root. Package `__init__.py` files on the path get edges too, since Python executes them on import. Modules outside the repo resolve to nothing. In test files, dotted string literals such as `'pkg.mod.Class.method'` (mock.patch targets) resolve to the longest dotted prefix that is a repo module. Co-Authored-By: Claude Opus 5.5 * fix(regen): report missing dependencies as REGEN_DEPS_MISSING, not REGEN_STALE When package.json declares dependencies and node_modules is missing, the scratch worktree runs the generator without them. A generator that degrades and still exits 0 then looked stale, and the fix line sent people to rerun the generator. gstack-regen check and the pregate regen check now report "dependencies missing; run " with its own result code and troubleshooting anchor; the install command follows the lockfile. The fixture reproduces the shape GSTA-31 saw in gbrain. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(pregate): keep Python import scanning linear and conservative Review of #3201 on the integrated tree found two quadratic paths on untrusted Python source and three ways a test could be left unselected: - The from-import module spec let two repetitions share a dot, so a failed match on a long run of dots backtracked quadratically. The spec is now dots followed by an optional word-led name. - Dotted literals and imports resolved every prefix with repeated split/join. Names are capped at 32 segments, deeper than any module path. - A literal that also named a repo file skipped its mock.patch reading; both readings now add edges. - A package and a same-named module both map (Python loads the package). - Imports after `;` or a compound statement's `:` are recognised. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(pregate): linear Python string blanking, non-empty from-spec, repo-bounded depth Second review round on the Python import scanner: - An empty from-spec let the whitespace on both sides of it compete, so `from` followed by a long run of spaces backtracked quadratically. The spec now starts with a dot or a word character. - Strings and comments are blanked by a single linear scanner that honours backslash escapes, so an escaped triple quote no longer ends a docstring early and swallows the imports after it. - The fixed 32-segment cap dropped real deep modules. Dotted names are now bounded by the repo's deepest path, which no module can exceed. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(changelog): unreleased entry for Python import lanes and REGEN_DEPS_MISSING Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * v1.91.126.0 stamp Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * release slot 1.91.126.0: back to unstamped (CHANGELOG entry unreleased) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(changelog): resolve the main-merge conflict (Unreleased entry above 1.91.127.0) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * v1.91.131.0 stamp Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: Scott Gigante Co-authored-by: Claude Opus 5.5 Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 21 +++++++ VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- bin/gstack-doctor-components.sh | 1 + bin/gstack-regen | 4 +- docs/pregate.md | 4 +- docs/troubleshooting.md | 12 ++++ lib/pregate/checks/regen.ts | 2 + lib/pregate/import-graph.ts | 108 +++++++++++++++++++++++++++++++- lib/regen.ts | 26 +++++++- lib/result-codes-release.ts | 9 ++- package.json | 2 +- test/gstack-regen.test.ts | 50 ++++++++++++++- test/pregate-selection.test.ts | 77 +++++++++++++++++++++++ 14 files changed, 306 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 796a13f7d..ac4418ce4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,26 @@ # Changelog +## [1.91.131.0] - 2026-10-18 + +**`gstack-pregate` finds the tests for Python files, and a missing `node_modules` no longer reads as "stale generated files".** In a Python repo, the `lanes` check failed almost every touched `.py` file with `PREGATE_NO_LANE`, because the import graph only parsed JavaScript and TypeScript. In a FastAPI service with about 1,500 non-test modules, Scott Gigante's change (#3201) cut the unmapped count from 1,473 to 780, and most of the rest are migrations and scripts no import reaches. Separately, a fresh checkout without `node_modules` made the pre-gate report `REGEN_STALE` and suggest rerunning the generator, which sent people the wrong way. + +### Added +- **Python import edges in `lanes`** (thanks @scottgigante-hubflow, #3201). `import a.b`, `from a.b import c`, parenthesised lists and relative `from . import x` resolve from the repository root to `a/b.py` or `a/b/__init__.py`, plus the package `__init__.py` files Python runs on the way in. Stdlib and site-packages imports add nothing. In test files, a dotted `mock.patch` target such as `'pkg.mod.Class.method'` maps to its module. +- **`REGEN_DEPS_MISSING`.** When a regen command exits 0 with different outputs, `package.json` declares dependencies and `node_modules` is missing, `gstack-regen check` and the pre-gate say "dependencies missing (no node_modules); run bun install, then rerun the check" instead of `REGEN_STALE`. The install command follows the lockfile. The pre-gate check is `incomplete`, never `pass`. + +### Fixed +- **The Python scanner stays linear on hostile input and does not drop tests.** Review found several problems before landing, all fixed: + - Long runs of dots or spaces after `from` no longer backtrack quadratically. + - Deep dotted names stay linear, and real deep modules still map. + - An escaped triple quote no longer swallows the imports after it. + - A literal that names a repo file still gets its `mock.patch` reading. + - A package and a same-named module both map. + - Imports after `;` or `:` count. + +### If you… +- **…run `gstack-pregate` in a Python repo:** touched modules now select the pytest files that import them. `conftest.py`, `src/` layouts and `importlib` calls still need `.gstack/pregate.json` `dependencies` declarations. +- **…see `REGEN_DEPS_MISSING`:** run the printed install command, then rerun the check. + ## [1.91.130.0] - 2026-10-18 **Every PR plan now shows what a tighter, exercised-path case map would run, without changing what actually runs.** Today a change to a bin or lib file reaches almost every gate case, because selection follows every mention of the file. The exercised-path map counts only what a case really executes: its touchfiles, its test's imports, the shell its skill runs, and the bins its recorded trials invoked. It runs in shadow mode. Replayed over every PR run that had a gate red between 2026-10-07 and 2026-10-17 (37 runs, 47 failed gate cases), it keeps all 47: `bun run test:exercised-map --replay`. diff --git a/VERSION b/VERSION index 97bde5585..a1846f406 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.130.0 +1.91.131.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 895f1b42e..e0a4e9ee8 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.130.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.131.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/bin/gstack-doctor-components.sh b/bin/gstack-doctor-components.sh index 463567aae..eea90fd19 100644 --- a/bin/gstack-doctor-components.sh +++ b/bin/gstack-doctor-components.sh @@ -267,6 +267,7 @@ gstack_result_anchor() { APPROVAL_SOURCE_UNAVAILABLE) printf '%s' 'approval-source-unavailable' ;; STATE_MATRIX_INCOMPLETE) printf '%s' 'state-matrix-incomplete' ;; WTREE_UNAVAILABLE) printf '%s' 'wtree-unavailable' ;; + REGEN_DEPS_MISSING) printf '%s' 'regen-deps-missing' ;; *) return 1 ;; esac } diff --git a/bin/gstack-regen b/bin/gstack-regen index 4281915f3..39657a47e 100755 --- a/bin/gstack-regen +++ b/bin/gstack-regen @@ -119,8 +119,8 @@ function main(argv: string[]): number { let outcome; try { outcome = runRegen({ repoRoot: root, registry: loaded.registry, mode: cmd, only: list(flag(rest, '--only')), touched: list(flag(rest, '--touched')) }); } catch (e: any) { process.stderr.write(resultLine(`gstack-regen ${cmd}`, 'PREGATE_INCOMPLETE', e.message) + '\n'); process.stdout.write(formatResult('regen', 'incomplete', loaded.source.label) + '\n'); return EXIT.fail; } - const bad = outcome.results.filter(r => r.status === 'stale' || r.status === 'failed'); - const status = outcome.results.some(r => r.status === 'failed') ? 'incomplete' : 'complete'; + const bad = outcome.results.filter(r => r.status === 'stale' || r.status === 'failed' || r.status === 'deps-missing'); + const status = outcome.results.some(r => r.status === 'failed' || r.status === 'deps-missing') ? 'incomplete' : 'complete'; if (json) { process.stdout.write(JSON.stringify({ mode: cmd, registry: loaded.source.label, wtree: outcome.wtree, results: outcome.results, ok: bad.length === 0 }) + '\n'); return bad.length ? EXIT.fail : EXIT.ok; } process.stdout.write(`registry: ${loaded.source.label}\n`); for (const r of outcome.results) process.stdout.write(renderEntry(r) + '\n'); diff --git a/docs/pregate.md b/docs/pregate.md index 6e9e4f0e0..b988c06ab 100644 --- a/docs/pregate.md +++ b/docs/pregate.md @@ -41,7 +41,9 @@ paths, at most 64 per list); commands are one line under 512 bytes. ` for entries whose inputs or outputs intersect) in a scratch `git worktree` seeded from the working tree, with `node_modules` linked rather than copied, and reports stale outputs (`REGEN_STALE`, exit 1) without touching - the tree. An entry with no `command` is tracked only (goldens regenerated by + the tree. When outputs differ but `package.json` declares dependencies and + `node_modules` is missing, the entry is `REGEN_DEPS_MISSING` (install, then + rerun) instead of stale. An entry with no `command` is tracked only (goldens regenerated by another entry). - `gstack-regen write` regenerates in place: what `gstack-restamp` and the Step 3 conflict rule use. On a conflict in a registered output, take either diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index d802e845b..e2719bca0 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -2789,6 +2789,18 @@ take either side, then regenerate. worktree has no `node_modules` of its own, so a command that needs them must reference the repository's (gstack's entries do through `bun run`). + +### `REGEN_DEPS_MISSING` + +**Meaning.** A registered command exited 0 and its outputs differ from the +committed ones, but `package.json` declares dependencies and the repository has +no `node_modules`, so the scratch worktree ran the generator without them. The +difference may come from the missing dependencies rather than stale files, so +the entry is reported as unverified, not `REGEN_STALE`. + +**Fix.** Run the printed install command (`bun install`, or the one matching the +repository's lockfile), then rerun `gstack-regen check` or the pre-gate. + ### `PREGATE_CONFIG_INVALID` diff --git a/lib/pregate/checks/regen.ts b/lib/pregate/checks/regen.ts index 510512230..dd63dd003 100644 --- a/lib/pregate/checks/regen.ts +++ b/lib/pregate/checks/regen.ts @@ -27,7 +27,9 @@ function run(ctx: PregateContext): CheckResult { const lines = results.map(renderEntry); const stale = results.filter(r => r.status === 'stale'); const failed = results.filter(r => r.status === 'failed'); + const depsMissing = results.filter(r => r.status === 'deps-missing'); ctx.explain.push(...results.map(r => `regen ${r.id}: selected because touched ∩ (${selected.find(e => e.id === r.id)?.inputs.join(',') || 'always'} ∪ outputs) ≠ ∅`)); + if (depsMissing.length) return { id: 'regen', stage: 'preflight', status: 'incomplete', inputs, lines, code: 'REGEN_DEPS_MISSING', detail: `dependencies missing; ${depsMissing[0]!.fix} (${depsMissing.length} entr${depsMissing.length === 1 ? 'y' : 'ies'} unverified: ${depsMissing.map(d => d.id).join(', ')})`, fix: depsMissing[0]!.fix }; if (failed.length) return { id: 'regen', stage: 'preflight', status: 'incomplete', inputs, lines, code: 'REGEN_COMMAND_FAILED', detail: `${failed.length} regen command(s) failed: ${failed.map(f => f.id).join(', ')}`, fix: failed.map(f => f.fix).join('; ') }; if (stale.length) return { id: 'regen', stage: 'preflight', status: 'fail', inputs, lines, code: 'REGEN_STALE', detail: `${stale.length} stale: ${stale.flatMap(s => s.changed.concat(s.outside)).join(' ')}`, fix: stale.map(s => s.fix).join('; ') }; return { id: 'regen', stage: 'preflight', status: 'pass', inputs, lines, detail: `${results.length} entr${results.length === 1 ? 'y' : 'ies'} fresh${results.some(r => r.status === 'listed') ? ' (hand-regenerated outputs listed)' : ''}` }; diff --git a/lib/pregate/import-graph.ts b/lib/pregate/import-graph.ts index 56ad8d989..9db6f4fd6 100644 --- a/lib/pregate/import-graph.ts +++ b/lib/pregate/import-graph.ts @@ -2,10 +2,13 @@ * pregate/import-graph — which test files depend on which repo files (plan * C8 `lanes`). The reverse graph unions four sources, each kept as provenance * for `--explain`: - * 1. static imports and requires (Bun.Transpiler.scanImports); + * 1. static imports and requires (Bun.Transpiler.scanImports), and Python + * `import` / `from … import` statements resolved from the repository root + * or, when relative, from the importing file's package; * 2. repo-path string literals, including `join('a', 'b')` segment runs * (a test that spawns `bin/gstack-doctor` names it in a string, never an - * import) and literal directories (`test/fixtures/x` → `x/**`); + * import) and literal directories (`test/fixtures/x` → `x/**`), and in + * Python, dotted `mock.patch` targets (`'pkg.mod.Class.method'`); * 3. the `bin/` → `test/*.test.*` convention (with and without * the `gstack-` prefix); * 4. explicit `.gstack/pregate.json` `dependencies` declarations. @@ -41,6 +44,10 @@ const scanner = new Bun.Transpiler({ loader: 'tsx' }); const QUOTED = /['"`]([A-Za-z0-9_@][A-Za-z0-9_./@()[\]+-]*)['"`]/g; const JOIN = /\bjoin\(([^()]*)\)/g; const RESOLVE_EXT = ['', '.ts', '.tsx', '.js', '.mjs', '.cjs', '/index.ts', '/index.js']; +// A statement starts a line or follows `;` or a compound statement's `:`. The module spec is +// non-empty and starts with a dot or a word character, so no two repetitions can share a dot or a space. +const PY_IMPORT = /(?:^|[;:])[ \t\f]*(?:from[ \t]+(\.+(?:\w[\w.]*)?|\w[\w.]*)[ \t]+import[ \t]+(\([^)]*\)|[^\n;]+)|import[ \t]+([^\n;]+))/gm; +const PY_DOTTED = /^[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)+$/; export function isTestFile(rel: string): boolean { return TEST_FILE_RE.test(rel); } @@ -185,6 +192,95 @@ function pathLike(config: PathAliases, ref: string): boolean { return chosen !== null || others.length > 0; } +const pathDepths = new WeakMap, number>(); +/** + * Segments in the repo's deepest path. No module name resolves deeper, so + * dotted-name resolution is bounded by the repo rather than by the source. + */ +function maxPathDepth(files: Set): number { + let depth = pathDepths.get(files); + if (depth === undefined) { depth = 0; for (const f of files) depth = Math.max(depth, f.split('/').length); pathDepths.set(files, depth); } + return depth; +} + +/** The leading dotted name of each comma-separated `name [as alias]` clause. */ +function pyNames(list: string): string[] { + return list.replace(/[()]/g, '').split(',').map(p => p.trim().split(/\s+/)[0]!).filter(n => n && n !== '*'); +} + +/** `from ` → the absolute dotted package it names, or null when it climbs above the repository root. */ +function pyFromBase(rel: string, spec: string): string | null { + const dots = /^\.*/.exec(spec)![0].length; + const rest = spec.slice(dots); + if (!dots) return rest; + const pkg = path.posix.dirname(rel).split('/').filter(s => s !== '.'); + if (dots - 1 > pkg.length) return null; + return [...pkg.slice(0, pkg.length - (dots - 1)), ...(rest ? [rest] : [])].join('.'); +} + +/** + * Python source with every string blanked to `''` and every comment dropped, in + * one linear pass, so a `#` inside a string and a quote inside a comment both + * stay inert. Backslash escapes are honoured; an unterminated string runs to + * its line end, or to the end of the file when triple-quoted. + */ +function pythonCode(source: string): string { + const parts: string[] = []; + let from = 0; + for (let i = 0; i < source.length;) { + const c = source[i]!; + if (c === '#') { parts.push(source.slice(from, i)); const end = source.indexOf('\n', i); i = from = end < 0 ? source.length : end; continue; } + if (c !== '"' && c !== "'") { i++; continue; } + parts.push(source.slice(from, i), "''"); + const close = source.startsWith(c + c + c, i) ? c + c + c : c; + let j = i + close.length; + while (j < source.length && !source.startsWith(close, j) && (close.length === 3 || source[j] !== '\n')) j += source[j] === '\\' ? 2 : 1; + i = from = Math.min(source.length, source.startsWith(close, j) ? j + close.length : j); + } + parts.push(source.slice(from)); + return parts.join(''); +} + +/** Dotted modules a Python source imports; `from a import b` also yields `a.b`, which may be a submodule. */ +function pythonImports(rel: string, source: string): string[] { + const code = pythonCode(source).replace(/\\\r?\n/g, ' '); + const out: string[] = []; + for (const m of code.matchAll(PY_IMPORT)) { + if (m[3]) { out.push(...pyNames(m[3])); continue; } + const base = pyFromBase(rel, m[1]!); + if (base === null) continue; + if (base) out.push(base); + for (const name of pyNames(m[2]!)) out.push(base ? `${base}.${name}` : name); + } + return out; +} + +/** + * A dotted module → its repo files (`a/b/__init__.py`, which Python prefers, + * and `a/b.py`; both when both exist) plus the package `__init__.py` files + * Python executes on the way; [] for a module outside the repo (stdlib, + * site-packages), including one deeper than any repo path. + */ +function pythonModuleFiles(files: Set, dotted: string): string[] { + const parts = dotted.split('.'); + if (parts.length > maxPathDepth(files) || parts.some(p => !p)) return []; + const p = parts.join('/'); + const found = [`${p}/__init__.py`, `${p}.py`].filter(f => files.has(f)); + if (!found.length) return []; + const inits = parts.slice(0, -1).map((_, i) => `${parts.slice(0, i + 1).join('/')}/__init__.py`).filter(f => files.has(f)); + return [...found, ...inits]; +} + +/** `'pkg.mod.Class.attr'` (a `mock.patch` target) → the longest dotted prefix that is a repo module. */ +function pythonPatchTarget(files: Set, ref: string): string[] { + const parts = ref.split('.').slice(0, maxPathDepth(files)); + for (let n = parts.length; n >= 1; n--) { + const hit = pythonModuleFiles(files, parts.slice(0, n).join('.')); + if (hit.length) return hit; + } + return []; +} + function addEdge(graph: Graph, dependency: string, dependent: string, via: Edge['via']): void { if (dependency === dependent) return; const list = graph.reverse.get(dependency) ?? []; @@ -210,11 +306,17 @@ export function referencesIn(files: Set, dirs: Set, rel: string, // The transpiler erases type-only imports; a types module still binds its importers' contracts. for (const m of source.matchAll(/^\s*(?:import|export)\s+type\b[^;'"]*?\bfrom\s+['"]([^'"]+)['"]/gm)) importOf(m[1]!); } + const python = rel.endsWith('.py'); + if (python) for (const mod of pythonImports(rel, source)) for (const t of pythonModuleFiles(files, mod)) out.push({ target: t, via: 'import' }); // Literals carry dependencies only from the test tree and from bin/ and shell // sources (a test spawns a bin; a bash bin calls a sibling). Prose and // manifests name everything and would make every file a hub. if (!scansLiterals(rel)) return out; - for (const m of source.matchAll(QUOTED)) for (const target of resolveRefs(files, dirs, rel, m[1]!, true, config)) out.push({ target, via: 'literal' }); + for (const m of source.matchAll(QUOTED)) { + const targets = resolveRefs(files, dirs, rel, m[1]!, true, config); + for (const target of targets) out.push({ target, via: 'literal' }); + if (python && PY_DOTTED.test(m[1]!)) for (const f of pythonPatchTarget(files, m[1]!)) out.push({ target: f, via: 'literal' }); + } for (const m of source.matchAll(JOIN)) { const segments = [...m[1]!.matchAll(/['"`]([^'"`]+)['"`]/g)].map(s => s[1]!); if (segments.length < 2) continue; diff --git a/lib/regen.ts b/lib/regen.ts index c3672a627..2a8538647 100644 --- a/lib/regen.ts +++ b/lib/regen.ts @@ -156,7 +156,7 @@ export function scratchWorktree(repoRoot: string, prefix = 'gstack-regen-'): Scr export interface EntryResult { id: string; command: string | null; - status: 'fresh' | 'stale' | 'failed' | 'listed' | 'written' | 'skipped'; + status: 'fresh' | 'stale' | 'failed' | 'listed' | 'written' | 'skipped' | 'deps-missing'; /** Output files whose bytes changed (stale) or were written. */ changed: string[]; /** Files the command wrote outside its declared outputs (a refusal in `check`). */ @@ -189,6 +189,20 @@ function diffSnapshots(a: Map, b: Map): string[] return [...changed].filter(p => p !== 'node_modules').sort(); } +/** + * The install command for a repository whose package.json declares + * dependencies but has no node_modules (the scratch worktree links none + * then), picked from its lockfile; null when nothing is missing. + */ +export function missingDependencies(repoRoot: string): string | null { + if (fs.existsSync(path.join(repoRoot, 'node_modules'))) return null; + let pkg: { dependencies?: Record; devDependencies?: Record }; + try { pkg = JSON.parse(fs.readFileSync(path.join(repoRoot, 'package.json'), 'utf8')) ?? {}; } catch { return null; } + if (!Object.keys(pkg.dependencies ?? {}).length && !Object.keys(pkg.devDependencies ?? {}).length) return null; + const lockfiles: Array<[string, string]> = [['bun.lock', 'bun install'], ['bun.lockb', 'bun install'], ['pnpm-lock.yaml', 'pnpm install'], ['yarn.lock', 'yarn install'], ['package-lock.json', 'npm install']]; + return lockfiles.find(([file]) => fs.existsSync(path.join(repoRoot, file)))?.[1] ?? 'bun install'; +} + function runCommand(cwd: string, command: string, env: NodeJS.ProcessEnv, timeoutMs: number): { exit: number | null; stderr: string } { const r = runBashSync(`set -o pipefail; ${command}`, { cwd, encoding: 'utf8', timeout: timeoutMs, maxBuffer: 64 * 1024 * 1024, env: { ...env, GSTACK_REGEN_SCRATCH: cwd } }); return { exit: r.status, stderr: ((r.stderr ?? '') + (r.error ? `\n${r.error.message}` : '')).trim().slice(-4000) }; @@ -207,7 +221,9 @@ export interface RunRegenOptions { * `check`: run every selected command in one scratch worktree and compare its * outputs with the working tree; `write`: run them in the repository. An * entry with `command: null` is `listed` (its fix line names the outputs an - * author refreshes by hand). Never runs a command the caller did not opt into: + * author refreshes by hand). A `check` whose outputs differ while the + * repository's dependencies are not installed is `deps-missing`, not `stale`: + * the generator may have run without them. Never runs a command the caller did not opt into: * the bin enforces --allow-repo-commands before calling this. */ export function runRegen(o: RunRegenOptions): { results: EntryResult[]; wtree: string | null } { @@ -215,6 +231,7 @@ export function runRegen(o: RunRegenOptions): { results: EntryResult[]; wtree: s const timeout = o.timeoutMs ?? 600_000; const results: EntryResult[] = []; let scratch: Scratch | null = null; + const install = o.mode === 'check' ? missingDependencies(o.repoRoot) : null; const fixFor = (e: RegenEntry) => e.command ? `${e.command} # then commit ${e.outputs.join(' ')}` : `regenerate by hand: ${e.outputs.join(' ')}${e.note ? ` (${e.note})` : ''}`; try { if (o.mode === 'check' && entries.some(e => e.command)) scratch = scratchWorktree(o.repoRoot); @@ -230,7 +247,9 @@ export function runRegen(o: RunRegenOptions): { results: EntryResult[]; wtree: s const ms = Date.now() - started; if (run.exit !== 0) { results.push({ id: e.id, command: e.command, status: 'failed', changed, outside, exit: run.exit, stderr: run.stderr, ms, fix: fixFor(e) }); continue; } if (o.mode === 'check') { - results.push({ id: e.id, command: e.command, status: changed.length || outside.length ? 'stale' : 'fresh', changed, outside, exit: 0, ms, fix: fixFor(e) }); + const differs = changed.length > 0 || outside.length > 0; + if (differs && install) results.push({ id: e.id, command: e.command, status: 'deps-missing', changed, outside, exit: 0, ms, fix: `run ${install}, then rerun the check` }); + else results.push({ id: e.id, command: e.command, status: differs ? 'stale' : 'fresh', changed, outside, exit: 0, ms, fix: fixFor(e) }); // Reset the scratch so the next entry's diff is its own. git(cwd, ['checkout', '-q', '--', '.']); git(cwd, ['clean', '-fdq']); } else results.push({ id: e.id, command: e.command, status: changed.length ? 'written' : 'fresh', changed, outside, exit: 0, ms, fix: fixFor(e) }); @@ -247,6 +266,7 @@ export function renderEntry(r: EntryResult): string { if (r.status === 'fresh' || r.status === 'skipped') return head; if (r.status === 'listed') return `${head} — ${r.fix}`; if (r.status === 'failed') return `${head} — exit ${r.exit}: ${(r.stderr ?? '').split('\n').pop() ?? ''}; fix: run \`${r.command}\` and read its output (REGEN_COMMAND_FAILED)`; + if (r.status === 'deps-missing') return `${head} — dependencies missing (no node_modules); ${r.fix} — differs without them: ${[...r.changed, ...r.outside].join(' ')} (REGEN_DEPS_MISSING)`; const parts = [head]; if (r.changed.length) parts.push(`stale: ${r.changed.join(' ')}`); if (r.outside.length) parts.push(`outside declared outputs: ${r.outside.join(' ')}`); diff --git a/lib/result-codes-release.ts b/lib/result-codes-release.ts index 19f291a69..42ff9b088 100644 --- a/lib/result-codes-release.ts +++ b/lib/result-codes-release.ts @@ -1,6 +1,7 @@ /** * Result codes for the release path: gstack-ship-queue and the ship-policy - * floor, pregate timing and its working-tree fingerprint, approval classes and + * floor, pregate timing and its working-tree fingerprint, regen with missing + * dependencies, approval classes and * the state-machine matrix. Kept beside lib/result-codes.ts (which spreads * this table into RESULT_CODES) so * the owner module stays within the module-size ratchet; the same rules apply: @@ -83,4 +84,10 @@ export const RELEASE_RESULT_CODES = { summary: 'cannot fingerprint the working tree: bin/gstack-wtree could not start (on Windows, no Git Bash was found) or it ran and failed', fix: 'on Windows point GSTACK_GIT_BASH at Git Bash\'s bash.exe (Git Bash: export GSTACK_GIT_BASH=\'C:/Program Files/Git/bin/bash.exe\'; PowerShell: $env:GSTACK_GIT_BASH = \'C:\\Program Files\\Git\\bin\\bash.exe\'), or install Git for Windows (winget install --id Git.Git -e); elsewhere rerun ./setup so bin/gstack-wtree is present; then rerun gstack-pregate', }, + // Fix wave W5 (GSTA-31): pregate regen without installed dependencies. + REGEN_DEPS_MISSING: { + anchor: 'regen-deps-missing', + summary: 'a regen command exited 0 with different outputs while package.json declares dependencies and node_modules is missing, so the outputs were not verified', + fix: 'install the dependencies (the printed command, such as bun install), then rerun the check', + }, } as const satisfies Record; diff --git a/package.json b/package.json index 1a72f3fc1..d0f2a1ac6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.130", + "version": "1.91.131", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/test/gstack-regen.test.ts b/test/gstack-regen.test.ts index 5c527b24e..f718d6ec9 100644 --- a/test/gstack-regen.test.ts +++ b/test/gstack-regen.test.ts @@ -9,13 +9,24 @@ import { afterEach, describe, expect, test } from 'bun:test'; import * as fs from 'node:fs'; import * as path from 'node:path'; import { EXIT } from '../lib/headless-artifacts'; -import { parseRegistry, selectEntries, validateRegistry } from '../lib/regen'; +import { missingDependencies, parseRegistry, selectEntries, validateRegistry } from '../lib/regen'; import { BASE_REGISTRY, cleanup, commit, makeFixtureRepo, readFile, runBin, write, type FixtureRepo } from './helpers/pregate-fixture'; const repos: FixtureRepo[] = []; const fixture = (opts?: Parameters[0]) => { const r = makeFixtureRepo(opts); repos.push(r); return r; }; afterEach(() => { while (repos.length) cleanup(repos.pop()!); }); +/** gbrain's shape (GSTA-31): the generator degrades without its dependency and still exits 0. */ +const DEPS_FIXTURE = { + registry: { entries: [{ id: 'digest', command: 'node gen.js', outputs: ['generated/**'], inputs: ['src/version.js', 'gen.js'] }] }, + baseFiles: { + 'package.json': JSON.stringify({ name: 'fixture', version: '1.0.0', dependencies: { 'fixture-dep': '1.0.0' } }, null, 2) + '\n', + 'bun.lock': '{}\n', + 'gen.js': "const fs=require('fs');const {VERSION}=require('./src/version.js');let style='plain';try{style=require('fixture-dep').style;}catch{}fs.writeFileSync('generated/digest.md',`# digest v${VERSION} (${style})\\n`);\n", + 'generated/digest.md': '# digest v1.0.0 (fancy)\n', + }, +}; + describe('validateRegistry (schema + containment, pure)', () => { test('the fixture registry validates; command null is a listed entry; inputs default to always', () => { expect(validateRegistry(BASE_REGISTRY)).toEqual([]); @@ -147,6 +158,43 @@ describe('gstack-regen (bin)', () => { expect(all.out).toContain('regen digest: stale'); }); + test('missing dependencies: a generator that exits 0 without node_modules is REGEN_DEPS_MISSING (install, then rerun), not REGEN_STALE; installed deps make it fresh', () => { + const repo = fixture(DEPS_FIXTURE); + commit(repo, { 'src/version.js': "module.exports = { VERSION: '1.0.0' };\n// touched\n" }); + const r = runBin(repo, 'gstack-regen', ['check', '--base', 'main', '--allow-repo-commands']); + expect(r.status).toBe(EXIT.fail); + expect(r.out).toMatch(/^regen digest: deps-missing \(\d+ ms\) — dependencies missing \(no node_modules\); run bun install, then rerun the check — differs without them: generated\/digest\.md \(REGEN_DEPS_MISSING\)$/m); + expect(r.out).not.toContain('REGEN_STALE'); + expect(r.out).toContain('GSTACK_RESULT: skill=regen status=incomplete'); + const json = JSON.parse(runBin(repo, 'gstack-regen', ['check', '--base', 'main', '--allow-repo-commands', '--json']).out); + expect(json.results.map((x: any) => [x.id, x.status])).toEqual([['digest', 'deps-missing']]); + expect(repo.git('status', '--porcelain')).toBe(''); + + write(repo.work, 'node_modules/fixture-dep/index.js', "exports.style = 'fancy';\n"); + const installed = runBin(repo, 'gstack-regen', ['check', '--base', 'main', '--allow-repo-commands']); + expect(installed.status).toBe(EXIT.ok); + expect(installed.out).toContain('regen digest: fresh'); + + // With dependencies installed, a real input change is still stale. + commit(repo, { 'src/version.js': "module.exports = { VERSION: '1.1.0' };\n" }); + const stale = runBin(repo, 'gstack-regen', ['check', '--base', 'main', '--allow-repo-commands']); + expect(stale.status).toBe(EXIT.fail); + expect(stale.out).toContain('regen digest: stale'); + expect(stale.out).toContain('(REGEN_STALE)'); + expect(stale.out).not.toContain('REGEN_DEPS_MISSING'); + }); + + test('missing dependencies: the install command follows the lockfile', () => { + const repo = fixture(DEPS_FIXTURE); + expect(missingDependencies(repo.work)).toBe('bun install'); + fs.rmSync(path.join(repo.work, 'bun.lock')); + write(repo.work, 'package-lock.json', '{}\n'); + expect(missingDependencies(repo.work)).toBe('npm install'); + fs.mkdirSync(path.join(repo.work, 'node_modules')); + expect(missingDependencies(repo.work)).toBeNull(); + expect(missingDependencies(fixture().work)).toBeNull(); + }); + test('validate, list and init: an invalid working-tree registry is REGEN_REGISTRY_INVALID; init refuses to overwrite', () => { const repo = fixture(); expect(runBin(repo, 'gstack-regen', ['validate']).out).toContain('REGISTRY_VALID: .gstack/generated.json'); diff --git a/test/pregate-selection.test.ts b/test/pregate-selection.test.ts index fe839ffe0..59404fd5f 100644 --- a/test/pregate-selection.test.ts +++ b/test/pregate-selection.test.ts @@ -81,6 +81,66 @@ describe('import graph (static imports ∪ repo-path literals ∪ bin convention expect(testsFor(g, 'test/fixtures/sample.md').provenance[0]).toContain('(literal)'); expect(testsFor(g, 'notes/scratch.md')).toMatchObject({ tests: [], declaredNone: false }); }); + + test('python: absolute, from-import submodule, relative and parenthesised imports and mock.patch targets map; stdlib, comments and docstrings do not', () => { + const repo = fixture(); + commit(repo, { + 'pkg/__init__.py': '', + 'pkg/core.py': 'import os\nfrom pkg.util import helper\n', + 'pkg/util.py': 'def helper():\n return 1\n', + 'pkg/multi.py': 'X = 1\n', + 'pkg/patched.py': 'class Thing:\n def run(self):\n return 1\n', + 'pkg/ghost.py': 'G = 1\n', + 'pkg/sub/__init__.py': '', + 'pkg/sub/sibling.py': 'S = 1\n', + 'pkg/sub/leaf.py': 'from .. import core\nfrom . import sibling\n', + 'tests/test_core.py': 'import pkg.core as core\n', + 'tests/test_leaf.py': 'from pkg.sub import leaf\n', + 'tests/test_multi.py': 'from pkg import (\n multi, # the module\n util,\n)\nimport json, os.path\n', + 'tests/test_patch.py': "from unittest import mock\n\n@mock.patch('pkg.patched.Thing.run')\ndef test_run(run):\n pass\n", + 'tests/test_ghost.py': '"""\nimport pkg.ghost\n"""\n# from pkg import ghost\nX = "from pkg import ghost"\n', + }); + const g = buildGraph(repo.work); + expect(testsFor(g, 'pkg/core.py').tests).toEqual(['tests/test_core.py', 'tests/test_leaf.py']); + expect(testsFor(g, 'pkg/util.py').tests).toEqual(['tests/test_core.py', 'tests/test_leaf.py', 'tests/test_multi.py']); + expect(testsFor(g, 'pkg/sub/sibling.py').tests).toEqual(['tests/test_leaf.py']); + expect(testsFor(g, 'pkg/multi.py').tests).toEqual(['tests/test_multi.py']); + expect(testsFor(g, 'pkg/patched.py').provenance).toEqual(['tests/test_patch.py ← pkg/patched.py (literal)']); + expect(testsFor(g, 'pkg/sub/__init__.py').tests).toEqual(['tests/test_leaf.py']); + expect(testsFor(g, 'pkg/__init__.py').tests).toEqual(['tests/test_core.py', 'tests/test_leaf.py', 'tests/test_multi.py', 'tests/test_patch.py']); + expect(testsFor(g, 'pkg/ghost.py').tests).toEqual([]); + expect([...g.reverse.keys()].filter(k => k.endsWith('.py')).sort()).toEqual([ + 'pkg/__init__.py', 'pkg/core.py', 'pkg/multi.py', 'pkg/patched.py', 'pkg/sub/__init__.py', 'pkg/sub/leaf.py', 'pkg/sub/sibling.py', 'pkg/util.py', + ]); + }); + + test('python: a package and a same-named module both map, statements after ; and : map, escaped triple quotes and deep modules map, a path literal does not hide its patch target, and adversarial input stays linear', () => { + const repo = fixture(); + commit(repo, { + 'app.py': 'A = 1\n', + 'app/__init__.py': '', + 'pkg/__init__.py': '', + 'pkg/core.py': 'C = 1\n', + 'pkg/patched.py': 'class Thing:\n pass\n', + 'pkg.patched.Thing.run': 'a file whose name is also a patch target\n', + 'tests/test_app.py': 'import app\n', + 'tests/test_semicolon.py': 'import os; import pkg.core\n', + 'tests/test_inline.py': 'def test_core():\n if True: from pkg.core import C\n', + 'tests/test_patch.py': "from unittest import mock\n\n@mock.patch('pkg.patched.Thing.run')\ndef test_run(run):\n pass\n", + 'tests/test_adversarial.py': `from ${'.'.repeat(100_000)}x\nfrom ${' '.repeat(100_000)}x\nX = '${Array.from({ length: 20_000 }, () => 'pkg').join('.')}'\n${'\\"""'.repeat(20_000)}\n`, + 'tests/test_escaped.py': 'DOC = """text \\"""\nstill doc"""\nimport pkg.core\nTAIL = """x"""\n', + [`${Array.from({ length: 33 }, (_, i) => `d${i}`).join('/')}/leaf.py`]: 'L = 1\n', + 'tests/test_deep.py': `import ${Array.from({ length: 33 }, (_, i) => `d${i}`).join('.')}.leaf\n`, + }); + const g = buildGraph(repo.work); + expect(testsFor(g, 'app/__init__.py').tests).toEqual(['tests/test_app.py']); + expect(testsFor(g, 'app.py').tests).toEqual(['tests/test_app.py']); + expect(testsFor(g, 'pkg/core.py').tests).toEqual(['tests/test_escaped.py', 'tests/test_inline.py', 'tests/test_semicolon.py']); + expect(testsFor(g, `${Array.from({ length: 33 }, (_, i) => `d${i}`).join('/')}/leaf.py`).tests).toEqual(['tests/test_deep.py']); + expect(testsFor(g, 'pkg/patched.py').tests).toEqual(['tests/test_patch.py']); + expect(testsFor(g, 'pkg.patched.Thing.run').tests).toEqual(['tests/test_patch.py']); + expect(testsFor(g, 'pkg/__init__.py').tests).toEqual(['tests/test_adversarial.py', 'tests/test_escaped.py', 'tests/test_inline.py', 'tests/test_patch.py', 'tests/test_semicolon.py']); + }); }); describe('literals (pure)', () => { @@ -189,6 +249,23 @@ describe('gstack-pregate (bin)', () => { expect(readOut(repo).stage2_ms).toBeUndefined(); }); + test('missing dependencies: regen is incomplete with REGEN_DEPS_MISSING and the install command, not REGEN_STALE', () => { + const repo = fixture({ + registry: { entries: [{ id: 'digest', command: 'node gen.js', outputs: ['generated/**'], inputs: ['src/version.js', 'gen.js'] }] }, + baseFiles: { + 'package.json': JSON.stringify({ name: 'fixture', version: '1.0.0', scripts: { test: 'node --test test/*.test.js' }, devDependencies: { 'fixture-dep': '1.0.0' } }, null, 2) + '\n', + 'gen.js': "const fs=require('fs');const {VERSION}=require('./src/version.js');let style='plain';try{style=require('fixture-dep').style;}catch{}fs.writeFileSync('generated/digest.md',`# digest v${VERSION} (${style})\\n`);\n", + 'generated/digest.md': '# digest v1.0.0 (fancy)\n', + }, + }); + commit(repo, { 'src/version.js': "module.exports = { VERSION: '1.0.0' };\n// touched\n" }); + const r = pregate(repo, ['--stage', 'preflight', '--only', 'regen']); + expect(r.status).not.toBe(EXIT.ok); + expect(r.out).toMatch(/^preflight {2}regen {5}incomplete .*dependencies missing; run bun install, then rerun the check \(1 entry unverified: digest\) \(REGEN_DEPS_MISSING\)/m); + expect(r.err).toContain('(REGEN_DEPS_MISSING)'); + expect(r.out + r.err).not.toContain('REGEN_STALE'); + }); + test('p0-4-scratch-script: zz-probe.sh at the root fails strays (downgraded to warn only when the base config says so); a conventional root file passes', () => { const strict = fixture({ pregate: { ...BASE_PREGATE, warn: [] } }); commit(strict, { 'zz-probe.sh': '#!/bin/sh\necho hi\n', 'CONTRIBUTING.md': '# c\n', 'notes.txt': 'x\n', 'newtop/thing.js': 'module.exports = 1;\n' });