From c5c2f28994c7fb3815384bbc576febdd55b6f070 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sat, 17 Oct 2026 18:07:35 -0700 Subject: [PATCH 1/2] v1.91.128.0 ci(seed): weekly standing recorder for the Linux and Windows duration seeds (#3253) * ci(seed): weekly standing recorder for the Linux and Windows duration seeds (W4) free-durations-record.yml runs Mondays 04:00 UTC (before test-health) and on dispatch. Linux records on ubicloud-standard-8, the free-suite shard runner, with scripts/ubicloud/setup-free-suite.sh; Windows records through windows-free-tests.yml's record_durations path (now also workflow_call). When either seed's file set drifts from main, the propose job force-updates bot/free-duration-seeds, opens or refreshes one seed PR stating the seed ratchet result and the files that failed while recording, and dispatches the free lanes on it. A red file keeps its duration; an interrupted recording uploads no seed (this also stops the Windows dispatch from uploading a stale copy), and more than 5 red files on either OS proposes nothing. The bot never edits the allowlist. open_pr=false on dispatch records and reports only. The workflow is classified free-lane only in scripts/test-pr-profile.ts, with a non-behavioral HARNESS_VERSION decision. Intentional pin change: test/windows-native-workflows.test.ts now expects workflow_call and the complete-recording guard on the Windows record job. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * ci(seed): keep maintainer commits on the open seed PR, publish only from main, survive long ratchet reports /review on #3253 (gpt-6.1-sol): a refresh force-pushed over maintainer repairs on the seed PR, a dispatch from a feature branch would publish that branch's commits, and grep | head under pipefail could abort the step on a long ratchet report. The refresh now leaves an open seed PR alone when it holds any commit besides the bot's one refresh commit and pushes with an exact lease; only main publishes; the report is cut with sed. Each has a functional test that fails on the old workflow. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(changelog): unreleased entry for the weekly duration-seed recorder Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * chore: stamp v1.91.128.0 Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- .github/workflows/free-durations-record.yml | 172 +++++++++++++++++++ .github/workflows/windows-free-tests.yml | 25 ++- CHANGELOG.md | 11 ++ VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- docs/TESTING_INTERNALS.md | 26 ++- package.json | 2 +- scripts/harness-version.json | 2 +- scripts/test-pr-profile.ts | 1 + test/free-durations-record-workflow.test.ts | 177 ++++++++++++++++++++ test/windows-native-workflows.test.ts | 11 +- 11 files changed, 413 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/free-durations-record.yml create mode 100644 test/free-durations-record-workflow.test.ts diff --git a/.github/workflows/free-durations-record.yml b/.github/workflows/free-durations-record.yml new file mode 100644 index 000000000..848be5dfc --- /dev/null +++ b/.github/workflows/free-durations-record.yml @@ -0,0 +1,172 @@ +name: Free-suite duration seeds + +# Weekly standing recorder for the two free-suite duration seeds that the +# shard packers read: scripts/free-test-durations.json (Linux, recorded on the +# same ubicloud-standard-8 runner as the free-suite shards) and +# scripts/free-test-durations-windows.json (windows-latest, through the +# record_durations path of windows-free-tests.yml). When either recording adds +# or drops a test file, it force-updates one bot branch and opens or refreshes +# a seed PR. A file that fails while recording turns its job red but keeps its +# duration (a red file is still a real cost) and is named in the PR body; an +# interrupted recording, or more than 5 red files on either OS, proposes +# nothing (docs/TESTING_INTERNALS.md#free-suite-duration-seed). + +on: + schedule: + - cron: '0 4 * * 1' # Monday 04:00 UTC, before the 10:00 test-health run + workflow_dispatch: + inputs: + open_pr: + description: Push the bot branch and open or refresh the seed PR (false records and reports only) + type: boolean + default: true + +concurrency: + group: free-durations-record + cancel-in-progress: false + +permissions: + contents: read + +jobs: + linux: + runs-on: ubicloud-standard-8 + timeout-minutes: 45 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - name: Prepare the same inputs as a free-suite shard + run: | + bash scripts/ubicloud/setup-free-suite.sh + echo "$HOME/.bun/bin" >> "$GITHUB_PATH" + - name: Time every free file alone + id: record + env: + GSTACK_EXPECT_BINARIES: '1' + run: | + set -o pipefail + xvfb-run -a bun run test:free --record-durations 2>&1 | tee "$RUNNER_TEMP/record.log" + - name: Keep the recording only when every file was timed + id: seed + if: always() && steps.record.outcome != 'skipped' + run: | + grep -q 'wrote [0-9]* durations to' "$RUNNER_TEMP/record.log" || { echo "::error::The recording was interrupted; no Linux seed to upload."; exit 1; } + mkdir -p "$RUNNER_TEMP/seed" + cp scripts/free-test-durations.json "$RUNNER_TEMP/seed/free-test-durations.json" + sed -n '/failed while recording:/,$p' "$RUNNER_TEMP/record.log" | sed 's/\r$//' | { grep -E '^ ✗ [^ ]+$' || true; } | sed 's/^ ✗ //' > "$RUNNER_TEMP/seed/record-failures.txt" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + if: always() + with: + name: linux-record-durations-logs + path: .context/free-test-logs/gstack-free-test-*.log + include-hidden-files: true + if-no-files-found: ignore + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + if: always() && steps.seed.outcome == 'success' + with: + name: free-test-durations + path: ${{ runner.temp }}/seed/ + if-no-files-found: error + + windows: + uses: ./.github/workflows/windows-free-tests.yml + with: + record_durations: true + + propose: + needs: [linux, windows] + if: ${{ !cancelled() }} + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: write + pull-requests: write + actions: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + fetch-depth: 0 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.4.2 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + name: free-test-durations + path: ${{ runner.temp }}/linux + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + name: free-test-durations-windows + path: ${{ runner.temp }}/windows + - name: Open or refresh the seed PR when either seed's file set changed + env: + GH_TOKEN: ${{ github.token }} + OPEN_PR: ${{ github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || inputs.open_pr) }} + run: | + set -euo pipefail + linux_failed=$(wc -l < "$RUNNER_TEMP/linux/record-failures.txt") + windows_failed=$(wc -l < "$RUNNER_TEMP/windows/record-failures.txt") + if [ "$linux_failed" -gt 5 ] || [ "$windows_failed" -gt 5 ]; then + echo "::error::$linux_failed Linux and $windows_failed Windows files failed while recording; more than 5 on either OS means a broken tree, so no seed is proposed." + exit 1 + fi + failures() { if [ -s "$1" ]; then paste -sd, "$1" | sed 's/,/, /g'; else echo none; fi; } + drift() { # committed recorded -> " " + jq -n --slurpfile old "$1" --slurpfile new "$2" \ + '[($new[0].durations|keys) - ($old[0].durations|keys), ($old[0].durations|keys) - ($new[0].durations|keys)] | map(length) | join(" ")' + } + read -r linux_added linux_removed <<<"$(drift scripts/free-test-durations.json "$RUNNER_TEMP/linux/free-test-durations.json" | tr -d '"')" + read -r windows_added windows_removed <<<"$(drift scripts/free-test-durations-windows.json "$RUNNER_TEMP/windows/free-test-durations-windows.json" | tr -d '"')" + echo "Linux seed: +$linux_added -$linux_removed files; Windows seed: +$windows_added -$windows_removed files" | tee -a "$GITHUB_STEP_SUMMARY" + if [ "$((linux_added + linux_removed + windows_added + windows_removed))" -eq 0 ]; then + echo "Both seeds cover the current test files; no PR." | tee -a "$GITHUB_STEP_SUMMARY" + exit 0 + fi + cp "$RUNNER_TEMP/linux/free-test-durations.json" scripts/free-test-durations.json + cp "$RUNNER_TEMP/windows/free-test-durations-windows.json" scripts/free-test-durations-windows.json + if ratchet=$(GSTACK_FREE_SEED_BASE="$(git rev-parse HEAD)" bun test test/free-seed-ratchet.test.ts 2>&1); then + ratchet_line="passes" + else + ratchet_line="FAILS; a maintainer must split the named file or add an annotated allowlist entry on this branch before merge:"$'\n\n'"\`\`\`"$'\n'"$(printf '%s\n' "$ratchet" | sed -n -E '/Free seed ratchet|^- /p' | sed -n '1,20p')"$'\n'"\`\`\`" + fi + failed_line="Linux: $(failures "$RUNNER_TEMP/linux/record-failures.txt"); Windows: $(failures "$RUNNER_TEMP/windows/record-failures.txt")" + echo "Seed growth ratchet: $ratchet_line" >> "$GITHUB_STEP_SUMMARY" + echo "Failed while recording: $failed_line" | tee -a "$GITHUB_STEP_SUMMARY" + if [ "$OPEN_PR" != true ]; then + echo "Report only (open_pr=false, or a ref other than main): no branch pushed." | tee -a "$GITHUB_STEP_SUMMARY" + exit 0 + fi + branch=bot/free-duration-seeds + subject="test(seed): refresh the free-suite duration seeds from the weekly recording" + number=$(gh pr list --head "$branch" --state open --json number -q '.[0].number // empty') + remote_head= + if git fetch -q origin "+refs/heads/$branch:refs/remotes/origin/$branch" 2>/dev/null; then + remote_head=$(git rev-parse "origin/$branch") + fi + if [ -n "$number" ] && [ -n "$remote_head" ] && [ "$(git log --format=%s "HEAD..$remote_head")" != "$subject" ]; then + echo "::warning::Seed PR #$number has commits besides the bot's one refresh commit; this run leaves them in place. Merge or close that PR and the next run refreshes the seeds." | tee -a "$GITHUB_STEP_SUMMARY" + exit 0 + fi + git switch -C "$branch" + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add scripts/free-test-durations.json scripts/free-test-durations-windows.json + git commit -m "$subject" + git push --force-with-lease="refs/heads/$branch:$remote_head" origin "HEAD:refs/heads/$branch" + cat > pr-body.md <&1 | tee "$RUNNER_TEMP/record.log" - name: Name the recording as the Windows seed - if: always() + id: seed + if: always() && steps.record.outcome != 'skipped' shell: bash - run: cp scripts/free-test-durations.json "$RUNNER_TEMP/free-test-durations-windows.json" + run: | + grep -q 'wrote [0-9]* durations to' "$RUNNER_TEMP/record.log" || { echo "::error::The recording was interrupted; no Windows seed to upload."; exit 1; } + cp scripts/free-test-durations.json "$RUNNER_TEMP/free-test-durations-windows.json" + sed -n '/failed while recording:/,$p' "$RUNNER_TEMP/record.log" | sed 's/\r$//' | { grep -E '^ ✗ [^ ]+$' || true; } | sed 's/^ ✗ //' > "$RUNNER_TEMP/record-failures.txt" - name: Upload full recording logs if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 @@ -251,8 +264,10 @@ jobs: include-hidden-files: true if-no-files-found: ignore - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 - if: always() + if: always() && steps.seed.outcome == 'success' with: name: free-test-durations-windows - path: ${{ runner.temp }}/free-test-durations-windows.json + path: | + ${{ runner.temp }}/free-test-durations-windows.json + ${{ runner.temp }}/record-failures.txt if-no-files-found: error diff --git a/CHANGELOG.md b/CHANGELOG.md index e2057130d..2db7e8018 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## [1.91.128.0] - 2026-10-18 + +**The two free-suite duration seeds now refresh themselves weekly.** The shard packers read a Linux and a Windows duration seed, and both went stale as test files came and went. They were refreshed by hand twice this week. A new weekly workflow records both seeds and opens one seed-refresh PR when either seed's file set changes. That PR goes through the normal review. + +### Added +- **`free-durations-record.yml`** runs Mondays at 04:00 UTC, before test-health, and on dispatch. It records the Linux seed on `ubicloud-standard-8`, the free-suite shard runner, and the Windows seed through `windows-free-tests.yml`'s `record_durations` path (now also callable with `workflow_call`). When either seed adds or drops a file, it opens or refreshes one PR on `bot/free-duration-seeds` and runs the free lanes on it. The PR body states the seed-ratchet result and names every file that failed while recording. +- **Guards:** the bot never edits `scripts/free-test-seed-allowlist.json`. While the open seed PR holds any commit besides the bot's own refresh, later runs leave it alone. Only runs on `main` publish, and `open_pr=false` records and reports only. An interrupted recording, or more than 5 red files on either OS, proposes nothing. + +### Fixed +- **An interrupted manual Windows recording no longer uploads the unchanged committed seed** under the Windows seed name. + ## [1.91.127.0] - 2026-10-17 **The paid PR gate stops failing on its own fixtures.** A 132-verdict PR gate was fully green only about a quarter of the time. Every red verdict since #3231 was diagnosed from its transcript and replayed through the real detector. This release fixes the remaining causes where they start: fixture actors that broke the production protocol, detectors that misread a correct run, and skill prompts that left the agent guessing. No threshold, trial count, case kind or quarantine changed. diff --git a/VERSION b/VERSION index 78a0be812..d6ddc385a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.127.0 +1.91.128.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index bf1d005c4..2e080b314 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.127.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.128.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/docs/TESTING_INTERNALS.md b/docs/TESTING_INTERNALS.md index 6bf03294b..2befad2fe 100644 --- a/docs/TESTING_INTERNALS.md +++ b/docs/TESTING_INTERNALS.md @@ -1164,10 +1164,11 @@ changed. Cut or unstarted reruns are incomplete, not passes. ### Free suite duration seed `scripts/free-test-durations.json` drives shard packing; files missing from it -pack at the seed's 99th percentile. Acceptable seeds come only from -`bun run test:ubicloud --record-durations` (each file timed in its own child on a -VM with the CI lane's environment; it needs `UBICLOUD_API_KEY` and copies the -refreshed seed back into the checkout). A 4-core laptop or small-sandbox +pack at the seed's 99th percentile. Acceptable seeds come only from a recording +in the CI lane's environment, each file timed in its own child: the weekly +`free-durations-record.yml` run, or `bun run test:ubicloud --record-durations` +(it needs `UBICLOUD_API_KEY` and copies the refreshed seed back into the +checkout). A 4-core laptop or small-sandbox recording is not acceptable: browser and display tests skip or slow down there, so the seed misstates them. Contributors without `UBICLOUD_API_KEY` leave the refresh to a maintainer. @@ -1188,6 +1189,20 @@ refresh to a maintainer. `{file, reason, todo}` and the test prints it. A merge-base without the allowlist file counts as the first allowlist. Fix a failure by splitting the file, then rerun `bun run test:ubicloud --record-durations`. +- `free-durations-record.yml` (Mondays 04:00 UTC, before test-health, and on + dispatch) records the Linux seed on `ubicloud-standard-8` with + `scripts/ubicloud/setup-free-suite.sh` and the Windows seed through + `windows-free-tests.yml`'s `record_durations` path. When either recording adds + or drops a test file, it force-updates the `bot/free-duration-seeds` branch, + opens or refreshes one seed PR whose body states the ratchet result, and + dispatches the free lanes on it. A file that fails while recording turns its + job red but keeps its duration, and the PR body names it; the seed PR's own + free checks decide whether the tree is healthy. An interrupted recording + uploads no seed, and more than 5 red files on either OS proposes nothing. The + bot never edits the allowlist: a ratchet failure is fixed on the branch by a + maintainer before the PR merges, and while the open seed PR holds any commit + besides the bot's one refresh commit, later runs leave it alone. Only runs on + `main` push the branch; a dispatch from another ref reports only. ### Windows free lane @@ -1195,7 +1210,8 @@ refresh to a maintainer. runs one strict serial shard per `windows-latest` job (`bun run test:windows --ci-run`), and verifies every shard receipt on Linux (`--windows-only --ci-verify`). Packing uses Windows-measured durations from -`scripts/free-test-durations-windows.json`; refresh them with +`scripts/free-test-durations-windows.json`; the weekly +`free-durations-record.yml` run refreshes them, or by hand run `gh workflow run windows-free-tests.yml --ref -f record_durations=true`, then download the `free-test-durations-windows` artifact and commit it. diff --git a/package.json b/package.json index 87e40ee61..5bc904faf 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.127", + "version": "1.91.128", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/scripts/harness-version.json b/scripts/harness-version.json index b3c02f3a5..ee40de983 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -16,7 +16,7 @@ "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":5}, "scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":5}, "scripts/test-paid-shards.ts": {"blob":"8ddee5f11ed7856876135520dd8025ab989e2db9","since":5}, - "scripts/test-pr-profile.ts": {"blob":"ddc1e5991636c8c94722cd32472edce26489b26d","since":5}, + "scripts/test-pr-profile.ts": {"blob":"67d3e163dd443bfc5e381586e28f7e83ed59125a","since":5,"nonBehavioral":"Classifies the new free-durations-record.yml workflow as free-lane only; no paid case reads it, so no verdict can change"}, "scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":5}, "test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":5}, "test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":5}, diff --git a/scripts/test-pr-profile.ts b/scripts/test-pr-profile.ts index ddc1e5991..67d3e163d 100644 --- a/scripts/test-pr-profile.ts +++ b/scripts/test-pr-profile.ts @@ -230,6 +230,7 @@ export const FREE_ONLY_PR_FILES = [ '.github/workflows/cso-runtime-qualification.yml', '.github/workflows/cso-scanner-images.yml', '.github/workflows/dependency-review.yml', + '.github/workflows/free-durations-record.yml', '.github/workflows/free-tests.yml', '.github/workflows/make-pdf-gate.yml', '.github/workflows/model-policy-freshness.yml', diff --git a/test/free-durations-record-workflow.test.ts b/test/free-durations-record-workflow.test.ts new file mode 100644 index 000000000..7a61d7c6b --- /dev/null +++ b/test/free-durations-record-workflow.test.ts @@ -0,0 +1,177 @@ +/** + * Static pins and a functional check for the weekly duration-seed recorder + * (.github/workflows/free-durations-record.yml): Linux on the free-suite + * runner, Windows through the record_durations path, and one bot seed PR only + * when a seed's file set drifts. + */ +import { afterEach, expect, test } from 'bun:test'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; + +const root = path.resolve(import.meta.dir, '..'); +const load = (file: string) => Bun.YAML.parse(readFileSync(path.join(root, '.github/workflows', file), 'utf8')) as any; +const recorder = load('free-durations-record.yml'); +const freeTests = load('free-tests.yml'); +const proposeStep = recorder.jobs.propose.steps.find((step: any) => step.name === 'Open or refresh the seed PR when either seed\'s file set changed'); +const dirs: string[] = []; +afterEach(() => { for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); }); + +test('the recorder runs weekly before test-health and on dispatch, read-only except the propose job', () => { + expect(recorder.on.schedule).toEqual([{ cron: '0 4 * * 1' }]); + expect(Object.keys(recorder.on).sort()).toEqual(['schedule', 'workflow_dispatch']); + expect(Object.keys(recorder.on.workflow_dispatch.inputs)).toEqual(['open_pr']); + expect(load('test-health.yml').on.schedule[0].cron).toBe('0 10 * * 1'); + expect(recorder.permissions).toEqual({ contents: 'read' }); + expect(recorder.concurrency['cancel-in-progress']).toBe(false); + expect(recorder.jobs.linux.permissions).toBeUndefined(); + expect(recorder.jobs.propose.permissions).toEqual({ contents: 'write', 'pull-requests': 'write', actions: 'write' }); +}); + +test('Linux records on the free-suite runner, turns red on any red file, and uploads only a complete recording', () => { + const linux = recorder.jobs.linux; + expect(linux['runs-on']).toBe(freeTests.jobs['free-suite']['runs-on']); + const runs = linux.steps.map((step: any) => step.run).filter(Boolean).join('\n'); + expect(runs).toContain('bash scripts/ubicloud/setup-free-suite.sh'); + const record = linux.steps.find((step: any) => step.name === 'Time every free file alone'); + expect(record.run).toBe('set -o pipefail\nxvfb-run -a bun run test:free --record-durations 2>&1 | tee "$RUNNER_TEMP/record.log"\n'); + expect(record['continue-on-error']).toBeUndefined(); + const keep = linux.steps.find((step: any) => step.id === 'seed'); + expect(keep.if).toBe("always() && steps.record.outcome != 'skipped'"); + expect(keep.run).toContain('grep -q \'wrote [0-9]* durations to\' "$RUNNER_TEMP/record.log" || {'); + expect(linux.steps.at(-1).if).toBe("always() && steps.seed.outcome == 'success'"); + expect(linux.steps.at(-1).with).toMatchObject({ name: 'free-test-durations', path: '${{ runner.temp }}/seed/' }); +}); + +test('Windows records through the windows-free-tests record_durations path, and propose needs both complete recordings', () => { + expect(recorder.jobs.windows).toEqual({ uses: './.github/workflows/windows-free-tests.yml', with: { record_durations: true } }); + expect(recorder.jobs.propose.needs).toEqual(['linux', 'windows']); + expect(recorder.jobs.propose.if).toBe('${{ !cancelled() }}'); + const downloads = recorder.jobs.propose.steps.filter((step: any) => step.uses?.startsWith('actions/download-artifact@')); + expect(downloads.map((step: any) => step.with.name)).toEqual(['free-test-durations', 'free-test-durations-windows']); + expect(downloads.every((step: any) => step['continue-on-error'] === undefined)).toBe(true); +}); + +test('both recorders list exactly the files the recorder reported red, and more than 5 on either OS proposes nothing', () => { + const windowsRecord = load('windows-free-tests.yml').jobs['windows-record-durations'].steps.find((step: any) => step.id === 'seed').run as string; + const linuxRecord = recorder.jobs.linux.steps.find((step: any) => step.id === 'seed').run as string; + const extract = (run: string) => run.split('\n').find(line => line.startsWith("sed -n '/failed while recording:/"))!; + expect(extract(linuxRecord).replace('seed/record-failures.txt', 'record-failures.txt')).toBe(extract(windowsRecord)); + const dir = mkdtempSync(path.join(os.tmpdir(), 'seed-failures-')); + dirs.push(dir); + const list = (log: string) => { + writeFileSync(path.join(dir, 'record.log'), log); + const result = spawnSync('bash', ['-c', `set -euo pipefail\n${extract(windowsRecord)}\ncat "$RUNNER_TEMP/record-failures.txt"`], { encoding: 'utf8', timeout: 10_000, env: { ...process.env, RUNNER_TEMP: dir } }); + expect(result.status, result.stderr).toBe(0); + return result.stdout; + }; + const red = ' ✗ test/a.test.ts — suite > case\r\n[test:free] wrote 3 durations to scripts\\free-test-durations.json\r\n[test:free] WARNING: 2 file(s) failed while recording:\r\n ✗ test/a.test.ts\r\n ✗ browse/test/b.test.ts\r\n'; + expect(list(red)).toBe('test/a.test.ts\nbrowse/test/b.test.ts\n'); + expect(list(' ✗ test/a.test.ts — suite > flaky then fixed\n[test:free] wrote 3 durations to scripts/free-test-durations.json\n')).toBe(''); + const script: string = proposeStep.run; + expect(script).toContain('if [ "$linux_failed" -gt 5 ] || [ "$windows_failed" -gt 5 ]; then'); + expect(script.indexOf('-gt 5 ]')).toBeLessThan(script.indexOf('drift() {')); + expect(script).toContain('Files that failed while recording (durations kept; this PR\'s own free checks decide): $failed_line'); +}); + +test('the propose step touches only the two seeds, never the allowlist, on one bot branch', () => { + const script: string = proposeStep.run; + expect(script).toContain('git add scripts/free-test-durations.json scripts/free-test-durations-windows.json\n'); + expect(script).not.toContain('free-test-seed-allowlist.json'); + expect(script).toContain('branch=bot/free-duration-seeds'); + expect(script).toContain('bun test test/free-seed-ratchet.test.ts'); + expect(script).toContain('gh workflow run free-tests.yml'); + expect(proposeStep.env.OPEN_PR).toBe("${{ github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || inputs.open_pr) }}"); + expect(script.indexOf('if [ "$OPEN_PR" != true ]')).toBeLessThan(script.indexOf('git push')); + expect(recorder.on.workflow_dispatch.inputs.open_pr).toMatchObject({ type: 'boolean', default: true }); +}); + +test('a long ratchet report reaches the PR body instead of aborting the step under pipefail', () => { + const script: string = proposeStep.run; + const line = script.split('\n').find(entry => entry.trimStart().startsWith('ratchet_line="FAILS'))!.trim(); + const ratchet = ['error: Free seed ratchet:', ...Array.from({ length: 3000 }, (_, i) => `- test/f${i}.test.ts is seeded at 61s, over the 60s ceiling. Fix: split it, then rerun the recording.`)].join('\n'); + const dir = mkdtempSync(path.join(os.tmpdir(), 'seed-ratchet-')); + dirs.push(dir); + writeFileSync(path.join(dir, 'ratchet.txt'), ratchet); + const result = spawnSync('bash', ['-c', `set -euo pipefail\nratchet=$(cat "$1")\n${line}\nprintf '%s' "$ratchet_line"`, 'ratchet', path.join(dir, 'ratchet.txt')], { encoding: 'utf8', timeout: 10_000 }); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain('- test/f18.test.ts is seeded at 61s, over'); + expect(result.stdout).not.toContain('- test/f19.test.ts'); +}); + +test('a refresh keeps a maintainer\'s commits on the open seed PR and leases the bot branch it replaces', () => { + const script: string = proposeStep.run; + const publish = script.slice(script.indexOf('branch=bot/free-duration-seeds'), script.indexOf('cat > pr-body.md')); + const dir = mkdtempSync(path.join(os.tmpdir(), 'seed-publish-')); + dirs.push(dir); + const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.startsWith('GIT_'))); + const git = (cwd: string, ...args: string[]) => { + const result = spawnSync('git', args, { cwd, encoding: 'utf8', timeout: 10_000, env }); + expect(result.status, result.stderr).toBe(0); + return result.stdout.trim(); + }; + const origin = path.join(dir, 'origin.git'); + const work = path.join(dir, 'work'); + git(dir, 'init', '-q', '--bare', '-b', 'main', origin); + git(dir, 'clone', '-q', origin, work); + const commit = (email: string, file: string, message: string) => { + git(work, 'config', 'user.name', email); + git(work, 'config', 'user.email', email); + writeFileSync(path.join(work, file), message); + git(work, 'add', file); + git(work, 'commit', '-q', '-m', message); + }; + for (const file of ['free-test-durations.json', 'free-test-durations-windows.json']) { + writeFileSync(path.join(work, file), '{}'); + } + git(work, 'checkout', '-q', '-b', 'main'); + commit('dev@example.com', 'README', 'main'); + git(work, 'push', '-q', 'origin', 'main'); + const subject = 'test(seed): refresh the free-suite duration seeds from the weekly recording'; + git(work, 'checkout', '-q', '-b', 'bot/free-duration-seeds'); + commit('bot@example.com', 'seed.txt', subject); + git(work, 'push', '-q', 'origin', 'bot/free-duration-seeds'); + const stub = path.join(dir, 'bin'); + spawnSync('mkdir', [stub], { timeout: 10_000 }); + writeFileSync(path.join(stub, 'gh'), '#!/bin/sh\necho 7\n', { mode: 0o755 }); + const run = () => { + git(work, 'checkout', '-q', 'main'); + spawnSync('mkdir', ['-p', path.join(work, 'scripts')], { timeout: 10_000 }); + writeFileSync(path.join(work, 'scripts/free-test-durations.json'), `{"run":${Date.now()}}`); + writeFileSync(path.join(work, 'scripts/free-test-durations-windows.json'), '{}'); + const result = spawnSync('bash', ['-c', `set -euo pipefail\n${publish}`], { cwd: work, encoding: 'utf8', timeout: 20_000, + env: { ...env, PATH: `${stub}:${process.env.PATH}`, GITHUB_STEP_SUMMARY: path.join(dir, 'summary.md') } }); + expect(result.status, result.stderr).toBe(0); + return git(origin, 'rev-parse', 'refs/heads/bot/free-duration-seeds'); + }; + const replaced = run(); + expect(git(origin, 'log', '--format=%s', `main..${replaced}`)).toBe(subject); + expect(git(origin, 'show', `${replaced}:scripts/free-test-durations.json`)).toContain('"run"'); + git(work, 'checkout', '-q', '-B', 'bot/free-duration-seeds', replaced); + commit('maintainer@example.com', 'allowlist.json', 'maintainer repair'); + git(work, 'push', '-q', 'origin', 'HEAD:refs/heads/bot/free-duration-seeds'); + const repaired = git(origin, 'rev-parse', 'refs/heads/bot/free-duration-seeds'); + expect(run()).toBe(repaired); + expect(readFileSync(path.join(dir, 'summary.md'), 'utf8')).toContain("Seed PR #7 has commits besides the bot's one refresh commit"); +}); + +test('the drift check counts added and removed files, so an unchanged file set proposes nothing', () => { + const script: string = proposeStep.run; + const drift = script.slice(script.indexOf('drift() {'), script.indexOf('\n}', script.indexOf('drift() {')) + 2); + const dir = mkdtempSync(path.join(os.tmpdir(), 'seed-drift-')); + dirs.push(dir); + const seed = (name: string, files: Record) => { + const file = path.join(dir, name); + writeFileSync(file, JSON.stringify({ version: 1, recordedAt: '2026-10-17T00:00:00.000Z', durations: files })); + return file; + }; + const committed = seed('committed.json', { 'test/a.test.ts': 100, 'test/b.test.ts': 200 }); + const run = (recorded: string) => spawnSync('bash', ['-c', `${drift}\ndrift "$1" "$2"`, 'drift', committed, recorded], { encoding: 'utf8', timeout: 10_000 }); + const same = run(seed('same.json', { 'test/a.test.ts': 900, 'test/b.test.ts': 50 })); + expect(same.status, same.stderr).toBe(0); + expect(same.stdout.trim()).toBe('"0 0"'); + const drifted = run(seed('drifted.json', { 'test/a.test.ts': 100, 'test/c.test.ts': 1, 'test/d.test.ts': 2 })); + expect(drifted.status, drifted.stderr).toBe(0); + expect(drifted.stdout.trim()).toBe('"2 1"'); +}); diff --git a/test/windows-native-workflows.test.ts b/test/windows-native-workflows.test.ts index 838d7c528..013e89fed 100644 --- a/test/windows-native-workflows.test.ts +++ b/test/windows-native-workflows.test.ts @@ -23,8 +23,9 @@ const native = load('native-qualification.yml'); const WINDOWS_CURATED_FLOOR = 638; test('the Windows lane plans, runs one strict shard per job and verifies every result', () => { - expect(Object.keys(windows.on).sort()).toEqual(['pull_request', 'workflow_dispatch']); + expect(Object.keys(windows.on).sort()).toEqual(['pull_request', 'workflow_call', 'workflow_dispatch']); expect(Object.keys(windows.on.workflow_dispatch.inputs)).toEqual(['record_durations']); + expect(Object.keys(windows.on.workflow_call.inputs)).toEqual(['record_durations']); const plan = windows.jobs['windows-plan']; expect(plan['runs-on']).toBe('ubuntu-24.04'); const planStep = plan.steps.find((step: any) => step.id === 'plan'); @@ -62,15 +63,17 @@ test('a record_durations dispatch times each Windows-safe file alone and uploads expect(job.if).toBe('${{ inputs.record_durations }}'); expect(job['runs-on']).toBe('windows-latest'); const record = job.steps.find((step: any) => step.name === 'Time every Windows-safe file alone'); - expect(record.run).toBe('bun run test:windows --record-durations --wall-timeout 1200'); + expect(record.run).toBe('set -o pipefail\nbun run test:windows --record-durations --wall-timeout 1200 2>&1 | tee "$RUNNER_TEMP/record.log"\n'); expect(record.env.GSTACK_FREE_TEST_DURATIONS).toBeUndefined(); const rename = job.steps.find((step: any) => step.name === 'Name the recording as the Windows seed'); - expect(rename.if).toBe('always()'); - expect(rename.run).toBe('cp scripts/free-test-durations.json "$RUNNER_TEMP/free-test-durations-windows.json"'); + expect(rename.if).toBe("always() && steps.record.outcome != 'skipped'"); + expect(rename.run).toContain('grep -q \'wrote [0-9]* durations to\' "$RUNNER_TEMP/record.log" || {'); + expect(rename.run).toContain('cp scripts/free-test-durations.json "$RUNNER_TEMP/free-test-durations-windows.json"'); const logs = job.steps.find((step: any) => step.name === 'Upload full recording logs'); expect(logs.if).toBe('always()'); expect(logs.with['include-hidden-files']).toBe(true); expect(job.steps.at(-1).with.name).toBe('free-test-durations-windows'); + expect(job.steps.at(-1).if).toBe("always() && steps.seed.outcome == 'success'"); }); test('Windows shards use the pinned Node runtime and retain complete logs on every run', () => { From a669dafcb43b9291aea50dd9f47042bac7d5da5e Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sat, 17 Oct 2026 18:22:15 -0700 Subject: [PATCH 2/2] v1.91.129.0 fix(pr-selection): scan shebang and .ts imports, fail closed on scan errors, report cost per run (#3257) * fix(pr-selection): scan shebang and .ts imports, fail closed on scan errors, classify repo config safely A1-1: strip a leading #! before scanImports at every site, treat bun/node shebang scripts as code, pick the loader by extension (the TSX loader misread 15 .ts files), and restore the full gate when any other import scan fails. Broadens selection only: 0 (file, case) pairs lost vs main, 41 files move from no-consumer to consumed. A1-2: FREE_ONLY_PR_FILES gains six repository-config files, capy/ is derivable, and a full-gate file is labelled 'full-gate input' instead of 'real unknown dependency'. The replay MISS check honours the same audited free-only exemptions as the selector. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * feat(test-health): report gate cases and known cost per PR run, lead with cost per merged PR (A1-3) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(testing): correct the PR fallback section, document the lockfile decision (A1-3, A1-4) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * chore(harness): record the A1 test-pr-profile.ts change as non-behavioral (selection only) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * chore(release): v1.91.127.0 Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Revert "chore(release): v1.91.127.0" This reverts commit 79e3ef6c6330b3e21e0355f90a2f465d1ebb54cf. * chore(release): v1.91.129.0 Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 16 ++++++ VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- docs/TESTING_INTERNALS.md | 39 ++++++++++++--- package.json | 2 +- scripts/harness-version.json | 2 +- scripts/pr-dependencies.ts | 55 +++++++++++++++------ scripts/replay-pr-selection.ts | 8 +-- scripts/test-health-report.ts | 42 +++++++++++++--- scripts/test-pr-profile.ts | 19 ++++++- test/free-fixtures.test.ts | 4 +- test/paid-free-boundary.test.ts | 2 +- test/pr-dependencies.test.ts | 87 +++++++++++++++++++++++++++++++++ test/test-health-report.test.ts | 26 +++++++++- 14 files changed, 265 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2db7e8018..0ecfedfec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## [1.91.129.0] - 2026-10-18 + +**PR paid-lane selection reads every script it depends on, and fails closed when it can't.** Selection derives which paid cases a changed file reaches by scanning imports. It skipped extensionless `bun`/`node` scripts, read 15 `.ts` files with the TSX loader, and treated a scan error as "no consumers", which under-selects. Across all 2,223 code files in the repo, scans now finish with 0 errors, and no (file, case) pair that main selected is lost. + +### Fixed +- **Shebang scripts and `.ts` files are scanned correctly.** A leading `#!` line is stripped before scanning, extensionless `bun`/`node` scripts count as code, and the loader is chosen by file extension. 41 files that reached no paid case now reach the cases that use them. +- **A failed import scan restores the full gate** with the label `import scan failed` and the file to fix, instead of silently selecting nothing. +- **Repository config no longer looks like an unknown dependency.** `.gstack/ship-policy.example.json`, `.github/dependabot.yml`, `.osv-scanner.toml`, `setup.ps1`, `.gitlab-ci.yml` and `.github/ISSUE_TEMPLATE/**` are audited free-only files; `capy/` paths are derived like other source; a full-gate file is labelled `full-gate input`. + +### Added +- **`bun run test:health` reports cost per merged PR first**, plus gate cases per PR run and known cost per PR run, split by selection mode. + +### For contributors +- The selection replay (`scripts/replay-pr-selection.ts`) honours the same free-only exemptions as the selector when it counts misses. +- docs/TESTING_INTERNALS.md#pr-paid-lane-fallback describes the fallback classes as they now behave, and records why the lockfile stays a full-gate input. + ## [1.91.128.0] - 2026-10-18 **The two free-suite duration seeds now refresh themselves weekly.** The shard packers read a Linux and a Windows duration seed, and both went stale as test files came and went. They were refreshed by hand twice this week. A new weekly workflow records both seeds and opens one seed-refresh PR when either seed's file set changes. That PR goes through the normal review. diff --git a/VERSION b/VERSION index d6ddc385a..de898c0c6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.128.0 +1.91.129.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 2e080b314..fbec08ef6 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.128.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.129.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/docs/TESTING_INTERNALS.md b/docs/TESTING_INTERNALS.md index 2befad2fe..7b7e7956a 100644 --- a/docs/TESTING_INTERNALS.md +++ b/docs/TESTING_INTERNALS.md @@ -1009,17 +1009,44 @@ select nothing: docs, any non-paid `*.test.ts(x)`, `FREE_ONLY_PR_FILES` (the free and paid duration seeds, `scripts/ubicloud/**`, every workflow that runs no paid evals, and free-only tools), and the fixtures in `FREE_FIXTURES` (`test/helpers/free-fixtures-data.ts`, each fixture with its free consumers). -`PAID_WORKFLOW_FILES` (`evals.yml`, `evals-periodic.yml`, `evals-marathon.yml`) -and any other unknown file restore the full gate (`prCoverage.mode = -full-fallback`). The job summary and PR comment list each file that caused a -fallback with one of three labels: +Unmapped files under `DERIVABLE_PREFIXES` (bin/, lib/, scripts/, browse/, +design/, make-pdf/, hosts/, extension/, model-overlays/, agents/, test/helpers/, +capy/) are not unknown: `scripts/pr-dependencies.ts` derives the cases whose +reference closure reaches them (`prCoverage.mode = dependents`), or reports that +no paid case reaches them. The derivation scans the imports of every code file, +including `#!`-shebang `.ts` files and extensionless Bun or Node scripts; if any +import scan fails, every unmapped file restores the full gate (label **import +scan failed**). `PR_FULL_GATE_FILES`, `PAID_WORKFLOW_FILES` (`evals.yml`, +`evals-periodic.yml`, `evals-marathon.yml`) and any other unknown file restore +the full gate (`prCoverage.mode = full-fallback`). The job summary and PR comment +list each unmapped file that caused a fallback with one of these labels: - **needs touchfile entry**: a paid test, helper or fixture no case maps; add it to the owning case in `E2E_TOUCHFILES` (`test/helpers/touchfiles-data.ts`). - **add to FREE_ONLY_PR_FILES**: a file only free lanes read; add it to the list in `scripts/test-pr-profile.ts` (or a fixture to `FREE_FIXTURES`). -- **real unknown dependency**: shared code (bin/, lib/, resolvers, hosts, setup) - whose dependents are not mapped; the full gate is the correct result. +- **full-gate input**: a `PR_FULL_GATE_FILES` entry every case consumes; the + full gate is intended and no mapping narrows it. +- **deleted but still referenced** and **import scan failed**: fix the named + reference or parse error. +- **real unknown dependency**: a file outside the derivable directories that no + list classifies (for example `ETHOS.md`); map it, exempt it, or derive its + directory. + +The fallback rate is a label, not the bill: a dependents run whose derived +closure reaches every gate case costs about what a full-fallback run does. +`bun run test:health` leads with the cost per merged PR and reports +`pr-cases-per-run` (mean selected gate cases and judges per PR run, per mode) +and `pr-known-cost-per-run` (mean known trial cost per PR run, per mode) beside +`pr-fallback-rate`. + +`package.json` (beyond a version-only change) and `bun.lock` stay full-gate +inputs. Over the 225 PR runs from 2026-10-07 to 2026-10-17, no run fell back on +them alone, dependabot PRs never reach the paid lane (`evals.yml` skips +`github.actor == 'dependabot[bot]'`), and every runtime package's importer +(`test/helpers/agent-sdk-runner.ts`, `test/helpers/llm-judge.ts`, `browse/src/*`, +`make-pdf/src/*`) reaches every gate case or is in the global set, so narrowing +them would select the same cases. `test/free-fixtures.test.ts` holds the inventory: each `test/fixtures/**` path belongs to a paid touchfile or `FREE_FIXTURES`, and each workflow to the paid or diff --git a/package.json b/package.json index 5bc904faf..5bb38fc1d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.128", + "version": "1.91.129", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/scripts/harness-version.json b/scripts/harness-version.json index ee40de983..f37b3d4c3 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -16,7 +16,7 @@ "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":5}, "scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":5}, "scripts/test-paid-shards.ts": {"blob":"8ddee5f11ed7856876135520dd8025ab989e2db9","since":5}, - "scripts/test-pr-profile.ts": {"blob":"67d3e163dd443bfc5e381586e28f7e83ed59125a","since":5,"nonBehavioral":"Classifies the new free-durations-record.yml workflow as free-lane only; no paid case reads it, so no verdict can change"}, + "scripts/test-pr-profile.ts": {"blob":"64cd7acfc0f2a4f53472bbdc8a50216984230c5a","since":5,"nonBehavioral":"PR selection only (A1 merged with #3253's free-only entry): shebang and per-extension import scanning, fail-closed scan errors, free-only repository-config files, capy/ derivable, the full-gate fallback label; it changes which cases a PR runs, never how any case is judged"}, "scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":5}, "test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":5}, "test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":5}, diff --git a/scripts/pr-dependencies.ts b/scripts/pr-dependencies.ts index fbf1ef43d..a6dc94e3a 100644 --- a/scripts/pr-dependencies.ts +++ b/scripts/pr-dependencies.ts @@ -49,6 +49,12 @@ export interface DerivedDependencies { tracked: Set; /** The reference chain from an E2E case's declared inputs to `file` (for fix text and audits), else null. */ trace(caseId: string, file: string): string[] | null; + /** + * Files whose import scan failed for a reason other than a leading `#!` + * line (their imports are unknown). PR selection restores the full gate + * while this is non-empty; it fills as the derivation reads files. + */ + scanErrors: Map; /** * Tracked files whose reference text still names `missing`, a path absent * from the tree (deleted in the diff): its path or extensionless import @@ -60,8 +66,10 @@ export interface DerivedDependencies { } const CODE = /\.(?:[cm]?[jt]s|tsx|jsx)$/; +/** An extensionless script Bun or Node runs: its imports are code, not shell. */ +const CODE_SHEBANG = /^#![^\n]*\b(?:bun|node)\b/; const TEXT = /\.(?:[cm]?[jt]s|tsx|jsx|sh|md|tmpl|json|ya?ml|toml|txt|py)$/; -const REPO_PATH = /(?:^|[^\w.@-])((?:bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents|test\/helpers|test\/fixtures)\/[\w.@-]+(?:\/[\w.@-]+)*)/g; +const REPO_PATH = /(?:^|[^\w.@-])((?:bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents|capy|test\/helpers|test\/fixtures)\/[\w.@-]+(?:\/[\w.@-]+)*)/g; /** `path.join(ROOT, 'lib', 'x.ts')`: a top-level directory and its entry as separate string literals. */ const JOINED_PATH = /['"`](bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents)['"`]\s*,\s*['"`]([\w.@-]+)['"`]/g; const BIN_NAME = /\bgstack-[a-z0-9][a-z0-9-]*[a-z0-9]\b/g; @@ -85,7 +93,26 @@ export const SELECTION_DATA_MODULES: readonly string[] = [ ]; const DEFAULT_JUDGE_OWNER = 'test/skill-llm-eval.test.ts'; -const transpiler = new Bun.Transpiler({ loader: 'tsx' }); +const transpilers = { + ts: new Bun.Transpiler({ loader: 'ts' }), tsx: new Bun.Transpiler({ loader: 'tsx' }), + js: new Bun.Transpiler({ loader: 'js' }), jsx: new Bun.Transpiler({ loader: 'jsx' }), +}; + +/** Code by extension, or an extensionless file whose shebang runs Bun or Node. */ +export function isCodeFile(file: string, text: string): boolean { + return CODE.test(file) || (!/\.[^/]+$/.test(path.posix.basename(file)) && CODE_SHEBANG.test(text)); +} + +/** + * Relative import specifiers of a code file. A leading `#!` line is blanked + * first (`scanImports` rejects it), and each extension gets its own loader + * (the TSX loader misreads `.ts` generics and casts). Any other parse error + * throws, so callers fail closed. + */ +export function scanCodeImports(file: string, text: string): string[] { + const loader = /\.tsx$/.test(file) ? 'tsx' : /\.jsx$/.test(file) ? 'jsx' : /\.[cm]?js$/.test(file) ? 'js' : 'ts'; + return transpilers[loader].scanImports(text.replace(/^#![^\n]*/, '')).map(item => item.path); +} /** * Text a reference scan reads: whole-line comments are prose about other @@ -168,15 +195,17 @@ export function deriveDependencies(maps: DependencyMaps, root: string): DerivedD }; const dirs = new Set(tracked.flatMap(file => file.split('/').slice(0, -1).map((_, i, parts) => parts.slice(0, i + 1).join('/')))); + const scanErrors = new Map(); + const importsOf = (file: string): string[] => { + try { return scanCodeImports(file, read(file)); } catch (error) { scanErrors.set(file, String(error)); return []; } + }; const edges = new Map(); const refs = (file: string): string[] => { if (edges.has(file)) return edges.get(file)!; const raw = read(file); const text = (SELECTION_DATA_MODULES as readonly string[]).includes(file) ? '' : referenceText(file, raw); const out = new Set(); - if (CODE.test(file)) { - try { for (const item of transpiler.scanImports(raw)) { const hit = resolveSpecifier(root, item.path, file); if (hit) out.add(hit); } } catch { /* literal scan below still applies */ } - } + if (isCodeFile(file, raw)) for (const specifier of importsOf(file)) { const hit = resolveSpecifier(root, specifier, file); if (hit) out.add(hit); } for (const match of text.matchAll(REPO_PATH)) { const ref = match[1]!.replace(/[.]+$/, ''); if (trackedSet.has(ref)) out.add(ref); @@ -238,19 +267,17 @@ export function deriveDependencies(maps: DependencyMaps, root: string): DerivedD const file = pending.pop()!; if (global.has(file)) continue; global.add(file); - if (!CODE.test(file)) continue; - try { for (const item of transpiler.scanImports(read(file))) { const hit = resolveSpecifier(root, item.path, file); if (hit && !global.has(hit)) pending.push(hit); } } catch { /* unparseable: its own entry stays global */ } + if (!isCodeFile(file, read(file))) continue; + for (const specifier of importsOf(file)) { const hit = resolveSpecifier(root, specifier, file); if (hit && !global.has(hit)) pending.push(hit); } } const importTargets = new Map(); const targetsOf = (file: string): string[] => { if (!importTargets.has(file)) { const targets: string[] = []; - if (CODE.test(file) && !(SELECTION_DATA_MODULES as readonly string[]).includes(file)) { - try { - for (const item of transpiler.scanImports(read(file))) { - if (item.path.startsWith('.')) targets.push(path.posix.normalize(path.posix.join(path.posix.dirname(file), item.path))); - } - } catch { /* literal mentions below still apply */ } + if (isCodeFile(file, read(file)) && !(SELECTION_DATA_MODULES as readonly string[]).includes(file)) { + for (const specifier of importsOf(file)) { + if (specifier.startsWith('.')) targets.push(path.posix.normalize(path.posix.join(path.posix.dirname(file), specifier))); + } } importTargets.set(file, targets); } @@ -287,7 +314,7 @@ export function deriveDependencies(maps: DependencyMaps, root: string): DerivedD while (parents.has(chain[0]!)) chain.unshift(parents.get(chain[0]!)!); return chain; }; - return { e2e, judges: consumers(maps.judgeTouchfiles, DEFAULT_JUDGE_OWNER), everyCase, global, tracked: trackedSet, trace, referencers }; + return { e2e, judges: consumers(maps.judgeTouchfiles, DEFAULT_JUDGE_OWNER), everyCase, global, tracked: trackedSet, trace, referencers, scanErrors }; } const cache = new Map(); diff --git a/scripts/replay-pr-selection.ts b/scripts/replay-pr-selection.ts index 71e79d5de..3e6ed6903 100644 --- a/scripts/replay-pr-selection.ts +++ b/scripts/replay-pr-selection.ts @@ -26,7 +26,7 @@ import { packageChangeOnlyVersion } from './test-pr-profile'; import { E2E_TIERS, E2E_TOUCHFILES, GLOBAL_TOUCHFILES } from '../test/helpers/touchfiles-data'; import { matchGlob } from '../test/helpers/test-selection'; import { derivedDependencies } from './pr-dependencies'; -import { PR_PROFILE_MAPS } from './test-pr-profile'; +import { knownNonBehaviorFile, PR_PROFILE_MAPS } from './test-pr-profile'; export interface ReplayInput { runId: number; @@ -59,7 +59,9 @@ export interface ReplayRow { /** * Whether a case depends on any changed file under today's dependency graph: * its touchfiles, the globals, and the derived reference closure - * (scripts/pr-dependencies.ts, its global runner closure included). + * (scripts/pr-dependencies.ts, its global runner closure included). Like + * the selector, a touchfile match wins over the audited free-only exemptions, + * and an exempt file reaches no case through the derived closure. */ export function caseTouchesDiff(id: string, changedFiles: readonly string[]): boolean { const patterns = [...(E2E_TOUCHFILES[id] ?? []), ...GLOBAL_TOUCHFILES]; @@ -67,7 +69,7 @@ export function caseTouchesDiff(id: string, changedFiles: readonly string[]): bo // A path absent from today's tree reaches a case through the live files that still reference it. const via = (file: string) => derived.tracked.has(file) ? [file] : derived.referencers(file); return changedFiles.some(file => patterns.some(pattern => matchGlob(file, pattern)) - || via(file).some(source => derived.global.has(source) || !!derived.e2e.get(source)?.has(id))); + || (!knownNonBehaviorFile(file) && via(file).some(source => derived.global.has(source) || !!derived.e2e.get(source)?.has(id)))); } export function replayRun(input: ReplayInput, oldSelection: readonly string[]): ReplayRow { diff --git a/scripts/test-health-report.ts b/scripts/test-health-report.ts index e4dc5da46..cf11e4f3e 100644 --- a/scripts/test-health-report.ts +++ b/scripts/test-health-report.ts @@ -33,7 +33,7 @@ export const COMPLEXITY_PATHS = ['scripts', 'test/helpers', '.github/workflows'] export interface Metric { id: string; title: string; rule: string; value?: string; detail?: string[]; unavailable?: string } export interface Check { id: string; title: string; status: 'pass' | 'fail' | 'unavailable'; detail: string; fix: string } export type IssueAction = 'upsert' | 'keep-open' | 'close'; -export interface HealthReport { repo: string; sinceDays: number; generatedAt: string; metrics: Metric[]; checks: Check[]; issueAction: IssueAction } +export interface HealthReport { repo: string; sinceDays: number; generatedAt: string; headline?: string; metrics: Metric[]; checks: Check[]; issueAction: IssueAction } export interface HealthInputs { client: GhClient; @@ -188,14 +188,17 @@ export function collectHealth(inputs: HealthInputs): HealthReport { const need = (value: T | Error): T => { if (value instanceof Error) throw value; return value; }; const prRecords = attempt(() => need(prRuns).filter(run => run.conclusion !== 'cancelled') .map(run => ({ run, trials: records(artifactDirs(run, name => name.startsWith('trial-outcomes'))) }))); + const prManifests = attempt(() => need(prRuns).flatMap(run => { + const file = artifactDirs(run, name => name === 'paid-plan').map(dir => path.join(dir, 'manifest.json')).find(candidate => fs.existsSync(candidate)); + return file ? [{ run, manifest: readJson(file) }] : []; + })); const metrics: Metric[] = []; + let costPerMergedPr: string | undefined; - metrics.push(metric('pr-fallback-rate', 'PR full-fallback rate', 'share of evals.yml pull_request runs whose paid-plan manifest has prCoverage.mode = full-fallback', () => { + metrics.push(metric('pr-fallback-rate', 'PR full-fallback rate', 'share of evals.yml pull_request runs whose paid-plan manifest has prCoverage.mode = full-fallback (a label, not spend: dependents runs select about as much)', () => { const modes = new Map(); - for (const run of need(prRuns)) { - const manifest = artifactDirs(run, name => name === 'paid-plan').flatMap(dir => [path.join(dir, 'manifest.json')]).find(file => fs.existsSync(file)); - if (!manifest) continue; - const mode = String(readJson(manifest).prCoverage?.mode ?? 'none'); + for (const { manifest } of need(prManifests)) { + const mode = String(manifest.prCoverage?.mode ?? 'none'); modes.set(mode, (modes.get(mode) ?? 0) + 1); } const total = [...modes.values()].reduce((a, b) => a + b, 0); @@ -204,6 +207,27 @@ export function collectHealth(inputs: HealthInputs): HealthReport { detail: [...modes].map(([mode, count]) => `${mode}: ${count}`) }; })); + metrics.push(metric('pr-cases-per-run', 'Selected gate cases and judges per PR run', 'mean length of prCoverage.e2e (gate cases) and prCoverage.judges in the paid-plan manifests of evals.yml pull_request runs, overall and per mode', () => { + const planned = need(prManifests).filter(({ manifest }) => Array.isArray(manifest.prCoverage?.e2e) && Array.isArray(manifest.prCoverage?.judges)); + if (planned.length === 0) throw new Error('no paid-plan manifest with prCoverage.e2e and prCoverage.judges in the window'); + const mean = (entries: typeof planned) => `${(entries.reduce((sum, { manifest }) => sum + manifest.prCoverage.e2e.length, 0) / entries.length).toFixed(1)} gate cases, ${(entries.reduce((sum, { manifest }) => sum + manifest.prCoverage.judges.length, 0) / entries.length).toFixed(1)} judges`; + const modes = [...new Set(planned.map(({ manifest }) => String(manifest.prCoverage.mode)))].sort(); + return { value: `${mean(planned)} per run (${planned.length} runs)`, + detail: modes.map(mode => { const entries = planned.filter(({ manifest }) => String(manifest.prCoverage.mode) === mode); return `${mode}: ${mean(entries)} (${entries.length} runs)`; }) }; + })); + + metrics.push(metric('pr-known-cost-per-run', 'Known cost per PR run', 'mean over non-cancelled evals.yml pull_request runs of executed trial cost_usd with a known cost (a run with no trial records counts $0), overall and per manifest mode; unknown-cost trials (PTY) are counted, not priced', () => { + const entries = need(prRecords); + if (entries.length === 0) throw new Error('no non-cancelled PR runs in the window'); + const known = (trials: TrialRecord[]) => trials.filter(record => record.execution === 'executed' && record.cost_known !== false).reduce((sum, record) => sum + record.cost_usd, 0); + const unknown = entries.flatMap(entry => entry.trials).filter(record => record.execution === 'executed' && record.cost_known === false).length; + const modeOf = new Map((prManifests instanceof Error ? [] : prManifests).map(({ run, manifest }) => [run.id, String(manifest.prCoverage?.mode ?? 'none')])); + const mean = (list: typeof entries) => `$${(list.reduce((sum, entry) => sum + known(entry.trials), 0) / list.length).toFixed(2)}`; + const modes = [...new Set(entries.map(entry => modeOf.get(entry.run.id) ?? 'no manifest'))].sort(); + return { value: `${mean(entries)} per run (${entries.length} runs; ${unknown} executed trial(s) with unknown cost)`, + detail: modes.map(mode => { const list = entries.filter(entry => (modeOf.get(entry.run.id) ?? 'no manifest') === mode); return `${mode}: ${mean(list)} (${list.length} runs)`; }) }; + })); + metrics.push(metric('pr-reused-trials', 'Reused PR trial records', 'trial-outcomes records with execution = reused across non-cancelled PR runs', () => { const all = need(prRecords).flatMap(entry => entry.trials); const reused = all.filter(record => record.execution === 'reused').length; @@ -216,6 +240,7 @@ export function collectHealth(inputs: HealthInputs): HealthReport { .reduce((sum, record) => sum + record.cost_usd, 0); const revisions = new Set(entries.filter(entry => entry.run.conclusion === 'success' || entry.run.conclusion === 'failure').map(entry => entry.run.sha)).size; const merged = listMergedPulls(client, repo, since); + if (merged) costPerMergedPr = `$${(spend / merged).toFixed(2)} (${merged} merged PRs, $${spend.toFixed(2)} recorded)`; return { value: `$${spend.toFixed(2)} recorded`, detail: [ `per merged PR: ${merged ? `$${(spend / merged).toFixed(2)} (${merged} merged)` : 'n/a (0 merged)'}`, `per validated revision: ${revisions ? `$${(spend / revisions).toFixed(2)} (${revisions} revisions)` : 'n/a'}`] }; @@ -267,7 +292,8 @@ export function collectHealth(inputs: HealthInputs): HealthReport { })); const checks = [flakeCheck(flakes), unseededCheck(root)]; - return { repo, sinceDays, generatedAt: new Date(now).toISOString(), metrics, checks, issueAction: issueActionFor(checks) }; + const headline = `Cost per merged PR: ${costPerMergedPr ?? `unavailable (${metrics.find(entry => entry.id === 'pr-spend')?.unavailable ?? '0 merged PRs'})`}`; + return { repo, sinceDays, generatedAt: new Date(now).toISOString(), headline, metrics, checks, issueAction: issueActionFor(checks) }; } function listMergedPulls(client: GhClient, repo: string, since: string): number { @@ -340,7 +366,7 @@ function censusMetrics(inputs: HealthInputs, artifactDirs: (run: CiRun, match: ( // --- Rendering --- export function formatHealth(report: HealthReport): string { - const lines = [`test:health ${report.repo}, last ${report.sinceDays} day(s), ${report.generatedAt}`]; + const lines = [`test:health ${report.repo}, last ${report.sinceDays} day(s), ${report.generatedAt}`, ...(report.headline ? [report.headline] : [])]; for (const entry of report.metrics) { lines.push(`- ${entry.title}: ${entry.unavailable ? `unavailable: ${entry.unavailable}` : entry.value}`); lines.push(` rule: ${entry.rule}`); diff --git a/scripts/test-pr-profile.ts b/scripts/test-pr-profile.ts index 67d3e163d..64cd7acfc 100644 --- a/scripts/test-pr-profile.ts +++ b/scripts/test-pr-profile.ts @@ -216,6 +216,12 @@ export const FREE_ONLY_PR_FILES = [ '.gstack/generated.json', // gstack's own regen registry: gstack-regen/gstack-pregate read it from origin/ of the repo they run in; paid cases run in fixture repos. '.gstack/pregate.json', // gstack's own pre-gate policy: same trust boundary, same reason. '.gstack/ship-policy.json', // gstack's own ship policy: gstack-ship-policy/gstack-restamp read it from origin/ of the repo they run in; same reason. + '.gstack/ship-policy.example.json', // The example beside it: only .gitignore and docs/ship-policy.md name it. + '.github/dependabot.yml', // Dependabot's own config: read by GitHub, and by free test/compare-pr-version.test.ts; dependabot runs skip the paid lane. + '.osv-scanner.toml', // Read by .github/workflows/osv-scanner.yml and the free OSV tests (test/osv-*.test.ts). + 'setup.ps1', // The Windows installer: run only by .github/workflows/windows-setup-e2e.yml; the paid lane is Linux. + '.gitlab-ci.yml', // gstack's own GitLab config: skills probe the fixture repo's .gitlab-ci.yml, never this one. + '.github/ISSUE_TEMPLATE/**', // Issue templates: read by GitHub and free test/gstack-doctor.test.ts. 'scripts/retired-command.ts', // One-release stubs for retired package scripts; no paid case imports it. 'careful/bin/check-careful.sh', // /careful PreToolUse hook: covered by test/hook-scripts.test.ts; no paid fixture copies or runs it. // Reporting and launch tools: they read CI history or start a lane, never run inside a paid case. @@ -274,9 +280,9 @@ export const PR_FULL_GATE_FILES = [ export const PERIODIC_ONLY_PR_FILES = ['.github/workflows/evals-periodic.yml', '.github/workflows/evals-marathon.yml'] as const; /** Directories the reference derivation covers: an unmapped file there with no consumer is consumed by no paid case. */ -export const DERIVABLE_PREFIXES = ['bin/', 'lib/', 'scripts/', 'browse/', 'design/', 'make-pdf/', 'hosts/', 'extension/', 'model-overlays/', 'agents/', 'test/helpers/'] as const; +export const DERIVABLE_PREFIXES = ['bin/', 'lib/', 'scripts/', 'browse/', 'design/', 'make-pdf/', 'hosts/', 'extension/', 'model-overlays/', 'agents/', 'test/helpers/', 'capy/'] as const; -function knownNonBehaviorFile(file: string): boolean { +export function knownNonBehaviorFile(file: string): boolean { // A mapped dependency still wins over these exemptions. New helper, // runtime, dependency, or paid-workflow files are deliberately not exempted. return /^(?:docs\/|(?:README|CONTRIBUTING|ARCHITECTURE|CHANGELOG|TODOS)\.md$|VERSION$)/.test(file) @@ -294,6 +300,9 @@ export const FALLBACK_FIX_ANCHOR = 'docs/TESTING_INTERNALS.md#pr-paid-lane-fallb /** Why one changed file restored the full gate, and the edit that narrows it next time (DX-6). */ export function unknownFileLabel(file: string): { label: string; fix: string } { + if (matches(file, PR_FULL_GATE_FILES)) { + return { label: 'full-gate input', fix: `${file} is in PR_FULL_GATE_FILES (scripts/test-pr-profile.ts): every paid case consumes it, so the full gate is intended and no mapping narrows it` }; + } if (isPaidTestFile(file) || file.startsWith('test/helpers/')) { return { label: 'needs touchfile entry', fix: `register ${file} under the cases that consume it in test/helpers/touchfiles-data.ts` }; } @@ -381,6 +390,12 @@ export function selectPrProfile(options: { if (found.e2e.size + found.judges.size > 0) consumers.set(file, found); else noConsumerFiles.push(file); } + // A failed import scan leaves some closure unknown: every unmapped file restores the full gate (fail closed). + const scanFailed = derived && unmapped.length ? [...derived.scanErrors.keys()].sort() : []; + if (scanFailed.length) { + consumers.clear(); derivedGlobal.length = 0; noConsumerFiles.length = 0; + for (const file of unmapped) deletedLabels.set(file, { label: 'import scan failed', fix: `scripts/pr-dependencies.ts could not scan the imports of ${scanFailed.slice(0, 3).join(', ')}${scanFailed.length > 3 ? ` and ${scanFailed.length - 3} more` : ''}; fix the parse error so ${file}'s consumers can be derived` }); + } const derivedFiles = [...consumers.keys()]; const unknownFiles = unmapped.filter(file => !derivedGlobal.includes(file) && !consumers.has(file) && !noConsumerFiles.includes(file)); const sharedInputs = files.filter(file => depends(file, PR_FULL_GATE_FILES)); diff --git a/test/free-fixtures.test.ts b/test/free-fixtures.test.ts index c9cb1d933..4feeb8c37 100644 --- a/test/free-fixtures.test.ts +++ b/test/free-fixtures.test.ts @@ -85,7 +85,7 @@ describe('workflow inventory (CEO-20)', () => { test('the inventory rule also reads .yaml files', () => { expect(unknownFileLabel('.github/workflows/new-check.yaml').label).toBe('add to FREE_ONLY_PR_FILES'); - expect(unknownFileLabel('.github/workflows/evals.yml').label).toBe('real unknown dependency'); + expect(unknownFileLabel('.github/workflows/evals.yml').label).toBe('full-gate input'); }); }); @@ -111,7 +111,7 @@ describe('PR selection over the W1a single-file fallbacks', () => { const result = select(['.github/workflows/evals.yml']); expect(result.mode).toBe('full-fallback'); expect(result.unknownFileLabels).toEqual([{ file: '.github/workflows/evals.yml', ...unknownFileLabel('.github/workflows/evals.yml') }]); - expect(result.reasons[0]).toContain('.github/workflows/evals.yml (real unknown dependency)'); + expect(result.reasons[0]).toContain('.github/workflows/evals.yml (full-gate input)'); }); test('a mapped free fixture is exempt; a paid-registered fixture selects its cases; an unmapped one falls back', () => { diff --git a/test/paid-free-boundary.test.ts b/test/paid-free-boundary.test.ts index 402ac6917..dd4ea3eb9 100644 --- a/test/paid-free-boundary.test.ts +++ b/test/paid-free-boundary.test.ts @@ -114,7 +114,7 @@ describe('paid/free dependency boundary', () => { expect(result.selection).toEqual({ e2e: [], judges: [] }); } // Tracked files outside every mapping and outside the derivable directories restore the full gate. - for (const file of ['ETHOS.md', 'SECURITY.md', 'conductor.json', '.osv-scanner.toml']) { + for (const file of ['ETHOS.md', 'SECURITY.md', 'conductor.json', 'NOTICE.md']) { const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: [FREE_ONLY_PR_FILES[0], file] }); expect(result.coverage?.mode, file).toBe('full-fallback'); expect(result.coverage?.unknownFiles).toContain(file); diff --git a/test/pr-dependencies.test.ts b/test/pr-dependencies.test.ts index 2470405e9..a3ddcf73d 100644 --- a/test/pr-dependencies.test.ts +++ b/test/pr-dependencies.test.ts @@ -96,6 +96,93 @@ describe('reference derivation on a fixture tree', () => { }); }); +describe('shebang files and import scan failures (A1-1)', () => { + const tree = (extra: Record = {}) => fixtureRepo({ + 'test/skill-e2e-alpha.test.ts': "import '../lib/entry';\n", 'test/skill-e2e-beta.test.ts': '', + 'lib/entry.ts': '#!/usr/bin/env bun\nimport { y } from \'./typed\';\nexport const x = y;\n', + 'lib/typed.ts': 'export const y = (v: T) => v as unknown as T;\nimport \'./typed-dep\';\n', 'lib/typed-dep.ts': '', + 'alpha/SKILL.md.tmpl': 'Run `gstack-sb`.\n', 'bin/gstack-sb': '#!/usr/bin/env -S bun run\nimport { z } from \'../lib/sb-dep\';\n', 'lib/sb-dep.ts': '', + 'bin/gstack-sh': '#!/bin/sh\nimport ../lib/sh-only.ts\n', 'lib/sh-only.ts': '', + 'scripts/build.sh': '', 'scripts/resolvers/index.ts': '', 'package.json': '{}', ...extra, + }); + const maps = { e2eTouchfiles: { alpha: ['test/skill-e2e-alpha.test.ts', 'alpha/**'], beta: ['test/skill-e2e-beta.test.ts'] }, judgeTouchfiles: {}, globalTouchfiles: [] as string[] }; + const profileMaps = { ...maps, tiers: { alpha: 'gate', beta: 'gate' } as const }; + const pick = (derived: ReturnType, changedFiles: string[]) => + selectPrProfile({ maps: profileMaps, profile: [], selectedE2E: [], selectedJudges: [], changedFiles, derived }); + + test('a shebang .ts file, a bun-shebang extensionless bin and a .ts generic arrow are all import-scanned', () => { + const derived = deriveDependencies(maps, tree()); + expect([...derived.e2e.get('lib/typed.ts') ?? []]).toEqual(['alpha']); + expect([...derived.e2e.get('lib/typed-dep.ts') ?? []]).toEqual(['alpha']); + expect([...derived.e2e.get('lib/sb-dep.ts') ?? []]).toEqual(['alpha']); + expect(derived.e2e.get('lib/sh-only.ts')).toBeUndefined(); + expect(derived.scanErrors.size).toBe(0); + expect(pick(derived, ['lib/sb-dep.ts'])).toMatchObject({ mode: 'dependents', e2e: ['alpha'] }); + }); + + test('lib/fs-utils.ts selects the cases of bin/gstack-redact-prepush, which imports it', () => { + const bin = gate.filter(id => real.e2e.get('bin/gstack-redact-prepush')?.has(id)); + expect(bin.length).toBeGreaterThan(0); + const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: ['lib/fs-utils.ts'], packageVersionOnly: true }); + expect(result.coverage?.mode).toBe('dependents'); + for (const id of bin) expect(result.selection.e2e, id).toContain(id); + expect(real.scanErrors.size).toBe(0); + expect(real.global.has('test/helpers/bootstrap-retention.ts')).toBe(true); + }); + + test('any other scan error fails closed at every site: case closures, the global walk and deleted-file referencers', () => { + const broken = "import { from './nowhere';\n"; + const inClosure = deriveDependencies(maps, tree({ 'lib/typed-dep.ts': broken })); + expect([...inClosure.scanErrors.keys()]).toEqual(['lib/typed-dep.ts']); + const closed = pick(inClosure, ['lib/sb-dep.ts']); + expect(closed).toMatchObject({ mode: 'full-fallback', e2e: ['alpha', 'beta'], unknownFiles: ['lib/sb-dep.ts'] }); + expect(closed.unknownFileLabels[0]).toMatchObject({ label: 'import scan failed' }); + expect(closed.unknownFileLabels[0]!.fix).toContain('lib/typed-dep.ts'); + + const inGlobal = deriveDependencies({ ...maps, globalTouchfiles: ['lib/runner.ts'] }, tree({ 'lib/runner.ts': broken })); + expect([...inGlobal.scanErrors.keys()]).toEqual(['lib/runner.ts']); + expect(pick(inGlobal, ['lib/sb-dep.ts']).mode).toBe('full-fallback'); + + const unreached = deriveDependencies(maps, tree({ 'lib/orphan.ts': broken })); + expect(unreached.scanErrors.size).toBe(0); + expect(pick(unreached, ['lib/gone.ts'])).toMatchObject({ mode: 'full-fallback', unknownFiles: ['lib/gone.ts'] }); + expect([...unreached.scanErrors.keys()]).toEqual(['lib/orphan.ts']); + }); +}); + +describe('safe classification (A1-2)', () => { + test('capy/support.env derives to the bins that read it instead of restoring the full gate', () => { + for (const bin of ['bin/gstack-install-verdict.sh', 'bin/gstack-doctor-check.sh']) { + for (const id of real.e2e.get(bin) ?? []) expect(real.e2e.get('capy/support.env')?.has(id), `${bin} ${id}`).toBe(true); + } + const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: ['capy/support.env'], packageVersionOnly: true }); + expect(result.coverage?.unknownFiles).toEqual([]); + expect(result.coverage?.mode).toBe('dependents'); + expect(DERIVABLE_PREFIXES).toContain('capy/'); + }); + + test('repository config read only by free lanes or GitHub stays in the PR profile', () => { + const files = ['.github/dependabot.yml', '.osv-scanner.toml', 'setup.ps1', '.gitlab-ci.yml', '.github/ISSUE_TEMPLATE/bug_report.md', '.gstack/ship-policy.example.json']; + for (const file of files) expect(real.tracked.has(file), file).toBe(true); + const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: files, packageVersionOnly: true }); + expect(result.coverage?.mode).toBe('pr'); + expect(result.selection).toEqual({ e2e: [], judges: [] }); + }); + + test('a full-gate file is never labelled a real unknown dependency', () => { + for (const file of ['.github/workflows/evals.yml', 'test-setup.ts', '.gitattributes']) { + const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: [file], packageVersionOnly: true }); + expect(result.coverage?.mode, file).toBe('full-fallback'); + for (const label of result.coverage!.unknownFileLabels) { + expect(label.label, file).toBe('full-gate input'); + expect(label.fix).toContain('PR_FULL_GATE_FILES'); + } + expect(unknownFileLabel(file).label).toBe('full-gate input'); + } + expect(unknownFileLabel('ETHOS.md').label).toBe('real unknown dependency'); + }); +}); + describe('derived dependencies of this checkout', () => { test('a bin script a skill template invokes selects that skill\'s cases', () => { const consumers = real.e2e.get('bin/gstack-next-version') ?? new Set(); diff --git a/test/test-health-report.test.ts b/test/test-health-report.test.ts index 75970cef3..38a2e31f4 100644 --- a/test/test-health-report.test.ts +++ b/test/test-health-report.test.ts @@ -191,7 +191,7 @@ describe('collectHealth over recorded fixture data', () => { [/runs\/61\/artifacts/, () => ({ artifacts: [] })], [/workflows\/windows-free-tests\.yml\/runs/, () => ({ workflow_runs: windowsRuns })], ], { - 11: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'full-fallback' } }) }, + 11: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'full-fallback', e2e: ['gate-a', 'gate-b'], judges: ['judge-a'] } }) }, 12: { 'trial-outcomes.jsonl': prTrials }, 51: { 'trial-outcomes.jsonl': census }, 61: { 'flake-ledger.jsonl': ledger('test/a.test.ts') }, @@ -205,6 +205,10 @@ describe('collectHealth over recorded fixture data', () => { expect(value('net-complexity').unavailable).toContain('git grep HEAD failed'); expect(value('pr-reused-trials').value).toBe('1 of 2 trial records'); expect(value('pr-spend')).toMatchObject({ value: '$2.50 recorded', detail: ['per merged PR: $2.50 (1 merged)', 'per validated revision: $2.50 (1 revisions)'] }); + expect(value('pr-cases-per-run')).toMatchObject({ value: '2.0 gate cases, 1.0 judges per run (1 runs)', detail: ['full-fallback: 2.0 gate cases, 1.0 judges (1 runs)'] }); + expect(value('pr-known-cost-per-run')).toMatchObject({ value: '$2.50 per run (1 runs; 0 executed trial(s) with unknown cost)', detail: ['full-fallback: $2.50 (1 runs)'] }); + expect(report.headline).toBe('Cost per merged PR: $2.50 (1 merged PRs, $2.50 recorded)'); + expect(formatHealth(report).split('\n')[1]).toBe('Cost per merged PR: $2.50 (1 merged PRs, $2.50 recorded)'); expect(value('cancelled-slice-minutes').value).toBe('30 min/week (1 cancelled run(s))'); expect(value('push-to-verdict').value).toBe('p50 10.0 min, p95 10.0 min (1 runs)'); expect(value('census-periodic').value).toBe('1.00 expected failed trials/run (1/3 over 1 run(s)); P(red) 33.3% in force, 33.3% single-trial'); @@ -223,6 +227,26 @@ describe('collectHealth over recorded fixture data', () => { expect(formatHealth(report)).toContain('rule: share of evals.yml pull_request runs'); }); + test('known cost per PR run prices only known-cost executed trials and counts the unknown ones', () => { + const trials = formatTrialOutcomes([trialRecord('gate-a', 'passed', { tier: 'gate', lane: 'gate/pr', cost_usd: 4 }), + trialRecord('gate-pty', 'passed', { tier: 'gate', lane: 'gate/pr', cost_usd: 0, cost_known: false })]); + const client = fakeClient([ + [/workflows\/evals\.yml\/runs/, () => ({ workflow_runs: [run(1, { event: 'pull_request', head_branch: 'a' }), run(2, { event: 'pull_request', head_branch: 'b' })] })], + [/runs\/1\/artifacts/, () => ({ artifacts: [{ id: 11, name: 'paid-plan', size_in_bytes: 10 }, { id: 12, name: 'trial-outcomes-pr-a1', size_in_bytes: 10 }] })], + [/runs\/2\/artifacts/, () => ({ artifacts: [{ id: 21, name: 'paid-plan', size_in_bytes: 10 }] })], + [/pulls\?state=closed/, () => []], + ], { + 11: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'dependents', e2e: ['gate-a', 'gate-pty', 'gate-c'], judges: ['j'] } }) }, + 12: { 'trial-outcomes.jsonl': trials }, + 21: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'pr', e2e: [], judges: [] } }) }, + }); + const report = collectHealth({ client, repo: 'o/r', now: NOW, sinceDays: 7, cacheDir: tmp('health-cache-'), root: freeRoot(3, 0) }); + const value = (id: string) => report.metrics.find(entry => entry.id === id)!; + expect(value('pr-known-cost-per-run')).toMatchObject({ value: '$2.00 per run (2 runs; 1 executed trial(s) with unknown cost)', detail: ['dependents: $4.00 (1 runs)', 'pr: $0.00 (1 runs)'] }); + expect(value('pr-cases-per-run')).toMatchObject({ value: '1.5 gate cases, 0.5 judges per run (2 runs)', detail: ['dependents: 3.0 gate cases, 1.0 judges (1 runs)', 'pr: 0.0 gate cases, 0.0 judges (1 runs)'] }); + expect(report.headline).toBe('Cost per merged PR: unavailable (0 merged PRs)'); + }); + test('the Windows wall counts only successful pull_request runs and says so when there are none', () => { const wall = (windowsRuns: unknown[]) => collectHealth({ client: fixture(windowsRuns), repo: 'o/r', now: NOW, sinceDays: 7, cacheDir: tmp('health-cache-'), root: freeRoot(3, 0) }) .metrics.find(entry => entry.id === 'free-wall')!;