mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-05 00:03:15 +01:00
Merge remote-tracking branch 'origin/main' into capy/fw3-a3
This commit is contained in:
commit
4f2e9ed830
36 files changed
+4402
-71
No files matched your search
@@ -723,6 +723,34 @@ jobs:
|
||||
if-no-files-found: ignore
|
||||
retention-days: 90
|
||||
|
||||
# Exercised-path map (shadow, non-blocking; docs/TESTING_INTERNALS.md#exercised-path-map):
|
||||
# union this census's own recorded commands into the map, report MAP_GAP lines and
|
||||
# cases that recorded no commands, and print the refresh command. Never fails the run.
|
||||
- name: Refresh the exercised-path map from this census
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
run: |
|
||||
mkdir -p /tmp/exercised-map
|
||||
dirs=()
|
||||
for dir in /tmp/gate-census-report /tmp/paid-report; do [ -d "$dir" ] && dirs+=(--from-dir "$dir"); done
|
||||
status=0
|
||||
bun --no-install run scripts/exercised-map.ts "${dirs[@]}" \
|
||||
--run-id "$GITHUB_RUN_ID" --out /tmp/exercised-map/exercised-commands.json --report /tmp/exercised-map/report.md || status=$?
|
||||
if [ "$status" -ne 0 ]; then
|
||||
echo "exercised-path map refresh failed with exit $status (non-blocking; no map uploaded)" | tee -a "$GITHUB_STEP_SUMMARY"
|
||||
fi
|
||||
if [ -f /tmp/exercised-map/report.md ]; then cat /tmp/exercised-map/report.md >> "$GITHUB_STEP_SUMMARY"; fi
|
||||
|
||||
- name: Upload the refreshed exercised-path map
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
with:
|
||||
name: exercised-map-a${{ github.run_attempt }}
|
||||
path: /tmp/exercised-map
|
||||
if-no-files-found: ignore
|
||||
retention-days: 90
|
||||
|
||||
# Weekly pass-rate gate over the last 10 weekly runs (drift, rule cases
|
||||
# behaving like behavior, quarantine exit/expiry/cap). Fails closed when
|
||||
# history cannot be fetched.
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
name: Free-suite duration seeds
|
||||
|
||||
# Weekly standing recorder for the two free-suite duration seeds that the
|
||||
# shard packers read: scripts/free-test-durations.json (Linux, recorded on the
|
||||
# same ubicloud-standard-8 runner as the free-suite shards) and
|
||||
# scripts/free-test-durations-windows.json (windows-latest, through the
|
||||
# record_durations path of windows-free-tests.yml). When either recording adds
|
||||
# or drops a test file, it force-updates one bot branch and opens or refreshes
|
||||
# a seed PR. A file that fails while recording turns its job red but keeps its
|
||||
# duration (a red file is still a real cost) and is named in the PR body; an
|
||||
# interrupted recording, or more than 5 red files on either OS, proposes
|
||||
# nothing (docs/TESTING_INTERNALS.md#free-suite-duration-seed).
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 4 * * 1' # Monday 04:00 UTC, before the 10:00 test-health run
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
open_pr:
|
||||
description: Push the bot branch and open or refresh the seed PR (false records and reports only)
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
concurrency:
|
||||
group: free-durations-record
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
linux:
|
||||
runs-on: ubicloud-standard-8
|
||||
timeout-minutes: 45
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Prepare the same inputs as a free-suite shard
|
||||
run: |
|
||||
bash scripts/ubicloud/setup-free-suite.sh
|
||||
echo "$HOME/.bun/bin" >> "$GITHUB_PATH"
|
||||
- name: Time every free file alone
|
||||
id: record
|
||||
env:
|
||||
GSTACK_EXPECT_BINARIES: '1'
|
||||
run: |
|
||||
set -o pipefail
|
||||
xvfb-run -a bun run test:free --record-durations 2>&1 | tee "$RUNNER_TEMP/record.log"
|
||||
- name: Keep the recording only when every file was timed
|
||||
id: seed
|
||||
if: always() && steps.record.outcome != 'skipped'
|
||||
run: |
|
||||
grep -q 'wrote [0-9]* durations to' "$RUNNER_TEMP/record.log" || { echo "::error::The recording was interrupted; no Linux seed to upload."; exit 1; }
|
||||
mkdir -p "$RUNNER_TEMP/seed"
|
||||
cp scripts/free-test-durations.json "$RUNNER_TEMP/seed/free-test-durations.json"
|
||||
sed -n '/failed while recording:/,$p' "$RUNNER_TEMP/record.log" | sed 's/\r$//' | { grep -E '^ ✗ [^ ]+$' || true; } | sed 's/^ ✗ //' > "$RUNNER_TEMP/seed/record-failures.txt"
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
if: always()
|
||||
with:
|
||||
name: linux-record-durations-logs
|
||||
path: .context/free-test-logs/gstack-free-test-*.log
|
||||
include-hidden-files: true
|
||||
if-no-files-found: ignore
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
if: always() && steps.seed.outcome == 'success'
|
||||
with:
|
||||
name: free-test-durations
|
||||
path: ${{ runner.temp }}/seed/
|
||||
if-no-files-found: error
|
||||
|
||||
windows:
|
||||
uses: ./.github/workflows/windows-free-tests.yml
|
||||
with:
|
||||
record_durations: true
|
||||
|
||||
propose:
|
||||
needs: [linux, windows]
|
||||
if: ${{ !cancelled() }}
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
actions: write
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.4.2
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
||||
with:
|
||||
name: free-test-durations
|
||||
path: ${{ runner.temp }}/linux
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
||||
with:
|
||||
name: free-test-durations-windows
|
||||
path: ${{ runner.temp }}/windows
|
||||
- name: Open or refresh the seed PR when either seed's file set changed
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
OPEN_PR: ${{ github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || inputs.open_pr) }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
linux_failed=$(wc -l < "$RUNNER_TEMP/linux/record-failures.txt")
|
||||
windows_failed=$(wc -l < "$RUNNER_TEMP/windows/record-failures.txt")
|
||||
if [ "$linux_failed" -gt 5 ] || [ "$windows_failed" -gt 5 ]; then
|
||||
echo "::error::$linux_failed Linux and $windows_failed Windows files failed while recording; more than 5 on either OS means a broken tree, so no seed is proposed."
|
||||
exit 1
|
||||
fi
|
||||
failures() { if [ -s "$1" ]; then paste -sd, "$1" | sed 's/,/, /g'; else echo none; fi; }
|
||||
drift() { # committed recorded -> "<added> <removed>"
|
||||
jq -n --slurpfile old "$1" --slurpfile new "$2" \
|
||||
'[($new[0].durations|keys) - ($old[0].durations|keys), ($old[0].durations|keys) - ($new[0].durations|keys)] | map(length) | join(" ")'
|
||||
}
|
||||
read -r linux_added linux_removed <<<"$(drift scripts/free-test-durations.json "$RUNNER_TEMP/linux/free-test-durations.json" | tr -d '"')"
|
||||
read -r windows_added windows_removed <<<"$(drift scripts/free-test-durations-windows.json "$RUNNER_TEMP/windows/free-test-durations-windows.json" | tr -d '"')"
|
||||
echo "Linux seed: +$linux_added -$linux_removed files; Windows seed: +$windows_added -$windows_removed files" | tee -a "$GITHUB_STEP_SUMMARY"
|
||||
if [ "$((linux_added + linux_removed + windows_added + windows_removed))" -eq 0 ]; then
|
||||
echo "Both seeds cover the current test files; no PR." | tee -a "$GITHUB_STEP_SUMMARY"
|
||||
exit 0
|
||||
fi
|
||||
cp "$RUNNER_TEMP/linux/free-test-durations.json" scripts/free-test-durations.json
|
||||
cp "$RUNNER_TEMP/windows/free-test-durations-windows.json" scripts/free-test-durations-windows.json
|
||||
if ratchet=$(GSTACK_FREE_SEED_BASE="$(git rev-parse HEAD)" bun test test/free-seed-ratchet.test.ts 2>&1); then
|
||||
ratchet_line="passes"
|
||||
else
|
||||
ratchet_line="FAILS; a maintainer must split the named file or add an annotated allowlist entry on this branch before merge:"$'\n\n'"\`\`\`"$'\n'"$(printf '%s\n' "$ratchet" | sed -n -E '/Free seed ratchet|^- /p' | sed -n '1,20p')"$'\n'"\`\`\`"
|
||||
fi
|
||||
failed_line="Linux: $(failures "$RUNNER_TEMP/linux/record-failures.txt"); Windows: $(failures "$RUNNER_TEMP/windows/record-failures.txt")"
|
||||
echo "Seed growth ratchet: $ratchet_line" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "Failed while recording: $failed_line" | tee -a "$GITHUB_STEP_SUMMARY"
|
||||
if [ "$OPEN_PR" != true ]; then
|
||||
echo "Report only (open_pr=false, or a ref other than main): no branch pushed." | tee -a "$GITHUB_STEP_SUMMARY"
|
||||
exit 0
|
||||
fi
|
||||
branch=bot/free-duration-seeds
|
||||
subject="test(seed): refresh the free-suite duration seeds from the weekly recording"
|
||||
number=$(gh pr list --head "$branch" --state open --json number -q '.[0].number // empty')
|
||||
remote_head=
|
||||
if git fetch -q origin "+refs/heads/$branch:refs/remotes/origin/$branch" 2>/dev/null; then
|
||||
remote_head=$(git rev-parse "origin/$branch")
|
||||
fi
|
||||
if [ -n "$number" ] && [ -n "$remote_head" ] && [ "$(git log --format=%s "HEAD..$remote_head")" != "$subject" ]; then
|
||||
echo "::warning::Seed PR #$number has commits besides the bot's one refresh commit; this run leaves them in place. Merge or close that PR and the next run refreshes the seeds." | tee -a "$GITHUB_STEP_SUMMARY"
|
||||
exit 0
|
||||
fi
|
||||
git switch -C "$branch"
|
||||
git config user.name github-actions[bot]
|
||||
git config user.email 41898282+github-actions[bot]@users.noreply.github.com
|
||||
git add scripts/free-test-durations.json scripts/free-test-durations-windows.json
|
||||
git commit -m "$subject"
|
||||
git push --force-with-lease="refs/heads/$branch:$remote_head" origin "HEAD:refs/heads/$branch"
|
||||
cat > pr-body.md <<EOF
|
||||
Weekly recording from run $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID on $GITHUB_SHA.
|
||||
|
||||
- Linux seed (ubicloud-standard-8): +$linux_added / -$linux_removed files
|
||||
- Windows seed (windows-latest): +$windows_added / -$windows_removed files
|
||||
- Files that failed while recording (durations kept; this PR's own free checks decide): $failed_line
|
||||
- Seed growth ratchet (60 s ceiling, shrink-only allowlist): $ratchet_line
|
||||
|
||||
Data only. Stamp and merge per docs/ship-policy.md; the stamp push runs the full PR checks.
|
||||
EOF
|
||||
if [ -n "$number" ]; then
|
||||
gh pr edit "$number" --body-file pr-body.md
|
||||
else
|
||||
gh pr create --base main --head "$branch" --title "test(seed): refresh the free-suite duration seeds" --body-file pr-body.md
|
||||
fi
|
||||
gh workflow run free-tests.yml --repo "$GITHUB_REPOSITORY" --ref "$branch"
|
||||
gh workflow run windows-free-tests.yml --repo "$GITHUB_REPOSITORY" --ref "$branch"
|
||||
@@ -29,6 +29,12 @@ on:
|
||||
description: Record per-file Windows durations (artifact) instead of running the lane
|
||||
type: boolean
|
||||
default: false
|
||||
# free-durations-record.yml calls the recording path weekly.
|
||||
workflow_call:
|
||||
inputs:
|
||||
record_durations:
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
concurrency:
|
||||
group: windows-free-${{ github.event.pull_request.number || github.run_id }}
|
||||
@@ -231,15 +237,22 @@ jobs:
|
||||
# would leak into every timed test, and seed-reading tests would then read
|
||||
# the half-written recording instead of their fixtures.
|
||||
- name: Time every Windows-safe file alone
|
||||
id: record
|
||||
shell: bash
|
||||
env:
|
||||
TEMP: ${{ runner.temp }}
|
||||
TMP: ${{ runner.temp }}
|
||||
run: bun run test:windows --record-durations --wall-timeout 1200
|
||||
run: |
|
||||
set -o pipefail
|
||||
bun run test:windows --record-durations --wall-timeout 1200 2>&1 | tee "$RUNNER_TEMP/record.log"
|
||||
- name: Name the recording as the Windows seed
|
||||
if: always()
|
||||
id: seed
|
||||
if: always() && steps.record.outcome != 'skipped'
|
||||
shell: bash
|
||||
run: cp scripts/free-test-durations.json "$RUNNER_TEMP/free-test-durations-windows.json"
|
||||
run: |
|
||||
grep -q 'wrote [0-9]* durations to' "$RUNNER_TEMP/record.log" || { echo "::error::The recording was interrupted; no Windows seed to upload."; exit 1; }
|
||||
cp scripts/free-test-durations.json "$RUNNER_TEMP/free-test-durations-windows.json"
|
||||
sed -n '/failed while recording:/,$p' "$RUNNER_TEMP/record.log" | sed 's/\r$//' | { grep -E '^ ✗ [^ ]+$' || true; } | sed 's/^ ✗ //' > "$RUNNER_TEMP/record-failures.txt"
|
||||
- name: Upload full recording logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
@@ -251,8 +264,10 @@ jobs:
|
||||
include-hidden-files: true
|
||||
if-no-files-found: ignore
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
if: always()
|
||||
if: always() && steps.seed.outcome == 'success'
|
||||
with:
|
||||
name: free-test-durations-windows
|
||||
path: ${{ runner.temp }}/free-test-durations-windows.json
|
||||
path: |
|
||||
${{ runner.temp }}/free-test-durations-windows.json
|
||||
${{ runner.temp }}/record-failures.txt
|
||||
if-no-files-found: error
|
||||
@@ -1,5 +1,68 @@
|
||||
# Changelog
|
||||
|
||||
## [1.91.131.0] - 2026-10-18
|
||||
|
||||
**`gstack-pregate` finds the tests for Python files, and a missing `node_modules` no longer reads as "stale generated files".** In a Python repo, the `lanes` check failed almost every touched `.py` file with `PREGATE_NO_LANE`, because the import graph only parsed JavaScript and TypeScript. In a FastAPI service with about 1,500 non-test modules, Scott Gigante's change (#3201) cut the unmapped count from 1,473 to 780, and most of the rest are migrations and scripts no import reaches. Separately, a fresh checkout without `node_modules` made the pre-gate report `REGEN_STALE` and suggest rerunning the generator, which sent people the wrong way.
|
||||
|
||||
### Added
|
||||
- **Python import edges in `lanes`** (thanks @scottgigante-hubflow, #3201). `import a.b`, `from a.b import c`, parenthesised lists and relative `from . import x` resolve from the repository root to `a/b.py` or `a/b/__init__.py`, plus the package `__init__.py` files Python runs on the way in. Stdlib and site-packages imports add nothing. In test files, a dotted `mock.patch` target such as `'pkg.mod.Class.method'` maps to its module.
|
||||
- **`REGEN_DEPS_MISSING`.** When a regen command exits 0 with different outputs, `package.json` declares dependencies and `node_modules` is missing, `gstack-regen check` and the pre-gate say "dependencies missing (no node_modules); run bun install, then rerun the check" instead of `REGEN_STALE`. The install command follows the lockfile. The pre-gate check is `incomplete`, never `pass`.
|
||||
|
||||
### Fixed
|
||||
- **The Python scanner stays linear on hostile input and does not drop tests.** Review found several problems before landing, all fixed:
|
||||
- Long runs of dots or spaces after `from` no longer backtrack quadratically.
|
||||
- Deep dotted names stay linear, and real deep modules still map.
|
||||
- An escaped triple quote no longer swallows the imports after it.
|
||||
- A literal that names a repo file still gets its `mock.patch` reading.
|
||||
- A package and a same-named module both map.
|
||||
- Imports after `;` or `:` count.
|
||||
|
||||
### If you…
|
||||
- **…run `gstack-pregate` in a Python repo:** touched modules now select the pytest files that import them. `conftest.py`, `src/` layouts and `importlib` calls still need `.gstack/pregate.json` `dependencies` declarations.
|
||||
- **…see `REGEN_DEPS_MISSING`:** run the printed install command, then rerun the check.
|
||||
|
||||
## [1.91.130.0] - 2026-10-18
|
||||
|
||||
**Every PR plan now shows what a tighter, exercised-path case map would run, without changing what actually runs.** Today a change to a bin or lib file reaches almost every gate case, because selection follows every mention of the file. The exercised-path map counts only what a case really executes: its touchfiles, its test's imports, the shell its skill runs, and the bins its recorded trials invoked. It runs in shadow mode. Replayed over every PR run that had a gate red between 2026-10-07 and 2026-10-17 (37 runs, 47 failed gate cases), it keeps all 47: `bun run test:exercised-map --replay`.
|
||||
|
||||
### Added
|
||||
- **A shadow line in each PR plan's coverage summary.** It names the gate cases the map would select, drop and add, with per-file provenance and a predicted saving. The saving is a lower bound, because about 39% of observed case runs (the PTY trials) have no known cost.
|
||||
- **`bun run test:health` reports the shadow saving and shadow misses per week.** A red on a case the shadow would drop counts as a miss unless the case passed later at the same head SHA, which lists it as a flaky drop.
|
||||
- **MAP_GAP lines in the paid report.** A command a case ran outside its map is printed as `MAP_GAP <case> <bin>`, marked non-blocking for this PR, and widens the map at the next refresh. It never affects a verdict.
|
||||
- **`bun run test:exercised-map`** refreshes the committed map (`test/helpers/exercised-commands.json`) from a census run (`--from-run <id>`) or from local artifacts (`--from-dir`). Refreshes only add commands. Unavailable or partial input writes nothing and exits with a stable code: 2 usage, 3 unavailable, 4 incomplete.
|
||||
|
||||
### For contributors
|
||||
- The weekly census unions its own recorded commands into the map and uploads it as `exercised-map-a<attempt>`.
|
||||
- A PR that changes only the map selects no paid case while the map is in shadow mode.
|
||||
- docs/TESTING_INTERNALS.md#exercised-path-map documents the edge classes, the refresh contract and the activation bar.
|
||||
|
||||
## [1.91.129.0] - 2026-10-18
|
||||
|
||||
**PR paid-lane selection reads every script it depends on, and fails closed when it can't.** Selection derives which paid cases a changed file reaches by scanning imports. It skipped extensionless `bun`/`node` scripts, read 15 `.ts` files with the TSX loader, and treated a scan error as "no consumers", which under-selects. Across all 2,223 code files in the repo, scans now finish with 0 errors, and no (file, case) pair that main selected is lost.
|
||||
|
||||
### Fixed
|
||||
- **Shebang scripts and `.ts` files are scanned correctly.** A leading `#!` line is stripped before scanning, extensionless `bun`/`node` scripts count as code, and the loader is chosen by file extension. 41 files that reached no paid case now reach the cases that use them.
|
||||
- **A failed import scan restores the full gate** with the label `import scan failed` and the file to fix, instead of silently selecting nothing.
|
||||
- **Repository config no longer looks like an unknown dependency.** `.gstack/ship-policy.example.json`, `.github/dependabot.yml`, `.osv-scanner.toml`, `setup.ps1`, `.gitlab-ci.yml` and `.github/ISSUE_TEMPLATE/**` are audited free-only files; `capy/` paths are derived like other source; a full-gate file is labelled `full-gate input`.
|
||||
|
||||
### Added
|
||||
- **`bun run test:health` reports cost per merged PR first**, plus gate cases per PR run and known cost per PR run, split by selection mode.
|
||||
|
||||
### For contributors
|
||||
- The selection replay (`scripts/replay-pr-selection.ts`) honours the same free-only exemptions as the selector when it counts misses.
|
||||
- docs/TESTING_INTERNALS.md#pr-paid-lane-fallback describes the fallback classes as they now behave, and records why the lockfile stays a full-gate input.
|
||||
|
||||
## [1.91.128.0] - 2026-10-18
|
||||
|
||||
**The two free-suite duration seeds now refresh themselves weekly.** The shard packers read a Linux and a Windows duration seed, and both went stale as test files came and went. They were refreshed by hand twice this week. A new weekly workflow records both seeds and opens one seed-refresh PR when either seed's file set changes. That PR goes through the normal review.
|
||||
|
||||
### Added
|
||||
- **`free-durations-record.yml`** runs Mondays at 04:00 UTC, before test-health, and on dispatch. It records the Linux seed on `ubicloud-standard-8`, the free-suite shard runner, and the Windows seed through `windows-free-tests.yml`'s `record_durations` path (now also callable with `workflow_call`). When either seed adds or drops a file, it opens or refreshes one PR on `bot/free-duration-seeds` and runs the free lanes on it. The PR body states the seed-ratchet result and names every file that failed while recording.
|
||||
- **Guards:** the bot never edits `scripts/free-test-seed-allowlist.json`. While the open seed PR holds any commit besides the bot's own refresh, later runs leave it alone. Only runs on `main` publish, and `open_pr=false` records and reports only. An interrupted recording, or more than 5 red files on either OS, proposes nothing.
|
||||
|
||||
### Fixed
|
||||
- **An interrupted manual Windows recording no longer uploads the unchanged committed seed** under the Windows seed name.
|
||||
|
||||
## [1.91.127.0] - 2026-10-17
|
||||
|
||||
**The paid PR gate stops failing on its own fixtures.** A 132-verdict PR gate was fully green only about a quarter of the time. Every red verdict since #3231 was diagnosed from its transcript and replayed through the real detector. This release fixes the remaining causes where they start: fixture actors that broke the production protocol, detectors that misread a correct run, and skill prompts that left the agent guessing. No threshold, trial count, case kind or quarantine changed.
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# gstack digest v1.91.127.0 — regenerate/re-copy after upgrading gstack
|
||||
# gstack digest v1.91.131.0 — regenerate/re-copy after upgrading gstack
|
||||
|
||||
Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed
|
||||
for agent hosts without a full skill install. The full skills add workflows,
|
||||
|
||||
@@ -267,6 +267,7 @@ gstack_result_anchor() {
|
||||
APPROVAL_SOURCE_UNAVAILABLE) printf '%s' 'approval-source-unavailable' ;;
|
||||
STATE_MATRIX_INCOMPLETE) printf '%s' 'state-matrix-incomplete' ;;
|
||||
WTREE_UNAVAILABLE) printf '%s' 'wtree-unavailable' ;;
|
||||
REGEN_DEPS_MISSING) printf '%s' 'regen-deps-missing' ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
+2
-2
@@ -119,8 +119,8 @@ function main(argv: string[]): number {
|
||||
let outcome;
|
||||
try { outcome = runRegen({ repoRoot: root, registry: loaded.registry, mode: cmd, only: list(flag(rest, '--only')), touched: list(flag(rest, '--touched')) }); }
|
||||
catch (e: any) { process.stderr.write(resultLine(`gstack-regen ${cmd}`, 'PREGATE_INCOMPLETE', e.message) + '\n'); process.stdout.write(formatResult('regen', 'incomplete', loaded.source.label) + '\n'); return EXIT.fail; }
|
||||
const bad = outcome.results.filter(r => r.status === 'stale' || r.status === 'failed');
|
||||
const status = outcome.results.some(r => r.status === 'failed') ? 'incomplete' : 'complete';
|
||||
const bad = outcome.results.filter(r => r.status === 'stale' || r.status === 'failed' || r.status === 'deps-missing');
|
||||
const status = outcome.results.some(r => r.status === 'failed' || r.status === 'deps-missing') ? 'incomplete' : 'complete';
|
||||
if (json) { process.stdout.write(JSON.stringify({ mode: cmd, registry: loaded.source.label, wtree: outcome.wtree, results: outcome.results, ok: bad.length === 0 }) + '\n'); return bad.length ? EXIT.fail : EXIT.ok; }
|
||||
process.stdout.write(`registry: ${loaded.source.label}\n`);
|
||||
for (const r of outcome.results) process.stdout.write(renderEntry(r) + '\n');
|
||||
|
||||
+126
-11
@@ -1009,17 +1009,44 @@ select nothing: docs, any non-paid `*.test.ts(x)`, `FREE_ONLY_PR_FILES` (the
|
||||
free and paid duration seeds, `scripts/ubicloud/**`, every workflow that runs no
|
||||
paid evals, and free-only tools), and the fixtures in `FREE_FIXTURES`
|
||||
(`test/helpers/free-fixtures-data.ts`, each fixture with its free consumers).
|
||||
`PAID_WORKFLOW_FILES` (`evals.yml`, `evals-periodic.yml`, `evals-marathon.yml`)
|
||||
and any other unknown file restore the full gate (`prCoverage.mode =
|
||||
full-fallback`). The job summary and PR comment list each file that caused a
|
||||
fallback with one of three labels:
|
||||
Unmapped files under `DERIVABLE_PREFIXES` (bin/, lib/, scripts/, browse/,
|
||||
design/, make-pdf/, hosts/, extension/, model-overlays/, agents/, test/helpers/,
|
||||
capy/) are not unknown: `scripts/pr-dependencies.ts` derives the cases whose
|
||||
reference closure reaches them (`prCoverage.mode = dependents`), or reports that
|
||||
no paid case reaches them. The derivation scans the imports of every code file,
|
||||
including `#!`-shebang `.ts` files and extensionless Bun or Node scripts; if any
|
||||
import scan fails, every unmapped file restores the full gate (label **import
|
||||
scan failed**). `PR_FULL_GATE_FILES`, `PAID_WORKFLOW_FILES` (`evals.yml`,
|
||||
`evals-periodic.yml`, `evals-marathon.yml`) and any other unknown file restore
|
||||
the full gate (`prCoverage.mode = full-fallback`). The job summary and PR comment
|
||||
list each unmapped file that caused a fallback with one of these labels:
|
||||
|
||||
- **needs touchfile entry**: a paid test, helper or fixture no case maps; add it
|
||||
to the owning case in `E2E_TOUCHFILES` (`test/helpers/touchfiles-data.ts`).
|
||||
- **add to FREE_ONLY_PR_FILES**: a file only free lanes read; add it to the list
|
||||
in `scripts/test-pr-profile.ts` (or a fixture to `FREE_FIXTURES`).
|
||||
- **real unknown dependency**: shared code (bin/, lib/, resolvers, hosts, setup)
|
||||
whose dependents are not mapped; the full gate is the correct result.
|
||||
- **full-gate input**: a `PR_FULL_GATE_FILES` entry every case consumes; the
|
||||
full gate is intended and no mapping narrows it.
|
||||
- **deleted but still referenced** and **import scan failed**: fix the named
|
||||
reference or parse error.
|
||||
- **real unknown dependency**: a file outside the derivable directories that no
|
||||
list classifies (for example `ETHOS.md`); map it, exempt it, or derive its
|
||||
directory.
|
||||
|
||||
The fallback rate is a label, not the bill: a dependents run whose derived
|
||||
closure reaches every gate case costs about what a full-fallback run does.
|
||||
`bun run test:health` leads with the cost per merged PR and reports
|
||||
`pr-cases-per-run` (mean selected gate cases and judges per PR run, per mode)
|
||||
and `pr-known-cost-per-run` (mean known trial cost per PR run, per mode) beside
|
||||
`pr-fallback-rate`.
|
||||
|
||||
`package.json` (beyond a version-only change) and `bun.lock` stay full-gate
|
||||
inputs. Over the 225 PR runs from 2026-10-07 to 2026-10-17, no run fell back on
|
||||
them alone, dependabot PRs never reach the paid lane (`evals.yml` skips
|
||||
`github.actor == 'dependabot[bot]'`), and every runtime package's importer
|
||||
(`test/helpers/agent-sdk-runner.ts`, `test/helpers/llm-judge.ts`, `browse/src/*`,
|
||||
`make-pdf/src/*`) reaches every gate case or is in the global set, so narrowing
|
||||
them would select the same cases.
|
||||
|
||||
`test/free-fixtures.test.ts` holds the inventory: each `test/fixtures/**` path
|
||||
belongs to a paid touchfile or `FREE_FIXTURES`, and each workflow to the paid or
|
||||
@@ -1027,6 +1054,78 @@ free-only list. Editing a paid test file runs its own gate cases on the PR when
|
||||
their Bun test is named by the case id; the file's other cases stay deferred with
|
||||
a reason that says to name the test by its case id.
|
||||
|
||||
### Exercised-path map
|
||||
|
||||
The derived closure (above) reaches almost every gate case from most bin/ and
|
||||
lib/ files, because it follows prose, string and doc-comment mentions. The
|
||||
exercised-path map is a tighter consumer set that runs in **shadow mode**: it
|
||||
is computed and reported on every PR plan, and it never changes which cases a
|
||||
PR runs. Activating it is a later decision that needs PTY command capture for
|
||||
the cases that record no commands, two weeks of shadow data with zero shadow
|
||||
misses, and Garry's call on the measured saving.
|
||||
|
||||
A case's exercised consumers (`exercisedConsumers` in
|
||||
`scripts/pr-dependencies.ts`) are the union of its touchfile matches, the
|
||||
closure of its test file and helpers, the executable shell of its loaded skill
|
||||
(template and generated SKILL.md) and of every bin, lib and script that shell
|
||||
runs, and the bins its recorded trials invoked. Shell edges are command-position
|
||||
`gstack-*` names and repo paths, `.`/`source`, `./setup`, quoted path
|
||||
literals, `path.join` pairs, data files, `bun run` scripts and compiled outfiles
|
||||
(`browse/dist/browse`, `$B`, `design/dist/design`, `$D`, `make-pdf/dist/pdf`).
|
||||
Comments, echo/printf/die messages, case patterns, bare `VAR=gstack-x` strings
|
||||
and slash-command skill names are not edges; command substitutions are kept, an
|
||||
expanding heredoc keeps its substitutions and a heredoc an interpreter reads
|
||||
stays whole. The closure stops at `PR_FULL_GATE_FILES` and the global set, which
|
||||
stay full. `test/exercised-map.test.ts` pins that this function equals or is a
|
||||
reviewed superset of the measured prototype (`test/helpers/exercised-prototype.ts`).
|
||||
|
||||
The observed half is `test/helpers/exercised-commands.json` (generated; per case
|
||||
the bins and compiled outfiles its recorded trials ran, observation counts, the
|
||||
source run ids and a mean known cost per run). Refreshes are union-only: an
|
||||
observation never removes a command and a missing or empty artifact never erases
|
||||
one; removing a command is a reviewed edit. While the map is in shadow mode it
|
||||
changes no selection, so a PR that changes only the map selects no paid case.
|
||||
|
||||
Dollar figures are lower bounds. Known cost excludes PTY trials, whose cost is
|
||||
unknown: about 39% of observed case runs in the committed map. The shadow block
|
||||
and `test:health` both say so.
|
||||
|
||||
- **Shadow block:** `prCoverage.shadow` in the paid-plan manifest lists the gate
|
||||
cases the map would select, would drop and would add, a provenance line per
|
||||
changed file ("selects N case(s): touchfile (k), import (k), skill bash via
|
||||
bin/gstack-x (k), observed bin/gstack-y (k)"), the predicted saving and the
|
||||
unknown-cost share. The coverage summary prints one line of it.
|
||||
- **Shadow misses (`bun run test:health`, metric `pr-shadow-map`):** a red case
|
||||
(more executed failed than passed trials in the run) that the shadow would
|
||||
drop is an activation miss unless the case passed in a later run of the same
|
||||
branch at the same head SHA. That later pass marks it a flaky drop, which is
|
||||
listed but not counted. The D3 activation bar counts activation misses only.
|
||||
- **MAP_GAP:** the paid report runs the same bin matcher over each executed
|
||||
case's recorded commands and prints, and writes to `map-gaps.txt`,
|
||||
`MAP_GAP <case> <command>: non-blocking for this PR; the next census refresh
|
||||
widens the map (docs/TESTING_INTERNALS.md#exercised-path-map)` for a command
|
||||
outside the case's static-plus-observed closure. It is its own result: never a
|
||||
case verdict, a lane problem or an INFRA re-dispatch input.
|
||||
- **Census refresh:** the weekly census unions its own recorded commands into the
|
||||
map (no REST artifact downloads), uploads it as `exercised-map-a<attempt>`,
|
||||
reports MAP_GAP lines and the cases that recorded no commands, and prints the
|
||||
refresh command. Run `bun run test:exercised-map --from-run <id>`, or
|
||||
`--from-dir <dir>` on a locally downloaded `exercised-map-a<attempt>` artifact
|
||||
when the download hits HTTP 403, and open the change as an ordinary PR.
|
||||
`bun run test:exercised-map --check` validates the committed file.
|
||||
- **Refresh contract:** a refresh whose input is unavailable or partial writes
|
||||
nothing, keeps the committed union and exits with a stable code that names
|
||||
what was missing: `2` usage; `3` input unavailable (a missing directory, or a
|
||||
failed or HTTP 403 download); `4` input incomplete (no `shards/`, no
|
||||
`trial-outcomes.jsonl`, an executed case without a shard artifact, or a missing
|
||||
or malformed `exercised-commands.json`). The census step reports a failed
|
||||
refresh in its job summary and uploads no map.
|
||||
- **Replay receipt:** `bun run test:exercised-map --replay` replays the recorded
|
||||
red PR runs in `test/fixtures/exercised-map-replay.json` (every executed failed
|
||||
gate trial in 225 `evals.yml` pull_request runs, 2026-10-07 to 2026-10-17)
|
||||
through today's selector and its shadow, and exits 1 on any MISS: a failed
|
||||
gate case the shadow would not run. `test/exercised-map.test.ts` pins zero.
|
||||
|
||||
### PR receipts, recovery and debounce
|
||||
|
||||
Passes and failures of single-trial shards are written as receipts at the
|
||||
@@ -1164,10 +1263,11 @@ changed. Cut or unstarted reruns are incomplete, not passes.
|
||||
### Free suite duration seed
|
||||
|
||||
`scripts/free-test-durations.json` drives shard packing; files missing from it
|
||||
pack at the seed's 99th percentile. Acceptable seeds come only from
|
||||
`bun run test:ubicloud --record-durations` (each file timed in its own child on a
|
||||
VM with the CI lane's environment; it needs `UBICLOUD_API_KEY` and copies the
|
||||
refreshed seed back into the checkout). A 4-core laptop or small-sandbox
|
||||
pack at the seed's 99th percentile. Acceptable seeds come only from a recording
|
||||
in the CI lane's environment, each file timed in its own child: the weekly
|
||||
`free-durations-record.yml` run, or `bun run test:ubicloud --record-durations`
|
||||
(it needs `UBICLOUD_API_KEY` and copies the refreshed seed back into the
|
||||
checkout). A 4-core laptop or small-sandbox
|
||||
recording is not acceptable: browser and display tests skip or slow down there,
|
||||
so the seed misstates them. Contributors without `UBICLOUD_API_KEY` leave the
|
||||
refresh to a maintainer.
|
||||
@@ -1188,6 +1288,20 @@ refresh to a maintainer.
|
||||
`{file, reason, todo}` and the test prints it. A merge-base without the
|
||||
allowlist file counts as the first allowlist. Fix a failure by splitting the
|
||||
file, then rerun `bun run test:ubicloud --record-durations`.
|
||||
- `free-durations-record.yml` (Mondays 04:00 UTC, before test-health, and on
|
||||
dispatch) records the Linux seed on `ubicloud-standard-8` with
|
||||
`scripts/ubicloud/setup-free-suite.sh` and the Windows seed through
|
||||
`windows-free-tests.yml`'s `record_durations` path. When either recording adds
|
||||
or drops a test file, it force-updates the `bot/free-duration-seeds` branch,
|
||||
opens or refreshes one seed PR whose body states the ratchet result, and
|
||||
dispatches the free lanes on it. A file that fails while recording turns its
|
||||
job red but keeps its duration, and the PR body names it; the seed PR's own
|
||||
free checks decide whether the tree is healthy. An interrupted recording
|
||||
uploads no seed, and more than 5 red files on either OS proposes nothing. The
|
||||
bot never edits the allowlist: a ratchet failure is fixed on the branch by a
|
||||
maintainer before the PR merges, and while the open seed PR holds any commit
|
||||
besides the bot's one refresh commit, later runs leave it alone. Only runs on
|
||||
`main` push the branch; a dispatch from another ref reports only.
|
||||
|
||||
### Windows free lane
|
||||
|
||||
@@ -1195,7 +1309,8 @@ refresh to a maintainer.
|
||||
runs one strict serial shard per `windows-latest` job
|
||||
(`bun run test:windows --ci-run`), and verifies every shard receipt on Linux
|
||||
(`--windows-only --ci-verify`). Packing uses Windows-measured durations from
|
||||
`scripts/free-test-durations-windows.json`; refresh them with
|
||||
`scripts/free-test-durations-windows.json`; the weekly
|
||||
`free-durations-record.yml` run refreshes them, or by hand run
|
||||
`gh workflow run windows-free-tests.yml --ref <branch> -f record_durations=true`,
|
||||
then download the `free-test-durations-windows` artifact and commit it.
|
||||
|
||||
|
||||
+3
-1
@@ -41,7 +41,9 @@ paths, at most 64 per list); commands are one line under 512 bytes.
|
||||
<paths>` for entries whose inputs or outputs intersect) in a scratch
|
||||
`git worktree` seeded from the working tree, with `node_modules` linked rather
|
||||
than copied, and reports stale outputs (`REGEN_STALE`, exit 1) without touching
|
||||
the tree. An entry with no `command` is tracked only (goldens regenerated by
|
||||
the tree. When outputs differ but `package.json` declares dependencies and
|
||||
`node_modules` is missing, the entry is `REGEN_DEPS_MISSING` (install, then
|
||||
rerun) instead of stale. An entry with no `command` is tracked only (goldens regenerated by
|
||||
another entry).
|
||||
- `gstack-regen write` regenerates in place: what `gstack-restamp` and the
|
||||
Step 3 conflict rule use. On a conflict in a registered output, take either
|
||||
|
||||
@@ -2789,6 +2789,18 @@ take either side, then regenerate.
|
||||
worktree has no `node_modules` of its own, so a command that needs them must
|
||||
reference the repository's (gstack's entries do through `bun run`).
|
||||
|
||||
<a id="regen-deps-missing"></a>
|
||||
### `REGEN_DEPS_MISSING`
|
||||
|
||||
**Meaning.** A registered command exited 0 and its outputs differ from the
|
||||
committed ones, but `package.json` declares dependencies and the repository has
|
||||
no `node_modules`, so the scratch worktree ran the generator without them. The
|
||||
difference may come from the missing dependencies rather than stale files, so
|
||||
the entry is reported as unverified, not `REGEN_STALE`.
|
||||
|
||||
**Fix.** Run the printed install command (`bun install`, or the one matching the
|
||||
repository's lockfile), then rerun `gstack-regen check` or the pre-gate.
|
||||
|
||||
<a id="pregate-config-invalid"></a>
|
||||
### `PREGATE_CONFIG_INVALID`
|
||||
|
||||
|
||||
@@ -27,7 +27,9 @@ function run(ctx: PregateContext): CheckResult {
|
||||
const lines = results.map(renderEntry);
|
||||
const stale = results.filter(r => r.status === 'stale');
|
||||
const failed = results.filter(r => r.status === 'failed');
|
||||
const depsMissing = results.filter(r => r.status === 'deps-missing');
|
||||
ctx.explain.push(...results.map(r => `regen ${r.id}: selected because touched ∩ (${selected.find(e => e.id === r.id)?.inputs.join(',') || 'always'} ∪ outputs) ≠ ∅`));
|
||||
if (depsMissing.length) return { id: 'regen', stage: 'preflight', status: 'incomplete', inputs, lines, code: 'REGEN_DEPS_MISSING', detail: `dependencies missing; ${depsMissing[0]!.fix} (${depsMissing.length} entr${depsMissing.length === 1 ? 'y' : 'ies'} unverified: ${depsMissing.map(d => d.id).join(', ')})`, fix: depsMissing[0]!.fix };
|
||||
if (failed.length) return { id: 'regen', stage: 'preflight', status: 'incomplete', inputs, lines, code: 'REGEN_COMMAND_FAILED', detail: `${failed.length} regen command(s) failed: ${failed.map(f => f.id).join(', ')}`, fix: failed.map(f => f.fix).join('; ') };
|
||||
if (stale.length) return { id: 'regen', stage: 'preflight', status: 'fail', inputs, lines, code: 'REGEN_STALE', detail: `${stale.length} stale: ${stale.flatMap(s => s.changed.concat(s.outside)).join(' ')}`, fix: stale.map(s => s.fix).join('; ') };
|
||||
return { id: 'regen', stage: 'preflight', status: 'pass', inputs, lines, detail: `${results.length} entr${results.length === 1 ? 'y' : 'ies'} fresh${results.some(r => r.status === 'listed') ? ' (hand-regenerated outputs listed)' : ''}` };
|
||||
|
||||
+105
-3
@@ -2,10 +2,13 @@
|
||||
* pregate/import-graph — which test files depend on which repo files (plan
|
||||
* C8 `lanes`). The reverse graph unions four sources, each kept as provenance
|
||||
* for `--explain`:
|
||||
* 1. static imports and requires (Bun.Transpiler.scanImports);
|
||||
* 1. static imports and requires (Bun.Transpiler.scanImports), and Python
|
||||
* `import` / `from … import` statements resolved from the repository root
|
||||
* or, when relative, from the importing file's package;
|
||||
* 2. repo-path string literals, including `join('a', 'b')` segment runs
|
||||
* (a test that spawns `bin/gstack-doctor` names it in a string, never an
|
||||
* import) and literal directories (`test/fixtures/x` → `x/**`);
|
||||
* import) and literal directories (`test/fixtures/x` → `x/**`), and in
|
||||
* Python, dotted `mock.patch` targets (`'pkg.mod.Class.method'`);
|
||||
* 3. the `bin/<name>` → `test/<name>*.test.*` convention (with and without
|
||||
* the `gstack-` prefix);
|
||||
* 4. explicit `.gstack/pregate.json` `dependencies` declarations.
|
||||
@@ -41,6 +44,10 @@ const scanner = new Bun.Transpiler({ loader: 'tsx' });
|
||||
const QUOTED = /['"`]([A-Za-z0-9_@][A-Za-z0-9_./@()[\]+-]*)['"`]/g;
|
||||
const JOIN = /\bjoin\(([^()]*)\)/g;
|
||||
const RESOLVE_EXT = ['', '.ts', '.tsx', '.js', '.mjs', '.cjs', '/index.ts', '/index.js'];
|
||||
// A statement starts a line or follows `;` or a compound statement's `:`. The module spec is
|
||||
// non-empty and starts with a dot or a word character, so no two repetitions can share a dot or a space.
|
||||
const PY_IMPORT = /(?:^|[;:])[ \t\f]*(?:from[ \t]+(\.+(?:\w[\w.]*)?|\w[\w.]*)[ \t]+import[ \t]+(\([^)]*\)|[^\n;]+)|import[ \t]+([^\n;]+))/gm;
|
||||
const PY_DOTTED = /^[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)+$/;
|
||||
|
||||
export function isTestFile(rel: string): boolean { return TEST_FILE_RE.test(rel); }
|
||||
|
||||
@@ -185,6 +192,95 @@ function pathLike(config: PathAliases, ref: string): boolean {
|
||||
return chosen !== null || others.length > 0;
|
||||
}
|
||||
|
||||
const pathDepths = new WeakMap<Set<string>, number>();
|
||||
/**
|
||||
* Segments in the repo's deepest path. No module name resolves deeper, so
|
||||
* dotted-name resolution is bounded by the repo rather than by the source.
|
||||
*/
|
||||
function maxPathDepth(files: Set<string>): number {
|
||||
let depth = pathDepths.get(files);
|
||||
if (depth === undefined) { depth = 0; for (const f of files) depth = Math.max(depth, f.split('/').length); pathDepths.set(files, depth); }
|
||||
return depth;
|
||||
}
|
||||
|
||||
/** The leading dotted name of each comma-separated `name [as alias]` clause. */
|
||||
function pyNames(list: string): string[] {
|
||||
return list.replace(/[()]/g, '').split(',').map(p => p.trim().split(/\s+/)[0]!).filter(n => n && n !== '*');
|
||||
}
|
||||
|
||||
/** `from <dots><rest>` → the absolute dotted package it names, or null when it climbs above the repository root. */
|
||||
function pyFromBase(rel: string, spec: string): string | null {
|
||||
const dots = /^\.*/.exec(spec)![0].length;
|
||||
const rest = spec.slice(dots);
|
||||
if (!dots) return rest;
|
||||
const pkg = path.posix.dirname(rel).split('/').filter(s => s !== '.');
|
||||
if (dots - 1 > pkg.length) return null;
|
||||
return [...pkg.slice(0, pkg.length - (dots - 1)), ...(rest ? [rest] : [])].join('.');
|
||||
}
|
||||
|
||||
/**
|
||||
* Python source with every string blanked to `''` and every comment dropped, in
|
||||
* one linear pass, so a `#` inside a string and a quote inside a comment both
|
||||
* stay inert. Backslash escapes are honoured; an unterminated string runs to
|
||||
* its line end, or to the end of the file when triple-quoted.
|
||||
*/
|
||||
function pythonCode(source: string): string {
|
||||
const parts: string[] = [];
|
||||
let from = 0;
|
||||
for (let i = 0; i < source.length;) {
|
||||
const c = source[i]!;
|
||||
if (c === '#') { parts.push(source.slice(from, i)); const end = source.indexOf('\n', i); i = from = end < 0 ? source.length : end; continue; }
|
||||
if (c !== '"' && c !== "'") { i++; continue; }
|
||||
parts.push(source.slice(from, i), "''");
|
||||
const close = source.startsWith(c + c + c, i) ? c + c + c : c;
|
||||
let j = i + close.length;
|
||||
while (j < source.length && !source.startsWith(close, j) && (close.length === 3 || source[j] !== '\n')) j += source[j] === '\\' ? 2 : 1;
|
||||
i = from = Math.min(source.length, source.startsWith(close, j) ? j + close.length : j);
|
||||
}
|
||||
parts.push(source.slice(from));
|
||||
return parts.join('');
|
||||
}
|
||||
|
||||
/** Dotted modules a Python source imports; `from a import b` also yields `a.b`, which may be a submodule. */
|
||||
function pythonImports(rel: string, source: string): string[] {
|
||||
const code = pythonCode(source).replace(/\\\r?\n/g, ' ');
|
||||
const out: string[] = [];
|
||||
for (const m of code.matchAll(PY_IMPORT)) {
|
||||
if (m[3]) { out.push(...pyNames(m[3])); continue; }
|
||||
const base = pyFromBase(rel, m[1]!);
|
||||
if (base === null) continue;
|
||||
if (base) out.push(base);
|
||||
for (const name of pyNames(m[2]!)) out.push(base ? `${base}.${name}` : name);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* A dotted module → its repo files (`a/b/__init__.py`, which Python prefers,
|
||||
* and `a/b.py`; both when both exist) plus the package `__init__.py` files
|
||||
* Python executes on the way; [] for a module outside the repo (stdlib,
|
||||
* site-packages), including one deeper than any repo path.
|
||||
*/
|
||||
function pythonModuleFiles(files: Set<string>, dotted: string): string[] {
|
||||
const parts = dotted.split('.');
|
||||
if (parts.length > maxPathDepth(files) || parts.some(p => !p)) return [];
|
||||
const p = parts.join('/');
|
||||
const found = [`${p}/__init__.py`, `${p}.py`].filter(f => files.has(f));
|
||||
if (!found.length) return [];
|
||||
const inits = parts.slice(0, -1).map((_, i) => `${parts.slice(0, i + 1).join('/')}/__init__.py`).filter(f => files.has(f));
|
||||
return [...found, ...inits];
|
||||
}
|
||||
|
||||
/** `'pkg.mod.Class.attr'` (a `mock.patch` target) → the longest dotted prefix that is a repo module. */
|
||||
function pythonPatchTarget(files: Set<string>, ref: string): string[] {
|
||||
const parts = ref.split('.').slice(0, maxPathDepth(files));
|
||||
for (let n = parts.length; n >= 1; n--) {
|
||||
const hit = pythonModuleFiles(files, parts.slice(0, n).join('.'));
|
||||
if (hit.length) return hit;
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
function addEdge(graph: Graph, dependency: string, dependent: string, via: Edge['via']): void {
|
||||
if (dependency === dependent) return;
|
||||
const list = graph.reverse.get(dependency) ?? [];
|
||||
@@ -210,11 +306,17 @@ export function referencesIn(files: Set<string>, dirs: Set<string>, rel: string,
|
||||
// The transpiler erases type-only imports; a types module still binds its importers' contracts.
|
||||
for (const m of source.matchAll(/^\s*(?:import|export)\s+type\b[^;'"]*?\bfrom\s+['"]([^'"]+)['"]/gm)) importOf(m[1]!);
|
||||
}
|
||||
const python = rel.endsWith('.py');
|
||||
if (python) for (const mod of pythonImports(rel, source)) for (const t of pythonModuleFiles(files, mod)) out.push({ target: t, via: 'import' });
|
||||
// Literals carry dependencies only from the test tree and from bin/ and shell
|
||||
// sources (a test spawns a bin; a bash bin calls a sibling). Prose and
|
||||
// manifests name everything and would make every file a hub.
|
||||
if (!scansLiterals(rel)) return out;
|
||||
for (const m of source.matchAll(QUOTED)) for (const target of resolveRefs(files, dirs, rel, m[1]!, true, config)) out.push({ target, via: 'literal' });
|
||||
for (const m of source.matchAll(QUOTED)) {
|
||||
const targets = resolveRefs(files, dirs, rel, m[1]!, true, config);
|
||||
for (const target of targets) out.push({ target, via: 'literal' });
|
||||
if (python && PY_DOTTED.test(m[1]!)) for (const f of pythonPatchTarget(files, m[1]!)) out.push({ target: f, via: 'literal' });
|
||||
}
|
||||
for (const m of source.matchAll(JOIN)) {
|
||||
const segments = [...m[1]!.matchAll(/['"`]([^'"`]+)['"`]/g)].map(s => s[1]!);
|
||||
if (segments.length < 2) continue;
|
||||
|
||||
+23
-3
@@ -156,7 +156,7 @@ export function scratchWorktree(repoRoot: string, prefix = 'gstack-regen-'): Scr
|
||||
|
||||
export interface EntryResult {
|
||||
id: string; command: string | null;
|
||||
status: 'fresh' | 'stale' | 'failed' | 'listed' | 'written' | 'skipped';
|
||||
status: 'fresh' | 'stale' | 'failed' | 'listed' | 'written' | 'skipped' | 'deps-missing';
|
||||
/** Output files whose bytes changed (stale) or were written. */
|
||||
changed: string[];
|
||||
/** Files the command wrote outside its declared outputs (a refusal in `check`). */
|
||||
@@ -189,6 +189,20 @@ function diffSnapshots(a: Map<string, string>, b: Map<string, string>): string[]
|
||||
return [...changed].filter(p => p !== 'node_modules').sort();
|
||||
}
|
||||
|
||||
/**
|
||||
* The install command for a repository whose package.json declares
|
||||
* dependencies but has no node_modules (the scratch worktree links none
|
||||
* then), picked from its lockfile; null when nothing is missing.
|
||||
*/
|
||||
export function missingDependencies(repoRoot: string): string | null {
|
||||
if (fs.existsSync(path.join(repoRoot, 'node_modules'))) return null;
|
||||
let pkg: { dependencies?: Record<string, unknown>; devDependencies?: Record<string, unknown> };
|
||||
try { pkg = JSON.parse(fs.readFileSync(path.join(repoRoot, 'package.json'), 'utf8')) ?? {}; } catch { return null; }
|
||||
if (!Object.keys(pkg.dependencies ?? {}).length && !Object.keys(pkg.devDependencies ?? {}).length) return null;
|
||||
const lockfiles: Array<[string, string]> = [['bun.lock', 'bun install'], ['bun.lockb', 'bun install'], ['pnpm-lock.yaml', 'pnpm install'], ['yarn.lock', 'yarn install'], ['package-lock.json', 'npm install']];
|
||||
return lockfiles.find(([file]) => fs.existsSync(path.join(repoRoot, file)))?.[1] ?? 'bun install';
|
||||
}
|
||||
|
||||
function runCommand(cwd: string, command: string, env: NodeJS.ProcessEnv, timeoutMs: number): { exit: number | null; stderr: string } {
|
||||
const r = runBashSync(`set -o pipefail; ${command}`, { cwd, encoding: 'utf8', timeout: timeoutMs, maxBuffer: 64 * 1024 * 1024, env: { ...env, GSTACK_REGEN_SCRATCH: cwd } });
|
||||
return { exit: r.status, stderr: ((r.stderr ?? '') + (r.error ? `\n${r.error.message}` : '')).trim().slice(-4000) };
|
||||
@@ -207,7 +221,9 @@ export interface RunRegenOptions {
|
||||
* `check`: run every selected command in one scratch worktree and compare its
|
||||
* outputs with the working tree; `write`: run them in the repository. An
|
||||
* entry with `command: null` is `listed` (its fix line names the outputs an
|
||||
* author refreshes by hand). Never runs a command the caller did not opt into:
|
||||
* author refreshes by hand). A `check` whose outputs differ while the
|
||||
* repository's dependencies are not installed is `deps-missing`, not `stale`:
|
||||
* the generator may have run without them. Never runs a command the caller did not opt into:
|
||||
* the bin enforces --allow-repo-commands before calling this.
|
||||
*/
|
||||
export function runRegen(o: RunRegenOptions): { results: EntryResult[]; wtree: string | null } {
|
||||
@@ -215,6 +231,7 @@ export function runRegen(o: RunRegenOptions): { results: EntryResult[]; wtree: s
|
||||
const timeout = o.timeoutMs ?? 600_000;
|
||||
const results: EntryResult[] = [];
|
||||
let scratch: Scratch | null = null;
|
||||
const install = o.mode === 'check' ? missingDependencies(o.repoRoot) : null;
|
||||
const fixFor = (e: RegenEntry) => e.command ? `${e.command} # then commit ${e.outputs.join(' ')}` : `regenerate by hand: ${e.outputs.join(' ')}${e.note ? ` (${e.note})` : ''}`;
|
||||
try {
|
||||
if (o.mode === 'check' && entries.some(e => e.command)) scratch = scratchWorktree(o.repoRoot);
|
||||
@@ -230,7 +247,9 @@ export function runRegen(o: RunRegenOptions): { results: EntryResult[]; wtree: s
|
||||
const ms = Date.now() - started;
|
||||
if (run.exit !== 0) { results.push({ id: e.id, command: e.command, status: 'failed', changed, outside, exit: run.exit, stderr: run.stderr, ms, fix: fixFor(e) }); continue; }
|
||||
if (o.mode === 'check') {
|
||||
results.push({ id: e.id, command: e.command, status: changed.length || outside.length ? 'stale' : 'fresh', changed, outside, exit: 0, ms, fix: fixFor(e) });
|
||||
const differs = changed.length > 0 || outside.length > 0;
|
||||
if (differs && install) results.push({ id: e.id, command: e.command, status: 'deps-missing', changed, outside, exit: 0, ms, fix: `run ${install}, then rerun the check` });
|
||||
else results.push({ id: e.id, command: e.command, status: differs ? 'stale' : 'fresh', changed, outside, exit: 0, ms, fix: fixFor(e) });
|
||||
// Reset the scratch so the next entry's diff is its own.
|
||||
git(cwd, ['checkout', '-q', '--', '.']); git(cwd, ['clean', '-fdq']);
|
||||
} else results.push({ id: e.id, command: e.command, status: changed.length ? 'written' : 'fresh', changed, outside, exit: 0, ms, fix: fixFor(e) });
|
||||
@@ -247,6 +266,7 @@ export function renderEntry(r: EntryResult): string {
|
||||
if (r.status === 'fresh' || r.status === 'skipped') return head;
|
||||
if (r.status === 'listed') return `${head} — ${r.fix}`;
|
||||
if (r.status === 'failed') return `${head} — exit ${r.exit}: ${(r.stderr ?? '').split('\n').pop() ?? ''}; fix: run \`${r.command}\` and read its output (REGEN_COMMAND_FAILED)`;
|
||||
if (r.status === 'deps-missing') return `${head} — dependencies missing (no node_modules); ${r.fix} — differs without them: ${[...r.changed, ...r.outside].join(' ')} (REGEN_DEPS_MISSING)`;
|
||||
const parts = [head];
|
||||
if (r.changed.length) parts.push(`stale: ${r.changed.join(' ')}`);
|
||||
if (r.outside.length) parts.push(`outside declared outputs: ${r.outside.join(' ')}`);
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
/**
|
||||
* Result codes for the release path: gstack-ship-queue and the ship-policy
|
||||
* floor, pregate timing and its working-tree fingerprint, approval classes and
|
||||
* floor, pregate timing and its working-tree fingerprint, regen with missing
|
||||
* dependencies, approval classes and
|
||||
* the state-machine matrix. Kept beside lib/result-codes.ts (which spreads
|
||||
* this table into RESULT_CODES) so
|
||||
* the owner module stays within the module-size ratchet; the same rules apply:
|
||||
@@ -83,4 +84,10 @@ export const RELEASE_RESULT_CODES = {
|
||||
summary: 'cannot fingerprint the working tree: bin/gstack-wtree could not start (on Windows, no Git Bash was found) or it ran and failed',
|
||||
fix: 'on Windows point GSTACK_GIT_BASH at Git Bash\'s bash.exe (Git Bash: export GSTACK_GIT_BASH=\'C:/Program Files/Git/bin/bash.exe\'; PowerShell: $env:GSTACK_GIT_BASH = \'C:\\Program Files\\Git\\bin\\bash.exe\'), or install Git for Windows (winget install --id Git.Git -e); elsewhere rerun ./setup so bin/gstack-wtree is present; then rerun gstack-pregate',
|
||||
},
|
||||
// Fix wave W5 (GSTA-31): pregate regen without installed dependencies.
|
||||
REGEN_DEPS_MISSING: {
|
||||
anchor: 'regen-deps-missing',
|
||||
summary: 'a regen command exited 0 with different outputs while package.json declares dependencies and node_modules is missing, so the outputs were not verified',
|
||||
fix: 'install the dependencies (the printed command, such as bun install), then rerun the check',
|
||||
},
|
||||
} as const satisfies Record<string, ResultCode>;
|
||||
+2
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "gstack",
|
||||
"version": "1.91.127",
|
||||
"version": "1.91.131",
|
||||
"description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.",
|
||||
"license": "MIT",
|
||||
"type": "module",
|
||||
@@ -50,6 +50,7 @@
|
||||
"eval:bg:gate": "bun run scripts/eval-bg.ts gate",
|
||||
"eval:bg:periodic": "bun run scripts/eval-bg.ts periodic",
|
||||
"test:health": "bun run scripts/test-health-report.ts",
|
||||
"test:exercised-map": "bun run scripts/exercised-map.ts",
|
||||
"eval:list": "bun run scripts/eval-list.ts",
|
||||
"eval:compare": "bun run scripts/eval-compare.ts",
|
||||
"eval:summary": "bun run scripts/eval-summary.ts",
|
||||
|
||||
@@ -0,0 +1,384 @@
|
||||
#!/usr/bin/env bun
|
||||
/**
|
||||
* test:exercised-map — maintain test/helpers/exercised-commands.json, the
|
||||
* observed half of the exercised-path consumer map (scripts/pr-dependencies.ts
|
||||
* exercisedConsumers; shadow mode, docs/TESTING_INTERNALS.md#exercised-path-map).
|
||||
*
|
||||
* Refreshes are union-only: an observation adds commands, never removes one.
|
||||
* A refresh whose input is unavailable or partial writes nothing, keeps the
|
||||
* committed union and exits non-zero with a stable code naming what was
|
||||
* missing: 2 usage, 3 input unavailable (missing directory, failed or HTTP 403
|
||||
* download), 4 input incomplete (no shards, no trial outcomes, an executed case
|
||||
* without a shard artifact, a missing or malformed exercised-commands.json).
|
||||
* Removing a command is a reviewed edit of the JSON file.
|
||||
*
|
||||
* Usage:
|
||||
* bun run test:exercised-map --from-run <id> [--out <file>]
|
||||
* download that census run's exercised-map artifact and union it into the committed map
|
||||
* bun run test:exercised-map --from-dir <downloaded exercised-map artifact dir> [--out <file>]
|
||||
* the same from a locally downloaded artifact (the fallback when the download hits HTTP 403)
|
||||
* bun run test:exercised-map --from-dir <dir> [--from-dir <dir>...] --run-id <id> [--out <file>] [--report <file>]
|
||||
* read paid-slice / gate-census report directories (shards/<shard>/...: transcripts, junit.xml,
|
||||
* PTY terminal.visible.log, trial-outcomes.jsonl) and union their commands into the map
|
||||
* bun run test:exercised-map --check
|
||||
* validate the committed map
|
||||
* bun run test:exercised-map --replay [<fixture>]
|
||||
* replay recorded red PR runs (default test/fixtures/exercised-map-replay.json) through
|
||||
* today's selector and its shadow; a MISS is a failed gate case the shadow would not run
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import {
|
||||
checkoutExercisedConsumers, EXERCISED_MAP_PATH, parseExercisedMap, type ExercisedCase, type ExercisedConsumers, type ExercisedMap,
|
||||
} from './pr-dependencies';
|
||||
import { caseOwnerFile, EXERCISED_MAP_ANCHOR, PR_FULL_GATE_FILES, PR_PROFILE_MAPS } from './test-pr-profile';
|
||||
import { CASE_TEST_NAMES } from './lib/paid-cases';
|
||||
import { computePaidCaseSelection } from './test-paid-shards';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
export const EXERCISED_MAP_ARTIFACT = 'exercised-map';
|
||||
|
||||
/** Commands one shard directory recorded: per Bun test name, and shard-level (no test name). */
|
||||
export interface ShardCommands { shard: string; executed: string[]; perTest: Record<string, string[]>; shardLevel: string[] }
|
||||
|
||||
function walk(dir: string): string[] {
|
||||
return fs.readdirSync(dir, { withFileTypes: true }).flatMap(entry => entry.isDirectory() ? walk(path.join(dir, entry.name)) : [path.join(dir, entry.name)]);
|
||||
}
|
||||
|
||||
function toolCommands(node: unknown, out: string[]): void {
|
||||
if (Array.isArray(node)) { for (const item of node) toolCommands(item, out); return; }
|
||||
if (!node || typeof node !== 'object') return;
|
||||
const record = node as Record<string, any>;
|
||||
if (record.type === 'tool_use' && typeof record.input?.command === 'string') out.push(record.input.command);
|
||||
for (const value of Object.values(record)) if (value && typeof value === 'object') toolCommands(value, out);
|
||||
}
|
||||
|
||||
/** Every `shards/<shard>/` directory under `dir`, with the Bash commands its transcripts, JSONL logs and PTY screens recorded. */
|
||||
export function readShardCommands(dir: string): ShardCommands[] {
|
||||
if (!fs.existsSync(dir)) return [];
|
||||
const shardDirs = walk(dir).map(file => path.relative(dir, file).split(path.sep)).flatMap(parts => {
|
||||
const at = parts.lastIndexOf('shards');
|
||||
return at >= 0 && parts.length > at + 2 ? [path.join(dir, ...parts.slice(0, at + 2))] : [];
|
||||
});
|
||||
return [...new Set(shardDirs)].sort().map(shardDir => {
|
||||
const record: ShardCommands = { shard: path.basename(shardDir), executed: [], perTest: {}, shardLevel: [] };
|
||||
for (const file of walk(shardDir)) {
|
||||
const base = path.basename(file);
|
||||
try {
|
||||
if (base === 'junit.xml') {
|
||||
for (const match of fs.readFileSync(file, 'utf8').matchAll(/<testcase\b([^>]*?)(\/>|>([\s\S]*?)<\/testcase>)/g)) {
|
||||
const name = /\bname="([^"]*)"/.exec(match[1]!)?.[1];
|
||||
if (name && !/<skipped/.test(match[3] ?? '')) record.executed.push(name.replace(/"/g, '"').replace(/&/g, '&'));
|
||||
}
|
||||
} else if (base.endsWith('.json')) {
|
||||
const parsed = JSON.parse(fs.readFileSync(file, 'utf8'));
|
||||
if (Array.isArray(parsed?.tests) && parsed.tests.some((test: any) => test?.transcript)) {
|
||||
for (const test of parsed.tests) { const out: string[] = []; toolCommands(test.transcript ?? [], out); if (out.length) (record.perTest[test.name] ??= []).push(...out); }
|
||||
} else toolCommands(parsed, record.shardLevel);
|
||||
} else if (base.endsWith('.jsonl')) {
|
||||
for (const line of fs.readFileSync(file, 'utf8').split('\n')) if (line.includes('tool_use')) { try { toolCommands(JSON.parse(line), record.shardLevel); } catch { /* partial line */ } }
|
||||
} else if (base === 'terminal.visible.log') {
|
||||
for (const match of fs.readFileSync(file, 'utf8').matchAll(/Bash\(([^\r\n]{0,400})/g)) record.shardLevel.push(`PTY:${match[1]}`);
|
||||
}
|
||||
} catch { /* unreadable evidence file: its commands are simply not observed */ }
|
||||
}
|
||||
record.shardLevel = [...new Set(record.shardLevel)];
|
||||
for (const name of Object.keys(record.perTest)) record.perTest[name] = [...new Set(record.perTest[name])];
|
||||
return record;
|
||||
});
|
||||
}
|
||||
|
||||
const testNameCase = (name: string, ids: string[]): string | null =>
|
||||
ids.includes(name) ? name : Object.keys(CASE_TEST_NAMES).find(id => CASE_TEST_NAMES[id] === name && ids.includes(id)) ?? ids.find(id => name.includes(id)) ?? null;
|
||||
|
||||
/**
|
||||
* Raw commands per case. A case shard (`<stem>--<id>`) is that case; a file
|
||||
* shard's per-test commands go to the case its test name names, and its
|
||||
* file-level commands to every case of the file that ran (by JUnit name, else
|
||||
* all of them): conservative over-attribution, as in the measured prototype.
|
||||
*/
|
||||
export function attributeCommands(shards: ShardCommands[], maps = PR_PROFILE_MAPS): Map<string, string[]> {
|
||||
const ids = Object.keys(maps.e2eTouchfiles);
|
||||
const byFile = new Map<string, string[]>();
|
||||
for (const id of ids) { const owner = caseOwnerFile(id, maps); if (owner) byFile.set(owner, [...byFile.get(owner) ?? [], id]); }
|
||||
const out = new Map<string, string[]>();
|
||||
const add = (id: string, commands: string[]) => out.set(id, [...out.get(id) ?? [], ...commands]);
|
||||
for (const shard of shards) {
|
||||
const [stem, caseKey] = shard.shard.split('--');
|
||||
const caseId = caseKey?.replace(/\.t\d+$/, '');
|
||||
const owned = byFile.get(`test/${stem}.test.ts`) ?? [];
|
||||
const ran = owned.filter(id => shard.executed.some(name => testNameCase(name, [id]) === id));
|
||||
const targets = caseId && ids.includes(caseId) ? [caseId] : ran.length ? ran : owned;
|
||||
for (const id of targets) add(id, shard.shardLevel);
|
||||
for (const [name, commands] of Object.entries(shard.perTest)) {
|
||||
const id = testNameCase(name, targets) ?? testNameCase(name, ids);
|
||||
for (const target of id ? [id] : targets) add(target, commands);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Known trial cost per case summed over the trial-outcomes.jsonl files under `dir` (executed trials only). */
|
||||
export function readCaseCosts(dir: string): Map<string, number> {
|
||||
const out = new Map<string, number>();
|
||||
if (!fs.existsSync(dir)) return out;
|
||||
for (const file of walk(dir).filter(file => path.basename(file) === 'trial-outcomes.jsonl')) {
|
||||
for (const line of fs.readFileSync(file, 'utf8').split('\n').filter(Boolean)) {
|
||||
try {
|
||||
const record = JSON.parse(line);
|
||||
if (record.execution === 'executed' && record.cost_known !== false && typeof record.case === 'string') out.set(record.case, (out.get(record.case) ?? 0) + (Number(record.cost_usd) || 0));
|
||||
} catch { /* malformed line */ }
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** One run's observations as a map: the bins each attributed case invoked, and its known cost. */
|
||||
export function observeRun(commandsByCase: Map<string, string[]>, ex: Pick<ExercisedConsumers, 'binsInCommand'>, run: { runId: string; observedAt: string; costs?: Map<string, number> }): ExercisedMap {
|
||||
const cases: Record<string, ExercisedCase> = {};
|
||||
for (const [id, commands] of commandsByCase) {
|
||||
cases[id] = { observations: 1, withCommands: commands.length ? 1 : 0, commands: [...new Set(commands.flatMap(command => ex.binsInCommand(command)))].sort() };
|
||||
}
|
||||
for (const [id, cost] of run.costs ?? []) {
|
||||
cases[id] ??= { observations: 1, withCommands: 0, commands: [] };
|
||||
Object.assign(cases[id], { costUsd: Math.round(cost * 10000) / 10000, costRuns: 1 });
|
||||
}
|
||||
return { schema: 1, generatedAt: run.observedAt, observedThrough: run.observedAt, sourceRuns: [run.runId], cases };
|
||||
}
|
||||
|
||||
/**
|
||||
* Union-only merge: commands are unioned, counts add for runs the base has
|
||||
* not seen, costs average by run count, and nothing the base holds is ever
|
||||
* removed (an absent or empty case in `fresh` changes nothing).
|
||||
*/
|
||||
export function mergeExercisedMap(base: ExercisedMap, fresh: ExercisedMap, generatedAt = new Date().toISOString()): ExercisedMap {
|
||||
const counts = fresh.sourceRuns.some(run => !base.sourceRuns.includes(run));
|
||||
const cases: Record<string, ExercisedCase> = structuredClone(base.cases);
|
||||
for (const [id, entry] of Object.entries(fresh.cases)) {
|
||||
const old = cases[id];
|
||||
const merged: ExercisedCase = {
|
||||
observations: (old?.observations ?? 0) + (counts ? entry.observations : 0),
|
||||
withCommands: (old?.withCommands ?? 0) + (counts ? entry.withCommands : 0),
|
||||
commands: [...new Set([...old?.commands ?? [], ...entry.commands])].sort(),
|
||||
};
|
||||
const oldRuns = old?.costRuns ?? 0, freshRuns = counts && entry.costUsd !== undefined ? entry.costRuns ?? 1 : 0;
|
||||
if (freshRuns) Object.assign(merged, { costUsd: Math.round(((old?.costUsd ?? 0) * oldRuns + entry.costUsd! * freshRuns) / (oldRuns + freshRuns) * 10000) / 10000, costRuns: oldRuns + freshRuns });
|
||||
else if (old?.costUsd !== undefined) Object.assign(merged, { costUsd: old.costUsd, costRuns: oldRuns });
|
||||
cases[id] = merged;
|
||||
}
|
||||
return {
|
||||
schema: 1, generatedAt,
|
||||
observedThrough: [base.observedThrough, fresh.observedThrough].sort().at(-1)!,
|
||||
sourceRuns: [...new Set([...base.sourceRuns, ...fresh.sourceRuns])].sort(),
|
||||
cases: Object.fromEntries(Object.entries(cases).sort(([a], [b]) => a.localeCompare(b))),
|
||||
};
|
||||
}
|
||||
|
||||
export function formatExercisedMap(map: ExercisedMap): string {
|
||||
return `${JSON.stringify(map, null, 2)}\n`;
|
||||
}
|
||||
|
||||
/** `MAP_GAP <case> <command>`: a recorded command whose files all lie outside the case's static-plus-observed exercised closure. */
|
||||
export function mapGaps(commandsByCase: Map<string, string[]>, ex: ExercisedConsumers): Array<{ id: string; command: string }> {
|
||||
const gaps: Array<{ id: string; command: string }> = [];
|
||||
for (const [id, commands] of [...commandsByCase].sort(([a], [b]) => a.localeCompare(b))) {
|
||||
const closure = ex.closure(id);
|
||||
for (const command of [...new Set(commands.flatMap(raw => ex.binsInCommand(raw)))].sort()) {
|
||||
const files = ex.commandFiles(command);
|
||||
if (files.length && !files.some(file => closure.has(file))) gaps.push({ id, command });
|
||||
}
|
||||
}
|
||||
return gaps;
|
||||
}
|
||||
|
||||
/** Executed cases that recorded no command although their loaded skill's shell runs bins: blind to gap detection. */
|
||||
export function blindCases(commandsByCase: Map<string, string[]>, ex: ExercisedConsumers): string[] {
|
||||
return [...commandsByCase].filter(([id, commands]) => commands.length === 0
|
||||
&& [...ex.closure(id)].some(([file, label]) => file.startsWith('bin/') && label.startsWith('skill bash via'))).map(([id]) => id).sort();
|
||||
}
|
||||
|
||||
/** Stable exit codes of test:exercised-map (docs/TESTING_INTERNALS.md#exercised-path-map). */
|
||||
export const REFRESH_EXIT = { usage: 2, unavailable: 3, incomplete: 4 } as const;
|
||||
|
||||
/** A refresh that cannot be trusted: nothing is written and the committed union stays as it was. */
|
||||
export class RefreshError extends Error {
|
||||
constructor(readonly exitCode: typeof REFRESH_EXIT.unavailable | typeof REFRESH_EXIT.incomplete, message: string) { super(message); }
|
||||
}
|
||||
|
||||
/** One MAP_GAP line, saying who it is for and what follows from it. */
|
||||
export function formatMapGap(gap: { id: string; command: string }): string {
|
||||
return `MAP_GAP ${gap.id} ${gap.command}: non-blocking for this PR; the next census refresh widens the map (${EXERCISED_MAP_ANCHOR})`;
|
||||
}
|
||||
|
||||
function writeAtomically(file: string, text: string): void {
|
||||
fs.mkdirSync(path.dirname(path.resolve(file)), { recursive: true });
|
||||
const temp = `${file}.${process.pid}.tmp`;
|
||||
fs.writeFileSync(temp, text);
|
||||
fs.renameSync(temp, file);
|
||||
}
|
||||
|
||||
/**
|
||||
* Census refresh from local report directories (paid-slice artifacts under
|
||||
* shards/, plus trial-outcomes.jsonl). Every executed case must have shard
|
||||
* evidence; a missing directory, no shards, no trial outcomes or an executed
|
||||
* case without a shard throws before anything is written.
|
||||
*/
|
||||
export function refreshFromReportDirs(dirs: string[], runId: string, base: ExercisedMap, ex: ExercisedConsumers, observedAt = new Date().toISOString(), maps = PR_PROFILE_MAPS) {
|
||||
const missing = dirs.filter(dir => !fs.existsSync(dir));
|
||||
if (missing.length) throw new RefreshError(REFRESH_EXIT.unavailable, `report director${missing.length > 1 ? 'ies' : 'y'} not found: ${missing.join(', ')}`);
|
||||
const shards = dirs.flatMap(readShardCommands);
|
||||
if (shards.length === 0) throw new RefreshError(REFRESH_EXIT.incomplete, `no shards/<shard>/ directories under ${dirs.join(', ')}; download every paid-slice-* artifact of the run first`);
|
||||
const outcomes = dirs.flatMap(dir => walk(dir).filter(file => path.basename(file) === 'trial-outcomes.jsonl'));
|
||||
if (outcomes.length === 0) throw new RefreshError(REFRESH_EXIT.incomplete, `no trial-outcomes.jsonl under ${dirs.join(', ')}, so the executed cases cannot be checked against the shard evidence`);
|
||||
const commands = attributeCommands(shards, maps);
|
||||
const executed = new Set(outcomes.flatMap(file => fs.readFileSync(file, 'utf8').split('\n').flatMap(line => {
|
||||
try { const record = JSON.parse(line); return record.execution === 'executed' && typeof record.case === 'string' ? [record.case as string] : []; } catch { return []; }
|
||||
})));
|
||||
const uncovered = [...executed].filter(id => Object.hasOwn(maps.e2eTouchfiles, id) && !commands.has(id)).sort();
|
||||
if (uncovered.length) throw new RefreshError(REFRESH_EXIT.incomplete, `${uncovered.length} executed case(s) have no shard artifact: ${uncovered.slice(0, 10).join(', ')}${uncovered.length > 10 ? ' ...' : ''}`);
|
||||
const costs = new Map<string, number>();
|
||||
for (const dir of dirs) for (const [id, cost] of readCaseCosts(dir)) costs.set(id, (costs.get(id) ?? 0) + cost);
|
||||
const merged = mergeExercisedMap(base, observeRun(commands, ex, { runId, observedAt, costs }), observedAt);
|
||||
return { merged, commands };
|
||||
}
|
||||
|
||||
function census(dirs: string[], runId: string, out: string, reportPath: string | undefined): void {
|
||||
const ex = checkoutExercisedConsumers(PR_PROFILE_MAPS, ROOT, PR_FULL_GATE_FILES);
|
||||
const base = ex.map!;
|
||||
const { merged, commands } = refreshFromReportDirs(dirs, runId, base, ex);
|
||||
writeAtomically(out, formatExercisedMap(merged));
|
||||
const added = Object.entries(merged.cases).flatMap(([id, entry]) => entry.commands.filter(command => !base.cases[id]?.commands.includes(command)).map(command => `${id} ${command}`));
|
||||
const gaps = mapGaps(commands, ex);
|
||||
const blind = blindCases(commands, ex);
|
||||
const lines = [
|
||||
'## Exercised-path map (shadow, non-blocking)', '',
|
||||
`- ${commands.size} executed case(s), ${[...commands.values()].reduce((sum, list) => sum + list.length, 0)} recorded command(s); ${added.length} new observed edge(s).`,
|
||||
`- Refresh: \`bun run test:exercised-map --from-run ${runId}\`, then open the change as an ordinary PR (shadow-only data: it selects no paid case).`,
|
||||
`- MAP_GAP (a command outside its case's static-plus-observed closure): ${gaps.length}`,
|
||||
...gaps.map(gap => ` - ${formatMapGap(gap)}`),
|
||||
`- Cases that recorded no commands although their skill runs bins (PTY capture is the activation prerequisite): ${blind.length ? blind.join(', ') : 'none'}`,
|
||||
...added.slice(0, 50).map(edge => ` - new: ${edge}`),
|
||||
`- Docs: ${EXERCISED_MAP_ANCHOR}`,
|
||||
];
|
||||
console.log(lines.join('\n'));
|
||||
if (reportPath) fs.writeFileSync(reportPath, `${lines.join('\n')}\n`);
|
||||
}
|
||||
|
||||
/** Union every exercised-commands.json of a downloaded exercised-map artifact into `base`; any missing or malformed file throws. */
|
||||
export function mergeArtifactDir(dir: string, base: ExercisedMap, generatedAt = new Date().toISOString()): ExercisedMap {
|
||||
const files = fs.existsSync(dir) ? walk(dir).filter(file => path.basename(file) === 'exercised-commands.json') : [];
|
||||
if (files.length === 0) throw new RefreshError(REFRESH_EXIT.incomplete, `no exercised-commands.json in ${dir} (the census refresh step failed or its ${EXERCISED_MAP_ARTIFACT}-a<attempt> artifact is partial)`);
|
||||
let map = base;
|
||||
for (const file of files.sort()) {
|
||||
let fresh: ExercisedMap;
|
||||
try { fresh = parseExercisedMap(fs.readFileSync(file, 'utf8')); }
|
||||
catch (error) { throw new RefreshError(REFRESH_EXIT.incomplete, `${path.relative(dir, file)} is not a complete exercised map: ${error instanceof Error ? error.message : String(error)}`); }
|
||||
map = mergeExercisedMap(map, fresh, generatedAt);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
export type ArtifactDownload = (runId: string, dir: string) => { status: number | null; stderr: string };
|
||||
|
||||
const ghDownload: ArtifactDownload = (runId, dir) => {
|
||||
const result = spawnSync('gh', ['run', 'download', runId, '--pattern', `${EXERCISED_MAP_ARTIFACT}-*`, '--dir', dir], { encoding: 'utf8', timeout: 120_000 });
|
||||
return { status: result.status, stderr: result.stderr || result.error?.message || '' };
|
||||
};
|
||||
|
||||
/** Download a census run's exercised-map artifact and union it into `target`; on any failure nothing is written. */
|
||||
export function refreshFromRun(runId: string, target: string, download: ArtifactDownload = ghDownload): ExercisedMap {
|
||||
if (!/^\d+$/.test(runId)) throw new RefreshError(REFRESH_EXIT.unavailable, `--from-run needs a numeric GitHub run id, got ${runId}`);
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'exercised-map-'));
|
||||
try {
|
||||
const result = download(runId, dir);
|
||||
if (result.status !== 0) {
|
||||
const reason = result.stderr.trim();
|
||||
const hint = /\b403\b/.test(reason)
|
||||
? `HTTP 403 (the installation's artifact download limit); download the ${EXERCISED_MAP_ARTIFACT}-a<attempt> artifact from the run page and pass its directory with --from-dir`
|
||||
: `check \`gh auth status\` and that run ${runId} uploaded an ${EXERCISED_MAP_ARTIFACT}-a<attempt> artifact`;
|
||||
throw new RefreshError(REFRESH_EXIT.unavailable, `gh run download ${runId} failed: ${reason || `exit ${result.status}`}; ${hint}`);
|
||||
}
|
||||
const map = mergeArtifactDir(dir, parseExercisedMap(fs.readFileSync(target, 'utf8')));
|
||||
writeAtomically(target, formatExercisedMap(map));
|
||||
return map;
|
||||
} finally { fs.rmSync(dir, { recursive: true, force: true }); }
|
||||
}
|
||||
|
||||
export const REPLAY_FIXTURE = 'test/fixtures/exercised-map-replay.json';
|
||||
|
||||
export interface ReplayRun { runId: number; pr: number; packageVersionOnly: boolean; failedGateCases: string[]; changedFiles: string[] }
|
||||
|
||||
/** One recorded red run under today's selection and its shadow: which failed gate cases each would run. */
|
||||
export interface ShadowReplayRow { runId: number; pr: number; mode: string; today: number; shadow: number; savingUsd: number; caughtToday: string[]; misses: string[]; retired: string[] }
|
||||
|
||||
export function replayShadow(runs: readonly ReplayRun[]): ShadowReplayRow[] {
|
||||
const gateIds = Object.keys(PR_PROFILE_MAPS.tiers).filter(id => PR_PROFILE_MAPS.tiers[id] === 'gate');
|
||||
return runs.map(run => {
|
||||
const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: run.changedFiles, packageVersionOnly: run.packageVersionOnly });
|
||||
const today = new Set(result.selection.e2e ?? gateIds);
|
||||
const shadow = result.coverage?.shadow;
|
||||
if (shadow?.error) throw new Error(`run ${run.runId}: shadow unavailable (${shadow.error})`);
|
||||
const shadowSet = new Set(shadow?.e2e ?? today);
|
||||
const live = run.failedGateCases.filter(id => PR_PROFILE_MAPS.tiers[id] === 'gate');
|
||||
return { runId: run.runId, pr: run.pr, mode: result.coverage?.mode ?? 'full', today: today.size, shadow: shadowSet.size, savingUsd: shadow?.predictedSavingUsd ?? 0,
|
||||
caughtToday: live.filter(id => today.has(id)), misses: live.filter(id => !shadowSet.has(id)), retired: run.failedGateCases.filter(id => !live.includes(id)) };
|
||||
});
|
||||
}
|
||||
|
||||
export function formatShadowReplay(rows: readonly ShadowReplayRow[]): string[] {
|
||||
const pairs = (list: readonly ShadowReplayRow[]) => list.reduce((sum, row) => sum + new Set([...row.caughtToday, ...row.misses]).size, 0);
|
||||
const misses = rows.flatMap(row => row.misses.map(id => `${row.runId} #${row.pr} ${id}`));
|
||||
const modes = new Map<string, number>();
|
||||
for (const row of rows) modes.set(row.mode, (modes.get(row.mode) ?? 0) + 1);
|
||||
const mean = (pick: (row: ShadowReplayRow) => number) => rows.length ? (rows.reduce((sum, row) => sum + pick(row), 0) / rows.length).toFixed(1) : 'n/a';
|
||||
return [
|
||||
`Exercised-path shadow replay over ${rows.length} recorded red PR run(s) on this checkout`,
|
||||
`- failed gate (run, case) pairs: ${pairs(rows)}, ${pairs(rows.filter(row => row.mode !== 'full-fallback'))} of them in runs that do not fall back to the full gate (${rows.reduce((sum, row) => sum + row.retired.length, 0)} more on cases no longer in the gate)`,
|
||||
`- today's selection runs ${rows.reduce((sum, row) => sum + row.caughtToday.length, 0)} of them; modes ${[...modes].map(([mode, n]) => `${mode} ${n}`).join(', ')}`,
|
||||
`- mean gate cases: today ${mean(row => row.today)}, shadow ${mean(row => row.shadow)}; predicted shadow saving over these runs $${rows.reduce((sum, row) => sum + row.savingUsd, 0).toFixed(2)}`,
|
||||
`- MISSES (failed, not in the shadow selection): ${misses.length}`,
|
||||
...misses.map(miss => ` - ${miss}`),
|
||||
];
|
||||
}
|
||||
|
||||
const USAGE = 'usage: bun run test:exercised-map --from-run <id> [--out <file>] | --from-dir <dir>... [--run-id <id>] [--out <file>] [--report <file>] | --check | --replay [<fixture>]';
|
||||
|
||||
if (import.meta.main) {
|
||||
const argv = process.argv.slice(2);
|
||||
const values = (flag: string) => argv.flatMap((arg, index) => arg === flag && argv[index + 1] ? [argv[index + 1]!] : []);
|
||||
const out = values('--out')[0] ?? path.join(ROOT, EXERCISED_MAP_PATH);
|
||||
try {
|
||||
if (argv.includes('--check')) {
|
||||
const map = parseExercisedMap(fs.readFileSync(path.join(ROOT, EXERCISED_MAP_PATH), 'utf8'));
|
||||
console.log(`${EXERCISED_MAP_PATH}: ${Object.keys(map.cases).length} cases, ${map.sourceRuns.length} source runs, observed through ${map.observedThrough}`);
|
||||
} else if (argv.includes('--replay')) {
|
||||
const fixture = JSON.parse(fs.readFileSync(path.resolve(ROOT, values('--replay')[0] ?? REPLAY_FIXTURE), 'utf8')) as { runs: ReplayRun[] };
|
||||
const rows = replayShadow(fixture.runs);
|
||||
console.log(formatShadowReplay(rows).join('\n'));
|
||||
if (rows.some(row => row.misses.length)) process.exit(1);
|
||||
} else if (values('--from-run').length) {
|
||||
const runId = values('--from-run')[0]!;
|
||||
const map = refreshFromRun(runId, out);
|
||||
console.log(`union-merged run ${runId}'s ${EXERCISED_MAP_ARTIFACT} artifact into ${path.relative(ROOT, out)} (${Object.keys(map.cases).length} cases, observed through ${map.observedThrough}); review the diff and open it as a PR`);
|
||||
} else if (values('--from-dir').length) {
|
||||
const dirs = values('--from-dir');
|
||||
if (dirs.length === 1 && fs.existsSync(dirs[0]!) && readShardCommands(dirs[0]!).length === 0 && walk(dirs[0]!).some(file => path.basename(file) === 'exercised-commands.json')) {
|
||||
const map = mergeArtifactDir(dirs[0]!, parseExercisedMap(fs.readFileSync(out, 'utf8')));
|
||||
writeAtomically(out, formatExercisedMap(map));
|
||||
console.log(`union-merged the ${EXERCISED_MAP_ARTIFACT} artifact in ${dirs[0]} into ${path.relative(ROOT, out)}; review the diff and open it as a PR`);
|
||||
} else {
|
||||
const runId = values('--run-id')[0];
|
||||
if (!runId) { console.error(`test:exercised-map: --from-dir with report directories needs --run-id <id> (the run whose artifacts these are)\n${USAGE}`); process.exit(REFRESH_EXIT.usage); }
|
||||
census(dirs, runId, out, values('--report')[0]);
|
||||
}
|
||||
} else {
|
||||
console.error(USAGE);
|
||||
process.exit(REFRESH_EXIT.usage);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`test:exercised-map: ${error instanceof Error ? error.message : String(error)}${error instanceof RefreshError ? '; nothing was written, the committed union is unchanged' : ''}`);
|
||||
process.exit(error instanceof RefreshError ? error.exitCode : 1);
|
||||
}
|
||||
}
|
||||
@@ -12,11 +12,11 @@
|
||||
"files": {
|
||||
"scripts/lib/paid-cases.ts": {"blob":"9bacd52446192a7ecf3dfaae901ea443cf9c9ae2","since":5},
|
||||
"scripts/lib/paid-plan.ts": {"blob":"45721ce128a112666a0caed6c13c66637b74ca54","since":5},
|
||||
"scripts/lib/paid-report.ts": {"blob":"3fda96a8b1f7f3d55797934847353f254cc4d054","since":5},
|
||||
"scripts/lib/paid-report.ts": {"blob":"b66c41237611ccecaaa7bd6a4d7b7aac8dc548cd","since":5,"nonBehavioral":"A2 exercised-path map: test-pr-profile.ts adds a shadow block and classifies the shadow-only map file as selecting no case (selection only); paid-report.ts writes non-blocking MAP_GAP lines after the verdict inputs, inside try/catch, never a verdict or lane problem"},
|
||||
"scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":5},
|
||||
"scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":5},
|
||||
"scripts/test-paid-shards.ts": {"blob":"8ddee5f11ed7856876135520dd8025ab989e2db9","since":5},
|
||||
"scripts/test-pr-profile.ts": {"blob":"ddc1e5991636c8c94722cd32472edce26489b26d","since":5},
|
||||
"scripts/test-pr-profile.ts": {"blob":"2029a89e14f511866cc91bebfee00b4fe9e7cf63","since":5,"nonBehavioral":"A2 exercised-path map: test-pr-profile.ts adds a shadow block and classifies the shadow-only map file as selecting no case (selection only); paid-report.ts writes non-blocking MAP_GAP lines after the verdict inputs, inside try/catch, never a verdict or lane problem"},
|
||||
"scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":5},
|
||||
"test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":5},
|
||||
"test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":5},
|
||||
|
||||
@@ -22,6 +22,9 @@ import { type ManifestEntry, PAID_TEST_DURATIONS_FILE, type PaidRunManifest, typ
|
||||
import { DEFAULT_JOBS, type PaidCaseSelection, type PaidTier, ROOT, type ShardTrialRecord } from './paid-types';
|
||||
import { collectPaidTestFiles, isAllSkippedPass, shardSlug, paidSelectionEnv } from './paid-select';
|
||||
import type { RunShardsOptions } from '../test-paid-shards';
|
||||
import { attributeCommands, formatMapGap, mapGaps, readShardCommands } from '../exercised-map';
|
||||
import { checkoutExercisedConsumers } from '../pr-dependencies';
|
||||
import { PR_FULL_GATE_FILES, PR_PROFILE_MAPS } from '../test-pr-profile';
|
||||
|
||||
// ─── Local diagnosis: one case through the CI panel runner (A9) ────────────
|
||||
|
||||
@@ -482,6 +485,30 @@ export function infraOnly(problems: readonly string[]): boolean {
|
||||
return problems.length > 0 && problems.every(problem => INFRA_PROBLEMS.some(re => re.test(problem)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Shadow gap detection (docs/TESTING_INTERNALS.md#exercised-path-map): a
|
||||
* command an executed case ran outside its static-plus-observed exercised
|
||||
* closure prints a MAP_GAP line (formatMapGap: non-blocking for this PR, the
|
||||
* next census refresh widens the map) and lands in map-gaps.txt. It is its own
|
||||
* result: never a case verdict, a lane problem or a re-dispatch input.
|
||||
*/
|
||||
export function reportMapGaps(reportDir: string, executedCases: ReadonlySet<string>, rootDir = ROOT): string[] {
|
||||
const out = path.join(reportDir, 'map-gaps.txt');
|
||||
try {
|
||||
const commands = new Map([...attributeCommands(readShardCommands(reportDir))].filter(([id, list]) => executedCases.has(id) && list.length > 0));
|
||||
const lines = commands.size === 0 ? [] : mapGaps(commands, checkoutExercisedConsumers(PR_PROFILE_MAPS, rootDir, PR_FULL_GATE_FILES)).map(formatMapGap);
|
||||
fs.writeFileSync(out, lines.length ? `${lines.join('\n')}\n` : '');
|
||||
console.log(`[test:paid] exercised-path map (shadow, non-blocking): ${lines.length} MAP_GAP line(s) over ${commands.size} executed case(s) with recorded commands`);
|
||||
for (const line of lines) console.log(` ${line}`);
|
||||
return lines;
|
||||
} catch (error) {
|
||||
const reason = error instanceof Error ? error.message : String(error);
|
||||
fs.writeFileSync(out, `MAP_GAP unavailable: ${reason}\n`);
|
||||
console.log(`[test:paid] exercised-path map (shadow, non-blocking): MAP_GAP unavailable: ${reason}`);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Report mode: reconcile slice artifacts against the manifest (fail-closed),
|
||||
* compute every panel verdict with panelVerdict(), write collector-outcomes
|
||||
@@ -743,6 +770,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
|
||||
...(t.exit_reason ? { exit_reason: t.exit_reason } : {}), ...(t.error ? { error: t.error } : {}) })) })),
|
||||
// evals.yml wraps these in a fixed fence for the PR comment.
|
||||
failures: failureLines.map(line => sanitizeFixedFenceLine(line)) }, null, 2) + '\n');
|
||||
reportMapGaps(reportDir, new Set(executed.map(record => record.case)), rootDir);
|
||||
if (verdict.problems.length) {
|
||||
console.error(`[test:paid] report: ${verdict.problems.length} problem(s):`);
|
||||
for (const problem of verdict.problems) console.error(` ✗ ${problem}`);
|
||||
|
||||
+335
-14
@@ -49,6 +49,12 @@ export interface DerivedDependencies {
|
||||
tracked: Set<string>;
|
||||
/** The reference chain from an E2E case's declared inputs to `file` (for fix text and audits), else null. */
|
||||
trace(caseId: string, file: string): string[] | null;
|
||||
/**
|
||||
* Files whose import scan failed for a reason other than a leading `#!`
|
||||
* line (their imports are unknown). PR selection restores the full gate
|
||||
* while this is non-empty; it fills as the derivation reads files.
|
||||
*/
|
||||
scanErrors: Map<string, string>;
|
||||
/**
|
||||
* Tracked files whose reference text still names `missing`, a path absent
|
||||
* from the tree (deleted in the diff): its path or extensionless import
|
||||
@@ -60,8 +66,10 @@ export interface DerivedDependencies {
|
||||
}
|
||||
|
||||
const CODE = /\.(?:[cm]?[jt]s|tsx|jsx)$/;
|
||||
/** An extensionless script Bun or Node runs: its imports are code, not shell. */
|
||||
const CODE_SHEBANG = /^#![^\n]*\b(?:bun|node)\b/;
|
||||
const TEXT = /\.(?:[cm]?[jt]s|tsx|jsx|sh|md|tmpl|json|ya?ml|toml|txt|py)$/;
|
||||
const REPO_PATH = /(?:^|[^\w.@-])((?:bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents|test\/helpers|test\/fixtures)\/[\w.@-]+(?:\/[\w.@-]+)*)/g;
|
||||
const REPO_PATH = /(?:^|[^\w.@-])((?:bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents|capy|test\/helpers|test\/fixtures)\/[\w.@-]+(?:\/[\w.@-]+)*)/g;
|
||||
/** `path.join(ROOT, 'lib', 'x.ts')`: a top-level directory and its entry as separate string literals. */
|
||||
const JOINED_PATH = /['"`](bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents)['"`]\s*,\s*['"`]([\w.@-]+)['"`]/g;
|
||||
const BIN_NAME = /\bgstack-[a-z0-9][a-z0-9-]*[a-z0-9]\b/g;
|
||||
@@ -82,10 +90,31 @@ export const SELECTION_DATA_MODULES: readonly string[] = [
|
||||
'scripts/build.sh',
|
||||
// The reason-code table: its `fix:` prose names the command a person reruns, not a file the module consumes.
|
||||
'lib/result-codes.ts', 'lib/result-codes-release.ts',
|
||||
// The exercised-path map: observed commands per case, read only by exercisedConsumers() (shadow data; a change selects no paid case).
|
||||
'test/helpers/exercised-commands.json',
|
||||
];
|
||||
const DEFAULT_JUDGE_OWNER = 'test/skill-llm-eval.test.ts';
|
||||
|
||||
const transpiler = new Bun.Transpiler({ loader: 'tsx' });
|
||||
const transpilers = {
|
||||
ts: new Bun.Transpiler({ loader: 'ts' }), tsx: new Bun.Transpiler({ loader: 'tsx' }),
|
||||
js: new Bun.Transpiler({ loader: 'js' }), jsx: new Bun.Transpiler({ loader: 'jsx' }),
|
||||
};
|
||||
|
||||
/** Code by extension, or an extensionless file whose shebang runs Bun or Node. */
|
||||
export function isCodeFile(file: string, text: string): boolean {
|
||||
return CODE.test(file) || (!/\.[^/]+$/.test(path.posix.basename(file)) && CODE_SHEBANG.test(text));
|
||||
}
|
||||
|
||||
/**
|
||||
* Relative import specifiers of a code file. A leading `#!` line is blanked
|
||||
* first (`scanImports` rejects it), and each extension gets its own loader
|
||||
* (the TSX loader misreads `.ts` generics and casts). Any other parse error
|
||||
* throws, so callers fail closed.
|
||||
*/
|
||||
export function scanCodeImports(file: string, text: string): string[] {
|
||||
const loader = /\.tsx$/.test(file) ? 'tsx' : /\.jsx$/.test(file) ? 'jsx' : /\.[cm]?js$/.test(file) ? 'js' : 'ts';
|
||||
return transpilers[loader].scanImports(text.replace(/^#![^\n]*/, '')).map(item => item.path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Text a reference scan reads: whole-line comments are prose about other
|
||||
@@ -168,15 +197,17 @@ export function deriveDependencies(maps: DependencyMaps, root: string): DerivedD
|
||||
};
|
||||
const dirs = new Set(tracked.flatMap(file => file.split('/').slice(0, -1).map((_, i, parts) => parts.slice(0, i + 1).join('/'))));
|
||||
|
||||
const scanErrors = new Map<string, string>();
|
||||
const importsOf = (file: string): string[] => {
|
||||
try { return scanCodeImports(file, read(file)); } catch (error) { scanErrors.set(file, String(error)); return []; }
|
||||
};
|
||||
const edges = new Map<string, string[]>();
|
||||
const refs = (file: string): string[] => {
|
||||
if (edges.has(file)) return edges.get(file)!;
|
||||
const raw = read(file);
|
||||
const text = (SELECTION_DATA_MODULES as readonly string[]).includes(file) ? '' : referenceText(file, raw);
|
||||
const out = new Set<string>();
|
||||
if (CODE.test(file)) {
|
||||
try { for (const item of transpiler.scanImports(raw)) { const hit = resolveSpecifier(root, item.path, file); if (hit) out.add(hit); } } catch { /* literal scan below still applies */ }
|
||||
}
|
||||
if (isCodeFile(file, raw)) for (const specifier of importsOf(file)) { const hit = resolveSpecifier(root, specifier, file); if (hit) out.add(hit); }
|
||||
for (const match of text.matchAll(REPO_PATH)) {
|
||||
const ref = match[1]!.replace(/[.]+$/, '');
|
||||
if (trackedSet.has(ref)) out.add(ref);
|
||||
@@ -238,19 +269,17 @@ export function deriveDependencies(maps: DependencyMaps, root: string): DerivedD
|
||||
const file = pending.pop()!;
|
||||
if (global.has(file)) continue;
|
||||
global.add(file);
|
||||
if (!CODE.test(file)) continue;
|
||||
try { for (const item of transpiler.scanImports(read(file))) { const hit = resolveSpecifier(root, item.path, file); if (hit && !global.has(hit)) pending.push(hit); } } catch { /* unparseable: its own entry stays global */ }
|
||||
if (!isCodeFile(file, read(file))) continue;
|
||||
for (const specifier of importsOf(file)) { const hit = resolveSpecifier(root, specifier, file); if (hit && !global.has(hit)) pending.push(hit); }
|
||||
}
|
||||
const importTargets = new Map<string, string[]>();
|
||||
const targetsOf = (file: string): string[] => {
|
||||
if (!importTargets.has(file)) {
|
||||
const targets: string[] = [];
|
||||
if (CODE.test(file) && !(SELECTION_DATA_MODULES as readonly string[]).includes(file)) {
|
||||
try {
|
||||
for (const item of transpiler.scanImports(read(file))) {
|
||||
if (item.path.startsWith('.')) targets.push(path.posix.normalize(path.posix.join(path.posix.dirname(file), item.path)));
|
||||
}
|
||||
} catch { /* literal mentions below still apply */ }
|
||||
if (isCodeFile(file, read(file)) && !(SELECTION_DATA_MODULES as readonly string[]).includes(file)) {
|
||||
for (const specifier of importsOf(file)) {
|
||||
if (specifier.startsWith('.')) targets.push(path.posix.normalize(path.posix.join(path.posix.dirname(file), specifier)));
|
||||
}
|
||||
}
|
||||
importTargets.set(file, targets);
|
||||
}
|
||||
@@ -287,7 +316,7 @@ export function deriveDependencies(maps: DependencyMaps, root: string): DerivedD
|
||||
while (parents.has(chain[0]!)) chain.unshift(parents.get(chain[0]!)!);
|
||||
return chain;
|
||||
};
|
||||
return { e2e, judges: consumers(maps.judgeTouchfiles, DEFAULT_JUDGE_OWNER), everyCase, global, tracked: trackedSet, trace, referencers };
|
||||
return { e2e, judges: consumers(maps.judgeTouchfiles, DEFAULT_JUDGE_OWNER), everyCase, global, tracked: trackedSet, trace, referencers, scanErrors };
|
||||
}
|
||||
|
||||
const cache = new Map<string, DerivedDependencies>();
|
||||
@@ -299,3 +328,295 @@ export function derivedDependencies(maps: DependencyMaps, root: string): Derived
|
||||
return cache.get(key)!;
|
||||
}
|
||||
|
||||
// ─── Exercised-path consumer map (W2-A2, shadow mode) ──────────────────────
|
||||
|
||||
/** The committed per-case observed commands (generated by scripts/exercised-map.ts; union-only refreshes). */
|
||||
export const EXERCISED_MAP_PATH = 'test/helpers/exercised-commands.json';
|
||||
|
||||
export interface ExercisedCase {
|
||||
/** Recorded trials (run, case) the commands were taken from. */
|
||||
observations: number;
|
||||
/** Of those, trials that recorded at least one command. */
|
||||
withCommands: number;
|
||||
/** Bins and compiled outfiles the case invoked (binsInCommand over every recorded command). */
|
||||
commands: string[];
|
||||
/** Mean known trial cost per run, for the shadow cost estimate. */
|
||||
costUsd?: number;
|
||||
costRuns?: number;
|
||||
}
|
||||
|
||||
export interface ExercisedMap {
|
||||
schema: 1;
|
||||
generatedAt: string;
|
||||
observedThrough: string;
|
||||
sourceRuns: string[];
|
||||
cases: Record<string, ExercisedCase>;
|
||||
}
|
||||
|
||||
export function isExercisedMap(value: unknown): value is ExercisedMap {
|
||||
const map = value as ExercisedMap;
|
||||
return !!map && map.schema === 1 && typeof map.generatedAt === 'string' && typeof map.observedThrough === 'string'
|
||||
&& Array.isArray(map.sourceRuns) && map.sourceRuns.every(run => typeof run === 'string')
|
||||
&& !!map.cases && typeof map.cases === 'object' && Object.values(map.cases).every(entry => !!entry
|
||||
&& Number.isSafeInteger(entry.observations) && Number.isSafeInteger(entry.withCommands)
|
||||
&& Array.isArray(entry.commands) && entry.commands.every(command => typeof command === 'string'));
|
||||
}
|
||||
|
||||
/** Parse an exercised map's text; throws on malformed input so callers fail closed. */
|
||||
export function parseExercisedMap(text: string): ExercisedMap {
|
||||
const parsed = JSON.parse(text);
|
||||
if (!isExercisedMap(parsed)) throw new Error(`${EXERCISED_MAP_PATH}: not a schema-1 exercised map`);
|
||||
return parsed;
|
||||
}
|
||||
|
||||
export interface ExercisedConsumers {
|
||||
/** Repository file -> case ids whose exercised closure contains it. */
|
||||
e2e: Map<string, Set<string>>;
|
||||
/** One case's exercised closure: file -> how it is reached (touchfile, import, skill bash via <file>, observed <command>, declared via <file>, host). */
|
||||
closure(caseId: string): Map<string, string>;
|
||||
/** Bins (repo paths) and compiled outfiles one recorded command invokes. */
|
||||
binsInCommand(command: string): string[];
|
||||
/** The tracked files a bin or compiled outfile runs. */
|
||||
commandFiles(command: string): string[];
|
||||
/** The observed map the consumers were built with. */
|
||||
map: ExercisedMap | null;
|
||||
}
|
||||
|
||||
const SHELL_SHEBANG = /^#![^\n]*\b(?:ba|z|da)?sh\b/;
|
||||
const MESSAGE_CALL = /^(?:echo|printf|say|warn|die|fail|err|log|info|note|_dc_row|row)\b/;
|
||||
const CMD_START = /(?:^|[;|&({\x60]|\$\(|\b(?:then|do|else|exec|command|nohup|bash|sh|bun|node|timeout\s+\S+|env(?:\s+\w+=\S*)*|if|while|until|!|time))\s*"?$/;
|
||||
const CMD_PREFIX = /(?:"?(?:\$\{?\w+\}?|\$\(dirname\s+"?\$0"?\)|~|\.)?(?:\/(?:[\w.@-]+\/)*|(?:[\w.@-]+\/)+))?"?$/;
|
||||
const INTERPRETER = /\b(?:bash|sh|zsh|bun|node|eval|source|python3?)\b/;
|
||||
|
||||
/** Text between `$(`…`)` and backticks in a span: substitutions run even inside a message. */
|
||||
function substitutions(span: string): string[] {
|
||||
const out: string[] = [];
|
||||
for (let i = 0; i < span.length; i++) {
|
||||
if (span[i] === '\\') { i++; continue; }
|
||||
if (span[i] === '\x60') { const end = span.indexOf('\x60', i + 1); if (end < 0) break; out.push(`$(${span.slice(i + 1, end)})`); i = end; continue; }
|
||||
if (span[i] !== '$' || span[i + 1] !== '(') continue;
|
||||
let depth = 0, j = i + 1;
|
||||
for (; j < span.length; j++) { if (span[j] === '(') depth++; else if (span[j] === ')' && --depth === 0) break; }
|
||||
out.push(span.slice(i, j + 1));
|
||||
i = j;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Message calls (echo, printf, die …) are prose; their command substitutions still execute and are kept. */
|
||||
function stripMessages(text: string): string {
|
||||
let out = '', i = 0;
|
||||
while (i < text.length) {
|
||||
const prev = i === 0 ? '\n' : text[i - 1]!;
|
||||
const call = /[\s;|&(\x60{]/.test(prev) ? MESSAGE_CALL.exec(text.slice(i, i + 8)) : null;
|
||||
if (!call) { out += text[i]; i++; continue; }
|
||||
let j = i + call[0].length, quote: string | null = null, depth = 0;
|
||||
while (j < text.length) {
|
||||
const c = text[j]!;
|
||||
if (c === '\\') { j += 2; continue; }
|
||||
if (quote) { if (c === quote) quote = null; else if (quote === '"' && c === '$' && text[j + 1] === '(') { depth++; j++; } else if (c === ')' && depth) depth--; j++; continue; }
|
||||
if (c === '"' || c === "'") { quote = c; j++; continue; }
|
||||
if (c === '$' && text[j + 1] === '(') { depth++; j += 2; continue; }
|
||||
if (c === ')' && depth) { depth--; j++; continue; }
|
||||
if (c === '\n' || c === ';' || c === '|' || c === '&' || c === ')') break;
|
||||
j++;
|
||||
}
|
||||
out += ` ${substitutions(text.slice(i, j)).join('; ')} `;
|
||||
i = j;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Executable shell text: comments, case patterns and message calls are
|
||||
* prose. A heredoc an interpreter reads stays whole; an expanding (unquoted)
|
||||
* heredoc keeps its command substitutions; a quoted one fed to anything else
|
||||
* is dropped.
|
||||
*/
|
||||
export function executableShell(text: string): string {
|
||||
return stripMessages(text
|
||||
.replace(/^(.*?)<<-?\s*(['"]?)(\w+)\2([^\n]*)\n([\s\S]*?)\n\s*\3\s*$/gm, (whole, head: string, quote: string, _word: string, rest: string, body: string) =>
|
||||
INTERPRETER.test(head + rest) ? whole : quote ? head : [head, ...substitutions(body)].join('\n'))
|
||||
.split('\n').filter(line => !/^\s*#(?!!)/.test(line))
|
||||
.map(line => line.replace(/(^\s*|\bin\s+|;;\s*)[\w*."'-]+(?:\s*\|\s*[\w*."'-]+)*\)(?=\s|$)/g, '$1').replace(/\s#\s.*$/, ''))
|
||||
.join('\n'));
|
||||
}
|
||||
|
||||
function commandPosition(before: string): boolean {
|
||||
const head = before.replace(CMD_PREFIX, '');
|
||||
return CMD_START.test(head) || (/\/$/.test(before) && /\b\w+="?$/.test(head));
|
||||
}
|
||||
|
||||
/**
|
||||
* Consumers by exercised paths: a case consumes its touchfile matches, the
|
||||
* closure of its test file, the executable shell of its loaded skill
|
||||
* (template and generated SKILL.md) and of every bin, lib and script that
|
||||
* shell runs, and the bins its recorded trials invoked (`observed`). The
|
||||
* closure stops at `stopAt` (the full-gate files) and the global set,
|
||||
* which stay full.
|
||||
*/
|
||||
export function exercisedConsumers(maps: DependencyMaps, root: string, options: { observed: ExercisedMap | null; stopAt: readonly string[] }): ExercisedConsumers {
|
||||
const base = derivedDependencies(maps, root);
|
||||
const tracked = [...base.tracked];
|
||||
const texts = new Map<string, string>();
|
||||
const read = (file: string): string => {
|
||||
if (!texts.has(file)) {
|
||||
let text = '';
|
||||
try { if (base.tracked.has(file) && fs.statSync(path.join(root, file)).size < 3_000_000) text = fs.readFileSync(path.join(root, file), 'utf8'); } catch { /* unreadable */ }
|
||||
texts.set(file, text);
|
||||
}
|
||||
return texts.get(file)!;
|
||||
};
|
||||
const isShell = (file: string) => !isCodeFile(file, read(file)) && (/\.sh$/.test(file) || file === 'setup' || SHELL_SHEBANG.test(read(file)));
|
||||
const dirs = new Set(tracked.flatMap(file => file.split('/').slice(0, -1).map((_, i, parts) => parts.slice(0, i + 1).join('/'))));
|
||||
const expandRef = (ref: string): string[] => {
|
||||
const clean = ref.replace(/[.]+$/, '');
|
||||
if (base.tracked.has(clean)) return [clean];
|
||||
return dirs.has(clean) ? tracked.filter(file => file.startsWith(`${clean}/`)) : [];
|
||||
};
|
||||
const expanded = new Map<string, string[]>();
|
||||
const expand = (pattern: string): string[] => {
|
||||
if (!pattern.includes('*')) return base.tracked.has(pattern) ? [pattern] : [];
|
||||
if (!expanded.has(pattern)) expanded.set(pattern, tracked.filter(file => matchGlob(file, pattern)));
|
||||
return expanded.get(pattern)!;
|
||||
};
|
||||
const binByName = new Map<string, string>();
|
||||
for (const file of tracked.filter(file => file.startsWith('bin/') || file.startsWith('browse/bin/'))) {
|
||||
binByName.set(path.posix.basename(file), file);
|
||||
binByName.set(path.posix.basename(file).replace(/\.(?:ts|sh|js)$/, ''), file);
|
||||
}
|
||||
const binaries = [...read('scripts/build.sh').matchAll(/--compile\b[^\n]*?\s([\w./-]+\.ts)\s+--outfile\s+([\w./-]+)/g)]
|
||||
.map(match => ({ outfile: match[2]!, tree: match[1]!.startsWith('bin/') ? [match[1]!] : tracked.filter(file => matchGlob(file, `${path.posix.dirname(match[1]!)}/**`)) }));
|
||||
const outfileHit = (outfile: string, text: string) => text.includes(outfile)
|
||||
|| (outfile === 'browse/dist/browse' && /\$B\b|\$\{B\}|find-browse/.test(text)) || (outfile === 'design/dist/design' && /\$D\b|\$\{D\}/.test(text));
|
||||
let scripts: Record<string, string> = {};
|
||||
try { scripts = JSON.parse(read('package.json')).scripts ?? {}; } catch { /* no package scripts */ }
|
||||
|
||||
const shellEdges = (text: string, from: string, out: Set<string>) => {
|
||||
for (const match of text.matchAll(/gstack-[a-z0-9][a-z0-9-]*[a-z0-9](?:\.(?:sh|ts))?\b/g)) {
|
||||
const start = text.lastIndexOf('\n', match.index! - 1) + 1;
|
||||
const bin = binByName.get(match[0]);
|
||||
if (bin && commandPosition(text.slice(start, match.index))) out.add(bin);
|
||||
}
|
||||
for (const match of text.matchAll(/(?:bin|lib|scripts|browse|design|make-pdf)\/[\w.@-]+(?:\/[\w.@-]+)*/g)) {
|
||||
const before = text.slice(text.lastIndexOf('\n', match.index! - 1) + 1, match.index);
|
||||
if (/[\w.@-]$/.test(before)) continue;
|
||||
if (commandPosition(before) || /\bbun\s+(?:run\s+)?"?(?:\$\{?\w+\}?\/)?$/.test(before)) for (const file of expandRef(match[0])) out.add(file);
|
||||
}
|
||||
for (const match of text.matchAll(/(?:bin|lib|scripts|browse|design|make-pdf|hosts|capy|test\/fixtures|test\/helpers)\/[\w.@-]+(?:\/[\w.@-]+)*\.(?:json|jsonl|env|txt|ya?ml|toml|csv|tsv)\b/g)) for (const file of expandRef(match[0])) out.add(file);
|
||||
for (const match of text.matchAll(/(?:^|[\s;&|({])(?:\.|source)\s+([^;|&\n]+)/gm)) {
|
||||
for (const ref of match[1]!.matchAll(/(?:bin|lib|scripts)\/[\w.@-]+(?:\/[\w.@-]+)*\.sh\b/g)) for (const file of expandRef(ref[0])) out.add(file);
|
||||
const name = [...match[1]!.matchAll(/([\w.-]+\.sh|gstack-[\w.-]+)/g)].map(found => found[1]!).at(-1);
|
||||
if (!name) continue;
|
||||
const sibling = path.posix.join(path.posix.dirname(from), name);
|
||||
if (base.tracked.has(sibling)) out.add(sibling);
|
||||
else if (binByName.has(name)) out.add(binByName.get(name)!);
|
||||
}
|
||||
if (/(?:^|[\s;&|(])"?(?:\.|\$\{?\w+\}?)\/setup\b/m.test(text) && base.tracked.has('setup')) out.add('setup');
|
||||
for (const { outfile, tree } of binaries) if (outfileHit(outfile, text)) for (const file of tree) out.add(file);
|
||||
for (const match of text.matchAll(PACKAGE_RUN)) {
|
||||
const command = scripts[match[1]!];
|
||||
if (command) for (const ref of command.matchAll(REPO_PATH)) for (const file of expandRef(ref[1]!)) out.add(file);
|
||||
}
|
||||
};
|
||||
const edges = new Map<string, string[]>();
|
||||
const execRefs = (file: string): string[] => {
|
||||
if (edges.has(file)) return edges.get(file)!;
|
||||
const raw = read(file);
|
||||
const out = new Set<string>();
|
||||
if (raw && !(SELECTION_DATA_MODULES as readonly string[]).includes(file)) {
|
||||
if (isCodeFile(file, raw)) {
|
||||
try { for (const specifier of scanCodeImports(file, raw)) { const hit = resolveSpecifier(root, specifier, file); if (hit) out.add(hit); } }
|
||||
catch (error) { base.scanErrors.set(file, String(error)); }
|
||||
const code = raw.replace(/^#![^\n]*/, '').replace(/\/\*[\s\S]*?\*\//g, '').replace(/^[ \t]*\/\/.*$/gm, '');
|
||||
for (const match of code.matchAll(/['"\x60](?:\$\{[^}]+\}\/)?((?:bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents|test\/helpers|test\/fixtures)\/[\w.@-]+(?:\/[\w.@-]+)*)['"\x60]/g)) for (const hit of expandRef(match[1]!)) out.add(hit);
|
||||
for (const match of code.matchAll(JOINED_PATH)) for (const hit of expandRef(`${match[1]}/${match[2]}`)) out.add(hit);
|
||||
if (/join\([^)\n]*['"]setup['"]\s*\)/.test(code) && base.tracked.has('setup')) out.add('setup');
|
||||
for (const { outfile, tree } of binaries) if (code.includes(`'${outfile}'`) || code.includes(`"${outfile}"`)) for (const hit of tree) out.add(hit);
|
||||
} else if (isShell(file)) {
|
||||
shellEdges(executableShell(raw), file, out);
|
||||
} else if (/\.(?:md|tmpl)$/.test(file)) {
|
||||
shellEdges(executableShell([...raw.matchAll(/\x60\x60\x60(?:bash|sh|shell)\n([\s\S]*?)\x60\x60\x60/g)].map(match => match[1]!).join('\n')), file, out);
|
||||
}
|
||||
}
|
||||
out.delete(file);
|
||||
edges.set(file, [...out]);
|
||||
return edges.get(file)!;
|
||||
};
|
||||
const stopped = new Map<string, boolean>();
|
||||
const stops = (file: string): boolean => {
|
||||
if (!stopped.has(file)) stopped.set(file, base.global.has(file) || options.stopAt.some(pattern => matchGlob(file, pattern)));
|
||||
return stopped.get(file)!;
|
||||
};
|
||||
const binsInCommand = (command: string): string[] => {
|
||||
const out = new Set<string>();
|
||||
const pty = command.startsWith('PTY:');
|
||||
for (const match of command.matchAll(/gstack-[a-z0-9-]*[a-z0-9](?:\.(?:sh|ts))?/g)) {
|
||||
let token = match[0];
|
||||
if (pty) while (token.length > 7 && !binByName.has(token)) { const cut = token.lastIndexOf('-'); token = cut > 6 ? token.slice(0, cut) : ''; }
|
||||
const at = match.index!;
|
||||
if (command[at - 1] === '/' && at >= 2 && /[\s"'\x60(]/.test(command[at - 2]!)) continue;
|
||||
const bin = binByName.get(token);
|
||||
if (bin) out.add(bin);
|
||||
}
|
||||
for (const match of command.matchAll(/browse\/bin\/([a-z-]+)/g)) { const bin = binByName.get(match[1]!); if (bin) out.add(bin); }
|
||||
for (const { outfile } of binaries) if (outfileHit(outfile, command) || (outfile === 'make-pdf/dist/pdf' && command.includes(outfile))) out.add(outfile);
|
||||
return [...out];
|
||||
};
|
||||
const commandFiles = (command: string) => base.tracked.has(command) ? [command] : binaries.find(binary => binary.outfile === command)?.tree ?? [];
|
||||
|
||||
const hostIds = tracked.filter(file => /^hosts\/[\w-]+\.ts$/.test(file) && file !== 'hosts/index.ts').map(file => path.posix.basename(file, '.ts'));
|
||||
const closures = new Map<string, Map<string, string>>();
|
||||
const caseClosure = (id: string): Map<string, string> => {
|
||||
if (closures.has(id)) return closures.get(id)!;
|
||||
const patterns = maps.e2eTouchfiles[id] ?? [];
|
||||
const owners = patterns.filter(pattern => OWNER.test(pattern) && isPaidTestFile(pattern));
|
||||
const declared = patterns.flatMap(expand);
|
||||
const generated = declared.filter(file => file.endsWith('.tmpl')).map(file => file.replace(/\.tmpl$/, '')).filter(file => base.tracked.has(file));
|
||||
const touchfiles = new Set([...declared, ...generated]);
|
||||
const skillDoc = (file: string) => /\.(?:md|tmpl)$/.test(file);
|
||||
const seeds: Array<[string, string]> = [
|
||||
...owners.map(file => [file, 'import'] as [string, string]),
|
||||
...[...declared, ...generated].map(file => [file, skillDoc(file) ? 'skill' : `declared via ${file}`] as [string, string]),
|
||||
...(options.observed?.cases[id]?.commands ?? []).flatMap(command => commandFiles(command).map(file => [file, `observed ${command}`] as [string, string])),
|
||||
];
|
||||
const ownerText = owners.map(read).join('\n');
|
||||
for (const host of hostIds) {
|
||||
if (new RegExp(`['"\x60]${host}['"\x60]|--host[ =]${host}\\b`).test(ownerText)) for (const file of [`hosts/${host}.ts`, ...expand(`hosts/${host}/**`)]) seeds.push([file, 'host']);
|
||||
}
|
||||
const reached = new Map<string, string>();
|
||||
const queue = seeds.filter(([file]) => base.tracked.has(file));
|
||||
for (let head = 0; head < queue.length; head++) {
|
||||
const [file, label] = queue[head]!;
|
||||
if (reached.has(file)) continue;
|
||||
reached.set(file, touchfiles.has(file) ? 'touchfile' : label);
|
||||
if (stops(file)) continue;
|
||||
for (const next of execRefs(file)) {
|
||||
if (reached.has(next) || (/\.test\.tsx?$/.test(next) && isPaidTestFile(next))) continue;
|
||||
queue.push([next, label === 'skill' ? `skill bash via ${next}` : label]);
|
||||
}
|
||||
}
|
||||
closures.set(id, reached);
|
||||
return reached;
|
||||
};
|
||||
const e2e = new Map<string, Set<string>>();
|
||||
for (const id of Object.keys(maps.e2eTouchfiles)) {
|
||||
for (const file of caseClosure(id).keys()) {
|
||||
if (!e2e.has(file)) e2e.set(file, new Set());
|
||||
e2e.get(file)!.add(id);
|
||||
}
|
||||
}
|
||||
return { e2e, closure: caseClosure, binsInCommand, commandFiles, map: options.observed };
|
||||
}
|
||||
|
||||
const exercisedCache = new Map<string, ExercisedConsumers>();
|
||||
|
||||
/** The exercised consumers of the checkout at `root` with its committed map, computed once per process; throws when the map is malformed. */
|
||||
export function checkoutExercisedConsumers(maps: DependencyMaps, root: string, stopAt: readonly string[]): ExercisedConsumers {
|
||||
const key = `${root}\0${Object.keys(maps.e2eTouchfiles).length}`;
|
||||
if (!exercisedCache.has(key)) {
|
||||
const observed = parseExercisedMap(fs.readFileSync(path.join(root, EXERCISED_MAP_PATH), 'utf8'));
|
||||
exercisedCache.set(key, exercisedConsumers(maps, root, { observed, stopAt }));
|
||||
}
|
||||
return exercisedCache.get(key)!;
|
||||
}
|
||||
@@ -26,7 +26,7 @@ import { packageChangeOnlyVersion } from './test-pr-profile';
|
||||
import { E2E_TIERS, E2E_TOUCHFILES, GLOBAL_TOUCHFILES } from '../test/helpers/touchfiles-data';
|
||||
import { matchGlob } from '../test/helpers/test-selection';
|
||||
import { derivedDependencies } from './pr-dependencies';
|
||||
import { PR_PROFILE_MAPS } from './test-pr-profile';
|
||||
import { knownNonBehaviorFile, PR_PROFILE_MAPS } from './test-pr-profile';
|
||||
|
||||
export interface ReplayInput {
|
||||
runId: number;
|
||||
@@ -59,7 +59,9 @@ export interface ReplayRow {
|
||||
/**
|
||||
* Whether a case depends on any changed file under today's dependency graph:
|
||||
* its touchfiles, the globals, and the derived reference closure
|
||||
* (scripts/pr-dependencies.ts, its global runner closure included).
|
||||
* (scripts/pr-dependencies.ts, its global runner closure included). Like
|
||||
* the selector, a touchfile match wins over the audited free-only exemptions,
|
||||
* and an exempt file reaches no case through the derived closure.
|
||||
*/
|
||||
export function caseTouchesDiff(id: string, changedFiles: readonly string[]): boolean {
|
||||
const patterns = [...(E2E_TOUCHFILES[id] ?? []), ...GLOBAL_TOUCHFILES];
|
||||
@@ -67,7 +69,7 @@ export function caseTouchesDiff(id: string, changedFiles: readonly string[]): bo
|
||||
// A path absent from today's tree reaches a case through the live files that still reference it.
|
||||
const via = (file: string) => derived.tracked.has(file) ? [file] : derived.referencers(file);
|
||||
return changedFiles.some(file => patterns.some(pattern => matchGlob(file, pattern))
|
||||
|| via(file).some(source => derived.global.has(source) || !!derived.e2e.get(source)?.has(id)));
|
||||
|| (!knownNonBehaviorFile(file) && via(file).some(source => derived.global.has(source) || !!derived.e2e.get(source)?.has(id))));
|
||||
}
|
||||
|
||||
export function replayRun(input: ReplayInput, oldSelection: readonly string[]): ReplayRow {
|
||||
|
||||
@@ -33,7 +33,7 @@ export const COMPLEXITY_PATHS = ['scripts', 'test/helpers', '.github/workflows']
|
||||
export interface Metric { id: string; title: string; rule: string; value?: string; detail?: string[]; unavailable?: string }
|
||||
export interface Check { id: string; title: string; status: 'pass' | 'fail' | 'unavailable'; detail: string; fix: string }
|
||||
export type IssueAction = 'upsert' | 'keep-open' | 'close';
|
||||
export interface HealthReport { repo: string; sinceDays: number; generatedAt: string; metrics: Metric[]; checks: Check[]; issueAction: IssueAction }
|
||||
export interface HealthReport { repo: string; sinceDays: number; generatedAt: string; headline?: string; metrics: Metric[]; checks: Check[]; issueAction: IssueAction }
|
||||
|
||||
export interface HealthInputs {
|
||||
client: GhClient;
|
||||
@@ -188,14 +188,17 @@ export function collectHealth(inputs: HealthInputs): HealthReport {
|
||||
const need = <T>(value: T | Error): T => { if (value instanceof Error) throw value; return value; };
|
||||
const prRecords = attempt(() => need(prRuns).filter(run => run.conclusion !== 'cancelled')
|
||||
.map(run => ({ run, trials: records(artifactDirs(run, name => name.startsWith('trial-outcomes'))) })));
|
||||
const prManifests = attempt(() => need(prRuns).flatMap(run => {
|
||||
const file = artifactDirs(run, name => name === 'paid-plan').map(dir => path.join(dir, 'manifest.json')).find(candidate => fs.existsSync(candidate));
|
||||
return file ? [{ run, manifest: readJson(file) }] : [];
|
||||
}));
|
||||
const metrics: Metric[] = [];
|
||||
let costPerMergedPr: string | undefined;
|
||||
|
||||
metrics.push(metric('pr-fallback-rate', 'PR full-fallback rate', 'share of evals.yml pull_request runs whose paid-plan manifest has prCoverage.mode = full-fallback', () => {
|
||||
metrics.push(metric('pr-fallback-rate', 'PR full-fallback rate', 'share of evals.yml pull_request runs whose paid-plan manifest has prCoverage.mode = full-fallback (a label, not spend: dependents runs select about as much)', () => {
|
||||
const modes = new Map<string, number>();
|
||||
for (const run of need(prRuns)) {
|
||||
const manifest = artifactDirs(run, name => name === 'paid-plan').flatMap(dir => [path.join(dir, 'manifest.json')]).find(file => fs.existsSync(file));
|
||||
if (!manifest) continue;
|
||||
const mode = String(readJson(manifest).prCoverage?.mode ?? 'none');
|
||||
for (const { manifest } of need(prManifests)) {
|
||||
const mode = String(manifest.prCoverage?.mode ?? 'none');
|
||||
modes.set(mode, (modes.get(mode) ?? 0) + 1);
|
||||
}
|
||||
const total = [...modes.values()].reduce((a, b) => a + b, 0);
|
||||
@@ -204,6 +207,51 @@ export function collectHealth(inputs: HealthInputs): HealthReport {
|
||||
detail: [...modes].map(([mode, count]) => `${mode}: ${count}`) };
|
||||
}));
|
||||
|
||||
metrics.push(metric('pr-cases-per-run', 'Selected gate cases and judges per PR run', 'mean length of prCoverage.e2e (gate cases) and prCoverage.judges in the paid-plan manifests of evals.yml pull_request runs, overall and per mode', () => {
|
||||
const planned = need(prManifests).filter(({ manifest }) => Array.isArray(manifest.prCoverage?.e2e) && Array.isArray(manifest.prCoverage?.judges));
|
||||
if (planned.length === 0) throw new Error('no paid-plan manifest with prCoverage.e2e and prCoverage.judges in the window');
|
||||
const mean = (entries: typeof planned) => `${(entries.reduce((sum, { manifest }) => sum + manifest.prCoverage.e2e.length, 0) / entries.length).toFixed(1)} gate cases, ${(entries.reduce((sum, { manifest }) => sum + manifest.prCoverage.judges.length, 0) / entries.length).toFixed(1)} judges`;
|
||||
const modes = [...new Set(planned.map(({ manifest }) => String(manifest.prCoverage.mode)))].sort();
|
||||
return { value: `${mean(planned)} per run (${planned.length} runs)`,
|
||||
detail: modes.map(mode => { const entries = planned.filter(({ manifest }) => String(manifest.prCoverage.mode) === mode); return `${mode}: ${mean(entries)} (${entries.length} runs)`; }) };
|
||||
}));
|
||||
|
||||
metrics.push(metric('pr-known-cost-per-run', 'Known cost per PR run', 'mean over non-cancelled evals.yml pull_request runs of executed trial cost_usd with a known cost (a run with no trial records counts $0), overall and per manifest mode; unknown-cost trials (PTY) are counted, not priced', () => {
|
||||
const entries = need(prRecords);
|
||||
if (entries.length === 0) throw new Error('no non-cancelled PR runs in the window');
|
||||
const known = (trials: TrialRecord[]) => trials.filter(record => record.execution === 'executed' && record.cost_known !== false).reduce((sum, record) => sum + record.cost_usd, 0);
|
||||
const unknown = entries.flatMap(entry => entry.trials).filter(record => record.execution === 'executed' && record.cost_known === false).length;
|
||||
const modeOf = new Map((prManifests instanceof Error ? [] : prManifests).map(({ run, manifest }) => [run.id, String(manifest.prCoverage?.mode ?? 'none')]));
|
||||
const mean = (list: typeof entries) => `$${(list.reduce((sum, entry) => sum + known(entry.trials), 0) / list.length).toFixed(2)}`;
|
||||
const modes = [...new Set(entries.map(entry => modeOf.get(entry.run.id) ?? 'no manifest'))].sort();
|
||||
return { value: `${mean(entries)} per run (${entries.length} runs; ${unknown} executed trial(s) with unknown cost)`,
|
||||
detail: modes.map(mode => { const list = entries.filter(entry => (modeOf.get(entry.run.id) ?? 'no manifest') === mode); return `${mode}: ${mean(list)} (${list.length} runs)`; }) };
|
||||
}));
|
||||
|
||||
metrics.push(metric('pr-shadow-map', 'Exercised-path map shadow (not active)', 'over evals.yml pull_request runs whose paid-plan manifest carries prCoverage.shadow: the predicted saving (map per-case known cost of dropped minus added cases; a lower bound, since known cost excludes PTY trials with unknown cost) and shadow drops of red cases (more executed failed than passed trials in the run, on a case the shadow would drop). A drop counts toward the D3 activation bar as a miss unless the case passed in a later run of the same branch at the same head SHA, which marks it a flaky drop. Scaled to 7 days; docs/TESTING_INTERNALS.md#exercised-path-map', () => {
|
||||
const shadowed = need(prManifests).filter(({ manifest }) => manifest.prCoverage?.shadow && !manifest.prCoverage.shadow.error);
|
||||
if (shadowed.length === 0) throw new Error('no paid-plan manifest with a prCoverage.shadow block in the window');
|
||||
const records = prRecords instanceof Error ? [] : prRecords;
|
||||
const executed = (trials: TrialRecord[], id: string, outcome: string) => trials.filter(record => record.case === id && record.execution === 'executed' && record.outcome === outcome).length;
|
||||
const saving = shadowed.reduce((sum, { manifest }) => sum + (Number(manifest.prCoverage.shadow.predictedSavingUsd) || 0), 0);
|
||||
const shares = shadowed.map(({ manifest }) => Number(manifest.prCoverage.shadow.unknownCostShare)).filter(share => Number.isFinite(share) && share > 0);
|
||||
const misses: string[] = [];
|
||||
const flaky: string[] = [];
|
||||
for (const { run, manifest } of shadowed) {
|
||||
const trials = records.find(entry => entry.run.id === run.id)?.trials ?? [];
|
||||
for (const id of (manifest.prCoverage.shadow.wouldDrop ?? []) as string[]) {
|
||||
if (executed(trials, id, 'failed') <= executed(trials, id, 'passed')) continue;
|
||||
const laterPass = records.find(entry => entry.run.branch === run.branch && entry.run.sha === run.sha
|
||||
&& (entry.run.createdAt > run.createdAt || (entry.run.id === run.id && entry.run.attempt > run.attempt)) && executed(entry.trials, id, 'passed') > 0);
|
||||
if (laterPass) flaky.push(`${run.id}:${id} (passed in run ${laterPass.run.id})`); else misses.push(`${run.id}:${id}`);
|
||||
}
|
||||
}
|
||||
const week = 7 / sinceDays;
|
||||
const bound = shares.length ? `a lower bound: known cost excludes the ${Math.round(100 * Math.max(...shares))}% of observed case runs with unknown cost (PTY)` : 'a lower bound: known cost excludes PTY trials with unknown cost';
|
||||
return { value: `$${(saving * week).toFixed(2)} predicted saving/week (${bound}); ${(misses.length * week).toFixed(1)} activation miss(es)/week, ${flaky.length} flaky drop(s) not counted (${shadowed.length} runs)`,
|
||||
detail: [...misses.slice(0, 20).map(miss => `shadow miss: ${miss}`), ...flaky.slice(0, 20).map(drop => `flaky drop: ${drop}`)] };
|
||||
}));
|
||||
|
||||
metrics.push(metric('pr-reused-trials', 'Reused PR trial records', 'trial-outcomes records with execution = reused across non-cancelled PR runs', () => {
|
||||
const all = need(prRecords).flatMap(entry => entry.trials);
|
||||
const reused = all.filter(record => record.execution === 'reused').length;
|
||||
@@ -216,6 +264,7 @@ export function collectHealth(inputs: HealthInputs): HealthReport {
|
||||
.reduce((sum, record) => sum + record.cost_usd, 0);
|
||||
const revisions = new Set(entries.filter(entry => entry.run.conclusion === 'success' || entry.run.conclusion === 'failure').map(entry => entry.run.sha)).size;
|
||||
const merged = listMergedPulls(client, repo, since);
|
||||
if (merged) costPerMergedPr = `$${(spend / merged).toFixed(2)} (${merged} merged PRs, $${spend.toFixed(2)} recorded)`;
|
||||
return { value: `$${spend.toFixed(2)} recorded`, detail: [
|
||||
`per merged PR: ${merged ? `$${(spend / merged).toFixed(2)} (${merged} merged)` : 'n/a (0 merged)'}`,
|
||||
`per validated revision: ${revisions ? `$${(spend / revisions).toFixed(2)} (${revisions} revisions)` : 'n/a'}`] };
|
||||
@@ -267,7 +316,8 @@ export function collectHealth(inputs: HealthInputs): HealthReport {
|
||||
}));
|
||||
|
||||
const checks = [flakeCheck(flakes), unseededCheck(root)];
|
||||
return { repo, sinceDays, generatedAt: new Date(now).toISOString(), metrics, checks, issueAction: issueActionFor(checks) };
|
||||
const headline = `Cost per merged PR: ${costPerMergedPr ?? `unavailable (${metrics.find(entry => entry.id === 'pr-spend')?.unavailable ?? '0 merged PRs'})`}`;
|
||||
return { repo, sinceDays, generatedAt: new Date(now).toISOString(), headline, metrics, checks, issueAction: issueActionFor(checks) };
|
||||
}
|
||||
|
||||
function listMergedPulls(client: GhClient, repo: string, since: string): number {
|
||||
@@ -340,7 +390,7 @@ function censusMetrics(inputs: HealthInputs, artifactDirs: (run: CiRun, match: (
|
||||
// --- Rendering ---
|
||||
|
||||
export function formatHealth(report: HealthReport): string {
|
||||
const lines = [`test:health ${report.repo}, last ${report.sinceDays} day(s), ${report.generatedAt}`];
|
||||
const lines = [`test:health ${report.repo}, last ${report.sinceDays} day(s), ${report.generatedAt}`, ...(report.headline ? [report.headline] : [])];
|
||||
for (const entry of report.metrics) {
|
||||
lines.push(`- ${entry.title}: ${entry.unavailable ? `unavailable: ${entry.unavailable}` : entry.value}`);
|
||||
lines.push(` rule: ${entry.rule}`);
|
||||
|
||||
+106
-6
@@ -5,7 +5,9 @@ import { E2E_TOUCHFILES, E2E_TIERS, GLOBAL_TOUCHFILES, LLM_JUDGE_TOUCHFILES } fr
|
||||
import { matchGlob, TOUCHFILES_DATA_PATH } from '../test/helpers/test-selection';
|
||||
import { isPaidTestFile } from '../test/helpers/paid-test-set';
|
||||
import { FREE_FIXTURES } from '../test/helpers/free-fixtures-data';
|
||||
import { derivedDependencies, type DerivedDependencies } from './pr-dependencies';
|
||||
import {
|
||||
checkoutExercisedConsumers, derivedDependencies, EXERCISED_MAP_PATH, type DerivedDependencies, type ExercisedConsumers,
|
||||
} from './pr-dependencies';
|
||||
|
||||
/** Existing short behavioral probes; intersect with changed-input selection. */
|
||||
export const PR_PROFILE_CASE_IDS = [
|
||||
@@ -157,6 +159,27 @@ export interface PrProfileSelection {
|
||||
missingCoverage: string[];
|
||||
needsFullValidation: boolean;
|
||||
reasons: string[];
|
||||
/** What the exercised-path map would select (shadow mode: never changes e2e or judges). */
|
||||
shadow?: ShadowSelection;
|
||||
}
|
||||
|
||||
/** The exercised-path map's selection beside the real one (docs/TESTING_INTERNALS.md#exercised-path-map). */
|
||||
export interface ShadowSelection {
|
||||
/** Gate cases the map would select. */
|
||||
e2e: string[];
|
||||
/** Selected gate cases the map would not select. */
|
||||
wouldDrop: string[];
|
||||
/** Gate cases the map would select that today's selection does not. */
|
||||
wouldAdd: string[];
|
||||
/** Per changed file with exercised consumers: how many gate cases it reaches, and through which edge. */
|
||||
provenance: Array<{ file: string; summary: string }>;
|
||||
/** Mean known cost of the dropped cases minus the added ones (the map's per-case costUsd; an unpriced case counts $0): a lower bound. */
|
||||
predictedSavingUsd: number;
|
||||
unpricedCases: number;
|
||||
/** Share of observed case runs whose trials carry no known cost (PTY trials); their spend is missing from every dollar figure. */
|
||||
unknownCostShare: number;
|
||||
/** Why no shadow could be computed (the real selection is unaffected). */
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/** Reject stale profile registrations and deferred cases with no scheduled home. */
|
||||
@@ -216,6 +239,12 @@ export const FREE_ONLY_PR_FILES = [
|
||||
'.gstack/generated.json', // gstack's own regen registry: gstack-regen/gstack-pregate read it from origin/<base> of the repo they run in; paid cases run in fixture repos.
|
||||
'.gstack/pregate.json', // gstack's own pre-gate policy: same trust boundary, same reason.
|
||||
'.gstack/ship-policy.json', // gstack's own ship policy: gstack-ship-policy/gstack-restamp read it from origin/<base> of the repo they run in; same reason.
|
||||
'.gstack/ship-policy.example.json', // The example beside it: only .gitignore and docs/ship-policy.md name it.
|
||||
'.github/dependabot.yml', // Dependabot's own config: read by GitHub, and by free test/compare-pr-version.test.ts; dependabot runs skip the paid lane.
|
||||
'.osv-scanner.toml', // Read by .github/workflows/osv-scanner.yml and the free OSV tests (test/osv-*.test.ts).
|
||||
'setup.ps1', // The Windows installer: run only by .github/workflows/windows-setup-e2e.yml; the paid lane is Linux.
|
||||
'.gitlab-ci.yml', // gstack's own GitLab config: skills probe the fixture repo's .gitlab-ci.yml, never this one.
|
||||
'.github/ISSUE_TEMPLATE/**', // Issue templates: read by GitHub and free test/gstack-doctor.test.ts.
|
||||
'scripts/retired-command.ts', // One-release stubs for retired package scripts; no paid case imports it.
|
||||
'careful/bin/check-careful.sh', // /careful PreToolUse hook: covered by test/hook-scripts.test.ts; no paid fixture copies or runs it.
|
||||
// Reporting and launch tools: they read CI history or start a lane, never run inside a paid case.
|
||||
@@ -230,6 +259,7 @@ export const FREE_ONLY_PR_FILES = [
|
||||
'.github/workflows/cso-runtime-qualification.yml',
|
||||
'.github/workflows/cso-scanner-images.yml',
|
||||
'.github/workflows/dependency-review.yml',
|
||||
'.github/workflows/free-durations-record.yml',
|
||||
'.github/workflows/free-tests.yml',
|
||||
'.github/workflows/make-pdf-gate.yml',
|
||||
'.github/workflows/model-policy-freshness.yml',
|
||||
@@ -273,9 +303,9 @@ export const PR_FULL_GATE_FILES = [
|
||||
export const PERIODIC_ONLY_PR_FILES = ['.github/workflows/evals-periodic.yml', '.github/workflows/evals-marathon.yml'] as const;
|
||||
|
||||
/** Directories the reference derivation covers: an unmapped file there with no consumer is consumed by no paid case. */
|
||||
export const DERIVABLE_PREFIXES = ['bin/', 'lib/', 'scripts/', 'browse/', 'design/', 'make-pdf/', 'hosts/', 'extension/', 'model-overlays/', 'agents/', 'test/helpers/'] as const;
|
||||
export const DERIVABLE_PREFIXES = ['bin/', 'lib/', 'scripts/', 'browse/', 'design/', 'make-pdf/', 'hosts/', 'extension/', 'model-overlays/', 'agents/', 'test/helpers/', 'capy/'] as const;
|
||||
|
||||
function knownNonBehaviorFile(file: string): boolean {
|
||||
export function knownNonBehaviorFile(file: string): boolean {
|
||||
// A mapped dependency still wins over these exemptions. New helper,
|
||||
// runtime, dependency, or paid-workflow files are deliberately not exempted.
|
||||
return /^(?:docs\/|(?:README|CONTRIBUTING|ARCHITECTURE|CHANGELOG|TODOS)\.md$|VERSION$)/.test(file)
|
||||
@@ -290,9 +320,13 @@ function knownNonBehaviorFile(file: string): boolean {
|
||||
}
|
||||
|
||||
export const FALLBACK_FIX_ANCHOR = 'docs/TESTING_INTERNALS.md#pr-paid-lane-fallback';
|
||||
export const EXERCISED_MAP_ANCHOR = 'docs/TESTING_INTERNALS.md#exercised-path-map';
|
||||
|
||||
/** Why one changed file restored the full gate, and the edit that narrows it next time (DX-6). */
|
||||
export function unknownFileLabel(file: string): { label: string; fix: string } {
|
||||
if (matches(file, PR_FULL_GATE_FILES)) {
|
||||
return { label: 'full-gate input', fix: `${file} is in PR_FULL_GATE_FILES (scripts/test-pr-profile.ts): every paid case consumes it, so the full gate is intended and no mapping narrows it` };
|
||||
}
|
||||
if (isPaidTestFile(file) || file.startsWith('test/helpers/')) {
|
||||
return { label: 'needs touchfile entry', fix: `register ${file} under the cases that consume it in test/helpers/touchfiles-data.ts` };
|
||||
}
|
||||
@@ -338,6 +372,8 @@ export function selectPrProfile(options: {
|
||||
readSource?: (file: string) => string;
|
||||
/** Derived references; default: derived from the checkout for the real maps, none for fixture maps. */
|
||||
derived?: DerivedDependencies | null;
|
||||
/** Exercised-path consumers for the shadow block; default: the checkout's committed map for the real maps, none for fixture maps. */
|
||||
exercised?: ExercisedConsumers | null;
|
||||
}): PrProfileSelection {
|
||||
const maps = options.maps ?? PR_PROFILE_MAPS;
|
||||
const profile = options.profile ?? PR_PROFILE_CASE_IDS;
|
||||
@@ -351,7 +387,7 @@ export function selectPrProfile(options: {
|
||||
...Object.values(maps.e2eTouchfiles).flat(), ...Object.values(maps.judgeTouchfiles).flat(),
|
||||
...maps.globalTouchfiles,
|
||||
])];
|
||||
const unmapped = files.filter(file => file !== TOUCHFILES_DATA_PATH
|
||||
const unmapped = files.filter(file => file !== TOUCHFILES_DATA_PATH && file !== EXERCISED_MAP_PATH
|
||||
&& !depends(file, dependencyPatterns) && !knownNonBehaviorFile(file));
|
||||
// Unmapped files are placed by derived references where the derivation covers them; the rest restore the full gate.
|
||||
const derived = options.derived !== undefined ? options.derived : maps === PR_PROFILE_MAPS ? derivedDependencies(maps, ROOT) : null;
|
||||
@@ -380,6 +416,12 @@ export function selectPrProfile(options: {
|
||||
if (found.e2e.size + found.judges.size > 0) consumers.set(file, found);
|
||||
else noConsumerFiles.push(file);
|
||||
}
|
||||
// A failed import scan leaves some closure unknown: every unmapped file restores the full gate (fail closed).
|
||||
const scanFailed = derived && unmapped.length ? [...derived.scanErrors.keys()].sort() : [];
|
||||
if (scanFailed.length) {
|
||||
consumers.clear(); derivedGlobal.length = 0; noConsumerFiles.length = 0;
|
||||
for (const file of unmapped) deletedLabels.set(file, { label: 'import scan failed', fix: `scripts/pr-dependencies.ts could not scan the imports of ${scanFailed.slice(0, 3).join(', ')}${scanFailed.length > 3 ? ` and ${scanFailed.length - 3} more` : ''}; fix the parse error so ${file}'s consumers can be derived` });
|
||||
}
|
||||
const derivedFiles = [...consumers.keys()];
|
||||
const unknownFiles = unmapped.filter(file => !derivedGlobal.includes(file) && !consumers.has(file) && !noConsumerFiles.includes(file));
|
||||
const sharedInputs = files.filter(file => depends(file, PR_FULL_GATE_FILES));
|
||||
@@ -422,25 +464,80 @@ export function selectPrProfile(options: {
|
||||
if (sharedInputs.length) reasons.push(`Shared runtime/build inputs restore every gate case and judge: ${sharedInputs.join(', ')}`);
|
||||
if (dependentsMode) reasons.push(`Derived references select every dependent gate case: ${derivedFiles.map(file => `${file} (${consumers.get(file)!.e2e.size} cases)`).join(', ')}`);
|
||||
else if (!fallback) reasons.push('Changed-input selection intersected with the fast PR profile; selected judges retained');
|
||||
if (files.includes(EXERCISED_MAP_PATH)) reasons.push(`${EXERCISED_MAP_PATH} is shadow-only data (it changes no selection yet), so it selects no paid case (${EXERCISED_MAP_ANCHOR})`);
|
||||
if (derivedGlobal.length) reasons.push(`Paid-runner inputs (global touchfile import closure) select every case: ${derivedGlobal.join(', ')}`);
|
||||
if (noConsumerFiles.length) reasons.push(`No paid case's reference closure reaches (or, deleted, still references): ${noConsumerFiles.join(', ')}`);
|
||||
if (directCases.length) reasons.push(`Edited paid test files select their own gate cases: ${directCases.join(', ')}`);
|
||||
if (deferred.length) reasons.push(`${deferred.length} selected behaviors remain scheduled/release coverage, not PR passes`);
|
||||
if (deferredPromptFiles.length) reasons.push(`No quick live coverage; known broad prompt checks deferred: ${deferredPromptFiles.join(', ')}`);
|
||||
if (missingCoverage.length) reasons.push(`Full validation required for prompts without a relevant PR check: ${missingCoverage.join(', ')}`);
|
||||
const exercised = options.exercised !== undefined ? options.exercised : maps === PR_PROFILE_MAPS ? checkoutExercised() : null;
|
||||
const shadow = exercised instanceof Error ? emptyShadow(e2e, exercised.message)
|
||||
: exercised ? shadowSelection({ ...options, maps, profile }, { e2e, fallback, derived, exercised, files }) : undefined;
|
||||
return {
|
||||
mode: fallback ? 'full-fallback' : dependentsMode ? 'dependents' : 'pr', e2e, judges, deferred,
|
||||
unknownFiles, unknownFileLabels: unknownFiles.map(file => ({ file, ...(deletedLabels.get(file) ?? unknownFileLabel(file)) })), directCases,
|
||||
derivedFiles: derivedFiles.map(file => ({ file, e2e: consumers.get(file)!.e2e.size, judges: consumers.get(file)!.judges.size })), noConsumerFiles,
|
||||
deferredPromptFiles, missingCoverage, needsFullValidation: missingCoverage.length > 0, reasons,
|
||||
deferredPromptFiles, missingCoverage, needsFullValidation: missingCoverage.length > 0, reasons, ...(shadow ? { shadow } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function checkoutExercised(): ExercisedConsumers | Error {
|
||||
try { return checkoutExercisedConsumers(PR_PROFILE_MAPS, ROOT, PR_FULL_GATE_FILES); }
|
||||
catch (error) { return error instanceof Error ? error : new Error(String(error)); }
|
||||
}
|
||||
|
||||
function emptyShadow(e2e: string[], error: string): ShadowSelection {
|
||||
return { e2e, wouldDrop: [], wouldAdd: [], provenance: [], predictedSavingUsd: 0, unpricedCases: 0, unknownCostShare: 0, error };
|
||||
}
|
||||
|
||||
/**
|
||||
* Shadow selection (variant cu of the measured prototype): the same policy
|
||||
* with the exercised map in place of the derived closure, unioned with the
|
||||
* exercised consumers of every changed file that is not global or full-gate.
|
||||
* A full-gate fallback stays full.
|
||||
*/
|
||||
function shadowSelection(options: Parameters<typeof selectPrProfile>[0] & { maps: PrProfileMaps },
|
||||
today: { e2e: string[]; fallback: boolean; derived: DerivedDependencies | null; exercised: ExercisedConsumers; files: string[] }): ShadowSelection {
|
||||
const { maps } = options;
|
||||
const { exercised } = today;
|
||||
const observedCases = Object.values(exercised.map?.cases ?? {});
|
||||
const observations = observedCases.reduce((sum, entry) => sum + entry.observations, 0);
|
||||
const unknownCostShare = observations ? Math.round(1000 * (1 - observedCases.reduce((sum, entry) => sum + (entry.costRuns ?? 0), 0) / observations)) / 1000 : 0;
|
||||
if (today.fallback) return { e2e: today.e2e, wouldDrop: [], wouldAdd: [], provenance: [], predictedSavingUsd: 0, unpricedCases: 0, unknownCostShare };
|
||||
const gate = (ids: Iterable<string>) => [...ids].filter(id => maps.tiers[id] === 'gate');
|
||||
const reach = new Map<string, string[]>();
|
||||
const inner = selectPrProfile({ ...options, derived: today.derived ? { ...today.derived, e2e: exercised.e2e } : null, exercised: null });
|
||||
const sources = [...new Set([...today.files, ...Object.values(options.sourceAliases ?? {})])];
|
||||
const shadowSet = new Set(inner.e2e);
|
||||
for (const file of sources) {
|
||||
if (today.derived?.global.has(file) || matches(file, PR_FULL_GATE_FILES) || knownNonBehaviorFile(file)) continue;
|
||||
const via = !today.derived || today.derived.tracked.has(file) ? [file] : today.derived.referencers(file);
|
||||
const ids = gate(new Set(via.flatMap(source => [...exercised.e2e.get(source) ?? []])));
|
||||
if (ids.length) reach.set(file, ids);
|
||||
for (const id of ids) shadowSet.add(id);
|
||||
}
|
||||
const e2e = [...shadowSet].sort();
|
||||
const wouldDrop = today.e2e.filter(id => !shadowSet.has(id));
|
||||
const wouldAdd = e2e.filter(id => !today.e2e.includes(id));
|
||||
const cost = (ids: string[]) => ids.reduce((sum, id) => sum + (exercised.map?.cases[id]?.costUsd ?? 0), 0);
|
||||
const provenance = [...reach].map(([file, ids]) => {
|
||||
const groups = new Map<string, number>();
|
||||
for (const id of ids) {
|
||||
const label = exercised.closure(id).get(file) ?? 'referencer';
|
||||
groups.set(label, (groups.get(label) ?? 0) + 1);
|
||||
}
|
||||
return { file, summary: `selects ${ids.length} case(s): ${[...groups].sort((a, b) => b[1] - a[1]).slice(0, 6).map(([label, n]) => `${label} (${n})`).join(', ')}` };
|
||||
});
|
||||
return { e2e, wouldDrop, wouldAdd, provenance, predictedSavingUsd: Math.round((cost(wouldDrop) - cost(wouldAdd)) * 100) / 100,
|
||||
unpricedCases: [...wouldDrop, ...wouldAdd].filter(id => exercised.map?.cases[id]?.costUsd === undefined).length, unknownCostShare };
|
||||
}
|
||||
|
||||
/** The manifest fields the coverage summary reads (a PaidRunManifest subset). */
|
||||
export interface CoverageSummaryInput {
|
||||
profile?: string;
|
||||
selection?: { e2e: string[] | null; judges: string[] | null };
|
||||
prCoverage?: Pick<PrProfileSelection, 'mode' | 'deferred' | 'unknownFiles'> & Partial<Pick<PrProfileSelection, 'unknownFileLabels' | 'derivedFiles' | 'noConsumerFiles'>>;
|
||||
prCoverage?: Pick<PrProfileSelection, 'mode' | 'deferred' | 'unknownFiles'> & Partial<Pick<PrProfileSelection, 'unknownFileLabels' | 'derivedFiles' | 'noConsumerFiles' | 'shadow'>>;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -461,6 +558,9 @@ export function formatPrCoverageSummary(manifest: CoverageSummaryInput, totals?:
|
||||
lines.push(`- Derived dependents (scripts/pr-dependencies.ts) of ${coverage.derivedFiles.length} unmapped file(s):`);
|
||||
for (const item of coverage.derivedFiles.slice(0, 40)) lines.push(` - \`${item.file}\`: ${item.e2e} case(s), ${item.judges} judge(s)`);
|
||||
}
|
||||
const shadow = coverage?.shadow;
|
||||
if (shadow) lines.push(shadow.error ? `- Shadow exercised-path map: unavailable (${shadow.error}); selection unaffected`
|
||||
: `- Shadow exercised-path map (not active): would select ${shadow.e2e.length} gate case(s), dropping ${shadow.wouldDrop.length} and adding ${shadow.wouldAdd.length}; predicted saving $${shadow.predictedSavingUsd.toFixed(2)}${shadow.unpricedCases ? ` (${shadow.unpricedCases} unpriced)` : ''}, a lower bound: known cost excludes the ${Math.round(100 * (shadow.unknownCostShare ?? 0))}% of observed case runs with unknown cost (PTY) (${EXERCISED_MAP_ANCHOR})`);
|
||||
if (coverage?.noConsumerFiles?.length) lines.push(`- No paid case reaches: ${coverage.noConsumerFiles.slice(0, 40).map(file => `\`${file}\``).join(', ')}`);
|
||||
if (coverage?.mode === 'full-fallback' && coverage.unknownFiles.length) {
|
||||
lines.push(`- Full gate restored by ${coverage.unknownFiles.length} file(s) (fix: ${FALLBACK_FIX_ANCHOR}):`);
|
||||
|
||||
@@ -0,0 +1,357 @@
|
||||
/**
|
||||
* Exercised-path consumer map (W2-A2, shadow mode): edge classes on a
|
||||
* synthetic tree, the five variant-c miss shapes as negative controls,
|
||||
* union-only refreshes, the refresh contract on unavailable or partial input,
|
||||
* shadow-only map data selecting nothing, shadow output that never changes the
|
||||
* real selection, MAP_GAP detection, and parity with the
|
||||
* measured prototype (docs/TESTING_INTERNALS.md#exercised-path-map).
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import {
|
||||
deriveDependencies, EXERCISED_MAP_PATH, exercisedConsumers, parseExercisedMap, type ExercisedMap,
|
||||
} from '../scripts/pr-dependencies';
|
||||
import {
|
||||
formatPrCoverageSummary, PR_FULL_GATE_FILES, PR_PROFILE_MAPS, selectPrProfile,
|
||||
} from '../scripts/test-pr-profile';
|
||||
import {
|
||||
attributeCommands, blindCases, formatExercisedMap, formatMapGap, formatShadowReplay, mapGaps, mergeExercisedMap, observeRun, readCaseCosts, readShardCommands,
|
||||
REFRESH_EXIT, RefreshError, refreshFromReportDirs, refreshFromRun, REPLAY_FIXTURE, replayShadow, type ArtifactDownload, type ReplayRun,
|
||||
} from '../scripts/exercised-map';
|
||||
import { reportMapGaps } from '../scripts/lib/paid-report';
|
||||
import { computePaidCaseSelection } from '../scripts/test-paid-shards';
|
||||
import { prototypeConsumers } from './helpers/exercised-prototype';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
|
||||
function fixtureRepo(files: Record<string, string>): string {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'exercised-map-'));
|
||||
for (const [file, text] of Object.entries(files)) {
|
||||
fs.mkdirSync(path.dirname(path.join(root, file)), { recursive: true });
|
||||
fs.writeFileSync(path.join(root, file), text);
|
||||
}
|
||||
for (const args of [['init', '-q'], ['add', '-A']]) {
|
||||
const result = spawnSync('git', args, { cwd: root, encoding: 'utf8', timeout: 10_000 });
|
||||
if (result.status !== 0) throw new Error(`git ${args[0]} failed: ${result.stderr}`);
|
||||
}
|
||||
return root;
|
||||
}
|
||||
|
||||
const map = (cases: ExercisedMap['cases'], sourceRuns = ['1'], observedThrough = '2026-10-01T00:00:00Z'): ExercisedMap =>
|
||||
({ schema: 1, generatedAt: observedThrough, observedThrough, sourceRuns, cases });
|
||||
|
||||
const FENCE = '```';
|
||||
const skill = [
|
||||
'# alpha', '', 'Prose naming gstack-prose and /gstack-slash is not run.', '', FENCE + 'bash',
|
||||
'gstack-start --quiet',
|
||||
'echo "see gstack-echoed for details"',
|
||||
'echo "now $(gstack-subst --print)"',
|
||||
'# gstack-commented',
|
||||
'case "$x" in gstack-casepat) true ;; esac',
|
||||
'VAR=gstack-bare',
|
||||
'"$ROOT/bin/gstack-pathed" run',
|
||||
'. "$ROOT/lib/sourced.sh"',
|
||||
"cat <<'EOF'", 'gstack-quoted-heredoc', 'EOF',
|
||||
'cat <<EOF', '$(gstack-expanding)', 'gstack-heredoc-prose', 'EOF',
|
||||
"bash <<'EOF'", 'gstack-interp', 'EOF',
|
||||
'$B goto https://example.com',
|
||||
'bun run build-thing',
|
||||
'./setup --quiet',
|
||||
FENCE, '',
|
||||
].join('\n');
|
||||
const bins = ['echoed', 'subst', 'commented', 'casepat', 'bare', 'pathed', 'quoted-heredoc', 'expanding', 'heredoc-prose', 'interp', 'prose', 'slash', 'observed', 'gapper'];
|
||||
const tree = {
|
||||
'test/skill-e2e-alpha.test.ts': "import './helpers/h';\n",
|
||||
'test/helpers/h.ts': "const a = path.join(ROOT, 'lib', 'joined.ts');\nconst b = 'lib/quoted.ts';\nexport { a, b };\n",
|
||||
'lib/joined.ts': '', 'lib/quoted.ts': '', 'lib/deep.ts': '', 'lib/sourced.sh': '#!/bin/sh\n', 'lib/unreached.ts': '',
|
||||
'alpha/SKILL.md.tmpl': skill,
|
||||
'bin/gstack-start': '#!/usr/bin/env bash\ngstack-helper "$@"\n',
|
||||
'bin/gstack-helper': "#!/usr/bin/env bun\nimport '../lib/deep';\n",
|
||||
'bin/gstack-config': '#!/usr/bin/env bash\necho config\n',
|
||||
...Object.fromEntries(bins.map(name => [`bin/gstack-${name}`, '#!/bin/sh\n'])),
|
||||
'test/skill-e2e-beta.test.ts': '', 'test/skill-e2e-gamma.test.ts': '', 'test/skill-e2e-delta.test.ts': '',
|
||||
'gamma/SKILL.md.tmpl': `# gamma\n\n${FENCE}bash\ngstack-config get proactive\n${FENCE}\n`,
|
||||
'scripts/build.sh': 'bun build --compile browse/src/cli.ts --outfile browse/dist/browse\n', 'browse/src/cli.ts': "import './page';\n", 'browse/src/page.ts': '',
|
||||
'scripts/thing.ts': '', 'scripts/resolvers/index.ts': '', 'setup': '#!/usr/bin/env bash\n',
|
||||
'package.json': JSON.stringify({ scripts: { 'build-thing': 'bun run scripts/thing.ts' } }),
|
||||
};
|
||||
const maps = {
|
||||
e2eTouchfiles: {
|
||||
alpha: ['test/skill-e2e-alpha.test.ts', 'alpha/**'], beta: ['test/skill-e2e-beta.test.ts'],
|
||||
gamma: ['test/skill-e2e-gamma.test.ts', 'gamma/**'], delta: ['test/skill-e2e-delta.test.ts', 'bin/gstack-config', 'lib/deep.ts'],
|
||||
},
|
||||
judgeTouchfiles: {}, globalTouchfiles: [] as string[],
|
||||
};
|
||||
const profileMaps = { ...maps, tiers: { alpha: 'gate', beta: 'gate', gamma: 'gate', delta: 'gate' } as const };
|
||||
const observed = map({ beta: { observations: 3, withCommands: 2, commands: ['bin/gstack-observed'], costUsd: 2, costRuns: 3 },
|
||||
alpha: { observations: 4, withCommands: 4, commands: [], costUsd: 5, costRuns: 4 }, gamma: { observations: 2, withCommands: 0, commands: [] } });
|
||||
|
||||
describe('exercised consumers on a synthetic tree', () => {
|
||||
const root = fixtureRepo(tree);
|
||||
const ex = exercisedConsumers(maps, root, { observed, stopAt: ['setup'] });
|
||||
const alpha = ex.closure('alpha');
|
||||
|
||||
test('each kept edge class reaches its file, with its provenance', () => {
|
||||
expect(alpha.get('alpha/SKILL.md.tmpl')).toBe('touchfile');
|
||||
expect(alpha.get('bin/gstack-start')).toBe('skill bash via bin/gstack-start');
|
||||
expect(alpha.get('lib/deep.ts')).toBe('skill bash via bin/gstack-start');
|
||||
expect(alpha.get('lib/joined.ts')).toBe('import');
|
||||
expect(alpha.get('lib/quoted.ts')).toBe('import');
|
||||
for (const file of ['bin/gstack-subst', 'bin/gstack-pathed', 'lib/sourced.sh', 'bin/gstack-expanding', 'bin/gstack-interp', 'browse/src/page.ts', 'scripts/thing.ts', 'setup']) {
|
||||
expect(alpha.has(file), file).toBe(true);
|
||||
}
|
||||
expect(ex.closure('beta').get('bin/gstack-observed')).toBe('observed bin/gstack-observed');
|
||||
});
|
||||
|
||||
test('messages, comments, case patterns, bare assignments, quoted and expanded heredoc prose, prose and slash commands are not edges', () => {
|
||||
for (const name of ['echoed', 'commented', 'casepat', 'bare', 'quoted-heredoc', 'heredoc-prose', 'prose', 'slash']) {
|
||||
expect(alpha.has(`bin/gstack-${name}`), name).toBe(false);
|
||||
}
|
||||
expect(ex.e2e.get('lib/unreached.ts')).toBeUndefined();
|
||||
});
|
||||
|
||||
test('the closure stops at a full-gate file and at the global set', () => {
|
||||
const stopped = exercisedConsumers({ ...maps, globalTouchfiles: ['bin/gstack-start'] }, fixtureRepo(tree), { observed, stopAt: ['setup'] });
|
||||
expect(stopped.closure('alpha').has('bin/gstack-start')).toBe(true);
|
||||
expect(stopped.closure('alpha').has('lib/deep.ts')).toBe(false);
|
||||
});
|
||||
|
||||
test('bins in recorded commands: paths, bare names, PTY lines and compiled outfiles', () => {
|
||||
expect(ex.binsInCommand('gstack-start --x && "$B" goto y')).toEqual(['bin/gstack-start', 'browse/dist/browse']);
|
||||
expect(ex.binsInCommand('PTY:gstack-config-get proactive')).toEqual(['bin/gstack-config']);
|
||||
expect(ex.binsInCommand('"$ROOT/bin/gstack-start" --x')).toEqual(['bin/gstack-start']);
|
||||
expect(ex.binsInCommand('run /gstack-start next')).toEqual([]);
|
||||
expect(ex.commandFiles('browse/dist/browse').sort()).toEqual(['browse/src/cli.ts', 'browse/src/page.ts']);
|
||||
});
|
||||
|
||||
test('the variant-c miss shapes stay selected in the shadow (negative controls)', () => {
|
||||
const shadow = (changedFiles: string[], selectedE2E: string[]) => selectPrProfile({
|
||||
maps: profileMaps, profile: ['alpha', 'beta', 'gamma', 'delta'], selectedE2E, selectedJudges: [], changedFiles,
|
||||
derived: deriveDependencies(maps, root), exercised: ex,
|
||||
}).shadow!;
|
||||
// A preamble hub a touchfile maps only for another case (skill-start/end x3; gstack-config for the PTY case with no observations).
|
||||
expect(shadow(['bin/gstack-config'], ['delta']).e2e).toEqual(['delta', 'gamma']);
|
||||
expect(observed.cases.gamma!.withCommands).toBe(0);
|
||||
// A lib reached only through a bin the skill runs (redact-patterns via gstack-redact).
|
||||
expect(shadow(['lib/deep.ts'], ['delta']).e2e).toEqual(['alpha', 'delta']);
|
||||
expect(shadow(['lib/deep.ts'], ['delta']).provenance).toEqual([{ file: 'lib/deep.ts', summary: 'selects 2 case(s): skill bash via bin/gstack-start (1), touchfile (1)' }]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('the five recorded variant-c misses on this checkout', () => {
|
||||
const shadowOf = (file: string) => computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: [file], packageVersionOnly: true }).coverage!.shadow!;
|
||||
|
||||
test('skill-start reds stay in the shadow of a gstack-skill-start change', () => {
|
||||
const shadow = shadowOf('bin/gstack-skill-start');
|
||||
expect(shadow.error).toBeUndefined();
|
||||
for (const id of ['qa-functional-webhook-fix', 'qa-functional-cli-report', 'ship-exploratory-plan-checks']) expect(shadow.e2e, id).toContain(id);
|
||||
}, 60_000);
|
||||
|
||||
test('gstack-config reaches the PTY case that records no commands; redact-patterns reaches ship-docsync-store', () => {
|
||||
expect(shadowOf('bin/gstack-config').e2e).toContain('plan-mode-no-op');
|
||||
const committed = parseExercisedMap(fs.readFileSync(path.join(ROOT, EXERCISED_MAP_PATH), 'utf8'));
|
||||
const ex = exercisedConsumers(PR_PROFILE_MAPS, ROOT, { observed: committed, stopAt: PR_FULL_GATE_FILES });
|
||||
expect(ex.closure('plan-mode-no-op').get('bin/gstack-config')).toStartWith('skill bash via');
|
||||
expect(ex.closure('ship-docsync-store').has('lib/redact-patterns.ts')).toBe(true);
|
||||
}, 60_000);
|
||||
});
|
||||
|
||||
describe('shadow output never changes the real selection', () => {
|
||||
test('mode, cases, judges and reasons equal the selection without a shadow', () => {
|
||||
for (const files of [['bin/gstack-config'], ['design/src/evolve.ts'], ['review/SKILL.md.tmpl'], ['lib/fs-utils.ts', 'docs/x.md'], ['ETHOS.md']]) {
|
||||
const withShadow = selectPrProfile({ selectedE2E: [], selectedJudges: [], changedFiles: files });
|
||||
const without = selectPrProfile({ selectedE2E: [], selectedJudges: [], changedFiles: files, exercised: null });
|
||||
expect(withShadow.shadow, files.join()).toBeDefined();
|
||||
const { shadow, ...rest } = withShadow;
|
||||
expect(rest, files.join()).toEqual(without);
|
||||
if (withShadow.mode === 'full-fallback') expect(shadow!.e2e).toEqual(withShadow.e2e);
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
test('the coverage summary prints one shadow line; a broken map is reported, not fatal', () => {
|
||||
const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: ['design/src/evolve.ts'], packageVersionOnly: true });
|
||||
const lines = formatPrCoverageSummary({ profile: 'pr', selection: result.selection, prCoverage: result.coverage });
|
||||
expect(lines.filter(line => line.includes('Shadow exercised-path map (not active)'))).toHaveLength(1);
|
||||
const broken = selectPrProfile({ selectedE2E: [], selectedJudges: [], changedFiles: ['design/src/evolve.ts'], exercised: null });
|
||||
expect(formatPrCoverageSummary({ prCoverage: { ...broken, shadow: { e2e: broken.e2e, wouldDrop: [], wouldAdd: [], provenance: [], predictedSavingUsd: 0, unpricedCases: 0, unknownCostShare: 0, error: 'bad map' } } })
|
||||
.some(line => line.includes('unavailable (bad map); selection unaffected'))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('union-only refreshes', () => {
|
||||
const base = map({ a: { observations: 2, withCommands: 1, commands: ['bin/x'], costUsd: 4, costRuns: 2 } }, ['1']);
|
||||
|
||||
test('disjoint branches merge to the same union in either order, and nothing is removed', () => {
|
||||
const left = map({ a: { observations: 1, withCommands: 1, commands: ['bin/y'] } }, ['2'], '2026-10-02T00:00:00Z');
|
||||
const right = map({ b: { observations: 1, withCommands: 1, commands: ['bin/z'], costUsd: 1, costRuns: 1 } }, ['3'], '2026-10-03T00:00:00Z');
|
||||
const one = mergeExercisedMap(mergeExercisedMap(base, left, 't'), right, 't');
|
||||
const two = mergeExercisedMap(mergeExercisedMap(base, right, 't'), left, 't');
|
||||
expect(one).toEqual(two);
|
||||
expect(one.cases.a!.commands).toEqual(['bin/x', 'bin/y']);
|
||||
expect(one.cases.b!.commands).toEqual(['bin/z']);
|
||||
expect(one.sourceRuns).toEqual(['1', '2', '3']);
|
||||
expect(one.observedThrough).toBe('2026-10-03T00:00:00Z');
|
||||
expect(one.cases.a!.costUsd).toBe(4);
|
||||
});
|
||||
|
||||
test('empty or missing artifacts never erase consumers, and a run already merged is not counted twice', () => {
|
||||
expect(mergeExercisedMap(base, map({}, ['9']), 't').cases).toEqual(base.cases);
|
||||
expect(mergeExercisedMap(base, map({ a: { observations: 1, withCommands: 0, commands: [] } }, ['9']), 't').cases.a!.commands).toEqual(['bin/x']);
|
||||
expect(mergeExercisedMap(base, map({ a: { observations: 5, withCommands: 5, commands: ['bin/x'], costUsd: 9, costRuns: 1 } }, ['1']), 't').cases.a)
|
||||
.toEqual(base.cases.a!);
|
||||
const costed = mergeExercisedMap(base, map({ a: { observations: 1, withCommands: 1, commands: [], costUsd: 7, costRuns: 1 } }, ['4']), 't');
|
||||
expect(costed.cases.a).toMatchObject({ observations: 3, costUsd: 5, costRuns: 3 });
|
||||
expect(parseExercisedMap(formatExercisedMap(costed))).toEqual(costed);
|
||||
});
|
||||
});
|
||||
|
||||
describe('shadow-only map data', () => {
|
||||
test('a map-only diff selects no paid case and says why', () => {
|
||||
const selection = selectPrProfile({ selectedE2E: [], selectedJudges: [], changedFiles: [EXERCISED_MAP_PATH] });
|
||||
expect(selection).toMatchObject({ mode: 'pr', e2e: [], judges: [], unknownFiles: [], derivedFiles: [], noConsumerFiles: [] });
|
||||
expect(selection.shadow!.e2e).toEqual([]);
|
||||
expect(selection.reasons).toContain(`${EXERCISED_MAP_PATH} is shadow-only data (it changes no selection yet), so it selects no paid case (docs/TESTING_INTERNALS.md#exercised-path-map)`);
|
||||
const plan = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: [EXERCISED_MAP_PATH, 'package.json'], packageVersionOnly: true });
|
||||
expect(plan.selection.e2e).toEqual([]);
|
||||
expect(plan.selection.judges).toEqual([]);
|
||||
}, 60_000);
|
||||
});
|
||||
|
||||
describe('refresh contract: unavailable or partial input writes nothing', () => {
|
||||
const root = fixtureRepo(tree);
|
||||
const ex = exercisedConsumers(maps, root, { observed, stopAt: ['setup'] });
|
||||
const report = (files: Record<string, string>) => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'exercised-refresh-'));
|
||||
for (const [file, text] of Object.entries(files)) { fs.mkdirSync(path.dirname(path.join(dir, file)), { recursive: true }); fs.writeFileSync(path.join(dir, file), text); }
|
||||
return dir;
|
||||
};
|
||||
const outcome = (id: string) => JSON.stringify({ case: id, execution: 'executed', cost_usd: 1 });
|
||||
const shard = { 'paid-slice-1-a1/shards/skill-e2e-beta/junit.xml': '<testsuite><testcase name="beta"/></testsuite>' };
|
||||
const code = (run: () => unknown) => { try { run(); return 0; } catch (error) { return error instanceof RefreshError ? error.exitCode : -1; } };
|
||||
|
||||
test('report directories: missing is unavailable; no shards, no trial outcomes or an executed case without a shard is incomplete', () => {
|
||||
expect(code(() => refreshFromReportDirs([path.join(os.tmpdir(), 'no-such-census-dir')], '1', observed, ex, 't', profileMaps))).toBe(REFRESH_EXIT.unavailable);
|
||||
expect(code(() => refreshFromReportDirs([report({ 'trial-outcomes.jsonl': outcome('beta') })], '1', observed, ex, 't', profileMaps))).toBe(REFRESH_EXIT.incomplete);
|
||||
expect(code(() => refreshFromReportDirs([report(shard)], '1', observed, ex, 't', profileMaps))).toBe(REFRESH_EXIT.incomplete);
|
||||
expect(() => refreshFromReportDirs([report({ ...shard, 'trial-outcomes.jsonl': `${outcome('beta')}\n${outcome('alpha')}` })], '1', observed, ex, 't', profileMaps))
|
||||
.toThrow('1 executed case(s) have no shard artifact: alpha');
|
||||
const { merged } = refreshFromReportDirs([report({ ...shard, 'trial-outcomes.jsonl': outcome('beta') })], '9', observed, ex, 't', profileMaps);
|
||||
expect(merged.sourceRuns).toEqual(['1', '9']);
|
||||
expect(merged.cases.beta!.observations).toBe(4);
|
||||
});
|
||||
|
||||
test('--from-run: a 403, a missing map or a malformed map leaves the target byte-identical; a complete artifact is unioned', () => {
|
||||
const target = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'exercised-target-')), 'exercised-commands.json');
|
||||
fs.writeFileSync(target, formatExercisedMap(observed));
|
||||
const before = fs.readFileSync(target, 'utf8');
|
||||
const artifact = (files: Record<string, string>): ArtifactDownload => (_runId, dir) => {
|
||||
for (const [file, text] of Object.entries(files)) { fs.mkdirSync(path.dirname(path.join(dir, file)), { recursive: true }); fs.writeFileSync(path.join(dir, file), text); }
|
||||
return { status: 0, stderr: '' };
|
||||
};
|
||||
let forbidden: unknown;
|
||||
try { refreshFromRun('42', target, () => ({ status: 1, stderr: 'HTTP 403: Resource not accessible by integration' })); } catch (error) { forbidden = error; }
|
||||
expect(forbidden).toBeInstanceOf(RefreshError);
|
||||
expect((forbidden as RefreshError).exitCode).toBe(REFRESH_EXIT.unavailable);
|
||||
expect((forbidden as Error).message).toContain('pass its directory with --from-dir');
|
||||
expect(code(() => refreshFromRun('42', target, artifact({ 'exercised-map-a1/report.md': 'x' })))).toBe(REFRESH_EXIT.incomplete);
|
||||
expect(code(() => refreshFromRun('42', target, artifact({ 'exercised-map-a1/exercised-commands.json': '{"schema":1,"cases":' })))).toBe(REFRESH_EXIT.incomplete);
|
||||
expect(code(() => refreshFromRun('42', target, artifact({ 'exercised-map-a1/exercised-commands.json': formatExercisedMap(map({}, ['5'])), 'exercised-map-a2/exercised-commands.json': '{}' })))).toBe(REFRESH_EXIT.incomplete);
|
||||
expect(code(() => refreshFromRun('not-a-run', target))).toBe(REFRESH_EXIT.unavailable);
|
||||
expect(fs.readFileSync(target, 'utf8')).toBe(before);
|
||||
const fresh = map({ beta: { observations: 1, withCommands: 1, commands: ['bin/gstack-new'] } }, ['5'], '2026-10-09T00:00:00Z');
|
||||
refreshFromRun('5', target, artifact({ 'exercised-map-a1/exercised-commands.json': formatExercisedMap(fresh) }));
|
||||
const after = parseExercisedMap(fs.readFileSync(target, 'utf8'));
|
||||
expect(after.cases.beta!.commands).toEqual(['bin/gstack-new', 'bin/gstack-observed']);
|
||||
expect(after.sourceRuns).toEqual(['1', '5']);
|
||||
});
|
||||
|
||||
test('the CLI exits 2 on usage and 3 on a missing report directory, writing nothing', () => {
|
||||
const out = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'exercised-cli-')), 'map.json');
|
||||
const cli = (args: string[]) => spawnSync('bun', ['run', path.join(ROOT, 'scripts/exercised-map.ts'), ...args], { cwd: ROOT, encoding: 'utf8', timeout: 60_000 });
|
||||
expect(cli([]).status).toBe(REFRESH_EXIT.usage);
|
||||
const missing = cli(['--from-dir', path.join(os.tmpdir(), 'no-such-census-dir'), '--run-id', '1', '--out', out]);
|
||||
expect(missing.status).toBe(REFRESH_EXIT.unavailable);
|
||||
expect(missing.stderr).toContain('nothing was written, the committed union is unchanged');
|
||||
expect(fs.existsSync(out)).toBe(false);
|
||||
}, 90_000);
|
||||
});
|
||||
|
||||
describe('recorded commands and MAP_GAP', () => {
|
||||
const root = fixtureRepo(tree);
|
||||
const ex = exercisedConsumers(maps, root, { observed, stopAt: ['setup'] });
|
||||
const report = fs.mkdtempSync(path.join(os.tmpdir(), 'exercised-report-'));
|
||||
const write = (file: string, text: string) => { fs.mkdirSync(path.dirname(path.join(report, file)), { recursive: true }); fs.writeFileSync(path.join(report, file), text); };
|
||||
write('paid-slice-1-a1/shards/skill-e2e-alpha--alpha.t1/result.json', JSON.stringify({ tests: [{ name: 'alpha', transcript: [{ type: 'tool_use', input: { command: 'gstack-start && $B goto x' } }] }] }));
|
||||
write('paid-slice-1-a1/shards/skill-e2e-beta/junit.xml', '<testsuite><testcase name="beta"/></testsuite>');
|
||||
write('paid-slice-1-a1/shards/skill-e2e-beta/session.jsonl', `${JSON.stringify({ type: 'tool_use', input: { command: 'bin/gstack-gapper --x' } })}\n`);
|
||||
write('paid-slice-1-a1/shards/skill-e2e-gamma/junit.xml', '<testsuite><testcase name="gamma"/></testsuite>');
|
||||
write('paid-slice-1-a1/trial-outcomes.jsonl', [{ case: 'alpha', execution: 'executed', cost_usd: 1.5 }, { case: 'alpha', execution: 'executed', cost_usd: 0, cost_known: false }, { case: 'beta', execution: 'reused', cost_usd: 3 }].map(record => JSON.stringify(record)).join('\n'));
|
||||
|
||||
test('commands are read per shard and attributed to their cases', () => {
|
||||
const commands = attributeCommands(readShardCommands(report), profileMaps);
|
||||
expect(commands.get('alpha')).toEqual(['gstack-start && $B goto x']);
|
||||
expect(commands.get('beta')).toEqual(['bin/gstack-gapper --x']);
|
||||
expect(commands.get('gamma')).toEqual([]);
|
||||
expect(observeRun(commands, ex, { runId: '7', observedAt: 't', costs: readCaseCosts(report) }).cases.alpha)
|
||||
.toEqual({ observations: 1, withCommands: 1, commands: ['bin/gstack-start', 'browse/dist/browse'], costUsd: 1.5, costRuns: 1 });
|
||||
});
|
||||
|
||||
test('a command outside the static-plus-observed closure is a MAP_GAP; a skill-running case with no commands is blind', () => {
|
||||
const commands = attributeCommands(readShardCommands(report), profileMaps);
|
||||
expect(mapGaps(commands, ex)).toEqual([{ id: 'beta', command: 'bin/gstack-gapper' }]);
|
||||
expect(blindCases(commands, ex)).toEqual(['gamma']);
|
||||
});
|
||||
|
||||
test('the paid report writes MAP_GAP lines without failing, and an unreadable checkout degrades to unavailable', () => {
|
||||
const live = fs.mkdtempSync(path.join(os.tmpdir(), 'exercised-live-'));
|
||||
const committed = parseExercisedMap(fs.readFileSync(path.join(ROOT, EXERCISED_MAP_PATH), 'utf8'));
|
||||
const ex = exercisedConsumers(PR_PROFILE_MAPS, ROOT, { observed: committed, stopAt: PR_FULL_GATE_FILES });
|
||||
const outside = fs.readdirSync(path.join(ROOT, 'bin')).map(name => `bin/${name}`).find(file => /^bin\/gstack-[a-z-]+$/.test(file) && !ex.closure('ship-docsync-store').has(file))!;
|
||||
fs.mkdirSync(path.join(live, 'paid-slice-2-a1/shards/skill-e2e-ship-docsync--ship-docsync-store.t1'), { recursive: true });
|
||||
fs.writeFileSync(path.join(live, 'paid-slice-2-a1/shards/skill-e2e-ship-docsync--ship-docsync-store.t1/r.json'),
|
||||
JSON.stringify({ tests: [{ name: 'ship-docsync-store', transcript: [{ type: 'tool_use', input: { command: `${path.basename(outside)} --help` } }] }] }));
|
||||
const line = `MAP_GAP ship-docsync-store ${outside}: non-blocking for this PR; the next census refresh widens the map (docs/TESTING_INTERNALS.md#exercised-path-map)`;
|
||||
expect(formatMapGap({ id: 'ship-docsync-store', command: outside })).toBe(line);
|
||||
expect(reportMapGaps(live, new Set(['ship-docsync-store']), ROOT)).toEqual([line]);
|
||||
expect(fs.readFileSync(path.join(live, 'map-gaps.txt'), 'utf8')).toBe(`${line}\n`);
|
||||
expect(reportMapGaps(live, new Set(['ship-docsync-store']), path.join(live, 'no-checkout'))).toEqual([]);
|
||||
expect(fs.readFileSync(path.join(live, 'map-gaps.txt'), 'utf8')).toStartWith('MAP_GAP unavailable:');
|
||||
}, 60_000);
|
||||
});
|
||||
|
||||
describe('parity with the measured prototype', () => {
|
||||
/** Files the shipped function reaches beyond the prototype, each with the reviewed edge class behind it. */
|
||||
const REVIEWED_SUPERSET: Record<string, string> = {
|
||||
setup: './setup, $ROOT/setup and join(..., "setup") exec edges (plan A2-2); setup is a full-gate file, so the closure stops there',
|
||||
};
|
||||
|
||||
test('the shipped map equals or is a reviewed superset of the prototype map on this checkout', () => {
|
||||
const committed = parseExercisedMap(fs.readFileSync(path.join(ROOT, EXERCISED_MAP_PATH), 'utf8'));
|
||||
const prototype = prototypeConsumers(PR_PROFILE_MAPS, ROOT, committed, PR_FULL_GATE_FILES);
|
||||
const shipped = exercisedConsumers(PR_PROFILE_MAPS, ROOT, { observed: committed, stopAt: PR_FULL_GATE_FILES });
|
||||
const missing = [...prototype].flatMap(([file, ids]) => [...ids].filter(id => !shipped.e2e.get(file)?.has(id)).map(id => `${file} ${id}`));
|
||||
expect(missing).toEqual([]);
|
||||
const unreviewed = [...new Set([...shipped.e2e].filter(([file, ids]) => [...ids].some(id => !prototype.get(file)?.has(id))).map(([file]) => file))]
|
||||
.filter(file => !Object.hasOwn(REVIEWED_SUPERSET, file));
|
||||
expect(unreviewed.map(file => `${file}: reached beyond the prototype; review the edge class and add it to REVIEWED_SUPERSET with its reason`)).toEqual([]);
|
||||
}, 60_000);
|
||||
});
|
||||
|
||||
describe('replay of the recorded red PR runs', () => {
|
||||
test('every recorded failed gate case stays in the shadow selection (the PR receipt: bun run test:exercised-map --replay)', () => {
|
||||
const fixture = JSON.parse(fs.readFileSync(path.join(ROOT, REPLAY_FIXTURE), 'utf8')) as { failedGateCases: number; runs: ReplayRun[] };
|
||||
expect(fixture.runs.reduce((sum, run) => sum + run.failedGateCases.length, 0)).toBe(fixture.failedGateCases);
|
||||
const rows = replayShadow(fixture.runs);
|
||||
expect(rows.flatMap(row => row.misses.map(id => `${row.runId}:${id}`))).toEqual([]);
|
||||
expect(formatShadowReplay(rows).at(-1)).toBe('- MISSES (failed, not in the shadow selection): 0');
|
||||
}, 120_000);
|
||||
|
||||
test('a red the shadow drops is reported as a miss', () => {
|
||||
const [row] = replayShadow([{ runId: 1, pr: 1, packageVersionOnly: true, failedGateCases: ['plan-mode-no-op'], changedFiles: ['docs/designs/x.md'] }]);
|
||||
expect(row!.misses).toEqual(['plan-mode-no-op']);
|
||||
expect(formatShadowReplay([row!]).slice(-2)).toEqual(['- MISSES (failed, not in the shadow selection): 1', ' - 1 #1 plan-mode-no-op']);
|
||||
});
|
||||
});
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"source": "evals.yml pull_request runs 2026-10-07..2026-10-17 (225 runs): every run with an executed failed gate trial, deduplicated per (run, case); changed files are the three-dot compare of the merge-base with the run head",
|
||||
"failedGateCases": 47,
|
||||
"runs": [
|
||||
{"runId":37560420246,"pr":3059,"headSha":"d6250a058af971dd697036acd6e05f7740e8e210","packageVersionOnly":true,"failedGateCases":["plan-mode-no-op"],"changedFiles":[".github/actionlint.yaml",".github/workflows/eval-sweep.yml","CHANGELOG.md","CONTRIBUTING.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-config","docs/TESTING_INTERNALS.md","package.json","scripts/eval-flake-rank.ts","scripts/harness-version.json","scripts/lib/measure-bar.ts","scripts/ship-measure-sweep.ts","scripts/ship-measure.ts","scripts/test-pr-profile.ts","ship/sections/measure.md","ship/sections/measure.md.tmpl","test/ci-image-tag-binding.test.ts","test/eval-sweep-workflow.test.ts","test/fixtures/module-size-ratchet.json","test/helpers/carve-guards.ts","test/helpers/touchfiles-data.ts","test/ship-measure-bar.test.ts","test/ship-measure-loop-skill.test.ts","test/ship-measure-seeded-fixture.test.ts","test/ship-measure-sweep.test.ts","test/ship-measure.test.ts"]},
|
||||
{"runId":37564096977,"pr":3059,"headSha":"36c0cf1579a2f823783c2ecee5c509e872b0c20a","packageVersionOnly":true,"failedGateCases":["qa-functional-cli-report"],"changedFiles":[".github/actionlint.yaml",".github/workflows/eval-sweep.yml","CHANGELOG.md","CONTRIBUTING.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-config","docs/TESTING_INTERNALS.md","package.json","scripts/eval-flake-rank.ts","scripts/harness-version.json","scripts/lib/measure-bar.ts","scripts/ship-measure-sweep.ts","scripts/ship-measure.ts","scripts/test-pr-profile.ts","ship/sections/measure.md","ship/sections/measure.md.tmpl","test/ci-image-tag-binding.test.ts","test/eval-sweep-workflow.test.ts","test/fixtures/module-size-ratchet.json","test/helpers/carve-guards.ts","test/helpers/touchfiles-data.ts","test/ship-measure-bar.test.ts","test/ship-measure-loop-skill.test.ts","test/ship-measure-seeded-fixture.test.ts","test/ship-measure-sweep.test.ts","test/ship-measure.test.ts","test/skill-e2e-plan-mode-no-op.test.ts"]},
|
||||
{"runId":37691665506,"pr":3069,"headSha":"068a3099baf660e995c693b5625b4b3e4546b7a8","packageVersionOnly":true,"failedGateCases":["shared-libs-read-only"],"changedFiles":[".github/ISSUE_TEMPLATE/bug_report.md",".github/docker/Dockerfile.ci",".github/workflows/evals-periodic.yml",".github/workflows/free-tests.yml",".github/workflows/measure-journal-read.yml","CHANGELOG.md","README.md","SKILL.md","TODOS.md","VERSION","agents-digest/gstack-AGENTS.md","autoplan/SKILL.md","autoplan/bin/guard-journal.ts","autoplan/bin/guard-log.ts","autoplan/bin/guard-reasons.ts","autoplan/bin/phase-publication-hook.ts","autoplan/sections/ceo-phase.md","autoplan/sections/ceo-phase.md.tmpl","autoplan/sections/design-phase.md","autoplan/sections/design-phase.md.tmpl","autoplan/sections/dx-phase.md","autoplan/sections/dx-phase.md.tmpl","autoplan/sections/eng-phase.md","autoplan/sections/eng-phase.md.tmpl","autoplan/sections/phase-close.md","autoplan/sections/phase-close.md.tmpl","benchmark-models/SKILL.md","benchmark/SKILL.md","bin/gstack-artifacts-init","bin/gstack-brain-lock.sh","bin/gstack-brain-restore","bin/gstack-brain-sync","bin/gstack-codex-probe","bin/gstack-codex-status.sh","bin/gstack-config","bin/gstack-doctor","bin/gstack-gbrain-source-wireup","bin/gstack-greptile-early","bin/gstack-post","bin/gstack-pr-title-rewrite.sh","bin/gstack-question-preference","bin/gstack-redact-prepush","bin/gstack-skill-start","bin/gstack-slug","browse/SKILL.md","browse/test/extension-sender-auth.test.ts","canary/SKILL.md","codex/SKILL.md","codex/SKILL.md.tmpl","codex/sections/challenge-mode.md","codex/sections/challenge-mode.md.tmpl","codex/sections/consult-mode.md","codex/sections/consult-mode.md.tmpl","codex/sections/review-mode.md","codex/sections/review-mode.md.tmpl","context-restore/SKILL.md","context-restore/SKILL.md.tmpl","context-save/SKILL.md","context-save/SKILL.md.tmpl","design-consultation/SKILL.md","design-consultation/sections/proposal-and-preview.md","design-html/SKILL.md","design-review/SKILL.md","design-shotgun/SKILL.md","devex-review/SKILL.md","diagram/SKILL.md","docs/autoplan-guard-troubleshooting.md","docs/designs/FOLLOWUP_WAVE_2026_10_07.md","docs/gbrain-sync-errors.md","docs/gbrain-sync.md","docs/gbrain-write-surfaces.md","docs/troubleshooting.md","document-generate/SKILL.md","document-release/SKILL.md","document-release/sections/release-body.md","health/SKILL.md","investigate/SKILL.md","ios-clean/SKILL.md","ios-design-review/SKILL.md","ios-fix/SKILL.md","ios-qa/SKILL.md","ios-sync/SKILL.md","land-and-deploy/SKILL.md","landing-report/SKILL.md","learn/SKILL.md","lib/claude-journal-records.ts","lib/claude-owned-journal.ts","lib/claude-public-transcript.ts","lib/greptile-early.ts","lib/gstack-post.ts","lib/outside-review-result.ts","make-pdf/SKILL.md","office-hours/SKILL.md","office-hours/sections/design-and-handoff.md","open-gstack-browser/SKILL.md","package.json","pair-agent/SKILL.md","plan-ceo-review/SKILL.md","plan-ceo-review/sections/review-sections.md","plan-design-review/SKILL.md","plan-devex-review/SKILL.md","plan-devex-review/sections/review-sections.md","plan-eng-review/SKILL.md","plan-eng-review/sections/review-sections.md","plan-tune/SKILL.md","plan-tune/SKILL.md.tmpl","qa-only/SKILL.md","qa/SKILL.md","qa/sections/browser-setup.md","retro/SKILL.md","review/SKILL.md","review/greptile-triage.md","review/sections/adversarial.md","review/sections/review-army.md","scrape/SKILL.md","scripts/harness-version.json","scripts/lib/windows-curation.ts","scripts/measure-journal-read.ts","scripts/resolvers/aside.ts","scripts/resolvers/constants.ts","scripts/resolvers/design.ts","scripts/resolvers/gbrain.ts","scripts/resolvers/index.ts","scripts/resolvers/outside-voice-steps.ts","scripts/resolvers/outside-voice.ts","scripts/resolvers/preamble/generate-brain-sync-block.ts","scripts/resolvers/question-tuning.ts","scripts/resolvers/redact-doc.ts","scripts/resolvers/review-army.ts","scripts/resolvers/spec-review.ts","scripts/test-pr-profile.ts","scripts/typecheck-test-baseline.json","setup","setup-browser-cookies/SKILL.md","setup-deploy/SKILL.md","setup-gbrain/SKILL.md","ship/SKILL.md","ship/SKILL.md.tmpl","ship/sections/adversarial.md","ship/sections/documentation.md","ship/sections/greptile-early.md","ship/sections/greptile-early.md.tmpl","ship/sections/greptile.md","ship/sections/greptile.md.tmpl","ship/sections/manifest.json","ship/sections/pr-body.md","ship/sections/pr-body.md.tmpl","ship/sections/review-army.md","skillify/SKILL.md","spec/SKILL.md","spec/sections/gate-and-file.md","spec/sections/gate-and-file.md.tmpl","sync-gbrain/SKILL.md","test-audit/SKILL.md","test/aside-driver.test.ts","test/autoplan-bounded-journal.test.ts","test/autoplan-dual-voice-evidence.test.ts","test/autoplan-guard-log.test.ts","test/autoplan-guard-reasons.test.ts","test/autoplan-long-session-observer.test.ts","test/autoplan-publication-guard.test.ts","test/autoplan-publication-hook.test.ts","test/autoplan-publication-payload.test.ts","test/auLine truncated
|
||||
{"runId":37694473528,"pr":3069,"headSha":"c2d1aa1976a1403a0a6f7846a748080f61c8c726","packageVersionOnly":true,"failedGateCases":["qa-quick","shared-libs-read-only","shared-libs-unsupported-git"],"changedFiles":[".github/ISSUE_TEMPLATE/bug_report.md",".github/docker/Dockerfile.ci",".github/workflows/evals-periodic.yml",".github/workflows/free-tests.yml",".github/workflows/measure-journal-read.yml","CHANGELOG.md","README.md","SKILL.md","TODOS.md","VERSION","agents-digest/gstack-AGENTS.md","autoplan/SKILL.md","autoplan/bin/guard-journal.ts","autoplan/bin/guard-log.ts","autoplan/bin/guard-reasons.ts","autoplan/bin/phase-publication-hook.ts","autoplan/sections/ceo-phase.md","autoplan/sections/ceo-phase.md.tmpl","autoplan/sections/design-phase.md","autoplan/sections/design-phase.md.tmpl","autoplan/sections/dx-phase.md","autoplan/sections/dx-phase.md.tmpl","autoplan/sections/eng-phase.md","autoplan/sections/eng-phase.md.tmpl","autoplan/sections/phase-close.md","autoplan/sections/phase-close.md.tmpl","benchmark-models/SKILL.md","benchmark/SKILL.md","bin/gstack-artifacts-init","bin/gstack-brain-lock.sh","bin/gstack-brain-restore","bin/gstack-brain-sync","bin/gstack-codex-probe","bin/gstack-codex-status.sh","bin/gstack-config","bin/gstack-doctor","bin/gstack-gbrain-source-wireup","bin/gstack-greptile-early","bin/gstack-post","bin/gstack-pr-title-rewrite.sh","bin/gstack-question-preference","bin/gstack-redact-prepush","bin/gstack-skill-start","bin/gstack-slug","browse/SKILL.md","browse/test/extension-sender-auth.test.ts","canary/SKILL.md","codex/SKILL.md","codex/SKILL.md.tmpl","codex/sections/challenge-mode.md","codex/sections/challenge-mode.md.tmpl","codex/sections/consult-mode.md","codex/sections/consult-mode.md.tmpl","codex/sections/review-mode.md","codex/sections/review-mode.md.tmpl","context-restore/SKILL.md","context-restore/SKILL.md.tmpl","context-save/SKILL.md","context-save/SKILL.md.tmpl","design-consultation/SKILL.md","design-consultation/sections/proposal-and-preview.md","design-html/SKILL.md","design-review/SKILL.md","design-shotgun/SKILL.md","devex-review/SKILL.md","diagram/SKILL.md","docs/autoplan-guard-troubleshooting.md","docs/designs/FOLLOWUP_WAVE_2026_10_07.md","docs/gbrain-sync-errors.md","docs/gbrain-sync.md","docs/gbrain-write-surfaces.md","docs/troubleshooting.md","document-generate/SKILL.md","document-release/SKILL.md","document-release/sections/release-body.md","health/SKILL.md","investigate/SKILL.md","ios-clean/SKILL.md","ios-design-review/SKILL.md","ios-fix/SKILL.md","ios-qa/SKILL.md","ios-sync/SKILL.md","land-and-deploy/SKILL.md","landing-report/SKILL.md","learn/SKILL.md","lib/claude-journal-records.ts","lib/claude-owned-journal.ts","lib/claude-public-transcript.ts","lib/greptile-early.ts","lib/gstack-post.ts","lib/outside-review-result.ts","make-pdf/SKILL.md","office-hours/SKILL.md","office-hours/sections/design-and-handoff.md","open-gstack-browser/SKILL.md","package.json","pair-agent/SKILL.md","plan-ceo-review/SKILL.md","plan-ceo-review/sections/review-sections.md","plan-design-review/SKILL.md","plan-devex-review/SKILL.md","plan-devex-review/sections/review-sections.md","plan-eng-review/SKILL.md","plan-eng-review/sections/review-sections.md","plan-tune/SKILL.md","plan-tune/SKILL.md.tmpl","qa-only/SKILL.md","qa/SKILL.md","qa/sections/browser-setup.md","retro/SKILL.md","review/SKILL.md","review/greptile-triage.md","review/sections/adversarial.md","review/sections/review-army.md","scrape/SKILL.md","scripts/harness-version.json","scripts/lib/windows-curation.ts","scripts/measure-journal-read.ts","scripts/resolvers/aside.ts","scripts/resolvers/constants.ts","scripts/resolvers/design.ts","scripts/resolvers/gbrain.ts","scripts/resolvers/index.ts","scripts/resolvers/outside-voice-steps.ts","scripts/resolvers/outside-voice.ts","scripts/resolvers/preamble/generate-brain-sync-block.ts","scripts/resolvers/question-tuning.ts","scripts/resolvers/redact-doc.ts","scripts/resolvers/review-army.ts","scripts/resolvers/spec-review.ts","scripts/test-pr-profile.ts","scripts/typecheck-test-baseline.json","setup","setup-browser-cookies/SKILL.md","setup-deploy/SKILL.md","setup-gbrain/SKILL.md","ship/SKILL.md","ship/SKILL.md.tmpl","ship/sections/adversarial.md","ship/sections/documentation.md","ship/sections/greptile-early.md","ship/sections/greptile-early.md.tmpl","ship/sections/greptile.md","ship/sections/greptile.md.tmpl","ship/sections/manifest.json","ship/sections/pr-body.md","ship/sections/pr-body.md.tmpl","ship/sections/review-army.md","skillify/SKILL.md","spec/SKILL.md","spec/sections/gate-and-file.md","spec/sections/gate-and-file.md.tmpl","sync-gbrain/SKILL.md","test-audit/SKILL.md","test/aside-driver.test.ts","test/autoplan-bounded-journal.test.ts","test/autoplan-dual-voice-evidence.test.ts","test/autoplan-guard-log.test.ts","test/autoplan-guard-reasons.test.ts","test/autoplan-long-session-observer.test.ts","test/autoplan-publication-guard.test.ts","test/autoplan-publication-hook.test.ts","test/autopLine truncated
|
||||
{"runId":37731666423,"pr":3073,"headSha":"872e1542e42d40b3a95356d5284c6e4a3553c31f","packageVersionOnly":true,"failedGateCases":["plan-ceo-finding-floor","ship-docsync-missing-asset"],"changedFiles":[".github/workflows/model-policy-freshness.yml","CHANGELOG.md","CONTRIBUTING.md","README.md","VERSION","agents-digest/gstack-AGENTS.md","autoplan/SKILL.md","autoplan/SKILL.md.tmpl","autoplan/sections/ceo-phase.md","autoplan/sections/design-phase.md","autoplan/sections/dx-phase.md","autoplan/sections/eng-phase.md","bin/gstack-codex-probe","bin/gstack-config","bin/gstack-models","claude-code/SKILL.md.tmpl","codex/SKILL.md","codex/SKILL.md.tmpl","codex/sections/challenge-mode.md","codex/sections/challenge-mode.md.tmpl","codex/sections/consult-mode.md","codex/sections/consult-mode.md.tmpl","codex/sections/review-mode.md","codex/sections/review-mode.md.tmpl","docs/model-policy.md","docs/skills.md","docs/troubleshooting.md","lib/claude-code-migration.ts","lib/claude-code.ts","lib/model-catalog.ts","lib/model-policy-cli.ts","lib/model-policy-freshness-github.ts","lib/model-policy-freshness-parser-worker.ts","lib/model-policy-freshness-publication.ts","lib/model-policy-freshness-sources.ts","lib/model-policy-freshness.ts","lib/model-policy-notice.ts","lib/model-policy.ts","package.json","plan-ceo-review/sections/review-sections.md","plan-design-review/SKILL.md","plan-devex-review/sections/review-sections.md","plan-eng-review/sections/review-sections.md","plan-eng-review/sections/review-sections.md.tmpl","review/SKILL.md","scripts/harness-version.json","scripts/model-policy-freshness.ts","scripts/preflight-codex-overlap.ts","scripts/resolve-codex-generation-model.ts","scripts/resolvers/constants.ts","scripts/resolvers/design.ts","scripts/resolvers/index.ts","scripts/resolvers/outside-voice-steps.ts","scripts/resolvers/outside-voice.ts","scripts/resolvers/plan-review.ts","scripts/resolvers/qa.ts","scripts/test-pr-profile.ts","ship/sections/review-army.md","spec/SKILL.md","spec/SKILL.md.tmpl","spec/sections/gate-and-file.md","spec/sections/gate-and-file.md.tmpl","test/claude-code-migration.test.ts","test/claude-code-skill.test.ts","test/codex-hardening.test.ts","test/codex-probe-cache.test.ts","test/codex-under-codex-detection.test.ts","test/fixtures/model-policy-freshness.ts","test/fixtures/model-policy-freshness/anthropic-lifecycle.md","test/fixtures/model-policy-freshness/anthropic-models.md","test/fixtures/model-policy-freshness/openai-lifecycle.md","test/fixtures/model-policy-freshness/openai-models.md","test/fixtures/qa-caller-read-probe-mtb1007a.json","test/helpers/free-fixtures-data.ts","test/helpers/pty/launch.ts","test/helpers/qa-callers-fixture.ts","test/model-catalog-isolation.test.ts","test/model-policy-cli.test.ts","test/model-policy-freshness.test.ts","test/model-policy-workflow.test.ts","test/model-policy.test.ts","test/model-role-invocation.test.ts","test/outside-voice-invocation.test.ts","test/outside-voice-preflight.test.ts","test/plan-review-cases.test.ts","test/pty-output-wake.test.ts","test/qa-exploratory-callers.test.ts","test/qa-probe-gates.test.ts","test/run-in-background-guidance.test.ts","test/setup-claude-code-migration.test.ts","test/team-mode.test.ts"]},
|
||||
{"runId":37739845694,"pr":3077,"headSha":"e2e725ad3ddefac67b57f91c2a000848de56b752","packageVersionOnly":true,"failedGateCases":["ship-docsync-missing-asset"],"changedFiles":["CHANGELOG.md","TODOS.md","VERSION","agents-digest/gstack-AGENTS.md","careful/SKILL.md","careful/SKILL.md.tmpl","careful/bin/check-careful.sh","careful/bin/hook-extract.sh","docs/TESTING_INTERNALS.md","docs/skills.md","freeze/SKILL.md","freeze/SKILL.md.tmpl","freeze/bin/check-freeze.sh","guard/SKILL.md","guard/SKILL.md.tmpl","investigate/SKILL.md","investigate/SKILL.md.tmpl","package.json","scripts/harness-version.json","scripts/lib/windows-curation.ts","scripts/test-pr-profile.ts","test/fixtures/context-budget.json","test/hook-scripts.test.ts"]},
|
||||
{"runId":37743622252,"pr":3079,"headSha":"4637fae03baad83603ab7d554f4137c4702b4bd4","packageVersionOnly":true,"failedGateCases":["qa-quick"],"changedFiles":[".github/workflows/free-tests.yml",".github/workflows/make-pdf-gate.yml",".github/workflows/windows-setup-e2e.yml",".gitignore","CHANGELOG.md","README.md","TODOS.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-doctor","bin/gstack-global-discover.ts","bin/gstack-launch-probe.sh","browse/src/cli.ts","browse/src/find-browse.ts","browse/test/sidebar-ux.test.ts","docs/troubleshooting.md","extension/sidepanel-terminal.js","extension/sidepanel.css","make-pdf/src/cli.ts","package.json","scripts/build-cso.sh","scripts/build.sh","setup","test/binary-version-probe.test.ts","test/build-cso-nonfatal.test.ts","test/build-cso-result-record.test.ts","test/cso-windows-launcher.test.ts","test/gstack-doctor.test.ts","test/gstack-launch-probe.test.ts","test/setup-cso-degrade.test.ts","test/setup-cso-windows-probe.test.ts","test/setup-windows-launch-hint.test.ts"]},
|
||||
{"runId":37762901988,"pr":3079,"headSha":"b29f02250967df02f0e02dc4c8310155771cdf41","packageVersionOnly":true,"failedGateCases":["ship-exploratory-unavailable"],"changedFiles":[".github/workflows/free-tests.yml",".github/workflows/make-pdf-gate.yml",".github/workflows/windows-setup-e2e.yml",".gitignore","CHANGELOG.md","README.md","TODOS.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-doctor","bin/gstack-global-discover.ts","bin/gstack-launch-probe.sh","browse/src/cli.ts","browse/src/find-browse.ts","browse/test/sidebar-ux.test.ts","docs/troubleshooting.md","extension/sidepanel-terminal.js","extension/sidepanel.css","make-pdf/src/cli.ts","package.json","scripts/build-cso.sh","scripts/build.sh","setup","test/binary-version-probe.test.ts","test/build-cso-nonfatal.test.ts","test/build-cso-result-record.test.ts","test/cso-windows-launcher.test.ts","test/gstack-doctor.test.ts","test/gstack-launch-probe.test.ts","test/setup-cso-degrade.test.ts","test/setup-cso-windows-probe.test.ts","test/setup-windows-launch-hint.test.ts"]},
|
||||
{"runId":38087752336,"pr":3114,"headSha":"3e51fb06ba5c9c8943e7211ba32fb93fcccfe402","packageVersionOnly":false,"failedGateCases":["qa-functional-cli-report"],"changedFiles":["AGENTS.md","ETHOS.md","README.md","SKILL.md","autoplan/SKILL.md","benchmark-models/SKILL.md","benchmark/SKILL.md","bin/gstack-evidence","bin/gstack-gate-diff","bin/gstack-review-log","browse/SKILL.md","canary/SKILL.md","codex/SKILL.md","context-restore/SKILL.md","context-save/SKILL.md","design-consultation/SKILL.md","design-html/SKILL.md","design-review/SKILL.md","design-shotgun/SKILL.md","devex-review/SKILL.md","diagram/SKILL.md","docs/designs/HONEST_WORK_GATE_INTEGRITY.md","docs/designs/gate-diff-calibration.md","document-generate/SKILL.md","document-release/SKILL.md","health/SKILL.md","investigate/SKILL.md","ios-clean/SKILL.md","ios-design-review/SKILL.md","ios-fix/SKILL.md","ios-qa/SKILL.md","ios-sync/SKILL.md","land-and-deploy/SKILL.md","landing-report/SKILL.md","learn/SKILL.md","lib/gate-diff/classify.ts","lib/gate-diff/git.ts","lib/gate-diff/index.ts","lib/gate-diff/render.ts","lib/gate-diff/tags.ts","lib/gate-diff/types.ts","lib/review-evidence.ts","make-pdf/SKILL.md","office-hours/SKILL.md","open-gstack-browser/SKILL.md","pair-agent/SKILL.md","plan-ceo-review/SKILL.md","plan-ceo-review/sections/review-sections.md","plan-ceo-review/sections/review-sections.md.tmpl","plan-design-review/SKILL.md","plan-devex-review/SKILL.md","plan-eng-review/SKILL.md","plan-eng-review/sections/review-sections.md","plan-tune/SKILL.md","qa-only/SKILL.md","qa/SKILL.md","retro/SKILL.md","review/SKILL.md","review/SKILL.md.tmpl","review/checklist.md","review/gate-integrity.md","review/specialists/testing.md","scrape/SKILL.md","scripts/gate-diff-calibrate.ts","scripts/harness-version.json","scripts/question-registry.ts","scripts/resolvers/dispatch-rules.ts","scripts/resolvers/gate-scan.ts","scripts/resolvers/index.ts","scripts/resolvers/plan-gates.ts","scripts/resolvers/preamble/generate-completion-status.ts","scripts/resolvers/preamble/generate-evidence-directive.ts","scripts/resolvers/test-value.ts","scripts/resolvers/testing.ts","scripts/test-pr-profile.ts","setup-browser-cookies/SKILL.md","setup-deploy/SKILL.md","setup-gbrain/SKILL.md","ship/SKILL.md","ship/SKILL.md.tmpl","ship/sections/plan-completion.md","ship/sections/plan-completion.md.tmpl","ship/sections/pr-body.md","ship/sections/pr-body.md.tmpl","ship/sections/review-army.md","ship/sections/review-army.md.tmpl","ship/sections/test-coverage.md","ship/sections/test-coverage.md.tmpl","ship/sections/tests.md","ship/sections/tests.md.tmpl","skillify/SKILL.md","spec/SKILL.md","spec/SKILL.md.tmpl","spec/sections/gate-and-file.md","spec/sections/gate-and-file.md.tmpl","sync-gbrain/SKILL.md","test-audit/SKILL.md","test/dispatch-rules.test.ts","test/eng-finding-retry-budget.test.ts","test/env-host-runtime-assets.test.ts","test/evidence.test.ts","test/fixtures/context-budget.json","test/fixtures/gate-diff/adv-forged-measured.base.patch","test/fixtures/gate-diff/adv-forged-measured.patch","test/fixtures/gate-diff/adv-golden-beside-unrelated.base.patch","test/fixtures/gate-diff/adv-golden-beside-unrelated.patch","test/fixtures/gate-diff/adv-kotlin-ignore.base.patch","test/fixtures/gate-diff/adv-kotlin-ignore.patch","test/fixtures/gate-diff/control-added-test.base.patch","test/fixtures/gate-diff/control-added-test.patch","test/fixtures/gate-diff/control-rename.base.patch","test/fixtures/gate-diff/control-rename.patch","test/fixtures/gate-diff/control-snapshot-with-owner.base.patch","test/fixtures/gate-diff/control-snapshot-with-owner.patch","test/fixtures/gate-diff/control-spec-skill-dir.base.patch","test/fixtures/gate-diff/control-spec-skill-dir.patch","test/fixtures/gate-diff/control-test-deleted-with-source.base.patch","test/fixtures/gate-diff/control-test-deleted-with-source.patch","test/fixtures/gate-diff/package-json-scripts.base.patch","test/fixtures/gate-diff/package-json-scripts.patch","test/fixtures/gate-diff/rh1-removed.base.patch","test/fixtures/gate-diff/rh1-removed.patch","test/fixtures/gate-diff/rh1-skip.base.patch","test/fixtures/gate-diff/rh1-skip.patch","test/fixtures/gate-diff/rh12-env.base.patch","test/fixtures/gate-diff/rh12-env.patch","test/fixtures/gate-diff/rh13-pragma.base.patch","test/fixtures/gate-diff/rh13-pragma.patch","test/fixtures/gate-diff/rh14-bypass.base.patch","test/fixtures/gate-diff/rh14-bypass.patch","test/fixtures/gate-diff/rh15-bound.base.patch","test/fixtures/gate-diff/rh15-bound.patch","test/fixtures/gate-diff/rh15-timeout.base.patch","test/fixtures/gate-diff/rh15-timeout.patch","test/fixtures/gate-diff/rh16-swallow.base.patch","test/fixtures/gate-diff/rh16-swallow.patch","test/fixtures/gate-diff/rh3-golden.base.patch","test/fixtures/gate-diff/rh3-golden.patch","test/fixtures/gate-diff/rh3-snapshot-no-owner.base.patch","test/fixtures/gate-diff/rh3-snapshot-no-owner.patch","test/fixtures/gate-diff/rh4-placeholder.base.patch","test/fixtures/gate-diff/rh4-placeLine truncated
|
||||
{"runId":38095294863,"pr":3116,"headSha":"82828943447b168adbbdc173c2095fcf33ba6f53","packageVersionOnly":true,"failedGateCases":["ship-exploratory-plan-checks"],"changedFiles":["CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","benchmark/SKILL.md","bin/gstack-browser-ensure","bin/gstack-bun-version.sh","bin/gstack-capy-install","bin/gstack-doctor","bin/gstack-doctor-check.sh","bin/gstack-doctor-components.sh","bin/gstack-runtime-pins.ts","browse/SKILL.md","canary/SKILL.md","design-consultation/SKILL.md","design-review/SKILL.md","devex-review/SKILL.md","docs/capy-install.md","docs/state-root.md","docs/troubleshooting.md","land-and-deploy/SKILL.md","lib/doctor-components.ts","lib/result-codes.ts","lib/runtime-pins.ts","open-gstack-browser/SKILL.md","package.json","pair-agent/SKILL.md","qa/sections/browser-setup.md","scrape/SKILL.md","scripts/gen-doctor-components.ts","scripts/gen-skill-docs.ts","scripts/resolvers/browse.ts","scripts/typecheck-test-baseline.json","setup","setup-browser-cookies/SKILL.md","test/bun-version-drift.test.ts","test/doctor-components.test.ts","test/egress-receipt-wiring.test.ts","test/fixtures/module-size-ratchet.json","test/gen-skill-docs-checks.test.ts","test/gen-skill-docs-prune-stale.test.ts","test/gstack-browser-ensure.test.ts","test/gstack-capy-install.test.ts","test/gstack-doctor.test.ts","test/helpers/manual-judge-review-fixture.ts","test/host-conformance.test.ts","test/runtime-pins.test.ts","test/troubleshooting-anchors.test.ts"]},
|
||||
{"runId":38097941308,"pr":3117,"headSha":"d0febdf79b1608f3de47f3c1ec099f7829f709f5","packageVersionOnly":true,"failedGateCases":["qa-functional-cli-report"],"changedFiles":["CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","autoplan/SKILL.md","autoplan/SKILL.md.tmpl","autoplan/sections/manifest.json","autoplan/sections/unattended.md","autoplan/sections/unattended.md.tmpl","bin/gstack-autoplan","bin/gstack-codex-login","bin/gstack-doctor-components.sh","bin/gstack-office-hours-review","bin/gstack-outside-voice","docs/troubleshooting.md","docs/unattended.md","lib/autoplan-export.ts","lib/autoplan-gate.ts","lib/autoplan-prompts.ts","lib/autoplan-reconcile.ts","lib/autoplan-run.ts","lib/autoplan-state.ts","lib/headless-artifacts.ts","lib/office-hours-review.ts","lib/outside-voice-api.ts","lib/outside-voice-runner.ts","lib/result-codes.ts","lib/spec-review-cap.ts","lib/spend-ledger.ts","office-hours/sections/design-and-handoff.md","package.json","plan-ceo-review/SKILL.md","plan-ceo-review/sections/checklist.md","plan-ceo-review/sections/checklist.md.tmpl","plan-ceo-review/sections/manifest.json","plan-design-review/sections/checklist.md","plan-design-review/sections/checklist.md.tmpl","plan-design-review/sections/manifest.json","plan-devex-review/sections/checklist.md","plan-devex-review/sections/checklist.md.tmpl","plan-devex-review/sections/manifest.json","plan-eng-review/sections/checklist.md","plan-eng-review/sections/checklist.md.tmpl","plan-eng-review/sections/manifest.json","scripts/resolvers/sections.ts","scripts/resolvers/spec-review.ts","test/autoplan-answer-invalidation.test.ts","test/autoplan-run-concurrency.test.ts","test/autoplan-run-recovery.test.ts","test/egress-receipt-wiring.test.ts","test/fixtures/module-size-ratchet.json","test/fixtures/multi-agent-wave/fixture-reviewer.ts","test/helpers/autoplan-run-fixture.ts","test/helpers/free-fixtures-data.ts","test/helpers/office-hours-completion.ts","test/helpers/workflow-judge-input.ts","test/office-hours-completion.test.ts","test/office-hours-review.test.ts","test/outside-runner-contract.test.ts","test/review-checklists.test.ts","test/spend-ledger.test.ts","test/workflow-judge-input.test.ts"]},
|
||||
{"runId":38101502677,"pr":3119,"headSha":"c9a1b4c273a2479dd62d10ba1477d0fe5c6c5824","packageVersionOnly":true,"failedGateCases":["land-and-deploy-review-gate","qa-functional-cli-fix"],"changedFiles":["CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","autoplan/SKILL.md","autoplan/SKILL.md.tmpl","autoplan/sections/ceo-phase.md","autoplan/sections/ceo-phase.md.tmpl","autoplan/sections/eng-phase.md","autoplan/sections/eng-phase.md.tmpl","benchmark/SKILL.md","benchmark/SKILL.md.tmpl","bin/gstack-doctor-components.sh","bin/gstack-plan-reality","cso/SKILL.md","cso/SKILL.md.tmpl","docs/troubleshooting.md","investigate/SKILL.md","investigate/SKILL.md.tmpl","investigate/sections/backlog.md","investigate/sections/backlog.md.tmpl","investigate/sections/eval-bisect.md","investigate/sections/eval-bisect.md.tmpl","investigate/sections/flake.md","investigate/sections/flake.md.tmpl","investigate/sections/manifest.json","investigate/sections/polluter.md","investigate/sections/polluter.md.tmpl","investigate/sections/race.md","investigate/sections/race.md.tmpl","lib/plan-reality.ts","lib/result-codes.ts","package.json","plan-ceo-review/SKILL.md","plan-ceo-review/SKILL.md.tmpl","plan-ceo-review/sections/checklist.md","plan-ceo-review/sections/checklist.md.tmpl","plan-ceo-review/sections/review-sections.md","plan-ceo-review/sections/review-sections.md.tmpl","plan-eng-review/sections/checklist.md","plan-eng-review/sections/checklist.md.tmpl","plan-eng-review/sections/review-sections.md","plan-eng-review/sections/review-sections.md.tmpl","review/SKILL.md","review/SKILL.md.tmpl","review/checklist.md","review/sections/plan-completion.md","scripts/resolvers/index.ts","scripts/resolvers/plan-gates.ts","scripts/resolvers/plan-reality.ts","scripts/resolvers/sideways.ts","test/carve-section-loading.test.ts","test/fixtures/module-size-ratchet.json","test/fixtures/multi-agent-wave/INCIDENTS.md","test/fixtures/multi-agent-wave/reality-rows.json","test/helpers/carve-guards.ts","test/helpers/free-fixtures-data.ts","test/helpers/parity-harness.ts","test/helpers/touchfiles-data.ts","test/investigate-recipes.test.ts","test/plan-reality.test.ts","test/qa-supervision-selection.test.ts","test/sideways-review.test.ts","test/skill-fixture.test.ts"]},
|
||||
{"runId":38101616162,"pr":3121,"headSha":"819ef8e083162096a5259625b6cc74465d3889df","packageVersionOnly":true,"failedGateCases":["design-review-plugin-handoff"],"changedFiles":[".gitignore",".gstack/generated.json",".gstack/pregate.json","CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-doctor-components.sh","bin/gstack-issue-links","bin/gstack-pregate","bin/gstack-regen","bin/gstack-ship-receipt","docs/pregate.md","docs/troubleshooting.md","lib/headless-artifacts.ts","lib/issue-links.ts","lib/pregate/checks/lanes.ts","lib/pregate/checks/literals.ts","lib/pregate/checks/regen.ts","lib/pregate/checks/secrets.ts","lib/pregate/checks/strays.ts","lib/pregate/config.ts","lib/pregate/context.ts","lib/pregate/import-graph.ts","lib/pregate/index.ts","lib/pregate/lane-selection.ts","lib/pregate/types.ts","lib/pregate/workflows.ts","lib/regen.ts","lib/result-codes.ts","lib/runtime-pins.ts","lib/ship-receipt.ts","package.json","scripts/gen-ship-goldens.ts","ship/sections/pr-body.md","ship/sections/pr-body.md.tmpl","ship/sections/tests.md","ship/sections/tests.md.tmpl","test/fixtures/module-size-ratchet.json","test/fixtures/multi-agent-wave/INCIDENTS.md","test/gstack-regen.test.ts","test/helpers/pregate-fixture.ts","test/pregate-invalidation.test.ts","test/pregate-selection.test.ts","test/ship-issue-intent.test.ts"]},
|
||||
{"runId":38336400566,"pr":3166,"headSha":"8bf348ddee3a31bed6e914f0cd95e49bf12f3d8e","packageVersionOnly":true,"failedGateCases":["ship-docsync-launch-failure"],"changedFiles":["CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-doctor-components.sh","bin/gstack-eval-plan","bin/gstack-pregate","bin/gstack-redact-prepush","docs/designs/GSTACK_FIELD_WAVE_2026_10_12.md","docs/eval-calls.md","docs/pregate.md","docs/ship-policy.md","docs/troubleshooting.md","docs/unattended.md","eval-plan/SKILL.md","eval-plan/SKILL.md.tmpl","lib/coordinator-contract.ts","lib/eval-plan.ts","lib/pregate/checks/guards.ts","lib/pregate/checks/lanes.ts","lib/pregate/config.ts","lib/pregate/index.ts","lib/pregate/regate.ts","lib/pregate/runtime.ts","lib/pricing.ts","lib/restamp.ts","lib/result-codes.ts","lib/sealed-guard.ts","lib/ship-policy.ts","lib/tree-receipt.ts","package.json","scripts/lib/windows-curation.ts","scripts/pr-dependencies.ts","spec/sections/gate-and-file.md","test/benchmark-runner.test.ts","test/fixtures/module-size-ratchet.json","test/helpers/capy-host.ts","test/helpers/placeholder-allowlist.ts","test/lane-coordination.test.ts","test/pr-dependencies.test.ts","test/pregate-guards.test.ts","test/pregate-invalidation.test.ts","test/pregate-regate.test.ts","test/pregate-selection.test.ts","test/pricing-eval-plan.test.ts","test/restamp-command-boundary.test.ts","test/sealed-guard.test.ts","test/stamp-templates.test.ts"]},
|
||||
{"runId":38337974933,"pr":3164,"headSha":"f6997e51f4351963fc198bf389872a92ee5b95c1","packageVersionOnly":true,"failedGateCases":["tpa-apple-ban"],"changedFiles":["CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-bun-version.sh","bin/gstack-capy-install","bin/gstack-capy-session","bin/gstack-doctor","bin/gstack-doctor-check.sh","bin/gstack-doctor-components.sh","bin/gstack-install-verdict.sh","bin/gstack-skill-end","bin/gstack-skill-start","bin/gstack-upgrade","capy/skill/gstack/SKILL.md","capy/skill/gstack/scripts/install.sh","capy/support.env","docs/CHANGELOG_STYLE.md","docs/capy-install.md","docs/designs/GSTACK_FIELD_WAVE_2026_10_12.md","docs/troubleshooting.md","docs/unattended.md","lib/doctor-components.ts","lib/install-verdict.ts","lib/result-codes.ts","package.json","scripts/harness-version.json","scripts/test-pr-profile.ts","test/capy-skill-wrapper.test.ts","test/fixtures/context-budget.json","test/gstack-capy-install.test.ts","test/gstack-doctor.test.ts","test/gstack-skill-start.test.ts","test/gstack-upgrade-bin.test.ts","test/helpers/capy-host.ts","test/install-verdict.test.ts","test/test-pr-profile.test.ts"]},
|
||||
{"runId":38345572852,"pr":3164,"headSha":"2dc7afe70762ddefd69c5ebbf15eea4cc3669fa4","packageVersionOnly":true,"failedGateCases":["qa-functional-webhook-report"],"changedFiles":["CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-bun-version.sh","bin/gstack-capy-install","bin/gstack-capy-session","bin/gstack-doctor","bin/gstack-doctor-check.sh","bin/gstack-doctor-components.sh","bin/gstack-install-verdict.sh","bin/gstack-skill-end","bin/gstack-skill-start","bin/gstack-upgrade","capy/skill/gstack/SKILL.md","capy/skill/gstack/scripts/install.sh","capy/support.env","docs/CHANGELOG_STYLE.md","docs/capy-install.md","docs/designs/GSTACK_FIELD_WAVE_2026_10_12.md","docs/troubleshooting.md","docs/unattended.md","lib/doctor-components.ts","lib/install-verdict.ts","lib/result-codes.ts","package.json","scripts/harness-version.json","scripts/test-pr-profile.ts","test/capy-skill-wrapper.test.ts","test/fixtures/context-budget.json","test/gstack-capy-install.test.ts","test/gstack-doctor.test.ts","test/gstack-skill-start.test.ts","test/gstack-upgrade-bin.test.ts","test/helpers/capy-host.ts","test/install-verdict.test.ts","test/test-pr-profile.test.ts"]},
|
||||
{"runId":38418037093,"pr":3170,"headSha":"44d380c494816ea915db718c7e3e98889e8f3256","packageVersionOnly":true,"failedGateCases":["shared-libs-read-only"],"changedFiles":["CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","autoplan/DRIVER.md","autoplan/DRIVER.md.tmpl","autoplan/SKILL.md","autoplan/SKILL.md.tmpl","bin/gstack-artifact","bin/gstack-autoplan","bin/gstack-autoplan-snapshot.ts","bin/gstack-doctor-components.sh","bin/gstack-plan-reality","docs/troubleshooting.md","docs/unattended.md","lib/autoplan-admission.ts","lib/autoplan-gate.ts","lib/autoplan-migrate.ts","lib/autoplan-plan-text.ts","lib/autoplan-prompts.ts","lib/autoplan-reconcile.ts","lib/autoplan-revise.ts","lib/autoplan-run.ts","lib/autoplan-scope.ts","lib/autoplan-state.ts","lib/gate-list.ts","lib/headless-artifacts.ts","lib/plan-reality.ts","lib/result-codes.ts","lib/source-root.ts","package.json","scripts/gen-skill-docs.ts","scripts/resolvers/autoplan-driver.ts","scripts/resolvers/index.ts","test/autoplan-admission.test.ts","test/autoplan-answer-invalidation.test.ts","test/autoplan-capy-e2e.test.ts","test/autoplan-driver.test.ts","test/autoplan-gate-judgment.test.ts","test/autoplan-record.test.ts","test/autoplan-revise.test.ts","test/autoplan-run-concurrency.test.ts","test/autoplan-scope.test.ts","test/fixtures/module-size-ratchet.json","test/gen-skill-docs-checks.test.ts","test/helpers/autoplan-capy.ts","test/plan-reality.test.ts","test/review-checklists.test.ts"]},
|
||||
{"runId":38428252213,"pr":3171,"headSha":"e2b08d308cd72f6d9bfbfcf514f0ece0891da153","packageVersionOnly":true,"failedGateCases":["land-and-deploy-workflow","ship-docsync-stale-before","ship-exploratory-plan-checks"],"changedFiles":["CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-doctor-check.sh","capy/skill/gstack/SKILL.md","docs/troubleshooting.md","lib/result-codes.ts","package.json","test/gstack-doctor.test.ts"]},
|
||||
{"runId":38478947158,"pr":3173,"headSha":"b6389e0631c99449be8f6ac8e9466f26b3daa611","packageVersionOnly":true,"failedGateCases":["ship-exploratory-plan-checks"],"changedFiles":["CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","autoplan/DRIVER.md","autoplan/DRIVER.md.tmpl","bin/gstack-autoplan","bin/gstack-capy-session","bin/gstack-doctor-components.sh","bin/gstack-gbrain-sync.ts","bin/gstack-harness-pid.sh","bin/gstack-lease","bin/gstack-skill-start","bin/gstack-upgrade","capy/skill/gstack/SKILL.md","capy/support.env","docs/capy-install.md","docs/designs/FIX_WAVE_2026_10_12.md","docs/troubleshooting.md","docs/unattended.md","lib/autoplan-admission.ts","lib/autoplan-export.ts","lib/autoplan-gate.ts","lib/autoplan-prompts.ts","lib/autoplan-run.ts","lib/gate-list.ts","lib/headless-artifacts.ts","lib/owner-brief.ts","lib/restamp.ts","lib/result-codes.ts","package.json","scripts/harness-version.json","test-setup.ts","test/auto-decide-fixture.test.ts","test/autoplan-answer-invalidation.test.ts","test/autoplan-capy-e2e.test.ts","test/autoplan-gate-judgment.test.ts","test/autoplan-preconfigured-onboarding-ar.test.ts","test/autoplan-residuals.test.ts","test/autoplan-revise.test.ts","test/capy-skill-wrapper.test.ts","test/ceo-finding-fixture.test.ts","test/gstack-lease.test.ts","test/gstack-skill-start.test.ts","test/gstack-upgrade-bin.test.ts","test/helpers/capy-host.ts","test/helpers/hermetic-env.ts"]},
|
||||
{"runId":38829994639,"pr":3209,"headSha":"2961edad7f56c982d78c97a85c87015bab29b155","packageVersionOnly":false,"failedGateCases":["qa-functional-cli-report"],"changedFiles":[".github/dependabot.yml",".github/workflows/arm-setup-smoke.yml",".github/workflows/cso-runtime-images.yml",".github/workflows/cso-runtime-promote.yml",".github/workflows/cso-runtime-qualification.yml",".github/workflows/cso-scanner-images.yml",".github/workflows/eval-sweep.yml",".github/workflows/evals-marathon.yml",".github/workflows/evals-periodic.yml",".github/workflows/evals.yml",".github/workflows/model-policy-freshness.yml",".github/workflows/native-qualification.yml",".github/workflows/scorecard.yml",".github/workflows/version-gate.yml",".github/workflows/windows-free-tests.yml",".github/workflows/windows-pregate-e2e.yml",".github/workflows/windows-setup-e2e.yml",".osv-scanner.toml","CHANGELOG.md","README.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-browser-ensure","bin/gstack-bun-version.sh","bin/gstack-ceo-plan-find","bin/gstack-claude-root.sh","bin/gstack-config","bin/gstack-doctor-components.sh","bin/gstack-evidence","bin/gstack-ff-update","bin/gstack-gate-diff","bin/gstack-gbrain-sync.ts","bin/gstack-memorable","bin/gstack-sanitize-echo.sh","bin/gstack-session-update","bin/gstack-skill-end","bin/gstack-timeline-log","browse/src/browser-manager.ts","browse/src/meta-commands.ts","browse/src/server.ts","browse/src/terminal-agent.ts","browse/test/tab-external-close-dispatch.test.ts","browse/test/tab-external-close.test.ts","browse/test/terminal-agent-signal-cleanup.test.ts","bun.lock","docs/autoplan-guard-troubleshooting.md","docs/designs/FIX_WAVE_2026_10_15.md","docs/howto-ios-testing-with-gstack.md","docs/troubleshooting.md","gstack-upgrade/SKILL.md","gstack-upgrade/SKILL.md.tmpl","hosts/claude/hooks/spawn-bin.ts","ios-qa/daemon/src/devicectl.ts","ios-qa/daemon/src/tunnel-bootstrap.ts","ios-qa/daemon/test/devicectl-launch-env.test.ts","lib/claude-journal-records.ts","lib/cso/docker.ts","lib/cso/process.ts","lib/cso/scanner-executor.ts","lib/cso/watchdog.c","lib/gate-diff/git.ts","lib/gbrain-exec.ts","lib/pregate/checks/secrets.ts","lib/pregate/import-graph.ts","lib/pregate/lane-selection.ts","lib/redact-engine.ts","lib/regen.ts","lib/restamp.ts","lib/result-codes.ts","lib/sanitize-echo.ts","lib/script-invocation.ts","package.json","plan-ceo-review/sections/review-sections.md","plan-devex-review/sections/review-sections.md","plan-eng-review/SKILL.md","plan-eng-review/SKILL.md.tmpl","plan-eng-review/sections/review-sections.md","scripts/harness-version.json","scripts/resolvers/outside-voice-steps.ts","scripts/test-pr-profile.ts","setup","setup.ps1","test/autoplan-compaction-preserved-segment.test.ts","test/bun-version-shells.test.ts","test/ceo-plan-find-cross-branch.test.ts","test/cso-docker-argv0.test.ts","test/cso-json-redaction.test.ts","test/cso-scanner-release.test.ts","test/dependency-security.test.ts","test/design-doc-data-guard.test.ts","test/fixtures/module-size-ratchet.json","test/fixtures/session-update-heartbeat.sh","test/fixtures/windows-bash-resolution.json","test/gbrain-source-gitignore.test.ts","test/gbrain-spawn-windows-shell.test.ts","test/gbrain-sync-no-bash-message.test.ts","test/gstack-browser-ensure.test.ts","test/gstack-ceo-plan-find.test.ts","test/gstack-ff-update.test.ts","test/gstack-memorable.test.ts","test/gstack-upgrade-fresh-shell.test.ts","test/held-install-recovery.test.ts","test/helpers/auq-sdk-capture.ts","test/helpers/carve-guards.ts","test/helpers/free-fixtures-data.ts","test/helpers/touchfiles-data.ts","test/model-policy-workflow.test.ts","test/osv-ignore-expiry.test.ts","test/outside-voice-native-deadline.test.ts","test/plan-tune-cathedral.test.ts","test/pregate-import-graph-paths.test.ts","test/pregate-manifest-paths.test.ts","test/readme-skill-lists.test.ts","test/redact-prepush-rebase-force-push.test.ts","test/redact-prepush-scan-range.test.ts","test/run-in-background-guidance.test.ts","test/sanitize-echo.test.ts","test/script-invocation-tripwire.test.ts","test/script-invocation.test.ts","test/section-capture-native-tools.test.ts","test/session-update-autostash.test.ts","test/session-update-heartbeat.test.ts","test/session-update-held-banner.test.ts","test/session-update-hook-hold.test.ts","test/session-update-stages.test.ts","test/setup-claude-code-migration.test.ts","test/setup-cursor-legacy-skills.test.ts","test/setup-hook-canonical-paths.test.ts","test/setup-hook-syntax-gate.test.ts","test/setup-playwright-best-effort.test.ts","test/setup-ps1.test.ts","test/ship-measure-sweep.test.ts","test/ship-measure.test.ts","test/skill-end-completion-durability.test.ts","test/state-root-config-merge.test.ts","test/team-mode.test.ts","test/test-value-bar.test.ts","test/timeline-stop-hook-slug.test.ts","test/upgrade-setup-recovery.test.ts","test/upgrade-template-pins.test.ts","test/workflow-action-pins.test.ts","test/wtree-consumers.test.ts"]},
|
||||
{"runId":38969924945,"pr":3225,"headSha":"09fe44fe435e7ccf6291f688d4883bcdcb5464da","packageVersionOnly":false,"failedGateCases":["qa-only-no-fix","ship-docsync-store"],"changedFiles":["benchmark/SKILL.md","bin/gstack-upgrade","browse/SKILL.md","canary/SKILL.md","design-consultation/SKILL.md","design-review/SKILL.md","devex-review/SKILL.md","land-and-deploy/SKILL.md","lib/redact-patterns.ts","plan-design-review/sections/review-sections.md","plan-design-review/sections/review-sections.md.tmpl","qa/sections/browser-setup.md","scrape/SKILL.md","scripts/resolvers/aside.ts","test/aside-driver.test.ts","test/cso-git-hardening.test.ts","test/gstack-upgrade-path-spelling.test.ts","test/redact-engine.test.ts","test/review-entry-and-design-clarity-au.test.ts"]},
|
||||
{"runId":38973329985,"pr":3225,"headSha":"c130a85ad516ff288a2d0840dfb3180b07b83eca","packageVersionOnly":false,"failedGateCases":["ship-exploratory-small-cli"],"changedFiles":["benchmark/SKILL.md","bin/gstack-upgrade","browse/SKILL.md","canary/SKILL.md","design-consultation/SKILL.md","design-review/SKILL.md","devex-review/SKILL.md","land-and-deploy/SKILL.md","lib/pregate/checks/secrets.ts","lib/redact-patterns.ts","plan-design-review/sections/review-sections.md","plan-design-review/sections/review-sections.md.tmpl","plan-devex-review/sections/review-sections.md","plan-devex-review/sections/review-sections.md.tmpl","qa/sections/browser-setup.md","scrape/SKILL.md","scripts/resolvers/aside.ts","test/aside-driver.test.ts","test/cso-git-hardening.test.ts","test/gstack-upgrade-path-spelling.test.ts","test/pregate-secrets-source-context.test.ts","test/redact-engine.test.ts","test/review-entry-and-design-clarity-au.test.ts"]},
|
||||
{"runId":38974669950,"pr":3190,"headSha":"03968f0a7c3f54d4181a20c37ee77594de26974f","packageVersionOnly":true,"failedGateCases":["qa-functional-webhook-fix"],"changedFiles":[".github/workflows/free-tests.yml","CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","autoplan/DRIVER.md","autoplan/DRIVER.md.tmpl","autoplan/sections/ceo-phase.md","autoplan/sections/eng-phase.md","bin/gstack-autoplan","bin/gstack-bun-version.sh","bin/gstack-capy-install","bin/gstack-capy-session","bin/gstack-doctor-check.sh","bin/gstack-doctor-components.sh","bin/gstack-gate","bin/gstack-install-verdict.sh","bin/gstack-issue-links","bin/gstack-plan-reality","bin/gstack-pregate","bin/gstack-ship-receipt","bin/gstack-upgrade","capy/skill/gstack/SKILL.md","capy/skill/gstack/scripts/install.sh","docs/capy-install.md","docs/pregate.md","docs/troubleshooting.md","docs/unattended.md","lib/autoplan-checkpoint.ts","lib/autoplan-export.ts","lib/autoplan-gate.ts","lib/autoplan-plan-text.ts","lib/autoplan-prompts.ts","lib/autoplan-reconcile.ts","lib/autoplan-revise.ts","lib/autoplan-run.ts","lib/autoplan-state.ts","lib/doctor-components.ts","lib/gate-list.ts","lib/headless-artifacts.ts","lib/install-verdict.ts","lib/issue-links.ts","lib/plan-reality.ts","lib/pregate/checks/guards.ts","lib/pregate/checks/lanes.ts","lib/pregate/config.ts","lib/pregate/context.ts","lib/pregate/index.ts","lib/pregate/runtime.ts","lib/pregate/supervise.ts","lib/pregate/types.ts","lib/result-codes.ts","lib/runtime-pins.ts","lib/ship-receipt.ts","package.json","scripts/lib/windows-curation.ts","scripts/resolvers/autoplan-driver.ts","scripts/resolvers/plan-reality.ts","scripts/ubicloud/ubi-runner.sh","setup","ship/SKILL.md","ship/SKILL.md.tmpl","ship/sections/handoff.md","ship/sections/handoff.md.tmpl","ship/sections/pr-body.md","ship/sections/pr-body.md.tmpl","ship/sections/tests.md","ship/sections/tests.md.tmpl","test/autoplan-answer-invalidation.test.ts","test/autoplan-capy-e2e.test.ts","test/autoplan-checkpoint.test.ts","test/autoplan-driver.test.ts","test/autoplan-gate-artifacts.test.ts","test/autoplan-gate-judgment.test.ts","test/autoplan-pairing.test.ts","test/autoplan-record.test.ts","test/autoplan-residuals.test.ts","test/autoplan-revise.test.ts","test/autoplan-run-concurrency.test.ts","test/autoplan-run-recovery.test.ts","test/bun-resolve.test.ts","test/bun-version-drift.test.ts","test/capy-skill-wrapper.test.ts","test/cso-distribution.test.ts","test/fixtures/autoplan-pairing/run-20261014-pairs.json","test/fixtures/context-budget.json","test/fixtures/golden/agy-ship-SKILL.md","test/fixtures/golden/claude-ship-SKILL.md","test/fixtures/golden/codex-ship-SKILL.md","test/fixtures/golden/factory-ship-SKILL.md","test/gate-list.test.ts","test/gstack-capy-install.test.ts","test/gstack-doctor.test.ts","test/gstack-upgrade-bin.test.ts","test/helpers/autoplan-capy.ts","test/helpers/autoplan-run-fixture.ts","test/helpers/carve-guards.ts","test/helpers/free-fixtures-data.ts","test/helpers/pregate-fixture.ts","test/host-conformance.test.ts","test/install-verdict.test.ts","test/plan-reality.test.ts","test/pregate-guards.test.ts","test/pregate-remote.test.ts","test/pregate-supervise.test.ts","test/redact-prepush-hook.test.ts","test/runtime-pins.test.ts","test/script-invocation-tripwire.test.ts","test/setup-hook-syntax-gate.test.ts","test/setup-playwright-best-effort.test.ts","test/ship-issue-intent.test.ts","test/ubicloud-runner.test.ts"]},
|
||||
{"runId":38976253774,"pr":3190,"headSha":"7d68dedae91ffadf494f4cd9a1ab5c253f4b73d6","packageVersionOnly":true,"failedGateCases":["autoplan-guard-pty"],"changedFiles":[".github/workflows/free-tests.yml","CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","autoplan/DRIVER.md","autoplan/DRIVER.md.tmpl","autoplan/sections/ceo-phase.md","autoplan/sections/eng-phase.md","bin/gstack-autoplan","bin/gstack-bun-version.sh","bin/gstack-capy-install","bin/gstack-capy-session","bin/gstack-doctor-check.sh","bin/gstack-doctor-components.sh","bin/gstack-gate","bin/gstack-install-verdict.sh","bin/gstack-issue-links","bin/gstack-plan-reality","bin/gstack-pregate","bin/gstack-ship-receipt","bin/gstack-upgrade","capy/skill/gstack/SKILL.md","capy/skill/gstack/scripts/install.sh","docs/capy-install.md","docs/pregate.md","docs/troubleshooting.md","docs/unattended.md","lib/autoplan-checkpoint.ts","lib/autoplan-export.ts","lib/autoplan-gate.ts","lib/autoplan-plan-text.ts","lib/autoplan-prompts.ts","lib/autoplan-reconcile.ts","lib/autoplan-revise.ts","lib/autoplan-run.ts","lib/autoplan-state.ts","lib/doctor-components.ts","lib/gate-list.ts","lib/headless-artifacts.ts","lib/install-verdict.ts","lib/issue-links.ts","lib/plan-reality.ts","lib/pregate/checks/guards.ts","lib/pregate/checks/lanes.ts","lib/pregate/config.ts","lib/pregate/context.ts","lib/pregate/index.ts","lib/pregate/runtime.ts","lib/pregate/supervise.ts","lib/pregate/types.ts","lib/result-codes.ts","lib/runtime-pins.ts","lib/ship-receipt.ts","package.json","scripts/lib/windows-curation.ts","scripts/resolvers/autoplan-driver.ts","scripts/resolvers/plan-reality.ts","scripts/ubicloud/ubi-runner.sh","setup","ship/SKILL.md","ship/SKILL.md.tmpl","ship/sections/handoff.md","ship/sections/handoff.md.tmpl","ship/sections/pr-body.md","ship/sections/pr-body.md.tmpl","ship/sections/tests.md","ship/sections/tests.md.tmpl","test/autoplan-answer-invalidation.test.ts","test/autoplan-capy-e2e.test.ts","test/autoplan-checkpoint.test.ts","test/autoplan-driver.test.ts","test/autoplan-gate-artifacts.test.ts","test/autoplan-gate-judgment.test.ts","test/autoplan-pairing.test.ts","test/autoplan-record.test.ts","test/autoplan-residuals.test.ts","test/autoplan-revise.test.ts","test/autoplan-run-concurrency.test.ts","test/autoplan-run-recovery.test.ts","test/bun-resolve.test.ts","test/bun-version-drift.test.ts","test/capy-skill-wrapper.test.ts","test/cso-distribution.test.ts","test/fixtures/autoplan-pairing/run-20261014-pairs.json","test/fixtures/context-budget.json","test/fixtures/golden/agy-ship-SKILL.md","test/fixtures/golden/claude-ship-SKILL.md","test/fixtures/golden/codex-ship-SKILL.md","test/fixtures/golden/factory-ship-SKILL.md","test/gate-list.test.ts","test/gstack-capy-install.test.ts","test/gstack-doctor.test.ts","test/gstack-upgrade-bin.test.ts","test/helpers/autoplan-capy.ts","test/helpers/autoplan-run-fixture.ts","test/helpers/carve-guards.ts","test/helpers/free-fixtures-data.ts","test/helpers/pregate-fixture.ts","test/helpers/qa-functional-fixture.ts","test/host-conformance.test.ts","test/install-verdict.test.ts","test/plan-reality.test.ts","test/pregate-guards.test.ts","test/pregate-remote.test.ts","test/pregate-supervise.test.ts","test/redact-prepush-hook.test.ts","test/runtime-pins.test.ts","test/script-invocation-tripwire.test.ts","test/setup-hook-syntax-gate.test.ts","test/setup-playwright-best-effort.test.ts","test/ship-issue-intent.test.ts","test/ubicloud-runner.test.ts"]},
|
||||
{"runId":38980005793,"pr":3225,"headSha":"3f4a98ecac3fc434f8aeef61bc9def17080718d9","packageVersionOnly":false,"failedGateCases":["design-review-detector-shim-dom","qa-quick"],"changedFiles":["benchmark/SKILL.md","bin/gstack-upgrade","browse/SKILL.md","canary/SKILL.md","design-consultation/SKILL.md","design-review/SKILL.md","devex-review/SKILL.md","land-and-deploy/SKILL.md","lib/pregate/checks/secrets.ts","lib/redact-patterns.ts","plan-design-review/sections/review-sections.md","plan-design-review/sections/review-sections.md.tmpl","plan-devex-review/sections/review-sections.md","plan-devex-review/sections/review-sections.md.tmpl","qa/sections/browser-setup.md","scrape/SKILL.md","scripts/resolvers/aside.ts","test/aside-driver.test.ts","test/cso-git-hardening.test.ts","test/gstack-upgrade-path-spelling.test.ts","test/pregate-secrets-source-context.test.ts","test/redact-engine.test.ts","test/review-entry-and-design-clarity-au.test.ts"]},
|
||||
{"runId":38981523623,"pr":3231,"headSha":"3ba8b460645b972f91042bb73e6a7a07eb4b05ab","packageVersionOnly":false,"failedGateCases":["ship-exploratory-plan-checks"],"changedFiles":["bin/gstack-skill-end","bin/gstack-skill-start","test/detector-corpus-ship-docsync-launch-failure.test.ts","test/docsync-command-grammar.test.ts","test/fixtures/detector-corpora/ship-docsync-launch-failure/38336400566-t1-fail-derived-current-claim.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38336400566-t1-fail-derived-prose-claim.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38336400566-t1-fail.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38829994639-t1-pass.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38972920715-t1-pass.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/inventory.json","test/fixtures/docsync-store-38969924945-bash.json","test/gstack-skill-start.test.ts","test/helpers/docsync-fault-eval.ts","test/helpers/docsync-observer.ts","test/helpers/free-fixtures-data.ts","test/helpers/qa-browser-deadline-evidence.ts","test/qa-browser-deadline-evidence.test.ts","test/test-of-test-ratchet.test.ts"]},
|
||||
{"runId":38983306684,"pr":3231,"headSha":"84c29e9ee3325c19626edff677dc1ff2ff4f21d9","packageVersionOnly":false,"failedGateCases":["qa-functional-cli-report","ship-docsync-missing-asset"],"changedFiles":["bin/gstack-skill-end","bin/gstack-skill-start","review/sections/review-army.md","scripts/resolvers/review-army.ts","ship/sections/review-army.md","test/detector-corpus-ship-docsync-launch-failure.test.ts","test/docsync-command-grammar.test.ts","test/fixtures/detector-corpora/ship-docsync-launch-failure/38336400566-t1-fail-derived-current-claim.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38336400566-t1-fail-derived-prose-claim.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38336400566-t1-fail.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38829994639-t1-pass.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38972920715-t1-pass.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/inventory.json","test/fixtures/docsync-store-38969924945-bash.json","test/gstack-skill-start.test.ts","test/helpers/docsync-fault-eval.ts","test/helpers/docsync-observer.ts","test/helpers/free-fixtures-data.ts","test/helpers/qa-browser-deadline-evidence.ts","test/qa-browser-deadline-evidence.test.ts","test/review-workflow-clarity.test.ts","test/test-of-test-ratchet.test.ts"]},
|
||||
{"runId":38984977679,"pr":3226,"headSha":"122638084111bcea20fc3f62e4a8cd286cbaa437","packageVersionOnly":false,"failedGateCases":["ship-docsync-failure"],"changedFiles":["README.md","autoplan/SKILL.md","autoplan/SKILL.md.tmpl","autoplan/sections/dx-phase.md","autoplan/sections/dx-phase.md.tmpl","bin/gstack-autoplan-snapshot.ts","bin/gstack-doctor-components.sh","bin/gstack-error-capture","devex-review/SKILL.md","devex-review/SKILL.md.tmpl","docs/agent-ready-errors.md","docs/skills.md","docs/troubleshooting.md","lib/autoplan-prompts.ts","lib/dx-error-capture.ts","lib/dx-error-contract.ts","lib/qa-deadline.ts","lib/result-codes.ts","plan-devex-review/SKILL.md","plan-devex-review/SKILL.md.tmpl","plan-devex-review/dx-hall-of-fame.md","plan-devex-review/sections/checklist.md","plan-devex-review/sections/checklist.md.tmpl","plan-devex-review/sections/review-sections.md","plan-devex-review/sections/review-sections.md.tmpl","scripts/resolvers/dx.ts","scripts/resolvers/index.ts","test/dx-error-capture.test.ts","test/dx-error-contract-grader.test.ts","test/dx-error-contract.test.ts","test/eng-finding-retry-budget.test.ts","test/fixtures/dx-error-contract-case.ts","test/helpers/touchfiles-data.ts","test/skill-e2e-dx-error-contract.test.ts"]},
|
||||
{"runId":38986120559,"pr":3231,"headSha":"6fd20bef0fe9387cf13dee88cb144e7b3914575b","packageVersionOnly":true,"failedGateCases":["qa-functional-webhook-fix"],"changedFiles":["CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-skill-end","bin/gstack-skill-start","package.json","review/sections/review-army.md","scripts/resolvers/review-army.ts","ship/sections/review-army.md","test/detector-corpus-ship-docsync-launch-failure.test.ts","test/docsync-command-grammar.test.ts","test/fixtures/detector-corpora/ship-docsync-launch-failure/38336400566-t1-fail-derived-current-claim.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38336400566-t1-fail-derived-prose-claim.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38336400566-t1-fail.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38829994639-t1-pass.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/38972920715-t1-pass.json","test/fixtures/detector-corpora/ship-docsync-launch-failure/inventory.json","test/fixtures/docsync-store-38969924945-bash.json","test/gstack-skill-start.test.ts","test/helpers/docsync-fault-eval.ts","test/helpers/docsync-observer.ts","test/helpers/free-fixtures-data.ts","test/helpers/qa-browser-deadline-evidence.ts","test/qa-browser-deadline-evidence.test.ts","test/review-workflow-clarity.test.ts","test/test-of-test-ratchet.test.ts"]},
|
||||
{"runId":38988927989,"pr":3232,"headSha":"641916c650257b7fb5719ec45bef70b00066874e","packageVersionOnly":false,"failedGateCases":["tpa-absent-linux"],"changedFiles":["bin/gstack-capy-install","bin/gstack-capy-session","bin/gstack-lease","bin/gstack-skill-start","capy/skill/gstack/SKILL.md","docs/capy-install.md","docs/troubleshooting.md","test/capy-skill-wrapper.test.ts","test/gstack-capy-install.test.ts","test/gstack-lease.test.ts","test/gstack-upgrade-bin.test.ts"]},
|
||||
{"runId":38990111230,"pr":3226,"headSha":"6c07d85c82d17bda81860cd0d78616de3dc14cc2","packageVersionOnly":false,"failedGateCases":["ship-docsync-stale-after"],"changedFiles":["README.md","autoplan/SKILL.md","autoplan/SKILL.md.tmpl","autoplan/sections/dx-phase.md","autoplan/sections/dx-phase.md.tmpl","bin/gstack-autoplan-snapshot.ts","bin/gstack-doctor-components.sh","bin/gstack-error-capture","devex-review/SKILL.md","devex-review/SKILL.md.tmpl","docs/agent-ready-errors.md","docs/skills.md","docs/troubleshooting.md","lib/autoplan-prompts.ts","lib/dx-error-capture.ts","lib/dx-error-contract.ts","lib/qa-deadline.ts","lib/result-codes.ts","plan-devex-review/SKILL.md","plan-devex-review/SKILL.md.tmpl","plan-devex-review/dx-hall-of-fame.md","plan-devex-review/sections/checklist.md","plan-devex-review/sections/checklist.md.tmpl","plan-devex-review/sections/review-sections.md","plan-devex-review/sections/review-sections.md.tmpl","scripts/resolvers/dx.ts","scripts/resolvers/index.ts","test/dx-error-capture.test.ts","test/dx-error-contract-grader.test.ts","test/dx-error-contract.test.ts","test/eng-finding-retry-budget.test.ts","test/fixtures/dx-error-contract-case.ts","test/helpers/touchfiles-data.ts","test/skill-e2e-dx-error-contract.test.ts"]},
|
||||
{"runId":38993076767,"pr":3234,"headSha":"b40d8418f9435804a4803981dcb01c7288d87e73","packageVersionOnly":false,"failedGateCases":["ship-docsync-store","ship-exploratory-small-cli"],"changedFiles":["autoplan/DRIVER.md","autoplan/DRIVER.md.tmpl","bin/gstack-artifact","bin/gstack-autoplan","bin/gstack-capy-install","bin/gstack-doctor-check.sh","bin/gstack-doctor-components.sh","bin/gstack-eval-plan","bin/gstack-install-verdict.sh","capy/skill/gstack/SKILL.md","capy/skill/gstack/scripts/install.sh","docs/capy-install.md","docs/designs/field-list-2-2026-10-16/PLAN.md","docs/designs/field-list-2-2026-10-16/ROUND2-OBLIGATIONS.md","docs/eval-models.md","docs/troubleshooting.md","docs/unattended.md","eval-plan/SKILL.md","eval-plan/SKILL.md.tmpl","lib/autoplan-budget.ts","lib/autoplan-checkpoint.ts","lib/autoplan-confirm.ts","lib/autoplan-export.ts","lib/autoplan-gate.ts","lib/autoplan-prompts.ts","lib/autoplan-public.ts","lib/autoplan-reconcile.ts","lib/autoplan-revise.ts","lib/autoplan-run.ts","lib/autoplan-state.ts","lib/coordinator-contract.ts","lib/doctor-components.ts","lib/eval-models.ts","lib/eval-plan.ts","lib/gate-list.ts","lib/headless-artifacts.ts","lib/install-verdict.ts","lib/result-codes.ts","lib/spend-ledger.ts","scripts/resolvers/autoplan-driver.ts","ship/sections/handoff.md","ship/sections/handoff.md.tmpl","spec/sections/gate-and-file.md","test/autoplan-budget.test.ts","test/autoplan-checkpoint.test.ts","test/autoplan-durability.test.ts","test/autoplan-public-export.test.ts","test/capy-review-voice.test.ts","test/capy-skill-wrapper.test.ts","test/doctor-components.test.ts","test/eval-models.test.ts","test/eval-plan-prereg-diagnostics.test.ts","test/fixtures/context-budget.json","test/fixtures/module-size-ratchet.json","test/gstack-capy-install.test.ts","test/gstack-doctor.test.ts"]},
|
||||
{"runId":39001107574,"pr":3236,"headSha":"79554ccaba38b47a06d55f43dc8fdbb922b738b9","packageVersionOnly":true,"failedGateCases":["qa-functional-cli-report"],"changedFiles":[".github/workflows/windows-pregate-e2e.yml","CHANGELOG.md","VERSION","agents-digest/gstack-AGENTS.md","bin/gstack-doctor","bin/gstack-doctor-components.sh","bin/gstack-pregate","bin/gstack-windows-bash.ts","docs/ADDING_A_HOST.md","docs/troubleshooting.md","lib/pregate/checks/lanes.ts","lib/pregate/context.ts","lib/pregate/import-graph.ts","lib/pregate/index.ts","lib/regen.ts","lib/result-codes-release.ts","lib/script-invocation.ts","package.json","scripts/lib/windows-curation.ts","scripts/resolvers/preamble.ts","scripts/resolvers/preamble/generate-completion-status.ts","test/fixtures/golden/agy-ship-SKILL.md","test/fixtures/golden/codex-ship-SKILL.md","test/fixtures/golden/factory-ship-SKILL.md","test/fixtures/skill-md-size-allowlist.json","test/gen-skill-docs.test.ts","test/gstack-doctor.test.ts","test/helpers/free-fixtures-data.ts","test/helpers/pregate-fixture.ts","test/helpers/restamp-fixture.ts","test/pregate-bats-nested.test.ts","test/pregate-wtree-refusal.test.ts","test/review-entry-and-design-clarity-au.test.ts","test/script-invocation-tripwire.test.ts","test/script-invocation.test.ts","test/skill-md-size-guard.test.ts"]},
|
||||
{"runId":39002269281,"pr":3239,"headSha":"8e878d9ee9f06a196ff04ccfb0a4d836c13cb1aa","packageVersionOnly":false,"failedGateCases":["ship-exploratory-plan-checks"],"changedFiles":[".github/dependabot.yml",".github/workflows/version-gate.yml",".osv-scanner.toml","TODOS.md","bun.lock","lib/diagram-render/bun.lock","lib/diagram-render/package.json","make-pdf/src/image-size.ts","make-pdf/src/orchestrator.ts","make-pdf/test/docx-image-size.test.ts","make-pdf/test/fixtures/docx-images/doc.md","make-pdf/test/fixtures/docx-images/document.golden.xml","make-pdf/test/fixtures/docx-images/square.gif","make-pdf/test/fixtures/docx-images/tall.jpg","make-pdf/test/fixtures/docx-images/wide.png","package.json","scripts/compare-pr-version.ts","scripts/typecheck-test-baseline.json","test/ceo-finding-fixture.test.ts","test/compare-pr-version.test.ts","test/osv-config-wiring.test.ts","test/osv-ignore-expiry.test.ts","test/workflow-action-pins.test.ts"]},
|
||||
{"runId":39004466422,"pr":3239,"headSha":"65ca746bda533a348a66032a6541647b43339416","packageVersionOnly":false,"failedGateCases":["plan-mode-no-op"],"changedFiles":[".github/dependabot.yml",".github/workflows/version-gate.yml",".osv-scanner.toml","TODOS.md","bun.lock","lib/diagram-render/bun.lock","lib/diagram-render/package.json","make-pdf/src/docx-images.ts","make-pdf/src/html-to-vdom.d.ts","make-pdf/src/orchestrator.ts","make-pdf/test/docx-image-size.test.ts","make-pdf/test/fixtures/docx-images/doc.md","make-pdf/test/fixtures/docx-images/document.golden.xml","make-pdf/test/fixtures/docx-images/square.gif","make-pdf/test/fixtures/docx-images/tall.jpg","make-pdf/test/fixtures/docx-images/wide.png","package.json","scripts/compare-pr-version.ts","scripts/typecheck-test-baseline.json","test/ceo-finding-fixture.test.ts","test/compare-pr-version.test.ts","test/osv-config-wiring.test.ts","test/osv-ignore-expiry.test.ts","test/workflow-action-pins.test.ts","tsconfig.test.json"]},
|
||||
{"runId":39006896632,"pr":3239,"headSha":"3449a9cd5fd1ed51fd9242bde3a5636ae8d97b77","packageVersionOnly":false,"failedGateCases":["qa-functional-webhook-report"],"changedFiles":[".github/dependabot.yml",".github/workflows/version-gate.yml",".osv-scanner.toml","CHANGELOG.md","TODOS.md","VERSION","agents-digest/gstack-AGENTS.md","bun.lock","lib/diagram-render/bun.lock","lib/diagram-render/package.json","make-pdf/src/docx-images.ts","make-pdf/src/html-to-vdom.d.ts","make-pdf/src/orchestrator.ts","make-pdf/test/docx-image-size.test.ts","make-pdf/test/fixtures/docx-images/doc.md","make-pdf/test/fixtures/docx-images/document.golden.xml","make-pdf/test/fixtures/docx-images/square.gif","make-pdf/test/fixtures/docx-images/tall.jpg","make-pdf/test/fixtures/docx-images/wide.png","package.json","scripts/compare-pr-version.ts","scripts/typecheck-test-baseline.json","test/ceo-finding-fixture.test.ts","test/compare-pr-version.test.ts","test/osv-config-wiring.test.ts","test/osv-ignore-expiry.test.ts","test/workflow-action-pins.test.ts","tsconfig.test.json"]},
|
||||
{"runId":39011101113,"pr":3239,"headSha":"599d3f044952ad9ca5f92e6e4f78d342278b44e9","packageVersionOnly":false,"failedGateCases":["tpa-absent-linux"],"changedFiles":[".github/dependabot.yml",".github/workflows/version-gate.yml",".osv-scanner.toml","CHANGELOG.md","TODOS.md","VERSION","agents-digest/gstack-AGENTS.md","bun.lock","lib/diagram-render/bun.lock","lib/diagram-render/package.json","make-pdf/src/docx-images.ts","make-pdf/src/html-to-vdom.d.ts","make-pdf/src/orchestrator.ts","make-pdf/test/docx-image-size.test.ts","make-pdf/test/fixtures/docx-images/doc.md","make-pdf/test/fixtures/docx-images/document.golden.xml","make-pdf/test/fixtures/docx-images/square.gif","make-pdf/test/fixtures/docx-images/tall.jpg","make-pdf/test/fixtures/docx-images/wide.png","package.json","scripts/compare-pr-version.ts","scripts/typecheck-test-baseline.json","test/ceo-finding-fixture.test.ts","test/compare-pr-version.test.ts","test/osv-config-wiring.test.ts","test/osv-ignore-expiry.test.ts","test/workflow-action-pins.test.ts","tsconfig.test.json"]}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
/**
|
||||
* Static pins and a functional check for the weekly duration-seed recorder
|
||||
* (.github/workflows/free-durations-record.yml): Linux on the free-suite
|
||||
* runner, Windows through the record_durations path, and one bot seed PR only
|
||||
* when a seed's file set drifts.
|
||||
*/
|
||||
import { afterEach, expect, test } from 'bun:test';
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
|
||||
const root = path.resolve(import.meta.dir, '..');
|
||||
const load = (file: string) => Bun.YAML.parse(readFileSync(path.join(root, '.github/workflows', file), 'utf8')) as any;
|
||||
const recorder = load('free-durations-record.yml');
|
||||
const freeTests = load('free-tests.yml');
|
||||
const proposeStep = recorder.jobs.propose.steps.find((step: any) => step.name === 'Open or refresh the seed PR when either seed\'s file set changed');
|
||||
const dirs: string[] = [];
|
||||
afterEach(() => { for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); });
|
||||
|
||||
test('the recorder runs weekly before test-health and on dispatch, read-only except the propose job', () => {
|
||||
expect(recorder.on.schedule).toEqual([{ cron: '0 4 * * 1' }]);
|
||||
expect(Object.keys(recorder.on).sort()).toEqual(['schedule', 'workflow_dispatch']);
|
||||
expect(Object.keys(recorder.on.workflow_dispatch.inputs)).toEqual(['open_pr']);
|
||||
expect(load('test-health.yml').on.schedule[0].cron).toBe('0 10 * * 1');
|
||||
expect(recorder.permissions).toEqual({ contents: 'read' });
|
||||
expect(recorder.concurrency['cancel-in-progress']).toBe(false);
|
||||
expect(recorder.jobs.linux.permissions).toBeUndefined();
|
||||
expect(recorder.jobs.propose.permissions).toEqual({ contents: 'write', 'pull-requests': 'write', actions: 'write' });
|
||||
});
|
||||
|
||||
test('Linux records on the free-suite runner, turns red on any red file, and uploads only a complete recording', () => {
|
||||
const linux = recorder.jobs.linux;
|
||||
expect(linux['runs-on']).toBe(freeTests.jobs['free-suite']['runs-on']);
|
||||
const runs = linux.steps.map((step: any) => step.run).filter(Boolean).join('\n');
|
||||
expect(runs).toContain('bash scripts/ubicloud/setup-free-suite.sh');
|
||||
const record = linux.steps.find((step: any) => step.name === 'Time every free file alone');
|
||||
expect(record.run).toBe('set -o pipefail\nxvfb-run -a bun run test:free --record-durations 2>&1 | tee "$RUNNER_TEMP/record.log"\n');
|
||||
expect(record['continue-on-error']).toBeUndefined();
|
||||
const keep = linux.steps.find((step: any) => step.id === 'seed');
|
||||
expect(keep.if).toBe("always() && steps.record.outcome != 'skipped'");
|
||||
expect(keep.run).toContain('grep -q \'wrote [0-9]* durations to\' "$RUNNER_TEMP/record.log" || {');
|
||||
expect(linux.steps.at(-1).if).toBe("always() && steps.seed.outcome == 'success'");
|
||||
expect(linux.steps.at(-1).with).toMatchObject({ name: 'free-test-durations', path: '${{ runner.temp }}/seed/' });
|
||||
});
|
||||
|
||||
test('Windows records through the windows-free-tests record_durations path, and propose needs both complete recordings', () => {
|
||||
expect(recorder.jobs.windows).toEqual({ uses: './.github/workflows/windows-free-tests.yml', with: { record_durations: true } });
|
||||
expect(recorder.jobs.propose.needs).toEqual(['linux', 'windows']);
|
||||
expect(recorder.jobs.propose.if).toBe('${{ !cancelled() }}');
|
||||
const downloads = recorder.jobs.propose.steps.filter((step: any) => step.uses?.startsWith('actions/download-artifact@'));
|
||||
expect(downloads.map((step: any) => step.with.name)).toEqual(['free-test-durations', 'free-test-durations-windows']);
|
||||
expect(downloads.every((step: any) => step['continue-on-error'] === undefined)).toBe(true);
|
||||
});
|
||||
|
||||
test('both recorders list exactly the files the recorder reported red, and more than 5 on either OS proposes nothing', () => {
|
||||
const windowsRecord = load('windows-free-tests.yml').jobs['windows-record-durations'].steps.find((step: any) => step.id === 'seed').run as string;
|
||||
const linuxRecord = recorder.jobs.linux.steps.find((step: any) => step.id === 'seed').run as string;
|
||||
const extract = (run: string) => run.split('\n').find(line => line.startsWith("sed -n '/failed while recording:/"))!;
|
||||
expect(extract(linuxRecord).replace('seed/record-failures.txt', 'record-failures.txt')).toBe(extract(windowsRecord));
|
||||
const dir = mkdtempSync(path.join(os.tmpdir(), 'seed-failures-'));
|
||||
dirs.push(dir);
|
||||
const list = (log: string) => {
|
||||
writeFileSync(path.join(dir, 'record.log'), log);
|
||||
const result = spawnSync('bash', ['-c', `set -euo pipefail\n${extract(windowsRecord)}\ncat "$RUNNER_TEMP/record-failures.txt"`], { encoding: 'utf8', timeout: 10_000, env: { ...process.env, RUNNER_TEMP: dir } });
|
||||
expect(result.status, result.stderr).toBe(0);
|
||||
return result.stdout;
|
||||
};
|
||||
const red = ' ✗ test/a.test.ts — suite > case\r\n[test:free] wrote 3 durations to scripts\\free-test-durations.json\r\n[test:free] WARNING: 2 file(s) failed while recording:\r\n ✗ test/a.test.ts\r\n ✗ browse/test/b.test.ts\r\n';
|
||||
expect(list(red)).toBe('test/a.test.ts\nbrowse/test/b.test.ts\n');
|
||||
expect(list(' ✗ test/a.test.ts — suite > flaky then fixed\n[test:free] wrote 3 durations to scripts/free-test-durations.json\n')).toBe('');
|
||||
const script: string = proposeStep.run;
|
||||
expect(script).toContain('if [ "$linux_failed" -gt 5 ] || [ "$windows_failed" -gt 5 ]; then');
|
||||
expect(script.indexOf('-gt 5 ]')).toBeLessThan(script.indexOf('drift() {'));
|
||||
expect(script).toContain('Files that failed while recording (durations kept; this PR\'s own free checks decide): $failed_line');
|
||||
});
|
||||
|
||||
test('the propose step touches only the two seeds, never the allowlist, on one bot branch', () => {
|
||||
const script: string = proposeStep.run;
|
||||
expect(script).toContain('git add scripts/free-test-durations.json scripts/free-test-durations-windows.json\n');
|
||||
expect(script).not.toContain('free-test-seed-allowlist.json');
|
||||
expect(script).toContain('branch=bot/free-duration-seeds');
|
||||
expect(script).toContain('bun test test/free-seed-ratchet.test.ts');
|
||||
expect(script).toContain('gh workflow run free-tests.yml');
|
||||
expect(proposeStep.env.OPEN_PR).toBe("${{ github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || inputs.open_pr) }}");
|
||||
expect(script.indexOf('if [ "$OPEN_PR" != true ]')).toBeLessThan(script.indexOf('git push'));
|
||||
expect(recorder.on.workflow_dispatch.inputs.open_pr).toMatchObject({ type: 'boolean', default: true });
|
||||
});
|
||||
|
||||
test('a long ratchet report reaches the PR body instead of aborting the step under pipefail', () => {
|
||||
const script: string = proposeStep.run;
|
||||
const line = script.split('\n').find(entry => entry.trimStart().startsWith('ratchet_line="FAILS'))!.trim();
|
||||
const ratchet = ['error: Free seed ratchet:', ...Array.from({ length: 3000 }, (_, i) => `- test/f${i}.test.ts is seeded at 61s, over the 60s ceiling. Fix: split it, then rerun the recording.`)].join('\n');
|
||||
const dir = mkdtempSync(path.join(os.tmpdir(), 'seed-ratchet-'));
|
||||
dirs.push(dir);
|
||||
writeFileSync(path.join(dir, 'ratchet.txt'), ratchet);
|
||||
const result = spawnSync('bash', ['-c', `set -euo pipefail\nratchet=$(cat "$1")\n${line}\nprintf '%s' "$ratchet_line"`, 'ratchet', path.join(dir, 'ratchet.txt')], { encoding: 'utf8', timeout: 10_000 });
|
||||
expect(result.status, result.stderr).toBe(0);
|
||||
expect(result.stdout).toContain('- test/f18.test.ts is seeded at 61s, over');
|
||||
expect(result.stdout).not.toContain('- test/f19.test.ts');
|
||||
});
|
||||
|
||||
test('a refresh keeps a maintainer\'s commits on the open seed PR and leases the bot branch it replaces', () => {
|
||||
const script: string = proposeStep.run;
|
||||
const publish = script.slice(script.indexOf('branch=bot/free-duration-seeds'), script.indexOf('cat > pr-body.md'));
|
||||
const dir = mkdtempSync(path.join(os.tmpdir(), 'seed-publish-'));
|
||||
dirs.push(dir);
|
||||
const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.startsWith('GIT_')));
|
||||
const git = (cwd: string, ...args: string[]) => {
|
||||
const result = spawnSync('git', args, { cwd, encoding: 'utf8', timeout: 10_000, env });
|
||||
expect(result.status, result.stderr).toBe(0);
|
||||
return result.stdout.trim();
|
||||
};
|
||||
const origin = path.join(dir, 'origin.git');
|
||||
const work = path.join(dir, 'work');
|
||||
git(dir, 'init', '-q', '--bare', '-b', 'main', origin);
|
||||
git(dir, 'clone', '-q', origin, work);
|
||||
const commit = (email: string, file: string, message: string) => {
|
||||
git(work, 'config', 'user.name', email);
|
||||
git(work, 'config', 'user.email', email);
|
||||
writeFileSync(path.join(work, file), message);
|
||||
git(work, 'add', file);
|
||||
git(work, 'commit', '-q', '-m', message);
|
||||
};
|
||||
for (const file of ['free-test-durations.json', 'free-test-durations-windows.json']) {
|
||||
writeFileSync(path.join(work, file), '{}');
|
||||
}
|
||||
git(work, 'checkout', '-q', '-b', 'main');
|
||||
commit('dev@example.com', 'README', 'main');
|
||||
git(work, 'push', '-q', 'origin', 'main');
|
||||
const subject = 'test(seed): refresh the free-suite duration seeds from the weekly recording';
|
||||
git(work, 'checkout', '-q', '-b', 'bot/free-duration-seeds');
|
||||
commit('bot@example.com', 'seed.txt', subject);
|
||||
git(work, 'push', '-q', 'origin', 'bot/free-duration-seeds');
|
||||
const stub = path.join(dir, 'bin');
|
||||
spawnSync('mkdir', [stub], { timeout: 10_000 });
|
||||
writeFileSync(path.join(stub, 'gh'), '#!/bin/sh\necho 7\n', { mode: 0o755 });
|
||||
const run = () => {
|
||||
git(work, 'checkout', '-q', 'main');
|
||||
spawnSync('mkdir', ['-p', path.join(work, 'scripts')], { timeout: 10_000 });
|
||||
writeFileSync(path.join(work, 'scripts/free-test-durations.json'), `{"run":${Date.now()}}`);
|
||||
writeFileSync(path.join(work, 'scripts/free-test-durations-windows.json'), '{}');
|
||||
const result = spawnSync('bash', ['-c', `set -euo pipefail\n${publish}`], { cwd: work, encoding: 'utf8', timeout: 20_000,
|
||||
env: { ...env, PATH: `${stub}:${process.env.PATH}`, GITHUB_STEP_SUMMARY: path.join(dir, 'summary.md') } });
|
||||
expect(result.status, result.stderr).toBe(0);
|
||||
return git(origin, 'rev-parse', 'refs/heads/bot/free-duration-seeds');
|
||||
};
|
||||
const replaced = run();
|
||||
expect(git(origin, 'log', '--format=%s', `main..${replaced}`)).toBe(subject);
|
||||
expect(git(origin, 'show', `${replaced}:scripts/free-test-durations.json`)).toContain('"run"');
|
||||
git(work, 'checkout', '-q', '-B', 'bot/free-duration-seeds', replaced);
|
||||
commit('maintainer@example.com', 'allowlist.json', 'maintainer repair');
|
||||
git(work, 'push', '-q', 'origin', 'HEAD:refs/heads/bot/free-duration-seeds');
|
||||
const repaired = git(origin, 'rev-parse', 'refs/heads/bot/free-duration-seeds');
|
||||
expect(run()).toBe(repaired);
|
||||
expect(readFileSync(path.join(dir, 'summary.md'), 'utf8')).toContain("Seed PR #7 has commits besides the bot's one refresh commit");
|
||||
});
|
||||
|
||||
test('the drift check counts added and removed files, so an unchanged file set proposes nothing', () => {
|
||||
const script: string = proposeStep.run;
|
||||
const drift = script.slice(script.indexOf('drift() {'), script.indexOf('\n}', script.indexOf('drift() {')) + 2);
|
||||
const dir = mkdtempSync(path.join(os.tmpdir(), 'seed-drift-'));
|
||||
dirs.push(dir);
|
||||
const seed = (name: string, files: Record<string, number>) => {
|
||||
const file = path.join(dir, name);
|
||||
writeFileSync(file, JSON.stringify({ version: 1, recordedAt: '2026-10-17T00:00:00.000Z', durations: files }));
|
||||
return file;
|
||||
};
|
||||
const committed = seed('committed.json', { 'test/a.test.ts': 100, 'test/b.test.ts': 200 });
|
||||
const run = (recorded: string) => spawnSync('bash', ['-c', `${drift}\ndrift "$1" "$2"`, 'drift', committed, recorded], { encoding: 'utf8', timeout: 10_000 });
|
||||
const same = run(seed('same.json', { 'test/a.test.ts': 900, 'test/b.test.ts': 50 }));
|
||||
expect(same.status, same.stderr).toBe(0);
|
||||
expect(same.stdout.trim()).toBe('"0 0"');
|
||||
const drifted = run(seed('drifted.json', { 'test/a.test.ts': 100, 'test/c.test.ts': 1, 'test/d.test.ts': 2 }));
|
||||
expect(drifted.status, drifted.stderr).toBe(0);
|
||||
expect(drifted.stdout.trim()).toBe('"2 1"');
|
||||
});
|
||||
@@ -85,7 +85,7 @@ describe('workflow inventory (CEO-20)', () => {
|
||||
|
||||
test('the inventory rule also reads .yaml files', () => {
|
||||
expect(unknownFileLabel('.github/workflows/new-check.yaml').label).toBe('add to FREE_ONLY_PR_FILES');
|
||||
expect(unknownFileLabel('.github/workflows/evals.yml').label).toBe('real unknown dependency');
|
||||
expect(unknownFileLabel('.github/workflows/evals.yml').label).toBe('full-gate input');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -111,7 +111,7 @@ describe('PR selection over the W1a single-file fallbacks', () => {
|
||||
const result = select(['.github/workflows/evals.yml']);
|
||||
expect(result.mode).toBe('full-fallback');
|
||||
expect(result.unknownFileLabels).toEqual([{ file: '.github/workflows/evals.yml', ...unknownFileLabel('.github/workflows/evals.yml') }]);
|
||||
expect(result.reasons[0]).toContain('.github/workflows/evals.yml (real unknown dependency)');
|
||||
expect(result.reasons[0]).toContain('.github/workflows/evals.yml (full-gate input)');
|
||||
});
|
||||
|
||||
test('a mapped free fixture is exempt; a paid-registered fixture selects its cases; an unmapped one falls back', () => {
|
||||
|
||||
@@ -9,13 +9,24 @@ import { afterEach, describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { EXIT } from '../lib/headless-artifacts';
|
||||
import { parseRegistry, selectEntries, validateRegistry } from '../lib/regen';
|
||||
import { missingDependencies, parseRegistry, selectEntries, validateRegistry } from '../lib/regen';
|
||||
import { BASE_REGISTRY, cleanup, commit, makeFixtureRepo, readFile, runBin, write, type FixtureRepo } from './helpers/pregate-fixture';
|
||||
|
||||
const repos: FixtureRepo[] = [];
|
||||
const fixture = (opts?: Parameters<typeof makeFixtureRepo>[0]) => { const r = makeFixtureRepo(opts); repos.push(r); return r; };
|
||||
afterEach(() => { while (repos.length) cleanup(repos.pop()!); });
|
||||
|
||||
/** gbrain's shape (GSTA-31): the generator degrades without its dependency and still exits 0. */
|
||||
const DEPS_FIXTURE = {
|
||||
registry: { entries: [{ id: 'digest', command: 'node gen.js', outputs: ['generated/**'], inputs: ['src/version.js', 'gen.js'] }] },
|
||||
baseFiles: {
|
||||
'package.json': JSON.stringify({ name: 'fixture', version: '1.0.0', dependencies: { 'fixture-dep': '1.0.0' } }, null, 2) + '\n',
|
||||
'bun.lock': '{}\n',
|
||||
'gen.js': "const fs=require('fs');const {VERSION}=require('./src/version.js');let style='plain';try{style=require('fixture-dep').style;}catch{}fs.writeFileSync('generated/digest.md',`# digest v${VERSION} (${style})\\n`);\n",
|
||||
'generated/digest.md': '# digest v1.0.0 (fancy)\n',
|
||||
},
|
||||
};
|
||||
|
||||
describe('validateRegistry (schema + containment, pure)', () => {
|
||||
test('the fixture registry validates; command null is a listed entry; inputs default to always', () => {
|
||||
expect(validateRegistry(BASE_REGISTRY)).toEqual([]);
|
||||
@@ -147,6 +158,43 @@ describe('gstack-regen (bin)', () => {
|
||||
expect(all.out).toContain('regen digest: stale');
|
||||
});
|
||||
|
||||
test('missing dependencies: a generator that exits 0 without node_modules is REGEN_DEPS_MISSING (install, then rerun), not REGEN_STALE; installed deps make it fresh', () => {
|
||||
const repo = fixture(DEPS_FIXTURE);
|
||||
commit(repo, { 'src/version.js': "module.exports = { VERSION: '1.0.0' };\n// touched\n" });
|
||||
const r = runBin(repo, 'gstack-regen', ['check', '--base', 'main', '--allow-repo-commands']);
|
||||
expect(r.status).toBe(EXIT.fail);
|
||||
expect(r.out).toMatch(/^regen digest: deps-missing \(\d+ ms\) — dependencies missing \(no node_modules\); run bun install, then rerun the check — differs without them: generated\/digest\.md \(REGEN_DEPS_MISSING\)$/m);
|
||||
expect(r.out).not.toContain('REGEN_STALE');
|
||||
expect(r.out).toContain('GSTACK_RESULT: skill=regen status=incomplete');
|
||||
const json = JSON.parse(runBin(repo, 'gstack-regen', ['check', '--base', 'main', '--allow-repo-commands', '--json']).out);
|
||||
expect(json.results.map((x: any) => [x.id, x.status])).toEqual([['digest', 'deps-missing']]);
|
||||
expect(repo.git('status', '--porcelain')).toBe('');
|
||||
|
||||
write(repo.work, 'node_modules/fixture-dep/index.js', "exports.style = 'fancy';\n");
|
||||
const installed = runBin(repo, 'gstack-regen', ['check', '--base', 'main', '--allow-repo-commands']);
|
||||
expect(installed.status).toBe(EXIT.ok);
|
||||
expect(installed.out).toContain('regen digest: fresh');
|
||||
|
||||
// With dependencies installed, a real input change is still stale.
|
||||
commit(repo, { 'src/version.js': "module.exports = { VERSION: '1.1.0' };\n" });
|
||||
const stale = runBin(repo, 'gstack-regen', ['check', '--base', 'main', '--allow-repo-commands']);
|
||||
expect(stale.status).toBe(EXIT.fail);
|
||||
expect(stale.out).toContain('regen digest: stale');
|
||||
expect(stale.out).toContain('(REGEN_STALE)');
|
||||
expect(stale.out).not.toContain('REGEN_DEPS_MISSING');
|
||||
});
|
||||
|
||||
test('missing dependencies: the install command follows the lockfile', () => {
|
||||
const repo = fixture(DEPS_FIXTURE);
|
||||
expect(missingDependencies(repo.work)).toBe('bun install');
|
||||
fs.rmSync(path.join(repo.work, 'bun.lock'));
|
||||
write(repo.work, 'package-lock.json', '{}\n');
|
||||
expect(missingDependencies(repo.work)).toBe('npm install');
|
||||
fs.mkdirSync(path.join(repo.work, 'node_modules'));
|
||||
expect(missingDependencies(repo.work)).toBeNull();
|
||||
expect(missingDependencies(fixture().work)).toBeNull();
|
||||
});
|
||||
|
||||
test('validate, list and init: an invalid working-tree registry is REGEN_REGISTRY_INVALID; init refuses to overwrite', () => {
|
||||
const repo = fixture();
|
||||
expect(runBin(repo, 'gstack-regen', ['validate']).out).toContain('REGISTRY_VALID: .gstack/generated.json');
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,149 @@
|
||||
/**
|
||||
* Frozen parity oracle for the exercised-path map (W2-A2): the measured
|
||||
* prototype (GSTA-45 investigation, exercised.ts at 521fd42, REPORT-2) with
|
||||
* only its module-level root and base derivation turned into parameters, its
|
||||
* path-prefix regex rewritten to an equivalent non-backtracking form, its
|
||||
* import scan switched to A1's scanCodeImports (the prototype ran on the
|
||||
* shebang-fixed base; A1 also fixed the TSX loader misreading .ts generics),
|
||||
* and the per-case build of its replay (variant cu) appended. Do not edit its
|
||||
* edge rules: test/exercised-map.test.ts pins that the shipped
|
||||
* exercisedConsumers() equals or is a reviewed superset of this map.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { derivedDependencies, scanCodeImports, SELECTION_DATA_MODULES, type DependencyMaps, type ExercisedMap } from '../../scripts/pr-dependencies';
|
||||
import { matchGlob } from './test-selection';
|
||||
import { isPaidTestFile } from './paid-test-set';
|
||||
|
||||
export function prototypeConsumers(maps: DependencyMaps, ROOT: string, observedMap: ExercisedMap | null, PR_FULL_GATE_FILES: readonly string[]): Map<string, Set<string>> {
|
||||
const base = derivedDependencies(maps, ROOT);
|
||||
const tracked: string[] = [...base.tracked];
|
||||
const trackedSet = base.tracked as Set<string>;
|
||||
const cache = new Map<string, string>();
|
||||
const read = (f: string) => { if (!cache.has(f)) { let t = ''; try { if (trackedSet.has(f) && fs.statSync(path.join(ROOT, f)).size < 3_000_000) t = fs.readFileSync(path.join(ROOT, f), 'utf8'); } catch {} cache.set(f, t); } return cache.get(f)!; };
|
||||
const CODE = /\.(?:[cm]?[jt]s|tsx|jsx)$/;
|
||||
const isBunShebang = (f: string) => /^#!.*\b(?:bun|node)\b/.test(read(f).slice(0, 200));
|
||||
const isCode = (f: string) => CODE.test(f) || isBunShebang(f);
|
||||
const isShell = (f: string) => !isCode(f) && (/\.sh$/.test(f) || f === 'setup' || /^#!.*\b(?:ba|z|da)?sh\b/.test(read(f).slice(0, 200)));
|
||||
const resolveSpec = (spec: string, from: string) => {
|
||||
if (!spec.startsWith('.')) return null;
|
||||
const b = path.posix.normalize(path.posix.join(path.posix.dirname(from), spec));
|
||||
for (const c of [b, b + '.ts', b + '.tsx', b + '.js', b + '.mjs', b + '/index.ts', b + '/index.js']) if (trackedSet.has(c)) return c;
|
||||
return null;
|
||||
};
|
||||
const dirs = new Set(tracked.flatMap(f => f.split('/').slice(0, -1).map((_, i, p) => p.slice(0, i + 1).join('/'))));
|
||||
const REPO_PATH = /(?:^|[^\w.@-])((?:bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents|test\/helpers|test\/fixtures|capy)\/[\w.@-]+(?:\/[\w.@-]+)*)/g;
|
||||
const JOINED = /['"\x60](bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents)['"\x60]\s*,\s*['"\x60]([\w.@-]+)['"\x60]/g;
|
||||
const binByName = new Map<string, string>();
|
||||
for (const f of tracked) if (f.startsWith('bin/') || f.startsWith('browse/bin/')) { const b = path.posix.basename(f); binByName.set(b, f); binByName.set(b.replace(/\.(ts|sh|js)$/, ''), f); }
|
||||
const BINARIES = [...read('scripts/build.sh').matchAll(/--compile\b[^\n]*?\s([\w./-]+\.ts)\s+--outfile\s+([\w./-]+)/g)].map(m => ({ outfile: m[2]!, entry: m[1]!, tree: m[1]!.startsWith('bin/') ? [m[1]!] : tracked.filter(f => matchGlob(f, path.posix.dirname(m[1]!) + '/**')) }));
|
||||
let pkgScripts: Record<string, string> = {}; try { pkgScripts = JSON.parse(read('package.json')).scripts ?? {}; } catch {}
|
||||
const expandRef = (ref: string) => { ref = ref.replace(/[.]+$/, ''); if (trackedSet.has(ref)) return [ref]; if (dirs.has(ref)) return tracked.filter(f => f.startsWith(ref + '/')); return []; };
|
||||
const stripCode = (t: string) => t.replace(/\/\*[\s\S]*?\*\//g, '').replace(/^[ \t]*\/\/.*$/gm, '');
|
||||
const stripMessages = (t: string) => {
|
||||
let out = '', i = 0;
|
||||
while (i < t.length) {
|
||||
const m = /^(?:echo|printf|say|warn|die|fail|err|log|info|note|_dc_row|row)\b/.exec(t.slice(i, i + 8));
|
||||
const prev = i === 0 ? '\n' : t[i - 1]!;
|
||||
if (m && /[\s;|&(\x60{]/.test(prev)) {
|
||||
let j = i + m[0].length, q: string | null = null;
|
||||
while (j < t.length) { const c = t[j]!; if (q) { if (c === '\\') { j += 2; continue; } if (c === q) q = null; j++; continue; } if (c === '"' || c === "'") { q = c; j++; continue; } if (c === '\n' || c === ';' || c === '|' || c === '&' || c === ')') break; j++; }
|
||||
out += ' '; i = j; continue;
|
||||
}
|
||||
out += t[i]; i++;
|
||||
}
|
||||
return out;
|
||||
};
|
||||
const stripShell = (t: string) => stripMessages(t
|
||||
.replace(/<<-?\s*['"]?(\w+)['"]?[^\n]*\n[\s\S]*?\n\s*\1\s*$/gm, '')
|
||||
.split('\n').filter(l => !/^\s*#(?!!)/.test(l))
|
||||
.map(l => l.replace(/(^\s*|\bin\s+|;;\s*)[\w*."'-]+(?:\s*\|\s*[\w*."'-]+)*\)(?=\s|$)/g, '$1').replace(/\s#\s.*$/, ''))
|
||||
.join('\n'));
|
||||
const CMD_START = /(?:^|[;|&({\x60]|\$\(|\b(?:then|do|else|exec|command|nohup|bash|sh|bun|node|timeout\s+\S+|env(?:\s+\w+=\S*)*|if|while|until|!|time))\s*"?$/;
|
||||
// Changed from the prototype: its `(?:\/?[\w.@-]+)*\/` path prefix backtracks exponentially; this is the same language (words separated by single slashes, ending in a slash) in an unambiguous form.
|
||||
const PREFIX = /(?:"?(?:\$\{?\w+\}?|\$\(dirname\s+"?\$0"?\)|~|\.)?(?:\/(?:[\w.@-]+\/)*|(?:[\w.@-]+\/)+))?"?$/;
|
||||
const cmdOk = (before: string) => { const b = before.replace(PREFIX, ''); return CMD_START.test(b) || (/\/$/.test(before) && /\b\w+="?$/.test(b)); };
|
||||
const shellExec = (text: string, from: string, out: Set<string>) => {
|
||||
for (const m of text.matchAll(/gstack-[a-z0-9][a-z0-9-]*[a-z0-9](?:\.(?:sh|ts))?\b/g)) {
|
||||
const i = m.index!; const ls = text.lastIndexOf('\n', i - 1) + 1;
|
||||
if (!cmdOk(text.slice(ls, i))) continue;
|
||||
const b = binByName.get(m[0]); if (b) out.add(b);
|
||||
}
|
||||
for (const m of text.matchAll(/(?:bin|lib|scripts|browse|design|make-pdf)\/[\w.@-]+(?:\/[\w.@-]+)*/g)) {
|
||||
const i = m.index!; const ls = text.lastIndexOf('\n', i - 1) + 1; const before = text.slice(ls, i);
|
||||
if (/[\w.@-]$/.test(before)) continue;
|
||||
if (!cmdOk(before) && !/\bbun\s+(?:run\s+)?"?(?:\$\{?\w+\}?\/)?$/.test(before)) continue;
|
||||
for (const f of expandRef(m[0])) out.add(f);
|
||||
}
|
||||
for (const m of text.matchAll(/(?:bin|lib|scripts|browse|design|make-pdf|hosts|capy|test\/fixtures|test\/helpers)\/[\w.@-]+(?:\/[\w.@-]+)*\.(?:json|jsonl|env|txt|ya?ml|toml|csv|tsv)\b/g)) for (const f of expandRef(m[0])) out.add(f);
|
||||
for (const m of text.matchAll(/(?:^|[\s;&|({])(?:\.|source)\s+([^;|&\n]+)/gm)) { const names = [...m[1]!.matchAll(/([\w.-]+\.sh|gstack-[\w.-]+)/g)].map(x => x[1]!); if (!names.length) continue; const b = names[names.length - 1]!; const sib = path.posix.join(path.posix.dirname(from), b); if (trackedSet.has(sib)) out.add(sib); else if (binByName.get(b)) out.add(binByName.get(b)!); }
|
||||
for (const bin of BINARIES) if (text.includes(bin.outfile) || (bin.outfile === 'browse/dist/browse' && /\$B\b|find-browse/.test(text)) || (bin.outfile === 'design/dist/design' && /\$D\b/.test(text))) for (const f of bin.tree) out.add(f);
|
||||
for (const m of text.matchAll(/\b(?:bun|npm) run ([a-z][\w:.-]*)/g)) { const c = pkgScripts[m[1]!]; if (c) for (const r of c.matchAll(REPO_PATH)) for (const f of expandRef(r[1]!)) out.add(f); }
|
||||
};
|
||||
const bashBlocks = (md: string) => [...md.matchAll(/\x60\x60\x60(?:bash|sh|shell)\n([\s\S]*?)\x60\x60\x60/g)].map(m => m[1]!).join('\n');
|
||||
const edges = new Map<string, string[]>();
|
||||
const execRefs = (f: string): string[] => {
|
||||
if (edges.has(f)) return edges.get(f)!;
|
||||
const out = new Set<string>();
|
||||
const raw = read(f);
|
||||
if ((SELECTION_DATA_MODULES as readonly string[]).includes(f) || !raw) { edges.set(f, []); return []; }
|
||||
if (isCode(f)) {
|
||||
try { for (const spec of scanCodeImports(f, raw)) { const h = resolveSpec(spec, f); if (h) out.add(h); } } catch {}
|
||||
const code = stripCode(raw.replace(/^#!.*/, ''));
|
||||
for (const m of code.matchAll(/['"\x60](?:\$\{[^}]+\}\/)?((?:bin|lib|scripts|browse|design|make-pdf|hosts|extension|model-overlays|agents|test\/helpers|test\/fixtures)\/[\w.@-]+(?:\/[\w.@-]+)*)['"\x60]/g)) for (const x of expandRef(m[1]!)) out.add(x);
|
||||
for (const m of code.matchAll(JOINED)) for (const x of expandRef(m[1] + '/' + m[2])) out.add(x);
|
||||
for (const bin of BINARIES) if (code.includes("'" + bin.outfile + "'") || code.includes('"' + bin.outfile + '"')) for (const x of bin.tree) out.add(x);
|
||||
} else if (isShell(f)) {
|
||||
shellExec(stripShell(raw), f, out);
|
||||
} else if (/\.(md|tmpl)$/.test(f)) {
|
||||
shellExec(stripShell(bashBlocks(raw)), f, out);
|
||||
}
|
||||
out.delete(f);
|
||||
const list = [...out]; edges.set(f, list); return list;
|
||||
};
|
||||
const isFullGate = (f: string) => PR_FULL_GATE_FILES.some((p: string) => matchGlob(f, p));
|
||||
const closure = (entries: Iterable<string>) => {
|
||||
const seen = new Set<string>(); const q = [...entries].filter(f => trackedSet.has(f));
|
||||
while (q.length) { const f = q.pop()!; if (seen.has(f)) continue; seen.add(f); if (base.global.has(f) || isFullGate(f)) continue; for (const n of execRefs(f)) if (!seen.has(n) && !(/\.test\.tsx?$/.test(n) && isPaidTestFile(n))) q.push(n); }
|
||||
return seen;
|
||||
};
|
||||
const expand = (p: string) => p.includes('*') ? tracked.filter(f => matchGlob(f, p)) : trackedSet.has(p) ? [p] : [];
|
||||
const OWNER = /^test\/[^/]+\.test\.ts$/;
|
||||
function caseEntries(patterns: string[], defaultOwner: string | null) {
|
||||
const owners = patterns.filter(p => OWNER.test(p) && isPaidTestFile(p));
|
||||
const files = [...(owners.length ? owners : defaultOwner ? [defaultOwner] : []), ...patterns.flatMap(expand)];
|
||||
const gen = files.filter(f => f.endsWith('.tmpl')).map(f => f.replace(/\.tmpl$/, '')).filter(f => trackedSet.has(f));
|
||||
return [...files, ...gen];
|
||||
}
|
||||
function binsInCommand(cmd: string): string[] {
|
||||
const out = new Set<string>();
|
||||
const pty = cmd.startsWith('PTY:');
|
||||
for (const m of cmd.matchAll(/gstack-[a-z0-9-]*[a-z0-9](?:\.(?:sh|ts))?/g)) {
|
||||
let tok = m[0]!;
|
||||
if (pty) { while (tok.length > 7 && !binByName.has(tok)) { const k = tok.lastIndexOf('-'); tok = k > 6 ? tok.slice(0, k) : ''; } }
|
||||
const i = m.index!; if (cmd[i - 1] === '/' && i >= 2 && /[\s"'\x60(]/.test(cmd[i - 2]!)) continue;
|
||||
const b = binByName.get(tok); if (b) out.add(b);
|
||||
}
|
||||
for (const m of cmd.matchAll(/browse\/bin\/([a-z-]+)/g)) { const b = binByName.get(m[1]!); if (b) out.add(b); }
|
||||
for (const bin of BINARIES) {
|
||||
const hit = cmd.includes(bin.outfile) || (bin.outfile === 'browse/dist/browse' && /\$B\b|\$\{B\}|find-browse/.test(cmd)) || (bin.outfile === 'design/dist/design' && /\$D\b|\$\{D\}/.test(cmd)) || (bin.outfile === 'make-pdf/dist/pdf' && /make-pdf\/dist\/pdf/.test(cmd));
|
||||
if (hit) out.add(bin.outfile);
|
||||
}
|
||||
return [...out];
|
||||
}
|
||||
const binaryTree = (outfile: string) => BINARIES.find(b => b.outfile === outfile)?.tree ?? [];
|
||||
|
||||
const observed = observedMap?.cases ?? {};
|
||||
const hostIds = tracked.filter((f: string) => /^hosts\/[\w-]+\.ts$/.test(f) && f !== 'hosts/index.ts').map((f: string) => path.basename(f, '.ts'));
|
||||
const byFile = new Map<string, Set<string>>();
|
||||
for (const [id, patterns] of Object.entries(maps.e2eTouchfiles)) {
|
||||
const entries = caseEntries(patterns, null);
|
||||
for (const b of observed[id]?.commands ?? []) { if (trackedSet.has(b)) entries.push(b); else entries.push(...binaryTree(b)); }
|
||||
const files = closure(entries);
|
||||
const ownerText = entries.filter((f: string) => /^test\/[^/]+\.test\.ts$/.test(f)).map(read).join('\n');
|
||||
for (const h of hostIds) if (new RegExp('[\'"\x60]' + h + '[\'"\x60]|--host[ =]' + h + '\\b').test(ownerText)) for (const f of closure(['hosts/' + h + '.ts', ...expand('hosts/' + h + '/**')])) files.add(f);
|
||||
for (const f of files) { if (!byFile.has(f)) byFile.set(f, new Set()); byFile.get(f)!.add(id); }
|
||||
}
|
||||
void binsInCommand; void stripShell; void execRefs;
|
||||
return byFile;
|
||||
}
|
||||
@@ -116,6 +116,7 @@ export const FREE_FIXTURES: Readonly<Record<string, readonly string[]>> = {
|
||||
'test/fixtures/eng-test-plan-edit-cli.js': ['test/eng-test-plan-edit-approval.test.ts'],
|
||||
'test/fixtures/eng-test-plan-edit-dacc.json': ['test/eng-test-plan-edit-approval.test.ts'],
|
||||
'test/fixtures/eval-history/wave-reds-8-censuses.json': ['test/eval-history.test.ts'],
|
||||
'test/fixtures/exercised-map-replay.json': ['scripts/exercised-map.ts'],
|
||||
'test/fixtures/exit-propagation/**': ['test/exit-propagation.test.ts'],
|
||||
'test/fixtures/free-shard-capture.ts': ['test/test-free-shards-capture.test.ts'],
|
||||
'test/fixtures/golden/**': ['test/host-config.test.ts', 'test/ship-review-loop.test.ts'],
|
||||
|
||||
@@ -114,7 +114,7 @@ describe('paid/free dependency boundary', () => {
|
||||
expect(result.selection).toEqual({ e2e: [], judges: [] });
|
||||
}
|
||||
// Tracked files outside every mapping and outside the derivable directories restore the full gate.
|
||||
for (const file of ['ETHOS.md', 'SECURITY.md', 'conductor.json', '.osv-scanner.toml']) {
|
||||
for (const file of ['ETHOS.md', 'SECURITY.md', 'conductor.json', 'NOTICE.md']) {
|
||||
const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: [FREE_ONLY_PR_FILES[0], file] });
|
||||
expect(result.coverage?.mode, file).toBe('full-fallback');
|
||||
expect(result.coverage?.unknownFiles).toContain(file);
|
||||
|
||||
@@ -96,6 +96,93 @@ describe('reference derivation on a fixture tree', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('shebang files and import scan failures (A1-1)', () => {
|
||||
const tree = (extra: Record<string, string> = {}) => fixtureRepo({
|
||||
'test/skill-e2e-alpha.test.ts': "import '../lib/entry';\n", 'test/skill-e2e-beta.test.ts': '',
|
||||
'lib/entry.ts': '#!/usr/bin/env bun\nimport { y } from \'./typed\';\nexport const x = y;\n',
|
||||
'lib/typed.ts': 'export const y = <T,>(v: T) => v as unknown as T;\nimport \'./typed-dep\';\n', 'lib/typed-dep.ts': '',
|
||||
'alpha/SKILL.md.tmpl': 'Run `gstack-sb`.\n', 'bin/gstack-sb': '#!/usr/bin/env -S bun run\nimport { z } from \'../lib/sb-dep\';\n', 'lib/sb-dep.ts': '',
|
||||
'bin/gstack-sh': '#!/bin/sh\nimport ../lib/sh-only.ts\n', 'lib/sh-only.ts': '',
|
||||
'scripts/build.sh': '', 'scripts/resolvers/index.ts': '', 'package.json': '{}', ...extra,
|
||||
});
|
||||
const maps = { e2eTouchfiles: { alpha: ['test/skill-e2e-alpha.test.ts', 'alpha/**'], beta: ['test/skill-e2e-beta.test.ts'] }, judgeTouchfiles: {}, globalTouchfiles: [] as string[] };
|
||||
const profileMaps = { ...maps, tiers: { alpha: 'gate', beta: 'gate' } as const };
|
||||
const pick = (derived: ReturnType<typeof deriveDependencies>, changedFiles: string[]) =>
|
||||
selectPrProfile({ maps: profileMaps, profile: [], selectedE2E: [], selectedJudges: [], changedFiles, derived });
|
||||
|
||||
test('a shebang .ts file, a bun-shebang extensionless bin and a .ts generic arrow are all import-scanned', () => {
|
||||
const derived = deriveDependencies(maps, tree());
|
||||
expect([...derived.e2e.get('lib/typed.ts') ?? []]).toEqual(['alpha']);
|
||||
expect([...derived.e2e.get('lib/typed-dep.ts') ?? []]).toEqual(['alpha']);
|
||||
expect([...derived.e2e.get('lib/sb-dep.ts') ?? []]).toEqual(['alpha']);
|
||||
expect(derived.e2e.get('lib/sh-only.ts')).toBeUndefined();
|
||||
expect(derived.scanErrors.size).toBe(0);
|
||||
expect(pick(derived, ['lib/sb-dep.ts'])).toMatchObject({ mode: 'dependents', e2e: ['alpha'] });
|
||||
});
|
||||
|
||||
test('lib/fs-utils.ts selects the cases of bin/gstack-redact-prepush, which imports it', () => {
|
||||
const bin = gate.filter(id => real.e2e.get('bin/gstack-redact-prepush')?.has(id));
|
||||
expect(bin.length).toBeGreaterThan(0);
|
||||
const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: ['lib/fs-utils.ts'], packageVersionOnly: true });
|
||||
expect(result.coverage?.mode).toBe('dependents');
|
||||
for (const id of bin) expect(result.selection.e2e, id).toContain(id);
|
||||
expect(real.scanErrors.size).toBe(0);
|
||||
expect(real.global.has('test/helpers/bootstrap-retention.ts')).toBe(true);
|
||||
});
|
||||
|
||||
test('any other scan error fails closed at every site: case closures, the global walk and deleted-file referencers', () => {
|
||||
const broken = "import { from './nowhere';\n";
|
||||
const inClosure = deriveDependencies(maps, tree({ 'lib/typed-dep.ts': broken }));
|
||||
expect([...inClosure.scanErrors.keys()]).toEqual(['lib/typed-dep.ts']);
|
||||
const closed = pick(inClosure, ['lib/sb-dep.ts']);
|
||||
expect(closed).toMatchObject({ mode: 'full-fallback', e2e: ['alpha', 'beta'], unknownFiles: ['lib/sb-dep.ts'] });
|
||||
expect(closed.unknownFileLabels[0]).toMatchObject({ label: 'import scan failed' });
|
||||
expect(closed.unknownFileLabels[0]!.fix).toContain('lib/typed-dep.ts');
|
||||
|
||||
const inGlobal = deriveDependencies({ ...maps, globalTouchfiles: ['lib/runner.ts'] }, tree({ 'lib/runner.ts': broken }));
|
||||
expect([...inGlobal.scanErrors.keys()]).toEqual(['lib/runner.ts']);
|
||||
expect(pick(inGlobal, ['lib/sb-dep.ts']).mode).toBe('full-fallback');
|
||||
|
||||
const unreached = deriveDependencies(maps, tree({ 'lib/orphan.ts': broken }));
|
||||
expect(unreached.scanErrors.size).toBe(0);
|
||||
expect(pick(unreached, ['lib/gone.ts'])).toMatchObject({ mode: 'full-fallback', unknownFiles: ['lib/gone.ts'] });
|
||||
expect([...unreached.scanErrors.keys()]).toEqual(['lib/orphan.ts']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('safe classification (A1-2)', () => {
|
||||
test('capy/support.env derives to the bins that read it instead of restoring the full gate', () => {
|
||||
for (const bin of ['bin/gstack-install-verdict.sh', 'bin/gstack-doctor-check.sh']) {
|
||||
for (const id of real.e2e.get(bin) ?? []) expect(real.e2e.get('capy/support.env')?.has(id), `${bin} ${id}`).toBe(true);
|
||||
}
|
||||
const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: ['capy/support.env'], packageVersionOnly: true });
|
||||
expect(result.coverage?.unknownFiles).toEqual([]);
|
||||
expect(result.coverage?.mode).toBe('dependents');
|
||||
expect(DERIVABLE_PREFIXES).toContain('capy/');
|
||||
});
|
||||
|
||||
test('repository config read only by free lanes or GitHub stays in the PR profile', () => {
|
||||
const files = ['.github/dependabot.yml', '.osv-scanner.toml', 'setup.ps1', '.gitlab-ci.yml', '.github/ISSUE_TEMPLATE/bug_report.md', '.gstack/ship-policy.example.json'];
|
||||
for (const file of files) expect(real.tracked.has(file), file).toBe(true);
|
||||
const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: files, packageVersionOnly: true });
|
||||
expect(result.coverage?.mode).toBe('pr');
|
||||
expect(result.selection).toEqual({ e2e: [], judges: [] });
|
||||
});
|
||||
|
||||
test('a full-gate file is never labelled a real unknown dependency', () => {
|
||||
for (const file of ['.github/workflows/evals.yml', 'test-setup.ts', '.gitattributes']) {
|
||||
const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: [file], packageVersionOnly: true });
|
||||
expect(result.coverage?.mode, file).toBe('full-fallback');
|
||||
for (const label of result.coverage!.unknownFileLabels) {
|
||||
expect(label.label, file).toBe('full-gate input');
|
||||
expect(label.fix).toContain('PR_FULL_GATE_FILES');
|
||||
}
|
||||
expect(unknownFileLabel(file).label).toBe('full-gate input');
|
||||
}
|
||||
expect(unknownFileLabel('ETHOS.md').label).toBe('real unknown dependency');
|
||||
});
|
||||
});
|
||||
|
||||
describe('derived dependencies of this checkout', () => {
|
||||
test('a bin script a skill template invokes selects that skill\'s cases', () => {
|
||||
const consumers = real.e2e.get('bin/gstack-next-version') ?? new Set();
|
||||
|
||||
@@ -81,6 +81,66 @@ describe('import graph (static imports ∪ repo-path literals ∪ bin convention
|
||||
expect(testsFor(g, 'test/fixtures/sample.md').provenance[0]).toContain('(literal)');
|
||||
expect(testsFor(g, 'notes/scratch.md')).toMatchObject({ tests: [], declaredNone: false });
|
||||
});
|
||||
|
||||
test('python: absolute, from-import submodule, relative and parenthesised imports and mock.patch targets map; stdlib, comments and docstrings do not', () => {
|
||||
const repo = fixture();
|
||||
commit(repo, {
|
||||
'pkg/__init__.py': '',
|
||||
'pkg/core.py': 'import os\nfrom pkg.util import helper\n',
|
||||
'pkg/util.py': 'def helper():\n return 1\n',
|
||||
'pkg/multi.py': 'X = 1\n',
|
||||
'pkg/patched.py': 'class Thing:\n def run(self):\n return 1\n',
|
||||
'pkg/ghost.py': 'G = 1\n',
|
||||
'pkg/sub/__init__.py': '',
|
||||
'pkg/sub/sibling.py': 'S = 1\n',
|
||||
'pkg/sub/leaf.py': 'from .. import core\nfrom . import sibling\n',
|
||||
'tests/test_core.py': 'import pkg.core as core\n',
|
||||
'tests/test_leaf.py': 'from pkg.sub import leaf\n',
|
||||
'tests/test_multi.py': 'from pkg import (\n multi, # the module\n util,\n)\nimport json, os.path\n',
|
||||
'tests/test_patch.py': "from unittest import mock\n\n@mock.patch('pkg.patched.Thing.run')\ndef test_run(run):\n pass\n",
|
||||
'tests/test_ghost.py': '"""\nimport pkg.ghost\n"""\n# from pkg import ghost\nX = "from pkg import ghost"\n',
|
||||
});
|
||||
const g = buildGraph(repo.work);
|
||||
expect(testsFor(g, 'pkg/core.py').tests).toEqual(['tests/test_core.py', 'tests/test_leaf.py']);
|
||||
expect(testsFor(g, 'pkg/util.py').tests).toEqual(['tests/test_core.py', 'tests/test_leaf.py', 'tests/test_multi.py']);
|
||||
expect(testsFor(g, 'pkg/sub/sibling.py').tests).toEqual(['tests/test_leaf.py']);
|
||||
expect(testsFor(g, 'pkg/multi.py').tests).toEqual(['tests/test_multi.py']);
|
||||
expect(testsFor(g, 'pkg/patched.py').provenance).toEqual(['tests/test_patch.py ← pkg/patched.py (literal)']);
|
||||
expect(testsFor(g, 'pkg/sub/__init__.py').tests).toEqual(['tests/test_leaf.py']);
|
||||
expect(testsFor(g, 'pkg/__init__.py').tests).toEqual(['tests/test_core.py', 'tests/test_leaf.py', 'tests/test_multi.py', 'tests/test_patch.py']);
|
||||
expect(testsFor(g, 'pkg/ghost.py').tests).toEqual([]);
|
||||
expect([...g.reverse.keys()].filter(k => k.endsWith('.py')).sort()).toEqual([
|
||||
'pkg/__init__.py', 'pkg/core.py', 'pkg/multi.py', 'pkg/patched.py', 'pkg/sub/__init__.py', 'pkg/sub/leaf.py', 'pkg/sub/sibling.py', 'pkg/util.py',
|
||||
]);
|
||||
});
|
||||
|
||||
test('python: a package and a same-named module both map, statements after ; and : map, escaped triple quotes and deep modules map, a path literal does not hide its patch target, and adversarial input stays linear', () => {
|
||||
const repo = fixture();
|
||||
commit(repo, {
|
||||
'app.py': 'A = 1\n',
|
||||
'app/__init__.py': '',
|
||||
'pkg/__init__.py': '',
|
||||
'pkg/core.py': 'C = 1\n',
|
||||
'pkg/patched.py': 'class Thing:\n pass\n',
|
||||
'pkg.patched.Thing.run': 'a file whose name is also a patch target\n',
|
||||
'tests/test_app.py': 'import app\n',
|
||||
'tests/test_semicolon.py': 'import os; import pkg.core\n',
|
||||
'tests/test_inline.py': 'def test_core():\n if True: from pkg.core import C\n',
|
||||
'tests/test_patch.py': "from unittest import mock\n\n@mock.patch('pkg.patched.Thing.run')\ndef test_run(run):\n pass\n",
|
||||
'tests/test_adversarial.py': `from ${'.'.repeat(100_000)}x\nfrom ${' '.repeat(100_000)}x\nX = '${Array.from({ length: 20_000 }, () => 'pkg').join('.')}'\n${'\\"""'.repeat(20_000)}\n`,
|
||||
'tests/test_escaped.py': 'DOC = """text \\"""\nstill doc"""\nimport pkg.core\nTAIL = """x"""\n',
|
||||
[`${Array.from({ length: 33 }, (_, i) => `d${i}`).join('/')}/leaf.py`]: 'L = 1\n',
|
||||
'tests/test_deep.py': `import ${Array.from({ length: 33 }, (_, i) => `d${i}`).join('.')}.leaf\n`,
|
||||
});
|
||||
const g = buildGraph(repo.work);
|
||||
expect(testsFor(g, 'app/__init__.py').tests).toEqual(['tests/test_app.py']);
|
||||
expect(testsFor(g, 'app.py').tests).toEqual(['tests/test_app.py']);
|
||||
expect(testsFor(g, 'pkg/core.py').tests).toEqual(['tests/test_escaped.py', 'tests/test_inline.py', 'tests/test_semicolon.py']);
|
||||
expect(testsFor(g, `${Array.from({ length: 33 }, (_, i) => `d${i}`).join('/')}/leaf.py`).tests).toEqual(['tests/test_deep.py']);
|
||||
expect(testsFor(g, 'pkg/patched.py').tests).toEqual(['tests/test_patch.py']);
|
||||
expect(testsFor(g, 'pkg.patched.Thing.run').tests).toEqual(['tests/test_patch.py']);
|
||||
expect(testsFor(g, 'pkg/__init__.py').tests).toEqual(['tests/test_adversarial.py', 'tests/test_escaped.py', 'tests/test_inline.py', 'tests/test_patch.py', 'tests/test_semicolon.py']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('literals (pure)', () => {
|
||||
@@ -189,6 +249,23 @@ describe('gstack-pregate (bin)', () => {
|
||||
expect(readOut(repo).stage2_ms).toBeUndefined();
|
||||
});
|
||||
|
||||
test('missing dependencies: regen is incomplete with REGEN_DEPS_MISSING and the install command, not REGEN_STALE', () => {
|
||||
const repo = fixture({
|
||||
registry: { entries: [{ id: 'digest', command: 'node gen.js', outputs: ['generated/**'], inputs: ['src/version.js', 'gen.js'] }] },
|
||||
baseFiles: {
|
||||
'package.json': JSON.stringify({ name: 'fixture', version: '1.0.0', scripts: { test: 'node --test test/*.test.js' }, devDependencies: { 'fixture-dep': '1.0.0' } }, null, 2) + '\n',
|
||||
'gen.js': "const fs=require('fs');const {VERSION}=require('./src/version.js');let style='plain';try{style=require('fixture-dep').style;}catch{}fs.writeFileSync('generated/digest.md',`# digest v${VERSION} (${style})\\n`);\n",
|
||||
'generated/digest.md': '# digest v1.0.0 (fancy)\n',
|
||||
},
|
||||
});
|
||||
commit(repo, { 'src/version.js': "module.exports = { VERSION: '1.0.0' };\n// touched\n" });
|
||||
const r = pregate(repo, ['--stage', 'preflight', '--only', 'regen']);
|
||||
expect(r.status).not.toBe(EXIT.ok);
|
||||
expect(r.out).toMatch(/^preflight {2}regen {5}incomplete .*dependencies missing; run bun install, then rerun the check \(1 entry unverified: digest\) \(REGEN_DEPS_MISSING\)/m);
|
||||
expect(r.err).toContain('(REGEN_DEPS_MISSING)');
|
||||
expect(r.out + r.err).not.toContain('REGEN_STALE');
|
||||
});
|
||||
|
||||
test('p0-4-scratch-script: zz-probe.sh at the root fails strays (downgraded to warn only when the base config says so); a conventional root file passes', () => {
|
||||
const strict = fixture({ pregate: { ...BASE_PREGATE, warn: [] } });
|
||||
commit(strict, { 'zz-probe.sh': '#!/bin/sh\necho hi\n', 'CONTRIBUTING.md': '# c\n', 'notes.txt': 'x\n', 'newtop/thing.js': 'module.exports = 1;\n' });
|
||||
|
||||
@@ -191,7 +191,8 @@ describe('collectHealth over recorded fixture data', () => {
|
||||
[/runs\/61\/artifacts/, () => ({ artifacts: [] })],
|
||||
[/workflows\/windows-free-tests\.yml\/runs/, () => ({ workflow_runs: windowsRuns })],
|
||||
], {
|
||||
11: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'full-fallback' } }) },
|
||||
11: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'full-fallback', e2e: ['gate-a', 'gate-b'], judges: ['judge-a'],
|
||||
shadow: { e2e: ['gate-a', 'gate-b'], wouldDrop: [], wouldAdd: [], provenance: [], predictedSavingUsd: 0, unpricedCases: 0 } } }) },
|
||||
12: { 'trial-outcomes.jsonl': prTrials },
|
||||
51: { 'trial-outcomes.jsonl': census },
|
||||
61: { 'flake-ledger.jsonl': ledger('test/a.test.ts') },
|
||||
@@ -205,6 +206,11 @@ describe('collectHealth over recorded fixture data', () => {
|
||||
expect(value('net-complexity').unavailable).toContain('git grep HEAD failed');
|
||||
expect(value('pr-reused-trials').value).toBe('1 of 2 trial records');
|
||||
expect(value('pr-spend')).toMatchObject({ value: '$2.50 recorded', detail: ['per merged PR: $2.50 (1 merged)', 'per validated revision: $2.50 (1 revisions)'] });
|
||||
expect(value('pr-cases-per-run')).toMatchObject({ value: '2.0 gate cases, 1.0 judges per run (1 runs)', detail: ['full-fallback: 2.0 gate cases, 1.0 judges (1 runs)'] });
|
||||
expect(value('pr-known-cost-per-run')).toMatchObject({ value: '$2.50 per run (1 runs; 0 executed trial(s) with unknown cost)', detail: ['full-fallback: $2.50 (1 runs)'] });
|
||||
expect(value('pr-shadow-map')).toMatchObject({ value: '$0.00 predicted saving/week (a lower bound: known cost excludes PTY trials with unknown cost); 0.0 activation miss(es)/week, 0 flaky drop(s) not counted (1 runs)', detail: [] });
|
||||
expect(report.headline).toBe('Cost per merged PR: $2.50 (1 merged PRs, $2.50 recorded)');
|
||||
expect(formatHealth(report).split('\n')[1]).toBe('Cost per merged PR: $2.50 (1 merged PRs, $2.50 recorded)');
|
||||
expect(value('cancelled-slice-minutes').value).toBe('30 min/week (1 cancelled run(s))');
|
||||
expect(value('push-to-verdict').value).toBe('p50 10.0 min, p95 10.0 min (1 runs)');
|
||||
expect(value('census-periodic').value).toBe('1.00 expected failed trials/run (1/3 over 1 run(s)); P(red) 33.3% in force, 33.3% single-trial');
|
||||
@@ -223,6 +229,58 @@ describe('collectHealth over recorded fixture data', () => {
|
||||
expect(formatHealth(report)).toContain('rule: share of evals.yml pull_request runs');
|
||||
});
|
||||
|
||||
test('known cost per PR run prices only known-cost executed trials and counts the unknown ones', () => {
|
||||
const trials = formatTrialOutcomes([trialRecord('gate-a', 'passed', { tier: 'gate', lane: 'gate/pr', cost_usd: 4 }),
|
||||
trialRecord('gate-pty', 'passed', { tier: 'gate', lane: 'gate/pr', cost_usd: 0, cost_known: false })]);
|
||||
const client = fakeClient([
|
||||
[/workflows\/evals\.yml\/runs/, () => ({ workflow_runs: [run(1, { event: 'pull_request', head_branch: 'a' }), run(2, { event: 'pull_request', head_branch: 'b' })] })],
|
||||
[/runs\/1\/artifacts/, () => ({ artifacts: [{ id: 11, name: 'paid-plan', size_in_bytes: 10 }, { id: 12, name: 'trial-outcomes-pr-a1', size_in_bytes: 10 }] })],
|
||||
[/runs\/2\/artifacts/, () => ({ artifacts: [{ id: 21, name: 'paid-plan', size_in_bytes: 10 }] })],
|
||||
[/pulls\?state=closed/, () => []],
|
||||
], {
|
||||
11: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'dependents', e2e: ['gate-a', 'gate-pty', 'gate-c'], judges: ['j'] } }) },
|
||||
12: { 'trial-outcomes.jsonl': trials },
|
||||
21: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'pr', e2e: [], judges: [] } }) },
|
||||
});
|
||||
const report = collectHealth({ client, repo: 'o/r', now: NOW, sinceDays: 7, cacheDir: tmp('health-cache-'), root: freeRoot(3, 0) });
|
||||
const value = (id: string) => report.metrics.find(entry => entry.id === id)!;
|
||||
expect(value('pr-known-cost-per-run')).toMatchObject({ value: '$2.00 per run (2 runs; 1 executed trial(s) with unknown cost)', detail: ['dependents: $4.00 (1 runs)', 'pr: $0.00 (1 runs)'] });
|
||||
expect(value('pr-cases-per-run')).toMatchObject({ value: '1.5 gate cases, 0.5 judges per run (2 runs)', detail: ['dependents: 3.0 gate cases, 1.0 judges (1 runs)', 'pr: 0.0 gate cases, 0.0 judges (1 runs)'] });
|
||||
expect(report.headline).toBe('Cost per merged PR: unavailable (0 merged PRs)');
|
||||
});
|
||||
|
||||
test('the shadow map metric counts red drops as activation misses unless the case passed later at the same head SHA, and labels dollars a lower bound', () => {
|
||||
const gate = (caseId: string, outcome: 'passed' | 'failed', extra: Partial<TrialOutcomeRecord> = {}) => trialRecord(caseId, outcome, { tier: 'gate', lane: 'gate/pr', ...extra });
|
||||
const first = formatTrialOutcomes([gate('gate-a', 'failed', { cost_usd: 3 }), gate('gate-b', 'failed'), gate('gate-c', 'failed'),
|
||||
gate('gate-d', 'failed', { execution: 'reused', cost_usd: 0 }), gate('gate-e', 'passed')]);
|
||||
const rerun = formatTrialOutcomes([gate('gate-b', 'passed'), gate('gate-a', 'failed')]);
|
||||
const shadow = (wouldDrop: string[], predictedSavingUsd: number, unknownCostShare = 0.391) =>
|
||||
({ e2e: [], wouldDrop, wouldAdd: [], provenance: [], predictedSavingUsd, unpricedCases: 0, unknownCostShare });
|
||||
const client = fakeClient([
|
||||
[/workflows\/evals\.yml\/runs/, () => ({ workflow_runs: [
|
||||
run(1, { event: 'pull_request', head_branch: 'a', head_sha: 'same', created_at: day(3) }),
|
||||
run(2, { event: 'pull_request', head_branch: 'a', head_sha: 'same', created_at: day(2) }),
|
||||
run(3, { event: 'pull_request', head_branch: 'c' }),
|
||||
run(4, { event: 'pull_request', head_branch: 'd' })] })],
|
||||
[/runs\/1\/artifacts/, () => ({ artifacts: [{ id: 11, name: 'paid-plan', size_in_bytes: 10 }, { id: 12, name: 'trial-outcomes-pr-a1', size_in_bytes: 10 }] })],
|
||||
[/runs\/2\/artifacts/, () => ({ artifacts: [{ id: 21, name: 'paid-plan', size_in_bytes: 10 }, { id: 22, name: 'trial-outcomes-pr-a1', size_in_bytes: 10 }] })],
|
||||
[/runs\/3\/artifacts/, () => ({ artifacts: [{ id: 31, name: 'paid-plan', size_in_bytes: 10 }] })],
|
||||
[/runs\/4\/artifacts/, () => ({ artifacts: [{ id: 41, name: 'paid-plan', size_in_bytes: 10 }] })],
|
||||
[/pulls\?state=closed/, () => []],
|
||||
], {
|
||||
11: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'dependents', e2e: [], judges: [], shadow: shadow(['gate-a', 'gate-b', 'gate-d', 'gate-e'], 4.5) } }) },
|
||||
12: { 'trial-outcomes.jsonl': first },
|
||||
21: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'pr', e2e: [], judges: [], shadow: shadow([], -1.5) } }) },
|
||||
22: { 'trial-outcomes.jsonl': rerun },
|
||||
31: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'pr', e2e: [], judges: [], shadow: { ...shadow([], 99), error: 'bad map' } } }) },
|
||||
41: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'pr', e2e: [], judges: [] } }) },
|
||||
});
|
||||
const report = collectHealth({ client, repo: 'o/r', now: NOW, sinceDays: 7, cacheDir: tmp('health-cache-'), root: freeRoot(3, 0) });
|
||||
expect(report.metrics.find(entry => entry.id === 'pr-shadow-map')).toMatchObject({
|
||||
value: '$3.00 predicted saving/week (a lower bound: known cost excludes the 39% of observed case runs with unknown cost (PTY)); 1.0 activation miss(es)/week, 1 flaky drop(s) not counted (2 runs)',
|
||||
detail: ['shadow miss: 1:gate-a', 'flaky drop: 1:gate-b (passed in run 2)'] });
|
||||
});
|
||||
|
||||
test('the Windows wall counts only successful pull_request runs and says so when there are none', () => {
|
||||
const wall = (windowsRuns: unknown[]) => collectHealth({ client: fixture(windowsRuns), repo: 'o/r', now: NOW, sinceDays: 7, cacheDir: tmp('health-cache-'), root: freeRoot(3, 0) })
|
||||
.metrics.find(entry => entry.id === 'free-wall')!;
|
||||
|
||||
@@ -23,8 +23,9 @@ const native = load('native-qualification.yml');
|
||||
const WINDOWS_CURATED_FLOOR = 638;
|
||||
|
||||
test('the Windows lane plans, runs one strict shard per job and verifies every result', () => {
|
||||
expect(Object.keys(windows.on).sort()).toEqual(['pull_request', 'workflow_dispatch']);
|
||||
expect(Object.keys(windows.on).sort()).toEqual(['pull_request', 'workflow_call', 'workflow_dispatch']);
|
||||
expect(Object.keys(windows.on.workflow_dispatch.inputs)).toEqual(['record_durations']);
|
||||
expect(Object.keys(windows.on.workflow_call.inputs)).toEqual(['record_durations']);
|
||||
const plan = windows.jobs['windows-plan'];
|
||||
expect(plan['runs-on']).toBe('ubuntu-24.04');
|
||||
const planStep = plan.steps.find((step: any) => step.id === 'plan');
|
||||
@@ -62,15 +63,17 @@ test('a record_durations dispatch times each Windows-safe file alone and uploads
|
||||
expect(job.if).toBe('${{ inputs.record_durations }}');
|
||||
expect(job['runs-on']).toBe('windows-latest');
|
||||
const record = job.steps.find((step: any) => step.name === 'Time every Windows-safe file alone');
|
||||
expect(record.run).toBe('bun run test:windows --record-durations --wall-timeout 1200');
|
||||
expect(record.run).toBe('set -o pipefail\nbun run test:windows --record-durations --wall-timeout 1200 2>&1 | tee "$RUNNER_TEMP/record.log"\n');
|
||||
expect(record.env.GSTACK_FREE_TEST_DURATIONS).toBeUndefined();
|
||||
const rename = job.steps.find((step: any) => step.name === 'Name the recording as the Windows seed');
|
||||
expect(rename.if).toBe('always()');
|
||||
expect(rename.run).toBe('cp scripts/free-test-durations.json "$RUNNER_TEMP/free-test-durations-windows.json"');
|
||||
expect(rename.if).toBe("always() && steps.record.outcome != 'skipped'");
|
||||
expect(rename.run).toContain('grep -q \'wrote [0-9]* durations to\' "$RUNNER_TEMP/record.log" || {');
|
||||
expect(rename.run).toContain('cp scripts/free-test-durations.json "$RUNNER_TEMP/free-test-durations-windows.json"');
|
||||
const logs = job.steps.find((step: any) => step.name === 'Upload full recording logs');
|
||||
expect(logs.if).toBe('always()');
|
||||
expect(logs.with['include-hidden-files']).toBe(true);
|
||||
expect(job.steps.at(-1).with.name).toBe('free-test-durations-windows');
|
||||
expect(job.steps.at(-1).if).toBe("always() && steps.seed.outcome == 'success'");
|
||||
});
|
||||
|
||||
test('Windows shards use the pinned Node runtime and retain complete logs on every run', () => {
|
||||
|
||||
Reference in new issue
Block a user