diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml index 451f3a150..e7b81e55b 100644 --- a/.github/workflows/evals.yml +++ b/.github/workflows/evals.yml @@ -278,10 +278,14 @@ jobs: # Free Tests red on this head fails the run without paid work (re-run this # workflow after a green Free Tests rerun). Dispatches never wait: they are # explicit (eval:bg, census). Version Gate is not an input, so a stamp push - # waits for its own Free Tests only and always gets its gate. + # waits for its own Free Tests only and always gets its gate. A pr-summary + # run (a repository with a train block) plans no paid slices, so it skips + # the wait: the train gate and the dispatched screen carry the paid work, + # and Free Tests stays the required free check. readiness: runs-on: ubuntu-24.04 - if: github.event_name == 'pull_request' && github.actor != 'dependabot[bot]' && github.event.pull_request.head.repo.full_name == github.repository + needs: [route] + if: github.event_name == 'pull_request' && github.actor != 'dependabot[bot]' && github.event.pull_request.head.repo.full_name == github.repository && needs.route.outputs.mode != 'pr-summary' timeout-minutes: 90 permissions: contents: read diff --git a/.github/workflows/free-tests.yml b/.github/workflows/free-tests.yml index 92ec29173..443888a40 100644 --- a/.github/workflows/free-tests.yml +++ b/.github/workflows/free-tests.yml @@ -23,6 +23,22 @@ on: push: branches: [main] workflow_dispatch: + inputs: + free_runner: + description: 'Runner-sizing experiments only: Ubicloud runner for the free-suite shards (ubicloud-standard-2, -4 or -8; empty keeps FREE_SUITE_RUNNER). Never cite such a run as a PR or train Free Tests result.' + type: string + default: '' + free_shards: + description: 'Runner-sizing experiments only: shard count, 1-40 (empty keeps FREE_SUITE_SHARDS)' + type: string + default: '' + +# The one source of the free-suite runner and shard count. pull_request and +# push runs never see workflow_dispatch inputs, so free-plan resolves these +# (or a dispatch override) into outputs that free-suite reads. +env: + FREE_SUITE_RUNNER: ubicloud-standard-2 + FREE_SUITE_SHARDS: '20' # Keyed on the PR number, not head_ref: a bare branch name carries no fork # prefix, so same-name branches from two forks would share one group and a @@ -43,6 +59,7 @@ jobs: timeout-minutes: 5 outputs: matrix: ${{ steps.plan.outputs.matrix }} + runner: ${{ steps.plan.outputs.runner }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: @@ -56,9 +73,27 @@ jobs: - id: plan name: Inventory and balance the complete free suite # Warns in the job summary when more than 5 files are unseeded. + env: + RUNNER_OVERRIDE: ${{ inputs.free_runner }} + SHARDS_OVERRIDE: ${{ inputs.free_shards }} run: | - matrix=$(bun run scripts/test-free-shards.ts --ci-plan "$RUNNER_TEMP/free-plan.json" --shards 20) + set -euo pipefail + runner="${RUNNER_OVERRIDE:-$FREE_SUITE_RUNNER}" + shards="${SHARDS_OVERRIDE:-$FREE_SUITE_SHARDS}" + case "$runner" in + ubicloud-standard-2|ubicloud-standard-4|ubicloud-standard-8) ;; + *) echo "::error::free_runner '$runner' is not ubicloud-standard-2, -4 or -8"; exit 1 ;; + esac + case "$shards" in + ''|*[!0-9]*) echo "::error::free_shards '$shards' is not a whole number"; exit 1 ;; + esac + if [ "$shards" -lt 1 ] || [ "$shards" -gt 40 ]; then echo "::error::free_shards $shards is outside 1-40"; exit 1; fi + if [ "$runner" != "$FREE_SUITE_RUNNER" ] || [ "$shards" != "$FREE_SUITE_SHARDS" ]; then + echo "- Runner-sizing experiment: $shards shards on $runner (defaults $FREE_SUITE_SHARDS on $FREE_SUITE_RUNNER); not a PR or train Free Tests result" >> "$GITHUB_STEP_SUMMARY" + fi + matrix=$(bun run scripts/test-free-shards.ts --ci-plan "$RUNNER_TEMP/free-plan.json" --shards "$shards") echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + echo "runner=$runner" >> "$GITHUB_OUTPUT" - name: Seed growth ratchet (60 s ceiling, shrink-only allowlist) run: | GSTACK_FREE_SEED_BASE="$(git merge-base HEAD origin/main)" bun test test/free-seed-ratchet.test.ts @@ -329,7 +364,7 @@ jobs: free-suite: needs: free-plan - runs-on: ubicloud-standard-8 + runs-on: ${{ needs.free-plan.outputs.runner }} timeout-minutes: 20 strategy: fail-fast: false @@ -476,6 +511,18 @@ jobs: - name: Build trusted CSO helper used by free contract tests run: bun run build:cso + # Runner sizing: the suite runs one test at a time, so a shard needs RAM + # and disk more than cores. Sample both while it runs and report the peak. + - name: Start the memory and disk sampler + run: | + cat > "$RUNNER_TEMP/sample-resources.sh" <<'EOF' + while :; do + printf '%s %s %s\n' "$(date +%s)" "$(free -k | awk 'NR == 2 { print $3 }')" "$(df -Pk / | awk 'NR == 2 { print $4 }')" + sleep 2 + done + EOF + nohup sh "$RUNNER_TEMP/sample-resources.sh" > "$RUNNER_TEMP/resources.log" 2>&1 & + # GSTACK_EXPECT_BINARIES=1 arms make-pdf/test/e2e/ci-prereqs.test.ts: # if a future edit drops the gate build (or poppler), the lane FAILS # instead of the gates silently self-skipping back to false green. @@ -494,6 +541,12 @@ jobs: GSTACK_FREE_RETRY_FLAKY: "1" GSTACK_FLAKE_LEDGER: ${{ runner.temp }}/flake-ledger.jsonl + - name: Report peak memory and minimum free disk + if: always() + run: | + if [ ! -s "$RUNNER_TEMP/resources.log" ]; then echo "RESOURCES: no samples"; exit 0; fi + awk '{ if ($2 > used) used = $2; if (free == "" || $3 < free) free = $3 } END { printf "RESOURCES: peak_used_mib=%d min_free_disk_mib=%d samples=%d\n", used / 1024, free / 1024, NR }' "$RUNNER_TEMP/resources.log" | tee -a "$GITHUB_STEP_SUMMARY" + - name: Upload strict shard result if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 diff --git a/CHANGELOG.md b/CHANGELOG.md index 05b172940..220029eb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,111 @@ # Changelog +## [1.91.374.0] - 2026-10-31 + +**/ship pauses on a red base instead of failing.** When main is red for a reason outside a PR, /ship's Step 5 used to fail or ask the fix/TODO/blame/skip question, and unattended lanes either burned runs or held by hand. Truther's two main-red fixes (#815 and the cover-contract branch) each failed only on the other's broken cases, so neither could pass /ship alone until a Captain folded them into #818 by hand (GSTA-157, from the 2026-10-29 speed feedback). + +### Added +- **`gstack-base-red classify`** (bin/gstack-base-red, lib/base-red.ts): reads a new `base_red: { setup, select, timeout_sec }` block from the base branch's ship policy, then calls a failure base-red only when every failure the lane reported is named, the PR touches neither the failing test file nor any path its signature names, and `setup` then `select {files}` fails with the same test id and first error line at the base head and at HEAD in scratch worktrees under the state root. Anything else stays in-branch and /ship triages exactly as before. `--pause` writes a pause record (repository and branch hashed, 24-hour expiry). +- **`gstack-base-red wait --pause `**: on each new base head it reruns the paused files there and exits 0 (`BASE_GREEN`) on the first head that passes; a head that still fails keeps it waiting. Exits 3 on `BASE_RED_WAIT_TIMEOUT` or `BASE_RED_PAUSE_EXPIRED`. +- **Conditional receipts**: `gstack-ship-receipt write --base-red ` adds a `base_red` block; `result` then prints `SHIP_PAUSED: base-red …` with `status=incomplete`. `bindReceipt` refuses a conditional receipt by default (`QUEUE_RECEIPT_CONDITIONAL`), so `ready`, `train build`, `verify --train` and `carry` never accept one. +- **Policy key `base_red`** (`BASE_RED_KEY_VERSION`) with validation: `select` needs a `{files}` slot, commands are one line under 512 characters, `timeout_sec` is 10–7200. + +### Changed +- **/ship Step 5** (ship/sections/tests.md.tmpl) runs `classify` before the T3 question when T1 calls every failure pre-existing, carries base-red cases as that run's only waived failures into Step 16, and documents `/ship --resume ` after `BASE_GREEN` (Step 5 reruns on HEAD merged with the new base in a scratch worktree; a green run rewrites the receipt without `base_red`). docs/captain.md names the paused state. + +### Tests +- `test/base-red.test.ts`: base-red on a real fixture with a bare origin and no leftover worktree; in-branch without running anything when the PR touches the test or a path its signature names; in-branch when the test passes on the base or fails there differently; unconfigured, unnamed and failing-setup cases; the hashed pause path; conditional receipts refused by default; `result` reports `SHIP_PAUSED` and `status=incomplete`; `carry` refuses; `wait` skips a still-red base move and returns on the first green head, and expires or times out; policy validation. + +## [1.91.373.0] - 2026-10-31 + +- **No-gstack arm in every head-to-head eval.** `/eval-plan` preregistrations now say in a required `## Comparison` section whether any arm loads gstack or another skill stack (`stack: loaded`) or none does (`stack: none` with a `why:` line). A loaded comparison must add a `## No-gstack arm` section and one `#no-gstack` arm per model. That arm uses the same model and inputs, a plain prompt with no skill text, and the same panel and judge, and its result is reported beside the variants. `gstack-eval-plan prereg --check` refuses a loaded comparison without it (`PREREG_NO_GSTACK_ARM`). Arms are now identities `[#]`, priced by the model, so the pilot reserves a variant and its no-gstack arm on the same model separately; a repeated identity is refused (`EVAL_ARM_DUPLICATE`). The rule is written up in `docs/TESTING_INTERNALS.md`. + +## [1.91.372.0] - 2026-10-31 + +**A test that fails once in CI and passes when run again no longer turns the job red, and every such flake is recorded for the daily flaky-fix pass.** `gstack-flaky run` wraps any test command that writes a JUnit report (Playwright, `bun test`, jest, vitest, pytest). When a few tests fail, it runs the command once more. A test that failed and then passes is a flaky pass, so the job stays green, an Actions `::warning` and a step-summary row name it, and one line goes on a ledger. A rerun counts only when the runner itself reports a full green run, so a real failure, a coverage threshold, a test the first run never reached, a crash or a cancelled job stays red. Before this, each one-off flake meant rerunning the whole CI job by hand (five times in Truther's last evidence window, three of them for one test). + +### Added +- **`bin/gstack-flaky run --junit -- `**: reruns the whole command once when at most `--max-reruns` (3) tests failed and the runner made no retries of its own. `--rerun-scope test` opts in to rerunning only the failed tests (Playwright `--grep`/`--project`, bun `--test-name-pattern`), for suites with no coverage gate and no fail-on-flaky policy; it still falls back to the whole command when it sees an early stop, a coverage table or the runner's own retries. +- **Ledger** at `GSTACK_FLAKE_LEDGER_JUNIT` (default `/flake-ledger-junit.jsonl`), kept apart from the free suite's file-level ledger: one redacted line per flaky pass and per test that failed its rerun, with tree, commit, branch, run URL, attempts, scope and the first lines of the error. +- **`gstack-flaky ids --junit `** prints the test ids a report holds (`:::: title>`). +- Exit codes: 0 pass (flaky passes included), 1 a test failed, 2 usage, 3 no usable report, a crash or a killed runner, each with a `FLAKY_` and a `fix:` line. A missing, stale, empty or malformed report never passes, and an existing `--junit` file that is not a JUnit report is refused rather than deleted. + +### Docs +- `docs/flaky.md`: the GitHub Actions setup, runners, rerun scope and its preconditions, rules, output and exit codes. `docs/TESTING_INTERNALS.md#flake-ledger` points other repositories to it. + +### Tests +- `test/flaky-run.test.ts` with captured Playwright 1.64 and bun 1.4 JUnit reports under `test/fixtures/junit/`: flaky pass, real red, rerun scopes, coverage-threshold and early-stop runs, unreported failures, cancelled runs, report deletion and ledger redaction. + +### Gate integrity +- `test/helpers/free-fixtures-data.ts`: registers `test/fixtures/junit/**` to `test/flaky-run.test.ts`. No assertion, threshold or skip changed. + +## [1.91.371.0] - 2026-10-31 + +**A PR push in a repository with a merge train no longer waits ten minutes for a check that does no work.** Under a train policy every PR push runs E2E Evals in `pr-summary` mode, which plans no paid slices, yet its `readiness` job still waited out the 600-second quiet window and green Free Tests before the run could finish. Now `readiness` waits for `route` and skips in `pr-summary` mode, so E2E Evals on a PR push finishes in about a minute (58 s on #3609) instead of after the 10-minute wait. + +### Changed +- **`.github/workflows/evals.yml`**: `readiness` needs `route` and runs only when the mode is not `pr-summary`. In `pr` mode (a repository without a train) it still gates recovery and planning as before. In `pr-summary` mode a red Free Tests now shows on Free Tests alone, which stays the required free check; the train gate and the dispatched screen still carry the paid work. +- **`bun run test:health`**: the "PR lane push-to-verdict" rule text says `pr-summary` runs skip the wait since GSTA-149, so its p50 drops by about 10 minutes from this change on. +- **docs/TESTING_INTERNALS.md** describes the skip. + +### Tests +- `test/evals-pr-lane-workflow.test.ts` pins `readiness.needs` and the full `if`. `test/evals-train-workflow.test.ts` checks that a `pr-summary` push runs only `build-image`, `route` and `screen-summary`, and that `pr` mode still runs `readiness`. + +## [1.91.370.0] - 2026-10-31 + +**The free suite now runs on `ubicloud-standard-2` instead of `ubicloud-standard-8`, at the same 20 shards, for about a quarter of the runner cost.** Each shard runs its files one at a time, so it waits on disk and processes rather than CPU. In three paired rounds at one commit, the median per-shard time rose by 3 to 8 s, inside the round-to-round spread of −90 to +65 s, while runner-minutes stayed about equal (464 vs 475). Peak memory stayed under 2.5 GiB and free disk above 15 GiB. A 16-shard arm was slower beyond that spread, so the shard count stays at 20. + +### Changed +- **`.github/workflows/free-tests.yml`**: one workflow-level `FREE_SUITE_RUNNER` (`ubicloud-standard-2`) and `FREE_SUITE_SHARDS` (`20`) set the free-suite size for pull_request, push and dispatch runs alike. `free-plan` resolves them into its `runner` output and the plan's shard count, and `free-suite` reads `runs-on` from that output. The duration-seed recorder stays on `ubicloud-standard-8`. +- **Sizing experiments without a workflow edit**: `workflow_dispatch` takes `free_runner` (`ubicloud-standard-2`, `-4` or `-8` only) and `free_shards` (1–40). `free-plan` refuses any other value before planning, and a run that overrides either one says in its job summary that it is a sizing experiment, not a PR or train Free Tests result. +- **Every free-suite shard prints `RESOURCES: peak_used_mib=… min_free_disk_mib=… samples=…`** from a memory and disk sample every 2 s, even when the suite fails. If a shard shows OOM, ENOSPC or an unexplained Chromium launch failure, move `FREE_SUITE_RUNNER` to `ubicloud-standard-4`. + +### Tests +- `test/free-tests-workflow-wiring.test.ts`: runs the real `free-plan` step with a stubbed `bun`. It checks that the constant passes through when no override is given, that a valid override is honored, and that `ubuntu-24.04`, `ubicloud-standard-8 x`, `0`, `41` and `16; true` are refused before planning. It also checks that the sampler starts before the suite and that the report step runs `always()`. +- `test/free-durations-record-workflow.test.ts`: the recorder and `FREE_SUITE_RUNNER` must both be Ubicloud standard runners. They no longer have to be the same size, because the recorder stays on standard-8. + +## [1.91.369.0] - 2026-10-31 + +**`gstack-wtree` works again in a repository that gitignores `.bun/`.** Since GST-421, its `git add -A` carried an exclude pathspec naming `.bun`, and git refuses a pathspec naming an ignored path with exit 1. Every fingerprint call then failed, which broke `gstack-review-log --start`, the evidence checks and /ship in harness-evals (HARN-1, HARN-7, harness-evals#148; GST-449). + +### Fixed +- **The `.bun/` exclude applies only when the root `.bun/` exists and is not ignored.** The check reads the ignore rules alone (`git check-ignore --no-index`), so a file still tracked under an ignored `.bun/` doesn't bring the failure back. An ignored `.bun/` already stays out of `git add -A`, so the fingerprint is the same either way. + +### Tests +- `test/evidence.test.ts`: with `.bun/` or `/.bun` in `.gitignore`, `gstack-wtree` prints a tree and a lane that writes `.bun/` grades FRESH. With a file tracked under an ignored `.bun/`, the tree matches a plain `git add -A`, and editing or deleting that file changes it. The existing unignored `.bun/` cases still pass. + +## [1.91.368.0] - 2026-10-31 + +**`gstack-pregate --remote` no longer dies at 300 s while its gate waits on CI.** CAPY-1 (qm#2837) had a remote gate that waits for a CI run and prints nothing while it waits, so pregate killed it as stalled after 300 s, whatever `--timeout` said, and the heartbeat gave no hint of what the gate was doing (GST-431). + +### Fixed +- **The remote gate is bounded by its timeout only** (`--timeout`, else `timeout_s`, else 600 s; the gate also gets it as `PREGATE_TIMEOUT_S`). The stall timer still applies to the lanes runner and guards. +- **The remote gate's heartbeat names its last output line**, so a gate that prints the CI run it is waiting on shows it every 60 s. + +### Tests +- `test/pregate-remote.test.ts`: a gate quiet past `--stall-timeout` but inside `--timeout` clears its obligation; a gate past `--timeout` stays uncleared with `PREGATE_INCOMPLETE`. +- `test/pregate-supervise.test.ts`: a heartbeat with `lastLine` names the command's last output line. + +## [1.91.367.0] - 2026-10-31 + +**/ship's review collection no longer counts a reviewer that says it could not review as clean.** In the GSTA-196 replay a specialist replied in prose that the diff base did not exist and it was not returning NO FINDINGS. `collect` parsed that reply to zero findings, and `verify` marked the brief collected, so the review would have read clean from a reader that never saw the diff. What reviewers see is unchanged: every brief renders byte for byte as before. + +### Fixed +- **`gstack-review-dispatch collect` needs the clean marker a reader's own brief asks for before it counts a reply with no findings.** That is `NO FINDINGS` alone on a line for the specialists, the batched specialist, the targeted reader, the Red Team and fix-check; the `Recommendation:` line for the outside voice; and either one for the native adversarial voice. A reply with no finding and no marker (a refusal, an empty reply, a reader that never got the diff) is recorded with `review: false`, prints `COLLECT_NOT_REVIEWED` and exits 1. Re-sending the brief and collecting a real review voids it, and the outside voice, whose failure never blocks, is collected with `--unavailable ''` and verifies as `UNAVAILABLE`, never as a review. A failure text that still holds findings is refused there and collected normally, so nothing is lost. +- **`gstack-review-dispatch verify` reports that reader's brief `NOT_COLLECTED` and its output `NOT_REVIEWED`, so VERIFY reads dirty.** When the output file is still there and unchanged, verify checks the marker in it again, so a ledger written before this change is judged by the same rule. The GSTA-189 replay's combined ledger still verifies clean, with five `NO FINDINGS` specialists and one outside voice ending in `Recommendation:`. +- **A paragraph finding keeps its detail bullets.** The GSTA-196 native adversarial voice wrote four findings, each with unlabeled bullets and then a `Severity:` line. Every bullet closed the open finding, so each `Severity:` line opened a new one: 4 findings collected as 8. They now collect as 4, each at the stronger of its two labels. Only an unnumbered bullet with no file and line of its own counts as detail; a numbered item is still its own finding. +- **Every other finding collects exactly as before.** Main's parser and this one were run over every recorded reader output: 25 raw outputs from the GSTA-189 and GSTA-196 replays and 2,172 finding records from arms C6, L6, A3668 (both suites), G1 and L. 2,196 of the 2,197 collect the same findings with the same labels. The one difference is the GSTA-196 native sample, where 8 fragments join into 4 findings, each at its strongest fragment's label. + +## [1.91.366.0] - 2026-10-31 + +**A partial `gstack-post pr-body` post keeps the PR's intro.** Since GST-412, any body file with a `## ` section replaced the text above the PR's first heading, so a lane that posted only `## Screenshots` deleted a human intro or a `Review-loop exception:` line under the receipt (GStack Minder: "a partial pr-body post must keep the human intro"). + +### Fixed +- **Only a full post owns the text above the first heading.** A file with a receipt block above its first `## ` section, as /ship composes it, still replaces stale lead text and stale notes under the receipt. Any other file keeps them; its own lead text still replaces the PR's lead text. + +### Tests +- `test/gstack-post.test.ts`: a partial post keeps the intro and the note under the receipt, even with a caption of its own or a receipt further down; the full-post cases now carry a receipt, as /ship's body does. + ## [1.91.365.0] - 2026-10-31 **An unclosed code fence in an outside review no longer hides a later `[P1]` or not-ready verdict (GST-436).** Before, an outside-voice reply that said no issues were found, opened a code fence it never closed, and then listed a `[P1]` finding or ended with `Verdict: not ready` was classified clean, because everything after the fence was treated as quoted code. Now a P0 or P1 severity or a last `Verdict: not ready` after an unclosed fence still blocks. Text inside that fence still can't make a reply clean, so a quoted `Verdict: ready`, `[P3]` or `No issues found.` there is ignored, as before. diff --git a/TODOS.md b/TODOS.md index 778672f8b..89b7d0396 100644 --- a/TODOS.md +++ b/TODOS.md @@ -4,7 +4,6 @@ ### P3: GST-421 and GST-412 follow-ups (filed 2026-10-31, from the PR B /ship review) -- **Partial `gstack-post pr-body` posts drop the intro.** Since GST-412, any body file with a `## ` section replaces the text above the current body's first heading, so a lane that posts only `## Screenshots` deletes a human intro or a `Review-loop exception:` line written there (stderr names it; GitHub's edit history keeps it). Decide whether the preamble should be managed only when the file brings its own lead text or a receipt, or have /ship put the exception line under a named section. **Priority:** P3. - **Check the Approvals table's own cells.** `--check-prose` skips the whole rendered Approvals block, including the Asked, Covers and Still pending cells the lane types. Narrow the skip to the Exact words column. **Priority:** P3. - **The root `.bun/` blind spot.** `gstack-wtree` leaves every untracked file under a root `.bun/` out of the fingerprint, so an edit there never makes evidence STALE. Narrow the exclusion to Bun's cache subfolder if anything starts loading code from it. **Priority:** P3. diff --git a/VERSION b/VERSION index d04265b19..e849722ad 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.365.0 +1.91.374.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index f731d61ff..2817a10b0 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.365.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.374.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/bin/gstack-doctor-components.sh b/bin/gstack-doctor-components.sh index ae4f534d1..4e70a2be6 100644 --- a/bin/gstack-doctor-components.sh +++ b/bin/gstack-doctor-components.sh @@ -194,7 +194,6 @@ gstack_result_anchor() { ISSUE_UNREADABLE) printf '%s' 'issue-unreadable' ;; ISSUE_LOOKUP_FAILED) printf '%s' 'issue-lookup-failed' ;; ISSUE_LINKS_MISSING) printf '%s' 'issue-links-missing' ;; - PREREG_INCOMPLETE) printf '%s' 'prereg-incomplete' ;; EVAL_PRICE_MISSING) printf '%s' 'eval-price-missing' ;; EVAL_DRY_RUN_FAILED) printf '%s' 'eval-dry-run-failed' ;; EVAL_ARMS_UNRESOLVED) printf '%s' 'eval-arms-unresolved' ;; @@ -240,7 +239,6 @@ gstack_result_anchor() { OUTSIDE_VOICE_UNAVAILABLE) printf '%s' 'outside-voice-unavailable' ;; ARTIFACT_MACHINE_PATH) printf '%s' 'artifact-machine-path' ;; ARTIFACT_PUBLIC_TARGET_EXISTS) printf '%s' 'artifact-public-target-exists' ;; - PREREG_DIAGNOSTIC_UNTESTED) printf '%s' 'prereg-diagnostic-untested' ;; EVAL_MODELS_INVALID) printf '%s' 'eval-models-invalid' ;; EVAL_MODEL_DENIED) printf '%s' 'eval-model-denied' ;; EVAL_MODEL_OLD_GENERATION) printf '%s' 'eval-model-old-generation' ;; @@ -352,6 +350,10 @@ gstack_result_anchor() { BUDGET_LEDGER_INVALID) printf '%s' 'budget-ledger-invalid' ;; BUDGET_CHECK_SKIPPED) printf '%s' 'budget-check-skipped' ;; HANDOFF_PIN_MOVED) printf '%s' 'handoff-pin-moved' ;; + PREREG_INCOMPLETE) printf '%s' 'prereg-incomplete' ;; + PREREG_DIAGNOSTIC_UNTESTED) printf '%s' 'prereg-diagnostic-untested' ;; + PREREG_NO_GSTACK_ARM) printf '%s' 'prereg-no-gstack-arm' ;; + EVAL_ARM_DUPLICATE) printf '%s' 'eval-arm-duplicate' ;; *) return 1 ;; esac } diff --git a/bin/gstack-eval-plan b/bin/gstack-eval-plan index 20eb4b474..1dc081a76 100755 --- a/bin/gstack-eval-plan +++ b/bin/gstack-eval-plan @@ -6,7 +6,7 @@ * gstack-eval-plan arms [--arms ] [--policy-doc ] [--input-tokens N] [--output-tokens N] [--json] * gstack-eval-plan dry-run --input --arms [--fields a,b] [--output-budget N] * [--retry-prompt ] [--max-rows N] [--out ] [--json] - * gstack-eval-plan pilot start --out --cap --arms --dry-run [--n 3] + * gstack-eval-plan pilot start --out --cap --arms --dry-run [--n 3] * gstack-eval-plan pilot settle --out --attempt --usd * gstack-eval-plan pilot release --out --attempt * gstack-eval-plan pilot report --out --total [--workers W] [--json] @@ -31,7 +31,7 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import { spawnSync } from 'node:child_process'; -import { checkPrereg, DEFAULT_TOKENS, dryRun, pilotReport, preregTemplate, priceGate, releasePilot, renderArms, renderDryRun, renderGatePrice, renderPilotReport, renderPreregDiagnostics, resolveArms, resolvePlanSource, runPreregDiagnostics, settlePilot, startPilot } from '../lib/eval-plan'; +import { armModel, checkPrereg, DEFAULT_TOKENS, PREREG_SECTIONS, dryRun, pilotReport, preregTemplate, priceGate, releasePilot, renderArms, renderDryRun, renderGatePrice, renderPilotReport, renderPreregDiagnostics, resolveArms, resolvePlanSource, runPreregDiagnostics, settlePilot, startPilot } from '../lib/eval-plan'; import { checkModelRules, EVAL_SCOPES, type EvalScope, type ModelRulesCheck } from '../lib/eval-models'; import { EXIT, exitTableHelp } from '../lib/headless-artifacts'; import { resultLine } from '../lib/result-codes'; @@ -50,7 +50,7 @@ const HELP = `gstack-eval-plan — preregistration, priced arms, $0 dry run, pri dry-run --input rows.jsonl --arms m,m [--fields a,b] [--output-budget N] [--retry-prompt f] [--max-rows N] [--cap usd] [--out dir] [--json] stub model over the real rows; checklist: price table, output budget, retry prompt text, and with --cap reservation-fits-cap (one call: largest row + output budget, priced) - pilot start --out dir --cap usd --arms m,m --dry-run dry-run.json [--n 3] + pilot start --out dir --cap usd --arms m,m#no-gstack --dry-run dry-run.json [--n 3] pilot settle --out dir --attempt id --usd pilot release --out dir --attempt id pilot report --out dir --total items [--workers W] [--json] spend table + extrapolation with a 95% band, then the approval ask price-gate --gate-file | --gate "" [--plan-file f] [--base b] [--cap-usd N] [--smoke-cap-usd N] [--allow-unpriced ""] [--out dir] [--json] @@ -90,7 +90,7 @@ function lintModels(args: string[], uses: Array<{ model: string; scope: EvalScop if (from !== 'base' && from !== 'head') throw new Error('--models-from needs base or head'); const top = spawnSync('git', ['rev-parse', '--show-toplevel'], { encoding: 'utf8', timeout: 30_000 }); const repoRoot = top.status === 0 ? top.stdout.trim() : null; - const check = checkModelRules(repoRoot, uses, { from, base: repoRoot ? resolveBaseBranch(repoRoot, flag(args, '--base')) : 'main' }); + const check = checkModelRules(repoRoot, uses.map(u => (armModel(u.model) === u.model ? u : { ...u, model: armModel(u.model), where: `${u.where} (${u.model})` })), { from, base: repoRoot ? resolveBaseBranch(repoRoot, flag(args, '--base')) : 'main' }); if (!args.includes('--json')) process.stdout.write(check.lines.join('\n') + '\n'); for (const e of check.errors) process.stderr.write(resultLine(`gstack-eval-plan ${command}`, e.code, e.message) + '\n'); return check; @@ -116,11 +116,13 @@ function doPrereg(rest: string[]): number { const run = runnable && allowed ? runPreregDiagnostics(d, repoRoot) : null; if (rest.includes('--json')) process.stdout.write(JSON.stringify({ file, ...r, diagnostics_run: run }) + '\n'); else { - process.stdout.write(`PREREG: ${r.ok ? 'ok' : 'incomplete'} present=${r.present.length}/5${r.missing.length ? ` missing=${r.missing.join(',')}` : ''}${r.empty.length ? ` empty=${r.empty.join(',')}` : ''}${d ? ` diagnostics=${d.errors.length ? 'invalid' : d.expected.length}` : ''} file=${file}\n`); + process.stdout.write(`PREREG: ${r.ok ? 'ok' : 'incomplete'} present=${r.present.length}/${PREREG_SECTIONS.length}${r.missing.length ? ` missing=${r.missing.join(',')}` : ''}${r.empty.length ? ` empty=${r.empty.join(',')}` : ''}${r.comparison.stack ? ` stack=${r.comparison.stack}` : ''}${r.no_gstack.required ? ` no_gstack_arm=${r.no_gstack.ok ? 'ok' : r.no_gstack.problems[0] === 'missing section' ? 'missing' : 'incomplete'}` : ''}${d ? ` diagnostics=${d.errors.length ? 'invalid' : d.expected.length}` : ''} file=${file}\n`); if (runnable && !allowed) process.stdout.write(`PREREG_COMMAND: summarizer=${JSON.stringify(d.summarizer)} fixture=${d.fixture} cwd=${repoRoot}\n`); if (run) process.stdout.write(renderPreregDiagnostics(run)); } - if (!r.ok) process.stderr.write(resultLine('gstack-eval-plan prereg', 'PREREG_INCOMPLETE', [...r.missing.map(m => `missing ${m}`), ...r.empty.map(e => `empty ${e}`), ...(d?.errors ?? []).map(e => `diagnostics: ${e}`)].join('; ')) + '\n'); + const incomplete = [...r.missing.map(m => `missing ${m}`), ...r.empty.map(e => `empty ${e}`), ...r.comparison.errors.map(e => `comparison: ${e}`), ...(d?.errors ?? []).map(e => `diagnostics: ${e}`)]; + if (incomplete.length) process.stderr.write(resultLine('gstack-eval-plan prereg', 'PREREG_INCOMPLETE', incomplete.join('; ')) + '\n'); + if (!r.no_gstack.ok) process.stderr.write(resultLine('gstack-eval-plan prereg', 'PREREG_NO_GSTACK_ARM', `no-gstack arm: ${r.no_gstack.problems.join('; ')}`) + '\n'); if (run && !run.ok) process.stderr.write(resultLine('gstack-eval-plan prereg', 'PREREG_DIAGNOSTIC_UNTESTED', run.error ?? run.rows.filter(row => row.status !== 'ok').map(row => `${row.name} ${row.status} (expected ${row.expected ?? 'undeclared'}, got ${row.got ?? 'none'})`).join('; ')) + '\n'); if (!r.ok || (run && !run.ok)) return EXIT.fail; if (runnable && !allowed) { process.stderr.write(resultLine('gstack-eval-plan prereg', 'REPO_COMMANDS_NOT_ALLOWED', `the Diagnostics section declares a summarizer; it runs only with --allow-repo-commands, and the preregistration is not checked until it does`) + '\n'); return EXIT.refused; } diff --git a/bin/gstack-flaky b/bin/gstack-flaky new file mode 100755 index 000000000..40239d156 --- /dev/null +++ b/bin/gstack-flaky @@ -0,0 +1,125 @@ +#!/usr/bin/env -S bun --no-env-file --config=/dev/null +/** + * gstack-flaky (GSTA-158): rerun failed tests once and record the flake, for + * any runner that writes JUnit (Playwright, bun, jest, pytest, vitest). CREW + * rule 16 for every repository: "you can just run the failed test again and + * note that it is flaky". The rules live in lib/flaky-run.ts; docs/flaky.md is + * the how-to and the error-code table. + */ +import * as fs from 'fs'; +import { DEFAULT_MAX_RERUNS, appendLedger, ledgerPath, renderResult, runFlaky, safeText, stepSummary, workflowEscape } from '../lib/flaky-run'; +import { JunitError, readJunit, type JunitRunner } from '../lib/junit'; + +const USAGE = `Usage: + gstack-flaky run --junit [--runner playwright|bun|generic] [--max-reruns ] + [--rerun-scope command|test] [--ledger ] -- + gstack-flaky ids --junit [--runner playwright|bun|generic] + gstack-flaky --help + +run: runs the command (an argv, no shell), reads the JUnit report it writes at +--junit, and reruns each failed test once when the runner made no retries and +at most --max-reruns (${DEFAULT_MAX_RERUNS}) tests failed. The rerun repeats the whole +command and counts only on the runner's full green verdict (exit 0, no failed +or missing test). --rerun-scope test reruns the failed test alone (playwright: + --grep --project <p>; bun: <file> --test-name-pattern <title>) +when the runner's summary proves the first run complete and nothing but the +failed tests can fail it (no coverage, no runner-flaky test); otherwise, and +always for generic runners, the whole command reruns. A test that passes its +rerun once, with the rerun exiting 0, is a flaky pass: green, and one line on the ledger (--ledger, else +GSTACK_FLAKE_LEDGER_JUNIT, else <state root>/flake-ledger-junit.jsonl). --junit +holds the first run's report afterwards; reruns' reports sit beside it as +<name>.rerun-<n>.xml. The command runs with PLAYWRIGHT_JUNIT_INCLUDE_RETRIES=1 +so Playwright's own retries are reported. + +ids: prints the id of every test in a report (<file>::<project>::<title>). + +Output: one FLAKY: line per test that failed at least once, FLAKY_RERUN: +scope=command (<why>) when the rerun repeated the whole command, then +FLAKY_RESULT: tests=<n> failed=<n> flaky=<n> reruns=<n> command_exit=<n> verdict=pass|fail ledger=<file|none> +Exit: 0 pass (flaky passes included), 1 a test failed, 2 usage, +3 no usable report, a crash or a killed runner (FLAKY_<CODE> and a fix: line on stderr). +`; + +const FIX: Record<string, string> = { + FLAKY_NO_RESULTS: 'make the command write JUnit to the --junit path (playwright: PLAYWRIGHT_JUNIT_OUTPUT_NAME=<file> with --reporter=junit; bun: --reporter=junit --reporter-outfile=<file>)', + FLAKY_JUNIT_INVALID: 'the report is not well-formed XML; check that one process writes it and nothing else appends to it', + FLAKY_RETRIES_HIDDEN: 'set PLAYWRIGHT_JUNIT_INCLUDE_RETRIES=1 where the report is written', + FLAKY_COMMAND_FAILED: 'read the command output above: the suite crashed or a non-test step failed; that is a real red', + FLAKY_COMMAND_KILLED: 'the runner was killed by a signal (a cancelled job, an OOM kill); nothing was rerun, rerun the job', + FLAKY_COMMAND_NOT_STARTED: 'check the command after -- (it runs without a shell; wrap it as bash -c \'<script> "$@"\' _ when it needs one)', + FLAKY_UNREPORTED_FAILURES: 'the runner counted failures its JUnit omits (a test file that failed to load, an error outside any test); fix that first, it is a real red', +}; + +function die(msg: string, code = 2): never { + process.stderr.write(`gstack-flaky: ${msg}\n`); + process.exit(code); +} + +function coded(e: JunitError): never { + process.stderr.write(`gstack-flaky: ${e.message} (${e.code})\nfix: ${FIX[e.code] ?? 'see docs/flaky.md'}\n`); + process.exit(3); +} + +function parse(argv: string[]) { + const o: { junit?: string; runner?: JunitRunner; maxReruns: number; rerunScope: 'command' | 'test'; ledger?: string; command: string[] } = { maxReruns: DEFAULT_MAX_RERUNS, rerunScope: 'command', command: [] }; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]!; + const val = () => { const v = argv[++i]; if (v === undefined) die(`${a} needs a value`); return v; }; + if (a === '--') { o.command = argv.slice(i + 1); break; } + if (a === '--junit') o.junit = val(); + else if (a === '--runner') { + const r = val(); + if (r !== 'playwright' && r !== 'bun' && r !== 'generic') die(`--runner must be playwright, bun or generic, got ${JSON.stringify(r)}`); + o.runner = r; + } else if (a === '--max-reruns') { + const n = Number(val()); + if (!Number.isInteger(n) || n < 0) die('--max-reruns must be a whole number'); + o.maxReruns = n; + } else if (a === '--rerun-scope') { + const v = val(); + if (v !== 'command' && v !== 'test') die(`--rerun-scope must be command or test, got ${JSON.stringify(v)}`); + o.rerunScope = v; + } else if (a === '--ledger') o.ledger = val(); + else die(`unknown option ${a}\n${USAGE}`); + } + return o; +} + +const [sub, ...rest] = process.argv.slice(2); +if (!sub || sub === '--help' || sub === '-h') { process.stdout.write(USAGE); process.exit(sub ? 0 : 2); } + +if (sub === 'ids') { + const o = parse(rest); + if (!o.junit) die('ids needs --junit <file>'); + try { + for (const t of readJunit(o.junit, o.runner).tests) process.stdout.write(`${safeText(t.id)}\n`); + } catch (e) { if (e instanceof JunitError) coded(e); throw e; } + process.exit(0); +} + +if (sub !== 'run') die(`unknown command ${sub}\n${USAGE}`); +const o = parse(rest); +if (!o.junit) die('run needs --junit <file.xml>'); +if (!/\.xml$/i.test(o.junit) || fs.statSync(o.junit, { throwIfNoEntry: false })?.isDirectory()) die(`--junit must name a .xml file (it is deleted before every attempt), got ${JSON.stringify(o.junit)}`); +let existing: string | false | undefined; +try { existing = fs.statSync(o.junit, { throwIfNoEntry: false }) && fs.readFileSync(o.junit, 'utf-8').slice(0, 4096); } catch (e) { + die(`--junit ${JSON.stringify(o.junit)} exists but cannot be read (${(e as Error).message}); gstack-flaky deletes that path before every attempt, so it refuses to touch it`); +} +if (existing && existing.trim() && !/<testsuites?[\s>/]/.test(existing)) die(`--junit ${JSON.stringify(o.junit)} exists and is not a JUnit report; gstack-flaky deletes that path before every attempt, so it refuses to touch it`); +if (!o.command.length) die('run needs the test command after --'); +let result; +try { + result = await runFlaky({ command: o.command, junit: o.junit, runner: o.runner, maxReruns: o.maxReruns, rerunScope: o.rerunScope }); +} catch (e) { if (e instanceof JunitError) coded(e); throw e; } +const ledger = ledgerPath(o.ledger); +appendLedger(result.ledger, ledger, s => process.stderr.write(`gstack-flaky: ${s}\n`)); +for (const l of renderResult(result, ledger)) process.stdout.write(`${l}\n`); +const summary = process.env.GITHUB_STEP_SUMMARY && stepSummary(result); +if (summary) { try { fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY!, summary); } catch { /* a summary is a convenience, never the verdict */ } } +if (process.env.GITHUB_ACTIONS === 'true') { + for (const f of result.outcomes.filter(x => x.status === 'flaky-pass')) process.stdout.write(`::warning title=flaky test::${workflowEscape(safeText(f.test.id))} passed on attempt ${f.attempts.length}\n`); +} +if (result.code) { + process.stderr.write(`gstack-flaky: ${result.message} (${result.code})\nfix: ${FIX[result.code] ?? 'see docs/flaky.md'}\n`); +} +process.exit(result.exitCode); diff --git a/bin/gstack-pregate b/bin/gstack-pregate index d11a0730c..b3f5297eb 100755 --- a/bin/gstack-pregate +++ b/bin/gstack-pregate @@ -75,7 +75,7 @@ const HELP = `gstack-pregate — cheap checks before anything paid or remote (do timeout catches a silent one sooner --stall-timeout <s> kill a test command's process group after this long with no output and record the check incomplete (PREGATE_STALLED); default stall_timeout_s from the base .gstack/pregate.json, - else ${DEFAULT_STALL_TIMEOUT_S}; 0 disables + else ${DEFAULT_STALL_TIMEOUT_S}; 0 disables; the --remote gate is exempt (only --timeout bounds it) --remote run the base branch's remote_gate now instead of local tests (needs --allow-repo-commands; under --policy-from head the gate is still the base copy's); it clears the requires-remote obligation only on its PREGATE_REMOTE: tree=<tested tree> tests=<n> failed=0 line with n > 0 diff --git a/bin/gstack-review-dispatch b/bin/gstack-review-dispatch index 9aaf07aa2..d7dd38f51 100755 --- a/bin/gstack-review-dispatch +++ b/bin/gstack-review-dispatch @@ -20,7 +20,7 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import { EXIT, exitTableHelp } from '../lib/headless-artifacts'; import { BRIEF_ROLES, BriefError, briefFile, promptHash, renderBrief, sectionTexts, type BriefRole, type Slots } from '../lib/review-brief'; -import { collect, readJsonl, readLedger, readerKey, recordJudgment, recordSent, appendEvent, verify, type JudgedLine, type MergedLine, type VerifyRow } from '../lib/review-ledger'; +import { cleanMarkers, collect, mayBeUnavailable, parseFindings, readJsonl, readLedger, readerKey, recordJudgment, recordSent, appendEvent, verify, type JudgedLine, type MergedLine, type VerifyRow } from '../lib/review-ledger'; import { buildManifest, chunkCount, chunkOf, manifestPath, markReader, readManifest, renderStage, writeManifest, STAGES, type Extra, type Manifest, type PassReader, type ReaderState, type Stage } from '../lib/review-pass'; import type { Dispatch } from '../lib/review-depth'; import { runScriptSync } from '../lib/script-invocation'; @@ -42,7 +42,11 @@ const HELP = `gstack-review-dispatch — verbatim reviewer briefs and loss-free sent --ledger <f> --role <role> [--name <n>] --sent <file> the exact prompt a dispatch passed: BRIEF_SENT: … match, or BRIEF_MISMATCH: … (exit 1; the pass is not reviewed) collect --ledger <f> --role <role> [--name <n>] --output <file> --merged <merged.jsonl> - the reader's raw output: hashes it and appends each finding to merged.jsonl with its label as written + the reader's raw output: hashes it and appends each finding to merged.jsonl with its label as written. A reply with + no finding must carry its brief's clean marker (NO FINDINGS; the Recommendation: line for the voices); without one it is + COLLECT_NOT_REVIEWED (exit 1): a refusal, an empty reply or a reader that never saw the diff, and verify reports NOT_COLLECTED + until the brief is re-sent and answered with a review, which voids it. --unavailable '<reason>' (the outside voice only; refused when the + output holds findings) records its failure text as missing coverage: COLLECT_UNAVAILABLE, and verify prints UNAVAILABLE and unavailable=N, never a review judge --ledger <f> --merged <merged.jsonl> --judged <judged.jsonl> judged lines {"id","severity","reason"?,"duplicate_of"?}: every merged id kept at its severity or above (a duplicate folds into one at least as severe); added and promoted findings need a reason. JUDGMENT: ok …, or @@ -268,7 +272,16 @@ function main(): number { const output = readFile(outFile, 'reader output'); const merged = flag('--merged'); if (!reader || output === undefined || !merged) { process.stderr.write('gstack-review-dispatch collect: needs --role, --output <file> and --merged <file>\n'); return EXIT.usage; } - const r = collect(ledger, reader, output, merged, path.resolve(outFile!)); + const unavailable = flag('--unavailable'); + if (unavailable !== undefined && !mayBeUnavailable(reader)) { process.stderr.write(`gstack-review-dispatch collect: --unavailable is for the outside voice only; ${reader}'s missing coverage leaves the review incomplete, never clean\n`); return EXIT.usage; } + if (unavailable !== undefined && !unavailable.trim()) { process.stderr.write('gstack-review-dispatch collect: --unavailable needs a reason\n'); return EXIT.usage; } + if (unavailable !== undefined && parseFindings(reader, output).length) { process.stderr.write(`gstack-review-dispatch collect: ${reader}'s output holds findings; collect it without --unavailable so none is lost\n`); return EXIT.usage; } + const r = collect(ledger, reader, output, merged, path.resolve(outFile!), unavailable); + if (unavailable !== undefined) { process.stdout.write(`COLLECT_UNAVAILABLE: reader=${reader} sha256=${r.sha} reason=${JSON.stringify(unavailable)} (missing outside coverage: unavailable, never clean)\n`); return EXIT.ok; } + if (!r.review) { + process.stdout.write(`COLLECT_NOT_REVIEWED: reader=${reader} findings=0 sha256=${r.sha} (no finding and no ${cleanMarkers(reader).map(([m]) => m).join(' or ')} line: a refusal, an empty reply or a reader that never saw the diff is not a review; verify reports this brief NOT_COLLECTED)\n`); + return EXIT.fail; + } process.stdout.write(`COLLECT: reader=${reader} findings=${r.lines.length} sha256=${r.sha} merged=${merged}\n`); return EXIT.ok; } @@ -290,7 +303,7 @@ function main(): number { process.stdout.write(`TARGETED_RECORD: classes=${record.classes.join(',')} status=${record.status} findings=${record.findings} brief=${record.brief_sha.slice(0, 12)} rules=${record.rules}\n`); if (flag('--targeted-out')) fs.writeFileSync(flag('--targeted-out')!, JSON.stringify(record) + '\n'); } - process.stdout.write(`VERIFY: ${v.clean ? 'clean' : 'dirty'} briefs=${v.briefs} outputs=${v.rows.filter(r => r.kind === 'OUTPUT').length} voided=${v.voided}\n`); + process.stdout.write(`VERIFY: ${v.clean ? 'clean' : 'dirty'} briefs=${v.briefs} outputs=${v.rows.filter(r => r.kind === 'OUTPUT').length} voided=${v.voided}${v.unavailable ? ` unavailable=${v.unavailable}` : ''}\n`); return v.clean ? EXIT.ok : EXIT.fail; } process.stderr.write(`gstack-review-dispatch: unknown command ${cmd}\n${HELP}\n`); diff --git a/bin/gstack-wtree b/bin/gstack-wtree index a843f75d4..c8034e375 100755 --- a/bin/gstack-wtree +++ b/bin/gstack-wtree @@ -7,7 +7,8 @@ # that index. An untracked `.bun/` folder at the repository root (Bun's install # cache, which a test run can create) is left out so a passing run does not # look like a mid-run edit (GST-421); tracked files under it still count, and a -# tree without that folder hashes exactly as a plain `git add -A` would. +# tree without that folder (or with it gitignored) hashes exactly as a plain +# `git add -A` would. # Properties that make this the right staleness fingerprint, vs # `git rev-parse HEAD^{tree}`: # @@ -87,7 +88,14 @@ if [ -n "$REAL_INDEX" ] && [ -f "$REAL_INDEX" ] && cp "$REAL_INDEX" "$TMPIDX" 2> else git -C "$TOP" read-tree HEAD 2>/dev/null || exit 1 fi -if ! ERR=$(git -C "$TOP" add -u 2>&1 >/dev/null) || ! ERR=$(git -C "$TOP" add -A -- . ':(exclude,top).bun' 2>&1 >/dev/null); then +# The exclude is added only when the root .bun/ exists and is not ignored: an +# exclude pathspec naming an ignored path makes `git add -A` exit 1 (GST-449), +# and an ignored .bun/ stays out of a plain `git add -A` anyway. --no-index +# asks the ignore rules alone; without it a tracked file under .bun/ makes git +# call .bun not ignored. +EXCLUDE_BUN="" +if [ -e "$TOP/.bun" ] && ! git -C "$TOP" check-ignore --no-index -q .bun 2>/dev/null; then EXCLUDE_BUN=':(exclude,top).bun'; fi +if ! ERR=$(git -C "$TOP" add -u 2>&1 >/dev/null) || ! ERR=$(git -C "$TOP" add -A -- . ${EXCLUDE_BUN:+"$EXCLUDE_BUN"} 2>&1 >/dev/null); then echo "gstack-wtree: cannot stage the working tree: $(printf '%s' "$ERR" | head -1)" >&2 exit 1 fi diff --git a/changelog.d/gsta157-base-red-pause.md b/changelog.d/gsta157-base-red-pause.md deleted file mode 100644 index 5372ebb21..000000000 --- a/changelog.d/gsta157-base-red-pause.md +++ /dev/null @@ -1,13 +0,0 @@ -**/ship pauses on a red base instead of failing.** When main is red for a reason outside a PR, /ship's Step 5 used to fail or ask the fix/TODO/blame/skip question, and unattended lanes either burned runs or held by hand. Truther's two main-red fixes (#815 and the cover-contract branch) each failed only on the other's broken cases, so neither could pass /ship alone until a Captain folded them into #818 by hand (GSTA-157, from the 2026-10-29 speed feedback). - -### Added -- **`gstack-base-red classify`** (bin/gstack-base-red, lib/base-red.ts): reads a new `base_red: { setup, select, timeout_sec }` block from the base branch's ship policy, then calls a failure base-red only when every failure the lane reported is named, the PR touches neither the failing test file nor any path its signature names, and `setup` then `select {files}` fails with the same test id and first error line at the base head and at HEAD in scratch worktrees under the state root. Anything else stays in-branch and /ship triages exactly as before. `--pause` writes a pause record (repository and branch hashed, 24-hour expiry). -- **`gstack-base-red wait --pause <file>`**: on each new base head it reruns the paused files there and exits 0 (`BASE_GREEN`) on the first head that passes; a head that still fails keeps it waiting. Exits 3 on `BASE_RED_WAIT_TIMEOUT` or `BASE_RED_PAUSE_EXPIRED`. -- **Conditional receipts**: `gstack-ship-receipt write --base-red <pause file>` adds a `base_red` block; `result` then prints `SHIP_PAUSED: base-red …` with `status=incomplete`. `bindReceipt` refuses a conditional receipt by default (`QUEUE_RECEIPT_CONDITIONAL`), so `ready`, `train build`, `verify --train` and `carry` never accept one. -- **Policy key `base_red`** (`BASE_RED_KEY_VERSION`) with validation: `select` needs a `{files}` slot, commands are one line under 512 characters, `timeout_sec` is 10–7200. - -### Changed -- **/ship Step 5** (ship/sections/tests.md.tmpl) runs `classify` before the T3 question when T1 calls every failure pre-existing, carries base-red cases as that run's only waived failures into Step 16, and documents `/ship --resume <record>` after `BASE_GREEN` (Step 5 reruns on HEAD merged with the new base in a scratch worktree; a green run rewrites the receipt without `base_red`). docs/captain.md names the paused state. - -### Tests -- `test/base-red.test.ts`: base-red on a real fixture with a bare origin and no leftover worktree; in-branch without running anything when the PR touches the test or a path its signature names; in-branch when the test passes on the base or fails there differently; unconfigured, unnamed and failing-setup cases; the hashed pause path; conditional receipts refused by default; `result` reports `SHIP_PAUSED` and `status=incomplete`; `carry` refuses; `wait` skips a still-red base move and returns on the first green head, and expires or times out; policy validation. diff --git a/docs/TESTING_INTERNALS.md b/docs/TESTING_INTERNALS.md index cb477888d..2ed948c98 100644 --- a/docs/TESTING_INTERNALS.md +++ b/docs/TESTING_INTERNALS.md @@ -197,7 +197,15 @@ shards get duration-aware walls (`max(base, predicted × 3, files × 5s)`). The legacy `--shards N --shard i` path keeps stable hash indices. Required CI uses one duration-packed `--ci-plan`, 20 ordinary `--ci-run` shards plus a separate exclusive-fixture shard, and a `--ci-verify` aggregate. CI shards still run on -independent machines without ordering unrelated jobs. The public +independent machines without ordering unrelated jobs. Two workflow-level +constants in `free-tests.yml` set that size for every event: `FREE_SUITE_RUNNER` +(`ubicloud-standard-2`) and `FREE_SUITE_SHARDS` (`'20'`), which `free-plan` +resolves into the shards' runner and shard count. A `workflow_dispatch` run may +override them with `free_runner` (`ubicloud-standard-2`, `-4` or `-8`) and +`free_shards` (1–40) for runner-sizing experiments only; its job summary says +so, and it is never cited as a PR or train Free Tests result. Each shard prints +`RESOURCES: peak_used_mib=… min_free_disk_mib=… samples=…` from a memory and +disk sample every 2 seconds, even when the suite fails. The public `TREE_MUTATING` map now classifies exclusive host-state fixtures; its sole entry is `test/bootstrap-retention.test.ts`, whose same-UID nondumpable actors affect host-wide procfs permission checks. Locally, this file runs only after @@ -1274,7 +1282,10 @@ re-run keeps the creation time, so it does not wait out the window again). A head that is no longer the PR's head reports "superseded by <sha>" and plans nothing. Selection, trials and verdict rules do not change. Version Gate is not an input, so a stamp push waits only for its own Free Tests and always gets a -gate. Dispatches (`eval:bg`, the census) never wait. +gate. Dispatches (`eval:bg`, the census) never wait. Neither does a +`pr-summary` run (a repository whose policy declares a train): it plans no paid +slices, so `readiness` is skipped, the run ends with the screening summary, and +a red Free Tests shows on Free Tests alone. **Verdict reuse (R2a restamp-only, R2 main moved; a gate edit).** A PR head reuses the verdict of the newest executed gate of the same PR instead of @@ -1424,6 +1435,13 @@ prints a run's ledgers as a table. A flaky file is fixed at source; the retry never masks it. `bun run eval:pass-rates` lists only the local ledger (`flakeLedgerPath()`, overridden by `GSTACK_FLAKE_LEDGER`). +Other repositories get the same rule per test with `bin/gstack-flaky run` +([docs/flaky.md](flaky.md)): it reads a JUnit report from any runner, reruns +the command once when tests failed (counting only the runner's full green +verdict), and writes `flaky-pass` and `rerun-fail` lines to its +own ledger (`flake-ledger-junit.jsonl`, or `GSTACK_FLAKE_LEDGER_JUNIT`), kept apart +from this file-level ledger. + The aggregate job of `free-tests.yml` downloads every `flake-ledger-*` artifact of the run and appends `bun run scripts/test-health-report.ts flake-summary <dir>` to its job summary. The Windows lane uploads `flake-ledger-windows-<shard>`. @@ -1504,7 +1522,9 @@ refresh to a maintainer. - `free-durations-record.yml` (Mondays 04:00 UTC, before test-health, and on dispatch) records the Linux seed on `ubicloud-standard-8` with `scripts/ubicloud/setup-free-suite.sh` and the Windows seed through - `windows-free-tests.yml`'s `record_durations` path. When either recording adds + `windows-free-tests.yml`'s `record_durations` path. The recorder stays on + `ubicloud-standard-8` while the free-suite shards run on `FREE_SUITE_RUNNER` + (`ubicloud-standard-2`); both are Ubicloud standard runners. When either recording adds or drops a test file, it force-updates the `bot/free-duration-seeds` branch, opens or refreshes one seed PR whose body states the ratchet result, and dispatches Free Tests, Windows Free Tests and Quality gate on it. Those @@ -1624,6 +1644,51 @@ lower-case words in path components under `test/fixtures/`; a substring match separate change with its own touchfile updates; the list is here so that change can be scoped. +## No-gstack arm + +Garry, 2026-10-31: "all of our head to head tests have to test no gstack from +here so that that case is tested obviously Do it for science". A variant that +beats another variant can still lose to the plain model, and only a no-gstack +arm shows that. + +**Scope.** Every preregistered comparison where any arm loads gstack or another +skill stack: review variants, preamble loadouts, skill diets scored by behavior +evals, and model panels run under gstack or under another stack. The +preregistration says which kind it is in `## Comparison`: `stack: loaded`, or +`stack: none` with a `why:` line (a scorer calibration, for example, where no +arm loads any skill text). `gstack-eval-plan prereg --check` refuses a loaded +comparison without the arm (`PREREG_NO_GSTACK_ARM`). + +**How the arm is built.** One arm per model in the comparison, named +`<model>#no-gstack` in the Arms section and listed on the `arms:` line of +`## No-gstack arm`. Arm identities are `<model>[#<loadout>]` and are priced by +the model, so the pilot reserves the no-gstack arm beside its variants under its +own attempts. The arm gets: + +- the same model, settings, inputs, fixture and tools as the variants; +- a plain prompt: the task text only, with no SKILL.md, preamble, hooks, + installed skills or routing lines from gstack or any other stack; +- the same panel: the same `E2E_KINDS` kind and trial count, the same judge and + rubric, and the same blinding labels (the judge reads it as `A` or `B` like + any other arm, per [Candidate-side blinding](#candidate-side-blinding)); +- the same budget treatment: it is priced and reserved in the pilot like every + other arm. + +**Reporting.** The no-gstack result is one row beside the variants in every +results table and every verdict, including when it wins. A comparison that +drops the row has not reported its result. The prereg check cannot see the +report, so the `report:` line states where the row goes and the reviewer of +the result holds the author to it. + +**Precedent and its gaps.** `test/helpers/arm-benchmark-harness.ts` already has +a `without-skill` arm beside `with-skill`, and it is the closest existing +control. It falls short of this rule in three ways: each arm and task runs +once rather than as a panel; the judge (`armJudge` in +`test/helpers/llm-judge.ts`) scores each diff on its own, without `A`/`B` +labels; and the candidate's temporary folder is named after the arm +(`arm-<fixture>-<arm>-*`). Existing paid cases are not retrofitted by the +change that added this rule; an audit of them is tracked separately. + ## Running evals as an agent: detach Moved verbatim from CLAUDE.md (#2096 size limit). The short rule stays in diff --git a/docs/flaky.md b/docs/flaky.md new file mode 100644 index 000000000..1a56b798e --- /dev/null +++ b/docs/flaky.md @@ -0,0 +1,137 @@ +# Flaky tests: rerun once and record it (`gstack-flaky`) + +`gstack-flaky run` runs your test command, reads the JUnit report it writes, +and when tests failed, runs the command once more. A test that passes on its +rerun is a **flaky pass**: the job stays green and the flake is recorded on a ledger, so +the daily flaky pass can fix it at the source. A test that fails twice is red. +This is CREW rule 16 ("run the failed test again and note that it is flaky") +for any runner that writes JUnit: Playwright, `bun test`, jest, vitest and +pytest. gstack's own free suite keeps its file-level retry +(`GSTACK_FREE_RETRY_FLAKY`, [Flake ledger](TESTING_INTERNALS.md#flake-ledger)). + +## Get started in GitHub Actions + +```yaml +- uses: oven-sh/setup-bun@v2 + with: { bun-version: 1.4.4 } +- name: Install gstack-flaky (pinned) + env: { GSTACK_REF: <a gstack commit sha you have tested> } + run: | + git clone --quiet --filter=blob:none https://github.com/garrytan/gstack.git "$RUNNER_TEMP/gstack" + git -C "$RUNNER_TEMP/gstack" checkout --quiet "$GSTACK_REF" +- name: E2E + env: + PLAYWRIGHT_JUNIT_OUTPUT_NAME: results/junit.xml + GSTACK_FLAKE_LEDGER_JUNIT: ${{ runner.temp }}/flaky-junit-ledger.jsonl + run: | + "$RUNNER_TEMP/gstack/bin/gstack-flaky" run --junit results/junit.xml -- npx playwright test --reporter=line,junit +- if: always() + uses: actions/upload-artifact@v4 + with: { name: flaky-junit-ledger-e2e, path: "${{ runner.temp }}/flaky-junit-ledger.jsonl", if-no-files-found: ignore, retention-days: 90 } +``` + +Replace `GSTACK_REF` with a gstack commit you have tested; the bin needs only +Bun and git. In Actions it also writes a step-summary table of the flaky +passes and one `::warning` annotation per flaky test, so they are visible on +the run before anyone reads the ledger. + +## Runners + +| Runner | Command after `--` | Notes | +|---|---|---| +| Playwright | `npx playwright test --reporter=line,junit` with `PLAYWRIGHT_JUNIT_OUTPUT_NAME=<file>` | `gstack-flaky` sets `PLAYWRIGHT_JUNIT_INCLUDE_RETRIES=1` so Playwright's own `retries` are reported; with retries configured, those attempts stand and nothing is rerun again. | +| bun | `bun test --reporter=junit --reporter-outfile=<file>` | | +| jest, vitest, pytest (`--runner generic`) | your command, writing JUnit | Always reruns the whole command, even with `--rerun-scope test`. | + +The command after `--` runs without a shell. When it needs one, wrap it as +`bash -c '<script> "$@"' _` so a targeted rerun's selection arguments reach +the runner. + +## Rerun scope + +**Default: the whole command.** The rerun repeats your command exactly, and +counts only on the runner's own full green verdict: it exits 0, no test fails, +every test of the first run is in its report, and the runner counts no failure +outside the report. That verdict covers every reason a run can fail, including +the ones a narrowed rerun cannot see: a test the first run never reached +(`--bail`, `--max-failures`, `-x`), a coverage threshold (however coverage is +configured, `bunfig.toml` and lcov included), a fail-on-flaky policy, a +global setup error. A test that failed in the first run and passes once in this +rerun is a flaky pass. It costs one more run of the suite, which is still far +cheaper than rerunning the CI job by hand. + +**Opt-in: `--rerun-scope test`.** Reruns the failed test alone (Playwright: +`<file> --grep <title> --project <project>`; bun: `<file> --test-name-pattern <title>`) +and clears it when that rerun exits 0. Use it only when the first run's exit +status can come from failed tests alone: no coverage threshold in any form, no +`--fail-on-flaky-tests`, no other step inside the command that can fail. As a +safety net, `gstack-flaky` still reruns the whole command +(`FLAKY_RERUN: scope=command (<why>)`) when the runner exited anything but +1, its summary is missing, it stopped early (bun `Bailed out`, Playwright +`did not run`, `interrupted`, `Testing stopped early`, a global timeout), it +printed a coverage table or got `--coverage`, or it retried flaky tests itself. +It cannot see a coverage threshold configured without console output (for +example `coverageReporter = ["lcov"]` in `bunfig.toml`), which is why this +scope is opt-in. + +## Rules + +- **One rerun, small failures only.** A failed test gets one rerun when the + runner made no retries and at most `--max-reruns` (3) tests failed. More + failures than that is a real red, and nothing is rerun. When the run is + already red for a test that cannot be rerun, a whole-command rerun is + skipped. +- **A rerun clears a test only cleanly.** The rerun's command must exit 0 and + the test must run once and pass on its first attempt; a rerun the runner + retried again does not count. +- **Never green by accident.** The report is deleted before every attempt, and + a report that attempt did not write counts as missing. A missing, empty, + zero-test or malformed report fails, as does a non-zero exit with no failed + test (a crash or a non-test step) and a runner killed by a signal. Cancelling + `gstack-flaky` (SIGINT, SIGTERM) is passed to the runner and latched: nothing + is rerun after it, whatever exit the runner chooses. When the runner's + summary counts failures its JUnit omits (a bun file that fails to load, a + Playwright error outside any test), the run is red and nothing is rerun. +- **Reports.** After the run, `--junit` holds the first run's report, so a + later upload or test reporter sees what failed. Each rerun's report is kept + beside it as `<name>.rerun-<n>.xml`. Other reporters' outputs (Playwright's + `test-results/` and HTML report) are rewritten by a rerun; upload the JUnit + report as the record. An existing `--junit` file that is not a JUnit report, + or cannot be read, is refused (usage, exit 2), never deleted. +- **Ids.** A test is `<file>::<project>::<describe > title>`, with the project + empty outside Playwright, so a test in two browser projects is two tests. + `gstack-flaky ids --junit <file>` prints them. +- **Ledger.** One JSON line per flaky pass (`kind: flaky-pass`) and per test + that failed its rerun (`rerun-fail`), with the tree, commit, branch, run URL, + attempts, the rerun's scope (`test` or `command`) and the first lines of + the error. Ids, paths and failure text go through gstack's redaction engine + first; when the scan cannot run, they are withheld. The path is `--ledger`, + else `GSTACK_FLAKE_LEDGER_JUNIT`, else + `<state root>/flake-ledger-junit.jsonl`, and `FLAKY_RESULT` names it. It is + kept apart from the free suite's file-level ledger (`GSTACK_FLAKE_LEDGER`). + A ledger write error is a warning and never changes the verdict. + +## Output and exit codes + +One `FLAKY: <flaky-pass|fail|rerun-fail> <id> attempts=<…> (<reason>)` line per +test that failed at least once, `FLAKY_RERUN: scope=command (<why>)` when the +rerun repeated the whole command, then +`FLAKY_RESULT: tests=<n> failed=<n> flaky=<n> reruns=<n> command_exit=<n> verdict=pass|fail ledger=<file|none>`. + +| Exit | Meaning | +|---|---| +| 0 | Every test passed, flaky passes included | +| 1 | A test failed (after its rerun, or with no rerun), including `FLAKY_UNREPORTED_FAILURES` | +| 2 | Usage error, including a `--junit` path that is not a `.xml` file, or an existing file there that is not a readable JUnit report | +| 3 | No usable report, a crash, or a killed runner (a code below on stderr) | + +| Code | Exit | Cause | Fix | +|---|---|---|---| +| `FLAKY_NO_RESULTS` | 3 | No report at `--junit`, an empty one, or one with no testcase | Make the command write JUnit to that path | +| `FLAKY_JUNIT_INVALID` | 3 | The report is not well-formed XML | Make sure one process writes it and nothing appends to it | +| `FLAKY_RETRIES_HIDDEN` | 3 | Playwright retried but its report omits the retries | Set `PLAYWRIGHT_JUNIT_INCLUDE_RETRIES=1` where the report is written | +| `FLAKY_COMMAND_FAILED` | 3 | Non-zero exit with no failed test | Read the command output: a real red | +| `FLAKY_COMMAND_KILLED` | 3 | The runner was killed by a signal (a cancelled job, an OOM kill) | Nothing was rerun; rerun the job | +| `FLAKY_COMMAND_NOT_STARTED` | 3 | The command could not be started | Check the argv after `--` | +| `FLAKY_UNREPORTED_FAILURES` | 1 | The runner counted failures or errors its JUnit omits | Fix the load error or the error outside any test: a real red | +| `FLAKY_LEDGER_WRITE_FAILED` | — | The ledger could not be appended (warning) | Check the path; the verdict stands | diff --git a/docs/pregate.md b/docs/pregate.md index b3b5ef4ed..b9d7596a3 100644 --- a/docs/pregate.md +++ b/docs/pregate.md @@ -85,7 +85,7 @@ the byte-pinned `/ship` renders in `test/fixtures/golden/`). | `serial_patterns` | test-file globs the `lanes` check runs one process per file, apart from the unit files (default `**/*.serial.test.*`) | | `sealed` | sealed or eval output that must never reach a public destination; read by the pre-push hook and `gstack-pregate sealed` (see [sealed output](#sealed)) | | `price_file` | repo-relative path of the paid gate's call-plan data file (`GSTACK_EVAL_CALLS v1`); `gstack-eval-plan price-gate` prices from it and never executes anything ([docs/eval-calls.md](eval-calls.md)) | -| `stall_timeout_s` | seconds a test command may print nothing before its process group is killed and the check is `incomplete` (`PREGATE_STALLED`); default 300, `0` disables; `--stall-timeout` overrides (see [supervision](#supervision)) | +| `stall_timeout_s` | seconds a test command may print nothing before its process group is killed and the check is `incomplete` (`PREGATE_STALLED`); default 300, `0` disables; `--stall-timeout` overrides; the `--remote` gate is exempt (see [supervision](#supervision)) | | `timeout_s` | seconds each runner process may take (the unit-file group, each serial file, each guard without its own `timeout_s`, and the remote gate's deadline) when `--timeout` is not passed; 1 to 3600, default 600. `gstack-ship-preflight` and the `finish` fallback run pregate without `--timeout`, so this is how a repo with long lanes sets theirs | | `remote_gate` | the repo's remote test gate command; under 8 cores the local test tier becomes a `requires-remote` obligation naming it, and `--remote` runs it (see [remote gate](#remote-gate)) | | `allowlist` | `[{ finding, paths?, pattern?, reason }]`: benign findings the `secrets` check accepts and names as allowlisted, by gstack-redact finding id or non-secret category, path globs and a pattern anchored at the finding; never `env.value` (see [allowlist](#allowlist)) | @@ -411,7 +411,9 @@ test-tier command: - is killed as **stalled** when its log does not grow for the stall timeout (`--stall-timeout <s>`, else `stall_timeout_s`, else 300; `0` disables): the check is `incomplete` with `PREGATE_STALLED` and names the last file - started; the next serial file still runs; + started; the next serial file still runs. The `--remote` gate is exempt: a + gate that waits on CI is quiet by design, so only its timeout bounds it, and + its heartbeats add `; last output: <the gate's last line>`; - is killed as **timed out** when that runner process reaches its timeout (`--timeout <s>`, else `timeout_s`, else 600; per runner process, so many serial files that each fit still pass): the check is `incomplete`, its detail @@ -452,7 +454,11 @@ the supervision above. Close the obligation one of two ways: `PREGATE_REMOTE: tree=<tree> tests=<n> failed=<n> [spent=<usd>]`, reporting the tree it actually tested. The obligation clears only when that tree is `PREGATE_TREE`, `tests` > 0, `failed` = 0 and the gate exits 0; anything else - leaves it uncleared and the check `incomplete`. The cap defaults to + leaves it uncleared and the check `incomplete`. The gate runs until it exits + or reaches the timeout (`--timeout`, else `timeout_s`, else 600 s); the stall + timer does not apply, and each 60 s heartbeat names the gate's last output + line, so a gate that prints what it is waiting on (a CI run id and its + status) shows it. The cap defaults to `$GSTACK_SPEND_CAP_USD`; there is no default spend. Spend prints as `spent, reserved, unknown`: a gate that prints no `spent=` is `unknown`, counted at the cap. diff --git a/docs/review-policy.md b/docs/review-policy.md index d86c1b71d..301d187e5 100644 --- a/docs/review-policy.md +++ b/docs/review-policy.md @@ -473,7 +473,14 @@ ones, and the arm lost blockers. `bin/gstack-review-dispatch` closes both ends: second line for the same field starts another: `Severity` raises its label, `Blocks: yes` makes it CRITICAL and `Blocks: no` never lowers it; an inline `Blocks: yes` is CRITICAL too, and with no finding above it is one. The native - adversarial voice's FIXABLE collects as CRITICAL (normalizer v2, GSTA-167). A phantom finding costs a fold; a lost one costs the gate. + adversarial voice's FIXABLE collects as CRITICAL (normalizer v2, GSTA-167). An unnumbered, unlabeled bullet with no file:line + under a paragraph finding whose Severity or Blocks line has not come yet is that finding's detail, so the field line after it + lands on the finding; a numbered item is its own finding. + A phantom finding costs a fold; a lost one costs the gate. + A reply with no finding counts as a review only with the clean marker its own brief asks for: `NO FINDINGS` alone on a + line for the specialists, the batched specialist, the targeted reader, the Red Team and fix-check, the `Recommendation:` + line for the outside voice, either for the native adversarial voice. Without one (a refusal, an empty reply, a reader + that never saw the diff) collect records `review: false`, prints `COLLECT_NOT_REVIEWED` and exits 1. 4. Lead judgment writes `judged.jsonl`; `judge` refuses a set that drops a merged finding, lowers its severity, folds it into a less severe duplicate, or adds or promotes one without a reason (`JUDGMENT_REJECTED`). A merged file missing any id @@ -481,10 +488,14 @@ ones, and the arm lost blockers. `bin/gstack-review-dispatch` closes both ends: 5. `verify --ledger <f>` replays all of it from the run's ledger: rendered against dispatched hashes per render (each render is one dispatch of its reader, so a two-pass ledger checks pass-1 sends against pass-1 briefs; a later render prints as - `<reader>@2`), a collected output for every sent reader (`NOT_COLLECTED` - otherwise; a reader that failed is collected from its failure text, where any - label word becomes a finding the judge folds, and its coverage is logged - `unavailable` in the review-log row). A mismatched send re-sent + `<reader>@2`), a collected output that is a review for every sent reader (`NOT_COLLECTED` + otherwise, and the output row `NOT_REVIEWED`; the marker is checked again in the output file when it is unchanged, so an + older ledger meets the same rule; a non-review the same dispatch later replaced with a review, by re-sending the brief, is `VOIDED` + and counted in `voided=N`). The outside voice may fail without blocking: `collect --unavailable '<reason>'` records its failure text + as missing coverage (`COLLECT_UNAVAILABLE`); an empty reason or an output that holds findings is refused (exit 2), so a partial + answer is collected normally and loses nothing. Verify prints its brief and output `UNAVAILABLE` and `unavailable=N` on the + verdict line, and its coverage is logged `unavailable` in the review-log row; any other reader is refused (exit 2), because its + missing coverage leaves the review incomplete. A mismatched send re-sent byte-exact before anything was collected is void and counted (`voided=N` on the verdict line); one that produced an output stays `MISMATCH`. Then output hashes against the merged lines they produced, and the judgment rerun against the diff --git a/docs/standalone-binaries.md b/docs/standalone-binaries.md index 0c4bc8e52..e160f5549 100644 --- a/docs/standalone-binaries.md +++ b/docs/standalone-binaries.md @@ -18,6 +18,7 @@ Beyond the slash-command skills, gstack ships standalone CLIs for workflows that | `gstack-review-log` | **Review-pass receipts** — `--start <skill>` captures the working-tree fingerprint before a diff review; `'<JSON>' --finish <token>` consumes that single-use, repository/branch/skill-scoped receipt. Binding requires matching start/end content and reviewer-reported `completed:true` and `converged:true`; it is not independent proof that a model read the source. `--check-shared-libs <token>` reads a current finding as JSON on stdin and checks prior Skip decisions against the actual branch, capture and eligible raw source blobs. It returns `reusable:false` when proof is missing or unsafe, including older records without logger-versioned coverage. Final review logging computes shared-code fingerprints and coverage rather than trusting supplied proof. | | `gstack-review-read` | **Review freshness** — emits review records with computed `review_freshness.status` and `reason`: CURRENT, STALE, or UNVERIFIED for diff reviews. `/ship` and `/land-and-deploy` use the same grade; a matching commit alone never certifies a diff review. [Dashboard rules](skills.md#review-readiness-dashboard). | | `gstack-evidence` | **Verification-evidence ledger** — `run --label <lane> -- <cmd>` transparently wraps any test command (the child's exit code always passes through) and records what ran against which working-tree fingerprint (tracked and untracked files that are not gitignored, minus an untracked `.bun/` folder at the repo root, which Bun can create during a run); a run whose files changed while it ran is recorded without a fingerprint and with up to five of the changed paths, which `check` names in its STALE reason. `check` grades each label FRESH/STALE/MISSING with `--expect-cmd`, `--max-age`, and `--allow-paths` binding. /ship and /land-and-deploy cite fresh evidence instead of re-running suites. Per-run logs are 0600, capped at 2MB, pruned after 30 days; the ledger and logs stay machine-local by design. | +| `gstack-flaky` | **Flaky-test rerun for any JUnit runner** — `run --junit <file.xml> -- <command>` runs your test command (Playwright, `bun test`, jest, vitest, pytest) and, when at most `--max-reruns` (3) tests failed and the runner made no retries of its own, reruns the whole command once (`--rerun-scope test` opts in to rerunning only the failed tests). A test that fails and then passes on that rerun is a flaky pass: the run stays green and one redacted line per flaky pass or failed rerun goes on a ledger (`GSTACK_FLAKE_LEDGER_JUNIT`, default `<state root>/flake-ledger-junit.jsonl`). A missing or malformed report, a crash or a killed runner never passes. `ids --junit <file>` prints a report's test ids. Exit 0 pass, 1 a test failed, 2 usage, 3 no usable report. [Guide](flaky.md). | | `gstack-pr-wait` | **PR event watcher** — `gstack-pr-wait <pr> [--repo owner/name] [--until green\|ready\|any-change] [--interval 20] [--timeout 60] [--exclude CHECK,...]` polls one pull request through `gh` and prints one stdout line per real event (a check failed with its link, all checks finished, a comment or review from someone other than the author, a review bot finished on the head, the base moved and its new commits touch this PR's files, BEHIND or DIRTY, merged or closed), with progress on stderr, so a monitor wakes the agent once per event instead of on a timer. `green` gates on every check on the head (an advisory bot's review is never waited for, its checks gate like any other), as `gstack-ci-gate` does; a check the base branch requires (rulesets and classic branch protection) is pending until it appears, never green by omission; two identical polls confirm a check set is complete; `--exclude` leaves a named non-required check out of the verdict as `gstack-ci-gate --exclude` does (a required name is refused and gates). Exit 0 when the condition is met, 1 when it needs action or `--timeout` passes (`PR_WAIT_BLOCKED`), 2 on bad input, a second watcher for the same PR or a local error (`PR_WAIT_REFUSED`). State under `<state root>/pr-wait/`, baseline discarded after 6 hours. Review bots are configuration: `gstack-config set pr_wait_gating_bots greptile,macroscope`, `pr_wait_advisory_bots coderabbit` (advisory bots are reported, never waited on). `/land-and-deploy` Step 3 is one call of it. Design ported from tstack's pr-wait (MIT, Theo Browne; NOTICE.md). [Exit codes](troubleshooting.md#gstack-pr-wait). | | `gstack-issue-guard` | **Tracker-text trust envelope** — fetches GitHub issue/PR text (`issue <n>`, `pr-body`, `pr-comments`, or `--stdin`) and wraps it in a labeled envelope so agents treat it as data: injection-shaped lines get labeled even through fullwidth and invisible-character evasion, and forged envelope banners are defused. Every tracker-text ingress in gstack routes through it, enforced by a CI scanner. | | `gstack-ios-qa-daemon` | **iOS QA daemon** — Mac-side broker between an agent and a connected iPhone or iPad over USB CoreDevice. Loopback by default; `--tailnet` opens a Tailscale-facing listener with identity-gated capability tiers. Single-instance via flock on `~/.gstack/ios-qa-daemon.pid`. See [docs/howto-ios-testing-with-gstack.md](howto-ios-testing-with-gstack.md). | diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 0abe1935f..a8c51edff 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -2522,12 +2522,15 @@ same-titled section (title compared case-insensitively), the ````gstack-ship-receipt```` block is a section of its own so a receipt rewrite replaces only the receipt, sections the file does not rewrite stay in place, and new sections are appended after them. The text above the first -`## ` heading belongs to the file whenever the file has a `## ` section: the -current body's text there (the opening lines, and any lines between the receipt -and the first heading, or the whole body when it has no `## ` heading) is -replaced by the file's own text above its first heading, or dropped when the -file has none. A file with no `## ` section (a receipt-only or text-only post) -leaves that text alone. One stderr line names what was replaced, kept, added +`## ` heading belongs to the file when the file is a full post: a +receipt block above its first `## ` section, as /ship composes it. Then the current body's text there (the opening lines, and +any lines between the receipt and the first heading, or the whole body when it +has no `## ` heading) is replaced by the file's own text above its first +heading, or dropped when the file has none. A partial post (`## ` sections +with no receipt above them, such as a lane posting `## Screenshots`) and a +file with no `## ` section (a receipt-only or text-only post) leave that text alone, so a human +intro or a `Review-loop exception:` line under the receipt survives; only +opening lines the file brings of its own replace the PR's opening lines. One stderr line names what was replaced, kept, added and dropped, a dropped piece with its first line, or with that line withheld when the redaction scan flags it (`gstack-post: pr-body merged by section: replaced What changed; kept ship receipt, Why, Approvals; added @@ -3464,8 +3467,7 @@ the lane log, then rerun or split the selection. A tool missing from PATH is <a id="pregate-stalled"></a> ### `PREGATE_STALLED` -**Meaning.** A test command the lanes check (or a guard, or the `--remote` -gate) launched wrote nothing to its log for the stall timeout (default 300 s, +**Meaning.** A test command the lanes check (or a guard) launched wrote nothing to its log for the stall timeout (default 300 s, `stall_timeout_s` in the base branch's `.gstack/pregate.json` or the user-owned file, or `--stall-timeout`). pregate killed its process group and names the last test file started. The check is incomplete, never pass. @@ -3474,7 +3476,9 @@ file started. The check is incomplete, never pass. child, a server that never exits, a synchronous wait), fix it and rerun `gstack-pregate --stage tests`. A test that is legitimately quiet for longer needs a larger `--stall-timeout`; `0` disables the stall kill and leaves only -`--timeout`. +`--timeout`. The `--remote` gate never stalls: it is bounded by `--timeout` +(else `timeout_s`), so a gate that waits on CI needs a timeout long enough for +the CI run. <a id="pregate-receipt-invalid"></a> ### `PREGATE_RECEIPT_INVALID` @@ -3640,14 +3644,44 @@ parentheses; grep the code. ### `PREREG_INCOMPLETE` **Meaning.** `gstack-eval-plan prereg --check <file>` found a required -preregistration section missing or empty: `## Bars`, `## Arms`, `## Decoys`, -`## Stop rules` or `## Held-out exposure` (every held-out set the plan -touches, its prior exposure and the reservation on it). +preregistration section missing or empty: `## Bars`, `## Arms`, +`## Comparison`, `## Decoys`, `## Stop rules` or `## Held-out exposure` +(every held-out set the plan touches, its prior exposure and the reservation +on it). It also fires when `## Comparison` has no `stack:` line, a value other +than `loaded` or `none`, or `stack: none` without a `why:` line, and when a +`## Diagnostics` declaration is malformed. **Fix.** Fill the named sections before any priced call. A plan that cannot name its stop rule or its held-out exposure is not preregistered; the pilot refuses to start without a passing check. +<a id="prereg-no-gstack-arm"></a> +### `PREREG_NO_GSTACK_ARM` + +**Meaning.** The preregistration says `stack: loaded` (some arm loads gstack or +another skill stack) and its `## No-gstack arm` section is missing, lacks one +of `arms:`, `inputs:`, `prompt:`, `panel:` or `report:`, lists an arm that +is not a `<model>#no-gstack` Arms line, or leaves a model in the comparison +without its own no-gstack arm. Arms match on whole identities, so +`claude-opus-5#no-gstack` does not cover `claude-opus-5-5`. + +**Fix.** Add one `<model>#no-gstack` line per model under `## Arms`, list them +on the `arms:` line, and fill the other four keys: the same inputs, a plain +prompt with no skill text, the same panel and judge, and where the result row +goes beside the variants. When no arm loads a skill stack, write +`stack: none` with a `why:` line instead. The rule is in +[docs/TESTING_INTERNALS.md](TESTING_INTERNALS.md#no-gstack-arm). + +<a id="eval-arm-duplicate"></a> +### `EVAL_ARM_DUPLICATE` + +**Meaning.** `gstack-eval-plan pilot start --arms` named the same arm identity +twice, so two arms would share one set of pilot attempts. + +**Fix.** Give each arm its own identity: `<model>` or `<model>#<loadout>`, for +example `claude-opus-5-5` and `claude-opus-5-5#no-gstack`. Nothing was +reserved. + <a id="eval-price-missing"></a> ### `EVAL_PRICE_MISSING` diff --git a/eval-plan/SKILL.md b/eval-plan/SKILL.md index 9200ee065..5ec40e511 100644 --- a/eval-plan/SKILL.md +++ b/eval-plan/SKILL.md @@ -135,10 +135,12 @@ Write (or open) the preregistration file. Each section is required and must be f | Section | What it holds | |---|---| | Bars | one line per metric: name, pass bar, direction, what a miss means for the decision | -| Arms | one line per arm: model id (in the price table), settings, per-item cost from Step 1b | +| Arms | one line per arm: the arm identity first (`<model>` or `<model>#<loadout>`), settings, per-item cost from Step 1b | +| Comparison | `stack: loaded` when any arm loads gstack or another skill stack; otherwise `stack: none` and a `why:` line | | Decoys | inputs the scorer must reject or score low; how many, how they were made, what a decoy pass proves | | Stop rules | the spend cap, a bar missed on the pilot, a scorer below its bar, `budget_exhausted` | | Held-out exposure | every held-out set the plan touches: prior exposure (which runs, which models), the reservation on it | +| No-gstack arm | required with `stack: loaded`: `arms:`, `inputs:`, `prompt:`, `panel:`, `report:` (below) | ```bash ~/.claude/skills/gstack/bin/gstack-eval-plan prereg --init <path> @@ -149,6 +151,16 @@ Write (or open) the preregistration file. Each section is required and must be f (`PREREG_INCOMPLETE`, exit 1); fill them and rerun. Do not continue on an incomplete preregistration. +**No-gstack arm.** A head-to-head where any arm loads gstack or another skill stack +(review variants, preamble loadouts, skill diets, a model panel under a stack) also +runs each model with no skill stack at all. Add `<model>#no-gstack` to Arms for every +model, and fill `## No-gstack arm`: `arms:` lists those arms; `inputs:` the same rows +and fixture; `prompt:` the task text only, with no SKILL.md, preamble, hooks or installed +skills, and the same tools; `panel:` the same E2E kind, trials, judge and blinding labels; +`report:` its row beside the variants in every results table and verdict. +`PREREG_NO_GSTACK_ARM` (exit 1) names what is missing. The PREREG line shows +`stack=loaded no_gstack_arm=ok` when it is complete, or `stack=none` for an opt-out. + When the summarizer counts diagnostics, declare them before the freeze commit in a `## Diagnostics` section: `summarizer: <command naming {fixture}>`, `fixture: <synthetic records file>`, and one `| diagnostic | expected |` row per diagnostic, with the count the @@ -169,7 +181,9 @@ falling back to the repo's own policy document (`--policy-doc <path>`), or from `--arms <picked-models>` when the user named them. Each `ARM:` line carries its source and the per-item cost at the stated token estimate; `price=MISSING` (`EVAL_PRICE_MISSING`) means the model has no row in `lib/pricing.ts`: add the row from the provider's pricing -page or drop the arm. Copy the priced arms into the Arms section. +page or drop the arm. Copy the priced arms into the Arms section. An arm identity +`<model>#<loadout>` (for example `claude-opus-5-5#no-gstack`) is priced by its model and +gets its own pilot attempts; naming one identity twice is `EVAL_ARM_DUPLICATE`. When the repo commits `.gstack/eval-models.json` (allow, deny, per-scope lists, families newest first; `docs/eval-models.md` in gstack), `arms`, `dry-run`, `pilot start` and diff --git a/eval-plan/SKILL.md.tmpl b/eval-plan/SKILL.md.tmpl index 39ad672ee..019499eca 100644 --- a/eval-plan/SKILL.md.tmpl +++ b/eval-plan/SKILL.md.tmpl @@ -57,10 +57,12 @@ Write (or open) the preregistration file. Each section is required and must be f | Section | What it holds | |---|---| | Bars | one line per metric: name, pass bar, direction, what a miss means for the decision | -| Arms | one line per arm: model id (in the price table), settings, per-item cost from Step 1b | +| Arms | one line per arm: the arm identity first (`<model>` or `<model>#<loadout>`), settings, per-item cost from Step 1b | +| Comparison | `stack: loaded` when any arm loads gstack or another skill stack; otherwise `stack: none` and a `why:` line | | Decoys | inputs the scorer must reject or score low; how many, how they were made, what a decoy pass proves | | Stop rules | the spend cap, a bar missed on the pilot, a scorer below its bar, `budget_exhausted` | | Held-out exposure | every held-out set the plan touches: prior exposure (which runs, which models), the reservation on it | +| No-gstack arm | required with `stack: loaded`: `arms:`, `inputs:`, `prompt:`, `panel:`, `report:` (below) | ```bash ~/.claude/skills/gstack/bin/gstack-eval-plan prereg --init <path> @@ -71,6 +73,16 @@ Write (or open) the preregistration file. Each section is required and must be f (`PREREG_INCOMPLETE`, exit 1); fill them and rerun. Do not continue on an incomplete preregistration. +**No-gstack arm.** A head-to-head where any arm loads gstack or another skill stack +(review variants, preamble loadouts, skill diets, a model panel under a stack) also +runs each model with no skill stack at all. Add `<model>#no-gstack` to Arms for every +model, and fill `## No-gstack arm`: `arms:` lists those arms; `inputs:` the same rows +and fixture; `prompt:` the task text only, with no SKILL.md, preamble, hooks or installed +skills, and the same tools; `panel:` the same E2E kind, trials, judge and blinding labels; +`report:` its row beside the variants in every results table and verdict. +`PREREG_NO_GSTACK_ARM` (exit 1) names what is missing. The PREREG line shows +`stack=loaded no_gstack_arm=ok` when it is complete, or `stack=none` for an opt-out. + When the summarizer counts diagnostics, declare them before the freeze commit in a `## Diagnostics` section: `summarizer: <command naming {fixture}>`, `fixture: <synthetic records file>`, and one `| diagnostic | expected |` row per diagnostic, with the count the @@ -91,7 +103,9 @@ falling back to the repo's own policy document (`--policy-doc <path>`), or from `--arms <picked-models>` when the user named them. Each `ARM:` line carries its source and the per-item cost at the stated token estimate; `price=MISSING` (`EVAL_PRICE_MISSING`) means the model has no row in `lib/pricing.ts`: add the row from the provider's pricing -page or drop the arm. Copy the priced arms into the Arms section. +page or drop the arm. Copy the priced arms into the Arms section. An arm identity +`<model>#<loadout>` (for example `claude-opus-5-5#no-gstack`) is priced by its model and +gets its own pilot attempts; naming one identity twice is `EVAL_ARM_DUPLICATE`. When the repo commits `.gstack/eval-models.json` (allow, deny, per-scope lists, families newest first; `docs/eval-models.md` in gstack), `arms`, `dry-run`, `pilot start` and diff --git a/lib/eval-plan.ts b/lib/eval-plan.ts index 33542c5fa..6bb74db94 100644 --- a/lib/eval-plan.ts +++ b/lib/eval-plan.ts @@ -23,8 +23,14 @@ import { readLedger, release, reserve, settle, SpendCapExceeded, summarize, type // --------------------------------------------------------------------------- // preregistration // --------------------------------------------------------------------------- -export const PREREG_SECTIONS = ['Bars', 'Arms', 'Decoys', 'Stop rules', 'Held-out exposure'] as const; -export interface PreregCheck { ok: boolean; present: string[]; missing: string[]; empty: string[]; diagnostics: PreregDiagnostics | null } +export const PREREG_SECTIONS = ['Bars', 'Arms', 'Comparison', 'Decoys', 'Stop rules', 'Held-out exposure'] as const; +/** `## Comparison`: `stack: loaded` when any arm loads gstack or another skill stack, `stack: none` with a `why:` otherwise. */ +export interface ComparisonCheck { stack: 'loaded' | 'none' | null; why: string | null; errors: string[] } +/** `## No-gstack arm`, required when `stack: loaded`: one `<model>#no-gstack` Arms line per model, and the inputs, prompt, panel and report it shares with the variants. */ +export interface NoGstackArmCheck { required: boolean; ok: boolean; arms: string[]; problems: string[] } +export interface PreregCheck { ok: boolean; present: string[]; missing: string[]; empty: string[]; comparison: ComparisonCheck; no_gstack: NoGstackArmCheck; diagnostics: PreregDiagnostics | null } +export const NO_GSTACK_LOADOUT = 'no-gstack'; +export const NO_GSTACK_KEYS = ['arms', 'inputs', 'prompt', 'panel', 'report'] as const; export function preregTemplate(title = 'Eval preregistration'): string { return `# ${title} @@ -33,7 +39,13 @@ export function preregTemplate(title = 'Eval preregistration'): string { <!-- One line per metric: name, the pass bar, the direction, and what a miss means for the decision. --> ## Arms -<!-- One line per arm: model id (must be in lib/pricing.ts), settings, and the per-item cost from \`gstack-eval-plan arms\`. --> +<!-- One line per arm: the arm identity first (\`<model>\` or \`<model>#<loadout>\`, the model in lib/pricing.ts, e.g. \`claude-opus-5-5#no-gstack\`), settings, and the per-item cost from \`gstack-eval-plan arms\`. --> + +## Comparison +<!-- \`stack: loaded\` when any arm loads gstack or another skill stack (review variants, preamble loadouts, skill diets, a model panel under a stack); otherwise \`stack: none\` and a \`why:\` line. --> + +## No-gstack arm +<!-- Required with \`stack: loaded\`: \`arms:\` one \`<model>#no-gstack\` Arms line per model; \`inputs:\` the same rows and fixture; \`prompt:\` the task only (no SKILL.md, preamble, hooks or installed skills); \`panel:\` the same E2E kind, trials and judge; \`report:\` its row beside the variants. --> ## Decoys <!-- Inputs the scorer must reject or score low; how many, how they were made, what a decoy pass proves. --> @@ -47,14 +59,71 @@ export function preregTemplate(title = 'Eval preregistration'): string { `; } -/** A section's content lines (blank lines and one-line `<!-- -->` guidance dropped); null when the heading is absent. */ +/** A section's content lines (blank lines and `<!-- -->` guidance, one-line or multi-line, dropped); null when the heading is absent. */ function sectionBody(text: string, name: string): string[] | null { const lines = text.split('\n'); const headings = lines.map((l, i) => ({ i, m: /^#{1,6}\s+(.+?)\s*$/.exec(l) })).filter(x => x.m) as Array<{ i: number; m: RegExpExecArray }>; const idx = headings.findIndex(h => h.m[1]!.toLowerCase().replace(/[^a-z]/g, '') === name.toLowerCase().replace(/[^a-z]/g, '')); if (idx < 0) return null; const end = idx + 1 < headings.length ? headings[idx + 1]!.i : lines.length; - return lines.slice(headings[idx]!.i + 1, end).filter(l => l.trim() && !/^\s*<!--.*-->\s*$/.test(l)); + return lines.slice(headings[idx]!.i + 1, end).join('\n').replace(/<!--[\s\S]*?-->/g, '').split('\n').filter(l => l.trim()); +} + +/** The model an arm identity `<model>[#<loadout>]` is priced by. */ +export const armModel = (id: string) => id.split('#')[0]!; +const armLoadout = (id: string) => (id.includes('#') ? id.slice(id.indexOf('#') + 1) : null); + +/** The first token of each Arms line (bullets, table pipes and backticks stripped; a table's header row skipped). */ +export function armIdentities(text: string): string[] { + const body = sectionBody(text, 'Arms') ?? []; + const row = (l: string) => /^\s*\|/.test(l); + const separator = (l: string | undefined) => !!l && /^[\s:|-]+$/.test(l) && l.includes('-'); + const ids: string[] = []; + body.forEach((line, i) => { + if (separator(line) || (row(line) && separator(body[i + 1]))) return; + const token = line.replace(/^\s*(?:[-*+]|\d+[.)])?\s*\|?\s*/, '').split(/[\s|]+/)[0]!.replace(/[`,;:]+$/g, '').replace(/^`/, ''); + if (token) ids.push(token); + }); + return ids; +} + +const keyValues = (body: string[]) => { + const kv: Record<string, string> = {}; + for (const line of body) { + const m = /^\s*(?:[-*]\s+)?([a-z]+):(.*)$/i.exec(line); + if (m) kv[m[1]!.toLowerCase()] = m[2]!.trim(); + } + return kv; +}; + +function checkComparison(text: string): ComparisonCheck { + const body = sectionBody(text, 'Comparison'); + if (!body?.length) return { stack: null, why: null, errors: [] }; + const kv = keyValues(body); + const why = kv.why || null; + if (kv.stack === undefined) return { stack: null, why, errors: ['no stack: line (stack: loaded or stack: none)'] }; + if (kv.stack !== 'loaded' && kv.stack !== 'none') return { stack: null, why, errors: [`stack: ${kv.stack} is neither loaded nor none`] }; + return { stack: kv.stack, why, errors: kv.stack === 'none' && !why ? ['stack: none needs a why: line saying why no arm loads a skill stack'] : [] }; +} + +function checkNoGstackArm(text: string, stack: ComparisonCheck['stack']): NoGstackArmCheck { + if (stack !== 'loaded') return { required: false, ok: true, arms: [], problems: [] }; + const body = sectionBody(text, 'No-gstack arm'); + if (!body?.length) return { required: true, ok: false, arms: [], problems: ['missing section'] }; + const kv = keyValues(body); + const problems = NO_GSTACK_KEYS.filter(k => !kv[k]).map(k => `missing ${k}`); + const listed = (kv.arms ?? '').split(/[\s,]+/).map(a => a.replace(/^`|`$/g, '')).filter(Boolean); + const ids = armIdentities(text); + for (const a of listed) { + if (armLoadout(a) !== NO_GSTACK_LOADOUT) problems.push(`${a} is not a <model>#${NO_GSTACK_LOADOUT} identity`); + else if (!ids.includes(a)) problems.push(`${a} is not an Arms line`); + } + const loaded = [...new Set(ids.filter(id => armLoadout(id) !== NO_GSTACK_LOADOUT).map(armModel))]; + for (const m of loaded) { + const arm = `${m}#${NO_GSTACK_LOADOUT}`; + if (!ids.includes(arm) || !listed.includes(arm)) problems.push(`no no-gstack arm for ${m}`); + } + return { required: true, ok: problems.length === 0, arms: listed, problems }; } export function checkPrereg(text: string): PreregCheck { @@ -63,8 +132,10 @@ export function checkPrereg(text: string): PreregCheck { const body = sectionBody(text, name); if (body === null) missing.push(name); else if (body.length === 0) empty.push(name); else present.push(name); } + const comparison = checkComparison(text); + const no_gstack = checkNoGstackArm(text, comparison.stack); const diagnostics = parsePreregDiagnostics(text); - return { ok: missing.length === 0 && empty.length === 0 && !diagnostics?.errors.length, present, missing, empty, diagnostics }; + return { ok: missing.length === 0 && empty.length === 0 && !comparison.errors.length && no_gstack.ok && !diagnostics?.errors.length, present, missing, empty, comparison, no_gstack, diagnostics }; } // --------------------------------------------------------------------------- @@ -152,8 +223,8 @@ export const DEFAULT_TOKENS: TokenEstimate = { input: 4_000, output: 2_000 }; const POLICY_DOCS = ['docs/model-policy.md', 'MODEL_POLICY.md', 'docs/MODELS.md']; function priceArm(model: string, source: string, provider: string | null, tokens: TokenEstimate): Arm { - const row = pricingRow(model); - return { model, provider, source, priced: !!row, input_per_mtok: row?.input_per_mtok ?? null, output_per_mtok: row?.output_per_mtok ?? null, per_item_usd: row ? estimateCostUsd(tokens, model) : null }; + const row = pricingRow(armModel(model)); + return { model, provider, source, priced: !!row, input_per_mtok: row?.input_per_mtok ?? null, output_per_mtok: row?.output_per_mtok ?? null, per_item_usd: row ? estimateCostUsd(tokens, armModel(model)) : null }; } /** Models named in the repo's policy document that the price table knows. */ @@ -249,10 +320,10 @@ export function dryRun(o: DryRunOptions): DryRunResult { if (missing.length) bad.push({ line: i + 1, ok: false, missing, input_tokens: t }); }); const retryText = o.retryPrompt && fs.existsSync(o.retryPrompt) ? fs.readFileSync(o.retryPrompt, 'utf8').trim() : null; - const missingPrice = o.arms.filter(a => !hasPricing(a)); + const missingPrice = o.arms.filter(a => !hasPricing(armModel(a))); const perItem: Record<string, number | null> = {}; const avgIn = rows ? Math.ceil(inputTokens / rows) : 0; - for (const a of o.arms) perItem[a] = hasPricing(a) ? estimateCostUsd({ input: avgIn, output: budget }, a) : null; + for (const a of o.arms) perItem[a] = hasPricing(armModel(a)) ? estimateCostUsd({ input: avgIn, output: budget }, armModel(a)) : null; const checks = { price_table: { ok: missingPrice.length === 0 && o.arms.length > 0, missing: missingPrice }, output_budget: { ok: budget >= REASONING_OUTPUT_FLOOR, budget, floor: REASONING_OUTPUT_FLOOR }, @@ -260,7 +331,7 @@ export function dryRun(o: DryRunOptions): DryRunResult { } as DryRunResult['checks']; if (o.cap !== undefined) { const perArm: Record<string, number | null> = {}; - for (const a of o.arms) perArm[a] = hasPricing(a) ? estimateCostUsd({ input: maxInput, output: budget }, a) : null; + for (const a of o.arms) perArm[a] = hasPricing(armModel(a)) ? estimateCostUsd({ input: maxInput, output: budget }, armModel(a)) : null; checks.reservation_fits_cap = { ok: Object.values(perArm).every(v => v === null || v <= o.cap!), cap: o.cap, max_input_tokens: maxInput, per_arm: perArm }; } if (o.modelRules) checks.model_rules = { ok: o.modelRules.ok, source: o.modelRules.source, refused: o.modelRules.errors.map(e => e.message) }; @@ -303,15 +374,19 @@ export function startPilot(o: PilotStartOptions): { record: PilotRecord; error?: const dry = JSON.parse(fs.readFileSync(o.dryRun, 'utf8')) as DryRunResult; if (!dry.pass) return { record: null as unknown as PilotRecord, error: { code: 'EVAL_DRY_RUN_FAILED', message: `${o.dryRun} did not pass; the pilot never starts on a failed dry run` } }; const tokens = o.tokens ?? { input: dry.rows ? Math.ceil(dry.est_input_tokens / dry.rows) : DEFAULT_TOKENS.input, output: dry.checks.output_budget.budget }; + const record: PilotRecord = { schema_version: 1, n: o.n, arms: o.arms, cap_usd: o.cap, dry_run: o.dryRun, attempts: [], started_at: (o.now ?? new Date()).toISOString(), status: 'reserved' }; + const keyOf = (a: string) => `${armModel(a).replace(/[^A-Za-z0-9.-]/g, '_')}${armLoadout(a) !== null ? `.${armLoadout(a)!.replace(/[^A-Za-z0-9.-]/g, '_')}` : ''}`; + const keys = o.arms.map(keyOf); + const duplicates = [...new Set(o.arms.filter((_, i) => keys.indexOf(keys[i]!) !== i).map(a => o.arms.filter(b => keyOf(b) === keyOf(a)).join(' = ')))]; + if (duplicates.length) return { record, error: { code: 'EVAL_ARM_DUPLICATE', message: `arm ${duplicates.join(', ')} is listed more than once or shares one ledger attempt name; give each arm its own <model>#<loadout> identity` } }; fs.mkdirSync(o.out, { recursive: true }); const ledgerFile = path.join(o.out, PILOT_FILES.spend); - const record: PilotRecord = { schema_version: 1, n: o.n, arms: o.arms, cap_usd: o.cap, dry_run: o.dryRun, attempts: [], started_at: (o.now ?? new Date()).toISOString(), status: 'reserved' }; let error: { code: ResultCodeName; message: string } | undefined; outer: for (const model of o.arms) { - if (!hasPricing(model)) { error = { code: 'EVAL_PRICE_MISSING', message: `${model} has no row in lib/pricing.ts` }; break; } + if (!hasPricing(armModel(model))) { error = { code: 'EVAL_PRICE_MISSING', message: `${armModel(model)} has no row in lib/pricing.ts` }; break; } for (let i = 1; i <= o.n; i++) { - const estimate = estimateCostUsd(tokens, model); - const attempt = `pilot-${model.replace(/[^A-Za-z0-9.-]/g, '_')}-${i}`; + const estimate = estimateCostUsd(tokens, armModel(model)); + const attempt = `pilot-${keyOf(model)}-${i}`; try { reserve(ledgerFile, attempt, estimate, { cap: o.cap, label: `pilot ${model} #${i}`, now: o.now }); } catch (e) { if (e instanceof SpendCapExceeded) { record.status = 'budget_exhausted'; error = { code: 'EVAL_BUDGET_EXHAUSTED', message: e.message }; break outer; } diff --git a/lib/flaky-run.ts b/lib/flaky-run.ts new file mode 100644 index 000000000..4c2f0d65e --- /dev/null +++ b/lib/flaky-run.ts @@ -0,0 +1,368 @@ +/** + * gstack-flaky run (GSTA-158): run a test command, read its JUnit, rerun once + * when tests failed, and record flaky passes on a ledger. This is CREW rule 16 + * ("run the failed test again and note that it is flaky") for any runner that + * writes JUnit, the per-test form of the free suite's GSTACK_FREE_RETRY_FLAKY. + * + * Rules (docs/flaky.md): + * - A failed test gets one rerun when the runner made no retries of its own and + * at most maxReruns tests failed; more failures are a real red and nothing + * is rerun. When the runner retried (Playwright `retries` with retry-aware + * JUnit), its attempts stand and nothing is rerun again. + * - The rerun repeats the whole command by default, and counts only on the + * runner's own full green verdict: exit 0, no failed test, every first-run + * test present, nothing counted outside the report. Only that verdict covers + * every reason a run can fail (a test it never reached, a coverage + * threshold, a fail-on-flaky policy). + * - rerunScope 'test' (opt-in) reruns the failed test alone, and only with + * proof that the first run was complete: the runner's own summary, its + * test-failure exit status 1, no early stop, no coverage and no runner-flaky + * test; otherwise, and always for generic runners, the whole command reruns. + * - A rerun makes a flaky pass only when its command exited 0 and the test ran + * exactly once and passed on its first attempt. The command is an argv run + * without a shell; a targeted rerun appends the adapter's selection, each one + * argv element. + * - The JUnit file is removed before every attempt; a report that attempt did + * not write is missing. Missing, unreadable or empty JUnit is never green; a + * non-zero exit with no failed test and a runner killed by a signal are never + * green. When the runner's own summary counts failures or errors the JUnit + * omits (bun files that fail to load), the run is red and nothing is rerun. + * The first report is restored at the --junit path when the run ends; each + * rerun's report is kept beside it as <name>.rerun-<n>.xml. + */ +import * as fs from 'fs'; +import * as path from 'path'; +import { spawn as spawnChild, spawnSync } from 'child_process'; +import { finalAttempt, JunitError, readJunit, type JunitReport, type JunitRunner, type JunitTest } from './junit'; +import { redactFindingSpans } from './redact-engine'; +import { resolveStateRoot } from './state-root'; + +export type LedgerKind = 'flaky-pass' | 'rerun-fail'; + +export interface LedgerLine { + ts: string; + runner: 'junit'; + kind: LedgerKind; + id: string; + file: string; + /** 'runner' when the runner's own retries decided, 'gstack-flaky' for our rerun. */ + rerun_by: 'runner' | 'gstack-flaky'; + /** For gstack-flaky reruns: the failed test alone, or the whole command when the first run could not be proven complete. */ + rerun_scope?: 'test' | 'command'; + attempts: string[]; + tree?: string; + git_sha?: string; + branch?: string; + run_url?: string; + excerpt: string[]; +} + +export interface RunOptions { + command: string[]; + junit: string; + runner?: JunitRunner; + maxReruns: number; + /** 'command' (default): rerun the whole command. 'test': rerun the failed test alone when the first run is proven complete. */ + rerunScope?: 'command' | 'test'; + cwd?: string; + env?: NodeJS.ProcessEnv; + /** Runs one attempt; resolves with its exit status and the tail of its combined output. */ + spawn?: (argv: string[], env: NodeJS.ProcessEnv, cwd: string) => Promise<Attempted>; + now?: () => Date; +} + +export interface TestOutcome { + test: JunitTest; + status: 'flaky-pass' | 'fail' | 'rerun-fail'; + attempts: string[]; + reason?: string; +} + +export interface Attempted { exit: number; output: string; signal?: string } + +export interface RunResult { + exitCode: 0 | 1 | 3; + code?: string; + message?: string; + tests: number; + commandExit: number; + outcomes: TestOutcome[]; + reruns: number; + /** Why the rerun was the whole command (the default, or why a requested targeted rerun was refused); absent for targeted reruns. */ + wholeRerun?: string; + ledger: LedgerLine[]; +} + +export const DEFAULT_MAX_RERUNS = 3; + +const escapeRegex = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + +/** Selection arguments a targeted rerun appends for one test, each one argv element. */ +export function rerunSelection(runner: 'playwright' | 'bun', t: JunitTest): string[] { + if (runner === 'bun') return [t.file, '--test-name-pattern', `^${escapeRegex(t.path.join(' '))}$`]; + const args = [escapeRegex(t.file), '--grep', `(?:^|\\s)${escapeRegex(t.path.join(' '))}(?:\\s|$)`]; + return t.project ? [...args, '--project', t.project] : args; +} + +const OUTPUT_TAIL = 256 * 1024; + +/** Set once gstack-flaky itself is asked to stop; every attempt after that counts as killed, whatever exit the runner chose. */ +let cancelled: string | undefined; + +/** Stream the child's output live (a quiet runner trips CI idle watchdogs) and keep its tail for reconciliation. */ +function defaultSpawn(argv: string[], env: NodeJS.ProcessEnv, cwd: string): Promise<Attempted> { + return new Promise((resolve, reject) => { + let tail = ''; + const keep = (chunk: Buffer) => { tail = (tail + chunk.toString('utf-8')).slice(-OUTPUT_TAIL); }; + const child = spawnChild(argv[0]!, argv.slice(1), { stdio: ['inherit', 'pipe', 'pipe'], env, cwd }); + const forward = (s: NodeJS.Signals) => { cancelled ??= s; child.kill(s); }; + const release = () => { process.off('SIGINT', forward); process.off('SIGTERM', forward); }; + process.on('SIGINT', forward); + process.on('SIGTERM', forward); + child.stdout!.on('data', (c: Buffer) => { process.stdout.write(c); keep(c); }); + child.stderr!.on('data', (c: Buffer) => { process.stderr.write(c); keep(c); }); + child.on('error', e => { release(); reject(new JunitError('FLAKY_COMMAND_NOT_STARTED', `could not start ${argv[0]}: ${e.message}`)); }); + child.on('close', (code, signal) => { + release(); + const killed = signal ?? cancelled; + resolve({ exit: code ?? 128, output: tail, ...(killed ? { signal: killed } : {}) }); + }); + }); +} + +const plain = (output: string) => output.replace(/\x1b\[[0-9;]*m/g, ''); + +/** + * Failures a runner counted that its JUnit omits: bun leaves files that fail to + * load out of the report ("N errors"), Playwright reports errors outside any + * test. A rerun cannot clear those, so they keep the run red. + */ +export function unreportedFailures(runner: JunitRunner, output: string, reportedFailed: number): string | null { + const text = plain(output); + const last = (re: RegExp) => { const m = [...text.matchAll(re)].pop(); return m ? Number(m[1]) : 0; }; + if (runner === 'bun') { + const errors = last(/^\s*(\d+) errors?\s*$/gm); + const fails = last(/^\s*(\d+) fail\s*$/gm); + if (errors > 0) return `bun reported ${errors} error(s) outside the JUnit report (a file failed to load or threw between tests)`; + if (fails > reportedFailed) return `bun counted ${fails} failing tests, the JUnit report ${reportedFailed}`; + } + if (runner === 'playwright') { + const outside = last(/(\d+) errors? (?:was|were) not a part of any test/g); + const fails = last(/^\s*(\d+) failed\s*$/gm); + if (outside > 0) return `Playwright reported ${outside} error(s) outside any test`; + if (fails > reportedFailed) return `Playwright counted ${fails} failed tests, the JUnit report ${reportedFailed}`; + } + return null; +} + +/** + * Why the first run cannot be trusted to have run every test, so a rerun must + * repeat the whole command; null when the runner's own output proves it + * complete and its exit status explained by the failed tests alone. A run that + * stopped early reports only the tests it reached, and a targeted rerun would + * never run the rest; a coverage threshold or a fail-on-flaky policy fails the + * run for a reason a narrowed rerun cannot reproduce. + */ +export function incompleteRun(runner: JunitRunner, output: string, exit: number, command: string[] = [], runnerFlaky = 0): string | null { + if (runner === 'generic') return 'a generic runner cannot prove every test ran'; + if (exit !== 1) return `the command exited ${exit}, not the runner's test-failure status 1`; + const text = plain(output); + if (command.some(a => a === '--coverage' || a.startsWith('--coverage=')) || /%\s*Funcs\s*\|\s*%\s*Lines|coverage threshold/i.test(text)) return 'coverage is measured, and a narrowed rerun cannot reproduce a coverage failure'; + if (runnerFlaky) return `the runner retried ${runnerFlaky} flaky test(s), which a fail-on-flaky policy can fail the run on`; + if (runner === 'bun') { + if (/Bailed out after/.test(text)) return 'bun stopped early (--bail)'; + if (!/^\s*Ran \d+ tests? across \d+ files?/m.test(text)) return 'bun printed no run summary'; + return null; + } + const notRun = text.match(/\b(\d+) (did not run|interrupted)\b/); + if (notRun) return `Playwright reports ${notRun[1]} ${notRun[2]}`; + if (/Testing stopped early|Timed out waiting [\d.]+s for the (?:entire )?test run/.test(text)) return 'Playwright stopped early'; + if (!/^\s*\d+ failed\s*$/m.test(text)) return 'Playwright printed no summary (add a console reporter: --reporter=line,junit)'; + return null; +} + +function git(args: string[], cwd: string): string | undefined { + const r = spawnSync('git', args, { cwd, encoding: 'utf-8', timeout: 10_000 }); + return r.status === 0 ? r.stdout.trim() || undefined : undefined; +} + +export function provenance(cwd: string, env: NodeJS.ProcessEnv): Pick<LedgerLine, 'tree' | 'git_sha' | 'branch' | 'run_url'> { + const runUrl = env.GITHUB_SERVER_URL && env.GITHUB_REPOSITORY && env.GITHUB_RUN_ID + ? `${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}${env.GITHUB_RUN_ATTEMPT ? `/attempts/${env.GITHUB_RUN_ATTEMPT}` : ''}` + : undefined; + return { + tree: git(['rev-parse', 'HEAD^{tree}'], cwd), + git_sha: git(['rev-parse', 'HEAD'], cwd), + branch: env.GITHUB_HEAD_REF || env.GITHUB_REF_NAME || git(['rev-parse', '--abbrev-ref', 'HEAD'], cwd), + run_url: runUrl, + }; +} + +/** Test ids and paths leave this process only through the redaction engine: a parameterized title can carry a credential. */ +export function safeText(s: string): string { + return redactFindingSpans(s) ?? '[withheld: the redaction scan could not run]'; +} + +/** Escape a value for a GitHub workflow command, so a test title cannot end the command or start another. */ +export const workflowEscape = (s: string) => s.replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A'); + +/** Failure text leaves this process only through the redaction engine; when it cannot scan, nothing leaves. */ +export function safeExcerpt(lines: string[]): string[] { + if (!lines.length) return []; + const redacted = redactFindingSpans(lines.join('\n')); + return redacted === null ? ['[excerpt withheld: the redaction scan could not run]'] : redacted.split('\n'); +} + +/** One attempt: remove the report, run, and read a report this attempt wrote. */ +async function attempt(opts: RunOptions, argv: string[], cwd: string, env: NodeJS.ProcessEnv, runner = opts.runner): Promise<Attempted & { report?: JunitReport; error?: JunitError }> { + const junit = path.resolve(cwd, opts.junit); + fs.rmSync(junit, { force: true }); + const ran = await (opts.spawn ?? defaultSpawn)(argv, env, cwd); + if (!fs.existsSync(junit)) return { ...ran, error: new JunitError('FLAKY_NO_RESULTS', `the command wrote no JUnit report at ${opts.junit} (exit ${ran.exit})`) }; + try { return { ...ran, report: readJunit(junit, runner) }; } catch (e) { + if (e instanceof JunitError) return { ...ran, error: e }; + throw e; + } +} + +/** + * What keeps a rerun from clearing anything even when it exited 0: failures + * its runner counted outside the report and, for a whole-command rerun, any + * failed test or any first-run test missing from its report. + */ +function rerunProblem(again: Attempted & { report?: JunitReport }, first: JunitReport, whole: boolean): string | undefined { + if (!again.report) return undefined; + const failedNow = again.report.tests.filter(x => finalAttempt(x) === 'fail'); + const unreported = unreportedFailures(first.runner, again.output, failedNow.length); + if (unreported) return unreported; + if (!whole) return undefined; + if (failedNow.length) return `${failedNow.length} test(s) failed in the whole-command rerun`; + const seen = new Set(again.report.tests.map(x => x.id)); + const missing = first.tests.filter(x => !seen.has(x.id)).length; + return missing ? `${missing} test(s) of the first run are missing from the whole-command rerun` : undefined; +} + +/** A rerun clears a test only when its command exited 0 and the test ran once and passed on its first attempt. */ +function rerunVerdict(again: Attempted & { report?: JunitReport; error?: JunitError }, t: JunitTest, problem?: string): { status: 'flaky-pass' | 'rerun-fail'; attempts: string[]; reason?: string } { + const match = again.report?.tests.find(x => x.id === t.id); + if (!match) return { status: 'rerun-fail', attempts: [...t.attempts, 'fail'], reason: again.error ? `rerun: ${again.error.message}` : 'rerun: the report does not contain this test' }; + const attempts = [...t.attempts, ...match.attempts]; + if (again.signal) return { status: 'rerun-fail', attempts, reason: `rerun: killed by ${again.signal}` }; + if (again.exit !== 0) return { status: 'rerun-fail', attempts, reason: `rerun: the command exited ${again.exit}` }; + if (problem) return { status: 'rerun-fail', attempts, reason: `rerun: ${problem}` }; + if (match.executions !== 1 || match.attempts.join() !== 'pass') return { status: 'rerun-fail', attempts, reason: `rerun: the test did not pass once on its first attempt (${match.attempts.join(',')})` }; + return { status: 'flaky-pass', attempts }; +} + +export async function runFlaky(opts: RunOptions): Promise<RunResult> { + const cwd = opts.cwd ?? process.cwd(); + const env = { ...(opts.env ?? process.env), PLAYWRIGHT_JUNIT_INCLUDE_RETRIES: '1' }; + const now = opts.now ?? (() => new Date()); + const empty = { tests: 0, outcomes: [], reruns: 0, ledger: [] }; + const first = await attempt(opts, opts.command, cwd, env); + if (first.signal) return { exitCode: 3, code: 'FLAKY_COMMAND_KILLED', message: `the command was killed by ${first.signal}`, commandExit: first.exit, ...empty }; + if (!first.report) return { exitCode: 3, code: first.error!.code, message: first.error!.message, commandExit: first.exit, ...empty }; + const report = first.report; + if (report.retriesHidden) { + return { exitCode: 3, code: 'FLAKY_RETRIES_HIDDEN', commandExit: first.exit, ...empty, tests: report.tests.length, + message: 'Playwright retried but its JUnit omits the retries; set PLAYWRIGHT_JUNIT_INCLUDE_RETRIES=1 (gstack-flaky run sets it for the command it starts)' }; + } + const failed = report.tests.filter(t => finalAttempt(t) === 'fail'); + const runnerFlaky = report.tests.filter(t => finalAttempt(t) !== 'fail' && t.attempts.includes('fail')); + if (first.exit !== 0 && failed.length === 0) { + return { exitCode: 3, code: 'FLAKY_COMMAND_FAILED', commandExit: first.exit, ...empty, tests: report.tests.length, + message: `the command exited ${first.exit} but its JUnit reports no failed test; a crash or a non-test step failed` }; + } + const unreported = unreportedFailures(report.runner, first.output, failed.length); + if (unreported) { + return { exitCode: 1, code: 'FLAKY_UNREPORTED_FAILURES', commandExit: first.exit, ...empty, tests: report.tests.length, + outcomes: failed.map(t => ({ test: t, status: 'fail' as const, attempts: t.attempts, reason: 'not rerun: the runner reported failures outside the JUnit report' })), + message: `${unreported}; nothing was rerun` }; + } + const prov = provenance(cwd, env); + const line = (t: JunitTest, kind: LedgerKind, by: LedgerLine['rerun_by'], attempts: string[], scope?: LedgerLine['rerun_scope']): LedgerLine => + ({ ts: now().toISOString(), runner: 'junit', kind, id: safeText(t.id), file: safeText(t.file), rerun_by: by, ...(scope ? { rerun_scope: scope } : {}), attempts, ...prov, excerpt: safeExcerpt(t.excerpt) }); + const ledger: LedgerLine[] = runnerFlaky.map(t => line(t, 'flaky-pass', 'runner', t.attempts)); + const outcomes: TestOutcome[] = runnerFlaky.map(t => ({ test: t, status: 'flaky-pass' as const, attempts: t.attempts })); + const eligible: JunitTest[] = []; + for (const t of failed) { + if (t.executions > 1) outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: `it ran ${t.executions} times in this report and one failed; not rerun` }); + else if (t.attempts.length > 1) outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: `the runner already retried (${t.attempts.length} attempts)` }); + else if (failed.length > opts.maxReruns) outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: `${failed.length} tests failed, more than --max-reruns ${opts.maxReruns}; not rerun` }); + else eligible.push(t); + } + const wholeRerun = !eligible.length ? undefined + : opts.rerunScope !== 'test' ? 'the default; --rerun-scope test reruns the failed test alone' + : incompleteRun(report.runner, first.output, first.exit, opts.command, runnerFlaky.length) ?? undefined; + const junit = path.resolve(cwd, opts.junit); + const firstXml = fs.readFileSync(junit); + let reruns = 0; + const rerun = async (argv: string[]) => { + reruns++; + const again = await attempt(opts, argv, cwd, env, report.runner); + if (fs.existsSync(junit)) fs.renameSync(junit, `${junit.replace(/\.xml$/i, '')}.rerun-${reruns}.xml`); + return again; + }; + const record = (t: JunitTest, v: ReturnType<typeof rerunVerdict>, scope: 'test' | 'command') => { + outcomes.push({ test: t, status: v.status, attempts: v.attempts, ...(v.reason ? { reason: v.reason } : {}) }); + ledger.push(line(t, v.status, 'gstack-flaky', v.attempts, scope)); + }; + try { + if (wholeRerun && outcomes.some(o => o.status === 'fail')) { + for (const t of eligible) outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: `not rerun: the run is already red, and a rerun would repeat the whole command (${wholeRerun})` }); + } else if (wholeRerun) { + const again = await rerun(opts.command); + const problem = rerunProblem(again, report, true); + for (const t of eligible) record(t, rerunVerdict(again, t, problem), 'command'); + } else if (eligible.length) { + let interrupted = false; + for (const t of eligible) { + if (interrupted) { outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: 'not rerun: an earlier rerun was killed' }); continue; } + const again = await rerun([...opts.command, ...rerunSelection(report.runner as 'playwright' | 'bun', t)]); + interrupted = !!again.signal; + record(t, rerunVerdict(again, t, rerunProblem(again, report, false)), 'test'); + } + } + } finally { + if (reruns) fs.writeFileSync(junit, firstXml); + } + const red = outcomes.some(o => o.status !== 'flaky-pass'); + return { exitCode: red ? 1 : 0, tests: report.tests.length, commandExit: first.exit, outcomes, reruns, ...(wholeRerun && reruns ? { wholeRerun } : {}), ledger }; +} + +export function ledgerPath(flag: string | undefined, env: NodeJS.ProcessEnv = process.env): string { + if (flag) return flag; + if (env.GSTACK_FLAKE_LEDGER_JUNIT) return env.GSTACK_FLAKE_LEDGER_JUNIT; + return path.join(resolveStateRoot(env), 'flake-ledger-junit.jsonl'); +} + +/** Append ledger lines; an I/O error is a warning and never changes the verdict. */ +export function appendLedger(lines: LedgerLine[], file: string, warn: (s: string) => void): void { + if (!lines.length) return; + try { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.appendFileSync(file, lines.map(l => JSON.stringify(l)).join('\n') + '\n'); + } catch (e) { + warn(`FLAKY_LEDGER_WRITE_FAILED: could not append ${file} (${(e as Error).message}); the verdict is unaffected`); + } +} + +/** Lines the bin prints: one per non-passing test, then the result line. */ +export function renderResult(r: RunResult, ledger: string): string[] { + const out: string[] = []; + for (const o of r.outcomes) out.push(`FLAKY: ${o.status} ${safeText(o.test.id)} attempts=${o.attempts.join(',')}${o.reason ? ` (${o.reason})` : ''}`); + const flaky = r.outcomes.filter(o => o.status === 'flaky-pass').length; + const failed = r.outcomes.length - flaky; + if (r.wholeRerun) out.push(`FLAKY_RERUN: scope=command (${r.wholeRerun})`); + out.push(`FLAKY_RESULT: tests=${r.tests} failed=${failed} flaky=${flaky} reruns=${r.reruns} command_exit=${r.commandExit} verdict=${r.exitCode === 0 ? 'pass' : 'fail'} ledger=${r.ledger.length ? ledger : 'none'}`); + return out; +} + +/** A GitHub step summary for the flaky passes, so they are seen before any report reads the ledger. */ +export function stepSummary(r: RunResult): string { + const rows = r.outcomes.filter(o => o.status === 'flaky-pass'); + if (!rows.length) return ''; + const cell = (s: string) => s.replace(/\|/g, '\\|').replace(/`/g, "'").replace(/[\r\n]+/g, ' '); + return ['### Flaky passes (gstack-flaky)', '', '| Test | Attempts | First error |', '|---|---|---|', + ...rows.map(o => `| \`${cell(safeText(o.test.id))}\` | ${o.attempts.join(', ')} | ${cell(safeExcerpt(o.test.excerpt)[0] ?? '')} |`), ''].join('\n'); +} diff --git a/lib/gstack-post.ts b/lib/gstack-post.ts index dca9b81f5..1860170ac 100644 --- a/lib/gstack-post.ts +++ b/lib/gstack-post.ts @@ -9,8 +9,9 @@ * `pr-body` merges by `## ` section into the PR's current body (read through * the same runner): a section in the file replaces the same-titled one, the * ship-receipt block is a section of its own, sections the file does not - * rewrite are kept, new ones are appended, and a file with a `## ` section - * replaces the text above the first heading; `--replace` overwrites instead. + * rewrite are kept, new ones are appended, and a file with a receipt above + * its first `## ` section replaces the text above the first heading; + * `--replace` overwrites instead. * The scan runs on the merged bytes, which are the bytes sent. * * Exit codes: 0 posted; 1 HIGH finding (or oversize input) refused; @@ -212,25 +213,28 @@ function splitSections(body: string): Section[] { * A file section whose title the current body holds more than once (an old * description pasted under the new one) matches no single section, so the * merge refuses instead of replacing one copy and keeping the stale other. - * A file with at least one `## ` section manages the preamble (GST-412): every + * A full post (a file with a receipt block above its first `## ` section, as + * /ship composes it) manages the preamble (GST-412): every * non-blank text section before the current body's first heading (the leading * text, text after a receipt block) is dropped, the file's own text before its * first heading takes its place (leading text first, text after the file's * receipt right after the receipt), a dropped piece the file brings no text for * is listed in `dropped` with its first line (withheld when the redaction scan * flags it, since the merged bytes no longer hold it), and the receipt keeps the rules - * above. A file without a `## ` section merges as described above. + * above. Any other file (`## ` sections with no receipt above them, or no + * `## ` section) merges as described above, so the current intro is kept. */ export function mergeBodySections(current: string, incoming: string): { body: string; replaced: string[]; kept: string[]; added: string[]; dropped: string[] } { const now = splitSections(current); const next = splitSections(incoming).filter(n => !(n.key.startsWith("text:") && n.text.trim() === "")); const ambiguous = next.filter(n => now.filter(s => s.key === n.key).length > 1).map(n => `"${n.label}"`); if (ambiguous.length) throw new UsageError(`the PR's current body holds ${[...new Set(ambiguous)].join(", ")} more than once, so the file's copy matches no single section; nothing was posted. Rerun with --replace and the whole body`); - const managed = next.some(n => n.key.startsWith("h2:")); const preamble = (list: Section[]) => { const end = list.findIndex(s => s.key.startsWith("h2:")); return list.slice(0, end < 0 ? list.length : end).filter(s => s.key.startsWith("text:") && s.text.trim() !== ""); }; + const firstHeading = next.findIndex(n => n.key.startsWith("h2:")); + const managed = firstHeading > 0 && next.slice(0, firstHeading).some(n => n.key === "receipt"); const stale = managed ? preamble(now) : []; const ours = managed ? preamble(next) : []; const role = (s: Section) => (s.key === "text:lead" ? "lead" : "trailing"); diff --git a/lib/junit.ts b/lib/junit.ts new file mode 100644 index 000000000..0f4d504d3 --- /dev/null +++ b/lib/junit.ts @@ -0,0 +1,204 @@ +/** + * JUnit XML reader for gstack-flaky (GSTA-158). One reader for every runner + * that writes JUnit: Playwright (`--reporter=junit`, retry-aware with + * PLAYWRIGHT_JUNIT_INCLUDE_RETRIES=1), `bun test --reporter=junit`, jest, + * pytest and vitest. No XML dependency: the subset JUnit uses (elements, + * attributes, text, CDATA, comments, processing instructions) is tokenized + * here. + * + * A test's id is `<file>::<project>::<title>`. The title is the describe path + * plus the test name joined by " > ", so Playwright's " › " and bun's nested + * testsuites give the same shape. The project is Playwright's project name + * (its testsuite `hostname`) and empty for every other runner, so a test + * that runs in two browser projects is two ids. + * + * Attempts: Playwright's retry-aware report lists each failed retry as + * `rerunFailure` (final result failed) or `flakyFailure` (final result + * passed). A Playwright failure whose text shows "Retry #" without those + * elements means retries ran but were not reported (`retriesHidden`). A test + * that appears more than once (repeat-each, a duplicate name) fails when any + * of its executions failed. + */ +import * as fs from 'fs'; + +export type JunitRunner = 'playwright' | 'bun' | 'generic'; +export type Attempt = 'pass' | 'fail' | 'skip'; + +export interface JunitTest { + id: string; + file: string; + project: string; + /** Describe path plus test name. */ + path: string[]; + attempts: Attempt[]; + /** Testcase elements with this id in the report (more than one: repeat-each or a duplicate name). */ + executions: number; + /** First lines of the final failure, or of the first failed attempt. */ + excerpt: string[]; +} + +export interface JunitReport { + runner: JunitRunner; + tests: JunitTest[]; + retriesHidden: boolean; +} + +export class JunitError extends Error { + constructor(readonly code: string, message: string) { super(message); } +} + +interface XmlNode { name: string; attrs: Record<string, string>; children: XmlNode[]; text: string; parent: XmlNode | null } + +const ENTITIES: Record<string, string> = { lt: '<', gt: '>', amp: '&', quot: '"', apos: "'" }; +function decode(s: string): string { + return s.replace(/&(#x[0-9a-fA-F]+|#\d+|[a-z]+);/g, (m, e: string) => { + if (e[0] === '#') { + const code = e[1] === 'x' ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10); + return Number.isFinite(code) && code <= 0x10ffff ? String.fromCodePoint(code) : m; + } + return ENTITIES[e] ?? m; + }); +} + +const TAG = /^<(\/?)([A-Za-z_][\w:.-]*)((?:\s+[^\s=/>]+\s*=\s*(?:"[^"]*"|'[^']*'))*)\s*(\/?)>/; +const ATTR = /([^\s=/>]+)\s*=\s*(?:"([^"]*)"|'([^']*)')/g; + +export function parseXml(xml: string): XmlNode { + const root: XmlNode = { name: '#root', attrs: {}, children: [], text: '', parent: null }; + let cur = root; + let i = 0; + while (i < xml.length) { + const lt = xml.indexOf('<', i); + if (lt < 0) { cur.text += decode(xml.slice(i)); break; } + if (lt > i) cur.text += decode(xml.slice(i, lt)); + const rest = xml.slice(lt, lt + 9); + if (rest.startsWith('<![CDATA[')) { + const end = xml.indexOf(']]>', lt); + if (end < 0) throw new JunitError('FLAKY_JUNIT_INVALID', 'unterminated CDATA section'); + cur.text += xml.slice(lt + 9, end); + i = end + 3; + continue; + } + if (rest.startsWith('<!--')) { + const end = xml.indexOf('-->', lt); + if (end < 0) throw new JunitError('FLAKY_JUNIT_INVALID', 'unterminated comment'); + i = end + 3; + continue; + } + if (rest.startsWith('<?') || rest.startsWith('<!')) { + const end = xml.indexOf('>', lt); + if (end < 0) throw new JunitError('FLAKY_JUNIT_INVALID', 'unterminated declaration'); + i = end + 1; + continue; + } + const m = TAG.exec(xml.slice(lt)); + if (!m) throw new JunitError('FLAKY_JUNIT_INVALID', `malformed tag at offset ${lt}`); + const [whole, close, name, attrText, selfClose] = m; + if (close) { + if (cur.name !== name) throw new JunitError('FLAKY_JUNIT_INVALID', `</${name}> closes <${cur.name}> at offset ${lt}`); + cur = cur.parent!; + } else { + const attrs: Record<string, string> = {}; + for (const a of attrText!.matchAll(ATTR)) attrs[a[1]!] = decode(a[2] ?? a[3] ?? ''); + const node: XmlNode = { name: name!, attrs, children: [], text: '', parent: cur }; + cur.children.push(node); + if (!selfClose) cur = node; + } + i = lt + whole!.length; + } + if (cur !== root) throw new JunitError('FLAKY_JUNIT_INVALID', `<${cur.name}> is never closed`); + return root; +} + +function allNodes(node: XmlNode, name: string, out: XmlNode[] = []): XmlNode[] { + for (const c of node.children) { + if (c.name === name) out.push(c); + allNodes(c, name, out); + } + return out; +} + +export function detectRunner(root: XmlNode): JunitRunner { + const top = root.children.find(c => c.name === 'testsuites'); + if (top?.attrs.name === 'bun test') return 'bun'; + const cases = allNodes(root, 'testcase'); + if (cases.some(c => (c.attrs.name ?? '').includes(' › '))) return 'playwright'; + const suites = allNodes(root, 'testsuite'); + if (suites.length && suites.every(s => 'timestamp' in s.attrs && 'hostname' in s.attrs && !('file' in s.attrs)) + && cases.every(c => c.parent?.name === 'testsuite' && c.attrs.classname === c.parent.attrs.name)) return 'playwright'; + return 'generic'; +} + +function excerptOf(node: XmlNode | undefined): string[] { + if (!node) return []; + const body = (node.attrs.message ? node.attrs.message + '\n' : '') + node.text; + return body.replace(/\x1b\[[0-9;]*m/g, '').split('\n').map(l => l.trimEnd()).filter(l => l.trim()).slice(0, 8); +} + +export const testId = (file: string, project: string, path: string[]) => `${file}::${project}::${path.join(' > ')}`; + +function readCase(tc: XmlNode, runner: JunitRunner): Omit<JunitTest, 'id' | 'executions'> & { hidden: boolean } { + const suite = tc.parent!; + let file: string, project = '', path: string[]; + if (runner === 'playwright') { + file = tc.attrs.classname || suite.attrs.name || ''; + project = suite.attrs.hostname ?? ''; + path = (tc.attrs.name ?? '').split(' › '); + } else if (runner === 'bun') { + file = tc.attrs.file ?? ''; + path = []; + for (let s: XmlNode | null = suite; s && s.name === 'testsuite'; s = s.parent) { + if ('file' in s.attrs && s.attrs.name === s.attrs.file && s.parent?.name !== 'testsuite') break; + path.unshift(s.attrs.name ?? ''); + } + path.push(tc.attrs.name ?? ''); + } else { + file = tc.attrs.file || tc.attrs.classname || suite.attrs.file || suite.attrs.name || ''; + path = [tc.attrs.name ?? '']; + } + const kids = (n: string) => tc.children.filter(c => c.name === n); + const final = kids('failure')[0] ?? kids('error')[0]; + const skipped = kids('skipped').length > 0; + const flaky = [...kids('flakyFailure'), ...kids('flakyError')]; + const rerun = [...kids('rerunFailure'), ...kids('rerunError')]; + let attempts: Attempt[]; + if (final) attempts = [...rerun.map(() => 'fail' as const), 'fail']; + else if (skipped) attempts = ['skip']; + else attempts = [...flaky.map(() => 'fail' as const), 'pass']; + const hidden = runner === 'playwright' && !!final && rerun.length === 0 && /\bRetry #\d+/.test(final.text); + return { file, project, path, attempts, excerpt: excerptOf(final ?? flaky[0]), hidden }; +} + +export function parseJunit(xml: string, runner?: JunitRunner): JunitReport { + const root = parseXml(xml); + const kind = runner ?? detectRunner(root); + const byId = new Map<string, JunitTest>(); + let retriesHidden = false; + for (const tc of allNodes(root, 'testcase')) { + const { hidden, ...t } = readCase(tc, kind); + retriesHidden ||= hidden; + const id = testId(t.file, t.project, t.path); + const seen = byId.get(id); + if (seen) { + const failed = finalAttempt(seen) === 'fail' || t.attempts.at(-1) === 'fail'; + seen.attempts = [...seen.attempts, ...t.attempts].sort((a, b) => failed ? Number(a === 'fail') - Number(b === 'fail') : 0); + seen.executions++; + if (!seen.excerpt.length) seen.excerpt = t.excerpt; + } else byId.set(id, { id, ...t, executions: 1 }); + } + if (byId.size === 0) throw new JunitError('FLAKY_NO_RESULTS', 'the JUnit report holds no testcase'); + return { runner: kind, tests: [...byId.values()], retriesHidden }; +} + +/** Read and parse one JUnit file; missing, unreadable and empty files are errors, never green. */ +export function readJunit(file: string, runner?: JunitRunner): JunitReport { + let xml: string; + try { xml = fs.readFileSync(file, 'utf-8'); } catch (e) { + throw new JunitError('FLAKY_NO_RESULTS', `cannot read ${file}: ${(e as Error).message}`); + } + if (!xml.trim()) throw new JunitError('FLAKY_NO_RESULTS', `${file} is empty`); + return parseJunit(xml, runner); +} + +/** The final outcome of a test: its last attempt. */ +export const finalAttempt = (t: JunitTest): Attempt => t.attempts[t.attempts.length - 1]!; diff --git a/lib/pregate/checks/lanes.ts b/lib/pregate/checks/lanes.ts index 759b3588a..f1206404c 100644 --- a/lib/pregate/checks/lanes.ts +++ b/lib/pregate/checks/lanes.ts @@ -406,7 +406,9 @@ export function remoteReportProblem(report: RemoteReport, tree: string): string * must print `PREGATE_REMOTE: tree=<tree> tests=<n> failed=<n> [spent=<usd>]`; * the obligation clears only when that tree is the tested tree, tests > 0, * failed = 0 and the gate exited 0. Anything else leaves it uncleared and - * the check incomplete. + * the check incomplete. The gate is bounded by that timeout only: a gate + * that waits on CI is quiet by design, so the stall timer does not apply, + * and its heartbeat names the gate's last output line. */ async function runRemote(ctx: PregateContext, sel: Selection, base: CheckResult): Promise<CheckResult> { const lines = base.lines ?? []; @@ -418,7 +420,7 @@ async function runRemote(ctx: PregateContext, sel: Selection, base: CheckResult) const filesFile = path.join(dir, 'files.txt'); fs.writeFileSync(log, ''); fs.writeFileSync(filesFile, sel.localTests.join('\n') + '\n'); - const r = await supervise({ command: obligation.command!, cwd: ctx.repoRoot, env: { ...ctx.env, PREGATE_TREE: ctx.wtree, PREGATE_FILES_FILE: filesFile, PREGATE_SPEND_CAP_USD: String(cap), PREGATE_TIMEOUT_S: String(Math.floor(ctx.timeoutMs / 1000)) }, log, deadline: Date.now() + ctx.timeoutMs, stallMs: ctx.stallTimeoutMs, label: `remote gate ${obligation.lane}`, progress: ctx.progress }); + const r = await supervise({ command: obligation.command!, cwd: ctx.repoRoot, env: { ...ctx.env, PREGATE_TREE: ctx.wtree, PREGATE_FILES_FILE: filesFile, PREGATE_SPEND_CAP_USD: String(cap), PREGATE_TIMEOUT_S: String(Math.floor(ctx.timeoutMs / 1000)) }, log, deadline: Date.now() + ctx.timeoutMs, stallMs: 0, lastLine: true, label: `remote gate ${obligation.lane}`, progress: ctx.progress }); const output = readRange(log, r.start, r.end); const reported = parseRemoteReport(output); const spent = reported?.spent; @@ -426,16 +428,17 @@ async function runRemote(ctx: PregateContext, sel: Selection, base: CheckResult) ctx.progress(`PREGATE_REMOTE_SPEND: ${spend.slice('spend: '.length)}`); const runLines = [...lines, `remote gate: ${obligation.command}`, `log: ${log}`, spend, ...output.trim().split('\n').slice(-15).map(l => ` ${l}`)]; const inputs = { ...base.inputs, command: sha256(obligation.command!) }; - const why = r.outcome !== 'exited' ? `the remote gate did not finish (${r.outcome === 'stalled' ? `no output for ${ctx.stallTimeoutMs / 1000} s` : r.outcome})` + const why = r.outcome === 'timeout' ? `the remote gate did not finish (timed out after ${ctx.timeoutMs / 1000} s, ${ctx.timeoutSource})` + : r.outcome !== 'exited' ? `the remote gate did not finish (${r.outcome})` : !reported ? `the remote gate exited ${r.exit} without a PREGATE_REMOTE: line` : remoteReportProblem(reported, ctx.wtree) ?? (r.exit !== 0 ? `the remote gate exited ${r.exit}` : null); // A gate that printed no report and names a quota, network or missing-tool error in its preamble never ran the tests: the environment, not the diff. // Only the gate's first lines are read (a VM refusal or a network error comes before any test streams); exit 127 counts only when the gate's own binary is off PATH. - const environment = why && !reported && r.outcome !== 'stalled' && r.outcome !== 'timeout' + const environment = why && !reported && r.outcome !== 'timeout' ? r.outcome === 'spawn-failed' ? { ...(classifyEnvironment(r.error ?? '', null) ?? launchCause('missing-binary', r.error ?? 'the remote gate could not be started')), proof: 'launch' as const } : classifyCommandEnvironment(obligation.command!, output.split('\n').slice(0, REMOTE_PREAMBLE_LINES).join('\n'), r.exit, ctx.env) : null; if (environment) return environmentResult({ ...base, inputs, lines: runLines, remote: [...sel.remote, obligation] }, environment, `${why}; requires-remote ${obligation.lane} stays uncleared`); - if (why) return { ...base, inputs, lines: runLines, status: 'incomplete', remote: [...sel.remote, obligation], code: r.outcome === 'stalled' ? 'PREGATE_STALLED' : 'PREGATE_INCOMPLETE', detail: `${why}; requires-remote ${obligation.lane} stays uncleared`, fix: `read ${log}; the obligation clears only on PREGATE_REMOTE: tree=${ctx.wtree.slice(0, 12)}… tests>0 failed=0 with exit 0` }; + if (why) return { ...base, inputs, lines: runLines, status: 'incomplete', remote: [...sel.remote, obligation], code: 'PREGATE_INCOMPLETE', detail: `${why}; requires-remote ${obligation.lane} stays uncleared`, fix: `read ${log}; the obligation clears only on PREGATE_REMOTE: tree=${ctx.wtree.slice(0, 12)}… tests>0 failed=0 with exit 0` }; const cleared = { ...obligation, receipt: `remote-gate tree=${ctx.wtree.slice(0, 12)} tests=${reported!.tests}`, cleared_by: `gstack-pregate --remote ${new Date().toISOString()}` }; return { ...base, inputs, lines: runLines, status: 'requires-remote', remote: [...sel.remote, cleared], detail: `remote gate ${obligation.lane}: ${reported!.tests} tests, 0 failed on tree ${ctx.wtree.slice(0, 12)}; ${spend}` }; } diff --git a/lib/pregate/supervise.ts b/lib/pregate/supervise.ts index 17639f97f..a7ff3ed88 100644 --- a/lib/pregate/supervise.ts +++ b/lib/pregate/supervise.ts @@ -11,7 +11,7 @@ * headers) reports `PREGATE_TEST: <n>/<total> <file> start|pass|fail <s>` * per file; a run of one file reports it from the launch itself for any * runner. Without either, only heartbeats: a line after each `heartbeatMs` - * with no progress line. + * with no progress line; with `lastLine` it names the last line the command printed. * * Platforms: POSIX signals the negative pid (the group `detached` creates * with setsid). Windows has no process groups here: the kill is @@ -43,6 +43,8 @@ export interface SuperviseOptions { files?: readonly string[]; adapter?: ProgressAdapter | null; counter?: Counter; + /** Heartbeats name the command's last output line (the remote gate, which has no per-file progress). */ + lastLine?: boolean; progress: (line: string) => void; } @@ -183,7 +185,13 @@ export function supervise(o: SuperviseOptions): Promise<Supervised> { if (outcome) return; if (now >= o.deadline) { stop('timeout'); return; } if (o.stallMs > 0 && now - lastGrowth >= o.stallMs) { stop('stalled'); return; } - if (now - lastProgress >= heartbeatMs) emit(`PREGATE_HEARTBEAT: ${o.label} running ${Math.round((now - started) / 1000)}s; last file ${lastFile ?? 'none reported'}; log ${size - start} bytes, last growth ${Math.round((now - lastGrowth) / 1000)}s ago (${o.log})`); + if (now - lastProgress < heartbeatMs) return; + let last: string | undefined; + if (o.lastLine) { + try { last = readRange(o.log, start, size, 4096).replace(/\x1b\[[0-9;?]*[A-Za-z]/g, '').split(/\r\n?|\n/).map(l => l.replace(/[\x00-\x1f\x7f]/g, '').trim()).filter(Boolean).at(-1); } + catch { /* log removed */ } + } + emit(`PREGATE_HEARTBEAT: ${o.label} running ${Math.round((now - started) / 1000)}s; last file ${lastFile ?? 'none reported'}; log ${size - start} bytes, last growth ${Math.round((now - lastGrowth) / 1000)}s ago (${o.log})${last ? `; last output: ${last.slice(0, 160)}` : ''}`); }, pollMs); if (child.pid) ACTIVE.add(child.pid); child.on('error', e => finish(null, null, e.message)); diff --git a/lib/result-codes-eval.ts b/lib/result-codes-eval.ts new file mode 100644 index 000000000..70102382b --- /dev/null +++ b/lib/result-codes-eval.ts @@ -0,0 +1,33 @@ +/** + * Result codes for the /eval-plan preregistration (bin/gstack-eval-plan prereg + * --check, the priced pilot's arm list). Kept beside lib/result-codes.ts, which + * spreads this table into RESULT_CODES; the same rules apply: one stable + * docs/troubleshooting.md anchor, a summary and a fix clause each. The + * no-gstack arm rule is documented in docs/TESTING_INTERNALS.md#no-gstack-arm. + */ +import type { ResultCode } from './result-codes'; + +export const EVAL_RESULT_CODES = { + PREREG_INCOMPLETE: { + anchor: 'prereg-incomplete', + summary: 'the preregistration file lacks a required section, a section is empty, or the Comparison or Diagnostics declaration is malformed', + fix: 'fill every section `gstack-eval-plan prereg --check` names (bars, arms, comparison, decoys, stop rules, held-out exposure) before any priced call', + }, + // Field list 2 (2026-10-16), W8: preregistered diagnostics run before the freeze commit. + PREREG_DIAGNOSTIC_UNTESTED: { + anchor: 'prereg-diagnostic-untested', + summary: 'the declared summarizer, run on the declared synthetic fixture, did not report every preregistered diagnostic at its expected count', + fix: 'fix the summarizer or the declared definition (expected count, fixture) until every PREREG_DIAGNOSTIC line says ok, then commit the freeze; never freeze a diagnostic the summarizer has not counted', + }, + // Garry, 2026-10-31: "all of our head to head tests have to test no gstack from here". + PREREG_NO_GSTACK_ARM: { + anchor: 'prereg-no-gstack-arm', + summary: 'the Comparison says stack: loaded and the No-gstack arm section is missing, lacks a key, or does not give every model its own <model>#no-gstack Arms line', + fix: 'add `## No-gstack arm` with arms:, inputs:, prompt:, panel: and report:, and list each <model>#no-gstack arm under Arms; when no arm loads a skill stack, say stack: none with a why: line', + }, + EVAL_ARM_DUPLICATE: { + anchor: 'eval-arm-duplicate', + summary: 'the pilot was given the same arm identity twice, so two arms would share one set of attempts', + fix: 'give each arm its own identity, <model> or <model>#<loadout> (for example claude-opus-5-5#no-gstack beside claude-opus-5-5); nothing was reserved', + }, +} as const satisfies Record<string, ResultCode>; diff --git a/lib/result-codes.ts b/lib/result-codes.ts index 082924de1..beaeb779b 100644 --- a/lib/result-codes.ts +++ b/lib/result-codes.ts @@ -13,6 +13,7 @@ import { RELEASE_RESULT_CODES } from './result-codes-release'; import { CAPY_RESULT_CODES } from './result-codes-capy'; import { BUDGET_RESULT_CODES } from './result-codes-budget'; +import { EVAL_RESULT_CODES } from './result-codes-eval'; export interface ResultCode { /** Stable `<a id>` in docs/troubleshooting.md. */ @@ -435,11 +436,6 @@ export const RESULT_CODES = { }, // PR E (multi-agent wave): gstack-eval-plan, gstack-lane-check, gstack-evidence verify, // gstack-owner-brief, gstack-contributor-mode, gstack-banned-terms. - PREREG_INCOMPLETE: { - anchor: 'prereg-incomplete', - summary: 'the preregistration file lacks a required section or a section is empty', - fix: 'fill every section `gstack-eval-plan prereg --check` names (bars, arms, decoys, stop rules, held-out exposure) before any priced call', - }, EVAL_PRICE_MISSING: { anchor: 'eval-price-missing', summary: 'an arm names a model with no row in the price table, so its admission cannot be estimated', @@ -671,12 +667,6 @@ export const RESULT_CODES = { summary: '`gstack-autoplan export --public <dir>` writes only into an empty directory or over an earlier public copy of the same run', fix: 'pass an empty or new --public directory, or the directory that already holds this run\'s public copy', }, - // Field list 2 (2026-10-16), W8: preregistered diagnostics run before the freeze commit. - PREREG_DIAGNOSTIC_UNTESTED: { - anchor: 'prereg-diagnostic-untested', - summary: 'the declared summarizer, run on the declared synthetic fixture, did not report every preregistered diagnostic at its expected count', - fix: 'fix the summarizer or the declared definition (expected count, fixture) until every PREREG_DIAGNOSTIC line says ok, then commit the freeze; never freeze a diagnostic the summarizer has not counted', - }, // Field list 2 (2026-10-16), W9: .gstack/eval-models.json. EVAL_MODELS_INVALID: { anchor: 'eval-models-invalid', @@ -772,6 +762,7 @@ export const RESULT_CODES = { ...RELEASE_RESULT_CODES, ...CAPY_RESULT_CODES, ...BUDGET_RESULT_CODES, + ...EVAL_RESULT_CODES, } as const satisfies Record<string, ResultCode>; export type ResultCodeName = keyof typeof RESULT_CODES; diff --git a/lib/review-ledger.ts b/lib/review-ledger.ts index 0c9a96e9f..0d945d0c4 100644 --- a/lib/review-ledger.ts +++ b/lib/review-ledger.ts @@ -4,7 +4,7 @@ * * rendered a brief gstack-review-dispatch render wrote (role, name, file, sha256) * sent the prompt a dispatch program actually passed, by hash; a mismatch fails the pass - * output a reader's raw output (sha256) and the merged.jsonl ids collect wrote from it + * output a reader's raw output (sha256), the merged.jsonl ids collect wrote from it, and whether it was a review at all * judgment each lead-judgment change: an added or promoted finding, with its reason * judge the verdict on the parent's judged set (ok, or the dropped and downgraded ids) * @@ -72,11 +72,12 @@ export interface MergedLine { id: string; reader: string; label: string; raw: st * of the two labels). A `Severity` or `Blocks` field line continues the open finding, once per field: `Severity` raises its label, * `Blocks: yes` makes it CRITICAL, `Blocks: no` never lowers it; with no finding open it is one, carrying the paragraph's subject. * A list item, a heading, a bold or post-blank paragraph at the margin, or a line naming a file:line after the open finding's fields - * starts the next finding. An inline `Blocks: yes` is CRITICAL too. The native voice's FIXABLE collects as CRITICAL; other labels - * stay as written for lead judgment. When in doubt the parser merges rather than drops: a lost blocker costs the gate. + * starts the next finding, except an unlabeled, unnumbered bullet with no file:line under a paragraph finding whose fields have not + * come yet, which is its detail and joins its text (GSTA-196: 4 findings with detail bullets collected as 8). An inline + * `Blocks: yes` is CRITICAL too. The native voice's FIXABLE collects as CRITICAL; other labels stay as written for lead judgment. When in doubt the parser merges rather than drops: a lost blocker costs the gate. */ export function parseFindings(reader: string, output: string): Array<{ label: string; raw: string }> { - type Finding = { label: string; raw: string; fields: Set<string> }; + type Finding = { label: string; raw: string; fields: Set<string>; list: boolean }; const out: Finding[] = []; let section: string | null = null; let current: Finding | null = null; @@ -85,7 +86,7 @@ export function parseFindings(reader: string, output: string): Array<{ label: st const role = reader.split('/')[0]; const named = (l: string) => (role === 'adversarial-native' && l.toUpperCase() === 'FIXABLE' ? 'CRITICAL' : l); const strongest = (t: string) => [...t.matchAll(ALL_LABELS)].map(m => named(m[1]!)).reduce<string | undefined>((a, b) => (a === undefined ? b : stronger(a, b)), undefined); - const push = (label: string, raw: string, fields: string[] = []): Finding => { const f = { label, raw, fields: new Set(fields) }; out.push(f); return f; }; + const push = (label: string, raw: string, fields: string[] = []): Finding => { const f = { label, raw, fields: new Set(fields), list: LIST_ITEM.test(raw) }; out.push(f); return f; }; for (const line of output.replace(/\r\n/g, '\n').split('\n')) { const t = line.trim(); if (!t) { gap = true; continue; } @@ -118,7 +119,9 @@ export function parseFindings(reader: string, output: string): Array<{ label: st if (typeof o.severity === 'string') { current = push(o.severity, t); continue; } } catch { /* prose that starts with a brace */ } } - const fresh = listItem || (!indented && (t.startsWith('**') || afterGap)) || (!!current?.fields.size && locus); + const detail = listItem && /^[-*+]\s/.test(t) && !locus && !label && !section && !!current && !current.list && !current.fields.size; + if (detail && !blocks) { current!.raw += ` | ${t}`; continue; } + const fresh = (listItem && !detail) || (!indented && !detail && (t.startsWith('**') || afterGap)) || (!!current?.fields.size && locus); if (fresh) { current = null; head = t; } const inherited = section && listItem ? section : undefined; const found = label && inherited ? stronger(inherited, label) : label ?? inherited; @@ -133,15 +136,39 @@ export function parseFindings(reader: string, output: string): Array<{ label: st return out.map(({ label, raw }) => ({ label, raw })); } -/** collect: hash the raw output, append its findings to merged.jsonl and record both. */ -export function collect(ledger: string, reader: string, output: string, mergedFile: string, outputFile?: string): { sha: string; lines: MergedLine[] } { +/** + * The clean marker each reader's own brief prescribes for a review that found nothing: `NO FINDINGS` alone on a line for the + * specialists, the batched specialist and the targeted reader (ship/sections/review-army.md, the specialist brief; review/specialists/*.md), + * the Red Team (its brief in ship/sections/review-army.md) and fix-check (scripts/resolvers/fix-check.ts); the `Recommendation:` line + * for the outside voice (ship/sections/adversarial.md, the outside prompt); either for the native adversarial voice, whose brief asks + * for both. A reply with no finding and no marker (a refusal, an empty reply, a reader that never saw the diff) is not a review. + */ +const NO_FINDINGS: [string, RegExp] = ['NO FINDINGS', /^[`*]*NO FINDINGS\.?[`*]*$/]; +const RECOMMENDATION: [string, RegExp] = ['Recommendation:', /^[`*]*Recommendation:/i]; +const CLEAN_MARKERS: Record<string, Array<[string, RegExp]>> = { + specialist: [NO_FINDINGS], 'specialist-batched': [NO_FINDINGS], targeted: [NO_FINDINGS], 'red-team': [NO_FINDINGS], 'fix-check': [NO_FINDINGS], + 'adversarial-native': [NO_FINDINGS, RECOMMENDATION], 'outside-voice': [RECOMMENDATION], +}; +/** The markers a reader's zero-finding reply may carry; a role outside the brief roles takes either. */ +export const cleanMarkers = (reader: string) => CLEAN_MARKERS[reader.split('/')[0]!] ?? [NO_FINDINGS, RECOMMENDATION]; +const hasCleanMarker = (reader: string, output: string) => output.replace(/\r\n/g, '\n').split('\n').some(l => cleanMarkers(reader).some(([, re]) => re.test(l.trim()))); + +/** + * collect: hash the raw output, append its findings to merged.jsonl and record both; a reply with no finding and no clean marker records + * review: false. With unavailable (the outside voice only) the output is its failure text, recorded as missing coverage. + */ +export function collect(ledger: string, reader: string, output: string, mergedFile: string, outputFile?: string, unavailable?: string): { sha: string; lines: MergedLine[]; review: boolean } { const sha = promptHash(output); const prior = readLedger(ledger).filter(e => e.event === 'output' && e.reader === reader).length; - const lines = parseFindings(reader, output).map((f, i) => ({ id: `${reader}#${prior ? `${prior + 1}.` : ''}${i + 1}`, reader, label: f.label, raw: f.raw, output_sha256: sha })); + const lines = unavailable ? [] : parseFindings(reader, output).map((f, i) => ({ id: `${reader}#${prior ? `${prior + 1}.` : ''}${i + 1}`, reader, label: f.label, raw: f.raw, output_sha256: sha })); + const review = !unavailable && (lines.length > 0 || hasCleanMarker(reader, output)); fs.appendFileSync(mergedFile, lines.map(l => JSON.stringify(l) + '\n').join(''), { mode: 0o600 }); - appendEvent(ledger, { event: 'output', reader, sha256: sha, file: outputFile ? path.resolve(outputFile) : null, merged: path.resolve(mergedFile), ids: lines.map(l => l.id) }); - return { sha, lines }; + appendEvent(ledger, { event: 'output', reader, sha256: sha, file: outputFile ? path.resolve(outputFile) : null, merged: path.resolve(mergedFile), ids: lines.map(l => l.id), review, ...(unavailable ? { unavailable } : {}) }); + return { sha, lines, review }; } +/** The outside voice may fail without blocking (ship/sections/adversarial.md): its failure is recorded as unavailable coverage, never as a review. */ +export const mayBeUnavailable = (reader: string) => reader.split('/')[0] === 'outside-voice'; +const unavailableOutput = (o: LedgerEvent) => typeof o.unavailable === 'string' && mayBeUnavailable(String(o.reader)); export function readJsonl<T>(file: string): T[] { return fs.readFileSync(file, 'utf8').split('\n').filter(l => l.trim()).map(l => JSON.parse(l) as T); @@ -190,13 +217,24 @@ export function recordJudgment(ledger: string, merged: MergedLine[], judged: Jud return res; } +/** + * Whether a recorded output was a review: it has findings, or its file (unchanged since collect) carries the reader's clean marker, so + * a ledger written before collect recorded the field is judged by the same rule; with no readable file, collect's own review field. + */ +function reviewed(o: LedgerEvent): boolean { + if (Array.isArray(o.ids) && o.ids.length) return true; + let text: string | undefined; + try { text = typeof o.file === 'string' ? fs.readFileSync(o.file, 'utf8') : undefined; } catch { /* unreadable */ } + return text !== undefined && promptHash(text) === o.sha256 ? hasCleanMarker(String(o.reader), text) : o.review === true; +} + /** Every id collect recorded, in order. */ const collectedIds = (events: LedgerEvent[]) => events.filter(e => e.event === 'output').flatMap(e => (Array.isArray(e.ids) ? (e.ids as string[]) : [])); export interface VerifyRow { kind: 'BRIEF' | 'OUTPUT' | 'JUDGE'; reader: string; status: string; detail: string } /** Replay every check from a finished run's ledger: briefs dispatched as rendered, outputs merged as returned, judgment without loss. */ -export function verify(ledger: string): { rows: VerifyRow[]; clean: boolean; briefs: number; voided: number } { +export function verify(ledger: string): { rows: VerifyRow[]; clean: boolean; briefs: number; voided: number; unavailable: number } { const events = readLedger(ledger); const rows: VerifyRow[] = []; const short = (s: unknown) => (typeof s === 'string' ? s.slice(0, 12) : '-'); @@ -225,10 +263,12 @@ export function verify(ledger: string): { rows: VerifyRow[]; clean: boolean; bri } voided += epVoided; const lastMatch = [...sent].reverse().find(({ e }) => e.sha256 === r.sha256); - const collected = !!lastMatch && inside.some(({ e, i }) => e.event === 'output' && i > lastMatch.i); - const status = !sent.length ? 'NOT_SENT' : mismatched ? 'MISMATCH' : collected ? 'MATCH' : 'NOT_COLLECTED'; + const outputs = lastMatch ? inside.filter(({ e, i }) => e.event === 'output' && i > lastMatch.i) : []; + const collected = outputs.some(({ e }) => reviewed(e)); + const unavailable = !collected && outputs.some(({ e }) => unavailableOutput(e)); + const status = !sent.length ? 'NOT_SENT' : mismatched ? 'MISMATCH' : collected ? 'MATCH' : unavailable ? 'UNAVAILABLE' : 'NOT_COLLECTED'; const tampered = fileNow && fileNow !== r.sha256 ? ' file_changed_since_render' : ''; - rows.push({ kind: 'BRIEF', reader: p.label, status: tampered ? 'MISMATCH' : status, detail: `rendered=${short(r.sha256)} sent=${sent.map(({ e }) => short(e.sha256)).join(',') || '-'}${epVoided ? ` voided=${epVoided}` : ''}${tampered}` }); + rows.push({ kind: 'BRIEF', reader: p.label, status: tampered ? 'MISMATCH' : status, detail: `rendered=${short(r.sha256)} sent=${sent.map(({ e }) => short(e.sha256)).join(',') || '-'}${epVoided ? ` voided=${epVoided}` : ''}${outputs.length && !collected && !unavailable ? ' not_reviewed' : ''}${tampered}` }); } for (const [i, o] of events.entries()) { if (o.event !== 'output') continue; @@ -245,6 +285,15 @@ export function verify(ledger: string): { rows: VerifyRow[]; clean: boolean; bri try { lines = typeof o.merged === 'string' ? readJsonl<MergedLine>(o.merged).filter(l => ids.includes(l.id)) : []; } catch { notes.push('merged_unreadable'); status = 'MISMATCH'; } if (lines.length !== ids.length) { status = 'MISMATCH'; notes.push(`merged_lines=${lines.length}/${ids.length}`); } if (lines.some(l => l.output_sha256 !== o.sha256)) { status = 'MISMATCH'; notes.push('merged_line_from_other_output'); } + if (status === 'MATCH' && !reviewed(o)) { + // A non-review the same dispatch later replaced with a review (the brief re-sent verbatim, then answered) is void, counted like a voided send. + const later = p ? events.slice(i + 1, p.next) : []; + const review = later.findIndex(e => e.event === 'output' && e.reader === reader && reviewed(e)); + const replaced = review > 0 && later.slice(0, review).some(e => e.event === 'sent' && e.reader === reader && e.sha256 === events[p!.at]!.sha256); + if (unavailableOutput(o)) { status = 'UNAVAILABLE'; notes.push(`reason=${JSON.stringify(o.unavailable)}`); } + else if (replaced) { status = 'VOIDED'; voided++; notes.push('replaced_by_a_review'); } + else { status = 'NOT_REVIEWED'; notes.push('no_finding_no_clean_marker'); } + } rows.push({ kind: 'OUTPUT', reader: p?.label ?? reader, status, detail: `sha=${short(o.sha256)} findings=${ids.length}${notes.length ? ' ' + notes.join(' ') : ''}` }); } const lastJudge = events.map(e => e.event).lastIndexOf('judge'); @@ -266,6 +315,6 @@ export function verify(ledger: string): { rows: VerifyRow[]; clean: boolean; bri const status = !v.ok ? 'REJECTED' : notes.length ? 'MISMATCH' : 'OK'; rows.push({ kind: 'JUDGE', reader: '-', status, detail: `dropped=${JSON.stringify(v.dropped)} downgraded=${JSON.stringify(v.downgraded)} added=${v.added} promoted=${v.promoted} folded=${v.folded}${notes.length ? ' ' + notes.join(' ') : ''}` }); } else if (events.some(e => e.event === 'output')) rows.push({ kind: 'JUDGE', reader: '-', status: 'MISSING', detail: 'outputs were collected but no judged set was checked' }); - const clean = episodes.length > 0 && rows.every(r => r.status === 'MATCH' || r.status === 'OK'); - return { rows, clean, briefs: episodes.length, voided }; + const clean = episodes.length > 0 && rows.every(r => ['MATCH', 'OK', 'VOIDED', 'UNAVAILABLE'].includes(r.status)); + return { rows, clean, briefs: episodes.length, voided, unavailable: rows.filter(r => r.kind === 'BRIEF' && r.status === 'UNAVAILABLE').length }; } diff --git a/lib/ship-policy.ts b/lib/ship-policy.ts index 6ed75ea3c..e4232c959 100644 --- a/lib/ship-policy.ts +++ b/lib/ship-policy.ts @@ -190,7 +190,7 @@ export const DOCS_CAPS_KEY_VERSION = '1.91.270.0'; export const CHANGELOG_FRAGMENTS_KEY_VERSION = '1.91.336.0'; /** `bump: never` is a value of an older key, so the per-key floor check misses it; compatErrors checks it by value. */ export const BUMP_NEVER_KEY_VERSION = '1.91.353.0'; -export const BASE_RED_KEY_VERSION = '1.92.0.0'; +export const BASE_RED_KEY_VERSION = '1.91.374.0'; export const POLICY_KEY_INTRODUCED: Record<PolicyKey, string> = { bump: '1.91.73.0', queue_mode: '1.91.73.0', release_tool: '1.91.73.0', release_outputs: '1.91.73.0', stamp_paths: '1.91.73.0', changelog: '1.91.73.0', mirror: '1.91.73.0', history: '1.91.73.0', preregistration_shas: '1.91.73.0', stamp_sensitive_gates: '1.91.73.0', diff --git a/package.json b/package.json index 521d596fe..36f46b3a6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.365", + "version": "1.91.374", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/scripts/pr-dependencies.ts b/scripts/pr-dependencies.ts index b0a107407..e40ae2c35 100644 --- a/scripts/pr-dependencies.ts +++ b/scripts/pr-dependencies.ts @@ -89,7 +89,7 @@ export const SELECTION_DATA_MODULES: readonly string[] = [ // The build entry list: every compiled binary, which skills reach through their `--outfile` path instead. 'scripts/build.sh', // The reason-code table: its `fix:` prose names the command a person reruns, not a file the module consumes. - 'lib/result-codes.ts', 'lib/result-codes-release.ts', 'lib/result-codes-capy.ts', 'lib/result-codes-budget.ts', + 'lib/result-codes.ts', 'lib/result-codes-release.ts', 'lib/result-codes-capy.ts', 'lib/result-codes-budget.ts', 'lib/result-codes-eval.ts', // The exercised-path map: observed commands per case, read only by exercisedConsumers() (shadow data; a change selects no paid case). 'test/helpers/exercised-commands.json', ]; diff --git a/scripts/test-health-report.ts b/scripts/test-health-report.ts index ae09e9614..8d9e55168 100644 --- a/scripts/test-health-report.ts +++ b/scripts/test-health-report.ts @@ -277,7 +277,7 @@ export function collectHealth(inputs: HealthInputs): HealthReport { return { value: `${(total / 60_000 * 7 / sinceDays).toFixed(0)} min/week (${cancelled.length} cancelled run(s))` }; })); - metrics.push(metric('push-to-verdict', 'PR lane push-to-verdict', 'created_at to updated_at of evals.yml pull_request runs that reached success or failure (includes queue and the readiness wait)', () => { + metrics.push(metric('push-to-verdict', 'PR lane push-to-verdict', 'created_at to updated_at of evals.yml pull_request runs that reached success or failure (includes queue and the readiness wait, which pr-summary runs skip since GSTA-149, so their verdicts land about 10 min sooner)', () => { const walls = need(prRuns).filter(run => run.conclusion === 'success' || run.conclusion === 'failure').map(run => span(run.createdAt, run.updatedAt)!); if (walls.length === 0) throw new Error('no PR run reached a verdict in the window'); return { value: `p50 ${minutes(percentile(walls, 0.5))}, p95 ${minutes(percentile(walls, 0.95))} (${walls.length} runs)` }; diff --git a/scripts/ubicloud/pregate-remote.sh b/scripts/ubicloud/pregate-remote.sh index 0759ee9b1..68e5b5692 100755 --- a/scripts/ubicloud/pregate-remote.sh +++ b/scripts/ubicloud/pregate-remote.sh @@ -185,7 +185,7 @@ args=(run -s "$size" --src "$root" --setup "$prelude" [ -z "${GSTACK_FREE_JOBS:-}" ] || args+=(--env "GSTACK_FREE_JOBS=$GSTACK_FREE_JOBS") # The VM run gets its own session (python3, which ubi-runner already needs, for -# setsid on Linux and macOS): pregate's --timeout and stall kill signal this +# setsid on Linux and macOS): pregate's --timeout kill signals this # script's process group and then SIGKILL it, which must never cut ubi-runner's # teardown short. Output goes to files (a dead reader would SIGPIPE ubi-runner), # stamped for the phase timings; this script streams it. A watchdog sends diff --git a/test/eval-models.test.ts b/test/eval-models.test.ts index 6e22a4e03..e907b7e6e 100644 --- a/test/eval-models.test.ts +++ b/test/eval-models.test.ts @@ -142,6 +142,19 @@ describe('gstack-eval-plan lints every arm (bin, fixture repo with an origin)', expect(head.err).toContain('(EVAL_MODEL_GENERATION_UNRESOLVED)'); }); + test('a <model>#<loadout> arm is linted as its model: a deny still refuses it and an exact allow accepts it', () => { + seed(GBRAIN_RULES); + const r = run(['arms', '--arms', 'claude-opus-5-5#no-gstack,gpt-5.4-mini#no-gstack']); + expect(r.code).toBe(1); + expect(r.out).toContain('EVAL_MODEL: claude-opus-5-5 scope=full ok (allow claude-*; newest in families.claude-opus) where=flag:--arms (claude-opus-5-5#no-gstack)'); + expect(r.out).toContain('EVAL_MODEL: gpt-5.4-mini scope=full denied (deny gpt-5.4-mini) where=flag:--arms (gpt-5.4-mini#no-gstack)'); + expect(r.err).toContain('(EVAL_MODEL_DENIED)'); + fs.writeFileSync(path.join(repo, '.gstack', 'eval-models.json'), JSON.stringify({ schema_version: 1, allow: ['claude-opus-5-5'], deny: [] })); + const exact = run(['arms', '--arms', 'claude-opus-5-5,claude-opus-5-5#no-gstack', '--models-from', 'head']); + expect(exact.code).toBe(0); + expect(exact.out).toContain('where=flag:--arms (claude-opus-5-5#no-gstack)'); + }); + test('dry-run and pilot start: Fable fails the full scope (no reservation), passes --scope smoke; the dry-run record carries the check', () => { seed(GBRAIN_RULES); const out = path.join(host.base, 'out'); diff --git a/test/eval-plan-no-gstack-arm.test.ts b/test/eval-plan-no-gstack-arm.test.ts new file mode 100644 index 000000000..c04e4de9b --- /dev/null +++ b/test/eval-plan-no-gstack-arm.test.ts @@ -0,0 +1,199 @@ +/** + * The no-gstack arm (Garry, 2026-10-31: "all of our head to head tests have to + * test no gstack from here"). A preregistration says in `## Comparison` whether + * any arm loads gstack or another skill stack (`stack: loaded`) or none does + * (`stack: none` with a `why:`). A loaded comparison needs a `## No-gstack arm` + * section naming one `<model>#no-gstack` arm per model, each also an Arms line, + * plus the inputs, prompt, panel and report it shares with the variants. Arms are + * identities `<model>[#<loadout>]`, priced by the model, so the no-gstack arm on + * the same model gets its own pilot reservations. + */ +import { afterEach, describe, expect, test } from 'bun:test'; +import { spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { armModel, checkPrereg, dryRun, preregTemplate, resolveArms, startPilot } from '../lib/eval-plan'; + +const BIN = path.resolve(import.meta.dir, '..', 'bin', 'gstack-eval-plan'); +const dirs: string[] = []; +const tmp = () => { const d = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-no-gstack-arm-')); dirs.push(d); return d; }; +afterEach(() => { while (dirs.length) fs.rmSync(dirs.pop()!, { recursive: true, force: true }); }); + +const BASE = ['Bars', 'Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`).join('\n'); +const ARMS = (...ids: string[]) => `\n## Arms\n${ids.map(id => `- ${id} default settings, $0.01 per item`).join('\n')}\n`; +const LOADED = '\n## Comparison\nstack: loaded\n'; +const NO_GSTACK = (o: Partial<Record<'arms' | 'inputs' | 'prompt' | 'panel' | 'report', string>> = {}) => { + const v = { arms: 'claude-opus-5-5#no-gstack', inputs: 'the same rows and fixture as the variants', prompt: 'the task text only; no SKILL.md, preamble, hooks or installed skills', panel: 'behavior, 3 trials, the same judge and labels', report: 'one row beside the variants in the results table', ...o }; + return `\n## No-gstack arm\n${Object.entries(v).filter(([, x]) => x !== undefined).map(([k, x]) => `${k}: ${x}`).join('\n')}\n`; +}; +const VARIANTS = ARMS('claude-opus-5-5#review-g', 'claude-opus-5-5#review-h', 'claude-opus-5-5#no-gstack'); + +describe('the Comparison declaration', () => { + test('Comparison is a required section; the blank template fails and multi-line guidance never counts as content', () => { + const t = preregTemplate('x'); + expect(t).toContain('## Comparison'); + expect(t).toContain('## No-gstack arm'); + const blank = checkPrereg(t); + expect(blank.empty).toContain('Comparison'); + expect(checkPrereg(BASE + VARIANTS + '\n## Comparison\n<!-- stack: loaded\n more guidance -->\n').empty).toContain('Comparison'); + }); + + test('stack: none needs a why: line and passes without the arm; an unknown value fails', () => { + const none = checkPrereg(BASE + ARMS('claude-opus-5-5') + '\n## Comparison\nstack: none\nwhy: scorer calibration; no arm loads a skill stack\n'); + expect(none.ok).toBe(true); + expect(none.comparison.stack).toBe('none'); + expect(none.no_gstack.required).toBe(false); + const noWhy = checkPrereg(BASE + ARMS('claude-opus-5-5') + '\n## Comparison\nstack: none\n'); + expect(noWhy.ok).toBe(false); + expect(noWhy.comparison.errors.join(';')).toContain('why:'); + const odd = checkPrereg(BASE + ARMS('claude-opus-5-5') + '\n## Comparison\nstack: maybe\n'); + expect(odd.ok).toBe(false); + expect(odd.comparison.errors.join(';')).toContain('stack: maybe'); + }); +}); + +describe('the No-gstack arm section', () => { + test('a loaded comparison passes with a complete section and fails without it', () => { + expect(checkPrereg(BASE + VARIANTS + LOADED + NO_GSTACK()).ok).toBe(true); + const missing = checkPrereg(BASE + VARIANTS + LOADED); + expect(missing.ok).toBe(false); + expect(missing.no_gstack).toMatchObject({ required: true, ok: false }); + expect(missing.no_gstack.problems).toEqual(['missing section']); + }); + + for (const key of ['arms', 'inputs', 'prompt', 'panel', 'report'] as const) { + test(`a missing or blank ${key}: line fails naming it`, () => { + for (const v of [undefined, ' ']) { + const r = checkPrereg(BASE + VARIANTS + LOADED + NO_GSTACK({ [key]: v } as Record<string, string>)); + expect(r.ok).toBe(false); + expect(r.no_gstack.problems).toContain(`missing ${key}`); + } + }); + } + + test('every model that loads a stack needs its own <model>#no-gstack arm: a two-model panel needs two', () => { + const panel = ARMS('claude-opus-5-5', 'gpt-6.1-sol', 'claude-opus-5-5#no-gstack'); + const one = checkPrereg(BASE + panel + LOADED + NO_GSTACK()); + expect(one.ok).toBe(false); + expect(one.no_gstack.problems).toEqual(['no no-gstack arm for gpt-6.1-sol']); + const both = checkPrereg(BASE + ARMS('claude-opus-5-5', 'gpt-6.1-sol', 'claude-opus-5-5#no-gstack', 'gpt-6.1-sol#no-gstack') + LOADED + NO_GSTACK({ arms: 'claude-opus-5-5#no-gstack, gpt-6.1-sol#no-gstack' })); + expect(both.ok).toBe(true); + }); + + test('a model panel under another skill stack (not gstack) is loaded too and needs the same arms', () => { + const other = ARMS('claude-opus-5-5#other-stack', 'gpt-6.1-sol#other-stack'); + const r = checkPrereg(BASE + other + LOADED + NO_GSTACK({ arms: 'claude-opus-5-5#no-gstack' })); + expect(r.ok).toBe(false); + expect(r.no_gstack.problems).toEqual(['claude-opus-5-5#no-gstack is not an Arms line', 'no no-gstack arm for claude-opus-5-5', 'no no-gstack arm for gpt-6.1-sol']); + }); + + test('a listed arm must be an Arms line and a <model>#no-gstack identity, matched on whole identities', () => { + const notListed = checkPrereg(BASE + ARMS('claude-opus-5-5#review-g') + LOADED + NO_GSTACK()); + expect(notListed.no_gstack.problems).toContain('claude-opus-5-5#no-gstack is not an Arms line'); + const prefix = checkPrereg(BASE + ARMS('claude-opus-5-5#review-g', 'claude-opus-5#no-gstack') + LOADED + NO_GSTACK({ arms: 'claude-opus-5#no-gstack' })); + expect(prefix.no_gstack.problems).toContain('no no-gstack arm for claude-opus-5-5'); + const wrongLoadout = checkPrereg(BASE + ARMS('claude-opus-5-5#review-g', 'claude-opus-5-5#plain') + LOADED + NO_GSTACK({ arms: 'claude-opus-5-5#plain' })); + expect(wrongLoadout.no_gstack.problems).toContain('claude-opus-5-5#plain is not a <model>#no-gstack identity'); + const prose = checkPrereg(BASE + '\n## Arms\n- claude-opus-5-5#review-g, compared with claude-opus-5-5#no-gstack in prose\n' + LOADED + NO_GSTACK()); + expect(prose.no_gstack.problems).toContain('claude-opus-5-5#no-gstack is not an Arms line'); + }); + + test('a table in Arms: the header row is not an arm', () => { + const table = '\n## Arms\n| model | settings |\n|---|---|\n| claude-opus-5-5#review-g | default |\n| claude-opus-5-5#no-gstack | default |\n'; + expect(checkPrereg(BASE + table + LOADED + NO_GSTACK()).ok).toBe(true); + }); +}); + +describe('arm identities in pricing and the pilot', () => { + test('armModel strips the loadout; arms and the dry run price <model>#no-gstack by its model', () => { + expect(armModel('anthropic/claude-opus-5-5#no-gstack')).toBe('anthropic/claude-opus-5-5'); + expect(armModel('gpt-6.1-sol')).toBe('gpt-6.1-sol'); + const d = tmp(); + const arms = resolveArms({ cwd: d, explicit: ['claude-opus-5-5#no-gstack'] }); + expect(arms.errors).toEqual([]); + expect(arms.arms[0]).toMatchObject({ model: 'claude-opus-5-5#no-gstack', priced: true }); + const rows = path.join(d, 'rows.jsonl'); + fs.writeFileSync(rows, JSON.stringify({ prompt: 'hello there' }) + '\n'); + const dry = dryRun({ input: rows, arms: ['claude-opus-5-5', 'claude-opus-5-5#no-gstack'] }); + expect(dry.checks.price_table.ok).toBe(true); + expect(dry.per_item_usd['claude-opus-5-5#no-gstack']).toBe(dry.per_item_usd['claude-opus-5-5']); + }); + + const passingDryRun = (d: string) => { + const rows = path.join(d, 'rows.jsonl'); + fs.writeFileSync(rows, JSON.stringify({ prompt: 'hello there' }) + '\n'); + const file = path.join(d, 'dry-run.json'); + const r = dryRun({ input: rows, arms: ['claude-opus-5-5'], retryPrompt: (fs.writeFileSync(path.join(d, 'retry.md'), 'try again\n'), path.join(d, 'retry.md')) }); + fs.writeFileSync(file, JSON.stringify(r)); + expect(r.pass, JSON.stringify(r.checks)).toBe(true); + return file; + }; + + test('the pilot reserves a variant and its no-gstack arm on the same model under distinct attempts; the total is the sum', () => { + const d = tmp(); + const r = startPilot({ out: path.join(d, 'pilot'), cap: 100, n: 2, arms: ['claude-opus-5-5', 'claude-opus-5-5#no-gstack'], dryRun: passingDryRun(d) }); + expect(r.error).toBeUndefined(); + const ids = r.record.attempts.map(a => a.attempt); + expect(new Set(ids).size).toBe(4); + const ledger = JSON.parse(fs.readFileSync(path.join(d, 'pilot', fs.readdirSync(path.join(d, 'pilot')).find(f => f.includes('spend'))!), 'utf8')); + const reserved = JSON.stringify(ledger); + for (const id of ids) expect(reserved).toContain(id); + const each = r.record.attempts[0]!.estimate_usd; + expect(r.record.attempts.every(a => a.estimate_usd === each)).toBe(true); + }); + + test('a duplicate arm identity is refused before anything is reserved (EVAL_ARM_DUPLICATE)', () => { + const d = tmp(); + const r = startPilot({ out: path.join(d, 'pilot'), cap: 100, n: 1, arms: ['claude-opus-5-5', 'claude-opus-5-5'], dryRun: passingDryRun(d) }); + expect(r.error?.code).toBe('EVAL_ARM_DUPLICATE'); + expect(r.record.attempts).toEqual([]); + }); + + test('distinct identities that share one ledger attempt name are refused before anything is reserved', () => { + for (const arms of [['claude-opus-5-5#review/a', 'claude-opus-5-5#review_a'], ['gpt-6#1-sol', 'gpt-6.1-sol']]) { + const d = tmp(); + const r = startPilot({ out: path.join(d, 'pilot'), cap: 100, n: 1, arms, dryRun: passingDryRun(d) }); + expect(r.error?.code).toBe('EVAL_ARM_DUPLICATE'); + expect(r.error?.message).toContain(arms.join(' = ')); + expect(r.record.attempts).toEqual([]); + expect(fs.existsSync(path.join(d, 'pilot'))).toBe(false); + } + }); +}); + +describe('gstack-eval-plan prereg --check', () => { + const run = (text: string) => { + const d = tmp(); + const file = path.join(d, 'prereg.md'); + fs.writeFileSync(file, text); + const r = spawnSync(BIN, ['prereg', '--check', file], { cwd: d, encoding: 'utf8', timeout: 60_000 }); + return { code: r.status, out: r.stdout ?? '', err: r.stderr ?? '' }; + }; + + test('a loaded comparison without the arm: one PREREG line out of 6, PREREG_NO_GSTACK_ARM on stderr, exit 1', () => { + const r = run(BASE + VARIANTS + LOADED); + expect(r.code).toBe(1); + expect(r.out).toMatch(/^PREREG: incomplete present=6\/6 stack=loaded no_gstack_arm=missing file=/m); + expect(r.err).toContain('missing section'); + expect(r.err).toContain('(PREREG_NO_GSTACK_ARM)'); + expect(r.err).not.toContain('(PREREG_INCOMPLETE)'); + }); + + test('missing sections and arm problems both print, PREREG_INCOMPLETE first', () => { + const r = run(VARIANTS + LOADED); + expect(r.code).toBe(1); + const a = r.err.indexOf('(PREREG_INCOMPLETE)'), b = r.err.indexOf('(PREREG_NO_GSTACK_ARM)'); + expect(a).toBeGreaterThanOrEqual(0); + expect(b).toBeGreaterThan(a); + }); + + test('complete: PREREG: ok with stack=loaded no_gstack_arm=ok; an opt-out prints stack=none', () => { + const ok = run(BASE + VARIANTS + LOADED + NO_GSTACK()); + expect(ok.code, ok.out + ok.err).toBe(0); + expect(ok.out).toMatch(/^PREREG: ok present=6\/6 stack=loaded no_gstack_arm=ok file=/m); + const none = run(BASE + ARMS('claude-opus-5-5') + '\n## Comparison\nstack: none\nwhy: scorer calibration\n'); + expect(none.code, none.out + none.err).toBe(0); + expect(none.out).toMatch(/^PREREG: ok present=6\/6 stack=none file=/m); + }); +}); diff --git a/test/eval-plan-prereg-diagnostics.test.ts b/test/eval-plan-prereg-diagnostics.test.ts index 7271cc7a8..9e9e18a20 100644 --- a/test/eval-plan-prereg-diagnostics.test.ts +++ b/test/eval-plan-prereg-diagnostics.test.ts @@ -14,7 +14,7 @@ import { checkPrereg, parsePreregDiagnostics, preregTemplate } from '../lib/eval import { capyHost, type CapyHost } from './helpers/capy-host'; const BIN = path.resolve(import.meta.dir, '..', 'bin', 'gstack-eval-plan'); -const FILLED = ['Bars', 'Arms', 'Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`).join('\n'); +const FILLED = ['Bars', 'Arms', 'Decoys', 'Stop rules', 'Held-out exposure'].map(s => `## ${s}\n- filled in\n`).join('\n') + '\n## Comparison\nstack: none\nwhy: one arm, no skill stack\n'; const SUMMARIZER = [ 'touch ran.marker', 'r=$(grep -c \'"kind":"refusal"\' "$1")', @@ -54,7 +54,7 @@ describe('prereg --check: the Diagnostics section', () => { const r = check(FILLED + diagnostics([['refusals', '2'], ['truncated', '1']])); expect(r.code).toBe(3); expect(r.ran).toBe(false); - expect(r.out).toContain('PREREG: ok present=5/5 diagnostics=2 file='); + expect(r.out).toContain('PREREG: ok present=6/6 stack=none diagnostics=2 file='); expect(r.out).toContain(`PREREG_COMMAND: summarizer="sh summarize.sh {fixture}" fixture=fixtures/synthetic records.jsonl cwd=${fs.realpathSync(repo)}`); expect(r.err).toContain('(REPO_COMMANDS_NOT_ALLOWED)'); expect(r.out).not.toContain('PREREG_DIAGNOSTICS:'); @@ -104,7 +104,7 @@ describe('prereg --check: the Diagnostics section', () => { const r = check(FILLED + diagnostics([['refusals', 'two']], { summarizer: 'sh summarize.sh fixtures/x.jsonl' }), '--allow-repo-commands'); expect(r.code).toBe(1); expect(r.ran).toBe(false); - expect(r.out).toContain('PREREG: incomplete present=5/5 diagnostics=invalid'); + expect(r.out).toContain('PREREG: incomplete present=6/6 stack=none diagnostics=invalid'); expect(r.err).toContain('diagnostics: row "| refusals | two |"'); expect(r.err).toContain('diagnostics: the summarizer command never names the fixture ({fixture})'); expect(r.err).toContain('diagnostics: no | diagnostic | expected | rows'); @@ -114,7 +114,7 @@ describe('prereg --check: the Diagnostics section', () => { test('without a Diagnostics section the check is unchanged; the template mentions the section without adding a heading', () => { const r = check(FILLED); expect(r.code).toBe(0); - expect(r.out).toBe(`PREREG: ok present=5/5 file=${path.join(repo, 'evals', 'prereg.md')}\n`); + expect(r.out).toBe(`PREREG: ok present=6/6 stack=none file=${path.join(repo, 'evals', 'prereg.md')}\n`); expect(parsePreregDiagnostics(FILLED)).toBeNull(); expect(parsePreregDiagnostics(FILLED + '\n## Diagnostics\n<!-- none -->\n')).toBeNull(); expect(preregTemplate()).toContain('## Diagnostics'); diff --git a/test/evals-pr-lane-workflow.test.ts b/test/evals-pr-lane-workflow.test.ts index 439357bf5..385b40667 100644 --- a/test/evals-pr-lane-workflow.test.ts +++ b/test/evals-pr-lane-workflow.test.ts @@ -83,7 +83,8 @@ describe('evals.yml readiness-gated dispatch (R1, GSTA-48)', () => { test('a tiny PR-only job waits for a quiet PR and green Free Tests before recovery and planning', () => { expect(readiness['runs-on']).toBe('ubuntu-24.04'); - expect(readiness.if).toBe("github.event_name == 'pull_request' && github.actor != 'dependabot[bot]' && github.event.pull_request.head.repo.full_name == github.repository"); + expect(readiness.if).toBe("github.event_name == 'pull_request' && github.actor != 'dependabot[bot]' && github.event.pull_request.head.repo.full_name == github.repository && needs.route.outputs.mode != 'pr-summary'"); + expect(readiness.needs).toEqual(['route']); expect(readiness.permissions).toEqual({ contents: 'read', actions: 'read', 'pull-requests': 'read' }); expect(Number(readiness['timeout-minutes'])).toBeGreaterThan(80); const wait = step('readiness', 'Wait for a quiet PR and green Free Tests on this head')!; diff --git a/test/evals-train-workflow.test.ts b/test/evals-train-workflow.test.ts index 891063a13..cf3c8b71e 100644 --- a/test/evals-train-workflow.test.ts +++ b/test/evals-train-workflow.test.ts @@ -193,9 +193,10 @@ describe('without a train block nothing changes (the inert proof)', () => { }); describe('under a train block (the gate edits, each only on its mode)', () => { - test('a PR push starts no paid slice: route, readiness and the free screening summary only', () => { + test('a PR push starts no paid slice and waits for nothing: route, the image and the free screening summary only', () => { const sim = simulate(evals, { github: pr(), inputs: {}, outputs: { route: { ...ROUTE, ...train, mode: 'pr-summary' } } }); - expect(sim.jobs).toEqual(['build-image', 'readiness', 'route', 'screen-summary']); + expect(sim.jobs).toEqual(['build-image', 'route', 'screen-summary']); + expect(simulate(evals, { github: pr(), inputs: {}, outputs: { route: { ...ROUTE, mode: 'pr' } } }).jobs).toContain('readiness'); const summary = evals.jobs['screen-summary']!; expect(summary.permissions).toEqual({ contents: 'read' }); expect(JSON.stringify(summary)).not.toMatch(/secrets\.|restore-deps|bun install/); diff --git a/test/evidence.test.ts b/test/evidence.test.ts index 479095f6a..e40d6769c 100644 --- a/test/evidence.test.ts +++ b/test/evidence.test.ts @@ -304,6 +304,71 @@ describe('gstack-evidence check', () => { expect(chk.status).toBe(0); }); + // Value: protects=a repository that gitignores .bun/ still gets a fingerprint (GST-449): an exclude pathspec naming an ignored path made + // `git add -A` exit 1, so gstack-wtree failed and every review-log start and evidence check lost its fingerprint; fails_when=gstack-wtree + // exits non-zero or the lane records no wtree with .bun/ (or /.bun) in .gitignore; why_new=the GST-421 test covered only an unignored + // .bun/ (harness-evals#148); seam=none; no_claim=Windows git + test('GST-449: with .bun/ gitignored, gstack-wtree still fingerprints and a lane that writes .bun/ grades FRESH', () => { + for (const pattern of ['.bun/', '/.bun']) { + fs.writeFileSync(path.join(repoDir, '.gitignore'), `${pattern}\n`); + git('add .gitignore'); + git(`commit -q -m "ignore ${pattern}"`); + fs.mkdirSync(path.join(repoDir, '.bun'), { recursive: true }); + fs.writeFileSync(path.join(repoDir, '.bun', 'x'), 'cache\n'); + const wt = runBin(path.join(ROOT, 'bin', 'gstack-wtree'), [], { cwd: repoDir }); + expect(wt.stderr).toBe(''); + expect(wt.status).toBe(0); + expect(wt.stdout.trim()).toMatch(/^[0-9a-f]{40}$/); + const r = run(['run', '--label', 'tests', '--', 'echo cache2 > .bun/y && echo green']); + expect(r.status).toBe(0); + expect(records().pop().wtree).toMatch(/^[0-9a-f]{40}$/); + expect(run(['check', '--label', 'tests']).stdout).toContain('EVIDENCE: FRESH label=tests'); + fs.rmSync(path.join(repoDir, '.bun'), { recursive: true, force: true }); + } + }); + + // Value: protects=a repository that ignores .bun/ while still tracking a file under it gets a fingerprint (GST-449 review): the + // index-aware check-ignore called .bun not ignored, so the exclude pathspec came back and `git add -A` exited 1; fails_when=gstack-wtree + // exits non-zero, or its tree differs from a plain `git add -A`, or an edit or delete of the tracked file leaves it unchanged; + // why_new=the pattern loop above starts from an untracked .bun/ only; seam=none; no_claim=Windows git + test('GST-449: an ignored .bun/ that still tracks a file fingerprints like a plain git add -A, and edits to the tracked file count', () => { + fs.mkdirSync(path.join(repoDir, '.bun'), { recursive: true }); + fs.writeFileSync(path.join(repoDir, '.bun', 'tracked'), 'v1\n'); + git('add -f .bun/tracked'); + git('commit -q -m "track .bun/tracked"'); + fs.writeFileSync(path.join(repoDir, '.gitignore'), '.bun/\n'); + git('add .gitignore'); + git('commit -q -m "ignore .bun/"'); + fs.writeFileSync(path.join(repoDir, '.bun', 'cache'), 'cache\n'); + const wtree = () => { + const wt = runBin(path.join(ROOT, 'bin', 'gstack-wtree'), [], { cwd: repoDir }); + expect(wt.stderr).toBe(''); + expect(wt.status).toBe(0); + return wt.stdout.trim(); + }; + const plain = () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-wtree-idx-')); + try { + const env = { ...process.env, GIT_INDEX_FILE: path.join(dir, 'index') }; + spawnSync('git', ['read-tree', 'HEAD'], { cwd: repoDir, env, timeout: 10_000 }); + spawnSync('git', ['add', '-A'], { cwd: repoDir, env, timeout: 10_000 }); + return spawnSync('git', ['write-tree'], { cwd: repoDir, env, encoding: 'utf-8', timeout: 10_000 }).stdout.trim(); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }; + const clean = wtree(); + expect(clean).toBe(plain()); + fs.writeFileSync(path.join(repoDir, '.bun', 'tracked'), 'v2\n'); + const edited = wtree(); + expect(edited).not.toBe(clean); + expect(edited).toBe(plain()); + fs.rmSync(path.join(repoDir, '.bun', 'tracked')); + const deleted = wtree(); + expect(deleted).not.toBe(clean); + expect(deleted).toBe(plain()); + }); + test('GST-421: a lane that writes untracked source grades STALE and the reason names the path', () => { const r = run(['run', '--label', 'tests', '--', 'mkdir -p src && echo "export {}" > src/new.ts && echo green']); expect(r.status).toBe(0); diff --git a/test/fixtures/junit/bun.xml b/test/fixtures/junit/bun.xml new file mode 100644 index 000000000..7b8f851b1 --- /dev/null +++ b/test/fixtures/junit/bun.xml @@ -0,0 +1,17 @@ +<?xml version="1.0" encoding="UTF-8"?> +<testsuites name="bun test" tests="4" assertions="2" failures="1" skipped="1" time="0.02640424"> + <testsuite name="a.test.ts" file="a.test.ts" tests="4" assertions="2" failures="1" skipped="1" time="0.014448064" hostname="ci-host"> + <testsuite name="outer" file="a.test.ts" line="2" tests="3" assertions="2" failures="1" skipped="1" time="0" hostname="ci-host"> + <testsuite name="inner" file="a.test.ts" line="2" tests="2" assertions="2" failures="1" skipped="0" time="0.006" hostname="ci-host"> + <testcase name="passes" classname="inner > outer" time="0.000069" file="a.test.ts" line="2" assertions="1" /> + <testcase name="fails" classname="inner > outer" time="0.006322" file="a.test.ts" line="2" assertions="1"> + <failure type="AssertionError" message="expect(received).toBe(expected) Expected: 2 Received: 1 ">AssertionError: expect(received).toBe(expected) Expected: 2 Received: 1 at a.test.ts:2:131 </failure> + </testcase> + </testsuite> + <testcase name="skipped" classname="outer" time="0" file="a.test.ts" line="2" assertions="0"> + <skipped /> + </testcase> + </testsuite> + <testcase name="top level" classname="" time="0.000039" file="a.test.ts" line="3" assertions="0" /> + </testsuite> +</testsuites> diff --git a/test/fixtures/junit/playwright-r0.xml b/test/fixtures/junit/playwright-r0.xml new file mode 100644 index 000000000..8fb6599e0 --- /dev/null +++ b/test/fixtures/junit/playwright-r0.xml @@ -0,0 +1,138 @@ +<testsuites id="" name="" tests="10" failures="4" skipped="2" errors="0" time="3.23467"> +<testsuite name="a.spec.ts" timestamp="2026-10-29T16:02:06.214Z" hostname="alpha" tests="5" failures="2" skipped="1" time="0.03" errors="0"> +<testcase name="group one › passes" classname="a.spec.ts" time="0.011"> +</testcase> +<testcase name="group one › fails always" classname="a.spec.ts" time="0.004"> +<system-out> +<![CDATA[ +[[ATTACHMENT|pw/test-results/a-group-one-fails-always-alpha/error-context.md]] +]]> +</system-out> +<failure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe"> +<![CDATA[ [alpha] › a.spec.ts:5:7 › group one › fails always ─────────────────────────────────────────────── + + Error: expect(received).toBe(expected) // Object.is equality + + Expected: 2 + Received: 1 + + 3 | test.describe('group one', () => { + 4 | test('passes', () => { expect(1).toBe(1); }); + > 5 | test('fails always', () => { expect(1).toBe(2); }); + | ^ + 6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } }); + 7 | test.skip('skipped', () => {}); + 8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); }); + at /repo/tests/a.spec.ts:5:42 + + Error Context: ../test-results/a-group-one-fails-always-alpha/error-context.md +]]> +</failure> +</testcase> +<testcase name="group one › flaky once" classname="a.spec.ts" time="0.009"> +<system-out> +<![CDATA[ +[[ATTACHMENT|pw/test-results/a-group-one-flaky-once-alpha/error-context.md]] +]]> +</system-out> +<failure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe"> +<![CDATA[ [alpha] › a.spec.ts:6:7 › group one › flaky once ───────────────────────────────────────────────── + + Error: expect(received).toBe(expected) // Object.is equality + + Expected: "second" + Received: "first" + + 4 | test('passes', () => { expect(1).toBe(1); }); + 5 | test('fails always', () => { expect(1).toBe(2); }); + > 6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } }); + | ^ + 7 | test.skip('skipped', () => {}); + 8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); }); + 9 | }); + at /repo/tests/a.spec.ts:6:152 + + Error Context: ../test-results/a-group-one-flaky-once-alpha/error-context.md +]]> +</failure> +</testcase> +<testcase name="group one › skipped" classname="a.spec.ts"> +<properties> +<property name="skip" value=""> +</property> +</properties> +<skipped> +</skipped> +</testcase> +<testcase name="group one › title with $(rm -rf) [x] (y)? *" classname="a.spec.ts" time="0.006"> +</testcase> +</testsuite> +<testsuite name="a.spec.ts" timestamp="2026-10-29T16:02:06.214Z" hostname="beta" tests="5" failures="2" skipped="1" time="0.027" errors="0"> +<testcase name="group one › passes" classname="a.spec.ts" time="0.005"> +</testcase> +<testcase name="group one › fails always" classname="a.spec.ts" time="0.002"> +<system-out> +<![CDATA[ +[[ATTACHMENT|pw/test-results/a-group-one-fails-always-beta/error-context.md]] +]]> +</system-out> +<failure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe"> +<![CDATA[ [beta] › a.spec.ts:5:7 › group one › fails always ──────────────────────────────────────────────── + + Error: expect(received).toBe(expected) // Object.is equality + + Expected: 2 + Received: 1 + + 3 | test.describe('group one', () => { + 4 | test('passes', () => { expect(1).toBe(1); }); + > 5 | test('fails always', () => { expect(1).toBe(2); }); + | ^ + 6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } }); + 7 | test.skip('skipped', () => {}); + 8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); }); + at /repo/tests/a.spec.ts:5:42 + + Error Context: ../test-results/a-group-one-fails-always-beta/error-context.md +]]> +</failure> +</testcase> +<testcase name="group one › flaky once" classname="a.spec.ts" time="0.013"> +<system-out> +<![CDATA[ +[[ATTACHMENT|pw/test-results/a-group-one-flaky-once-beta/error-context.md]] +]]> +</system-out> +<failure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe"> +<![CDATA[ [beta] › a.spec.ts:6:7 › group one › flaky once ────────────────────────────────────────────────── + + Error: expect(received).toBe(expected) // Object.is equality + + Expected: "second" + Received: "first" + + 4 | test('passes', () => { expect(1).toBe(1); }); + 5 | test('fails always', () => { expect(1).toBe(2); }); + > 6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } }); + | ^ + 7 | test.skip('skipped', () => {}); + 8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); }); + 9 | }); + at /repo/tests/a.spec.ts:6:152 + + Error Context: ../test-results/a-group-one-flaky-once-beta/error-context.md +]]> +</failure> +</testcase> +<testcase name="group one › skipped" classname="a.spec.ts"> +<properties> +<property name="skip" value=""> +</property> +</properties> +<skipped> +</skipped> +</testcase> +<testcase name="group one › title with $(rm -rf) [x] (y)? *" classname="a.spec.ts" time="0.007"> +</testcase> +</testsuite> +</testsuites> \ No newline at end of file diff --git a/test/fixtures/junit/playwright-r2-inc.xml b/test/fixtures/junit/playwright-r2-inc.xml new file mode 100644 index 000000000..6f043478c --- /dev/null +++ b/test/fixtures/junit/playwright-r2-inc.xml @@ -0,0 +1,244 @@ +<testsuites id="" name="" tests="10" failures="2" skipped="2" errors="0" time="4.805343"> +<testsuite name="a.spec.ts" timestamp="2026-10-29T16:02:16.450Z" hostname="alpha" tests="5" failures="1" skipped="1" time="0.043" errors="0"> +<testcase name="group one › passes" classname="a.spec.ts" time="0.007"> +</testcase> +<testcase name="group one › fails always" classname="a.spec.ts" time="0.005"> +<system-out> +<![CDATA[ +[[ATTACHMENT|pw/test-results/a-group-one-fails-always-alpha/error-context.md]] +]]> +</system-out> +<rerunFailure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe" time="0.008"> +<stackTrace> +<![CDATA[Error: expect(received).toBe(expected) // Object.is equality + +Expected: 2 +Received: 1 + at /repo/tests/a.spec.ts:5:42]]> +</stackTrace> +<system-out> +<![CDATA[ +[[ATTACHMENT|pw/test-results/a-group-one-fails-always-alpha-retry1/error-context.md]] +]]> +</system-out> +</rerunFailure> +<rerunFailure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe" time="0.009"> +<stackTrace> +<![CDATA[Error: expect(received).toBe(expected) // Object.is equality + +Expected: 2 +Received: 1 + at /repo/tests/a.spec.ts:5:42]]> +</stackTrace> +<system-out> +<![CDATA[ +[[ATTACHMENT|pw/test-results/a-group-one-fails-always-alpha-retry2/error-context.md]] +]]> +</system-out> +</rerunFailure> +<failure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe"> +<![CDATA[ [alpha] › a.spec.ts:5:7 › group one › fails always ─────────────────────────────────────────────── + + Error: expect(received).toBe(expected) // Object.is equality + + Expected: 2 + Received: 1 + + 3 | test.describe('group one', () => { + 4 | test('passes', () => { expect(1).toBe(1); }); + > 5 | test('fails always', () => { expect(1).toBe(2); }); + | ^ + 6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } }); + 7 | test.skip('skipped', () => {}); + 8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); }); + at /repo/tests/a.spec.ts:5:42 + + Error Context: ../test-results/a-group-one-fails-always-alpha/error-context.md + + Retry #1 ─────────────────────────────────────────────────────────────────────────────────────── + + Error: expect(received).toBe(expected) // Object.is equality + + Expected: 2 + Received: 1 + + 3 | test.describe('group one', () => { + 4 | test('passes', () => { expect(1).toBe(1); }); + > 5 | test('fails always', () => { expect(1).toBe(2); }); + | ^ + 6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } }); + 7 | test.skip('skipped', () => {}); + 8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); }); + at /repo/tests/a.spec.ts:5:42 + + Error Context: ../test-results/a-group-one-fails-always-alpha-retry1/error-context.md + + Retry #2 ─────────────────────────────────────────────────────────────────────────────────────── + + Error: expect(received).toBe(expected) // Object.is equality + + Expected: 2 + Received: 1 + + 3 | test.describe('group one', () => { + 4 | test('passes', () => { expect(1).toBe(1); }); + > 5 | test('fails always', () => { expect(1).toBe(2); }); + | ^ + 6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } }); + 7 | test.skip('skipped', () => {}); + 8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); }); + at /repo/tests/a.spec.ts:5:42 + + Error Context: ../test-results/a-group-one-fails-always-alpha-retry2/error-context.md +]]> +</failure> +</testcase> +<testcase name="group one › flaky once" classname="a.spec.ts" time="0.002"> +<flakyFailure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe" time="0.009"> +<stackTrace> +<![CDATA[Error: expect(received).toBe(expected) // Object.is equality + +Expected: "second" +Received: "first" + at /repo/tests/a.spec.ts:6:152]]> +</stackTrace> +<system-out> +<![CDATA[ +[[ATTACHMENT|pw/test-results/a-group-one-flaky-once-alpha/error-context.md]] +]]> +</system-out> +</flakyFailure> +</testcase> +<testcase name="group one › skipped" classname="a.spec.ts"> +<properties> +<property name="skip" value=""> +</property> +</properties> +<skipped> +</skipped> +</testcase> +<testcase name="group one › title with $(rm -rf) [x] (y)? *" classname="a.spec.ts" time="0.003"> +</testcase> +</testsuite> +<testsuite name="a.spec.ts" timestamp="2026-10-29T16:02:16.450Z" hostname="beta" tests="5" failures="1" skipped="1" time="0.033" errors="0"> +<testcase name="group one › passes" classname="a.spec.ts" time="0.004"> +</testcase> +<testcase name="group one › fails always" classname="a.spec.ts" time="0.002"> +<system-out> +<![CDATA[ +[[ATTACHMENT|pw/test-results/a-group-one-fails-always-beta/error-context.md]] +]]> +</system-out> +<rerunFailure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe" time="0.006"> +<stackTrace> +<![CDATA[Error: expect(received).toBe(expected) // Object.is equality + +Expected: 2 +Received: 1 + at /repo/tests/a.spec.ts:5:42]]> +</stackTrace> +<system-out> +<![CDATA[ +[[ATTACHMENT|pw/test-results/a-group-one-fails-always-beta-retry1/error-context.md]] +]]> +</system-out> +</rerunFailure> +<rerunFailure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe" time="0.006"> +<stackTrace> +<![CDATA[Error: expect(received).toBe(expected) // Object.is equality + +Expected: 2 +Received: 1 + at /repo/tests/a.spec.ts:5:42]]> +</stackTrace> +<system-out> +<![CDATA[ +[[ATTACHMENT|pw/test-results/a-group-one-fails-always-beta-retry2/error-context.md]] +]]> +</system-out> +</rerunFailure> +<failure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe"> +<![CDATA[ [beta] › a.spec.ts:5:7 › group one › fails always ──────────────────────────────────────────────── + + Error: expect(received).toBe(expected) // Object.is equality + + Expected: 2 + Received: 1 + + 3 | test.describe('group one', () => { + 4 | test('passes', () => { expect(1).toBe(1); }); + > 5 | test('fails always', () => { expect(1).toBe(2); }); + | ^ + 6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } }); + 7 | test.skip('skipped', () => {}); + 8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); }); + at /repo/tests/a.spec.ts:5:42 + + Error Context: ../test-results/a-group-one-fails-always-beta/error-context.md + + Retry #1 ─────────────────────────────────────────────────────────────────────────────────────── + + Error: expect(received).toBe(expected) // Object.is equality + + Expected: 2 + Received: 1 + + 3 | test.describe('group one', () => { + 4 | test('passes', () => { expect(1).toBe(1); }); + > 5 | test('fails always', () => { expect(1).toBe(2); }); + | ^ + 6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } }); + 7 | test.skip('skipped', () => {}); + 8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); }); + at /repo/tests/a.spec.ts:5:42 + + Error Context: ../test-results/a-group-one-fails-always-beta-retry1/error-context.md + + Retry #2 ─────────────────────────────────────────────────────────────────────────────────────── + + Error: expect(received).toBe(expected) // Object.is equality + + Expected: 2 + Received: 1 + + 3 | test.describe('group one', () => { + 4 | test('passes', () => { expect(1).toBe(1); }); + > 5 | test('fails always', () => { expect(1).toBe(2); }); + | ^ + 6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } }); + 7 | test.skip('skipped', () => {}); + 8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); }); + at /repo/tests/a.spec.ts:5:42 + + Error Context: ../test-results/a-group-one-fails-always-beta-retry2/error-context.md +]]> +</failure> +</testcase> +<testcase name="group one › flaky once" classname="a.spec.ts" time="0.002"> +<flakyFailure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe" time="0.01"> +<stackTrace> +<![CDATA[Error: expect(received).toBe(expected) // Object.is equality + +Expected: "second" +Received: "first" + at /repo/tests/a.spec.ts:6:152]]> +</stackTrace> +<system-out> +<![CDATA[ +[[ATTACHMENT|pw/test-results/a-group-one-flaky-once-beta/error-context.md]] +]]> +</system-out> +</flakyFailure> +</testcase> +<testcase name="group one › skipped" classname="a.spec.ts"> +<properties> +<property name="skip" value=""> +</property> +</properties> +<skipped> +</skipped> +</testcase> +<testcase name="group one › title with $(rm -rf) [x] (y)? *" classname="a.spec.ts" time="0.003"> +</testcase> +</testsuite> +</testsuites> \ No newline at end of file diff --git a/test/fixtures/junit/playwright-r2.xml b/test/fixtures/junit/playwright-r2.xml new file mode 100644 index 000000000..40877b668 --- /dev/null +++ b/test/fixtures/junit/playwright-r2.xml @@ -0,0 +1,178 @@ +<testsuites id="" name="" tests="10" failures="2" skipped="2" errors="0" time="4.985760999999999"> +<testsuite name="a.spec.ts" timestamp="2026-10-29T16:02:10.182Z" hostname="alpha" tests="5" failures="1" skipped="1" time="0.044" errors="0"> +<testcase name="group one › passes" classname="a.spec.ts" time="0.005"> +</testcase> +<testcase name="group one › fails always" classname="a.spec.ts" time="0.021"> +<system-out> +<![CDATA[ +[[ATTACHMENT|pw/test-results/a-group-one-fails-always-alpha/error-context.md]] + +[[ATTACHMENT|pw/test-results/a-group-one-fails-always-alpha-retry1/error-context.md]] + +[[ATTACHMENT|pw/test-results/a-group-one-fails-always-alpha-retry2/error-context.md]] +]]> +</system-out> +<failure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe"> +<![CDATA[ [alpha] › a.spec.ts:5:7 › group one › fails always ─────────────────────────────────────────────── + + Error: expect(received).toBe(expected) // Object.is equality + + Expected: 2 + Received: 1 + + 3 | test.describe('group one', () => { + 4 | test('passes', () => { expect(1).toBe(1); }); + > 5 | test('fails always', () => { expect(1).toBe(2); }); + | ^ + 6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } }); + 7 | test.skip('skipped', () => {}); + 8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); }); + at /repo/tests/a.spec.ts:5:42 + + Error Context: ../test-results/a-group-one-fails-always-alpha/error-context.md + + Retry #1 ─────────────────────────────────────────────────────────────────────────────────────── + + Error: expect(received).toBe(expected) // Object.is equality + + Expected: 2 + Received: 1 + + 3 | test.describe('group one', () => { + 4 | test('passes', () => { expect(1).toBe(1); }); + > 5 | test('fails always', () => { expect(1).toBe(2); }); + | ^ + 6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } }); + 7 | test.skip('skipped', () => {}); + 8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); }); + at /repo/tests/a.spec.ts:5:42 + + Error Context: ../test-results/a-group-one-fails-always-alpha-retry1/error-context.md + + Retry #2 ─────────────────────────────────────────────────────────────────────────────────────── + + Error: expect(received).toBe(expected) // Object.is equality + + Expected: 2 + Received: 1 + + 3 | test.describe('group one', () => { + 4 | test('passes', () => { expect(1).toBe(1); }); + > 5 | test('fails always', () => { expect(1).toBe(2); }); + | ^ + 6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } }); + 7 | test.skip('skipped', () => {}); + 8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); }); + at /repo/tests/a.spec.ts:5:42 + + Error Context: ../test-results/a-group-one-fails-always-alpha-retry2/error-context.md +]]> +</failure> +</testcase> +<testcase name="group one › flaky once" classname="a.spec.ts" time="0.014"> +<system-out> +<![CDATA[ +[[ATTACHMENT|pw/test-results/a-group-one-flaky-once-alpha/error-context.md]] +]]> +</system-out> +</testcase> +<testcase name="group one › skipped" classname="a.spec.ts"> +<properties> +<property name="skip" value=""> +</property> +</properties> +<skipped> +</skipped> +</testcase> +<testcase name="group one › title with $(rm -rf) [x] (y)? *" classname="a.spec.ts" time="0.004"> +</testcase> +</testsuite> +<testsuite name="a.spec.ts" timestamp="2026-10-29T16:02:10.182Z" hostname="beta" tests="5" failures="1" skipped="1" time="0.04" errors="0"> +<testcase name="group one › passes" classname="a.spec.ts" time="0.005"> +</testcase> +<testcase name="group one › fails always" classname="a.spec.ts" time="0.019"> +<system-out> +<![CDATA[ +[[ATTACHMENT|pw/test-results/a-group-one-fails-always-beta/error-context.md]] + +[[ATTACHMENT|pw/test-results/a-group-one-fails-always-beta-retry1/error-context.md]] + +[[ATTACHMENT|pw/test-results/a-group-one-fails-always-beta-retry2/error-context.md]] +]]> +</system-out> +<failure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe"> +<![CDATA[ [beta] › a.spec.ts:5:7 › group one › fails always ──────────────────────────────────────────────── + + Error: expect(received).toBe(expected) // Object.is equality + + Expected: 2 + Received: 1 + + 3 | test.describe('group one', () => { + 4 | test('passes', () => { expect(1).toBe(1); }); + > 5 | test('fails always', () => { expect(1).toBe(2); }); + | ^ + 6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } }); + 7 | test.skip('skipped', () => {}); + 8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); }); + at /repo/tests/a.spec.ts:5:42 + + Error Context: ../test-results/a-group-one-fails-always-beta/error-context.md + + Retry #1 ─────────────────────────────────────────────────────────────────────────────────────── + + Error: expect(received).toBe(expected) // Object.is equality + + Expected: 2 + Received: 1 + + 3 | test.describe('group one', () => { + 4 | test('passes', () => { expect(1).toBe(1); }); + > 5 | test('fails always', () => { expect(1).toBe(2); }); + | ^ + 6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } }); + 7 | test.skip('skipped', () => {}); + 8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); }); + at /repo/tests/a.spec.ts:5:42 + + Error Context: ../test-results/a-group-one-fails-always-beta-retry1/error-context.md + + Retry #2 ─────────────────────────────────────────────────────────────────────────────────────── + + Error: expect(received).toBe(expected) // Object.is equality + + Expected: 2 + Received: 1 + + 3 | test.describe('group one', () => { + 4 | test('passes', () => { expect(1).toBe(1); }); + > 5 | test('fails always', () => { expect(1).toBe(2); }); + | ^ + 6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } }); + 7 | test.skip('skipped', () => {}); + 8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); }); + at /repo/tests/a.spec.ts:5:42 + + Error Context: ../test-results/a-group-one-fails-always-beta-retry2/error-context.md +]]> +</failure> +</testcase> +<testcase name="group one › flaky once" classname="a.spec.ts" time="0.012"> +<system-out> +<![CDATA[ +[[ATTACHMENT|pw/test-results/a-group-one-flaky-once-beta/error-context.md]] +]]> +</system-out> +</testcase> +<testcase name="group one › skipped" classname="a.spec.ts"> +<properties> +<property name="skip" value=""> +</property> +</properties> +<skipped> +</skipped> +</testcase> +<testcase name="group one › title with $(rm -rf) [x] (y)? *" classname="a.spec.ts" time="0.004"> +</testcase> +</testsuite> +</testsuites> \ No newline at end of file diff --git a/test/fixtures/module-size-ratchet.json b/test/fixtures/module-size-ratchet.json index a91653450..d56450722 100644 --- a/test/fixtures/module-size-ratchet.json +++ b/test/fixtures/module-size-ratchet.json @@ -101,6 +101,7 @@ { "file": "lib/result-codes-release.ts", "movedFrom": "lib/result-codes.ts (the release-path rows: ship queue, pregate timing, approval classes, state matrix; field list 2 wave B)" }, { "file": "lib/result-codes-capy.ts", "movedFrom": "lib/result-codes.ts (the Capy rows: approval record, Captain brief, PR screenshots; GSTA-61)" }, { "file": "lib/result-codes-budget.ts", "movedFrom": "lib/result-codes.ts (the /ship token budget and handoff rows; GSTA-152)" }, + { "file": "lib/result-codes-eval.ts", "movedFrom": "lib/result-codes.ts (the /eval-plan preregistration rows and the no-gstack arm; GSTA-168)" }, { "file": "lib/ship-budget.ts", "movedFrom": "(new: /ship token budget ledger, meters and admission, GSTA-152)" }, { "file": "lib/ship-budget-cli.ts", "movedFrom": "(new: bin/gstack-ship-budget start/check/settle/close/resume, GSTA-152)" }, { "file": "lib/ship-handoff.ts", "movedFrom": "(new: /ship fresh-context handoff, verify, return and reconcile, GSTA-152)" }, diff --git a/test/fixtures/review-ledger/gsta-196-native.txt b/test/fixtures/review-ledger/gsta-196-native.txt new file mode 100644 index 000000000..8c273413b --- /dev/null +++ b/test/fixtures/review-ledger/gsta-196-native.txt @@ -0,0 +1,43 @@ +Adversarial review (native) of `1b68e637c07c..0a8efed`. HEAD is still `0a8efeda8f79`, the working tree is clean and the same 10 files changed, so the findings below are the ones I reported earlier, rechecked against this range. + +**Coverage:** I read the full diffs of the non-fixture source: `.github/workflows/{ci,sealed,train-gate}.yml`, `scripts/gate.ts`, `gate/manifest.json`, `package.json` and `README.md`. **I reviewed the test files in summary mode only**, so their coverage of the review is reduced: +- `tests/ci/gate.test.ts` is new (+299). It covers the train chain walk, `routeDispatch` refusals, `publishable` binding and the gate-surface rule, and `runCases`/junit judging. +- `tests/ci/sealed-workflow.test.ts` (+19/−6) adds the push trigger. +- `tests/ci/workflow-hygiene.test.ts` is new (+85). It covers SHA pins, read-only tokens, no secrets, the checksummed Bun install, the gstack pin and parity between the manifest and ci.yml. + +**1. A PR in the train controls the workflow that judges the train (INVESTIGATE).** `workflow_dispatch` runs the copy of `train-gate.yml` at the dispatched ref, `train/<nonce>`, and that copy contains every train PR's edits. So a PR that edits `train-gate.yml` changes the `route`, `test` and `publish` jobs themselves. +- It can drop the call to the base's `gate.ts publish`. It can also post the `full_gate_check` check run unconditionally with the publish job's `checks: write`, or give itself more `permissions:` and use repository secrets. +- The header says "a pull request in the train cannot change how the train is judged". In practice only the case list and `gate.ts` come from the base. +- The "a gate surface PR rides alone" rule (`publishable`, `slots > 1`) is enforced by the publish job, which is code the PR can rewrite. So the rule holds only against PRs that don't break it. +- `ci.yml`'s new `workflow_dispatch` has the same property. +- Real mitigations would run the judging workflow from the default branch, for example `repository_dispatch` or a dispatch on `main` that takes the train head as an input, or have gstack verify which workflow file produced the check. Either one changes the protocol with gstack. +- Today an attacker needs a malicious workflow edit to pass review and READY, and no workflow references a secret. + +Severity: High. Blocks: no. + +**2. A case timeout leaks processes and can lose the record (FIXABLE).** `runCases` calls `spawnSync(..., { timeout })` (`scripts/gate.ts:252`), which kills only the `bash` wrapper. Grandchildren such as `bun test` survive with inherited stdio and keep running into later cases. +- A 20-minute case timeout (`CASE_TIMEOUT_MS`) plus the other cases can go past the job's 30-minute `timeout-minutes`. +- GitHub then kills `gate.ts` before it writes `train-record.json`. That run has no record and fails as infrastructure, not as a red verdict. +- Fix: spawn each case detached and kill its whole process group on timeout, and keep the per-case cap well under the job limit (or size the job to cases × cap). + +Severity: Low. Blocks: no. + +**3. Local `run` hides a broken train chain (FIXABLE).** In `gate.ts run` (`:386`), when `trainBase` returns `{ error }`, the code falls back to `merge-base HEAD origin/<default>` without saying so. A malformed train run locally is judged against the wrong base. +- CI always passes `--base`, so this affects only local `bun run gate`. +- Fix: fail on an error result, the way `route` and `publish` already do. + +Severity: Low. Blocks: no. + +**4. No train policy is on the base, so trains cannot run yet (INVESTIGATE).** `.gstack/ship-policy.json` does not exist at HEAD. Until a commit with a `train` block lands on main, `route` refuses every train and bisect dispatch with "needs a train block". The workflow cannot gate a train at merge, and this PR does not add the file. +- Related: `bun.lock` is not in `gate_surface`. A multi-PR train can change resolved dependency versions within `package.json` ranges, for example `@types/bun ^1.4.4`, which changes what `typecheck` and `unit` run against. + +Severity: Low. Blocks: no. + +**Ruled out:** +- `grep -q` after `tee` in the sealed case is not a SIGPIPE risk. `SEALED:` is the last and only line on gstack-pregate's stdout. +- The empty wiki `--base` on dispatch is safe: `resolveBase` falls back to `origin/HEAD`. +- `Bun.Glob("**/tsconfig*.json")` matches `tsconfig.json` at the repository root. +- The bisect artifact name matches gstack's `bisectArtifact`. +- `route` refuses when the bash T1 lookup and `trainBase` disagree. + +Recommendation: Ship as-is and send finding 1 to the train-protocol owner because the strongest issue, a train PR rewriting `train-gate.yml` to post `full_gate_check` itself, needs a malicious workflow edit to pass review, and fixing it means changing the dispatch protocol with gstack, not this diff. \ No newline at end of file diff --git a/test/fixtures/review-ledger/gsta-196-refusal.txt b/test/fixtures/review-ledger/gsta-196-refusal.txt new file mode 100644 index 000000000..c1eb77c28 --- /dev/null +++ b/test/fixtures/review-ledger/gsta-196-refusal.txt @@ -0,0 +1,5 @@ +I couldn't run the review because the diff base doesn't exist. `git diff 1b68e637c07cea38f01c01120e02d184575da82c` fails with "bad object", and fetching that sha from origin is refused with "not our ref", so the commit is neither in this checkout nor on the remote. + +This checkout is on `main` at 047dad7 (v1.91.357.0), tracking origin/main with no uncommitted changes, so there's nothing local to review against either. I'm not returning NO FINDINGS, because that would claim a clean diff I never saw. + +To continue I need one of these: a diff base that exists, or the branch or sha holding the c3-36-p1 change. It may only exist on another machine and was never pushed. I've read all three checklists (testing, security, state-machines) and will apply them as soon as I have a diff. \ No newline at end of file diff --git a/test/flaky-run.test.ts b/test/flaky-run.test.ts new file mode 100644 index 000000000..c890b5883 --- /dev/null +++ b/test/flaky-run.test.ts @@ -0,0 +1,459 @@ +/** + * gstack-flaky run (GSTA-158): the JUnit reader against reports captured from + * Playwright 1.64 and bun 1.4 (test/fixtures/junit, paths rewritten to /repo), + * the rerun-once rules, and the bin end to end with a fake runner. + */ +import { afterAll, describe, expect, test } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { spawn, spawnSync } from 'child_process'; +import { JunitError, parseJunit, readJunit, testId } from '../lib/junit'; +import { appendLedger, incompleteRun, ledgerPath, renderResult, rerunSelection, runFlaky, safeExcerpt, stepSummary, unreportedFailures, workflowEscape } from '../lib/flaky-run'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const FIX = path.join(ROOT, 'test', 'fixtures', 'junit'); +const fixture = (name: string) => fs.readFileSync(path.join(FIX, name), 'utf-8'); +const TMP = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-flaky-')); +const tmp = () => fs.mkdtempSync(path.join(TMP, 't-')); +afterAll(() => fs.rmSync(TMP, { recursive: true, force: true })); + +describe('JUnit reader', () => { + test('bun: nested describe path, skip, failure excerpt', () => { + const r = parseJunit(fixture('bun.xml')); + expect(r.runner).toBe('bun'); + expect(r.tests.map(t => [t.id, t.attempts.join(',')])).toEqual([ + ['a.test.ts::::outer > inner > passes', 'pass'], + ['a.test.ts::::outer > inner > fails', 'fail'], + ['a.test.ts::::outer > skipped', 'skip'], + ['a.test.ts::::top level', 'pass'], + ]); + expect(r.tests[1]!.excerpt[0]).toContain('expect(received).toBe(expected)'); + }); + + test('playwright: one id per project, retry-aware attempts', () => { + const r = parseJunit(fixture('playwright-r2-inc.xml')); + expect(r.runner).toBe('playwright'); + expect(r.retriesHidden).toBe(false); + const byId = Object.fromEntries(r.tests.map(t => [t.id, t.attempts.join(',')])); + expect(byId['a.spec.ts::alpha::group one > fails always']).toBe('fail,fail,fail'); + expect(byId['a.spec.ts::beta::group one > fails always']).toBe('fail,fail,fail'); + expect(byId['a.spec.ts::alpha::group one > flaky once']).toBe('fail,pass'); + expect(byId['a.spec.ts::alpha::group one > skipped']).toBe('skip'); + expect(byId['a.spec.ts::alpha::group one > title with $(rm -rf) [x] (y)? *']).toBe('pass'); + }); + + test('playwright retries without retry data are detected, not read as one attempt', () => { + expect(parseJunit(fixture('playwright-r2.xml')).retriesHidden).toBe(true); + expect(parseJunit(fixture('playwright-r0.xml')).retriesHidden).toBe(false); + }); + + test('missing, empty, zero-testcase and malformed reports are errors, never green', () => { + const dir = tmp(); + const code = (fn: () => unknown) => { try { fn(); return 'ok'; } catch (e) { return (e as JunitError).code; } }; + expect(code(() => readJunit(path.join(dir, 'absent.xml')))).toBe('FLAKY_NO_RESULTS'); + fs.writeFileSync(path.join(dir, 'empty.xml'), ' \n'); + expect(code(() => readJunit(path.join(dir, 'empty.xml')))).toBe('FLAKY_NO_RESULTS'); + expect(code(() => parseJunit('<testsuites name="x"><testsuite name="s" tests="0"/></testsuites>'))).toBe('FLAKY_NO_RESULTS'); + expect(code(() => parseJunit('<testsuites><testsuite name="s"><testcase name="a"></testsuite></testsuites>'))).toBe('FLAKY_JUNIT_INVALID'); + }); + + test('generic runners (jest, pytest): file from the testcase, entities decoded', () => { + const r = parseJunit('<?xml version="1.0"?><testsuites><testsuite name="pytest"><testcase classname="tests.test_a" name="test_x[a&b]" file="tests/test_a.py"><error message="boom"/></testcase></testsuite></testsuites>'); + expect(r.runner).toBe('generic'); + expect(r.tests[0]!.id).toBe('tests/test_a.py::::test_x[a&b]'); + expect(r.tests[0]!.attempts).toEqual(['fail']); + }); +}); + +describe('rerun selection', () => { + const pw = parseJunit(fixture('playwright-r0.xml')).tests.find(t => t.path.at(-1)!.startsWith('title with') && t.project === 'beta')!; + test('playwright: file and title regex-escaped, project kept, each one argv element', () => { + expect(rerunSelection('playwright', pw)).toEqual([ + 'a\\.spec\\.ts', '--grep', '(?:^|\\s)group one title with \\$\\(rm -rf\\) \\[x\\] \\(y\\)\\? \\*(?:\\s|$)', '--project', 'beta', + ]); + }); + test('bun: anchored space-joined path', () => { + const t = parseJunit(fixture('bun.xml')).tests[1]!; + expect(rerunSelection('bun', t)).toEqual(['a.test.ts', '--test-name-pattern', '^outer inner fails$']); + }); +}); + +describe('complete-run proof for a targeted rerun', () => { + test('a complete run needs exit 1 and the runner summary without an early stop', () => { + expect(incompleteRun('bun', ' 4 pass\n 1 fail\nRan 5 tests across 2 files. [9ms]\n', 1)).toBeNull(); + expect(incompleteRun('bun', 'Bailed out after 1 failure\nRan 1 test across 1 file.\n', 1)).toContain('--bail'); + expect(incompleteRun('bun', ' 1 fail\n', 1)).toContain('no run summary'); + expect(incompleteRun('bun', 'Ran 5 tests across 2 files.\n', 2)).toContain('exited 2'); + expect(incompleteRun('playwright', ' 1 failed\n [alpha] x\n 4 passed (1s)\n', 1)).toBeNull(); + expect(incompleteRun('playwright', 'Testing stopped early after 1 maximum allowed failures.\n 1 failed\n 2 did not run\n', 1)).toContain('2 did not run'); + expect(incompleteRun('playwright', ' 1 failed\n 3 interrupted\n', 1)).toContain('3 interrupted'); + expect(incompleteRun('playwright', 'Timed out waiting 600s for the test suite to run\nTimed out waiting 600s for the entire test run\n 1 failed\n', 1)).toContain('stopped early'); + expect(incompleteRun('playwright', '', 1)).toContain('no summary'); + expect(incompleteRun('generic', 'Ran 5 tests', 1)).toContain('generic'); + expect(incompleteRun('bun', 'All files | % Funcs | % Lines |\nRan 5 tests across 2 files.\n', 1)).toContain('coverage'); + expect(incompleteRun('bun', 'Ran 5 tests across 2 files.\n', 1, ['bun', 'test', '--coverage'])).toContain('coverage'); + expect(incompleteRun('playwright', ' 1 failed\n 1 flaky\n 4 passed\n', 1, [], 1)).toContain('fail-on-flaky'); + }); +}); + +/** A fake runner: argv selection decides which canned report the attempt writes. */ +const BUN_SUMMARY = 'Ran 2 tests across 1 file. [3ms]\n'; + +function fakeRun(reports: (argv: string[], n: number) => string | null, exits: (argv: string[], n: number) => number, outputs: (argv: string[], n: number) => string = () => BUN_SUMMARY, signals: (n: number) => string | undefined = () => undefined) { + const calls: string[][] = []; + return { + calls, + spawn(junit: string) { + return async (argv: string[], _env: NodeJS.ProcessEnv, cwd: string) => { + calls.push(argv); + const xml = reports(argv, calls.length); + if (xml !== null) fs.writeFileSync(path.resolve(cwd, junit), xml); + const signal = signals(calls.length); + return { exit: exits(argv, calls.length), output: outputs(argv, calls.length), ...(signal ? { signal } : {}) }; + }; + }, + }; +} + +const suite = (cases: string) => `<testsuites name="bun test"><testsuite name="a.test.ts" file="a.test.ts">${cases}</testsuite></testsuites>`; +const pass = (name: string) => `<testcase name="${name}" classname="" file="a.test.ts"/>`; +const fail = (name: string, msg = 'boom') => `<testcase name="${name}" classname="" file="a.test.ts"><failure message="${msg}"/></testcase>`; + +describe('runFlaky', () => { + const base = (spawn: ReturnType<typeof fakeRun>, dir: string, extra: Partial<Parameters<typeof runFlaky>[0]> = {}) => + runFlaky({ command: ['bun', 'test'], junit: 'r.xml', maxReruns: 3, cwd: dir, spawn: spawn.spawn('r.xml'), env: {}, now: () => new Date('2026-10-29T00:00:00Z'), ...extra }); + + test('a green run with no failures is green and reruns nothing', async () => { + const dir = tmp(); + const f = fakeRun(() => suite(pass('a') + pass('b')), () => 0); + expect(await base(f, dir)).toMatchObject({ exitCode: 0, reruns: 0, ledger: [] }); + expect(f.calls).toHaveLength(1); + }); + + test('a coverage failure beside a flaky test is not cleared by a narrowed rerun', async () => { + const dir = tmp(); + const cov = () => '----|---------|---------|\nAll files | % Funcs | % Lines |\nRan 2 tests across 1 file.\n'; + const f = fakeRun((_a, n) => n === 1 ? suite(pass('a') + fail('b')) : suite(pass('a') + pass('b')), () => 1, cov); + const r = await base(f, dir, { rerunScope: 'test' }); + expect(f.calls[1]).toEqual(['bun', 'test']); + expect(r).toMatchObject({ exitCode: 1, wholeRerun: 'coverage is measured, and a narrowed rerun cannot reproduce a coverage failure' }); + }); + + test('by default the rerun is the whole command and counts only on its full green verdict', async () => { + const dir = tmp(); + const green = fakeRun((_a, n) => n === 1 ? suite(pass('a') + fail('b')) : suite(pass('a') + pass('b')), (_a, n) => n === 1 ? 1 : 0); + const r = await base(green, dir); + expect(green.calls).toEqual([['bun', 'test'], ['bun', 'test']]); + expect(r).toMatchObject({ exitCode: 0, wholeRerun: 'the default; --rerun-scope test reruns the failed test alone' }); + expect(r.ledger[0]).toMatchObject({ kind: 'flaky-pass', rerun_scope: 'command' }); + const otherFails = fakeRun((_a, n) => n === 1 ? suite(pass('a') + fail('b')) : suite(fail('a') + pass('b')), (_a, n) => n === 1 ? 1 : 0); + expect((await base(otherFails, dir)).outcomes[0]).toMatchObject({ status: 'rerun-fail', reason: 'rerun: 1 test(s) failed in the whole-command rerun' }); + }); + + test('a failed test that passes on rerun is a flaky pass and goes on the ledger', async () => { + const dir = tmp(); + const f = fakeRun((argv, n) => n === 1 ? suite(pass('a') + fail('b')) : suite(pass('b')), (_a, n) => n === 1 ? 1 : 0); + const r = await base(f, dir, { rerunScope: 'test' }); + expect(r.exitCode).toBe(0); + expect(f.calls[1]).toEqual(['bun', 'test', 'a.test.ts', '--test-name-pattern', '^b$']); + expect(r.outcomes.map(o => [o.status, o.attempts.join(',')])).toEqual([['flaky-pass', 'fail,pass']]); + expect(r.ledger).toHaveLength(1); + expect(r.ledger[0]).toMatchObject({ kind: 'flaky-pass', id: 'a.test.ts::::b', rerun_by: 'gstack-flaky', attempts: ['fail', 'pass'], excerpt: ['boom'] }); + expect(renderResult(r, '/l.jsonl').at(-1)).toBe('FLAKY_RESULT: tests=2 failed=0 flaky=1 reruns=1 command_exit=1 verdict=pass ledger=/l.jsonl'); + expect(stepSummary(r)).toContain('a.test.ts::::b'); + }); + + test('a test that fails its rerun is red and is ledgered as rerun-fail', async () => { + const dir = tmp(); + const f = fakeRun(() => suite(fail('b')), () => 1); + const r = await base(f, dir); + expect(r.exitCode).toBe(1); + expect(r.outcomes[0]).toMatchObject({ status: 'rerun-fail', attempts: ['fail', 'fail'] }); + expect(r.ledger[0]!.kind).toBe('rerun-fail'); + }); + + test('more failures than --max-reruns: a real red, nothing rerun', async () => { + const dir = tmp(); + const f = fakeRun(() => suite(fail('a') + fail('b') + fail('c') + fail('d')), () => 1); + const r = await base(f, dir); + expect(r.exitCode).toBe(1); + expect(f.calls).toHaveLength(1); + expect(r.reruns).toBe(0); + expect(r.outcomes.every(o => o.status === 'fail' && /more than --max-reruns 3/.test(o.reason!))).toBe(true); + expect(r.ledger).toEqual([]); + }); + + test('runner retries stand: no second rerun, and a runner flaky pass is ledgered', async () => { + const dir = tmp(); + const f = fakeRun(() => fixture('playwright-r2-inc.xml'), () => 1); + const r = await base(f, dir); + expect(f.calls).toHaveLength(1); + expect(r.exitCode).toBe(1); + expect(r.ledger.filter(l => l.kind === 'flaky-pass').map(l => [l.id, l.rerun_by])).toEqual([ + ['a.spec.ts::alpha::group one > flaky once', 'runner'], ['a.spec.ts::beta::group one > flaky once', 'runner'], + ]); + expect(r.outcomes.filter(o => o.status === 'fail').every(o => /already retried \(3 attempts\)/.test(o.reason!))).toBe(true); + }); + + test('hidden Playwright retries refuse with a code instead of a guess', async () => { + const dir = tmp(); + const r = await base(fakeRun(() => fixture('playwright-r2.xml'), () => 1), dir); + expect([r.exitCode, r.code]).toEqual([3, 'FLAKY_RETRIES_HIDDEN']); + }); + + test('no report, a stale report, and a crash with no failed test are never green', async () => { + const dir = tmp(); + expect(await base(fakeRun(() => null, () => 0), dir)).toMatchObject({ exitCode: 3, code: 'FLAKY_NO_RESULTS' }); + fs.writeFileSync(path.join(dir, 'r.xml'), suite(pass('stale'))); + expect(await base(fakeRun(() => null, () => 0), dir)).toMatchObject({ exitCode: 3, code: 'FLAKY_NO_RESULTS' }); + expect(await base(fakeRun(() => suite(pass('a')), () => 2), dir)).toMatchObject({ exitCode: 3, code: 'FLAKY_COMMAND_FAILED', commandExit: 2 }); + }); + + test('a rerun that does not report the targeted test fails it; other tests in the rerun are ignored', async () => { + const dir = tmp(); + const f = fakeRun((_a, n) => n === 1 ? suite(fail('b')) : suite(pass('other')), (_a, n) => n === 1 ? 1 : 0); + const r = await base(f, dir); + expect(r.exitCode).toBe(1); + expect(r.outcomes[0]).toMatchObject({ status: 'rerun-fail', reason: 'rerun: the report does not contain this test' }); + }); + + test('a rerun whose command exits non-zero is not a flaky pass, even with a passing report', async () => { + const dir = tmp(); + const f = fakeRun((_a, n) => n === 1 ? suite(fail('b')) : suite(pass('b')), () => 1); + expect((await base(f, dir)).outcomes[0]).toMatchObject({ status: 'rerun-fail', reason: 'rerun: the command exited 1' }); + }); + + test('a test reported twice with one failure is red and not rerun', async () => { + const dir = tmp(); + const r = parseJunit(suite(pass('b') + fail('b'))); + expect(r.tests[0]).toMatchObject({ executions: 2, attempts: ['pass', 'fail'] }); + const f = fakeRun(() => suite(fail('b') + pass('b') + fail('c')), () => 1); + const out = await base(f, dir, { rerunScope: 'test' }); + expect(out.outcomes.find(o => o.test.id.endsWith('::b'))).toMatchObject({ status: 'fail', reason: 'it ran 2 times in this report and one failed; not rerun' }); + expect(f.calls).toHaveLength(2); + }); + + test('a runner-caught flake is ledgered even when a duplicate execution of the test was skipped', async () => { + const dir = tmp(); + const caught = '<testcase name="b" classname="" file="a.test.ts"><flakyFailure message="x"/></testcase>'; + const r = await base(fakeRun(() => suite(caught + '<testcase name="b" classname="" file="a.test.ts"><skipped/></testcase>'), () => 0), dir); + expect(r.exitCode).toBe(0); + expect(r.ledger.map(l => [l.kind, l.rerun_by])).toEqual([['flaky-pass', 'runner']]); + }); + + test('the Playwright environment flag reaches every attempt', async () => { + const dir = tmp(); + const seen: string[] = []; + await runFlaky({ command: ['x'], junit: 'r.xml', maxReruns: 3, cwd: dir, env: {}, spawn: async (_a, env, cwd) => { + seen.push(env.PLAYWRIGHT_JUNIT_INCLUDE_RETRIES ?? ''); + fs.writeFileSync(path.join(cwd, 'r.xml'), suite(seen.length === 1 ? fail('b') : pass('b'))); + return { exit: seen.length === 1 ? 1 : 0, output: '' }; + } }); + expect(seen).toEqual(['1', '1']); + }); + + test('bun load errors the JUnit omits keep the run red and nothing is rerun', async () => { + const dir = tmp(); + const f = fakeRun(() => suite(fail('b')), () => 1, () => '\n 0 pass\n 3 fail\n 2 errors\nRan 3 tests across 3 files.\n'); + const r = await base(f, dir); + expect(r).toMatchObject({ exitCode: 1, code: 'FLAKY_UNREPORTED_FAILURES', reruns: 0 }); + expect(f.calls).toHaveLength(1); + }); + + test('runner summaries: Playwright errors outside tests and count mismatches are unreported failures', () => { + expect(unreportedFailures('playwright', ' 1 error was not a part of any test, see above for details\n 1 failed\n', 1)).toContain('outside any test'); + expect(unreportedFailures('playwright', ' 2 failed\n 10 passed\n', 1)).toContain('counted 2 failed'); + expect(unreportedFailures('playwright', ' 1 failed\n [chromium] x\n 10 passed\n', 1)).toBeNull(); + expect(unreportedFailures('bun', ' 4 pass\n 1 fail\n', 1)).toBeNull(); + expect(unreportedFailures('generic', '5 failed', 0)).toBeNull(); + }); + + test('a run that stopped early reruns the whole command, so tests it never reached must pass too', async () => { + const dir = tmp(); + const bail = () => 'Bailed out after 1 failure\nRan 1 test across 1 file.\n'; + const green = fakeRun((_a, n) => n === 1 ? suite(fail('b')) : suite(pass('b') + pass('never-reached')), (_a, n) => n === 1 ? 1 : 0, bail); + const r = await base(green, dir, { rerunScope: 'test' }); + expect(green.calls).toEqual([['bun', 'test'], ['bun', 'test']]); + expect(r).toMatchObject({ exitCode: 0, wholeRerun: 'bun stopped early (--bail)' }); + expect(r.ledger[0]).toMatchObject({ kind: 'flaky-pass', rerun_scope: 'command' }); + expect(renderResult(r, '/l')).toContain('FLAKY_RERUN: scope=command (bun stopped early (--bail))'); + const red = fakeRun((_a, n) => n === 1 ? suite(fail('b')) : suite(pass('b') + fail('never-reached')), () => 1, bail); + expect((await base(red, dir, { rerunScope: 'test' })).outcomes[0]).toMatchObject({ status: 'rerun-fail', reason: 'rerun: the command exited 1' }); + }); + + test('generic runners rerun the whole command even when a targeted rerun is asked for', async () => { + const dir = tmp(); + const f = fakeRun((_a, n) => n === 1 ? suite(fail('b')) : suite(pass('b')), (_a, n) => n === 1 ? 1 : 0); + const r = await base(f, dir, { runner: 'generic', rerunScope: 'test' }); + expect(f.calls[1]).toEqual(['bun', 'test']); + expect(r).toMatchObject({ exitCode: 0, wholeRerun: 'a generic runner cannot prove every test ran' }); + }); + + test('a whole-command rerun clears nothing when any test fails in it, a first-run test is missing, it is killed or it retried', async () => { + const dir = tmp(); + const generic = (second: string, exit = 0, signal?: string) => base(fakeRun((_a, n) => n === 1 ? suite(fail('a') + pass('b')) : second, () => exit && 1, undefined, n => n === 2 ? signal : undefined), dir, { runner: 'generic' }); + expect((await generic(suite(pass('a') + fail('b')))).outcomes[0]).toMatchObject({ status: 'rerun-fail', reason: 'rerun: 1 test(s) failed in the whole-command rerun' }); + expect((await generic(suite(pass('a')))).outcomes[0]).toMatchObject({ status: 'rerun-fail', reason: 'rerun: 1 test(s) of the first run are missing from the whole-command rerun' }); + expect((await generic(suite(pass('a') + pass('b')), 0, 'SIGTERM')).outcomes[0]).toMatchObject({ status: 'rerun-fail', reason: 'rerun: killed by SIGTERM' }); + const retried = await generic(suite('<testcase name="a" classname="" file="a.test.ts"><flakyFailure message="x"/></testcase>' + pass('b'))); + expect(retried.outcomes.find(o => o.status === 'rerun-fail')).toMatchObject({ attempts: ['fail', 'fail', 'pass'] }); + expect(retried.ledger.find(l => l.kind === 'rerun-fail')).toMatchObject({ rerun_scope: 'command' }); + expect(retried.exitCode).toBe(1); + }); + + test('an incomplete run that is already red is not rerun as a whole', async () => { + const dir = tmp(); + const f = fakeRun(() => suite(fail('b') + pass('b') + fail('c')), () => 1, () => ''); + const r = await base(f, dir); + expect(f.calls).toHaveLength(1); + expect(r.outcomes.find(o => o.test.id.endsWith('::c'))!.reason).toContain('already red'); + }); + + test('a rerun the runner retried again does not clear the test', async () => { + const dir = tmp(); + const f = fakeRun((_a, n) => n === 1 ? suite(fail('b')) : suite('<testcase name="b" classname="" file="a.test.ts"><flakyFailure message="x"/></testcase>'), (_a, n) => n === 1 ? 1 : 0); + expect((await base(f, dir)).outcomes[0]).toMatchObject({ status: 'rerun-fail', attempts: ['fail', 'fail', 'pass'] }); + }); + + test('a killed runner is never green and stops further reruns', async () => { + const dir = tmp(); + expect(await base(fakeRun(() => suite(fail('b')), () => 128, undefined, () => 'SIGTERM'), dir)).toMatchObject({ exitCode: 3, code: 'FLAKY_COMMAND_KILLED' }); + const f = fakeRun((_a, n) => n === 1 ? suite(fail('b') + fail('c')) : suite(pass('b') + pass('c')), (_a, n) => n === 1 ? 1 : 128, undefined, n => n === 2 ? 'SIGTERM' : undefined); + const r = await base(f, dir, { rerunScope: 'test' }); + expect(f.calls).toHaveLength(2); + expect(r.outcomes.map(o => o.reason)).toEqual(['rerun: killed by SIGTERM', 'not rerun: an earlier rerun was killed']); + expect(r.exitCode).toBe(1); + }); + + test('the first report is restored at --junit and each rerun report is kept beside it', async () => { + const dir = tmp(); + const firstXml = suite(pass('a') + fail('b')); + await base(fakeRun((_a, n) => n === 1 ? firstXml : suite(pass('b')), (_a, n) => n === 1 ? 1 : 0), dir); + expect(fs.readFileSync(path.join(dir, 'r.xml'), 'utf-8')).toBe(firstXml); + expect(fs.readFileSync(path.join(dir, 'r.rerun-1.xml'), 'utf-8')).toBe(suite(pass('b'))); + }); + + test('credentials in a test title never reach the ledger, the result lines or the step summary', async () => { + const dir = tmp(); + const key = ['sk', 'ant', 'api03', 'B'.repeat(93) + 'AA'].join('-'); + const r = await base(fakeRun((_a, n) => n === 1 ? suite(fail(`login with ${key}`)) : suite(pass(`login with ${key}`)), (_a, n) => n === 1 ? 1 : 0), dir); + expect(r.exitCode).toBe(0); + expect(JSON.stringify(r.ledger)).not.toContain(key); + expect(renderResult(r, '/l').join('\n')).not.toContain(key); + expect(stepSummary(r)).not.toContain(key); + }); + + test('workflow commands escape a title that would end or start a command', () => { + expect(workflowEscape('a%b\r\n::error::x')).toBe('a%25b%0D%0A::error::x'); + }); + + test('ledger: a write error is a warning; the path comes from --ledger, then GSTACK_FLAKE_LEDGER_JUNIT, then the state root', () => { + const dir = tmp(); + fs.writeFileSync(path.join(dir, 'file'), ''); + const warns: string[] = []; + appendLedger([{ ts: 't', runner: 'junit', kind: 'flaky-pass', id: 'x', file: 'x', rerun_by: 'gstack-flaky', attempts: [], excerpt: [] }], path.join(dir, 'file', 'l.jsonl'), w => warns.push(w)); + expect(warns[0]).toContain('FLAKY_LEDGER_WRITE_FAILED'); + expect(ledgerPath('/flag', { GSTACK_FLAKE_LEDGER_JUNIT: '/env' })).toBe('/flag'); + expect(ledgerPath(undefined, { GSTACK_FLAKE_LEDGER_JUNIT: '/env', GSTACK_FLAKE_LEDGER: '/free' })).toBe('/env'); + expect(ledgerPath(undefined, { GSTACK_STATE_ROOT: dir, GSTACK_FLAKE_LEDGER: '/free' })).toBe(path.join(dir, 'flake-ledger-junit.jsonl')); + }); + + test('credentials in failure text never reach the ledger', () => { + const key = ['sk', 'ant', 'api03', 'A'.repeat(93) + 'AA'].join('-'); + const out = safeExcerpt([`Error: request failed with key ${key}`]); + expect(out.join('\n')).not.toContain(key); + }); +}); + +describe('bin/gstack-flaky', () => { + test('run: a fake bun runner flakes once, the bin reruns it and exits 0 with a ledger line', () => { + const dir = tmp(); + const runner = path.join(dir, 'runner.sh'); + fs.writeFileSync(runner, [ + '#!/usr/bin/env bash', + 'n=$(cat count 2>/dev/null || echo 0); echo $((n+1)) > count', + `if [ "$n" = 0 ]; then printf '%s' '${suite(pass('a') + fail('b'))}' > r.xml; echo 'Ran 2 tests across 1 file.' >&2; exit 1; fi`, + '[ "$1" = a.test.ts ] && [ "$2" = --test-name-pattern ] && [ "$3" = "^b$" ] || exit 9', + `printf '%s' '${suite(pass('b'))}' > r.xml`, + ].join('\n'), { mode: 0o755 }); + const ledger = path.join(dir, 'ledger.jsonl'); + const r = spawnSync(path.join(ROOT, 'bin', 'gstack-flaky'), ['run', '--junit', 'r.xml', '--runner', 'bun', '--rerun-scope', 'test', '--ledger', ledger, '--', runner], { cwd: dir, encoding: 'utf-8', timeout: 30_000, env: { ...process.env, GITHUB_ACTIONS: '', GITHUB_STEP_SUMMARY: '' } }); + expect(r.stdout).toContain('FLAKY: flaky-pass a.test.ts::::b attempts=fail,pass'); + expect(r.stdout).toContain(`verdict=pass ledger=${ledger}`); + expect(r.status).toBe(0); + expect(JSON.parse(fs.readFileSync(ledger, 'utf-8').trim()).kind).toBe('flaky-pass'); + }); + + test('usage, a command that cannot start, and missing reports exit with their codes', () => { + const dir = tmp(); + const bin = path.join(ROOT, 'bin', 'gstack-flaky'); + expect(spawnSync(bin, ['run', '--', 'true'], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }).status).toBe(2); + fs.writeFileSync(path.join(dir, 'keep.ts'), 'x'); + expect(spawnSync(bin, ['run', '--junit', 'keep.ts', '--', 'true'], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }).status).toBe(2); + expect(fs.existsSync(path.join(dir, 'keep.ts'))).toBe(true); + fs.writeFileSync(path.join(dir, 'pom.xml'), '<project><modelVersion>4.0.0</modelVersion></project>'); + expect(spawnSync(bin, ['run', '--junit', 'pom.xml', '--', 'true'], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }).status).toBe(2); + expect(fs.existsSync(path.join(dir, 'pom.xml'))).toBe(true); + expect(spawnSync(bin, ['run', '--junit', 'r.xml', '--rerun-scope', 'file', '--', 'true'], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }).status).toBe(2); + fs.writeFileSync(path.join(dir, 'locked.xml'), '<testsuites/>', { mode: 0o000 }); + if (process.getuid?.() !== 0) { + const locked = spawnSync(bin, ['run', '--junit', 'locked.xml', '--', 'true'], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }); + expect([locked.status, locked.stderr.includes('cannot be read')]).toEqual([2, true]); + } + const missing = spawnSync(bin, ['run', '--junit', 'r.xml', '--', path.join(dir, 'no-such-runner')], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }); + expect(missing.status).toBe(3); + expect(missing.stderr).toContain('(FLAKY_COMMAND_NOT_STARTED)'); + const r = spawnSync(bin, ['run', '--junit', 'r.xml', '--', 'true'], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }); + expect(r.status).toBe(3); + expect(r.stderr).toContain('(FLAKY_NO_RESULTS)'); + expect(r.stderr).toContain('fix: '); + }); + + test('a runner that dies by a signal exits 3 FLAKY_COMMAND_KILLED and is not rerun', () => { + const dir = tmp(); + const runner = path.join(dir, 'runner.sh'); + fs.writeFileSync(runner, ['#!/usr/bin/env bash', 'echo x >> count', `printf '%s' '${suite(fail('b'))}' > r.xml`, 'kill -TERM $$'].join('\n'), { mode: 0o755 }); + const r = spawnSync(path.join(ROOT, 'bin', 'gstack-flaky'), ['run', '--junit', 'r.xml', '--runner', 'bun', '--ledger', path.join(dir, 'l.jsonl'), '--', runner], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }); + expect(r.status).toBe(3); + expect(r.stderr).toContain('(FLAKY_COMMAND_KILLED)'); + expect(fs.readFileSync(path.join(dir, 'count'), 'utf-8').trim().split('\n')).toHaveLength(1); + }); + + test('cancelling gstack-flaky is latched even when the runner traps the signal and exits by code', async () => { + const dir = tmp(); + const runner = path.join(dir, 'runner.sh'); + fs.writeFileSync(runner, [ + '#!/usr/bin/env bash', + 'echo x >> count', + 'trap "cp report.xml r.xml; exit 143" TERM', + 'touch ready', + 'for i in $(seq 1 300); do sleep 0.1; done', + ].join('\n'), { mode: 0o755 }); + fs.writeFileSync(path.join(dir, 'report.xml'), suite(fail('b'))); + const child = spawn(path.join(ROOT, 'bin', 'gstack-flaky'), ['run', '--junit', 'r.xml', '--runner', 'generic', '--ledger', path.join(dir, 'l.jsonl'), '--', runner], { cwd: dir, stdio: ['ignore', 'ignore', 'pipe'] }); + let stderr = ''; + child.stderr.on('data', (c: Buffer) => { stderr += c.toString(); }); + const done = new Promise<number | null>(resolve => child.on('close', code => resolve(code))); + const deadline = Date.now() + 20_000; + while (!fs.existsSync(path.join(dir, 'ready')) && Date.now() < deadline) await Bun.sleep(50); + child.kill('SIGTERM'); + expect(await done).toBe(3); + expect(stderr).toContain('(FLAKY_COMMAND_KILLED)'); + expect(fs.readFileSync(path.join(dir, 'count'), 'utf-8').trim().split('\n')).toHaveLength(1); + expect(fs.existsSync(path.join(dir, 'l.jsonl'))).toBe(false); + }, 30_000); + + test('every FLAKY_ code the bin and the library emit is in the docs table', () => { + const doc = fs.readFileSync(path.join(ROOT, 'docs', 'flaky.md'), 'utf-8'); + const src = fs.readFileSync(path.join(ROOT, 'bin', 'gstack-flaky'), 'utf-8') + fs.readFileSync(path.join(ROOT, 'lib', 'flaky-run.ts'), 'utf-8') + fs.readFileSync(path.join(ROOT, 'lib', 'junit.ts'), 'utf-8'); + const codes = [...new Set(src.match(/FLAKY_[A-Z_]+(?=['":])/g))].filter(c => c !== 'FLAKY_RESULT' && c !== 'FLAKY_RERUN'); + expect(codes.length).toBeGreaterThan(5); + expect(codes.filter(c => !doc.includes(`| \`${c}\` |`))).toEqual([]); + }); + + test('ids prints one id per test', () => { + const r = spawnSync(path.join(ROOT, 'bin', 'gstack-flaky'), ['ids', '--junit', path.join(FIX, 'bun.xml')], { encoding: 'utf-8', timeout: 30_000 }); + expect(r.stdout.trim().split('\n')).toEqual([testId('a.test.ts', '', ['outer', 'inner', 'passes']), 'a.test.ts::::outer > inner > fails', 'a.test.ts::::outer > skipped', 'a.test.ts::::top level']); + }); +}); diff --git a/test/free-durations-record-workflow.test.ts b/test/free-durations-record-workflow.test.ts index a86d0fcbc..9bd99d135 100644 --- a/test/free-durations-record-workflow.test.ts +++ b/test/free-durations-record-workflow.test.ts @@ -29,9 +29,10 @@ test('the recorder runs weekly before test-health and on dispatch, read-only exc expect(recorder.jobs.propose.permissions).toEqual({ contents: 'write', 'pull-requests': 'write', actions: 'write' }); }); -test('Linux records on the free-suite runner, turns red on any red file, and uploads only a complete recording', () => { +test('Linux records on the free-suite runner image, turns red on any red file, and uploads only a complete recording', () => { const linux = recorder.jobs.linux; - expect(linux['runs-on']).toBe(freeTests.jobs['free-suite']['runs-on']); + expect(linux['runs-on']).toMatch(/^ubicloud-standard-\d+$/); + expect(freeTests.env.FREE_SUITE_RUNNER).toMatch(/^ubicloud-standard-\d+$/); const runs = linux.steps.map((step: any) => step.run).filter(Boolean).join('\n'); expect(runs).toContain('bash scripts/ubicloud/setup-free-suite.sh'); const record = linux.steps.find((step: any) => step.name === 'Time every free file alone'); diff --git a/test/free-tests-workflow-wiring.test.ts b/test/free-tests-workflow-wiring.test.ts index cd8322971..613936f0d 100644 --- a/test/free-tests-workflow-wiring.test.ts +++ b/test/free-tests-workflow-wiring.test.ts @@ -166,6 +166,66 @@ describe('free-tests workflow wiring', () => { } }); + test.skipIf(!Bun.which('bash'))('free-plan resolves the one runner and shard constant for every event, and validates dispatch overrides', () => { + const workflow = Bun.YAML.parse(source) as any; + const planner = workflow.jobs['free-plan']; + const plan = planner.steps.find((step: any) => step.id === 'plan'); + expect(workflow.env.FREE_SUITE_RUNNER).toMatch(/^ubicloud-standard-(2|4|8)$/); + expect(workflow.env.FREE_SUITE_SHARDS).toMatch(/^[1-9][0-9]?$/); + expect(planner.outputs.runner).toBe('${{ steps.plan.outputs.runner }}'); + expect(workflow.jobs['free-suite']['runs-on']).toBe('${{ needs.free-plan.outputs.runner }}'); + expect(plan.env).toEqual({ RUNNER_OVERRIDE: '${{ inputs.free_runner }}', SHARDS_OVERRIDE: '${{ inputs.free_shards }}' }); + expect(plan.run).not.toContain('${{'); + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'free plan ')); + const bin = path.join(directory, 'bin'); + fs.mkdirSync(bin); + fs.writeFileSync(path.join(bin, 'bun'), '#!/bin/sh\nprintf "%s\\n" "$*" > "$RUNNER_TEMP/bun-args"\necho \'{"shard":[1]}\'\n', { mode: 0o755 }); + const toShell = (p: string) => p.split(path.sep).join('/'); + const resolve = (overrides: Record<string, string>) => { + const output = path.join(directory, 'output'); + fs.writeFileSync(output, ''); + const result = spawnSync('bash', ['-c', plan.run], { + env: { ...process.env, ...workflow.env, RUNNER_OVERRIDE: '', SHARDS_OVERRIDE: '', ...overrides, + PATH: `${toShell(bin)}:${process.env.PATH}`, RUNNER_TEMP: toShell(directory), + GITHUB_OUTPUT: toShell(output), GITHUB_STEP_SUMMARY: toShell(path.join(directory, 'summary')) }, + encoding: 'utf8', timeout: 5000, + }); + const outputs = Object.fromEntries(fs.readFileSync(output, 'utf8').trim().split('\n').filter(Boolean).map(line => line.split(/=(.*)/s).slice(0, 2))); + const args = fs.existsSync(path.join(directory, 'bun-args')) ? fs.readFileSync(path.join(directory, 'bun-args'), 'utf8') : ''; + fs.rmSync(path.join(directory, 'bun-args'), { force: true }); + return { status: result.status, outputs, args }; + }; + try { + const unset = resolve({}); + expect(unset.status).toBe(0); + expect(unset.outputs.runner).toBe(workflow.env.FREE_SUITE_RUNNER); + expect(unset.args).toContain(`--shards ${workflow.env.FREE_SUITE_SHARDS}`); + const override = resolve({ RUNNER_OVERRIDE: 'ubicloud-standard-4', SHARDS_OVERRIDE: '16' }); + expect(override.status).toBe(0); + expect(override.outputs.runner).toBe('ubicloud-standard-4'); + expect(override.args).toContain('--shards 16'); + const refusals: Record<string, string>[] = [{ RUNNER_OVERRIDE: 'ubuntu-24.04' }, { RUNNER_OVERRIDE: 'ubicloud-standard-8 x' }, { SHARDS_OVERRIDE: '0' }, { SHARDS_OVERRIDE: '41' }, { SHARDS_OVERRIDE: '16; true' }]; + for (const bad of refusals) { + const refused = resolve(bad); + expect(refused.status, JSON.stringify(bad)).not.toBe(0); + expect(refused.outputs.runner, JSON.stringify(bad)).toBeUndefined(); + expect(refused.args, JSON.stringify(bad)).toBe(''); + } + } finally { fs.rmSync(directory, { recursive: true, force: true }); } + }); + + test('every shard reports its peak memory and minimum free disk, even when the suite fails', () => { + const steps = (Bun.YAML.parse(source) as any).jobs['free-suite'].steps; + const start = steps.findIndex((step: any) => step.name === 'Start the memory and disk sampler'); + const suite = steps.findIndex((step: any) => step.name === 'Run free suite'); + const report = steps.findIndex((step: any) => step.name === 'Report peak memory and minimum free disk'); + expect(start).toBeGreaterThan(-1); + expect(start).toBeLessThan(suite); + expect(report).toBeGreaterThan(suite); + expect(steps[report].if).toBe('always()'); + expect(steps[report].run).toContain('RESOURCES: peak_used_mib='); + }); + test('least-privilege token: contents read-only, credentials not persisted', () => { // The job executes PR-controlled code (install lifecycle scripts + the // suite itself). A default-grant GITHUB_TOKEN persisted into .git/config diff --git a/test/gstack-doctor.test.ts b/test/gstack-doctor.test.ts index d39ab1440..1becf6c50 100644 --- a/test/gstack-doctor.test.ts +++ b/test/gstack-doctor.test.ts @@ -27,7 +27,7 @@ const COPIED = [ 'autoplan/SKILL.md', // its frontmatter registers the autoplan hook the hooks-row test breaks 'bin/gstack-launch-probe.sh', 'bin/gstack-doctor-check.sh', 'bin/gstack-doctor-components.sh', 'bin/gstack-runtime-pins.ts', 'lib/runtime-pins.ts', - 'bin/gstack-windows-bash.ts', 'lib/script-invocation.ts', 'lib/bun-isolation.ts', 'lib/result-codes.ts', 'lib/result-codes-release.ts', 'lib/result-codes-capy.ts', 'lib/result-codes-budget.ts', // the windows bash row + 'bin/gstack-windows-bash.ts', 'lib/script-invocation.ts', 'lib/bun-isolation.ts', 'lib/result-codes.ts', 'lib/result-codes-release.ts', 'lib/result-codes-capy.ts', 'lib/result-codes-budget.ts', 'lib/result-codes-eval.ts', // the windows bash row ]; const bases: string[] = []; afterEach(() => { for (const b of bases.splice(0)) fs.rmSync(b, { recursive: true, force: true }); }); diff --git a/test/gstack-post.test.ts b/test/gstack-post.test.ts index 67bd093db..72e6ecbf7 100644 --- a/test/gstack-post.test.ts +++ b/test/gstack-post.test.ts @@ -446,21 +446,21 @@ describe("gstack-post pr-body merges the PR body by section", () => { expect(after.stderr()).toBe("gstack-post: pr-body merged by section: replaced ship receipt, Why; kept unclosed receipt fence; added none\n"); }); - // Value: protects=a file with sections owns the text above the first heading (GST-412): stale lead text and stale text between the receipt and the + // Value: protects=a full post (a receipt above its first heading, as /ship composes it) owns the text above the first heading (GST-412): stale lead text and stale text between the receipt and the // first heading are dropped and named on stderr with their first line, the file's own preamble takes their place, a receipt-only or text-only file // keeps the older merge, a repost is byte-identical and an attribution callout is never doubled; fails_when=pr-body keeps a stale intro above the // receipt or a stale note under it after /ship reposts the sections; why_new=lead text was kept unless the file brought its own; seam=fake run; // no_claim=real gh - test("a file with sections replaces every text section above the current first heading and names each dropped piece", () => { + test("a full post replaces every text section above the current first heading and names each dropped piece", () => { const stale = withBody("Draft intro from gh pr create, long enough to need truncation in the summary line\nsecond line\n\n" + CURRENT, { "/b": RECEIPT("bbbb") + "\n## What changed\n\n- new\n" }); expect(run(["pr-body", "5", "--body-file", "/b"], stale)).toBe(EXIT.posted); expect(sent(stale).input).toBe(RECEIPT("bbbb") + "\n## What changed\n\n- new\n\n## Why\n\nBecause.\n\n## Approvals\n\n- owner: do it\n"); expect(stale.stderr()).toBe("gstack-post: pr-body merged by section: replaced ship receipt, What changed; kept Why, Approvals; added none; dropped leading text (\"Draft intro from gh pr create, long enough to need truncati…\")\n"); - const between = withBody(RECEIPT("aaaa") + "Old note under the receipt\n\n## What changed\n\n- old\n", { "/b": "## What changed\n\n- new\n" }); + const between = withBody(RECEIPT("aaaa") + "Old note under the receipt\n\n## What changed\n\n- old\n", { "/b": RECEIPT("bbbb") + "## What changed\n\n- new\n" }); expect(run(["pr-body", "5", "--body-file", "/b"], between)).toBe(EXIT.posted); - expect(sent(between).input).toBe(RECEIPT("aaaa") + "\n## What changed\n\n- new\n"); - expect(between.stderr()).toBe("gstack-post: pr-body merged by section: replaced What changed; kept ship receipt; added none; dropped trailing text (\"Old note under the receipt\")\n"); + expect(sent(between).input).toBe(RECEIPT("bbbb") + "\n## What changed\n\n- new\n"); + expect(between.stderr()).toBe("gstack-post: pr-body merged by section: replaced ship receipt, What changed; kept none; added none; dropped trailing text (\"Old note under the receipt\")\n"); const own = withBody("Stale intro\n\n" + RECEIPT("aaaa") + "Stale note\n\n## Why\n\nold\n", { "/b": "Fresh intro\n\n" + RECEIPT("bbbb") + "Fresh note\n\n## Why\n\nnew\n" }); expect(run(["pr-body", "5", "--body-file", "/b"], own)).toBe(EXIT.posted); @@ -469,6 +469,23 @@ describe("gstack-post pr-body merges the PR body by section", () => { expect(own.stderr()).toBe("gstack-post: pr-body merged by section: replaced leading text, ship receipt, trailing text, Why; kept none; added none\n"); }); + // Value: protects=a partial post (no receipt above its first `## ` heading, even with a caption of its own or a receipt further down) keeps + // the human intro and the note under the receipt; fails_when=a lane posting only `## Screenshots` + // deletes the intro or a Review-loop exception line; why_new=GST-412 made any file with a section own the preamble (GstackMinder, + // 2026-10-31: "a partial pr-body post must keep the human intro"); seam=fake run; no_claim=real gh + test("a partial post keeps the current intro and the note under the receipt", () => { + const partial = withBody("Human intro\n\n" + RECEIPT("aaaa") + "Review-loop exception: accepted by owner\n\n## What changed\n\n- old\n", { "/b": "## Screenshots\n\n![shot](u)\n" }); + expect(run(["pr-body", "5", "--body-file", "/b"], partial)).toBe(EXIT.posted); + expect(sent(partial).input).toBe("Human intro\n\n" + RECEIPT("aaaa") + "Review-loop exception: accepted by owner\n\n## What changed\n\n- old\n\n## Screenshots\n\n![shot](u)\n"); + expect(partial.stderr()).toBe("gstack-post: pr-body merged by section: replaced none; kept leading text, ship receipt, trailing text, What changed; added Screenshots\n"); + + for (const file of ["Screenshots below.\n\n## Screenshots\n\n![shot](u)\n", "## Screenshots\n\n![shot](u)\n\n" + RECEIPT("bbbb")]) { + const merged = mergeBodySections("Human intro\n\n" + RECEIPT("aaaa") + "Review-loop exception: accepted by owner\n\n## What changed\n\n- old\n", file); + expect(merged.body).toContain("Review-loop exception: accepted by owner"); + expect(merged.dropped).toEqual([]); + } + }); + // Value: protects=the dropped-piece excerpt on stderr never repeats a secret the merge removed from the body (the scan only reads the merged bytes); // fails_when=a stale intro holding a credential is echoed to stderr while it leaves the PR; why_new=the excerpt is new in GST-412; seam=fake run; no_claim=real gh test("a dropped piece whose first line the redaction scan flags is named without its text", () => { @@ -504,21 +521,21 @@ describe("gstack-post pr-body merges the PR body by section", () => { }); // Value: protects=--replace keeps the plain overwrite for a caller that composed the whole body, a current body with no H2 headings is kept as the leading - // section by a file without sections and dropped (and named) by a file with sections (GST-412), an empty current body posts the new body byte for byte, + // section by a file without sections and dropped (and named) by a full post (GST-412), an empty current body posts the new body byte for byte, // the scan runs on the merged bytes, and an unreadable current body is exit 3 with the --replace hint rather than an overwrite; fails_when=--replace // merges, headingless text is dropped by a receipt-only post or kept under new sections, the merge changes a lone new body, a secret in a kept section // passes the scan, or a failed read overwrites; why_new=first pr-body merge tests; seam=fake run; no_claim=real gh - test("--replace overwrites; a body without H2 headings is kept by a file without sections and dropped by one with sections; the merged bytes are what is scanned; a failed read refuses", () => { + test("--replace overwrites; a body without H2 headings is kept by a file without a receipt and dropped by a full post; the merged bytes are what is scanned; a failed read refuses", () => { const replaced = withBody(CURRENT, { "/b": "## What changed\n\nonly this\n" }); expect(run(["pr-body", "5", "--replace", "--body-file", "/b"], replaced)).toBe(EXIT.posted); expect(sent(replaced).input).toBe("## What changed\n\nonly this\n"); expect(replaced.stderr()).toBe(""); expect(replaced.calls.some(c => c.cmd === "gh" && c.args[1] === "view")).toBe(false); - const plain = withBody("Just a paragraph.\nAnother line.", { "/b": "## Tests\n\n- one\n" }); + const plain = withBody("Just a paragraph.\nAnother line.", { "/b": RECEIPT("bbbb") + "\n## Tests\n\n- one\n" }); expect(run(["pr-body", "5", "--body-file", "/b"], plain)).toBe(EXIT.posted); - expect(sent(plain).input).toBe("## Tests\n\n- one\n"); - expect(plain.stderr()).toBe("gstack-post: pr-body merged by section: replaced none; kept none; added Tests; dropped leading text (\"Just a paragraph.\")\n"); + expect(sent(plain).input).toBe(RECEIPT("bbbb") + "\n## Tests\n\n- one\n"); + expect(plain.stderr()).toBe("gstack-post: pr-body merged by section: replaced none; kept none; added ship receipt, Tests; dropped leading text (\"Just a paragraph.\")\n"); const plainText = withBody("Just a paragraph.\nAnother line.", { "/b": RECEIPT("bbbb") }); expect(run(["pr-body", "5", "--body-file", "/b"], plainText)).toBe(EXIT.posted); expect(sent(plainText).input).toBe(RECEIPT("bbbb") + "\nJust a paragraph.\nAnother line."); diff --git a/test/helpers/free-fixtures-data.ts b/test/helpers/free-fixtures-data.ts index d62e50265..dfe62b544 100644 --- a/test/helpers/free-fixtures-data.ts +++ b/test/helpers/free-fixtures-data.ts @@ -151,6 +151,7 @@ export const FREE_FIXTURES: Readonly<Record<string, readonly string[]>> = { 'test/fixtures/investigate-owned-question-39195396438.json': ['test/workflow-boundaries-fixture.test.ts'], 'test/fixtures/judge-calibration/**': ['scripts/judge-calibration.ts', 'test/judge-calibration.test.ts'], 'test/fixtures/judge-panel-replay-2026-10-05.json': ['test/judge-panel-median.test.ts'], + 'test/fixtures/junit/**': ['test/flaky-run.test.ts'], 'test/fixtures/launch-env-echo/**': ['test/launch-env-echo-ratchet.test.ts'], 'test/fixtures/merge-train-3345/**': ['test/changelog-merge-damage.test.ts', 'test/compare-pr-version.test.ts'], 'test/fixtures/merge-train-3453/**': ['test/changelog-merge-damage.test.ts'], @@ -195,6 +196,7 @@ export const FREE_FIXTURES: Readonly<Record<string, readonly string[]>> = { 'test/fixtures/redact-diagram/**': ['test/redact-context-exemptions.test.ts'], 'test/fixtures/remote-identity-vectors.json': ['test/bin-context-windows-slug.test.ts', 'test/gstack-slug-parity.test.ts', 'test/remote-identity-vectors.test.ts'], 'test/fixtures/review-depth/**': ['test/review-dispatch-compare.test.ts'], + 'test/fixtures/review-ledger/**': ['test/review-dispatch.test.ts'], 'test/fixtures/review-design-lite-reports-ci-36633323521.json': ['test/review-finalization-budget.test.ts'], 'test/fixtures/review-gate-integrity-opus-reports.json': ['test/gate-integrity-templates.test.ts'], 'test/fixtures/session-update-heartbeat.sh': ['test/session-update-heartbeat.test.ts'], diff --git a/test/plan-tune-cathedral.test.ts b/test/plan-tune-cathedral.test.ts index e0c73fcf3..20109dc42 100644 --- a/test/plan-tune-cathedral.test.ts +++ b/test/plan-tune-cathedral.test.ts @@ -70,7 +70,7 @@ function scaffoldFixture(workDir: string): { workDir: string; stateRoot: string; // These imports are resolved relative to the copied install, not the source checkout. const libDir = path.join(workDir, 'lib'); fs.mkdirSync(libDir, { recursive: true }); - for (const file of ['jsonl-store.ts', 'is-conductor.ts', 'state-root.ts', 'bin-context.ts', 'remote-identity.ts', 'script-invocation.ts', 'bun-isolation.ts', 'result-codes.ts', 'result-codes-release.ts', 'result-codes-capy.ts', 'result-codes-budget.ts']) { + for (const file of ['jsonl-store.ts', 'is-conductor.ts', 'state-root.ts', 'bin-context.ts', 'remote-identity.ts', 'script-invocation.ts', 'bun-isolation.ts', 'result-codes.ts', 'result-codes-release.ts', 'result-codes-capy.ts', 'result-codes-budget.ts', 'result-codes-eval.ts']) { fs.copyFileSync(path.join(ROOT, 'lib', file), path.join(libDir, file)); } diff --git a/test/pregate-remote.test.ts b/test/pregate-remote.test.ts index 88308ed8a..ca781b02e 100644 --- a/test/pregate-remote.test.ts +++ b/test/pregate-remote.test.ts @@ -22,6 +22,7 @@ const GATE = 'node scripts/remote-gate.js'; const GATE_SCRIPT = [ "const fs = require('node:fs');", "fs.writeFileSync('../gate-env.json', JSON.stringify({ tree: process.env.PREGATE_TREE, cap: process.env.PREGATE_SPEND_CAP_USD, timeout: process.env.PREGATE_TIMEOUT_S, files: fs.readFileSync(process.env.PREGATE_FILES_FILE, 'utf8') }));", + "if (process.env.GATE_QUIET_MS) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, Number(process.env.GATE_QUIET_MS));", "if (process.env.GATE_SILENT) process.exit(0);", "const spent = process.env.GATE_SPENT ? ` spent=${process.env.GATE_SPENT}` : '';", "console.log(`PREGATE_REMOTE: tree=${process.env.GATE_TREE || process.env.PREGATE_TREE} tests=${process.env.GATE_TESTS ?? 3} failed=${process.env.GATE_FAILED ?? 0}${spent}`);", @@ -187,6 +188,22 @@ describe('gstack-pregate --remote (C-c)', () => { expect(fs.existsSync(path.join(repo.root, 'gate-env.json'))).toBe(false); }, 60_000); + // Value: protects=a remote gate that waits quietly on CI is bounded by its deadline (--timeout, else timeout_s), not killed by the + // test-output stall timer; a gate past its deadline still stays uncleared (GST-431); fails_when=the stall timer kills a quiet gate + // inside its deadline, or a gate past --timeout clears; why_new=a CI-waiting gate died at 300 s on qm (CAPY-1); seam=fake gate; + // no_claim=the Ubicloud VM + test('a gate that is quiet past the stall timeout but inside its deadline clears; past --timeout it stays uncleared', () => { + const repo = gated(); + const quiet = remote(repo, ['--remote', '--allow-repo-commands', '--spend-cap', '2', '--stall-timeout', '1', '--timeout', '60'], { GATE_QUIET_MS: '2500' }); + expect(quiet.status).toBe(EXIT.ok); + expect(pregateFile(repo).verdict).toBe('requires-remote cleared'); + const late = remote(repo, ['--remote', '--allow-repo-commands', '--spend-cap', '2', '--timeout', '2'], { GATE_QUIET_MS: '8000' }); + expect(late.status).toBe(EXIT.fail); + expect(lanes(repo)).toMatchObject({ status: 'incomplete', code: 'PREGATE_INCOMPLETE' }); + expect(lanes(repo).detail).toContain('the remote gate did not finish (timed out after 2 s, --timeout)'); + expect(pregateFile(repo).requires_remote[0].receipt).toBeNull(); + }, 60_000); + test('a wrong tree, zero tests, a failure, a missing line or a non-zero exit leaves the obligation uncleared and the check incomplete; unreported spend counts at the cap', () => { const repo = gated(); const cases: Array<[Record<string, string>, string]> = [ diff --git a/test/pregate-supervise.test.ts b/test/pregate-supervise.test.ts index abdf3909b..70921184a 100644 --- a/test/pregate-supervise.test.ts +++ b/test/pregate-supervise.test.ts @@ -91,6 +91,28 @@ describe('supervise: process group, log file, stall and deadline', () => { expect(r.lines.filter(l => l.startsWith('PREGATE_HEARTBEAT: fixture running ')).length).toBeGreaterThanOrEqual(2); expect(r.lines.some(l => l.startsWith('PREGATE_TEST:'))).toBe(false); }); + + // Value: protects=a remote gate's heartbeat says what the gate last printed (GST-431); fails_when=the heartbeat for a gate that + // reports its CI wait shows only "last file none reported"; why_new=a CI-waiting gate looked dead on qm (CAPY-1); seam=none; no_claim=none + test('with lastLine, a heartbeat names the command\'s last output line', async () => { + const dir = scratch(); + const r = await run(dir, 'echo "waiting on run 7 (in_progress)"; sleep 1.2; echo done', { heartbeatMs: 300, lastLine: true }); + expect(r.outcome).toBe('exited'); + expect(r.lines.some(l => l.startsWith('PREGATE_HEARTBEAT: fixture running ') && l.includes('last output: waiting on run 7 (in_progress)'))).toBe(true); + const redraw = await run(dir, "printf 'run 7 queued\\r\\033[2Krun 7 in_progress\\n'; sleep 1.2", { heartbeatMs: 300, lastLine: true }); + expect(redraw.lines.some(l => l.endsWith('last output: run 7 in_progress'))).toBe(true); + expect(redraw.lines.some(l => l.includes('\x1b') || l.includes('\r'))).toBe(false); + }); + + // Value: protects=reading the last line is best effort: a log that disappears mid-run never throws out of the supervisor's timer, which would + // abandon the gate's deadline and cleanup (GST-431 review); fails_when=lastLine sampling throws ENOENT; why_new=lastLine is new; seam=none; no_claim=none + test('with lastLine, a log removed mid-run still ends in a normal outcome', async () => { + const dir = scratch(); + const log = path.join(dir, 'run.log'); + fs.writeFileSync(log, ''); + const r = await supervise({ command: `echo hi; sleep 0.4; rm -f "${log}"; sleep 1.2; echo done`, cwd: dir, env: process.env, log, deadline: Date.now() + 30_000, stallMs: 0, graceMs: 300, pollMs: 50, heartbeatMs: 300, lastLine: true, label: 'fixture', progress: () => {} }); + expect(r.outcome).toBe('exited'); + }); }); describe('supervise: per-file progress adapters', () => { diff --git a/test/pricing-eval-plan.test.ts b/test/pricing-eval-plan.test.ts index 661ed5742..8784d88a0 100644 --- a/test/pricing-eval-plan.test.ts +++ b/test/pricing-eval-plan.test.ts @@ -50,7 +50,8 @@ describe('preregistration', () => { expect(check.missing).toEqual([]); expect(check.empty.length).toBe(PREREG_SECTIONS.length); expect(check.ok).toBe(false); - const filled = t.replace(/^(##\s+.+)$/gm, '$1\n- filled in'); + expect(PREREG_SECTIONS).toContain('Comparison'); + const filled = t.replace(/^(##\s+.+)$/gm, '$1\n- filled in').replace('## Comparison\n- filled in', '## Comparison\nstack: none\nwhy: one arm, no skill stack'); expect(checkPrereg(filled)).toMatchObject({ ok: true, missing: [], empty: [] }); expect(checkPrereg('# nothing here\n').missing).toEqual([...PREREG_SECTIONS]); }); diff --git a/test/review-dispatch.test.ts b/test/review-dispatch.test.ts index 826f502c6..b76de4a38 100644 --- a/test/review-dispatch.test.ts +++ b/test/review-dispatch.test.ts @@ -4,7 +4,7 @@ import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { BriefError, promptHash, renderBrief, sectionTexts } from '../lib/review-brief'; -import { judge, parseFindings, type MergedLine } from '../lib/review-ledger'; +import { collect, judge, parseFindings, type MergedLine } from '../lib/review-ledger'; const ROOT = path.resolve(import.meta.dir, '..'); const BIN = path.join(ROOT, 'bin', 'gstack-review-dispatch'); @@ -326,6 +326,148 @@ describe('gstack-review-dispatch sent, collect, judge and verify: nothing change expect(parseFindings('outside-voice', '1. Retry loses writes (queue.ts:78). Severity: Medium\n Fix: reject the batch. Blocks: yes')).toEqual([{ label: 'CRITICAL', raw: '1. Retry loses writes (queue.ts:78). Severity: Medium | Fix: reject the batch. Blocks: yes' }]); }); + // Value: protects=a reader reply with no finding and no clean marker (a refusal, an empty reply, a reader that never saw the diff) is + // recorded as a non-review: collect exits 1 and verify reports the brief NOT_COLLECTED, never clean; fails_when=collect hashes the + // GSTA-196 refusal ("I'm not returning NO FINDINGS"), an empty file or an "I could not access the diff" reply as findings=0 and verify + // reads MATCH (replay C3-36: the batched specialist found no diff base and its brief read MATCH); + // why_new=collect recorded whatever parseFindings returned and verify counted any output as collected; seam=collect and verify; + // no_claim=that a reply carrying a marker actually reviewed the diff + test('a reply with no finding and no clean marker is a non-review: collect exits 1 and verify is NOT_COLLECTED, never clean', () => { + const refusal = fs.readFileSync(path.join(ROOT, 'test/fixtures/review-ledger/gsta-196-refusal.txt'), 'utf8'); + expect(promptHash(refusal)).toBe('7150c3b9ffe2698f77a988fa422b66d7004e1939d12e81185e8d556a47dff784'); + const b = tmp(); + const batched = field(run(['render', '--role', 'specialist-batched', '--checklists', 'testing,security,state-machines', '--out', b, '--diff-base', SHA]).stdout, 'file'); + const cases = [ + { reader: 'specialist-batched', dir: b, brief: batched, output: refusal, args: ['--role', 'specialist-batched'] }, + ...['', 'I could not access the diff: git diff failed with "bad object", so there is nothing here to review.\n'].map(output => { + const p = pass(); + return { reader: 'specialist/testing', dir: p.out, brief: p.file, output, args: ['--role', 'specialist', '--name', 'testing'] }; + }), + ]; + for (const c of cases) { + const ledger = path.join(c.dir, 'review-ledger.jsonl'); + const merged = path.join(c.dir, 'merged.jsonl'); + const out = path.join(c.dir, 'reply.out'); + fs.writeFileSync(out, c.output); + expect(run(['sent', '--ledger', ledger, ...c.args, '--sent', c.brief]).status).toBe(0); + const col = run(['collect', '--ledger', ledger, ...c.args, '--output', out, '--merged', merged]); + expect([col.status, col.stdout.startsWith(`COLLECT_NOT_REVIEWED: reader=${c.reader} findings=0 sha256=${promptHash(c.output)} `)]).toEqual([1, true]); + const event = JSON.parse(fs.readFileSync(ledger, 'utf8').trim().split('\n').at(-1)!); + expect([event.event, event.ids, event.review]).toEqual(['output', [], false]); + fs.writeFileSync(path.join(c.dir, 'judged.jsonl'), ''); + expect(run(['judge', '--ledger', ledger, '--merged', merged, '--judged', path.join(c.dir, 'judged.jsonl')]).status).toBe(0); + const v = run(['verify', '--ledger', ledger]); + expect([v.status, v.stdout.includes(`BRIEF ${c.reader} NOT_COLLECTED rendered=`), v.stdout.includes(`OUTPUT ${c.reader} NOT_REVIEWED`), v.stdout.includes(' not_reviewed\n'), v.stdout.includes('VERIFY: dirty')]).toEqual([1, true, true, true, true]); + } + }); + + // Value: protects=a clean reply carrying its section's marker and a reply with findings still collect with exit 0 and verify clean under + // the non-review rule; fails_when=the rule makes a real NO FINDINGS specialist or a findings reply dirty or exit 1; + // why_new=the non-review rule above could over-reach; seam=collect and verify; no_claim=other roles (the matrix below covers them) + test('a NO FINDINGS reply and a findings reply still collect with exit 0 and verify clean', () => { + for (const [output, findings] of [['NO FINDINGS\n', 0], ['{"severity":"CRITICAL","path":"a.ts","line":3,"category":"logic","summary":"off by one","specialist":"testing"}\n', 1]] as const) { + const p = pass(); + run(['sent', '--ledger', p.ledger, '--role', 'specialist', '--name', 'testing', '--sent', p.file]); + const col = run(['collect', '--ledger', p.ledger, '--role', 'specialist', '--name', 'testing', '--output', p.w('reply.out', output), '--merged', p.merged]); + expect([col.status, col.stdout.startsWith(`COLLECT: reader=specialist/testing findings=${findings} `)]).toEqual([0, true]); + expect(JSON.parse(fs.readFileSync(p.ledger, 'utf8').trim().split('\n').at(-1)!).review).toBe(true); + const judged = p.w('judged.jsonl', findings ? JSON.stringify({ id: 'specialist/testing#1', severity: 'CRITICAL' }) + '\n' : ''); + expect(run(['judge', '--ledger', p.ledger, '--merged', p.merged, '--judged', judged]).status).toBe(0); + const v = run(['verify', '--ledger', p.ledger]); + expect([v.status, v.stdout.includes(`OUTPUT specialist/testing MATCH sha=${promptHash(output).slice(0, 12)} findings=${findings}`), v.stdout.includes('VERIFY: clean briefs=1 outputs=1')]).toEqual([0, true, true]); + } + }); + + // Value: protects=each role's zero-finding reply counts as a review only with the marker its own section prescribes (NO FINDINGS for + // specialists, the batched specialist, Red Team, fix-check and targeted; the Recommendation: line for the outside voice; either for the + // native adversarial voice); fails_when=a marker named inside a sentence, another role's marker or an empty reply reads as clean, or a + // prescribed marker reads as a non-review; why_new=no marker check existed; seam=collect; no_claim=that the marker's reason is true + test('each role takes only the clean marker its section prescribes', () => { + const d = tmp(); + const outsideClean = 'No actionable findings in the supplied diff. Static review only.\n\nRecommendation: approve because the diff introduces no demonstrated bypass.\n'; + const cases: Array<[string, string, boolean]> = [ + ['specialist/testing', 'NO FINDINGS\n', true], ['specialist-batched', '`NO FINDINGS`\n', true], ['targeted', 'NO FINDINGS\n', true], ['fix-check', 'NO FINDINGS\n', true], + ['red-team', 'Fixtures were reviewed in summary mode only.\n\nNO FINDINGS\n', true], + ['adversarial-native', 'Fixtures were reviewed in summary mode.\n\nRecommendation: Ship as-is because no finding survived tracing\n', true], ['adversarial-native', 'NO FINDINGS\n', true], + ['outside-voice', outsideClean, true], ['outside-voice', 'NO FINDINGS\n', false], + ['specialist/testing', 'Recommendation: ship because it is clean\n', false], ['specialist/testing', "I'm not returning NO FINDINGS, because I never saw the diff.\n", false], + ['red-team', '', false], ['fix-check', 'The fixes look fine to me.\n', false], ['adversarial-native', 'I could not read the diff.\n', false], + ]; + for (const [i, [reader, output, review]] of cases.entries()) { + const ledger = path.join(d, `l${i}.jsonl`); + const r = collect(ledger, reader, output, path.join(d, `m${i}.jsonl`)); + expect([reader, output, r.lines.length, r.review]).toEqual([reader, output, 0, review]); + expect(JSON.parse(fs.readFileSync(ledger, 'utf8').trim()).review).toBe(review); + } + }); + + // Value: protects=a paragraph finding's unlabeled detail bullets and the Severity/Blocks line after them stay one finding (GSTA-196 + // native voice: 4 findings collected as 8, the closing Severity lines opening phantoms judge had to fold); a list item's sibling and + // a labeled bullet still start their own findings; fails_when=the sample collects more or fewer than 4, or a labeled bullet merges + // away; why_new=every margin list item closed the open finding; seam=parseFindings; no_claim=indented detail lists (already attached) + test('unlabeled detail bullets and the Severity line after them stay on the open paragraph finding', () => { + const native = fs.readFileSync(path.join(ROOT, 'test/fixtures/review-ledger/gsta-196-native.txt'), 'utf8'); + expect(parseFindings('adversarial-native', native).map(f => [f.label, f.raw.slice(0, 12), f.raw.endsWith('| Severity: ' + (f.label === 'High' ? 'High' : 'Low') + '. Blocks: no.')])) + .toEqual([['High', '**1. A PR in', true], ['CRITICAL', '**2. A case ', true], ['CRITICAL', '**3. Local `', true], ['Low', '**4. No trai', true]]); + expect(parseFindings('outside-voice', '1. FIXABLE: race in a.ts:3\n- Unlabeled sibling at b.ts:9\nSeverity: Medium').map(f => f.label)).toEqual(['FIXABLE', 'Medium']); + expect(parseFindings('outside-voice', '**Retry storm in r.ts:2 (INVESTIGATE).**\n- detail\n- High: a second defect in s.ts:4\nSeverity: Medium').map(f => f.label)).toEqual(['INVESTIGATE', 'High']); + expect(parseFindings('outside-voice', 'Low: cache TTL mismatch (cache.ts:4)\n\n2. Token never expires (auth.ts:9)\nSeverity: High\nBlocks: yes').map(f => [f.label, f.raw])) + .toEqual([['Low', 'Low: cache TTL mismatch (cache.ts:4)'], ['CRITICAL', '2. Token never expires (auth.ts:9) | Severity: High | Blocks: yes']]); + expect(parseFindings('outside-voice', '**Race in a.ts:3 (High).**\n- the writer holds no lock\nSeverity: Medium').map(f => f.raw)).toEqual(['**Race in a.ts:3 (High).** | - the writer holds no lock | Severity: Medium']); + expect(parseFindings('outside-voice', '**Race in a.ts:3 (Medium).**\n\n- the writer holds no lock\n\nSeverity: Medium. Blocks: yes').map(f => [f.label, f.raw])) + .toEqual([['CRITICAL', '**Race in a.ts:3 (Medium).** | - the writer holds no lock | Severity: Medium. Blocks: yes']]); + }); + + // Value: protects=a non-review the same dispatch later answers with a review is voided (counted, VERIFY clean), and a failed outside + // voice recorded with --unavailable reads UNAVAILABLE (missing coverage, never a review) without blocking, while --unavailable is + // refused for any other reader; fails_when=one outside-voice timeout or one refusal makes VERIFY dirty for good, or --unavailable + // lets a specialist skip coverage; why_new=#3677 Step 11 adversarial voice; seam=collect and verify; no_claim=the review-log row + test('a refusal replaced by a review is voided; an outside voice failure is UNAVAILABLE; --unavailable is the outside voice only', () => { + const p = pass(); + const send = () => run(['sent', '--ledger', p.ledger, '--role', 'specialist', '--name', 'testing', '--sent', p.file]); + const take = (f: string) => run(['collect', '--ledger', p.ledger, '--role', 'specialist', '--name', 'testing', '--output', f, '--merged', p.merged]); + send(); + expect(take(p.w('refusal.out', 'I could not access the diff.\n')).status).toBe(1); + expect(run(['verify', '--ledger', p.ledger]).stdout).toContain('VERIFY: dirty'); + send(); + expect(take(p.w('clean.out', 'NO FINDINGS\n')).status).toBe(0); + fs.writeFileSync(path.join(p.out, 'judged.jsonl'), ''); + run(['judge', '--ledger', p.ledger, '--merged', p.merged, '--judged', path.join(p.out, 'judged.jsonl')]); + const v = run(['verify', '--ledger', p.ledger]); + expect([v.status, v.stdout.includes('replaced_by_a_review'), v.stdout.includes('VERIFY: clean briefs=1 outputs=2 voided=1')]).toEqual([0, true, true]); + const head = spawnSync('git', ['rev-parse', 'HEAD'], { cwd: ROOT, encoding: 'utf8', timeout: 10_000 }).stdout.trim(); + const o = tmp(); + const ov = field(run(['render', '--role', 'outside-voice', '--out', o, '--diff-base', head]).stdout, 'file'); + const ledger = path.join(o, 'review-ledger.jsonl'), merged = path.join(o, 'merged.jsonl'); + run(['sent', '--ledger', ledger, '--role', 'outside-voice', '--sent', ov]); + const fail = path.join(o, 'timeout.out'); + fs.writeFileSync(fail, 'Codex timed out after 9 minutes and was terminated; this pass produced NO findings.\n'); + const c = run(['collect', '--ledger', ledger, '--role', 'outside-voice', '--output', fail, '--merged', merged, '--unavailable', 'timeout']); + expect([c.status, c.stdout.startsWith('COLLECT_UNAVAILABLE: reader=outside-voice ')]).toEqual([0, true]); + fs.writeFileSync(path.join(o, 'judged.jsonl'), ''); + run(['judge', '--ledger', ledger, '--merged', merged, '--judged', path.join(o, 'judged.jsonl')]); + const w = run(['verify', '--ledger', ledger]); + expect([w.status, w.stdout.includes('BRIEF outside-voice UNAVAILABLE'), w.stdout.includes('OUTPUT outside-voice UNAVAILABLE'), w.stdout.includes(' unavailable=1\n')]).toEqual([0, true, true, true]); + const q = pass(); + run(['sent', '--ledger', q.ledger, '--role', 'specialist', '--name', 'testing', '--sent', q.file]); + const refused = run(['collect', '--ledger', q.ledger, '--role', 'specialist', '--name', 'testing', '--output', q.w('t.out', 'timed out\n'), '--merged', q.merged, '--unavailable', 'timeout']); + expect([refused.status, refused.stderr.includes('--unavailable is for the outside voice only')]).toEqual([2, true]); + const partial = path.join(o, 'partial.out'); + fs.writeFileSync(partial, '1. **CRITICAL**: token leak in a.ts:3\nCodex then timed out.\n'); + const kept = run(['collect', '--ledger', ledger, '--role', 'outside-voice', '--output', partial, '--merged', merged, '--unavailable', 'timeout']); + expect([kept.status, kept.stderr.includes('holds findings; collect it without --unavailable')]).toEqual([2, true]); + expect(run(['collect', '--ledger', ledger, '--role', 'outside-voice', '--output', fail, '--merged', merged, '--unavailable', ' ']).status).toBe(2); + const n = pass(); + const takeN = (f: string) => run(['collect', '--ledger', n.ledger, '--role', 'specialist', '--name', 'testing', '--output', f, '--merged', n.merged]); + run(['sent', '--ledger', n.ledger, '--role', 'specialist', '--name', 'testing', '--sent', n.file]); + takeN(n.w('refusal.out', 'I could not access the diff.\n')); + takeN(n.w('clean.out', 'NO FINDINGS\n')); + fs.writeFileSync(path.join(n.out, 'judged.jsonl'), ''); + run(['judge', '--ledger', n.ledger, '--merged', n.merged, '--judged', path.join(n.out, 'judged.jsonl')]); + const unsent = run(['verify', '--ledger', n.ledger]); + expect([unsent.status, unsent.stdout.includes(' NOT_REVIEWED '), unsent.stdout.includes('VERIFY: dirty')]).toEqual([1, true, true]); + }); + test('verify on a ledger that kept no prompts says so and exits 3', () => { const d = tmp(); const ledger = path.join(d, 'ledger.jsonl'); diff --git a/test/review-pass.test.ts b/test/review-pass.test.ts index 624a57452..9bfc99c89 100644 --- a/test/review-pass.test.ts +++ b/test/review-pass.test.ts @@ -128,6 +128,7 @@ function repo(): { dir: string; env: NodeJS.ProcessEnv } { interface Sub { title: string; prompt: string; status: string; polls: number; model?: unknown; summary?: string; late?: boolean } /** A fake Capy host: the shell is real (the repository's bins), subagents are scripted. */ +const cleanReply = (title: string) => (title.endsWith('outside-voice') ? 'Recommendation: ship as-is because no finding survived' : 'NO FINDINGS'); function host(dir: string, env: NodeJS.ProcessEnv, o: { output?: (title: string) => string; fail?: (title: string) => boolean; reject?: (title: string) => boolean; createThrows?: (title: string) => 'before' | 'after' | undefined; pollsToFinish?: number; lateSummary?: boolean } = {}) { let clock = 0; const subs = new Map<string, Sub>(); @@ -135,12 +136,12 @@ function host(dir: string, env: NodeJS.ProcessEnv, o: { output?: (title: string) const starts: number[][] = []; const advance = () => { for (const s of subs.values()) { - if (s.late && !s.summary) { s.summary = o.output?.(s.title) ?? 'NO FINDINGS'; continue; } + if (s.late && !s.summary) { s.summary = o.output?.(s.title) ?? cleanReply(s.title); continue; } if (s.status !== 'working') continue; if (++s.polls < (o.pollsToFinish ?? 1)) continue; if (o.fail?.(s.title)) s.status = 'failed'; else if (o.lateSummary && !s.late) { s.status = 'done'; s.late = true; } - else { s.status = 'done'; s.summary = o.output?.(s.title) ?? 'NO FINDINGS'; } + else { s.status = 'done'; s.summary = o.output?.(s.title) ?? cleanReply(s.title); } } }; const tools = { diff --git a/test/timeline-stop-hook-slug.test.ts b/test/timeline-stop-hook-slug.test.ts index 0c9975788..857df2304 100644 --- a/test/timeline-stop-hook-slug.test.ts +++ b/test/timeline-stop-hook-slug.test.ts @@ -33,6 +33,7 @@ const HOOK_FILES = [ 'lib/result-codes-release.ts', 'lib/result-codes-capy.ts', 'lib/result-codes-budget.ts', + 'lib/result-codes-eval.ts', ]; const SLUG = 'stop-hook-slug-test'; const BUDGET_MS = 2000;