diff --git a/CHANGELOG.md b/CHANGELOG.md index 845e26fed..2977c693d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Changelog +## [1.91.372.0] - 2026-10-31 + +**A test that fails once in CI and passes when run again no longer turns the job red, and every such flake is recorded for the daily flaky-fix pass.** `gstack-flaky run` wraps any test command that writes a JUnit report (Playwright, `bun test`, jest, vitest, pytest). When a few tests fail, it runs the command once more. A test that failed and then passes is a flaky pass, so the job stays green, an Actions `::warning` and a step-summary row name it, and one line goes on a ledger. A rerun counts only when the runner itself reports a full green run, so a real failure, a coverage threshold, a test the first run never reached, a crash or a cancelled job stays red. Before this, each one-off flake meant rerunning the whole CI job by hand (five times in Truther's last evidence window, three of them for one test). + +### Added +- **`bin/gstack-flaky run --junit -- `**: reruns the whole command once when at most `--max-reruns` (3) tests failed and the runner made no retries of its own. `--rerun-scope test` opts in to rerunning only the failed tests (Playwright `--grep`/`--project`, bun `--test-name-pattern`), for suites with no coverage gate and no fail-on-flaky policy; it still falls back to the whole command when it sees an early stop, a coverage table or the runner's own retries. +- **Ledger** at `GSTACK_FLAKE_LEDGER_JUNIT` (default `/flake-ledger-junit.jsonl`), kept apart from the free suite's file-level ledger: one redacted line per flaky pass and per test that failed its rerun, with tree, commit, branch, run URL, attempts, scope and the first lines of the error. +- **`gstack-flaky ids --junit `** prints the test ids a report holds (`:::: title>`). +- Exit codes: 0 pass (flaky passes included), 1 a test failed, 2 usage, 3 no usable report, a crash or a killed runner, each with a `FLAKY_` and a `fix:` line. A missing, stale, empty or malformed report never passes, and an existing `--junit` file that is not a JUnit report is refused rather than deleted. + +### Docs +- `docs/flaky.md`: the GitHub Actions setup, runners, rerun scope and its preconditions, rules, output and exit codes. `docs/TESTING_INTERNALS.md#flake-ledger` points other repositories to it. + +### Tests +- `test/flaky-run.test.ts` with captured Playwright 1.64 and bun 1.4 JUnit reports under `test/fixtures/junit/`: flaky pass, real red, rerun scopes, coverage-threshold and early-stop runs, unreported failures, cancelled runs, report deletion and ledger redaction. + +### Gate integrity +- `test/helpers/free-fixtures-data.ts`: registers `test/fixtures/junit/**` to `test/flaky-run.test.ts`. No assertion, threshold or skip changed. + ## [1.91.371.0] - 2026-10-31 **A PR push in a repository with a merge train no longer waits ten minutes for a check that does no work.** Under a train policy every PR push runs E2E Evals in `pr-summary` mode, which plans no paid slices, yet its `readiness` job still waited out the 600-second quiet window and green Free Tests before the run could finish. Now `readiness` waits for `route` and skips in `pr-summary` mode, so E2E Evals on a PR push finishes in about a minute (58 s on #3609) instead of after the 10-minute wait. diff --git a/VERSION b/VERSION index 74cc5d7cc..4c1d8c4ff 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.371.0 +1.91.372.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 1cfbabfca..fe7b8182c 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.371.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.372.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/bin/gstack-flaky b/bin/gstack-flaky new file mode 100755 index 000000000..40239d156 --- /dev/null +++ b/bin/gstack-flaky @@ -0,0 +1,125 @@ +#!/usr/bin/env -S bun --no-env-file --config=/dev/null +/** + * gstack-flaky (GSTA-158): rerun failed tests once and record the flake, for + * any runner that writes JUnit (Playwright, bun, jest, pytest, vitest). CREW + * rule 16 for every repository: "you can just run the failed test again and + * note that it is flaky". The rules live in lib/flaky-run.ts; docs/flaky.md is + * the how-to and the error-code table. + */ +import * as fs from 'fs'; +import { DEFAULT_MAX_RERUNS, appendLedger, ledgerPath, renderResult, runFlaky, safeText, stepSummary, workflowEscape } from '../lib/flaky-run'; +import { JunitError, readJunit, type JunitRunner } from '../lib/junit'; + +const USAGE = `Usage: + gstack-flaky run --junit [--runner playwright|bun|generic] [--max-reruns ] + [--rerun-scope command|test] [--ledger ] -- + gstack-flaky ids --junit [--runner playwright|bun|generic] + gstack-flaky --help + +run: runs the command (an argv, no shell), reads the JUnit report it writes at +--junit, and reruns each failed test once when the runner made no retries and +at most --max-reruns (${DEFAULT_MAX_RERUNS}) tests failed. The rerun repeats the whole +command and counts only on the runner's full green verdict (exit 0, no failed +or missing test). --rerun-scope test reruns the failed test alone (playwright: + --grep --project <p>; bun: <file> --test-name-pattern <title>) +when the runner's summary proves the first run complete and nothing but the +failed tests can fail it (no coverage, no runner-flaky test); otherwise, and +always for generic runners, the whole command reruns. A test that passes its +rerun once, with the rerun exiting 0, is a flaky pass: green, and one line on the ledger (--ledger, else +GSTACK_FLAKE_LEDGER_JUNIT, else <state root>/flake-ledger-junit.jsonl). --junit +holds the first run's report afterwards; reruns' reports sit beside it as +<name>.rerun-<n>.xml. The command runs with PLAYWRIGHT_JUNIT_INCLUDE_RETRIES=1 +so Playwright's own retries are reported. + +ids: prints the id of every test in a report (<file>::<project>::<title>). + +Output: one FLAKY: line per test that failed at least once, FLAKY_RERUN: +scope=command (<why>) when the rerun repeated the whole command, then +FLAKY_RESULT: tests=<n> failed=<n> flaky=<n> reruns=<n> command_exit=<n> verdict=pass|fail ledger=<file|none> +Exit: 0 pass (flaky passes included), 1 a test failed, 2 usage, +3 no usable report, a crash or a killed runner (FLAKY_<CODE> and a fix: line on stderr). +`; + +const FIX: Record<string, string> = { + FLAKY_NO_RESULTS: 'make the command write JUnit to the --junit path (playwright: PLAYWRIGHT_JUNIT_OUTPUT_NAME=<file> with --reporter=junit; bun: --reporter=junit --reporter-outfile=<file>)', + FLAKY_JUNIT_INVALID: 'the report is not well-formed XML; check that one process writes it and nothing else appends to it', + FLAKY_RETRIES_HIDDEN: 'set PLAYWRIGHT_JUNIT_INCLUDE_RETRIES=1 where the report is written', + FLAKY_COMMAND_FAILED: 'read the command output above: the suite crashed or a non-test step failed; that is a real red', + FLAKY_COMMAND_KILLED: 'the runner was killed by a signal (a cancelled job, an OOM kill); nothing was rerun, rerun the job', + FLAKY_COMMAND_NOT_STARTED: 'check the command after -- (it runs without a shell; wrap it as bash -c \'<script> "$@"\' _ when it needs one)', + FLAKY_UNREPORTED_FAILURES: 'the runner counted failures its JUnit omits (a test file that failed to load, an error outside any test); fix that first, it is a real red', +}; + +function die(msg: string, code = 2): never { + process.stderr.write(`gstack-flaky: ${msg}\n`); + process.exit(code); +} + +function coded(e: JunitError): never { + process.stderr.write(`gstack-flaky: ${e.message} (${e.code})\nfix: ${FIX[e.code] ?? 'see docs/flaky.md'}\n`); + process.exit(3); +} + +function parse(argv: string[]) { + const o: { junit?: string; runner?: JunitRunner; maxReruns: number; rerunScope: 'command' | 'test'; ledger?: string; command: string[] } = { maxReruns: DEFAULT_MAX_RERUNS, rerunScope: 'command', command: [] }; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]!; + const val = () => { const v = argv[++i]; if (v === undefined) die(`${a} needs a value`); return v; }; + if (a === '--') { o.command = argv.slice(i + 1); break; } + if (a === '--junit') o.junit = val(); + else if (a === '--runner') { + const r = val(); + if (r !== 'playwright' && r !== 'bun' && r !== 'generic') die(`--runner must be playwright, bun or generic, got ${JSON.stringify(r)}`); + o.runner = r; + } else if (a === '--max-reruns') { + const n = Number(val()); + if (!Number.isInteger(n) || n < 0) die('--max-reruns must be a whole number'); + o.maxReruns = n; + } else if (a === '--rerun-scope') { + const v = val(); + if (v !== 'command' && v !== 'test') die(`--rerun-scope must be command or test, got ${JSON.stringify(v)}`); + o.rerunScope = v; + } else if (a === '--ledger') o.ledger = val(); + else die(`unknown option ${a}\n${USAGE}`); + } + return o; +} + +const [sub, ...rest] = process.argv.slice(2); +if (!sub || sub === '--help' || sub === '-h') { process.stdout.write(USAGE); process.exit(sub ? 0 : 2); } + +if (sub === 'ids') { + const o = parse(rest); + if (!o.junit) die('ids needs --junit <file>'); + try { + for (const t of readJunit(o.junit, o.runner).tests) process.stdout.write(`${safeText(t.id)}\n`); + } catch (e) { if (e instanceof JunitError) coded(e); throw e; } + process.exit(0); +} + +if (sub !== 'run') die(`unknown command ${sub}\n${USAGE}`); +const o = parse(rest); +if (!o.junit) die('run needs --junit <file.xml>'); +if (!/\.xml$/i.test(o.junit) || fs.statSync(o.junit, { throwIfNoEntry: false })?.isDirectory()) die(`--junit must name a .xml file (it is deleted before every attempt), got ${JSON.stringify(o.junit)}`); +let existing: string | false | undefined; +try { existing = fs.statSync(o.junit, { throwIfNoEntry: false }) && fs.readFileSync(o.junit, 'utf-8').slice(0, 4096); } catch (e) { + die(`--junit ${JSON.stringify(o.junit)} exists but cannot be read (${(e as Error).message}); gstack-flaky deletes that path before every attempt, so it refuses to touch it`); +} +if (existing && existing.trim() && !/<testsuites?[\s>/]/.test(existing)) die(`--junit ${JSON.stringify(o.junit)} exists and is not a JUnit report; gstack-flaky deletes that path before every attempt, so it refuses to touch it`); +if (!o.command.length) die('run needs the test command after --'); +let result; +try { + result = await runFlaky({ command: o.command, junit: o.junit, runner: o.runner, maxReruns: o.maxReruns, rerunScope: o.rerunScope }); +} catch (e) { if (e instanceof JunitError) coded(e); throw e; } +const ledger = ledgerPath(o.ledger); +appendLedger(result.ledger, ledger, s => process.stderr.write(`gstack-flaky: ${s}\n`)); +for (const l of renderResult(result, ledger)) process.stdout.write(`${l}\n`); +const summary = process.env.GITHUB_STEP_SUMMARY && stepSummary(result); +if (summary) { try { fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY!, summary); } catch { /* a summary is a convenience, never the verdict */ } } +if (process.env.GITHUB_ACTIONS === 'true') { + for (const f of result.outcomes.filter(x => x.status === 'flaky-pass')) process.stdout.write(`::warning title=flaky test::${workflowEscape(safeText(f.test.id))} passed on attempt ${f.attempts.length}\n`); +} +if (result.code) { + process.stderr.write(`gstack-flaky: ${result.message} (${result.code})\nfix: ${FIX[result.code] ?? 'see docs/flaky.md'}\n`); +} +process.exit(result.exitCode); diff --git a/docs/TESTING_INTERNALS.md b/docs/TESTING_INTERNALS.md index 5334f468f..ecd162ccc 100644 --- a/docs/TESTING_INTERNALS.md +++ b/docs/TESTING_INTERNALS.md @@ -1435,6 +1435,13 @@ prints a run's ledgers as a table. A flaky file is fixed at source; the retry never masks it. `bun run eval:pass-rates` lists only the local ledger (`flakeLedgerPath()`, overridden by `GSTACK_FLAKE_LEDGER`). +Other repositories get the same rule per test with `bin/gstack-flaky run` +([docs/flaky.md](flaky.md)): it reads a JUnit report from any runner, reruns +the command once when tests failed (counting only the runner's full green +verdict), and writes `flaky-pass` and `rerun-fail` lines to its +own ledger (`flake-ledger-junit.jsonl`, or `GSTACK_FLAKE_LEDGER_JUNIT`), kept apart +from this file-level ledger. + The aggregate job of `free-tests.yml` downloads every `flake-ledger-*` artifact of the run and appends `bun run scripts/test-health-report.ts flake-summary <dir>` to its job summary. The Windows lane uploads `flake-ledger-windows-<shard>`. diff --git a/docs/flaky.md b/docs/flaky.md new file mode 100644 index 000000000..1a56b798e --- /dev/null +++ b/docs/flaky.md @@ -0,0 +1,137 @@ +# Flaky tests: rerun once and record it (`gstack-flaky`) + +`gstack-flaky run` runs your test command, reads the JUnit report it writes, +and when tests failed, runs the command once more. A test that passes on its +rerun is a **flaky pass**: the job stays green and the flake is recorded on a ledger, so +the daily flaky pass can fix it at the source. A test that fails twice is red. +This is CREW rule 16 ("run the failed test again and note that it is flaky") +for any runner that writes JUnit: Playwright, `bun test`, jest, vitest and +pytest. gstack's own free suite keeps its file-level retry +(`GSTACK_FREE_RETRY_FLAKY`, [Flake ledger](TESTING_INTERNALS.md#flake-ledger)). + +## Get started in GitHub Actions + +```yaml +- uses: oven-sh/setup-bun@v2 + with: { bun-version: 1.4.4 } +- name: Install gstack-flaky (pinned) + env: { GSTACK_REF: <a gstack commit sha you have tested> } + run: | + git clone --quiet --filter=blob:none https://github.com/garrytan/gstack.git "$RUNNER_TEMP/gstack" + git -C "$RUNNER_TEMP/gstack" checkout --quiet "$GSTACK_REF" +- name: E2E + env: + PLAYWRIGHT_JUNIT_OUTPUT_NAME: results/junit.xml + GSTACK_FLAKE_LEDGER_JUNIT: ${{ runner.temp }}/flaky-junit-ledger.jsonl + run: | + "$RUNNER_TEMP/gstack/bin/gstack-flaky" run --junit results/junit.xml -- npx playwright test --reporter=line,junit +- if: always() + uses: actions/upload-artifact@v4 + with: { name: flaky-junit-ledger-e2e, path: "${{ runner.temp }}/flaky-junit-ledger.jsonl", if-no-files-found: ignore, retention-days: 90 } +``` + +Replace `GSTACK_REF` with a gstack commit you have tested; the bin needs only +Bun and git. In Actions it also writes a step-summary table of the flaky +passes and one `::warning` annotation per flaky test, so they are visible on +the run before anyone reads the ledger. + +## Runners + +| Runner | Command after `--` | Notes | +|---|---|---| +| Playwright | `npx playwright test --reporter=line,junit` with `PLAYWRIGHT_JUNIT_OUTPUT_NAME=<file>` | `gstack-flaky` sets `PLAYWRIGHT_JUNIT_INCLUDE_RETRIES=1` so Playwright's own `retries` are reported; with retries configured, those attempts stand and nothing is rerun again. | +| bun | `bun test --reporter=junit --reporter-outfile=<file>` | | +| jest, vitest, pytest (`--runner generic`) | your command, writing JUnit | Always reruns the whole command, even with `--rerun-scope test`. | + +The command after `--` runs without a shell. When it needs one, wrap it as +`bash -c '<script> "$@"' _` so a targeted rerun's selection arguments reach +the runner. + +## Rerun scope + +**Default: the whole command.** The rerun repeats your command exactly, and +counts only on the runner's own full green verdict: it exits 0, no test fails, +every test of the first run is in its report, and the runner counts no failure +outside the report. That verdict covers every reason a run can fail, including +the ones a narrowed rerun cannot see: a test the first run never reached +(`--bail`, `--max-failures`, `-x`), a coverage threshold (however coverage is +configured, `bunfig.toml` and lcov included), a fail-on-flaky policy, a +global setup error. A test that failed in the first run and passes once in this +rerun is a flaky pass. It costs one more run of the suite, which is still far +cheaper than rerunning the CI job by hand. + +**Opt-in: `--rerun-scope test`.** Reruns the failed test alone (Playwright: +`<file> --grep <title> --project <project>`; bun: `<file> --test-name-pattern <title>`) +and clears it when that rerun exits 0. Use it only when the first run's exit +status can come from failed tests alone: no coverage threshold in any form, no +`--fail-on-flaky-tests`, no other step inside the command that can fail. As a +safety net, `gstack-flaky` still reruns the whole command +(`FLAKY_RERUN: scope=command (<why>)`) when the runner exited anything but +1, its summary is missing, it stopped early (bun `Bailed out`, Playwright +`did not run`, `interrupted`, `Testing stopped early`, a global timeout), it +printed a coverage table or got `--coverage`, or it retried flaky tests itself. +It cannot see a coverage threshold configured without console output (for +example `coverageReporter = ["lcov"]` in `bunfig.toml`), which is why this +scope is opt-in. + +## Rules + +- **One rerun, small failures only.** A failed test gets one rerun when the + runner made no retries and at most `--max-reruns` (3) tests failed. More + failures than that is a real red, and nothing is rerun. When the run is + already red for a test that cannot be rerun, a whole-command rerun is + skipped. +- **A rerun clears a test only cleanly.** The rerun's command must exit 0 and + the test must run once and pass on its first attempt; a rerun the runner + retried again does not count. +- **Never green by accident.** The report is deleted before every attempt, and + a report that attempt did not write counts as missing. A missing, empty, + zero-test or malformed report fails, as does a non-zero exit with no failed + test (a crash or a non-test step) and a runner killed by a signal. Cancelling + `gstack-flaky` (SIGINT, SIGTERM) is passed to the runner and latched: nothing + is rerun after it, whatever exit the runner chooses. When the runner's + summary counts failures its JUnit omits (a bun file that fails to load, a + Playwright error outside any test), the run is red and nothing is rerun. +- **Reports.** After the run, `--junit` holds the first run's report, so a + later upload or test reporter sees what failed. Each rerun's report is kept + beside it as `<name>.rerun-<n>.xml`. Other reporters' outputs (Playwright's + `test-results/` and HTML report) are rewritten by a rerun; upload the JUnit + report as the record. An existing `--junit` file that is not a JUnit report, + or cannot be read, is refused (usage, exit 2), never deleted. +- **Ids.** A test is `<file>::<project>::<describe > title>`, with the project + empty outside Playwright, so a test in two browser projects is two tests. + `gstack-flaky ids --junit <file>` prints them. +- **Ledger.** One JSON line per flaky pass (`kind: flaky-pass`) and per test + that failed its rerun (`rerun-fail`), with the tree, commit, branch, run URL, + attempts, the rerun's scope (`test` or `command`) and the first lines of + the error. Ids, paths and failure text go through gstack's redaction engine + first; when the scan cannot run, they are withheld. The path is `--ledger`, + else `GSTACK_FLAKE_LEDGER_JUNIT`, else + `<state root>/flake-ledger-junit.jsonl`, and `FLAKY_RESULT` names it. It is + kept apart from the free suite's file-level ledger (`GSTACK_FLAKE_LEDGER`). + A ledger write error is a warning and never changes the verdict. + +## Output and exit codes + +One `FLAKY: <flaky-pass|fail|rerun-fail> <id> attempts=<…> (<reason>)` line per +test that failed at least once, `FLAKY_RERUN: scope=command (<why>)` when the +rerun repeated the whole command, then +`FLAKY_RESULT: tests=<n> failed=<n> flaky=<n> reruns=<n> command_exit=<n> verdict=pass|fail ledger=<file|none>`. + +| Exit | Meaning | +|---|---| +| 0 | Every test passed, flaky passes included | +| 1 | A test failed (after its rerun, or with no rerun), including `FLAKY_UNREPORTED_FAILURES` | +| 2 | Usage error, including a `--junit` path that is not a `.xml` file, or an existing file there that is not a readable JUnit report | +| 3 | No usable report, a crash, or a killed runner (a code below on stderr) | + +| Code | Exit | Cause | Fix | +|---|---|---|---| +| `FLAKY_NO_RESULTS` | 3 | No report at `--junit`, an empty one, or one with no testcase | Make the command write JUnit to that path | +| `FLAKY_JUNIT_INVALID` | 3 | The report is not well-formed XML | Make sure one process writes it and nothing appends to it | +| `FLAKY_RETRIES_HIDDEN` | 3 | Playwright retried but its report omits the retries | Set `PLAYWRIGHT_JUNIT_INCLUDE_RETRIES=1` where the report is written | +| `FLAKY_COMMAND_FAILED` | 3 | Non-zero exit with no failed test | Read the command output: a real red | +| `FLAKY_COMMAND_KILLED` | 3 | The runner was killed by a signal (a cancelled job, an OOM kill) | Nothing was rerun; rerun the job | +| `FLAKY_COMMAND_NOT_STARTED` | 3 | The command could not be started | Check the argv after `--` | +| `FLAKY_UNREPORTED_FAILURES` | 1 | The runner counted failures or errors its JUnit omits | Fix the load error or the error outside any test: a real red | +| `FLAKY_LEDGER_WRITE_FAILED` | — | The ledger could not be appended (warning) | Check the path; the verdict stands | diff --git a/docs/standalone-binaries.md b/docs/standalone-binaries.md index 0c4bc8e52..e160f5549 100644 --- a/docs/standalone-binaries.md +++ b/docs/standalone-binaries.md @@ -18,6 +18,7 @@ Beyond the slash-command skills, gstack ships standalone CLIs for workflows that | `gstack-review-log` | **Review-pass receipts** — `--start <skill>` captures the working-tree fingerprint before a diff review; `'<JSON>' --finish <token>` consumes that single-use, repository/branch/skill-scoped receipt. Binding requires matching start/end content and reviewer-reported `completed:true` and `converged:true`; it is not independent proof that a model read the source. `--check-shared-libs <token>` reads a current finding as JSON on stdin and checks prior Skip decisions against the actual branch, capture and eligible raw source blobs. It returns `reusable:false` when proof is missing or unsafe, including older records without logger-versioned coverage. Final review logging computes shared-code fingerprints and coverage rather than trusting supplied proof. | | `gstack-review-read` | **Review freshness** — emits review records with computed `review_freshness.status` and `reason`: CURRENT, STALE, or UNVERIFIED for diff reviews. `/ship` and `/land-and-deploy` use the same grade; a matching commit alone never certifies a diff review. [Dashboard rules](skills.md#review-readiness-dashboard). | | `gstack-evidence` | **Verification-evidence ledger** — `run --label <lane> -- <cmd>` transparently wraps any test command (the child's exit code always passes through) and records what ran against which working-tree fingerprint (tracked and untracked files that are not gitignored, minus an untracked `.bun/` folder at the repo root, which Bun can create during a run); a run whose files changed while it ran is recorded without a fingerprint and with up to five of the changed paths, which `check` names in its STALE reason. `check` grades each label FRESH/STALE/MISSING with `--expect-cmd`, `--max-age`, and `--allow-paths` binding. /ship and /land-and-deploy cite fresh evidence instead of re-running suites. Per-run logs are 0600, capped at 2MB, pruned after 30 days; the ledger and logs stay machine-local by design. | +| `gstack-flaky` | **Flaky-test rerun for any JUnit runner** — `run --junit <file.xml> -- <command>` runs your test command (Playwright, `bun test`, jest, vitest, pytest) and, when at most `--max-reruns` (3) tests failed and the runner made no retries of its own, reruns the whole command once (`--rerun-scope test` opts in to rerunning only the failed tests). A test that fails and then passes on that rerun is a flaky pass: the run stays green and one redacted line per flaky pass or failed rerun goes on a ledger (`GSTACK_FLAKE_LEDGER_JUNIT`, default `<state root>/flake-ledger-junit.jsonl`). A missing or malformed report, a crash or a killed runner never passes. `ids --junit <file>` prints a report's test ids. Exit 0 pass, 1 a test failed, 2 usage, 3 no usable report. [Guide](flaky.md). | | `gstack-pr-wait` | **PR event watcher** — `gstack-pr-wait <pr> [--repo owner/name] [--until green\|ready\|any-change] [--interval 20] [--timeout 60] [--exclude CHECK,...]` polls one pull request through `gh` and prints one stdout line per real event (a check failed with its link, all checks finished, a comment or review from someone other than the author, a review bot finished on the head, the base moved and its new commits touch this PR's files, BEHIND or DIRTY, merged or closed), with progress on stderr, so a monitor wakes the agent once per event instead of on a timer. `green` gates on every check on the head (an advisory bot's review is never waited for, its checks gate like any other), as `gstack-ci-gate` does; a check the base branch requires (rulesets and classic branch protection) is pending until it appears, never green by omission; two identical polls confirm a check set is complete; `--exclude` leaves a named non-required check out of the verdict as `gstack-ci-gate --exclude` does (a required name is refused and gates). Exit 0 when the condition is met, 1 when it needs action or `--timeout` passes (`PR_WAIT_BLOCKED`), 2 on bad input, a second watcher for the same PR or a local error (`PR_WAIT_REFUSED`). State under `<state root>/pr-wait/`, baseline discarded after 6 hours. Review bots are configuration: `gstack-config set pr_wait_gating_bots greptile,macroscope`, `pr_wait_advisory_bots coderabbit` (advisory bots are reported, never waited on). `/land-and-deploy` Step 3 is one call of it. Design ported from tstack's pr-wait (MIT, Theo Browne; NOTICE.md). [Exit codes](troubleshooting.md#gstack-pr-wait). | | `gstack-issue-guard` | **Tracker-text trust envelope** — fetches GitHub issue/PR text (`issue <n>`, `pr-body`, `pr-comments`, or `--stdin`) and wraps it in a labeled envelope so agents treat it as data: injection-shaped lines get labeled even through fullwidth and invisible-character evasion, and forged envelope banners are defused. Every tracker-text ingress in gstack routes through it, enforced by a CI scanner. | | `gstack-ios-qa-daemon` | **iOS QA daemon** — Mac-side broker between an agent and a connected iPhone or iPad over USB CoreDevice. Loopback by default; `--tailnet` opens a Tailscale-facing listener with identity-gated capability tiers. Single-instance via flock on `~/.gstack/ios-qa-daemon.pid`. See [docs/howto-ios-testing-with-gstack.md](howto-ios-testing-with-gstack.md). | diff --git a/lib/flaky-run.ts b/lib/flaky-run.ts new file mode 100644 index 000000000..4c2f0d65e --- /dev/null +++ b/lib/flaky-run.ts @@ -0,0 +1,368 @@ +/** + * gstack-flaky run (GSTA-158): run a test command, read its JUnit, rerun once + * when tests failed, and record flaky passes on a ledger. This is CREW rule 16 + * ("run the failed test again and note that it is flaky") for any runner that + * writes JUnit, the per-test form of the free suite's GSTACK_FREE_RETRY_FLAKY. + * + * Rules (docs/flaky.md): + * - A failed test gets one rerun when the runner made no retries of its own and + * at most maxReruns tests failed; more failures are a real red and nothing + * is rerun. When the runner retried (Playwright `retries` with retry-aware + * JUnit), its attempts stand and nothing is rerun again. + * - The rerun repeats the whole command by default, and counts only on the + * runner's own full green verdict: exit 0, no failed test, every first-run + * test present, nothing counted outside the report. Only that verdict covers + * every reason a run can fail (a test it never reached, a coverage + * threshold, a fail-on-flaky policy). + * - rerunScope 'test' (opt-in) reruns the failed test alone, and only with + * proof that the first run was complete: the runner's own summary, its + * test-failure exit status 1, no early stop, no coverage and no runner-flaky + * test; otherwise, and always for generic runners, the whole command reruns. + * - A rerun makes a flaky pass only when its command exited 0 and the test ran + * exactly once and passed on its first attempt. The command is an argv run + * without a shell; a targeted rerun appends the adapter's selection, each one + * argv element. + * - The JUnit file is removed before every attempt; a report that attempt did + * not write is missing. Missing, unreadable or empty JUnit is never green; a + * non-zero exit with no failed test and a runner killed by a signal are never + * green. When the runner's own summary counts failures or errors the JUnit + * omits (bun files that fail to load), the run is red and nothing is rerun. + * The first report is restored at the --junit path when the run ends; each + * rerun's report is kept beside it as <name>.rerun-<n>.xml. + */ +import * as fs from 'fs'; +import * as path from 'path'; +import { spawn as spawnChild, spawnSync } from 'child_process'; +import { finalAttempt, JunitError, readJunit, type JunitReport, type JunitRunner, type JunitTest } from './junit'; +import { redactFindingSpans } from './redact-engine'; +import { resolveStateRoot } from './state-root'; + +export type LedgerKind = 'flaky-pass' | 'rerun-fail'; + +export interface LedgerLine { + ts: string; + runner: 'junit'; + kind: LedgerKind; + id: string; + file: string; + /** 'runner' when the runner's own retries decided, 'gstack-flaky' for our rerun. */ + rerun_by: 'runner' | 'gstack-flaky'; + /** For gstack-flaky reruns: the failed test alone, or the whole command when the first run could not be proven complete. */ + rerun_scope?: 'test' | 'command'; + attempts: string[]; + tree?: string; + git_sha?: string; + branch?: string; + run_url?: string; + excerpt: string[]; +} + +export interface RunOptions { + command: string[]; + junit: string; + runner?: JunitRunner; + maxReruns: number; + /** 'command' (default): rerun the whole command. 'test': rerun the failed test alone when the first run is proven complete. */ + rerunScope?: 'command' | 'test'; + cwd?: string; + env?: NodeJS.ProcessEnv; + /** Runs one attempt; resolves with its exit status and the tail of its combined output. */ + spawn?: (argv: string[], env: NodeJS.ProcessEnv, cwd: string) => Promise<Attempted>; + now?: () => Date; +} + +export interface TestOutcome { + test: JunitTest; + status: 'flaky-pass' | 'fail' | 'rerun-fail'; + attempts: string[]; + reason?: string; +} + +export interface Attempted { exit: number; output: string; signal?: string } + +export interface RunResult { + exitCode: 0 | 1 | 3; + code?: string; + message?: string; + tests: number; + commandExit: number; + outcomes: TestOutcome[]; + reruns: number; + /** Why the rerun was the whole command (the default, or why a requested targeted rerun was refused); absent for targeted reruns. */ + wholeRerun?: string; + ledger: LedgerLine[]; +} + +export const DEFAULT_MAX_RERUNS = 3; + +const escapeRegex = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + +/** Selection arguments a targeted rerun appends for one test, each one argv element. */ +export function rerunSelection(runner: 'playwright' | 'bun', t: JunitTest): string[] { + if (runner === 'bun') return [t.file, '--test-name-pattern', `^${escapeRegex(t.path.join(' '))}$`]; + const args = [escapeRegex(t.file), '--grep', `(?:^|\\s)${escapeRegex(t.path.join(' '))}(?:\\s|$)`]; + return t.project ? [...args, '--project', t.project] : args; +} + +const OUTPUT_TAIL = 256 * 1024; + +/** Set once gstack-flaky itself is asked to stop; every attempt after that counts as killed, whatever exit the runner chose. */ +let cancelled: string | undefined; + +/** Stream the child's output live (a quiet runner trips CI idle watchdogs) and keep its tail for reconciliation. */ +function defaultSpawn(argv: string[], env: NodeJS.ProcessEnv, cwd: string): Promise<Attempted> { + return new Promise((resolve, reject) => { + let tail = ''; + const keep = (chunk: Buffer) => { tail = (tail + chunk.toString('utf-8')).slice(-OUTPUT_TAIL); }; + const child = spawnChild(argv[0]!, argv.slice(1), { stdio: ['inherit', 'pipe', 'pipe'], env, cwd }); + const forward = (s: NodeJS.Signals) => { cancelled ??= s; child.kill(s); }; + const release = () => { process.off('SIGINT', forward); process.off('SIGTERM', forward); }; + process.on('SIGINT', forward); + process.on('SIGTERM', forward); + child.stdout!.on('data', (c: Buffer) => { process.stdout.write(c); keep(c); }); + child.stderr!.on('data', (c: Buffer) => { process.stderr.write(c); keep(c); }); + child.on('error', e => { release(); reject(new JunitError('FLAKY_COMMAND_NOT_STARTED', `could not start ${argv[0]}: ${e.message}`)); }); + child.on('close', (code, signal) => { + release(); + const killed = signal ?? cancelled; + resolve({ exit: code ?? 128, output: tail, ...(killed ? { signal: killed } : {}) }); + }); + }); +} + +const plain = (output: string) => output.replace(/\x1b\[[0-9;]*m/g, ''); + +/** + * Failures a runner counted that its JUnit omits: bun leaves files that fail to + * load out of the report ("N errors"), Playwright reports errors outside any + * test. A rerun cannot clear those, so they keep the run red. + */ +export function unreportedFailures(runner: JunitRunner, output: string, reportedFailed: number): string | null { + const text = plain(output); + const last = (re: RegExp) => { const m = [...text.matchAll(re)].pop(); return m ? Number(m[1]) : 0; }; + if (runner === 'bun') { + const errors = last(/^\s*(\d+) errors?\s*$/gm); + const fails = last(/^\s*(\d+) fail\s*$/gm); + if (errors > 0) return `bun reported ${errors} error(s) outside the JUnit report (a file failed to load or threw between tests)`; + if (fails > reportedFailed) return `bun counted ${fails} failing tests, the JUnit report ${reportedFailed}`; + } + if (runner === 'playwright') { + const outside = last(/(\d+) errors? (?:was|were) not a part of any test/g); + const fails = last(/^\s*(\d+) failed\s*$/gm); + if (outside > 0) return `Playwright reported ${outside} error(s) outside any test`; + if (fails > reportedFailed) return `Playwright counted ${fails} failed tests, the JUnit report ${reportedFailed}`; + } + return null; +} + +/** + * Why the first run cannot be trusted to have run every test, so a rerun must + * repeat the whole command; null when the runner's own output proves it + * complete and its exit status explained by the failed tests alone. A run that + * stopped early reports only the tests it reached, and a targeted rerun would + * never run the rest; a coverage threshold or a fail-on-flaky policy fails the + * run for a reason a narrowed rerun cannot reproduce. + */ +export function incompleteRun(runner: JunitRunner, output: string, exit: number, command: string[] = [], runnerFlaky = 0): string | null { + if (runner === 'generic') return 'a generic runner cannot prove every test ran'; + if (exit !== 1) return `the command exited ${exit}, not the runner's test-failure status 1`; + const text = plain(output); + if (command.some(a => a === '--coverage' || a.startsWith('--coverage=')) || /%\s*Funcs\s*\|\s*%\s*Lines|coverage threshold/i.test(text)) return 'coverage is measured, and a narrowed rerun cannot reproduce a coverage failure'; + if (runnerFlaky) return `the runner retried ${runnerFlaky} flaky test(s), which a fail-on-flaky policy can fail the run on`; + if (runner === 'bun') { + if (/Bailed out after/.test(text)) return 'bun stopped early (--bail)'; + if (!/^\s*Ran \d+ tests? across \d+ files?/m.test(text)) return 'bun printed no run summary'; + return null; + } + const notRun = text.match(/\b(\d+) (did not run|interrupted)\b/); + if (notRun) return `Playwright reports ${notRun[1]} ${notRun[2]}`; + if (/Testing stopped early|Timed out waiting [\d.]+s for the (?:entire )?test run/.test(text)) return 'Playwright stopped early'; + if (!/^\s*\d+ failed\s*$/m.test(text)) return 'Playwright printed no summary (add a console reporter: --reporter=line,junit)'; + return null; +} + +function git(args: string[], cwd: string): string | undefined { + const r = spawnSync('git', args, { cwd, encoding: 'utf-8', timeout: 10_000 }); + return r.status === 0 ? r.stdout.trim() || undefined : undefined; +} + +export function provenance(cwd: string, env: NodeJS.ProcessEnv): Pick<LedgerLine, 'tree' | 'git_sha' | 'branch' | 'run_url'> { + const runUrl = env.GITHUB_SERVER_URL && env.GITHUB_REPOSITORY && env.GITHUB_RUN_ID + ? `${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}${env.GITHUB_RUN_ATTEMPT ? `/attempts/${env.GITHUB_RUN_ATTEMPT}` : ''}` + : undefined; + return { + tree: git(['rev-parse', 'HEAD^{tree}'], cwd), + git_sha: git(['rev-parse', 'HEAD'], cwd), + branch: env.GITHUB_HEAD_REF || env.GITHUB_REF_NAME || git(['rev-parse', '--abbrev-ref', 'HEAD'], cwd), + run_url: runUrl, + }; +} + +/** Test ids and paths leave this process only through the redaction engine: a parameterized title can carry a credential. */ +export function safeText(s: string): string { + return redactFindingSpans(s) ?? '[withheld: the redaction scan could not run]'; +} + +/** Escape a value for a GitHub workflow command, so a test title cannot end the command or start another. */ +export const workflowEscape = (s: string) => s.replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A'); + +/** Failure text leaves this process only through the redaction engine; when it cannot scan, nothing leaves. */ +export function safeExcerpt(lines: string[]): string[] { + if (!lines.length) return []; + const redacted = redactFindingSpans(lines.join('\n')); + return redacted === null ? ['[excerpt withheld: the redaction scan could not run]'] : redacted.split('\n'); +} + +/** One attempt: remove the report, run, and read a report this attempt wrote. */ +async function attempt(opts: RunOptions, argv: string[], cwd: string, env: NodeJS.ProcessEnv, runner = opts.runner): Promise<Attempted & { report?: JunitReport; error?: JunitError }> { + const junit = path.resolve(cwd, opts.junit); + fs.rmSync(junit, { force: true }); + const ran = await (opts.spawn ?? defaultSpawn)(argv, env, cwd); + if (!fs.existsSync(junit)) return { ...ran, error: new JunitError('FLAKY_NO_RESULTS', `the command wrote no JUnit report at ${opts.junit} (exit ${ran.exit})`) }; + try { return { ...ran, report: readJunit(junit, runner) }; } catch (e) { + if (e instanceof JunitError) return { ...ran, error: e }; + throw e; + } +} + +/** + * What keeps a rerun from clearing anything even when it exited 0: failures + * its runner counted outside the report and, for a whole-command rerun, any + * failed test or any first-run test missing from its report. + */ +function rerunProblem(again: Attempted & { report?: JunitReport }, first: JunitReport, whole: boolean): string | undefined { + if (!again.report) return undefined; + const failedNow = again.report.tests.filter(x => finalAttempt(x) === 'fail'); + const unreported = unreportedFailures(first.runner, again.output, failedNow.length); + if (unreported) return unreported; + if (!whole) return undefined; + if (failedNow.length) return `${failedNow.length} test(s) failed in the whole-command rerun`; + const seen = new Set(again.report.tests.map(x => x.id)); + const missing = first.tests.filter(x => !seen.has(x.id)).length; + return missing ? `${missing} test(s) of the first run are missing from the whole-command rerun` : undefined; +} + +/** A rerun clears a test only when its command exited 0 and the test ran once and passed on its first attempt. */ +function rerunVerdict(again: Attempted & { report?: JunitReport; error?: JunitError }, t: JunitTest, problem?: string): { status: 'flaky-pass' | 'rerun-fail'; attempts: string[]; reason?: string } { + const match = again.report?.tests.find(x => x.id === t.id); + if (!match) return { status: 'rerun-fail', attempts: [...t.attempts, 'fail'], reason: again.error ? `rerun: ${again.error.message}` : 'rerun: the report does not contain this test' }; + const attempts = [...t.attempts, ...match.attempts]; + if (again.signal) return { status: 'rerun-fail', attempts, reason: `rerun: killed by ${again.signal}` }; + if (again.exit !== 0) return { status: 'rerun-fail', attempts, reason: `rerun: the command exited ${again.exit}` }; + if (problem) return { status: 'rerun-fail', attempts, reason: `rerun: ${problem}` }; + if (match.executions !== 1 || match.attempts.join() !== 'pass') return { status: 'rerun-fail', attempts, reason: `rerun: the test did not pass once on its first attempt (${match.attempts.join(',')})` }; + return { status: 'flaky-pass', attempts }; +} + +export async function runFlaky(opts: RunOptions): Promise<RunResult> { + const cwd = opts.cwd ?? process.cwd(); + const env = { ...(opts.env ?? process.env), PLAYWRIGHT_JUNIT_INCLUDE_RETRIES: '1' }; + const now = opts.now ?? (() => new Date()); + const empty = { tests: 0, outcomes: [], reruns: 0, ledger: [] }; + const first = await attempt(opts, opts.command, cwd, env); + if (first.signal) return { exitCode: 3, code: 'FLAKY_COMMAND_KILLED', message: `the command was killed by ${first.signal}`, commandExit: first.exit, ...empty }; + if (!first.report) return { exitCode: 3, code: first.error!.code, message: first.error!.message, commandExit: first.exit, ...empty }; + const report = first.report; + if (report.retriesHidden) { + return { exitCode: 3, code: 'FLAKY_RETRIES_HIDDEN', commandExit: first.exit, ...empty, tests: report.tests.length, + message: 'Playwright retried but its JUnit omits the retries; set PLAYWRIGHT_JUNIT_INCLUDE_RETRIES=1 (gstack-flaky run sets it for the command it starts)' }; + } + const failed = report.tests.filter(t => finalAttempt(t) === 'fail'); + const runnerFlaky = report.tests.filter(t => finalAttempt(t) !== 'fail' && t.attempts.includes('fail')); + if (first.exit !== 0 && failed.length === 0) { + return { exitCode: 3, code: 'FLAKY_COMMAND_FAILED', commandExit: first.exit, ...empty, tests: report.tests.length, + message: `the command exited ${first.exit} but its JUnit reports no failed test; a crash or a non-test step failed` }; + } + const unreported = unreportedFailures(report.runner, first.output, failed.length); + if (unreported) { + return { exitCode: 1, code: 'FLAKY_UNREPORTED_FAILURES', commandExit: first.exit, ...empty, tests: report.tests.length, + outcomes: failed.map(t => ({ test: t, status: 'fail' as const, attempts: t.attempts, reason: 'not rerun: the runner reported failures outside the JUnit report' })), + message: `${unreported}; nothing was rerun` }; + } + const prov = provenance(cwd, env); + const line = (t: JunitTest, kind: LedgerKind, by: LedgerLine['rerun_by'], attempts: string[], scope?: LedgerLine['rerun_scope']): LedgerLine => + ({ ts: now().toISOString(), runner: 'junit', kind, id: safeText(t.id), file: safeText(t.file), rerun_by: by, ...(scope ? { rerun_scope: scope } : {}), attempts, ...prov, excerpt: safeExcerpt(t.excerpt) }); + const ledger: LedgerLine[] = runnerFlaky.map(t => line(t, 'flaky-pass', 'runner', t.attempts)); + const outcomes: TestOutcome[] = runnerFlaky.map(t => ({ test: t, status: 'flaky-pass' as const, attempts: t.attempts })); + const eligible: JunitTest[] = []; + for (const t of failed) { + if (t.executions > 1) outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: `it ran ${t.executions} times in this report and one failed; not rerun` }); + else if (t.attempts.length > 1) outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: `the runner already retried (${t.attempts.length} attempts)` }); + else if (failed.length > opts.maxReruns) outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: `${failed.length} tests failed, more than --max-reruns ${opts.maxReruns}; not rerun` }); + else eligible.push(t); + } + const wholeRerun = !eligible.length ? undefined + : opts.rerunScope !== 'test' ? 'the default; --rerun-scope test reruns the failed test alone' + : incompleteRun(report.runner, first.output, first.exit, opts.command, runnerFlaky.length) ?? undefined; + const junit = path.resolve(cwd, opts.junit); + const firstXml = fs.readFileSync(junit); + let reruns = 0; + const rerun = async (argv: string[]) => { + reruns++; + const again = await attempt(opts, argv, cwd, env, report.runner); + if (fs.existsSync(junit)) fs.renameSync(junit, `${junit.replace(/\.xml$/i, '')}.rerun-${reruns}.xml`); + return again; + }; + const record = (t: JunitTest, v: ReturnType<typeof rerunVerdict>, scope: 'test' | 'command') => { + outcomes.push({ test: t, status: v.status, attempts: v.attempts, ...(v.reason ? { reason: v.reason } : {}) }); + ledger.push(line(t, v.status, 'gstack-flaky', v.attempts, scope)); + }; + try { + if (wholeRerun && outcomes.some(o => o.status === 'fail')) { + for (const t of eligible) outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: `not rerun: the run is already red, and a rerun would repeat the whole command (${wholeRerun})` }); + } else if (wholeRerun) { + const again = await rerun(opts.command); + const problem = rerunProblem(again, report, true); + for (const t of eligible) record(t, rerunVerdict(again, t, problem), 'command'); + } else if (eligible.length) { + let interrupted = false; + for (const t of eligible) { + if (interrupted) { outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: 'not rerun: an earlier rerun was killed' }); continue; } + const again = await rerun([...opts.command, ...rerunSelection(report.runner as 'playwright' | 'bun', t)]); + interrupted = !!again.signal; + record(t, rerunVerdict(again, t, rerunProblem(again, report, false)), 'test'); + } + } + } finally { + if (reruns) fs.writeFileSync(junit, firstXml); + } + const red = outcomes.some(o => o.status !== 'flaky-pass'); + return { exitCode: red ? 1 : 0, tests: report.tests.length, commandExit: first.exit, outcomes, reruns, ...(wholeRerun && reruns ? { wholeRerun } : {}), ledger }; +} + +export function ledgerPath(flag: string | undefined, env: NodeJS.ProcessEnv = process.env): string { + if (flag) return flag; + if (env.GSTACK_FLAKE_LEDGER_JUNIT) return env.GSTACK_FLAKE_LEDGER_JUNIT; + return path.join(resolveStateRoot(env), 'flake-ledger-junit.jsonl'); +} + +/** Append ledger lines; an I/O error is a warning and never changes the verdict. */ +export function appendLedger(lines: LedgerLine[], file: string, warn: (s: string) => void): void { + if (!lines.length) return; + try { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.appendFileSync(file, lines.map(l => JSON.stringify(l)).join('\n') + '\n'); + } catch (e) { + warn(`FLAKY_LEDGER_WRITE_FAILED: could not append ${file} (${(e as Error).message}); the verdict is unaffected`); + } +} + +/** Lines the bin prints: one per non-passing test, then the result line. */ +export function renderResult(r: RunResult, ledger: string): string[] { + const out: string[] = []; + for (const o of r.outcomes) out.push(`FLAKY: ${o.status} ${safeText(o.test.id)} attempts=${o.attempts.join(',')}${o.reason ? ` (${o.reason})` : ''}`); + const flaky = r.outcomes.filter(o => o.status === 'flaky-pass').length; + const failed = r.outcomes.length - flaky; + if (r.wholeRerun) out.push(`FLAKY_RERUN: scope=command (${r.wholeRerun})`); + out.push(`FLAKY_RESULT: tests=${r.tests} failed=${failed} flaky=${flaky} reruns=${r.reruns} command_exit=${r.commandExit} verdict=${r.exitCode === 0 ? 'pass' : 'fail'} ledger=${r.ledger.length ? ledger : 'none'}`); + return out; +} + +/** A GitHub step summary for the flaky passes, so they are seen before any report reads the ledger. */ +export function stepSummary(r: RunResult): string { + const rows = r.outcomes.filter(o => o.status === 'flaky-pass'); + if (!rows.length) return ''; + const cell = (s: string) => s.replace(/\|/g, '\\|').replace(/`/g, "'").replace(/[\r\n]+/g, ' '); + return ['### Flaky passes (gstack-flaky)', '', '| Test | Attempts | First error |', '|---|---|---|', + ...rows.map(o => `| \`${cell(safeText(o.test.id))}\` | ${o.attempts.join(', ')} | ${cell(safeExcerpt(o.test.excerpt)[0] ?? '')} |`), ''].join('\n'); +} diff --git a/lib/junit.ts b/lib/junit.ts new file mode 100644 index 000000000..0f4d504d3 --- /dev/null +++ b/lib/junit.ts @@ -0,0 +1,204 @@ +/** + * JUnit XML reader for gstack-flaky (GSTA-158). One reader for every runner + * that writes JUnit: Playwright (`--reporter=junit`, retry-aware with + * PLAYWRIGHT_JUNIT_INCLUDE_RETRIES=1), `bun test --reporter=junit`, jest, + * pytest and vitest. No XML dependency: the subset JUnit uses (elements, + * attributes, text, CDATA, comments, processing instructions) is tokenized + * here. + * + * A test's id is `<file>::<project>::<title>`. The title is the describe path + * plus the test name joined by " > ", so Playwright's " › " and bun's nested + * testsuites give the same shape. The project is Playwright's project name + * (its testsuite `hostname`) and empty for every other runner, so a test + * that runs in two browser projects is two ids. + * + * Attempts: Playwright's retry-aware report lists each failed retry as + * `rerunFailure` (final result failed) or `flakyFailure` (final result + * passed). A Playwright failure whose text shows "Retry #" without those + * elements means retries ran but were not reported (`retriesHidden`). A test + * that appears more than once (repeat-each, a duplicate name) fails when any + * of its executions failed. + */ +import * as fs from 'fs'; + +export type JunitRunner = 'playwright' | 'bun' | 'generic'; +export type Attempt = 'pass' | 'fail' | 'skip'; + +export interface JunitTest { + id: string; + file: string; + project: string; + /** Describe path plus test name. */ + path: string[]; + attempts: Attempt[]; + /** Testcase elements with this id in the report (more than one: repeat-each or a duplicate name). */ + executions: number; + /** First lines of the final failure, or of the first failed attempt. */ + excerpt: string[]; +} + +export interface JunitReport { + runner: JunitRunner; + tests: JunitTest[]; + retriesHidden: boolean; +} + +export class JunitError extends Error { + constructor(readonly code: string, message: string) { super(message); } +} + +interface XmlNode { name: string; attrs: Record<string, string>; children: XmlNode[]; text: string; parent: XmlNode | null } + +const ENTITIES: Record<string, string> = { lt: '<', gt: '>', amp: '&', quot: '"', apos: "'" }; +function decode(s: string): string { + return s.replace(/&(#x[0-9a-fA-F]+|#\d+|[a-z]+);/g, (m, e: string) => { + if (e[0] === '#') { + const code = e[1] === 'x' ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10); + return Number.isFinite(code) && code <= 0x10ffff ? String.fromCodePoint(code) : m; + } + return ENTITIES[e] ?? m; + }); +} + +const TAG = /^<(\/?)([A-Za-z_][\w:.-]*)((?:\s+[^\s=/>]+\s*=\s*(?:"[^"]*"|'[^']*'))*)\s*(\/?)>/; +const ATTR = /([^\s=/>]+)\s*=\s*(?:"([^"]*)"|'([^']*)')/g; + +export function parseXml(xml: string): XmlNode { + const root: XmlNode = { name: '#root', attrs: {}, children: [], text: '', parent: null }; + let cur = root; + let i = 0; + while (i < xml.length) { + const lt = xml.indexOf('<', i); + if (lt < 0) { cur.text += decode(xml.slice(i)); break; } + if (lt > i) cur.text += decode(xml.slice(i, lt)); + const rest = xml.slice(lt, lt + 9); + if (rest.startsWith('<![CDATA[')) { + const end = xml.indexOf(']]>', lt); + if (end < 0) throw new JunitError('FLAKY_JUNIT_INVALID', 'unterminated CDATA section'); + cur.text += xml.slice(lt + 9, end); + i = end + 3; + continue; + } + if (rest.startsWith('<!--')) { + const end = xml.indexOf('-->', lt); + if (end < 0) throw new JunitError('FLAKY_JUNIT_INVALID', 'unterminated comment'); + i = end + 3; + continue; + } + if (rest.startsWith('<?') || rest.startsWith('<!')) { + const end = xml.indexOf('>', lt); + if (end < 0) throw new JunitError('FLAKY_JUNIT_INVALID', 'unterminated declaration'); + i = end + 1; + continue; + } + const m = TAG.exec(xml.slice(lt)); + if (!m) throw new JunitError('FLAKY_JUNIT_INVALID', `malformed tag at offset ${lt}`); + const [whole, close, name, attrText, selfClose] = m; + if (close) { + if (cur.name !== name) throw new JunitError('FLAKY_JUNIT_INVALID', `</${name}> closes <${cur.name}> at offset ${lt}`); + cur = cur.parent!; + } else { + const attrs: Record<string, string> = {}; + for (const a of attrText!.matchAll(ATTR)) attrs[a[1]!] = decode(a[2] ?? a[3] ?? ''); + const node: XmlNode = { name: name!, attrs, children: [], text: '', parent: cur }; + cur.children.push(node); + if (!selfClose) cur = node; + } + i = lt + whole!.length; + } + if (cur !== root) throw new JunitError('FLAKY_JUNIT_INVALID', `<${cur.name}> is never closed`); + return root; +} + +function allNodes(node: XmlNode, name: string, out: XmlNode[] = []): XmlNode[] { + for (const c of node.children) { + if (c.name === name) out.push(c); + allNodes(c, name, out); + } + return out; +} + +export function detectRunner(root: XmlNode): JunitRunner { + const top = root.children.find(c => c.name === 'testsuites'); + if (top?.attrs.name === 'bun test') return 'bun'; + const cases = allNodes(root, 'testcase'); + if (cases.some(c => (c.attrs.name ?? '').includes(' › '))) return 'playwright'; + const suites = allNodes(root, 'testsuite'); + if (suites.length && suites.every(s => 'timestamp' in s.attrs && 'hostname' in s.attrs && !('file' in s.attrs)) + && cases.every(c => c.parent?.name === 'testsuite' && c.attrs.classname === c.parent.attrs.name)) return 'playwright'; + return 'generic'; +} + +function excerptOf(node: XmlNode | undefined): string[] { + if (!node) return []; + const body = (node.attrs.message ? node.attrs.message + '\n' : '') + node.text; + return body.replace(/\x1b\[[0-9;]*m/g, '').split('\n').map(l => l.trimEnd()).filter(l => l.trim()).slice(0, 8); +} + +export const testId = (file: string, project: string, path: string[]) => `${file}::${project}::${path.join(' > ')}`; + +function readCase(tc: XmlNode, runner: JunitRunner): Omit<JunitTest, 'id' | 'executions'> & { hidden: boolean } { + const suite = tc.parent!; + let file: string, project = '', path: string[]; + if (runner === 'playwright') { + file = tc.attrs.classname || suite.attrs.name || ''; + project = suite.attrs.hostname ?? ''; + path = (tc.attrs.name ?? '').split(' › '); + } else if (runner === 'bun') { + file = tc.attrs.file ?? ''; + path = []; + for (let s: XmlNode | null = suite; s && s.name === 'testsuite'; s = s.parent) { + if ('file' in s.attrs && s.attrs.name === s.attrs.file && s.parent?.name !== 'testsuite') break; + path.unshift(s.attrs.name ?? ''); + } + path.push(tc.attrs.name ?? ''); + } else { + file = tc.attrs.file || tc.attrs.classname || suite.attrs.file || suite.attrs.name || ''; + path = [tc.attrs.name ?? '']; + } + const kids = (n: string) => tc.children.filter(c => c.name === n); + const final = kids('failure')[0] ?? kids('error')[0]; + const skipped = kids('skipped').length > 0; + const flaky = [...kids('flakyFailure'), ...kids('flakyError')]; + const rerun = [...kids('rerunFailure'), ...kids('rerunError')]; + let attempts: Attempt[]; + if (final) attempts = [...rerun.map(() => 'fail' as const), 'fail']; + else if (skipped) attempts = ['skip']; + else attempts = [...flaky.map(() => 'fail' as const), 'pass']; + const hidden = runner === 'playwright' && !!final && rerun.length === 0 && /\bRetry #\d+/.test(final.text); + return { file, project, path, attempts, excerpt: excerptOf(final ?? flaky[0]), hidden }; +} + +export function parseJunit(xml: string, runner?: JunitRunner): JunitReport { + const root = parseXml(xml); + const kind = runner ?? detectRunner(root); + const byId = new Map<string, JunitTest>(); + let retriesHidden = false; + for (const tc of allNodes(root, 'testcase')) { + const { hidden, ...t } = readCase(tc, kind); + retriesHidden ||= hidden; + const id = testId(t.file, t.project, t.path); + const seen = byId.get(id); + if (seen) { + const failed = finalAttempt(seen) === 'fail' || t.attempts.at(-1) === 'fail'; + seen.attempts = [...seen.attempts, ...t.attempts].sort((a, b) => failed ? Number(a === 'fail') - Number(b === 'fail') : 0); + seen.executions++; + if (!seen.excerpt.length) seen.excerpt = t.excerpt; + } else byId.set(id, { id, ...t, executions: 1 }); + } + if (byId.size === 0) throw new JunitError('FLAKY_NO_RESULTS', 'the JUnit report holds no testcase'); + return { runner: kind, tests: [...byId.values()], retriesHidden }; +} + +/** Read and parse one JUnit file; missing, unreadable and empty files are errors, never green. */ +export function readJunit(file: string, runner?: JunitRunner): JunitReport { + let xml: string; + try { xml = fs.readFileSync(file, 'utf-8'); } catch (e) { + throw new JunitError('FLAKY_NO_RESULTS', `cannot read ${file}: ${(e as Error).message}`); + } + if (!xml.trim()) throw new JunitError('FLAKY_NO_RESULTS', `${file} is empty`); + return parseJunit(xml, runner); +} + +/** The final outcome of a test: its last attempt. */ +export const finalAttempt = (t: JunitTest): Attempt => t.attempts[t.attempts.length - 1]!; diff --git a/package.json b/package.json index 8bad65817..3884072da 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.371", + "version": "1.91.372", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/test/fixtures/junit/bun.xml b/test/fixtures/junit/bun.xml new file mode 100644 index 000000000..7b8f851b1 --- /dev/null +++ b/test/fixtures/junit/bun.xml @@ -0,0 +1,17 @@ +<?xml version="1.0" encoding="UTF-8"?> +<testsuites name="bun test" tests="4" assertions="2" failures="1" skipped="1" time="0.02640424"> + <testsuite name="a.test.ts" file="a.test.ts" tests="4" assertions="2" failures="1" skipped="1" time="0.014448064" hostname="ci-host"> + <testsuite name="outer" file="a.test.ts" line="2" tests="3" assertions="2" failures="1" skipped="1" time="0" hostname="ci-host"> + <testsuite name="inner" file="a.test.ts" line="2" tests="2" assertions="2" failures="1" skipped="0" time="0.006" hostname="ci-host"> + <testcase name="passes" classname="inner > outer" time="0.000069" file="a.test.ts" line="2" assertions="1" /> + <testcase name="fails" classname="inner > outer" time="0.006322" file="a.test.ts" line="2" assertions="1"> + <failure type="AssertionError" message="expect(received).toBe(expected) Expected: 2 Received: 1 ">AssertionError: expect(received).toBe(expected) Expected: 2 Received: 1 at a.test.ts:2:131 </failure> + </testcase> + </testsuite> + <testcase name="skipped" classname="outer" time="0" file="a.test.ts" line="2" assertions="0"> + <skipped /> + </testcase> + </testsuite> + <testcase name="top level" classname="" time="0.000039" file="a.test.ts" line="3" assertions="0" /> + </testsuite> +</testsuites> diff --git a/test/fixtures/junit/playwright-r0.xml b/test/fixtures/junit/playwright-r0.xml new file mode 100644 index 000000000..8fb6599e0 --- /dev/null +++ b/test/fixtures/junit/playwright-r0.xml @@ -0,0 +1,138 @@ +<testsuites id="" name="" tests="10" failures="4" skipped="2" errors="0" time="3.23467"> +<testsuite name="a.spec.ts" timestamp="2026-10-29T16:02:06.214Z" hostname="alpha" tests="5" failures="2" skipped="1" time="0.03" errors="0"> +<testcase name="group one › passes" classname="a.spec.ts" time="0.011"> +</testcase> +<testcase name="group one › fails always" classname="a.spec.ts" time="0.004"> +<system-out> +<![CDATA[ +[[ATTACHMENT|pw/test-results/a-group-one-fails-always-alpha/error-context.md]] +]]> +</system-out> +<failure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe"> +<![CDATA[ [alpha] › a.spec.ts:5:7 › group one › fails always ─────────────────────────────────────────────── + + Error: expect(received).toBe(expected) // Object.is equality + + Expected: 2 + Received: 1 + + 3 | test.describe('group one', () => { + 4 | test('passes', () => { expect(1).toBe(1); }); + > 5 | test('fails always', () => { expect(1).toBe(2); }); + | ^ + 6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } }); + 7 | test.skip('skipped', () => {}); + 8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); }); + at /repo/tests/a.spec.ts:5:42 + + Error Context: ../test-results/a-group-one-fails-always-alpha/error-context.md +]]> +</failure> +</testcase> +<testcase name="group one › flaky once" classname="a.spec.ts" time="0.009"> +<system-out> +<![CDATA[ +[[ATTACHMENT|pw/test-results/a-group-one-flaky-once-alpha/error-context.md]] +]]> +</system-out> +<failure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe"> +<![CDATA[ [alpha] › a.spec.ts:6:7 › group one › flaky once ───────────────────────────────────────────────── + + Error: expect(received).toBe(expected) // Object.is equality + + Expected: "second" + Received: "first" + + 4 | test('passes', () => { expect(1).toBe(1); }); + 5 | test('fails always', () => { expect(1).toBe(2); }); + > 6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } }); + | ^ + 7 | test.skip('skipped', () => {}); + 8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); }); + 9 | }); + at /repo/tests/a.spec.ts:6:152 + + Error Context: ../test-results/a-group-one-flaky-once-alpha/error-context.md +]]> +</failure> +</testcase> +<testcase name="group one › skipped" classname="a.spec.ts"> +<properties> +<property name="skip" value=""> +</property> +</properties> +<skipped> +</skipped> +</testcase> +<testcase name="group one › title with $(rm -rf) [x] (y)? *" classname="a.spec.ts" time="0.006"> +</testcase> +</testsuite> +<testsuite name="a.spec.ts" timestamp="2026-10-29T16:02:06.214Z" hostname="beta" tests="5" failures="2" skipped="1" time="0.027" errors="0"> +<testcase name="group one › passes" classname="a.spec.ts" time="0.005"> +</testcase> +<testcase name="group one › fails always" classname="a.spec.ts" time="0.002"> +<system-out> +<![CDATA[ +[[ATTACHMENT|pw/test-results/a-group-one-fails-always-beta/error-context.md]] +]]> +</system-out> +<failure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe"> +<![CDATA[ [beta] › a.spec.ts:5:7 › group one › fails always ──────────────────────────────────────────────── + + Error: expect(received).toBe(expected) // Object.is equality + + Expected: 2 + Received: 1 + + 3 | test.describe('group one', () => { + 4 | test('passes', () => { expect(1).toBe(1); }); + > 5 | test('fails always', () => { expect(1).toBe(2); }); + | ^ + 6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } }); + 7 | test.skip('skipped', () => {}); + 8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); }); + at /repo/tests/a.spec.ts:5:42 + + Error Context: ../test-results/a-group-one-fails-always-beta/error-context.md +]]> +</failure> +</testcase> +<testcase name="group one › flaky once" classname="a.spec.ts" time="0.013"> +<system-out> +<![CDATA[ +[[ATTACHMENT|pw/test-results/a-group-one-flaky-once-beta/error-context.md]] +]]> +</system-out> +<failure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe"> +<![CDATA[ [beta] › a.spec.ts:6:7 › group one › flaky once ────────────────────────────────────────────────── + + Error: expect(received).toBe(expected) // Object.is equality + + Expected: "second" + Received: "first" + + 4 | test('passes', () => { expect(1).toBe(1); }); + 5 | test('fails always', () => { expect(1).toBe(2); }); + > 6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } }); + | ^ + 7 | test.skip('skipped', () => {}); + 8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); }); + 9 | }); + at /repo/tests/a.spec.ts:6:152 + + Error Context: ../test-results/a-group-one-flaky-once-beta/error-context.md +]]> +</failure> +</testcase> +<testcase name="group one › skipped" classname="a.spec.ts"> +<properties> +<property name="skip" value=""> +</property> +</properties> +<skipped> +</skipped> +</testcase> +<testcase name="group one › title with $(rm -rf) [x] (y)? *" classname="a.spec.ts" time="0.007"> +</testcase> +</testsuite> +</testsuites> \ No newline at end of file diff --git a/test/fixtures/junit/playwright-r2-inc.xml b/test/fixtures/junit/playwright-r2-inc.xml new file mode 100644 index 000000000..6f043478c --- /dev/null +++ b/test/fixtures/junit/playwright-r2-inc.xml @@ -0,0 +1,244 @@ +<testsuites id="" name="" tests="10" failures="2" skipped="2" errors="0" time="4.805343"> +<testsuite name="a.spec.ts" timestamp="2026-10-29T16:02:16.450Z" hostname="alpha" tests="5" failures="1" skipped="1" time="0.043" errors="0"> +<testcase name="group one › passes" classname="a.spec.ts" time="0.007"> +</testcase> +<testcase name="group one › fails always" classname="a.spec.ts" time="0.005"> +<system-out> +<![CDATA[ +[[ATTACHMENT|pw/test-results/a-group-one-fails-always-alpha/error-context.md]] +]]> +</system-out> +<rerunFailure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe" time="0.008"> +<stackTrace> +<![CDATA[Error: expect(received).toBe(expected) // Object.is equality + +Expected: 2 +Received: 1 + at /repo/tests/a.spec.ts:5:42]]> +</stackTrace> +<system-out> +<![CDATA[ +[[ATTACHMENT|pw/test-results/a-group-one-fails-always-alpha-retry1/error-context.md]] +]]> +</system-out> +</rerunFailure> +<rerunFailure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe" time="0.009"> +<stackTrace> +<![CDATA[Error: expect(received).toBe(expected) // Object.is equality + +Expected: 2 +Received: 1 + at /repo/tests/a.spec.ts:5:42]]> +</stackTrace> +<system-out> +<![CDATA[ +[[ATTACHMENT|pw/test-results/a-group-one-fails-always-alpha-retry2/error-context.md]] +]]> +</system-out> +</rerunFailure> +<failure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe"> +<![CDATA[ [alpha] › a.spec.ts:5:7 › group one › fails always ─────────────────────────────────────────────── + + Error: expect(received).toBe(expected) // Object.is equality + + Expected: 2 + Received: 1 + + 3 | test.describe('group one', () => { + 4 | test('passes', () => { expect(1).toBe(1); }); + > 5 | test('fails always', () => { expect(1).toBe(2); }); + | ^ + 6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } }); + 7 | test.skip('skipped', () => {}); + 8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); }); + at /repo/tests/a.spec.ts:5:42 + + Error Context: ../test-results/a-group-one-fails-always-alpha/error-context.md + + Retry #1 ─────────────────────────────────────────────────────────────────────────────────────── + + Error: expect(received).toBe(expected) // Object.is equality + + Expected: 2 + Received: 1 + + 3 | test.describe('group one', () => { + 4 | test('passes', () => { expect(1).toBe(1); }); + > 5 | test('fails always', () => { expect(1).toBe(2); }); + | ^ + 6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } }); + 7 | test.skip('skipped', () => {}); + 8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); }); + at /repo/tests/a.spec.ts:5:42 + + Error Context: ../test-results/a-group-one-fails-always-alpha-retry1/error-context.md + + Retry #2 ─────────────────────────────────────────────────────────────────────────────────────── + + Error: expect(received).toBe(expected) // Object.is equality + + Expected: 2 + Received: 1 + + 3 | test.describe('group one', () => { + 4 | test('passes', () => { expect(1).toBe(1); }); + > 5 | test('fails always', () => { expect(1).toBe(2); }); + | ^ + 6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } }); + 7 | test.skip('skipped', () => {}); + 8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); }); + at /repo/tests/a.spec.ts:5:42 + + Error Context: ../test-results/a-group-one-fails-always-alpha-retry2/error-context.md +]]> +</failure> +</testcase> +<testcase name="group one › flaky once" classname="a.spec.ts" time="0.002"> +<flakyFailure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe" time="0.009"> +<stackTrace> +<![CDATA[Error: expect(received).toBe(expected) // Object.is equality + +Expected: "second" +Received: "first" + at /repo/tests/a.spec.ts:6:152]]> +</stackTrace> +<system-out> +<![CDATA[ +[[ATTACHMENT|pw/test-results/a-group-one-flaky-once-alpha/error-context.md]] +]]> +</system-out> +</flakyFailure> +</testcase> +<testcase name="group one › skipped" classname="a.spec.ts"> +<properties> +<property name="skip" value=""> +</property> +</properties> +<skipped> +</skipped> +</testcase> +<testcase name="group one › title with $(rm -rf) [x] (y)? *" classname="a.spec.ts" time="0.003"> +</testcase> +</testsuite> +<testsuite name="a.spec.ts" timestamp="2026-10-29T16:02:16.450Z" hostname="beta" tests="5" failures="1" skipped="1" time="0.033" errors="0"> +<testcase name="group one › passes" classname="a.spec.ts" time="0.004"> +</testcase> +<testcase name="group one › fails always" classname="a.spec.ts" time="0.002"> +<system-out> +<![CDATA[ +[[ATTACHMENT|pw/test-results/a-group-one-fails-always-beta/error-context.md]] +]]> +</system-out> +<rerunFailure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe" time="0.006"> +<stackTrace> +<![CDATA[Error: expect(received).toBe(expected) // Object.is equality + +Expected: 2 +Received: 1 + at /repo/tests/a.spec.ts:5:42]]> +</stackTrace> +<system-out> +<![CDATA[ +[[ATTACHMENT|pw/test-results/a-group-one-fails-always-beta-retry1/error-context.md]] +]]> +</system-out> +</rerunFailure> +<rerunFailure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe" time="0.006"> +<stackTrace> +<![CDATA[Error: expect(received).toBe(expected) // Object.is equality + +Expected: 2 +Received: 1 + at /repo/tests/a.spec.ts:5:42]]> +</stackTrace> +<system-out> +<![CDATA[ +[[ATTACHMENT|pw/test-results/a-group-one-fails-always-beta-retry2/error-context.md]] +]]> +</system-out> +</rerunFailure> +<failure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe"> +<![CDATA[ [beta] › a.spec.ts:5:7 › group one › fails always ──────────────────────────────────────────────── + + Error: expect(received).toBe(expected) // Object.is equality + + Expected: 2 + Received: 1 + + 3 | test.describe('group one', () => { + 4 | test('passes', () => { expect(1).toBe(1); }); + > 5 | test('fails always', () => { expect(1).toBe(2); }); + | ^ + 6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } }); + 7 | test.skip('skipped', () => {}); + 8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); }); + at /repo/tests/a.spec.ts:5:42 + + Error Context: ../test-results/a-group-one-fails-always-beta/error-context.md + + Retry #1 ─────────────────────────────────────────────────────────────────────────────────────── + + Error: expect(received).toBe(expected) // Object.is equality + + Expected: 2 + Received: 1 + + 3 | test.describe('group one', () => { + 4 | test('passes', () => { expect(1).toBe(1); }); + > 5 | test('fails always', () => { expect(1).toBe(2); }); + | ^ + 6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } }); + 7 | test.skip('skipped', () => {}); + 8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); }); + at /repo/tests/a.spec.ts:5:42 + + Error Context: ../test-results/a-group-one-fails-always-beta-retry1/error-context.md + + Retry #2 ─────────────────────────────────────────────────────────────────────────────────────── + + Error: expect(received).toBe(expected) // Object.is equality + + Expected: 2 + Received: 1 + + 3 | test.describe('group one', () => { + 4 | test('passes', () => { expect(1).toBe(1); }); + > 5 | test('fails always', () => { expect(1).toBe(2); }); + | ^ + 6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } }); + 7 | test.skip('skipped', () => {}); + 8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); }); + at /repo/tests/a.spec.ts:5:42 + + Error Context: ../test-results/a-group-one-fails-always-beta-retry2/error-context.md +]]> +</failure> +</testcase> +<testcase name="group one › flaky once" classname="a.spec.ts" time="0.002"> +<flakyFailure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe" time="0.01"> +<stackTrace> +<![CDATA[Error: expect(received).toBe(expected) // Object.is equality + +Expected: "second" +Received: "first" + at /repo/tests/a.spec.ts:6:152]]> +</stackTrace> +<system-out> +<![CDATA[ +[[ATTACHMENT|pw/test-results/a-group-one-flaky-once-beta/error-context.md]] +]]> +</system-out> +</flakyFailure> +</testcase> +<testcase name="group one › skipped" classname="a.spec.ts"> +<properties> +<property name="skip" value=""> +</property> +</properties> +<skipped> +</skipped> +</testcase> +<testcase name="group one › title with $(rm -rf) [x] (y)? *" classname="a.spec.ts" time="0.003"> +</testcase> +</testsuite> +</testsuites> \ No newline at end of file diff --git a/test/fixtures/junit/playwright-r2.xml b/test/fixtures/junit/playwright-r2.xml new file mode 100644 index 000000000..40877b668 --- /dev/null +++ b/test/fixtures/junit/playwright-r2.xml @@ -0,0 +1,178 @@ +<testsuites id="" name="" tests="10" failures="2" skipped="2" errors="0" time="4.985760999999999"> +<testsuite name="a.spec.ts" timestamp="2026-10-29T16:02:10.182Z" hostname="alpha" tests="5" failures="1" skipped="1" time="0.044" errors="0"> +<testcase name="group one › passes" classname="a.spec.ts" time="0.005"> +</testcase> +<testcase name="group one › fails always" classname="a.spec.ts" time="0.021"> +<system-out> +<![CDATA[ +[[ATTACHMENT|pw/test-results/a-group-one-fails-always-alpha/error-context.md]] + +[[ATTACHMENT|pw/test-results/a-group-one-fails-always-alpha-retry1/error-context.md]] + +[[ATTACHMENT|pw/test-results/a-group-one-fails-always-alpha-retry2/error-context.md]] +]]> +</system-out> +<failure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe"> +<![CDATA[ [alpha] › a.spec.ts:5:7 › group one › fails always ─────────────────────────────────────────────── + + Error: expect(received).toBe(expected) // Object.is equality + + Expected: 2 + Received: 1 + + 3 | test.describe('group one', () => { + 4 | test('passes', () => { expect(1).toBe(1); }); + > 5 | test('fails always', () => { expect(1).toBe(2); }); + | ^ + 6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } }); + 7 | test.skip('skipped', () => {}); + 8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); }); + at /repo/tests/a.spec.ts:5:42 + + Error Context: ../test-results/a-group-one-fails-always-alpha/error-context.md + + Retry #1 ─────────────────────────────────────────────────────────────────────────────────────── + + Error: expect(received).toBe(expected) // Object.is equality + + Expected: 2 + Received: 1 + + 3 | test.describe('group one', () => { + 4 | test('passes', () => { expect(1).toBe(1); }); + > 5 | test('fails always', () => { expect(1).toBe(2); }); + | ^ + 6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } }); + 7 | test.skip('skipped', () => {}); + 8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); }); + at /repo/tests/a.spec.ts:5:42 + + Error Context: ../test-results/a-group-one-fails-always-alpha-retry1/error-context.md + + Retry #2 ─────────────────────────────────────────────────────────────────────────────────────── + + Error: expect(received).toBe(expected) // Object.is equality + + Expected: 2 + Received: 1 + + 3 | test.describe('group one', () => { + 4 | test('passes', () => { expect(1).toBe(1); }); + > 5 | test('fails always', () => { expect(1).toBe(2); }); + | ^ + 6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } }); + 7 | test.skip('skipped', () => {}); + 8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); }); + at /repo/tests/a.spec.ts:5:42 + + Error Context: ../test-results/a-group-one-fails-always-alpha-retry2/error-context.md +]]> +</failure> +</testcase> +<testcase name="group one › flaky once" classname="a.spec.ts" time="0.014"> +<system-out> +<![CDATA[ +[[ATTACHMENT|pw/test-results/a-group-one-flaky-once-alpha/error-context.md]] +]]> +</system-out> +</testcase> +<testcase name="group one › skipped" classname="a.spec.ts"> +<properties> +<property name="skip" value=""> +</property> +</properties> +<skipped> +</skipped> +</testcase> +<testcase name="group one › title with $(rm -rf) [x] (y)? *" classname="a.spec.ts" time="0.004"> +</testcase> +</testsuite> +<testsuite name="a.spec.ts" timestamp="2026-10-29T16:02:10.182Z" hostname="beta" tests="5" failures="1" skipped="1" time="0.04" errors="0"> +<testcase name="group one › passes" classname="a.spec.ts" time="0.005"> +</testcase> +<testcase name="group one › fails always" classname="a.spec.ts" time="0.019"> +<system-out> +<![CDATA[ +[[ATTACHMENT|pw/test-results/a-group-one-fails-always-beta/error-context.md]] + +[[ATTACHMENT|pw/test-results/a-group-one-fails-always-beta-retry1/error-context.md]] + +[[ATTACHMENT|pw/test-results/a-group-one-fails-always-beta-retry2/error-context.md]] +]]> +</system-out> +<failure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe"> +<![CDATA[ [beta] › a.spec.ts:5:7 › group one › fails always ──────────────────────────────────────────────── + + Error: expect(received).toBe(expected) // Object.is equality + + Expected: 2 + Received: 1 + + 3 | test.describe('group one', () => { + 4 | test('passes', () => { expect(1).toBe(1); }); + > 5 | test('fails always', () => { expect(1).toBe(2); }); + | ^ + 6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } }); + 7 | test.skip('skipped', () => {}); + 8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); }); + at /repo/tests/a.spec.ts:5:42 + + Error Context: ../test-results/a-group-one-fails-always-beta/error-context.md + + Retry #1 ─────────────────────────────────────────────────────────────────────────────────────── + + Error: expect(received).toBe(expected) // Object.is equality + + Expected: 2 + Received: 1 + + 3 | test.describe('group one', () => { + 4 | test('passes', () => { expect(1).toBe(1); }); + > 5 | test('fails always', () => { expect(1).toBe(2); }); + | ^ + 6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } }); + 7 | test.skip('skipped', () => {}); + 8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); }); + at /repo/tests/a.spec.ts:5:42 + + Error Context: ../test-results/a-group-one-fails-always-beta-retry1/error-context.md + + Retry #2 ─────────────────────────────────────────────────────────────────────────────────────── + + Error: expect(received).toBe(expected) // Object.is equality + + Expected: 2 + Received: 1 + + 3 | test.describe('group one', () => { + 4 | test('passes', () => { expect(1).toBe(1); }); + > 5 | test('fails always', () => { expect(1).toBe(2); }); + | ^ + 6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } }); + 7 | test.skip('skipped', () => {}); + 8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); }); + at /repo/tests/a.spec.ts:5:42 + + Error Context: ../test-results/a-group-one-fails-always-beta-retry2/error-context.md +]]> +</failure> +</testcase> +<testcase name="group one › flaky once" classname="a.spec.ts" time="0.012"> +<system-out> +<![CDATA[ +[[ATTACHMENT|pw/test-results/a-group-one-flaky-once-beta/error-context.md]] +]]> +</system-out> +</testcase> +<testcase name="group one › skipped" classname="a.spec.ts"> +<properties> +<property name="skip" value=""> +</property> +</properties> +<skipped> +</skipped> +</testcase> +<testcase name="group one › title with $(rm -rf) [x] (y)? *" classname="a.spec.ts" time="0.004"> +</testcase> +</testsuite> +</testsuites> \ No newline at end of file diff --git a/test/flaky-run.test.ts b/test/flaky-run.test.ts new file mode 100644 index 000000000..c890b5883 --- /dev/null +++ b/test/flaky-run.test.ts @@ -0,0 +1,459 @@ +/** + * gstack-flaky run (GSTA-158): the JUnit reader against reports captured from + * Playwright 1.64 and bun 1.4 (test/fixtures/junit, paths rewritten to /repo), + * the rerun-once rules, and the bin end to end with a fake runner. + */ +import { afterAll, describe, expect, test } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { spawn, spawnSync } from 'child_process'; +import { JunitError, parseJunit, readJunit, testId } from '../lib/junit'; +import { appendLedger, incompleteRun, ledgerPath, renderResult, rerunSelection, runFlaky, safeExcerpt, stepSummary, unreportedFailures, workflowEscape } from '../lib/flaky-run'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const FIX = path.join(ROOT, 'test', 'fixtures', 'junit'); +const fixture = (name: string) => fs.readFileSync(path.join(FIX, name), 'utf-8'); +const TMP = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-flaky-')); +const tmp = () => fs.mkdtempSync(path.join(TMP, 't-')); +afterAll(() => fs.rmSync(TMP, { recursive: true, force: true })); + +describe('JUnit reader', () => { + test('bun: nested describe path, skip, failure excerpt', () => { + const r = parseJunit(fixture('bun.xml')); + expect(r.runner).toBe('bun'); + expect(r.tests.map(t => [t.id, t.attempts.join(',')])).toEqual([ + ['a.test.ts::::outer > inner > passes', 'pass'], + ['a.test.ts::::outer > inner > fails', 'fail'], + ['a.test.ts::::outer > skipped', 'skip'], + ['a.test.ts::::top level', 'pass'], + ]); + expect(r.tests[1]!.excerpt[0]).toContain('expect(received).toBe(expected)'); + }); + + test('playwright: one id per project, retry-aware attempts', () => { + const r = parseJunit(fixture('playwright-r2-inc.xml')); + expect(r.runner).toBe('playwright'); + expect(r.retriesHidden).toBe(false); + const byId = Object.fromEntries(r.tests.map(t => [t.id, t.attempts.join(',')])); + expect(byId['a.spec.ts::alpha::group one > fails always']).toBe('fail,fail,fail'); + expect(byId['a.spec.ts::beta::group one > fails always']).toBe('fail,fail,fail'); + expect(byId['a.spec.ts::alpha::group one > flaky once']).toBe('fail,pass'); + expect(byId['a.spec.ts::alpha::group one > skipped']).toBe('skip'); + expect(byId['a.spec.ts::alpha::group one > title with $(rm -rf) [x] (y)? *']).toBe('pass'); + }); + + test('playwright retries without retry data are detected, not read as one attempt', () => { + expect(parseJunit(fixture('playwright-r2.xml')).retriesHidden).toBe(true); + expect(parseJunit(fixture('playwright-r0.xml')).retriesHidden).toBe(false); + }); + + test('missing, empty, zero-testcase and malformed reports are errors, never green', () => { + const dir = tmp(); + const code = (fn: () => unknown) => { try { fn(); return 'ok'; } catch (e) { return (e as JunitError).code; } }; + expect(code(() => readJunit(path.join(dir, 'absent.xml')))).toBe('FLAKY_NO_RESULTS'); + fs.writeFileSync(path.join(dir, 'empty.xml'), ' \n'); + expect(code(() => readJunit(path.join(dir, 'empty.xml')))).toBe('FLAKY_NO_RESULTS'); + expect(code(() => parseJunit('<testsuites name="x"><testsuite name="s" tests="0"/></testsuites>'))).toBe('FLAKY_NO_RESULTS'); + expect(code(() => parseJunit('<testsuites><testsuite name="s"><testcase name="a"></testsuite></testsuites>'))).toBe('FLAKY_JUNIT_INVALID'); + }); + + test('generic runners (jest, pytest): file from the testcase, entities decoded', () => { + const r = parseJunit('<?xml version="1.0"?><testsuites><testsuite name="pytest"><testcase classname="tests.test_a" name="test_x[a&b]" file="tests/test_a.py"><error message="boom"/></testcase></testsuite></testsuites>'); + expect(r.runner).toBe('generic'); + expect(r.tests[0]!.id).toBe('tests/test_a.py::::test_x[a&b]'); + expect(r.tests[0]!.attempts).toEqual(['fail']); + }); +}); + +describe('rerun selection', () => { + const pw = parseJunit(fixture('playwright-r0.xml')).tests.find(t => t.path.at(-1)!.startsWith('title with') && t.project === 'beta')!; + test('playwright: file and title regex-escaped, project kept, each one argv element', () => { + expect(rerunSelection('playwright', pw)).toEqual([ + 'a\\.spec\\.ts', '--grep', '(?:^|\\s)group one title with \\$\\(rm -rf\\) \\[x\\] \\(y\\)\\? \\*(?:\\s|$)', '--project', 'beta', + ]); + }); + test('bun: anchored space-joined path', () => { + const t = parseJunit(fixture('bun.xml')).tests[1]!; + expect(rerunSelection('bun', t)).toEqual(['a.test.ts', '--test-name-pattern', '^outer inner fails$']); + }); +}); + +describe('complete-run proof for a targeted rerun', () => { + test('a complete run needs exit 1 and the runner summary without an early stop', () => { + expect(incompleteRun('bun', ' 4 pass\n 1 fail\nRan 5 tests across 2 files. [9ms]\n', 1)).toBeNull(); + expect(incompleteRun('bun', 'Bailed out after 1 failure\nRan 1 test across 1 file.\n', 1)).toContain('--bail'); + expect(incompleteRun('bun', ' 1 fail\n', 1)).toContain('no run summary'); + expect(incompleteRun('bun', 'Ran 5 tests across 2 files.\n', 2)).toContain('exited 2'); + expect(incompleteRun('playwright', ' 1 failed\n [alpha] x\n 4 passed (1s)\n', 1)).toBeNull(); + expect(incompleteRun('playwright', 'Testing stopped early after 1 maximum allowed failures.\n 1 failed\n 2 did not run\n', 1)).toContain('2 did not run'); + expect(incompleteRun('playwright', ' 1 failed\n 3 interrupted\n', 1)).toContain('3 interrupted'); + expect(incompleteRun('playwright', 'Timed out waiting 600s for the test suite to run\nTimed out waiting 600s for the entire test run\n 1 failed\n', 1)).toContain('stopped early'); + expect(incompleteRun('playwright', '', 1)).toContain('no summary'); + expect(incompleteRun('generic', 'Ran 5 tests', 1)).toContain('generic'); + expect(incompleteRun('bun', 'All files | % Funcs | % Lines |\nRan 5 tests across 2 files.\n', 1)).toContain('coverage'); + expect(incompleteRun('bun', 'Ran 5 tests across 2 files.\n', 1, ['bun', 'test', '--coverage'])).toContain('coverage'); + expect(incompleteRun('playwright', ' 1 failed\n 1 flaky\n 4 passed\n', 1, [], 1)).toContain('fail-on-flaky'); + }); +}); + +/** A fake runner: argv selection decides which canned report the attempt writes. */ +const BUN_SUMMARY = 'Ran 2 tests across 1 file. [3ms]\n'; + +function fakeRun(reports: (argv: string[], n: number) => string | null, exits: (argv: string[], n: number) => number, outputs: (argv: string[], n: number) => string = () => BUN_SUMMARY, signals: (n: number) => string | undefined = () => undefined) { + const calls: string[][] = []; + return { + calls, + spawn(junit: string) { + return async (argv: string[], _env: NodeJS.ProcessEnv, cwd: string) => { + calls.push(argv); + const xml = reports(argv, calls.length); + if (xml !== null) fs.writeFileSync(path.resolve(cwd, junit), xml); + const signal = signals(calls.length); + return { exit: exits(argv, calls.length), output: outputs(argv, calls.length), ...(signal ? { signal } : {}) }; + }; + }, + }; +} + +const suite = (cases: string) => `<testsuites name="bun test"><testsuite name="a.test.ts" file="a.test.ts">${cases}</testsuite></testsuites>`; +const pass = (name: string) => `<testcase name="${name}" classname="" file="a.test.ts"/>`; +const fail = (name: string, msg = 'boom') => `<testcase name="${name}" classname="" file="a.test.ts"><failure message="${msg}"/></testcase>`; + +describe('runFlaky', () => { + const base = (spawn: ReturnType<typeof fakeRun>, dir: string, extra: Partial<Parameters<typeof runFlaky>[0]> = {}) => + runFlaky({ command: ['bun', 'test'], junit: 'r.xml', maxReruns: 3, cwd: dir, spawn: spawn.spawn('r.xml'), env: {}, now: () => new Date('2026-10-29T00:00:00Z'), ...extra }); + + test('a green run with no failures is green and reruns nothing', async () => { + const dir = tmp(); + const f = fakeRun(() => suite(pass('a') + pass('b')), () => 0); + expect(await base(f, dir)).toMatchObject({ exitCode: 0, reruns: 0, ledger: [] }); + expect(f.calls).toHaveLength(1); + }); + + test('a coverage failure beside a flaky test is not cleared by a narrowed rerun', async () => { + const dir = tmp(); + const cov = () => '----|---------|---------|\nAll files | % Funcs | % Lines |\nRan 2 tests across 1 file.\n'; + const f = fakeRun((_a, n) => n === 1 ? suite(pass('a') + fail('b')) : suite(pass('a') + pass('b')), () => 1, cov); + const r = await base(f, dir, { rerunScope: 'test' }); + expect(f.calls[1]).toEqual(['bun', 'test']); + expect(r).toMatchObject({ exitCode: 1, wholeRerun: 'coverage is measured, and a narrowed rerun cannot reproduce a coverage failure' }); + }); + + test('by default the rerun is the whole command and counts only on its full green verdict', async () => { + const dir = tmp(); + const green = fakeRun((_a, n) => n === 1 ? suite(pass('a') + fail('b')) : suite(pass('a') + pass('b')), (_a, n) => n === 1 ? 1 : 0); + const r = await base(green, dir); + expect(green.calls).toEqual([['bun', 'test'], ['bun', 'test']]); + expect(r).toMatchObject({ exitCode: 0, wholeRerun: 'the default; --rerun-scope test reruns the failed test alone' }); + expect(r.ledger[0]).toMatchObject({ kind: 'flaky-pass', rerun_scope: 'command' }); + const otherFails = fakeRun((_a, n) => n === 1 ? suite(pass('a') + fail('b')) : suite(fail('a') + pass('b')), (_a, n) => n === 1 ? 1 : 0); + expect((await base(otherFails, dir)).outcomes[0]).toMatchObject({ status: 'rerun-fail', reason: 'rerun: 1 test(s) failed in the whole-command rerun' }); + }); + + test('a failed test that passes on rerun is a flaky pass and goes on the ledger', async () => { + const dir = tmp(); + const f = fakeRun((argv, n) => n === 1 ? suite(pass('a') + fail('b')) : suite(pass('b')), (_a, n) => n === 1 ? 1 : 0); + const r = await base(f, dir, { rerunScope: 'test' }); + expect(r.exitCode).toBe(0); + expect(f.calls[1]).toEqual(['bun', 'test', 'a.test.ts', '--test-name-pattern', '^b$']); + expect(r.outcomes.map(o => [o.status, o.attempts.join(',')])).toEqual([['flaky-pass', 'fail,pass']]); + expect(r.ledger).toHaveLength(1); + expect(r.ledger[0]).toMatchObject({ kind: 'flaky-pass', id: 'a.test.ts::::b', rerun_by: 'gstack-flaky', attempts: ['fail', 'pass'], excerpt: ['boom'] }); + expect(renderResult(r, '/l.jsonl').at(-1)).toBe('FLAKY_RESULT: tests=2 failed=0 flaky=1 reruns=1 command_exit=1 verdict=pass ledger=/l.jsonl'); + expect(stepSummary(r)).toContain('a.test.ts::::b'); + }); + + test('a test that fails its rerun is red and is ledgered as rerun-fail', async () => { + const dir = tmp(); + const f = fakeRun(() => suite(fail('b')), () => 1); + const r = await base(f, dir); + expect(r.exitCode).toBe(1); + expect(r.outcomes[0]).toMatchObject({ status: 'rerun-fail', attempts: ['fail', 'fail'] }); + expect(r.ledger[0]!.kind).toBe('rerun-fail'); + }); + + test('more failures than --max-reruns: a real red, nothing rerun', async () => { + const dir = tmp(); + const f = fakeRun(() => suite(fail('a') + fail('b') + fail('c') + fail('d')), () => 1); + const r = await base(f, dir); + expect(r.exitCode).toBe(1); + expect(f.calls).toHaveLength(1); + expect(r.reruns).toBe(0); + expect(r.outcomes.every(o => o.status === 'fail' && /more than --max-reruns 3/.test(o.reason!))).toBe(true); + expect(r.ledger).toEqual([]); + }); + + test('runner retries stand: no second rerun, and a runner flaky pass is ledgered', async () => { + const dir = tmp(); + const f = fakeRun(() => fixture('playwright-r2-inc.xml'), () => 1); + const r = await base(f, dir); + expect(f.calls).toHaveLength(1); + expect(r.exitCode).toBe(1); + expect(r.ledger.filter(l => l.kind === 'flaky-pass').map(l => [l.id, l.rerun_by])).toEqual([ + ['a.spec.ts::alpha::group one > flaky once', 'runner'], ['a.spec.ts::beta::group one > flaky once', 'runner'], + ]); + expect(r.outcomes.filter(o => o.status === 'fail').every(o => /already retried \(3 attempts\)/.test(o.reason!))).toBe(true); + }); + + test('hidden Playwright retries refuse with a code instead of a guess', async () => { + const dir = tmp(); + const r = await base(fakeRun(() => fixture('playwright-r2.xml'), () => 1), dir); + expect([r.exitCode, r.code]).toEqual([3, 'FLAKY_RETRIES_HIDDEN']); + }); + + test('no report, a stale report, and a crash with no failed test are never green', async () => { + const dir = tmp(); + expect(await base(fakeRun(() => null, () => 0), dir)).toMatchObject({ exitCode: 3, code: 'FLAKY_NO_RESULTS' }); + fs.writeFileSync(path.join(dir, 'r.xml'), suite(pass('stale'))); + expect(await base(fakeRun(() => null, () => 0), dir)).toMatchObject({ exitCode: 3, code: 'FLAKY_NO_RESULTS' }); + expect(await base(fakeRun(() => suite(pass('a')), () => 2), dir)).toMatchObject({ exitCode: 3, code: 'FLAKY_COMMAND_FAILED', commandExit: 2 }); + }); + + test('a rerun that does not report the targeted test fails it; other tests in the rerun are ignored', async () => { + const dir = tmp(); + const f = fakeRun((_a, n) => n === 1 ? suite(fail('b')) : suite(pass('other')), (_a, n) => n === 1 ? 1 : 0); + const r = await base(f, dir); + expect(r.exitCode).toBe(1); + expect(r.outcomes[0]).toMatchObject({ status: 'rerun-fail', reason: 'rerun: the report does not contain this test' }); + }); + + test('a rerun whose command exits non-zero is not a flaky pass, even with a passing report', async () => { + const dir = tmp(); + const f = fakeRun((_a, n) => n === 1 ? suite(fail('b')) : suite(pass('b')), () => 1); + expect((await base(f, dir)).outcomes[0]).toMatchObject({ status: 'rerun-fail', reason: 'rerun: the command exited 1' }); + }); + + test('a test reported twice with one failure is red and not rerun', async () => { + const dir = tmp(); + const r = parseJunit(suite(pass('b') + fail('b'))); + expect(r.tests[0]).toMatchObject({ executions: 2, attempts: ['pass', 'fail'] }); + const f = fakeRun(() => suite(fail('b') + pass('b') + fail('c')), () => 1); + const out = await base(f, dir, { rerunScope: 'test' }); + expect(out.outcomes.find(o => o.test.id.endsWith('::b'))).toMatchObject({ status: 'fail', reason: 'it ran 2 times in this report and one failed; not rerun' }); + expect(f.calls).toHaveLength(2); + }); + + test('a runner-caught flake is ledgered even when a duplicate execution of the test was skipped', async () => { + const dir = tmp(); + const caught = '<testcase name="b" classname="" file="a.test.ts"><flakyFailure message="x"/></testcase>'; + const r = await base(fakeRun(() => suite(caught + '<testcase name="b" classname="" file="a.test.ts"><skipped/></testcase>'), () => 0), dir); + expect(r.exitCode).toBe(0); + expect(r.ledger.map(l => [l.kind, l.rerun_by])).toEqual([['flaky-pass', 'runner']]); + }); + + test('the Playwright environment flag reaches every attempt', async () => { + const dir = tmp(); + const seen: string[] = []; + await runFlaky({ command: ['x'], junit: 'r.xml', maxReruns: 3, cwd: dir, env: {}, spawn: async (_a, env, cwd) => { + seen.push(env.PLAYWRIGHT_JUNIT_INCLUDE_RETRIES ?? ''); + fs.writeFileSync(path.join(cwd, 'r.xml'), suite(seen.length === 1 ? fail('b') : pass('b'))); + return { exit: seen.length === 1 ? 1 : 0, output: '' }; + } }); + expect(seen).toEqual(['1', '1']); + }); + + test('bun load errors the JUnit omits keep the run red and nothing is rerun', async () => { + const dir = tmp(); + const f = fakeRun(() => suite(fail('b')), () => 1, () => '\n 0 pass\n 3 fail\n 2 errors\nRan 3 tests across 3 files.\n'); + const r = await base(f, dir); + expect(r).toMatchObject({ exitCode: 1, code: 'FLAKY_UNREPORTED_FAILURES', reruns: 0 }); + expect(f.calls).toHaveLength(1); + }); + + test('runner summaries: Playwright errors outside tests and count mismatches are unreported failures', () => { + expect(unreportedFailures('playwright', ' 1 error was not a part of any test, see above for details\n 1 failed\n', 1)).toContain('outside any test'); + expect(unreportedFailures('playwright', ' 2 failed\n 10 passed\n', 1)).toContain('counted 2 failed'); + expect(unreportedFailures('playwright', ' 1 failed\n [chromium] x\n 10 passed\n', 1)).toBeNull(); + expect(unreportedFailures('bun', ' 4 pass\n 1 fail\n', 1)).toBeNull(); + expect(unreportedFailures('generic', '5 failed', 0)).toBeNull(); + }); + + test('a run that stopped early reruns the whole command, so tests it never reached must pass too', async () => { + const dir = tmp(); + const bail = () => 'Bailed out after 1 failure\nRan 1 test across 1 file.\n'; + const green = fakeRun((_a, n) => n === 1 ? suite(fail('b')) : suite(pass('b') + pass('never-reached')), (_a, n) => n === 1 ? 1 : 0, bail); + const r = await base(green, dir, { rerunScope: 'test' }); + expect(green.calls).toEqual([['bun', 'test'], ['bun', 'test']]); + expect(r).toMatchObject({ exitCode: 0, wholeRerun: 'bun stopped early (--bail)' }); + expect(r.ledger[0]).toMatchObject({ kind: 'flaky-pass', rerun_scope: 'command' }); + expect(renderResult(r, '/l')).toContain('FLAKY_RERUN: scope=command (bun stopped early (--bail))'); + const red = fakeRun((_a, n) => n === 1 ? suite(fail('b')) : suite(pass('b') + fail('never-reached')), () => 1, bail); + expect((await base(red, dir, { rerunScope: 'test' })).outcomes[0]).toMatchObject({ status: 'rerun-fail', reason: 'rerun: the command exited 1' }); + }); + + test('generic runners rerun the whole command even when a targeted rerun is asked for', async () => { + const dir = tmp(); + const f = fakeRun((_a, n) => n === 1 ? suite(fail('b')) : suite(pass('b')), (_a, n) => n === 1 ? 1 : 0); + const r = await base(f, dir, { runner: 'generic', rerunScope: 'test' }); + expect(f.calls[1]).toEqual(['bun', 'test']); + expect(r).toMatchObject({ exitCode: 0, wholeRerun: 'a generic runner cannot prove every test ran' }); + }); + + test('a whole-command rerun clears nothing when any test fails in it, a first-run test is missing, it is killed or it retried', async () => { + const dir = tmp(); + const generic = (second: string, exit = 0, signal?: string) => base(fakeRun((_a, n) => n === 1 ? suite(fail('a') + pass('b')) : second, () => exit && 1, undefined, n => n === 2 ? signal : undefined), dir, { runner: 'generic' }); + expect((await generic(suite(pass('a') + fail('b')))).outcomes[0]).toMatchObject({ status: 'rerun-fail', reason: 'rerun: 1 test(s) failed in the whole-command rerun' }); + expect((await generic(suite(pass('a')))).outcomes[0]).toMatchObject({ status: 'rerun-fail', reason: 'rerun: 1 test(s) of the first run are missing from the whole-command rerun' }); + expect((await generic(suite(pass('a') + pass('b')), 0, 'SIGTERM')).outcomes[0]).toMatchObject({ status: 'rerun-fail', reason: 'rerun: killed by SIGTERM' }); + const retried = await generic(suite('<testcase name="a" classname="" file="a.test.ts"><flakyFailure message="x"/></testcase>' + pass('b'))); + expect(retried.outcomes.find(o => o.status === 'rerun-fail')).toMatchObject({ attempts: ['fail', 'fail', 'pass'] }); + expect(retried.ledger.find(l => l.kind === 'rerun-fail')).toMatchObject({ rerun_scope: 'command' }); + expect(retried.exitCode).toBe(1); + }); + + test('an incomplete run that is already red is not rerun as a whole', async () => { + const dir = tmp(); + const f = fakeRun(() => suite(fail('b') + pass('b') + fail('c')), () => 1, () => ''); + const r = await base(f, dir); + expect(f.calls).toHaveLength(1); + expect(r.outcomes.find(o => o.test.id.endsWith('::c'))!.reason).toContain('already red'); + }); + + test('a rerun the runner retried again does not clear the test', async () => { + const dir = tmp(); + const f = fakeRun((_a, n) => n === 1 ? suite(fail('b')) : suite('<testcase name="b" classname="" file="a.test.ts"><flakyFailure message="x"/></testcase>'), (_a, n) => n === 1 ? 1 : 0); + expect((await base(f, dir)).outcomes[0]).toMatchObject({ status: 'rerun-fail', attempts: ['fail', 'fail', 'pass'] }); + }); + + test('a killed runner is never green and stops further reruns', async () => { + const dir = tmp(); + expect(await base(fakeRun(() => suite(fail('b')), () => 128, undefined, () => 'SIGTERM'), dir)).toMatchObject({ exitCode: 3, code: 'FLAKY_COMMAND_KILLED' }); + const f = fakeRun((_a, n) => n === 1 ? suite(fail('b') + fail('c')) : suite(pass('b') + pass('c')), (_a, n) => n === 1 ? 1 : 128, undefined, n => n === 2 ? 'SIGTERM' : undefined); + const r = await base(f, dir, { rerunScope: 'test' }); + expect(f.calls).toHaveLength(2); + expect(r.outcomes.map(o => o.reason)).toEqual(['rerun: killed by SIGTERM', 'not rerun: an earlier rerun was killed']); + expect(r.exitCode).toBe(1); + }); + + test('the first report is restored at --junit and each rerun report is kept beside it', async () => { + const dir = tmp(); + const firstXml = suite(pass('a') + fail('b')); + await base(fakeRun((_a, n) => n === 1 ? firstXml : suite(pass('b')), (_a, n) => n === 1 ? 1 : 0), dir); + expect(fs.readFileSync(path.join(dir, 'r.xml'), 'utf-8')).toBe(firstXml); + expect(fs.readFileSync(path.join(dir, 'r.rerun-1.xml'), 'utf-8')).toBe(suite(pass('b'))); + }); + + test('credentials in a test title never reach the ledger, the result lines or the step summary', async () => { + const dir = tmp(); + const key = ['sk', 'ant', 'api03', 'B'.repeat(93) + 'AA'].join('-'); + const r = await base(fakeRun((_a, n) => n === 1 ? suite(fail(`login with ${key}`)) : suite(pass(`login with ${key}`)), (_a, n) => n === 1 ? 1 : 0), dir); + expect(r.exitCode).toBe(0); + expect(JSON.stringify(r.ledger)).not.toContain(key); + expect(renderResult(r, '/l').join('\n')).not.toContain(key); + expect(stepSummary(r)).not.toContain(key); + }); + + test('workflow commands escape a title that would end or start a command', () => { + expect(workflowEscape('a%b\r\n::error::x')).toBe('a%25b%0D%0A::error::x'); + }); + + test('ledger: a write error is a warning; the path comes from --ledger, then GSTACK_FLAKE_LEDGER_JUNIT, then the state root', () => { + const dir = tmp(); + fs.writeFileSync(path.join(dir, 'file'), ''); + const warns: string[] = []; + appendLedger([{ ts: 't', runner: 'junit', kind: 'flaky-pass', id: 'x', file: 'x', rerun_by: 'gstack-flaky', attempts: [], excerpt: [] }], path.join(dir, 'file', 'l.jsonl'), w => warns.push(w)); + expect(warns[0]).toContain('FLAKY_LEDGER_WRITE_FAILED'); + expect(ledgerPath('/flag', { GSTACK_FLAKE_LEDGER_JUNIT: '/env' })).toBe('/flag'); + expect(ledgerPath(undefined, { GSTACK_FLAKE_LEDGER_JUNIT: '/env', GSTACK_FLAKE_LEDGER: '/free' })).toBe('/env'); + expect(ledgerPath(undefined, { GSTACK_STATE_ROOT: dir, GSTACK_FLAKE_LEDGER: '/free' })).toBe(path.join(dir, 'flake-ledger-junit.jsonl')); + }); + + test('credentials in failure text never reach the ledger', () => { + const key = ['sk', 'ant', 'api03', 'A'.repeat(93) + 'AA'].join('-'); + const out = safeExcerpt([`Error: request failed with key ${key}`]); + expect(out.join('\n')).not.toContain(key); + }); +}); + +describe('bin/gstack-flaky', () => { + test('run: a fake bun runner flakes once, the bin reruns it and exits 0 with a ledger line', () => { + const dir = tmp(); + const runner = path.join(dir, 'runner.sh'); + fs.writeFileSync(runner, [ + '#!/usr/bin/env bash', + 'n=$(cat count 2>/dev/null || echo 0); echo $((n+1)) > count', + `if [ "$n" = 0 ]; then printf '%s' '${suite(pass('a') + fail('b'))}' > r.xml; echo 'Ran 2 tests across 1 file.' >&2; exit 1; fi`, + '[ "$1" = a.test.ts ] && [ "$2" = --test-name-pattern ] && [ "$3" = "^b$" ] || exit 9', + `printf '%s' '${suite(pass('b'))}' > r.xml`, + ].join('\n'), { mode: 0o755 }); + const ledger = path.join(dir, 'ledger.jsonl'); + const r = spawnSync(path.join(ROOT, 'bin', 'gstack-flaky'), ['run', '--junit', 'r.xml', '--runner', 'bun', '--rerun-scope', 'test', '--ledger', ledger, '--', runner], { cwd: dir, encoding: 'utf-8', timeout: 30_000, env: { ...process.env, GITHUB_ACTIONS: '', GITHUB_STEP_SUMMARY: '' } }); + expect(r.stdout).toContain('FLAKY: flaky-pass a.test.ts::::b attempts=fail,pass'); + expect(r.stdout).toContain(`verdict=pass ledger=${ledger}`); + expect(r.status).toBe(0); + expect(JSON.parse(fs.readFileSync(ledger, 'utf-8').trim()).kind).toBe('flaky-pass'); + }); + + test('usage, a command that cannot start, and missing reports exit with their codes', () => { + const dir = tmp(); + const bin = path.join(ROOT, 'bin', 'gstack-flaky'); + expect(spawnSync(bin, ['run', '--', 'true'], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }).status).toBe(2); + fs.writeFileSync(path.join(dir, 'keep.ts'), 'x'); + expect(spawnSync(bin, ['run', '--junit', 'keep.ts', '--', 'true'], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }).status).toBe(2); + expect(fs.existsSync(path.join(dir, 'keep.ts'))).toBe(true); + fs.writeFileSync(path.join(dir, 'pom.xml'), '<project><modelVersion>4.0.0</modelVersion></project>'); + expect(spawnSync(bin, ['run', '--junit', 'pom.xml', '--', 'true'], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }).status).toBe(2); + expect(fs.existsSync(path.join(dir, 'pom.xml'))).toBe(true); + expect(spawnSync(bin, ['run', '--junit', 'r.xml', '--rerun-scope', 'file', '--', 'true'], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }).status).toBe(2); + fs.writeFileSync(path.join(dir, 'locked.xml'), '<testsuites/>', { mode: 0o000 }); + if (process.getuid?.() !== 0) { + const locked = spawnSync(bin, ['run', '--junit', 'locked.xml', '--', 'true'], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }); + expect([locked.status, locked.stderr.includes('cannot be read')]).toEqual([2, true]); + } + const missing = spawnSync(bin, ['run', '--junit', 'r.xml', '--', path.join(dir, 'no-such-runner')], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }); + expect(missing.status).toBe(3); + expect(missing.stderr).toContain('(FLAKY_COMMAND_NOT_STARTED)'); + const r = spawnSync(bin, ['run', '--junit', 'r.xml', '--', 'true'], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }); + expect(r.status).toBe(3); + expect(r.stderr).toContain('(FLAKY_NO_RESULTS)'); + expect(r.stderr).toContain('fix: '); + }); + + test('a runner that dies by a signal exits 3 FLAKY_COMMAND_KILLED and is not rerun', () => { + const dir = tmp(); + const runner = path.join(dir, 'runner.sh'); + fs.writeFileSync(runner, ['#!/usr/bin/env bash', 'echo x >> count', `printf '%s' '${suite(fail('b'))}' > r.xml`, 'kill -TERM $$'].join('\n'), { mode: 0o755 }); + const r = spawnSync(path.join(ROOT, 'bin', 'gstack-flaky'), ['run', '--junit', 'r.xml', '--runner', 'bun', '--ledger', path.join(dir, 'l.jsonl'), '--', runner], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }); + expect(r.status).toBe(3); + expect(r.stderr).toContain('(FLAKY_COMMAND_KILLED)'); + expect(fs.readFileSync(path.join(dir, 'count'), 'utf-8').trim().split('\n')).toHaveLength(1); + }); + + test('cancelling gstack-flaky is latched even when the runner traps the signal and exits by code', async () => { + const dir = tmp(); + const runner = path.join(dir, 'runner.sh'); + fs.writeFileSync(runner, [ + '#!/usr/bin/env bash', + 'echo x >> count', + 'trap "cp report.xml r.xml; exit 143" TERM', + 'touch ready', + 'for i in $(seq 1 300); do sleep 0.1; done', + ].join('\n'), { mode: 0o755 }); + fs.writeFileSync(path.join(dir, 'report.xml'), suite(fail('b'))); + const child = spawn(path.join(ROOT, 'bin', 'gstack-flaky'), ['run', '--junit', 'r.xml', '--runner', 'generic', '--ledger', path.join(dir, 'l.jsonl'), '--', runner], { cwd: dir, stdio: ['ignore', 'ignore', 'pipe'] }); + let stderr = ''; + child.stderr.on('data', (c: Buffer) => { stderr += c.toString(); }); + const done = new Promise<number | null>(resolve => child.on('close', code => resolve(code))); + const deadline = Date.now() + 20_000; + while (!fs.existsSync(path.join(dir, 'ready')) && Date.now() < deadline) await Bun.sleep(50); + child.kill('SIGTERM'); + expect(await done).toBe(3); + expect(stderr).toContain('(FLAKY_COMMAND_KILLED)'); + expect(fs.readFileSync(path.join(dir, 'count'), 'utf-8').trim().split('\n')).toHaveLength(1); + expect(fs.existsSync(path.join(dir, 'l.jsonl'))).toBe(false); + }, 30_000); + + test('every FLAKY_ code the bin and the library emit is in the docs table', () => { + const doc = fs.readFileSync(path.join(ROOT, 'docs', 'flaky.md'), 'utf-8'); + const src = fs.readFileSync(path.join(ROOT, 'bin', 'gstack-flaky'), 'utf-8') + fs.readFileSync(path.join(ROOT, 'lib', 'flaky-run.ts'), 'utf-8') + fs.readFileSync(path.join(ROOT, 'lib', 'junit.ts'), 'utf-8'); + const codes = [...new Set(src.match(/FLAKY_[A-Z_]+(?=['":])/g))].filter(c => c !== 'FLAKY_RESULT' && c !== 'FLAKY_RERUN'); + expect(codes.length).toBeGreaterThan(5); + expect(codes.filter(c => !doc.includes(`| \`${c}\` |`))).toEqual([]); + }); + + test('ids prints one id per test', () => { + const r = spawnSync(path.join(ROOT, 'bin', 'gstack-flaky'), ['ids', '--junit', path.join(FIX, 'bun.xml')], { encoding: 'utf-8', timeout: 30_000 }); + expect(r.stdout.trim().split('\n')).toEqual([testId('a.test.ts', '', ['outer', 'inner', 'passes']), 'a.test.ts::::outer > inner > fails', 'a.test.ts::::outer > skipped', 'a.test.ts::::top level']); + }); +}); diff --git a/test/helpers/free-fixtures-data.ts b/test/helpers/free-fixtures-data.ts index 202d54c87..dfe62b544 100644 --- a/test/helpers/free-fixtures-data.ts +++ b/test/helpers/free-fixtures-data.ts @@ -151,6 +151,7 @@ export const FREE_FIXTURES: Readonly<Record<string, readonly string[]>> = { 'test/fixtures/investigate-owned-question-39195396438.json': ['test/workflow-boundaries-fixture.test.ts'], 'test/fixtures/judge-calibration/**': ['scripts/judge-calibration.ts', 'test/judge-calibration.test.ts'], 'test/fixtures/judge-panel-replay-2026-10-05.json': ['test/judge-panel-median.test.ts'], + 'test/fixtures/junit/**': ['test/flaky-run.test.ts'], 'test/fixtures/launch-env-echo/**': ['test/launch-env-echo-ratchet.test.ts'], 'test/fixtures/merge-train-3345/**': ['test/changelog-merge-damage.test.ts', 'test/compare-pr-version.test.ts'], 'test/fixtures/merge-train-3453/**': ['test/changelog-merge-damage.test.ts'],