From 0246abd2bae463fc968a1c955df5007d4e458c5e Mon Sep 17 00:00:00 2001 From: garrytan Date: Thu, 29 Oct 2026 18:19:22 +0000 Subject: [PATCH] feat(flaky): quarantine registry for repeat flakes (GSTA-158 P2) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 8 + bin/gstack-flaky | 44 +- captain/templates/automation-flaky-pass.md | 9 +- docs/captain.md | 4 +- docs/flaky.md | 125 +++++- docs/standalone-binaries.md | 2 +- lib/flaky-run.ts | 130 ++++-- lib/junit.ts | 8 + lib/quarantine.ts | 444 +++++++++++++++++++++ test/flaky-run.test.ts | 2 +- test/quarantine.test.ts | 427 ++++++++++++++++++++ 11 files changed, 1164 insertions(+), 39 deletions(-) create mode 100644 lib/quarantine.ts create mode 100644 test/quarantine.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 83ca83714..ce5322ec2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,14 +10,22 @@ - **`gstack-flaky ids --junit `** prints the test ids a report holds (`:::: title>`). - Exit codes: 0 pass (flaky passes included), 1 a test failed, 2 usage, 3 no usable report, a crash or a killed runner, each with a `FLAKY_` and a `fix:` line. A missing, stale, empty or malformed report never passes, and an existing `--junit` file that is not a JUnit report is refused rather than deleted. +### Added +- **Quarantine registry, `gstack-flaky quarantine add|renew|release|check|report`** (`.gstack/quarantine.json`, GSTA-158 P2): a test that keeps flaking while someone fixes it gets a larger attempt budget instead of a skip. When every test an attempt failed is quarantined, the whole command may run again, up to 3 attempts; the run is green only when an attempt reaches the runner's own full green verdict, so a coverage threshold or other exit outside the report stays red. A quarantined test that fails all 3 is a `hard-break`, a quarantined id missing from a run that included its file is red, and quarantined tests are listed (`FLAKY_RESULT ... quarantined=`) and ledgered (`quarantined-pass`, `quarantined-fail`, `hard-break`) on every run, never counted as passes. +- **Entries are owned and expire**: owner, ledger link, class (`detector`, `harness`, `timing`, `model-latency`; `product` and `infra` refused), evidence, 7 days by default and 14 at most, one renewal of up to 7 days with new evidence. An expired entry stops applying with no edit and prints the renew and release commands. Live entries are capped at 10% of the suite (at least 1, and 5 under 50 tests). +- **`run --base `** reads the registry from the base branch, so a change cannot quarantine the test it breaks; **`check --base`** refuses an entry added in the change that edits its test's file, an entry removed without a release record, a removed release record and rewritten history. **`report --ledger `** lists entries soonest-expiring first and marks those with 10 consecutive clean passes ready to release. gstack's own `CASE_QUARANTINE` / `EVAL_POLICY` is unchanged and shares no schema with it. + ### Docs - `docs/flaky.md`: the GitHub Actions setup, runners, rerun scope and its preconditions, rules, output and exit codes. `docs/TESTING_INTERNALS.md#flake-ledger` points other repositories to it. +- `docs/flaky.md#quarantine`: quarantine commands, rules, expiry, cap, classes, the `CASE_QUARANTINE` separation and the daily-pass `gh run download` + `report` loop; `captain/templates/automation-flaky-pass.md` and `docs/captain.md` run `report` and fix expiring entries first. ### Tests - `test/flaky-run.test.ts` with captured Playwright 1.64 and bun 1.4 JUnit reports under `test/fixtures/junit/`: flaky pass, real red, rerun scopes, coverage-threshold and early-stop runs, unreported failures, cancelled runs, report deletion and ledger redaction. +- `test/quarantine.test.ts`: add refusals (cap, more than 14 days, no evidence, product or infra class, unknown id), one renewal, release, `check --base` against git fixtures, expiry, a hard break across 3 attempts, the false-green regression (exit 1 outside the report while only a quarantined test fails stays red), missing ids with their file present or absent, and the registry read from the base branch. ### Gate integrity - `test/helpers/free-fixtures-data.ts`: registers `test/fixtures/junit/**` to `test/flaky-run.test.ts`. No assertion, threshold or skip changed. +- `test/flaky-run.test.ts`: the exact `FLAKY_RESULT` line it expects gains the new `quarantined=0` field. No other assertion, threshold or skip changed. ## [1.91.320.0] - 2026-10-29 diff --git a/bin/gstack-flaky b/bin/gstack-flaky index 40239d156..e0eebf142 100755 --- a/bin/gstack-flaky +++ b/bin/gstack-flaky @@ -9,11 +9,13 @@ import * as fs from 'fs'; import { DEFAULT_MAX_RERUNS, appendLedger, ledgerPath, renderResult, runFlaky, safeText, stepSummary, workflowEscape } from '../lib/flaky-run'; import { JunitError, readJunit, type JunitRunner } from '../lib/junit'; +import { liveQuarantine, QuarantineError, quarantineMain, readRegistry } from '../lib/quarantine'; const USAGE = `Usage: gstack-flaky run --junit [--runner playwright|bun|generic] [--max-reruns ] - [--rerun-scope command|test] [--ledger ] -- + [--rerun-scope command|test] [--ledger ] [--base ] -- gstack-flaky ids --junit [--runner playwright|bun|generic] + gstack-flaky quarantine add|renew|release|check|report ... (gstack-flaky quarantine --help) gstack-flaky --help run: runs the command (an argv, no shell), reads the JUnit report it writes at @@ -31,11 +33,18 @@ holds the first run's report afterwards; reruns' reports sit beside it as .rerun-.xml. The command runs with PLAYWRIGHT_JUNIT_INCLUDE_RETRIES=1 so Playwright's own retries are reported. +Quarantine: tests listed in .gstack/quarantine.json (read from --base when +given, so PR CI passes its target branch) still run and never count as passes. +When every test an attempt failed is a live quarantined test, the whole command +may run again, up to 3 attempts; the run is green only on an attempt with the +runner's full green verdict, and a quarantined test that fails all 3 is a hard +break (red). A quarantined id missing from a run that included its file is red. + ids: prints the id of every test in a report (::::). Output: one FLAKY: line per test that failed at least once, FLAKY_RERUN: scope=command (<why>) when the rerun repeated the whole command, then -FLAKY_RESULT: tests=<n> failed=<n> flaky=<n> reruns=<n> command_exit=<n> verdict=pass|fail ledger=<file|none> +FLAKY_RESULT: tests=<n> failed=<n> flaky=<n> quarantined=<n> reruns=<n> command_exit=<n> verdict=pass|fail ledger=<file|none> Exit: 0 pass (flaky passes included), 1 a test failed, 2 usage, 3 no usable report, a crash or a killed runner (FLAKY_<CODE> and a fix: line on stderr). `; @@ -48,6 +57,10 @@ const FIX: Record<string, string> = { FLAKY_COMMAND_KILLED: 'the runner was killed by a signal (a cancelled job, an OOM kill); nothing was rerun, rerun the job', FLAKY_COMMAND_NOT_STARTED: 'check the command after -- (it runs without a shell; wrap it as bash -c \'<script> "$@"\' _ when it needs one)', FLAKY_UNREPORTED_FAILURES: 'the runner counted failures its JUnit omits (a test file that failed to load, an error outside any test); fix that first, it is a real red', + FLAKY_QUARANTINE_INVALID: 'fix .gstack/quarantine.json: it must be {"schema":1,"entries":{...},"released":[...]} (gstack-flaky quarantine check names the problem)', + FLAKY_QUARANTINE_BASE: 'fetch the base branch before the run (git fetch origin <branch>, or actions/checkout fetch-depth: 0) and pass it as --base origin/<branch>', + FLAKY_QUARANTINE_REFUSED: 'read the reason above; docs/flaky.md#quarantine lists the rules', + FLAKY_QUARANTINE_USAGE: 'gstack-flaky quarantine --help', }; function die(msg: string, code = 2): never { @@ -60,8 +73,13 @@ function coded(e: JunitError): never { process.exit(3); } +function quarantineFailed(e: QuarantineError): never { + process.stderr.write(`gstack-flaky: ${e.message} (${e.code})\nfix: ${FIX[e.code] ?? 'see docs/flaky.md#quarantine'}\n`); + process.exit(e.exitCode); +} + function parse(argv: string[]) { - const o: { junit?: string; runner?: JunitRunner; maxReruns: number; rerunScope: 'command' | 'test'; ledger?: string; command: string[] } = { maxReruns: DEFAULT_MAX_RERUNS, rerunScope: 'command', command: [] }; + const o: { junit?: string; runner?: JunitRunner; maxReruns: number; rerunScope: 'command' | 'test'; ledger?: string; base?: string; command: string[] } = { maxReruns: DEFAULT_MAX_RERUNS, rerunScope: 'command', command: [] }; for (let i = 0; i < argv.length; i++) { const a = argv[i]!; const val = () => { const v = argv[++i]; if (v === undefined) die(`${a} needs a value`); return v; }; @@ -80,6 +98,7 @@ function parse(argv: string[]) { if (v !== 'command' && v !== 'test') die(`--rerun-scope must be command or test, got ${JSON.stringify(v)}`); o.rerunScope = v; } else if (a === '--ledger') o.ledger = val(); + else if (a === '--base') o.base = val(); else die(`unknown option ${a}\n${USAGE}`); } return o; @@ -88,6 +107,16 @@ function parse(argv: string[]) { const [sub, ...rest] = process.argv.slice(2); if (!sub || sub === '--help' || sub === '-h') { process.stdout.write(USAGE); process.exit(sub ? 0 : 2); } +if (sub === 'quarantine') { + try { + process.exit(quarantineMain(rest, { cwd: process.cwd(), out: s => process.stdout.write(s), now: new Date() })); + } catch (e) { + if (e instanceof QuarantineError) quarantineFailed(e); + if (e instanceof JunitError) coded(e); + throw e; + } +} + if (sub === 'ids') { const o = parse(rest); if (!o.junit) die('ids needs --junit <file>'); @@ -107,9 +136,15 @@ try { existing = fs.statSync(o.junit, { throwIfNoEntry: false }) && fs.readFileS } if (existing && existing.trim() && !/<testsuites?[\s>/]/.test(existing)) die(`--junit ${JSON.stringify(o.junit)} exists and is not a JUnit report; gstack-flaky deletes that path before every attempt, so it refuses to touch it`); if (!o.command.length) die('run needs the test command after --'); +let quarantined: Set<string>; +try { + const live = liveQuarantine(readRegistry(process.cwd(), o.base), new Date()); + for (const n of live.notices) process.stdout.write(`${n}\n`); + quarantined = live.ids; +} catch (e) { if (e instanceof QuarantineError) quarantineFailed(e); throw e; } let result; try { - result = await runFlaky({ command: o.command, junit: o.junit, runner: o.runner, maxReruns: o.maxReruns, rerunScope: o.rerunScope }); + result = await runFlaky({ command: o.command, junit: o.junit, runner: o.runner, maxReruns: o.maxReruns, rerunScope: o.rerunScope, quarantined }); } catch (e) { if (e instanceof JunitError) coded(e); throw e; } const ledger = ledgerPath(o.ledger); appendLedger(result.ledger, ledger, s => process.stderr.write(`gstack-flaky: ${s}\n`)); @@ -118,6 +153,7 @@ const summary = process.env.GITHUB_STEP_SUMMARY && stepSummary(result); if (summary) { try { fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY!, summary); } catch { /* a summary is a convenience, never the verdict */ } } if (process.env.GITHUB_ACTIONS === 'true') { for (const f of result.outcomes.filter(x => x.status === 'flaky-pass')) process.stdout.write(`::warning title=flaky test::${workflowEscape(safeText(f.test.id))} passed on attempt ${f.attempts.length}\n`); + for (const f of result.outcomes.filter(x => x.status === 'hard-break')) process.stdout.write(`::error title=quarantined test hard break::${workflowEscape(safeText(f.test.id))} failed all ${f.attempts.length} attempts\n`); } if (result.code) { process.stderr.write(`gstack-flaky: ${result.message} (${result.code})\nfix: ${FIX[result.code] ?? 'see docs/flaky.md'}\n`); diff --git a/captain/templates/automation-flaky-pass.md b/captain/templates/automation-flaky-pass.md index c51b78944..7afbd5f26 100644 --- a/captain/templates/automation-flaky-pass.md +++ b/captain/templates/automation-flaky-pass.md @@ -16,6 +16,13 @@ ledger's Open section: in-run rerun turned green (date, case, tier, PR and head, failing attempt, passing attempt, one-line symptom from the failure log). 2. The free suites' retried cases in the same window, from the runners' retry output. +3. For repositories that run `gstack-flaky`: download the window's `flaky-junit-ledger-*` + artifacts (`gh run download <run-id> --pattern 'flaky-junit-ledger-*' --dir ledgers/<run-id>`) + and run `gstack-flaky quarantine report --ledger ledgers`. Fix its expired and EXPIRING + entries first: fix the test, or renew once with new evidence and the owner named, never + a second time. Release each READY_TO_RELEASE entry with + `gstack-flaky quarantine release <id> --reason <why> --pr <url>` in a PR of its own. + A hard-break line is a real red, not a flake: fix it like any failing test. Never add a case that failed twice (that is a real failure, not a flake), and never a harness, infra or incomplete red. Then group the Open section by case, worst repeat @@ -33,5 +40,5 @@ spend of every paid run you made. ``` What it reads: the ledger, the gate runs' flake-records artifacts, the free suites' -retry output, the crew rules file. What it starts: one fix PR per cause, each READY'd to +retry output, the `gstack-flaky` ledgers and quarantine report, the crew rules file. What it starts: one fix PR per cause, each READY'd to the controller. What it never does: rerun a gate, change a threshold, or merge. diff --git a/docs/captain.md b/docs/captain.md index e955acfb7..5fb3de733 100644 --- a/docs/captain.md +++ b/docs/captain.md @@ -332,7 +332,9 @@ A scheduled automation starts one lane a day with the runs' flake-records artifacts and the free suites' retried cases, writes one ledger line per event, groups by case, and fixes the worst repeat first with a PR that shows the failure red before green. It never reruns a gate and never touches a threshold, trial -count or case kind. On Capy, create it with the automation tool, scheduled in the owner's +count or case kind. Where a repository runs `gstack-flaky`, the lane also reads +`gstack-flaky quarantine report --ledger <downloaded ledgers>` and fixes expiring quarantine +entries before anything else ([flaky.md](flaky.md#the-daily-flaky-pass)). On Capy, create it with the automation tool, scheduled in the owner's timezone, running as a thread in the crew so its report lands in `reports/`. The lane's side of the same rule: in CI the gate reruns a case that failed on its own diff --git a/docs/flaky.md b/docs/flaky.md index 1a56b798e..eeea3e8ab 100644 --- a/docs/flaky.md +++ b/docs/flaky.md @@ -6,7 +6,9 @@ rerun is a **flaky pass**: the job stays green and the flake is recorded on a le the daily flaky pass can fix it at the source. A test that fails twice is red. This is CREW rule 16 ("run the failed test again and note that it is flaky") for any runner that writes JUnit: Playwright, `bun test`, jest, vitest and -pytest. gstack's own free suite keeps its file-level retry +pytest. A test that keeps flaking while it is fixed can be +[quarantined](#quarantine): it still runs, with up to 3 attempts, and never counts +as a pass. gstack's own free suite keeps its file-level retry (`GSTACK_FREE_RETRY_FLAKY`, [Flake ledger](TESTING_INTERNALS.md#flake-ledger)). ## Get started in GitHub Actions @@ -74,6 +76,104 @@ It cannot see a coverage threshold configured without console output (for example `coverageReporter = ["lcov"]` in `bunfig.toml`), which is why this scope is opt-in. +## Quarantine + +A test that keeps flaking while someone fixes it can be **quarantined** in +`.gstack/quarantine.json` at the repository root. Quarantine is a larger +attempt budget, never a skip: the test still runs on every run, is listed as +quarantined in the output, and never counts as a pass. + +**How a quarantined test is judged.** The verdict rule from +[Rerun scope](#rerun-scope) does not change: an attempt counts only when the +runner itself reports a full green run. When every test an attempt failed is a +live quarantined test, and nothing else is wrong with that attempt (no kill, no +failure outside the report, no first-run test missing), the whole command may +run again, up to 3 attempts in total. The run is green only when one of those +attempts is fully green. A quarantined test that fails all 3 is a **hard +break**, and the run is red. A test that is not quarantined keeps its one +rerun. A red attempt never turns green because its only reported failure is a +quarantined test, because the runner's exit status can come from things the +report does not show (a coverage threshold, a fail-on-flaky policy, a global +setup). With `--rerun-scope test` the quarantined test gets the same 3 +attempts on the targeted path, with the same fallback to the whole command. +When the runner retries on its own (Playwright `retries`), its attempts are +the budget and nothing is rerun again. + +**Missing results.** A quarantined id missing from a run that included its +file (and, for Playwright, its project) is red. A run that did not include the +file, such as a shard or a diff selection, is not. + +**Read from the base branch.** `run --base <ref>` reads the registry from +that commit instead of the working tree. Pull-request CI passes its target +branch, so a change cannot quarantine the test it breaks. `check --base` +compares against the merge base, so check out full history for it: + +```yaml +- uses: actions/checkout@v4 + with: { fetch-depth: 0 } +- name: Quarantine registry + run: '"$RUNNER_TEMP/gstack/bin/gstack-flaky" quarantine check --base "origin/${{ github.base_ref }}"' +- name: E2E + run: | + "$RUNNER_TEMP/gstack/bin/gstack-flaky" run --base "origin/${{ github.base_ref }}" --junit results/junit.xml -- npx playwright test --reporter=line,junit +``` + +### Commands + +| Command | What it does | +|---|---| +| `gstack-flaky quarantine add <id> --owner <who> --ledger <url> --class <class> --reason <text> --evidence <url>… --junit <file> [--days <n>]` | Adds an entry that expires in `--days` (default 7, at most 14). `--junit` counts the suite for the cap and confirms the id exists; `gstack-flaky ids` prints the ids. | +| `gstack-flaky quarantine renew <id> --owner <who> --evidence <url> [--days <n>]` | One renewal of up to 7 days from the later of now and the current expiry, with evidence not already on the entry and the owner named again. A second renewal is refused. | +| `gstack-flaky quarantine release <id> --reason <text> --pr <url>` | Moves the entry to `released` with the date, the reason and the PR. | +| `gstack-flaky quarantine check [--base <ref>] [--junit <file>]` | Validates every entry and the cap; with `--base`, also what this change did since the merge base. Exits 1 with one `QUARANTINE_REFUSED:` line per problem. | +| `gstack-flaky quarantine report [--ledger <file\|dir>…] [--base <ref>]` | Lists expired entries, then live entries soonest-expiring first with days left. With downloaded ledgers it adds recent outcomes and marks entries with 10 consecutive clean passes `READY_TO_RELEASE`; without `--ledger` it says the exit rule was not checked. | + +### Rules + +- **Every entry is owned and expires.** An entry holds `owner`, `ledger` + (the ledger line or issue tracking the fix), `failureClass`, `reason`, + `evidence`, `enteredAt`, `expiresAt`, `renewals` and `suiteTests` (the + suite size `add` counted). It lasts 7 days by default and at most 14 from + entry, plus one renewal of at most 7 days. +- **Expired means not quarantined.** An expired entry stops applying without + an edit: `run` treats the test as an ordinary test and prints + `FLAKY_QUARANTINE_EXPIRED` with the renew and release commands. An entry + that breaks a rule (for example one that lasts 30 days) is ignored the same + way, with `FLAKY_QUARANTINE_IGNORED`. +- **Cap.** Live entries are capped at 10% of the suite's tests, at least 1, + and 5 for a suite under 50 tests. `add` refuses past the cap; `check` + enforces it against `--junit` when given, else against the smallest + `suiteTests` on a live entry. +- **Classes.** `detector`, `harness`, `timing` and `model-latency`. + `product` is refused (a product defect is fixed, not quarantined), and so is + `infra` (an infrastructure red is rerun or fixed as infrastructure). +- **`check --base` refuses** an entry added in the same change that edits its + test's file, an entry that disappears without a release record, a release + record that was removed, and a changed `enteredAt` or rewritten renewal + history. +- **Exit rule.** 10 consecutive clean passes (`quarantined-pass` with no + failed attempt) on the ledgers since the entry was added. Release the entry + with the fix PR. +- **Separate from `CASE_QUARANTINE`.** gstack's own paid-eval quarantine + (`CASE_QUARANTINE` and `EVAL_POLICY`, + [TESTING_INTERNALS.md](TESTING_INTERNALS.md)) is a different registry with + its own rules for eval cases. The two share no schema, and `gstack-flaky` + neither reads nor writes it. + +### The daily flaky pass + +Download the ledger artifacts of the last week's runs and read them with +`report`; fix expired and `EXPIRING` entries first (fix the test, or renew +once with new evidence), release `READY_TO_RELEASE` entries, then fix the +worst repeat on the ledger: + +```bash +for id in $(gh run list --workflow e2e.yml --limit 50 --json databaseId -q '.[].databaseId'); do + gh run download "$id" --pattern 'flaky-junit-ledger-*' --dir "ledgers/$id" || true +done +gstack-flaky quarantine report --ledger ledgers +``` + ## Rules - **One rerun, small failures only.** A failed test gets one rerun when the @@ -101,8 +201,9 @@ scope is opt-in. - **Ids.** A test is `<file>::<project>::<describe > title>`, with the project empty outside Playwright, so a test in two browser projects is two tests. `gstack-flaky ids --junit <file>` prints them. -- **Ledger.** One JSON line per flaky pass (`kind: flaky-pass`) and per test - that failed its rerun (`rerun-fail`), with the tree, commit, branch, run URL, +- **Ledger.** One JSON line per flaky pass (`kind: flaky-pass`), per test + that failed its rerun (`rerun-fail`) and per quarantined test on every run + (`quarantined-pass`, `quarantined-fail`, `hard-break`), with the tree, commit, branch, run URL, attempts, the rerun's scope (`test` or `command`) and the first lines of the error. Ids, paths and failure text go through gstack's redaction engine first; when the scan cannot run, they are withheld. The path is `--ledger`, @@ -113,15 +214,17 @@ scope is opt-in. ## Output and exit codes -One `FLAKY: <flaky-pass|fail|rerun-fail> <id> attempts=<…> (<reason>)` line per -test that failed at least once, `FLAKY_RERUN: scope=command (<why>)` when the -rerun repeated the whole command, then -`FLAKY_RESULT: tests=<n> failed=<n> flaky=<n> reruns=<n> command_exit=<n> verdict=pass|fail ledger=<file|none>`. +One `FLAKY: <flaky-pass|fail|rerun-fail|quarantined-pass|quarantined-fail|hard-break> <id> attempts=<…> (<reason>)` +line per test that failed at least once and per quarantined test, +`FLAKY_RERUN: scope=command (<why>)` when the rerun repeated the whole command, then +`FLAKY_RESULT: tests=<n> failed=<n> flaky=<n> quarantined=<n> reruns=<n> command_exit=<n> verdict=pass|fail ledger=<file|none>`. +`failed` counts every outcome but `flaky-pass` and `quarantined-pass`; `quarantined` +counts the quarantined tests' outcomes, which are never counted as passes. | Exit | Meaning | |---|---| | 0 | Every test passed, flaky passes included | -| 1 | A test failed (after its rerun, or with no rerun), including `FLAKY_UNREPORTED_FAILURES` | +| 1 | A test failed (after its rerun, or with no rerun), including `FLAKY_UNREPORTED_FAILURES`, a quarantined test's hard break or missing result, and a refused `quarantine` command | | 2 | Usage error, including a `--junit` path that is not a `.xml` file, or an existing file there that is not a readable JUnit report | | 3 | No usable report, a crash, or a killed runner (a code below on stderr) | @@ -135,3 +238,9 @@ rerun repeated the whole command, then | `FLAKY_COMMAND_NOT_STARTED` | 3 | The command could not be started | Check the argv after `--` | | `FLAKY_UNREPORTED_FAILURES` | 1 | The runner counted failures or errors its JUnit omits | Fix the load error or the error outside any test: a real red | | `FLAKY_LEDGER_WRITE_FAILED` | — | The ledger could not be appended (warning) | Check the path; the verdict stands | +| `FLAKY_QUARANTINE_INVALID` | 3 | `.gstack/quarantine.json` is not valid JSON or not a schema 1 registry | Fix the file; `quarantine check` names the problem | +| `FLAKY_QUARANTINE_BASE` | 3 | `--base` does not resolve to a commit (exit 2 for a value starting with `-`), or `check` finds no merge base | Fetch the base branch; `check --base` needs `fetch-depth: 0` | +| `FLAKY_QUARANTINE_REFUSED` | 1 | `add`, `renew` or `release` broke a rule | Read the reason; [Quarantine](#quarantine) lists the rules | +| `FLAKY_QUARANTINE_USAGE` | 2 | An unknown `quarantine` command or option, or a missing value | `gstack-flaky quarantine --help` | +| `FLAKY_QUARANTINE_EXPIRED` | — | Notice: an expired entry no longer applies | Fix the test, renew once, or release; the line prints both commands | +| `FLAKY_QUARANTINE_IGNORED` | — | Notice: an entry breaks a rule and does not apply | Fix the entry; `quarantine check` names the rule | diff --git a/docs/standalone-binaries.md b/docs/standalone-binaries.md index 201521adc..ea21814b5 100644 --- a/docs/standalone-binaries.md +++ b/docs/standalone-binaries.md @@ -18,7 +18,7 @@ Beyond the slash-command skills, gstack ships standalone CLIs for workflows that | `gstack-review-log` | **Review-pass receipts** — `--start <skill>` captures the working-tree fingerprint before a diff review; `'<JSON>' --finish <token>` consumes that single-use, repository/branch/skill-scoped receipt. Binding requires matching start/end content and reviewer-reported `completed:true` and `converged:true`; it is not independent proof that a model read the source. `--check-shared-libs <token>` reads a current finding as JSON on stdin and checks prior Skip decisions against the actual branch, capture and eligible raw source blobs. It returns `reusable:false` when proof is missing or unsafe, including older records without logger-versioned coverage. Final review logging computes shared-code fingerprints and coverage rather than trusting supplied proof. | | `gstack-review-read` | **Review freshness** — emits review records with computed `review_freshness.status` and `reason`: CURRENT, STALE, or UNVERIFIED for diff reviews. `/ship` and `/land-and-deploy` use the same grade; a matching commit alone never certifies a diff review. [Dashboard rules](skills.md#review-readiness-dashboard). | | `gstack-evidence` | **Verification-evidence ledger** — `run --label <lane> -- <cmd>` transparently wraps any test command (the child's exit code always passes through) and records what ran against which working-tree fingerprint; `check` grades each label FRESH/STALE/MISSING with `--expect-cmd`, `--max-age`, and `--allow-paths` binding. /ship and /land-and-deploy cite fresh evidence instead of re-running suites. Per-run logs are 0600, capped at 2MB, pruned after 30 days; the ledger and logs stay machine-local by design. | -| `gstack-flaky` | **Flaky-test rerun for any JUnit runner** — `run --junit <file.xml> -- <command>` runs your test command (Playwright, `bun test`, jest, vitest, pytest) and, when at most `--max-reruns` (3) tests failed and the runner made no retries of its own, reruns the whole command once (`--rerun-scope test` opts in to rerunning only the failed tests). A test that fails and then passes on that rerun is a flaky pass: the run stays green and one redacted line per flaky pass or failed rerun goes on a ledger (`GSTACK_FLAKE_LEDGER_JUNIT`, default `<state root>/flake-ledger-junit.jsonl`). A missing or malformed report, a crash or a killed runner never passes. `ids --junit <file>` prints a report's test ids. Exit 0 pass, 1 a test failed, 2 usage, 3 no usable report. [Guide](flaky.md). | +| `gstack-flaky` | **Flaky-test rerun for any JUnit runner** — `run --junit <file.xml> -- <command>` runs your test command (Playwright, `bun test`, jest, vitest, pytest) and, when at most `--max-reruns` (3) tests failed and the runner made no retries of its own, reruns the whole command once (`--rerun-scope test` opts in to rerunning only the failed tests). A test that fails and then passes on that rerun is a flaky pass: the run stays green and one redacted line per flaky pass or failed rerun goes on a ledger (`GSTACK_FLAKE_LEDGER_JUNIT`, default `<state root>/flake-ledger-junit.jsonl`). A missing or malformed report, a crash or a killed runner never passes. `quarantine add|renew|release|check|report` keeps `.gstack/quarantine.json`: a quarantined test still runs, gets up to 3 attempts of the whole command, passes only on a full green attempt and is never counted as a pass (`run --base <ref>` reads the registry from the base branch). `ids --junit <file>` prints a report's test ids. Exit 0 pass, 1 a test failed, 2 usage, 3 no usable report. [Guide](flaky.md). | | `gstack-pr-wait` | **PR event watcher** — `gstack-pr-wait <pr> [--repo owner/name] [--until green\|ready\|any-change] [--interval 20] [--timeout 60] [--exclude CHECK,...]` polls one pull request through `gh` and prints one stdout line per real event (a check failed with its link, all checks finished, a comment or review from someone other than the author, a review bot finished on the head, the base moved and its new commits touch this PR's files, BEHIND or DIRTY, merged or closed), with progress on stderr, so a monitor wakes the agent once per event instead of on a timer. `green` gates on every check on the head (an advisory bot's review is never waited for, its checks gate like any other), as `gstack-ci-gate` does; a check the base branch requires (rulesets and classic branch protection) is pending until it appears, never green by omission; two identical polls confirm a check set is complete; `--exclude` leaves a named non-required check out of the verdict as `gstack-ci-gate --exclude` does (a required name is refused and gates). Exit 0 when the condition is met, 1 when it needs action or `--timeout` passes (`PR_WAIT_BLOCKED`), 2 on bad input, a second watcher for the same PR or a local error (`PR_WAIT_REFUSED`). State under `<state root>/pr-wait/`, baseline discarded after 6 hours. Review bots are configuration: `gstack-config set pr_wait_gating_bots greptile,macroscope`, `pr_wait_advisory_bots coderabbit` (advisory bots are reported, never waited on). `/land-and-deploy` Step 3 is one call of it. Design ported from tstack's pr-wait (MIT, Theo Browne; NOTICE.md). [Exit codes](troubleshooting.md#gstack-pr-wait). | | `gstack-issue-guard` | **Tracker-text trust envelope** — fetches GitHub issue/PR text (`issue <n>`, `pr-body`, `pr-comments`, or `--stdin`) and wraps it in a labeled envelope so agents treat it as data: injection-shaped lines get labeled even through fullwidth and invisible-character evasion, and forged envelope banners are defused. Every tracker-text ingress in gstack routes through it, enforced by a CI scanner. | | `gstack-ios-qa-daemon` | **iOS QA daemon** — Mac-side broker between an agent and a connected iPhone or iPad over USB CoreDevice. Loopback by default; `--tailnet` opens a Tailscale-facing listener with identity-gated capability tiers. Single-instance via flock on `~/.gstack/ios-qa-daemon.pid`. See [docs/howto-ios-testing-with-gstack.md](howto-ios-testing-with-gstack.md). | diff --git a/lib/flaky-run.ts b/lib/flaky-run.ts index 4c2f0d65e..a92ba91d8 100644 --- a/lib/flaky-run.ts +++ b/lib/flaky-run.ts @@ -29,15 +29,23 @@ * omits (bun files that fail to load), the run is red and nothing is rerun. * The first report is restored at the --junit path when the run ends; each * rerun's report is kept beside it as <name>.rerun-<n>.xml. + * - Quarantine (lib/quarantine.ts) is a larger attempt budget, never a skip: + * a live quarantined test is listed and ledgered on every run and never + * counts as a pass. When every test a whole-command attempt failed is + * quarantined and nothing else is wrong, the command may run again, up to + * QUARANTINE_ATTEMPTS in total; the run is green only on an attempt with the + * runner's full green verdict, so a quarantined failure never turns a red + * attempt green. One that fails every attempt is a hard break. A quarantined + * id missing from a run that included its file is red. */ import * as fs from 'fs'; import * as path from 'path'; import { spawn as spawnChild, spawnSync } from 'child_process'; -import { finalAttempt, JunitError, readJunit, type JunitReport, type JunitRunner, type JunitTest } from './junit'; +import { finalAttempt, JunitError, parseTestId, readJunit, type Attempt, type JunitReport, type JunitRunner, type JunitTest } from './junit'; import { redactFindingSpans } from './redact-engine'; import { resolveStateRoot } from './state-root'; -export type LedgerKind = 'flaky-pass' | 'rerun-fail'; +export type LedgerKind = 'flaky-pass' | 'rerun-fail' | 'quarantined-pass' | 'quarantined-fail' | 'hard-break'; export interface LedgerLine { ts: string; @@ -64,6 +72,8 @@ export interface RunOptions { maxReruns: number; /** 'command' (default): rerun the whole command. 'test': rerun the failed test alone when the first run is proven complete. */ rerunScope?: 'command' | 'test'; + /** Live quarantined test ids (lib/quarantine.ts liveQuarantine): up to QUARANTINE_ATTEMPTS attempts, never a skip. */ + quarantined?: ReadonlySet<string>; cwd?: string; env?: NodeJS.ProcessEnv; /** Runs one attempt; resolves with its exit status and the tail of its combined output. */ @@ -73,7 +83,7 @@ export interface RunOptions { export interface TestOutcome { test: JunitTest; - status: 'flaky-pass' | 'fail' | 'rerun-fail'; + status: 'flaky-pass' | 'fail' | 'rerun-fail' | 'quarantined-pass' | 'quarantined-fail' | 'hard-break'; attempts: string[]; reason?: string; } @@ -88,6 +98,8 @@ export interface RunResult { commandExit: number; outcomes: TestOutcome[]; reruns: number; + /** Outcomes of quarantined tests (listed, never counted as passes). */ + quarantined: number; /** Why the rerun was the whole command (the default, or why a requested targeted rerun was refused); absent for targeted reruns. */ wholeRerun?: string; ledger: LedgerLine[]; @@ -243,7 +255,7 @@ function rerunProblem(again: Attempted & { report?: JunitReport }, first: JunitR } /** A rerun clears a test only when its command exited 0 and the test ran once and passed on its first attempt. */ -function rerunVerdict(again: Attempted & { report?: JunitReport; error?: JunitError }, t: JunitTest, problem?: string): { status: 'flaky-pass' | 'rerun-fail'; attempts: string[]; reason?: string } { +function rerunVerdict(again: Attempted & { report?: JunitReport; error?: JunitError }, t: JunitTest, problem?: string): { status: 'flaky-pass' | 'rerun-fail'; attempts: Attempt[]; reason?: string } { const match = again.report?.tests.find(x => x.id === t.id); if (!match) return { status: 'rerun-fail', attempts: [...t.attempts, 'fail'], reason: again.error ? `rerun: ${again.error.message}` : 'rerun: the report does not contain this test' }; const attempts = [...t.attempts, ...match.attempts]; @@ -254,11 +266,47 @@ function rerunVerdict(again: Attempted & { report?: JunitReport; error?: JunitEr return { status: 'flaky-pass', attempts }; } +export const QUARANTINE_ATTEMPTS = 3; + +const PASSING: ReadonlySet<TestOutcome['status']> = new Set(['flaky-pass', 'quarantined-pass']); + +type Ran = Attempted & { report?: JunitReport; error?: JunitError }; + +/** The runner's own full green verdict for a whole-command attempt: exit 0, no failed or missing test, nothing counted outside the report. */ +function fullGreen(again: Ran, first: JunitReport): boolean { + return !!again.report && !again.signal && again.exit === 0 && !rerunProblem(again, first, true); +} + +/** + * Whether a red whole-command attempt may be followed by another: only within + * the quarantine budget, and only when every test it failed is a live + * quarantined test that ran once and nothing else is wrong (a kill, a missing + * report, failures outside the report, a first-run test missing). + */ +function mayRunAgain(again: Ran, first: JunitReport, quarantined: ReadonlySet<string>, attemptsSoFar: number): boolean { + if (attemptsSoFar >= QUARANTINE_ATTEMPTS || again.signal || !again.report) return false; + const failedNow = again.report.tests.filter(x => finalAttempt(x) === 'fail'); + if (!failedNow.length || !failedNow.every(x => quarantined.has(x.id) && x.executions === 1 && x.attempts.length === 1)) return false; + if (unreportedFailures(first.runner, again.output, failedNow.length)) return false; + const seen = new Set(again.report.tests.map(x => x.id)); + return first.tests.every(x => seen.has(x.id)); +} + +/** A test's attempts over the first run and the given whole-command reruns; a rerun that did not report it counts as a failure. */ +const history = (t: JunitTest, runs: Ran[]): Attempt[] => + [...t.attempts, ...runs.flatMap(r => r.report?.tests.find(x => x.id === t.id)?.attempts ?? ['fail' as const])]; + +/** A quarantined test passes only when it never failed or an attempt with the runner's full verdict cleared it. */ +const quarantinedStatus = (attempts: string[], cleared: boolean): 'quarantined-pass' | 'quarantined-fail' | 'hard-break' => + cleared || !attempts.includes('fail') ? 'quarantined-pass' + : attempts.length >= QUARANTINE_ATTEMPTS && attempts.every(a => a === 'fail') ? 'hard-break' : 'quarantined-fail'; + export async function runFlaky(opts: RunOptions): Promise<RunResult> { const cwd = opts.cwd ?? process.cwd(); const env = { ...(opts.env ?? process.env), PLAYWRIGHT_JUNIT_INCLUDE_RETRIES: '1' }; const now = opts.now ?? (() => new Date()); - const empty = { tests: 0, outcomes: [], reruns: 0, ledger: [] }; + const quarantined = opts.quarantined ?? new Set<string>(); + const empty = { tests: 0, outcomes: [], reruns: 0, quarantined: 0, ledger: [] }; const first = await attempt(opts, opts.command, cwd, env); if (first.signal) return { exitCode: 3, code: 'FLAKY_COMMAND_KILLED', message: `the command was killed by ${first.signal}`, commandExit: first.exit, ...empty }; if (!first.report) return { exitCode: 3, code: first.error!.code, message: first.error!.message, commandExit: first.exit, ...empty }; @@ -282,15 +330,39 @@ export async function runFlaky(opts: RunOptions): Promise<RunResult> { const prov = provenance(cwd, env); const line = (t: JunitTest, kind: LedgerKind, by: LedgerLine['rerun_by'], attempts: string[], scope?: LedgerLine['rerun_scope']): LedgerLine => ({ ts: now().toISOString(), runner: 'junit', kind, id: safeText(t.id), file: safeText(t.file), rerun_by: by, ...(scope ? { rerun_scope: scope } : {}), attempts, ...prov, excerpt: safeExcerpt(t.excerpt) }); - const ledger: LedgerLine[] = runnerFlaky.map(t => line(t, 'flaky-pass', 'runner', t.attempts)); - const outcomes: TestOutcome[] = runnerFlaky.map(t => ({ test: t, status: 'flaky-pass' as const, attempts: t.attempts })); + const isQ = (t: JunitTest) => quarantined.has(t.id); + const outcomes: TestOutcome[] = []; + const ledger: LedgerLine[] = []; + for (const t of runnerFlaky.filter(x => !isQ(x))) { + outcomes.push({ test: t, status: 'flaky-pass', attempts: t.attempts }); + ledger.push(line(t, 'flaky-pass', 'runner', t.attempts)); + } + const settleQ = (t: JunitTest, attempts: string[], cleared: boolean, reason?: string, scope?: LedgerLine['rerun_scope']) => { + const status = quarantinedStatus(attempts, cleared); + const why = status === 'hard-break' ? `failed all ${attempts.length} attempts: a hard break, not a flake` : status === 'quarantined-fail' ? reason : undefined; + outcomes.push({ test: t, status, attempts, ...(why ? { reason: why } : {}) }); + ledger.push(line(t, status, 'gstack-flaky', attempts, scope)); + }; + const settle = (t: JunitTest, attempts: string[], reason: string) => isQ(t) ? settleQ(t, attempts, false, reason) : outcomes.push({ test: t, status: 'fail', attempts, reason }); + const present = new Set(report.tests.map(t => t.id)); + const included = new Set(report.tests.map(t => `${t.file}::${t.project}`)); + for (const id of quarantined) { + const p = parseTestId(id); + if (!p || present.has(id) || !included.has(`${p.file}::${p.project}`)) continue; + const t: JunitTest = { id, file: p.file, project: p.project, path: p.title.split(' > '), attempts: [], executions: 0, excerpt: [] }; + outcomes.push({ test: t, status: 'quarantined-fail', attempts: [], reason: 'quarantined, and missing from a run that included its file' }); + ledger.push(line(t, 'quarantined-fail', 'gstack-flaky', [])); + } + const others = failed.filter(t => !isQ(t)).length; const eligible: JunitTest[] = []; for (const t of failed) { - if (t.executions > 1) outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: `it ran ${t.executions} times in this report and one failed; not rerun` }); - else if (t.attempts.length > 1) outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: `the runner already retried (${t.attempts.length} attempts)` }); - else if (failed.length > opts.maxReruns) outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: `${failed.length} tests failed, more than --max-reruns ${opts.maxReruns}; not rerun` }); + const why = t.executions > 1 ? `it ran ${t.executions} times in this report and one failed; not rerun` + : t.attempts.length > 1 ? `the runner already retried (${t.attempts.length} attempts)` + : !isQ(t) && others > opts.maxReruns ? `${others} tests failed, more than --max-reruns ${opts.maxReruns}; not rerun` : undefined; + if (why) settle(t, t.attempts, why); else eligible.push(t); } + const passedQ = report.tests.filter(t => isQ(t) && finalAttempt(t) !== 'fail' && t.attempts.some(a => a !== 'skip')); const wholeRerun = !eligible.length ? undefined : opts.rerunScope !== 'test' ? 'the default; --rerun-scope test reruns the failed test alone' : incompleteRun(report.runner, first.output, first.exit, opts.command, runnerFlaky.length) ?? undefined; @@ -304,30 +376,42 @@ export async function runFlaky(opts: RunOptions): Promise<RunResult> { return again; }; const record = (t: JunitTest, v: ReturnType<typeof rerunVerdict>, scope: 'test' | 'command') => { + if (isQ(t)) return settleQ(t, v.attempts, v.status === 'flaky-pass', v.reason, scope); outcomes.push({ test: t, status: v.status, attempts: v.attempts, ...(v.reason ? { reason: v.reason } : {}) }); ledger.push(line(t, v.status, 'gstack-flaky', v.attempts, scope)); }; + const runs: Ran[] = []; try { - if (wholeRerun && outcomes.some(o => o.status === 'fail')) { - for (const t of eligible) outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: `not rerun: the run is already red, and a rerun would repeat the whole command (${wholeRerun})` }); + if (wholeRerun && outcomes.some(o => !PASSING.has(o.status))) { + for (const t of eligible) settle(t, t.attempts, `not rerun: the run is already red, and a rerun would repeat the whole command (${wholeRerun})`); } else if (wholeRerun) { - const again = await rerun(opts.command); - const problem = rerunProblem(again, report, true); - for (const t of eligible) record(t, rerunVerdict(again, t, problem), 'command'); + do { + runs.push(await rerun(opts.command)); + } while (!fullGreen(runs.at(-1)!, report) && mayRunAgain(runs.at(-1)!, report, quarantined, 1 + runs.length)); + const decider = runs.at(-1)!; + const problem = rerunProblem(decider, report, true); + for (const t of eligible) record(t, rerunVerdict(decider, { ...t, attempts: history(t, runs.slice(0, -1)) }, problem), 'command'); } else if (eligible.length) { let interrupted = false; for (const t of eligible) { - if (interrupted) { outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: 'not rerun: an earlier rerun was killed' }); continue; } - const again = await rerun([...opts.command, ...rerunSelection(report.runner as 'playwright' | 'bun', t)]); - interrupted = !!again.signal; - record(t, rerunVerdict(again, t, rerunProblem(again, report, false)), 'test'); + if (interrupted) { settle(t, t.attempts, 'not rerun: an earlier rerun was killed'); continue; } + let v: ReturnType<typeof rerunVerdict> = { status: 'rerun-fail', attempts: t.attempts }; + for (let budget = isQ(t) ? QUARANTINE_ATTEMPTS - t.attempts.length : 1; budget > 0 && v.status !== 'flaky-pass' && !interrupted; budget--) { + const again = await rerun([...opts.command, ...rerunSelection(report.runner as 'playwright' | 'bun', t)]); + interrupted = !!again.signal; + v = rerunVerdict(again, { ...t, attempts: v.attempts }, rerunProblem(again, report, false)); + } + record(t, v, 'test'); } } } finally { if (reruns) fs.writeFileSync(junit, firstXml); } - const red = outcomes.some(o => o.status !== 'flaky-pass'); - return { exitCode: red ? 1 : 0, tests: report.tests.length, commandExit: first.exit, outcomes, reruns, ...(wholeRerun && reruns ? { wholeRerun } : {}), ledger }; + const green = runs.length ? fullGreen(runs.at(-1)!, report) : reruns > 0 || first.exit === 0; + for (const t of passedQ) settleQ(t, history(t, runs), green, 'the attempt that decided the run was not a full green run', runs.length ? 'command' : undefined); + const red = !green || outcomes.some(o => !PASSING.has(o.status)); + const q = outcomes.filter(o => o.status === 'quarantined-pass' || o.status === 'quarantined-fail' || o.status === 'hard-break').length; + return { exitCode: red ? 1 : 0, tests: report.tests.length, commandExit: first.exit, outcomes, reruns, quarantined: q, ...(wholeRerun && reruns ? { wholeRerun } : {}), ledger }; } export function ledgerPath(flag: string | undefined, env: NodeJS.ProcessEnv = process.env): string { @@ -352,9 +436,9 @@ export function renderResult(r: RunResult, ledger: string): string[] { const out: string[] = []; for (const o of r.outcomes) out.push(`FLAKY: ${o.status} ${safeText(o.test.id)} attempts=${o.attempts.join(',')}${o.reason ? ` (${o.reason})` : ''}`); const flaky = r.outcomes.filter(o => o.status === 'flaky-pass').length; - const failed = r.outcomes.length - flaky; + const failed = r.outcomes.filter(o => !PASSING.has(o.status)).length; if (r.wholeRerun) out.push(`FLAKY_RERUN: scope=command (${r.wholeRerun})`); - out.push(`FLAKY_RESULT: tests=${r.tests} failed=${failed} flaky=${flaky} reruns=${r.reruns} command_exit=${r.commandExit} verdict=${r.exitCode === 0 ? 'pass' : 'fail'} ledger=${r.ledger.length ? ledger : 'none'}`); + out.push(`FLAKY_RESULT: tests=${r.tests} failed=${failed} flaky=${flaky} quarantined=${r.quarantined} reruns=${r.reruns} command_exit=${r.commandExit} verdict=${r.exitCode === 0 ? 'pass' : 'fail'} ledger=${r.ledger.length ? ledger : 'none'}`); return out; } diff --git a/lib/junit.ts b/lib/junit.ts index 0f4d504d3..5440f4b48 100644 --- a/lib/junit.ts +++ b/lib/junit.ts @@ -137,6 +137,14 @@ function excerptOf(node: XmlNode | undefined): string[] { export const testId = (file: string, project: string, path: string[]) => `${file}::${project}::${path.join(' > ')}`; +/** The parts of a test id, or null when it is not `<file>::<project>::<title>`. */ +export function parseTestId(id: string): { file: string; project: string; title: string } | null { + const i = id.indexOf('::'); + const j = i < 0 ? -1 : id.indexOf('::', i + 2); + if (i <= 0 || j < 0 || j + 2 >= id.length) return null; + return { file: id.slice(0, i), project: id.slice(i + 2, j), title: id.slice(j + 2) }; +} + function readCase(tc: XmlNode, runner: JunitRunner): Omit<JunitTest, 'id' | 'executions'> & { hidden: boolean } { const suite = tc.parent!; let file: string, project = '', path: string[]; diff --git a/lib/quarantine.ts b/lib/quarantine.ts new file mode 100644 index 000000000..04ddb2dc3 --- /dev/null +++ b/lib/quarantine.ts @@ -0,0 +1,444 @@ +/** + * gstack-flaky quarantine (GSTA-158 P2): a registry of known-flaky tests at + * `.gstack/quarantine.json`. A quarantined test still runs; it gets a larger + * attempt budget (lib/flaky-run.ts, QUARANTINE_ATTEMPTS) under the same full + * green verdict, and it is never counted as a pass. This file holds the + * registry: schema, validation, add/renew/release, `check` against the base + * branch, and `report` over downloaded ledgers. + * + * Rules (docs/flaky.md § Quarantine): + * - Every entry has an owner, a ledger link, a failure class (detector, + * harness, timing, model-latency; product defects and infra are refused), + * evidence and an expiry: 7 days by default, at most 14 from entry, and one + * renewal of at most 7 days with new evidence and the owner named again. + * - An expired or invalid entry is not quarantined; nothing needs editing. + * - Live entries are capped at max(1, 10% of the suite's tests), and 5 for a + * suite under 50 tests. + * - `run --base` reads the registry from the base branch so a change cannot + * quarantine the test it breaks; `check --base` refuses an entry added in + * the change that edits its test's file, an entry that disappears without a + * release record, a release record that was removed, and rewritten history. + * - gstack's own CASE_QUARANTINE / EVAL_POLICY (paid evals) is a separate + * registry; the two share no schema. + */ +import * as fs from 'fs'; +import * as path from 'path'; +import { spawnSync } from 'child_process'; +import { parseTestId, readJunit, type JunitRunner } from './junit'; +import { safeText } from './flaky-run'; + +export const REGISTRY_PATH = '.gstack/quarantine.json'; +export const FAILURE_CLASSES = ['detector', 'harness', 'timing', 'model-latency'] as const; +export type FailureClass = typeof FAILURE_CLASSES[number]; +export const DEFAULT_DAYS = 7; +export const MAX_DAYS = 14; +export const RENEW_MAX_DAYS = 7; +export const EXIT_RULE_PASSES = 10; +const DAY = 86_400_000; + +export interface Renewal { renewedAt: string; previousExpiresAt: string; expiresAt: string; owner: string; evidence: string[] } + +export interface QuarantineEntry { + owner: string; + /** Link to the flaky ledger line or issue that tracks the fix. */ + ledger: string; + failureClass: FailureClass; + reason: string; + evidence: string[]; + enteredAt: string; + expiresAt: string; + renewals: Renewal[]; + /** The suite's test count when the entry was added (`add --junit`), the base of the cap. */ + suiteTests: number; +} + +export type ReleasedEntry = Omit<QuarantineEntry, 'reason'> & { id: string; releasedAt: string; reason: string; pr: string }; + +export interface Registry { schema: 1; entries: Record<string, QuarantineEntry>; released: ReleasedEntry[] } + +export class QuarantineError extends Error { + constructor(readonly code: string, message: string, readonly exitCode: 1 | 2 | 3 = 3) { super(message); } +} + +export const emptyRegistry = (): Registry => ({ schema: 1, entries: {}, released: [] }); + +/** Live entries allowed for a suite of `tests` tests. */ +export const suiteCap = (tests: number) => tests < 50 ? 5 : Math.max(1, Math.floor(tests / 10)); + +function git(args: string[], cwd: string): { ok: boolean; out: string } { + const r = spawnSync('git', args, { cwd, encoding: 'utf-8', timeout: 15_000 }); + return { ok: r.status === 0, out: r.stdout ?? '' }; +} + +export function repoRoot(cwd: string): string { + const r = git(['rev-parse', '--show-toplevel'], cwd); + return r.ok && r.out.trim() ? r.out.trim() : cwd; +} + +export function parseRegistry(text: string, source: string): Registry { + let data: unknown; + try { data = JSON.parse(text); } catch (e) { + throw new QuarantineError('FLAKY_QUARANTINE_INVALID', `${source} is not valid JSON (${(e as Error).message})`); + } + const r = data as Partial<Registry> | null; + if (!r || typeof r !== 'object' || r.schema !== 1 || !r.entries || typeof r.entries !== 'object' || Array.isArray(r.entries) || !Array.isArray(r.released)) { + throw new QuarantineError('FLAKY_QUARANTINE_INVALID', `${source} is not a schema 1 quarantine registry ({"schema":1,"entries":{},"released":[]})`); + } + return r as Registry; +} + +function resolveCommit(ref: string, cwd: string): string { + if (!ref || ref.startsWith('-')) throw new QuarantineError('FLAKY_QUARANTINE_BASE', `--base ${JSON.stringify(ref)} is not a git ref`, 2); + const r = git(['rev-parse', '--verify', '--quiet', `${ref}^{commit}`], cwd); + if (!r.ok) throw new QuarantineError('FLAKY_QUARANTINE_BASE', `--base ${ref} does not resolve to a commit in ${cwd}`); + return r.out.trim(); +} + +function registryAt(commit: string, root: string, label: string): Registry { + if (!git(['cat-file', '-e', `${commit}:${REGISTRY_PATH}`], root).ok) return emptyRegistry(); + const shown = git(['show', `${commit}:${REGISTRY_PATH}`], root); + if (!shown.ok) throw new QuarantineError('FLAKY_QUARANTINE_BASE', `cannot read ${REGISTRY_PATH} at ${label}`); + return parseRegistry(shown.out, `${REGISTRY_PATH} at ${label}`); +} + +/** The registry in the working tree, or at `base` when given; a missing file is an empty registry. */ +export function readRegistry(cwd: string, base?: string): Registry { + const root = repoRoot(cwd); + if (base !== undefined) return registryAt(resolveCommit(base, root), root, base); + const file = path.join(root, REGISTRY_PATH); + if (!fs.existsSync(file)) return emptyRegistry(); + return parseRegistry(fs.readFileSync(file, 'utf-8'), REGISTRY_PATH); +} + +export function writeRegistry(cwd: string, reg: Registry): string { + const file = path.join(repoRoot(cwd), REGISTRY_PATH); + const entries = Object.fromEntries(Object.keys(reg.entries).sort().map(id => [id, reg.entries[id]!])); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, JSON.stringify({ schema: 1, entries, released: reg.released }, null, 2) + '\n'); + return file; +} + +const filled = (s: unknown): s is string => typeof s === 'string' && s.trim().length > 0; +const filledList = (l: unknown): l is string[] => Array.isArray(l) && l.length > 0 && l.every(filled); + +export function classRefusal(c: unknown): string | undefined { + if (FAILURE_CLASSES.includes(c as FailureClass)) return undefined; + if (c === 'product') return 'class product is refused: a product defect is a real red; fix it instead of quarantining it'; + if (c === 'infra') return 'class infra is refused: an infrastructure red is rerun or fixed as infra, never quarantined'; + return `class ${JSON.stringify(c)} is not one of ${FAILURE_CLASSES.join(', ')}`; +} + +/** What makes one entry invalid; an invalid entry is never quarantined and `check` refuses it. */ +export function entryProblems(id: string, e: QuarantineEntry, now: Date): string[] { + if (!e || typeof e !== 'object') return ['not an object']; + const p: string[] = []; + if (!parseTestId(id)) p.push('not a test id (<file>::<project>::<title>; gstack-flaky ids prints them)'); + if (!filled(e.owner)) p.push('no owner'); + if (!filled(e.ledger)) p.push('no ledger link'); + if (!filled(e.reason)) p.push('no reason'); + const cls = classRefusal(e.failureClass); + if (cls) p.push(cls); + if (!filledList(e.evidence)) p.push('no evidence'); + if (!(Number.isInteger(e.suiteTests) && e.suiteTests > 0)) p.push('no suiteTests (add records the suite size from --junit)'); + const entered = Date.parse(e.enteredAt); + const expires = Date.parse(e.expiresAt); + if (Number.isNaN(entered) || Number.isNaN(expires)) return [...p, 'enteredAt or expiresAt is not a date']; + if (entered > now.getTime() + DAY) p.push('enteredAt is in the future'); + if (!Array.isArray(e.renewals)) return [...p, 'renewals is not a list']; + if (e.renewals.length > 1) p.push(`renewed ${e.renewals.length} times; one renewal is allowed, then fix the test or release it`); + const firstExpiry = e.renewals.length ? Date.parse(e.renewals[0]!.previousExpiresAt) : expires; + if (!(firstExpiry > entered)) p.push('expires before it entered'); + if (firstExpiry - entered > MAX_DAYS * DAY) p.push(`expires more than ${MAX_DAYS} days after it entered`); + for (const r of e.renewals) { + if (!filled(r?.owner)) p.push('a renewal names no owner'); + if (!filledList(r?.evidence) || r.evidence.every(x => filledList(e.evidence) && e.evidence.includes(x))) p.push('a renewal carries no new evidence'); + const from = Math.max(Date.parse(r?.renewedAt), Date.parse(r?.previousExpiresAt)); + const until = Date.parse(r?.expiresAt); + if (Number.isNaN(from) || Number.isNaN(until) || until - from > RENEW_MAX_DAYS * DAY) p.push(`a renewal is not a date range of at most ${RENEW_MAX_DAYS} days`); + } + const last = e.renewals.at(-1); + if (last && last.expiresAt !== e.expiresAt) p.push('expiresAt differs from the last renewal'); + return p; +} + +export const isLive = (e: QuarantineEntry, now: Date) => Date.parse(e.expiresAt) > now.getTime(); + +/** Ids `run` gives the quarantine budget, plus one notice per entry it ignores (expired or invalid). */ +export function liveQuarantine(reg: Registry, now: Date): { ids: Set<string>; notices: string[] } { + const ids = new Set<string>(); + const notices: string[] = []; + for (const [id, e] of Object.entries(reg.entries)) { + const problems = entryProblems(id, e, now); + if (problems.length) notices.push(`FLAKY_QUARANTINE_IGNORED: ${safeText(id)} is not quarantined: ${problems.join('; ')}`); + else if (!isLive(e, now)) notices.push(expiredNotice(id, e)); + else ids.add(id); + } + return { ids, notices }; +} + +const shq = (s: string) => "'" + s.split("'").join(`'"'"'`) + "'"; +export const renewCommand = (id: string) => `gstack-flaky quarantine renew ${shq(safeText(id))} --owner <you> --evidence <new evidence url>`; +export const releaseCommand = (id: string) => `gstack-flaky quarantine release ${shq(safeText(id))} --reason <why> --pr <fix pr url>`; + +export function expiredNotice(id: string, e: QuarantineEntry): string { + return `FLAKY_QUARANTINE_EXPIRED: ${safeText(id)} expired ${e.expiresAt.slice(0, 10)} and is not quarantined; renew: ${renewCommand(id)} | release: ${releaseCommand(id)}`; +} + +export interface AddInput { owner?: string; ledger?: string; failureClass?: string; reason?: string; evidence: string[]; days?: number; suiteIds: string[] } + +const refuse = (msg: string): never => { throw new QuarantineError('FLAKY_QUARANTINE_REFUSED', msg, 1); }; + +export function addEntry(reg: Registry, id: string, a: AddInput, now: Date): QuarantineEntry { + const days = a.days ?? DEFAULT_DAYS; + if (!Number.isInteger(days) || days < 1 || days > MAX_DAYS) refuse(`--days must be a whole number from 1 to ${MAX_DAYS}, got ${a.days}`); + const cls = classRefusal(a.failureClass); + if (cls) refuse(cls); + if (!a.evidence.length) refuse('no evidence: pass --evidence <url> (a failing run, a ledger line) at least once'); + if (!a.suiteIds.includes(id)) refuse(`${safeText(id)} is not a test in the --junit report (gstack-flaky ids --junit <file> prints the ids)`); + if (reg.entries[id]) refuse(`${safeText(id)} already has an entry; renew or release it`); + const live = Object.values(reg.entries).filter(e => isLive(e, now)).length; + const cap = suiteCap(a.suiteIds.length); + if (live + 1 > cap) refuse(`the registry holds ${live} live entries and the cap for a ${a.suiteIds.length}-test suite is ${cap}; fix or release one first`); + const entry: QuarantineEntry = { + owner: a.owner ?? '', ledger: a.ledger ?? '', failureClass: a.failureClass as FailureClass, reason: a.reason ?? '', evidence: a.evidence, + enteredAt: now.toISOString(), expiresAt: new Date(now.getTime() + days * DAY).toISOString(), renewals: [], suiteTests: a.suiteIds.length, + }; + const problems = entryProblems(id, entry, now); + if (problems.length) refuse(problems.join('; ')); + reg.entries[id] = entry; + return entry; +} + +export function renewEntry(reg: Registry, id: string, r: { owner?: string; evidence: string[]; days?: number }, now: Date): QuarantineEntry { + const e = reg.entries[id] ?? refuse(`${safeText(id)} has no entry to renew`); + if (e.renewals.length) refuse(`${safeText(id)} was renewed once already; a second renewal is refused: fix the test or release it`); + if (!filled(r.owner)) refuse('renew names the owner again: pass --owner'); + if (!r.evidence.some(x => filled(x) && !e.evidence.includes(x))) refuse('renew needs new evidence: pass --evidence <url> not already on the entry'); + const days = r.days ?? RENEW_MAX_DAYS; + if (!Number.isInteger(days) || days < 1 || days > RENEW_MAX_DAYS) refuse(`a renewal is 1 to ${RENEW_MAX_DAYS} days, got ${r.days}`); + const from = Math.max(now.getTime(), Date.parse(e.expiresAt)); + const expiresAt = new Date(from + days * DAY).toISOString(); + e.renewals.push({ renewedAt: now.toISOString(), previousExpiresAt: e.expiresAt, expiresAt, owner: r.owner!, evidence: r.evidence }); + e.expiresAt = expiresAt; + return e; +} + +export function releaseEntry(reg: Registry, id: string, r: { reason?: string; pr?: string }, now: Date): ReleasedEntry { + const e = reg.entries[id] ?? refuse(`${safeText(id)} has no entry to release`); + if (!filled(r.reason) || !filled(r.pr)) refuse('release needs --reason and --pr (the fix, or the decision to stop quarantining)'); + const record: ReleasedEntry = { id, ...e, releasedAt: now.toISOString(), reason: r.reason!, pr: r.pr! }; + reg.released.push(record); + delete reg.entries[id]; + return record; +} + +const liveEntries = (reg: Registry, now: Date) => Object.values(reg.entries).filter(e => e && typeof e === 'object' && isLive(e, now)); +const smallestSuite = (live: QuarantineEntry[]) => Math.min(...live.map(e => e.suiteTests).filter(n => Number.isInteger(n) && n > 0)); + +/** Structural problems of a whole registry plus the cap, for `check`. */ +export function registryProblems(reg: Registry, now: Date, suiteTests?: number): string[] { + const p: string[] = []; + for (const [id, e] of Object.entries(reg.entries)) for (const x of entryProblems(id, e, now)) p.push(`${safeText(id)}: ${x}`); + for (const r of reg.released) { + if (!r || !filled(r.id) || Number.isNaN(Date.parse(r.releasedAt)) || !filled(r.reason) || !filled(r.pr)) p.push(`released record ${JSON.stringify(r?.id)}: needs id, releasedAt, reason and pr`); + } + const live = liveEntries(reg, now); + const tests = suiteTests ?? smallestSuite(live); + if (live.length && Number.isFinite(tests) && live.length > suiteCap(tests)) p.push(`${live.length} live entries, over the cap of ${suiteCap(tests)} for a ${tests}-test suite`); + return p; +} + +const editsFile = (changed: string[], file: string) => changed.some(f => f === file || f.endsWith(`/${file}`)); + +/** What a change did to the registry that `check --base` refuses. */ +export function baseProblems(base: Registry, head: Registry, changed: string[]): string[] { + const p: string[] = []; + for (const [id, e] of Object.entries(head.entries)) { + const was = base.entries[id]; + const t = parseTestId(id); + if (!was) { + if (t && editsFile(changed, t.file)) p.push(`${safeText(id)}: added in the same change that edits ${t.file}; a change cannot quarantine the test it touches (land the entry on its own, or fix the test)`); + continue; + } + if (was.enteredAt !== e?.enteredAt) p.push(`${safeText(id)}: enteredAt changed from ${was.enteredAt}; release the entry instead of restarting it`); + if (JSON.stringify((e?.renewals ?? []).slice(0, was.renewals.length)) !== JSON.stringify(was.renewals)) p.push(`${safeText(id)}: its renewal history was rewritten`); + } + for (const [id, was] of Object.entries(base.entries)) { + if (!(id in head.entries) && !head.released.some(r => r?.id === id && r.enteredAt === was.enteredAt)) p.push(`${safeText(id)}: removed without a release record (use: ${releaseCommand(id)})`); + } + for (const r of base.released) { + if (!head.released.some(h => h?.id === r.id && h.releasedAt === r.releasedAt)) p.push(`${safeText(r.id)}: its release record of ${r.releasedAt} was removed`); + } + return p; +} + +/** `check`: the working-tree registry alone, and with `base` against what this change did since the merge base. */ +export function checkRegistry(cwd: string, o: { base?: string; suiteTests?: number }, now: Date): { problems: string[]; live: number; cap?: number } { + const head = readRegistry(cwd); + const problems = registryProblems(head, now, o.suiteTests); + if (o.base !== undefined) { + const root = repoRoot(cwd); + const commit = resolveCommit(o.base, root); + const mb = git(['merge-base', commit, 'HEAD'], root); + if (!mb.ok) throw new QuarantineError('FLAKY_QUARANTINE_BASE', `no merge base between ${o.base} and HEAD (a shallow clone?)`); + const changed = git(['diff', '--name-only', mb.out.trim()], root); + if (!changed.ok) throw new QuarantineError('FLAKY_QUARANTINE_BASE', `cannot diff against the merge base of ${o.base}`); + problems.push(...baseProblems(registryAt(mb.out.trim(), root, `the merge base of ${o.base}`), head, changed.out.split('\n').filter(Boolean))); + } + const live = liveEntries(head, now); + const tests = o.suiteTests ?? smallestSuite(live); + return { problems, live: live.length, ...(Number.isFinite(tests) ? { cap: suiteCap(tests) } : {}) }; +} + +export interface LedgerRecord { ts: string; kind: string; id: string; attempts?: string[] } + +/** Read every JSON line of the given ledger files, and of every .jsonl file under the given directories. */ +export function readLedgers(paths: string[]): { records: LedgerRecord[]; skipped: number } { + const files: string[] = []; + const walk = (p: string) => { + const st = fs.statSync(p, { throwIfNoEntry: false }); + if (!st) throw new QuarantineError('FLAKY_QUARANTINE_USAGE', `--ledger ${p} does not exist`, 2); + if (!st.isDirectory()) { files.push(p); return; } + for (const n of fs.readdirSync(p).sort()) { + const c = path.join(p, n); + if (fs.statSync(c).isDirectory() || n.endsWith('.jsonl')) walk(c); + } + }; + paths.forEach(walk); + const records: LedgerRecord[] = []; + let skipped = 0; + for (const f of files) { + for (const l of fs.readFileSync(f, 'utf-8').split('\n')) { + if (!l.trim()) continue; + try { + const r = JSON.parse(l) as LedgerRecord; + if (filled(r?.id) && filled(r.kind) && filled(r.ts)) records.push(r); else skipped++; + } catch { skipped++; } + } + } + return { records, skipped }; +} + +const cleanPass = (r: LedgerRecord) => r.kind === 'quarantined-pass' && !(r.attempts ?? []).includes('fail'); +const shortKind = (r: LedgerRecord) => cleanPass(r) ? 'pass' : r.kind === 'quarantined-pass' ? 'flaky' : r.kind === 'quarantined-fail' ? 'fail' : r.kind; + +/** `report`: expired entries first, then live entries soonest-expiring first, with ledger history when given. */ +export function reportLines(reg: Registry, now: Date, ledgers?: LedgerRecord[]): string[] { + const out: string[] = []; + const all = Object.entries(reg.entries).sort((a, b) => Date.parse(a[1].expiresAt) - Date.parse(b[1].expiresAt)); + const expired = all.filter(([, e]) => !isLive(e, now)); + const live = all.filter(([, e]) => isLive(e, now)); + for (const [id, e] of expired) out.push(expiredNotice(id, e)); + let expiring = 0, ready = 0; + for (const [id, e] of live) { + const daysLeft = Math.ceil((Date.parse(e.expiresAt) - now.getTime()) / DAY); + if (daysLeft <= 2) expiring++; + let history = ''; + if (ledgers) { + const sid = safeText(id); + const mine = ledgers.filter(r => r.id === sid && Date.parse(r.ts) >= Date.parse(e.enteredAt)).sort((a, b) => Date.parse(a.ts) - Date.parse(b.ts)); + let streak = 0; + for (let i = mine.length - 1; i >= 0 && cleanPass(mine[i]!); i--) streak++; + const isReady = streak >= EXIT_RULE_PASSES; + if (isReady) ready++; + history = ` recent=${mine.slice(-5).map(shortKind).join(',') || 'none'} clean_streak=${streak}${isReady ? ` READY_TO_RELEASE: ${releaseCommand(id)}` : ''}`; + } + const renewal = e.renewals.length ? 'renewed once; fix or release' : `renew: ${renewCommand(id)}`; + out.push(`QUARANTINE: ${safeText(id)} owner=${safeText(e.owner)} class=${e.failureClass} days_left=${daysLeft} expires=${e.expiresAt.slice(0, 10)} ledger=${safeText(e.ledger)}${history}${daysLeft <= 2 ? ` EXPIRING (${renewal})` : ''}`); + } + out.push(`QUARANTINE_REPORT: live=${live.length} expired=${expired.length} expiring=${expiring} ready=${ledgers ? ready : 'unchecked'} exit_rule=${ledgers ? `checked (${EXIT_RULE_PASSES} consecutive clean passes)` : 'not checked (no --ledger given)'}`); + return out; +} + +export const QUARANTINE_USAGE = `Usage: + gstack-flaky quarantine add <id> --owner <who> --ledger <url> --class detector|harness|timing|model-latency + --reason <text> --evidence <url> [--evidence <url>...] --junit <file> [--runner <r>] [--days <1-${MAX_DAYS}>] + gstack-flaky quarantine renew <id> --owner <who> --evidence <new url> [--days <1-${RENEW_MAX_DAYS}>] + gstack-flaky quarantine release <id> --reason <text> --pr <url> + gstack-flaky quarantine check [--base <ref>] [--junit <file>] + gstack-flaky quarantine report [--ledger <file|dir>...] [--base <ref>] + +add/renew/release edit ${REGISTRY_PATH} at the repository root. check validates it +(owner, ledger, class, evidence, expiry, one renewal, the cap) and, with --base, +refuses an entry added in the change that edits its test's file, an entry removed +without a release record, and a removed release record. report lists expired and +live entries, soonest-expiring first; with --ledger (files or directories of +downloaded ledgers) it shows recent outcomes and marks entries with ${EXIT_RULE_PASSES} +consecutive clean passes READY_TO_RELEASE. +Exit: 0 ok, 1 refused, 2 usage, 3 unreadable registry or base. +`; + +interface CliIO { cwd: string; out: (s: string) => void; now: Date } + +const usage = (msg: string): never => { throw new QuarantineError('FLAKY_QUARANTINE_USAGE', msg, 2); }; + +function parseArgs(argv: string[]) { + const flags: Record<string, string[]> = {}; + const positional: string[] = []; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]!; + if (!a.startsWith('--')) { positional.push(a); continue; } + const v = argv[++i]; + if (v === undefined) usage(`${a} needs a value`); + (flags[a.slice(2)] ??= []).push(v!); + } + return { flags, positional }; +} + +function intFlag(v: string | undefined, name: string): number | undefined { + if (v === undefined) return undefined; + const n = Number(v); + if (!Number.isInteger(n)) usage(`--${name} must be a whole number`); + return n; +} + +const ALLOWED: Record<string, string[]> = { + add: ['owner', 'ledger', 'class', 'reason', 'evidence', 'junit', 'runner', 'days'], renew: ['owner', 'evidence', 'days'], + release: ['reason', 'pr'], check: ['base', 'junit', 'runner'], report: ['ledger', 'base'], +}; + +/** The `gstack-flaky quarantine` subcommands; returns the exit code. A JunitError from --junit propagates to the bin. */ +export function quarantineMain(argv: string[], io: CliIO): number { + const [sub, ...rest] = argv; + if (!sub || sub === '--help' || sub === '-h') { io.out(QUARANTINE_USAGE); return sub ? 0 : 2; } + if (!ALLOWED[sub]) usage(`unknown quarantine command ${sub}\n${QUARANTINE_USAGE}`); + const { flags, positional } = parseArgs(rest); + const unknown = Object.keys(flags).find(f => !ALLOWED[sub]!.includes(f)); + if (unknown) usage(`${sub} does not take --${unknown}`); + const one = (f: string) => flags[f]?.at(-1); + const runner = one('runner') as JunitRunner | undefined; + if (runner !== undefined && !['playwright', 'bun', 'generic'].includes(runner)) usage('--runner must be playwright, bun or generic'); + const needsId = sub === 'add' || sub === 'renew' || sub === 'release'; + if (needsId !== (positional.length === 1) || positional.length > 1) usage(needsId ? `${sub} takes one test id` : `${sub} takes no test id`); + if (sub === 'check') { + const junit = one('junit'); + const r = checkRegistry(io.cwd, { base: one('base'), suiteTests: junit ? readJunit(junit, runner).tests.length : undefined }, io.now); + for (const p of r.problems) io.out(`QUARANTINE_REFUSED: ${p}\n`); + io.out(`QUARANTINE_CHECK: live=${r.live} cap=${r.cap ?? 'none'} base=${one('base') ?? 'none'} verdict=${r.problems.length ? 'fail' : 'pass'}\n`); + return r.problems.length ? 1 : 0; + } + if (sub === 'report') { + const ledgers = flags.ledger ? readLedgers(flags.ledger) : undefined; + if (ledgers?.skipped) io.out(`QUARANTINE_LEDGER: skipped ${ledgers.skipped} unreadable line(s)\n`); + for (const l of reportLines(readRegistry(io.cwd, one('base')), io.now, ledgers?.records)) io.out(`${l}\n`); + return 0; + } + const id = positional[0]!; + const reg = readRegistry(io.cwd); + let line: string; + if (sub === 'add') { + const junit = one('junit') ?? refuse('add needs --junit <file> to count the suite and confirm the id exists'); + const suiteIds = readJunit(junit, runner).tests.map(t => t.id); + const e = addEntry(reg, id, { owner: one('owner'), ledger: one('ledger'), failureClass: one('class'), reason: one('reason'), evidence: flags.evidence ?? [], days: intFlag(one('days'), 'days'), suiteIds }, io.now); + line = `QUARANTINE_ADDED: ${safeText(id)} owner=${safeText(e.owner)} class=${e.failureClass} expires=${e.expiresAt}`; + } else if (sub === 'renew') { + const e = renewEntry(reg, id, { owner: one('owner'), evidence: flags.evidence ?? [], days: intFlag(one('days'), 'days') }, io.now); + line = `QUARANTINE_RENEWED: ${safeText(id)} owner=${safeText(e.renewals[0]!.owner)} expires=${e.expiresAt} (no further renewal)`; + } else { + const r = releaseEntry(reg, id, { reason: one('reason'), pr: one('pr') }, io.now); + line = `QUARANTINE_RELEASED: ${safeText(id)} pr=${safeText(r.pr)}`; + } + writeRegistry(io.cwd, reg); + io.out(`${line} registry=${REGISTRY_PATH}\n`); + return 0; +} diff --git a/test/flaky-run.test.ts b/test/flaky-run.test.ts index c890b5883..cd8f2e5b8 100644 --- a/test/flaky-run.test.ts +++ b/test/flaky-run.test.ts @@ -160,7 +160,7 @@ describe('runFlaky', () => { expect(r.outcomes.map(o => [o.status, o.attempts.join(',')])).toEqual([['flaky-pass', 'fail,pass']]); expect(r.ledger).toHaveLength(1); expect(r.ledger[0]).toMatchObject({ kind: 'flaky-pass', id: 'a.test.ts::::b', rerun_by: 'gstack-flaky', attempts: ['fail', 'pass'], excerpt: ['boom'] }); - expect(renderResult(r, '/l.jsonl').at(-1)).toBe('FLAKY_RESULT: tests=2 failed=0 flaky=1 reruns=1 command_exit=1 verdict=pass ledger=/l.jsonl'); + expect(renderResult(r, '/l.jsonl').at(-1)).toBe('FLAKY_RESULT: tests=2 failed=0 flaky=1 quarantined=0 reruns=1 command_exit=1 verdict=pass ledger=/l.jsonl'); expect(stepSummary(r)).toContain('a.test.ts::::b'); }); diff --git a/test/quarantine.test.ts b/test/quarantine.test.ts new file mode 100644 index 000000000..747afbd77 --- /dev/null +++ b/test/quarantine.test.ts @@ -0,0 +1,427 @@ +/** + * gstack-flaky quarantine (GSTA-158 P2): registry refusals, renew and release, + * `check --base` against git fixtures, expiry, the quarantine attempt budget + * under the runner's full green verdict (never a false green), missing ids, + * the registry read from the base branch, and `report` over ledgers. + */ +import { afterAll, describe, expect, test } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { spawnSync } from 'child_process'; +import { addEntry, checkRegistry, emptyRegistry, entryProblems, liveQuarantine, readLedgers, readRegistry, registryProblems, releaseEntry, renewEntry, reportLines, suiteCap, writeRegistry, type AddInput, type Registry } from '../lib/quarantine'; +import { renderResult, runFlaky } from '../lib/flaky-run'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const BIN = path.join(ROOT, 'bin', 'gstack-flaky'); +const TMP = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-quarantine-')); +const tmp = () => fs.mkdtempSync(path.join(TMP, 't-')); +afterAll(() => fs.rmSync(TMP, { recursive: true, force: true })); + +const NOW = new Date('2026-10-29T00:00:00Z'); +const DAY = 86_400_000; +const at = (days: number) => new Date(NOW.getTime() + days * DAY); +const ids = (n: number, file = 'a.test.ts') => Array.from({ length: n }, (_, i) => `${file}::::t${i}`); +const add = (reg: Registry, id: string, over: Partial<AddInput> = {}, now = NOW) => + addEntry(reg, id, { owner: 'garry', ledger: 'https://ledger/1', failureClass: 'timing', reason: 'slow CI disk', evidence: ['https://run/1'], suiteIds: ids(100), ...over }, now); +const refusal = (fn: () => unknown) => { try { fn(); return 'ok'; } catch (e) { return (e as Error).message; } }; + +describe('registry: add, renew, release', () => { + test('add refuses more than 14 days, no evidence, product or infra class, an unknown id, a duplicate and the cap', () => { + const reg = emptyRegistry(); + expect(refusal(() => add(reg, 'a.test.ts::::t0', { days: 15 }))).toContain('from 1 to 14'); + expect(refusal(() => add(reg, 'a.test.ts::::t0', { evidence: [] }))).toContain('no evidence'); + expect(refusal(() => add(reg, 'a.test.ts::::t0', { failureClass: 'product' }))).toContain('class product is refused'); + expect(refusal(() => add(reg, 'a.test.ts::::t0', { failureClass: 'infra' }))).toContain('class infra is refused'); + expect(refusal(() => add(reg, 'a.test.ts::::t0', { failureClass: 'flaky' }))).toContain('is not one of detector, harness, timing, model-latency'); + expect(refusal(() => add(reg, 'a.test.ts::::nope'))).toContain('is not a test in the --junit report'); + expect(refusal(() => add(reg, 'a.test.ts::::t0', { owner: ' ' }))).toContain('no owner'); + expect(Object.keys(reg.entries)).toEqual([]); + const e = add(reg, 'a.test.ts::::t0'); + expect(e).toMatchObject({ enteredAt: NOW.toISOString(), expiresAt: at(7).toISOString(), renewals: [], suiteTests: 100 }); + expect(add(reg, 'a.test.ts::::t1', { days: 14 }).expiresAt).toBe(at(14).toISOString()); + expect(refusal(() => add(reg, 'a.test.ts::::t0'))).toContain('already has an entry'); + const small = emptyRegistry(); + for (let i = 0; i < 5; i++) add(small, `a.test.ts::::t${i}`, { suiteIds: ids(10) }); + expect(refusal(() => add(small, 'a.test.ts::::t5', { suiteIds: ids(10) }))).toContain('the cap for a 10-test suite is 5'); + expect(refusal(() => add(small, 'a.test.ts::::t5', { suiteIds: ids(10) }, at(8)))).toBe('ok'); + }); + + test('the cap is 10% of the suite, at least 1, and 5 under 50 tests', () => { + expect([1, 10, 49, 50, 99, 100, 1000].map(suiteCap)).toEqual([5, 5, 5, 5, 9, 10, 100]); + }); + + test('renew once, with new evidence and the owner named again; a second renewal is refused', () => { + const reg = emptyRegistry(); + add(reg, 'a.test.ts::::t0'); + const id = 'a.test.ts::::t0'; + expect(refusal(() => renewEntry(reg, id, { evidence: ['https://run/2'] }, at(6)))).toContain('names the owner again'); + expect(refusal(() => renewEntry(reg, id, { owner: 'garry', evidence: ['https://run/1'] }, at(6)))).toContain('new evidence'); + expect(refusal(() => renewEntry(reg, id, { owner: 'garry', evidence: ['https://run/2'], days: 8 }, at(6)))).toContain('1 to 7 days'); + const e = renewEntry(reg, id, { owner: 'garry', evidence: ['https://run/2'] }, at(6)); + expect(e.expiresAt).toBe(at(14).toISOString()); + expect(entryProblems(id, e, at(6))).toEqual([]); + expect(refusal(() => renewEntry(reg, id, { owner: 'garry', evidence: ['https://run/3'] }, at(13)))).toContain('a second renewal is refused'); + const expired = emptyRegistry(); + add(expired, id); + expect(renewEntry(expired, id, { owner: 'sam', evidence: ['https://run/9'], days: 3 }, at(10)).expiresAt).toBe(at(13).toISOString()); + const twice = { ...e, renewals: [...e.renewals, { ...e.renewals[0]!, evidence: ['https://run/3'] }] }; + expect(entryProblems(id, twice, at(6)).join()).toContain('one renewal is allowed'); + }); + + test('release moves the entry to released with its reason and PR', () => { + const reg = emptyRegistry(); + add(reg, 'a.test.ts::::t0'); + expect(refusal(() => releaseEntry(reg, 'a.test.ts::::t0', { reason: 'fixed' }, at(2)))).toContain('--reason and --pr'); + const r = releaseEntry(reg, 'a.test.ts::::t0', { reason: 'fixed the wait', pr: 'https://github.com/o/r/pull/9' }, at(2)); + expect(reg.entries).toEqual({}); + expect(reg.released).toEqual([r]); + expect(r).toMatchObject({ id: 'a.test.ts::::t0', owner: 'garry', reason: 'fixed the wait', releasedAt: at(2).toISOString(), enteredAt: NOW.toISOString() }); + expect(refusal(() => releaseEntry(reg, 'a.test.ts::::t0', { reason: 'x', pr: 'y' }, at(2)))).toContain('no entry to release'); + }); + + test('expired and invalid entries are not quarantined; expiry prints the renew and release commands', () => { + const reg = emptyRegistry(); + add(reg, 'a.test.ts::::t0'); + expect([...liveQuarantine(reg, at(6)).ids]).toEqual(['a.test.ts::::t0']); + const later = liveQuarantine(reg, at(7)); + expect(later.ids.size).toBe(0); + expect(later.notices[0]).toContain('FLAKY_QUARANTINE_EXPIRED: a.test.ts::::t0 expired 2026-11-05 and is not quarantined'); + expect(later.notices[0]).toContain("renew: gstack-flaky quarantine renew 'a.test.ts::::t0' --owner"); + expect(later.notices[0]).toContain("release: gstack-flaky quarantine release 'a.test.ts::::t0' --reason"); + reg.entries['a.test.ts::::t1'] = { ...reg.entries['a.test.ts::::t0']!, expiresAt: at(30).toISOString() }; + reg.entries['a.test.ts::::t2'] = { ...reg.entries['a.test.ts::::t0']!, failureClass: 'infra' as never }; + const live = liveQuarantine(reg, at(1)); + expect([...live.ids]).toEqual(['a.test.ts::::t0']); + expect(live.notices.join('\n')).toContain('a.test.ts::::t1 is not quarantined: expires more than 14 days after it entered'); + expect(live.notices.join('\n')).toContain('a.test.ts::::t2 is not quarantined: class infra is refused'); + }); + + test('check enforces the cap and the entry rules on the whole registry', () => { + const reg = emptyRegistry(); + for (let i = 0; i < 5; i++) add(reg, `a.test.ts::::t${i}`, { suiteIds: ids(10) }); + expect(registryProblems(reg, NOW)).toEqual([]); + reg.entries['a.test.ts::::t5'] = { ...reg.entries['a.test.ts::::t0']! }; + expect(registryProblems(reg, NOW)).toEqual(['6 live entries, over the cap of 5 for a 10-test suite']); + expect(registryProblems(reg, NOW, 60)).toEqual([]); + expect(registryProblems(reg, at(8))).toEqual([]); + }); +}); + +const git = (dir: string, ...args: string[]) => { + const r = spawnSync('git', ['-c', 'user.name=t', '-c', 'user.email=t@example.com', '-c', 'commit.gpgsign=false', ...args], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }); + if (r.status !== 0) throw new Error(`git ${args.join(' ')}: ${r.stderr}`); + return r.stdout; +}; + +/** A repository whose main branch quarantines test/a.test.ts::::slow and has released test/a.test.ts::::old; HEAD is a change branch. */ +function fixtureRepo(now = NOW) { + const dir = tmp(); + git(dir, 'init', '-q', '-b', 'main'); + fs.mkdirSync(path.join(dir, 'test')); + fs.writeFileSync(path.join(dir, 'test', 'a.test.ts'), '// a\n'); + fs.writeFileSync(path.join(dir, 'test', 'b.test.ts'), '// b\n'); + const reg = emptyRegistry(); + const suiteIds = ['test/a.test.ts::::slow', 'test/a.test.ts::::old', 'test/b.test.ts::::other']; + add(reg, 'test/a.test.ts::::old', { suiteIds }, now); + releaseEntry(reg, 'test/a.test.ts::::old', { reason: 'fixed', pr: '#1' }, now); + add(reg, 'test/a.test.ts::::slow', { suiteIds }, now); + writeRegistry(dir, reg); + git(dir, 'add', '-A'); + git(dir, 'commit', '-qm', 'seed'); + git(dir, 'checkout', '-qb', 'change'); + return { dir, reg, suiteIds }; +} + +const clone = (r: Registry): Registry => JSON.parse(JSON.stringify(r)); + +describe('check --base against a git fixture', () => { + test('an unchanged registry and a proper release pass', () => { + const { dir, reg } = fixtureRepo(); + expect(checkRegistry(dir, { base: 'main' }, NOW).problems).toEqual([]); + const next = clone(reg); + releaseEntry(next, 'test/a.test.ts::::slow', { reason: 'fixed', pr: '#2' }, at(1)); + writeRegistry(dir, next); + expect(checkRegistry(dir, { base: 'main' }, at(1)).problems).toEqual([]); + }); + + test('a deleted entry, a removed release record and rewritten history are refused', () => { + const { dir, reg } = fixtureRepo(); + const deleted = clone(reg); + delete deleted.entries['test/a.test.ts::::slow']; + writeRegistry(dir, deleted); + expect(checkRegistry(dir, { base: 'main' }, NOW).problems.join('\n')).toContain('test/a.test.ts::::slow: removed without a release record'); + const unreleased = clone(reg); + unreleased.released = []; + writeRegistry(dir, unreleased); + expect(checkRegistry(dir, { base: 'main' }, NOW).problems.join('\n')).toContain('test/a.test.ts::::old: its release record of'); + const restarted = clone(reg); + restarted.entries['test/a.test.ts::::slow']!.enteredAt = at(1).toISOString(); + writeRegistry(dir, restarted); + expect(checkRegistry(dir, { base: 'main' }, at(1)).problems.join('\n')).toContain('enteredAt changed'); + }); + + test('an entry added in the same change that edits its test file is refused; without the edit it passes', () => { + const { dir, reg, suiteIds } = fixtureRepo(); + const next = clone(reg); + add(next, 'test/b.test.ts::::other', { suiteIds }); + writeRegistry(dir, next); + expect(checkRegistry(dir, { base: 'main' }, NOW).problems).toEqual([]); + fs.writeFileSync(path.join(dir, 'test', 'b.test.ts'), '// b, now broken\n'); + git(dir, 'commit', '-qam', 'break b and quarantine it'); + expect(checkRegistry(dir, { base: 'main' }, NOW).problems).toEqual([ + 'test/b.test.ts::::other: added in the same change that edits test/b.test.ts; a change cannot quarantine the test it touches (land the entry on its own, or fix the test)', + ]); + }); + + test('the bin: check exits 1 with QUARANTINE_REFUSED lines, and an unknown base exits 3', () => { + const { dir, reg } = fixtureRepo(new Date()); + const deleted = clone(reg); + delete deleted.entries['test/a.test.ts::::slow']; + writeRegistry(dir, deleted); + const r = spawnSync(BIN, ['quarantine', 'check', '--base', 'main'], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }); + expect(r.status).toBe(1); + expect(r.stdout).toContain('QUARANTINE_REFUSED: test/a.test.ts::::slow: removed without a release record'); + expect(r.stdout).toContain('verdict=fail'); + const missing = spawnSync(BIN, ['quarantine', 'check', '--base', 'no-such-branch'], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }); + expect([missing.status, missing.stderr.includes('(FLAKY_QUARANTINE_BASE)')]).toEqual([3, true]); + }); +}); + +/** A fake runner: the attempt number decides the report, exit status, output and signal. */ +function fakeRun(reports: (n: number, argv: string[]) => string | null, exits: (n: number) => number, outputs: (n: number) => string = () => 'Ran 3 tests across 1 file. [3ms]\n', signals: (n: number) => string | undefined = () => undefined) { + const calls: string[][] = []; + const spawn = async (argv: string[], _env: NodeJS.ProcessEnv, cwd: string) => { + calls.push(argv); + const xml = reports(calls.length, argv); + if (xml !== null) fs.writeFileSync(path.join(cwd, 'r.xml'), xml); + const signal = signals(calls.length); + return { exit: exits(calls.length), output: outputs(calls.length), ...(signal ? { signal } : {}) }; + }; + return { calls, spawn }; +} + +const suite = (cases: string, file = 'a.test.ts') => `<testsuites name="bun test"><testsuite name="${file}" file="${file}">${cases}</testsuite></testsuites>`; +const pass = (name: string) => `<testcase name="${name}" classname="" file="a.test.ts"/>`; +const fail = (name: string) => `<testcase name="${name}" classname="" file="a.test.ts"><failure message="boom"/></testcase>`; +const Q = new Set(['a.test.ts::::q']); +const run = (f: ReturnType<typeof fakeRun>, extra: Partial<Parameters<typeof runFlaky>[0]> = {}) => + runFlaky({ command: ['bun', 'test'], junit: 'r.xml', maxReruns: 3, cwd: tmp(), spawn: f.spawn, env: {}, now: () => NOW, quarantined: Q, ...extra }); + +describe('the quarantine attempt budget under the full green verdict', () => { + test('a quarantined test that fails all 3 attempts is a hard break', async () => { + const f = fakeRun(() => suite(pass('a') + fail('q')), () => 1); + const r = await run(f); + expect(f.calls).toHaveLength(3); + expect(r.exitCode).toBe(1); + expect(r.outcomes).toMatchObject([{ status: 'hard-break', attempts: ['fail', 'fail', 'fail'], reason: 'failed all 3 attempts: a hard break, not a flake' }]); + expect(r.ledger.map(l => [l.kind, l.id, l.rerun_scope])).toEqual([['hard-break', 'a.test.ts::::q', 'command']]); + expect(renderResult(r, '/l').at(-1)).toBe('FLAKY_RESULT: tests=2 failed=1 flaky=0 quarantined=1 reruns=2 command_exit=1 verdict=fail ledger=/l'); + }); + + test('it passes on attempt 3 only on the runner\'s full green verdict, listed as quarantined and not as a pass', async () => { + const f = fakeRun(n => suite(pass('a') + (n < 3 ? fail('q') : pass('q'))), n => n < 3 ? 1 : 0); + const r = await run(f); + expect(f.calls).toHaveLength(3); + expect(r.exitCode).toBe(0); + expect(r.outcomes).toMatchObject([{ status: 'quarantined-pass', attempts: ['fail', 'fail', 'pass'] }]); + expect(r.ledger[0]).toMatchObject({ kind: 'quarantined-pass', attempts: ['fail', 'fail', 'pass'] }); + expect(renderResult(r, '/l')).toEqual(['FLAKY: quarantined-pass a.test.ts::::q attempts=fail,fail,pass', 'FLAKY_RERUN: scope=command (the default; --rerun-scope test reruns the failed test alone)', + 'FLAKY_RESULT: tests=2 failed=0 flaky=0 quarantined=1 reruns=2 command_exit=1 verdict=pass ledger=/l']); + }); + + test('false-green regression: exit 1 from outside the report never turns green because only a quarantined test failed', async () => { + const greenReportLaterExit1 = fakeRun(n => suite(pass('a') + (n === 1 ? fail('q') : pass('q'))), () => 1); + const a = await run(greenReportLaterExit1); + expect(greenReportLaterExit1.calls).toHaveLength(2); + expect(a.exitCode).toBe(1); + expect(a.outcomes).toMatchObject([{ status: 'quarantined-fail', attempts: ['fail', 'pass'], reason: 'rerun: the command exited 1' }]); + const alwaysExit1 = fakeRun(() => suite(pass('a') + fail('q')), () => 1, () => 'All files | % Funcs | % Lines |\nRan 2 tests across 1 file.\n'); + const b = await run(alwaysExit1, { rerunScope: 'test' }); + expect(alwaysExit1.calls.every(c => c.join(' ') === 'bun test')).toBe(true); + expect([b.exitCode, b.outcomes[0]!.status]).toEqual([1, 'hard-break']); + }); + + test('the bin keeps a coverage-style exit 1 red while the only reported failure is quarantined', () => { + const dir = tmp(); + git(dir, 'init', '-q', '-b', 'main'); + const reg = emptyRegistry(); + add(reg, 'a.test.ts::::q', { suiteIds: ['a.test.ts::::a', 'a.test.ts::::q'] }, new Date()); + writeRegistry(dir, reg); + const runner = path.join(dir, 'runner.sh'); + fs.writeFileSync(runner, ['#!/usr/bin/env bash', 'echo x >> count', `printf '%s' '${suite(pass('a') + fail('q'))}' > r.xml`, 'exit 1'].join('\n'), { mode: 0o755 }); + const r = spawnSync(BIN, ['run', '--junit', 'r.xml', '--ledger', path.join(dir, 'l.jsonl'), '--', runner], { cwd: dir, encoding: 'utf-8', timeout: 30_000, env: { ...process.env, GITHUB_ACTIONS: '', GITHUB_STEP_SUMMARY: '' } }); + expect(r.status).toBe(1); + expect(r.stdout).toContain('FLAKY: hard-break a.test.ts::::q attempts=fail,fail,fail'); + expect(r.stdout).toContain('quarantined=1 reruns=2 command_exit=1 verdict=fail'); + expect(fs.readFileSync(path.join(dir, 'count'), 'utf-8').trim().split('\n')).toHaveLength(3); + }); + + test('non-quarantined failures keep one rerun; a new non-quarantined failure ends the budget', async () => { + const both = fakeRun(n => suite((n === 1 ? fail('a') : fail('a')) + (n === 1 ? fail('q') : pass('q'))), () => 1); + const r = await run(both); + expect(both.calls).toHaveLength(2); + expect(r.outcomes.map(o => [o.test.id, o.status])).toEqual([['a.test.ts::::a', 'rerun-fail'], ['a.test.ts::::q', 'quarantined-fail']]); + const newFailure = fakeRun(n => suite((n === 1 ? pass('a') : fail('a')) + fail('q')), () => 1); + const s = await run(newFailure); + expect(newFailure.calls).toHaveLength(2); + expect([s.exitCode, s.outcomes[0]!.status]).toEqual([1, 'quarantined-fail']); + }); + + test('a flaky test and a quarantined test both clear on the attempt that is fully green', async () => { + const f = fakeRun(n => suite((n === 1 ? fail('a') : pass('a')) + (n < 3 ? fail('q') : pass('q'))), n => n < 3 ? 1 : 0); + const r = await run(f); + expect(f.calls).toHaveLength(3); + expect(r.exitCode).toBe(0); + expect(r.outcomes.map(o => [o.status, o.attempts.join(',')])).toEqual([['flaky-pass', 'fail,pass,pass'], ['quarantined-pass', 'fail,fail,pass']]); + }); + + test('a killed attempt or a missing first-run test ends the budget red', async () => { + const killed = fakeRun(() => suite(pass('a') + fail('q')), n => n === 1 ? 1 : 128, undefined, n => n === 2 ? 'SIGTERM' : undefined); + expect((await run(killed)).exitCode).toBe(1); + expect(killed.calls).toHaveLength(2); + const shrunk = fakeRun(n => n === 1 ? suite(pass('a') + fail('q')) : suite(fail('q')), () => 1); + expect((await run(shrunk)).exitCode).toBe(1); + expect(shrunk.calls).toHaveLength(2); + }); + + test('a quarantined test that passes is listed, ledgered for the exit rule, and the run is green', async () => { + const r = await run(fakeRun(() => suite(pass('a') + pass('q')), () => 0)); + expect(r).toMatchObject({ exitCode: 0, reruns: 0, quarantined: 1 }); + expect(r.ledger.map(l => [l.kind, l.attempts])).toEqual([['quarantined-pass', ['pass']]]); + expect(renderResult(r, '/l').at(-1)).toContain('failed=0 flaky=0 quarantined=1'); + }); + + test('quarantined failures do not count against --max-reruns', async () => { + const f = fakeRun(n => suite(pass('a') + (n === 1 ? fail('q') : pass('q'))), n => n === 1 ? 1 : 0); + expect((await run(f, { maxReruns: 0 })).exitCode).toBe(0); + }); + + test('--rerun-scope test gives the quarantined test the same budget on the targeted path', async () => { + const f = fakeRun((n, argv) => argv.length === 2 ? suite(pass('a') + fail('q')) : suite(n < 3 ? fail('q') : pass('q')), n => n < 3 ? 1 : 0); + const r = await run(f, { rerunScope: 'test' }); + expect(f.calls.slice(1)).toEqual([['bun', 'test', 'a.test.ts', '--test-name-pattern', '^q$'], ['bun', 'test', 'a.test.ts', '--test-name-pattern', '^q$']]); + expect(r.exitCode).toBe(0); + expect(r.outcomes).toMatchObject([{ status: 'quarantined-pass', attempts: ['fail', 'fail', 'pass'] }]); + const notQ = fakeRun((n, argv) => argv.length === 2 ? suite(pass('q') + fail('a')) : suite(fail('a')), () => 1); + expect((await run(notQ, { rerunScope: 'test' })).exitCode).toBe(1); + expect(notQ.calls).toHaveLength(2); + }); + + test('an expired entry gives no budget: one rerun, as for any test', async () => { + const reg = emptyRegistry(); + add(reg, 'a.test.ts::::q', { suiteIds: ['a.test.ts::::q'] }); + const f = fakeRun(n => suite(pass('a') + (n < 3 ? fail('q') : pass('q'))), n => n < 3 ? 1 : 0); + const r = await run(f, { quarantined: liveQuarantine(reg, at(8)).ids }); + expect(f.calls).toHaveLength(2); + expect([r.exitCode, r.outcomes[0]!.status, r.quarantined]).toEqual([1, 'rerun-fail', 0]); + }); +}); + +describe('missing quarantined ids', () => { + test('a quarantined id missing from a run that included its file blocks', async () => { + const f = fakeRun(() => suite(pass('a') + pass('b')), () => 0); + const r = await run(f, { quarantined: new Set(['a.test.ts::::gone']) }); + expect(f.calls).toHaveLength(1); + expect(r.exitCode).toBe(1); + expect(r.outcomes).toMatchObject([{ status: 'quarantined-fail', reason: 'quarantined, and missing from a run that included its file' }]); + expect(r.ledger[0]).toMatchObject({ kind: 'quarantined-fail', id: 'a.test.ts::::gone' }); + }); + + test('a run that did not include its file (a shard, a diff selection) does not block', async () => { + const r = await run(fakeRun(() => suite(pass('a')), () => 0), { quarantined: new Set(['b.test.ts::::gone', 'a.test.ts:: chromium ::x']) }); + expect(r).toMatchObject({ exitCode: 0, outcomes: [], quarantined: 0 }); + }); +}); + +describe('the registry read from the base branch', () => { + test('readRegistry --base sees the base copy, not the change\'s own entry', () => { + const { dir, reg, suiteIds } = fixtureRepo(); + const next = clone(reg); + add(next, 'test/b.test.ts::::other', { suiteIds }); + writeRegistry(dir, next); + expect(Object.keys(readRegistry(dir).entries).sort()).toEqual(['test/a.test.ts::::slow', 'test/b.test.ts::::other']); + expect(Object.keys(readRegistry(dir, 'main').entries)).toEqual(['test/a.test.ts::::slow']); + }); + + test('the bin: a change that quarantines its own failing test gets no budget with --base', () => { + const dir = tmp(); + git(dir, 'init', '-q', '-b', 'main'); + git(dir, 'commit', '-q', '--allow-empty', '-m', 'seed'); + git(dir, 'checkout', '-qb', 'change'); + fs.writeFileSync(path.join(dir, 'seed.xml'), suite(pass('a') + pass('b'))); + const addRun = spawnSync(BIN, ['quarantine', 'add', 'a.test.ts::::b', '--owner', 'garry', '--ledger', 'https://l/1', '--class', 'timing', '--reason', 'slow', '--evidence', 'https://run/1', '--junit', 'seed.xml'], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }); + expect([addRun.status, addRun.stdout.startsWith('QUARANTINE_ADDED: a.test.ts::::b owner=garry class=timing')]).toEqual([0, true]); + const runner = path.join(dir, 'runner.sh'); + fs.writeFileSync(runner, [ + '#!/usr/bin/env bash', + 'n=$(cat count 2>/dev/null || echo 0); echo $((n+1)) > count', + `if [ "$n" -lt 2 ]; then printf '%s' '${suite(pass('a') + fail('b'))}' > r.xml; exit 1; fi`, + `printf '%s' '${suite(pass('a') + pass('b'))}' > r.xml`, + ].join('\n'), { mode: 0o755 }); + const flaky = (extra: string[]) => { + fs.rmSync(path.join(dir, 'count'), { force: true }); + return spawnSync(BIN, ['run', '--junit', 'r.xml', '--ledger', path.join(dir, 'l.jsonl'), ...extra, '--', runner], { cwd: dir, encoding: 'utf-8', timeout: 30_000, env: { ...process.env, GITHUB_ACTIONS: '', GITHUB_STEP_SUMMARY: '' } }); + }; + const own = flaky([]); + expect([own.status, own.stdout.includes('FLAKY: quarantined-pass a.test.ts::::b attempts=fail,fail,pass')]).toEqual([0, true]); + const fromBase = flaky(['--base', 'main']); + expect([fromBase.status, fromBase.stdout.includes('FLAKY: rerun-fail a.test.ts::::b')]).toEqual([1, true]); + expect(fs.readFileSync(path.join(dir, 'count'), 'utf-8').trim()).toBe('2'); + const badBase = flaky(['--base', 'no-such-branch']); + expect([badBase.status, badBase.stderr.includes('(FLAKY_QUARANTINE_BASE)')]).toEqual([3, true]); + }); +}); + +describe('quarantine CLI and report', () => { + test('add, renew, release through the bin; refusals exit 1 and usage exits 2', () => { + const dir = tmp(); + fs.writeFileSync(path.join(dir, 'seed.xml'), suite(pass('a') + pass('b'))); + const q = (...args: string[]) => spawnSync(BIN, ['quarantine', ...args], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }); + const base = ['--owner', 'garry', '--ledger', 'https://l/1', '--reason', 'slow', '--evidence', 'https://run/1', '--junit', 'seed.xml']; + const product = q('add', 'a.test.ts::::b', ...base, '--class', 'product'); + expect([product.status, product.stderr.includes('class product is refused'), product.stderr.includes('(FLAKY_QUARANTINE_REFUSED)')]).toEqual([1, true, true]); + expect(q('add', 'a.test.ts::::b', ...base, '--class', 'harness').status).toBe(0); + expect(q('renew', 'a.test.ts::::b', '--owner', 'garry', '--evidence', 'https://run/2').status).toBe(0); + expect(q('renew', 'a.test.ts::::b', '--owner', 'garry', '--evidence', 'https://run/3').status).toBe(1); + expect(q('check').stdout).toContain('QUARANTINE_CHECK: live=1 cap=5 base=none verdict=pass'); + expect(q('release', 'a.test.ts::::b', '--reason', 'fixed', '--pr', '#7').status).toBe(0); + const saved = JSON.parse(fs.readFileSync(path.join(dir, '.gstack', 'quarantine.json'), 'utf-8')) as Registry; + expect([Object.keys(saved.entries), saved.released.map(r => [r.id, r.pr, r.renewals.length])]).toEqual([[], [['a.test.ts::::b', '#7', 1]]]); + expect(q('add', 'a.test.ts::::b', '--bogus', 'x').status).toBe(2); + expect(q('frobnicate').status).toBe(2); + }); + + test('report: expired first, then soonest-expiring; the exit rule needs 10 consecutive clean passes from the ledgers', () => { + const reg = emptyRegistry(); + add(reg, 'a.test.ts::::t0', { days: 14 }); + add(reg, 'a.test.ts::::t1', { days: 2 }); + add(reg, 'a.test.ts::::t2', { days: 1 }, at(-5)); + const without = reportLines(reg, NOW); + expect(without.map(l => l.split(' ').slice(0, 2).join(' '))).toEqual(['FLAKY_QUARANTINE_EXPIRED: a.test.ts::::t2', 'QUARANTINE: a.test.ts::::t1', 'QUARANTINE: a.test.ts::::t0', 'QUARANTINE_REPORT: live=2']); + expect(without[1]).toContain('days_left=2'); + expect(without[1]).toContain("EXPIRING (renew: gstack-flaky quarantine renew 'a.test.ts::::t1'"); + expect(without.at(-1)).toBe('QUARANTINE_REPORT: live=2 expired=1 expiring=1 ready=unchecked exit_rule=not checked (no --ledger given)'); + const dir = tmp(); + fs.mkdirSync(path.join(dir, 'run-1', 'flaky-junit-ledger-e2e'), { recursive: true }); + const lineFor = (id: string, i: number, kind = 'quarantined-pass', attempts = ['pass']) => JSON.stringify({ ts: new Date(NOW.getTime() + i * 1000).toISOString(), kind, id, attempts }); + const t0 = [lineFor('a.test.ts::::t0', 0, 'hard-break', ['fail', 'fail', 'fail']), ...Array.from({ length: 10 }, (_, i) => lineFor('a.test.ts::::t0', i + 1))]; + const t1 = [...Array.from({ length: 9 }, (_, i) => lineFor('a.test.ts::::t1', i)), lineFor('a.test.ts::::t1', 9, 'quarantined-pass', ['fail', 'pass'])]; + fs.writeFileSync(path.join(dir, 'run-1', 'flaky-junit-ledger-e2e', 'flaky-junit-ledger.jsonl'), [...t0, ...t1, 'not json'].join('\n') + '\n'); + const ledgers = readLedgers([dir]); + expect([ledgers.records.length, ledgers.skipped]).toEqual([21, 1]); + const withLedger = reportLines(reg, NOW, ledgers.records); + expect(withLedger.find(l => l.includes('t0 '))).toContain("recent=pass,pass,pass,pass,pass clean_streak=10 READY_TO_RELEASE: gstack-flaky quarantine release 'a.test.ts::::t0'"); + expect(withLedger.find(l => l.includes('t1 '))).toContain('recent=pass,pass,pass,pass,flaky clean_streak=0'); + expect(withLedger.at(-1)).toBe('QUARANTINE_REPORT: live=2 expired=1 expiring=1 ready=1 exit_rule=checked (10 consecutive clean passes)'); + }); + + test('every FLAKY_QUARANTINE_ code is in the docs table, and the docs say the registries share no schema with CASE_QUARANTINE', () => { + const doc = fs.readFileSync(path.join(ROOT, 'docs', 'flaky.md'), 'utf-8'); + const src = fs.readFileSync(path.join(ROOT, 'lib', 'quarantine.ts'), 'utf-8') + fs.readFileSync(path.join(ROOT, 'bin', 'gstack-flaky'), 'utf-8'); + const codes = [...new Set(src.match(/FLAKY_QUARANTINE_[A-Z_]+/g))]; + expect(codes.length).toBeGreaterThan(4); + expect(codes.filter(c => !doc.includes(`| \`${c}\` |`))).toEqual([]); + expect(doc).toContain('CASE_QUARANTINE'); + }); +});