mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-03 23:37:53 +01:00
feat(flaky): quarantine registry for repeat flakes (GSTA-158 P2)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed
1 parent
c668d584ad
commit
0246abd2ba
11 files changed
+1164
-39
No files matched your search
@@ -10,14 +10,22 @@
|
||||
- **`gstack-flaky ids --junit <file>`** prints the test ids a report holds (`<file>::<project>::<describe > title>`).
|
||||
- Exit codes: 0 pass (flaky passes included), 1 a test failed, 2 usage, 3 no usable report, a crash or a killed runner, each with a `FLAKY_<CODE>` and a `fix:` line. A missing, stale, empty or malformed report never passes, and an existing `--junit` file that is not a JUnit report is refused rather than deleted.
|
||||
|
||||
### Added
|
||||
- **Quarantine registry, `gstack-flaky quarantine add|renew|release|check|report`** (`.gstack/quarantine.json`, GSTA-158 P2): a test that keeps flaking while someone fixes it gets a larger attempt budget instead of a skip. When every test an attempt failed is quarantined, the whole command may run again, up to 3 attempts; the run is green only when an attempt reaches the runner's own full green verdict, so a coverage threshold or other exit outside the report stays red. A quarantined test that fails all 3 is a `hard-break`, a quarantined id missing from a run that included its file is red, and quarantined tests are listed (`FLAKY_RESULT ... quarantined=<n>`) and ledgered (`quarantined-pass`, `quarantined-fail`, `hard-break`) on every run, never counted as passes.
|
||||
- **Entries are owned and expire**: owner, ledger link, class (`detector`, `harness`, `timing`, `model-latency`; `product` and `infra` refused), evidence, 7 days by default and 14 at most, one renewal of up to 7 days with new evidence. An expired entry stops applying with no edit and prints the renew and release commands. Live entries are capped at 10% of the suite (at least 1, and 5 under 50 tests).
|
||||
- **`run --base <ref>`** reads the registry from the base branch, so a change cannot quarantine the test it breaks; **`check --base`** refuses an entry added in the change that edits its test's file, an entry removed without a release record, a removed release record and rewritten history. **`report --ledger <file|dir>`** lists entries soonest-expiring first and marks those with 10 consecutive clean passes ready to release. gstack's own `CASE_QUARANTINE` / `EVAL_POLICY` is unchanged and shares no schema with it.
|
||||
|
||||
### Docs
|
||||
- `docs/flaky.md`: the GitHub Actions setup, runners, rerun scope and its preconditions, rules, output and exit codes. `docs/TESTING_INTERNALS.md#flake-ledger` points other repositories to it.
|
||||
- `docs/flaky.md#quarantine`: quarantine commands, rules, expiry, cap, classes, the `CASE_QUARANTINE` separation and the daily-pass `gh run download` + `report` loop; `captain/templates/automation-flaky-pass.md` and `docs/captain.md` run `report` and fix expiring entries first.
|
||||
|
||||
### Tests
|
||||
- `test/flaky-run.test.ts` with captured Playwright 1.64 and bun 1.4 JUnit reports under `test/fixtures/junit/`: flaky pass, real red, rerun scopes, coverage-threshold and early-stop runs, unreported failures, cancelled runs, report deletion and ledger redaction.
|
||||
- `test/quarantine.test.ts`: add refusals (cap, more than 14 days, no evidence, product or infra class, unknown id), one renewal, release, `check --base` against git fixtures, expiry, a hard break across 3 attempts, the false-green regression (exit 1 outside the report while only a quarantined test fails stays red), missing ids with their file present or absent, and the registry read from the base branch.
|
||||
|
||||
### Gate integrity
|
||||
- `test/helpers/free-fixtures-data.ts`: registers `test/fixtures/junit/**` to `test/flaky-run.test.ts`. No assertion, threshold or skip changed.
|
||||
- `test/flaky-run.test.ts`: the exact `FLAKY_RESULT` line it expects gains the new `quarantined=0` field. No other assertion, threshold or skip changed.
|
||||
|
||||
## [1.91.320.0] - 2026-10-29
|
||||
|
||||
|
||||
+40
-4
@@ -9,11 +9,13 @@
|
||||
import * as fs from 'fs';
|
||||
import { DEFAULT_MAX_RERUNS, appendLedger, ledgerPath, renderResult, runFlaky, safeText, stepSummary, workflowEscape } from '../lib/flaky-run';
|
||||
import { JunitError, readJunit, type JunitRunner } from '../lib/junit';
|
||||
import { liveQuarantine, QuarantineError, quarantineMain, readRegistry } from '../lib/quarantine';
|
||||
|
||||
const USAGE = `Usage:
|
||||
gstack-flaky run --junit <file.xml> [--runner playwright|bun|generic] [--max-reruns <n>]
|
||||
[--rerun-scope command|test] [--ledger <file>] -- <test command...>
|
||||
[--rerun-scope command|test] [--ledger <file>] [--base <ref>] -- <test command...>
|
||||
gstack-flaky ids --junit <file> [--runner playwright|bun|generic]
|
||||
gstack-flaky quarantine add|renew|release|check|report ... (gstack-flaky quarantine --help)
|
||||
gstack-flaky --help
|
||||
|
||||
run: runs the command (an argv, no shell), reads the JUnit report it writes at
|
||||
@@ -31,11 +33,18 @@ holds the first run's report afterwards; reruns' reports sit beside it as
|
||||
<name>.rerun-<n>.xml. The command runs with PLAYWRIGHT_JUNIT_INCLUDE_RETRIES=1
|
||||
so Playwright's own retries are reported.
|
||||
|
||||
Quarantine: tests listed in .gstack/quarantine.json (read from --base <ref> when
|
||||
given, so PR CI passes its target branch) still run and never count as passes.
|
||||
When every test an attempt failed is a live quarantined test, the whole command
|
||||
may run again, up to 3 attempts; the run is green only on an attempt with the
|
||||
runner's full green verdict, and a quarantined test that fails all 3 is a hard
|
||||
break (red). A quarantined id missing from a run that included its file is red.
|
||||
|
||||
ids: prints the id of every test in a report (<file>::<project>::<title>).
|
||||
|
||||
Output: one FLAKY: line per test that failed at least once, FLAKY_RERUN:
|
||||
scope=command (<why>) when the rerun repeated the whole command, then
|
||||
FLAKY_RESULT: tests=<n> failed=<n> flaky=<n> reruns=<n> command_exit=<n> verdict=pass|fail ledger=<file|none>
|
||||
FLAKY_RESULT: tests=<n> failed=<n> flaky=<n> quarantined=<n> reruns=<n> command_exit=<n> verdict=pass|fail ledger=<file|none>
|
||||
Exit: 0 pass (flaky passes included), 1 a test failed, 2 usage,
|
||||
3 no usable report, a crash or a killed runner (FLAKY_<CODE> and a fix: line on stderr).
|
||||
`;
|
||||
@@ -48,6 +57,10 @@ const FIX: Record<string, string> = {
|
||||
FLAKY_COMMAND_KILLED: 'the runner was killed by a signal (a cancelled job, an OOM kill); nothing was rerun, rerun the job',
|
||||
FLAKY_COMMAND_NOT_STARTED: 'check the command after -- (it runs without a shell; wrap it as bash -c \'<script> "$@"\' _ when it needs one)',
|
||||
FLAKY_UNREPORTED_FAILURES: 'the runner counted failures its JUnit omits (a test file that failed to load, an error outside any test); fix that first, it is a real red',
|
||||
FLAKY_QUARANTINE_INVALID: 'fix .gstack/quarantine.json: it must be {"schema":1,"entries":{...},"released":[...]} (gstack-flaky quarantine check names the problem)',
|
||||
FLAKY_QUARANTINE_BASE: 'fetch the base branch before the run (git fetch origin <branch>, or actions/checkout fetch-depth: 0) and pass it as --base origin/<branch>',
|
||||
FLAKY_QUARANTINE_REFUSED: 'read the reason above; docs/flaky.md#quarantine lists the rules',
|
||||
FLAKY_QUARANTINE_USAGE: 'gstack-flaky quarantine --help',
|
||||
};
|
||||
|
||||
function die(msg: string, code = 2): never {
|
||||
@@ -60,8 +73,13 @@ function coded(e: JunitError): never {
|
||||
process.exit(3);
|
||||
}
|
||||
|
||||
function quarantineFailed(e: QuarantineError): never {
|
||||
process.stderr.write(`gstack-flaky: ${e.message} (${e.code})\nfix: ${FIX[e.code] ?? 'see docs/flaky.md#quarantine'}\n`);
|
||||
process.exit(e.exitCode);
|
||||
}
|
||||
|
||||
function parse(argv: string[]) {
|
||||
const o: { junit?: string; runner?: JunitRunner; maxReruns: number; rerunScope: 'command' | 'test'; ledger?: string; command: string[] } = { maxReruns: DEFAULT_MAX_RERUNS, rerunScope: 'command', command: [] };
|
||||
const o: { junit?: string; runner?: JunitRunner; maxReruns: number; rerunScope: 'command' | 'test'; ledger?: string; base?: string; command: string[] } = { maxReruns: DEFAULT_MAX_RERUNS, rerunScope: 'command', command: [] };
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i]!;
|
||||
const val = () => { const v = argv[++i]; if (v === undefined) die(`${a} needs a value`); return v; };
|
||||
@@ -80,6 +98,7 @@ function parse(argv: string[]) {
|
||||
if (v !== 'command' && v !== 'test') die(`--rerun-scope must be command or test, got ${JSON.stringify(v)}`);
|
||||
o.rerunScope = v;
|
||||
} else if (a === '--ledger') o.ledger = val();
|
||||
else if (a === '--base') o.base = val();
|
||||
else die(`unknown option ${a}\n${USAGE}`);
|
||||
}
|
||||
return o;
|
||||
@@ -88,6 +107,16 @@ function parse(argv: string[]) {
|
||||
const [sub, ...rest] = process.argv.slice(2);
|
||||
if (!sub || sub === '--help' || sub === '-h') { process.stdout.write(USAGE); process.exit(sub ? 0 : 2); }
|
||||
|
||||
if (sub === 'quarantine') {
|
||||
try {
|
||||
process.exit(quarantineMain(rest, { cwd: process.cwd(), out: s => process.stdout.write(s), now: new Date() }));
|
||||
} catch (e) {
|
||||
if (e instanceof QuarantineError) quarantineFailed(e);
|
||||
if (e instanceof JunitError) coded(e);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
if (sub === 'ids') {
|
||||
const o = parse(rest);
|
||||
if (!o.junit) die('ids needs --junit <file>');
|
||||
@@ -107,9 +136,15 @@ try { existing = fs.statSync(o.junit, { throwIfNoEntry: false }) && fs.readFileS
|
||||
}
|
||||
if (existing && existing.trim() && !/<testsuites?[\s>/]/.test(existing)) die(`--junit ${JSON.stringify(o.junit)} exists and is not a JUnit report; gstack-flaky deletes that path before every attempt, so it refuses to touch it`);
|
||||
if (!o.command.length) die('run needs the test command after --');
|
||||
let quarantined: Set<string>;
|
||||
try {
|
||||
const live = liveQuarantine(readRegistry(process.cwd(), o.base), new Date());
|
||||
for (const n of live.notices) process.stdout.write(`${n}\n`);
|
||||
quarantined = live.ids;
|
||||
} catch (e) { if (e instanceof QuarantineError) quarantineFailed(e); throw e; }
|
||||
let result;
|
||||
try {
|
||||
result = await runFlaky({ command: o.command, junit: o.junit, runner: o.runner, maxReruns: o.maxReruns, rerunScope: o.rerunScope });
|
||||
result = await runFlaky({ command: o.command, junit: o.junit, runner: o.runner, maxReruns: o.maxReruns, rerunScope: o.rerunScope, quarantined });
|
||||
} catch (e) { if (e instanceof JunitError) coded(e); throw e; }
|
||||
const ledger = ledgerPath(o.ledger);
|
||||
appendLedger(result.ledger, ledger, s => process.stderr.write(`gstack-flaky: ${s}\n`));
|
||||
@@ -118,6 +153,7 @@ const summary = process.env.GITHUB_STEP_SUMMARY && stepSummary(result);
|
||||
if (summary) { try { fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY!, summary); } catch { /* a summary is a convenience, never the verdict */ } }
|
||||
if (process.env.GITHUB_ACTIONS === 'true') {
|
||||
for (const f of result.outcomes.filter(x => x.status === 'flaky-pass')) process.stdout.write(`::warning title=flaky test::${workflowEscape(safeText(f.test.id))} passed on attempt ${f.attempts.length}\n`);
|
||||
for (const f of result.outcomes.filter(x => x.status === 'hard-break')) process.stdout.write(`::error title=quarantined test hard break::${workflowEscape(safeText(f.test.id))} failed all ${f.attempts.length} attempts\n`);
|
||||
}
|
||||
if (result.code) {
|
||||
process.stderr.write(`gstack-flaky: ${result.message} (${result.code})\nfix: ${FIX[result.code] ?? 'see docs/flaky.md'}\n`);
|
||||
|
||||
@@ -16,6 +16,13 @@ ledger's Open section:
|
||||
in-run rerun turned green (date, case, tier, PR and head, failing attempt, passing
|
||||
attempt, one-line symptom from the failure log).
|
||||
2. The free suites' retried cases in the same window, from the runners' retry output.
|
||||
3. For repositories that run `gstack-flaky`: download the window's `flaky-junit-ledger-*`
|
||||
artifacts (`gh run download <run-id> --pattern 'flaky-junit-ledger-*' --dir ledgers/<run-id>`)
|
||||
and run `gstack-flaky quarantine report --ledger ledgers`. Fix its expired and EXPIRING
|
||||
entries first: fix the test, or renew once with new evidence and the owner named, never
|
||||
a second time. Release each READY_TO_RELEASE entry with
|
||||
`gstack-flaky quarantine release <id> --reason <why> --pr <url>` in a PR of its own.
|
||||
A hard-break line is a real red, not a flake: fix it like any failing test.
|
||||
|
||||
Never add a case that failed twice (that is a real failure, not a flake), and never a
|
||||
harness, infra or incomplete red. Then group the Open section by case, worst repeat
|
||||
@@ -33,5 +40,5 @@ spend of every paid run you made.
|
||||
```
|
||||
|
||||
What it reads: the ledger, the gate runs' flake-records artifacts, the free suites'
|
||||
retry output, the crew rules file. What it starts: one fix PR per cause, each READY'd to
|
||||
retry output, the `gstack-flaky` ledgers and quarantine report, the crew rules file. What it starts: one fix PR per cause, each READY'd to
|
||||
the controller. What it never does: rerun a gate, change a threshold, or merge.
|
||||
+3
-1
@@ -332,7 +332,9 @@ A scheduled automation starts one lane a day with the
|
||||
runs' flake-records artifacts and the free suites' retried cases, writes one ledger line
|
||||
per event, groups by case, and fixes the worst repeat first with a PR that shows the
|
||||
failure red before green. It never reruns a gate and never touches a threshold, trial
|
||||
count or case kind. On Capy, create it with the automation tool, scheduled in the owner's
|
||||
count or case kind. Where a repository runs `gstack-flaky`, the lane also reads
|
||||
`gstack-flaky quarantine report --ledger <downloaded ledgers>` and fixes expiring quarantine
|
||||
entries before anything else ([flaky.md](flaky.md#the-daily-flaky-pass)). On Capy, create it with the automation tool, scheduled in the owner's
|
||||
timezone, running as a thread in the crew so its report lands in `reports/`.
|
||||
|
||||
The lane's side of the same rule: in CI the gate reruns a case that failed on its own
|
||||
|
||||
+117
-8
@@ -6,7 +6,9 @@ rerun is a **flaky pass**: the job stays green and the flake is recorded on a le
|
||||
the daily flaky pass can fix it at the source. A test that fails twice is red.
|
||||
This is CREW rule 16 ("run the failed test again and note that it is flaky")
|
||||
for any runner that writes JUnit: Playwright, `bun test`, jest, vitest and
|
||||
pytest. gstack's own free suite keeps its file-level retry
|
||||
pytest. A test that keeps flaking while it is fixed can be
|
||||
[quarantined](#quarantine): it still runs, with up to 3 attempts, and never counts
|
||||
as a pass. gstack's own free suite keeps its file-level retry
|
||||
(`GSTACK_FREE_RETRY_FLAKY`, [Flake ledger](TESTING_INTERNALS.md#flake-ledger)).
|
||||
|
||||
## Get started in GitHub Actions
|
||||
@@ -74,6 +76,104 @@ It cannot see a coverage threshold configured without console output (for
|
||||
example `coverageReporter = ["lcov"]` in `bunfig.toml`), which is why this
|
||||
scope is opt-in.
|
||||
|
||||
## Quarantine
|
||||
|
||||
A test that keeps flaking while someone fixes it can be **quarantined** in
|
||||
`.gstack/quarantine.json` at the repository root. Quarantine is a larger
|
||||
attempt budget, never a skip: the test still runs on every run, is listed as
|
||||
quarantined in the output, and never counts as a pass.
|
||||
|
||||
**How a quarantined test is judged.** The verdict rule from
|
||||
[Rerun scope](#rerun-scope) does not change: an attempt counts only when the
|
||||
runner itself reports a full green run. When every test an attempt failed is a
|
||||
live quarantined test, and nothing else is wrong with that attempt (no kill, no
|
||||
failure outside the report, no first-run test missing), the whole command may
|
||||
run again, up to 3 attempts in total. The run is green only when one of those
|
||||
attempts is fully green. A quarantined test that fails all 3 is a **hard
|
||||
break**, and the run is red. A test that is not quarantined keeps its one
|
||||
rerun. A red attempt never turns green because its only reported failure is a
|
||||
quarantined test, because the runner's exit status can come from things the
|
||||
report does not show (a coverage threshold, a fail-on-flaky policy, a global
|
||||
setup). With `--rerun-scope test` the quarantined test gets the same 3
|
||||
attempts on the targeted path, with the same fallback to the whole command.
|
||||
When the runner retries on its own (Playwright `retries`), its attempts are
|
||||
the budget and nothing is rerun again.
|
||||
|
||||
**Missing results.** A quarantined id missing from a run that included its
|
||||
file (and, for Playwright, its project) is red. A run that did not include the
|
||||
file, such as a shard or a diff selection, is not.
|
||||
|
||||
**Read from the base branch.** `run --base <ref>` reads the registry from
|
||||
that commit instead of the working tree. Pull-request CI passes its target
|
||||
branch, so a change cannot quarantine the test it breaks. `check --base`
|
||||
compares against the merge base, so check out full history for it:
|
||||
|
||||
```yaml
|
||||
- uses: actions/checkout@v4
|
||||
with: { fetch-depth: 0 }
|
||||
- name: Quarantine registry
|
||||
run: '"$RUNNER_TEMP/gstack/bin/gstack-flaky" quarantine check --base "origin/${{ github.base_ref }}"'
|
||||
- name: E2E
|
||||
run: |
|
||||
"$RUNNER_TEMP/gstack/bin/gstack-flaky" run --base "origin/${{ github.base_ref }}" --junit results/junit.xml -- npx playwright test --reporter=line,junit
|
||||
```
|
||||
|
||||
### Commands
|
||||
|
||||
| Command | What it does |
|
||||
|---|---|
|
||||
| `gstack-flaky quarantine add <id> --owner <who> --ledger <url> --class <class> --reason <text> --evidence <url>… --junit <file> [--days <n>]` | Adds an entry that expires in `--days` (default 7, at most 14). `--junit` counts the suite for the cap and confirms the id exists; `gstack-flaky ids` prints the ids. |
|
||||
| `gstack-flaky quarantine renew <id> --owner <who> --evidence <url> [--days <n>]` | One renewal of up to 7 days from the later of now and the current expiry, with evidence not already on the entry and the owner named again. A second renewal is refused. |
|
||||
| `gstack-flaky quarantine release <id> --reason <text> --pr <url>` | Moves the entry to `released` with the date, the reason and the PR. |
|
||||
| `gstack-flaky quarantine check [--base <ref>] [--junit <file>]` | Validates every entry and the cap; with `--base`, also what this change did since the merge base. Exits 1 with one `QUARANTINE_REFUSED:` line per problem. |
|
||||
| `gstack-flaky quarantine report [--ledger <file\|dir>…] [--base <ref>]` | Lists expired entries, then live entries soonest-expiring first with days left. With downloaded ledgers it adds recent outcomes and marks entries with 10 consecutive clean passes `READY_TO_RELEASE`; without `--ledger` it says the exit rule was not checked. |
|
||||
|
||||
### Rules
|
||||
|
||||
- **Every entry is owned and expires.** An entry holds `owner`, `ledger`
|
||||
(the ledger line or issue tracking the fix), `failureClass`, `reason`,
|
||||
`evidence`, `enteredAt`, `expiresAt`, `renewals` and `suiteTests` (the
|
||||
suite size `add` counted). It lasts 7 days by default and at most 14 from
|
||||
entry, plus one renewal of at most 7 days.
|
||||
- **Expired means not quarantined.** An expired entry stops applying without
|
||||
an edit: `run` treats the test as an ordinary test and prints
|
||||
`FLAKY_QUARANTINE_EXPIRED` with the renew and release commands. An entry
|
||||
that breaks a rule (for example one that lasts 30 days) is ignored the same
|
||||
way, with `FLAKY_QUARANTINE_IGNORED`.
|
||||
- **Cap.** Live entries are capped at 10% of the suite's tests, at least 1,
|
||||
and 5 for a suite under 50 tests. `add` refuses past the cap; `check`
|
||||
enforces it against `--junit` when given, else against the smallest
|
||||
`suiteTests` on a live entry.
|
||||
- **Classes.** `detector`, `harness`, `timing` and `model-latency`.
|
||||
`product` is refused (a product defect is fixed, not quarantined), and so is
|
||||
`infra` (an infrastructure red is rerun or fixed as infrastructure).
|
||||
- **`check --base` refuses** an entry added in the same change that edits its
|
||||
test's file, an entry that disappears without a release record, a release
|
||||
record that was removed, and a changed `enteredAt` or rewritten renewal
|
||||
history.
|
||||
- **Exit rule.** 10 consecutive clean passes (`quarantined-pass` with no
|
||||
failed attempt) on the ledgers since the entry was added. Release the entry
|
||||
with the fix PR.
|
||||
- **Separate from `CASE_QUARANTINE`.** gstack's own paid-eval quarantine
|
||||
(`CASE_QUARANTINE` and `EVAL_POLICY`,
|
||||
[TESTING_INTERNALS.md](TESTING_INTERNALS.md)) is a different registry with
|
||||
its own rules for eval cases. The two share no schema, and `gstack-flaky`
|
||||
neither reads nor writes it.
|
||||
|
||||
### The daily flaky pass
|
||||
|
||||
Download the ledger artifacts of the last week's runs and read them with
|
||||
`report`; fix expired and `EXPIRING` entries first (fix the test, or renew
|
||||
once with new evidence), release `READY_TO_RELEASE` entries, then fix the
|
||||
worst repeat on the ledger:
|
||||
|
||||
```bash
|
||||
for id in $(gh run list --workflow e2e.yml --limit 50 --json databaseId -q '.[].databaseId'); do
|
||||
gh run download "$id" --pattern 'flaky-junit-ledger-*' --dir "ledgers/$id" || true
|
||||
done
|
||||
gstack-flaky quarantine report --ledger ledgers
|
||||
```
|
||||
|
||||
## Rules
|
||||
|
||||
- **One rerun, small failures only.** A failed test gets one rerun when the
|
||||
@@ -101,8 +201,9 @@ scope is opt-in.
|
||||
- **Ids.** A test is `<file>::<project>::<describe > title>`, with the project
|
||||
empty outside Playwright, so a test in two browser projects is two tests.
|
||||
`gstack-flaky ids --junit <file>` prints them.
|
||||
- **Ledger.** One JSON line per flaky pass (`kind: flaky-pass`) and per test
|
||||
that failed its rerun (`rerun-fail`), with the tree, commit, branch, run URL,
|
||||
- **Ledger.** One JSON line per flaky pass (`kind: flaky-pass`), per test
|
||||
that failed its rerun (`rerun-fail`) and per quarantined test on every run
|
||||
(`quarantined-pass`, `quarantined-fail`, `hard-break`), with the tree, commit, branch, run URL,
|
||||
attempts, the rerun's scope (`test` or `command`) and the first lines of
|
||||
the error. Ids, paths and failure text go through gstack's redaction engine
|
||||
first; when the scan cannot run, they are withheld. The path is `--ledger`,
|
||||
@@ -113,15 +214,17 @@ scope is opt-in.
|
||||
|
||||
## Output and exit codes
|
||||
|
||||
One `FLAKY: <flaky-pass|fail|rerun-fail> <id> attempts=<…> (<reason>)` line per
|
||||
test that failed at least once, `FLAKY_RERUN: scope=command (<why>)` when the
|
||||
rerun repeated the whole command, then
|
||||
`FLAKY_RESULT: tests=<n> failed=<n> flaky=<n> reruns=<n> command_exit=<n> verdict=pass|fail ledger=<file|none>`.
|
||||
One `FLAKY: <flaky-pass|fail|rerun-fail|quarantined-pass|quarantined-fail|hard-break> <id> attempts=<…> (<reason>)`
|
||||
line per test that failed at least once and per quarantined test,
|
||||
`FLAKY_RERUN: scope=command (<why>)` when the rerun repeated the whole command, then
|
||||
`FLAKY_RESULT: tests=<n> failed=<n> flaky=<n> quarantined=<n> reruns=<n> command_exit=<n> verdict=pass|fail ledger=<file|none>`.
|
||||
`failed` counts every outcome but `flaky-pass` and `quarantined-pass`; `quarantined`
|
||||
counts the quarantined tests' outcomes, which are never counted as passes.
|
||||
|
||||
| Exit | Meaning |
|
||||
|---|---|
|
||||
| 0 | Every test passed, flaky passes included |
|
||||
| 1 | A test failed (after its rerun, or with no rerun), including `FLAKY_UNREPORTED_FAILURES` |
|
||||
| 1 | A test failed (after its rerun, or with no rerun), including `FLAKY_UNREPORTED_FAILURES`, a quarantined test's hard break or missing result, and a refused `quarantine` command |
|
||||
| 2 | Usage error, including a `--junit` path that is not a `.xml` file, or an existing file there that is not a readable JUnit report |
|
||||
| 3 | No usable report, a crash, or a killed runner (a code below on stderr) |
|
||||
|
||||
@@ -135,3 +238,9 @@ rerun repeated the whole command, then
|
||||
| `FLAKY_COMMAND_NOT_STARTED` | 3 | The command could not be started | Check the argv after `--` |
|
||||
| `FLAKY_UNREPORTED_FAILURES` | 1 | The runner counted failures or errors its JUnit omits | Fix the load error or the error outside any test: a real red |
|
||||
| `FLAKY_LEDGER_WRITE_FAILED` | — | The ledger could not be appended (warning) | Check the path; the verdict stands |
|
||||
| `FLAKY_QUARANTINE_INVALID` | 3 | `.gstack/quarantine.json` is not valid JSON or not a schema 1 registry | Fix the file; `quarantine check` names the problem |
|
||||
| `FLAKY_QUARANTINE_BASE` | 3 | `--base` does not resolve to a commit (exit 2 for a value starting with `-`), or `check` finds no merge base | Fetch the base branch; `check --base` needs `fetch-depth: 0` |
|
||||
| `FLAKY_QUARANTINE_REFUSED` | 1 | `add`, `renew` or `release` broke a rule | Read the reason; [Quarantine](#quarantine) lists the rules |
|
||||
| `FLAKY_QUARANTINE_USAGE` | 2 | An unknown `quarantine` command or option, or a missing value | `gstack-flaky quarantine --help` |
|
||||
| `FLAKY_QUARANTINE_EXPIRED` | — | Notice: an expired entry no longer applies | Fix the test, renew once, or release; the line prints both commands |
|
||||
| `FLAKY_QUARANTINE_IGNORED` | — | Notice: an entry breaks a rule and does not apply | Fix the entry; `quarantine check` names the rule |
|
||||
@@ -18,7 +18,7 @@ Beyond the slash-command skills, gstack ships standalone CLIs for workflows that
|
||||
| `gstack-review-log` | **Review-pass receipts** — `--start <skill>` captures the working-tree fingerprint before a diff review; `'<JSON>' --finish <token>` consumes that single-use, repository/branch/skill-scoped receipt. Binding requires matching start/end content and reviewer-reported `completed:true` and `converged:true`; it is not independent proof that a model read the source. `--check-shared-libs <token>` reads a current finding as JSON on stdin and checks prior Skip decisions against the actual branch, capture and eligible raw source blobs. It returns `reusable:false` when proof is missing or unsafe, including older records without logger-versioned coverage. Final review logging computes shared-code fingerprints and coverage rather than trusting supplied proof. |
|
||||
| `gstack-review-read` | **Review freshness** — emits review records with computed `review_freshness.status` and `reason`: CURRENT, STALE, or UNVERIFIED for diff reviews. `/ship` and `/land-and-deploy` use the same grade; a matching commit alone never certifies a diff review. [Dashboard rules](skills.md#review-readiness-dashboard). |
|
||||
| `gstack-evidence` | **Verification-evidence ledger** — `run --label <lane> -- <cmd>` transparently wraps any test command (the child's exit code always passes through) and records what ran against which working-tree fingerprint; `check` grades each label FRESH/STALE/MISSING with `--expect-cmd`, `--max-age`, and `--allow-paths` binding. /ship and /land-and-deploy cite fresh evidence instead of re-running suites. Per-run logs are 0600, capped at 2MB, pruned after 30 days; the ledger and logs stay machine-local by design. |
|
||||
| `gstack-flaky` | **Flaky-test rerun for any JUnit runner** — `run --junit <file.xml> -- <command>` runs your test command (Playwright, `bun test`, jest, vitest, pytest) and, when at most `--max-reruns` (3) tests failed and the runner made no retries of its own, reruns the whole command once (`--rerun-scope test` opts in to rerunning only the failed tests). A test that fails and then passes on that rerun is a flaky pass: the run stays green and one redacted line per flaky pass or failed rerun goes on a ledger (`GSTACK_FLAKE_LEDGER_JUNIT`, default `<state root>/flake-ledger-junit.jsonl`). A missing or malformed report, a crash or a killed runner never passes. `ids --junit <file>` prints a report's test ids. Exit 0 pass, 1 a test failed, 2 usage, 3 no usable report. [Guide](flaky.md). |
|
||||
| `gstack-flaky` | **Flaky-test rerun for any JUnit runner** — `run --junit <file.xml> -- <command>` runs your test command (Playwright, `bun test`, jest, vitest, pytest) and, when at most `--max-reruns` (3) tests failed and the runner made no retries of its own, reruns the whole command once (`--rerun-scope test` opts in to rerunning only the failed tests). A test that fails and then passes on that rerun is a flaky pass: the run stays green and one redacted line per flaky pass or failed rerun goes on a ledger (`GSTACK_FLAKE_LEDGER_JUNIT`, default `<state root>/flake-ledger-junit.jsonl`). A missing or malformed report, a crash or a killed runner never passes. `quarantine add|renew|release|check|report` keeps `.gstack/quarantine.json`: a quarantined test still runs, gets up to 3 attempts of the whole command, passes only on a full green attempt and is never counted as a pass (`run --base <ref>` reads the registry from the base branch). `ids --junit <file>` prints a report's test ids. Exit 0 pass, 1 a test failed, 2 usage, 3 no usable report. [Guide](flaky.md). |
|
||||
| `gstack-pr-wait` | **PR event watcher** — `gstack-pr-wait <pr> [--repo owner/name] [--until green\|ready\|any-change] [--interval 20] [--timeout 60] [--exclude CHECK,...]` polls one pull request through `gh` and prints one stdout line per real event (a check failed with its link, all checks finished, a comment or review from someone other than the author, a review bot finished on the head, the base moved and its new commits touch this PR's files, BEHIND or DIRTY, merged or closed), with progress on stderr, so a monitor wakes the agent once per event instead of on a timer. `green` gates on every check on the head (an advisory bot's review is never waited for, its checks gate like any other), as `gstack-ci-gate` does; a check the base branch requires (rulesets and classic branch protection) is pending until it appears, never green by omission; two identical polls confirm a check set is complete; `--exclude` leaves a named non-required check out of the verdict as `gstack-ci-gate --exclude` does (a required name is refused and gates). Exit 0 when the condition is met, 1 when it needs action or `--timeout` passes (`PR_WAIT_BLOCKED`), 2 on bad input, a second watcher for the same PR or a local error (`PR_WAIT_REFUSED`). State under `<state root>/pr-wait/`, baseline discarded after 6 hours. Review bots are configuration: `gstack-config set pr_wait_gating_bots greptile,macroscope`, `pr_wait_advisory_bots coderabbit` (advisory bots are reported, never waited on). `/land-and-deploy` Step 3 is one call of it. Design ported from tstack's pr-wait (MIT, Theo Browne; NOTICE.md). [Exit codes](troubleshooting.md#gstack-pr-wait). |
|
||||
| `gstack-issue-guard` | **Tracker-text trust envelope** — fetches GitHub issue/PR text (`issue <n>`, `pr-body`, `pr-comments`, or `--stdin`) and wraps it in a labeled envelope so agents treat it as data: injection-shaped lines get labeled even through fullwidth and invisible-character evasion, and forged envelope banners are defused. Every tracker-text ingress in gstack routes through it, enforced by a CI scanner. |
|
||||
| `gstack-ios-qa-daemon` | **iOS QA daemon** — Mac-side broker between an agent and a connected iPhone or iPad over USB CoreDevice. Loopback by default; `--tailnet` opens a Tailscale-facing listener with identity-gated capability tiers. Single-instance via flock on `~/.gstack/ios-qa-daemon.pid`. See [docs/howto-ios-testing-with-gstack.md](howto-ios-testing-with-gstack.md). |
|
||||
|
||||
+107
-23
@@ -29,15 +29,23 @@
|
||||
* omits (bun files that fail to load), the run is red and nothing is rerun.
|
||||
* The first report is restored at the --junit path when the run ends; each
|
||||
* rerun's report is kept beside it as <name>.rerun-<n>.xml.
|
||||
* - Quarantine (lib/quarantine.ts) is a larger attempt budget, never a skip:
|
||||
* a live quarantined test is listed and ledgered on every run and never
|
||||
* counts as a pass. When every test a whole-command attempt failed is
|
||||
* quarantined and nothing else is wrong, the command may run again, up to
|
||||
* QUARANTINE_ATTEMPTS in total; the run is green only on an attempt with the
|
||||
* runner's full green verdict, so a quarantined failure never turns a red
|
||||
* attempt green. One that fails every attempt is a hard break. A quarantined
|
||||
* id missing from a run that included its file is red.
|
||||
*/
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { spawn as spawnChild, spawnSync } from 'child_process';
|
||||
import { finalAttempt, JunitError, readJunit, type JunitReport, type JunitRunner, type JunitTest } from './junit';
|
||||
import { finalAttempt, JunitError, parseTestId, readJunit, type Attempt, type JunitReport, type JunitRunner, type JunitTest } from './junit';
|
||||
import { redactFindingSpans } from './redact-engine';
|
||||
import { resolveStateRoot } from './state-root';
|
||||
|
||||
export type LedgerKind = 'flaky-pass' | 'rerun-fail';
|
||||
export type LedgerKind = 'flaky-pass' | 'rerun-fail' | 'quarantined-pass' | 'quarantined-fail' | 'hard-break';
|
||||
|
||||
export interface LedgerLine {
|
||||
ts: string;
|
||||
@@ -64,6 +72,8 @@ export interface RunOptions {
|
||||
maxReruns: number;
|
||||
/** 'command' (default): rerun the whole command. 'test': rerun the failed test alone when the first run is proven complete. */
|
||||
rerunScope?: 'command' | 'test';
|
||||
/** Live quarantined test ids (lib/quarantine.ts liveQuarantine): up to QUARANTINE_ATTEMPTS attempts, never a skip. */
|
||||
quarantined?: ReadonlySet<string>;
|
||||
cwd?: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
/** Runs one attempt; resolves with its exit status and the tail of its combined output. */
|
||||
@@ -73,7 +83,7 @@ export interface RunOptions {
|
||||
|
||||
export interface TestOutcome {
|
||||
test: JunitTest;
|
||||
status: 'flaky-pass' | 'fail' | 'rerun-fail';
|
||||
status: 'flaky-pass' | 'fail' | 'rerun-fail' | 'quarantined-pass' | 'quarantined-fail' | 'hard-break';
|
||||
attempts: string[];
|
||||
reason?: string;
|
||||
}
|
||||
@@ -88,6 +98,8 @@ export interface RunResult {
|
||||
commandExit: number;
|
||||
outcomes: TestOutcome[];
|
||||
reruns: number;
|
||||
/** Outcomes of quarantined tests (listed, never counted as passes). */
|
||||
quarantined: number;
|
||||
/** Why the rerun was the whole command (the default, or why a requested targeted rerun was refused); absent for targeted reruns. */
|
||||
wholeRerun?: string;
|
||||
ledger: LedgerLine[];
|
||||
@@ -243,7 +255,7 @@ function rerunProblem(again: Attempted & { report?: JunitReport }, first: JunitR
|
||||
}
|
||||
|
||||
/** A rerun clears a test only when its command exited 0 and the test ran once and passed on its first attempt. */
|
||||
function rerunVerdict(again: Attempted & { report?: JunitReport; error?: JunitError }, t: JunitTest, problem?: string): { status: 'flaky-pass' | 'rerun-fail'; attempts: string[]; reason?: string } {
|
||||
function rerunVerdict(again: Attempted & { report?: JunitReport; error?: JunitError }, t: JunitTest, problem?: string): { status: 'flaky-pass' | 'rerun-fail'; attempts: Attempt[]; reason?: string } {
|
||||
const match = again.report?.tests.find(x => x.id === t.id);
|
||||
if (!match) return { status: 'rerun-fail', attempts: [...t.attempts, 'fail'], reason: again.error ? `rerun: ${again.error.message}` : 'rerun: the report does not contain this test' };
|
||||
const attempts = [...t.attempts, ...match.attempts];
|
||||
@@ -254,11 +266,47 @@ function rerunVerdict(again: Attempted & { report?: JunitReport; error?: JunitEr
|
||||
return { status: 'flaky-pass', attempts };
|
||||
}
|
||||
|
||||
export const QUARANTINE_ATTEMPTS = 3;
|
||||
|
||||
const PASSING: ReadonlySet<TestOutcome['status']> = new Set(['flaky-pass', 'quarantined-pass']);
|
||||
|
||||
type Ran = Attempted & { report?: JunitReport; error?: JunitError };
|
||||
|
||||
/** The runner's own full green verdict for a whole-command attempt: exit 0, no failed or missing test, nothing counted outside the report. */
|
||||
function fullGreen(again: Ran, first: JunitReport): boolean {
|
||||
return !!again.report && !again.signal && again.exit === 0 && !rerunProblem(again, first, true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a red whole-command attempt may be followed by another: only within
|
||||
* the quarantine budget, and only when every test it failed is a live
|
||||
* quarantined test that ran once and nothing else is wrong (a kill, a missing
|
||||
* report, failures outside the report, a first-run test missing).
|
||||
*/
|
||||
function mayRunAgain(again: Ran, first: JunitReport, quarantined: ReadonlySet<string>, attemptsSoFar: number): boolean {
|
||||
if (attemptsSoFar >= QUARANTINE_ATTEMPTS || again.signal || !again.report) return false;
|
||||
const failedNow = again.report.tests.filter(x => finalAttempt(x) === 'fail');
|
||||
if (!failedNow.length || !failedNow.every(x => quarantined.has(x.id) && x.executions === 1 && x.attempts.length === 1)) return false;
|
||||
if (unreportedFailures(first.runner, again.output, failedNow.length)) return false;
|
||||
const seen = new Set(again.report.tests.map(x => x.id));
|
||||
return first.tests.every(x => seen.has(x.id));
|
||||
}
|
||||
|
||||
/** A test's attempts over the first run and the given whole-command reruns; a rerun that did not report it counts as a failure. */
|
||||
const history = (t: JunitTest, runs: Ran[]): Attempt[] =>
|
||||
[...t.attempts, ...runs.flatMap(r => r.report?.tests.find(x => x.id === t.id)?.attempts ?? ['fail' as const])];
|
||||
|
||||
/** A quarantined test passes only when it never failed or an attempt with the runner's full verdict cleared it. */
|
||||
const quarantinedStatus = (attempts: string[], cleared: boolean): 'quarantined-pass' | 'quarantined-fail' | 'hard-break' =>
|
||||
cleared || !attempts.includes('fail') ? 'quarantined-pass'
|
||||
: attempts.length >= QUARANTINE_ATTEMPTS && attempts.every(a => a === 'fail') ? 'hard-break' : 'quarantined-fail';
|
||||
|
||||
export async function runFlaky(opts: RunOptions): Promise<RunResult> {
|
||||
const cwd = opts.cwd ?? process.cwd();
|
||||
const env = { ...(opts.env ?? process.env), PLAYWRIGHT_JUNIT_INCLUDE_RETRIES: '1' };
|
||||
const now = opts.now ?? (() => new Date());
|
||||
const empty = { tests: 0, outcomes: [], reruns: 0, ledger: [] };
|
||||
const quarantined = opts.quarantined ?? new Set<string>();
|
||||
const empty = { tests: 0, outcomes: [], reruns: 0, quarantined: 0, ledger: [] };
|
||||
const first = await attempt(opts, opts.command, cwd, env);
|
||||
if (first.signal) return { exitCode: 3, code: 'FLAKY_COMMAND_KILLED', message: `the command was killed by ${first.signal}`, commandExit: first.exit, ...empty };
|
||||
if (!first.report) return { exitCode: 3, code: first.error!.code, message: first.error!.message, commandExit: first.exit, ...empty };
|
||||
@@ -282,15 +330,39 @@ export async function runFlaky(opts: RunOptions): Promise<RunResult> {
|
||||
const prov = provenance(cwd, env);
|
||||
const line = (t: JunitTest, kind: LedgerKind, by: LedgerLine['rerun_by'], attempts: string[], scope?: LedgerLine['rerun_scope']): LedgerLine =>
|
||||
({ ts: now().toISOString(), runner: 'junit', kind, id: safeText(t.id), file: safeText(t.file), rerun_by: by, ...(scope ? { rerun_scope: scope } : {}), attempts, ...prov, excerpt: safeExcerpt(t.excerpt) });
|
||||
const ledger: LedgerLine[] = runnerFlaky.map(t => line(t, 'flaky-pass', 'runner', t.attempts));
|
||||
const outcomes: TestOutcome[] = runnerFlaky.map(t => ({ test: t, status: 'flaky-pass' as const, attempts: t.attempts }));
|
||||
const isQ = (t: JunitTest) => quarantined.has(t.id);
|
||||
const outcomes: TestOutcome[] = [];
|
||||
const ledger: LedgerLine[] = [];
|
||||
for (const t of runnerFlaky.filter(x => !isQ(x))) {
|
||||
outcomes.push({ test: t, status: 'flaky-pass', attempts: t.attempts });
|
||||
ledger.push(line(t, 'flaky-pass', 'runner', t.attempts));
|
||||
}
|
||||
const settleQ = (t: JunitTest, attempts: string[], cleared: boolean, reason?: string, scope?: LedgerLine['rerun_scope']) => {
|
||||
const status = quarantinedStatus(attempts, cleared);
|
||||
const why = status === 'hard-break' ? `failed all ${attempts.length} attempts: a hard break, not a flake` : status === 'quarantined-fail' ? reason : undefined;
|
||||
outcomes.push({ test: t, status, attempts, ...(why ? { reason: why } : {}) });
|
||||
ledger.push(line(t, status, 'gstack-flaky', attempts, scope));
|
||||
};
|
||||
const settle = (t: JunitTest, attempts: string[], reason: string) => isQ(t) ? settleQ(t, attempts, false, reason) : outcomes.push({ test: t, status: 'fail', attempts, reason });
|
||||
const present = new Set(report.tests.map(t => t.id));
|
||||
const included = new Set(report.tests.map(t => `${t.file}::${t.project}`));
|
||||
for (const id of quarantined) {
|
||||
const p = parseTestId(id);
|
||||
if (!p || present.has(id) || !included.has(`${p.file}::${p.project}`)) continue;
|
||||
const t: JunitTest = { id, file: p.file, project: p.project, path: p.title.split(' > '), attempts: [], executions: 0, excerpt: [] };
|
||||
outcomes.push({ test: t, status: 'quarantined-fail', attempts: [], reason: 'quarantined, and missing from a run that included its file' });
|
||||
ledger.push(line(t, 'quarantined-fail', 'gstack-flaky', []));
|
||||
}
|
||||
const others = failed.filter(t => !isQ(t)).length;
|
||||
const eligible: JunitTest[] = [];
|
||||
for (const t of failed) {
|
||||
if (t.executions > 1) outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: `it ran ${t.executions} times in this report and one failed; not rerun` });
|
||||
else if (t.attempts.length > 1) outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: `the runner already retried (${t.attempts.length} attempts)` });
|
||||
else if (failed.length > opts.maxReruns) outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: `${failed.length} tests failed, more than --max-reruns ${opts.maxReruns}; not rerun` });
|
||||
const why = t.executions > 1 ? `it ran ${t.executions} times in this report and one failed; not rerun`
|
||||
: t.attempts.length > 1 ? `the runner already retried (${t.attempts.length} attempts)`
|
||||
: !isQ(t) && others > opts.maxReruns ? `${others} tests failed, more than --max-reruns ${opts.maxReruns}; not rerun` : undefined;
|
||||
if (why) settle(t, t.attempts, why);
|
||||
else eligible.push(t);
|
||||
}
|
||||
const passedQ = report.tests.filter(t => isQ(t) && finalAttempt(t) !== 'fail' && t.attempts.some(a => a !== 'skip'));
|
||||
const wholeRerun = !eligible.length ? undefined
|
||||
: opts.rerunScope !== 'test' ? 'the default; --rerun-scope test reruns the failed test alone'
|
||||
: incompleteRun(report.runner, first.output, first.exit, opts.command, runnerFlaky.length) ?? undefined;
|
||||
@@ -304,30 +376,42 @@ export async function runFlaky(opts: RunOptions): Promise<RunResult> {
|
||||
return again;
|
||||
};
|
||||
const record = (t: JunitTest, v: ReturnType<typeof rerunVerdict>, scope: 'test' | 'command') => {
|
||||
if (isQ(t)) return settleQ(t, v.attempts, v.status === 'flaky-pass', v.reason, scope);
|
||||
outcomes.push({ test: t, status: v.status, attempts: v.attempts, ...(v.reason ? { reason: v.reason } : {}) });
|
||||
ledger.push(line(t, v.status, 'gstack-flaky', v.attempts, scope));
|
||||
};
|
||||
const runs: Ran[] = [];
|
||||
try {
|
||||
if (wholeRerun && outcomes.some(o => o.status === 'fail')) {
|
||||
for (const t of eligible) outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: `not rerun: the run is already red, and a rerun would repeat the whole command (${wholeRerun})` });
|
||||
if (wholeRerun && outcomes.some(o => !PASSING.has(o.status))) {
|
||||
for (const t of eligible) settle(t, t.attempts, `not rerun: the run is already red, and a rerun would repeat the whole command (${wholeRerun})`);
|
||||
} else if (wholeRerun) {
|
||||
const again = await rerun(opts.command);
|
||||
const problem = rerunProblem(again, report, true);
|
||||
for (const t of eligible) record(t, rerunVerdict(again, t, problem), 'command');
|
||||
do {
|
||||
runs.push(await rerun(opts.command));
|
||||
} while (!fullGreen(runs.at(-1)!, report) && mayRunAgain(runs.at(-1)!, report, quarantined, 1 + runs.length));
|
||||
const decider = runs.at(-1)!;
|
||||
const problem = rerunProblem(decider, report, true);
|
||||
for (const t of eligible) record(t, rerunVerdict(decider, { ...t, attempts: history(t, runs.slice(0, -1)) }, problem), 'command');
|
||||
} else if (eligible.length) {
|
||||
let interrupted = false;
|
||||
for (const t of eligible) {
|
||||
if (interrupted) { outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: 'not rerun: an earlier rerun was killed' }); continue; }
|
||||
const again = await rerun([...opts.command, ...rerunSelection(report.runner as 'playwright' | 'bun', t)]);
|
||||
interrupted = !!again.signal;
|
||||
record(t, rerunVerdict(again, t, rerunProblem(again, report, false)), 'test');
|
||||
if (interrupted) { settle(t, t.attempts, 'not rerun: an earlier rerun was killed'); continue; }
|
||||
let v: ReturnType<typeof rerunVerdict> = { status: 'rerun-fail', attempts: t.attempts };
|
||||
for (let budget = isQ(t) ? QUARANTINE_ATTEMPTS - t.attempts.length : 1; budget > 0 && v.status !== 'flaky-pass' && !interrupted; budget--) {
|
||||
const again = await rerun([...opts.command, ...rerunSelection(report.runner as 'playwright' | 'bun', t)]);
|
||||
interrupted = !!again.signal;
|
||||
v = rerunVerdict(again, { ...t, attempts: v.attempts }, rerunProblem(again, report, false));
|
||||
}
|
||||
record(t, v, 'test');
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (reruns) fs.writeFileSync(junit, firstXml);
|
||||
}
|
||||
const red = outcomes.some(o => o.status !== 'flaky-pass');
|
||||
return { exitCode: red ? 1 : 0, tests: report.tests.length, commandExit: first.exit, outcomes, reruns, ...(wholeRerun && reruns ? { wholeRerun } : {}), ledger };
|
||||
const green = runs.length ? fullGreen(runs.at(-1)!, report) : reruns > 0 || first.exit === 0;
|
||||
for (const t of passedQ) settleQ(t, history(t, runs), green, 'the attempt that decided the run was not a full green run', runs.length ? 'command' : undefined);
|
||||
const red = !green || outcomes.some(o => !PASSING.has(o.status));
|
||||
const q = outcomes.filter(o => o.status === 'quarantined-pass' || o.status === 'quarantined-fail' || o.status === 'hard-break').length;
|
||||
return { exitCode: red ? 1 : 0, tests: report.tests.length, commandExit: first.exit, outcomes, reruns, quarantined: q, ...(wholeRerun && reruns ? { wholeRerun } : {}), ledger };
|
||||
}
|
||||
|
||||
export function ledgerPath(flag: string | undefined, env: NodeJS.ProcessEnv = process.env): string {
|
||||
@@ -352,9 +436,9 @@ export function renderResult(r: RunResult, ledger: string): string[] {
|
||||
const out: string[] = [];
|
||||
for (const o of r.outcomes) out.push(`FLAKY: ${o.status} ${safeText(o.test.id)} attempts=${o.attempts.join(',')}${o.reason ? ` (${o.reason})` : ''}`);
|
||||
const flaky = r.outcomes.filter(o => o.status === 'flaky-pass').length;
|
||||
const failed = r.outcomes.length - flaky;
|
||||
const failed = r.outcomes.filter(o => !PASSING.has(o.status)).length;
|
||||
if (r.wholeRerun) out.push(`FLAKY_RERUN: scope=command (${r.wholeRerun})`);
|
||||
out.push(`FLAKY_RESULT: tests=${r.tests} failed=${failed} flaky=${flaky} reruns=${r.reruns} command_exit=${r.commandExit} verdict=${r.exitCode === 0 ? 'pass' : 'fail'} ledger=${r.ledger.length ? ledger : 'none'}`);
|
||||
out.push(`FLAKY_RESULT: tests=${r.tests} failed=${failed} flaky=${flaky} quarantined=${r.quarantined} reruns=${r.reruns} command_exit=${r.commandExit} verdict=${r.exitCode === 0 ? 'pass' : 'fail'} ledger=${r.ledger.length ? ledger : 'none'}`);
|
||||
return out;
|
||||
}
|
||||
|
||||
|
||||
@@ -137,6 +137,14 @@ function excerptOf(node: XmlNode | undefined): string[] {
|
||||
|
||||
export const testId = (file: string, project: string, path: string[]) => `${file}::${project}::${path.join(' > ')}`;
|
||||
|
||||
/** The parts of a test id, or null when it is not `<file>::<project>::<title>`. */
|
||||
export function parseTestId(id: string): { file: string; project: string; title: string } | null {
|
||||
const i = id.indexOf('::');
|
||||
const j = i < 0 ? -1 : id.indexOf('::', i + 2);
|
||||
if (i <= 0 || j < 0 || j + 2 >= id.length) return null;
|
||||
return { file: id.slice(0, i), project: id.slice(i + 2, j), title: id.slice(j + 2) };
|
||||
}
|
||||
|
||||
function readCase(tc: XmlNode, runner: JunitRunner): Omit<JunitTest, 'id' | 'executions'> & { hidden: boolean } {
|
||||
const suite = tc.parent!;
|
||||
let file: string, project = '', path: string[];
|
||||
|
||||
@@ -0,0 +1,444 @@
|
||||
/**
|
||||
* gstack-flaky quarantine (GSTA-158 P2): a registry of known-flaky tests at
|
||||
* `.gstack/quarantine.json`. A quarantined test still runs; it gets a larger
|
||||
* attempt budget (lib/flaky-run.ts, QUARANTINE_ATTEMPTS) under the same full
|
||||
* green verdict, and it is never counted as a pass. This file holds the
|
||||
* registry: schema, validation, add/renew/release, `check` against the base
|
||||
* branch, and `report` over downloaded ledgers.
|
||||
*
|
||||
* Rules (docs/flaky.md § Quarantine):
|
||||
* - Every entry has an owner, a ledger link, a failure class (detector,
|
||||
* harness, timing, model-latency; product defects and infra are refused),
|
||||
* evidence and an expiry: 7 days by default, at most 14 from entry, and one
|
||||
* renewal of at most 7 days with new evidence and the owner named again.
|
||||
* - An expired or invalid entry is not quarantined; nothing needs editing.
|
||||
* - Live entries are capped at max(1, 10% of the suite's tests), and 5 for a
|
||||
* suite under 50 tests.
|
||||
* - `run --base` reads the registry from the base branch so a change cannot
|
||||
* quarantine the test it breaks; `check --base` refuses an entry added in
|
||||
* the change that edits its test's file, an entry that disappears without a
|
||||
* release record, a release record that was removed, and rewritten history.
|
||||
* - gstack's own CASE_QUARANTINE / EVAL_POLICY (paid evals) is a separate
|
||||
* registry; the two share no schema.
|
||||
*/
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { spawnSync } from 'child_process';
|
||||
import { parseTestId, readJunit, type JunitRunner } from './junit';
|
||||
import { safeText } from './flaky-run';
|
||||
|
||||
export const REGISTRY_PATH = '.gstack/quarantine.json';
|
||||
export const FAILURE_CLASSES = ['detector', 'harness', 'timing', 'model-latency'] as const;
|
||||
export type FailureClass = typeof FAILURE_CLASSES[number];
|
||||
export const DEFAULT_DAYS = 7;
|
||||
export const MAX_DAYS = 14;
|
||||
export const RENEW_MAX_DAYS = 7;
|
||||
export const EXIT_RULE_PASSES = 10;
|
||||
const DAY = 86_400_000;
|
||||
|
||||
export interface Renewal { renewedAt: string; previousExpiresAt: string; expiresAt: string; owner: string; evidence: string[] }
|
||||
|
||||
export interface QuarantineEntry {
|
||||
owner: string;
|
||||
/** Link to the flaky ledger line or issue that tracks the fix. */
|
||||
ledger: string;
|
||||
failureClass: FailureClass;
|
||||
reason: string;
|
||||
evidence: string[];
|
||||
enteredAt: string;
|
||||
expiresAt: string;
|
||||
renewals: Renewal[];
|
||||
/** The suite's test count when the entry was added (`add --junit`), the base of the cap. */
|
||||
suiteTests: number;
|
||||
}
|
||||
|
||||
export type ReleasedEntry = Omit<QuarantineEntry, 'reason'> & { id: string; releasedAt: string; reason: string; pr: string };
|
||||
|
||||
export interface Registry { schema: 1; entries: Record<string, QuarantineEntry>; released: ReleasedEntry[] }
|
||||
|
||||
export class QuarantineError extends Error {
|
||||
constructor(readonly code: string, message: string, readonly exitCode: 1 | 2 | 3 = 3) { super(message); }
|
||||
}
|
||||
|
||||
export const emptyRegistry = (): Registry => ({ schema: 1, entries: {}, released: [] });
|
||||
|
||||
/** Live entries allowed for a suite of `tests` tests. */
|
||||
export const suiteCap = (tests: number) => tests < 50 ? 5 : Math.max(1, Math.floor(tests / 10));
|
||||
|
||||
function git(args: string[], cwd: string): { ok: boolean; out: string } {
|
||||
const r = spawnSync('git', args, { cwd, encoding: 'utf-8', timeout: 15_000 });
|
||||
return { ok: r.status === 0, out: r.stdout ?? '' };
|
||||
}
|
||||
|
||||
export function repoRoot(cwd: string): string {
|
||||
const r = git(['rev-parse', '--show-toplevel'], cwd);
|
||||
return r.ok && r.out.trim() ? r.out.trim() : cwd;
|
||||
}
|
||||
|
||||
export function parseRegistry(text: string, source: string): Registry {
|
||||
let data: unknown;
|
||||
try { data = JSON.parse(text); } catch (e) {
|
||||
throw new QuarantineError('FLAKY_QUARANTINE_INVALID', `${source} is not valid JSON (${(e as Error).message})`);
|
||||
}
|
||||
const r = data as Partial<Registry> | null;
|
||||
if (!r || typeof r !== 'object' || r.schema !== 1 || !r.entries || typeof r.entries !== 'object' || Array.isArray(r.entries) || !Array.isArray(r.released)) {
|
||||
throw new QuarantineError('FLAKY_QUARANTINE_INVALID', `${source} is not a schema 1 quarantine registry ({"schema":1,"entries":{},"released":[]})`);
|
||||
}
|
||||
return r as Registry;
|
||||
}
|
||||
|
||||
function resolveCommit(ref: string, cwd: string): string {
|
||||
if (!ref || ref.startsWith('-')) throw new QuarantineError('FLAKY_QUARANTINE_BASE', `--base ${JSON.stringify(ref)} is not a git ref`, 2);
|
||||
const r = git(['rev-parse', '--verify', '--quiet', `${ref}^{commit}`], cwd);
|
||||
if (!r.ok) throw new QuarantineError('FLAKY_QUARANTINE_BASE', `--base ${ref} does not resolve to a commit in ${cwd}`);
|
||||
return r.out.trim();
|
||||
}
|
||||
|
||||
function registryAt(commit: string, root: string, label: string): Registry {
|
||||
if (!git(['cat-file', '-e', `${commit}:${REGISTRY_PATH}`], root).ok) return emptyRegistry();
|
||||
const shown = git(['show', `${commit}:${REGISTRY_PATH}`], root);
|
||||
if (!shown.ok) throw new QuarantineError('FLAKY_QUARANTINE_BASE', `cannot read ${REGISTRY_PATH} at ${label}`);
|
||||
return parseRegistry(shown.out, `${REGISTRY_PATH} at ${label}`);
|
||||
}
|
||||
|
||||
/** The registry in the working tree, or at `base` when given; a missing file is an empty registry. */
|
||||
export function readRegistry(cwd: string, base?: string): Registry {
|
||||
const root = repoRoot(cwd);
|
||||
if (base !== undefined) return registryAt(resolveCommit(base, root), root, base);
|
||||
const file = path.join(root, REGISTRY_PATH);
|
||||
if (!fs.existsSync(file)) return emptyRegistry();
|
||||
return parseRegistry(fs.readFileSync(file, 'utf-8'), REGISTRY_PATH);
|
||||
}
|
||||
|
||||
export function writeRegistry(cwd: string, reg: Registry): string {
|
||||
const file = path.join(repoRoot(cwd), REGISTRY_PATH);
|
||||
const entries = Object.fromEntries(Object.keys(reg.entries).sort().map(id => [id, reg.entries[id]!]));
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||
fs.writeFileSync(file, JSON.stringify({ schema: 1, entries, released: reg.released }, null, 2) + '\n');
|
||||
return file;
|
||||
}
|
||||
|
||||
const filled = (s: unknown): s is string => typeof s === 'string' && s.trim().length > 0;
|
||||
const filledList = (l: unknown): l is string[] => Array.isArray(l) && l.length > 0 && l.every(filled);
|
||||
|
||||
export function classRefusal(c: unknown): string | undefined {
|
||||
if (FAILURE_CLASSES.includes(c as FailureClass)) return undefined;
|
||||
if (c === 'product') return 'class product is refused: a product defect is a real red; fix it instead of quarantining it';
|
||||
if (c === 'infra') return 'class infra is refused: an infrastructure red is rerun or fixed as infra, never quarantined';
|
||||
return `class ${JSON.stringify(c)} is not one of ${FAILURE_CLASSES.join(', ')}`;
|
||||
}
|
||||
|
||||
/** What makes one entry invalid; an invalid entry is never quarantined and `check` refuses it. */
|
||||
export function entryProblems(id: string, e: QuarantineEntry, now: Date): string[] {
|
||||
if (!e || typeof e !== 'object') return ['not an object'];
|
||||
const p: string[] = [];
|
||||
if (!parseTestId(id)) p.push('not a test id (<file>::<project>::<title>; gstack-flaky ids prints them)');
|
||||
if (!filled(e.owner)) p.push('no owner');
|
||||
if (!filled(e.ledger)) p.push('no ledger link');
|
||||
if (!filled(e.reason)) p.push('no reason');
|
||||
const cls = classRefusal(e.failureClass);
|
||||
if (cls) p.push(cls);
|
||||
if (!filledList(e.evidence)) p.push('no evidence');
|
||||
if (!(Number.isInteger(e.suiteTests) && e.suiteTests > 0)) p.push('no suiteTests (add records the suite size from --junit)');
|
||||
const entered = Date.parse(e.enteredAt);
|
||||
const expires = Date.parse(e.expiresAt);
|
||||
if (Number.isNaN(entered) || Number.isNaN(expires)) return [...p, 'enteredAt or expiresAt is not a date'];
|
||||
if (entered > now.getTime() + DAY) p.push('enteredAt is in the future');
|
||||
if (!Array.isArray(e.renewals)) return [...p, 'renewals is not a list'];
|
||||
if (e.renewals.length > 1) p.push(`renewed ${e.renewals.length} times; one renewal is allowed, then fix the test or release it`);
|
||||
const firstExpiry = e.renewals.length ? Date.parse(e.renewals[0]!.previousExpiresAt) : expires;
|
||||
if (!(firstExpiry > entered)) p.push('expires before it entered');
|
||||
if (firstExpiry - entered > MAX_DAYS * DAY) p.push(`expires more than ${MAX_DAYS} days after it entered`);
|
||||
for (const r of e.renewals) {
|
||||
if (!filled(r?.owner)) p.push('a renewal names no owner');
|
||||
if (!filledList(r?.evidence) || r.evidence.every(x => filledList(e.evidence) && e.evidence.includes(x))) p.push('a renewal carries no new evidence');
|
||||
const from = Math.max(Date.parse(r?.renewedAt), Date.parse(r?.previousExpiresAt));
|
||||
const until = Date.parse(r?.expiresAt);
|
||||
if (Number.isNaN(from) || Number.isNaN(until) || until - from > RENEW_MAX_DAYS * DAY) p.push(`a renewal is not a date range of at most ${RENEW_MAX_DAYS} days`);
|
||||
}
|
||||
const last = e.renewals.at(-1);
|
||||
if (last && last.expiresAt !== e.expiresAt) p.push('expiresAt differs from the last renewal');
|
||||
return p;
|
||||
}
|
||||
|
||||
export const isLive = (e: QuarantineEntry, now: Date) => Date.parse(e.expiresAt) > now.getTime();
|
||||
|
||||
/** Ids `run` gives the quarantine budget, plus one notice per entry it ignores (expired or invalid). */
|
||||
export function liveQuarantine(reg: Registry, now: Date): { ids: Set<string>; notices: string[] } {
|
||||
const ids = new Set<string>();
|
||||
const notices: string[] = [];
|
||||
for (const [id, e] of Object.entries(reg.entries)) {
|
||||
const problems = entryProblems(id, e, now);
|
||||
if (problems.length) notices.push(`FLAKY_QUARANTINE_IGNORED: ${safeText(id)} is not quarantined: ${problems.join('; ')}`);
|
||||
else if (!isLive(e, now)) notices.push(expiredNotice(id, e));
|
||||
else ids.add(id);
|
||||
}
|
||||
return { ids, notices };
|
||||
}
|
||||
|
||||
const shq = (s: string) => "'" + s.split("'").join(`'"'"'`) + "'";
|
||||
export const renewCommand = (id: string) => `gstack-flaky quarantine renew ${shq(safeText(id))} --owner <you> --evidence <new evidence url>`;
|
||||
export const releaseCommand = (id: string) => `gstack-flaky quarantine release ${shq(safeText(id))} --reason <why> --pr <fix pr url>`;
|
||||
|
||||
export function expiredNotice(id: string, e: QuarantineEntry): string {
|
||||
return `FLAKY_QUARANTINE_EXPIRED: ${safeText(id)} expired ${e.expiresAt.slice(0, 10)} and is not quarantined; renew: ${renewCommand(id)} | release: ${releaseCommand(id)}`;
|
||||
}
|
||||
|
||||
export interface AddInput { owner?: string; ledger?: string; failureClass?: string; reason?: string; evidence: string[]; days?: number; suiteIds: string[] }
|
||||
|
||||
const refuse = (msg: string): never => { throw new QuarantineError('FLAKY_QUARANTINE_REFUSED', msg, 1); };
|
||||
|
||||
export function addEntry(reg: Registry, id: string, a: AddInput, now: Date): QuarantineEntry {
|
||||
const days = a.days ?? DEFAULT_DAYS;
|
||||
if (!Number.isInteger(days) || days < 1 || days > MAX_DAYS) refuse(`--days must be a whole number from 1 to ${MAX_DAYS}, got ${a.days}`);
|
||||
const cls = classRefusal(a.failureClass);
|
||||
if (cls) refuse(cls);
|
||||
if (!a.evidence.length) refuse('no evidence: pass --evidence <url> (a failing run, a ledger line) at least once');
|
||||
if (!a.suiteIds.includes(id)) refuse(`${safeText(id)} is not a test in the --junit report (gstack-flaky ids --junit <file> prints the ids)`);
|
||||
if (reg.entries[id]) refuse(`${safeText(id)} already has an entry; renew or release it`);
|
||||
const live = Object.values(reg.entries).filter(e => isLive(e, now)).length;
|
||||
const cap = suiteCap(a.suiteIds.length);
|
||||
if (live + 1 > cap) refuse(`the registry holds ${live} live entries and the cap for a ${a.suiteIds.length}-test suite is ${cap}; fix or release one first`);
|
||||
const entry: QuarantineEntry = {
|
||||
owner: a.owner ?? '', ledger: a.ledger ?? '', failureClass: a.failureClass as FailureClass, reason: a.reason ?? '', evidence: a.evidence,
|
||||
enteredAt: now.toISOString(), expiresAt: new Date(now.getTime() + days * DAY).toISOString(), renewals: [], suiteTests: a.suiteIds.length,
|
||||
};
|
||||
const problems = entryProblems(id, entry, now);
|
||||
if (problems.length) refuse(problems.join('; '));
|
||||
reg.entries[id] = entry;
|
||||
return entry;
|
||||
}
|
||||
|
||||
export function renewEntry(reg: Registry, id: string, r: { owner?: string; evidence: string[]; days?: number }, now: Date): QuarantineEntry {
|
||||
const e = reg.entries[id] ?? refuse(`${safeText(id)} has no entry to renew`);
|
||||
if (e.renewals.length) refuse(`${safeText(id)} was renewed once already; a second renewal is refused: fix the test or release it`);
|
||||
if (!filled(r.owner)) refuse('renew names the owner again: pass --owner');
|
||||
if (!r.evidence.some(x => filled(x) && !e.evidence.includes(x))) refuse('renew needs new evidence: pass --evidence <url> not already on the entry');
|
||||
const days = r.days ?? RENEW_MAX_DAYS;
|
||||
if (!Number.isInteger(days) || days < 1 || days > RENEW_MAX_DAYS) refuse(`a renewal is 1 to ${RENEW_MAX_DAYS} days, got ${r.days}`);
|
||||
const from = Math.max(now.getTime(), Date.parse(e.expiresAt));
|
||||
const expiresAt = new Date(from + days * DAY).toISOString();
|
||||
e.renewals.push({ renewedAt: now.toISOString(), previousExpiresAt: e.expiresAt, expiresAt, owner: r.owner!, evidence: r.evidence });
|
||||
e.expiresAt = expiresAt;
|
||||
return e;
|
||||
}
|
||||
|
||||
export function releaseEntry(reg: Registry, id: string, r: { reason?: string; pr?: string }, now: Date): ReleasedEntry {
|
||||
const e = reg.entries[id] ?? refuse(`${safeText(id)} has no entry to release`);
|
||||
if (!filled(r.reason) || !filled(r.pr)) refuse('release needs --reason and --pr (the fix, or the decision to stop quarantining)');
|
||||
const record: ReleasedEntry = { id, ...e, releasedAt: now.toISOString(), reason: r.reason!, pr: r.pr! };
|
||||
reg.released.push(record);
|
||||
delete reg.entries[id];
|
||||
return record;
|
||||
}
|
||||
|
||||
const liveEntries = (reg: Registry, now: Date) => Object.values(reg.entries).filter(e => e && typeof e === 'object' && isLive(e, now));
|
||||
const smallestSuite = (live: QuarantineEntry[]) => Math.min(...live.map(e => e.suiteTests).filter(n => Number.isInteger(n) && n > 0));
|
||||
|
||||
/** Structural problems of a whole registry plus the cap, for `check`. */
|
||||
export function registryProblems(reg: Registry, now: Date, suiteTests?: number): string[] {
|
||||
const p: string[] = [];
|
||||
for (const [id, e] of Object.entries(reg.entries)) for (const x of entryProblems(id, e, now)) p.push(`${safeText(id)}: ${x}`);
|
||||
for (const r of reg.released) {
|
||||
if (!r || !filled(r.id) || Number.isNaN(Date.parse(r.releasedAt)) || !filled(r.reason) || !filled(r.pr)) p.push(`released record ${JSON.stringify(r?.id)}: needs id, releasedAt, reason and pr`);
|
||||
}
|
||||
const live = liveEntries(reg, now);
|
||||
const tests = suiteTests ?? smallestSuite(live);
|
||||
if (live.length && Number.isFinite(tests) && live.length > suiteCap(tests)) p.push(`${live.length} live entries, over the cap of ${suiteCap(tests)} for a ${tests}-test suite`);
|
||||
return p;
|
||||
}
|
||||
|
||||
const editsFile = (changed: string[], file: string) => changed.some(f => f === file || f.endsWith(`/${file}`));
|
||||
|
||||
/** What a change did to the registry that `check --base` refuses. */
|
||||
export function baseProblems(base: Registry, head: Registry, changed: string[]): string[] {
|
||||
const p: string[] = [];
|
||||
for (const [id, e] of Object.entries(head.entries)) {
|
||||
const was = base.entries[id];
|
||||
const t = parseTestId(id);
|
||||
if (!was) {
|
||||
if (t && editsFile(changed, t.file)) p.push(`${safeText(id)}: added in the same change that edits ${t.file}; a change cannot quarantine the test it touches (land the entry on its own, or fix the test)`);
|
||||
continue;
|
||||
}
|
||||
if (was.enteredAt !== e?.enteredAt) p.push(`${safeText(id)}: enteredAt changed from ${was.enteredAt}; release the entry instead of restarting it`);
|
||||
if (JSON.stringify((e?.renewals ?? []).slice(0, was.renewals.length)) !== JSON.stringify(was.renewals)) p.push(`${safeText(id)}: its renewal history was rewritten`);
|
||||
}
|
||||
for (const [id, was] of Object.entries(base.entries)) {
|
||||
if (!(id in head.entries) && !head.released.some(r => r?.id === id && r.enteredAt === was.enteredAt)) p.push(`${safeText(id)}: removed without a release record (use: ${releaseCommand(id)})`);
|
||||
}
|
||||
for (const r of base.released) {
|
||||
if (!head.released.some(h => h?.id === r.id && h.releasedAt === r.releasedAt)) p.push(`${safeText(r.id)}: its release record of ${r.releasedAt} was removed`);
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
/** `check`: the working-tree registry alone, and with `base` against what this change did since the merge base. */
|
||||
export function checkRegistry(cwd: string, o: { base?: string; suiteTests?: number }, now: Date): { problems: string[]; live: number; cap?: number } {
|
||||
const head = readRegistry(cwd);
|
||||
const problems = registryProblems(head, now, o.suiteTests);
|
||||
if (o.base !== undefined) {
|
||||
const root = repoRoot(cwd);
|
||||
const commit = resolveCommit(o.base, root);
|
||||
const mb = git(['merge-base', commit, 'HEAD'], root);
|
||||
if (!mb.ok) throw new QuarantineError('FLAKY_QUARANTINE_BASE', `no merge base between ${o.base} and HEAD (a shallow clone?)`);
|
||||
const changed = git(['diff', '--name-only', mb.out.trim()], root);
|
||||
if (!changed.ok) throw new QuarantineError('FLAKY_QUARANTINE_BASE', `cannot diff against the merge base of ${o.base}`);
|
||||
problems.push(...baseProblems(registryAt(mb.out.trim(), root, `the merge base of ${o.base}`), head, changed.out.split('\n').filter(Boolean)));
|
||||
}
|
||||
const live = liveEntries(head, now);
|
||||
const tests = o.suiteTests ?? smallestSuite(live);
|
||||
return { problems, live: live.length, ...(Number.isFinite(tests) ? { cap: suiteCap(tests) } : {}) };
|
||||
}
|
||||
|
||||
export interface LedgerRecord { ts: string; kind: string; id: string; attempts?: string[] }
|
||||
|
||||
/** Read every JSON line of the given ledger files, and of every .jsonl file under the given directories. */
|
||||
export function readLedgers(paths: string[]): { records: LedgerRecord[]; skipped: number } {
|
||||
const files: string[] = [];
|
||||
const walk = (p: string) => {
|
||||
const st = fs.statSync(p, { throwIfNoEntry: false });
|
||||
if (!st) throw new QuarantineError('FLAKY_QUARANTINE_USAGE', `--ledger ${p} does not exist`, 2);
|
||||
if (!st.isDirectory()) { files.push(p); return; }
|
||||
for (const n of fs.readdirSync(p).sort()) {
|
||||
const c = path.join(p, n);
|
||||
if (fs.statSync(c).isDirectory() || n.endsWith('.jsonl')) walk(c);
|
||||
}
|
||||
};
|
||||
paths.forEach(walk);
|
||||
const records: LedgerRecord[] = [];
|
||||
let skipped = 0;
|
||||
for (const f of files) {
|
||||
for (const l of fs.readFileSync(f, 'utf-8').split('\n')) {
|
||||
if (!l.trim()) continue;
|
||||
try {
|
||||
const r = JSON.parse(l) as LedgerRecord;
|
||||
if (filled(r?.id) && filled(r.kind) && filled(r.ts)) records.push(r); else skipped++;
|
||||
} catch { skipped++; }
|
||||
}
|
||||
}
|
||||
return { records, skipped };
|
||||
}
|
||||
|
||||
const cleanPass = (r: LedgerRecord) => r.kind === 'quarantined-pass' && !(r.attempts ?? []).includes('fail');
|
||||
const shortKind = (r: LedgerRecord) => cleanPass(r) ? 'pass' : r.kind === 'quarantined-pass' ? 'flaky' : r.kind === 'quarantined-fail' ? 'fail' : r.kind;
|
||||
|
||||
/** `report`: expired entries first, then live entries soonest-expiring first, with ledger history when given. */
|
||||
export function reportLines(reg: Registry, now: Date, ledgers?: LedgerRecord[]): string[] {
|
||||
const out: string[] = [];
|
||||
const all = Object.entries(reg.entries).sort((a, b) => Date.parse(a[1].expiresAt) - Date.parse(b[1].expiresAt));
|
||||
const expired = all.filter(([, e]) => !isLive(e, now));
|
||||
const live = all.filter(([, e]) => isLive(e, now));
|
||||
for (const [id, e] of expired) out.push(expiredNotice(id, e));
|
||||
let expiring = 0, ready = 0;
|
||||
for (const [id, e] of live) {
|
||||
const daysLeft = Math.ceil((Date.parse(e.expiresAt) - now.getTime()) / DAY);
|
||||
if (daysLeft <= 2) expiring++;
|
||||
let history = '';
|
||||
if (ledgers) {
|
||||
const sid = safeText(id);
|
||||
const mine = ledgers.filter(r => r.id === sid && Date.parse(r.ts) >= Date.parse(e.enteredAt)).sort((a, b) => Date.parse(a.ts) - Date.parse(b.ts));
|
||||
let streak = 0;
|
||||
for (let i = mine.length - 1; i >= 0 && cleanPass(mine[i]!); i--) streak++;
|
||||
const isReady = streak >= EXIT_RULE_PASSES;
|
||||
if (isReady) ready++;
|
||||
history = ` recent=${mine.slice(-5).map(shortKind).join(',') || 'none'} clean_streak=${streak}${isReady ? ` READY_TO_RELEASE: ${releaseCommand(id)}` : ''}`;
|
||||
}
|
||||
const renewal = e.renewals.length ? 'renewed once; fix or release' : `renew: ${renewCommand(id)}`;
|
||||
out.push(`QUARANTINE: ${safeText(id)} owner=${safeText(e.owner)} class=${e.failureClass} days_left=${daysLeft} expires=${e.expiresAt.slice(0, 10)} ledger=${safeText(e.ledger)}${history}${daysLeft <= 2 ? ` EXPIRING (${renewal})` : ''}`);
|
||||
}
|
||||
out.push(`QUARANTINE_REPORT: live=${live.length} expired=${expired.length} expiring=${expiring} ready=${ledgers ? ready : 'unchecked'} exit_rule=${ledgers ? `checked (${EXIT_RULE_PASSES} consecutive clean passes)` : 'not checked (no --ledger given)'}`);
|
||||
return out;
|
||||
}
|
||||
|
||||
export const QUARANTINE_USAGE = `Usage:
|
||||
gstack-flaky quarantine add <id> --owner <who> --ledger <url> --class detector|harness|timing|model-latency
|
||||
--reason <text> --evidence <url> [--evidence <url>...] --junit <file> [--runner <r>] [--days <1-${MAX_DAYS}>]
|
||||
gstack-flaky quarantine renew <id> --owner <who> --evidence <new url> [--days <1-${RENEW_MAX_DAYS}>]
|
||||
gstack-flaky quarantine release <id> --reason <text> --pr <url>
|
||||
gstack-flaky quarantine check [--base <ref>] [--junit <file>]
|
||||
gstack-flaky quarantine report [--ledger <file|dir>...] [--base <ref>]
|
||||
|
||||
add/renew/release edit ${REGISTRY_PATH} at the repository root. check validates it
|
||||
(owner, ledger, class, evidence, expiry, one renewal, the cap) and, with --base,
|
||||
refuses an entry added in the change that edits its test's file, an entry removed
|
||||
without a release record, and a removed release record. report lists expired and
|
||||
live entries, soonest-expiring first; with --ledger (files or directories of
|
||||
downloaded ledgers) it shows recent outcomes and marks entries with ${EXIT_RULE_PASSES}
|
||||
consecutive clean passes READY_TO_RELEASE.
|
||||
Exit: 0 ok, 1 refused, 2 usage, 3 unreadable registry or base.
|
||||
`;
|
||||
|
||||
interface CliIO { cwd: string; out: (s: string) => void; now: Date }
|
||||
|
||||
const usage = (msg: string): never => { throw new QuarantineError('FLAKY_QUARANTINE_USAGE', msg, 2); };
|
||||
|
||||
function parseArgs(argv: string[]) {
|
||||
const flags: Record<string, string[]> = {};
|
||||
const positional: string[] = [];
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i]!;
|
||||
if (!a.startsWith('--')) { positional.push(a); continue; }
|
||||
const v = argv[++i];
|
||||
if (v === undefined) usage(`${a} needs a value`);
|
||||
(flags[a.slice(2)] ??= []).push(v!);
|
||||
}
|
||||
return { flags, positional };
|
||||
}
|
||||
|
||||
function intFlag(v: string | undefined, name: string): number | undefined {
|
||||
if (v === undefined) return undefined;
|
||||
const n = Number(v);
|
||||
if (!Number.isInteger(n)) usage(`--${name} must be a whole number`);
|
||||
return n;
|
||||
}
|
||||
|
||||
const ALLOWED: Record<string, string[]> = {
|
||||
add: ['owner', 'ledger', 'class', 'reason', 'evidence', 'junit', 'runner', 'days'], renew: ['owner', 'evidence', 'days'],
|
||||
release: ['reason', 'pr'], check: ['base', 'junit', 'runner'], report: ['ledger', 'base'],
|
||||
};
|
||||
|
||||
/** The `gstack-flaky quarantine` subcommands; returns the exit code. A JunitError from --junit propagates to the bin. */
|
||||
export function quarantineMain(argv: string[], io: CliIO): number {
|
||||
const [sub, ...rest] = argv;
|
||||
if (!sub || sub === '--help' || sub === '-h') { io.out(QUARANTINE_USAGE); return sub ? 0 : 2; }
|
||||
if (!ALLOWED[sub]) usage(`unknown quarantine command ${sub}\n${QUARANTINE_USAGE}`);
|
||||
const { flags, positional } = parseArgs(rest);
|
||||
const unknown = Object.keys(flags).find(f => !ALLOWED[sub]!.includes(f));
|
||||
if (unknown) usage(`${sub} does not take --${unknown}`);
|
||||
const one = (f: string) => flags[f]?.at(-1);
|
||||
const runner = one('runner') as JunitRunner | undefined;
|
||||
if (runner !== undefined && !['playwright', 'bun', 'generic'].includes(runner)) usage('--runner must be playwright, bun or generic');
|
||||
const needsId = sub === 'add' || sub === 'renew' || sub === 'release';
|
||||
if (needsId !== (positional.length === 1) || positional.length > 1) usage(needsId ? `${sub} takes one test id` : `${sub} takes no test id`);
|
||||
if (sub === 'check') {
|
||||
const junit = one('junit');
|
||||
const r = checkRegistry(io.cwd, { base: one('base'), suiteTests: junit ? readJunit(junit, runner).tests.length : undefined }, io.now);
|
||||
for (const p of r.problems) io.out(`QUARANTINE_REFUSED: ${p}\n`);
|
||||
io.out(`QUARANTINE_CHECK: live=${r.live} cap=${r.cap ?? 'none'} base=${one('base') ?? 'none'} verdict=${r.problems.length ? 'fail' : 'pass'}\n`);
|
||||
return r.problems.length ? 1 : 0;
|
||||
}
|
||||
if (sub === 'report') {
|
||||
const ledgers = flags.ledger ? readLedgers(flags.ledger) : undefined;
|
||||
if (ledgers?.skipped) io.out(`QUARANTINE_LEDGER: skipped ${ledgers.skipped} unreadable line(s)\n`);
|
||||
for (const l of reportLines(readRegistry(io.cwd, one('base')), io.now, ledgers?.records)) io.out(`${l}\n`);
|
||||
return 0;
|
||||
}
|
||||
const id = positional[0]!;
|
||||
const reg = readRegistry(io.cwd);
|
||||
let line: string;
|
||||
if (sub === 'add') {
|
||||
const junit = one('junit') ?? refuse('add needs --junit <file> to count the suite and confirm the id exists');
|
||||
const suiteIds = readJunit(junit, runner).tests.map(t => t.id);
|
||||
const e = addEntry(reg, id, { owner: one('owner'), ledger: one('ledger'), failureClass: one('class'), reason: one('reason'), evidence: flags.evidence ?? [], days: intFlag(one('days'), 'days'), suiteIds }, io.now);
|
||||
line = `QUARANTINE_ADDED: ${safeText(id)} owner=${safeText(e.owner)} class=${e.failureClass} expires=${e.expiresAt}`;
|
||||
} else if (sub === 'renew') {
|
||||
const e = renewEntry(reg, id, { owner: one('owner'), evidence: flags.evidence ?? [], days: intFlag(one('days'), 'days') }, io.now);
|
||||
line = `QUARANTINE_RENEWED: ${safeText(id)} owner=${safeText(e.renewals[0]!.owner)} expires=${e.expiresAt} (no further renewal)`;
|
||||
} else {
|
||||
const r = releaseEntry(reg, id, { reason: one('reason'), pr: one('pr') }, io.now);
|
||||
line = `QUARANTINE_RELEASED: ${safeText(id)} pr=${safeText(r.pr)}`;
|
||||
}
|
||||
writeRegistry(io.cwd, reg);
|
||||
io.out(`${line} registry=${REGISTRY_PATH}\n`);
|
||||
return 0;
|
||||
}
|
||||
@@ -160,7 +160,7 @@ describe('runFlaky', () => {
|
||||
expect(r.outcomes.map(o => [o.status, o.attempts.join(',')])).toEqual([['flaky-pass', 'fail,pass']]);
|
||||
expect(r.ledger).toHaveLength(1);
|
||||
expect(r.ledger[0]).toMatchObject({ kind: 'flaky-pass', id: 'a.test.ts::::b', rerun_by: 'gstack-flaky', attempts: ['fail', 'pass'], excerpt: ['boom'] });
|
||||
expect(renderResult(r, '/l.jsonl').at(-1)).toBe('FLAKY_RESULT: tests=2 failed=0 flaky=1 reruns=1 command_exit=1 verdict=pass ledger=/l.jsonl');
|
||||
expect(renderResult(r, '/l.jsonl').at(-1)).toBe('FLAKY_RESULT: tests=2 failed=0 flaky=1 quarantined=0 reruns=1 command_exit=1 verdict=pass ledger=/l.jsonl');
|
||||
expect(stepSummary(r)).toContain('a.test.ts::::b');
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,427 @@
|
||||
/**
|
||||
* gstack-flaky quarantine (GSTA-158 P2): registry refusals, renew and release,
|
||||
* `check --base` against git fixtures, expiry, the quarantine attempt budget
|
||||
* under the runner's full green verdict (never a false green), missing ids,
|
||||
* the registry read from the base branch, and `report` over ledgers.
|
||||
*/
|
||||
import { afterAll, describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { spawnSync } from 'child_process';
|
||||
import { addEntry, checkRegistry, emptyRegistry, entryProblems, liveQuarantine, readLedgers, readRegistry, registryProblems, releaseEntry, renewEntry, reportLines, suiteCap, writeRegistry, type AddInput, type Registry } from '../lib/quarantine';
|
||||
import { renderResult, runFlaky } from '../lib/flaky-run';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const BIN = path.join(ROOT, 'bin', 'gstack-flaky');
|
||||
const TMP = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-quarantine-'));
|
||||
const tmp = () => fs.mkdtempSync(path.join(TMP, 't-'));
|
||||
afterAll(() => fs.rmSync(TMP, { recursive: true, force: true }));
|
||||
|
||||
const NOW = new Date('2026-10-29T00:00:00Z');
|
||||
const DAY = 86_400_000;
|
||||
const at = (days: number) => new Date(NOW.getTime() + days * DAY);
|
||||
const ids = (n: number, file = 'a.test.ts') => Array.from({ length: n }, (_, i) => `${file}::::t${i}`);
|
||||
const add = (reg: Registry, id: string, over: Partial<AddInput> = {}, now = NOW) =>
|
||||
addEntry(reg, id, { owner: 'garry', ledger: 'https://ledger/1', failureClass: 'timing', reason: 'slow CI disk', evidence: ['https://run/1'], suiteIds: ids(100), ...over }, now);
|
||||
const refusal = (fn: () => unknown) => { try { fn(); return 'ok'; } catch (e) { return (e as Error).message; } };
|
||||
|
||||
describe('registry: add, renew, release', () => {
|
||||
test('add refuses more than 14 days, no evidence, product or infra class, an unknown id, a duplicate and the cap', () => {
|
||||
const reg = emptyRegistry();
|
||||
expect(refusal(() => add(reg, 'a.test.ts::::t0', { days: 15 }))).toContain('from 1 to 14');
|
||||
expect(refusal(() => add(reg, 'a.test.ts::::t0', { evidence: [] }))).toContain('no evidence');
|
||||
expect(refusal(() => add(reg, 'a.test.ts::::t0', { failureClass: 'product' }))).toContain('class product is refused');
|
||||
expect(refusal(() => add(reg, 'a.test.ts::::t0', { failureClass: 'infra' }))).toContain('class infra is refused');
|
||||
expect(refusal(() => add(reg, 'a.test.ts::::t0', { failureClass: 'flaky' }))).toContain('is not one of detector, harness, timing, model-latency');
|
||||
expect(refusal(() => add(reg, 'a.test.ts::::nope'))).toContain('is not a test in the --junit report');
|
||||
expect(refusal(() => add(reg, 'a.test.ts::::t0', { owner: ' ' }))).toContain('no owner');
|
||||
expect(Object.keys(reg.entries)).toEqual([]);
|
||||
const e = add(reg, 'a.test.ts::::t0');
|
||||
expect(e).toMatchObject({ enteredAt: NOW.toISOString(), expiresAt: at(7).toISOString(), renewals: [], suiteTests: 100 });
|
||||
expect(add(reg, 'a.test.ts::::t1', { days: 14 }).expiresAt).toBe(at(14).toISOString());
|
||||
expect(refusal(() => add(reg, 'a.test.ts::::t0'))).toContain('already has an entry');
|
||||
const small = emptyRegistry();
|
||||
for (let i = 0; i < 5; i++) add(small, `a.test.ts::::t${i}`, { suiteIds: ids(10) });
|
||||
expect(refusal(() => add(small, 'a.test.ts::::t5', { suiteIds: ids(10) }))).toContain('the cap for a 10-test suite is 5');
|
||||
expect(refusal(() => add(small, 'a.test.ts::::t5', { suiteIds: ids(10) }, at(8)))).toBe('ok');
|
||||
});
|
||||
|
||||
test('the cap is 10% of the suite, at least 1, and 5 under 50 tests', () => {
|
||||
expect([1, 10, 49, 50, 99, 100, 1000].map(suiteCap)).toEqual([5, 5, 5, 5, 9, 10, 100]);
|
||||
});
|
||||
|
||||
test('renew once, with new evidence and the owner named again; a second renewal is refused', () => {
|
||||
const reg = emptyRegistry();
|
||||
add(reg, 'a.test.ts::::t0');
|
||||
const id = 'a.test.ts::::t0';
|
||||
expect(refusal(() => renewEntry(reg, id, { evidence: ['https://run/2'] }, at(6)))).toContain('names the owner again');
|
||||
expect(refusal(() => renewEntry(reg, id, { owner: 'garry', evidence: ['https://run/1'] }, at(6)))).toContain('new evidence');
|
||||
expect(refusal(() => renewEntry(reg, id, { owner: 'garry', evidence: ['https://run/2'], days: 8 }, at(6)))).toContain('1 to 7 days');
|
||||
const e = renewEntry(reg, id, { owner: 'garry', evidence: ['https://run/2'] }, at(6));
|
||||
expect(e.expiresAt).toBe(at(14).toISOString());
|
||||
expect(entryProblems(id, e, at(6))).toEqual([]);
|
||||
expect(refusal(() => renewEntry(reg, id, { owner: 'garry', evidence: ['https://run/3'] }, at(13)))).toContain('a second renewal is refused');
|
||||
const expired = emptyRegistry();
|
||||
add(expired, id);
|
||||
expect(renewEntry(expired, id, { owner: 'sam', evidence: ['https://run/9'], days: 3 }, at(10)).expiresAt).toBe(at(13).toISOString());
|
||||
const twice = { ...e, renewals: [...e.renewals, { ...e.renewals[0]!, evidence: ['https://run/3'] }] };
|
||||
expect(entryProblems(id, twice, at(6)).join()).toContain('one renewal is allowed');
|
||||
});
|
||||
|
||||
test('release moves the entry to released with its reason and PR', () => {
|
||||
const reg = emptyRegistry();
|
||||
add(reg, 'a.test.ts::::t0');
|
||||
expect(refusal(() => releaseEntry(reg, 'a.test.ts::::t0', { reason: 'fixed' }, at(2)))).toContain('--reason and --pr');
|
||||
const r = releaseEntry(reg, 'a.test.ts::::t0', { reason: 'fixed the wait', pr: 'https://github.com/o/r/pull/9' }, at(2));
|
||||
expect(reg.entries).toEqual({});
|
||||
expect(reg.released).toEqual([r]);
|
||||
expect(r).toMatchObject({ id: 'a.test.ts::::t0', owner: 'garry', reason: 'fixed the wait', releasedAt: at(2).toISOString(), enteredAt: NOW.toISOString() });
|
||||
expect(refusal(() => releaseEntry(reg, 'a.test.ts::::t0', { reason: 'x', pr: 'y' }, at(2)))).toContain('no entry to release');
|
||||
});
|
||||
|
||||
test('expired and invalid entries are not quarantined; expiry prints the renew and release commands', () => {
|
||||
const reg = emptyRegistry();
|
||||
add(reg, 'a.test.ts::::t0');
|
||||
expect([...liveQuarantine(reg, at(6)).ids]).toEqual(['a.test.ts::::t0']);
|
||||
const later = liveQuarantine(reg, at(7));
|
||||
expect(later.ids.size).toBe(0);
|
||||
expect(later.notices[0]).toContain('FLAKY_QUARANTINE_EXPIRED: a.test.ts::::t0 expired 2026-11-05 and is not quarantined');
|
||||
expect(later.notices[0]).toContain("renew: gstack-flaky quarantine renew 'a.test.ts::::t0' --owner");
|
||||
expect(later.notices[0]).toContain("release: gstack-flaky quarantine release 'a.test.ts::::t0' --reason");
|
||||
reg.entries['a.test.ts::::t1'] = { ...reg.entries['a.test.ts::::t0']!, expiresAt: at(30).toISOString() };
|
||||
reg.entries['a.test.ts::::t2'] = { ...reg.entries['a.test.ts::::t0']!, failureClass: 'infra' as never };
|
||||
const live = liveQuarantine(reg, at(1));
|
||||
expect([...live.ids]).toEqual(['a.test.ts::::t0']);
|
||||
expect(live.notices.join('\n')).toContain('a.test.ts::::t1 is not quarantined: expires more than 14 days after it entered');
|
||||
expect(live.notices.join('\n')).toContain('a.test.ts::::t2 is not quarantined: class infra is refused');
|
||||
});
|
||||
|
||||
test('check enforces the cap and the entry rules on the whole registry', () => {
|
||||
const reg = emptyRegistry();
|
||||
for (let i = 0; i < 5; i++) add(reg, `a.test.ts::::t${i}`, { suiteIds: ids(10) });
|
||||
expect(registryProblems(reg, NOW)).toEqual([]);
|
||||
reg.entries['a.test.ts::::t5'] = { ...reg.entries['a.test.ts::::t0']! };
|
||||
expect(registryProblems(reg, NOW)).toEqual(['6 live entries, over the cap of 5 for a 10-test suite']);
|
||||
expect(registryProblems(reg, NOW, 60)).toEqual([]);
|
||||
expect(registryProblems(reg, at(8))).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
const git = (dir: string, ...args: string[]) => {
|
||||
const r = spawnSync('git', ['-c', 'user.name=t', '-c', 'user.email=t@example.com', '-c', 'commit.gpgsign=false', ...args], { cwd: dir, encoding: 'utf-8', timeout: 30_000 });
|
||||
if (r.status !== 0) throw new Error(`git ${args.join(' ')}: ${r.stderr}`);
|
||||
return r.stdout;
|
||||
};
|
||||
|
||||
/** A repository whose main branch quarantines test/a.test.ts::::slow and has released test/a.test.ts::::old; HEAD is a change branch. */
|
||||
function fixtureRepo(now = NOW) {
|
||||
const dir = tmp();
|
||||
git(dir, 'init', '-q', '-b', 'main');
|
||||
fs.mkdirSync(path.join(dir, 'test'));
|
||||
fs.writeFileSync(path.join(dir, 'test', 'a.test.ts'), '// a\n');
|
||||
fs.writeFileSync(path.join(dir, 'test', 'b.test.ts'), '// b\n');
|
||||
const reg = emptyRegistry();
|
||||
const suiteIds = ['test/a.test.ts::::slow', 'test/a.test.ts::::old', 'test/b.test.ts::::other'];
|
||||
add(reg, 'test/a.test.ts::::old', { suiteIds }, now);
|
||||
releaseEntry(reg, 'test/a.test.ts::::old', { reason: 'fixed', pr: '#1' }, now);
|
||||
add(reg, 'test/a.test.ts::::slow', { suiteIds }, now);
|
||||
writeRegistry(dir, reg);
|
||||
git(dir, 'add', '-A');
|
||||
git(dir, 'commit', '-qm', 'seed');
|
||||
git(dir, 'checkout', '-qb', 'change');
|
||||
return { dir, reg, suiteIds };
|
||||
}
|
||||
|
||||
const clone = (r: Registry): Registry => JSON.parse(JSON.stringify(r));
|
||||
|
||||
describe('check --base against a git fixture', () => {
|
||||
test('an unchanged registry and a proper release pass', () => {
|
||||
const { dir, reg } = fixtureRepo();
|
||||
expect(checkRegistry(dir, { base: 'main' }, NOW).problems).toEqual([]);
|
||||
const next = clone(reg);
|
||||
releaseEntry(next, 'test/a.test.ts::::slow', { reason: 'fixed', pr: '#2' }, at(1));
|
||||
writeRegistry(dir, next);
|
||||
expect(checkRegistry(dir, { base: 'main' }, at(1)).problems).toEqual([]);
|
||||
});
|
||||
|
||||
test('a deleted entry, a removed release record and rewritten history are refused', () => {
|
||||
const { dir, reg } = fixtureRepo();
|
||||
const deleted = clone(reg);
|
||||
delete deleted.entries['test/a.test.ts::::slow'];
|
||||
writeRegistry(dir, deleted);
|
||||
expect(checkRegistry(dir, { base: 'main' }, NOW).problems.join('\n')).toContain('test/a.test.ts::::slow: removed without a release record');
|
||||
const unreleased = clone(reg);
|
||||
unreleased.released = [];
|
||||
writeRegistry(dir, unreleased);
|
||||
expect(checkRegistry(dir, { base: 'main' }, NOW).problems.join('\n')).toContain('test/a.test.ts::::old: its release record of');
|
||||
const restarted = clone(reg);
|
||||
restarted.entries['test/a.test.ts::::slow']!.enteredAt = at(1).toISOString();
|
||||
writeRegistry(dir, restarted);
|
||||
expect(checkRegistry(dir, { base: 'main' }, at(1)).problems.join('\n')).toContain('enteredAt changed');
|
||||
});
|
||||
|
||||
test('an entry added in the same change that edits its test file is refused; without the edit it passes', () => {
|
||||
const { dir, reg, suiteIds } = fixtureRepo();
|
||||
const next = clone(reg);
|
||||
add(next, 'test/b.test.ts::::other', { suiteIds });
|
||||
writeRegistry(dir, next);
|
||||
expect(checkRegistry(dir, { base: 'main' }, NOW).problems).toEqual([]);
|
||||
fs.writeFileSync(path.join(dir, 'test', 'b.test.ts'), '// b, now broken\n');
|
||||
git(dir, 'commit', '-qam', 'break b and quarantine it');
|
||||
expect(checkRegistry(dir, { base: 'main' }, NOW).problems).toEqual([
|
||||
'test/b.test.ts::::other: added in the same change that edits test/b.test.ts; a change cannot quarantine the test it touches (land the entry on its own, or fix the test)',
|
||||
]);
|
||||
});
|
||||
|
||||
test('the bin: check exits 1 with QUARANTINE_REFUSED lines, and an unknown base exits 3', () => {
|
||||
const { dir, reg } = fixtureRepo(new Date());
|
||||
const deleted = clone(reg);
|
||||
delete deleted.entries['test/a.test.ts::::slow'];
|
||||
writeRegistry(dir, deleted);
|
||||
const r = spawnSync(BIN, ['quarantine', 'check', '--base', 'main'], { cwd: dir, encoding: 'utf-8', timeout: 30_000 });
|
||||
expect(r.status).toBe(1);
|
||||
expect(r.stdout).toContain('QUARANTINE_REFUSED: test/a.test.ts::::slow: removed without a release record');
|
||||
expect(r.stdout).toContain('verdict=fail');
|
||||
const missing = spawnSync(BIN, ['quarantine', 'check', '--base', 'no-such-branch'], { cwd: dir, encoding: 'utf-8', timeout: 30_000 });
|
||||
expect([missing.status, missing.stderr.includes('(FLAKY_QUARANTINE_BASE)')]).toEqual([3, true]);
|
||||
});
|
||||
});
|
||||
|
||||
/** A fake runner: the attempt number decides the report, exit status, output and signal. */
|
||||
function fakeRun(reports: (n: number, argv: string[]) => string | null, exits: (n: number) => number, outputs: (n: number) => string = () => 'Ran 3 tests across 1 file. [3ms]\n', signals: (n: number) => string | undefined = () => undefined) {
|
||||
const calls: string[][] = [];
|
||||
const spawn = async (argv: string[], _env: NodeJS.ProcessEnv, cwd: string) => {
|
||||
calls.push(argv);
|
||||
const xml = reports(calls.length, argv);
|
||||
if (xml !== null) fs.writeFileSync(path.join(cwd, 'r.xml'), xml);
|
||||
const signal = signals(calls.length);
|
||||
return { exit: exits(calls.length), output: outputs(calls.length), ...(signal ? { signal } : {}) };
|
||||
};
|
||||
return { calls, spawn };
|
||||
}
|
||||
|
||||
const suite = (cases: string, file = 'a.test.ts') => `<testsuites name="bun test"><testsuite name="${file}" file="${file}">${cases}</testsuite></testsuites>`;
|
||||
const pass = (name: string) => `<testcase name="${name}" classname="" file="a.test.ts"/>`;
|
||||
const fail = (name: string) => `<testcase name="${name}" classname="" file="a.test.ts"><failure message="boom"/></testcase>`;
|
||||
const Q = new Set(['a.test.ts::::q']);
|
||||
const run = (f: ReturnType<typeof fakeRun>, extra: Partial<Parameters<typeof runFlaky>[0]> = {}) =>
|
||||
runFlaky({ command: ['bun', 'test'], junit: 'r.xml', maxReruns: 3, cwd: tmp(), spawn: f.spawn, env: {}, now: () => NOW, quarantined: Q, ...extra });
|
||||
|
||||
describe('the quarantine attempt budget under the full green verdict', () => {
|
||||
test('a quarantined test that fails all 3 attempts is a hard break', async () => {
|
||||
const f = fakeRun(() => suite(pass('a') + fail('q')), () => 1);
|
||||
const r = await run(f);
|
||||
expect(f.calls).toHaveLength(3);
|
||||
expect(r.exitCode).toBe(1);
|
||||
expect(r.outcomes).toMatchObject([{ status: 'hard-break', attempts: ['fail', 'fail', 'fail'], reason: 'failed all 3 attempts: a hard break, not a flake' }]);
|
||||
expect(r.ledger.map(l => [l.kind, l.id, l.rerun_scope])).toEqual([['hard-break', 'a.test.ts::::q', 'command']]);
|
||||
expect(renderResult(r, '/l').at(-1)).toBe('FLAKY_RESULT: tests=2 failed=1 flaky=0 quarantined=1 reruns=2 command_exit=1 verdict=fail ledger=/l');
|
||||
});
|
||||
|
||||
test('it passes on attempt 3 only on the runner\'s full green verdict, listed as quarantined and not as a pass', async () => {
|
||||
const f = fakeRun(n => suite(pass('a') + (n < 3 ? fail('q') : pass('q'))), n => n < 3 ? 1 : 0);
|
||||
const r = await run(f);
|
||||
expect(f.calls).toHaveLength(3);
|
||||
expect(r.exitCode).toBe(0);
|
||||
expect(r.outcomes).toMatchObject([{ status: 'quarantined-pass', attempts: ['fail', 'fail', 'pass'] }]);
|
||||
expect(r.ledger[0]).toMatchObject({ kind: 'quarantined-pass', attempts: ['fail', 'fail', 'pass'] });
|
||||
expect(renderResult(r, '/l')).toEqual(['FLAKY: quarantined-pass a.test.ts::::q attempts=fail,fail,pass', 'FLAKY_RERUN: scope=command (the default; --rerun-scope test reruns the failed test alone)',
|
||||
'FLAKY_RESULT: tests=2 failed=0 flaky=0 quarantined=1 reruns=2 command_exit=1 verdict=pass ledger=/l']);
|
||||
});
|
||||
|
||||
test('false-green regression: exit 1 from outside the report never turns green because only a quarantined test failed', async () => {
|
||||
const greenReportLaterExit1 = fakeRun(n => suite(pass('a') + (n === 1 ? fail('q') : pass('q'))), () => 1);
|
||||
const a = await run(greenReportLaterExit1);
|
||||
expect(greenReportLaterExit1.calls).toHaveLength(2);
|
||||
expect(a.exitCode).toBe(1);
|
||||
expect(a.outcomes).toMatchObject([{ status: 'quarantined-fail', attempts: ['fail', 'pass'], reason: 'rerun: the command exited 1' }]);
|
||||
const alwaysExit1 = fakeRun(() => suite(pass('a') + fail('q')), () => 1, () => 'All files | % Funcs | % Lines |\nRan 2 tests across 1 file.\n');
|
||||
const b = await run(alwaysExit1, { rerunScope: 'test' });
|
||||
expect(alwaysExit1.calls.every(c => c.join(' ') === 'bun test')).toBe(true);
|
||||
expect([b.exitCode, b.outcomes[0]!.status]).toEqual([1, 'hard-break']);
|
||||
});
|
||||
|
||||
test('the bin keeps a coverage-style exit 1 red while the only reported failure is quarantined', () => {
|
||||
const dir = tmp();
|
||||
git(dir, 'init', '-q', '-b', 'main');
|
||||
const reg = emptyRegistry();
|
||||
add(reg, 'a.test.ts::::q', { suiteIds: ['a.test.ts::::a', 'a.test.ts::::q'] }, new Date());
|
||||
writeRegistry(dir, reg);
|
||||
const runner = path.join(dir, 'runner.sh');
|
||||
fs.writeFileSync(runner, ['#!/usr/bin/env bash', 'echo x >> count', `printf '%s' '${suite(pass('a') + fail('q'))}' > r.xml`, 'exit 1'].join('\n'), { mode: 0o755 });
|
||||
const r = spawnSync(BIN, ['run', '--junit', 'r.xml', '--ledger', path.join(dir, 'l.jsonl'), '--', runner], { cwd: dir, encoding: 'utf-8', timeout: 30_000, env: { ...process.env, GITHUB_ACTIONS: '', GITHUB_STEP_SUMMARY: '' } });
|
||||
expect(r.status).toBe(1);
|
||||
expect(r.stdout).toContain('FLAKY: hard-break a.test.ts::::q attempts=fail,fail,fail');
|
||||
expect(r.stdout).toContain('quarantined=1 reruns=2 command_exit=1 verdict=fail');
|
||||
expect(fs.readFileSync(path.join(dir, 'count'), 'utf-8').trim().split('\n')).toHaveLength(3);
|
||||
});
|
||||
|
||||
test('non-quarantined failures keep one rerun; a new non-quarantined failure ends the budget', async () => {
|
||||
const both = fakeRun(n => suite((n === 1 ? fail('a') : fail('a')) + (n === 1 ? fail('q') : pass('q'))), () => 1);
|
||||
const r = await run(both);
|
||||
expect(both.calls).toHaveLength(2);
|
||||
expect(r.outcomes.map(o => [o.test.id, o.status])).toEqual([['a.test.ts::::a', 'rerun-fail'], ['a.test.ts::::q', 'quarantined-fail']]);
|
||||
const newFailure = fakeRun(n => suite((n === 1 ? pass('a') : fail('a')) + fail('q')), () => 1);
|
||||
const s = await run(newFailure);
|
||||
expect(newFailure.calls).toHaveLength(2);
|
||||
expect([s.exitCode, s.outcomes[0]!.status]).toEqual([1, 'quarantined-fail']);
|
||||
});
|
||||
|
||||
test('a flaky test and a quarantined test both clear on the attempt that is fully green', async () => {
|
||||
const f = fakeRun(n => suite((n === 1 ? fail('a') : pass('a')) + (n < 3 ? fail('q') : pass('q'))), n => n < 3 ? 1 : 0);
|
||||
const r = await run(f);
|
||||
expect(f.calls).toHaveLength(3);
|
||||
expect(r.exitCode).toBe(0);
|
||||
expect(r.outcomes.map(o => [o.status, o.attempts.join(',')])).toEqual([['flaky-pass', 'fail,pass,pass'], ['quarantined-pass', 'fail,fail,pass']]);
|
||||
});
|
||||
|
||||
test('a killed attempt or a missing first-run test ends the budget red', async () => {
|
||||
const killed = fakeRun(() => suite(pass('a') + fail('q')), n => n === 1 ? 1 : 128, undefined, n => n === 2 ? 'SIGTERM' : undefined);
|
||||
expect((await run(killed)).exitCode).toBe(1);
|
||||
expect(killed.calls).toHaveLength(2);
|
||||
const shrunk = fakeRun(n => n === 1 ? suite(pass('a') + fail('q')) : suite(fail('q')), () => 1);
|
||||
expect((await run(shrunk)).exitCode).toBe(1);
|
||||
expect(shrunk.calls).toHaveLength(2);
|
||||
});
|
||||
|
||||
test('a quarantined test that passes is listed, ledgered for the exit rule, and the run is green', async () => {
|
||||
const r = await run(fakeRun(() => suite(pass('a') + pass('q')), () => 0));
|
||||
expect(r).toMatchObject({ exitCode: 0, reruns: 0, quarantined: 1 });
|
||||
expect(r.ledger.map(l => [l.kind, l.attempts])).toEqual([['quarantined-pass', ['pass']]]);
|
||||
expect(renderResult(r, '/l').at(-1)).toContain('failed=0 flaky=0 quarantined=1');
|
||||
});
|
||||
|
||||
test('quarantined failures do not count against --max-reruns', async () => {
|
||||
const f = fakeRun(n => suite(pass('a') + (n === 1 ? fail('q') : pass('q'))), n => n === 1 ? 1 : 0);
|
||||
expect((await run(f, { maxReruns: 0 })).exitCode).toBe(0);
|
||||
});
|
||||
|
||||
test('--rerun-scope test gives the quarantined test the same budget on the targeted path', async () => {
|
||||
const f = fakeRun((n, argv) => argv.length === 2 ? suite(pass('a') + fail('q')) : suite(n < 3 ? fail('q') : pass('q')), n => n < 3 ? 1 : 0);
|
||||
const r = await run(f, { rerunScope: 'test' });
|
||||
expect(f.calls.slice(1)).toEqual([['bun', 'test', 'a.test.ts', '--test-name-pattern', '^q$'], ['bun', 'test', 'a.test.ts', '--test-name-pattern', '^q$']]);
|
||||
expect(r.exitCode).toBe(0);
|
||||
expect(r.outcomes).toMatchObject([{ status: 'quarantined-pass', attempts: ['fail', 'fail', 'pass'] }]);
|
||||
const notQ = fakeRun((n, argv) => argv.length === 2 ? suite(pass('q') + fail('a')) : suite(fail('a')), () => 1);
|
||||
expect((await run(notQ, { rerunScope: 'test' })).exitCode).toBe(1);
|
||||
expect(notQ.calls).toHaveLength(2);
|
||||
});
|
||||
|
||||
test('an expired entry gives no budget: one rerun, as for any test', async () => {
|
||||
const reg = emptyRegistry();
|
||||
add(reg, 'a.test.ts::::q', { suiteIds: ['a.test.ts::::q'] });
|
||||
const f = fakeRun(n => suite(pass('a') + (n < 3 ? fail('q') : pass('q'))), n => n < 3 ? 1 : 0);
|
||||
const r = await run(f, { quarantined: liveQuarantine(reg, at(8)).ids });
|
||||
expect(f.calls).toHaveLength(2);
|
||||
expect([r.exitCode, r.outcomes[0]!.status, r.quarantined]).toEqual([1, 'rerun-fail', 0]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('missing quarantined ids', () => {
|
||||
test('a quarantined id missing from a run that included its file blocks', async () => {
|
||||
const f = fakeRun(() => suite(pass('a') + pass('b')), () => 0);
|
||||
const r = await run(f, { quarantined: new Set(['a.test.ts::::gone']) });
|
||||
expect(f.calls).toHaveLength(1);
|
||||
expect(r.exitCode).toBe(1);
|
||||
expect(r.outcomes).toMatchObject([{ status: 'quarantined-fail', reason: 'quarantined, and missing from a run that included its file' }]);
|
||||
expect(r.ledger[0]).toMatchObject({ kind: 'quarantined-fail', id: 'a.test.ts::::gone' });
|
||||
});
|
||||
|
||||
test('a run that did not include its file (a shard, a diff selection) does not block', async () => {
|
||||
const r = await run(fakeRun(() => suite(pass('a')), () => 0), { quarantined: new Set(['b.test.ts::::gone', 'a.test.ts:: chromium ::x']) });
|
||||
expect(r).toMatchObject({ exitCode: 0, outcomes: [], quarantined: 0 });
|
||||
});
|
||||
});
|
||||
|
||||
describe('the registry read from the base branch', () => {
|
||||
test('readRegistry --base sees the base copy, not the change\'s own entry', () => {
|
||||
const { dir, reg, suiteIds } = fixtureRepo();
|
||||
const next = clone(reg);
|
||||
add(next, 'test/b.test.ts::::other', { suiteIds });
|
||||
writeRegistry(dir, next);
|
||||
expect(Object.keys(readRegistry(dir).entries).sort()).toEqual(['test/a.test.ts::::slow', 'test/b.test.ts::::other']);
|
||||
expect(Object.keys(readRegistry(dir, 'main').entries)).toEqual(['test/a.test.ts::::slow']);
|
||||
});
|
||||
|
||||
test('the bin: a change that quarantines its own failing test gets no budget with --base', () => {
|
||||
const dir = tmp();
|
||||
git(dir, 'init', '-q', '-b', 'main');
|
||||
git(dir, 'commit', '-q', '--allow-empty', '-m', 'seed');
|
||||
git(dir, 'checkout', '-qb', 'change');
|
||||
fs.writeFileSync(path.join(dir, 'seed.xml'), suite(pass('a') + pass('b')));
|
||||
const addRun = spawnSync(BIN, ['quarantine', 'add', 'a.test.ts::::b', '--owner', 'garry', '--ledger', 'https://l/1', '--class', 'timing', '--reason', 'slow', '--evidence', 'https://run/1', '--junit', 'seed.xml'], { cwd: dir, encoding: 'utf-8', timeout: 30_000 });
|
||||
expect([addRun.status, addRun.stdout.startsWith('QUARANTINE_ADDED: a.test.ts::::b owner=garry class=timing')]).toEqual([0, true]);
|
||||
const runner = path.join(dir, 'runner.sh');
|
||||
fs.writeFileSync(runner, [
|
||||
'#!/usr/bin/env bash',
|
||||
'n=$(cat count 2>/dev/null || echo 0); echo $((n+1)) > count',
|
||||
`if [ "$n" -lt 2 ]; then printf '%s' '${suite(pass('a') + fail('b'))}' > r.xml; exit 1; fi`,
|
||||
`printf '%s' '${suite(pass('a') + pass('b'))}' > r.xml`,
|
||||
].join('\n'), { mode: 0o755 });
|
||||
const flaky = (extra: string[]) => {
|
||||
fs.rmSync(path.join(dir, 'count'), { force: true });
|
||||
return spawnSync(BIN, ['run', '--junit', 'r.xml', '--ledger', path.join(dir, 'l.jsonl'), ...extra, '--', runner], { cwd: dir, encoding: 'utf-8', timeout: 30_000, env: { ...process.env, GITHUB_ACTIONS: '', GITHUB_STEP_SUMMARY: '' } });
|
||||
};
|
||||
const own = flaky([]);
|
||||
expect([own.status, own.stdout.includes('FLAKY: quarantined-pass a.test.ts::::b attempts=fail,fail,pass')]).toEqual([0, true]);
|
||||
const fromBase = flaky(['--base', 'main']);
|
||||
expect([fromBase.status, fromBase.stdout.includes('FLAKY: rerun-fail a.test.ts::::b')]).toEqual([1, true]);
|
||||
expect(fs.readFileSync(path.join(dir, 'count'), 'utf-8').trim()).toBe('2');
|
||||
const badBase = flaky(['--base', 'no-such-branch']);
|
||||
expect([badBase.status, badBase.stderr.includes('(FLAKY_QUARANTINE_BASE)')]).toEqual([3, true]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('quarantine CLI and report', () => {
|
||||
test('add, renew, release through the bin; refusals exit 1 and usage exits 2', () => {
|
||||
const dir = tmp();
|
||||
fs.writeFileSync(path.join(dir, 'seed.xml'), suite(pass('a') + pass('b')));
|
||||
const q = (...args: string[]) => spawnSync(BIN, ['quarantine', ...args], { cwd: dir, encoding: 'utf-8', timeout: 30_000 });
|
||||
const base = ['--owner', 'garry', '--ledger', 'https://l/1', '--reason', 'slow', '--evidence', 'https://run/1', '--junit', 'seed.xml'];
|
||||
const product = q('add', 'a.test.ts::::b', ...base, '--class', 'product');
|
||||
expect([product.status, product.stderr.includes('class product is refused'), product.stderr.includes('(FLAKY_QUARANTINE_REFUSED)')]).toEqual([1, true, true]);
|
||||
expect(q('add', 'a.test.ts::::b', ...base, '--class', 'harness').status).toBe(0);
|
||||
expect(q('renew', 'a.test.ts::::b', '--owner', 'garry', '--evidence', 'https://run/2').status).toBe(0);
|
||||
expect(q('renew', 'a.test.ts::::b', '--owner', 'garry', '--evidence', 'https://run/3').status).toBe(1);
|
||||
expect(q('check').stdout).toContain('QUARANTINE_CHECK: live=1 cap=5 base=none verdict=pass');
|
||||
expect(q('release', 'a.test.ts::::b', '--reason', 'fixed', '--pr', '#7').status).toBe(0);
|
||||
const saved = JSON.parse(fs.readFileSync(path.join(dir, '.gstack', 'quarantine.json'), 'utf-8')) as Registry;
|
||||
expect([Object.keys(saved.entries), saved.released.map(r => [r.id, r.pr, r.renewals.length])]).toEqual([[], [['a.test.ts::::b', '#7', 1]]]);
|
||||
expect(q('add', 'a.test.ts::::b', '--bogus', 'x').status).toBe(2);
|
||||
expect(q('frobnicate').status).toBe(2);
|
||||
});
|
||||
|
||||
test('report: expired first, then soonest-expiring; the exit rule needs 10 consecutive clean passes from the ledgers', () => {
|
||||
const reg = emptyRegistry();
|
||||
add(reg, 'a.test.ts::::t0', { days: 14 });
|
||||
add(reg, 'a.test.ts::::t1', { days: 2 });
|
||||
add(reg, 'a.test.ts::::t2', { days: 1 }, at(-5));
|
||||
const without = reportLines(reg, NOW);
|
||||
expect(without.map(l => l.split(' ').slice(0, 2).join(' '))).toEqual(['FLAKY_QUARANTINE_EXPIRED: a.test.ts::::t2', 'QUARANTINE: a.test.ts::::t1', 'QUARANTINE: a.test.ts::::t0', 'QUARANTINE_REPORT: live=2']);
|
||||
expect(without[1]).toContain('days_left=2');
|
||||
expect(without[1]).toContain("EXPIRING (renew: gstack-flaky quarantine renew 'a.test.ts::::t1'");
|
||||
expect(without.at(-1)).toBe('QUARANTINE_REPORT: live=2 expired=1 expiring=1 ready=unchecked exit_rule=not checked (no --ledger given)');
|
||||
const dir = tmp();
|
||||
fs.mkdirSync(path.join(dir, 'run-1', 'flaky-junit-ledger-e2e'), { recursive: true });
|
||||
const lineFor = (id: string, i: number, kind = 'quarantined-pass', attempts = ['pass']) => JSON.stringify({ ts: new Date(NOW.getTime() + i * 1000).toISOString(), kind, id, attempts });
|
||||
const t0 = [lineFor('a.test.ts::::t0', 0, 'hard-break', ['fail', 'fail', 'fail']), ...Array.from({ length: 10 }, (_, i) => lineFor('a.test.ts::::t0', i + 1))];
|
||||
const t1 = [...Array.from({ length: 9 }, (_, i) => lineFor('a.test.ts::::t1', i)), lineFor('a.test.ts::::t1', 9, 'quarantined-pass', ['fail', 'pass'])];
|
||||
fs.writeFileSync(path.join(dir, 'run-1', 'flaky-junit-ledger-e2e', 'flaky-junit-ledger.jsonl'), [...t0, ...t1, 'not json'].join('\n') + '\n');
|
||||
const ledgers = readLedgers([dir]);
|
||||
expect([ledgers.records.length, ledgers.skipped]).toEqual([21, 1]);
|
||||
const withLedger = reportLines(reg, NOW, ledgers.records);
|
||||
expect(withLedger.find(l => l.includes('t0 '))).toContain("recent=pass,pass,pass,pass,pass clean_streak=10 READY_TO_RELEASE: gstack-flaky quarantine release 'a.test.ts::::t0'");
|
||||
expect(withLedger.find(l => l.includes('t1 '))).toContain('recent=pass,pass,pass,pass,flaky clean_streak=0');
|
||||
expect(withLedger.at(-1)).toBe('QUARANTINE_REPORT: live=2 expired=1 expiring=1 ready=1 exit_rule=checked (10 consecutive clean passes)');
|
||||
});
|
||||
|
||||
test('every FLAKY_QUARANTINE_ code is in the docs table, and the docs say the registries share no schema with CASE_QUARANTINE', () => {
|
||||
const doc = fs.readFileSync(path.join(ROOT, 'docs', 'flaky.md'), 'utf-8');
|
||||
const src = fs.readFileSync(path.join(ROOT, 'lib', 'quarantine.ts'), 'utf-8') + fs.readFileSync(path.join(ROOT, 'bin', 'gstack-flaky'), 'utf-8');
|
||||
const codes = [...new Set(src.match(/FLAKY_QUARANTINE_[A-Z_]+/g))];
|
||||
expect(codes.length).toBeGreaterThan(4);
|
||||
expect(codes.filter(c => !doc.includes(`| \`${c}\` |`))).toEqual([]);
|
||||
expect(doc).toContain('CASE_QUARANTINE');
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user