mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-09 18:28:15 +01:00
feat(flaky): gstack-flaky run reruns a failed JUnit test once and ledgers the flake (GSTA-158)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed
1 parent
319ef500ca
commit
5d3ccb69c3
10 files changed
+1458
No files matched your search
Executable
+110
@@ -0,0 +1,110 @@
|
||||
#!/usr/bin/env -S bun --no-env-file --config=/dev/null
|
||||
/**
|
||||
* gstack-flaky (GSTA-158): rerun a failed test once and record the flake, for
|
||||
* any runner that writes JUnit (Playwright, bun, jest, pytest, vitest). CREW
|
||||
* rule 16 for every repository: "you can just run the failed test again and
|
||||
* note that it is flaky". The rules live in lib/flaky-run.ts; docs/flaky.md is
|
||||
* the how-to and the error-code table.
|
||||
*/
|
||||
import * as fs from 'fs';
|
||||
import { DEFAULT_MAX_RERUNS, appendLedger, ledgerPath, renderResult, runFlaky, stepSummary } from '../lib/flaky-run';
|
||||
import { JunitError, readJunit, type JunitRunner } from '../lib/junit';
|
||||
|
||||
const USAGE = `Usage:
|
||||
gstack-flaky run --junit <file> [--runner playwright|bun|generic] [--max-reruns <n>]
|
||||
[--rerun-arg <template>]... [--ledger <file>] -- <test command...>
|
||||
gstack-flaky ids --junit <file> [--runner playwright|bun|generic]
|
||||
gstack-flaky --help
|
||||
|
||||
run: runs the command (an argv, no shell), reads the JUnit report it writes at
|
||||
--junit, and reruns each failed test once when the runner made no retries and
|
||||
at most --max-reruns (${DEFAULT_MAX_RERUNS}) tests failed. A rerun appends the runner's
|
||||
selection to the same command: playwright adds <file> --grep <title> --project
|
||||
<p>; bun adds <file> --test-name-pattern <title>; generic appends each
|
||||
--rerun-arg with {file}, {title} and {pattern} substituted. A test that passes
|
||||
on rerun is a flaky pass: green, and one line on the ledger (--ledger, else
|
||||
GSTACK_FLAKE_LEDGER, else <state root>/flake-ledger-junit.jsonl). The command
|
||||
runs with PLAYWRIGHT_JUNIT_INCLUDE_RETRIES=1 so Playwright's own retries are
|
||||
reported.
|
||||
|
||||
ids: prints the id of every test in a report (<file>::<project>::<title>).
|
||||
|
||||
Output: one FLAKY: line per test that failed at least once, then
|
||||
FLAKY_RESULT: tests=<n> failed=<n> flaky=<n> reruns=<n> command_exit=<n> verdict=pass|fail ledger=<file|none>
|
||||
Exit: 0 pass (flaky passes included), 1 a test failed, 2 usage,
|
||||
3 no usable report or a crash (FLAKY_<CODE> and a fix: line on stderr).
|
||||
`;
|
||||
|
||||
const FIX: Record<string, string> = {
|
||||
FLAKY_NO_RESULTS: 'make the command write JUnit to the --junit path (playwright: PLAYWRIGHT_JUNIT_OUTPUT_NAME=<file> with --reporter=junit; bun: --reporter=junit --reporter-outfile=<file>)',
|
||||
FLAKY_JUNIT_INVALID: 'the report is not well-formed XML; check that one process writes it and nothing else appends to it',
|
||||
FLAKY_RETRIES_HIDDEN: 'set PLAYWRIGHT_JUNIT_INCLUDE_RETRIES=1 where the report is written',
|
||||
FLAKY_COMMAND_FAILED: 'read the command output above: the suite crashed or a non-test step failed; that is a real red',
|
||||
FLAKY_COMMAND_NOT_STARTED: 'check the command after -- (it runs without a shell; wrap it as bash -c "<script>" when it needs one)',
|
||||
FLAKY_RERUN_UNSUPPORTED: 'pass --runner playwright|bun, or --rerun-arg templates that select one test (for example --rerun-arg {file} --rerun-arg -k --rerun-arg {title})',
|
||||
};
|
||||
|
||||
function die(msg: string, code = 2): never {
|
||||
process.stderr.write(`gstack-flaky: ${msg}\n`);
|
||||
process.exit(code);
|
||||
}
|
||||
|
||||
function coded(e: JunitError): never {
|
||||
process.stderr.write(`gstack-flaky: ${e.message} (${e.code})\nfix: ${FIX[e.code] ?? 'see docs/flaky.md'}\n`);
|
||||
process.exit(3);
|
||||
}
|
||||
|
||||
function parse(argv: string[]) {
|
||||
const o: { junit?: string; runner?: JunitRunner; maxReruns: number; rerunArgs: string[]; ledger?: string; command: string[] } = { maxReruns: DEFAULT_MAX_RERUNS, rerunArgs: [], command: [] };
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i]!;
|
||||
const val = () => { const v = argv[++i]; if (v === undefined) die(`${a} needs a value`); return v; };
|
||||
if (a === '--') { o.command = argv.slice(i + 1); break; }
|
||||
if (a === '--junit') o.junit = val();
|
||||
else if (a === '--runner') {
|
||||
const r = val();
|
||||
if (r !== 'playwright' && r !== 'bun' && r !== 'generic') die(`--runner must be playwright, bun or generic, got ${JSON.stringify(r)}`);
|
||||
o.runner = r;
|
||||
} else if (a === '--max-reruns') {
|
||||
const n = Number(val());
|
||||
if (!Number.isInteger(n) || n < 0) die('--max-reruns must be a whole number');
|
||||
o.maxReruns = n;
|
||||
} else if (a === '--rerun-arg') o.rerunArgs.push(val());
|
||||
else if (a === '--ledger') o.ledger = val();
|
||||
else die(`unknown option ${a}\n${USAGE}`);
|
||||
}
|
||||
return o;
|
||||
}
|
||||
|
||||
const [sub, ...rest] = process.argv.slice(2);
|
||||
if (!sub || sub === '--help' || sub === '-h') { process.stdout.write(USAGE); process.exit(sub ? 0 : 2); }
|
||||
|
||||
if (sub === 'ids') {
|
||||
const o = parse(rest);
|
||||
if (!o.junit) die('ids needs --junit <file>');
|
||||
try {
|
||||
for (const t of readJunit(o.junit, o.runner).tests) process.stdout.write(`${t.id}\n`);
|
||||
} catch (e) { if (e instanceof JunitError) coded(e); throw e; }
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (sub !== 'run') die(`unknown command ${sub}\n${USAGE}`);
|
||||
const o = parse(rest);
|
||||
if (!o.junit) die('run needs --junit <file>');
|
||||
if (!o.command.length) die('run needs the test command after --');
|
||||
let result;
|
||||
try {
|
||||
result = runFlaky({ command: o.command, junit: o.junit, runner: o.runner, maxReruns: o.maxReruns, rerunArgs: o.rerunArgs });
|
||||
} catch (e) { if (e instanceof JunitError) coded(e); throw e; }
|
||||
const ledger = ledgerPath(o.ledger);
|
||||
appendLedger(result.ledger, ledger, s => process.stderr.write(`gstack-flaky: ${s}\n`));
|
||||
for (const l of renderResult(result, ledger)) process.stdout.write(`${l}\n`);
|
||||
const summary = process.env.GITHUB_STEP_SUMMARY && stepSummary(result);
|
||||
if (summary) { try { fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY!, summary); } catch { /* a summary is a convenience, never the verdict */ } }
|
||||
if (process.env.GITHUB_ACTIONS === 'true') {
|
||||
for (const f of result.outcomes.filter(x => x.status === 'flaky-pass')) process.stdout.write(`::warning title=flaky test::${f.test.id} passed on attempt ${f.attempts.length}\n`);
|
||||
}
|
||||
if (result.code) {
|
||||
process.stderr.write(`gstack-flaky: ${result.message} (${result.code})\nfix: ${FIX[result.code] ?? 'see docs/flaky.md'}\n`);
|
||||
}
|
||||
process.exit(result.exitCode);
|
||||
@@ -1409,6 +1409,11 @@ prints a run's ledgers as a table. A flaky file is fixed at source; the retry
|
||||
never masks it. `bun run eval:pass-rates` lists only the local ledger
|
||||
(`flakeLedgerPath()`, overridden by `GSTACK_FLAKE_LEDGER`).
|
||||
|
||||
Other repositories get the same rule per test with `bin/gstack-flaky run`
|
||||
([docs/flaky.md](flaky.md)): it reads a JUnit report from any runner, reruns
|
||||
each failed test once, and writes `flaky-pass` and `rerun-fail` lines to its
|
||||
own ledger (`flake-ledger-junit.jsonl`).
|
||||
|
||||
The aggregate job of `free-tests.yml` downloads every `flake-ledger-*` artifact
|
||||
of the run and appends `bun run scripts/test-health-report.ts flake-summary <dir>`
|
||||
to its job summary. The Windows lane uploads `flake-ledger-windows-<shard>`.
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
# Flaky tests: rerun once and record it (`gstack-flaky`)
|
||||
|
||||
`gstack-flaky run` runs your test command, reads the JUnit report it writes,
|
||||
and reruns each failed test once. A test that passes on its rerun is a
|
||||
**flaky pass**: the job stays green and the flake is recorded on a ledger, so
|
||||
the daily flaky pass can fix it at the source. A test that fails twice is red.
|
||||
This is CREW rule 16 ("run the failed test again and note that it is flaky")
|
||||
for any runner that writes JUnit: Playwright, `bun test`, jest, vitest and
|
||||
pytest. gstack's own free suite keeps its file-level retry
|
||||
(`GSTACK_FREE_RETRY_FLAKY`, [Flake ledger](TESTING_INTERNALS.md#flake-ledger)).
|
||||
|
||||
## Get started in GitHub Actions
|
||||
|
||||
```yaml
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
with: { bun-version: 1.4.4 }
|
||||
- name: Install gstack-flaky (pinned)
|
||||
env: { GSTACK_REF: <a gstack commit sha you have tested> }
|
||||
run: |
|
||||
git clone --quiet --filter=blob:none https://github.com/garrytan/gstack.git "$RUNNER_TEMP/gstack"
|
||||
git -C "$RUNNER_TEMP/gstack" checkout --quiet "$GSTACK_REF"
|
||||
- name: E2E
|
||||
env:
|
||||
PLAYWRIGHT_JUNIT_OUTPUT_NAME: results/junit.xml
|
||||
GSTACK_FLAKE_LEDGER: ${{ runner.temp }}/flake-ledger.jsonl
|
||||
run: "$RUNNER_TEMP/gstack/bin/gstack-flaky" run --junit results/junit.xml -- npx playwright test --reporter=junit
|
||||
- if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with: { name: flake-ledger-e2e, path: "${{ runner.temp }}/flake-ledger.jsonl", if-no-files-found: ignore, retention-days: 90 }
|
||||
```
|
||||
|
||||
Pin a gstack commit you have tested; the bin needs only Bun and git. In Actions it also writes a step-summary table of the flaky passes and
|
||||
one `::warning` annotation per flaky test, so they are visible on the run
|
||||
before anyone reads the ledger.
|
||||
|
||||
## Runners
|
||||
|
||||
| Runner | Command after `--` | Notes |
|
||||
|---|---|---|
|
||||
| Playwright | `npx playwright test --reporter=junit` with `PLAYWRIGHT_JUNIT_OUTPUT_NAME=<file>` | `gstack-flaky` sets `PLAYWRIGHT_JUNIT_INCLUDE_RETRIES=1` so Playwright's own `retries` are reported. With retries configured, those attempts stand and nothing is rerun again. A rerun adds `<file> --grep <title> --project <project>`. |
|
||||
| bun | `bun test --reporter=junit --reporter-outfile=<file>` | A rerun adds `<file> --test-name-pattern <title>`. |
|
||||
| jest, vitest, pytest (`--runner generic`) | your command, writing JUnit | Give the rerun selection with `--rerun-arg` templates, for example pytest: `--rerun-arg {file} --rerun-arg -k --rerun-arg {title}`. `{file}`, `{title}` and `{pattern}` (the anchored, regex-escaped title) each fill one argument. |
|
||||
|
||||
The command after `--` runs without a shell. Wrap it as
|
||||
`bash -c '<script>'` when it needs one. A rerun runs the same command with the
|
||||
selection arguments appended, each one argument, so a title with shell or
|
||||
regex characters selects only itself.
|
||||
|
||||
## Rules
|
||||
|
||||
- **One rerun, small failures only.** A failed test gets one rerun when the
|
||||
runner made no retries and at most `--max-reruns` (3) tests failed. More
|
||||
failures than that is a real red, and nothing is rerun.
|
||||
- **Never green by accident.** The report is deleted before every attempt, and
|
||||
a report that attempt did not write counts as missing. A missing, empty,
|
||||
zero-test or malformed report fails, as does a non-zero exit with no failed
|
||||
test (a crash or a non-test step). Only the targeted test is read back from
|
||||
a rerun.
|
||||
- **Ids.** A test is `<file>::<project>::<describe > title>`, with the project
|
||||
empty outside Playwright, so a test in two browser projects is two tests.
|
||||
`gstack-flaky ids --junit <file>` prints them.
|
||||
- **Ledger.** One JSON line per flaky pass (`kind: flaky-pass`) and per test
|
||||
that failed its rerun (`rerun-fail`), with the tree, commit, branch, run URL,
|
||||
attempts and the first lines of the error. Failure text goes through gstack's
|
||||
redaction engine first; when the scan cannot run, the excerpt is withheld.
|
||||
The path is `--ledger`, else `GSTACK_FLAKE_LEDGER`, else
|
||||
`<state root>/flake-ledger-junit.jsonl`, and `FLAKY_RESULT` names it. A
|
||||
ledger write error is a warning and never changes the verdict.
|
||||
|
||||
## Output and exit codes
|
||||
|
||||
One `FLAKY: <flaky-pass|fail|rerun-fail> <id> attempts=<…> (<reason>)` line per
|
||||
test that failed at least once, then
|
||||
`FLAKY_RESULT: tests=<n> failed=<n> flaky=<n> reruns=<n> command_exit=<n> verdict=pass|fail ledger=<file|none>`.
|
||||
|
||||
| Exit | Meaning |
|
||||
|---|---|
|
||||
| 0 | Every test passed, flaky passes included |
|
||||
| 1 | A test failed (after its rerun, or with no rerun) |
|
||||
| 2 | Usage error |
|
||||
| 3 | No usable report, or the command crashed (a code below on stderr) |
|
||||
|
||||
| Code | Cause | Fix |
|
||||
|---|---|---|
|
||||
| `FLAKY_NO_RESULTS` | No report at `--junit`, an empty one, or one with no testcase | Make the command write JUnit to that path |
|
||||
| `FLAKY_JUNIT_INVALID` | The report is not well-formed XML | Make sure one process writes it and nothing appends to it |
|
||||
| `FLAKY_RETRIES_HIDDEN` | Playwright retried but its report omits the retries | Set `PLAYWRIGHT_JUNIT_INCLUDE_RETRIES=1` where the report is written |
|
||||
| `FLAKY_COMMAND_FAILED` | Non-zero exit with no failed test | Read the command output: a real red |
|
||||
| `FLAKY_COMMAND_NOT_STARTED` | The command could not be started | Check the argv after `--` |
|
||||
| `FLAKY_RERUN_UNSUPPORTED` | A generic runner without `--rerun-arg` | Add `--runner` or `--rerun-arg` templates |
|
||||
| `FLAKY_LEDGER_WRITE_FAILED` | The ledger could not be appended (warning) | Check the path; the verdict stands |
|
||||
@@ -0,0 +1,226 @@
|
||||
/**
|
||||
* gstack-flaky run (GSTA-158): run a test command, read its JUnit, rerun each
|
||||
* failed test once, and record flaky passes on a ledger. This is CREW rule 16
|
||||
* ("run the failed test again and note that it is flaky") for any runner that
|
||||
* writes JUnit, the per-test form of the free suite's GSTACK_FREE_RETRY_FLAKY.
|
||||
*
|
||||
* Rules (docs/flaky.md):
|
||||
* - A failed test gets one rerun when the runner made no retries of its own and
|
||||
* at most maxReruns tests failed; more failures are a real red and nothing
|
||||
* is rerun. A rerun that passes makes the test a flaky pass (green, ledgered).
|
||||
* - When the runner retried (Playwright `retries` with retry-aware JUnit), its
|
||||
* attempts stand and nothing is rerun again.
|
||||
* - The test command is an argv run without a shell. A rerun appends the
|
||||
* adapter's selection arguments, each one argv element, and only the
|
||||
* targeted test is read back from the rerun's report, and a rerun counts as
|
||||
* a pass only when its command also exited 0.
|
||||
* - The JUnit file is removed before every attempt; a report that attempt did
|
||||
* not write is missing. Missing, unreadable or empty JUnit is never green, and
|
||||
* a non-zero exit with no failed test is a crash, never green.
|
||||
*/
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { spawnSync } from 'child_process';
|
||||
import { finalAttempt, JunitError, readJunit, type JunitReport, type JunitRunner, type JunitTest } from './junit';
|
||||
import { redactFindingSpans } from './redact-engine';
|
||||
import { resolveStateRoot } from './state-root';
|
||||
|
||||
export type LedgerKind = 'flaky-pass' | 'rerun-fail';
|
||||
|
||||
export interface LedgerLine {
|
||||
ts: string;
|
||||
runner: 'junit';
|
||||
kind: LedgerKind;
|
||||
id: string;
|
||||
file: string;
|
||||
/** 'runner' when the runner's own retries decided, 'gstack-flaky' for our rerun. */
|
||||
rerun_by: 'runner' | 'gstack-flaky';
|
||||
attempts: string[];
|
||||
tree?: string;
|
||||
git_sha?: string;
|
||||
branch?: string;
|
||||
run_url?: string;
|
||||
excerpt: string[];
|
||||
}
|
||||
|
||||
export interface RunOptions {
|
||||
command: string[];
|
||||
junit: string;
|
||||
runner?: JunitRunner;
|
||||
maxReruns: number;
|
||||
/** Generic adapter: argv templates appended on rerun, {file} {title} {pattern} substituted. */
|
||||
rerunArgs?: string[];
|
||||
cwd?: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
spawn?: (argv: string[], env: NodeJS.ProcessEnv, cwd: string) => number;
|
||||
now?: () => Date;
|
||||
}
|
||||
|
||||
export interface TestOutcome {
|
||||
test: JunitTest;
|
||||
status: 'flaky-pass' | 'fail' | 'rerun-fail';
|
||||
attempts: string[];
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
export interface RunResult {
|
||||
exitCode: 0 | 1 | 3;
|
||||
code?: string;
|
||||
message?: string;
|
||||
tests: number;
|
||||
commandExit: number;
|
||||
outcomes: TestOutcome[];
|
||||
reruns: number;
|
||||
ledger: LedgerLine[];
|
||||
}
|
||||
|
||||
export const DEFAULT_MAX_RERUNS = 3;
|
||||
|
||||
const escapeRegex = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
|
||||
/** Selection arguments a rerun appends for one test, each one argv element. */
|
||||
export function rerunSelection(runner: JunitRunner, t: JunitTest, generic: string[] = []): string[] {
|
||||
if (runner === 'playwright') {
|
||||
const args = [escapeRegex(t.file), '--grep', `(?:^|\\s)${escapeRegex(t.path.join(' '))}(?:\\s|$)`];
|
||||
return t.project ? [...args, '--project', t.project] : args;
|
||||
}
|
||||
if (runner === 'bun') return [t.file, '--test-name-pattern', `^${escapeRegex(t.path.join(' '))}$`];
|
||||
if (!generic.length) throw new JunitError('FLAKY_RERUN_UNSUPPORTED', 'a generic JUnit runner needs --rerun-arg templates to select one test');
|
||||
const title = t.path.join(' ');
|
||||
const pattern = `^${escapeRegex(title)}$`;
|
||||
return generic.map(a => a.replaceAll('{file}', t.file).replaceAll('{title}', title).replaceAll('{pattern}', pattern));
|
||||
}
|
||||
|
||||
function defaultSpawn(argv: string[], env: NodeJS.ProcessEnv, cwd: string): number {
|
||||
const r = spawnSync(argv[0]!, argv.slice(1), { stdio: 'inherit', env, cwd });
|
||||
if (r.error) throw new JunitError('FLAKY_COMMAND_NOT_STARTED', `could not start ${argv[0]}: ${r.error.message}`);
|
||||
return r.status ?? (r.signal ? 128 : 1);
|
||||
}
|
||||
|
||||
function git(args: string[], cwd: string): string | undefined {
|
||||
const r = spawnSync('git', args, { cwd, encoding: 'utf-8', timeout: 10_000 });
|
||||
return r.status === 0 ? r.stdout.trim() || undefined : undefined;
|
||||
}
|
||||
|
||||
export function provenance(cwd: string, env: NodeJS.ProcessEnv): Pick<LedgerLine, 'tree' | 'git_sha' | 'branch' | 'run_url'> {
|
||||
const runUrl = env.GITHUB_SERVER_URL && env.GITHUB_REPOSITORY && env.GITHUB_RUN_ID
|
||||
? `${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}${env.GITHUB_RUN_ATTEMPT ? `/attempts/${env.GITHUB_RUN_ATTEMPT}` : ''}`
|
||||
: undefined;
|
||||
return {
|
||||
tree: git(['rev-parse', 'HEAD^{tree}'], cwd),
|
||||
git_sha: git(['rev-parse', 'HEAD'], cwd),
|
||||
branch: env.GITHUB_HEAD_REF || env.GITHUB_REF_NAME || git(['rev-parse', '--abbrev-ref', 'HEAD'], cwd),
|
||||
run_url: runUrl,
|
||||
};
|
||||
}
|
||||
|
||||
/** Failure text leaves this process only through the redaction engine; when it cannot scan, nothing leaves. */
|
||||
export function safeExcerpt(lines: string[]): string[] {
|
||||
if (!lines.length) return [];
|
||||
const redacted = redactFindingSpans(lines.join('\n'));
|
||||
return redacted === null ? ['[excerpt withheld: the redaction scan could not run]'] : redacted.split('\n');
|
||||
}
|
||||
|
||||
/** One attempt: remove the report, run, and read a report this attempt wrote. */
|
||||
function attempt(opts: RunOptions, argv: string[], cwd: string, env: NodeJS.ProcessEnv): { exit: number; report?: JunitReport; error?: JunitError } {
|
||||
const junit = path.resolve(cwd, opts.junit);
|
||||
fs.rmSync(junit, { force: true });
|
||||
const exit = (opts.spawn ?? defaultSpawn)(argv, env, cwd);
|
||||
if (!fs.existsSync(junit)) return { exit, error: new JunitError('FLAKY_NO_RESULTS', `the command wrote no JUnit report at ${opts.junit} (exit ${exit})`) };
|
||||
try { return { exit, report: readJunit(junit, opts.runner) }; } catch (e) {
|
||||
if (e instanceof JunitError) return { exit, error: e };
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
export function runFlaky(opts: RunOptions): RunResult {
|
||||
const cwd = opts.cwd ?? process.cwd();
|
||||
const env = { ...(opts.env ?? process.env), PLAYWRIGHT_JUNIT_INCLUDE_RETRIES: '1' };
|
||||
const now = opts.now ?? (() => new Date());
|
||||
const empty = { tests: 0, outcomes: [], reruns: 0, ledger: [] };
|
||||
const first = attempt(opts, opts.command, cwd, env);
|
||||
if (!first.report) return { exitCode: 3, code: first.error!.code, message: first.error!.message, commandExit: first.exit, ...empty };
|
||||
const report = first.report;
|
||||
if (report.retriesHidden) {
|
||||
return { exitCode: 3, code: 'FLAKY_RETRIES_HIDDEN', commandExit: first.exit, ...empty, tests: report.tests.length,
|
||||
message: 'Playwright retried but its JUnit omits the retries; set PLAYWRIGHT_JUNIT_INCLUDE_RETRIES=1 (gstack-flaky run sets it for the command it starts)' };
|
||||
}
|
||||
const failed = report.tests.filter(t => finalAttempt(t) === 'fail');
|
||||
const runnerFlaky = report.tests.filter(t => finalAttempt(t) === 'pass' && t.attempts.includes('fail'));
|
||||
if (first.exit !== 0 && failed.length === 0) {
|
||||
return { exitCode: 3, code: 'FLAKY_COMMAND_FAILED', commandExit: first.exit, ...empty, tests: report.tests.length,
|
||||
message: `the command exited ${first.exit} but its JUnit reports no failed test; a crash or a non-test step failed` };
|
||||
}
|
||||
const prov = provenance(cwd, env);
|
||||
const line = (t: JunitTest, kind: LedgerKind, by: LedgerLine['rerun_by'], attempts: string[]): LedgerLine =>
|
||||
({ ts: now().toISOString(), runner: 'junit', kind, id: t.id, file: t.file, rerun_by: by, attempts, ...prov, excerpt: safeExcerpt(t.excerpt) });
|
||||
const ledger: LedgerLine[] = runnerFlaky.map(t => line(t, 'flaky-pass', 'runner', t.attempts));
|
||||
const outcomes: TestOutcome[] = runnerFlaky.map(t => ({ test: t, status: 'flaky-pass' as const, attempts: t.attempts }));
|
||||
let reruns = 0;
|
||||
const rerunnable = failed.length <= opts.maxReruns;
|
||||
for (const t of failed) {
|
||||
if (t.executions > 1) { outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: `it ran ${t.executions} times in this report and one failed; not rerun` }); continue; }
|
||||
if (t.attempts.length > 1) { outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: `the runner already retried (${t.attempts.length} attempts)` }); continue; }
|
||||
if (!rerunnable) { outcomes.push({ test: t, status: 'fail', attempts: t.attempts, reason: `${failed.length} tests failed, more than --max-reruns ${opts.maxReruns}; not rerun` }); continue; }
|
||||
const argv = [...opts.command, ...rerunSelection(report.runner, t, opts.rerunArgs)];
|
||||
reruns++;
|
||||
const again = attempt(opts, argv, cwd, env);
|
||||
const match = again.report?.tests.find(x => x.id === t.id);
|
||||
if (!match) {
|
||||
outcomes.push({ test: t, status: 'rerun-fail', attempts: [...t.attempts, 'fail'], reason: again.error ? `rerun: ${again.error.message}` : 'rerun: the report does not contain this test' });
|
||||
ledger.push(line(t, 'rerun-fail', 'gstack-flaky', [...t.attempts, 'fail']));
|
||||
continue;
|
||||
}
|
||||
const attempts = [...t.attempts, ...match.attempts];
|
||||
if (again.exit !== 0) {
|
||||
outcomes.push({ test: t, status: 'rerun-fail', attempts, reason: `rerun: the command exited ${again.exit}` });
|
||||
ledger.push(line(t, 'rerun-fail', 'gstack-flaky', attempts));
|
||||
} else if (finalAttempt(match) === 'pass') {
|
||||
outcomes.push({ test: t, status: 'flaky-pass', attempts });
|
||||
ledger.push(line(t, 'flaky-pass', 'gstack-flaky', attempts));
|
||||
} else {
|
||||
outcomes.push({ test: t, status: 'rerun-fail', attempts });
|
||||
ledger.push(line(t, 'rerun-fail', 'gstack-flaky', attempts));
|
||||
}
|
||||
}
|
||||
const red = outcomes.some(o => o.status !== 'flaky-pass');
|
||||
return { exitCode: red ? 1 : 0, tests: report.tests.length, commandExit: first.exit, outcomes, reruns, ledger };
|
||||
}
|
||||
|
||||
export function ledgerPath(flag: string | undefined, env: NodeJS.ProcessEnv = process.env): string {
|
||||
if (flag) return flag;
|
||||
if (env.GSTACK_FLAKE_LEDGER) return env.GSTACK_FLAKE_LEDGER;
|
||||
return path.join(resolveStateRoot(env), 'flake-ledger-junit.jsonl');
|
||||
}
|
||||
|
||||
/** Append ledger lines; an I/O error is a warning and never changes the verdict. */
|
||||
export function appendLedger(lines: LedgerLine[], file: string, warn: (s: string) => void): boolean {
|
||||
if (!lines.length) return true;
|
||||
try {
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||
fs.appendFileSync(file, lines.map(l => JSON.stringify(l)).join('\n') + '\n');
|
||||
return true;
|
||||
} catch (e) {
|
||||
warn(`FLAKY_LEDGER_WRITE_FAILED: could not append ${file} (${(e as Error).message}); the verdict is unaffected`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** Lines the bin prints: one per non-passing test, then the result line. */
|
||||
export function renderResult(r: RunResult, ledger: string): string[] {
|
||||
const out: string[] = [];
|
||||
for (const o of r.outcomes) out.push(`FLAKY: ${o.status} ${o.test.id} attempts=${o.attempts.join(',')}${o.reason ? ` (${o.reason})` : ''}`);
|
||||
const flaky = r.outcomes.filter(o => o.status === 'flaky-pass').length;
|
||||
const failed = r.outcomes.length - flaky;
|
||||
out.push(`FLAKY_RESULT: tests=${r.tests} failed=${failed} flaky=${flaky} reruns=${r.reruns} command_exit=${r.commandExit} verdict=${r.exitCode === 0 ? 'pass' : 'fail'} ledger=${r.ledger.length ? ledger : 'none'}`);
|
||||
return out;
|
||||
}
|
||||
|
||||
/** A GitHub step summary for the flaky passes, so they are seen before any report reads the ledger. */
|
||||
export function stepSummary(r: RunResult): string {
|
||||
const rows = r.outcomes.filter(o => o.status === 'flaky-pass');
|
||||
if (!rows.length) return '';
|
||||
const cell = (s: string) => s.replace(/\|/g, '\\|').replace(/\n/g, ' ');
|
||||
return ['### Flaky passes (gstack-flaky)', '', '| Test | Attempts | First error |', '|---|---|---|',
|
||||
...rows.map(o => `| \`${cell(o.test.id)}\` | ${o.attempts.join(', ')} | ${cell(safeExcerpt(o.test.excerpt)[0] ?? '')} |`), ''].join('\n');
|
||||
}
|
||||
+204
@@ -0,0 +1,204 @@
|
||||
/**
|
||||
* JUnit XML reader for gstack-flaky (GSTA-158). One reader for every runner
|
||||
* that writes JUnit: Playwright (`--reporter=junit`, retry-aware with
|
||||
* PLAYWRIGHT_JUNIT_INCLUDE_RETRIES=1), `bun test --reporter=junit`, jest,
|
||||
* pytest and vitest. No XML dependency: the subset JUnit uses (elements,
|
||||
* attributes, text, CDATA, comments, processing instructions) is tokenized
|
||||
* here.
|
||||
*
|
||||
* A test's id is `<file>::<project>::<title>`. The title is the describe path
|
||||
* plus the test name joined by " > ", so Playwright's " › " and bun's nested
|
||||
* testsuites give the same shape. The project is Playwright's project name
|
||||
* (its testsuite `hostname`) and empty for every other runner, so a test
|
||||
* that runs in two browser projects is two ids.
|
||||
*
|
||||
* Attempts: Playwright's retry-aware report lists each failed retry as
|
||||
* `rerunFailure` (final result failed) or `flakyFailure` (final result
|
||||
* passed). A Playwright failure whose text shows "Retry #" without those
|
||||
* elements means retries ran but were not reported (`retriesHidden`). A test
|
||||
* that appears more than once (repeat-each, a duplicate name) fails when any
|
||||
* of its executions failed.
|
||||
*/
|
||||
import * as fs from 'fs';
|
||||
|
||||
export type JunitRunner = 'playwright' | 'bun' | 'generic';
|
||||
export type Attempt = 'pass' | 'fail' | 'skip';
|
||||
|
||||
export interface JunitTest {
|
||||
id: string;
|
||||
file: string;
|
||||
project: string;
|
||||
/** Describe path plus test name. */
|
||||
path: string[];
|
||||
attempts: Attempt[];
|
||||
/** Testcase elements with this id in the report (more than one: repeat-each or a duplicate name). */
|
||||
executions: number;
|
||||
/** First lines of the final failure, or of the first failed attempt. */
|
||||
excerpt: string[];
|
||||
}
|
||||
|
||||
export interface JunitReport {
|
||||
runner: JunitRunner;
|
||||
tests: JunitTest[];
|
||||
retriesHidden: boolean;
|
||||
}
|
||||
|
||||
export class JunitError extends Error {
|
||||
constructor(readonly code: string, message: string) { super(message); }
|
||||
}
|
||||
|
||||
interface XmlNode { name: string; attrs: Record<string, string>; children: XmlNode[]; text: string; parent: XmlNode | null }
|
||||
|
||||
const ENTITIES: Record<string, string> = { lt: '<', gt: '>', amp: '&', quot: '"', apos: "'" };
|
||||
function decode(s: string): string {
|
||||
return s.replace(/&(#x[0-9a-fA-F]+|#\d+|[a-z]+);/g, (m, e: string) => {
|
||||
if (e[0] === '#') {
|
||||
const code = e[1] === 'x' ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10);
|
||||
return Number.isFinite(code) && code <= 0x10ffff ? String.fromCodePoint(code) : m;
|
||||
}
|
||||
return ENTITIES[e] ?? m;
|
||||
});
|
||||
}
|
||||
|
||||
const TAG = /^<(\/?)([A-Za-z_][\w:.-]*)((?:\s+[^\s=/>]+\s*=\s*(?:"[^"]*"|'[^']*'))*)\s*(\/?)>/;
|
||||
const ATTR = /([^\s=/>]+)\s*=\s*(?:"([^"]*)"|'([^']*)')/g;
|
||||
|
||||
export function parseXml(xml: string): XmlNode {
|
||||
const root: XmlNode = { name: '#root', attrs: {}, children: [], text: '', parent: null };
|
||||
let cur = root;
|
||||
let i = 0;
|
||||
while (i < xml.length) {
|
||||
const lt = xml.indexOf('<', i);
|
||||
if (lt < 0) { cur.text += decode(xml.slice(i)); break; }
|
||||
if (lt > i) cur.text += decode(xml.slice(i, lt));
|
||||
const rest = xml.slice(lt, lt + 9);
|
||||
if (rest.startsWith('<![CDATA[')) {
|
||||
const end = xml.indexOf(']]>', lt);
|
||||
if (end < 0) throw new JunitError('FLAKY_JUNIT_INVALID', 'unterminated CDATA section');
|
||||
cur.text += xml.slice(lt + 9, end);
|
||||
i = end + 3;
|
||||
continue;
|
||||
}
|
||||
if (rest.startsWith('<!--')) {
|
||||
const end = xml.indexOf('-->', lt);
|
||||
if (end < 0) throw new JunitError('FLAKY_JUNIT_INVALID', 'unterminated comment');
|
||||
i = end + 3;
|
||||
continue;
|
||||
}
|
||||
if (rest.startsWith('<?') || rest.startsWith('<!')) {
|
||||
const end = xml.indexOf('>', lt);
|
||||
if (end < 0) throw new JunitError('FLAKY_JUNIT_INVALID', 'unterminated declaration');
|
||||
i = end + 1;
|
||||
continue;
|
||||
}
|
||||
const m = TAG.exec(xml.slice(lt));
|
||||
if (!m) throw new JunitError('FLAKY_JUNIT_INVALID', `malformed tag at offset ${lt}`);
|
||||
const [whole, close, name, attrText, selfClose] = m;
|
||||
if (close) {
|
||||
if (cur.name !== name) throw new JunitError('FLAKY_JUNIT_INVALID', `</${name}> closes <${cur.name}> at offset ${lt}`);
|
||||
cur = cur.parent!;
|
||||
} else {
|
||||
const attrs: Record<string, string> = {};
|
||||
for (const a of attrText!.matchAll(ATTR)) attrs[a[1]!] = decode(a[2] ?? a[3] ?? '');
|
||||
const node: XmlNode = { name: name!, attrs, children: [], text: '', parent: cur };
|
||||
cur.children.push(node);
|
||||
if (!selfClose) cur = node;
|
||||
}
|
||||
i = lt + whole!.length;
|
||||
}
|
||||
if (cur !== root) throw new JunitError('FLAKY_JUNIT_INVALID', `<${cur.name}> is never closed`);
|
||||
return root;
|
||||
}
|
||||
|
||||
function allNodes(node: XmlNode, name: string, out: XmlNode[] = []): XmlNode[] {
|
||||
for (const c of node.children) {
|
||||
if (c.name === name) out.push(c);
|
||||
allNodes(c, name, out);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function detectRunner(root: XmlNode): JunitRunner {
|
||||
const top = root.children.find(c => c.name === 'testsuites');
|
||||
if (top?.attrs.name === 'bun test') return 'bun';
|
||||
const cases = allNodes(root, 'testcase');
|
||||
if (cases.some(c => (c.attrs.name ?? '').includes(' › '))) return 'playwright';
|
||||
const suites = allNodes(root, 'testsuite');
|
||||
if (suites.length && suites.every(s => 'timestamp' in s.attrs && 'hostname' in s.attrs && !('file' in s.attrs))
|
||||
&& cases.every(c => c.parent?.name === 'testsuite' && c.attrs.classname === c.parent.attrs.name)) return 'playwright';
|
||||
return 'generic';
|
||||
}
|
||||
|
||||
function excerptOf(node: XmlNode | undefined): string[] {
|
||||
if (!node) return [];
|
||||
const body = (node.attrs.message ? node.attrs.message + '\n' : '') + node.text;
|
||||
return body.replace(/\x1b\[[0-9;]*m/g, '').split('\n').map(l => l.trimEnd()).filter(l => l.trim()).slice(0, 8);
|
||||
}
|
||||
|
||||
export const testId = (file: string, project: string, path: string[]) => `${file}::${project}::${path.join(' > ')}`;
|
||||
|
||||
function readCase(tc: XmlNode, runner: JunitRunner): Omit<JunitTest, 'id' | 'executions'> & { hidden: boolean } {
|
||||
const suite = tc.parent!;
|
||||
let file: string, project = '', path: string[];
|
||||
if (runner === 'playwright') {
|
||||
file = tc.attrs.classname || suite.attrs.name || '';
|
||||
project = suite.attrs.hostname ?? '';
|
||||
path = (tc.attrs.name ?? '').split(' › ');
|
||||
} else if (runner === 'bun') {
|
||||
file = tc.attrs.file ?? '';
|
||||
path = [];
|
||||
for (let s: XmlNode | null = suite; s && s.name === 'testsuite'; s = s.parent) {
|
||||
if ('file' in s.attrs && s.attrs.name === s.attrs.file && s.parent?.name !== 'testsuite') break;
|
||||
path.unshift(s.attrs.name ?? '');
|
||||
}
|
||||
path.push(tc.attrs.name ?? '');
|
||||
} else {
|
||||
file = tc.attrs.file || tc.attrs.classname || suite.attrs.file || suite.attrs.name || '';
|
||||
path = [tc.attrs.name ?? ''];
|
||||
}
|
||||
const kids = (n: string) => tc.children.filter(c => c.name === n);
|
||||
const final = kids('failure')[0] ?? kids('error')[0];
|
||||
const skipped = kids('skipped').length > 0;
|
||||
const flaky = [...kids('flakyFailure'), ...kids('flakyError')];
|
||||
const rerun = [...kids('rerunFailure'), ...kids('rerunError')];
|
||||
let attempts: Attempt[];
|
||||
if (final) attempts = [...rerun.map(() => 'fail' as const), 'fail'];
|
||||
else if (skipped) attempts = ['skip'];
|
||||
else attempts = [...flaky.map(() => 'fail' as const), 'pass'];
|
||||
const hidden = runner === 'playwright' && !!final && rerun.length === 0 && /\bRetry #\d+/.test(final.text);
|
||||
return { file, project, path, attempts, excerpt: excerptOf(final ?? flaky[0]), hidden };
|
||||
}
|
||||
|
||||
export function parseJunit(xml: string, runner?: JunitRunner): JunitReport {
|
||||
const root = parseXml(xml);
|
||||
const kind = runner ?? detectRunner(root);
|
||||
const byId = new Map<string, JunitTest>();
|
||||
let retriesHidden = false;
|
||||
for (const tc of allNodes(root, 'testcase')) {
|
||||
const { hidden, ...t } = readCase(tc, kind);
|
||||
retriesHidden ||= hidden;
|
||||
const id = testId(t.file, t.project, t.path);
|
||||
const seen = byId.get(id);
|
||||
if (seen) {
|
||||
const failed = finalAttempt(seen) === 'fail' || t.attempts.at(-1) === 'fail';
|
||||
seen.attempts = [...seen.attempts, ...t.attempts].sort((a, b) => failed ? Number(a === 'fail') - Number(b === 'fail') : 0);
|
||||
seen.executions++;
|
||||
if (!seen.excerpt.length) seen.excerpt = t.excerpt;
|
||||
} else byId.set(id, { id, ...t, executions: 1 });
|
||||
}
|
||||
if (byId.size === 0) throw new JunitError('FLAKY_NO_RESULTS', 'the JUnit report holds no testcase');
|
||||
return { runner: kind, tests: [...byId.values()], retriesHidden };
|
||||
}
|
||||
|
||||
/** Read and parse one JUnit file; missing, unreadable and empty files are errors, never green. */
|
||||
export function readJunit(file: string, runner?: JunitRunner): JunitReport {
|
||||
let xml: string;
|
||||
try { xml = fs.readFileSync(file, 'utf-8'); } catch (e) {
|
||||
throw new JunitError('FLAKY_NO_RESULTS', `cannot read ${file}: ${(e as Error).message}`);
|
||||
}
|
||||
if (!xml.trim()) throw new JunitError('FLAKY_NO_RESULTS', `${file} is empty`);
|
||||
return parseJunit(xml, runner);
|
||||
}
|
||||
|
||||
/** The final outcome of a test: its last attempt. */
|
||||
export const finalAttempt = (t: JunitTest): Attempt => t.attempts[t.attempts.length - 1]!;
|
||||
Vendored
+17
@@ -0,0 +1,17 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<testsuites name="bun test" tests="4" assertions="2" failures="1" skipped="1" time="0.02640424">
|
||||
<testsuite name="a.test.ts" file="a.test.ts" tests="4" assertions="2" failures="1" skipped="1" time="0.014448064" hostname="ci-host">
|
||||
<testsuite name="outer" file="a.test.ts" line="2" tests="3" assertions="2" failures="1" skipped="1" time="0" hostname="ci-host">
|
||||
<testsuite name="inner" file="a.test.ts" line="2" tests="2" assertions="2" failures="1" skipped="0" time="0.006" hostname="ci-host">
|
||||
<testcase name="passes" classname="inner > outer" time="0.000069" file="a.test.ts" line="2" assertions="1" />
|
||||
<testcase name="fails" classname="inner > outer" time="0.006322" file="a.test.ts" line="2" assertions="1">
|
||||
<failure type="AssertionError" message="expect(received).toBe(expected) Expected: 2 Received: 1 ">AssertionError: expect(received).toBe(expected) Expected: 2 Received: 1 at a.test.ts:2:131 </failure>
|
||||
</testcase>
|
||||
</testsuite>
|
||||
<testcase name="skipped" classname="outer" time="0" file="a.test.ts" line="2" assertions="0">
|
||||
<skipped />
|
||||
</testcase>
|
||||
</testsuite>
|
||||
<testcase name="top level" classname="" time="0.000039" file="a.test.ts" line="3" assertions="0" />
|
||||
</testsuite>
|
||||
</testsuites>
|
||||
+138
@@ -0,0 +1,138 @@
|
||||
<testsuites id="" name="" tests="10" failures="4" skipped="2" errors="0" time="3.23467">
|
||||
<testsuite name="a.spec.ts" timestamp="2026-10-29T16:02:06.214Z" hostname="alpha" tests="5" failures="2" skipped="1" time="0.03" errors="0">
|
||||
<testcase name="group one › passes" classname="a.spec.ts" time="0.011">
|
||||
</testcase>
|
||||
<testcase name="group one › fails always" classname="a.spec.ts" time="0.004">
|
||||
<system-out>
|
||||
<![CDATA[
|
||||
[[ATTACHMENT|pw/test-results/a-group-one-fails-always-alpha/error-context.md]]
|
||||
]]>
|
||||
</system-out>
|
||||
<failure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe">
|
||||
<![CDATA[ [alpha] › a.spec.ts:5:7 › group one › fails always ───────────────────────────────────────────────
|
||||
|
||||
Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
Expected: 2
|
||||
Received: 1
|
||||
|
||||
3 | test.describe('group one', () => {
|
||||
4 | test('passes', () => { expect(1).toBe(1); });
|
||||
> 5 | test('fails always', () => { expect(1).toBe(2); });
|
||||
| ^
|
||||
6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } });
|
||||
7 | test.skip('skipped', () => {});
|
||||
8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); });
|
||||
at /repo/tests/a.spec.ts:5:42
|
||||
|
||||
Error Context: ../test-results/a-group-one-fails-always-alpha/error-context.md
|
||||
]]>
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase name="group one › flaky once" classname="a.spec.ts" time="0.009">
|
||||
<system-out>
|
||||
<![CDATA[
|
||||
[[ATTACHMENT|pw/test-results/a-group-one-flaky-once-alpha/error-context.md]]
|
||||
]]>
|
||||
</system-out>
|
||||
<failure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe">
|
||||
<![CDATA[ [alpha] › a.spec.ts:6:7 › group one › flaky once ─────────────────────────────────────────────────
|
||||
|
||||
Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
Expected: "second"
|
||||
Received: "first"
|
||||
|
||||
4 | test('passes', () => { expect(1).toBe(1); });
|
||||
5 | test('fails always', () => { expect(1).toBe(2); });
|
||||
> 6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } });
|
||||
| ^
|
||||
7 | test.skip('skipped', () => {});
|
||||
8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); });
|
||||
9 | });
|
||||
at /repo/tests/a.spec.ts:6:152
|
||||
|
||||
Error Context: ../test-results/a-group-one-flaky-once-alpha/error-context.md
|
||||
]]>
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase name="group one › skipped" classname="a.spec.ts">
|
||||
<properties>
|
||||
<property name="skip" value="">
|
||||
</property>
|
||||
</properties>
|
||||
<skipped>
|
||||
</skipped>
|
||||
</testcase>
|
||||
<testcase name="group one › title with $(rm -rf) [x] (y)? *" classname="a.spec.ts" time="0.006">
|
||||
</testcase>
|
||||
</testsuite>
|
||||
<testsuite name="a.spec.ts" timestamp="2026-10-29T16:02:06.214Z" hostname="beta" tests="5" failures="2" skipped="1" time="0.027" errors="0">
|
||||
<testcase name="group one › passes" classname="a.spec.ts" time="0.005">
|
||||
</testcase>
|
||||
<testcase name="group one › fails always" classname="a.spec.ts" time="0.002">
|
||||
<system-out>
|
||||
<![CDATA[
|
||||
[[ATTACHMENT|pw/test-results/a-group-one-fails-always-beta/error-context.md]]
|
||||
]]>
|
||||
</system-out>
|
||||
<failure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe">
|
||||
<![CDATA[ [beta] › a.spec.ts:5:7 › group one › fails always ────────────────────────────────────────────────
|
||||
|
||||
Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
Expected: 2
|
||||
Received: 1
|
||||
|
||||
3 | test.describe('group one', () => {
|
||||
4 | test('passes', () => { expect(1).toBe(1); });
|
||||
> 5 | test('fails always', () => { expect(1).toBe(2); });
|
||||
| ^
|
||||
6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } });
|
||||
7 | test.skip('skipped', () => {});
|
||||
8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); });
|
||||
at /repo/tests/a.spec.ts:5:42
|
||||
|
||||
Error Context: ../test-results/a-group-one-fails-always-beta/error-context.md
|
||||
]]>
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase name="group one › flaky once" classname="a.spec.ts" time="0.013">
|
||||
<system-out>
|
||||
<![CDATA[
|
||||
[[ATTACHMENT|pw/test-results/a-group-one-flaky-once-beta/error-context.md]]
|
||||
]]>
|
||||
</system-out>
|
||||
<failure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe">
|
||||
<![CDATA[ [beta] › a.spec.ts:6:7 › group one › flaky once ──────────────────────────────────────────────────
|
||||
|
||||
Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
Expected: "second"
|
||||
Received: "first"
|
||||
|
||||
4 | test('passes', () => { expect(1).toBe(1); });
|
||||
5 | test('fails always', () => { expect(1).toBe(2); });
|
||||
> 6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } });
|
||||
| ^
|
||||
7 | test.skip('skipped', () => {});
|
||||
8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); });
|
||||
9 | });
|
||||
at /repo/tests/a.spec.ts:6:152
|
||||
|
||||
Error Context: ../test-results/a-group-one-flaky-once-beta/error-context.md
|
||||
]]>
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase name="group one › skipped" classname="a.spec.ts">
|
||||
<properties>
|
||||
<property name="skip" value="">
|
||||
</property>
|
||||
</properties>
|
||||
<skipped>
|
||||
</skipped>
|
||||
</testcase>
|
||||
<testcase name="group one › title with $(rm -rf) [x] (y)? *" classname="a.spec.ts" time="0.007">
|
||||
</testcase>
|
||||
</testsuite>
|
||||
</testsuites>
|
||||
+244
@@ -0,0 +1,244 @@
|
||||
<testsuites id="" name="" tests="10" failures="2" skipped="2" errors="0" time="4.805343">
|
||||
<testsuite name="a.spec.ts" timestamp="2026-10-29T16:02:16.450Z" hostname="alpha" tests="5" failures="1" skipped="1" time="0.043" errors="0">
|
||||
<testcase name="group one › passes" classname="a.spec.ts" time="0.007">
|
||||
</testcase>
|
||||
<testcase name="group one › fails always" classname="a.spec.ts" time="0.005">
|
||||
<system-out>
|
||||
<![CDATA[
|
||||
[[ATTACHMENT|pw/test-results/a-group-one-fails-always-alpha/error-context.md]]
|
||||
]]>
|
||||
</system-out>
|
||||
<rerunFailure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe" time="0.008">
|
||||
<stackTrace>
|
||||
<![CDATA[Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
Expected: 2
|
||||
Received: 1
|
||||
at /repo/tests/a.spec.ts:5:42]]>
|
||||
</stackTrace>
|
||||
<system-out>
|
||||
<![CDATA[
|
||||
[[ATTACHMENT|pw/test-results/a-group-one-fails-always-alpha-retry1/error-context.md]]
|
||||
]]>
|
||||
</system-out>
|
||||
</rerunFailure>
|
||||
<rerunFailure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe" time="0.009">
|
||||
<stackTrace>
|
||||
<![CDATA[Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
Expected: 2
|
||||
Received: 1
|
||||
at /repo/tests/a.spec.ts:5:42]]>
|
||||
</stackTrace>
|
||||
<system-out>
|
||||
<![CDATA[
|
||||
[[ATTACHMENT|pw/test-results/a-group-one-fails-always-alpha-retry2/error-context.md]]
|
||||
]]>
|
||||
</system-out>
|
||||
</rerunFailure>
|
||||
<failure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe">
|
||||
<![CDATA[ [alpha] › a.spec.ts:5:7 › group one › fails always ───────────────────────────────────────────────
|
||||
|
||||
Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
Expected: 2
|
||||
Received: 1
|
||||
|
||||
3 | test.describe('group one', () => {
|
||||
4 | test('passes', () => { expect(1).toBe(1); });
|
||||
> 5 | test('fails always', () => { expect(1).toBe(2); });
|
||||
| ^
|
||||
6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } });
|
||||
7 | test.skip('skipped', () => {});
|
||||
8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); });
|
||||
at /repo/tests/a.spec.ts:5:42
|
||||
|
||||
Error Context: ../test-results/a-group-one-fails-always-alpha/error-context.md
|
||||
|
||||
Retry #1 ───────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
Expected: 2
|
||||
Received: 1
|
||||
|
||||
3 | test.describe('group one', () => {
|
||||
4 | test('passes', () => { expect(1).toBe(1); });
|
||||
> 5 | test('fails always', () => { expect(1).toBe(2); });
|
||||
| ^
|
||||
6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } });
|
||||
7 | test.skip('skipped', () => {});
|
||||
8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); });
|
||||
at /repo/tests/a.spec.ts:5:42
|
||||
|
||||
Error Context: ../test-results/a-group-one-fails-always-alpha-retry1/error-context.md
|
||||
|
||||
Retry #2 ───────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
Expected: 2
|
||||
Received: 1
|
||||
|
||||
3 | test.describe('group one', () => {
|
||||
4 | test('passes', () => { expect(1).toBe(1); });
|
||||
> 5 | test('fails always', () => { expect(1).toBe(2); });
|
||||
| ^
|
||||
6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } });
|
||||
7 | test.skip('skipped', () => {});
|
||||
8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); });
|
||||
at /repo/tests/a.spec.ts:5:42
|
||||
|
||||
Error Context: ../test-results/a-group-one-fails-always-alpha-retry2/error-context.md
|
||||
]]>
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase name="group one › flaky once" classname="a.spec.ts" time="0.002">
|
||||
<flakyFailure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe" time="0.009">
|
||||
<stackTrace>
|
||||
<![CDATA[Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
Expected: "second"
|
||||
Received: "first"
|
||||
at /repo/tests/a.spec.ts:6:152]]>
|
||||
</stackTrace>
|
||||
<system-out>
|
||||
<![CDATA[
|
||||
[[ATTACHMENT|pw/test-results/a-group-one-flaky-once-alpha/error-context.md]]
|
||||
]]>
|
||||
</system-out>
|
||||
</flakyFailure>
|
||||
</testcase>
|
||||
<testcase name="group one › skipped" classname="a.spec.ts">
|
||||
<properties>
|
||||
<property name="skip" value="">
|
||||
</property>
|
||||
</properties>
|
||||
<skipped>
|
||||
</skipped>
|
||||
</testcase>
|
||||
<testcase name="group one › title with $(rm -rf) [x] (y)? *" classname="a.spec.ts" time="0.003">
|
||||
</testcase>
|
||||
</testsuite>
|
||||
<testsuite name="a.spec.ts" timestamp="2026-10-29T16:02:16.450Z" hostname="beta" tests="5" failures="1" skipped="1" time="0.033" errors="0">
|
||||
<testcase name="group one › passes" classname="a.spec.ts" time="0.004">
|
||||
</testcase>
|
||||
<testcase name="group one › fails always" classname="a.spec.ts" time="0.002">
|
||||
<system-out>
|
||||
<![CDATA[
|
||||
[[ATTACHMENT|pw/test-results/a-group-one-fails-always-beta/error-context.md]]
|
||||
]]>
|
||||
</system-out>
|
||||
<rerunFailure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe" time="0.006">
|
||||
<stackTrace>
|
||||
<![CDATA[Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
Expected: 2
|
||||
Received: 1
|
||||
at /repo/tests/a.spec.ts:5:42]]>
|
||||
</stackTrace>
|
||||
<system-out>
|
||||
<![CDATA[
|
||||
[[ATTACHMENT|pw/test-results/a-group-one-fails-always-beta-retry1/error-context.md]]
|
||||
]]>
|
||||
</system-out>
|
||||
</rerunFailure>
|
||||
<rerunFailure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe" time="0.006">
|
||||
<stackTrace>
|
||||
<![CDATA[Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
Expected: 2
|
||||
Received: 1
|
||||
at /repo/tests/a.spec.ts:5:42]]>
|
||||
</stackTrace>
|
||||
<system-out>
|
||||
<![CDATA[
|
||||
[[ATTACHMENT|pw/test-results/a-group-one-fails-always-beta-retry2/error-context.md]]
|
||||
]]>
|
||||
</system-out>
|
||||
</rerunFailure>
|
||||
<failure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe">
|
||||
<![CDATA[ [beta] › a.spec.ts:5:7 › group one › fails always ────────────────────────────────────────────────
|
||||
|
||||
Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
Expected: 2
|
||||
Received: 1
|
||||
|
||||
3 | test.describe('group one', () => {
|
||||
4 | test('passes', () => { expect(1).toBe(1); });
|
||||
> 5 | test('fails always', () => { expect(1).toBe(2); });
|
||||
| ^
|
||||
6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } });
|
||||
7 | test.skip('skipped', () => {});
|
||||
8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); });
|
||||
at /repo/tests/a.spec.ts:5:42
|
||||
|
||||
Error Context: ../test-results/a-group-one-fails-always-beta/error-context.md
|
||||
|
||||
Retry #1 ───────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
Expected: 2
|
||||
Received: 1
|
||||
|
||||
3 | test.describe('group one', () => {
|
||||
4 | test('passes', () => { expect(1).toBe(1); });
|
||||
> 5 | test('fails always', () => { expect(1).toBe(2); });
|
||||
| ^
|
||||
6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } });
|
||||
7 | test.skip('skipped', () => {});
|
||||
8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); });
|
||||
at /repo/tests/a.spec.ts:5:42
|
||||
|
||||
Error Context: ../test-results/a-group-one-fails-always-beta-retry1/error-context.md
|
||||
|
||||
Retry #2 ───────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
Expected: 2
|
||||
Received: 1
|
||||
|
||||
3 | test.describe('group one', () => {
|
||||
4 | test('passes', () => { expect(1).toBe(1); });
|
||||
> 5 | test('fails always', () => { expect(1).toBe(2); });
|
||||
| ^
|
||||
6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } });
|
||||
7 | test.skip('skipped', () => {});
|
||||
8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); });
|
||||
at /repo/tests/a.spec.ts:5:42
|
||||
|
||||
Error Context: ../test-results/a-group-one-fails-always-beta-retry2/error-context.md
|
||||
]]>
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase name="group one › flaky once" classname="a.spec.ts" time="0.002">
|
||||
<flakyFailure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe" time="0.01">
|
||||
<stackTrace>
|
||||
<![CDATA[Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
Expected: "second"
|
||||
Received: "first"
|
||||
at /repo/tests/a.spec.ts:6:152]]>
|
||||
</stackTrace>
|
||||
<system-out>
|
||||
<![CDATA[
|
||||
[[ATTACHMENT|pw/test-results/a-group-one-flaky-once-beta/error-context.md]]
|
||||
]]>
|
||||
</system-out>
|
||||
</flakyFailure>
|
||||
</testcase>
|
||||
<testcase name="group one › skipped" classname="a.spec.ts">
|
||||
<properties>
|
||||
<property name="skip" value="">
|
||||
</property>
|
||||
</properties>
|
||||
<skipped>
|
||||
</skipped>
|
||||
</testcase>
|
||||
<testcase name="group one › title with $(rm -rf) [x] (y)? *" classname="a.spec.ts" time="0.003">
|
||||
</testcase>
|
||||
</testsuite>
|
||||
</testsuites>
|
||||
+178
@@ -0,0 +1,178 @@
|
||||
<testsuites id="" name="" tests="10" failures="2" skipped="2" errors="0" time="4.985760999999999">
|
||||
<testsuite name="a.spec.ts" timestamp="2026-10-29T16:02:10.182Z" hostname="alpha" tests="5" failures="1" skipped="1" time="0.044" errors="0">
|
||||
<testcase name="group one › passes" classname="a.spec.ts" time="0.005">
|
||||
</testcase>
|
||||
<testcase name="group one › fails always" classname="a.spec.ts" time="0.021">
|
||||
<system-out>
|
||||
<![CDATA[
|
||||
[[ATTACHMENT|pw/test-results/a-group-one-fails-always-alpha/error-context.md]]
|
||||
|
||||
[[ATTACHMENT|pw/test-results/a-group-one-fails-always-alpha-retry1/error-context.md]]
|
||||
|
||||
[[ATTACHMENT|pw/test-results/a-group-one-fails-always-alpha-retry2/error-context.md]]
|
||||
]]>
|
||||
</system-out>
|
||||
<failure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe">
|
||||
<![CDATA[ [alpha] › a.spec.ts:5:7 › group one › fails always ───────────────────────────────────────────────
|
||||
|
||||
Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
Expected: 2
|
||||
Received: 1
|
||||
|
||||
3 | test.describe('group one', () => {
|
||||
4 | test('passes', () => { expect(1).toBe(1); });
|
||||
> 5 | test('fails always', () => { expect(1).toBe(2); });
|
||||
| ^
|
||||
6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } });
|
||||
7 | test.skip('skipped', () => {});
|
||||
8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); });
|
||||
at /repo/tests/a.spec.ts:5:42
|
||||
|
||||
Error Context: ../test-results/a-group-one-fails-always-alpha/error-context.md
|
||||
|
||||
Retry #1 ───────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
Expected: 2
|
||||
Received: 1
|
||||
|
||||
3 | test.describe('group one', () => {
|
||||
4 | test('passes', () => { expect(1).toBe(1); });
|
||||
> 5 | test('fails always', () => { expect(1).toBe(2); });
|
||||
| ^
|
||||
6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } });
|
||||
7 | test.skip('skipped', () => {});
|
||||
8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); });
|
||||
at /repo/tests/a.spec.ts:5:42
|
||||
|
||||
Error Context: ../test-results/a-group-one-fails-always-alpha-retry1/error-context.md
|
||||
|
||||
Retry #2 ───────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
Expected: 2
|
||||
Received: 1
|
||||
|
||||
3 | test.describe('group one', () => {
|
||||
4 | test('passes', () => { expect(1).toBe(1); });
|
||||
> 5 | test('fails always', () => { expect(1).toBe(2); });
|
||||
| ^
|
||||
6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } });
|
||||
7 | test.skip('skipped', () => {});
|
||||
8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); });
|
||||
at /repo/tests/a.spec.ts:5:42
|
||||
|
||||
Error Context: ../test-results/a-group-one-fails-always-alpha-retry2/error-context.md
|
||||
]]>
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase name="group one › flaky once" classname="a.spec.ts" time="0.014">
|
||||
<system-out>
|
||||
<![CDATA[
|
||||
[[ATTACHMENT|pw/test-results/a-group-one-flaky-once-alpha/error-context.md]]
|
||||
]]>
|
||||
</system-out>
|
||||
</testcase>
|
||||
<testcase name="group one › skipped" classname="a.spec.ts">
|
||||
<properties>
|
||||
<property name="skip" value="">
|
||||
</property>
|
||||
</properties>
|
||||
<skipped>
|
||||
</skipped>
|
||||
</testcase>
|
||||
<testcase name="group one › title with $(rm -rf) [x] (y)? *" classname="a.spec.ts" time="0.004">
|
||||
</testcase>
|
||||
</testsuite>
|
||||
<testsuite name="a.spec.ts" timestamp="2026-10-29T16:02:10.182Z" hostname="beta" tests="5" failures="1" skipped="1" time="0.04" errors="0">
|
||||
<testcase name="group one › passes" classname="a.spec.ts" time="0.005">
|
||||
</testcase>
|
||||
<testcase name="group one › fails always" classname="a.spec.ts" time="0.019">
|
||||
<system-out>
|
||||
<![CDATA[
|
||||
[[ATTACHMENT|pw/test-results/a-group-one-fails-always-beta/error-context.md]]
|
||||
|
||||
[[ATTACHMENT|pw/test-results/a-group-one-fails-always-beta-retry1/error-context.md]]
|
||||
|
||||
[[ATTACHMENT|pw/test-results/a-group-one-fails-always-beta-retry2/error-context.md]]
|
||||
]]>
|
||||
</system-out>
|
||||
<failure message="expect(received).toBe(expected) // Object.is equality" type="expect.toBe">
|
||||
<![CDATA[ [beta] › a.spec.ts:5:7 › group one › fails always ────────────────────────────────────────────────
|
||||
|
||||
Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
Expected: 2
|
||||
Received: 1
|
||||
|
||||
3 | test.describe('group one', () => {
|
||||
4 | test('passes', () => { expect(1).toBe(1); });
|
||||
> 5 | test('fails always', () => { expect(1).toBe(2); });
|
||||
| ^
|
||||
6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } });
|
||||
7 | test.skip('skipped', () => {});
|
||||
8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); });
|
||||
at /repo/tests/a.spec.ts:5:42
|
||||
|
||||
Error Context: ../test-results/a-group-one-fails-always-beta/error-context.md
|
||||
|
||||
Retry #1 ───────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
Expected: 2
|
||||
Received: 1
|
||||
|
||||
3 | test.describe('group one', () => {
|
||||
4 | test('passes', () => { expect(1).toBe(1); });
|
||||
> 5 | test('fails always', () => { expect(1).toBe(2); });
|
||||
| ^
|
||||
6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } });
|
||||
7 | test.skip('skipped', () => {});
|
||||
8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); });
|
||||
at /repo/tests/a.spec.ts:5:42
|
||||
|
||||
Error Context: ../test-results/a-group-one-fails-always-beta-retry1/error-context.md
|
||||
|
||||
Retry #2 ───────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
Error: expect(received).toBe(expected) // Object.is equality
|
||||
|
||||
Expected: 2
|
||||
Received: 1
|
||||
|
||||
3 | test.describe('group one', () => {
|
||||
4 | test('passes', () => { expect(1).toBe(1); });
|
||||
> 5 | test('fails always', () => { expect(1).toBe(2); });
|
||||
| ^
|
||||
6 | test('flaky once', ({}, info) => { const f = '/tmp/pwflake-' + info.project.name; if (!fs.existsSync(f)) { fs.writeFileSync(f, '1'); expect('first').toBe('second'); } });
|
||||
7 | test.skip('skipped', () => {});
|
||||
8 | test('title with $(rm -rf) [x] (y)? *', () => { expect(1).toBe(1); });
|
||||
at /repo/tests/a.spec.ts:5:42
|
||||
|
||||
Error Context: ../test-results/a-group-one-fails-always-beta-retry2/error-context.md
|
||||
]]>
|
||||
</failure>
|
||||
</testcase>
|
||||
<testcase name="group one › flaky once" classname="a.spec.ts" time="0.012">
|
||||
<system-out>
|
||||
<![CDATA[
|
||||
[[ATTACHMENT|pw/test-results/a-group-one-flaky-once-beta/error-context.md]]
|
||||
]]>
|
||||
</system-out>
|
||||
</testcase>
|
||||
<testcase name="group one › skipped" classname="a.spec.ts">
|
||||
<properties>
|
||||
<property name="skip" value="">
|
||||
</property>
|
||||
</properties>
|
||||
<skipped>
|
||||
</skipped>
|
||||
</testcase>
|
||||
<testcase name="group one › title with $(rm -rf) [x] (y)? *" classname="a.spec.ts" time="0.004">
|
||||
</testcase>
|
||||
</testsuite>
|
||||
</testsuites>
|
||||
@@ -0,0 +1,245 @@
|
||||
/**
|
||||
* gstack-flaky run (GSTA-158): the JUnit reader against reports captured from
|
||||
* Playwright 1.64 and bun 1.4 (test/fixtures/junit, paths rewritten to /repo),
|
||||
* the rerun-once rules, and the bin end to end with a fake runner.
|
||||
*/
|
||||
import { afterAll, describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { spawnSync } from 'child_process';
|
||||
import { JunitError, parseJunit, readJunit, testId } from '../lib/junit';
|
||||
import { renderResult, rerunSelection, runFlaky, safeExcerpt, stepSummary } from '../lib/flaky-run';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const FIX = path.join(ROOT, 'test', 'fixtures', 'junit');
|
||||
const fixture = (name: string) => fs.readFileSync(path.join(FIX, name), 'utf-8');
|
||||
const TMP = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-flaky-'));
|
||||
const tmp = () => fs.mkdtempSync(path.join(TMP, 't-'));
|
||||
afterAll(() => fs.rmSync(TMP, { recursive: true, force: true }));
|
||||
|
||||
describe('JUnit reader', () => {
|
||||
test('bun: nested describe path, skip, failure excerpt', () => {
|
||||
const r = parseJunit(fixture('bun.xml'));
|
||||
expect(r.runner).toBe('bun');
|
||||
expect(r.tests.map(t => [t.id, t.attempts.join(',')])).toEqual([
|
||||
['a.test.ts::::outer > inner > passes', 'pass'],
|
||||
['a.test.ts::::outer > inner > fails', 'fail'],
|
||||
['a.test.ts::::outer > skipped', 'skip'],
|
||||
['a.test.ts::::top level', 'pass'],
|
||||
]);
|
||||
expect(r.tests[1]!.excerpt[0]).toContain('expect(received).toBe(expected)');
|
||||
});
|
||||
|
||||
test('playwright: one id per project, retry-aware attempts', () => {
|
||||
const r = parseJunit(fixture('playwright-r2-inc.xml'));
|
||||
expect(r.runner).toBe('playwright');
|
||||
expect(r.retriesHidden).toBe(false);
|
||||
const byId = Object.fromEntries(r.tests.map(t => [t.id, t.attempts.join(',')]));
|
||||
expect(byId['a.spec.ts::alpha::group one > fails always']).toBe('fail,fail,fail');
|
||||
expect(byId['a.spec.ts::beta::group one > fails always']).toBe('fail,fail,fail');
|
||||
expect(byId['a.spec.ts::alpha::group one > flaky once']).toBe('fail,pass');
|
||||
expect(byId['a.spec.ts::alpha::group one > skipped']).toBe('skip');
|
||||
expect(byId['a.spec.ts::alpha::group one > title with $(rm -rf) [x] (y)? *']).toBe('pass');
|
||||
});
|
||||
|
||||
test('playwright retries without retry data are detected, not read as one attempt', () => {
|
||||
expect(parseJunit(fixture('playwright-r2.xml')).retriesHidden).toBe(true);
|
||||
expect(parseJunit(fixture('playwright-r0.xml')).retriesHidden).toBe(false);
|
||||
});
|
||||
|
||||
test('missing, empty, zero-testcase and malformed reports are errors, never green', () => {
|
||||
const dir = tmp();
|
||||
const code = (fn: () => unknown) => { try { fn(); return 'ok'; } catch (e) { return (e as JunitError).code; } };
|
||||
expect(code(() => readJunit(path.join(dir, 'absent.xml')))).toBe('FLAKY_NO_RESULTS');
|
||||
fs.writeFileSync(path.join(dir, 'empty.xml'), ' \n');
|
||||
expect(code(() => readJunit(path.join(dir, 'empty.xml')))).toBe('FLAKY_NO_RESULTS');
|
||||
expect(code(() => parseJunit('<testsuites name="x"><testsuite name="s" tests="0"/></testsuites>'))).toBe('FLAKY_NO_RESULTS');
|
||||
expect(code(() => parseJunit('<testsuites><testsuite name="s"><testcase name="a"></testsuite></testsuites>'))).toBe('FLAKY_JUNIT_INVALID');
|
||||
});
|
||||
|
||||
test('generic runners (jest, pytest): file from the testcase, entities decoded', () => {
|
||||
const r = parseJunit('<?xml version="1.0"?><testsuites><testsuite name="pytest"><testcase classname="tests.test_a" name="test_x[a&b]" file="tests/test_a.py"><error message="boom"/></testcase></testsuite></testsuites>');
|
||||
expect(r.runner).toBe('generic');
|
||||
expect(r.tests[0]!.id).toBe('tests/test_a.py::::test_x[a&b]');
|
||||
expect(r.tests[0]!.attempts).toEqual(['fail']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('rerun selection', () => {
|
||||
const pw = parseJunit(fixture('playwright-r0.xml')).tests.find(t => t.path.at(-1)!.startsWith('title with') && t.project === 'beta')!;
|
||||
test('playwright: file and title regex-escaped, project kept, each one argv element', () => {
|
||||
expect(rerunSelection('playwright', pw)).toEqual([
|
||||
'a\\.spec\\.ts', '--grep', '(?:^|\\s)group one title with \\$\\(rm -rf\\) \\[x\\] \\(y\\)\\? \\*(?:\\s|$)', '--project', 'beta',
|
||||
]);
|
||||
});
|
||||
test('bun: anchored space-joined path', () => {
|
||||
const t = parseJunit(fixture('bun.xml')).tests[1]!;
|
||||
expect(rerunSelection('bun', t)).toEqual(['a.test.ts', '--test-name-pattern', '^outer inner fails$']);
|
||||
});
|
||||
test('generic: templates substitute into single argv elements; none given is a coded refusal', () => {
|
||||
const t = parseJunit('<testsuites><testsuite name="s"><testcase name="a b; rm x" file="t.py"><failure/></testcase></testsuite></testsuites>').tests[0]!;
|
||||
expect(rerunSelection('generic', t, ['{file}', '-k', '{title}'])).toEqual(['t.py', '-k', 'a b; rm x']);
|
||||
expect(() => rerunSelection('generic', t)).toThrow(JunitError);
|
||||
});
|
||||
});
|
||||
|
||||
/** A fake runner: argv selection decides which canned report the attempt writes. */
|
||||
function fakeRun(reports: (argv: string[], n: number) => string | null, exits: (argv: string[], n: number) => number) {
|
||||
const calls: string[][] = [];
|
||||
return {
|
||||
calls,
|
||||
spawn(junit: string) {
|
||||
return (argv: string[], _env: NodeJS.ProcessEnv, cwd: string) => {
|
||||
calls.push(argv);
|
||||
const xml = reports(argv, calls.length);
|
||||
if (xml !== null) fs.writeFileSync(path.resolve(cwd, junit), xml);
|
||||
return exits(argv, calls.length);
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const suite = (cases: string) => `<testsuites name="bun test"><testsuite name="a.test.ts" file="a.test.ts">${cases}</testsuite></testsuites>`;
|
||||
const pass = (name: string) => `<testcase name="${name}" classname="" file="a.test.ts"/>`;
|
||||
const fail = (name: string, msg = 'boom') => `<testcase name="${name}" classname="" file="a.test.ts"><failure message="${msg}"/></testcase>`;
|
||||
|
||||
describe('runFlaky', () => {
|
||||
const base = (spawn: ReturnType<typeof fakeRun>, dir: string, extra: Partial<Parameters<typeof runFlaky>[0]> = {}) =>
|
||||
runFlaky({ command: ['bun', 'test'], junit: 'r.xml', maxReruns: 3, cwd: dir, spawn: spawn.spawn('r.xml'), env: {}, now: () => new Date('2026-10-29T00:00:00Z'), ...extra });
|
||||
|
||||
test('a failed test that passes on rerun is a flaky pass and goes on the ledger', () => {
|
||||
const dir = tmp();
|
||||
const f = fakeRun((argv, n) => n === 1 ? suite(pass('a') + fail('b')) : suite(pass('b')), (_a, n) => n === 1 ? 1 : 0);
|
||||
const r = base(f, dir);
|
||||
expect(r.exitCode).toBe(0);
|
||||
expect(f.calls[1]).toEqual(['bun', 'test', 'a.test.ts', '--test-name-pattern', '^b$']);
|
||||
expect(r.outcomes.map(o => [o.status, o.attempts.join(',')])).toEqual([['flaky-pass', 'fail,pass']]);
|
||||
expect(r.ledger).toHaveLength(1);
|
||||
expect(r.ledger[0]).toMatchObject({ kind: 'flaky-pass', id: 'a.test.ts::::b', rerun_by: 'gstack-flaky', attempts: ['fail', 'pass'], excerpt: ['boom'] });
|
||||
expect(renderResult(r, '/l.jsonl').at(-1)).toBe('FLAKY_RESULT: tests=2 failed=0 flaky=1 reruns=1 command_exit=1 verdict=pass ledger=/l.jsonl');
|
||||
expect(stepSummary(r)).toContain('a.test.ts::::b');
|
||||
});
|
||||
|
||||
test('a test that fails its rerun is red and is ledgered as rerun-fail', () => {
|
||||
const dir = tmp();
|
||||
const f = fakeRun(() => suite(fail('b')), () => 1);
|
||||
const r = base(f, dir);
|
||||
expect(r.exitCode).toBe(1);
|
||||
expect(r.outcomes[0]).toMatchObject({ status: 'rerun-fail', attempts: ['fail', 'fail'] });
|
||||
expect(r.ledger[0]!.kind).toBe('rerun-fail');
|
||||
});
|
||||
|
||||
test('more failures than --max-reruns: a real red, nothing rerun', () => {
|
||||
const dir = tmp();
|
||||
const f = fakeRun(() => suite(fail('a') + fail('b') + fail('c') + fail('d')), () => 1);
|
||||
const r = base(f, dir);
|
||||
expect(r.exitCode).toBe(1);
|
||||
expect(f.calls).toHaveLength(1);
|
||||
expect(r.reruns).toBe(0);
|
||||
expect(r.outcomes.every(o => o.status === 'fail' && /more than --max-reruns 3/.test(o.reason!))).toBe(true);
|
||||
expect(r.ledger).toEqual([]);
|
||||
});
|
||||
|
||||
test('runner retries stand: no second rerun, and a runner flaky pass is ledgered', () => {
|
||||
const dir = tmp();
|
||||
const f = fakeRun(() => fixture('playwright-r2-inc.xml'), () => 1);
|
||||
const r = base(f, dir);
|
||||
expect(f.calls).toHaveLength(1);
|
||||
expect(r.exitCode).toBe(1);
|
||||
expect(r.ledger.filter(l => l.kind === 'flaky-pass').map(l => [l.id, l.rerun_by])).toEqual([
|
||||
['a.spec.ts::alpha::group one > flaky once', 'runner'], ['a.spec.ts::beta::group one > flaky once', 'runner'],
|
||||
]);
|
||||
expect(r.outcomes.filter(o => o.status === 'fail').every(o => /already retried \(3 attempts\)/.test(o.reason!))).toBe(true);
|
||||
});
|
||||
|
||||
test('hidden Playwright retries refuse with a code instead of a guess', () => {
|
||||
const dir = tmp();
|
||||
const r = base(fakeRun(() => fixture('playwright-r2.xml'), () => 1), dir);
|
||||
expect([r.exitCode, r.code]).toEqual([3, 'FLAKY_RETRIES_HIDDEN']);
|
||||
});
|
||||
|
||||
test('no report, a stale report, and a crash with no failed test are never green', () => {
|
||||
const dir = tmp();
|
||||
expect(base(fakeRun(() => null, () => 0), dir)).toMatchObject({ exitCode: 3, code: 'FLAKY_NO_RESULTS' });
|
||||
fs.writeFileSync(path.join(dir, 'r.xml'), suite(pass('stale')));
|
||||
expect(base(fakeRun(() => null, () => 0), dir)).toMatchObject({ exitCode: 3, code: 'FLAKY_NO_RESULTS' });
|
||||
expect(base(fakeRun(() => suite(pass('a')), () => 2), dir)).toMatchObject({ exitCode: 3, code: 'FLAKY_COMMAND_FAILED', commandExit: 2 });
|
||||
});
|
||||
|
||||
test('a rerun that does not report the targeted test fails it; other tests in the rerun are ignored', () => {
|
||||
const dir = tmp();
|
||||
const f = fakeRun((_a, n) => n === 1 ? suite(fail('b')) : suite(pass('other')), (_a, n) => n === 1 ? 1 : 0);
|
||||
const r = base(f, dir);
|
||||
expect(r.exitCode).toBe(1);
|
||||
expect(r.outcomes[0]).toMatchObject({ status: 'rerun-fail', reason: 'rerun: the report does not contain this test' });
|
||||
});
|
||||
|
||||
test('a rerun whose command exits non-zero is not a flaky pass, even with a passing report', () => {
|
||||
const dir = tmp();
|
||||
const f = fakeRun((_a, n) => n === 1 ? suite(fail('b')) : suite(pass('b')), () => 1);
|
||||
expect(base(f, dir).outcomes[0]).toMatchObject({ status: 'rerun-fail', reason: 'rerun: the command exited 1' });
|
||||
});
|
||||
|
||||
test('a test reported twice with one failure is red and not rerun', () => {
|
||||
const dir = tmp();
|
||||
const r = parseJunit(suite(pass('b') + fail('b')));
|
||||
expect(r.tests[0]).toMatchObject({ executions: 2, attempts: ['pass', 'fail'] });
|
||||
const f = fakeRun(() => suite(fail('b') + pass('b') + fail('c')), () => 1);
|
||||
const out = base(f, dir);
|
||||
expect(out.outcomes.find(o => o.test.id.endsWith('::b'))).toMatchObject({ status: 'fail', reason: 'it ran 2 times in this report and one failed; not rerun' });
|
||||
expect(f.calls).toHaveLength(2);
|
||||
});
|
||||
|
||||
test('the Playwright environment flag reaches every attempt', () => {
|
||||
const dir = tmp();
|
||||
const seen: string[] = [];
|
||||
runFlaky({ command: ['x'], junit: 'r.xml', maxReruns: 3, cwd: dir, env: {}, spawn: (_a, env, cwd) => {
|
||||
seen.push(env.PLAYWRIGHT_JUNIT_INCLUDE_RETRIES ?? '');
|
||||
fs.writeFileSync(path.join(cwd, 'r.xml'), suite(seen.length === 1 ? fail('b') : pass('b')));
|
||||
return seen.length === 1 ? 1 : 0;
|
||||
} });
|
||||
expect(seen).toEqual(['1', '1']);
|
||||
});
|
||||
|
||||
test('credentials in failure text never reach the ledger', () => {
|
||||
const key = ['sk', 'ant', 'api03', 'A'.repeat(93) + 'AA'].join('-');
|
||||
const out = safeExcerpt([`Error: request failed with key ${key}`]);
|
||||
expect(out.join('\n')).not.toContain(key);
|
||||
});
|
||||
});
|
||||
|
||||
describe('bin/gstack-flaky', () => {
|
||||
test('run: a fake bun runner flakes once, the bin reruns it and exits 0 with a ledger line', () => {
|
||||
const dir = tmp();
|
||||
const runner = path.join(dir, 'runner.sh');
|
||||
fs.writeFileSync(runner, [
|
||||
'#!/usr/bin/env bash',
|
||||
'n=$(cat count 2>/dev/null || echo 0); echo $((n+1)) > count',
|
||||
`if [ "$n" = 0 ]; then printf '%s' '${suite(pass('a') + fail('b'))}' > r.xml; exit 1; fi`,
|
||||
'[ "$1" = a.test.ts ] && [ "$2" = --test-name-pattern ] && [ "$3" = "^b$" ] || exit 9',
|
||||
`printf '%s' '${suite(pass('b'))}' > r.xml`,
|
||||
].join('\n'), { mode: 0o755 });
|
||||
const ledger = path.join(dir, 'ledger.jsonl');
|
||||
const r = spawnSync(path.join(ROOT, 'bin', 'gstack-flaky'), ['run', '--junit', 'r.xml', '--runner', 'bun', '--ledger', ledger, '--', runner], { cwd: dir, encoding: 'utf-8', timeout: 30_000, env: { ...process.env, GITHUB_ACTIONS: '', GITHUB_STEP_SUMMARY: '' } });
|
||||
expect(r.stdout).toContain('FLAKY: flaky-pass a.test.ts::::b attempts=fail,pass');
|
||||
expect(r.stdout).toContain(`verdict=pass ledger=${ledger}`);
|
||||
expect(r.status).toBe(0);
|
||||
expect(JSON.parse(fs.readFileSync(ledger, 'utf-8').trim()).kind).toBe('flaky-pass');
|
||||
});
|
||||
|
||||
test('usage and missing reports exit with their codes', () => {
|
||||
const dir = tmp();
|
||||
const bin = path.join(ROOT, 'bin', 'gstack-flaky');
|
||||
expect(spawnSync(bin, ['run', '--', 'true'], { cwd: dir, encoding: 'utf-8', timeout: 30_000 }).status).toBe(2);
|
||||
const r = spawnSync(bin, ['run', '--junit', 'r.xml', '--', 'true'], { cwd: dir, encoding: 'utf-8', timeout: 30_000 });
|
||||
expect(r.status).toBe(3);
|
||||
expect(r.stderr).toContain('(FLAKY_NO_RESULTS)');
|
||||
expect(r.stderr).toContain('fix: ');
|
||||
});
|
||||
|
||||
test('ids prints one id per test', () => {
|
||||
const r = spawnSync(path.join(ROOT, 'bin', 'gstack-flaky'), ['ids', '--junit', path.join(FIX, 'bun.xml')], { encoding: 'utf-8', timeout: 30_000 });
|
||||
expect(r.stdout.trim().split('\n')).toEqual([testId('a.test.ts', '', ['outer', 'inner', 'passes']), 'a.test.ts::::outer > inner > fails', 'a.test.ts::::outer > skipped', 'a.test.ts::::top level']);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user