mirror of
https://github.com/elder-plinius/OBLITERATUS.git
synced 2026-08-17 16:37:30 +02:00
545 lines
21 KiB
Python
545 lines
21 KiB
Python
"""Tests for quality-policy immutability and mature-scope measurement."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from copy import deepcopy
|
|
from datetime import date
|
|
import json
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
import pytest
|
|
|
|
from scripts import check_quality_policy as quality
|
|
|
|
|
|
def test_mutation_campaign_preloads_native_modules_before_covered_line_discovery():
|
|
pyproject = Path("pyproject.toml").read_text()
|
|
mutmut_config = pyproject.split("[tool.mutmut]", maxsplit=1)[1].split(
|
|
"\n[", maxsplit=1,
|
|
)[0]
|
|
workflow = Path(".github/workflows/ci.yml").read_text()
|
|
|
|
assert "mutate_only_covered_lines = true" in mutmut_config
|
|
assert '"obliteratus/runtime_contracts.py"' in mutmut_config
|
|
assert '"obliteratus/persistence_contracts.py"' in mutmut_config
|
|
assert '"obliteratus/evaluation/lm_eval_integration.py"' in mutmut_config
|
|
assert '"obliteratus/reporting/report.py"' not in mutmut_config
|
|
assert '"tests/test_runtime_contracts.py"' in mutmut_config
|
|
assert '"tests/test_persistence_contracts.py"' in mutmut_config
|
|
assert '"tests/test_lm_eval_reporting_contracts.py"' in mutmut_config
|
|
assert '"tests/test_telemetry_failure_contracts.py"' not in mutmut_config
|
|
assert '"tests/test_evaluation_reporting_contracts.py"' in Path(
|
|
"scripts/run_repeat_gate.py",
|
|
).read_text()
|
|
assert '"tests/test_lm_eval_reporting_contracts.py"' in Path(
|
|
"scripts/run_repeat_gate.py",
|
|
).read_text()
|
|
assert '"tests/test_telemetry_failure_contracts.py"' in Path(
|
|
"scripts/run_repeat_gate.py",
|
|
).read_text()
|
|
assert "import torch, yaml; from mutmut.__main__ import cli; cli()" in workflow
|
|
|
|
|
|
def _policy():
|
|
return {
|
|
"minimums": dict(quality.BASELINE_FLOORS),
|
|
"critical_cpu_paths": ["obliteratus/pure.py"],
|
|
"mature_cpu_scope": {
|
|
"exclusions": [{
|
|
"path": "obliteratus/external.py",
|
|
"boundary": "network-service",
|
|
"rationale": "Requires a live external service.",
|
|
"conditional_issue": "https://github.com/elder-plinius/OBLITERATUS/issues/71",
|
|
"conditional_gate": "network-services",
|
|
}],
|
|
},
|
|
"test_evidence": {
|
|
"retention_days": 90,
|
|
"flake_window_days": 30,
|
|
"maximum_quarantine_days": 30,
|
|
"duration_budgets": {
|
|
"mandatory_cpu": {
|
|
"owner": "@maintainers",
|
|
"max_suite_seconds_by_python": {
|
|
"3.10": 240.0,
|
|
"3.11": 240.0,
|
|
"3.12": 240.0,
|
|
},
|
|
"max_testcase_seconds": 15.0,
|
|
"max_marker_seconds": {
|
|
"cpu": 120.0,
|
|
"integration": 120.0,
|
|
"unmarked": 120.0,
|
|
},
|
|
"maximum_owner_days": 90,
|
|
"owned_slow_tests": [],
|
|
},
|
|
"repeat_gate": {
|
|
"owner": "@maintainers",
|
|
"max_total_seconds": 180.0,
|
|
"max_pass_seconds": 75.0,
|
|
},
|
|
},
|
|
"flake_history": [],
|
|
"quarantines": [],
|
|
},
|
|
"threshold_exceptions": [],
|
|
}
|
|
|
|
|
|
def _coverage():
|
|
return {
|
|
"files": {
|
|
"obliteratus/pure.py": {
|
|
"summary": {
|
|
"num_statements": 100,
|
|
"covered_lines": 92,
|
|
"num_branches": 100,
|
|
"covered_branches": 80,
|
|
},
|
|
},
|
|
"obliteratus/external.py": {
|
|
"summary": {
|
|
"num_statements": 1000,
|
|
"covered_lines": 0,
|
|
"num_branches": 500,
|
|
"covered_branches": 0,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
def test_policy_and_exact_mature_floors_pass():
|
|
policy = _policy()
|
|
assert quality.validate_policy(policy) == []
|
|
measurement, failures = quality.validate_mature_cpu_scope(_coverage(), policy)
|
|
assert failures == []
|
|
assert measurement["line_percent"] == 92
|
|
assert measurement["branch_percent"] == 80
|
|
|
|
|
|
def test_floor_regression_requires_structured_reviewed_exception():
|
|
policy = _policy()
|
|
policy["minimums"]["mutation_score"] = 74
|
|
assert quality.validate_policy(policy) == [
|
|
"quality minimum mutation_score cannot move below 75 without an explicit reviewed exception",
|
|
]
|
|
policy["threshold_exceptions"] = [{
|
|
"threshold": "mutation_score",
|
|
"new_value": 74,
|
|
"reason": "Temporary tool regression",
|
|
"approved_issue": "https://github.com/elder-plinius/OBLITERATUS/issues/999",
|
|
"expires": "2026-09-01",
|
|
}]
|
|
assert quality.validate_policy(policy) == []
|
|
|
|
|
|
def _test_trend(*, seconds: float = 2.0):
|
|
return {
|
|
"python": "3.12",
|
|
"tests": {
|
|
"total": 2,
|
|
"suite_duration_seconds": 5.0,
|
|
"marker_metadata_complete": True,
|
|
"missing_marker_nodeids": [],
|
|
"durations": [
|
|
{
|
|
"nodeid": "tests.test_example::test_cpu",
|
|
"seconds": seconds,
|
|
"markers": ["cpu"],
|
|
},
|
|
{
|
|
"nodeid": "tests.test_example::test_plain",
|
|
"seconds": 1.0,
|
|
"markers": ["unmarked"],
|
|
},
|
|
],
|
|
"marker_durations": {
|
|
"cpu": {"tests": 1, "duration_seconds": seconds},
|
|
"unmarked": {"tests": 1, "duration_seconds": 1.0},
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
def test_duration_policy_requires_owned_complete_positive_budgets():
|
|
policy = _policy()
|
|
mandatory = policy["test_evidence"]["duration_budgets"]["mandatory_cpu"]
|
|
mandatory["owner"] = "maintainers"
|
|
mandatory["max_suite_seconds_by_python"].pop("3.10")
|
|
mandatory["max_marker_seconds"].pop("unmarked")
|
|
failures = quality.validate_policy(policy)
|
|
|
|
assert "duration budget mandatory_cpu requires an @owner" in failures
|
|
assert (
|
|
"duration budget mandatory_cpu must cover exactly Python 3.10, 3.11, and 3.12"
|
|
in failures
|
|
)
|
|
assert "duration budget mandatory_cpu must own the unmarked layer" in failures
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("mutation", "message"),
|
|
[
|
|
(lambda evidence: evidence.update(duration_budgets=None),
|
|
"test evidence duration_budgets must be an object"),
|
|
(lambda evidence: evidence["duration_budgets"].update(mandatory_cpu=None),
|
|
"duration budget mandatory_cpu must be an object"),
|
|
(lambda evidence: evidence["duration_budgets"]["mandatory_cpu"].update(
|
|
max_suite_seconds_by_python=None,
|
|
), "max_suite_seconds_by_python must be an object"),
|
|
(lambda evidence: evidence["duration_budgets"]["mandatory_cpu"][
|
|
"max_suite_seconds_by_python"
|
|
].update({"3.12": 0}), "max_suite_seconds_by_python.3.12 must be positive"),
|
|
(lambda evidence: evidence["duration_budgets"]["mandatory_cpu"].update(
|
|
max_testcase_seconds=0,
|
|
), "max_testcase_seconds must be positive"),
|
|
(lambda evidence: evidence["duration_budgets"]["mandatory_cpu"].update(
|
|
max_marker_seconds={},
|
|
), "max_marker_seconds must be a non-empty object"),
|
|
(lambda evidence: evidence["duration_budgets"]["mandatory_cpu"][
|
|
"max_marker_seconds"
|
|
].update(cpu=0), "invalid marker budget 'cpu'"),
|
|
(lambda evidence: evidence["duration_budgets"]["mandatory_cpu"].update(
|
|
maximum_owner_days=0,
|
|
), "maximum_owner_days must be an integer from 1 to 365"),
|
|
(lambda evidence: evidence["duration_budgets"]["mandatory_cpu"].update(
|
|
owned_slow_tests=None,
|
|
), "owned_slow_tests must be a list"),
|
|
(lambda evidence: evidence["duration_budgets"]["mandatory_cpu"].update(
|
|
owned_slow_tests=[None],
|
|
), "owned slow test 0 must be an object"),
|
|
(lambda evidence: evidence["duration_budgets"]["mandatory_cpu"].update(
|
|
owned_slow_tests=[{"nodeid": ""}],
|
|
), "owned slow test 0 requires a non-empty nodeid"),
|
|
(lambda evidence: evidence["duration_budgets"].update(repeat_gate=None),
|
|
"duration budget repeat_gate must be an object"),
|
|
(lambda evidence: evidence["duration_budgets"]["repeat_gate"].update(
|
|
owner="maintainers", max_total_seconds=0, max_pass_seconds=0,
|
|
), "duration budget repeat_gate requires an @owner"),
|
|
(lambda evidence: evidence["duration_budgets"]["repeat_gate"].update(
|
|
max_total_seconds=10, max_pass_seconds=11,
|
|
), "max_pass_seconds cannot exceed max_total_seconds"),
|
|
],
|
|
)
|
|
def test_duration_policy_rejects_malformed_budget_shapes(mutation, message):
|
|
evidence = deepcopy(_policy()["test_evidence"])
|
|
mutation(evidence)
|
|
|
|
assert any(
|
|
message in failure
|
|
for failure in quality._validate_duration_policy(evidence, today=date(2026, 8, 15))
|
|
)
|
|
|
|
|
|
def _slow_owner(**overrides):
|
|
owner = {
|
|
"nodeid": "tests.test_example::test_slow",
|
|
"owner": "@runtime-maintainers",
|
|
"reason": "Exercises an installed offline artifact.",
|
|
"issue": "https://github.com/elder-plinius/OBLITERATUS/issues/999",
|
|
"max_seconds": 20.0,
|
|
"opened": "2026-08-15",
|
|
"expires": "2026-09-15",
|
|
}
|
|
owner.update(overrides)
|
|
return owner
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("owners", "today", "message"),
|
|
[
|
|
([_slow_owner(), _slow_owner()], date(2026, 8, 15), "duplicate owned slow test"),
|
|
([_slow_owner(owner="maintainers", reason="", issue="https://example.com/1")],
|
|
date(2026, 8, 15), "requires an @owner"),
|
|
([_slow_owner(max_seconds=0)], date(2026, 8, 15), "max_seconds must be positive"),
|
|
([_slow_owner(max_seconds=15)], date(2026, 8, 15),
|
|
"max_seconds must exceed the default testcase budget"),
|
|
([_slow_owner(opened="2026-08-16")], date(2026, 8, 15), "cannot open in the future"),
|
|
([_slow_owner(expires="2026-08-15")], date(2026, 8, 15),
|
|
"must expire after it opens"),
|
|
([_slow_owner(expires="2026-12-01")], date(2026, 8, 15),
|
|
"exceeds the 90-day review window"),
|
|
([_slow_owner(opened="2026-05-01", expires="2026-05-02")], date(2026, 8, 15),
|
|
"expired on 2026-05-02"),
|
|
],
|
|
)
|
|
def test_slow_test_ownership_is_unique_complete_and_time_bounded(owners, today, message):
|
|
evidence = deepcopy(_policy()["test_evidence"])
|
|
evidence["duration_budgets"]["mandatory_cpu"]["owned_slow_tests"] = owners
|
|
|
|
assert any(
|
|
message in failure
|
|
for failure in quality._validate_duration_policy(evidence, today=today)
|
|
)
|
|
|
|
|
|
def test_duration_evidence_enforces_suite_marker_and_unowned_test_budgets():
|
|
policy = _policy()
|
|
trend = _test_trend(seconds=16.0)
|
|
trend["tests"]["suite_duration_seconds"] = 241.0
|
|
trend["tests"]["marker_durations"]["cpu"]["duration_seconds"] = 16.0
|
|
policy["test_evidence"]["duration_budgets"]["mandatory_cpu"][
|
|
"max_marker_seconds"
|
|
]["cpu"] = 15.0
|
|
|
|
failures = quality.validate_duration_evidence(trend, policy)
|
|
assert "test trend Python 3.12 suite duration 241.000s exceeds 240.000s" in failures
|
|
assert (
|
|
"test tests.test_example::test_cpu took 16.000s above the 15.000s default "
|
|
"and has no owned slow-test budget"
|
|
) in failures
|
|
assert "test marker cpu duration 16.000s exceeds 15.000s" in failures
|
|
|
|
|
|
def test_duration_evidence_fails_closed_without_junit_marker_metadata():
|
|
trend = _test_trend()
|
|
trend["tests"]["marker_metadata_complete"] = False
|
|
trend["tests"]["missing_marker_nodeids"] = ["tests.test_example::test_plain"]
|
|
|
|
assert quality.validate_duration_evidence(trend, _policy()) == [
|
|
"test trend duration marker metadata is incomplete for 1 testcase(s)",
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("trend", "message"),
|
|
[
|
|
({"tests": []}, "test trend tests must be an object"),
|
|
({"python": "3.13", "tests": _test_trend()["tests"]},
|
|
"unsupported Python version '3.13'"),
|
|
({"python": "3.12", "tests": {
|
|
**_test_trend()["tests"], "durations": [], "total": 0,
|
|
"marker_durations": {},
|
|
}}, "durations must be a non-empty list"),
|
|
({"python": "3.12", "tests": {
|
|
**_test_trend()["tests"], "missing_marker_nodeids": [None],
|
|
}}, "missing_marker_nodeids must be a string list"),
|
|
({"python": "3.12", "tests": {
|
|
**_test_trend()["tests"], "durations": [None], "total": 1,
|
|
"marker_durations": {},
|
|
}}, "test duration 0 must be an object"),
|
|
({"python": "3.12", "tests": {
|
|
**_test_trend()["tests"],
|
|
"durations": [{"nodeid": "", "seconds": 0, "markers": ["cpu"]}],
|
|
"total": 1, "marker_durations": {},
|
|
}}, "test duration 0 requires a non-empty nodeid"),
|
|
({"python": "3.12", "tests": {
|
|
**_test_trend()["tests"],
|
|
"durations": [
|
|
{"nodeid": "duplicate", "seconds": 0, "markers": ["cpu"]},
|
|
{"nodeid": "duplicate", "seconds": 0, "markers": ["cpu"]},
|
|
],
|
|
"marker_durations": {"cpu": {"tests": 2, "duration_seconds": 0.0}},
|
|
}}, "test trend repeats duration for duplicate"),
|
|
({"python": "3.12", "tests": {
|
|
**_test_trend()["tests"],
|
|
"durations": [{"nodeid": "bad", "seconds": "bad", "markers": []}],
|
|
"total": 1, "marker_durations": {},
|
|
}}, "test duration 0 seconds must be a non-negative finite number"),
|
|
({"python": "3.12", "tests": {
|
|
**_test_trend()["tests"], "total": 3,
|
|
}}, "test trend total must equal the number of duration records"),
|
|
({"python": "3.12", "tests": {
|
|
**_test_trend()["tests"], "marker_durations": [],
|
|
}}, "test trend marker_durations must be an object"),
|
|
({"python": "3.12", "tests": {
|
|
**_test_trend()["tests"],
|
|
"durations": [{"nodeid": "new", "seconds": 1, "markers": ["new-layer"]}],
|
|
"total": 1,
|
|
"marker_durations": {"new-layer": {"tests": 1, "duration_seconds": 1.0}},
|
|
}}, "test marker new-layer has no duration budget"),
|
|
({"python": "3.12", "tests": {
|
|
**_test_trend()["tests"],
|
|
"marker_durations": {
|
|
**_test_trend()["tests"]["marker_durations"],
|
|
"extra": {"tests": 1, "duration_seconds": 1.0},
|
|
},
|
|
}}, "test marker extra has summary without duration records"),
|
|
({"repeat": []}, "test trend repeat must be an object"),
|
|
({"repeat": {"total_duration_seconds": 0, "passes": []}},
|
|
"repeat gate passes must be a non-empty list"),
|
|
({"repeat": {"total_duration_seconds": 0, "passes": [None]}},
|
|
"repeat gate pass 1 must be an object"),
|
|
({}, "test trend contains neither tests nor repeat duration evidence"),
|
|
],
|
|
)
|
|
def test_duration_evidence_rejects_malformed_or_unowned_records(trend, message):
|
|
assert any(
|
|
message in failure
|
|
for failure in quality.validate_duration_evidence(trend, _policy())
|
|
)
|
|
|
|
|
|
def test_duration_evidence_rejects_boolean_duration():
|
|
trend = _test_trend()
|
|
trend["tests"]["durations"][0]["seconds"] = False
|
|
|
|
failures = quality.validate_duration_evidence(trend, _policy())
|
|
assert "test duration 0 seconds must be a non-negative finite number" in failures
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("evidence_text", "message"),
|
|
[
|
|
("not-json", "cannot read test trend evidence"),
|
|
("[]", "test trend evidence root must be an object"),
|
|
(json.dumps({}), "test trend contains neither tests nor repeat duration evidence"),
|
|
],
|
|
)
|
|
def test_cli_rejects_invalid_duration_evidence(
|
|
tmp_path, monkeypatch, capsys, evidence_text, message,
|
|
):
|
|
policy_path = tmp_path / "policy.json"
|
|
evidence_path = tmp_path / "trend.json"
|
|
policy_path.write_text(json.dumps(_policy()))
|
|
evidence_path.write_text(evidence_text)
|
|
monkeypatch.setattr(
|
|
sys,
|
|
"argv",
|
|
[
|
|
"check_quality_policy.py",
|
|
"--policy",
|
|
str(policy_path),
|
|
"--evidence",
|
|
str(evidence_path),
|
|
],
|
|
)
|
|
|
|
assert quality.main() == 1
|
|
assert message in capsys.readouterr().out
|
|
|
|
|
|
def test_owned_slow_test_is_bounded_and_time_limited():
|
|
policy = _policy()
|
|
mandatory = policy["test_evidence"]["duration_budgets"]["mandatory_cpu"]
|
|
mandatory["owned_slow_tests"] = [{
|
|
"nodeid": "tests.test_example::test_cpu",
|
|
"owner": "@runtime-maintainers",
|
|
"reason": "Exercises the installed offline model vertical slice.",
|
|
"issue": "https://github.com/elder-plinius/OBLITERATUS/issues/999",
|
|
"max_seconds": 20.0,
|
|
"opened": "2026-08-15",
|
|
"expires": "2026-09-15",
|
|
}]
|
|
assert quality.validate_policy(policy, today=date(2026, 8, 15)) == []
|
|
assert quality.validate_duration_evidence(_test_trend(seconds=19.0), policy) == []
|
|
|
|
failures = quality.validate_duration_evidence(_test_trend(seconds=21.0), policy)
|
|
assert failures == [
|
|
"owned slow test tests.test_example::test_cpu took 21.000s above its 20.000s budget",
|
|
]
|
|
|
|
|
|
def test_repeat_duration_evidence_enforces_total_and_pass_budgets():
|
|
trend = {
|
|
"repeat": {
|
|
"total_duration_seconds": 181.0,
|
|
"passes": [
|
|
{"duration_seconds": 74.0},
|
|
{"duration_seconds": 76.0},
|
|
{"duration_seconds": 20.0},
|
|
],
|
|
},
|
|
}
|
|
|
|
assert quality.validate_duration_evidence(trend, _policy()) == [
|
|
"repeat gate total duration 181.000s exceeds 180.000s",
|
|
"repeat gate pass 2 duration 76.000s exceeds 75.000s",
|
|
]
|
|
|
|
|
|
def test_exclusions_require_unique_traceable_environment_boundaries():
|
|
policy = _policy()
|
|
duplicate = deepcopy(policy["mature_cpu_scope"]["exclusions"][0])
|
|
duplicate["rationale"] = ""
|
|
policy["mature_cpu_scope"]["exclusions"].append(duplicate)
|
|
failures = quality.validate_policy(policy)
|
|
assert "mature CPU exclusion path is duplicated: obliteratus/external.py" in failures
|
|
assert "mature CPU exclusion 1 requires non-empty rationale" in failures
|
|
|
|
|
|
def test_mature_scope_rejects_regression_and_stale_exclusion():
|
|
policy = _policy()
|
|
report = _coverage()
|
|
report["files"]["obliteratus/pure.py"]["summary"]["covered_lines"] = 91
|
|
_, failures = quality.validate_mature_cpu_scope(report, policy)
|
|
assert failures == [
|
|
"mature CPU line coverage 91.00% is below the 92.00% floor",
|
|
]
|
|
del report["files"]["obliteratus/external.py"]
|
|
_, failures = quality.measure_mature_cpu_scope(report, policy)
|
|
assert failures == [
|
|
"coverage report is missing excluded source file obliteratus/external.py",
|
|
]
|
|
|
|
|
|
def test_second_flake_in_window_requires_active_quarantine():
|
|
policy = _policy()
|
|
nodeid = "tests.test_example::test_unstable"
|
|
policy["test_evidence"]["flake_history"] = [
|
|
{
|
|
"nodeid": nodeid,
|
|
"observed_on": "2026-08-01",
|
|
"head_sha": "a" * 40,
|
|
"gate": "repeat",
|
|
},
|
|
{
|
|
"nodeid": nodeid,
|
|
"observed_on": "2026-08-14",
|
|
"head_sha": "b" * 40,
|
|
"gate": "mandatory-cpu",
|
|
},
|
|
]
|
|
|
|
assert quality.validate_policy(policy, today=date(2026, 8, 14)) == [
|
|
f"test {nodeid} flaked 2 times in 30 days without an active quarantine",
|
|
]
|
|
|
|
policy["test_evidence"]["quarantines"] = [{
|
|
"nodeid": nodeid,
|
|
"owner": "@maintainers",
|
|
"reason": "Ordering-sensitive global state is being isolated.",
|
|
"issue": "https://github.com/elder-plinius/OBLITERATUS/issues/999",
|
|
"opened": "2026-08-14",
|
|
"expires": "2026-09-13",
|
|
}]
|
|
assert quality.validate_policy(policy, today=date(2026, 8, 14)) == []
|
|
|
|
|
|
def test_quarantine_requires_bounded_owned_issue_linked_entry():
|
|
policy = _policy()
|
|
policy["test_evidence"]["quarantines"] = [{
|
|
"nodeid": "tests.test_example::test_unstable",
|
|
"owner": "maintainers",
|
|
"reason": "",
|
|
"issue": "https://example.com/issue/1",
|
|
"opened": "2026-08-01",
|
|
"expires": "2026-10-01",
|
|
}]
|
|
|
|
failures = quality.validate_policy(policy, today=date(2026, 8, 14))
|
|
assert "test quarantine 0 requires an @owner" in failures
|
|
assert "test quarantine 0 requires a non-empty reason" in failures
|
|
assert "test quarantine 0 requires an OBLITERATUS issue URL" in failures
|
|
assert "test quarantine 0 exceeds the 30-day maximum" in failures
|
|
|
|
|
|
def test_flake_history_rejects_malformed_duplicate_and_future_entries():
|
|
policy = _policy()
|
|
entry = {
|
|
"nodeid": "tests.test_example::test_unstable",
|
|
"observed_on": "2026-08-15",
|
|
"head_sha": "short",
|
|
"gate": "",
|
|
}
|
|
policy["test_evidence"]["flake_history"] = [entry, deepcopy(entry)]
|
|
|
|
failures = quality.validate_policy(policy, today=date(2026, 8, 14))
|
|
assert "flake history 0 requires a 40-character head_sha" in failures
|
|
assert "flake history 0 requires a non-empty gate" in failures
|
|
assert "flake history 0 observed_on cannot be in the future" in failures
|
|
assert "duplicate flake history entry for tests.test_example::test_unstable" in failures
|