mirror of
https://github.com/elder-plinius/OBLITERATUS.git
synced 2026-08-17 16:37:30 +02:00
182 lines
6.2 KiB
Python
182 lines
6.2 KiB
Python
"""Tests for quality-policy immutability and mature-scope measurement."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import tomllib
|
|
from copy import deepcopy
|
|
from datetime import date
|
|
from pathlib import Path
|
|
|
|
from scripts import check_quality_policy as quality
|
|
|
|
|
|
def test_mutation_campaign_avoids_native_extension_reload_mode():
|
|
pyproject = tomllib.loads(Path("pyproject.toml").read_text())
|
|
|
|
assert pyproject["tool"]["mutmut"]["mutate_only_covered_lines"] is False
|
|
|
|
|
|
def _policy():
|
|
return {
|
|
"minimums": dict(quality.BASELINE_FLOORS),
|
|
"critical_cpu_paths": ["obliteratus/pure.py"],
|
|
"mature_cpu_scope": {
|
|
"exclusions": [{
|
|
"path": "obliteratus/external.py",
|
|
"boundary": "network-service",
|
|
"rationale": "Requires a live external service.",
|
|
"conditional_issue": "https://github.com/elder-plinius/OBLITERATUS/issues/71",
|
|
"conditional_gate": "network-services",
|
|
}],
|
|
},
|
|
"test_evidence": {
|
|
"retention_days": 90,
|
|
"flake_window_days": 30,
|
|
"maximum_quarantine_days": 30,
|
|
"flake_history": [],
|
|
"quarantines": [],
|
|
},
|
|
"threshold_exceptions": [],
|
|
}
|
|
|
|
|
|
def _coverage():
|
|
return {
|
|
"files": {
|
|
"obliteratus/pure.py": {
|
|
"summary": {
|
|
"num_statements": 100,
|
|
"covered_lines": 80,
|
|
"num_branches": 20,
|
|
"covered_branches": 15,
|
|
},
|
|
},
|
|
"obliteratus/external.py": {
|
|
"summary": {
|
|
"num_statements": 1000,
|
|
"covered_lines": 0,
|
|
"num_branches": 500,
|
|
"covered_branches": 0,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
def test_policy_and_exact_mature_floors_pass():
|
|
policy = _policy()
|
|
assert quality.validate_policy(policy) == []
|
|
measurement, failures = quality.validate_mature_cpu_scope(_coverage(), policy)
|
|
assert failures == []
|
|
assert measurement["line_percent"] == 80
|
|
assert measurement["branch_percent"] == 75
|
|
|
|
|
|
def test_floor_regression_requires_structured_reviewed_exception():
|
|
policy = _policy()
|
|
policy["minimums"]["mutation_score"] = 74
|
|
assert quality.validate_policy(policy) == [
|
|
"quality minimum mutation_score cannot move below 75 without an explicit reviewed exception",
|
|
]
|
|
policy["threshold_exceptions"] = [{
|
|
"threshold": "mutation_score",
|
|
"new_value": 74,
|
|
"reason": "Temporary tool regression",
|
|
"approved_issue": "https://github.com/elder-plinius/OBLITERATUS/issues/999",
|
|
"expires": "2026-09-01",
|
|
}]
|
|
assert quality.validate_policy(policy) == []
|
|
|
|
|
|
def test_exclusions_require_unique_traceable_environment_boundaries():
|
|
policy = _policy()
|
|
duplicate = deepcopy(policy["mature_cpu_scope"]["exclusions"][0])
|
|
duplicate["rationale"] = ""
|
|
policy["mature_cpu_scope"]["exclusions"].append(duplicate)
|
|
failures = quality.validate_policy(policy)
|
|
assert "mature CPU exclusion path is duplicated: obliteratus/external.py" in failures
|
|
assert "mature CPU exclusion 1 requires non-empty rationale" in failures
|
|
|
|
|
|
def test_mature_scope_rejects_regression_and_stale_exclusion():
|
|
policy = _policy()
|
|
report = _coverage()
|
|
report["files"]["obliteratus/pure.py"]["summary"]["covered_lines"] = 79
|
|
_, failures = quality.validate_mature_cpu_scope(report, policy)
|
|
assert failures == [
|
|
"mature CPU line coverage 79.00% is below the 80.00% floor",
|
|
]
|
|
del report["files"]["obliteratus/external.py"]
|
|
_, failures = quality.measure_mature_cpu_scope(report, policy)
|
|
assert failures == [
|
|
"coverage report is missing excluded source file obliteratus/external.py",
|
|
]
|
|
|
|
|
|
def test_second_flake_in_window_requires_active_quarantine():
|
|
policy = _policy()
|
|
nodeid = "tests.test_example::test_unstable"
|
|
policy["test_evidence"]["flake_history"] = [
|
|
{
|
|
"nodeid": nodeid,
|
|
"observed_on": "2026-08-01",
|
|
"head_sha": "a" * 40,
|
|
"gate": "repeat",
|
|
},
|
|
{
|
|
"nodeid": nodeid,
|
|
"observed_on": "2026-08-14",
|
|
"head_sha": "b" * 40,
|
|
"gate": "mandatory-cpu",
|
|
},
|
|
]
|
|
|
|
assert quality.validate_policy(policy, today=date(2026, 8, 14)) == [
|
|
f"test {nodeid} flaked 2 times in 30 days without an active quarantine",
|
|
]
|
|
|
|
policy["test_evidence"]["quarantines"] = [{
|
|
"nodeid": nodeid,
|
|
"owner": "@maintainers",
|
|
"reason": "Ordering-sensitive global state is being isolated.",
|
|
"issue": "https://github.com/elder-plinius/OBLITERATUS/issues/999",
|
|
"opened": "2026-08-14",
|
|
"expires": "2026-09-13",
|
|
}]
|
|
assert quality.validate_policy(policy, today=date(2026, 8, 14)) == []
|
|
|
|
|
|
def test_quarantine_requires_bounded_owned_issue_linked_entry():
|
|
policy = _policy()
|
|
policy["test_evidence"]["quarantines"] = [{
|
|
"nodeid": "tests.test_example::test_unstable",
|
|
"owner": "maintainers",
|
|
"reason": "",
|
|
"issue": "https://example.com/issue/1",
|
|
"opened": "2026-08-01",
|
|
"expires": "2026-10-01",
|
|
}]
|
|
|
|
failures = quality.validate_policy(policy, today=date(2026, 8, 14))
|
|
assert "test quarantine 0 requires an @owner" in failures
|
|
assert "test quarantine 0 requires a non-empty reason" in failures
|
|
assert "test quarantine 0 requires an OBLITERATUS issue URL" in failures
|
|
assert "test quarantine 0 exceeds the 30-day maximum" in failures
|
|
|
|
|
|
def test_flake_history_rejects_malformed_duplicate_and_future_entries():
|
|
policy = _policy()
|
|
entry = {
|
|
"nodeid": "tests.test_example::test_unstable",
|
|
"observed_on": "2026-08-15",
|
|
"head_sha": "short",
|
|
"gate": "",
|
|
}
|
|
policy["test_evidence"]["flake_history"] = [entry, deepcopy(entry)]
|
|
|
|
failures = quality.validate_policy(policy, today=date(2026, 8, 14))
|
|
assert "flake history 0 requires a 40-character head_sha" in failures
|
|
assert "flake history 0 requires a non-empty gate" in failures
|
|
assert "flake history 0 observed_on cannot be in the future" in failures
|
|
assert "duplicate flake history entry for tests.test_example::test_unstable" in failures
|