mirror of
https://github.com/elder-plinius/OBLITERATUS.git
synced 2026-08-18 00:47:23 +02:00
test: enforce Wave A quality lock
This commit is contained in:
Joseph Magly
@@ -69,3 +69,21 @@ def test_ci_requires_the_committed_lock_and_strict_policy_gate():
|
||||
assert "scripts/check_supply_chain_policy.py licenses" in workflow
|
||||
supply_chain_job = workflow.split(" supply-chain:\n", maxsplit=1)[1]
|
||||
assert "|| true" not in supply_chain_job
|
||||
|
||||
|
||||
def test_ci_enforces_exact_base_module_regression_and_risk_mapping():
|
||||
workflow = WORKFLOW.read_text(encoding="utf-8")
|
||||
|
||||
assert "Generate exact-base coverage for module regression comparison" in workflow
|
||||
assert 'git worktree add --detach "$BASE_WORKTREE" "$COVERAGE_BASE"' in workflow
|
||||
assert "--touched-module-no-regression" in workflow
|
||||
assert "--base-report test-results/base-coverage-py3.12.json" in workflow
|
||||
assert "scripts/check_test_risk_map.py" in workflow
|
||||
|
||||
|
||||
def test_ci_retains_normalized_test_and_quality_trends_for_ninety_days():
|
||||
workflow = WORKFLOW.read_text(encoding="utf-8")
|
||||
|
||||
assert "test-trend-py${{ matrix.python-version }}.json" in workflow
|
||||
assert "quality-trend-py3.12.json" in workflow
|
||||
assert workflow.count("retention-days: 90") >= 2
|
||||
|
||||
@@ -165,3 +165,142 @@ def test_changed_line_gate_accepts_exact_floor():
|
||||
assert MODULE.validate_changed_coverage(
|
||||
_report(), changed, minimum=75.0,
|
||||
) == []
|
||||
|
||||
|
||||
def _module_report(
|
||||
path: str = "obliteratus/example.py",
|
||||
*,
|
||||
covered_lines: int = 8,
|
||||
statements: int = 10,
|
||||
covered_branches: int = 3,
|
||||
branches: int = 4,
|
||||
) -> dict[str, object]:
|
||||
return {
|
||||
"files": {
|
||||
path: {
|
||||
"summary": {
|
||||
"covered_lines": covered_lines,
|
||||
"num_statements": statements,
|
||||
"covered_branches": covered_branches,
|
||||
"num_branches": branches,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_parse_changed_modules_handles_nul_renames_and_filters_nonproduction():
|
||||
raw = (
|
||||
b"M\0obliteratus/current.py\0"
|
||||
b"A\0tests/test_current.py\0"
|
||||
b"D\0obliteratus/deleted.py\0"
|
||||
b"R100\0obliteratus/old.py\0obliteratus/new.py\0"
|
||||
b"R100\0obliteratus/removed.py\0docs/removed.py\0"
|
||||
b"C100\0obliteratus/source.py\0obliteratus/copied.py\0"
|
||||
)
|
||||
|
||||
assert MODULE.parse_changed_modules(raw) == [
|
||||
MODULE.ChangedModule("M", "obliteratus/current.py", "obliteratus/current.py"),
|
||||
MODULE.ChangedModule("D", "obliteratus/deleted.py", "obliteratus/deleted.py"),
|
||||
MODULE.ChangedModule("R100", "obliteratus/old.py", "obliteratus/new.py"),
|
||||
MODULE.ChangedModule("D", "obliteratus/removed.py", "obliteratus/removed.py"),
|
||||
MODULE.ChangedModule("C100", None, "obliteratus/copied.py"),
|
||||
]
|
||||
|
||||
|
||||
def test_parse_changed_modules_rejects_truncated_and_unknown_statuses():
|
||||
import pytest
|
||||
|
||||
with pytest.raises(ValueError, match="truncated"):
|
||||
MODULE.parse_changed_modules(b"R100\0obliteratus/old.py\0")
|
||||
with pytest.raises(ValueError, match="unsupported"):
|
||||
MODULE.parse_changed_modules(b"U\0obliteratus/example.py\0")
|
||||
|
||||
|
||||
def test_touched_module_gate_reports_line_and_branch_regressions_separately():
|
||||
base = _module_report(covered_lines=9, covered_branches=4)
|
||||
head = _module_report(covered_lines=8, covered_branches=3)
|
||||
changes = [MODULE.ChangedModule("M", "obliteratus/example.py", "obliteratus/example.py")]
|
||||
|
||||
assert MODULE.validate_touched_module_regression(head, base, changes) == [
|
||||
"touched module obliteratus/example.py line coverage regressed from 90.00% to 80.00%",
|
||||
"touched module obliteratus/example.py branch coverage regressed from 100.00% to 75.00%",
|
||||
]
|
||||
|
||||
|
||||
def test_touched_module_gate_applies_tolerances_to_existing_module():
|
||||
base = _module_report(covered_lines=801, statements=1000, covered_branches=751, branches=1000)
|
||||
head = _module_report(covered_lines=800, statements=1000, covered_branches=750, branches=1000)
|
||||
changes = [MODULE.ChangedModule("M", "obliteratus/example.py", "obliteratus/example.py")]
|
||||
|
||||
assert MODULE.validate_touched_module_regression(
|
||||
head,
|
||||
base,
|
||||
changes,
|
||||
line_tolerance=0.1,
|
||||
branch_tolerance=0.1,
|
||||
) == []
|
||||
assert len(MODULE.validate_touched_module_regression(
|
||||
head,
|
||||
base,
|
||||
changes,
|
||||
line_tolerance=0.09,
|
||||
branch_tolerance=0.09,
|
||||
)) == 2
|
||||
|
||||
|
||||
def test_touched_module_gate_enforces_new_module_floors():
|
||||
head = _module_report(covered_lines=7, statements=10, covered_branches=2, branches=4)
|
||||
changes = [MODULE.ChangedModule("A", None, "obliteratus/example.py")]
|
||||
|
||||
assert MODULE.validate_touched_module_regression(head, {}, changes) == [
|
||||
"new module obliteratus/example.py line coverage 70.00% is below the 80.00% floor",
|
||||
"new module obliteratus/example.py branch coverage 50.00% is below the 75.00% floor",
|
||||
]
|
||||
|
||||
|
||||
def test_touched_module_gate_requires_floor_when_existing_module_adds_branches():
|
||||
base = _module_report(covered_branches=0, branches=0)
|
||||
head = _module_report(covered_branches=2, branches=4)
|
||||
changes = [MODULE.ChangedModule("M", "obliteratus/example.py", "obliteratus/example.py")]
|
||||
|
||||
assert MODULE.validate_touched_module_regression(head, base, changes) == [
|
||||
"touched module obliteratus/example.py added branches at 50.00% coverage, "
|
||||
"below the 75.00% floor",
|
||||
]
|
||||
|
||||
|
||||
def test_touched_module_gate_accepts_branch_removal_and_rename():
|
||||
base = _module_report("obliteratus/old.py", covered_branches=3, branches=4)
|
||||
head = _module_report("obliteratus/new.py", covered_branches=0, branches=0)
|
||||
changes = [MODULE.ChangedModule("R100", "obliteratus/old.py", "obliteratus/new.py")]
|
||||
|
||||
assert MODULE.validate_touched_module_regression(head, base, changes) == []
|
||||
|
||||
|
||||
def test_touched_module_gate_rejects_missing_and_malformed_measurements():
|
||||
changes = [MODULE.ChangedModule("M", "obliteratus/example.py", "obliteratus/example.py")]
|
||||
assert MODULE.validate_touched_module_regression({}, _module_report(), changes) == [
|
||||
"coverage report is missing touched module obliteratus/example.py",
|
||||
]
|
||||
|
||||
malformed = _module_report()
|
||||
malformed["files"]["obliteratus/example.py"]["summary"]["covered_lines"] = True
|
||||
assert MODULE.validate_touched_module_regression(malformed, _module_report(), changes) == [
|
||||
"coverage report has invalid covered_lines for touched module obliteratus/example.py",
|
||||
]
|
||||
|
||||
|
||||
def test_touched_module_gate_rejects_missing_base_measurement():
|
||||
changes = [MODULE.ChangedModule("M", "obliteratus/example.py", "obliteratus/example.py")]
|
||||
assert MODULE.validate_touched_module_regression(_module_report(), {}, changes) == [
|
||||
"base coverage report is missing touched module obliteratus/example.py",
|
||||
]
|
||||
|
||||
|
||||
def test_touched_module_gate_requires_reviewed_exception_for_deletion():
|
||||
changes = [MODULE.ChangedModule("D", "obliteratus/example.py", "obliteratus/example.py")]
|
||||
assert MODULE.validate_touched_module_regression({}, _module_report(), changes) == [
|
||||
"deleted production module obliteratus/example.py requires an explicit reviewed "
|
||||
"coverage-policy exception",
|
||||
]
|
||||
|
||||
@@ -43,6 +43,11 @@ def test_repeat_gate_records_each_pass(monkeypatch, tmp_path):
|
||||
return SimpleNamespace(returncode=0, stdout="2 passed\n", stderr="")
|
||||
|
||||
monkeypatch.setattr(run_repeat_gate.subprocess, "run", fake_run)
|
||||
monkeypatch.setattr(
|
||||
run_repeat_gate,
|
||||
"junit_snapshot",
|
||||
lambda _path: {"tests": 2, "failed_nodeids": [], "skipped_nodeids": []},
|
||||
)
|
||||
output = tmp_path / "repeat.json"
|
||||
assert run_repeat_gate.run_repeat_gate(
|
||||
["first.py", "second.py"], output=output, python="python-fixture",
|
||||
@@ -53,18 +58,61 @@ def test_repeat_gate_records_each_pass(monkeypatch, tmp_path):
|
||||
"0", "1", "8675309",
|
||||
]
|
||||
assert calls[0][0] == [
|
||||
"python-fixture", "-m", "pytest", "--no-cov", "-q", "first.py", "second.py",
|
||||
"python-fixture",
|
||||
"-m",
|
||||
"pytest",
|
||||
"--no-cov",
|
||||
"-q",
|
||||
f"--junitxml={tmp_path / 'repeat-pass-1.xml'}",
|
||||
"first.py",
|
||||
"second.py",
|
||||
]
|
||||
|
||||
|
||||
def test_repeat_gate_stops_and_preserves_failure_output(monkeypatch, tmp_path):
|
||||
def test_repeat_gate_completes_all_passes_and_preserves_failure_output(monkeypatch, tmp_path):
|
||||
def fake_run(*_args, **_kwargs):
|
||||
return SimpleNamespace(returncode=3, stdout="failed output", stderr="failure detail")
|
||||
|
||||
monkeypatch.setattr(run_repeat_gate.subprocess, "run", fake_run)
|
||||
monkeypatch.setattr(
|
||||
run_repeat_gate,
|
||||
"junit_snapshot",
|
||||
lambda _path: {
|
||||
"tests": 1,
|
||||
"failed_nodeids": ["tests.test_example::test_failure"],
|
||||
"skipped_nodeids": [],
|
||||
},
|
||||
)
|
||||
output = tmp_path / "repeat.json"
|
||||
assert run_repeat_gate.run_repeat_gate(["test.py"], output=output) == 3
|
||||
evidence = json.loads(output.read_text())
|
||||
assert evidence["status"] == "failed"
|
||||
assert len(evidence["passes"]) == 1
|
||||
assert len(evidence["passes"]) == 3
|
||||
assert evidence["passes"][0]["stderr"] == "failure detail"
|
||||
assert evidence["consistent_failures"] == [{
|
||||
"nodeid": "tests.test_example::test_failure",
|
||||
"occurrences": 3,
|
||||
}]
|
||||
|
||||
|
||||
def test_repeat_gate_identifies_intermittent_failures(monkeypatch, tmp_path):
|
||||
results = iter([
|
||||
SimpleNamespace(returncode=1, stdout="failed", stderr=""),
|
||||
SimpleNamespace(returncode=0, stdout="passed", stderr=""),
|
||||
SimpleNamespace(returncode=1, stdout="failed", stderr=""),
|
||||
])
|
||||
snapshots = iter([
|
||||
{"tests": 1, "failed_nodeids": ["tests.test_example::test_flake"], "skipped_nodeids": []},
|
||||
{"tests": 1, "failed_nodeids": [], "skipped_nodeids": []},
|
||||
{"tests": 1, "failed_nodeids": ["tests.test_example::test_flake"], "skipped_nodeids": []},
|
||||
])
|
||||
monkeypatch.setattr(run_repeat_gate.subprocess, "run", lambda *_args, **_kwargs: next(results))
|
||||
monkeypatch.setattr(run_repeat_gate, "junit_snapshot", lambda _path: next(snapshots))
|
||||
|
||||
output = tmp_path / "repeat.json"
|
||||
assert run_repeat_gate.run_repeat_gate(["test.py"], output=output) == 1
|
||||
evidence = json.loads(output.read_text())
|
||||
assert evidence["flake_candidates"] == [{
|
||||
"nodeid": "tests.test_example::test_flake",
|
||||
"occurrences": 2,
|
||||
}]
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from copy import deepcopy
|
||||
from datetime import date
|
||||
|
||||
from scripts import check_quality_policy as quality
|
||||
|
||||
@@ -10,6 +11,7 @@ from scripts import check_quality_policy as quality
|
||||
def _policy():
|
||||
return {
|
||||
"minimums": dict(quality.BASELINE_FLOORS),
|
||||
"critical_cpu_paths": ["obliteratus/pure.py"],
|
||||
"mature_cpu_scope": {
|
||||
"exclusions": [{
|
||||
"path": "obliteratus/external.py",
|
||||
@@ -19,6 +21,13 @@ def _policy():
|
||||
"conditional_gate": "network-services",
|
||||
}],
|
||||
},
|
||||
"test_evidence": {
|
||||
"retention_days": 90,
|
||||
"flake_window_days": 30,
|
||||
"maximum_quarantine_days": 30,
|
||||
"flake_history": [],
|
||||
"quarantines": [],
|
||||
},
|
||||
"threshold_exceptions": [],
|
||||
}
|
||||
|
||||
@@ -94,3 +103,71 @@ def test_mature_scope_rejects_regression_and_stale_exclusion():
|
||||
assert failures == [
|
||||
"coverage report is missing excluded source file obliteratus/external.py",
|
||||
]
|
||||
|
||||
|
||||
def test_second_flake_in_window_requires_active_quarantine():
|
||||
policy = _policy()
|
||||
nodeid = "tests.test_example::test_unstable"
|
||||
policy["test_evidence"]["flake_history"] = [
|
||||
{
|
||||
"nodeid": nodeid,
|
||||
"observed_on": "2026-08-01",
|
||||
"head_sha": "a" * 40,
|
||||
"gate": "repeat",
|
||||
},
|
||||
{
|
||||
"nodeid": nodeid,
|
||||
"observed_on": "2026-08-14",
|
||||
"head_sha": "b" * 40,
|
||||
"gate": "mandatory-cpu",
|
||||
},
|
||||
]
|
||||
|
||||
assert quality.validate_policy(policy, today=date(2026, 8, 14)) == [
|
||||
f"test {nodeid} flaked 2 times in 30 days without an active quarantine",
|
||||
]
|
||||
|
||||
policy["test_evidence"]["quarantines"] = [{
|
||||
"nodeid": nodeid,
|
||||
"owner": "@maintainers",
|
||||
"reason": "Ordering-sensitive global state is being isolated.",
|
||||
"issue": "https://github.com/elder-plinius/OBLITERATUS/issues/999",
|
||||
"opened": "2026-08-14",
|
||||
"expires": "2026-09-13",
|
||||
}]
|
||||
assert quality.validate_policy(policy, today=date(2026, 8, 14)) == []
|
||||
|
||||
|
||||
def test_quarantine_requires_bounded_owned_issue_linked_entry():
|
||||
policy = _policy()
|
||||
policy["test_evidence"]["quarantines"] = [{
|
||||
"nodeid": "tests.test_example::test_unstable",
|
||||
"owner": "maintainers",
|
||||
"reason": "",
|
||||
"issue": "https://example.com/issue/1",
|
||||
"opened": "2026-08-01",
|
||||
"expires": "2026-10-01",
|
||||
}]
|
||||
|
||||
failures = quality.validate_policy(policy, today=date(2026, 8, 14))
|
||||
assert "test quarantine 0 requires an @owner" in failures
|
||||
assert "test quarantine 0 requires a non-empty reason" in failures
|
||||
assert "test quarantine 0 requires an OBLITERATUS issue URL" in failures
|
||||
assert "test quarantine 0 exceeds the 30-day maximum" in failures
|
||||
|
||||
|
||||
def test_flake_history_rejects_malformed_duplicate_and_future_entries():
|
||||
policy = _policy()
|
||||
entry = {
|
||||
"nodeid": "tests.test_example::test_unstable",
|
||||
"observed_on": "2026-08-15",
|
||||
"head_sha": "short",
|
||||
"gate": "",
|
||||
}
|
||||
policy["test_evidence"]["flake_history"] = [entry, deepcopy(entry)]
|
||||
|
||||
failures = quality.validate_policy(policy, today=date(2026, 8, 14))
|
||||
assert "flake history 0 requires a 40-character head_sha" in failures
|
||||
assert "flake history 0 requires a non-empty gate" in failures
|
||||
assert "flake history 0 observed_on cannot be in the future" in failures
|
||||
assert "duplicate flake history entry for tests.test_example::test_unstable" in failures
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Tests for normalized retained test and quality evidence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from scripts import write_test_evidence
|
||||
|
||||
|
||||
def _coverage(line: float = 80.0, branch: float = 75.0):
|
||||
return {
|
||||
"totals": {
|
||||
"percent_statements_covered": line,
|
||||
"percent_branches_covered": branch,
|
||||
},
|
||||
"files": {
|
||||
"obliteratus/example.py": {
|
||||
"summary": {
|
||||
"covered_lines": 8,
|
||||
"num_statements": 10,
|
||||
"covered_branches": 3,
|
||||
"num_branches": 4,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _risk_map():
|
||||
return {"modules": [{"path": "obliteratus/example.py"}]}
|
||||
|
||||
|
||||
def test_coverage_snapshot_records_repository_and_risk_module_metrics():
|
||||
snapshot = write_test_evidence.coverage_snapshot(_coverage(), _risk_map())
|
||||
|
||||
assert snapshot == {
|
||||
"line_percent": 80.0,
|
||||
"branch_percent": 75.0,
|
||||
"risk_modules": {
|
||||
"obliteratus/example.py": {
|
||||
"line_percent": 80.0,
|
||||
"branch_percent": 75.0,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_junit_snapshot_records_failures_skips_and_slowest_tests(tmp_path):
|
||||
junit = tmp_path / "junit.xml"
|
||||
junit.write_text(
|
||||
'<testsuites><testsuite tests="3">'
|
||||
'<testcase classname="tests.test_a" name="test_pass" time="0.1"/>'
|
||||
'<testcase classname="tests.test_a" name="test_fail" time="0.3">'
|
||||
'<failure message="boom"/></testcase>'
|
||||
'<testcase classname="tests.test_b" name="test_skip" time="0.2">'
|
||||
'<skipped/></testcase></testsuite></testsuites>',
|
||||
)
|
||||
|
||||
snapshot = write_test_evidence.junit_snapshot(junit)
|
||||
assert snapshot["total"] == 3
|
||||
assert snapshot["passed"] == 1
|
||||
assert snapshot["failures"] == ["tests.test_a::test_fail"]
|
||||
assert snapshot["skipped"] == ["tests.test_b::test_skip"]
|
||||
assert snapshot["duration_seconds"] == 0.6
|
||||
assert snapshot["slowest"][0] == {
|
||||
"nodeid": "tests.test_a::test_fail",
|
||||
"seconds": 0.3,
|
||||
}
|
||||
|
||||
|
||||
def test_evidence_records_base_deltas_repeat_and_mutation():
|
||||
evidence = write_test_evidence.build_evidence(
|
||||
head_sha="b" * 40,
|
||||
base_sha="a" * 40,
|
||||
python_version="3.12",
|
||||
coverage=_coverage(80, 75),
|
||||
base_coverage=_coverage(79, 74),
|
||||
risk_map=_risk_map(),
|
||||
repeat={"schema_version": 1, "status": "passed", "passes": []},
|
||||
mutation={"killed": 8, "total": 10},
|
||||
generated_at="2026-08-14T00:00:00+00:00",
|
||||
)
|
||||
|
||||
assert evidence["coverage"]["delta"] == {
|
||||
"line_percentage_points": 1.0,
|
||||
"branch_percentage_points": 1.0,
|
||||
}
|
||||
assert evidence["repeat"]["status"] == "passed"
|
||||
assert evidence["mutation"] == {
|
||||
"killed": 8,
|
||||
"total": 10,
|
||||
"score_percent": 80.0,
|
||||
}
|
||||
|
||||
|
||||
def test_evidence_rejects_missing_inputs_and_malformed_measurements():
|
||||
with pytest.raises(ValueError, match="at least one"):
|
||||
write_test_evidence.build_evidence(
|
||||
head_sha="local",
|
||||
base_sha="",
|
||||
python_version="3.12",
|
||||
)
|
||||
|
||||
malformed = _coverage()
|
||||
malformed["files"]["obliteratus/example.py"]["summary"]["covered_lines"] = True
|
||||
with pytest.raises(ValueError, match="invalid covered_lines"):
|
||||
write_test_evidence.coverage_snapshot(malformed, _risk_map())
|
||||
|
||||
malformed = _coverage(line=float("nan"))
|
||||
with pytest.raises(ValueError, match="numeric line"):
|
||||
write_test_evidence.coverage_snapshot(malformed, _risk_map())
|
||||
|
||||
|
||||
def test_base_snapshot_allows_risk_module_added_by_head():
|
||||
snapshot = write_test_evidence.coverage_snapshot(
|
||||
{"totals": _coverage()["totals"], "files": {}},
|
||||
_risk_map(),
|
||||
allow_missing_risk_modules=True,
|
||||
)
|
||||
|
||||
assert snapshot["risk_modules"]["obliteratus/example.py"] == {
|
||||
"line_percent": None,
|
||||
"branch_percent": None,
|
||||
}
|
||||
|
||||
|
||||
def test_mutation_snapshot_rejects_impossible_counts():
|
||||
with pytest.raises(ValueError, match="valid killed and total"):
|
||||
write_test_evidence.mutation_snapshot({"killed": 2, "total": 1})
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Executable contracts for the source-to-test risk map."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from copy import deepcopy
|
||||
from pathlib import Path
|
||||
|
||||
from scripts import check_test_risk_map
|
||||
|
||||
|
||||
ROOT = Path(__file__).parents[1]
|
||||
|
||||
|
||||
def _write(path: Path, value: object) -> Path:
|
||||
path.write_text(json.dumps(value), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def test_committed_test_risk_map_is_complete():
|
||||
assert check_test_risk_map.validate(
|
||||
ROOT / "ci" / "test-risk-map.json",
|
||||
ROOT / "ci" / "test-quality-policy.json",
|
||||
ROOT / "ci" / "conditional-test-policy.json",
|
||||
) == []
|
||||
|
||||
|
||||
def test_risk_map_rejects_duplicate_missing_and_unowned_modules(tmp_path):
|
||||
risk = json.loads((ROOT / "ci" / "test-risk-map.json").read_text())
|
||||
risk["owner"] = ""
|
||||
duplicate = deepcopy(risk["modules"][0])
|
||||
duplicate["required_tests"] = ["tests/missing.py"]
|
||||
duplicate["risk_class"] = "unknown"
|
||||
risk["modules"].append(duplicate)
|
||||
|
||||
errors = check_test_risk_map.validate(
|
||||
_write(tmp_path / "risk.json", risk),
|
||||
ROOT / "ci" / "test-quality-policy.json",
|
||||
ROOT / "ci" / "conditional-test-policy.json",
|
||||
)
|
||||
|
||||
assert "test risk map requires a non-empty owner" in errors
|
||||
assert "duplicate risk module path: obliteratus/cli.py" in errors
|
||||
assert "risk module obliteratus/cli.py has invalid risk_class" in errors
|
||||
assert "risk module obliteratus/cli.py maps missing test path: tests/missing.py" in errors
|
||||
|
||||
|
||||
def test_risk_map_rejects_missing_exclusion_and_gate_mapping(tmp_path):
|
||||
risk = json.loads((ROOT / "ci" / "test-risk-map.json").read_text())
|
||||
risk["modules"] = [
|
||||
module for module in risk["modules"] if module["path"] != "obliteratus/remote.py"
|
||||
]
|
||||
loader = next(
|
||||
module for module in risk["modules"] if module["path"] == "obliteratus/models/loader.py"
|
||||
)
|
||||
loader["conditional_gates"].remove("cuda-runtime")
|
||||
|
||||
errors = check_test_risk_map.validate(
|
||||
_write(tmp_path / "risk.json", risk),
|
||||
ROOT / "ci" / "test-quality-policy.json",
|
||||
ROOT / "ci" / "conditional-test-policy.json",
|
||||
)
|
||||
|
||||
assert "CPU exclusion is missing from test risk map: obliteratus/remote.py" in errors
|
||||
assert "conditional path is missing from test risk map: obliteratus/remote.py" in errors
|
||||
assert "risk module obliteratus/models/loader.py is missing conditional gate cuda-runtime" in errors
|
||||
|
||||
|
||||
def test_risk_map_rejects_gate_that_does_not_cover_module(tmp_path):
|
||||
risk = json.loads((ROOT / "ci" / "test-risk-map.json").read_text())
|
||||
cli = next(module for module in risk["modules"] if module["path"] == "obliteratus/cli.py")
|
||||
cli["conditional_gates"] = ["remote-execution", "unknown-gate"]
|
||||
|
||||
errors = check_test_risk_map.validate(
|
||||
_write(tmp_path / "risk.json", risk),
|
||||
ROOT / "ci" / "test-quality-policy.json",
|
||||
ROOT / "ci" / "conditional-test-policy.json",
|
||||
)
|
||||
|
||||
assert "risk module obliteratus/cli.py is not covered by gate remote-execution" in errors
|
||||
assert "risk module obliteratus/cli.py references unknown gate unknown-gate" in errors
|
||||
|
||||
|
||||
def test_risk_map_cannot_drop_critical_cpu_path(tmp_path):
|
||||
risk = json.loads((ROOT / "ci" / "test-risk-map.json").read_text())
|
||||
risk["modules"] = [
|
||||
module for module in risk["modules"] if module["path"] != "obliteratus/cli.py"
|
||||
]
|
||||
|
||||
errors = check_test_risk_map.validate(
|
||||
_write(tmp_path / "risk.json", risk),
|
||||
ROOT / "ci" / "test-quality-policy.json",
|
||||
ROOT / "ci" / "conditional-test-policy.json",
|
||||
)
|
||||
|
||||
assert "critical CPU path is missing from test risk map: obliteratus/cli.py" in errors
|
||||
Reference in New Issue
Block a user