"""Tests for quality-policy immutability and mature-scope measurement.""" from __future__ import annotations from copy import deepcopy from datetime import date import json from pathlib import Path import sys import pytest from scripts import check_quality_policy as quality def test_mutation_campaign_preloads_native_modules_before_covered_line_discovery(): pyproject = Path("pyproject.toml").read_text() mutmut_config = pyproject.split("[tool.mutmut]", maxsplit=1)[1].split( "\n[", maxsplit=1, )[0] workflow = Path(".github/workflows/ci.yml").read_text() assert "mutate_only_covered_lines = true" in mutmut_config assert '"obliteratus/runtime_contracts.py"' in mutmut_config assert '"obliteratus/persistence_contracts.py"' in mutmut_config assert '"obliteratus/evaluation/lm_eval_integration.py"' in mutmut_config assert '"obliteratus/reporting/report.py"' not in mutmut_config assert '"tests/test_runtime_contracts.py"' in mutmut_config assert '"tests/test_persistence_contracts.py"' in mutmut_config assert '"tests/test_lm_eval_reporting_contracts.py"' in mutmut_config assert '"tests/test_telemetry_failure_contracts.py"' not in mutmut_config assert '"tests/test_evaluation_reporting_contracts.py"' in Path( "scripts/run_repeat_gate.py", ).read_text() assert '"tests/test_lm_eval_reporting_contracts.py"' in Path( "scripts/run_repeat_gate.py", ).read_text() assert '"tests/test_telemetry_failure_contracts.py"' in Path( "scripts/run_repeat_gate.py", ).read_text() assert "import torch, yaml; from mutmut.__main__ import cli; cli()" in workflow def _policy(): return { "minimums": dict(quality.BASELINE_FLOORS), "critical_cpu_paths": ["obliteratus/pure.py"], "mature_cpu_scope": { "exclusions": [{ "path": "obliteratus/external.py", "boundary": "network-service", "rationale": "Requires a live external service.", "conditional_issue": "https://github.com/elder-plinius/OBLITERATUS/issues/71", "conditional_gate": "network-services", }], }, "test_evidence": { "retention_days": 90, "flake_window_days": 30, "maximum_quarantine_days": 30, "duration_budgets": { "mandatory_cpu": { "owner": "@maintainers", "max_suite_seconds_by_python": { "3.10": 240.0, "3.11": 240.0, "3.12": 240.0, }, "max_testcase_seconds": 15.0, "max_marker_seconds": { "cpu": 120.0, "integration": 120.0, "unmarked": 120.0, }, "maximum_owner_days": 90, "owned_slow_tests": [], }, "repeat_gate": { "owner": "@maintainers", "max_total_seconds": 180.0, "max_pass_seconds": 75.0, }, }, "flake_history": [], "quarantines": [], }, "threshold_exceptions": [], } def _coverage(): return { "files": { "obliteratus/pure.py": { "summary": { "num_statements": 100, "covered_lines": 92, "num_branches": 100, "covered_branches": 80, }, }, "obliteratus/external.py": { "summary": { "num_statements": 1000, "covered_lines": 0, "num_branches": 500, "covered_branches": 0, }, }, }, } def test_policy_and_exact_mature_floors_pass(): policy = _policy() assert quality.validate_policy(policy) == [] measurement, failures = quality.validate_mature_cpu_scope(_coverage(), policy) assert failures == [] assert measurement["line_percent"] == 92 assert measurement["branch_percent"] == 80 def test_floor_regression_requires_structured_reviewed_exception(): policy = _policy() policy["minimums"]["mutation_score"] = 74 assert quality.validate_policy(policy) == [ "quality minimum mutation_score cannot move below 75 without an explicit reviewed exception", ] policy["threshold_exceptions"] = [{ "threshold": "mutation_score", "new_value": 74, "reason": "Temporary tool regression", "approved_issue": "https://github.com/elder-plinius/OBLITERATUS/issues/999", "expires": "2026-09-01", }] assert quality.validate_policy(policy) == [] def _test_trend(*, seconds: float = 2.0): return { "python": "3.12", "tests": { "total": 2, "suite_duration_seconds": 5.0, "marker_metadata_complete": True, "missing_marker_nodeids": [], "durations": [ { "nodeid": "tests.test_example::test_cpu", "seconds": seconds, "markers": ["cpu"], }, { "nodeid": "tests.test_example::test_plain", "seconds": 1.0, "markers": ["unmarked"], }, ], "marker_durations": { "cpu": {"tests": 1, "duration_seconds": seconds}, "unmarked": {"tests": 1, "duration_seconds": 1.0}, }, }, } def test_duration_policy_requires_owned_complete_positive_budgets(): policy = _policy() mandatory = policy["test_evidence"]["duration_budgets"]["mandatory_cpu"] mandatory["owner"] = "maintainers" mandatory["max_suite_seconds_by_python"].pop("3.10") mandatory["max_marker_seconds"].pop("unmarked") failures = quality.validate_policy(policy) assert "duration budget mandatory_cpu requires an @owner" in failures assert ( "duration budget mandatory_cpu must cover exactly Python 3.10, 3.11, and 3.12" in failures ) assert "duration budget mandatory_cpu must own the unmarked layer" in failures @pytest.mark.parametrize( ("mutation", "message"), [ (lambda evidence: evidence.update(duration_budgets=None), "test evidence duration_budgets must be an object"), (lambda evidence: evidence["duration_budgets"].update(mandatory_cpu=None), "duration budget mandatory_cpu must be an object"), (lambda evidence: evidence["duration_budgets"]["mandatory_cpu"].update( max_suite_seconds_by_python=None, ), "max_suite_seconds_by_python must be an object"), (lambda evidence: evidence["duration_budgets"]["mandatory_cpu"][ "max_suite_seconds_by_python" ].update({"3.12": 0}), "max_suite_seconds_by_python.3.12 must be positive"), (lambda evidence: evidence["duration_budgets"]["mandatory_cpu"].update( max_testcase_seconds=0, ), "max_testcase_seconds must be positive"), (lambda evidence: evidence["duration_budgets"]["mandatory_cpu"].update( max_marker_seconds={}, ), "max_marker_seconds must be a non-empty object"), (lambda evidence: evidence["duration_budgets"]["mandatory_cpu"][ "max_marker_seconds" ].update(cpu=0), "invalid marker budget 'cpu'"), (lambda evidence: evidence["duration_budgets"]["mandatory_cpu"].update( maximum_owner_days=0, ), "maximum_owner_days must be an integer from 1 to 365"), (lambda evidence: evidence["duration_budgets"]["mandatory_cpu"].update( owned_slow_tests=None, ), "owned_slow_tests must be a list"), (lambda evidence: evidence["duration_budgets"]["mandatory_cpu"].update( owned_slow_tests=[None], ), "owned slow test 0 must be an object"), (lambda evidence: evidence["duration_budgets"]["mandatory_cpu"].update( owned_slow_tests=[{"nodeid": ""}], ), "owned slow test 0 requires a non-empty nodeid"), (lambda evidence: evidence["duration_budgets"].update(repeat_gate=None), "duration budget repeat_gate must be an object"), (lambda evidence: evidence["duration_budgets"]["repeat_gate"].update( owner="maintainers", max_total_seconds=0, max_pass_seconds=0, ), "duration budget repeat_gate requires an @owner"), (lambda evidence: evidence["duration_budgets"]["repeat_gate"].update( max_total_seconds=10, max_pass_seconds=11, ), "max_pass_seconds cannot exceed max_total_seconds"), ], ) def test_duration_policy_rejects_malformed_budget_shapes(mutation, message): evidence = deepcopy(_policy()["test_evidence"]) mutation(evidence) assert any( message in failure for failure in quality._validate_duration_policy(evidence, today=date(2026, 8, 15)) ) def _slow_owner(**overrides): owner = { "nodeid": "tests.test_example::test_slow", "owner": "@runtime-maintainers", "reason": "Exercises an installed offline artifact.", "issue": "https://github.com/elder-plinius/OBLITERATUS/issues/999", "max_seconds": 20.0, "opened": "2026-08-15", "expires": "2026-09-15", } owner.update(overrides) return owner @pytest.mark.parametrize( ("owners", "today", "message"), [ ([_slow_owner(), _slow_owner()], date(2026, 8, 15), "duplicate owned slow test"), ([_slow_owner(owner="maintainers", reason="", issue="https://example.com/1")], date(2026, 8, 15), "requires an @owner"), ([_slow_owner(max_seconds=0)], date(2026, 8, 15), "max_seconds must be positive"), ([_slow_owner(max_seconds=15)], date(2026, 8, 15), "max_seconds must exceed the default testcase budget"), ([_slow_owner(opened="2026-08-16")], date(2026, 8, 15), "cannot open in the future"), ([_slow_owner(expires="2026-08-15")], date(2026, 8, 15), "must expire after it opens"), ([_slow_owner(expires="2026-12-01")], date(2026, 8, 15), "exceeds the 90-day review window"), ([_slow_owner(opened="2026-05-01", expires="2026-05-02")], date(2026, 8, 15), "expired on 2026-05-02"), ], ) def test_slow_test_ownership_is_unique_complete_and_time_bounded(owners, today, message): evidence = deepcopy(_policy()["test_evidence"]) evidence["duration_budgets"]["mandatory_cpu"]["owned_slow_tests"] = owners assert any( message in failure for failure in quality._validate_duration_policy(evidence, today=today) ) def test_duration_evidence_enforces_suite_marker_and_unowned_test_budgets(): policy = _policy() trend = _test_trend(seconds=16.0) trend["tests"]["suite_duration_seconds"] = 241.0 trend["tests"]["marker_durations"]["cpu"]["duration_seconds"] = 16.0 policy["test_evidence"]["duration_budgets"]["mandatory_cpu"][ "max_marker_seconds" ]["cpu"] = 15.0 failures = quality.validate_duration_evidence(trend, policy) assert "test trend Python 3.12 suite duration 241.000s exceeds 240.000s" in failures assert ( "test tests.test_example::test_cpu took 16.000s above the 15.000s default " "and has no owned slow-test budget" ) in failures assert "test marker cpu duration 16.000s exceeds 15.000s" in failures def test_duration_evidence_fails_closed_without_junit_marker_metadata(): trend = _test_trend() trend["tests"]["marker_metadata_complete"] = False trend["tests"]["missing_marker_nodeids"] = ["tests.test_example::test_plain"] assert quality.validate_duration_evidence(trend, _policy()) == [ "test trend duration marker metadata is incomplete for 1 testcase(s)", ] @pytest.mark.parametrize( ("trend", "message"), [ ({"tests": []}, "test trend tests must be an object"), ({"python": "3.13", "tests": _test_trend()["tests"]}, "unsupported Python version '3.13'"), ({"python": "3.12", "tests": { **_test_trend()["tests"], "durations": [], "total": 0, "marker_durations": {}, }}, "durations must be a non-empty list"), ({"python": "3.12", "tests": { **_test_trend()["tests"], "missing_marker_nodeids": [None], }}, "missing_marker_nodeids must be a string list"), ({"python": "3.12", "tests": { **_test_trend()["tests"], "durations": [None], "total": 1, "marker_durations": {}, }}, "test duration 0 must be an object"), ({"python": "3.12", "tests": { **_test_trend()["tests"], "durations": [{"nodeid": "", "seconds": 0, "markers": ["cpu"]}], "total": 1, "marker_durations": {}, }}, "test duration 0 requires a non-empty nodeid"), ({"python": "3.12", "tests": { **_test_trend()["tests"], "durations": [ {"nodeid": "duplicate", "seconds": 0, "markers": ["cpu"]}, {"nodeid": "duplicate", "seconds": 0, "markers": ["cpu"]}, ], "marker_durations": {"cpu": {"tests": 2, "duration_seconds": 0.0}}, }}, "test trend repeats duration for duplicate"), ({"python": "3.12", "tests": { **_test_trend()["tests"], "durations": [{"nodeid": "bad", "seconds": "bad", "markers": []}], "total": 1, "marker_durations": {}, }}, "test duration 0 seconds must be a non-negative finite number"), ({"python": "3.12", "tests": { **_test_trend()["tests"], "total": 3, }}, "test trend total must equal the number of duration records"), ({"python": "3.12", "tests": { **_test_trend()["tests"], "marker_durations": [], }}, "test trend marker_durations must be an object"), ({"python": "3.12", "tests": { **_test_trend()["tests"], "durations": [{"nodeid": "new", "seconds": 1, "markers": ["new-layer"]}], "total": 1, "marker_durations": {"new-layer": {"tests": 1, "duration_seconds": 1.0}}, }}, "test marker new-layer has no duration budget"), ({"python": "3.12", "tests": { **_test_trend()["tests"], "marker_durations": { **_test_trend()["tests"]["marker_durations"], "extra": {"tests": 1, "duration_seconds": 1.0}, }, }}, "test marker extra has summary without duration records"), ({"repeat": []}, "test trend repeat must be an object"), ({"repeat": {"total_duration_seconds": 0, "passes": []}}, "repeat gate passes must be a non-empty list"), ({"repeat": {"total_duration_seconds": 0, "passes": [None]}}, "repeat gate pass 1 must be an object"), ({}, "test trend contains neither tests nor repeat duration evidence"), ], ) def test_duration_evidence_rejects_malformed_or_unowned_records(trend, message): assert any( message in failure for failure in quality.validate_duration_evidence(trend, _policy()) ) def test_duration_evidence_rejects_boolean_duration(): trend = _test_trend() trend["tests"]["durations"][0]["seconds"] = False failures = quality.validate_duration_evidence(trend, _policy()) assert "test duration 0 seconds must be a non-negative finite number" in failures @pytest.mark.parametrize( ("evidence_text", "message"), [ ("not-json", "cannot read test trend evidence"), ("[]", "test trend evidence root must be an object"), (json.dumps({}), "test trend contains neither tests nor repeat duration evidence"), ], ) def test_cli_rejects_invalid_duration_evidence( tmp_path, monkeypatch, capsys, evidence_text, message, ): policy_path = tmp_path / "policy.json" evidence_path = tmp_path / "trend.json" policy_path.write_text(json.dumps(_policy())) evidence_path.write_text(evidence_text) monkeypatch.setattr( sys, "argv", [ "check_quality_policy.py", "--policy", str(policy_path), "--evidence", str(evidence_path), ], ) assert quality.main() == 1 assert message in capsys.readouterr().out def test_owned_slow_test_is_bounded_and_time_limited(): policy = _policy() mandatory = policy["test_evidence"]["duration_budgets"]["mandatory_cpu"] mandatory["owned_slow_tests"] = [{ "nodeid": "tests.test_example::test_cpu", "owner": "@runtime-maintainers", "reason": "Exercises the installed offline model vertical slice.", "issue": "https://github.com/elder-plinius/OBLITERATUS/issues/999", "max_seconds": 20.0, "opened": "2026-08-15", "expires": "2026-09-15", }] assert quality.validate_policy(policy, today=date(2026, 8, 15)) == [] assert quality.validate_duration_evidence(_test_trend(seconds=19.0), policy) == [] failures = quality.validate_duration_evidence(_test_trend(seconds=21.0), policy) assert failures == [ "owned slow test tests.test_example::test_cpu took 21.000s above its 20.000s budget", ] def test_repeat_duration_evidence_enforces_total_and_pass_budgets(): trend = { "repeat": { "total_duration_seconds": 181.0, "passes": [ {"duration_seconds": 74.0}, {"duration_seconds": 76.0}, {"duration_seconds": 20.0}, ], }, } assert quality.validate_duration_evidence(trend, _policy()) == [ "repeat gate total duration 181.000s exceeds 180.000s", "repeat gate pass 2 duration 76.000s exceeds 75.000s", ] def test_exclusions_require_unique_traceable_environment_boundaries(): policy = _policy() duplicate = deepcopy(policy["mature_cpu_scope"]["exclusions"][0]) duplicate["rationale"] = "" policy["mature_cpu_scope"]["exclusions"].append(duplicate) failures = quality.validate_policy(policy) assert "mature CPU exclusion path is duplicated: obliteratus/external.py" in failures assert "mature CPU exclusion 1 requires non-empty rationale" in failures def test_mature_scope_rejects_regression_and_stale_exclusion(): policy = _policy() report = _coverage() report["files"]["obliteratus/pure.py"]["summary"]["covered_lines"] = 91 _, failures = quality.validate_mature_cpu_scope(report, policy) assert failures == [ "mature CPU line coverage 91.00% is below the 92.00% floor", ] del report["files"]["obliteratus/external.py"] _, failures = quality.measure_mature_cpu_scope(report, policy) assert failures == [ "coverage report is missing excluded source file obliteratus/external.py", ] def test_second_flake_in_window_requires_active_quarantine(): policy = _policy() nodeid = "tests.test_example::test_unstable" policy["test_evidence"]["flake_history"] = [ { "nodeid": nodeid, "observed_on": "2026-08-01", "head_sha": "a" * 40, "gate": "repeat", }, { "nodeid": nodeid, "observed_on": "2026-08-14", "head_sha": "b" * 40, "gate": "mandatory-cpu", }, ] assert quality.validate_policy(policy, today=date(2026, 8, 14)) == [ f"test {nodeid} flaked 2 times in 30 days without an active quarantine", ] policy["test_evidence"]["quarantines"] = [{ "nodeid": nodeid, "owner": "@maintainers", "reason": "Ordering-sensitive global state is being isolated.", "issue": "https://github.com/elder-plinius/OBLITERATUS/issues/999", "opened": "2026-08-14", "expires": "2026-09-13", }] assert quality.validate_policy(policy, today=date(2026, 8, 14)) == [] def test_quarantine_requires_bounded_owned_issue_linked_entry(): policy = _policy() policy["test_evidence"]["quarantines"] = [{ "nodeid": "tests.test_example::test_unstable", "owner": "maintainers", "reason": "", "issue": "https://example.com/issue/1", "opened": "2026-08-01", "expires": "2026-10-01", }] failures = quality.validate_policy(policy, today=date(2026, 8, 14)) assert "test quarantine 0 requires an @owner" in failures assert "test quarantine 0 requires a non-empty reason" in failures assert "test quarantine 0 requires an OBLITERATUS issue URL" in failures assert "test quarantine 0 exceeds the 30-day maximum" in failures def test_flake_history_rejects_malformed_duplicate_and_future_entries(): policy = _policy() entry = { "nodeid": "tests.test_example::test_unstable", "observed_on": "2026-08-15", "head_sha": "short", "gate": "", } policy["test_evidence"]["flake_history"] = [entry, deepcopy(entry)] failures = quality.validate_policy(policy, today=date(2026, 8, 14)) assert "flake history 0 requires a 40-character head_sha" in failures assert "flake history 0 requires a non-empty gate" in failures assert "flake history 0 observed_on cannot be in the future" in failures assert "duplicate flake history entry for tests.test_example::test_unstable" in failures