"""Tests for quality-policy immutability and mature-scope measurement.""" from __future__ import annotations from copy import deepcopy from datetime import date import json from pathlib import Path import sys import pytest import yaml from scripts import check_mutation_targets from scripts import prepare_mutation_coverage from scripts import run_prepared_mutmut from scripts import check_quality_policy as quality def _workflow_step(name: str) -> dict: workflow = yaml.safe_load(Path(".github/workflows/ci.yml").read_text()) steps = workflow["jobs"]["quality-depth"]["steps"] matches = [step for step in steps if step.get("name") == name] assert len(matches) == 1 return matches[0] def test_mutation_campaign_uses_fork_safe_native_runtime_policy(): pyproject = Path("pyproject.toml").read_text() mutmut_config = pyproject.split("[tool.mutmut]", maxsplit=1)[1].split( "\n[", maxsplit=1, )[0] mutation_step = _workflow_step("Run bounded selective mutation gate") mutation_run = mutation_step["run"] mutation_env = mutation_step["env"] quality_job = yaml.safe_load(Path(".github/workflows/ci.yml").read_text())["jobs"][ "quality-depth" ] assert "mutate_only_covered_lines = true" in mutmut_config assert "timeout_constant = 2.0" in mutmut_config assert '"obliteratus/runtime_contracts.py"' in mutmut_config assert '"obliteratus/persistence_contracts.py"' in mutmut_config assert '"obliteratus/evaluation/lm_eval_integration.py"' in mutmut_config assert '"obliteratus/analysis/numerical_contracts.py"' in mutmut_config assert '"obliteratus/analysis/whitened_svd.py"' in mutmut_config assert "required_mutation_targets" in mutmut_config required_mutation_config = mutmut_config.split( "required_mutation_targets = [", maxsplit=1, )[1].split("]", maxsplit=1)[0] assert '"obliteratus/persistence_contracts.py"' in required_mutation_config assert '"obliteratus/reporting/report.py"' not in mutmut_config assert '"tests/test_runtime_contracts.py"' in mutmut_config assert '"tests/test_persistence_contracts.py"' in mutmut_config assert '"tests/test_lm_eval_reporting_contracts.py"' in mutmut_config assert '"tests/test_telemetry_failure_contracts.py"' not in mutmut_config assert '"tests/test_evaluation_reporting_contracts.py"' in Path( "scripts/run_repeat_gate.py", ).read_text() assert '"tests/test_lm_eval_reporting_contracts.py"' in Path( "scripts/run_repeat_gate.py", ).read_text() assert '"tests/test_telemetry_failure_contracts.py"' in Path( "scripts/run_repeat_gate.py", ).read_text() repeat_config = Path("scripts/run_repeat_gate.py").read_text() assert '"tests/test_checkpoint_atomicity.py"' in repeat_config assert '"tests/test_persistence_pipeline.py"' in repeat_config assert mutation_env == { "BLIS_NUM_THREADS": "1", "MKL_NUM_THREADS": "1", "NUMEXPR_NUM_THREADS": "1", "OMP_THREAD_LIMIT": "1", "OMP_NUM_THREADS": "1", "OPENBLAS_NUM_THREADS": "1", "VECLIB_MAXIMUM_THREADS": "1", } assert quality_job["timeout-minutes"] == 45 assert "scripts/check_mutation_targets.py prepare" in mutation_run assert "scripts/prepare_mutation_coverage.py prepare-coverage --max-children 8" in mutation_run assert "scripts/prepare_mutation_coverage.py prepare-stats --max-children 8" in mutation_run assert "scripts/run_prepared_mutmut.py run --max-children 8" in mutation_run assert "OBLITERATUS_MUTMUT_REUSE_COVERAGE=1" not in mutation_run assert "scripts/check_mutation_targets.py check" in mutation_run assert '"$QUALITY_ENV/bin/mutmut" run --max-children 8' not in mutation_run assert "/usr/bin/time" in mutation_run timed_block = mutation_run.split("/usr/bin/time", maxsplit=1)[1] assert timed_block.index("scripts/prepare_mutation_coverage.py prepare-coverage") < ( timed_block.index("scripts/prepare_mutation_coverage.py prepare-stats") ) < ( timed_block.index("scripts/run_prepared_mutmut.py run --max-children 8") ) assert "quality-evidence/mutation-time.txt" in timed_block assert "import torch, yaml; from mutmut.__main__ import cli; cli()" not in mutation_run def test_prepared_mutmut_runner_fails_closed_without_executable(monkeypatch, capsys): monkeypatch.setattr(sys, "argv", ["run_prepared_mutmut.py", "run"]) monkeypatch.setattr(run_prepared_mutmut.shutil, "which", lambda _name: None) assert run_prepared_mutmut.main() == 1 assert "mutmut executable is not on PATH" in capsys.readouterr().out def test_prepared_mutmut_runner_execs_with_all_isolation_hooks(monkeypatch): captured: dict[str, object] = {} existing_pythonpath = "/existing/pythonpath" monkeypatch.setenv("PYTHONPATH", existing_pythonpath) for name in ( "OBLITERATUS_MUTMUT_REUSE_COVERAGE", "OBLITERATUS_MUTMUT_REUSE_STATS", "OBLITERATUS_MUTMUT_SUBPROCESS_PREFLIGHT", ): monkeypatch.delenv(name, raising=False) monkeypatch.setattr(sys, "argv", ["run_prepared_mutmut.py", "run", "--max-children", "4"]) monkeypatch.setattr(run_prepared_mutmut.shutil, "which", lambda _name: "/venv/bin/mutmut") def fake_execv(executable, command): captured["executable"] = executable captured["command"] = command captured["pythonpath"] = run_prepared_mutmut.os.environ["PYTHONPATH"] captured["reuse_coverage"] = run_prepared_mutmut.os.environ[ "OBLITERATUS_MUTMUT_REUSE_COVERAGE" ] captured["reuse_stats"] = run_prepared_mutmut.os.environ[ "OBLITERATUS_MUTMUT_REUSE_STATS" ] captured["subprocess_preflight"] = run_prepared_mutmut.os.environ[ "OBLITERATUS_MUTMUT_SUBPROCESS_PREFLIGHT" ] raise RuntimeError("exec intercepted") monkeypatch.setattr(run_prepared_mutmut.os, "execv", fake_execv) with pytest.raises(RuntimeError, match="exec intercepted"): run_prepared_mutmut.main() assert captured == { "executable": "/venv/bin/mutmut", "command": ["/venv/bin/mutmut", "run", "--max-children", "4"], "pythonpath": run_prepared_mutmut.os.pathsep.join( [ str(run_prepared_mutmut.SITECUSTOMIZE), str(run_prepared_mutmut.PROJECT_ROOT), existing_pythonpath, ], ), "reuse_coverage": "1", "reuse_stats": "1", "subprocess_preflight": "1", } def test_mutation_score_floor_is_immutable_across_policy_ci_and_validator(): policy = json.loads(Path("ci/test-quality-policy.json").read_text(encoding="utf-8")) workflow = Path(".github/workflows/ci.yml").read_text(encoding="utf-8") assert quality.BASELINE_FLOORS["mutation_score"] == 85.0 assert policy["minimums"]["mutation_score"] == 85.0 assert "--minimum 85.0" in workflow def _policy(): return { "minimums": dict(quality.BASELINE_FLOORS), "critical_cpu_paths": ["obliteratus/pure.py"], "mature_cpu_scope": { "exclusions": [{ "path": "obliteratus/external.py", "boundary": "network-service", "rationale": "Requires a live external service.", "conditional_issue": "https://github.com/elder-plinius/OBLITERATUS/issues/71", "conditional_gate": "network-services", }], }, "test_evidence": { "retention_days": 90, "flake_window_days": 30, "maximum_quarantine_days": 30, "duration_budgets": { "mandatory_cpu": { "owner": "@maintainers", "max_suite_seconds_by_python": { "3.10": 240.0, "3.11": 240.0, "3.12": 240.0, }, "max_testcase_seconds": 15.0, "max_marker_seconds": { "cpu": 120.0, "integration": 120.0, "unmarked": 120.0, }, "maximum_owner_days": 90, "owned_slow_tests": [], }, "repeat_gate": { "owner": "@maintainers", "max_total_seconds": 180.0, "max_pass_seconds": 75.0, }, }, "flake_history": [], "quarantines": [], }, "threshold_exceptions": [], } def _coverage(): return { "files": { "obliteratus/pure.py": { "summary": { "num_statements": 100, "covered_lines": 92, "num_branches": 100, "covered_branches": 80, }, }, "obliteratus/external.py": { "summary": { "num_statements": 1000, "covered_lines": 0, "num_branches": 500, "covered_branches": 0, }, }, }, } def _mutation_project( tmp_path: Path, *, mutmut_table: str | None = None, source_paths: tuple[str, ...] = ("obliteratus/analysis/numerical_contracts.py",), meta_by_path: dict[str, str] | None = None, ) -> Path: """Create a deterministic mutation-target fixture under tmp_path.""" pyproject = tmp_path / "pyproject.toml" pyproject.write_text( mutmut_table or """ [tool.mutmut] required_mutation_targets = [ "obliteratus/analysis/numerical_contracts.py", ] """.lstrip(), encoding="utf-8", ) for source_path in source_paths: source = tmp_path / source_path source.parent.mkdir(parents=True, exist_ok=True) source.write_text("def covered_target():\n return 1\n", encoding="utf-8") for target, metadata in (meta_by_path or {}).items(): meta = tmp_path / "mutants" / f"{target}.meta" meta.parent.mkdir(parents=True, exist_ok=True) meta.write_text(metadata, encoding="utf-8") return pyproject def test_mutation_target_guard_uses_required_exact_python_targets(tmp_path): pyproject = _mutation_project( tmp_path, mutmut_table=""" [tool.mutmut] only_mutate = [ "obliteratus/config.py", "obliteratus/analysis/*.py", ] required_mutation_targets = [ "obliteratus/analysis/numerical_contracts.py", ] """.lstrip(), ) assert check_mutation_targets.exact_python_targets( check_mutation_targets.load_mutmut_config(pyproject), ) == [Path("obliteratus/analysis/numerical_contracts.py")] @pytest.mark.parametrize( "bad_target", [ "/tmp/escape.py", "../escape.py", "obliteratus/../escape.py", "obliteratus/analysis/*.py", "README.md", ], ) def test_mutation_target_guard_rejects_required_target_escapes_and_non_exact_paths( tmp_path, bad_target, ): pyproject = _mutation_project( tmp_path, mutmut_table=f""" [tool.mutmut] required_mutation_targets = [ {bad_target!r}, ] """.lstrip(), ) with pytest.raises(ValueError, match="invalid required mutation target"): check_mutation_targets.exact_python_targets( check_mutation_targets.load_mutmut_config(pyproject), ) def test_mutation_target_guard_rejects_malformed_config_and_metadata(tmp_path): pyproject = _mutation_project( tmp_path, mutmut_table='[tool]\nmutmut = "not-a-table"\n', ) with pytest.raises(ValueError, match=r"\[tool\.mutmut\] must be a table"): check_mutation_targets.load_mutmut_config(pyproject) with pytest.raises(ValueError, match="list of strings"): check_mutation_targets.exact_python_targets({"required_mutation_targets": ["ok.py", 3]}) malformed = tmp_path / "mutants/bad.py.meta" malformed.parent.mkdir(parents=True) malformed.write_text("{", encoding="utf-8") with pytest.raises(ValueError, match="is not valid JSON"): check_mutation_targets.mutant_count(malformed) missing_key = tmp_path / "mutants/missing.py.meta" missing_key.write_text('{"mutants": []}', encoding="utf-8") with pytest.raises(ValueError, match="exit_code_by_key object"): check_mutation_targets.mutant_count(missing_key) def test_mutation_target_guard_rejects_missing_configured_source(tmp_path): pyproject = _mutation_project(tmp_path, source_paths=()) with pytest.raises(ValueError, match="configured mutation target does not exist"): check_mutation_targets.stale_or_empty_targets( [Path("obliteratus/analysis/numerical_contracts.py")], project_root=pyproject.parent, ) def test_mutation_target_guard_removes_all_stale_artifact_kinds(tmp_path): target = "obliteratus/analysis/numerical_contracts.py" pyproject = _mutation_project( tmp_path, source_paths=(target,), meta_by_path={target: '{"exit_code_by_key": {}}'}, ) artifact_base = tmp_path / "mutants" / target artifact_base.parent.mkdir(parents=True, exist_ok=True) for suffix in ("", ".spans"): (tmp_path / "mutants" / f"{target}{suffix}").write_text("stale", encoding="utf-8") assert check_mutation_targets.prepare_required_targets(pyproject) == [Path(target)] assert not artifact_base.exists() assert not artifact_base.with_suffix(".py.meta").exists() assert not artifact_base.with_suffix(".py.spans").exists() def test_mutation_target_guard_never_unlinks_escaped_symlink_artifacts(tmp_path): target = "obliteratus/analysis/numerical_contracts.py" pyproject = _mutation_project( tmp_path, source_paths=(target,), meta_by_path={target: '{"exit_code_by_key": {}}'}, ) outside = tmp_path / "outside.py" outside.write_text("do not remove\n", encoding="utf-8") artifact = tmp_path / "mutants" / target artifact.parent.mkdir(parents=True, exist_ok=True) artifact.symlink_to(outside) with pytest.raises(ValueError, match="escapes project-owned mutants"): check_mutation_targets.prepare_required_targets(pyproject) assert outside.exists() assert artifact.is_symlink() def test_mutation_target_guard_rejects_symlinked_mutants_root_without_deleting_outside( tmp_path, ): target = "obliteratus/analysis/numerical_contracts.py" outside = tmp_path / "outside-mutants" outside.mkdir() sentinel = outside / "sentinel.txt" sentinel.write_text("do not delete\n", encoding="utf-8") (tmp_path / "mutants").symlink_to(outside, target_is_directory=True) pyproject = _mutation_project( tmp_path, source_paths=(target,), meta_by_path={}, ) with pytest.raises(ValueError, match="literal project-owned mutants directory"): check_mutation_targets.prepare_required_targets(pyproject) with pytest.raises(ValueError, match="literal project-owned mutants directory"): check_mutation_targets.validate_required_targets(pyproject) assert sentinel.read_text(encoding="utf-8") == "do not delete\n" @pytest.mark.parametrize("mutants_dir", [Path("/tmp/mutants"), Path("../mutants"), Path("mutants-copy")]) def test_mutation_target_guard_rejects_unowned_mutants_dir(tmp_path, mutants_dir): pyproject = _mutation_project(tmp_path) with pytest.raises(ValueError, match="mutants directory must be project-owned"): check_mutation_targets.prepare_required_targets(pyproject, mutants_dir=mutants_dir) def test_mutation_target_guard_cli_prepare_check_and_failure_paths(tmp_path, monkeypatch, capsys): target = "obliteratus/analysis/numerical_contracts.py" pyproject = _mutation_project( tmp_path, source_paths=(target,), meta_by_path={target: '{"exit_code_by_key": {"target__mutmut_1": null}}'}, ) monkeypatch.setattr( sys, "argv", ["check_mutation_targets.py", "prepare", "--pyproject", str(pyproject)], ) assert check_mutation_targets.main() == 0 assert "found no stale required targets" in capsys.readouterr().out monkeypatch.setattr( sys, "argv", ["check_mutation_targets.py", "check", "--pyproject", str(pyproject)], ) assert check_mutation_targets.main() == 0 assert "mutation target guard passed" in capsys.readouterr().out (tmp_path / "mutants" / f"{target}.meta").unlink() monkeypatch.setattr( sys, "argv", ["check_mutation_targets.py", "check", "--pyproject", str(pyproject)], ) assert check_mutation_targets.main() == 1 assert "produced zero mutants" in capsys.readouterr().out def test_mutation_target_guard_rejects_no_test_required_mutants(tmp_path, monkeypatch): monkeypatch.setattr( check_mutation_targets, "_installed_mutmut_version", lambda: check_mutation_targets.SUPPORTED_MUTMUT_VERSION, ) target = "obliteratus/analysis/numerical_contracts.py" pyproject = _mutation_project( tmp_path, source_paths=(target,), meta_by_path={ target: '{"exit_code_by_key": {"target__mutmut_1": 5, "target__mutmut_2": 33}}', }, ) assert check_mutation_targets.validate_required_targets(pyproject) == [ "configured mutation target has 2 mutant(s) with no tests: " "obliteratus/analysis/numerical_contracts.py", ] assert check_mutation_targets.MUTMUT_NO_TEST_EXIT_CODES == frozenset({5, 33}) assert check_mutation_targets.SUPPORTED_MUTMUT_VERSION == "3.7.0" @pytest.mark.parametrize("version", [None, "3.6.0", "3.8.0"]) def test_mutation_target_guard_fails_closed_before_no_test_code_interpretation( tmp_path, monkeypatch, version, ): target = "obliteratus/analysis/numerical_contracts.py" pyproject = _mutation_project( tmp_path, source_paths=(target,), meta_by_path={target: '{"exit_code_by_key": {"target__mutmut_1": 33}}'}, ) monkeypatch.setattr(check_mutation_targets, "_installed_mutmut_version", lambda: version) with pytest.raises(ValueError, match="unsupported mutmut version"): check_mutation_targets.validate_required_targets(pyproject) def test_mutation_target_guard_cli_reports_validation_errors(tmp_path, monkeypatch, capsys): pyproject = _mutation_project(tmp_path, source_paths=()) monkeypatch.setattr( sys, "argv", ["check_mutation_targets.py", "prepare", "--pyproject", str(pyproject)], ) assert check_mutation_targets.main() == 1 assert "configured mutation target does not exist" in capsys.readouterr().out def test_mutation_coverage_manifest_rejects_stale_source_hash(tmp_path, monkeypatch): source = tmp_path / "pkg/example.py" mutant = tmp_path / "mutants/pkg/example.py" source.parent.mkdir(parents=True) mutant.parent.mkdir(parents=True) source.write_text("def f():\n return 1\n", encoding="utf-8") mutant.write_text("mutant", encoding="utf-8") manifest = { "version": 1, "mutmut_version": prepare_mutation_coverage.SUPPORTED_MUTMUT_VERSION, "mutate_only_covered_lines": True, "source_paths": ["pkg/"], "only_mutate": ["pkg/example.py"], "pytest_add_cli_args": ["--no-cov"], "pytest_add_cli_args_test_selection": ["tests/test_example.py"], "required_mutation_targets": ["pkg/example.py"], "mutatable_paths": ["pkg/example.py"], "source_hashes": {"pkg/example.py": "stale"}, "selected_test_hashes": {"tests/test_example.py": "ok"}, "hook_hashes": { "scripts/prepare_mutation_coverage.py": "ok", "scripts/mutmut_coverage_sitecustomize/sitecustomize.py": "ok", }, } monkeypatch.chdir(tmp_path) (tmp_path / "mutants/.covered-lines-prepass.json").write_text( json.dumps(manifest), encoding="utf-8", ) source.write_text("def f():\n return 2\n", encoding="utf-8") monkeypatch.setattr(prepare_mutation_coverage, "assert_supported_mutmut", lambda: None) monkeypatch.setattr( prepare_mutation_coverage, "configured_paths", lambda: ([Path("pkg/example.py")], [Path("pkg/example.py")]), ) monkeypatch.setattr( prepare_mutation_coverage, "validate_prepared_artifacts", lambda *, mutatable, required: { **manifest, "source_hashes": { "pkg/example.py": prepare_mutation_coverage._sha256(source), }, }, ) with pytest.raises(RuntimeError, match="manifest is stale for source_hashes"): prepare_mutation_coverage.validate_manifest() def test_mutation_coverage_prepass_rejects_escaped_cleanup_artifacts(tmp_path, monkeypatch): outside = tmp_path / "outside.py" outside.write_text("do not remove\n", encoding="utf-8") artifact = tmp_path / "mutants/pkg/example.py" artifact.parent.mkdir(parents=True, exist_ok=True) artifact.symlink_to(outside) monkeypatch.chdir(tmp_path) with pytest.raises(RuntimeError, match="mutation artifact escapes project-owned mutants"): prepare_mutation_coverage.remove_mutation_artifacts([Path("pkg/example.py")]) assert outside.exists() assert artifact.is_symlink() def test_mutation_coverage_manifest_rejects_stale_selected_test_hash(tmp_path, monkeypatch): source = tmp_path / "pkg/example.py" test_file = tmp_path / "tests/test_example.py" hook = tmp_path / "scripts/mutmut_coverage_sitecustomize/sitecustomize.py" prepass = tmp_path / "scripts/prepare_mutation_coverage.py" mutant = tmp_path / "mutants/pkg/example.py" for path in (source, test_file, hook, prepass, mutant): path.parent.mkdir(parents=True, exist_ok=True) path.write_text("original\n", encoding="utf-8") manifest = { "version": 1, "mutmut_version": prepare_mutation_coverage.SUPPORTED_MUTMUT_VERSION, "mutate_only_covered_lines": True, "source_paths": ["pkg/"], "only_mutate": ["pkg/example.py"], "pytest_add_cli_args": ["--no-cov"], "pytest_add_cli_args_test_selection": ["tests/test_example.py"], "required_mutation_targets": ["pkg/example.py"], "mutatable_paths": ["pkg/example.py"], "source_hashes": {"pkg/example.py": prepare_mutation_coverage._sha256(source)}, "selected_test_hashes": {"tests/test_example.py": "stale"}, "hook_hashes": { "scripts/prepare_mutation_coverage.py": prepare_mutation_coverage._sha256(prepass), "scripts/mutmut_coverage_sitecustomize/sitecustomize.py": ( prepare_mutation_coverage._sha256(hook) ), }, } monkeypatch.chdir(tmp_path) (tmp_path / "mutants/.covered-lines-prepass.json").write_text( json.dumps(manifest), encoding="utf-8", ) monkeypatch.setattr(prepare_mutation_coverage, "assert_supported_mutmut", lambda: None) monkeypatch.setattr( prepare_mutation_coverage, "configured_paths", lambda: ([Path("pkg/example.py")], [Path("pkg/example.py")]), ) monkeypatch.setattr( prepare_mutation_coverage, "validate_prepared_artifacts", lambda *, mutatable, required: { **manifest, "selected_test_hashes": { "tests/test_example.py": prepare_mutation_coverage._sha256(test_file), }, }, ) with pytest.raises(RuntimeError, match="manifest is stale for selected_test_hashes"): prepare_mutation_coverage.validate_manifest() def test_mutation_execution_manifest_rejects_stale_stats_hash(tmp_path, monkeypatch): stats = tmp_path / "mutants/mutmut-stats.json" stats.parent.mkdir(parents=True) stats.write_text( json.dumps({ "tests_by_mangled_function_name": {"pkg.x_f": ["tests/test_example.py::test_f"]}, "duration_by_test": {"tests/test_example.py::test_f": 0.01}, "stats_time": 0.1, "function_hashes": {"pkg.x_f": "abc"}, "function_dependencies": {}, "config_fingerprint": {}, "watched_file_hashes": {}, "git_commit": None, }), encoding="utf-8", ) manifest = {"stats_hash": "stale"} monkeypatch.chdir(tmp_path) monkeypatch.setattr( prepare_mutation_coverage, "validate_coverage_manifest", lambda: manifest, ) with pytest.raises(RuntimeError, match="execution manifest is stale for stats_hash"): prepare_mutation_coverage.validate_execution_manifest() def test_policy_and_exact_mature_floors_pass(): policy = _policy() assert quality.validate_policy(policy) == [] measurement, failures = quality.validate_mature_cpu_scope(_coverage(), policy) assert failures == [] assert measurement["line_percent"] == 92 assert measurement["branch_percent"] == 80 def test_floor_regression_requires_structured_reviewed_exception(): policy = _policy() policy["minimums"]["mutation_score"] = 84 assert quality.validate_policy(policy) == [ "quality minimum mutation_score cannot move below 85 without an explicit reviewed exception", ] policy["threshold_exceptions"] = [{ "threshold": "mutation_score", "new_value": 84, "reason": "Temporary tool regression", "approved_issue": "https://github.com/elder-plinius/OBLITERATUS/issues/999", "expires": "2026-09-01", }] assert quality.validate_policy(policy) == [] def _test_trend(*, seconds: float = 2.0): return { "python": "3.12", "tests": { "total": 2, "suite_duration_seconds": 5.0, "marker_metadata_complete": True, "missing_marker_nodeids": [], "durations": [ { "nodeid": "tests.test_example::test_cpu", "seconds": seconds, "markers": ["cpu"], }, { "nodeid": "tests.test_example::test_plain", "seconds": 1.0, "markers": ["unmarked"], }, ], "marker_durations": { "cpu": {"tests": 1, "duration_seconds": seconds}, "unmarked": {"tests": 1, "duration_seconds": 1.0}, }, }, } def test_duration_policy_requires_owned_complete_positive_budgets(): policy = _policy() mandatory = policy["test_evidence"]["duration_budgets"]["mandatory_cpu"] mandatory["owner"] = "maintainers" mandatory["max_suite_seconds_by_python"].pop("3.10") mandatory["max_marker_seconds"].pop("unmarked") failures = quality.validate_policy(policy) assert "duration budget mandatory_cpu requires an @owner" in failures assert ( "duration budget mandatory_cpu must cover exactly Python 3.10, 3.11, and 3.12" in failures ) assert "duration budget mandatory_cpu must own the unmarked layer" in failures @pytest.mark.parametrize( ("mutation", "message"), [ (lambda evidence: evidence.update(duration_budgets=None), "test evidence duration_budgets must be an object"), (lambda evidence: evidence["duration_budgets"].update(mandatory_cpu=None), "duration budget mandatory_cpu must be an object"), (lambda evidence: evidence["duration_budgets"]["mandatory_cpu"].update( max_suite_seconds_by_python=None, ), "max_suite_seconds_by_python must be an object"), (lambda evidence: evidence["duration_budgets"]["mandatory_cpu"][ "max_suite_seconds_by_python" ].update({"3.12": 0}), "max_suite_seconds_by_python.3.12 must be positive"), (lambda evidence: evidence["duration_budgets"]["mandatory_cpu"].update( max_testcase_seconds=0, ), "max_testcase_seconds must be positive"), (lambda evidence: evidence["duration_budgets"]["mandatory_cpu"].update( max_marker_seconds={}, ), "max_marker_seconds must be a non-empty object"), (lambda evidence: evidence["duration_budgets"]["mandatory_cpu"][ "max_marker_seconds" ].update(cpu=0), "invalid marker budget 'cpu'"), (lambda evidence: evidence["duration_budgets"]["mandatory_cpu"].update( maximum_owner_days=0, ), "maximum_owner_days must be an integer from 1 to 365"), (lambda evidence: evidence["duration_budgets"]["mandatory_cpu"].update( owned_slow_tests=None, ), "owned_slow_tests must be a list"), (lambda evidence: evidence["duration_budgets"]["mandatory_cpu"].update( owned_slow_tests=[None], ), "owned slow test 0 must be an object"), (lambda evidence: evidence["duration_budgets"]["mandatory_cpu"].update( owned_slow_tests=[{"nodeid": ""}], ), "owned slow test 0 requires a non-empty nodeid"), (lambda evidence: evidence["duration_budgets"].update(repeat_gate=None), "duration budget repeat_gate must be an object"), (lambda evidence: evidence["duration_budgets"]["repeat_gate"].update( owner="maintainers", max_total_seconds=0, max_pass_seconds=0, ), "duration budget repeat_gate requires an @owner"), (lambda evidence: evidence["duration_budgets"]["repeat_gate"].update( max_total_seconds=10, max_pass_seconds=11, ), "max_pass_seconds cannot exceed max_total_seconds"), ], ) def test_duration_policy_rejects_malformed_budget_shapes(mutation, message): evidence = deepcopy(_policy()["test_evidence"]) mutation(evidence) assert any( message in failure for failure in quality._validate_duration_policy(evidence, today=date(2026, 8, 15)) ) def _slow_owner(**overrides): owner = { "nodeid": "tests.test_example::test_slow", "owner": "@runtime-maintainers", "reason": "Exercises an installed offline artifact.", "issue": "https://github.com/elder-plinius/OBLITERATUS/issues/999", "max_seconds": 20.0, "opened": "2026-08-15", "expires": "2026-09-15", } owner.update(overrides) return owner @pytest.mark.parametrize( ("owners", "today", "message"), [ ([_slow_owner(), _slow_owner()], date(2026, 8, 15), "duplicate owned slow test"), ([_slow_owner(owner="maintainers", reason="", issue="https://example.com/1")], date(2026, 8, 15), "requires an @owner"), ([_slow_owner(max_seconds=0)], date(2026, 8, 15), "max_seconds must be positive"), ([_slow_owner(max_seconds=15)], date(2026, 8, 15), "max_seconds must exceed the default testcase budget"), ([_slow_owner(opened="2026-08-16")], date(2026, 8, 15), "cannot open in the future"), ([_slow_owner(expires="2026-08-15")], date(2026, 8, 15), "must expire after it opens"), ([_slow_owner(expires="2026-12-01")], date(2026, 8, 15), "exceeds the 90-day review window"), ([_slow_owner(opened="2026-05-01", expires="2026-05-02")], date(2026, 8, 15), "expired on 2026-05-02"), ], ) def test_slow_test_ownership_is_unique_complete_and_time_bounded(owners, today, message): evidence = deepcopy(_policy()["test_evidence"]) evidence["duration_budgets"]["mandatory_cpu"]["owned_slow_tests"] = owners assert any( message in failure for failure in quality._validate_duration_policy(evidence, today=today) ) def test_duration_evidence_enforces_suite_marker_and_unowned_test_budgets(): policy = _policy() trend = _test_trend(seconds=16.0) trend["tests"]["suite_duration_seconds"] = 241.0 trend["tests"]["marker_durations"]["cpu"]["duration_seconds"] = 16.0 policy["test_evidence"]["duration_budgets"]["mandatory_cpu"][ "max_marker_seconds" ]["cpu"] = 15.0 failures = quality.validate_duration_evidence(trend, policy) assert "test trend Python 3.12 suite duration 241.000s exceeds 240.000s" in failures assert ( "test tests.test_example::test_cpu took 16.000s above the 15.000s default " "and has no owned slow-test budget" ) in failures assert "test marker cpu duration 16.000s exceeds 15.000s" in failures def test_duration_evidence_fails_closed_without_junit_marker_metadata(): trend = _test_trend() trend["tests"]["marker_metadata_complete"] = False trend["tests"]["missing_marker_nodeids"] = ["tests.test_example::test_plain"] assert quality.validate_duration_evidence(trend, _policy()) == [ "test trend duration marker metadata is incomplete for 1 testcase(s)", ] @pytest.mark.parametrize( ("trend", "message"), [ ({"tests": []}, "test trend tests must be an object"), ({"python": "3.13", "tests": _test_trend()["tests"]}, "unsupported Python version '3.13'"), ({"python": "3.12", "tests": { **_test_trend()["tests"], "durations": [], "total": 0, "marker_durations": {}, }}, "durations must be a non-empty list"), ({"python": "3.12", "tests": { **_test_trend()["tests"], "missing_marker_nodeids": [None], }}, "missing_marker_nodeids must be a string list"), ({"python": "3.12", "tests": { **_test_trend()["tests"], "durations": [None], "total": 1, "marker_durations": {}, }}, "test duration 0 must be an object"), ({"python": "3.12", "tests": { **_test_trend()["tests"], "durations": [{"nodeid": "", "seconds": 0, "markers": ["cpu"]}], "total": 1, "marker_durations": {}, }}, "test duration 0 requires a non-empty nodeid"), ({"python": "3.12", "tests": { **_test_trend()["tests"], "durations": [ {"nodeid": "duplicate", "seconds": 0, "markers": ["cpu"]}, {"nodeid": "duplicate", "seconds": 0, "markers": ["cpu"]}, ], "marker_durations": {"cpu": {"tests": 2, "duration_seconds": 0.0}}, }}, "test trend repeats duration for duplicate"), ({"python": "3.12", "tests": { **_test_trend()["tests"], "durations": [{"nodeid": "bad", "seconds": "bad", "markers": []}], "total": 1, "marker_durations": {}, }}, "test duration 0 seconds must be a non-negative finite number"), ({"python": "3.12", "tests": { **_test_trend()["tests"], "total": 3, }}, "test trend total must equal the number of duration records"), ({"python": "3.12", "tests": { **_test_trend()["tests"], "marker_durations": [], }}, "test trend marker_durations must be an object"), ({"python": "3.12", "tests": { **_test_trend()["tests"], "durations": [{"nodeid": "new", "seconds": 1, "markers": ["new-layer"]}], "total": 1, "marker_durations": {"new-layer": {"tests": 1, "duration_seconds": 1.0}}, }}, "test marker new-layer has no duration budget"), ({"python": "3.12", "tests": { **_test_trend()["tests"], "marker_durations": { **_test_trend()["tests"]["marker_durations"], "extra": {"tests": 1, "duration_seconds": 1.0}, }, }}, "test marker extra has summary without duration records"), ({"repeat": []}, "test trend repeat must be an object"), ({"repeat": {"total_duration_seconds": 0, "passes": []}}, "repeat gate passes must be a non-empty list"), ({"repeat": {"total_duration_seconds": 0, "passes": [None]}}, "repeat gate pass 1 must be an object"), ({}, "test trend contains neither tests nor repeat duration evidence"), ], ) def test_duration_evidence_rejects_malformed_or_unowned_records(trend, message): assert any( message in failure for failure in quality.validate_duration_evidence(trend, _policy()) ) def test_duration_evidence_rejects_boolean_duration(): trend = _test_trend() trend["tests"]["durations"][0]["seconds"] = False failures = quality.validate_duration_evidence(trend, _policy()) assert "test duration 0 seconds must be a non-negative finite number" in failures @pytest.mark.parametrize( ("evidence_text", "message"), [ ("not-json", "cannot read test trend evidence"), ("[]", "test trend evidence root must be an object"), (json.dumps({}), "test trend contains neither tests nor repeat duration evidence"), ], ) def test_cli_rejects_invalid_duration_evidence( tmp_path, monkeypatch, capsys, evidence_text, message, ): policy_path = tmp_path / "policy.json" evidence_path = tmp_path / "trend.json" policy_path.write_text(json.dumps(_policy())) evidence_path.write_text(evidence_text) monkeypatch.setattr( sys, "argv", [ "check_quality_policy.py", "--policy", str(policy_path), "--evidence", str(evidence_path), ], ) assert quality.main() == 1 assert message in capsys.readouterr().out def test_owned_slow_test_is_bounded_and_time_limited(): policy = _policy() mandatory = policy["test_evidence"]["duration_budgets"]["mandatory_cpu"] mandatory["owned_slow_tests"] = [{ "nodeid": "tests.test_example::test_cpu", "owner": "@runtime-maintainers", "reason": "Exercises the installed offline model vertical slice.", "issue": "https://github.com/elder-plinius/OBLITERATUS/issues/999", "max_seconds": 20.0, "opened": "2026-08-15", "expires": "2026-09-15", }] assert quality.validate_policy(policy, today=date(2026, 8, 15)) == [] assert quality.validate_duration_evidence(_test_trend(seconds=19.0), policy) == [] failures = quality.validate_duration_evidence(_test_trend(seconds=21.0), policy) assert failures == [ "owned slow test tests.test_example::test_cpu took 21.000s above its 20.000s budget", ] def test_repeat_duration_evidence_enforces_total_and_pass_budgets(): trend = { "repeat": { "total_duration_seconds": 181.0, "passes": [ {"duration_seconds": 74.0}, {"duration_seconds": 76.0}, {"duration_seconds": 20.0}, ], }, } assert quality.validate_duration_evidence(trend, _policy()) == [ "repeat gate total duration 181.000s exceeds 180.000s", "repeat gate pass 2 duration 76.000s exceeds 75.000s", ] def test_exclusions_require_unique_traceable_environment_boundaries(): policy = _policy() duplicate = deepcopy(policy["mature_cpu_scope"]["exclusions"][0]) duplicate["rationale"] = "" policy["mature_cpu_scope"]["exclusions"].append(duplicate) failures = quality.validate_policy(policy) assert "mature CPU exclusion path is duplicated: obliteratus/external.py" in failures assert "mature CPU exclusion 1 requires non-empty rationale" in failures def test_mature_scope_rejects_regression_and_stale_exclusion(): policy = _policy() report = _coverage() report["files"]["obliteratus/pure.py"]["summary"]["covered_lines"] = 91 _, failures = quality.validate_mature_cpu_scope(report, policy) assert failures == [ "mature CPU line coverage 91.00% is below the 92.00% floor", ] del report["files"]["obliteratus/external.py"] _, failures = quality.measure_mature_cpu_scope(report, policy) assert failures == [ "coverage report is missing excluded source file obliteratus/external.py", ] def test_second_flake_in_window_requires_active_quarantine(): policy = _policy() nodeid = "tests.test_example::test_unstable" policy["test_evidence"]["flake_history"] = [ { "nodeid": nodeid, "observed_on": "2026-08-01", "head_sha": "a" * 40, "gate": "repeat", }, { "nodeid": nodeid, "observed_on": "2026-08-14", "head_sha": "b" * 40, "gate": "mandatory-cpu", }, ] assert quality.validate_policy(policy, today=date(2026, 8, 14)) == [ f"test {nodeid} flaked 2 times in 30 days without an active quarantine", ] policy["test_evidence"]["quarantines"] = [{ "nodeid": nodeid, "owner": "@maintainers", "reason": "Ordering-sensitive global state is being isolated.", "issue": "https://github.com/elder-plinius/OBLITERATUS/issues/999", "opened": "2026-08-14", "expires": "2026-09-13", }] assert quality.validate_policy(policy, today=date(2026, 8, 14)) == [] def test_quarantine_requires_bounded_owned_issue_linked_entry(): policy = _policy() policy["test_evidence"]["quarantines"] = [{ "nodeid": "tests.test_example::test_unstable", "owner": "maintainers", "reason": "", "issue": "https://example.com/issue/1", "opened": "2026-08-01", "expires": "2026-10-01", }] failures = quality.validate_policy(policy, today=date(2026, 8, 14)) assert "test quarantine 0 requires an @owner" in failures assert "test quarantine 0 requires a non-empty reason" in failures assert "test quarantine 0 requires an OBLITERATUS issue URL" in failures assert "test quarantine 0 exceeds the 30-day maximum" in failures def test_flake_history_rejects_malformed_duplicate_and_future_entries(): policy = _policy() entry = { "nodeid": "tests.test_example::test_unstable", "observed_on": "2026-08-15", "head_sha": "short", "gate": "", } policy["test_evidence"]["flake_history"] = [entry, deepcopy(entry)] failures = quality.validate_policy(policy, today=date(2026, 8, 14)) assert "flake history 0 requires a 40-character head_sha" in failures assert "flake history 0 requires a non-empty gate" in failures assert "flake history 0 observed_on cannot be in the future" in failures assert "duplicate flake history entry for tests.test_example::test_unstable" in failures