mirror of
https://github.com/elder-plinius/OBLITERATUS.git
synced 2026-08-17 16:37:30 +02:00
test: add Gate 3 numerical oracle contracts
This commit is contained in:
Joseph Magly
@@ -10,6 +10,9 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
DEFAULT_MUTATION_SCORE_MINIMUM = 85.0
|
||||
|
||||
|
||||
def mutation_score(stats: dict[str, Any]) -> tuple[int, int, float]:
|
||||
"""Return killed, total, and percentage after validating mutmut statistics."""
|
||||
killed = stats.get("killed")
|
||||
@@ -48,7 +51,7 @@ def validate_mutation_stats(
|
||||
def _parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("stats", type=Path, help="mutmut-cicd-stats.json path")
|
||||
parser.add_argument("--minimum", type=float, required=True)
|
||||
parser.add_argument("--minimum", type=float, default=DEFAULT_MUTATION_SCORE_MINIMUM)
|
||||
return parser
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fail closed when configured mutmut targets produce no mutants."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from importlib import metadata
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
try: # Python 3.11+
|
||||
import tomllib
|
||||
except ModuleNotFoundError: # pragma: no cover - exercised on Python 3.10
|
||||
import tomli as tomllib # type: ignore[no-redef]
|
||||
|
||||
|
||||
GLOB_MARKERS = frozenset("*?[")
|
||||
SUPPORTED_MUTMUT_VERSION = "3.7.0"
|
||||
MUTMUT_NO_TEST_EXIT_CODES = frozenset({5, 33})
|
||||
|
||||
|
||||
def load_mutmut_config(pyproject: Path) -> dict[str, Any]:
|
||||
"""Return the `[tool.mutmut]` table from pyproject.toml."""
|
||||
data = tomllib.loads(pyproject.read_text(encoding="utf-8"))
|
||||
mutmut_config = data.get("tool", {}).get("mutmut", {})
|
||||
if not isinstance(mutmut_config, dict):
|
||||
raise ValueError("[tool.mutmut] must be a table")
|
||||
return mutmut_config
|
||||
|
||||
|
||||
def exact_python_targets(mutmut_config: dict[str, Any]) -> list[Path]:
|
||||
"""Return exact `.py` paths that must enumerate mutants."""
|
||||
raw_targets = mutmut_config.get("required_mutation_targets", mutmut_config.get("only_mutate", []))
|
||||
if not isinstance(raw_targets, list) or not all(isinstance(path, str) for path in raw_targets):
|
||||
raise ValueError(
|
||||
"[tool.mutmut].required_mutation_targets must be a list of strings",
|
||||
)
|
||||
|
||||
targets: list[Path] = []
|
||||
strict_required = "required_mutation_targets" in mutmut_config
|
||||
for raw_path in raw_targets:
|
||||
path = Path(raw_path)
|
||||
invalid = (
|
||||
path.is_absolute()
|
||||
or ".." in path.parts
|
||||
or not raw_path.endswith(".py")
|
||||
or any(marker in raw_path for marker in GLOB_MARKERS)
|
||||
)
|
||||
if invalid:
|
||||
if strict_required:
|
||||
raise ValueError(f"invalid required mutation target: {raw_path}")
|
||||
continue
|
||||
targets.append(path)
|
||||
return sorted(targets, key=str)
|
||||
|
||||
|
||||
def _owned_mutants_dir(project_root: Path, mutants_dir: Path) -> Path:
|
||||
if mutants_dir != Path("mutants"):
|
||||
raise ValueError(
|
||||
"mutants directory must be project-owned as the literal project-owned mutants directory",
|
||||
)
|
||||
root = project_root.resolve()
|
||||
configured = root / "mutants"
|
||||
if configured.is_symlink():
|
||||
raise ValueError(
|
||||
"mutants directory must be project-owned as the literal project-owned mutants directory",
|
||||
)
|
||||
return configured
|
||||
|
||||
|
||||
def _owned_artifact(project_root: Path, mutants_dir: Path, target: Path, suffix: str) -> Path:
|
||||
configured = _owned_mutants_dir(project_root, mutants_dir)
|
||||
artifact = configured / f"{target}{suffix}"
|
||||
resolved = artifact.resolve() if artifact.exists() else artifact.resolve(strict=False)
|
||||
try:
|
||||
resolved.relative_to(configured)
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"mutation artifact escapes project-owned mutants/: {artifact}") from exc
|
||||
return artifact
|
||||
|
||||
|
||||
def _validate_owned_source(project_root: Path, target: Path) -> Path:
|
||||
source_path = project_root / target
|
||||
if not source_path.is_file():
|
||||
raise ValueError(f"configured mutation target does not exist: {target}")
|
||||
try:
|
||||
source_path.resolve().relative_to(project_root.resolve())
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"configured mutation target escapes project root: {target}") from exc
|
||||
return source_path
|
||||
|
||||
|
||||
def mutant_count(meta_path: Path) -> int:
|
||||
"""Return the number of generated mutants recorded in a mutmut metadata file."""
|
||||
try:
|
||||
metadata = json.loads(meta_path.read_text(encoding="utf-8"))
|
||||
except FileNotFoundError:
|
||||
return 0
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(f"{meta_path} is not valid JSON: {exc}") from exc
|
||||
|
||||
exit_code_by_key = metadata.get("exit_code_by_key")
|
||||
if not isinstance(exit_code_by_key, dict):
|
||||
raise ValueError(f"{meta_path} does not contain an exit_code_by_key object")
|
||||
return len(exit_code_by_key)
|
||||
|
||||
|
||||
def _installed_mutmut_version() -> str | None:
|
||||
try:
|
||||
return metadata.version("mutmut")
|
||||
except metadata.PackageNotFoundError:
|
||||
return None
|
||||
|
||||
|
||||
def assert_supported_mutmut_no_test_codes() -> None:
|
||||
version = _installed_mutmut_version()
|
||||
if version != SUPPORTED_MUTMUT_VERSION:
|
||||
raise ValueError(
|
||||
f"unsupported mutmut version {version!r}; expected {SUPPORTED_MUTMUT_VERSION} "
|
||||
"before interpreting no-test exit codes",
|
||||
)
|
||||
|
||||
|
||||
def no_test_mutants(meta_path: Path) -> list[str]:
|
||||
"""Return required-target mutants that mutmut marked as having no covering tests."""
|
||||
try:
|
||||
metadata = json.loads(meta_path.read_text(encoding="utf-8"))
|
||||
except FileNotFoundError:
|
||||
return []
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(f"{meta_path} is not valid JSON: {exc}") from exc
|
||||
|
||||
exit_code_by_key = metadata.get("exit_code_by_key")
|
||||
if not isinstance(exit_code_by_key, dict):
|
||||
raise ValueError(f"{meta_path} does not contain an exit_code_by_key object")
|
||||
no_tests = sorted(
|
||||
key for key, exit_code in exit_code_by_key.items()
|
||||
if exit_code in MUTMUT_NO_TEST_EXIT_CODES
|
||||
)
|
||||
if no_tests:
|
||||
assert_supported_mutmut_no_test_codes()
|
||||
return no_tests
|
||||
|
||||
|
||||
def stale_or_empty_targets(
|
||||
targets: list[Path], *, project_root: Path = Path("."), mutants_dir: Path = Path("mutants"),
|
||||
) -> list[Path]:
|
||||
"""Return configured targets whose mutant metadata is missing or empty."""
|
||||
stale: list[Path] = []
|
||||
_owned_mutants_dir(project_root, mutants_dir)
|
||||
for target in targets:
|
||||
_validate_owned_source(project_root, target)
|
||||
if mutant_count(_owned_artifact(project_root, mutants_dir, target, ".meta")) == 0:
|
||||
stale.append(target)
|
||||
return stale
|
||||
|
||||
|
||||
def prepare_required_targets(
|
||||
pyproject: Path = Path("pyproject.toml"), *, mutants_dir: Path = Path("mutants"),
|
||||
) -> list[Path]:
|
||||
"""Remove stale copied mutant files so mutmut regenerates required target metadata."""
|
||||
targets = exact_python_targets(load_mutmut_config(pyproject))
|
||||
stale = stale_or_empty_targets(targets, project_root=pyproject.parent, mutants_dir=mutants_dir)
|
||||
for target in stale:
|
||||
for suffix in ("", ".meta", ".spans"):
|
||||
artifact = _owned_artifact(pyproject.parent, mutants_dir, target, suffix)
|
||||
if artifact.exists():
|
||||
artifact.unlink()
|
||||
return stale
|
||||
|
||||
|
||||
def validate_required_targets(
|
||||
pyproject: Path = Path("pyproject.toml"), *, mutants_dir: Path = Path("mutants"),
|
||||
) -> list[str]:
|
||||
"""Return fail-closed messages for exact mutation targets with zero generated mutants."""
|
||||
targets = exact_python_targets(load_mutmut_config(pyproject))
|
||||
failures: list[str] = []
|
||||
for target in stale_or_empty_targets(targets, project_root=pyproject.parent, mutants_dir=mutants_dir):
|
||||
failures.append(f"configured mutation target produced zero mutants: {target}")
|
||||
for target in targets:
|
||||
no_tests = no_test_mutants(_owned_artifact(pyproject.parent, mutants_dir, target, ".meta"))
|
||||
if no_tests:
|
||||
failures.append(
|
||||
f"configured mutation target has {len(no_tests)} mutant(s) with no tests: {target}",
|
||||
)
|
||||
return failures
|
||||
|
||||
|
||||
def _parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("command", choices=("prepare", "check"))
|
||||
parser.add_argument("--pyproject", type=Path, default=Path("pyproject.toml"))
|
||||
parser.add_argument("--mutants-dir", type=Path, default=Path("mutants"))
|
||||
return parser
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = _parser().parse_args()
|
||||
try:
|
||||
if args.command == "prepare":
|
||||
stale = prepare_required_targets(args.pyproject, mutants_dir=args.mutants_dir)
|
||||
if stale:
|
||||
joined = ", ".join(str(path) for path in stale)
|
||||
print(f"mutation target guard invalidated stale metadata for: {joined}")
|
||||
else:
|
||||
print("mutation target guard found no stale required targets")
|
||||
return 0
|
||||
|
||||
failures = validate_required_targets(args.pyproject, mutants_dir=args.mutants_dir)
|
||||
except (OSError, ValueError) as exc:
|
||||
print(f"mutation target guard failed: {exc}")
|
||||
return 1
|
||||
|
||||
if failures:
|
||||
for failure in failures:
|
||||
print(f"mutation target guard failed: {failure}")
|
||||
return 1
|
||||
print("mutation target guard passed")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -18,7 +18,7 @@ BASELINE_FLOORS = {
|
||||
"changed_line": 95.0,
|
||||
"mature_cpu_statement": 92.0,
|
||||
"mature_cpu_branch": 80.0,
|
||||
"mutation_score": 75.0,
|
||||
"mutation_score": 85.0,
|
||||
"warning_budget": 0.0,
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Fail-closed mutmut prepared-artifact reuse hooks for CI."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
|
||||
if (
|
||||
os.environ.get("OBLITERATUS_MUTMUT_REUSE_COVERAGE") == "1"
|
||||
or os.environ.get("OBLITERATUS_MUTMUT_REUSE_STATS") == "1"
|
||||
or os.environ.get("OBLITERATUS_MUTMUT_SUBPROCESS_PREFLIGHT") == "1"
|
||||
):
|
||||
from mutmut import __main__ as mutmut_main
|
||||
|
||||
from scripts.prepare_mutation_coverage import validate_execution_manifest
|
||||
from scripts.prepare_mutation_coverage import validate_manifest
|
||||
|
||||
def store_lines_covered_by_tests_from_prepared_artifacts() -> None:
|
||||
validate_manifest()
|
||||
print("Reusing prepared mutmut covered-line artifacts")
|
||||
|
||||
mutmut_main.store_lines_covered_by_tests = store_lines_covered_by_tests_from_prepared_artifacts
|
||||
|
||||
if os.environ.get("OBLITERATUS_MUTMUT_REUSE_STATS") == "1":
|
||||
|
||||
def collect_or_load_prepared_stats(
|
||||
runner,
|
||||
*,
|
||||
mutants_caught_by_type_checker=None,
|
||||
apply_config_invalidation=False,
|
||||
invalidate_stale_callers=True,
|
||||
) -> None:
|
||||
del runner, mutants_caught_by_type_checker
|
||||
del apply_config_invalidation, invalidate_stale_callers
|
||||
validate_execution_manifest()
|
||||
if not mutmut_main.load_stats():
|
||||
raise RuntimeError("prepared mutmut stats failed to load")
|
||||
print("Reusing prepared mutmut test-selection stats")
|
||||
|
||||
mutmut_main.collect_or_load_stats = collect_or_load_prepared_stats
|
||||
|
||||
if os.environ.get("OBLITERATUS_MUTMUT_SUBPROCESS_PREFLIGHT") == "1":
|
||||
_original_execute_pytest = mutmut_main.PytestRunner.execute_pytest
|
||||
|
||||
def execute_pytest_with_subprocess_preflight(self, params, **kwargs):
|
||||
if kwargs:
|
||||
return _original_execute_pytest(self, params, **kwargs)
|
||||
if os.environ.get("MUTANT_UNDER_TEST") not in {"", "fail"}:
|
||||
return _original_execute_pytest(self, params, **kwargs)
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
full_params = ["--rootdir=.", "--tb=native", *params, *self._pytest_add_cli_args]
|
||||
result = subprocess.run([sys.executable, "-m", "pytest", *full_params], check=False)
|
||||
if result.returncode == 4:
|
||||
raise mutmut_main.BadTestExecutionCommandsException(full_params)
|
||||
return int(result.returncode)
|
||||
|
||||
mutmut_main.PytestRunner.execute_pytest = execute_pytest_with_subprocess_preflight
|
||||
@@ -0,0 +1,451 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prepare mutmut artifacts in throwaway processes before execution."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import inspect
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from scripts.check_mutation_targets import exact_python_targets
|
||||
from scripts.check_mutation_targets import load_mutmut_config
|
||||
from scripts.check_mutation_targets import SUPPORTED_MUTMUT_VERSION
|
||||
|
||||
|
||||
MANIFEST_PATH = Path("mutants/.covered-lines-prepass.json")
|
||||
STATS_PATH = Path("mutants/mutmut-stats.json")
|
||||
HOOK_HASH_PATHS = (
|
||||
Path("scripts/prepare_mutation_coverage.py"),
|
||||
Path("scripts/run_prepared_mutmut.py"),
|
||||
Path("scripts/mutmut_coverage_sitecustomize/sitecustomize.py"),
|
||||
)
|
||||
META_KEYS = {
|
||||
"exit_code_by_key",
|
||||
"hash_by_function_name",
|
||||
"type_check_error_by_key",
|
||||
"durations_by_key",
|
||||
"estimated_durations_by_key",
|
||||
}
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def _validated_relative_file(raw_path: str, *, label: str) -> Path:
|
||||
path = Path(raw_path)
|
||||
if path.is_absolute() or ".." in path.parts:
|
||||
raise RuntimeError(f"{label} must be a project-relative file: {raw_path}")
|
||||
if not path.is_file():
|
||||
raise RuntimeError(f"{label} is missing: {raw_path}")
|
||||
return path
|
||||
|
||||
|
||||
def _hash_relative_files(raw_paths: list[str], *, label: str) -> dict[str, str]:
|
||||
return {
|
||||
str(path): _sha256(path)
|
||||
for path in (_validated_relative_file(raw_path, label=label) for raw_path in raw_paths)
|
||||
}
|
||||
|
||||
|
||||
def _validated_mutation_path(path: Path) -> Path:
|
||||
if path.is_absolute() or ".." in path.parts:
|
||||
raise RuntimeError(f"mutation path must be project-relative: {path}")
|
||||
return path
|
||||
|
||||
|
||||
def _owned_mutants_root() -> Path:
|
||||
root = Path.cwd().resolve()
|
||||
expected = root / "mutants"
|
||||
if expected.is_symlink():
|
||||
raise RuntimeError(
|
||||
"mutants directory must be project-owned as the literal project-owned mutants directory",
|
||||
)
|
||||
return expected
|
||||
|
||||
|
||||
def _owned_mutation_artifact(path: Path, suffix: str) -> Path:
|
||||
relative = _validated_mutation_path(path)
|
||||
root = _owned_mutants_root()
|
||||
artifact = Path("mutants") / f"{relative}{suffix}"
|
||||
resolved = artifact.resolve() if artifact.exists() else artifact.resolve(strict=False)
|
||||
try:
|
||||
resolved.relative_to(root)
|
||||
except ValueError as exc:
|
||||
raise RuntimeError(f"mutation artifact escapes project-owned mutants/: {artifact}") from exc
|
||||
return artifact
|
||||
|
||||
|
||||
def _mutmut_version() -> str:
|
||||
import mutmut
|
||||
|
||||
return getattr(mutmut, "__version__", "")
|
||||
|
||||
|
||||
def assert_supported_mutmut() -> None:
|
||||
"""Fail closed if the mutmut internals this script calls have changed."""
|
||||
from mutmut import __main__ as mutmut_main
|
||||
|
||||
if _mutmut_version() != SUPPORTED_MUTMUT_VERSION:
|
||||
raise RuntimeError(
|
||||
f"unsupported mutmut version {_mutmut_version()!r}; expected {SUPPORTED_MUTMUT_VERSION}",
|
||||
)
|
||||
|
||||
expected = {
|
||||
"copy_src_dir": ("() -> 'None'",),
|
||||
"copy_also_copy_files": ("() -> 'None'",),
|
||||
"setup_source_paths": ("() -> 'None'",),
|
||||
"create_mutants": ("(max_children: 'int') -> 'MutantGenerationStats'",),
|
||||
"collect_or_load_stats": (
|
||||
"(runner: 'TestRunner', *, mutants_caught_by_type_checker: 'dict[str, Any] | None' = None, "
|
||||
"apply_config_invalidation: 'bool' = False, invalidate_stale_callers: 'bool' = True) -> 'None'",
|
||||
"(runner, *, mutants_caught_by_type_checker=None, apply_config_invalidation=False, "
|
||||
"invalidate_stale_callers=True) -> 'None'",
|
||||
),
|
||||
"load_stats": ("() -> 'bool'",),
|
||||
}
|
||||
for name, signatures in expected.items():
|
||||
actual = str(inspect.signature(getattr(mutmut_main, name)))
|
||||
if actual not in signatures:
|
||||
raise RuntimeError(f"mutmut internal {name} signature changed: {actual}")
|
||||
|
||||
|
||||
def configured_paths() -> tuple[list[Path], list[Path]]:
|
||||
"""Return all mutatable paths and exact required paths from the loaded config."""
|
||||
from mutmut import __main__ as mutmut_main
|
||||
from mutmut.configuration import Config
|
||||
|
||||
Config.ensure_loaded()
|
||||
if not Config.get().mutate_only_covered_lines:
|
||||
raise RuntimeError("[tool.mutmut].mutate_only_covered_lines must be true")
|
||||
|
||||
mutatable = sorted(mutmut_main.walk_mutatable_files(), key=str)
|
||||
required = exact_python_targets(load_mutmut_config(Path("pyproject.toml")))
|
||||
missing = [path for path in required if path not in mutatable]
|
||||
if missing:
|
||||
joined = ", ".join(str(path) for path in missing)
|
||||
raise RuntimeError(f"required mutation target is not mutatable by mutmut config: {joined}")
|
||||
if not mutatable:
|
||||
raise RuntimeError("mutmut configuration produced no mutatable files")
|
||||
return mutatable, required
|
||||
|
||||
|
||||
def remove_mutation_artifacts(paths: list[Path]) -> None:
|
||||
"""Remove generated files whose freshness matters for covered-line reuse."""
|
||||
for path in paths:
|
||||
for suffix in ("", ".meta", ".spans"):
|
||||
artifact = _owned_mutation_artifact(path, suffix)
|
||||
if artifact.exists():
|
||||
artifact.unlink()
|
||||
if MANIFEST_PATH.exists():
|
||||
MANIFEST_PATH.unlink()
|
||||
|
||||
|
||||
def _read_meta(path: Path) -> dict[str, Any]:
|
||||
meta_path = Path("mutants") / f"{path}.meta"
|
||||
try:
|
||||
metadata = json.loads(meta_path.read_text(encoding="utf-8"))
|
||||
except FileNotFoundError as exc:
|
||||
raise RuntimeError(f"missing mutmut metadata for {path}") from exc
|
||||
except json.JSONDecodeError as exc:
|
||||
raise RuntimeError(f"malformed mutmut metadata for {path}: {exc}") from exc
|
||||
if not isinstance(metadata, dict) or set(metadata) != META_KEYS:
|
||||
raise RuntimeError(f"unexpected mutmut metadata schema for {path}: {sorted(metadata)}")
|
||||
if not isinstance(metadata["exit_code_by_key"], dict):
|
||||
raise RuntimeError(f"mutmut metadata exit_code_by_key is not an object for {path}")
|
||||
return metadata
|
||||
|
||||
|
||||
def _validate_spans(path: Path, metadata: dict[str, Any]) -> None:
|
||||
from mutmut.mutation.data import MutantLineSpans
|
||||
|
||||
spans = MutantLineSpans.load(path)
|
||||
if spans is None:
|
||||
raise RuntimeError(f"missing or unsupported mutmut spans index for {path}")
|
||||
generated_names = {key.rpartition(".")[2] for key in metadata["exit_code_by_key"]}
|
||||
missing = sorted(generated_names - set(spans.span_by_function_name))
|
||||
if missing:
|
||||
raise RuntimeError(f"mutmut spans index for {path} is missing mutants: {missing[:5]}")
|
||||
|
||||
|
||||
def validate_prepared_artifacts(
|
||||
*,
|
||||
mutatable: list[Path],
|
||||
required: list[Path],
|
||||
covered_lines: dict[str, set[int]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Validate generated artifacts and return a fresh-source manifest."""
|
||||
from mutmut.configuration import Config
|
||||
|
||||
source_hashes: dict[str, str] = {}
|
||||
selected_test_hashes: dict[str, str] = {}
|
||||
hook_hashes: dict[str, str] = {}
|
||||
covered_line_counts: dict[str, int] = {}
|
||||
mutant_counts: dict[str, int] = {}
|
||||
|
||||
for path in mutatable:
|
||||
source = _validated_mutation_path(Path(path))
|
||||
mutant = _owned_mutation_artifact(path, "")
|
||||
if not source.is_file():
|
||||
raise RuntimeError(f"configured mutation source does not exist: {path}")
|
||||
if not mutant.is_file() or source.stat().st_mtime >= mutant.stat().st_mtime:
|
||||
raise RuntimeError(f"mutmut artifact for {path} is missing or stale")
|
||||
metadata = _read_meta(path)
|
||||
_validate_spans(path, metadata)
|
||||
source_hashes[str(path)] = _sha256(source)
|
||||
mutant_counts[str(path)] = len(metadata["exit_code_by_key"])
|
||||
if covered_lines is not None:
|
||||
key = str((Path("mutants") / path).absolute())
|
||||
covered_line_counts[str(path)] = len(covered_lines.get(key, set()))
|
||||
|
||||
for path in required:
|
||||
if mutant_counts.get(str(path), 0) == 0:
|
||||
raise RuntimeError(f"required mutation target produced zero covered mutants: {path}")
|
||||
if covered_lines is not None and covered_line_counts.get(str(path), 0) == 0:
|
||||
raise RuntimeError(f"required mutation target had no covered lines: {path}")
|
||||
|
||||
config = Config.get()
|
||||
selected_test_hashes = _hash_relative_files(
|
||||
list(config.pytest_add_cli_args_test_selection),
|
||||
label="selected mutation test file",
|
||||
)
|
||||
hook_hashes = _hash_relative_files([str(path) for path in HOOK_HASH_PATHS], label="mutation hook")
|
||||
return {
|
||||
"version": 1,
|
||||
"mutmut_version": _mutmut_version(),
|
||||
"mutate_only_covered_lines": config.mutate_only_covered_lines,
|
||||
"source_paths": [str(path) for path in config.source_paths],
|
||||
"only_mutate": list(config.only_mutate),
|
||||
"pytest_add_cli_args": list(config.pytest_add_cli_args),
|
||||
"pytest_add_cli_args_test_selection": list(config.pytest_add_cli_args_test_selection),
|
||||
"required_mutation_targets": [str(path) for path in required],
|
||||
"mutatable_paths": [str(path) for path in mutatable],
|
||||
"source_hashes": source_hashes,
|
||||
"selected_test_hashes": selected_test_hashes,
|
||||
"hook_hashes": hook_hashes,
|
||||
"covered_line_counts": covered_line_counts,
|
||||
"mutant_counts": mutant_counts,
|
||||
}
|
||||
|
||||
|
||||
def prepare(max_children: int) -> dict[str, Any]:
|
||||
"""Collect coverage, generate covered-only mutants, validate them, and write a manifest."""
|
||||
import mutmut
|
||||
from mutmut import __main__ as mutmut_main
|
||||
from mutmut.code_coverage import gather_coverage
|
||||
from mutmut.state import state
|
||||
|
||||
assert_supported_mutmut()
|
||||
mutatable, required = configured_paths()
|
||||
Path("mutants").mkdir(exist_ok=True)
|
||||
remove_mutation_artifacts(mutatable)
|
||||
|
||||
mutmut_main.copy_src_dir()
|
||||
mutmut_main.copy_also_copy_files()
|
||||
mutmut_main.setup_source_paths()
|
||||
source_files = list(mutmut_main.walk_source_files())
|
||||
covered_lines = gather_coverage(mutmut_main.PytestRunner(), source_files)
|
||||
mutmut._covered_lines = covered_lines
|
||||
stats = mutmut_main.create_mutants(max_children)
|
||||
state().current_function_hashes.clear()
|
||||
|
||||
manifest = validate_prepared_artifacts(
|
||||
mutatable=mutatable,
|
||||
required=required,
|
||||
covered_lines=covered_lines,
|
||||
)
|
||||
manifest["generation_stats"] = {
|
||||
"mutated": stats.mutated,
|
||||
"ignored": stats.ignored,
|
||||
"unmodified": stats.unmodified,
|
||||
}
|
||||
MANIFEST_PATH.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
return manifest
|
||||
|
||||
|
||||
def validate_coverage_manifest() -> dict[str, Any]:
|
||||
"""Validate that a prior prepass still matches the current sources and config."""
|
||||
assert_supported_mutmut()
|
||||
mutatable, required = configured_paths()
|
||||
try:
|
||||
manifest = json.loads(MANIFEST_PATH.read_text(encoding="utf-8"))
|
||||
except FileNotFoundError as exc:
|
||||
raise RuntimeError(f"missing covered-line prepass manifest: {MANIFEST_PATH}") from exc
|
||||
except json.JSONDecodeError as exc:
|
||||
raise RuntimeError(f"malformed covered-line prepass manifest: {exc}") from exc
|
||||
|
||||
expected = validate_prepared_artifacts(mutatable=mutatable, required=required)
|
||||
comparable_keys = (
|
||||
"version",
|
||||
"mutmut_version",
|
||||
"mutate_only_covered_lines",
|
||||
"source_paths",
|
||||
"only_mutate",
|
||||
"pytest_add_cli_args",
|
||||
"pytest_add_cli_args_test_selection",
|
||||
"required_mutation_targets",
|
||||
"mutatable_paths",
|
||||
"source_hashes",
|
||||
"selected_test_hashes",
|
||||
"hook_hashes",
|
||||
)
|
||||
for key in comparable_keys:
|
||||
if manifest.get(key) != expected[key]:
|
||||
raise RuntimeError(f"covered-line prepass manifest is stale for {key}")
|
||||
return manifest
|
||||
|
||||
|
||||
def validate_manifest() -> dict[str, Any]:
|
||||
"""Backward-compatible name for covered-line manifest validation."""
|
||||
return validate_coverage_manifest()
|
||||
|
||||
|
||||
def _read_stats() -> dict[str, Any]:
|
||||
try:
|
||||
stats = json.loads(STATS_PATH.read_text(encoding="utf-8"))
|
||||
except FileNotFoundError as exc:
|
||||
raise RuntimeError(f"missing prepared mutmut stats: {STATS_PATH}") from exc
|
||||
except json.JSONDecodeError as exc:
|
||||
raise RuntimeError(f"malformed prepared mutmut stats: {exc}") from exc
|
||||
if not isinstance(stats, dict):
|
||||
raise RuntimeError("prepared mutmut stats root must be an object")
|
||||
return stats
|
||||
|
||||
|
||||
def validate_stats_artifacts() -> dict[str, Any]:
|
||||
"""Validate prepared mutmut test-selection stats and return manifest fields."""
|
||||
stats = _read_stats()
|
||||
expected_keys = {
|
||||
"tests_by_mangled_function_name",
|
||||
"duration_by_test",
|
||||
"stats_time",
|
||||
"function_hashes",
|
||||
"function_dependencies",
|
||||
"config_fingerprint",
|
||||
"watched_file_hashes",
|
||||
"git_commit",
|
||||
}
|
||||
if set(stats) != expected_keys:
|
||||
raise RuntimeError(f"unexpected mutmut stats schema: {sorted(stats)}")
|
||||
|
||||
tests_by_function = stats["tests_by_mangled_function_name"]
|
||||
duration_by_test = stats["duration_by_test"]
|
||||
function_hashes = stats["function_hashes"]
|
||||
if not isinstance(tests_by_function, dict) or not tests_by_function:
|
||||
raise RuntimeError("prepared mutmut stats have no test-to-function mapping")
|
||||
if not isinstance(duration_by_test, dict) or not duration_by_test:
|
||||
raise RuntimeError("prepared mutmut stats have no test duration mapping")
|
||||
if not isinstance(function_hashes, dict) or not function_hashes:
|
||||
raise RuntimeError("prepared mutmut stats have no function hash baseline")
|
||||
associated_tests = {
|
||||
test
|
||||
for tests in tests_by_function.values()
|
||||
if isinstance(tests, list)
|
||||
for test in tests
|
||||
}
|
||||
if not associated_tests:
|
||||
raise RuntimeError("prepared mutmut stats do not associate any tests with mutants")
|
||||
missing_durations = sorted(associated_tests - set(duration_by_test))
|
||||
if missing_durations:
|
||||
raise RuntimeError(
|
||||
"prepared mutmut stats reference tests without durations: "
|
||||
f"{missing_durations[:5]}",
|
||||
)
|
||||
return {
|
||||
"stats_hash": _sha256(STATS_PATH),
|
||||
"stats_function_count": len(function_hashes),
|
||||
"stats_test_count": len(duration_by_test),
|
||||
"stats_association_count": sum(
|
||||
len(tests) for tests in tests_by_function.values() if isinstance(tests, list)
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def prepare_stats(max_children: int) -> dict[str, Any]:
|
||||
"""Build mutmut test-selection stats in a fresh process, then exit."""
|
||||
from mutmut import __main__ as mutmut_main
|
||||
from mutmut.state import state
|
||||
|
||||
manifest = validate_coverage_manifest()
|
||||
if STATS_PATH.exists():
|
||||
STATS_PATH.unlink()
|
||||
|
||||
mutmut_main.copy_src_dir()
|
||||
mutmut_main.copy_also_copy_files()
|
||||
mutmut_main.setup_source_paths()
|
||||
mutmut_main.create_mutants(max_children)
|
||||
runner = mutmut_main.PytestRunner()
|
||||
runner.prepare_main_test_run()
|
||||
mutmut_main.collect_or_load_stats(
|
||||
runner,
|
||||
mutants_caught_by_type_checker={},
|
||||
apply_config_invalidation=False,
|
||||
invalidate_stale_callers=False,
|
||||
)
|
||||
if not state().current_function_hashes:
|
||||
raise RuntimeError("mutmut stats prepass did not populate function hashes")
|
||||
|
||||
manifest.update(validate_stats_artifacts())
|
||||
MANIFEST_PATH.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
return manifest
|
||||
|
||||
|
||||
def validate_execution_manifest() -> dict[str, Any]:
|
||||
"""Validate coverage artifacts plus prepared test-selection stats."""
|
||||
manifest = validate_coverage_manifest()
|
||||
stats_fields = validate_stats_artifacts()
|
||||
for key, value in stats_fields.items():
|
||||
if manifest.get(key) != value:
|
||||
raise RuntimeError(f"prepared mutmut execution manifest is stale for {key}")
|
||||
return manifest
|
||||
|
||||
|
||||
def _parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"command",
|
||||
choices=("prepare", "prepare-coverage", "prepare-stats", "validate", "validate-execution"),
|
||||
)
|
||||
parser.add_argument("--max-children", type=int, default=4)
|
||||
return parser
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = _parser().parse_args()
|
||||
try:
|
||||
if args.command in {"prepare", "prepare-coverage"}:
|
||||
manifest = prepare(args.max_children)
|
||||
total = sum(manifest["mutant_counts"].values())
|
||||
print(f"prepared covered-line mutation artifacts for {total} mutants")
|
||||
elif args.command == "prepare-stats":
|
||||
manifest = prepare_stats(args.max_children)
|
||||
print(
|
||||
"prepared mutmut test-selection stats for "
|
||||
f"{manifest['stats_association_count']} function/test associations",
|
||||
)
|
||||
elif args.command == "validate-execution":
|
||||
validate_execution_manifest()
|
||||
print("prepared mutation execution artifacts are fresh")
|
||||
else:
|
||||
validate_coverage_manifest()
|
||||
print("covered-line mutation artifacts are fresh")
|
||||
except Exception as exc:
|
||||
print(f"mutation coverage preparation failed: {exc}")
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
exit_code = main()
|
||||
sys.stdout.flush()
|
||||
sys.stderr.flush()
|
||||
os._exit(exit_code)
|
||||
@@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run mutmut with prepared coverage and stats artifacts in a fresh interpreter."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
SITECUSTOMIZE = PROJECT_ROOT / "scripts" / "mutmut_coverage_sitecustomize"
|
||||
|
||||
|
||||
def _parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("mutmut_args", nargs=argparse.REMAINDER)
|
||||
return parser
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = _parser().parse_args()
|
||||
mutmut = shutil.which("mutmut")
|
||||
if mutmut is None:
|
||||
print("prepared mutmut runner failed: mutmut executable is not on PATH")
|
||||
return 1
|
||||
|
||||
pythonpath = os.environ.get("PYTHONPATH")
|
||||
prefixes = [str(SITECUSTOMIZE), str(PROJECT_ROOT)]
|
||||
os.environ["PYTHONPATH"] = os.pathsep.join(
|
||||
[*prefixes, *([pythonpath] if pythonpath else [])],
|
||||
)
|
||||
os.environ["OBLITERATUS_MUTMUT_REUSE_COVERAGE"] = "1"
|
||||
os.environ["OBLITERATUS_MUTMUT_REUSE_STATS"] = "1"
|
||||
os.environ["OBLITERATUS_MUTMUT_SUBPROCESS_PREFLIGHT"] = "1"
|
||||
|
||||
command = [mutmut, *(args.mutmut_args or ["run"])]
|
||||
os.execv(mutmut, command)
|
||||
raise AssertionError("os.execv returned unexpectedly")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user