#!/usr/bin/env python3 """Validate the source-to-test risk map and conditional coverage graph.""" from __future__ import annotations import argparse import json import re from pathlib import Path from typing import Any RISK_CLASSES = {"cpu-contract", "mixed-runtime", "conditional-runtime"} CONTRACT_TYPES = { "ablation-strategy", "architecture-selection", "configuration", "device-boundary", "external-service", "model-mutation", "model-runtime", "numerical-invariant", "operator-ui", "orchestration", "package-entrypoint", "persistence", "public-interface", "remote-execution", "reproducibility", "research-input", "research-metric", "research-output", } SURFACE_ID = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") def _load_object(path: Path, label: str, errors: list[str]) -> dict[str, Any]: try: value = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: errors.append(f"cannot read {label}: {exc}") return {} if not isinstance(value, dict): errors.append(f"{label} root must be an object") return {} return value def _project_root(risk_path: Path, quality_path: Path) -> Path: """Resolve the checkout root even when a test supplies temporary policy files.""" candidates = [ risk_path.resolve().parent.parent, quality_path.resolve().parent.parent, Path.cwd().resolve(), ] for candidate in candidates: if (candidate / "obliteratus").is_dir() and (candidate / "tests").is_dir(): return candidate return Path.cwd().resolve() def _validate_test_paths( tests: object, *, label: str, project_root: Path, errors: list[str], ) -> None: if not isinstance(tests, list) or not tests: errors.append(f"{label} requires at least one test") return if len(tests) != len(set(map(str, tests))): errors.append(f"{label} has duplicate test paths") for test_path in tests: if not isinstance(test_path, str) or not test_path.startswith("tests/"): errors.append(f"{label} has invalid test path: {test_path!r}") elif not (project_root / test_path).is_file(): errors.append(f"{label} maps missing test path: {test_path}") def _inventory_sources( inventory: object, *, project_root: Path, errors: list[str], ) -> set[str]: if not isinstance(inventory, dict): errors.append("test risk map source_inventory must be an object") return set() roots = inventory.get("roots") if not isinstance(roots, list) or not roots: errors.append("test risk map source_inventory requires non-empty roots") return set() if len(roots) != len(set(map(str, roots))): errors.append("test risk map source_inventory has duplicate roots") sources: set[str] = set() for value in roots: if not isinstance(value, str) or not value or Path(value).is_absolute(): errors.append(f"source_inventory has invalid root: {value!r}") continue root = project_root / value if root.is_file(): if root.suffix != ".py": errors.append(f"source_inventory root is not Python source: {value}") else: sources.add(root.relative_to(project_root).as_posix()) elif root.is_dir(): sources.update( path.relative_to(project_root).as_posix() for path in root.rglob("*.py") if path.is_file() ) else: errors.append(f"source_inventory root does not exist: {value}") return sources def _coverage_roots( inventory: object, *, project_root: Path, errors: list[str], ) -> tuple[str, ...]: if not isinstance(inventory, dict): return () values = inventory.get("coverage_roots") if not isinstance(values, list) or not values: errors.append("test risk map source_inventory requires non-empty coverage_roots") return () if len(values) != len(set(map(str, values))): errors.append("test risk map source_inventory has duplicate coverage_roots") roots: list[str] = [] for value in values: if not isinstance(value, str) or not value or Path(value).is_absolute(): errors.append(f"source_inventory has invalid coverage root: {value!r}") continue if not (project_root / value).exists(): errors.append(f"source_inventory coverage root does not exist: {value}") continue roots.append(value.rstrip("/")) return tuple(roots) def _is_in_coverage_scope(path: str, roots: tuple[str, ...]) -> bool: return any(path == root or path.startswith(f"{root}/") for root in roots) def _validate_contract_surfaces( surfaces: object, *, inventory_sources: set[str], project_root: Path, errors: list[str], ) -> dict[str, str]: if not isinstance(surfaces, list) or not surfaces: errors.append("test risk map contract_surfaces must be a non-empty list") return {} surface_ids: set[str] = set() source_owner: dict[str, str] = {} for index, surface in enumerate(surfaces): label = f"contract surface {index}" if not isinstance(surface, dict): errors.append(f"{label} must be an object") continue surface_id = surface.get("id") if not isinstance(surface_id, str) or not SURFACE_ID.fullmatch(surface_id): errors.append(f"{label} has invalid id: {surface_id!r}") surface_id = f"index-{index}" elif surface_id in surface_ids: errors.append(f"duplicate contract surface id: {surface_id}") surface_ids.add(surface_id) label = f"contract surface {surface_id}" if not isinstance(surface.get("owner"), str) or not surface["owner"].strip(): errors.append(f"{label} requires a non-empty owner") if not isinstance(surface.get("description"), str) or not surface["description"].strip(): errors.append(f"{label} requires a non-empty description") contract_types = surface.get("contract_types") if not isinstance(contract_types, list) or not contract_types: errors.append(f"{label} requires at least one contract type") else: if len(contract_types) != len(set(map(str, contract_types))): errors.append(f"{label} has duplicate contract types") for contract_type in contract_types: if not isinstance(contract_type, str) or contract_type not in CONTRACT_TYPES: errors.append(f"{label} has invalid contract type: {contract_type!r}") _validate_test_paths( surface.get("required_tests"), label=label, project_root=project_root, errors=errors, ) paths = surface.get("paths") if not isinstance(paths, list) or not paths: errors.append(f"{label} requires at least one source path") continue if len(paths) != len(set(map(str, paths))): errors.append(f"{label} has duplicate source paths") for source_path in paths: if not isinstance(source_path, str) or not source_path: errors.append(f"{label} has invalid source path: {source_path!r}") continue if source_path not in inventory_sources: errors.append(f"{label} maps source outside inventory: {source_path}") previous = source_owner.get(source_path) if previous is not None: errors.append( f"production source path has multiple contract owners: " f"{source_path} ({previous}, {surface_id})" ) else: source_owner[source_path] = surface_id for source_path in sorted(inventory_sources - source_owner.keys()): errors.append(f"unmapped production source path: {source_path}") return source_owner def validate(risk_path: Path, quality_path: Path, conditional_path: Path) -> list[str]: """Return structural and cross-policy failures for the risk map.""" errors: list[str] = [] risk = _load_object(risk_path, "test risk map", errors) quality = _load_object(quality_path, "quality policy", errors) conditional = _load_object(conditional_path, "conditional policy", errors) if errors: return errors project_root = _project_root(risk_path, quality_path) if risk.get("schema_version") != 2: errors.append("test risk map schema_version must be 2") if not isinstance(risk.get("owner"), str) or not risk["owner"].strip(): errors.append("test risk map requires a non-empty owner") source_inventory = risk.get("source_inventory") inventory_sources = _inventory_sources( source_inventory, project_root=project_root, errors=errors, ) coverage_roots = _coverage_roots( source_inventory, project_root=project_root, errors=errors, ) source_contracts = _validate_contract_surfaces( risk.get("contract_surfaces"), inventory_sources=inventory_sources, project_root=project_root, errors=errors, ) gates = conditional.get("gates") gate_by_id = { gate.get("id"): gate for gate in gates if isinstance(gate, dict) and isinstance(gate.get("id"), str) } if isinstance(gates, list) else {} modules = risk.get("modules") if not isinstance(modules, list) or not modules: return errors + ["test risk map modules must be a non-empty list"] module_by_path: dict[str, dict[str, Any]] = {} for index, module in enumerate(modules): label = f"risk module {index}" if not isinstance(module, dict): errors.append(f"{label} must be an object") continue path = module.get("path") if not isinstance(path, str) or not path: errors.append(f"{label} requires a non-empty path") continue if path in module_by_path: errors.append(f"duplicate risk module path: {path}") module_by_path[path] = module if not (project_root / path).is_file(): errors.append(f"risk module maps missing source path: {path}") if path not in source_contracts: errors.append(f"risk module is missing a contract owner: {path}") if not _is_in_coverage_scope(path, coverage_roots): errors.append(f"risk module is outside measured coverage roots: {path}") risk_class = module.get("risk_class") if not isinstance(risk_class, str) or risk_class not in RISK_CLASSES: errors.append(f"risk module {path} has invalid risk_class") if not isinstance(module.get("risk"), str) or not module["risk"].strip(): errors.append(f"risk module {path} requires a non-empty risk") _validate_test_paths( module.get("required_tests"), label=f"risk module {path}", project_root=project_root, errors=errors, ) module_gates = module.get("conditional_gates") if not isinstance(module_gates, list): errors.append(f"risk module {path} conditional_gates must be a list") continue if len(module_gates) != len(set(map(str, module_gates))): errors.append(f"risk module {path} has duplicate conditional gates") for gate_id in module_gates: if not isinstance(gate_id, str): errors.append(f"risk module {path} has invalid conditional gate: {gate_id!r}") continue gate = gate_by_id.get(gate_id) if gate is None: errors.append(f"risk module {path} references unknown gate {gate_id}") elif path not in gate.get("coverage_paths", []): errors.append(f"risk module {path} is not covered by gate {gate_id}") exclusions = quality.get("mature_cpu_scope", {}).get("exclusions", []) if not isinstance(exclusions, list): errors.append("quality policy exclusions must be a list") exclusions = [] for exclusion in exclusions: if not isinstance(exclusion, dict): errors.append("quality policy exclusion must be an object") continue path = exclusion.get("path") gate_id = exclusion.get("conditional_gate") module = module_by_path.get(path) if module is None: errors.append(f"CPU exclusion is missing from test risk map: {path}") elif gate_id not in module.get("conditional_gates", []): errors.append(f"CPU exclusion {path} is missing conditional gate {gate_id}") critical_paths = quality.get("critical_cpu_paths", []) if not isinstance(critical_paths, list): errors.append("quality policy critical_cpu_paths must be a list") critical_paths = [] for path in critical_paths: if path not in module_by_path: errors.append(f"critical CPU path is missing from test risk map: {path}") for gate_id, gate in gate_by_id.items(): for path in gate.get("coverage_paths", []): module = module_by_path.get(path) if module is None: errors.append(f"conditional path is missing from test risk map: {path}") elif gate_id not in module.get("conditional_gates", []): errors.append(f"risk module {path} is missing conditional gate {gate_id}") return errors def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--risk-map", type=Path, default=Path("ci/test-risk-map.json")) parser.add_argument("--quality", type=Path, default=Path("ci/test-quality-policy.json")) parser.add_argument( "--conditional", type=Path, default=Path("ci/conditional-test-policy.json"), ) args = parser.parse_args() errors = validate(args.risk_map, args.quality, args.conditional) if errors: for error in errors: print(f"ERROR: {error}") return 1 print("test risk map: valid") return 0 if __name__ == "__main__": raise SystemExit(main())