mirror of
https://github.com/elder-plinius/OBLITERATUS.git
synced 2026-08-18 00:47:23 +02:00
373 lines
14 KiB
Python
373 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Validate the source-to-test risk map and conditional coverage graph."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import re
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
RISK_CLASSES = {"cpu-contract", "mixed-runtime", "conditional-runtime"}
|
|
CONTRACT_TYPES = {
|
|
"ablation-strategy",
|
|
"architecture-selection",
|
|
"configuration",
|
|
"device-boundary",
|
|
"external-service",
|
|
"model-mutation",
|
|
"model-runtime",
|
|
"numerical-invariant",
|
|
"operator-ui",
|
|
"orchestration",
|
|
"package-entrypoint",
|
|
"persistence",
|
|
"public-interface",
|
|
"remote-execution",
|
|
"reproducibility",
|
|
"research-input",
|
|
"research-metric",
|
|
"research-output",
|
|
}
|
|
SURFACE_ID = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$")
|
|
|
|
|
|
def _load_object(path: Path, label: str, errors: list[str]) -> dict[str, Any]:
|
|
try:
|
|
value = json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError) as exc:
|
|
errors.append(f"cannot read {label}: {exc}")
|
|
return {}
|
|
if not isinstance(value, dict):
|
|
errors.append(f"{label} root must be an object")
|
|
return {}
|
|
return value
|
|
|
|
|
|
def _project_root(risk_path: Path, quality_path: Path) -> Path:
|
|
"""Resolve the checkout root even when a test supplies temporary policy files."""
|
|
|
|
candidates = [
|
|
risk_path.resolve().parent.parent,
|
|
quality_path.resolve().parent.parent,
|
|
Path.cwd().resolve(),
|
|
]
|
|
for candidate in candidates:
|
|
if (candidate / "obliteratus").is_dir() and (candidate / "tests").is_dir():
|
|
return candidate
|
|
return Path.cwd().resolve()
|
|
|
|
|
|
def _validate_test_paths(
|
|
tests: object,
|
|
*,
|
|
label: str,
|
|
project_root: Path,
|
|
errors: list[str],
|
|
) -> None:
|
|
if not isinstance(tests, list) or not tests:
|
|
errors.append(f"{label} requires at least one test")
|
|
return
|
|
if len(tests) != len(set(map(str, tests))):
|
|
errors.append(f"{label} has duplicate test paths")
|
|
for test_path in tests:
|
|
if not isinstance(test_path, str) or not test_path.startswith("tests/"):
|
|
errors.append(f"{label} has invalid test path: {test_path!r}")
|
|
elif not (project_root / test_path).is_file():
|
|
errors.append(f"{label} maps missing test path: {test_path}")
|
|
|
|
|
|
def _inventory_sources(
|
|
inventory: object,
|
|
*,
|
|
project_root: Path,
|
|
errors: list[str],
|
|
) -> set[str]:
|
|
if not isinstance(inventory, dict):
|
|
errors.append("test risk map source_inventory must be an object")
|
|
return set()
|
|
roots = inventory.get("roots")
|
|
if not isinstance(roots, list) or not roots:
|
|
errors.append("test risk map source_inventory requires non-empty roots")
|
|
return set()
|
|
if len(roots) != len(set(map(str, roots))):
|
|
errors.append("test risk map source_inventory has duplicate roots")
|
|
|
|
sources: set[str] = set()
|
|
for value in roots:
|
|
if not isinstance(value, str) or not value or Path(value).is_absolute():
|
|
errors.append(f"source_inventory has invalid root: {value!r}")
|
|
continue
|
|
root = project_root / value
|
|
if root.is_file():
|
|
if root.suffix != ".py":
|
|
errors.append(f"source_inventory root is not Python source: {value}")
|
|
else:
|
|
sources.add(root.relative_to(project_root).as_posix())
|
|
elif root.is_dir():
|
|
sources.update(
|
|
path.relative_to(project_root).as_posix()
|
|
for path in root.rglob("*.py")
|
|
if path.is_file()
|
|
)
|
|
else:
|
|
errors.append(f"source_inventory root does not exist: {value}")
|
|
return sources
|
|
|
|
|
|
def _coverage_roots(
|
|
inventory: object,
|
|
*,
|
|
project_root: Path,
|
|
errors: list[str],
|
|
) -> tuple[str, ...]:
|
|
if not isinstance(inventory, dict):
|
|
return ()
|
|
values = inventory.get("coverage_roots")
|
|
if not isinstance(values, list) or not values:
|
|
errors.append("test risk map source_inventory requires non-empty coverage_roots")
|
|
return ()
|
|
if len(values) != len(set(map(str, values))):
|
|
errors.append("test risk map source_inventory has duplicate coverage_roots")
|
|
|
|
roots: list[str] = []
|
|
for value in values:
|
|
if not isinstance(value, str) or not value or Path(value).is_absolute():
|
|
errors.append(f"source_inventory has invalid coverage root: {value!r}")
|
|
continue
|
|
if not (project_root / value).exists():
|
|
errors.append(f"source_inventory coverage root does not exist: {value}")
|
|
continue
|
|
roots.append(value.rstrip("/"))
|
|
return tuple(roots)
|
|
|
|
|
|
def _is_in_coverage_scope(path: str, roots: tuple[str, ...]) -> bool:
|
|
return any(path == root or path.startswith(f"{root}/") for root in roots)
|
|
|
|
|
|
def _validate_contract_surfaces(
|
|
surfaces: object,
|
|
*,
|
|
inventory_sources: set[str],
|
|
project_root: Path,
|
|
errors: list[str],
|
|
) -> dict[str, str]:
|
|
if not isinstance(surfaces, list) or not surfaces:
|
|
errors.append("test risk map contract_surfaces must be a non-empty list")
|
|
return {}
|
|
|
|
surface_ids: set[str] = set()
|
|
source_owner: dict[str, str] = {}
|
|
for index, surface in enumerate(surfaces):
|
|
label = f"contract surface {index}"
|
|
if not isinstance(surface, dict):
|
|
errors.append(f"{label} must be an object")
|
|
continue
|
|
surface_id = surface.get("id")
|
|
if not isinstance(surface_id, str) or not SURFACE_ID.fullmatch(surface_id):
|
|
errors.append(f"{label} has invalid id: {surface_id!r}")
|
|
surface_id = f"index-{index}"
|
|
elif surface_id in surface_ids:
|
|
errors.append(f"duplicate contract surface id: {surface_id}")
|
|
surface_ids.add(surface_id)
|
|
label = f"contract surface {surface_id}"
|
|
|
|
if not isinstance(surface.get("owner"), str) or not surface["owner"].strip():
|
|
errors.append(f"{label} requires a non-empty owner")
|
|
if not isinstance(surface.get("description"), str) or not surface["description"].strip():
|
|
errors.append(f"{label} requires a non-empty description")
|
|
|
|
contract_types = surface.get("contract_types")
|
|
if not isinstance(contract_types, list) or not contract_types:
|
|
errors.append(f"{label} requires at least one contract type")
|
|
else:
|
|
if len(contract_types) != len(set(map(str, contract_types))):
|
|
errors.append(f"{label} has duplicate contract types")
|
|
for contract_type in contract_types:
|
|
if not isinstance(contract_type, str) or contract_type not in CONTRACT_TYPES:
|
|
errors.append(f"{label} has invalid contract type: {contract_type!r}")
|
|
|
|
_validate_test_paths(
|
|
surface.get("required_tests"),
|
|
label=label,
|
|
project_root=project_root,
|
|
errors=errors,
|
|
)
|
|
|
|
paths = surface.get("paths")
|
|
if not isinstance(paths, list) or not paths:
|
|
errors.append(f"{label} requires at least one source path")
|
|
continue
|
|
if len(paths) != len(set(map(str, paths))):
|
|
errors.append(f"{label} has duplicate source paths")
|
|
for source_path in paths:
|
|
if not isinstance(source_path, str) or not source_path:
|
|
errors.append(f"{label} has invalid source path: {source_path!r}")
|
|
continue
|
|
if source_path not in inventory_sources:
|
|
errors.append(f"{label} maps source outside inventory: {source_path}")
|
|
previous = source_owner.get(source_path)
|
|
if previous is not None:
|
|
errors.append(
|
|
f"production source path has multiple contract owners: "
|
|
f"{source_path} ({previous}, {surface_id})"
|
|
)
|
|
else:
|
|
source_owner[source_path] = surface_id
|
|
|
|
for source_path in sorted(inventory_sources - source_owner.keys()):
|
|
errors.append(f"unmapped production source path: {source_path}")
|
|
return source_owner
|
|
|
|
|
|
def validate(risk_path: Path, quality_path: Path, conditional_path: Path) -> list[str]:
|
|
"""Return structural and cross-policy failures for the risk map."""
|
|
|
|
errors: list[str] = []
|
|
risk = _load_object(risk_path, "test risk map", errors)
|
|
quality = _load_object(quality_path, "quality policy", errors)
|
|
conditional = _load_object(conditional_path, "conditional policy", errors)
|
|
if errors:
|
|
return errors
|
|
|
|
project_root = _project_root(risk_path, quality_path)
|
|
if risk.get("schema_version") != 2:
|
|
errors.append("test risk map schema_version must be 2")
|
|
if not isinstance(risk.get("owner"), str) or not risk["owner"].strip():
|
|
errors.append("test risk map requires a non-empty owner")
|
|
|
|
source_inventory = risk.get("source_inventory")
|
|
inventory_sources = _inventory_sources(
|
|
source_inventory,
|
|
project_root=project_root,
|
|
errors=errors,
|
|
)
|
|
coverage_roots = _coverage_roots(
|
|
source_inventory,
|
|
project_root=project_root,
|
|
errors=errors,
|
|
)
|
|
source_contracts = _validate_contract_surfaces(
|
|
risk.get("contract_surfaces"),
|
|
inventory_sources=inventory_sources,
|
|
project_root=project_root,
|
|
errors=errors,
|
|
)
|
|
|
|
gates = conditional.get("gates")
|
|
gate_by_id = {
|
|
gate.get("id"): gate
|
|
for gate in gates if isinstance(gate, dict) and isinstance(gate.get("id"), str)
|
|
} if isinstance(gates, list) else {}
|
|
|
|
modules = risk.get("modules")
|
|
if not isinstance(modules, list) or not modules:
|
|
return errors + ["test risk map modules must be a non-empty list"]
|
|
|
|
module_by_path: dict[str, dict[str, Any]] = {}
|
|
for index, module in enumerate(modules):
|
|
label = f"risk module {index}"
|
|
if not isinstance(module, dict):
|
|
errors.append(f"{label} must be an object")
|
|
continue
|
|
path = module.get("path")
|
|
if not isinstance(path, str) or not path:
|
|
errors.append(f"{label} requires a non-empty path")
|
|
continue
|
|
if path in module_by_path:
|
|
errors.append(f"duplicate risk module path: {path}")
|
|
module_by_path[path] = module
|
|
if not (project_root / path).is_file():
|
|
errors.append(f"risk module maps missing source path: {path}")
|
|
if path not in source_contracts:
|
|
errors.append(f"risk module is missing a contract owner: {path}")
|
|
if not _is_in_coverage_scope(path, coverage_roots):
|
|
errors.append(f"risk module is outside measured coverage roots: {path}")
|
|
risk_class = module.get("risk_class")
|
|
if not isinstance(risk_class, str) or risk_class not in RISK_CLASSES:
|
|
errors.append(f"risk module {path} has invalid risk_class")
|
|
if not isinstance(module.get("risk"), str) or not module["risk"].strip():
|
|
errors.append(f"risk module {path} requires a non-empty risk")
|
|
|
|
_validate_test_paths(
|
|
module.get("required_tests"),
|
|
label=f"risk module {path}",
|
|
project_root=project_root,
|
|
errors=errors,
|
|
)
|
|
|
|
module_gates = module.get("conditional_gates")
|
|
if not isinstance(module_gates, list):
|
|
errors.append(f"risk module {path} conditional_gates must be a list")
|
|
continue
|
|
if len(module_gates) != len(set(map(str, module_gates))):
|
|
errors.append(f"risk module {path} has duplicate conditional gates")
|
|
for gate_id in module_gates:
|
|
if not isinstance(gate_id, str):
|
|
errors.append(f"risk module {path} has invalid conditional gate: {gate_id!r}")
|
|
continue
|
|
gate = gate_by_id.get(gate_id)
|
|
if gate is None:
|
|
errors.append(f"risk module {path} references unknown gate {gate_id}")
|
|
elif path not in gate.get("coverage_paths", []):
|
|
errors.append(f"risk module {path} is not covered by gate {gate_id}")
|
|
|
|
exclusions = quality.get("mature_cpu_scope", {}).get("exclusions", [])
|
|
if not isinstance(exclusions, list):
|
|
errors.append("quality policy exclusions must be a list")
|
|
exclusions = []
|
|
for exclusion in exclusions:
|
|
if not isinstance(exclusion, dict):
|
|
errors.append("quality policy exclusion must be an object")
|
|
continue
|
|
path = exclusion.get("path")
|
|
gate_id = exclusion.get("conditional_gate")
|
|
module = module_by_path.get(path)
|
|
if module is None:
|
|
errors.append(f"CPU exclusion is missing from test risk map: {path}")
|
|
elif gate_id not in module.get("conditional_gates", []):
|
|
errors.append(f"CPU exclusion {path} is missing conditional gate {gate_id}")
|
|
|
|
critical_paths = quality.get("critical_cpu_paths", [])
|
|
if not isinstance(critical_paths, list):
|
|
errors.append("quality policy critical_cpu_paths must be a list")
|
|
critical_paths = []
|
|
for path in critical_paths:
|
|
if path not in module_by_path:
|
|
errors.append(f"critical CPU path is missing from test risk map: {path}")
|
|
|
|
for gate_id, gate in gate_by_id.items():
|
|
for path in gate.get("coverage_paths", []):
|
|
module = module_by_path.get(path)
|
|
if module is None:
|
|
errors.append(f"conditional path is missing from test risk map: {path}")
|
|
elif gate_id not in module.get("conditional_gates", []):
|
|
errors.append(f"risk module {path} is missing conditional gate {gate_id}")
|
|
return errors
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--risk-map", type=Path, default=Path("ci/test-risk-map.json"))
|
|
parser.add_argument("--quality", type=Path, default=Path("ci/test-quality-policy.json"))
|
|
parser.add_argument(
|
|
"--conditional",
|
|
type=Path,
|
|
default=Path("ci/conditional-test-policy.json"),
|
|
)
|
|
args = parser.parse_args()
|
|
errors = validate(args.risk_map, args.quality, args.conditional)
|
|
if errors:
|
|
for error in errors:
|
|
print(f"ERROR: {error}")
|
|
return 1
|
|
print("test risk map: valid")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|