test: enforce Wave A quality lock

scripts/check_coverage_thresholds.py

@@ -7,10 +7,28 @@ import json
 import math
 import re
 import subprocess
+from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
 
 
+@dataclass(frozen=True)
+class ChangedModule:
+    """One live production module changed between the base and head commits."""
+
+    status: str
+    base_path: str | None
+    head_path: str
+
+
+@dataclass(frozen=True)
+class ModuleCoverage:
+    """Unrounded line and branch percentages for one measured module."""
+
+    line: float
+    branch: float | None
+
+
 def validate_coverage(
     report: dict[str, Any], *, min_line: float, min_branch: float,
     file_floors: dict[str, float] | None = None,
@@ -114,6 +132,166 @@ def validate_changed_coverage(
     return []
 
 
+def parse_changed_modules(
+    raw: bytes, *, roots: tuple[str, ...] = ("obliteratus",),
+) -> list[ChangedModule]:
+    """Parse ``git diff --name-status -M -z`` for live Python modules."""
+
+    tokens = raw.decode("utf-8", "surrogateescape").split("\0")
+    if tokens and tokens[-1] == "":
+        tokens.pop()
+
+    normalized_roots = tuple(root.rstrip("/") for root in roots if root.rstrip("/"))
+
+    def included(path: str) -> bool:
+        return path.endswith(".py") and any(
+            path.startswith(f"{root}/") for root in normalized_roots
+        )
+
+    changes: list[ChangedModule] = []
+    index = 0
+    while index < len(tokens):
+        status = tokens[index]
+        index += 1
+        if not status:
+            raise ValueError("changed-module diff contains an empty status")
+        kind = status[0]
+        if kind in {"R", "C"}:
+            if index + 1 >= len(tokens):
+                raise ValueError(f"changed-module diff is truncated after {status!r}")
+            old_path, new_path = tokens[index], tokens[index + 1]
+            index += 2
+            if kind == "R" and included(old_path) and not included(new_path):
+                changes.append(ChangedModule("D", old_path, old_path))
+                continue
+            if not included(new_path):
+                continue
+            base_path = old_path if kind == "R" and included(old_path) else None
+            changes.append(ChangedModule(status, base_path, new_path))
+            continue
+
+        if index >= len(tokens):
+            raise ValueError(f"changed-module diff is truncated after {status!r}")
+        path = tokens[index]
+        index += 1
+        if not included(path):
+            continue
+        if kind == "D":
+            changes.append(ChangedModule(status, path, path))
+            continue
+        if kind not in {"A", "M", "T"}:
+            raise ValueError(f"unsupported changed-module status {status!r}")
+        changes.append(
+            ChangedModule(status, None if kind == "A" else path, path),
+        )
+    return changes
+
+
+def module_coverage(
+    report: dict[str, Any], path: str,
+) -> tuple[ModuleCoverage | None, str | None]:
+    """Return exact per-module line/branch coverage or a validation failure."""
+
+    files = report.get("files")
+    entry = files.get(path) if isinstance(files, dict) else None
+    if not isinstance(entry, dict):
+        return None, f"coverage report is missing touched module {path}"
+    summary = entry.get("summary")
+    if not isinstance(summary, dict):
+        return None, f"coverage report is missing summary for touched module {path}"
+
+    values: dict[str, int] = {}
+    for key in ("covered_lines", "num_statements", "covered_branches", "num_branches"):
+        value = summary.get(key)
+        if isinstance(value, bool) or not isinstance(value, int) or value < 0:
+            return None, f"coverage report has invalid {key} for touched module {path}"
+        values[key] = value
+
+    if values["num_statements"] == 0:
+        return None, f"coverage report measures no statements for touched module {path}"
+    if values["covered_lines"] > values["num_statements"]:
+        return None, f"coverage report overcounts covered lines for touched module {path}"
+    if values["covered_branches"] > values["num_branches"]:
+        return None, f"coverage report overcounts covered branches for touched module {path}"
+
+    line = values["covered_lines"] / values["num_statements"] * 100
+    branch = (
+        values["covered_branches"] / values["num_branches"] * 100
+        if values["num_branches"]
+        else None
+    )
+    return ModuleCoverage(line=line, branch=branch), None
+
+
+def validate_touched_module_regression(
+    head_report: dict[str, Any],
+    base_report: dict[str, Any],
+    changes: list[ChangedModule],
+    *,
+    line_tolerance: float = 0.0,
+    branch_tolerance: float = 0.0,
+    new_module_min_line: float = 80.0,
+    new_module_min_branch: float = 75.0,
+) -> list[str]:
+    """Reject line or branch regressions in each touched production module."""
+
+    def regressed(head: float, base: float, tolerance: float) -> bool:
+        return base - head > tolerance + 1e-9
+
+    failures: list[str] = []
+    for change in changes:
+        if change.status.startswith("D"):
+            failures.append(
+                f"deleted production module {change.base_path} requires an explicit "
+                "reviewed coverage-policy exception",
+            )
+            continue
+        head, head_failure = module_coverage(head_report, change.head_path)
+        if head_failure:
+            failures.append(head_failure)
+            continue
+        assert head is not None
+
+        if change.base_path is None:
+            if head.line < new_module_min_line:
+                failures.append(
+                    f"new module {change.head_path} line coverage {head.line:.2f}% "
+                    f"is below the {new_module_min_line:.2f}% floor",
+                )
+            if head.branch is not None and head.branch < new_module_min_branch:
+                failures.append(
+                    f"new module {change.head_path} branch coverage {head.branch:.2f}% "
+                    f"is below the {new_module_min_branch:.2f}% floor",
+                )
+            continue
+
+        base, base_failure = module_coverage(base_report, change.base_path)
+        if base_failure:
+            failures.append(f"base {base_failure}")
+            continue
+        assert base is not None
+
+        if regressed(head.line, base.line, line_tolerance):
+            failures.append(
+                f"touched module {change.head_path} line coverage regressed "
+                f"from {base.line:.2f}% to {head.line:.2f}%",
+            )
+        if base.branch is None and head.branch is not None:
+            if head.branch < new_module_min_branch:
+                failures.append(
+                    f"touched module {change.head_path} added branches at "
+                    f"{head.branch:.2f}% coverage, below the "
+                    f"{new_module_min_branch:.2f}% floor",
+                )
+        elif base.branch is not None and head.branch is not None:
+            if regressed(head.branch, base.branch, branch_tolerance):
+                failures.append(
+                    f"touched module {change.head_path} branch coverage regressed "
+                    f"from {base.branch:.2f}% to {head.branch:.2f}%",
+                )
+    return failures
+
+
 def _file_floor(value: str) -> tuple[str, float]:
     try:
         path, minimum = value.rsplit("=", 1)
@@ -142,6 +320,26 @@ def _parser() -> argparse.ArgumentParser:
         "--base-ref",
         help="git base commit/ref used to calculate changed executable lines",
     )
+    parser.add_argument(
+        "--base-report",
+        type=Path,
+        help="coverage.py JSON report generated from the base commit",
+    )
+    parser.add_argument(
+        "--touched-module-no-regression",
+        action="store_true",
+        help="compare each touched production module against --base-report",
+    )
+    parser.add_argument(
+        "--module-root",
+        action="append",
+        default=[],
+        help="production module root for touched-module comparison (default: obliteratus)",
+    )
+    parser.add_argument("--module-line-tolerance", type=float, default=0.0)
+    parser.add_argument("--module-branch-tolerance", type=float, default=0.0)
+    parser.add_argument("--new-module-min-line", type=float, default=80.0)
+    parser.add_argument("--new-module-min-branch", type=float, default=75.0)
     return parser
 
 
@@ -180,6 +378,48 @@ def main() -> int:
                 failures.extend(
                     validate_changed_coverage(report, changed, minimum=args.min_changed),
                 )
+    touched_count: int | None = None
+    if args.touched_module_no_regression:
+        if not args.base_ref or args.base_report is None:
+            failures.append(
+                "touched-module comparison requires --base-ref and --base-report",
+            )
+        elif args.module_line_tolerance < 0 or args.module_branch_tolerance < 0:
+            failures.append("touched-module tolerances must be non-negative")
+        else:
+            try:
+                base_report = json.loads(args.base_report.read_text(encoding="utf-8"))
+                pathspecs = [
+                    f":(glob){root.rstrip('/')}/**/*.py"
+                    for root in (args.module_root or ["obliteratus"])
+                ]
+                raw_changes = subprocess.run(
+                    [
+                        "git", "diff", "--name-status", "-M", "-z",
+                        args.base_ref, "HEAD", "--", *pathspecs,
+                    ],
+                    check=True,
+                    capture_output=True,
+                ).stdout
+                module_changes = parse_changed_modules(
+                    raw_changes,
+                    roots=tuple(args.module_root or ["obliteratus"]),
+                )
+            except (OSError, json.JSONDecodeError, subprocess.CalledProcessError, ValueError) as exc:
+                failures.append(f"cannot compare touched-module coverage: {exc}")
+            else:
+                touched_count = len(module_changes)
+                failures.extend(
+                    validate_touched_module_regression(
+                        report,
+                        base_report,
+                        module_changes,
+                        line_tolerance=args.module_line_tolerance,
+                        branch_tolerance=args.module_branch_tolerance,
+                        new_module_min_line=args.new_module_min_line,
+                        new_module_min_branch=args.new_module_min_branch,
+                    ),
+                )
     if failures:
         for failure in failures:
             print(f"coverage gate failed: {failure}")
@@ -196,6 +436,8 @@ def main() -> int:
         print(
             f"changed-line gate passed: {percentage:.2f}% ({covered}/{executable})",
         )
+    if touched_count is not None:
+        print(f"touched-module coverage gate passed: {touched_count} module(s)")
     return 0
 
 

scripts/check_quality_policy.py

@@ -6,6 +6,8 @@ from __future__ import annotations
 import argparse
 import json
 import math
+import re
+from datetime import date, timedelta
 from pathlib import Path
 from typing import Any
 
@@ -41,7 +43,125 @@ def _valid_exception(exceptions: Any, name: str, current: float) -> bool:
     return False
 
 
-def validate_policy(policy: dict[str, Any]) -> list[str]:
+def _policy_date(value: Any, label: str, failures: list[str]) -> date | None:
+    if not isinstance(value, str):
+        failures.append(f"{label} must be an ISO date")
+        return None
+    try:
+        return date.fromisoformat(value)
+    except ValueError:
+        failures.append(f"{label} must be an ISO date")
+        return None
+
+
+def _validate_test_evidence(
+    policy: dict[str, Any], *, today: date,
+) -> list[str]:
+    failures: list[str] = []
+    evidence = policy.get("test_evidence")
+    if not isinstance(evidence, dict):
+        return ["quality policy is missing the test_evidence object"]
+
+    numeric: dict[str, int] = {}
+    for key, minimum in (
+        ("retention_days", 30),
+        ("flake_window_days", 30),
+        ("maximum_quarantine_days", 1),
+    ):
+        value = evidence.get(key)
+        if isinstance(value, bool) or not isinstance(value, int) or value < minimum:
+            failures.append(f"test evidence {key} must be an integer at least {minimum}")
+        else:
+            numeric[key] = value
+
+    history = evidence.get("flake_history")
+    quarantines = evidence.get("quarantines")
+    if not isinstance(history, list):
+        failures.append("test evidence flake_history must be a list")
+        history = []
+    if not isinstance(quarantines, list):
+        failures.append("test evidence quarantines must be a list")
+        quarantines = []
+
+    observations: dict[str, list[date]] = {}
+    history_keys: set[tuple[str, str, str]] = set()
+    for index, entry in enumerate(history):
+        label = f"flake history {index}"
+        if not isinstance(entry, dict):
+            failures.append(f"{label} must be an object")
+            continue
+        nodeid = entry.get("nodeid")
+        head_sha = entry.get("head_sha")
+        gate = entry.get("gate")
+        if not isinstance(nodeid, str) or not nodeid.strip():
+            failures.append(f"{label} requires a non-empty nodeid")
+            continue
+        if not isinstance(head_sha, str) or re.fullmatch(r"[0-9a-f]{40}", head_sha) is None:
+            failures.append(f"{label} requires a 40-character head_sha")
+        if not isinstance(gate, str) or not gate.strip():
+            failures.append(f"{label} requires a non-empty gate")
+        observed = _policy_date(entry.get("observed_on"), f"{label} observed_on", failures)
+        if observed is not None:
+            if observed > today:
+                failures.append(f"{label} observed_on cannot be in the future")
+            observations.setdefault(nodeid, []).append(observed)
+        key = (nodeid, str(entry.get("observed_on")), str(head_sha))
+        if key in history_keys:
+            failures.append(f"duplicate flake history entry for {nodeid}")
+        history_keys.add(key)
+
+    active_quarantines: set[str] = set()
+    quarantine_nodeids: set[str] = set()
+    max_days = numeric.get("maximum_quarantine_days", 30)
+    for index, entry in enumerate(quarantines):
+        label = f"test quarantine {index}"
+        if not isinstance(entry, dict):
+            failures.append(f"{label} must be an object")
+            continue
+        nodeid = entry.get("nodeid")
+        if not isinstance(nodeid, str) or not nodeid.strip():
+            failures.append(f"{label} requires a non-empty nodeid")
+            continue
+        if nodeid in quarantine_nodeids:
+            failures.append(f"duplicate test quarantine for {nodeid}")
+        quarantine_nodeids.add(nodeid)
+        owner = entry.get("owner")
+        if not isinstance(owner, str) or not owner.startswith("@"):
+            failures.append(f"{label} requires an @owner")
+        if not isinstance(entry.get("reason"), str) or not entry["reason"].strip():
+            failures.append(f"{label} requires a non-empty reason")
+        issue = entry.get("issue")
+        if not isinstance(issue, str) or not issue.startswith(
+            "https://github.com/elder-plinius/OBLITERATUS/issues/",
+        ):
+            failures.append(f"{label} requires an OBLITERATUS issue URL")
+        opened = _policy_date(entry.get("opened"), f"{label} opened", failures)
+        expires = _policy_date(entry.get("expires"), f"{label} expires", failures)
+        if opened is not None and expires is not None:
+            if opened > today:
+                failures.append(f"{label} cannot open in the future")
+            if expires <= opened:
+                failures.append(f"{label} must expire after it opens")
+            elif expires - opened > timedelta(days=max_days):
+                failures.append(f"{label} exceeds the {max_days}-day maximum")
+            elif expires < today:
+                failures.append(f"{label} expired on {expires.isoformat()}")
+            else:
+                active_quarantines.add(nodeid)
+
+    window = numeric.get("flake_window_days", 30)
+    cutoff = today - timedelta(days=window - 1)
+    for nodeid, dates in observations.items():
+        recent = [observed for observed in dates if cutoff <= observed <= today]
+        if len(recent) >= 2 and nodeid not in active_quarantines:
+            failures.append(
+                f"test {nodeid} flaked {len(recent)} times in {window} days "
+                "without an active quarantine",
+            )
+    return failures
+
+
+def validate_policy(policy: dict[str, Any], *, today: date | None = None) -> list[str]:
     """Validate policy structure, immutable floors, and exclusion traceability."""
     failures: list[str] = []
     minimums = policy.get("minimums")
@@ -63,6 +183,19 @@ def validate_policy(policy: dict[str, Any]) -> list[str]:
                 "without an explicit reviewed exception",
             )
 
+    critical_paths = policy.get("critical_cpu_paths")
+    if not isinstance(critical_paths, list) or not critical_paths:
+        failures.append("quality policy requires non-empty critical_cpu_paths")
+    else:
+        seen_critical: set[str] = set()
+        for path in critical_paths:
+            if not isinstance(path, str) or not path.startswith("obliteratus/"):
+                failures.append(f"invalid critical CPU path: {path!r}")
+            elif path in seen_critical:
+                failures.append(f"critical CPU path is duplicated: {path}")
+            else:
+                seen_critical.add(path)
+
     scope = policy.get("mature_cpu_scope")
     exclusions = scope.get("exclusions") if isinstance(scope, dict) else None
     if not isinstance(exclusions, list):
@@ -88,6 +221,7 @@ def validate_policy(policy: dict[str, Any]) -> list[str]:
         issue = exclusion.get("conditional_issue")
         if issue != "https://github.com/elder-plinius/OBLITERATUS/issues/71":
             failures.append(f"{label} must link the conditional-test issue #71")
+    failures.extend(_validate_test_evidence(policy, today=today or date.today()))
     return failures
 
 

scripts/check_test_risk_map.py

@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+"""Validate the source-to-test risk map and conditional coverage graph."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from typing import Any
+
+
+RISK_CLASSES = {"cpu-contract", "mixed-runtime", "conditional-runtime"}
+
+
+def _load_object(path: Path, label: str, errors: list[str]) -> dict[str, Any]:
+    try:
+        value = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        errors.append(f"cannot read {label}: {exc}")
+        return {}
+    if not isinstance(value, dict):
+        errors.append(f"{label} root must be an object")
+        return {}
+    return value
+
+
+def validate(risk_path: Path, quality_path: Path, conditional_path: Path) -> list[str]:
+    """Return structural and cross-policy failures for the risk map."""
+
+    errors: list[str] = []
+    risk = _load_object(risk_path, "test risk map", errors)
+    quality = _load_object(quality_path, "quality policy", errors)
+    conditional = _load_object(conditional_path, "conditional policy", errors)
+    if errors:
+        return errors
+
+    if risk.get("schema_version") != 1:
+        errors.append("test risk map schema_version must be 1")
+    if not isinstance(risk.get("owner"), str) or not risk["owner"].strip():
+        errors.append("test risk map requires a non-empty owner")
+
+    gates = conditional.get("gates")
+    gate_by_id = {
+        gate.get("id"): gate
+        for gate in gates if isinstance(gate, dict) and isinstance(gate.get("id"), str)
+    } if isinstance(gates, list) else {}
+
+    modules = risk.get("modules")
+    if not isinstance(modules, list) or not modules:
+        return errors + ["test risk map modules must be a non-empty list"]
+
+    module_by_path: dict[str, dict[str, Any]] = {}
+    for index, module in enumerate(modules):
+        label = f"risk module {index}"
+        if not isinstance(module, dict):
+            errors.append(f"{label} must be an object")
+            continue
+        path = module.get("path")
+        if not isinstance(path, str) or not path:
+            errors.append(f"{label} requires a non-empty path")
+            continue
+        if path in module_by_path:
+            errors.append(f"duplicate risk module path: {path}")
+        module_by_path[path] = module
+        if not Path(path).is_file():
+            errors.append(f"risk module maps missing source path: {path}")
+        if module.get("risk_class") not in RISK_CLASSES:
+            errors.append(f"risk module {path} has invalid risk_class")
+        if not isinstance(module.get("risk"), str) or not module["risk"].strip():
+            errors.append(f"risk module {path} requires a non-empty risk")
+
+        tests = module.get("required_tests")
+        if not isinstance(tests, list) or not tests:
+            errors.append(f"risk module {path} requires at least one test")
+        else:
+            for test_path in tests:
+                if not isinstance(test_path, str) or not test_path.startswith("tests/"):
+                    errors.append(f"risk module {path} has invalid test path: {test_path!r}")
+                elif not Path(test_path).is_file():
+                    errors.append(f"risk module {path} maps missing test path: {test_path}")
+
+        module_gates = module.get("conditional_gates")
+        if not isinstance(module_gates, list):
+            errors.append(f"risk module {path} conditional_gates must be a list")
+            continue
+        if len(module_gates) != len(set(module_gates)):
+            errors.append(f"risk module {path} has duplicate conditional gates")
+        for gate_id in module_gates:
+            gate = gate_by_id.get(gate_id)
+            if gate is None:
+                errors.append(f"risk module {path} references unknown gate {gate_id}")
+            elif path not in gate.get("coverage_paths", []):
+                errors.append(f"risk module {path} is not covered by gate {gate_id}")
+
+    exclusions = quality.get("mature_cpu_scope", {}).get("exclusions", [])
+    if not isinstance(exclusions, list):
+        errors.append("quality policy exclusions must be a list")
+        exclusions = []
+    for exclusion in exclusions:
+        if not isinstance(exclusion, dict):
+            errors.append("quality policy exclusion must be an object")
+            continue
+        path = exclusion.get("path")
+        gate_id = exclusion.get("conditional_gate")
+        module = module_by_path.get(path)
+        if module is None:
+            errors.append(f"CPU exclusion is missing from test risk map: {path}")
+        elif gate_id not in module.get("conditional_gates", []):
+            errors.append(f"CPU exclusion {path} is missing conditional gate {gate_id}")
+
+    critical_paths = quality.get("critical_cpu_paths", [])
+    if not isinstance(critical_paths, list):
+        errors.append("quality policy critical_cpu_paths must be a list")
+        critical_paths = []
+    for path in critical_paths:
+        if path not in module_by_path:
+            errors.append(f"critical CPU path is missing from test risk map: {path}")
+
+    for gate_id, gate in gate_by_id.items():
+        for path in gate.get("coverage_paths", []):
+            module = module_by_path.get(path)
+            if module is None:
+                errors.append(f"conditional path is missing from test risk map: {path}")
+            elif gate_id not in module.get("conditional_gates", []):
+                errors.append(f"risk module {path} is missing conditional gate {gate_id}")
+    return errors
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--risk-map", type=Path, default=Path("ci/test-risk-map.json"))
+    parser.add_argument("--quality", type=Path, default=Path("ci/test-quality-policy.json"))
+    parser.add_argument(
+        "--conditional",
+        type=Path,
+        default=Path("ci/conditional-test-policy.json"),
+    )
+    args = parser.parse_args()
+    errors = validate(args.risk_map, args.quality, args.conditional)
+    if errors:
+        for error in errors:
+            print(f"ERROR: {error}")
+        return 1
+    print("test risk map: valid")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/run_repeat_gate.py

@@ -11,6 +11,7 @@ import sys
 import time
 from pathlib import Path
 from typing import Any, Sequence
+from xml.etree import ElementTree
 
 
 DEFAULT_TESTS = (
@@ -32,6 +33,26 @@ def test_orders(paths: Sequence[str]) -> list[list[str]]:
     return [forward, reverse, interleaved]
 
 
+def junit_snapshot(path: Path) -> dict[str, Any]:
+    """Return counts and stable failed/skipped node IDs for one repeat pass."""
+
+    root = ElementTree.parse(path).getroot()
+    cases = list(root.iter("testcase"))
+    failed: list[str] = []
+    skipped: list[str] = []
+    for case in cases:
+        nodeid = f"{case.attrib.get('classname', '<unknown>')}::{case.attrib.get('name', '<unknown>')}"
+        if case.find("failure") is not None or case.find("error") is not None:
+            failed.append(nodeid)
+        if case.find("skipped") is not None:
+            skipped.append(nodeid)
+    return {
+        "tests": len(cases),
+        "failed_nodeids": failed,
+        "skipped_nodeids": skipped,
+    }
+
+
 def run_repeat_gate(
     paths: Sequence[str], *, output: Path, python: str = sys.executable,
 ) -> int:
@@ -44,7 +65,10 @@ def run_repeat_gate(
         zip(test_orders(paths), HASH_SEEDS, strict=True),
         start=1,
     ):
-        command = [python, "-m", "pytest", "--no-cov", "-q", *order]
+        junit = output.parent / f"repeat-pass-{index}.xml"
+        command = [
+            python, "-m", "pytest", "--no-cov", "-q", f"--junitxml={junit}", *order,
+        ]
         environment = os.environ.copy()
         environment["PYTHONHASHSEED"] = hash_seed
         pass_started = time.monotonic()
@@ -56,28 +80,60 @@ def run_repeat_gate(
             check=False,
         )
         duration = time.monotonic() - pass_started
-        results.append({
+        try:
+            counts = junit_snapshot(junit)
+            junit_error = None
+        except (OSError, ElementTree.ParseError) as exc:
+            counts = {"tests": 0, "failed_nodeids": [], "skipped_nodeids": []}
+            junit_error = str(exc)
+        result = {
             "pass": index,
             "python_hash_seed": hash_seed,
             "tests": order,
             "duration_seconds": round(duration, 3),
             "return_code": completed.returncode,
+            "junit": str(junit),
+            **counts,
             "stdout": completed.stdout[-4000:],
             "stderr": completed.stderr[-4000:],
-        })
-        if completed.returncode != 0:
+        }
+        if junit_error is not None:
+            result["junit_error"] = junit_error
+        results.append(result)
+        if completed.returncode != 0 and exit_code == 0:
             exit_code = completed.returncode
-            break
+        if junit_error is not None and exit_code == 0:
+            exit_code = 1
+
+    occurrences: dict[str, int] = {}
+    for result in results:
+        for nodeid in result["failed_nodeids"]:
+            occurrences[nodeid] = occurrences.get(nodeid, 0) + 1
+    flake_candidates = [
+        {"nodeid": nodeid, "occurrences": count}
+        for nodeid, count in sorted(occurrences.items())
+        if count < len(results)
+    ]
+    consistent_failures = [
+        {"nodeid": nodeid, "occurrences": count}
+        for nodeid, count in sorted(occurrences.items())
+        if count == len(results)
+    ]
 
     evidence = {
         "schema_version": 1,
         "status": "passed" if exit_code == 0 else "failed",
         "total_duration_seconds": round(time.monotonic() - started, 3),
         "passes": results,
+        "flake_candidates": flake_candidates,
+        "consistent_failures": consistent_failures,
     }
     output.write_text(json.dumps(evidence, indent=2) + "\n", encoding="utf-8")
     if exit_code:
-        failed = results[-1]
+        failed = next(
+            result for result in results
+            if result["return_code"] != 0 or result.get("junit_error") is not None
+        )
         print(
             f"repeat gate failed on pass {failed['pass']} "
             f"with PYTHONHASHSEED={failed['python_hash_seed']}",

scripts/write_test_evidence.py

@@ -0,0 +1,239 @@
+#!/usr/bin/env python3
+"""Normalize coverage, JUnit, repeat, and mutation results into retained evidence."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import os
+import platform
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+from xml.etree import ElementTree
+
+
+def _read_object(path: Path, label: str) -> dict[str, Any]:
+    try:
+        value = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        raise ValueError(f"cannot read {label}: {exc}") from exc
+    if not isinstance(value, dict):
+        raise ValueError(f"{label} root must be an object")
+    return value
+
+
+def _percentage(summary: dict[str, Any], covered: str, total: str, path: str) -> float | None:
+    covered_value = summary.get(covered)
+    total_value = summary.get(total)
+    for key, value in ((covered, covered_value), (total, total_value)):
+        if isinstance(value, bool) or not isinstance(value, int) or value < 0:
+            raise ValueError(f"coverage report has invalid {key} for {path}")
+    if covered_value > total_value:
+        raise ValueError(f"coverage report overcounts {covered} for {path}")
+    return covered_value / total_value * 100 if total_value else None
+
+
+def coverage_snapshot(
+    report: dict[str, Any],
+    risk_map: dict[str, Any] | None = None,
+    *,
+    allow_missing_risk_modules: bool = False,
+) -> dict[str, Any]:
+    """Return repository and risk-module coverage metrics."""
+
+    totals = report.get("totals")
+    files = report.get("files")
+    if not isinstance(totals, dict) or not isinstance(files, dict):
+        raise ValueError("coverage report requires totals and files objects")
+    line = totals.get("percent_statements_covered")
+    branch = totals.get("percent_branches_covered")
+    for label, value in (("line", line), ("branch", branch)):
+        if (
+            isinstance(value, bool)
+            or not isinstance(value, (int, float))
+            or not math.isfinite(value)
+        ):
+            raise ValueError(f"coverage report requires numeric {label} coverage")
+
+    modules: dict[str, dict[str, float | None]] = {}
+    if risk_map is not None:
+        risk_modules = risk_map.get("modules")
+        if not isinstance(risk_modules, list):
+            raise ValueError("test risk map modules must be a list")
+        for item in risk_modules:
+            path = item.get("path") if isinstance(item, dict) else None
+            entry = files.get(path) if isinstance(path, str) else None
+            summary = entry.get("summary") if isinstance(entry, dict) else None
+            if not isinstance(path, str):
+                raise ValueError("test risk map module requires a path")
+            if not isinstance(summary, dict) and allow_missing_risk_modules:
+                modules[path] = {"line_percent": None, "branch_percent": None}
+                continue
+            if not isinstance(summary, dict):
+                raise ValueError(f"coverage report is missing risk module {path}")
+            modules[path] = {
+                "line_percent": _percentage(summary, "covered_lines", "num_statements", path),
+                "branch_percent": _percentage(
+                    summary, "covered_branches", "num_branches", path,
+                ),
+            }
+    return {
+        "line_percent": float(line),
+        "branch_percent": float(branch),
+        "risk_modules": modules,
+    }
+
+
+def junit_snapshot(path: Path) -> dict[str, Any]:
+    """Return stable counts, node IDs, and slow-test evidence from JUnit XML."""
+
+    try:
+        root = ElementTree.parse(path).getroot()
+    except (OSError, ElementTree.ParseError) as exc:
+        raise ValueError(f"cannot read JUnit XML: {exc}") from exc
+    cases = list(root.iter("testcase"))
+    failures: list[str] = []
+    errors: list[str] = []
+    skipped: list[str] = []
+    durations: list[dict[str, str | float]] = []
+    for case in cases:
+        nodeid = f"{case.attrib.get('classname', '<unknown>')}::{case.attrib.get('name', '<unknown>')}"
+        try:
+            duration = float(case.attrib.get("time", "0"))
+        except ValueError as exc:
+            raise ValueError(f"JUnit testcase {nodeid} has invalid time") from exc
+        durations.append({"nodeid": nodeid, "seconds": duration})
+        if case.find("failure") is not None:
+            failures.append(nodeid)
+        if case.find("error") is not None:
+            errors.append(nodeid)
+        if case.find("skipped") is not None:
+            skipped.append(nodeid)
+    failed = set(failures) | set(errors) | set(skipped)
+    durations.sort(key=lambda item: (-float(item["seconds"]), str(item["nodeid"])))
+    return {
+        "total": len(cases),
+        "passed": len(cases) - len(failed),
+        "failures": failures,
+        "errors": errors,
+        "skipped": skipped,
+        "duration_seconds": round(sum(float(item["seconds"]) for item in durations), 3),
+        "slowest": durations[:20],
+    }
+
+
+def mutation_snapshot(stats: dict[str, Any]) -> dict[str, int | float]:
+    killed = stats.get("killed")
+    total = stats.get("total")
+    if (
+        isinstance(killed, bool)
+        or not isinstance(killed, int)
+        or killed < 0
+        or isinstance(total, bool)
+        or not isinstance(total, int)
+        or total <= 0
+        or killed > total
+    ):
+        raise ValueError("mutation statistics require valid killed and total counts")
+    return {"killed": killed, "total": total, "score_percent": killed / total * 100}
+
+
+def build_evidence(
+    *,
+    head_sha: str,
+    base_sha: str,
+    python_version: str,
+    coverage: dict[str, Any] | None = None,
+    base_coverage: dict[str, Any] | None = None,
+    risk_map: dict[str, Any] | None = None,
+    junit: dict[str, Any] | None = None,
+    repeat: dict[str, Any] | None = None,
+    mutation: dict[str, Any] | None = None,
+    generated_at: str | None = None,
+) -> dict[str, Any]:
+    """Build the project-owned evidence schema from already-parsed inputs."""
+
+    evidence: dict[str, Any] = {
+        "schema_version": 1,
+        "generated_at": generated_at or datetime.now(timezone.utc).isoformat(),
+        "repository": os.environ.get("GITHUB_REPOSITORY", "elder-plinius/OBLITERATUS"),
+        "head_sha": head_sha,
+        "base_sha": base_sha,
+        "python": python_version,
+        "platform": platform.platform(),
+    }
+    if coverage is not None:
+        current = coverage_snapshot(coverage, risk_map)
+        coverage_evidence: dict[str, Any] = {"current": current}
+        if base_coverage is not None:
+            base = coverage_snapshot(
+                base_coverage,
+                risk_map,
+                allow_missing_risk_modules=True,
+            )
+            coverage_evidence["base"] = base
+            coverage_evidence["delta"] = {
+                "line_percentage_points": current["line_percent"] - base["line_percent"],
+                "branch_percentage_points": current["branch_percent"] - base["branch_percent"],
+            }
+        evidence["coverage"] = coverage_evidence
+    if junit is not None:
+        evidence["tests"] = junit
+    if repeat is not None:
+        if repeat.get("schema_version") != 1:
+            raise ValueError("repeat evidence schema_version must be 1")
+        evidence["repeat"] = repeat
+    if mutation is not None:
+        evidence["mutation"] = mutation_snapshot(mutation)
+    if not any(key in evidence for key in ("coverage", "tests", "repeat", "mutation")):
+        raise ValueError("at least one test evidence input is required")
+    return evidence
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--coverage", type=Path)
+    parser.add_argument("--base-coverage", type=Path)
+    parser.add_argument("--junit", type=Path)
+    parser.add_argument("--repeat", type=Path)
+    parser.add_argument("--mutation", type=Path)
+    parser.add_argument("--risk-map", type=Path, default=Path("ci/test-risk-map.json"))
+    parser.add_argument("--head-sha", default=os.environ.get("GITHUB_SHA", "local"))
+    parser.add_argument("--base-sha", default="")
+    parser.add_argument("--python-version", default=platform.python_version())
+    parser.add_argument("--output", type=Path, required=True)
+    args = parser.parse_args()
+    try:
+        coverage = _read_object(args.coverage, "coverage report") if args.coverage else None
+        base_coverage = (
+            _read_object(args.base_coverage, "base coverage report")
+            if args.base_coverage else None
+        )
+        risk_map = _read_object(args.risk_map, "test risk map") if coverage else None
+        junit = junit_snapshot(args.junit) if args.junit else None
+        repeat = _read_object(args.repeat, "repeat evidence") if args.repeat else None
+        mutation = _read_object(args.mutation, "mutation evidence") if args.mutation else None
+        evidence = build_evidence(
+            head_sha=args.head_sha,
+            base_sha=args.base_sha,
+            python_version=args.python_version,
+            coverage=coverage,
+            base_coverage=base_coverage,
+            risk_map=risk_map,
+            junit=junit,
+            repeat=repeat,
+            mutation=mutation,
+        )
+    except ValueError as exc:
+        print(f"test evidence failed: {exc}")
+        return 1
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(json.dumps(evidence, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    print(f"test evidence written: {args.output}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())