test: enforce Wave A quality lock

scripts/run_repeat_gate.py

@@ -11,6 +11,7 @@ import sys
 import time
 from pathlib import Path
 from typing import Any, Sequence
+from xml.etree import ElementTree
 
 
 DEFAULT_TESTS = (
@@ -32,6 +33,26 @@ def test_orders(paths: Sequence[str]) -> list[list[str]]:
     return [forward, reverse, interleaved]
 
 
+def junit_snapshot(path: Path) -> dict[str, Any]:
+    """Return counts and stable failed/skipped node IDs for one repeat pass."""
+
+    root = ElementTree.parse(path).getroot()
+    cases = list(root.iter("testcase"))
+    failed: list[str] = []
+    skipped: list[str] = []
+    for case in cases:
+        nodeid = f"{case.attrib.get('classname', '<unknown>')}::{case.attrib.get('name', '<unknown>')}"
+        if case.find("failure") is not None or case.find("error") is not None:
+            failed.append(nodeid)
+        if case.find("skipped") is not None:
+            skipped.append(nodeid)
+    return {
+        "tests": len(cases),
+        "failed_nodeids": failed,
+        "skipped_nodeids": skipped,
+    }
+
+
 def run_repeat_gate(
     paths: Sequence[str], *, output: Path, python: str = sys.executable,
 ) -> int:
@@ -44,7 +65,10 @@ def run_repeat_gate(
         zip(test_orders(paths), HASH_SEEDS, strict=True),
         start=1,
     ):
-        command = [python, "-m", "pytest", "--no-cov", "-q", *order]
+        junit = output.parent / f"repeat-pass-{index}.xml"
+        command = [
+            python, "-m", "pytest", "--no-cov", "-q", f"--junitxml={junit}", *order,
+        ]
         environment = os.environ.copy()
         environment["PYTHONHASHSEED"] = hash_seed
         pass_started = time.monotonic()
@@ -56,28 +80,60 @@ def run_repeat_gate(
             check=False,
         )
         duration = time.monotonic() - pass_started
-        results.append({
+        try:
+            counts = junit_snapshot(junit)
+            junit_error = None
+        except (OSError, ElementTree.ParseError) as exc:
+            counts = {"tests": 0, "failed_nodeids": [], "skipped_nodeids": []}
+            junit_error = str(exc)
+        result = {
             "pass": index,
             "python_hash_seed": hash_seed,
             "tests": order,
             "duration_seconds": round(duration, 3),
             "return_code": completed.returncode,
+            "junit": str(junit),
+            **counts,
             "stdout": completed.stdout[-4000:],
             "stderr": completed.stderr[-4000:],
-        })
-        if completed.returncode != 0:
+        }
+        if junit_error is not None:
+            result["junit_error"] = junit_error
+        results.append(result)
+        if completed.returncode != 0 and exit_code == 0:
             exit_code = completed.returncode
-            break
+        if junit_error is not None and exit_code == 0:
+            exit_code = 1
+
+    occurrences: dict[str, int] = {}
+    for result in results:
+        for nodeid in result["failed_nodeids"]:
+            occurrences[nodeid] = occurrences.get(nodeid, 0) + 1
+    flake_candidates = [
+        {"nodeid": nodeid, "occurrences": count}
+        for nodeid, count in sorted(occurrences.items())
+        if count < len(results)
+    ]
+    consistent_failures = [
+        {"nodeid": nodeid, "occurrences": count}
+        for nodeid, count in sorted(occurrences.items())
+        if count == len(results)
+    ]
 
     evidence = {
         "schema_version": 1,
         "status": "passed" if exit_code == 0 else "failed",
         "total_duration_seconds": round(time.monotonic() - started, 3),
         "passes": results,
+        "flake_candidates": flake_candidates,
+        "consistent_failures": consistent_failures,
     }
     output.write_text(json.dumps(evidence, indent=2) + "\n", encoding="utf-8")
     if exit_code:
-        failed = results[-1]
+        failed = next(
+            result for result in results
+            if result["return_code"] != 0 or result.get("junit_error") is not None
+        )
         print(
             f"repeat gate failed on pass {failed['pass']} "
             f"with PYTHONHASHSEED={failed['python_hash_seed']}",