test: enforce Wave A quality lock

scripts/check_quality_policy.py

@@ -6,6 +6,8 @@ from __future__ import annotations
 import argparse
 import json
 import math
+import re
+from datetime import date, timedelta
 from pathlib import Path
 from typing import Any
 
@@ -41,7 +43,125 @@ def _valid_exception(exceptions: Any, name: str, current: float) -> bool:
     return False
 
 
-def validate_policy(policy: dict[str, Any]) -> list[str]:
+def _policy_date(value: Any, label: str, failures: list[str]) -> date | None:
+    if not isinstance(value, str):
+        failures.append(f"{label} must be an ISO date")
+        return None
+    try:
+        return date.fromisoformat(value)
+    except ValueError:
+        failures.append(f"{label} must be an ISO date")
+        return None
+
+
+def _validate_test_evidence(
+    policy: dict[str, Any], *, today: date,
+) -> list[str]:
+    failures: list[str] = []
+    evidence = policy.get("test_evidence")
+    if not isinstance(evidence, dict):
+        return ["quality policy is missing the test_evidence object"]
+
+    numeric: dict[str, int] = {}
+    for key, minimum in (
+        ("retention_days", 30),
+        ("flake_window_days", 30),
+        ("maximum_quarantine_days", 1),
+    ):
+        value = evidence.get(key)
+        if isinstance(value, bool) or not isinstance(value, int) or value < minimum:
+            failures.append(f"test evidence {key} must be an integer at least {minimum}")
+        else:
+            numeric[key] = value
+
+    history = evidence.get("flake_history")
+    quarantines = evidence.get("quarantines")
+    if not isinstance(history, list):
+        failures.append("test evidence flake_history must be a list")
+        history = []
+    if not isinstance(quarantines, list):
+        failures.append("test evidence quarantines must be a list")
+        quarantines = []
+
+    observations: dict[str, list[date]] = {}
+    history_keys: set[tuple[str, str, str]] = set()
+    for index, entry in enumerate(history):
+        label = f"flake history {index}"
+        if not isinstance(entry, dict):
+            failures.append(f"{label} must be an object")
+            continue
+        nodeid = entry.get("nodeid")
+        head_sha = entry.get("head_sha")
+        gate = entry.get("gate")
+        if not isinstance(nodeid, str) or not nodeid.strip():
+            failures.append(f"{label} requires a non-empty nodeid")
+            continue
+        if not isinstance(head_sha, str) or re.fullmatch(r"[0-9a-f]{40}", head_sha) is None:
+            failures.append(f"{label} requires a 40-character head_sha")
+        if not isinstance(gate, str) or not gate.strip():
+            failures.append(f"{label} requires a non-empty gate")
+        observed = _policy_date(entry.get("observed_on"), f"{label} observed_on", failures)
+        if observed is not None:
+            if observed > today:
+                failures.append(f"{label} observed_on cannot be in the future")
+            observations.setdefault(nodeid, []).append(observed)
+        key = (nodeid, str(entry.get("observed_on")), str(head_sha))
+        if key in history_keys:
+            failures.append(f"duplicate flake history entry for {nodeid}")
+        history_keys.add(key)
+
+    active_quarantines: set[str] = set()
+    quarantine_nodeids: set[str] = set()
+    max_days = numeric.get("maximum_quarantine_days", 30)
+    for index, entry in enumerate(quarantines):
+        label = f"test quarantine {index}"
+        if not isinstance(entry, dict):
+            failures.append(f"{label} must be an object")
+            continue
+        nodeid = entry.get("nodeid")
+        if not isinstance(nodeid, str) or not nodeid.strip():
+            failures.append(f"{label} requires a non-empty nodeid")
+            continue
+        if nodeid in quarantine_nodeids:
+            failures.append(f"duplicate test quarantine for {nodeid}")
+        quarantine_nodeids.add(nodeid)
+        owner = entry.get("owner")
+        if not isinstance(owner, str) or not owner.startswith("@"):
+            failures.append(f"{label} requires an @owner")
+        if not isinstance(entry.get("reason"), str) or not entry["reason"].strip():
+            failures.append(f"{label} requires a non-empty reason")
+        issue = entry.get("issue")
+        if not isinstance(issue, str) or not issue.startswith(
+            "https://github.com/elder-plinius/OBLITERATUS/issues/",
+        ):
+            failures.append(f"{label} requires an OBLITERATUS issue URL")
+        opened = _policy_date(entry.get("opened"), f"{label} opened", failures)
+        expires = _policy_date(entry.get("expires"), f"{label} expires", failures)
+        if opened is not None and expires is not None:
+            if opened > today:
+                failures.append(f"{label} cannot open in the future")
+            if expires <= opened:
+                failures.append(f"{label} must expire after it opens")
+            elif expires - opened > timedelta(days=max_days):
+                failures.append(f"{label} exceeds the {max_days}-day maximum")
+            elif expires < today:
+                failures.append(f"{label} expired on {expires.isoformat()}")
+            else:
+                active_quarantines.add(nodeid)
+
+    window = numeric.get("flake_window_days", 30)
+    cutoff = today - timedelta(days=window - 1)
+    for nodeid, dates in observations.items():
+        recent = [observed for observed in dates if cutoff <= observed <= today]
+        if len(recent) >= 2 and nodeid not in active_quarantines:
+            failures.append(
+                f"test {nodeid} flaked {len(recent)} times in {window} days "
+                "without an active quarantine",
+            )
+    return failures
+
+
+def validate_policy(policy: dict[str, Any], *, today: date | None = None) -> list[str]:
     """Validate policy structure, immutable floors, and exclusion traceability."""
     failures: list[str] = []
     minimums = policy.get("minimums")
@@ -63,6 +183,19 @@ def validate_policy(policy: dict[str, Any]) -> list[str]:
                 "without an explicit reviewed exception",
             )
 
+    critical_paths = policy.get("critical_cpu_paths")
+    if not isinstance(critical_paths, list) or not critical_paths:
+        failures.append("quality policy requires non-empty critical_cpu_paths")
+    else:
+        seen_critical: set[str] = set()
+        for path in critical_paths:
+            if not isinstance(path, str) or not path.startswith("obliteratus/"):
+                failures.append(f"invalid critical CPU path: {path!r}")
+            elif path in seen_critical:
+                failures.append(f"critical CPU path is duplicated: {path}")
+            else:
+                seen_critical.add(path)
+
     scope = policy.get("mature_cpu_scope")
     exclusions = scope.get("exclusions") if isinstance(scope, dict) else None
     if not isinstance(exclusions, list):
@@ -88,6 +221,7 @@ def validate_policy(policy: dict[str, Any]) -> list[str]:
         issue = exclusion.get("conditional_issue")
         if issue != "https://github.com/elder-plinius/OBLITERATUS/issues/71":
             failures.append(f"{label} must link the conditional-test issue #71")
+    failures.extend(_validate_test_evidence(policy, today=today or date.today()))
     return failures