ci: add conditional environment test gates

scripts/check_conditional_policy.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+"""Validate conditional-gate policy and its CPU-coverage mappings."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+
+REQUIRED_GATE_FIELDS = {
+    "id", "job", "marker", "runner", "prerequisites", "expected_cost", "coverage_paths"
+}
+
+
+def validate(policy_path: Path, quality_path: Path, workflow_path: Path) -> list[str]:
+    errors: list[str] = []
+    policy = json.loads(policy_path.read_text())
+    quality = json.loads(quality_path.read_text())
+    workflow = workflow_path.read_text()
+
+    if policy.get("schema_version") != 1:
+        errors.append("conditional policy schema_version must be 1")
+    for key in ("owner", "cadence", "evidence_retention_days", "maximum_evidence_age_days"):
+        if not policy.get(key):
+            errors.append(f"conditional policy is missing {key}")
+
+    gates = policy.get("gates")
+    if not isinstance(gates, list) or not gates:
+        return errors + ["conditional policy gates must be a non-empty list"]
+
+    by_id: dict[str, dict] = {}
+    for index, gate in enumerate(gates):
+        missing = REQUIRED_GATE_FIELDS - set(gate)
+        if missing:
+            errors.append(f"gate {index} is missing fields: {sorted(missing)}")
+            continue
+        gate_id = gate["id"]
+        if gate_id in by_id:
+            errors.append(f"duplicate conditional gate id: {gate_id}")
+        by_id[gate_id] = gate
+        if not gate["coverage_paths"]:
+            errors.append(f"gate {gate_id} has no coverage paths")
+        if f"{gate['job']}:" not in workflow:
+            errors.append(f"workflow job {gate['job']!r} for {gate_id} was not found")
+        for source_path in gate["coverage_paths"]:
+            if not Path(source_path).is_file():
+                errors.append(f"gate {gate_id} maps missing source path: {source_path}")
+
+    exclusions = quality.get("mature_cpu_scope", {}).get("exclusions", [])
+    for exclusion in exclusions:
+        gate_id = exclusion.get("conditional_gate")
+        source_path = exclusion.get("path")
+        if gate_id not in by_id:
+            errors.append(f"CPU exclusion {source_path} references unknown gate {gate_id}")
+            continue
+        if source_path not in by_id[gate_id]["coverage_paths"]:
+            errors.append(f"CPU exclusion {source_path} is not mapped by gate {gate_id}")
+
+    required_workflow_tokens = (
+        "workflow_dispatch:", "schedule:", "release:", "permissions:", "contents: read",
+        "scripts/run_conditional_gate.py", "scripts/conditional_gate_summary.py",
+    )
+    for token in required_workflow_tokens:
+        if token not in workflow:
+            errors.append(f"conditional workflow is missing {token!r}")
+    return errors
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--policy", type=Path, default=Path("ci/conditional-test-policy.json"))
+    parser.add_argument("--quality", type=Path, default=Path("ci/test-quality-policy.json"))
+    parser.add_argument(
+        "--workflow", type=Path, default=Path(".github/workflows/conditional-tests.yml")
+    )
+    args = parser.parse_args()
+    errors = validate(args.policy, args.quality, args.workflow)
+    if errors:
+        for error in errors:
+            print(f"ERROR: {error}")
+        return 1
+    print("conditional test policy: valid")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/conditional_gate_summary.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+"""Build a final conditional-workflow result and freshness summary."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+
+
+def summarize(policy: dict, results: dict, selected: dict) -> tuple[list[dict], list[str]]:
+    """Return per-gate rows and selected-job failures."""
+    failures: list[str] = []
+    rows: list[dict] = []
+    failed_jobs: set[str] = set()
+    for gate in policy["gates"]:
+        job = gate["job"]
+        is_selected = bool(selected.get(job, False))
+        result = results.get(job, "unknown")
+        status = result if is_selected else "not_selected_no_fresh_evidence"
+        if is_selected and result != "success" and job not in failed_jobs:
+            failures.append(f"{job}: selected but result was {result}")
+            failed_jobs.add(job)
+        rows.append({"gate": gate["id"], "job": job, "selected": is_selected, "status": status})
+    return rows, failures
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--policy", type=Path, default=Path("ci/conditional-test-policy.json"))
+    parser.add_argument("--output", type=Path, default=Path("conditional-evidence/summary.json"))
+    parser.add_argument("--results-json", default=os.environ.get("CONDITIONAL_RESULTS", "{}"))
+    parser.add_argument("--selected-json", default=os.environ.get("CONDITIONAL_SELECTED", "{}"))
+    args = parser.parse_args()
+    policy = json.loads(args.policy.read_text())
+    results = json.loads(args.results_json)
+    selected = json.loads(args.selected_json)
+    rows, failures = summarize(policy, results, selected)
+    payload = {
+        "schema_version": 1,
+        "generated_at": datetime.now(timezone.utc).isoformat(),
+        "git_sha": os.environ.get("GITHUB_SHA", "local"),
+        "maximum_evidence_age_days": policy["maximum_evidence_age_days"],
+        "gates": rows,
+        "failures": failures,
+    }
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
+    summary_path = os.environ.get("GITHUB_STEP_SUMMARY")
+    if summary_path:
+        lines = ["## Conditional test evidence", "", "| Gate | Job | Status |", "|---|---|---|"]
+        lines.extend(f"| {row['gate']} | {row['job']} | {row['status']} |" for row in rows)
+        lines.extend(["", f"Evidence becomes stale after {policy['maximum_evidence_age_days']} days."])
+        with Path(summary_path).open("a") as handle:
+            handle.write("\n".join(lines) + "\n")
+    for failure in failures:
+        print(f"ERROR: {failure}")
+    return 1 if failures else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/run_conditional_gate.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+"""Run one conditional pytest gate and reject empty or silently skipped evidence."""
+
+from __future__ import annotations
+
+import argparse
+import importlib.util
+import json
+import os
+import subprocess
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+from xml.etree import ElementTree
+
+
+GATES = {
+    "model-download-runtime": "tests/conditional/test_model_download_runtime.py",
+    "external-evaluation": "tests/conditional/test_external_evaluation_runtime.py",
+    "network-services": "tests/conditional/test_network_services.py",
+    "operator-ui": "tests/conditional/test_operator_ui.py",
+    "cuda-runtime": "tests/conditional/test_cuda_runtime.py",
+    "bitsandbytes-runtime": "tests/conditional/test_cuda_runtime.py",
+    "mps-runtime": "tests/conditional/test_mps_runtime.py",
+    "mlx-runtime": "tests/conditional/test_mlx_runtime.py",
+    "remote-execution": "tests/conditional/test_remote_runtime.py",
+}
+
+
+def missing_prerequisites(gate: str) -> list[str]:
+    missing: list[str] = []
+    if gate in {"cuda-runtime", "bitsandbytes-runtime", "mps-runtime"}:
+        import torch
+
+        if gate.startswith("cuda") or gate.startswith("bitsandbytes"):
+            if not torch.cuda.is_available():
+                missing.append("a CUDA-capable PyTorch runtime")
+        elif not (hasattr(torch.backends, "mps") and torch.backends.mps.is_available()):
+            missing.append("an available Apple MPS backend")
+    if gate == "bitsandbytes-runtime" and importlib.util.find_spec("bitsandbytes") is None:
+        missing.append("bitsandbytes")
+    if gate == "mlx-runtime":
+        for module in ("mlx", "mlx_lm"):
+            if importlib.util.find_spec(module) is None:
+                missing.append(module)
+    if gate == "remote-execution":
+        for variable in (
+            "OBLITERATUS_REMOTE_HOST",
+            "OBLITERATUS_REMOTE_USER",
+            "OBLITERATUS_REMOTE_KEY",
+            "OBLITERATUS_REMOTE_KNOWN_HOSTS",
+        ):
+            if not os.environ.get(variable):
+                missing.append(variable)
+    return missing
+
+
+def counts(junit_path: Path) -> dict[str, int]:
+    root = ElementTree.parse(junit_path).getroot()
+    suites = [root] if root.tag == "testsuite" else list(root.findall("testsuite"))
+    return {
+        key: sum(int(suite.attrib.get(key, "0")) for suite in suites)
+        for key in ("tests", "failures", "errors", "skipped")
+    }
+
+
+def write_report(path: Path, gate: str, status: str, **extra: object) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    payload = {
+        "schema_version": 1,
+        "gate": gate,
+        "status": status,
+        "generated_at": datetime.now(timezone.utc).isoformat(),
+        "git_sha": os.environ.get("GITHUB_SHA", "local"),
+        **extra,
+    }
+    path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("gate", choices=sorted(GATES))
+    parser.add_argument("--evidence-dir", type=Path, default=Path("conditional-evidence"))
+    parser.add_argument("--allow-missing", action="store_true")
+    args = parser.parse_args()
+    report = args.evidence_dir / f"{args.gate}.json"
+    junit = args.evidence_dir / f"{args.gate}.xml"
+
+    missing = missing_prerequisites(args.gate)
+    if missing:
+        message = "Missing prerequisites: " + ", ".join(missing)
+        write_report(report, args.gate, "not_run", reason=message)
+        print(message, file=sys.stderr)
+        return 0 if args.allow_missing else 2
+
+    command = [
+        sys.executable,
+        "-m",
+        "pytest",
+        GATES[args.gate],
+        "--no-cov",
+        "-q",
+        f"--junitxml={junit}",
+    ]
+    result = subprocess.run(command, check=False)
+    if not junit.is_file():
+        write_report(report, args.gate, "failed", exit_code=result.returncode, reason="no JUnit")
+        return result.returncode or 1
+
+    result_counts = counts(junit)
+    passed = (
+        result.returncode == 0
+        and result_counts["tests"] > 0
+        and result_counts["failures"] == 0
+        and result_counts["errors"] == 0
+        and result_counts["skipped"] == 0
+    )
+    write_report(
+        report,
+        args.gate,
+        "passed" if passed else "failed",
+        exit_code=result.returncode,
+        counts=result_counts,
+    )
+    if not passed:
+        print(f"conditional gate did not produce unskipped green evidence: {result_counts}")
+    return 0 if passed else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())