OBLITERATUS/scripts/run_repeat_gate.py

#!/usr/bin/env python3
"""Repeat deterministic tests in varied orders and emit timing evidence."""

from __future__ import annotations

import argparse
import json
import os
import subprocess
import sys
import time
from pathlib import Path
from typing import Any, Sequence
from xml.etree import ElementTree


DEFAULT_TESTS = (
    "tests/test_bayesian_optimizer_contracts.py",
    "tests/test_bestiary_contracts.py",
    "tests/test_checkpoint_atomicity.py",
    "tests/test_config.py",
    "tests/test_config_properties.py",
    "tests/test_conditional_evidence_freshness.py",
    "tests/test_coverage_thresholds.py",
    "tests/test_evaluation_reporting_contracts.py",
    "tests/test_lm_eval_reporting_contracts.py",
    "tests/test_informed_pipeline_contracts.py",
    "tests/test_model_profile_contracts.py",
    "tests/test_models_client_contracts.py",
    "tests/test_numerical_contracts.py",
    "tests/test_package_export_contracts.py",
    "tests/test_persistence_contracts.py",
    "tests/test_persistence_pipeline.py",
    "tests/test_property_contracts.py",
    "tests/test_advanced_metrics.py",
    "tests/test_metrics.py",
    "tests/test_remote_contracts.py",
    "tests/test_remaining_cpu_contracts.py",
    "tests/test_runtime_contracts.py",
    "tests/test_strategy_navigation_contracts.py",
    "tests/test_sweep_contracts.py",
    "tests/test_telemetry_failure_contracts.py",
    "tests/test_watchtower_contracts.py",
)
HASH_SEEDS = ("0", "1", "8675309")


def test_orders(paths: Sequence[str]) -> list[list[str]]:
    """Return stable forward, reverse, and interleaved orders."""
    forward = list(paths)
    reverse = list(reversed(paths))
    interleaved = forward[::2] + forward[1::2]
    return [forward, reverse, interleaved]


def junit_snapshot(path: Path) -> dict[str, Any]:
    """Return counts and stable failed/skipped node IDs for one repeat pass."""

    root = ElementTree.parse(path).getroot()
    cases = list(root.iter("testcase"))
    failed: list[str] = []
    skipped: list[str] = []
    for case in cases:
        nodeid = f"{case.attrib.get('classname', '<unknown>')}::{case.attrib.get('name', '<unknown>')}"
        if case.find("failure") is not None or case.find("error") is not None:
            failed.append(nodeid)
        if case.find("skipped") is not None:
            skipped.append(nodeid)
    return {
        "tests": len(cases),
        "failed_nodeids": failed,
        "skipped_nodeids": skipped,
    }


def run_repeat_gate(
    paths: Sequence[str], *, output: Path, python: str = sys.executable,
) -> int:
    """Run three deterministic passes, always writing a JSON evidence record."""
    output.parent.mkdir(parents=True, exist_ok=True)
    results: list[dict[str, Any]] = []
    started = time.monotonic()
    exit_code = 0
    for index, (order, hash_seed) in enumerate(
        zip(test_orders(paths), HASH_SEEDS, strict=True),
        start=1,
    ):
        junit = output.parent / f"repeat-pass-{index}.xml"
        command = [
            python, "-m", "pytest", "--no-cov", "-q", f"--junitxml={junit}", *order,
        ]
        environment = os.environ.copy()
        environment["PYTHONHASHSEED"] = hash_seed
        pass_started = time.monotonic()
        completed = subprocess.run(
            command,
            capture_output=True,
            text=True,
            env=environment,
            check=False,
        )
        duration = time.monotonic() - pass_started
        try:
            counts = junit_snapshot(junit)
            junit_error = None
        except (OSError, ElementTree.ParseError) as exc:
            counts = {"tests": 0, "failed_nodeids": [], "skipped_nodeids": []}
            junit_error = str(exc)
        result = {
            "pass": index,
            "python_hash_seed": hash_seed,
            "tests": order,
            "duration_seconds": round(duration, 3),
            "return_code": completed.returncode,
            "junit": str(junit),
            **counts,
            "stdout": completed.stdout[-4000:],
            "stderr": completed.stderr[-4000:],
        }
        if junit_error is not None:
            result["junit_error"] = junit_error
        results.append(result)
        if completed.returncode != 0 and exit_code == 0:
            exit_code = completed.returncode
        if junit_error is not None and exit_code == 0:
            exit_code = 1

    occurrences: dict[str, int] = {}
    for result in results:
        for nodeid in result["failed_nodeids"]:
            occurrences[nodeid] = occurrences.get(nodeid, 0) + 1
    flake_candidates = [
        {"nodeid": nodeid, "occurrences": count}
        for nodeid, count in sorted(occurrences.items())
        if count < len(results)
    ]
    consistent_failures = [
        {"nodeid": nodeid, "occurrences": count}
        for nodeid, count in sorted(occurrences.items())
        if count == len(results)
    ]

    evidence = {
        "schema_version": 1,
        "status": "passed" if exit_code == 0 else "failed",
        "total_duration_seconds": round(time.monotonic() - started, 3),
        "passes": results,
        "flake_candidates": flake_candidates,
        "consistent_failures": consistent_failures,
    }
    output.write_text(json.dumps(evidence, indent=2) + "\n", encoding="utf-8")
    if exit_code:
        failed = next(
            result for result in results
            if result["return_code"] != 0 or result.get("junit_error") is not None
        )
        print(
            f"repeat gate failed on pass {failed['pass']} "
            f"with PYTHONHASHSEED={failed['python_hash_seed']}",
        )
        if failed["stdout"]:
            print(failed["stdout"])
        if failed["stderr"]:
            print(failed["stderr"], file=sys.stderr)
        return exit_code

    print(
        f"repeat gate passed: {len(results)} orders in "
        f"{evidence['total_duration_seconds']:.3f}s",
    )
    return 0


def _parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("tests", nargs="*", default=list(DEFAULT_TESTS))
    parser.add_argument("--output", type=Path, required=True)
    return parser


def main() -> int:
    args = _parser().parse_args()
    return run_repeat_gate(args.tests, output=args.output)


if __name__ == "__main__":
    raise SystemExit(main())