#!/usr/bin/env python3 """Repeat deterministic tests in varied orders and emit timing evidence.""" from __future__ import annotations import argparse import json import os import subprocess import sys import time from pathlib import Path from typing import Any, Sequence DEFAULT_TESTS = ( "tests/test_config.py", "tests/test_config_properties.py", "tests/test_coverage_thresholds.py", "tests/test_property_contracts.py", "tests/test_advanced_metrics.py", "tests/test_metrics.py", ) HASH_SEEDS = ("0", "1", "8675309") def test_orders(paths: Sequence[str]) -> list[list[str]]: """Return stable forward, reverse, and interleaved orders.""" forward = list(paths) reverse = list(reversed(paths)) interleaved = forward[::2] + forward[1::2] return [forward, reverse, interleaved] def run_repeat_gate( paths: Sequence[str], *, output: Path, python: str = sys.executable, ) -> int: """Run three deterministic passes, always writing a JSON evidence record.""" output.parent.mkdir(parents=True, exist_ok=True) results: list[dict[str, Any]] = [] started = time.monotonic() exit_code = 0 for index, (order, hash_seed) in enumerate( zip(test_orders(paths), HASH_SEEDS, strict=True), start=1, ): command = [python, "-m", "pytest", "--no-cov", "-q", *order] environment = os.environ.copy() environment["PYTHONHASHSEED"] = hash_seed pass_started = time.monotonic() completed = subprocess.run( command, capture_output=True, text=True, env=environment, check=False, ) duration = time.monotonic() - pass_started results.append({ "pass": index, "python_hash_seed": hash_seed, "tests": order, "duration_seconds": round(duration, 3), "return_code": completed.returncode, "stdout": completed.stdout[-4000:], "stderr": completed.stderr[-4000:], }) if completed.returncode != 0: exit_code = completed.returncode break evidence = { "schema_version": 1, "status": "passed" if exit_code == 0 else "failed", "total_duration_seconds": round(time.monotonic() - started, 3), "passes": results, } output.write_text(json.dumps(evidence, indent=2) + "\n", encoding="utf-8") if exit_code: failed = results[-1] print( f"repeat gate failed on pass {failed['pass']} " f"with PYTHONHASHSEED={failed['python_hash_seed']}", ) if failed["stdout"]: print(failed["stdout"]) if failed["stderr"]: print(failed["stderr"], file=sys.stderr) return exit_code print( f"repeat gate passed: {len(results)} orders in " f"{evidence['total_duration_seconds']:.3f}s", ) return 0 def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("tests", nargs="*", default=list(DEFAULT_TESTS)) parser.add_argument("--output", type=Path, required=True) return parser def main() -> int: args = _parser().parse_args() return run_repeat_gate(args.tests, output=args.output) if __name__ == "__main__": raise SystemExit(main())