"""Deterministic tournament lifecycle, checkpoint, and rendering contracts.""" from __future__ import annotations import json from pathlib import Path from types import SimpleNamespace from unittest.mock import Mock import pytest from obliteratus import tourney def _contender( method: str, score: float, *, output_dir: str = "", error: str | None = None, direction: str = "mean_diff", cert: str = "GREEN", ) -> tourney.Contender: return tourney.Contender( method=method, score=score, metrics={ "refusal_rate": max(0.0, 1.0 - score), "coherence": max(0.0, score), "kl_divergence": 0.1, "perplexity": 12.0, "direction_method": direction, "spectral_certification": cert, }, output_dir=output_dir, time_s=2.5, error=error, direction_method=direction, spectral_cert=cert, ) @pytest.mark.parametrize( ("certification", "expected"), [("GREEN", 1.0), ("YELLOW", 0.975), ("RED", 0.95), (None, 0.975)], ) def test_composite_score_preserves_documented_weighting(certification, expected): metrics = { "refusal_rate": 0.0, "coherence": 1.0, "kl_divergence": 0.0, "perplexity": 0.0, "spectral_certification": certification, "degenerate_count": 0, } assert tourney.composite_score(metrics) == pytest.approx(expected) def test_composite_score_missing_metrics_and_degenerate_outputs_fail_safe(): assert tourney.composite_score({}) == pytest.approx(0.225) assert tourney.composite_score({"degenerate_count": 3}) == pytest.approx(0.1875) def test_result_dictionary_sorts_contenders_without_mutating_round_order(): low = _contender("low", 0.2) high = _contender("high", 0.9) rnd = tourney.TourneyRound( round_num=1, name="Qualifier", contenders=[low, high], prompt_volume=64, advanced_to=["high"], eliminated=["low"], ) result = tourney.TourneyResult( model="org/model", winner=high, rounds=[rnd], total_time_s=12.5, timestamp="2026-08-16T00:00:00+00:00", ) payload = result.to_dict() assert payload["winner"]["method"] == "high" assert [item["method"] for item in payload["rounds"][0]["contenders"]] == [ "high", "low", ] assert rnd.contenders == [low, high] def test_checkpoint_round_trip_preserves_completed_and_partial_metadata(tmp_path): completed = _contender("complete", 0.8, direction="svd", cert="YELLOW") completed.round_eliminated = 1 partial = _contender("partial", 0.7, direction="pca", cert="RED") result = tourney.TourneyResult( model="org/model", rounds=[ tourney.TourneyRound( round_num=1, name="Qualifier", contenders=[completed], prompt_volume=64, advanced_to=[], eliminated=["complete"], ) ], ) path = tourney._save_checkpoint( output_dir=tmp_path, result=result, current_round_num=2, current_round_name="Semifinals", current_round_volume=128, current_round_advance=1, current_round_verify=30, completed_methods=[partial], remaining_methods=["remaining"], alive=["partial", "remaining"], model_name="org/model", dataset_key="builtin", quantization="4bit", methods=["complete", "partial", "remaining"], ) checkpoint = tourney._load_checkpoint(tmp_path) restored, partials, remaining, interrupted = tourney._restore_rounds(checkpoint) assert path == tmp_path / tourney.CHECKPOINT_FILENAME assert tourney._checkpoint_matches(checkpoint, "org/model", "builtin", "4bit") assert not tourney._checkpoint_matches(checkpoint, "other/model", "builtin", "4bit") assert restored.rounds[0].contenders[0].direction_method == "svd" assert restored.rounds[0].contenders[0].spectral_cert == "YELLOW" assert partials[0].direction_method == "pca" assert partials[0].spectral_cert == "RED" assert partials[0].round_eliminated == 0 assert remaining == ["remaining"] assert interrupted["verify_sample_size"] == 30 @pytest.mark.parametrize("payload", ["[]", "null", "{}", '{"version": 2}', "not-json"]) def test_checkpoint_loader_rejects_malformed_or_unsupported_roots(tmp_path, payload): (tmp_path / tourney.CHECKPOINT_FILENAME).write_text(payload, encoding="utf-8") assert tourney._load_checkpoint(tmp_path) is None def test_checkpoint_loader_returns_none_when_absent(tmp_path): assert tourney._load_checkpoint(tmp_path) is None @pytest.mark.parametrize("error", [OSError("unreadable"), UnicodeError("invalid encoding")]) def test_checkpoint_loader_fails_closed_when_document_cannot_be_read( monkeypatch, tmp_path, error, ): path = tmp_path / tourney.CHECKPOINT_FILENAME path.write_text('{"version": 1}', encoding="utf-8") monkeypatch.setattr(Path, "read_text", Mock(side_effect=error)) assert tourney._load_checkpoint(tmp_path) is None def test_markdown_and_html_render_all_outcomes_and_escape_html(): winner = _contender("", 0.85, direction="") middle = _contender("middle", 0.55, cert="YELLOW") low = _contender("low", 0.2, cert="INCONCLUSIVE") failed = _contender("failed", -1.0, error="boom") rnd = tourney.TourneyRound( round_num=1, name="", contenders=[failed, low, middle, winner], prompt_volume=64, advanced_to=[winner.method, middle.method], eliminated=[low.method, failed.method], ) result = tourney.TourneyResult( model="org/", winner=winner, rounds=[rnd], total_time_s=120, hub_repo="org/result", ) markdown = tourney.render_bracket(result) html = tourney.render_bracket_html(result) assert "Pushed to: [org/result]" in markdown assert "| 4 | failed | — | ERROR" in markdown assert "*out*" not in markdown # the only round is final assert "<model>" in html assert "<winner>" in html assert "<script>alert(1)</script>" in html assert "" not in html assert "card-score good" in html assert "card-score mid" in html assert "card-score bad" in html assert "badge-err" in html def test_renderers_handle_no_winner_and_model_card_requires_winner(): result = tourney.TourneyResult(model="model", total_time_s=0) assert "**No winner**" in tourney.render_bracket(result) assert "No winner determined" in tourney.render_bracket_html(result) assert tourney.generate_model_card(result) == "" def test_model_card_includes_winner_metrics_and_bracket(): winner = _contender("advanced", 0.9) result = tourney.TourneyResult( model="org/base-model", winner=winner, rounds=[ tourney.TourneyRound( round_num=1, name="Final", contenders=[winner], prompt_volume=64, advanced_to=["advanced"], ) ], timestamp="2026-08-16T00:00:00+00:00", ) card = tourney.generate_model_card(result) assert "base_model: org/base-model" in card assert "Winning Method: `advanced`" in card assert "# OBLITERATUS TOURNEY" in card def test_runner_initialization_cleans_fresh_output_and_preserves_resume(tmp_path): output = tmp_path / "tourney" output.mkdir() (output / "stale.txt").write_text("stale", encoding="utf-8") fresh = tourney.TourneyRunner("model", output_dir=str(output), methods=["a"]) assert fresh.output_dir == output assert not (output / "stale.txt").exists() (output / "checkpoint.txt").write_text("keep", encoding="utf-8") resumed = tourney.TourneyRunner( "model", output_dir=str(output), methods=["a"], resume=True, ) assert (resumed.output_dir / "checkpoint.txt").read_text(encoding="utf-8") == "keep" def test_runner_prompt_loading_is_bounded_by_shortest_source(monkeypatch, tmp_path): runner = tourney.TourneyRunner("model", output_dir=str(tmp_path), methods=["a"]) loader = Mock(return_value=(["h1", "h2", "h3"], ["s1", "s2"])) monkeypatch.setattr("obliteratus.prompts.load_dataset_source", loader) harmful, harmless = runner._load_prompts(10) assert harmful == ["h1", "h2"] assert harmless == ["s1", "s2"] loader.assert_called_once_with("builtin") @pytest.mark.parametrize( ("message", "expected"), [ ("GPU quota exceeded for this session", True), ("ZeroGPU token expired", True), ("ordinary model failure", False), ], ) def test_quota_error_classification_is_narrow(message, expected): assert tourney.TourneyRunner._is_quota_error(RuntimeError(message)) is expected def test_run_one_method_uses_optional_gpu_wrapper(monkeypatch, tmp_path): runner = tourney.TourneyRunner("model", output_dir=str(tmp_path), methods=["a"]) direct = Mock(return_value=_contender("a", 0.8)) monkeypatch.setattr(runner, "_run_method", direct) assert runner._run_one_method("a", ["h"], ["s"], "out", 20, None).method == "a" wrapper = Mock(side_effect=lambda fn, *args: fn(*args)) assert runner._run_one_method("a", ["h"], ["s"], "out", 20, wrapper).method == "a" wrapper.assert_called_once() def test_full_runner_ranks_rounds_cleans_losers_and_writes_results(monkeypatch, tmp_path): logs: list[str] = [] rounds: list[tourney.TourneyRound] = [] runner = tourney.TourneyRunner( "org/model", methods=["alpha", "beta", "gamma", "delta"], output_dir=str(tmp_path / "run"), on_log=logs.append, on_round=rounds.append, ) scores = {"alpha": 0.9, "beta": 0.7, "gamma": 0.4, "delta": 0.2} monkeypatch.setattr(runner, "_load_prompts", lambda volume: (["h"] * volume, ["s"] * volume)) monkeypatch.setattr( tourney.shutil, "disk_usage", lambda _path: SimpleNamespace(free=int(4.5e9)), ) def run_method(method, _harmful, _harmless, save_dir, _verify): Path(save_dir).mkdir(parents=True, exist_ok=True) return _contender(method, scores[method], output_dir=save_dir) monkeypatch.setattr(runner, "_run_method", run_method) result = runner.run() assert [rnd.name for rnd in result.rounds] == ["Qualifiers", "Semifinals", "Championship"] assert result.winner.method == "alpha" assert rounds == result.rounds assert (runner.output_dir / "tourney_results.json").exists() assert (runner.output_dir / "tourney_bracket.md").exists() assert (runner.output_dir / "r3_alpha").exists() assert not (runner.output_dir / "r3_beta").exists() assert any("Low disk space" in line for line in logs) def test_full_runner_does_not_crown_an_errored_only_contender(monkeypatch, tmp_path): runner = tourney.TourneyRunner("model", methods=["broken"], output_dir=str(tmp_path)) monkeypatch.setattr(runner, "_load_prompts", lambda _volume: (["h"], ["s"])) monkeypatch.setattr( runner, "_run_method", lambda method, *_args: _contender(method, -1.0, error="broken"), ) result = runner.run() assert result.winner is None assert json.loads((tmp_path / "tourney_results.json").read_text())["winner"] is None def test_run_iter_saves_exact_resume_point_on_quota_exhaustion(monkeypatch, tmp_path): runner = tourney.TourneyRunner( "model", methods=["alpha", "beta"], output_dir=str(tmp_path), ) monkeypatch.setattr(runner, "_load_prompts", lambda _volume: (["h"], ["s"])) def run_one(method, *_args): if method == "beta": raise RuntimeError("GPU quota exceeded") return _contender(method, 0.8) monkeypatch.setattr(runner, "_run_one_method", run_one) iterator = runner.run_iter() assert "running `alpha`" in next(iterator)[0] assert "running `beta`" in next(iterator)[0] with pytest.raises(RuntimeError, match="GPU quota exceeded"): next(iterator) checkpoint = tourney._load_checkpoint(tmp_path) interrupted = checkpoint["interrupted_round"] assert [item["method"] for item in interrupted["completed_methods"]] == ["alpha"] assert interrupted["remaining_methods"] == ["beta"] def test_run_iter_resumes_partial_round_without_repeating_completed_method(monkeypatch, tmp_path): partial = _contender("alpha", 0.9, direction="pca", cert="YELLOW") tourney._save_checkpoint( output_dir=tmp_path, result=tourney.TourneyResult(model="model"), current_round_num=1, current_round_name="Qualifiers", current_round_volume=64, current_round_advance=1, current_round_verify=20, completed_methods=[partial], remaining_methods=["beta"], alive=["alpha", "beta"], model_name="model", dataset_key="builtin", quantization=None, methods=["alpha", "beta"], ) runner = tourney.TourneyRunner( "model", methods=["alpha", "beta"], output_dir=str(tmp_path), resume=True, ) monkeypatch.setattr(runner, "_load_prompts", lambda _volume: (["h"], ["s"])) called: list[str] = [] def run_one(method, *_args): called.append(method) return _contender(method, 0.7) monkeypatch.setattr(runner, "_run_one_method", run_one) events = list(runner.run_iter()) assert events[0][0].startswith("**Resuming tournament**") assert events[-1][0] == "Tournament complete" result = events[-1][1] assert called == ["beta"] assert result.winner.method == "alpha" assert result.winner.direction_method == "pca" assert result.winner.spectral_cert == "YELLOW" assert not (tmp_path / tourney.CHECKPOINT_FILENAME).exists()