OBLITERATUS/tests/test_tourney_contracts.py

"""Deterministic tournament lifecycle, checkpoint, and rendering contracts."""

from __future__ import annotations

import json
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import Mock

import pytest

from obliteratus import tourney


def _contender(
    method: str,
    score: float,
    *,
    output_dir: str = "",
    error: str | None = None,
    direction: str = "mean_diff",
    cert: str = "GREEN",
) -> tourney.Contender:
    return tourney.Contender(
        method=method,
        score=score,
        metrics={
            "refusal_rate": max(0.0, 1.0 - score),
            "coherence": max(0.0, score),
            "kl_divergence": 0.1,
            "perplexity": 12.0,
            "direction_method": direction,
            "spectral_certification": cert,
        },
        output_dir=output_dir,
        time_s=2.5,
        error=error,
        direction_method=direction,
        spectral_cert=cert,
    )


@pytest.mark.parametrize(
    ("certification", "expected"),
    [("GREEN", 1.0), ("YELLOW", 0.975), ("RED", 0.95), (None, 0.975)],
)
def test_composite_score_preserves_documented_weighting(certification, expected):
    metrics = {
        "refusal_rate": 0.0,
        "coherence": 1.0,
        "kl_divergence": 0.0,
        "perplexity": 0.0,
        "spectral_certification": certification,
        "degenerate_count": 0,
    }

    assert tourney.composite_score(metrics) == pytest.approx(expected)


def test_composite_score_missing_metrics_and_degenerate_outputs_fail_safe():
    assert tourney.composite_score({}) == pytest.approx(0.225)
    assert tourney.composite_score({"degenerate_count": 3}) == pytest.approx(0.1875)


def test_result_dictionary_sorts_contenders_without_mutating_round_order():
    low = _contender("low", 0.2)
    high = _contender("high", 0.9)
    rnd = tourney.TourneyRound(
        round_num=1,
        name="Qualifier",
        contenders=[low, high],
        prompt_volume=64,
        advanced_to=["high"],
        eliminated=["low"],
    )
    result = tourney.TourneyResult(
        model="org/model",
        winner=high,
        rounds=[rnd],
        total_time_s=12.5,
        timestamp="2026-08-16T00:00:00+00:00",
    )

    payload = result.to_dict()

    assert payload["winner"]["method"] == "high"
    assert [item["method"] for item in payload["rounds"][0]["contenders"]] == [
        "high",
        "low",
    ]
    assert rnd.contenders == [low, high]


def test_checkpoint_round_trip_preserves_completed_and_partial_metadata(tmp_path):
    completed = _contender("complete", 0.8, direction="svd", cert="YELLOW")
    completed.round_eliminated = 1
    partial = _contender("partial", 0.7, direction="pca", cert="RED")
    result = tourney.TourneyResult(
        model="org/model",
        rounds=[
            tourney.TourneyRound(
                round_num=1,
                name="Qualifier",
                contenders=[completed],
                prompt_volume=64,
                advanced_to=[],
                eliminated=["complete"],
            )
        ],
    )

    path = tourney._save_checkpoint(
        output_dir=tmp_path,
        result=result,
        current_round_num=2,
        current_round_name="Semifinals",
        current_round_volume=128,
        current_round_advance=1,
        current_round_verify=30,
        completed_methods=[partial],
        remaining_methods=["remaining"],
        alive=["partial", "remaining"],
        model_name="org/model",
        dataset_key="builtin",
        quantization="4bit",
        methods=["complete", "partial", "remaining"],
    )

    checkpoint = tourney._load_checkpoint(tmp_path)
    restored, partials, remaining, interrupted = tourney._restore_rounds(checkpoint)

    assert path == tmp_path / tourney.CHECKPOINT_FILENAME
    assert tourney._checkpoint_matches(checkpoint, "org/model", "builtin", "4bit")
    assert not tourney._checkpoint_matches(checkpoint, "other/model", "builtin", "4bit")
    assert restored.rounds[0].contenders[0].direction_method == "svd"
    assert restored.rounds[0].contenders[0].spectral_cert == "YELLOW"
    assert partials[0].direction_method == "pca"
    assert partials[0].spectral_cert == "RED"
    assert partials[0].round_eliminated == 0
    assert remaining == ["remaining"]
    assert interrupted["verify_sample_size"] == 30


@pytest.mark.parametrize("payload", ["[]", "null", "{}", '{"version": 2}', "not-json"])
def test_checkpoint_loader_rejects_malformed_or_unsupported_roots(tmp_path, payload):
    (tmp_path / tourney.CHECKPOINT_FILENAME).write_text(payload, encoding="utf-8")

    assert tourney._load_checkpoint(tmp_path) is None


def test_checkpoint_loader_returns_none_when_absent(tmp_path):
    assert tourney._load_checkpoint(tmp_path) is None


@pytest.mark.parametrize("error", [OSError("unreadable"), UnicodeError("invalid encoding")])
def test_checkpoint_loader_fails_closed_when_document_cannot_be_read(
    monkeypatch,
    tmp_path,
    error,
):
    path = tmp_path / tourney.CHECKPOINT_FILENAME
    path.write_text('{"version": 1}', encoding="utf-8")
    monkeypatch.setattr(Path, "read_text", Mock(side_effect=error))

    assert tourney._load_checkpoint(tmp_path) is None


def test_markdown_and_html_render_all_outcomes_and_escape_html():
    winner = _contender("<winner>", 0.85, direction="<dir>")
    middle = _contender("middle", 0.55, cert="YELLOW")
    low = _contender("low", 0.2, cert="INCONCLUSIVE")
    failed = _contender("failed", -1.0, error="boom")
    rnd = tourney.TourneyRound(
        round_num=1,
        name="<script>alert(1)</script>",
        contenders=[failed, low, middle, winner],
        prompt_volume=64,
        advanced_to=[winner.method, middle.method],
        eliminated=[low.method, failed.method],
    )
    result = tourney.TourneyResult(
        model="org/<model>",
        winner=winner,
        rounds=[rnd],
        total_time_s=120,
        hub_repo="org/result",
    )

    markdown = tourney.render_bracket(result)
    html = tourney.render_bracket_html(result)

    assert "Pushed to: [org/result]" in markdown
    assert "| 4 | failed | — | ERROR" in markdown
    assert "*out*" not in markdown  # the only round is final
    assert "&lt;model&gt;" in html
    assert "&lt;winner&gt;" in html
    assert "&lt;script&gt;alert(1)&lt;/script&gt;" in html
    assert "<script>alert(1)</script>" not in html
    assert "card-score good" in html
    assert "card-score mid" in html
    assert "card-score bad" in html
    assert "badge-err" in html


def test_renderers_handle_no_winner_and_model_card_requires_winner():
    result = tourney.TourneyResult(model="model", total_time_s=0)

    assert "**No winner**" in tourney.render_bracket(result)
    assert "No winner determined" in tourney.render_bracket_html(result)
    assert tourney.generate_model_card(result) == ""


def test_model_card_includes_winner_metrics_and_bracket():
    winner = _contender("advanced", 0.9)
    result = tourney.TourneyResult(
        model="org/base-model",
        winner=winner,
        rounds=[
            tourney.TourneyRound(
                round_num=1,
                name="Final",
                contenders=[winner],
                prompt_volume=64,
                advanced_to=["advanced"],
            )
        ],
        timestamp="2026-08-16T00:00:00+00:00",
    )

    card = tourney.generate_model_card(result)

    assert "base_model: org/base-model" in card
    assert "Winning Method: `advanced`" in card
    assert "# OBLITERATUS TOURNEY" in card


def test_runner_initialization_cleans_fresh_output_and_preserves_resume(tmp_path):
    output = tmp_path / "tourney"
    output.mkdir()
    (output / "stale.txt").write_text("stale", encoding="utf-8")

    fresh = tourney.TourneyRunner("model", output_dir=str(output), methods=["a"])
    assert fresh.output_dir == output
    assert not (output / "stale.txt").exists()

    (output / "checkpoint.txt").write_text("keep", encoding="utf-8")
    resumed = tourney.TourneyRunner(
        "model",
        output_dir=str(output),
        methods=["a"],
        resume=True,
    )
    assert (resumed.output_dir / "checkpoint.txt").read_text(encoding="utf-8") == "keep"


def test_runner_prompt_loading_is_bounded_by_shortest_source(monkeypatch, tmp_path):
    runner = tourney.TourneyRunner("model", output_dir=str(tmp_path), methods=["a"])
    loader = Mock(return_value=(["h1", "h2", "h3"], ["s1", "s2"]))
    monkeypatch.setattr("obliteratus.prompts.load_dataset_source", loader)

    harmful, harmless = runner._load_prompts(10)

    assert harmful == ["h1", "h2"]
    assert harmless == ["s1", "s2"]
    loader.assert_called_once_with("builtin")


@pytest.mark.parametrize(
    ("message", "expected"),
    [
        ("GPU quota exceeded for this session", True),
        ("ZeroGPU token expired", True),
        ("ordinary model failure", False),
    ],
)
def test_quota_error_classification_is_narrow(message, expected):
    assert tourney.TourneyRunner._is_quota_error(RuntimeError(message)) is expected


def test_run_one_method_uses_optional_gpu_wrapper(monkeypatch, tmp_path):
    runner = tourney.TourneyRunner("model", output_dir=str(tmp_path), methods=["a"])
    direct = Mock(return_value=_contender("a", 0.8))
    monkeypatch.setattr(runner, "_run_method", direct)

    assert runner._run_one_method("a", ["h"], ["s"], "out", 20, None).method == "a"
    wrapper = Mock(side_effect=lambda fn, *args: fn(*args))
    assert runner._run_one_method("a", ["h"], ["s"], "out", 20, wrapper).method == "a"
    wrapper.assert_called_once()


def test_full_runner_ranks_rounds_cleans_losers_and_writes_results(monkeypatch, tmp_path):
    logs: list[str] = []
    rounds: list[tourney.TourneyRound] = []
    runner = tourney.TourneyRunner(
        "org/model",
        methods=["alpha", "beta", "gamma", "delta"],
        output_dir=str(tmp_path / "run"),
        on_log=logs.append,
        on_round=rounds.append,
    )
    scores = {"alpha": 0.9, "beta": 0.7, "gamma": 0.4, "delta": 0.2}
    monkeypatch.setattr(runner, "_load_prompts", lambda volume: (["h"] * volume, ["s"] * volume))
    monkeypatch.setattr(
        tourney.shutil,
        "disk_usage",
        lambda _path: SimpleNamespace(free=int(4.5e9)),
    )

    def run_method(method, _harmful, _harmless, save_dir, _verify):
        Path(save_dir).mkdir(parents=True, exist_ok=True)
        return _contender(method, scores[method], output_dir=save_dir)

    monkeypatch.setattr(runner, "_run_method", run_method)

    result = runner.run()

    assert [rnd.name for rnd in result.rounds] == ["Qualifiers", "Semifinals", "Championship"]
    assert result.winner.method == "alpha"
    assert rounds == result.rounds
    assert (runner.output_dir / "tourney_results.json").exists()
    assert (runner.output_dir / "tourney_bracket.md").exists()
    assert (runner.output_dir / "r3_alpha").exists()
    assert not (runner.output_dir / "r3_beta").exists()
    assert any("Low disk space" in line for line in logs)


def test_full_runner_does_not_crown_an_errored_only_contender(monkeypatch, tmp_path):
    runner = tourney.TourneyRunner("model", methods=["broken"], output_dir=str(tmp_path))
    monkeypatch.setattr(runner, "_load_prompts", lambda _volume: (["h"], ["s"]))
    monkeypatch.setattr(
        runner,
        "_run_method",
        lambda method, *_args: _contender(method, -1.0, error="broken"),
    )

    result = runner.run()

    assert result.winner is None
    assert json.loads((tmp_path / "tourney_results.json").read_text())["winner"] is None


def test_run_iter_saves_exact_resume_point_on_quota_exhaustion(monkeypatch, tmp_path):
    runner = tourney.TourneyRunner(
        "model",
        methods=["alpha", "beta"],
        output_dir=str(tmp_path),
    )
    monkeypatch.setattr(runner, "_load_prompts", lambda _volume: (["h"], ["s"]))

    def run_one(method, *_args):
        if method == "beta":
            raise RuntimeError("GPU quota exceeded")
        return _contender(method, 0.8)

    monkeypatch.setattr(runner, "_run_one_method", run_one)
    iterator = runner.run_iter()

    assert "running `alpha`" in next(iterator)[0]
    assert "running `beta`" in next(iterator)[0]
    with pytest.raises(RuntimeError, match="GPU quota exceeded"):
        next(iterator)

    checkpoint = tourney._load_checkpoint(tmp_path)
    interrupted = checkpoint["interrupted_round"]
    assert [item["method"] for item in interrupted["completed_methods"]] == ["alpha"]
    assert interrupted["remaining_methods"] == ["beta"]


def test_run_iter_resumes_partial_round_without_repeating_completed_method(monkeypatch, tmp_path):
    partial = _contender("alpha", 0.9, direction="pca", cert="YELLOW")
    tourney._save_checkpoint(
        output_dir=tmp_path,
        result=tourney.TourneyResult(model="model"),
        current_round_num=1,
        current_round_name="Qualifiers",
        current_round_volume=64,
        current_round_advance=1,
        current_round_verify=20,
        completed_methods=[partial],
        remaining_methods=["beta"],
        alive=["alpha", "beta"],
        model_name="model",
        dataset_key="builtin",
        quantization=None,
        methods=["alpha", "beta"],
    )
    runner = tourney.TourneyRunner(
        "model",
        methods=["alpha", "beta"],
        output_dir=str(tmp_path),
        resume=True,
    )
    monkeypatch.setattr(runner, "_load_prompts", lambda _volume: (["h"], ["s"]))
    called: list[str] = []

    def run_one(method, *_args):
        called.append(method)
        return _contender(method, 0.7)

    monkeypatch.setattr(runner, "_run_one_method", run_one)

    events = list(runner.run_iter())

    assert events[0][0].startswith("**Resuming tournament**")
    assert events[-1][0] == "Tournament complete"
    result = events[-1][1]
    assert called == ["beta"]
    assert result.winner.method == "alpha"
    assert result.winner.direction_method == "pca"
    assert result.winner.spectral_cert == "YELLOW"
    assert not (tmp_path / tourney.CHECKPOINT_FILENAME).exists()