"""Run the evaluator against the pinned tiny causal language model."""

from __future__ import annotations

import math

import pytest
from datasets import Dataset

from obliteratus.evaluation.evaluator import Evaluator
from obliteratus.models.loader import load_model


pytestmark = [pytest.mark.network, pytest.mark.download]
MODEL = "hf-internal-testing/tiny-random-gpt2"
REVISION = "71034c5d8bde858ff824298bdedc65515b97d2b9"


def test_pinned_tiny_model_evaluator_produces_finite_perplexity():
    handle = load_model(
        MODEL,
        revision=REVISION,
        device="cpu",
        trust_remote_code=False,
        skip_snapshot=True,
    )
    handle.tokenizer.pad_token = handle.tokenizer.eos_token
    dataset = Dataset.from_dict({"text": ["A short test sentence.", "A second sentence."]})
    result = Evaluator(handle, dataset, batch_size=2, max_length=24).evaluate()
    assert math.isfinite(result["perplexity"])
    assert result["perplexity"] > 0
    handle.cleanup()