mirror of
https://github.com/elder-plinius/OBLITERATUS.git
synced 2026-08-17 16:37:30 +02:00
33 lines
990 B
Python
33 lines
990 B
Python
"""Run the evaluator against the pinned tiny causal language model."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import math
|
|
|
|
import pytest
|
|
from datasets import Dataset
|
|
|
|
from obliteratus.evaluation.evaluator import Evaluator
|
|
from obliteratus.models.loader import load_model
|
|
|
|
|
|
pytestmark = [pytest.mark.network, pytest.mark.download]
|
|
MODEL = "hf-internal-testing/tiny-random-gpt2"
|
|
REVISION = "71034c5d8bde858ff824298bdedc65515b97d2b9"
|
|
|
|
|
|
def test_pinned_tiny_model_evaluator_produces_finite_perplexity():
|
|
handle = load_model(
|
|
MODEL,
|
|
revision=REVISION,
|
|
device="cpu",
|
|
trust_remote_code=False,
|
|
skip_snapshot=True,
|
|
)
|
|
handle.tokenizer.pad_token = handle.tokenizer.eos_token
|
|
dataset = Dataset.from_dict({"text": ["A short test sentence.", "A second sentence."]})
|
|
result = Evaluator(handle, dataset, batch_size=2, max_length=24).evaluate()
|
|
assert math.isfinite(result["perplexity"])
|
|
assert result["perplexity"] > 0
|
|
handle.cleanup()
|