mirror of
https://github.com/elder-plinius/OBLITERATUS.git
synced 2026-08-17 16:37:30 +02:00
259 lines
8.6 KiB
Python
259 lines
8.6 KiB
Python
"""Tests for conditional policy, runner, and evidence helpers."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from datetime import date
|
|
from pathlib import Path
|
|
|
|
from scripts import check_conditional_policy
|
|
from scripts import conditional_gate_summary
|
|
from scripts import run_conditional_gate
|
|
|
|
|
|
ROOT = Path(__file__).parents[1]
|
|
TODAY = date(2026, 8, 16)
|
|
|
|
|
|
def _waiver(gate: str, **overrides) -> dict:
|
|
value = {
|
|
"gate": gate,
|
|
"reason": "No runner is currently attached.",
|
|
"issue": "https://github.com/elder-plinius/OBLITERATUS/issues/110",
|
|
"opened": "2026-08-16",
|
|
"expires": "2026-09-15",
|
|
"blocked_claim": f"{gate} support is not claimed.",
|
|
}
|
|
value.update(overrides)
|
|
return value
|
|
|
|
|
|
def test_committed_conditional_policy_is_complete():
|
|
assert check_conditional_policy.validate(
|
|
ROOT / "ci" / "conditional-test-policy.json",
|
|
ROOT / "ci" / "test-quality-policy.json",
|
|
ROOT / ".github" / "workflows" / "conditional-tests.yml",
|
|
today=TODAY,
|
|
) == []
|
|
|
|
|
|
def test_cuda_job_replaces_locked_cpu_torch_with_same_version_cuda_build():
|
|
workflow = (ROOT / ".github" / "workflows" / "conditional-tests.yml").read_text()
|
|
cuda_job = workflow.split(" cuda:\n", maxsplit=1)[1].split(" mps:\n", maxsplit=1)[0]
|
|
|
|
assert "torch.__version__.split" in cuda_job
|
|
assert "UV_TORCH_BACKEND=cu130 uv pip install" in cuda_job
|
|
assert "--reinstall-package torch" in cuda_job
|
|
assert '"torch==$CUDA_TORCH_VERSION"' in cuda_job
|
|
assert "assert torch.version.cuda is not None" in cuda_job
|
|
assert 'uv pip check --python "$CONDITIONAL_ENV/bin/python"' in cuda_job
|
|
|
|
|
|
def test_policy_rejects_unknown_cpu_exclusion_gate(tmp_path):
|
|
policy = json.loads((ROOT / "ci" / "conditional-test-policy.json").read_text())
|
|
quality = {
|
|
"mature_cpu_scope": {
|
|
"exclusions": [{"path": "obliteratus/device.py", "conditional_gate": "missing"}]
|
|
}
|
|
}
|
|
policy_path = tmp_path / "policy.json"
|
|
quality_path = tmp_path / "quality.json"
|
|
policy_path.write_text(json.dumps(policy))
|
|
quality_path.write_text(json.dumps(quality))
|
|
errors = check_conditional_policy.validate(
|
|
policy_path,
|
|
quality_path,
|
|
ROOT / ".github" / "workflows" / "conditional-tests.yml",
|
|
)
|
|
assert errors == ["CPU exclusion obliteratus/device.py references unknown gate missing"]
|
|
|
|
|
|
def test_junit_counts_and_report_are_machine_readable(tmp_path):
|
|
junit = tmp_path / "result.xml"
|
|
junit.write_text(
|
|
'<testsuites><testsuite tests="3" failures="1" errors="0" skipped="1"/>'
|
|
'<testsuite tests="2" failures="0" errors="1" skipped="0"/></testsuites>'
|
|
)
|
|
assert run_conditional_gate.counts(junit) == {
|
|
"tests": 5,
|
|
"failures": 1,
|
|
"errors": 1,
|
|
"skipped": 1,
|
|
}
|
|
report = tmp_path / "report.json"
|
|
run_conditional_gate.write_report(report, "network-services", "passed", counts={"tests": 1})
|
|
evidence = json.loads(report.read_text())
|
|
assert evidence["gate"] == "network-services"
|
|
assert evidence["status"] == "passed"
|
|
assert evidence["counts"] == {"tests": 1}
|
|
|
|
|
|
def test_remote_prerequisites_are_actionable(monkeypatch):
|
|
names = {
|
|
"OBLITERATUS_REMOTE_HOST",
|
|
"OBLITERATUS_REMOTE_USER",
|
|
"OBLITERATUS_REMOTE_KEY",
|
|
"OBLITERATUS_REMOTE_KNOWN_HOSTS",
|
|
}
|
|
for name in names:
|
|
monkeypatch.delenv(name, raising=False)
|
|
assert set(run_conditional_gate.missing_prerequisites("remote-execution")) == names
|
|
for name in names:
|
|
monkeypatch.setenv(name, "fixture")
|
|
assert run_conditional_gate.missing_prerequisites("remote-execution") == []
|
|
|
|
|
|
def test_summary_rejects_selected_failure_and_marks_unselected_stale():
|
|
policy = {
|
|
"gates": [
|
|
{"id": "first", "job": "shared"},
|
|
{"id": "second", "job": "shared"},
|
|
{"id": "optional", "job": "optional"},
|
|
]
|
|
}
|
|
rows, failures = conditional_gate_summary.summarize(
|
|
policy,
|
|
{"shared": "failure", "optional": "skipped"},
|
|
{"shared": True, "optional": False},
|
|
)
|
|
assert failures == ["shared: selected but result was failure"]
|
|
assert rows[0]["status"] == "failure"
|
|
assert rows[1]["status"] == "failure"
|
|
assert rows[2]["status"] == "not_selected_no_fresh_evidence"
|
|
|
|
|
|
def test_summary_records_active_waiver_and_selected_success_overrides_it():
|
|
policy = {
|
|
"gates": [{"id": "cuda-runtime", "job": "cuda"}],
|
|
"environment_waivers": [_waiver("cuda-runtime")],
|
|
}
|
|
|
|
rows, failures = conditional_gate_summary.summarize(
|
|
policy,
|
|
{"cuda": "skipped"},
|
|
{"cuda": False},
|
|
today=TODAY,
|
|
)
|
|
assert failures == []
|
|
assert rows == [{
|
|
"gate": "cuda-runtime",
|
|
"job": "cuda",
|
|
"selected": False,
|
|
"status": "waived_no_support_claim",
|
|
"waiver": {
|
|
"issue": "https://github.com/elder-plinius/OBLITERATUS/issues/110",
|
|
"expires": "2026-09-15",
|
|
"blocked_claim": "cuda-runtime support is not claimed.",
|
|
},
|
|
}]
|
|
|
|
rows, failures = conditional_gate_summary.summarize(
|
|
policy,
|
|
{"cuda": "success"},
|
|
{"cuda": True},
|
|
today=TODAY,
|
|
)
|
|
assert failures == []
|
|
assert rows[0]["status"] == "success"
|
|
assert "waiver" not in rows[0]
|
|
|
|
|
|
def test_environment_waiver_validation_rejects_duplicate_expired_and_overlong_records():
|
|
policy = {
|
|
"gates": [
|
|
{"id": "cuda-runtime"},
|
|
{"id": "mps-runtime"},
|
|
{"id": "mlx-runtime"},
|
|
],
|
|
"environment_waivers": [
|
|
_waiver("cuda-runtime"),
|
|
_waiver("cuda-runtime"),
|
|
_waiver("mps-runtime", opened="2026-07-01", expires="2026-07-31"),
|
|
_waiver("mlx-runtime", expires="2026-09-16"),
|
|
],
|
|
}
|
|
|
|
active, errors = check_conditional_policy.validate_environment_waivers(
|
|
policy,
|
|
today=TODAY,
|
|
)
|
|
|
|
assert active == {"cuda-runtime": policy["environment_waivers"][0]}
|
|
assert "duplicate environment waiver gate: cuda-runtime" in errors
|
|
assert "environment waiver mps-runtime expired on 2026-07-31" in errors
|
|
assert "environment waiver mlx-runtime exceeds 30 days" in errors
|
|
|
|
|
|
def test_environment_waiver_validation_rejects_malformed_or_software_records():
|
|
policy = {
|
|
"gates": [
|
|
{"id": "network-services"},
|
|
{"id": "remote-execution"},
|
|
],
|
|
"environment_waivers": [
|
|
_waiver("network-services"),
|
|
_waiver("remote-execution", issue="not-an-issue", blocked_claim=""),
|
|
"not-an-object",
|
|
],
|
|
}
|
|
|
|
active, errors = check_conditional_policy.validate_environment_waivers(
|
|
policy,
|
|
today=TODAY,
|
|
)
|
|
|
|
assert active == {}
|
|
assert "software-only gate may not use an environment waiver: network-services" in errors
|
|
assert "environment waiver remote-execution issue must be a canonical issue URL" in errors
|
|
assert "environment waiver remote-execution blocked_claim must be non-empty" in errors
|
|
assert "environment waiver 2 must be an object" in errors
|
|
|
|
|
|
def test_environment_waiver_validation_rejects_missing_future_and_reversed_dates():
|
|
missing = _waiver("remote-execution")
|
|
missing.pop("reason")
|
|
policy = {
|
|
"gates": [
|
|
{"id": "remote-execution"},
|
|
{"id": "mps-runtime"},
|
|
{"id": "mlx-runtime"},
|
|
],
|
|
"environment_waivers": [
|
|
missing,
|
|
_waiver("mps-runtime", opened="2026-08-17", expires="2026-09-15"),
|
|
_waiver("mlx-runtime", opened="2026-08-16", expires="2026-08-15"),
|
|
],
|
|
}
|
|
|
|
active, errors = check_conditional_policy.validate_environment_waivers(
|
|
policy,
|
|
today=TODAY,
|
|
)
|
|
|
|
assert active == {}
|
|
assert "environment waiver 0 is missing fields: ['reason']" in errors
|
|
assert "environment waiver mps-runtime opens in the future: 2026-08-17" in errors
|
|
assert "environment waiver mlx-runtime expires before it opens" in errors
|
|
assert "environment waiver mlx-runtime expired on 2026-08-15" in errors
|
|
|
|
|
|
def test_summary_fails_an_expired_waiver():
|
|
policy = {
|
|
"gates": [{"id": "cuda-runtime", "job": "cuda"}],
|
|
"environment_waivers": [
|
|
_waiver("cuda-runtime", opened="2026-07-01", expires="2026-07-31"),
|
|
],
|
|
}
|
|
|
|
rows, failures = conditional_gate_summary.summarize(
|
|
policy,
|
|
{"cuda": "skipped"},
|
|
{"cuda": False},
|
|
today=TODAY,
|
|
)
|
|
|
|
assert rows[0]["status"] == "not_selected_no_fresh_evidence"
|
|
assert failures == [
|
|
"waiver policy: environment waiver cuda-runtime expired on 2026-07-31",
|
|
]
|