test: govern unavailable conditional environments

This commit is contained in:
Joseph Magly
2026-08-16 09:05:41 -04:00
parent b961623513
commit c90240f1e9
5 changed files with 352 additions and 9 deletions
+152
View File
@@ -3,6 +3,7 @@
from __future__ import annotations
import json
from datetime import date
from pathlib import Path
from scripts import check_conditional_policy
@@ -11,6 +12,20 @@ from scripts import run_conditional_gate
ROOT = Path(__file__).parents[1]
TODAY = date(2026, 8, 16)
def _waiver(gate: str, **overrides) -> dict:
value = {
"gate": gate,
"reason": "No runner is currently attached.",
"issue": "https://github.com/elder-plinius/OBLITERATUS/issues/110",
"opened": "2026-08-16",
"expires": "2026-09-15",
"blocked_claim": f"{gate} support is not claimed.",
}
value.update(overrides)
return value
def test_committed_conditional_policy_is_complete():
@@ -18,6 +33,7 @@ def test_committed_conditional_policy_is_complete():
ROOT / "ci" / "conditional-test-policy.json",
ROOT / "ci" / "test-quality-policy.json",
ROOT / ".github" / "workflows" / "conditional-tests.yml",
today=TODAY,
) == []
@@ -92,3 +108,139 @@ def test_summary_rejects_selected_failure_and_marks_unselected_stale():
assert rows[0]["status"] == "failure"
assert rows[1]["status"] == "failure"
assert rows[2]["status"] == "not_selected_no_fresh_evidence"
def test_summary_records_active_waiver_and_selected_success_overrides_it():
policy = {
"gates": [{"id": "cuda-runtime", "job": "cuda"}],
"environment_waivers": [_waiver("cuda-runtime")],
}
rows, failures = conditional_gate_summary.summarize(
policy,
{"cuda": "skipped"},
{"cuda": False},
today=TODAY,
)
assert failures == []
assert rows == [{
"gate": "cuda-runtime",
"job": "cuda",
"selected": False,
"status": "waived_no_support_claim",
"waiver": {
"issue": "https://github.com/elder-plinius/OBLITERATUS/issues/110",
"expires": "2026-09-15",
"blocked_claim": "cuda-runtime support is not claimed.",
},
}]
rows, failures = conditional_gate_summary.summarize(
policy,
{"cuda": "success"},
{"cuda": True},
today=TODAY,
)
assert failures == []
assert rows[0]["status"] == "success"
assert "waiver" not in rows[0]
def test_environment_waiver_validation_rejects_duplicate_expired_and_overlong_records():
policy = {
"gates": [
{"id": "cuda-runtime"},
{"id": "mps-runtime"},
{"id": "mlx-runtime"},
],
"environment_waivers": [
_waiver("cuda-runtime"),
_waiver("cuda-runtime"),
_waiver("mps-runtime", opened="2026-07-01", expires="2026-07-31"),
_waiver("mlx-runtime", expires="2026-09-16"),
],
}
active, errors = check_conditional_policy.validate_environment_waivers(
policy,
today=TODAY,
)
assert active == {"cuda-runtime": policy["environment_waivers"][0]}
assert "duplicate environment waiver gate: cuda-runtime" in errors
assert "environment waiver mps-runtime expired on 2026-07-31" in errors
assert "environment waiver mlx-runtime exceeds 30 days" in errors
def test_environment_waiver_validation_rejects_malformed_or_software_records():
policy = {
"gates": [
{"id": "network-services"},
{"id": "remote-execution"},
],
"environment_waivers": [
_waiver("network-services"),
_waiver("remote-execution", issue="not-an-issue", blocked_claim=""),
"not-an-object",
],
}
active, errors = check_conditional_policy.validate_environment_waivers(
policy,
today=TODAY,
)
assert active == {}
assert "software-only gate may not use an environment waiver: network-services" in errors
assert "environment waiver remote-execution issue must be a canonical issue URL" in errors
assert "environment waiver remote-execution blocked_claim must be non-empty" in errors
assert "environment waiver 2 must be an object" in errors
def test_environment_waiver_validation_rejects_missing_future_and_reversed_dates():
missing = _waiver("remote-execution")
missing.pop("reason")
policy = {
"gates": [
{"id": "remote-execution"},
{"id": "mps-runtime"},
{"id": "mlx-runtime"},
],
"environment_waivers": [
missing,
_waiver("mps-runtime", opened="2026-08-17", expires="2026-09-15"),
_waiver("mlx-runtime", opened="2026-08-16", expires="2026-08-15"),
],
}
active, errors = check_conditional_policy.validate_environment_waivers(
policy,
today=TODAY,
)
assert active == {}
assert "environment waiver 0 is missing fields: ['reason']" in errors
assert "environment waiver mps-runtime opens in the future: 2026-08-17" in errors
assert "environment waiver mlx-runtime expires before it opens" in errors
assert "environment waiver mlx-runtime expired on 2026-08-15" in errors
def test_summary_fails_an_expired_waiver():
policy = {
"gates": [{"id": "cuda-runtime", "job": "cuda"}],
"environment_waivers": [
_waiver("cuda-runtime", opened="2026-07-01", expires="2026-07-31"),
],
}
rows, failures = conditional_gate_summary.summarize(
policy,
{"cuda": "skipped"},
{"cuda": False},
today=TODAY,
)
assert rows[0]["status"] == "not_selected_no_fresh_evidence"
assert failures == [
"waiver policy: environment waiver cuda-runtime expired on 2026-07-31",
]