test: govern unavailable conditional environments

This commit is contained in:
Joseph Magly
2026-08-16 09:05:41 -04:00
parent b961623513
commit c90240f1e9
5 changed files with 352 additions and 9 deletions
+99 -1
View File
@@ -13,6 +13,9 @@ from pathlib import Path
REQUIRED_GATE_FIELDS = {
"id", "job", "marker", "runner", "prerequisites", "expected_cost", "coverage_paths"
}
REQUIRED_WAIVER_FIELDS = {
"gate", "reason", "issue", "opened", "expires", "blocked_claim"
}
SOFTWARE_ONLY_GATES = (
"model-download-runtime",
"external-evaluation",
@@ -24,9 +27,99 @@ ISSUE_URL = re.compile(
r"^https://github\.com/elder-plinius/OBLITERATUS/issues/[1-9][0-9]*$",
)
MAX_STALE_EXCEPTION_DAYS = 30
MAX_ENVIRONMENT_WAIVER_DAYS = 30
def validate(policy_path: Path, quality_path: Path, workflow_path: Path) -> list[str]:
def validate_environment_waivers(
policy: dict,
*,
today: date | None = None,
) -> tuple[dict[str, dict], list[str]]:
"""Return active gate waivers and structural/lifetime errors."""
errors: list[str] = []
active: dict[str, dict] = {}
today = today or date.today()
gates = policy.get("gates", [])
gate_ids = {
gate.get("id")
for gate in gates
if isinstance(gate, dict) and isinstance(gate.get("id"), str)
}
waivers = policy.get("environment_waivers", [])
if not isinstance(waivers, list):
return {}, ["conditional policy environment_waivers must be a list"]
seen: set[str] = set()
for index, waiver in enumerate(waivers):
label = f"environment waiver {index}"
if not isinstance(waiver, dict):
errors.append(f"{label} must be an object")
continue
missing = REQUIRED_WAIVER_FIELDS - set(waiver)
if missing:
errors.append(f"{label} is missing fields: {sorted(missing)}")
continue
gate_id = waiver["gate"]
if not isinstance(gate_id, str) or not gate_id:
errors.append(f"{label} gate must be a non-empty string")
continue
if gate_id in seen:
errors.append(f"duplicate environment waiver gate: {gate_id}")
continue
seen.add(gate_id)
valid = True
if gate_id not in gate_ids:
errors.append(f"environment waiver references unknown gate: {gate_id}")
valid = False
if gate_id in SOFTWARE_ONLY_GATES:
errors.append(f"software-only gate may not use an environment waiver: {gate_id}")
valid = False
for field in ("reason", "blocked_claim"):
value = waiver[field]
if not isinstance(value, str) or not value.strip():
errors.append(f"environment waiver {gate_id} {field} must be non-empty")
valid = False
issue = waiver["issue"]
if not isinstance(issue, str) or ISSUE_URL.fullmatch(issue) is None:
errors.append(f"environment waiver {gate_id} issue must be a canonical issue URL")
valid = False
try:
opened = date.fromisoformat(waiver["opened"])
expires = date.fromisoformat(waiver["expires"])
except (TypeError, ValueError):
errors.append(f"environment waiver {gate_id} opened/expires must be ISO dates")
continue
if opened > today:
errors.append(f"environment waiver {gate_id} opens in the future: {opened}")
valid = False
if expires < opened:
errors.append(f"environment waiver {gate_id} expires before it opens")
valid = False
elif expires > opened + timedelta(days=MAX_ENVIRONMENT_WAIVER_DAYS):
errors.append(
f"environment waiver {gate_id} exceeds {MAX_ENVIRONMENT_WAIVER_DAYS} days",
)
valid = False
if expires < today:
errors.append(f"environment waiver {gate_id} expired on {expires}")
valid = False
if valid:
active[gate_id] = waiver
return active, errors
def validate(
policy_path: Path,
quality_path: Path,
workflow_path: Path,
*,
today: date | None = None,
) -> list[str]:
errors: list[str] = []
policy = json.loads(policy_path.read_text())
quality = json.loads(quality_path.read_text())
@@ -44,6 +137,9 @@ def validate(policy_path: Path, quality_path: Path, workflow_path: Path) -> list
by_id: dict[str, dict] = {}
for index, gate in enumerate(gates):
if not isinstance(gate, dict):
errors.append(f"gate {index} must be an object")
continue
missing = REQUIRED_GATE_FIELDS - set(gate)
if missing:
errors.append(f"gate {index} is missing fields: {sorted(missing)}")
@@ -77,6 +173,8 @@ def validate(policy_path: Path, quality_path: Path, workflow_path: Path) -> list
for token in required_workflow_tokens:
if token not in workflow:
errors.append(f"conditional workflow is missing {token!r}")
_, waiver_errors = validate_environment_waivers(policy, today=today)
errors.extend(waiver_errors)
return errors