mirror of
https://github.com/elder-plinius/OBLITERATUS.git
synced 2026-08-17 16:37:30 +02:00
test: govern unavailable conditional environments
This commit is contained in:
Joseph Magly
@@ -13,6 +13,9 @@ from pathlib import Path
|
||||
REQUIRED_GATE_FIELDS = {
|
||||
"id", "job", "marker", "runner", "prerequisites", "expected_cost", "coverage_paths"
|
||||
}
|
||||
REQUIRED_WAIVER_FIELDS = {
|
||||
"gate", "reason", "issue", "opened", "expires", "blocked_claim"
|
||||
}
|
||||
SOFTWARE_ONLY_GATES = (
|
||||
"model-download-runtime",
|
||||
"external-evaluation",
|
||||
@@ -24,9 +27,99 @@ ISSUE_URL = re.compile(
|
||||
r"^https://github\.com/elder-plinius/OBLITERATUS/issues/[1-9][0-9]*$",
|
||||
)
|
||||
MAX_STALE_EXCEPTION_DAYS = 30
|
||||
MAX_ENVIRONMENT_WAIVER_DAYS = 30
|
||||
|
||||
|
||||
def validate(policy_path: Path, quality_path: Path, workflow_path: Path) -> list[str]:
|
||||
def validate_environment_waivers(
|
||||
policy: dict,
|
||||
*,
|
||||
today: date | None = None,
|
||||
) -> tuple[dict[str, dict], list[str]]:
|
||||
"""Return active gate waivers and structural/lifetime errors."""
|
||||
|
||||
errors: list[str] = []
|
||||
active: dict[str, dict] = {}
|
||||
today = today or date.today()
|
||||
gates = policy.get("gates", [])
|
||||
gate_ids = {
|
||||
gate.get("id")
|
||||
for gate in gates
|
||||
if isinstance(gate, dict) and isinstance(gate.get("id"), str)
|
||||
}
|
||||
waivers = policy.get("environment_waivers", [])
|
||||
if not isinstance(waivers, list):
|
||||
return {}, ["conditional policy environment_waivers must be a list"]
|
||||
|
||||
seen: set[str] = set()
|
||||
for index, waiver in enumerate(waivers):
|
||||
label = f"environment waiver {index}"
|
||||
if not isinstance(waiver, dict):
|
||||
errors.append(f"{label} must be an object")
|
||||
continue
|
||||
missing = REQUIRED_WAIVER_FIELDS - set(waiver)
|
||||
if missing:
|
||||
errors.append(f"{label} is missing fields: {sorted(missing)}")
|
||||
continue
|
||||
|
||||
gate_id = waiver["gate"]
|
||||
if not isinstance(gate_id, str) or not gate_id:
|
||||
errors.append(f"{label} gate must be a non-empty string")
|
||||
continue
|
||||
if gate_id in seen:
|
||||
errors.append(f"duplicate environment waiver gate: {gate_id}")
|
||||
continue
|
||||
seen.add(gate_id)
|
||||
valid = True
|
||||
if gate_id not in gate_ids:
|
||||
errors.append(f"environment waiver references unknown gate: {gate_id}")
|
||||
valid = False
|
||||
if gate_id in SOFTWARE_ONLY_GATES:
|
||||
errors.append(f"software-only gate may not use an environment waiver: {gate_id}")
|
||||
valid = False
|
||||
|
||||
for field in ("reason", "blocked_claim"):
|
||||
value = waiver[field]
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
errors.append(f"environment waiver {gate_id} {field} must be non-empty")
|
||||
valid = False
|
||||
issue = waiver["issue"]
|
||||
if not isinstance(issue, str) or ISSUE_URL.fullmatch(issue) is None:
|
||||
errors.append(f"environment waiver {gate_id} issue must be a canonical issue URL")
|
||||
valid = False
|
||||
|
||||
try:
|
||||
opened = date.fromisoformat(waiver["opened"])
|
||||
expires = date.fromisoformat(waiver["expires"])
|
||||
except (TypeError, ValueError):
|
||||
errors.append(f"environment waiver {gate_id} opened/expires must be ISO dates")
|
||||
continue
|
||||
if opened > today:
|
||||
errors.append(f"environment waiver {gate_id} opens in the future: {opened}")
|
||||
valid = False
|
||||
if expires < opened:
|
||||
errors.append(f"environment waiver {gate_id} expires before it opens")
|
||||
valid = False
|
||||
elif expires > opened + timedelta(days=MAX_ENVIRONMENT_WAIVER_DAYS):
|
||||
errors.append(
|
||||
f"environment waiver {gate_id} exceeds {MAX_ENVIRONMENT_WAIVER_DAYS} days",
|
||||
)
|
||||
valid = False
|
||||
if expires < today:
|
||||
errors.append(f"environment waiver {gate_id} expired on {expires}")
|
||||
valid = False
|
||||
|
||||
if valid:
|
||||
active[gate_id] = waiver
|
||||
return active, errors
|
||||
|
||||
|
||||
def validate(
|
||||
policy_path: Path,
|
||||
quality_path: Path,
|
||||
workflow_path: Path,
|
||||
*,
|
||||
today: date | None = None,
|
||||
) -> list[str]:
|
||||
errors: list[str] = []
|
||||
policy = json.loads(policy_path.read_text())
|
||||
quality = json.loads(quality_path.read_text())
|
||||
@@ -44,6 +137,9 @@ def validate(policy_path: Path, quality_path: Path, workflow_path: Path) -> list
|
||||
|
||||
by_id: dict[str, dict] = {}
|
||||
for index, gate in enumerate(gates):
|
||||
if not isinstance(gate, dict):
|
||||
errors.append(f"gate {index} must be an object")
|
||||
continue
|
||||
missing = REQUIRED_GATE_FIELDS - set(gate)
|
||||
if missing:
|
||||
errors.append(f"gate {index} is missing fields: {sorted(missing)}")
|
||||
@@ -77,6 +173,8 @@ def validate(policy_path: Path, quality_path: Path, workflow_path: Path) -> list
|
||||
for token in required_workflow_tokens:
|
||||
if token not in workflow:
|
||||
errors.append(f"conditional workflow is missing {token!r}")
|
||||
_, waiver_errors = validate_environment_waivers(policy, today=today)
|
||||
errors.extend(waiver_errors)
|
||||
return errors
|
||||
|
||||
|
||||
|
||||
@@ -6,24 +6,49 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from datetime import date, datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from scripts.check_conditional_policy import validate_environment_waivers
|
||||
except ModuleNotFoundError: # Direct execution adds scripts/, not the repository root.
|
||||
from check_conditional_policy import validate_environment_waivers
|
||||
|
||||
def summarize(policy: dict, results: dict, selected: dict) -> tuple[list[dict], list[str]]:
|
||||
|
||||
def summarize(
|
||||
policy: dict,
|
||||
results: dict,
|
||||
selected: dict,
|
||||
*,
|
||||
today: date | None = None,
|
||||
) -> tuple[list[dict], list[str]]:
|
||||
"""Return per-gate rows and selected-job failures."""
|
||||
failures: list[str] = []
|
||||
waivers, waiver_errors = validate_environment_waivers(policy, today=today)
|
||||
failures = [f"waiver policy: {error}" for error in waiver_errors]
|
||||
rows: list[dict] = []
|
||||
failed_jobs: set[str] = set()
|
||||
for gate in policy["gates"]:
|
||||
job = gate["job"]
|
||||
is_selected = bool(selected.get(job, False))
|
||||
result = results.get(job, "unknown")
|
||||
status = result if is_selected else "not_selected_no_fresh_evidence"
|
||||
waiver = waivers.get(gate["id"])
|
||||
if is_selected:
|
||||
status = result
|
||||
elif waiver is not None:
|
||||
status = "waived_no_support_claim"
|
||||
else:
|
||||
status = "not_selected_no_fresh_evidence"
|
||||
if is_selected and result != "success" and job not in failed_jobs:
|
||||
failures.append(f"{job}: selected but result was {result}")
|
||||
failed_jobs.add(job)
|
||||
rows.append({"gate": gate["id"], "job": job, "selected": is_selected, "status": status})
|
||||
row = {"gate": gate["id"], "job": job, "selected": is_selected, "status": status}
|
||||
if not is_selected and waiver is not None:
|
||||
row["waiver"] = {
|
||||
"issue": waiver["issue"],
|
||||
"expires": waiver["expires"],
|
||||
"blocked_claim": waiver["blocked_claim"],
|
||||
}
|
||||
rows.append(row)
|
||||
return rows, failures
|
||||
|
||||
|
||||
@@ -50,8 +75,23 @@ def main() -> int:
|
||||
args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
||||
summary_path = os.environ.get("GITHUB_STEP_SUMMARY")
|
||||
if summary_path:
|
||||
lines = ["## Conditional test evidence", "", "| Gate | Job | Status |", "|---|---|---|"]
|
||||
lines.extend(f"| {row['gate']} | {row['job']} | {row['status']} |" for row in rows)
|
||||
lines = [
|
||||
"## Conditional test evidence",
|
||||
"",
|
||||
"| Gate | Job | Status | Constraint |",
|
||||
"|---|---|---|---|",
|
||||
]
|
||||
for row in rows:
|
||||
waiver = row.get("waiver")
|
||||
constraint = ""
|
||||
if waiver is not None:
|
||||
constraint = (
|
||||
f"No support claim; expires {waiver['expires']}; "
|
||||
f"[tracker]({waiver['issue']})"
|
||||
)
|
||||
lines.append(
|
||||
f"| {row['gate']} | {row['job']} | {row['status']} | {constraint} |",
|
||||
)
|
||||
lines.extend(["", f"Evidence becomes stale after {policy['maximum_evidence_age_days']} days."])
|
||||
with Path(summary_path).open("a") as handle:
|
||||
handle.write("\n".join(lines) + "\n")
|
||||
|
||||
Reference in New Issue
Block a user