test: govern unavailable conditional environments

This commit is contained in:
Joseph Magly
2026-08-16 09:05:41 -04:00
parent b961623513
commit c90240f1e9
5 changed files with 352 additions and 9 deletions
+47 -7
View File
@@ -6,24 +6,49 @@ from __future__ import annotations
import argparse
import json
import os
from datetime import datetime, timezone
from datetime import date, datetime, timezone
from pathlib import Path
try:
from scripts.check_conditional_policy import validate_environment_waivers
except ModuleNotFoundError: # Direct execution adds scripts/, not the repository root.
from check_conditional_policy import validate_environment_waivers
def summarize(policy: dict, results: dict, selected: dict) -> tuple[list[dict], list[str]]:
def summarize(
policy: dict,
results: dict,
selected: dict,
*,
today: date | None = None,
) -> tuple[list[dict], list[str]]:
"""Return per-gate rows and selected-job failures."""
failures: list[str] = []
waivers, waiver_errors = validate_environment_waivers(policy, today=today)
failures = [f"waiver policy: {error}" for error in waiver_errors]
rows: list[dict] = []
failed_jobs: set[str] = set()
for gate in policy["gates"]:
job = gate["job"]
is_selected = bool(selected.get(job, False))
result = results.get(job, "unknown")
status = result if is_selected else "not_selected_no_fresh_evidence"
waiver = waivers.get(gate["id"])
if is_selected:
status = result
elif waiver is not None:
status = "waived_no_support_claim"
else:
status = "not_selected_no_fresh_evidence"
if is_selected and result != "success" and job not in failed_jobs:
failures.append(f"{job}: selected but result was {result}")
failed_jobs.add(job)
rows.append({"gate": gate["id"], "job": job, "selected": is_selected, "status": status})
row = {"gate": gate["id"], "job": job, "selected": is_selected, "status": status}
if not is_selected and waiver is not None:
row["waiver"] = {
"issue": waiver["issue"],
"expires": waiver["expires"],
"blocked_claim": waiver["blocked_claim"],
}
rows.append(row)
return rows, failures
@@ -50,8 +75,23 @@ def main() -> int:
args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
summary_path = os.environ.get("GITHUB_STEP_SUMMARY")
if summary_path:
lines = ["## Conditional test evidence", "", "| Gate | Job | Status |", "|---|---|---|"]
lines.extend(f"| {row['gate']} | {row['job']} | {row['status']} |" for row in rows)
lines = [
"## Conditional test evidence",
"",
"| Gate | Job | Status | Constraint |",
"|---|---|---|---|",
]
for row in rows:
waiver = row.get("waiver")
constraint = ""
if waiver is not None:
constraint = (
f"No support claim; expires {waiver['expires']}; "
f"[tracker]({waiver['issue']})"
)
lines.append(
f"| {row['gate']} | {row['job']} | {row['status']} | {constraint} |",
)
lines.extend(["", f"Evidence becomes stale after {policy['maximum_evidence_age_days']} days."])
with Path(summary_path).open("a") as handle:
handle.write("\n".join(lines) + "\n")