mirror of
https://github.com/elder-plinius/OBLITERATUS.git
synced 2026-08-18 00:47:23 +02:00
Enforce Gate 3 test quality budgets
This commit is contained in:
Joseph Magly
@@ -15,13 +15,15 @@ from typing import Any
|
||||
BASELINE_FLOORS = {
|
||||
"repository_statement": 75.0,
|
||||
"repository_branch": 60.0,
|
||||
"changed_line": 90.0,
|
||||
"changed_line": 95.0,
|
||||
"mature_cpu_statement": 92.0,
|
||||
"mature_cpu_branch": 80.0,
|
||||
"mutation_score": 75.0,
|
||||
"warning_budget": 0.0,
|
||||
}
|
||||
|
||||
SUPPORTED_PYTHON = {"3.10", "3.11", "3.12"}
|
||||
|
||||
|
||||
def _valid_exception(exceptions: Any, name: str, current: float) -> bool:
|
||||
if not isinstance(exceptions, list):
|
||||
@@ -158,6 +160,314 @@ def _validate_test_evidence(
|
||||
f"test {nodeid} flaked {len(recent)} times in {window} days "
|
||||
"without an active quarantine",
|
||||
)
|
||||
failures.extend(_validate_duration_policy(evidence, today=today))
|
||||
return failures
|
||||
|
||||
|
||||
def _positive_number(value: Any) -> bool:
|
||||
return (
|
||||
not isinstance(value, bool)
|
||||
and isinstance(value, (int, float))
|
||||
and math.isfinite(value)
|
||||
and value > 0
|
||||
)
|
||||
|
||||
|
||||
def _validate_duration_policy(evidence: dict[str, Any], *, today: date) -> list[str]:
|
||||
"""Validate owned suite, marker, testcase, and repeat duration budgets."""
|
||||
failures: list[str] = []
|
||||
budgets = evidence.get("duration_budgets")
|
||||
if not isinstance(budgets, dict):
|
||||
return ["test evidence duration_budgets must be an object"]
|
||||
|
||||
mandatory = budgets.get("mandatory_cpu")
|
||||
if not isinstance(mandatory, dict):
|
||||
failures.append("duration budget mandatory_cpu must be an object")
|
||||
else:
|
||||
if not isinstance(mandatory.get("owner"), str) or not mandatory["owner"].startswith("@"):
|
||||
failures.append("duration budget mandatory_cpu requires an @owner")
|
||||
suite_budgets = mandatory.get("max_suite_seconds_by_python")
|
||||
if not isinstance(suite_budgets, dict):
|
||||
failures.append(
|
||||
"duration budget mandatory_cpu max_suite_seconds_by_python must be an object",
|
||||
)
|
||||
else:
|
||||
versions = set(suite_budgets)
|
||||
if versions != SUPPORTED_PYTHON:
|
||||
failures.append(
|
||||
"duration budget mandatory_cpu must cover exactly Python 3.10, 3.11, and 3.12",
|
||||
)
|
||||
for version, value in suite_budgets.items():
|
||||
if not _positive_number(value):
|
||||
failures.append(
|
||||
"duration budget mandatory_cpu "
|
||||
f"max_suite_seconds_by_python.{version} must be positive",
|
||||
)
|
||||
testcase_budget = mandatory.get("max_testcase_seconds")
|
||||
if not _positive_number(testcase_budget):
|
||||
failures.append(
|
||||
"duration budget mandatory_cpu max_testcase_seconds must be positive",
|
||||
)
|
||||
marker_budgets = mandatory.get("max_marker_seconds")
|
||||
if not isinstance(marker_budgets, dict) or not marker_budgets:
|
||||
failures.append(
|
||||
"duration budget mandatory_cpu max_marker_seconds must be a non-empty object",
|
||||
)
|
||||
else:
|
||||
if "unmarked" not in marker_budgets:
|
||||
failures.append("duration budget mandatory_cpu must own the unmarked layer")
|
||||
for marker, value in marker_budgets.items():
|
||||
if not isinstance(marker, str) or not marker.strip() or not _positive_number(value):
|
||||
failures.append(
|
||||
f"duration budget mandatory_cpu has invalid marker budget {marker!r}",
|
||||
)
|
||||
|
||||
review_days = mandatory.get("maximum_owner_days")
|
||||
if (
|
||||
isinstance(review_days, bool)
|
||||
or not isinstance(review_days, int)
|
||||
or not 1 <= review_days <= 365
|
||||
):
|
||||
failures.append(
|
||||
"duration budget mandatory_cpu maximum_owner_days must be an integer from 1 to 365",
|
||||
)
|
||||
review_days = 90
|
||||
owners = mandatory.get("owned_slow_tests")
|
||||
if not isinstance(owners, list):
|
||||
failures.append("duration budget mandatory_cpu owned_slow_tests must be a list")
|
||||
owners = []
|
||||
seen: set[str] = set()
|
||||
for index, entry in enumerate(owners):
|
||||
label = f"owned slow test {index}"
|
||||
if not isinstance(entry, dict):
|
||||
failures.append(f"{label} must be an object")
|
||||
continue
|
||||
nodeid = entry.get("nodeid")
|
||||
if not isinstance(nodeid, str) or not nodeid.strip():
|
||||
failures.append(f"{label} requires a non-empty nodeid")
|
||||
continue
|
||||
if nodeid in seen:
|
||||
failures.append(f"duplicate owned slow test {nodeid}")
|
||||
seen.add(nodeid)
|
||||
if not isinstance(entry.get("owner"), str) or not entry["owner"].startswith("@"):
|
||||
failures.append(f"{label} requires an @owner")
|
||||
if not isinstance(entry.get("reason"), str) or not entry["reason"].strip():
|
||||
failures.append(f"{label} requires a non-empty reason")
|
||||
issue = entry.get("issue")
|
||||
if not isinstance(issue, str) or not issue.startswith(
|
||||
"https://github.com/elder-plinius/OBLITERATUS/issues/",
|
||||
):
|
||||
failures.append(f"{label} requires an OBLITERATUS issue URL")
|
||||
maximum = entry.get("max_seconds")
|
||||
if not _positive_number(maximum):
|
||||
failures.append(f"{label} max_seconds must be positive")
|
||||
elif _positive_number(testcase_budget) and maximum <= testcase_budget:
|
||||
failures.append(
|
||||
f"{label} max_seconds must exceed the default testcase budget",
|
||||
)
|
||||
opened = _policy_date(entry.get("opened"), f"{label} opened", failures)
|
||||
expires = _policy_date(entry.get("expires"), f"{label} expires", failures)
|
||||
if opened is not None and expires is not None:
|
||||
if opened > today:
|
||||
failures.append(f"{label} cannot open in the future")
|
||||
if expires <= opened:
|
||||
failures.append(f"{label} must expire after it opens")
|
||||
elif expires - opened > timedelta(days=review_days):
|
||||
failures.append(f"{label} exceeds the {review_days}-day review window")
|
||||
elif expires < today:
|
||||
failures.append(f"{label} expired on {expires.isoformat()}")
|
||||
|
||||
repeat = budgets.get("repeat_gate")
|
||||
if not isinstance(repeat, dict):
|
||||
failures.append("duration budget repeat_gate must be an object")
|
||||
else:
|
||||
if not isinstance(repeat.get("owner"), str) or not repeat["owner"].startswith("@"):
|
||||
failures.append("duration budget repeat_gate requires an @owner")
|
||||
total = repeat.get("max_total_seconds")
|
||||
per_pass = repeat.get("max_pass_seconds")
|
||||
if not _positive_number(total):
|
||||
failures.append("duration budget repeat_gate max_total_seconds must be positive")
|
||||
if not _positive_number(per_pass):
|
||||
failures.append("duration budget repeat_gate max_pass_seconds must be positive")
|
||||
if _positive_number(total) and _positive_number(per_pass) and per_pass > total:
|
||||
failures.append(
|
||||
"duration budget repeat_gate max_pass_seconds cannot exceed max_total_seconds",
|
||||
)
|
||||
return failures
|
||||
|
||||
|
||||
def _duration_value(value: Any, label: str, failures: list[str]) -> float | None:
|
||||
if not _positive_number(value) and value != 0:
|
||||
failures.append(f"{label} must be a non-negative finite number")
|
||||
return None
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
failures.append(f"{label} must be a non-negative finite number")
|
||||
return None
|
||||
return float(value)
|
||||
|
||||
|
||||
def validate_duration_evidence(
|
||||
trend: dict[str, Any], policy: dict[str, Any],
|
||||
) -> list[str]:
|
||||
"""Enforce mandatory CPU and repeat budgets on normalized trend evidence."""
|
||||
failures: list[str] = []
|
||||
budgets = policy["test_evidence"]["duration_budgets"]
|
||||
tests = trend.get("tests")
|
||||
if tests is not None:
|
||||
if not isinstance(tests, dict):
|
||||
failures.append("test trend tests must be an object")
|
||||
else:
|
||||
mandatory = budgets["mandatory_cpu"]
|
||||
python_version = trend.get("python")
|
||||
suite_budgets = mandatory["max_suite_seconds_by_python"]
|
||||
if python_version not in suite_budgets:
|
||||
failures.append(f"test trend has unsupported Python version {python_version!r}")
|
||||
suite_duration = _duration_value(
|
||||
tests.get("suite_duration_seconds"),
|
||||
"test trend suite_duration_seconds",
|
||||
failures,
|
||||
)
|
||||
if suite_duration is not None and python_version in suite_budgets:
|
||||
maximum = float(suite_budgets[python_version])
|
||||
if suite_duration > maximum:
|
||||
failures.append(
|
||||
f"test trend Python {python_version} suite duration "
|
||||
f"{suite_duration:.3f}s exceeds {maximum:.3f}s",
|
||||
)
|
||||
|
||||
durations = tests.get("durations")
|
||||
if not isinstance(durations, list) or not durations:
|
||||
failures.append("test trend durations must be a non-empty list")
|
||||
durations = []
|
||||
missing_markers = tests.get("missing_marker_nodeids")
|
||||
if not isinstance(missing_markers, list) or any(
|
||||
not isinstance(nodeid, str) or not nodeid.strip()
|
||||
for nodeid in missing_markers
|
||||
):
|
||||
failures.append("test trend missing_marker_nodeids must be a string list")
|
||||
missing_markers = []
|
||||
if tests.get("marker_metadata_complete") is not True or missing_markers:
|
||||
failures.append(
|
||||
"test trend duration marker metadata is incomplete"
|
||||
+ (f" for {len(missing_markers)} testcase(s)" if missing_markers else ""),
|
||||
)
|
||||
owners = {
|
||||
entry["nodeid"]: entry
|
||||
for entry in mandatory["owned_slow_tests"]
|
||||
}
|
||||
default_max = float(mandatory["max_testcase_seconds"])
|
||||
seen_nodeids: set[str] = set()
|
||||
computed_markers: dict[str, dict[str, int | float]] = {}
|
||||
for index, entry in enumerate(durations):
|
||||
label = f"test duration {index}"
|
||||
if not isinstance(entry, dict):
|
||||
failures.append(f"{label} must be an object")
|
||||
continue
|
||||
nodeid = entry.get("nodeid")
|
||||
if not isinstance(nodeid, str) or not nodeid.strip():
|
||||
failures.append(f"{label} requires a non-empty nodeid")
|
||||
continue
|
||||
if nodeid in seen_nodeids:
|
||||
failures.append(f"test trend repeats duration for {nodeid}")
|
||||
seen_nodeids.add(nodeid)
|
||||
seconds = _duration_value(entry.get("seconds"), f"{label} seconds", failures)
|
||||
markers = entry.get("markers")
|
||||
if not isinstance(markers, list) or not markers or any(
|
||||
not isinstance(marker, str) or not marker.strip() for marker in markers
|
||||
):
|
||||
failures.append(f"{label} markers must be a non-empty string list")
|
||||
markers = []
|
||||
if seconds is not None:
|
||||
if seconds > default_max:
|
||||
owner = owners.get(nodeid)
|
||||
if owner is None:
|
||||
failures.append(
|
||||
f"test {nodeid} took {seconds:.3f}s above the "
|
||||
f"{default_max:.3f}s default and has no owned slow-test budget",
|
||||
)
|
||||
elif seconds > float(owner["max_seconds"]):
|
||||
failures.append(
|
||||
f"owned slow test {nodeid} took {seconds:.3f}s above its "
|
||||
f"{float(owner['max_seconds']):.3f}s budget",
|
||||
)
|
||||
for marker in set(markers):
|
||||
summary = computed_markers.setdefault(
|
||||
marker, {"tests": 0, "duration_seconds": 0.0},
|
||||
)
|
||||
summary["tests"] = int(summary["tests"]) + 1
|
||||
summary["duration_seconds"] = (
|
||||
float(summary["duration_seconds"]) + seconds
|
||||
)
|
||||
|
||||
total = tests.get("total")
|
||||
if isinstance(total, bool) or not isinstance(total, int) or total != len(durations):
|
||||
failures.append("test trend total must equal the number of duration records")
|
||||
marker_evidence = tests.get("marker_durations")
|
||||
if not isinstance(marker_evidence, dict):
|
||||
failures.append("test trend marker_durations must be an object")
|
||||
marker_evidence = {}
|
||||
marker_budgets = mandatory["max_marker_seconds"]
|
||||
for marker, computed in computed_markers.items():
|
||||
if marker not in marker_budgets:
|
||||
failures.append(f"test marker {marker} has no duration budget")
|
||||
continue
|
||||
observed = _duration_value(
|
||||
computed["duration_seconds"],
|
||||
f"test marker {marker} duration",
|
||||
failures,
|
||||
)
|
||||
if observed is not None and observed > float(marker_budgets[marker]):
|
||||
failures.append(
|
||||
f"test marker {marker} duration {observed:.3f}s exceeds "
|
||||
f"{float(marker_budgets[marker]):.3f}s",
|
||||
)
|
||||
recorded = marker_evidence.get(marker)
|
||||
expected = {
|
||||
"tests": computed["tests"],
|
||||
"duration_seconds": round(float(computed["duration_seconds"]), 3),
|
||||
}
|
||||
if recorded != expected:
|
||||
failures.append(f"test marker {marker} duration summary is inconsistent")
|
||||
extra_markers = sorted(set(marker_evidence) - set(computed_markers))
|
||||
for marker in extra_markers:
|
||||
failures.append(f"test marker {marker} has summary without duration records")
|
||||
|
||||
repeat = trend.get("repeat")
|
||||
if repeat is not None:
|
||||
if not isinstance(repeat, dict):
|
||||
failures.append("test trend repeat must be an object")
|
||||
else:
|
||||
repeat_budget = budgets["repeat_gate"]
|
||||
total = _duration_value(
|
||||
repeat.get("total_duration_seconds"),
|
||||
"repeat gate total duration",
|
||||
failures,
|
||||
)
|
||||
if total is not None and total > float(repeat_budget["max_total_seconds"]):
|
||||
failures.append(
|
||||
f"repeat gate total duration {total:.3f}s exceeds "
|
||||
f"{float(repeat_budget['max_total_seconds']):.3f}s",
|
||||
)
|
||||
passes = repeat.get("passes")
|
||||
if not isinstance(passes, list) or not passes:
|
||||
failures.append("repeat gate passes must be a non-empty list")
|
||||
passes = []
|
||||
for index, entry in enumerate(passes, start=1):
|
||||
if not isinstance(entry, dict):
|
||||
failures.append(f"repeat gate pass {index} must be an object")
|
||||
continue
|
||||
duration = _duration_value(
|
||||
entry.get("duration_seconds"),
|
||||
f"repeat gate pass {index} duration",
|
||||
failures,
|
||||
)
|
||||
if duration is not None and duration > float(repeat_budget["max_pass_seconds"]):
|
||||
failures.append(
|
||||
f"repeat gate pass {index} duration {duration:.3f}s exceeds "
|
||||
f"{float(repeat_budget['max_pass_seconds']):.3f}s",
|
||||
)
|
||||
if tests is None and repeat is None:
|
||||
failures.append("test trend contains neither tests nor repeat duration evidence")
|
||||
return failures
|
||||
|
||||
|
||||
@@ -282,6 +592,7 @@ def _parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--policy", type=Path, required=True)
|
||||
parser.add_argument("--coverage", type=Path)
|
||||
parser.add_argument("--evidence", type=Path)
|
||||
return parser
|
||||
|
||||
|
||||
@@ -309,6 +620,16 @@ def main() -> int:
|
||||
else:
|
||||
measurement, coverage_failures = validate_mature_cpu_scope(report, policy)
|
||||
failures.extend(coverage_failures)
|
||||
if args.evidence is not None and not failures:
|
||||
try:
|
||||
trend = json.loads(args.evidence.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
failures.append(f"cannot read test trend evidence: {exc}")
|
||||
else:
|
||||
if not isinstance(trend, dict):
|
||||
failures.append("test trend evidence root must be an object")
|
||||
else:
|
||||
failures.extend(validate_duration_evidence(trend, policy))
|
||||
|
||||
if failures:
|
||||
for failure in failures:
|
||||
|
||||
@@ -97,14 +97,40 @@ def junit_snapshot(path: Path) -> dict[str, Any]:
|
||||
failures: list[str] = []
|
||||
errors: list[str] = []
|
||||
skipped: list[str] = []
|
||||
durations: list[dict[str, str | float]] = []
|
||||
durations: list[dict[str, Any]] = []
|
||||
marker_totals: dict[str, dict[str, int | float]] = {}
|
||||
missing_marker_nodeids: list[str] = []
|
||||
for case in cases:
|
||||
nodeid = f"{case.attrib.get('classname', '<unknown>')}::{case.attrib.get('name', '<unknown>')}"
|
||||
try:
|
||||
duration = float(case.attrib.get("time", "0"))
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"JUnit testcase {nodeid} has invalid time") from exc
|
||||
durations.append({"nodeid": nodeid, "seconds": duration})
|
||||
if not math.isfinite(duration) or duration < 0:
|
||||
raise ValueError(f"JUnit testcase {nodeid} has invalid time")
|
||||
marker_property = None
|
||||
properties = case.find("properties")
|
||||
if properties is not None:
|
||||
values = [
|
||||
item.attrib.get("value", "")
|
||||
for item in properties.findall("property")
|
||||
if item.attrib.get("name") == "duration_markers"
|
||||
]
|
||||
if len(values) > 1:
|
||||
raise ValueError(f"JUnit testcase {nodeid} repeats duration_markers")
|
||||
marker_property = values[0] if values else None
|
||||
if marker_property is None:
|
||||
missing_marker_nodeids.append(nodeid)
|
||||
markers = sorted({
|
||||
marker.strip()
|
||||
for marker in (marker_property or "unmarked").split(",")
|
||||
if marker.strip()
|
||||
})
|
||||
durations.append({"nodeid": nodeid, "seconds": duration, "markers": markers})
|
||||
for marker in markers:
|
||||
summary = marker_totals.setdefault(marker, {"tests": 0, "duration_seconds": 0.0})
|
||||
summary["tests"] = int(summary["tests"]) + 1
|
||||
summary["duration_seconds"] = float(summary["duration_seconds"]) + duration
|
||||
if case.find("failure") is not None:
|
||||
failures.append(nodeid)
|
||||
if case.find("error") is not None:
|
||||
@@ -113,6 +139,20 @@ def junit_snapshot(path: Path) -> dict[str, Any]:
|
||||
skipped.append(nodeid)
|
||||
failed = set(failures) | set(errors) | set(skipped)
|
||||
durations.sort(key=lambda item: (-float(item["seconds"]), str(item["nodeid"])))
|
||||
suites = list(root.iter("testsuite"))
|
||||
raw_suite_duration = suites[0].attrib.get("time") if suites else None
|
||||
try:
|
||||
suite_duration = (
|
||||
float(raw_suite_duration)
|
||||
if raw_suite_duration is not None
|
||||
else sum(float(item["seconds"]) for item in durations)
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise ValueError("JUnit testsuite has invalid time") from exc
|
||||
if not math.isfinite(suite_duration) or suite_duration < 0:
|
||||
raise ValueError("JUnit testsuite has invalid time")
|
||||
for summary in marker_totals.values():
|
||||
summary["duration_seconds"] = round(float(summary["duration_seconds"]), 3)
|
||||
return {
|
||||
"total": len(cases),
|
||||
"passed": len(cases) - len(failed),
|
||||
@@ -120,6 +160,11 @@ def junit_snapshot(path: Path) -> dict[str, Any]:
|
||||
"errors": errors,
|
||||
"skipped": skipped,
|
||||
"duration_seconds": round(sum(float(item["seconds"]) for item in durations), 3),
|
||||
"suite_duration_seconds": round(suite_duration, 3),
|
||||
"durations": durations,
|
||||
"marker_durations": dict(sorted(marker_totals.items())),
|
||||
"marker_metadata_complete": not missing_marker_nodeids,
|
||||
"missing_marker_nodeids": missing_marker_nodeids,
|
||||
"slowest": durations[:20],
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user