diff --git a/.aiwg/architecture/sketch-testing-quality-program.md b/.aiwg/architecture/sketch-testing-quality-program.md index ae596cb..d93b858 100644 --- a/.aiwg/architecture/sketch-testing-quality-program.md +++ b/.aiwg/architecture/sketch-testing-quality-program.md @@ -38,3 +38,11 @@ tests. The exact thresholds and exclusion graph remain owned by Software-only conditional evidence must match the candidate commit. A stale evidence exception requires a reason, a canonical numeric OBLITERATUS issue, and an expiry no more than 30 days away. + +Gate 3 adds duration ownership to the existing normalized evidence path. Pytest +attaches the registered test-layer markers to each JUnit testcase; the evidence +normalizer retains the suite wall time, every testcase duration, the slowest +summary, and marker aggregates without changing schema version 1 consumers. +The quality-policy gate enforces per-Python suite, per-marker, individual-test, +and fixed repeat-campaign budgets. GitHub Actions independently caps each +mandatory Python test job at ten minutes. diff --git a/.aiwg/requirements/UC-testing-quality-program.md b/.aiwg/requirements/UC-testing-quality-program.md index 64a84cf..ce27539 100644 --- a/.aiwg/requirements/UC-testing-quality-program.md +++ b/.aiwg/requirements/UC-testing-quality-program.md @@ -37,3 +37,11 @@ A maintainer can exercise a non-editable installed distribution with a local deterministic tiny model through CLI/config parsing, loader/device selection, pipeline execution, checkpoint save/reload, evaluation, and JSON/CSV report generation without external services or accelerator hardware. + +## UC-TQ-07 — Maintainer detects unowned test-runtime growth + +A maintainer can inspect retained suite, testcase, registered-marker, and repeat +durations for the exact candidate commit. CI rejects a suite, marker, repeat +pass, or testcase that exceeds its project-owned budget. A deliberately slow +test requires a repository issue, owner, reason, explicit ceiling, and expiring +review window rather than an implicit or permanent exemption. diff --git a/.aiwg/risks/risks-testing-quality-program.md b/.aiwg/risks/risks-testing-quality-program.md index 1d97c8b..8f57b62 100644 --- a/.aiwg/risks/risks-testing-quality-program.md +++ b/.aiwg/risks/risks-testing-quality-program.md @@ -11,3 +11,4 @@ | TQ-R7 | Installed package behavior differs from checkout | medium | high | clean wheel and sdist installation smoke tests | | TQ-R8 | Numerical tests are device/dtype brittle | medium | high | invariant/tolerance contracts and separate backend-specific evidence | | TQ-R9 | Conditional evidence is reused for a different commit | medium | high | candidate-SHA validation; exceptions require a reason, canonical issue, and expiry within 30 days | +| TQ-R10 | Test growth silently erodes contributor feedback time | medium | medium | retained per-test/marker timing, policy-owned suite and repeat budgets, expiring slow-test ownership, ten-minute job cap | diff --git a/.aiwg/testing/master-test-plan.md b/.aiwg/testing/master-test-plan.md index 2a569e3..558cd50 100644 --- a/.aiwg/testing/master-test-plan.md +++ b/.aiwg/testing/master-test-plan.md @@ -15,7 +15,7 @@ tests, 9 conditionally deselected tests, no skips, and no warnings. The slowest hosted pytest selection is Python 3.12 at 136.33 seconds; its complete CI job is 4 minutes 52 seconds. Branch-enabled coverage is 75.35% statements and 61.49% branches. The documented mature CPU-testable scope is 92.69% statements and -81.47% branches. The repository enforces 90% changed-line coverage, +81.47% branches. The repository enforces 95% changed-line coverage, per-touched-module line and branch no-regression, a 75% selective mutation floor, and zero unexpected warnings. The current bounded campaign kills 809 of 886 mutants (91.31%), and 388 high-consequence tests pass in each of three file @@ -155,7 +155,7 @@ or tests that merely inspect source text do not satisfy this contract. | Hardware, network, or remote path | deterministic fake-based contract test plus the mapped conditional gate | | Research claim or metric | provenance/interpretation contract and a test preventing silent semantic drift | -New or modified code must maintain at least 90% changed-line coverage. A touched +New or modified code must maintain at least 95% changed-line coverage. A touched module may not lose line or branch coverage unless the PR documents why the measurement is misleading and adds equivalent contract or conditional evidence. No flaky-test rerun may turn a required failure into success. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9fe0863..b616250 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -230,7 +230,7 @@ jobs: test: name: Tests py${{ matrix.python-version }} runs-on: ubuntu-latest - timeout-minutes: 45 + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -340,7 +340,7 @@ jobs: --min-file obliteratus/reporting/report.py=70 \ --min-file obliteratus/community.py=70 \ --min-file obliteratus/telemetry.py=70 \ - --min-changed 90 \ + --min-changed 95 \ --base-ref "$COVERAGE_BASE" \ "${module_args[@]}" @@ -370,6 +370,12 @@ jobs: --output "test-results/test-trend-py${{ matrix.python-version }}.json" \ "${base_args[@]}" + - name: Enforce normalized test duration budgets + run: | + "$TEST_ENV/bin/python" scripts/check_quality_policy.py \ + --policy ci/test-quality-policy.json \ + --evidence "test-results/test-trend-py${{ matrix.python-version }}.json" + - name: Upload test and coverage evidence if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -450,6 +456,12 @@ jobs: --output quality-evidence/quality-trend-py3.12.json \ "${evidence_args[@]}" + - name: Enforce repeat duration budgets + run: | + "$QUALITY_ENV/bin/python" scripts/check_quality_policy.py \ + --policy ci/test-quality-policy.json \ + --evidence quality-evidence/quality-trend-py3.12.json + - name: Upload quality-depth evidence if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e072247..235b0af 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -66,6 +66,12 @@ days. Flake observations and time-bounded quarantines are governed in owner, reason, repository issue, and expiry. Thresholds may move downward only through a time-bounded exception linked to a reviewed repository issue. +The same policy owns mandatory-suite, registered-marker, individual-test, and +repeat-pass duration budgets. The normalized trend artifact records every +testcase duration and aggregates durations by test-layer marker. A testcase +that exceeds the default budget must have a named owner, repository issue, +reason, explicit ceiling, and expiring review window; otherwise CI fails. + Hardware, model-download, network-service, operator-UI, and remote-provider tests run separately so the pull-request baseline stays offline and credential-free. See [`docs/conditional-testing.md`](docs/conditional-testing.md) for manual commands, diff --git a/README.md b/README.md index 59a7b07..a61214d 100644 --- a/README.md +++ b/README.md @@ -761,19 +761,20 @@ pip install -e ".[dev]" pytest ``` -The mandatory CPU suite contains more than 1,400 tests, including a +The mandatory CPU suite contains more than 1,500 tests, including a repository-owned synthetic model that exercises the offline pipeline, installed-wheel CLI, study runner, transactional checkpoint recovery, and resumable auto-obliteration state. The suite also covers model/device/quantization/MLX boundaries, all analysis modules, architecture detection, visualization sanitization, community contributions, edge cases, and evaluation metrics. CI enforces at least -70% repository statement coverage, 55% repository branch coverage, 90% -changed-line coverage, and 90% statement / 78% branch coverage for the documented +75% repository statement coverage, 60% repository branch coverage, 95% +changed-line coverage, and 92% statement / 80% branch coverage for the documented mature CPU-only scope. Deterministic property, order-repeat, and selective mutation gates provide additional depth for numerical and policy-critical behavior. CI also rejects line or branch regressions in each touched production module by measuring the exact base commit, retains normalized trend evidence for 90 days, -and validates the source-to-test ownership graph in `ci/test-risk-map.json`. +enforces owned suite/test/marker/repeat duration budgets, and validates the +source-to-test ownership graph in `ci/test-risk-map.json`. Eight environment-bound test files run through the separately documented conditional workflow for model downloads, network services, operator UI, CUDA, bitsandbytes, MPS, MLX, and least-privileged remote execution. diff --git a/WORKSPACE.md b/WORKSPACE.md index ba4be8e..46ca4cb 100644 --- a/WORKSPACE.md +++ b/WORKSPACE.md @@ -45,11 +45,13 @@ Canonical required checks: - the exact Ruff F and actionlint command set in [.github/workflows/ci.yml](.github/workflows/ci.yml); - `python -m pytest` with at least 75% repository line coverage and 60% branch coverage; -- at least 90% changed-line coverage plus no line or branch regression in any +- at least 95% changed-line coverage plus no line or branch regression in any touched production module, compared with coverage from the exact base commit; - at least 92% line and 80% branch coverage for the documented mature CPU-testable scope, plus a 75% selective mutation score and zero unexpected warnings; +- normalized per-test and per-marker duration evidence, owned slow-test + exceptions, fixed repeat-campaign budgets, and a ten-minute test-job cap; - `python -m build --sdist --wheel` - `python -c 'import obliteratus; print(obliteratus.__version__)'` - `python -m obliteratus --help` diff --git a/ci/test-quality-policy.json b/ci/test-quality-policy.json index 12184c1..d80e804 100644 --- a/ci/test-quality-policy.json +++ b/ci/test-quality-policy.json @@ -3,7 +3,7 @@ "minimums": { "repository_statement": 75.0, "repository_branch": 60.0, - "changed_line": 90.0, + "changed_line": 95.0, "mature_cpu_statement": 92.0, "mature_cpu_branch": 80.0, "mutation_score": 75.0, @@ -138,6 +138,29 @@ "retention_days": 90, "flake_window_days": 30, "maximum_quarantine_days": 30, + "duration_budgets": { + "mandatory_cpu": { + "owner": "@maintainers", + "max_suite_seconds_by_python": { + "3.10": 240.0, + "3.11": 240.0, + "3.12": 240.0 + }, + "max_testcase_seconds": 15.0, + "max_marker_seconds": { + "cpu": 120.0, + "integration": 60.0, + "unmarked": 120.0 + }, + "maximum_owner_days": 90, + "owned_slow_tests": [] + }, + "repeat_gate": { + "owner": "@maintainers", + "max_total_seconds": 180.0, + "max_pass_seconds": 75.0 + } + }, "flake_history": [], "quarantines": [] }, diff --git a/scripts/check_quality_policy.py b/scripts/check_quality_policy.py index 3f7408d..4a7dd57 100644 --- a/scripts/check_quality_policy.py +++ b/scripts/check_quality_policy.py @@ -15,13 +15,15 @@ from typing import Any BASELINE_FLOORS = { "repository_statement": 75.0, "repository_branch": 60.0, - "changed_line": 90.0, + "changed_line": 95.0, "mature_cpu_statement": 92.0, "mature_cpu_branch": 80.0, "mutation_score": 75.0, "warning_budget": 0.0, } +SUPPORTED_PYTHON = {"3.10", "3.11", "3.12"} + def _valid_exception(exceptions: Any, name: str, current: float) -> bool: if not isinstance(exceptions, list): @@ -158,6 +160,314 @@ def _validate_test_evidence( f"test {nodeid} flaked {len(recent)} times in {window} days " "without an active quarantine", ) + failures.extend(_validate_duration_policy(evidence, today=today)) + return failures + + +def _positive_number(value: Any) -> bool: + return ( + not isinstance(value, bool) + and isinstance(value, (int, float)) + and math.isfinite(value) + and value > 0 + ) + + +def _validate_duration_policy(evidence: dict[str, Any], *, today: date) -> list[str]: + """Validate owned suite, marker, testcase, and repeat duration budgets.""" + failures: list[str] = [] + budgets = evidence.get("duration_budgets") + if not isinstance(budgets, dict): + return ["test evidence duration_budgets must be an object"] + + mandatory = budgets.get("mandatory_cpu") + if not isinstance(mandatory, dict): + failures.append("duration budget mandatory_cpu must be an object") + else: + if not isinstance(mandatory.get("owner"), str) or not mandatory["owner"].startswith("@"): + failures.append("duration budget mandatory_cpu requires an @owner") + suite_budgets = mandatory.get("max_suite_seconds_by_python") + if not isinstance(suite_budgets, dict): + failures.append( + "duration budget mandatory_cpu max_suite_seconds_by_python must be an object", + ) + else: + versions = set(suite_budgets) + if versions != SUPPORTED_PYTHON: + failures.append( + "duration budget mandatory_cpu must cover exactly Python 3.10, 3.11, and 3.12", + ) + for version, value in suite_budgets.items(): + if not _positive_number(value): + failures.append( + "duration budget mandatory_cpu " + f"max_suite_seconds_by_python.{version} must be positive", + ) + testcase_budget = mandatory.get("max_testcase_seconds") + if not _positive_number(testcase_budget): + failures.append( + "duration budget mandatory_cpu max_testcase_seconds must be positive", + ) + marker_budgets = mandatory.get("max_marker_seconds") + if not isinstance(marker_budgets, dict) or not marker_budgets: + failures.append( + "duration budget mandatory_cpu max_marker_seconds must be a non-empty object", + ) + else: + if "unmarked" not in marker_budgets: + failures.append("duration budget mandatory_cpu must own the unmarked layer") + for marker, value in marker_budgets.items(): + if not isinstance(marker, str) or not marker.strip() or not _positive_number(value): + failures.append( + f"duration budget mandatory_cpu has invalid marker budget {marker!r}", + ) + + review_days = mandatory.get("maximum_owner_days") + if ( + isinstance(review_days, bool) + or not isinstance(review_days, int) + or not 1 <= review_days <= 365 + ): + failures.append( + "duration budget mandatory_cpu maximum_owner_days must be an integer from 1 to 365", + ) + review_days = 90 + owners = mandatory.get("owned_slow_tests") + if not isinstance(owners, list): + failures.append("duration budget mandatory_cpu owned_slow_tests must be a list") + owners = [] + seen: set[str] = set() + for index, entry in enumerate(owners): + label = f"owned slow test {index}" + if not isinstance(entry, dict): + failures.append(f"{label} must be an object") + continue + nodeid = entry.get("nodeid") + if not isinstance(nodeid, str) or not nodeid.strip(): + failures.append(f"{label} requires a non-empty nodeid") + continue + if nodeid in seen: + failures.append(f"duplicate owned slow test {nodeid}") + seen.add(nodeid) + if not isinstance(entry.get("owner"), str) or not entry["owner"].startswith("@"): + failures.append(f"{label} requires an @owner") + if not isinstance(entry.get("reason"), str) or not entry["reason"].strip(): + failures.append(f"{label} requires a non-empty reason") + issue = entry.get("issue") + if not isinstance(issue, str) or not issue.startswith( + "https://github.com/elder-plinius/OBLITERATUS/issues/", + ): + failures.append(f"{label} requires an OBLITERATUS issue URL") + maximum = entry.get("max_seconds") + if not _positive_number(maximum): + failures.append(f"{label} max_seconds must be positive") + elif _positive_number(testcase_budget) and maximum <= testcase_budget: + failures.append( + f"{label} max_seconds must exceed the default testcase budget", + ) + opened = _policy_date(entry.get("opened"), f"{label} opened", failures) + expires = _policy_date(entry.get("expires"), f"{label} expires", failures) + if opened is not None and expires is not None: + if opened > today: + failures.append(f"{label} cannot open in the future") + if expires <= opened: + failures.append(f"{label} must expire after it opens") + elif expires - opened > timedelta(days=review_days): + failures.append(f"{label} exceeds the {review_days}-day review window") + elif expires < today: + failures.append(f"{label} expired on {expires.isoformat()}") + + repeat = budgets.get("repeat_gate") + if not isinstance(repeat, dict): + failures.append("duration budget repeat_gate must be an object") + else: + if not isinstance(repeat.get("owner"), str) or not repeat["owner"].startswith("@"): + failures.append("duration budget repeat_gate requires an @owner") + total = repeat.get("max_total_seconds") + per_pass = repeat.get("max_pass_seconds") + if not _positive_number(total): + failures.append("duration budget repeat_gate max_total_seconds must be positive") + if not _positive_number(per_pass): + failures.append("duration budget repeat_gate max_pass_seconds must be positive") + if _positive_number(total) and _positive_number(per_pass) and per_pass > total: + failures.append( + "duration budget repeat_gate max_pass_seconds cannot exceed max_total_seconds", + ) + return failures + + +def _duration_value(value: Any, label: str, failures: list[str]) -> float | None: + if not _positive_number(value) and value != 0: + failures.append(f"{label} must be a non-negative finite number") + return None + if isinstance(value, bool) or not isinstance(value, (int, float)): + failures.append(f"{label} must be a non-negative finite number") + return None + return float(value) + + +def validate_duration_evidence( + trend: dict[str, Any], policy: dict[str, Any], +) -> list[str]: + """Enforce mandatory CPU and repeat budgets on normalized trend evidence.""" + failures: list[str] = [] + budgets = policy["test_evidence"]["duration_budgets"] + tests = trend.get("tests") + if tests is not None: + if not isinstance(tests, dict): + failures.append("test trend tests must be an object") + else: + mandatory = budgets["mandatory_cpu"] + python_version = trend.get("python") + suite_budgets = mandatory["max_suite_seconds_by_python"] + if python_version not in suite_budgets: + failures.append(f"test trend has unsupported Python version {python_version!r}") + suite_duration = _duration_value( + tests.get("suite_duration_seconds"), + "test trend suite_duration_seconds", + failures, + ) + if suite_duration is not None and python_version in suite_budgets: + maximum = float(suite_budgets[python_version]) + if suite_duration > maximum: + failures.append( + f"test trend Python {python_version} suite duration " + f"{suite_duration:.3f}s exceeds {maximum:.3f}s", + ) + + durations = tests.get("durations") + if not isinstance(durations, list) or not durations: + failures.append("test trend durations must be a non-empty list") + durations = [] + missing_markers = tests.get("missing_marker_nodeids") + if not isinstance(missing_markers, list) or any( + not isinstance(nodeid, str) or not nodeid.strip() + for nodeid in missing_markers + ): + failures.append("test trend missing_marker_nodeids must be a string list") + missing_markers = [] + if tests.get("marker_metadata_complete") is not True or missing_markers: + failures.append( + "test trend duration marker metadata is incomplete" + + (f" for {len(missing_markers)} testcase(s)" if missing_markers else ""), + ) + owners = { + entry["nodeid"]: entry + for entry in mandatory["owned_slow_tests"] + } + default_max = float(mandatory["max_testcase_seconds"]) + seen_nodeids: set[str] = set() + computed_markers: dict[str, dict[str, int | float]] = {} + for index, entry in enumerate(durations): + label = f"test duration {index}" + if not isinstance(entry, dict): + failures.append(f"{label} must be an object") + continue + nodeid = entry.get("nodeid") + if not isinstance(nodeid, str) or not nodeid.strip(): + failures.append(f"{label} requires a non-empty nodeid") + continue + if nodeid in seen_nodeids: + failures.append(f"test trend repeats duration for {nodeid}") + seen_nodeids.add(nodeid) + seconds = _duration_value(entry.get("seconds"), f"{label} seconds", failures) + markers = entry.get("markers") + if not isinstance(markers, list) or not markers or any( + not isinstance(marker, str) or not marker.strip() for marker in markers + ): + failures.append(f"{label} markers must be a non-empty string list") + markers = [] + if seconds is not None: + if seconds > default_max: + owner = owners.get(nodeid) + if owner is None: + failures.append( + f"test {nodeid} took {seconds:.3f}s above the " + f"{default_max:.3f}s default and has no owned slow-test budget", + ) + elif seconds > float(owner["max_seconds"]): + failures.append( + f"owned slow test {nodeid} took {seconds:.3f}s above its " + f"{float(owner['max_seconds']):.3f}s budget", + ) + for marker in set(markers): + summary = computed_markers.setdefault( + marker, {"tests": 0, "duration_seconds": 0.0}, + ) + summary["tests"] = int(summary["tests"]) + 1 + summary["duration_seconds"] = ( + float(summary["duration_seconds"]) + seconds + ) + + total = tests.get("total") + if isinstance(total, bool) or not isinstance(total, int) or total != len(durations): + failures.append("test trend total must equal the number of duration records") + marker_evidence = tests.get("marker_durations") + if not isinstance(marker_evidence, dict): + failures.append("test trend marker_durations must be an object") + marker_evidence = {} + marker_budgets = mandatory["max_marker_seconds"] + for marker, computed in computed_markers.items(): + if marker not in marker_budgets: + failures.append(f"test marker {marker} has no duration budget") + continue + observed = _duration_value( + computed["duration_seconds"], + f"test marker {marker} duration", + failures, + ) + if observed is not None and observed > float(marker_budgets[marker]): + failures.append( + f"test marker {marker} duration {observed:.3f}s exceeds " + f"{float(marker_budgets[marker]):.3f}s", + ) + recorded = marker_evidence.get(marker) + expected = { + "tests": computed["tests"], + "duration_seconds": round(float(computed["duration_seconds"]), 3), + } + if recorded != expected: + failures.append(f"test marker {marker} duration summary is inconsistent") + extra_markers = sorted(set(marker_evidence) - set(computed_markers)) + for marker in extra_markers: + failures.append(f"test marker {marker} has summary without duration records") + + repeat = trend.get("repeat") + if repeat is not None: + if not isinstance(repeat, dict): + failures.append("test trend repeat must be an object") + else: + repeat_budget = budgets["repeat_gate"] + total = _duration_value( + repeat.get("total_duration_seconds"), + "repeat gate total duration", + failures, + ) + if total is not None and total > float(repeat_budget["max_total_seconds"]): + failures.append( + f"repeat gate total duration {total:.3f}s exceeds " + f"{float(repeat_budget['max_total_seconds']):.3f}s", + ) + passes = repeat.get("passes") + if not isinstance(passes, list) or not passes: + failures.append("repeat gate passes must be a non-empty list") + passes = [] + for index, entry in enumerate(passes, start=1): + if not isinstance(entry, dict): + failures.append(f"repeat gate pass {index} must be an object") + continue + duration = _duration_value( + entry.get("duration_seconds"), + f"repeat gate pass {index} duration", + failures, + ) + if duration is not None and duration > float(repeat_budget["max_pass_seconds"]): + failures.append( + f"repeat gate pass {index} duration {duration:.3f}s exceeds " + f"{float(repeat_budget['max_pass_seconds']):.3f}s", + ) + if tests is None and repeat is None: + failures.append("test trend contains neither tests nor repeat duration evidence") return failures @@ -282,6 +592,7 @@ def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--policy", type=Path, required=True) parser.add_argument("--coverage", type=Path) + parser.add_argument("--evidence", type=Path) return parser @@ -309,6 +620,16 @@ def main() -> int: else: measurement, coverage_failures = validate_mature_cpu_scope(report, policy) failures.extend(coverage_failures) + if args.evidence is not None and not failures: + try: + trend = json.loads(args.evidence.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + failures.append(f"cannot read test trend evidence: {exc}") + else: + if not isinstance(trend, dict): + failures.append("test trend evidence root must be an object") + else: + failures.extend(validate_duration_evidence(trend, policy)) if failures: for failure in failures: diff --git a/scripts/write_test_evidence.py b/scripts/write_test_evidence.py index e1bf710..d1b74ef 100644 --- a/scripts/write_test_evidence.py +++ b/scripts/write_test_evidence.py @@ -97,14 +97,40 @@ def junit_snapshot(path: Path) -> dict[str, Any]: failures: list[str] = [] errors: list[str] = [] skipped: list[str] = [] - durations: list[dict[str, str | float]] = [] + durations: list[dict[str, Any]] = [] + marker_totals: dict[str, dict[str, int | float]] = {} + missing_marker_nodeids: list[str] = [] for case in cases: nodeid = f"{case.attrib.get('classname', '')}::{case.attrib.get('name', '')}" try: duration = float(case.attrib.get("time", "0")) except ValueError as exc: raise ValueError(f"JUnit testcase {nodeid} has invalid time") from exc - durations.append({"nodeid": nodeid, "seconds": duration}) + if not math.isfinite(duration) or duration < 0: + raise ValueError(f"JUnit testcase {nodeid} has invalid time") + marker_property = None + properties = case.find("properties") + if properties is not None: + values = [ + item.attrib.get("value", "") + for item in properties.findall("property") + if item.attrib.get("name") == "duration_markers" + ] + if len(values) > 1: + raise ValueError(f"JUnit testcase {nodeid} repeats duration_markers") + marker_property = values[0] if values else None + if marker_property is None: + missing_marker_nodeids.append(nodeid) + markers = sorted({ + marker.strip() + for marker in (marker_property or "unmarked").split(",") + if marker.strip() + }) + durations.append({"nodeid": nodeid, "seconds": duration, "markers": markers}) + for marker in markers: + summary = marker_totals.setdefault(marker, {"tests": 0, "duration_seconds": 0.0}) + summary["tests"] = int(summary["tests"]) + 1 + summary["duration_seconds"] = float(summary["duration_seconds"]) + duration if case.find("failure") is not None: failures.append(nodeid) if case.find("error") is not None: @@ -113,6 +139,20 @@ def junit_snapshot(path: Path) -> dict[str, Any]: skipped.append(nodeid) failed = set(failures) | set(errors) | set(skipped) durations.sort(key=lambda item: (-float(item["seconds"]), str(item["nodeid"]))) + suites = list(root.iter("testsuite")) + raw_suite_duration = suites[0].attrib.get("time") if suites else None + try: + suite_duration = ( + float(raw_suite_duration) + if raw_suite_duration is not None + else sum(float(item["seconds"]) for item in durations) + ) + except ValueError as exc: + raise ValueError("JUnit testsuite has invalid time") from exc + if not math.isfinite(suite_duration) or suite_duration < 0: + raise ValueError("JUnit testsuite has invalid time") + for summary in marker_totals.values(): + summary["duration_seconds"] = round(float(summary["duration_seconds"]), 3) return { "total": len(cases), "passed": len(cases) - len(failed), @@ -120,6 +160,11 @@ def junit_snapshot(path: Path) -> dict[str, Any]: "errors": errors, "skipped": skipped, "duration_seconds": round(sum(float(item["seconds"]) for item in durations), 3), + "suite_duration_seconds": round(suite_duration, 3), + "durations": durations, + "marker_durations": dict(sorted(marker_totals.items())), + "marker_metadata_complete": not missing_marker_nodeids, + "missing_marker_nodeids": missing_marker_nodeids, "slowest": durations[:20], } diff --git a/tests/conftest.py b/tests/conftest.py index 8c5b02d..27237aa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,6 +9,28 @@ import pytest _EXTERNAL_MARKERS = ("network", "download", "remote") +_EVIDENCE_MARKERS = ( + "cpu", + "integration", + "slow", + "gpu", + "mps", + "mlx", + "network", + "download", + "remote", + "operator_ui", +) + + +def pytest_collection_modifyitems(items): + """Attach stable test-layer markers to JUnit timing evidence.""" + for item in items: + markers = sorted( + name for name in _EVIDENCE_MARKERS + if item.get_closest_marker(name) is not None + ) + item.user_properties.append(("duration_markers", ",".join(markers) or "unmarked")) @pytest.fixture(autouse=True) diff --git a/tests/test_ci_policy.py b/tests/test_ci_policy.py index 67ecfe4..d2d39c0 100644 --- a/tests/test_ci_policy.py +++ b/tests/test_ci_policy.py @@ -90,6 +90,18 @@ def test_ci_repository_coverage_floors_match_quality_policy(): assert f"--min-line {policy['minimums']['repository_statement']:g}" in workflow assert f"--min-branch {policy['minimums']['repository_branch']:g}" in workflow + assert f"--min-changed {policy['minimums']['changed_line']:g}" in workflow + + +def test_ci_enforces_owned_duration_budgets_and_ten_minute_test_lane(): + workflow = WORKFLOW.read_text(encoding="utf-8") + test_job = workflow.split(" test:\n", maxsplit=1)[1].split( + " quality-depth:\n", maxsplit=1, + )[0] + + assert "timeout-minutes: 10" in test_job + assert '--evidence "test-results/test-trend-py${{ matrix.python-version }}.json"' in workflow + assert "--evidence quality-evidence/quality-trend-py3.12.json" in workflow def test_ci_retains_normalized_test_and_quality_trends_for_ninety_days(): diff --git a/tests/test_coverage_thresholds.py b/tests/test_coverage_thresholds.py index b39feff..cb640d7 100644 --- a/tests/test_coverage_thresholds.py +++ b/tests/test_coverage_thresholds.py @@ -161,10 +161,16 @@ def test_changed_line_measurement_handles_missing_fields_and_multiple_files(): def test_changed_line_gate_accepts_exact_floor(): - changed = {"obliteratus/example.py": {1, 2, 3, 4}} + report = _report() + report["files"]["obliteratus/example.py"]["executed_lines"] = list(range(1, 20)) + report["files"]["obliteratus/example.py"]["missing_lines"] = [20] + changed = {"obliteratus/example.py": set(range(1, 21))} assert MODULE.validate_changed_coverage( - _report(), changed, minimum=75.0, + report, changed, minimum=95.0, ) == [] + assert MODULE.validate_changed_coverage( + report, changed, minimum=95.01, + ) == ["changed-line coverage 95.00% (19/20) is below the 95.01% floor"] def _module_report( diff --git a/tests/test_quality_policy.py b/tests/test_quality_policy.py index c91299a..47418e8 100644 --- a/tests/test_quality_policy.py +++ b/tests/test_quality_policy.py @@ -4,7 +4,11 @@ from __future__ import annotations from copy import deepcopy from datetime import date +import json from pathlib import Path +import sys + +import pytest from scripts import check_quality_policy as quality @@ -54,6 +58,29 @@ def _policy(): "retention_days": 90, "flake_window_days": 30, "maximum_quarantine_days": 30, + "duration_budgets": { + "mandatory_cpu": { + "owner": "@maintainers", + "max_suite_seconds_by_python": { + "3.10": 240.0, + "3.11": 240.0, + "3.12": 240.0, + }, + "max_testcase_seconds": 15.0, + "max_marker_seconds": { + "cpu": 120.0, + "integration": 60.0, + "unmarked": 120.0, + }, + "maximum_owner_days": 90, + "owned_slow_tests": [], + }, + "repeat_gate": { + "owner": "@maintainers", + "max_total_seconds": 180.0, + "max_pass_seconds": 75.0, + }, + }, "flake_history": [], "quarantines": [], }, @@ -109,6 +136,321 @@ def test_floor_regression_requires_structured_reviewed_exception(): assert quality.validate_policy(policy) == [] +def _test_trend(*, seconds: float = 2.0): + return { + "python": "3.12", + "tests": { + "total": 2, + "suite_duration_seconds": 5.0, + "marker_metadata_complete": True, + "missing_marker_nodeids": [], + "durations": [ + { + "nodeid": "tests.test_example::test_cpu", + "seconds": seconds, + "markers": ["cpu"], + }, + { + "nodeid": "tests.test_example::test_plain", + "seconds": 1.0, + "markers": ["unmarked"], + }, + ], + "marker_durations": { + "cpu": {"tests": 1, "duration_seconds": seconds}, + "unmarked": {"tests": 1, "duration_seconds": 1.0}, + }, + }, + } + + +def test_duration_policy_requires_owned_complete_positive_budgets(): + policy = _policy() + mandatory = policy["test_evidence"]["duration_budgets"]["mandatory_cpu"] + mandatory["owner"] = "maintainers" + mandatory["max_suite_seconds_by_python"].pop("3.10") + mandatory["max_marker_seconds"].pop("unmarked") + failures = quality.validate_policy(policy) + + assert "duration budget mandatory_cpu requires an @owner" in failures + assert ( + "duration budget mandatory_cpu must cover exactly Python 3.10, 3.11, and 3.12" + in failures + ) + assert "duration budget mandatory_cpu must own the unmarked layer" in failures + + +@pytest.mark.parametrize( + ("mutation", "message"), + [ + (lambda evidence: evidence.update(duration_budgets=None), + "test evidence duration_budgets must be an object"), + (lambda evidence: evidence["duration_budgets"].update(mandatory_cpu=None), + "duration budget mandatory_cpu must be an object"), + (lambda evidence: evidence["duration_budgets"]["mandatory_cpu"].update( + max_suite_seconds_by_python=None, + ), "max_suite_seconds_by_python must be an object"), + (lambda evidence: evidence["duration_budgets"]["mandatory_cpu"][ + "max_suite_seconds_by_python" + ].update({"3.12": 0}), "max_suite_seconds_by_python.3.12 must be positive"), + (lambda evidence: evidence["duration_budgets"]["mandatory_cpu"].update( + max_testcase_seconds=0, + ), "max_testcase_seconds must be positive"), + (lambda evidence: evidence["duration_budgets"]["mandatory_cpu"].update( + max_marker_seconds={}, + ), "max_marker_seconds must be a non-empty object"), + (lambda evidence: evidence["duration_budgets"]["mandatory_cpu"][ + "max_marker_seconds" + ].update(cpu=0), "invalid marker budget 'cpu'"), + (lambda evidence: evidence["duration_budgets"]["mandatory_cpu"].update( + maximum_owner_days=0, + ), "maximum_owner_days must be an integer from 1 to 365"), + (lambda evidence: evidence["duration_budgets"]["mandatory_cpu"].update( + owned_slow_tests=None, + ), "owned_slow_tests must be a list"), + (lambda evidence: evidence["duration_budgets"]["mandatory_cpu"].update( + owned_slow_tests=[None], + ), "owned slow test 0 must be an object"), + (lambda evidence: evidence["duration_budgets"]["mandatory_cpu"].update( + owned_slow_tests=[{"nodeid": ""}], + ), "owned slow test 0 requires a non-empty nodeid"), + (lambda evidence: evidence["duration_budgets"].update(repeat_gate=None), + "duration budget repeat_gate must be an object"), + (lambda evidence: evidence["duration_budgets"]["repeat_gate"].update( + owner="maintainers", max_total_seconds=0, max_pass_seconds=0, + ), "duration budget repeat_gate requires an @owner"), + (lambda evidence: evidence["duration_budgets"]["repeat_gate"].update( + max_total_seconds=10, max_pass_seconds=11, + ), "max_pass_seconds cannot exceed max_total_seconds"), + ], +) +def test_duration_policy_rejects_malformed_budget_shapes(mutation, message): + evidence = deepcopy(_policy()["test_evidence"]) + mutation(evidence) + + assert any( + message in failure + for failure in quality._validate_duration_policy(evidence, today=date(2026, 8, 15)) + ) + + +def _slow_owner(**overrides): + owner = { + "nodeid": "tests.test_example::test_slow", + "owner": "@runtime-maintainers", + "reason": "Exercises an installed offline artifact.", + "issue": "https://github.com/elder-plinius/OBLITERATUS/issues/999", + "max_seconds": 20.0, + "opened": "2026-08-15", + "expires": "2026-09-15", + } + owner.update(overrides) + return owner + + +@pytest.mark.parametrize( + ("owners", "today", "message"), + [ + ([_slow_owner(), _slow_owner()], date(2026, 8, 15), "duplicate owned slow test"), + ([_slow_owner(owner="maintainers", reason="", issue="https://example.com/1")], + date(2026, 8, 15), "requires an @owner"), + ([_slow_owner(max_seconds=0)], date(2026, 8, 15), "max_seconds must be positive"), + ([_slow_owner(max_seconds=15)], date(2026, 8, 15), + "max_seconds must exceed the default testcase budget"), + ([_slow_owner(opened="2026-08-16")], date(2026, 8, 15), "cannot open in the future"), + ([_slow_owner(expires="2026-08-15")], date(2026, 8, 15), + "must expire after it opens"), + ([_slow_owner(expires="2026-12-01")], date(2026, 8, 15), + "exceeds the 90-day review window"), + ([_slow_owner(opened="2026-05-01", expires="2026-05-02")], date(2026, 8, 15), + "expired on 2026-05-02"), + ], +) +def test_slow_test_ownership_is_unique_complete_and_time_bounded(owners, today, message): + evidence = deepcopy(_policy()["test_evidence"]) + evidence["duration_budgets"]["mandatory_cpu"]["owned_slow_tests"] = owners + + assert any( + message in failure + for failure in quality._validate_duration_policy(evidence, today=today) + ) + + +def test_duration_evidence_enforces_suite_marker_and_unowned_test_budgets(): + policy = _policy() + trend = _test_trend(seconds=16.0) + trend["tests"]["suite_duration_seconds"] = 241.0 + trend["tests"]["marker_durations"]["cpu"]["duration_seconds"] = 16.0 + policy["test_evidence"]["duration_budgets"]["mandatory_cpu"][ + "max_marker_seconds" + ]["cpu"] = 15.0 + + failures = quality.validate_duration_evidence(trend, policy) + assert "test trend Python 3.12 suite duration 241.000s exceeds 240.000s" in failures + assert ( + "test tests.test_example::test_cpu took 16.000s above the 15.000s default " + "and has no owned slow-test budget" + ) in failures + assert "test marker cpu duration 16.000s exceeds 15.000s" in failures + + +def test_duration_evidence_fails_closed_without_junit_marker_metadata(): + trend = _test_trend() + trend["tests"]["marker_metadata_complete"] = False + trend["tests"]["missing_marker_nodeids"] = ["tests.test_example::test_plain"] + + assert quality.validate_duration_evidence(trend, _policy()) == [ + "test trend duration marker metadata is incomplete for 1 testcase(s)", + ] + + +@pytest.mark.parametrize( + ("trend", "message"), + [ + ({"tests": []}, "test trend tests must be an object"), + ({"python": "3.13", "tests": _test_trend()["tests"]}, + "unsupported Python version '3.13'"), + ({"python": "3.12", "tests": { + **_test_trend()["tests"], "durations": [], "total": 0, + "marker_durations": {}, + }}, "durations must be a non-empty list"), + ({"python": "3.12", "tests": { + **_test_trend()["tests"], "missing_marker_nodeids": [None], + }}, "missing_marker_nodeids must be a string list"), + ({"python": "3.12", "tests": { + **_test_trend()["tests"], "durations": [None], "total": 1, + "marker_durations": {}, + }}, "test duration 0 must be an object"), + ({"python": "3.12", "tests": { + **_test_trend()["tests"], + "durations": [{"nodeid": "", "seconds": 0, "markers": ["cpu"]}], + "total": 1, "marker_durations": {}, + }}, "test duration 0 requires a non-empty nodeid"), + ({"python": "3.12", "tests": { + **_test_trend()["tests"], + "durations": [ + {"nodeid": "duplicate", "seconds": 0, "markers": ["cpu"]}, + {"nodeid": "duplicate", "seconds": 0, "markers": ["cpu"]}, + ], + "marker_durations": {"cpu": {"tests": 2, "duration_seconds": 0.0}}, + }}, "test trend repeats duration for duplicate"), + ({"python": "3.12", "tests": { + **_test_trend()["tests"], + "durations": [{"nodeid": "bad", "seconds": "bad", "markers": []}], + "total": 1, "marker_durations": {}, + }}, "test duration 0 seconds must be a non-negative finite number"), + ({"python": "3.12", "tests": { + **_test_trend()["tests"], "total": 3, + }}, "test trend total must equal the number of duration records"), + ({"python": "3.12", "tests": { + **_test_trend()["tests"], "marker_durations": [], + }}, "test trend marker_durations must be an object"), + ({"python": "3.12", "tests": { + **_test_trend()["tests"], + "durations": [{"nodeid": "new", "seconds": 1, "markers": ["new-layer"]}], + "total": 1, + "marker_durations": {"new-layer": {"tests": 1, "duration_seconds": 1.0}}, + }}, "test marker new-layer has no duration budget"), + ({"python": "3.12", "tests": { + **_test_trend()["tests"], + "marker_durations": { + **_test_trend()["tests"]["marker_durations"], + "extra": {"tests": 1, "duration_seconds": 1.0}, + }, + }}, "test marker extra has summary without duration records"), + ({"repeat": []}, "test trend repeat must be an object"), + ({"repeat": {"total_duration_seconds": 0, "passes": []}}, + "repeat gate passes must be a non-empty list"), + ({"repeat": {"total_duration_seconds": 0, "passes": [None]}}, + "repeat gate pass 1 must be an object"), + ({}, "test trend contains neither tests nor repeat duration evidence"), + ], +) +def test_duration_evidence_rejects_malformed_or_unowned_records(trend, message): + assert any( + message in failure + for failure in quality.validate_duration_evidence(trend, _policy()) + ) + + +def test_duration_evidence_rejects_boolean_duration(): + trend = _test_trend() + trend["tests"]["durations"][0]["seconds"] = False + + failures = quality.validate_duration_evidence(trend, _policy()) + assert "test duration 0 seconds must be a non-negative finite number" in failures + + +@pytest.mark.parametrize( + ("evidence_text", "message"), + [ + ("not-json", "cannot read test trend evidence"), + ("[]", "test trend evidence root must be an object"), + (json.dumps({}), "test trend contains neither tests nor repeat duration evidence"), + ], +) +def test_cli_rejects_invalid_duration_evidence( + tmp_path, monkeypatch, capsys, evidence_text, message, +): + policy_path = tmp_path / "policy.json" + evidence_path = tmp_path / "trend.json" + policy_path.write_text(json.dumps(_policy())) + evidence_path.write_text(evidence_text) + monkeypatch.setattr( + sys, + "argv", + [ + "check_quality_policy.py", + "--policy", + str(policy_path), + "--evidence", + str(evidence_path), + ], + ) + + assert quality.main() == 1 + assert message in capsys.readouterr().out + + +def test_owned_slow_test_is_bounded_and_time_limited(): + policy = _policy() + mandatory = policy["test_evidence"]["duration_budgets"]["mandatory_cpu"] + mandatory["owned_slow_tests"] = [{ + "nodeid": "tests.test_example::test_cpu", + "owner": "@runtime-maintainers", + "reason": "Exercises the installed offline model vertical slice.", + "issue": "https://github.com/elder-plinius/OBLITERATUS/issues/999", + "max_seconds": 20.0, + "opened": "2026-08-15", + "expires": "2026-09-15", + }] + assert quality.validate_policy(policy, today=date(2026, 8, 15)) == [] + assert quality.validate_duration_evidence(_test_trend(seconds=19.0), policy) == [] + + failures = quality.validate_duration_evidence(_test_trend(seconds=21.0), policy) + assert failures == [ + "owned slow test tests.test_example::test_cpu took 21.000s above its 20.000s budget", + ] + + +def test_repeat_duration_evidence_enforces_total_and_pass_budgets(): + trend = { + "repeat": { + "total_duration_seconds": 181.0, + "passes": [ + {"duration_seconds": 74.0}, + {"duration_seconds": 76.0}, + {"duration_seconds": 20.0}, + ], + }, + } + + assert quality.validate_duration_evidence(trend, _policy()) == [ + "repeat gate total duration 181.000s exceeds 180.000s", + "repeat gate pass 2 duration 76.000s exceeds 75.000s", + ] + + def test_exclusions_require_unique_traceable_environment_boundaries(): policy = _policy() duplicate = deepcopy(policy["mature_cpu_scope"]["exclusions"][0]) diff --git a/tests/test_test_evidence.py b/tests/test_test_evidence.py index b54cf86..39299bf 100644 --- a/tests/test_test_evidence.py +++ b/tests/test_test_evidence.py @@ -48,11 +48,15 @@ def test_coverage_snapshot_records_repository_and_risk_module_metrics(): def test_junit_snapshot_records_failures_skips_and_slowest_tests(tmp_path): junit = tmp_path / "junit.xml" junit.write_text( - '' - '' + '' + '' + '' + '' '' + '' '' '' + '' '', ) @@ -62,12 +66,101 @@ def test_junit_snapshot_records_failures_skips_and_slowest_tests(tmp_path): assert snapshot["failures"] == ["tests.test_a::test_fail"] assert snapshot["skipped"] == ["tests.test_b::test_skip"] assert snapshot["duration_seconds"] == 0.6 + assert snapshot["suite_duration_seconds"] == 0.9 + assert snapshot["marker_durations"] == { + "cpu": {"tests": 2, "duration_seconds": 0.4}, + "integration": {"tests": 1, "duration_seconds": 0.3}, + "unmarked": {"tests": 1, "duration_seconds": 0.2}, + } + assert len(snapshot["durations"]) == 3 + assert snapshot["marker_metadata_complete"] is True + assert snapshot["missing_marker_nodeids"] == [] assert snapshot["slowest"][0] == { "nodeid": "tests.test_a::test_fail", "seconds": 0.3, + "markers": ["cpu", "integration"], } +@pytest.mark.parametrize("value", ["nan", "inf", "-1", "invalid"]) +def test_junit_snapshot_rejects_invalid_testcase_durations(tmp_path, value): + junit = tmp_path / "junit.xml" + junit.write_text( + '' + f'' + '', + ) + + with pytest.raises(ValueError, match="invalid time"): + write_test_evidence.junit_snapshot(junit) + + +def test_junit_snapshot_rejects_invalid_suite_duration(tmp_path): + junit = tmp_path / "junit.xml" + junit.write_text( + '' + '' + '', + ) + + with pytest.raises(ValueError, match="testsuite has invalid time"): + write_test_evidence.junit_snapshot(junit) + + +def test_junit_snapshot_exposes_missing_marker_metadata(tmp_path): + junit = tmp_path / "junit.xml" + junit.write_text( + '' + '' + '', + ) + + snapshot = write_test_evidence.junit_snapshot(junit) + assert snapshot["marker_metadata_complete"] is False + assert snapshot["missing_marker_nodeids"] == ["tests.test_a::test_plain"] + + +def test_junit_snapshot_rejects_duplicate_marker_properties(tmp_path): + junit = tmp_path / "junit.xml" + junit.write_text( + '' + '' + '' + '' + '' + '', + ) + + with pytest.raises(ValueError, match="repeats duration_markers"): + write_test_evidence.junit_snapshot(junit) + + +def test_junit_snapshot_normalizes_empty_marker_and_missing_suite_time(tmp_path): + junit = tmp_path / "junit.xml" + junit.write_text( + '' + '' + '' + '', + ) + + snapshot = write_test_evidence.junit_snapshot(junit) + assert snapshot["suite_duration_seconds"] == 0.1 + assert snapshot["durations"][0]["markers"] == ["unmarked"] + + +def test_junit_snapshot_rejects_non_numeric_suite_duration(tmp_path): + junit = tmp_path / "junit.xml" + junit.write_text( + '' + '' + '', + ) + + with pytest.raises(ValueError, match="testsuite has invalid time"): + write_test_evidence.junit_snapshot(junit) + + def test_evidence_records_base_deltas_repeat_and_mutation(): evidence = write_test_evidence.build_evidence( head_sha="b" * 40,