mirror of
https://github.com/elder-plinius/OBLITERATUS.git
synced 2026-08-30 14:40:38 +02:00
244 lines
9.9 KiB
Python
244 lines
9.9 KiB
Python
"""Repository contracts for immutable CI execution dependencies."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).parents[1]
|
|
WORKFLOW = ROOT / ".github" / "workflows" / "ci.yml"
|
|
MANIFEST = ROOT / "ci" / "digests.txt"
|
|
PR_POLICY = ROOT / "ci" / "pr-test-policy.json"
|
|
ACTION_REF = re.compile(
|
|
r"^\s*uses:\s*([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)@([0-9a-f]{40})\s+#\s+(\S+)\s*$",
|
|
)
|
|
|
|
|
|
def _manifest_entries() -> dict[tuple[str, str], tuple[str, str]]:
|
|
entries: dict[tuple[str, str], tuple[str, str]] = {}
|
|
for line in MANIFEST.read_text(encoding="utf-8").splitlines():
|
|
if not line or line.startswith("#"):
|
|
continue
|
|
kind, name, pin, version, _date, _rationale = line.split(maxsplit=5)
|
|
entries[(kind, name)] = (pin, version)
|
|
return entries
|
|
|
|
|
|
def test_every_external_action_is_sha_pinned_and_manifested():
|
|
entries = _manifest_entries()
|
|
uses_lines = [
|
|
line for line in WORKFLOW.read_text(encoding="utf-8").splitlines()
|
|
if "uses:" in line and "uses: ./" not in line
|
|
]
|
|
|
|
assert uses_lines
|
|
for line in uses_lines:
|
|
match = ACTION_REF.match(line)
|
|
assert match is not None, f"external action is not SHA-pinned with a version comment: {line}"
|
|
name, pin, version = match.groups()
|
|
assert entries[("action", name)] == (pin, version)
|
|
|
|
|
|
def test_tested_source_snapshot_is_created_once_attested_and_reused():
|
|
workflow = WORKFLOW.read_text(encoding="utf-8")
|
|
package = workflow.split(" package:\n", maxsplit=1)[1].split(
|
|
" lint:\n", maxsplit=1,
|
|
)[0]
|
|
supply_chain = workflow.split(" supply-chain:\n", maxsplit=1)[1]
|
|
|
|
assert "artifact-metadata: write" in package
|
|
assert "attestations: write" in package
|
|
assert "id-token: write" in package
|
|
assert "package-evidence/SHA256SUMS" in package
|
|
assert "package-evidence/obliteratus.cdx.json" in package
|
|
assert 'git archive \\' in package
|
|
assert '--format=zip' in package
|
|
assert 'subject-path: release/*.zip' in package
|
|
assert "python -m build" not in package
|
|
assert "*.whl" not in package
|
|
assert "*.tar.gz" not in package
|
|
assert package.count("uses: actions/attest@") == 2
|
|
assert "subject-checksums: package-evidence/SHA256SUMS" in package
|
|
assert "sbom-path: package-evidence/obliteratus.cdx.json" in package
|
|
|
|
assert "needs: package" in supply_chain
|
|
assert "uses: actions/download-artifact@" in supply_chain
|
|
assert "sha256sum -c package-evidence/SHA256SUMS" in supply_chain
|
|
assert "tested-source-snapshot-py3.12" in supply_chain
|
|
assert "find release -maxdepth 1 -type f -name '*.zip'" in supply_chain
|
|
assert "-m build" not in supply_chain
|
|
|
|
|
|
def test_actionlint_version_and_checksum_match_manifest():
|
|
workflow = WORKFLOW.read_text(encoding="utf-8")
|
|
pin, version = _manifest_entries()[("tool", "rhysd/actionlint")]
|
|
|
|
assert f'ACTIONLINT_VERSION: "{version.removeprefix("v")}"' in workflow
|
|
assert f'ACTIONLINT_SHA256: "{pin.removeprefix("sha256:")}"' in workflow
|
|
|
|
|
|
def test_ci_ruff_f_gate_covers_the_entire_scripts_tree():
|
|
workflow = WORKFLOW.read_text(encoding="utf-8")
|
|
ruff_step = workflow.split(" - name: Enforce Ruff F gate\n", maxsplit=1)[1].split(
|
|
" - name:", maxsplit=1,
|
|
)[0]
|
|
|
|
assert "python -m ruff check --select F app.py obliteratus tests scripts" in ruff_step
|
|
|
|
|
|
def test_uv_and_gitleaks_pins_match_manifest():
|
|
workflow = WORKFLOW.read_text(encoding="utf-8")
|
|
entries = _manifest_entries()
|
|
uv_pin, uv_version = entries[("tool", "astral-sh/uv")]
|
|
gitleaks_pin, gitleaks_version = entries[("tool", "gitleaks/gitleaks")]
|
|
|
|
assert uv_pin == f"pypi:{uv_version.removeprefix('v')}"
|
|
assert f'UV_VERSION: "{uv_version.removeprefix("v")}"' in workflow
|
|
assert f'GITLEAKS_VERSION: "{gitleaks_version.removeprefix("v")}"' in workflow
|
|
assert f'GITLEAKS_SHA256: "{gitleaks_pin.removeprefix("sha256:")}"' in workflow
|
|
|
|
|
|
def test_ci_requires_the_committed_lock_and_strict_policy_gate():
|
|
workflow = WORKFLOW.read_text(encoding="utf-8")
|
|
|
|
assert "uv lock --check" in workflow
|
|
assert "uv sync --locked" in workflow
|
|
assert "scripts/check_supply_chain_policy.py audit" in workflow
|
|
assert "scripts/check_supply_chain_policy.py secrets" in workflow
|
|
assert "scripts/check_supply_chain_policy.py licenses" in workflow
|
|
supply_chain_job = workflow.split(" supply-chain:\n", maxsplit=1)[1]
|
|
assert "|| true" not in supply_chain_job
|
|
|
|
|
|
def test_ci_enforces_exact_base_module_regression_and_risk_mapping():
|
|
workflow = WORKFLOW.read_text(encoding="utf-8")
|
|
|
|
assert "Generate exact-base coverage for module regression comparison" in workflow
|
|
assert 'git worktree add --detach "$BASE_WORKTREE" "$COVERAGE_BASE"' in workflow
|
|
assert "--touched-module-no-regression" in workflow
|
|
assert "--base-report test-results/base-coverage-py3.12.json" in workflow
|
|
assert "scripts/check_test_risk_map.py" in workflow
|
|
|
|
|
|
def test_ci_repository_coverage_floors_match_quality_policy():
|
|
workflow = WORKFLOW.read_text(encoding="utf-8")
|
|
policy = json.loads(
|
|
(ROOT / "ci" / "test-quality-policy.json").read_text(encoding="utf-8"),
|
|
)
|
|
|
|
assert f"--min-line {policy['minimums']['repository_statement']:g}" in workflow
|
|
assert f"--min-branch {policy['minimums']['repository_branch']:g}" in workflow
|
|
assert f"--min-changed {policy['minimums']['changed_line']:g}" in workflow
|
|
|
|
|
|
def test_pull_request_gate_is_fast_risk_mapped_and_uses_shared_floor():
|
|
workflow = WORKFLOW.read_text(encoding="utf-8")
|
|
policy = json.loads(PR_POLICY.read_text(encoding="utf-8"))
|
|
quality = json.loads(
|
|
(ROOT / "ci" / "test-quality-policy.json").read_text(encoding="utf-8"),
|
|
)
|
|
pr_core = workflow.split(" pr-core:\n", maxsplit=1)[1].split(
|
|
" test:\n", maxsplit=1,
|
|
)[0]
|
|
|
|
assert "name: Pull request core" in pr_core
|
|
assert "github.event_name == 'pull_request'" in pr_core
|
|
assert 'python-version: "3.12"' in pr_core
|
|
assert "scripts/select_pr_tests.py" in pr_core
|
|
assert '"${selected_tests[@]}"' in pr_core
|
|
assert "--cov=app" in pr_core
|
|
assert "--min-line 0" in pr_core
|
|
assert "--min-branch 0" in pr_core
|
|
assert f"--min-changed {policy['changed_line_coverage_floor']:g}" in pr_core
|
|
assert policy["changed_line_coverage_floor"] == 50.0
|
|
assert quality["minimums"]["changed_line"] == 50.0
|
|
for test in policy["always_tests"] + policy["infrastructure_tests"]:
|
|
assert (ROOT / test).is_file(), test
|
|
assert "tests/conditional/" in policy["excluded_test_prefixes"]
|
|
|
|
|
|
def test_arm64_preflight_proves_portability_without_claiming_jetson_cuda():
|
|
workflow = WORKFLOW.read_text(encoding="utf-8")
|
|
arm = workflow.split(" arm64-preflight:\n", maxsplit=1)[1].split(
|
|
" pr-core:\n",
|
|
maxsplit=1,
|
|
)[0]
|
|
|
|
assert "runs-on: ubuntu-24.04-arm" in arm
|
|
assert "github.event_name == 'pull_request'" in arm
|
|
assert "github.event_name == 'push' && github.ref == 'refs/heads/main'" in arm
|
|
assert "-m build --wheel" in arm
|
|
assert "import obliteratus" in arm
|
|
assert "-m obliteratus --help" in arm
|
|
assert "tests/test_device_boundaries.py" in arm
|
|
assert "tests/test_jetson_support_tooling.py" in arm
|
|
assert "jetson-runtime" not in arm
|
|
|
|
|
|
def test_release_depth_jobs_only_run_for_tags_or_manual_validation():
|
|
workflow = WORKFLOW.read_text(encoding="utf-8")
|
|
release_condition = (
|
|
"if: github.event_name == 'workflow_dispatch' || "
|
|
"startsWith(github.ref, 'refs/tags/v')"
|
|
)
|
|
|
|
assert ' - "v*"' in workflow
|
|
assert " workflow_dispatch:" in workflow
|
|
for start, end in (
|
|
(" package:\n", " lint:\n"),
|
|
(" test:\n", " checkpoint-windows:\n"),
|
|
(" checkpoint-windows:\n", " quality-depth:\n"),
|
|
(" quality-depth:\n", " supply-chain:\n"),
|
|
):
|
|
job = workflow.split(start, maxsplit=1)[1].split(end, maxsplit=1)[0]
|
|
assert release_condition in job
|
|
supply_chain = workflow.split(" supply-chain:\n", maxsplit=1)[1]
|
|
assert release_condition in supply_chain
|
|
|
|
|
|
def test_ci_enforces_owned_duration_budgets_and_ten_minute_test_lane():
|
|
workflow = WORKFLOW.read_text(encoding="utf-8")
|
|
test_job = workflow.split(" test:\n", maxsplit=1)[1].split(
|
|
" quality-depth:\n", maxsplit=1,
|
|
)[0]
|
|
|
|
assert "timeout-minutes: 10" in test_job
|
|
assert '--evidence "test-results/test-trend-py${{ matrix.python-version }}.json"' in workflow
|
|
assert "--evidence quality-evidence/quality-trend-py3.12.json" in workflow
|
|
|
|
|
|
def test_ci_retains_normalized_test_and_quality_trends_for_ninety_days():
|
|
workflow = WORKFLOW.read_text(encoding="utf-8")
|
|
|
|
assert "test-trend-py${{ matrix.python-version }}.json" in workflow
|
|
assert "quality-trend-py3.12.json" in workflow
|
|
assert workflow.count("retention-days: 90") >= 2
|
|
|
|
|
|
def test_ci_evidence_names_the_pull_request_head_not_the_synthetic_merge_commit():
|
|
workflow = WORKFLOW.read_text(encoding="utf-8")
|
|
|
|
assert "CANDIDATE_SHA: ${{ github.event.pull_request.head.sha || github.sha }}" in workflow
|
|
assert workflow.count('--head-sha "$CANDIDATE_SHA"') == 2
|
|
assert '--head-sha "$GITHUB_SHA"' not in workflow
|
|
|
|
|
|
def test_ci_runs_checkpoint_portability_contracts_on_windows():
|
|
workflow = WORKFLOW.read_text(encoding="utf-8")
|
|
checkpoint_job = workflow.split(" checkpoint-windows:\n", maxsplit=1)[1].split(
|
|
" quality-depth:\n",
|
|
maxsplit=1,
|
|
)[0]
|
|
|
|
assert "runs-on: windows-latest" in checkpoint_job
|
|
assert 'python-version: "3.12"' in checkpoint_job
|
|
assert "uv sync --locked --no-default-groups --extra dev --no-editable" in checkpoint_job
|
|
assert "tests/test_checkpoint_atomicity.py" in checkpoint_job
|
|
assert "tests/test_persistence_contracts.py" in checkpoint_job
|
|
assert "tests/test_persistence_pipeline.py" in checkpoint_job
|
|
assert "--no-cov" in checkpoint_job
|
|
assert "checkpoint-evidence-windows-py3.12" in checkpoint_job
|
|
assert "if-no-files-found: error" in checkpoint_job
|