ci: add reproducible supply-chain gates

This commit is contained in:
Joseph Magly
2026-08-14 12:27:15 -04:00
parent eec8a615f7
commit e7b658c0c8
13 changed files with 5393 additions and 44 deletions
+5 -4
View File
@@ -19,14 +19,15 @@ delivery:
allowedMergeMethods: ["rebase"]
validation:
quick:
- "python -m ruff check --select F obliteratus tests scripts/check_coverage_thresholds.py scripts/gemma4_12b_recursive_loop.py"
- "python -m ruff check --select F app.py obliteratus tests scripts/check_coverage_thresholds.py scripts/check_supply_chain_policy.py scripts/gemma4_12b_recursive_loop.py"
- "uv lock --check"
full:
- "python -m pytest"
- "python -m build --sdist --wheel"
- "python -c 'import obliteratus; print(obliteratus.__version__)'"
- "python -m obliteratus --help"
documentation:
- "python -m ruff check --select F obliteratus tests scripts/check_coverage_thresholds.py scripts/gemma4_12b_recursive_loop.py"
- "python -m ruff check --select F app.py obliteratus tests scripts/check_coverage_thresholds.py scripts/check_supply_chain_policy.py scripts/gemma4_12b_recursive_loop.py"
researchIntegrity:
- "python -m pytest tests/test_advanced_metrics.py tests/test_breakthrough_modules.py tests/test_community.py"
riskSurfaces:
@@ -47,9 +48,9 @@ riskSurfaces:
concerns: ["cli-contract", "ui-contract", "platform-compatibility", "notebook-reproducibility"]
requiredChecks: ["python -m pytest tests/test_cli.py tests/test_module_imports.py", "python -m obliteratus --help"]
- id: "ci-supply-chain"
paths: [".github/workflows/**", "pyproject.toml"]
paths: [".github/workflows/**", "ci/**", "docs/SUPPLY_CHAIN_POLICY.md", "pyproject.toml", "scripts/check_supply_chain_policy.py", "uv.lock"]
concerns: ["workflow-permissions", "dependency-pinning", "untrusted-pull-request-code"]
requiredChecks: ["python -m ruff check --select F obliteratus tests scripts/check_coverage_thresholds.py scripts/gemma4_12b_recursive_loop.py", "python -m build --sdist --wheel"]
requiredChecks: ["uv lock --check", "python -m pytest tests/test_ci_policy.py tests/test_supply_chain_policy.py", "python -m build --sdist --wheel"]
research:
corpusPaths: ["obliteratus/prompts.py", "community_results/**"]
evidencePaths: ["paper/**", "docs/**", "community_results/**"]
+165 -22
View File
@@ -16,12 +16,15 @@ concurrency:
env:
PIP_DISABLE_PIP_VERSION_CHECK: "1"
PIP_NO_INPUT: "1"
UV_VERSION: "0.12.4"
jobs:
package:
name: Package
runs-on: ubuntu-latest
timeout-minutes: 15
env:
BUILD_TOOLS: /tmp/obliteratus-ci-tools
steps:
- name: Check out repository
@@ -34,13 +37,16 @@ jobs:
cache: pip
cache-dependency-path: |
pyproject.toml
requirements*.txt
uv.lock
- name: Install build tooling
run: python -m pip install "build==1.2.2.post1" "twine==7.0.0"
run: |
python -m pip install "uv==${UV_VERSION}"
UV_PROJECT_ENVIRONMENT="$BUILD_TOOLS" uv sync --locked --only-group ci
- name: Build source and wheel distributions
run: python -m build --sdist --wheel
run: >-
"$BUILD_TOOLS/bin/python" -m build --sdist --wheel
- name: Verify wheel contents and entry point
run: |
@@ -78,7 +84,7 @@ jobs:
- name: Validate distribution metadata
run: |
mkdir -p package-evidence
python -m twine check dist/* | tee package-evidence/twine-check.txt
"$BUILD_TOOLS/bin/python" -m twine check dist/* | tee package-evidence/twine-check.txt
sha256sum dist/* | tee package-evidence/SHA256SUMS
- name: Verify installed wheel contract
@@ -168,10 +174,12 @@ jobs:
with:
python-version: "3.12"
cache: pip
cache-dependency-path: pyproject.toml
cache-dependency-path: |
pyproject.toml
uv.lock
- name: Install Ruff
run: python -m pip install "ruff==0.8.6"
run: python -m pip install "ruff==0.16.2"
- name: Install actionlint with checksum verification
env:
@@ -190,15 +198,17 @@ jobs:
- name: Enforce Ruff F gate
run: >-
python -m ruff check --select F obliteratus tests
python -m ruff check --select F app.py obliteratus tests
scripts/check_coverage_thresholds.py
scripts/check_supply_chain_policy.py
scripts/gemma4_12b_recursive_loop.py
- name: Report E501 legacy baseline
if: always()
run: >-
python -m ruff check --select E501 --statistics obliteratus tests
python -m ruff check --select E501 --statistics app.py obliteratus tests
scripts/check_coverage_thresholds.py
scripts/check_supply_chain_policy.py
scripts/gemma4_12b_recursive_loop.py || true
test:
@@ -219,6 +229,7 @@ jobs:
HF_HUB_OFFLINE: "1"
TOKENIZERS_PARALLELISM: "false"
TRANSFORMERS_OFFLINE: "1"
TEST_ENV: /tmp/obliteratus-test-env
steps:
- name: Check out repository
@@ -231,22 +242,17 @@ jobs:
cache: pip
cache-dependency-path: |
pyproject.toml
requirements*.txt
uv.lock
- name: Install CPU PyTorch
run: python -m pip install --index-url https://download.pytorch.org/whl/cpu "torch>=2.0"
- name: Install package and test tools
- name: Install locked package and test tools
run: |
python - <<'PY' > /tmp/torch-cpu-constraint.txt
import torch
print(f"torch=={torch.__version__}")
PY
python -m pip install -e ".[dev]" -c /tmp/torch-cpu-constraint.txt
python -m pip install "uv==${UV_VERSION}"
UV_PROJECT_ENVIRONMENT="$TEST_ENV" \
uv sync --locked --no-default-groups --extra dev --no-editable
- name: Smoke import and CLI
run: |
python - <<'PY'
"$TEST_ENV/bin/python" - <<'PY'
import obliteratus
version = getattr(obliteratus, "__version__", None)
@@ -255,12 +261,12 @@ jobs:
else:
print("obliteratus import: ok")
PY
python -m obliteratus --help
"$TEST_ENV/bin/python" -m obliteratus --help
- name: Run tests with coverage
run: |
mkdir -p test-results
python -m pytest \
"$TEST_ENV/bin/python" -m pytest \
-m "not slow and not gpu and not mps and not mlx and not network and not download and not remote" \
--cov-branch \
--cov-fail-under=0 \
@@ -270,7 +276,7 @@ jobs:
- name: Enforce line and branch coverage floors
run: >-
python scripts/check_coverage_thresholds.py
"$TEST_ENV/bin/python" scripts/check_coverage_thresholds.py
"test-results/coverage-py${{ matrix.python-version }}.json"
--min-line 49
--min-branch 36
@@ -283,3 +289,140 @@ jobs:
path: test-results/
if-no-files-found: error
retention-days: 14
supply-chain:
name: Supply chain
runs-on: ubuntu-latest
timeout-minutes: 30
env:
EVIDENCE: /tmp/supply-chain-evidence
GITLEAKS_SHA256: "551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb"
GITLEAKS_VERSION: "8.30.1"
RUNTIME_ENV: /tmp/obliteratus-runtime
SUPPLY_TOOLS: /tmp/obliteratus-supply-tools
steps:
- name: Check out repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Set up Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.12"
cache: pip
cache-dependency-path: |
pyproject.toml
uv.lock
- name: Install locked supply-chain tooling
run: |
python -m pip install "uv==${UV_VERSION}"
uv lock --check
UV_PROJECT_ENVIRONMENT="$RUNTIME_ENV" \
uv sync --locked --all-extras --no-default-groups --no-editable
UV_PROJECT_ENVIRONMENT="$SUPPLY_TOOLS" \
uv sync --locked --only-group ci
mkdir -p "$EVIDENCE"
cd "$RUNNER_TEMP"
"$RUNTIME_ENV/bin/python" -I - <<'PY' | tee "$EVIDENCE/spaces-import.txt"
import gradio
import app
assert type(app.demo).__name__ == "Blocks"
print(f"installed all-extras app import: {app.__file__}")
print(f"gradio: {gradio.__version__}")
PY
- name: Validate exception policy
run: >-
python scripts/check_supply_chain_policy.py policy
--policy ci/supply-chain-policy.json
- name: Scan checkout for secrets with redacted evidence
run: |
archive="$RUNNER_TEMP/gitleaks.tar.gz"
curl -fsSLo "$archive" \
"https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz"
echo "${GITLEAKS_SHA256} ${archive}" | sha256sum -c -
tar -xzf "$archive" -C "$RUNNER_TEMP" gitleaks
set +e
"$RUNNER_TEMP/gitleaks" dir "$GITHUB_WORKSPACE" \
--redact=100 \
--report-format json \
--report-path "$EVIDENCE/gitleaks.json" \
--no-banner \
--no-color
status=$?
set -e
echo "$status" > "$EVIDENCE/gitleaks.status"
- name: Collect vulnerability evidence for supported Python versions
run: |
for version in 3.10 3.11 3.12; do
set +e
uv --preview-features audit-command,json-output audit \
--locked \
--python-version "$version" \
--python-platform linux \
--output-format json > "$EVIDENCE/uv-audit-py${version}.json"
status=$?
set -e
echo "$status" > "$EVIDENCE/uv-audit-py${version}.status"
done
- name: Build wheel and bind CycloneDX SBOM
run: |
mkdir -p "$EVIDENCE/dist"
"$SUPPLY_TOOLS/bin/python" -m build --wheel --outdir "$EVIDENCE/dist"
uv --preview-features sbom-export export \
--locked \
--format cyclonedx1.5 \
--all-extras \
--no-dev \
--no-editable \
--output-file "$EVIDENCE/obliteratus.cdx.unbound.json"
mapfile -t wheels < <(find "$EVIDENCE/dist" -maxdepth 1 -type f -name '*.whl' -print)
if [ "${#wheels[@]}" -ne 1 ]; then
echo "expected exactly one wheel, found ${#wheels[@]}"
exit 1
fi
python scripts/check_supply_chain_policy.py sbom \
--input "$EVIDENCE/obliteratus.cdx.unbound.json" \
--wheel "${wheels[0]}" \
--output "$EVIDENCE/obliteratus.cdx.json"
sha256sum "${wheels[0]}" > "$EVIDENCE/distribution.SHA256SUM"
- name: Collect packaged-dependency license inventory
run: >-
"$SUPPLY_TOOLS/bin/pip-licenses"
--python "$RUNTIME_ENV/bin/python"
--format json
--output-file "$EVIDENCE/licenses.json"
- name: Enforce vulnerability, secret, and license policies
run: |
for version in 3.10 3.11 3.12; do
python scripts/check_supply_chain_policy.py audit \
--policy ci/supply-chain-policy.json \
--evidence "$EVIDENCE/uv-audit-py${version}.json" \
--scanner-status "$EVIDENCE/uv-audit-py${version}.status" \
--decision "$EVIDENCE/uv-audit-py${version}.decision.json"
done
python scripts/check_supply_chain_policy.py secrets \
--policy ci/supply-chain-policy.json \
--evidence "$EVIDENCE/gitleaks.json" \
--scanner-status "$EVIDENCE/gitleaks.status" \
--decision "$EVIDENCE/gitleaks.decision.json"
python scripts/check_supply_chain_policy.py licenses \
--policy ci/supply-chain-policy.json \
--evidence "$EVIDENCE/licenses.json" \
--decision "$EVIDENCE/licenses.decision.json"
- name: Upload supply-chain evidence
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: supply-chain-evidence-py3.12
path: ${{ runner.temp }}/supply-chain-evidence/
if-no-files-found: error
retention-days: 14
+8 -1
View File
@@ -42,7 +42,7 @@ accelerator, or remote-execution credentials.
Canonical required checks:
- `python -m ruff check --select F obliteratus tests scripts/check_coverage_thresholds.py scripts/gemma4_12b_recursive_loop.py`
- `python -m ruff check --select F app.py obliteratus tests scripts/check_coverage_thresholds.py scripts/check_supply_chain_policy.py scripts/gemma4_12b_recursive_loop.py`
- `python -m pytest` (includes the measured 49% coverage floor)
- CI additionally enforces the measured 36% branch-coverage floor from its
retained coverage JSON report.
@@ -55,6 +55,13 @@ in an independent environment outside the checkout, exercises both CLI entry
paths, and retains the distributions plus evidence. Immutable CI action/tool
pins are recorded in [ci/digests.txt](ci/digests.txt).
Python CI resolution is locked by `uv.lock`, including the official CPU-only
PyTorch source for Linux and Windows. The required Supply chain job scans all
supported Python versions for known vulnerabilities, scans the checkout for
secrets with fully redacted evidence, enforces the packaged-dependency license
allow list, and binds a CycloneDX SBOM to the built wheel. Exception and update
rules are documented in [docs/SUPPLY_CHAIN_POLICY.md](docs/SUPPLY_CHAIN_POLICY.md).
GPU, MPS, model-download, network, and remote-execution checks are conditional
release or risk-surface gates, not part of the default CPU job.
+12 -11
View File
@@ -892,7 +892,7 @@ def _get_vram_html() -> str:
reserved_html = (
f'<span style="color:#4a5568;">reserved: {mem.reserved_gb:.1f} GB</span>'
if mem.reserved_gb > 0
else f'<span style="color:#4a5568;">unified memory</span>'
else '<span style="color:#4a5568;">unified memory</span>'
)
return (
f'<div style="margin:6px auto 0;max-width:480px;">'
@@ -1020,7 +1020,7 @@ def _generate_analysis_figs(pipeline, model_label: str = "") -> list:
plot_cross_layer_heatmap,
plot_angular_drift,
)
import tempfile, os
import tempfile
analyzer = CrossLayerAlignmentAnalyzer()
result = analyzer.analyze(directions)
@@ -1836,7 +1836,6 @@ def obliterate(model_choice: str, method_choice: str,
releases it when the function returns.
"""
import os
import re
model_id = MODELS.get(model_choice, model_choice)
is_preset = model_choice in MODELS
@@ -2579,7 +2578,7 @@ def load_bench_into_chat(choice: str, progress=gr.Progress()):
with _lock:
if _state["status"] == "ready" and _state["model"] is not None:
yield (
f"**Ready!** Model already loaded — just type in the chat below.",
"**Ready!** Model already loaded — just type in the chat below.",
get_chat_header(),
)
return
@@ -2587,7 +2586,7 @@ def load_bench_into_chat(choice: str, progress=gr.Progress()):
checkpoint = _state.get("output_dir")
if checkpoint and Path(checkpoint).exists():
yield (
f"**Loading model** from saved checkpoint...",
"**Loading model** from saved checkpoint...",
"",
)
# If we have a checkpoint, attempt reload outside the lock
@@ -2609,7 +2608,7 @@ def load_bench_into_chat(choice: str, progress=gr.Progress()):
_state["tokenizer"] = tokenizer_loaded
_state["status"] = "ready"
yield (
f"**Loaded!** Model reloaded from checkpoint — ready to chat.",
"**Loaded!** Model reloaded from checkpoint — ready to chat.",
get_chat_header(),
)
return
@@ -3961,7 +3960,7 @@ _JS = """
}
"""
with gr.Blocks(theme=THEME, css=CSS, js=_JS, title="OBLITERATUS", fill_height=True) as demo:
with gr.Blocks(title="OBLITERATUS", fill_height=True) as demo:
gr.HTML("""
<div class="header-wrap">
@@ -4580,8 +4579,7 @@ Pre-configured benchmark configurations for common research questions.
gr.ChatInterface(
fn=chat_respond,
type="messages",
chatbot=gr.Chatbot(height="11vh", type="messages"),
chatbot=gr.Chatbot(height="11vh"),
additional_inputs=[system_prompt, temperature, top_p, max_tokens, repetition_penalty, context_length],
fill_height=True,
)
@@ -4630,13 +4628,13 @@ See exactly how abliteration changes model behavior on the same prompt.
with gr.Column():
ab_header_left = gr.Markdown("#### Original (Pre-Abliteration)")
ab_chatbot_left = gr.Chatbot(
height="20vh", type="messages",
height="20vh",
label="Original Model",
)
with gr.Column():
ab_header_right = gr.Markdown("#### Abliterated")
ab_chatbot_right = gr.Chatbot(
height="20vh", type="messages",
height="20vh",
label="Abliterated Model",
)
@@ -5240,8 +5238,11 @@ def launch(
share=share,
inbrowser=inbrowser,
auth=auth,
css=CSS,
max_threads=max_threads,
js=_JS,
quiet=quiet,
theme=THEME,
)
+2
View File
@@ -3,3 +3,5 @@ action actions/checkout 3d3c42e5aac5ba805825da76410c181273ba90b1
action actions/setup-python 5fda3b95a4ea91299a34e894583c3862153e4b97 v7.0.0 2026-08-14 baseline pin (#59)
action actions/upload-artifact 043fb46d1a93c77aae656e7c1c64a875d1fc6a0a v7.0.1 2026-08-14 test/package evidence (#64, #65)
tool rhysd/actionlint sha256:8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8 v1.7.12 2026-08-14 workflow syntax/policy gate (#65)
tool astral-sh/uv pypi:0.12.4 v0.12.4 2026-08-14 locked dependency and SBOM engine (#66)
tool gitleaks/gitleaks sha256:551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb v8.30.1 2026-08-14 redacted checkout secret scan (#66)
+53
View File
@@ -0,0 +1,53 @@
{
"schema_version": 1,
"vulnerability": {
"decision": "fail-all-severities",
"severity_handling": "OSV does not guarantee a normalized severity, so every reported vulnerability is blocking unless an unexpired exception matches it.",
"fail_severities": [
"unknown",
"low",
"medium",
"high",
"critical"
],
"max_fixed_suppression_days": 7,
"max_unfixed_suppression_days": 90,
"suppressions": []
},
"secret": {
"decision": "fail-all-findings",
"report_redaction_percent": 100,
"max_suppression_days": 30,
"suppressions": []
},
"license": {
"decision": "exact-allow-list",
"excluded_packages": [
"obliteratus"
],
"allowed_expressions": [
"Apache 2.0 License",
"Apache License 2.0",
"Apache Software License",
"Apache Software License; BSD License",
"Apache-2.0",
"Apache-2.0 AND Apache-2.0 WITH LLVM-exception AND BSD-2-Clause AND BSD-3-Clause AND BSL-1.0 AND MIT",
"Apache-2.0 AND CNRI-Python",
"Apache-2.0 AND MIT",
"Apache-2.0 OR BSD-2-Clause",
"BSD License",
"BSD-2-Clause",
"BSD-3-Clause",
"BSD-3-Clause AND 0BSD AND MIT AND Zlib AND CC0-1.0",
"ISC License (ISCL)",
"MIT",
"MIT License",
"MIT-CMU",
"MPL-2.0 AND (Apache-2.0 OR MIT)",
"MPL-2.0 AND MIT",
"Mozilla Public License 2.0 (MPL 2.0)",
"PSF-2.0",
"Python Software Foundation License"
]
}
}
+61
View File
@@ -0,0 +1,61 @@
# Supply-chain policy
The committed `uv.lock` is the reproducible dependency source for the supported
Python 3.103.12 test matrix. CI installs CPU-only PyTorch from PyTorch's
explicit package index and resolves every other package from PyPI. The lock
contains exact versions, source URLs, environment markers, and artifact hashes.
CI uses exact Python tool versions and checksum-pinned standalone binaries.
`ci/digests.txt` records executable and GitHub Action pins; build, test, and
license-tool transitive dependencies are covered by `uv.lock`.
## Required evidence
The Supply chain job retains these artifacts for 14 days:
- one redacted Gitleaks JSON report for the checkout;
- OSV audit JSON and scanner status for Python 3.10, 3.11, and 3.12 on Linux;
- a JSON license inventory for all packaged extras;
- a CycloneDX 1.5 SBOM bound to the built wheel by SHA-256;
- the policy decisions and built wheel used by those checks.
Every secret finding and every OSV vulnerability is blocking by default. OSV
does not provide a normalized severity for every advisory, so the policy treats
unknown, low, medium, high, and critical findings alike. This is stricter than
a high-only threshold and avoids silently passing advisories with missing
severity data.
License metadata must exactly match an expression in
`ci/supply-chain-policy.json`. OBLITERATUS itself is excluded from dependency
license evaluation because its AGPL license is the project license rather than
a third-party dependency decision.
## Exceptions
Exceptions live only in `ci/supply-chain-policy.json`; command-line ignores and
unconditional success conversion are forbidden.
- A vulnerability exception names an OSV/GHSA/CVE identifier, states a reason,
records `approved_on` and `expires`, and declares whether a fix is available.
Fixable findings may be excepted for at most 7 days; findings without a fix
may be excepted for at most 90 days.
- A secret exception names the redacted Gitleaks fingerprint, states a reason,
and records `approved_on` and `expires`. It may last at most 30 days.
- Expired, overlong, stale, malformed, or unused exceptions fail CI. A finding
that gains a fix cannot use an exception declared as unfixable.
- License exceptions are not supported. Add an exact expression only after a
maintainer verifies the package metadata and compatibility with AGPL-3.0-or-later.
## Updating the lock and tools
Use the version of uv recorded in `ci/digests.txt`:
```bash
uvx --from uv==0.12.4 uv lock --upgrade
uvx --from uv==0.12.4 uv lock --check
```
Review the complete lock diff, source indexes, new licenses, vulnerability
evidence, and SBOM diff. Update direct pins in `pyproject.toml` and executable
pins/checksums in `ci/digests.txt` in the same pull request. A normal dependency
or tool update must not add an exception merely to make CI green.
+23 -5
View File
@@ -1,5 +1,5 @@
[build-system]
requires = ["setuptools>=77.0", "wheel"]
requires = ["setuptools==80.9.0", "wheel==0.45.1"]
build-backend = "setuptools.build_meta"
[project]
@@ -46,11 +46,18 @@ dependencies = [
dev = [
"build==1.2.2.post1",
"mypy==1.13.0",
"pytest==8.3.4",
"pytest-cov==6.0.0",
"ruff==0.8.6",
"pytest==9.1.1",
"pytest-cov==7.1.0",
"ruff==0.16.2",
]
spaces = ["gradio>=6.7,<7.0"]
[dependency-groups]
ci = [
"build==1.2.2.post1",
"pip-licenses==5.5.5",
"twine==7.0.0",
]
spaces = ["gradio>=5.0,<6.0"]
[project.scripts]
obliteratus = "obliteratus.cli:main"
@@ -98,4 +105,15 @@ markers = [
]
[tool.uv]
constraint-dependencies = ["pillow>=12.2.0"]
exclude-newer = "3 days"
[tool.uv.sources]
torch = [
{ index = "pytorch-cpu", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
]
[[tool.uv.index]]
name = "pytorch-cpu"
url = "https://download.pytorch.org/whl/cpu"
explicit = true
+1 -1
View File
@@ -1,4 +1,4 @@
gradio>=5.0,<6.0
gradio>=6.7,<7.0
torch>=2.0
transformers>=4.40
datasets>=2.14
+382
View File
@@ -0,0 +1,382 @@
"""Validate supply-chain policy and convert scanner evidence into strict decisions."""
from __future__ import annotations
import argparse
import hashlib
import json
from datetime import date
from pathlib import Path
from typing import Any
ALL_SEVERITIES = {"unknown", "low", "medium", "high", "critical"}
def _read_json(path: Path) -> Any:
return json.loads(path.read_text(encoding="utf-8"))
def _parse_date(value: object, field: str, failures: list[str]) -> date | None:
if not isinstance(value, str):
failures.append(f"{field} must be an ISO date")
return None
try:
return date.fromisoformat(value)
except ValueError:
failures.append(f"{field} must be an ISO date")
return None
def validate_policy(policy: object, *, today: date | None = None) -> list[str]:
"""Return failures for malformed, expired, or overlong policy exceptions."""
current = today or date.today()
if not isinstance(policy, dict):
return ["policy must be a JSON object"]
failures: list[str] = []
if policy.get("schema_version") != 1:
failures.append("policy schema_version must be 1")
vulnerability = policy.get("vulnerability")
if not isinstance(vulnerability, dict):
failures.append("policy is missing vulnerability settings")
else:
severities = vulnerability.get("fail_severities")
if not isinstance(severities, list) or set(severities) != ALL_SEVERITIES:
failures.append("vulnerability policy must fail every normalized severity")
failures.extend(
_validate_suppressions(
vulnerability.get("suppressions"),
identifier="id",
current=current,
fixed_limit=vulnerability.get("max_fixed_suppression_days"),
unfixed_limit=vulnerability.get("max_unfixed_suppression_days"),
),
)
secret = policy.get("secret")
if not isinstance(secret, dict):
failures.append("policy is missing secret settings")
else:
if secret.get("report_redaction_percent") != 100:
failures.append("secret reports must use 100 percent redaction")
failures.extend(
_validate_suppressions(
secret.get("suppressions"),
identifier="fingerprint",
current=current,
fixed_limit=secret.get("max_suppression_days"),
),
)
license_policy = policy.get("license")
if not isinstance(license_policy, dict):
failures.append("policy is missing license settings")
else:
allowed = license_policy.get("allowed_expressions")
if not isinstance(allowed, list) or not allowed or not all(
isinstance(item, str) and item for item in allowed
):
failures.append("license allowed_expressions must be a non-empty string list")
return failures
def _validate_suppressions(
suppressions: object,
*,
identifier: str,
current: date,
fixed_limit: object,
unfixed_limit: object | None = None,
) -> list[str]:
if not isinstance(suppressions, list):
return [f"{identifier} suppressions must be a list"]
failures: list[str] = []
seen: set[str] = set()
for index, item in enumerate(suppressions):
prefix = f"suppression[{index}]"
if not isinstance(item, dict):
failures.append(f"{prefix} must be an object")
continue
value = item.get(identifier)
if not isinstance(value, str) or not value:
failures.append(f"{prefix}.{identifier} must be a non-empty string")
elif value in seen:
failures.append(f"duplicate suppression {identifier}: {value}")
else:
seen.add(value)
if not isinstance(item.get("reason"), str) or not item["reason"].strip():
failures.append(f"{prefix}.reason must be a non-empty string")
approved = _parse_date(item.get("approved_on"), f"{prefix}.approved_on", failures)
expires = _parse_date(item.get("expires"), f"{prefix}.expires", failures)
if approved is None or expires is None:
continue
if expires < current:
failures.append(f"{prefix} expired on {expires.isoformat()}")
if expires < approved:
failures.append(f"{prefix}.expires precedes approved_on")
continue
limit = fixed_limit
if unfixed_limit is not None:
fix_available = item.get("fix_available")
if not isinstance(fix_available, bool):
failures.append(f"{prefix}.fix_available must be boolean")
continue
limit = fixed_limit if fix_available else unfixed_limit
if not isinstance(limit, int) or isinstance(limit, bool) or limit < 1:
failures.append(f"{prefix} suppression limit must be a positive integer")
elif (expires - approved).days > limit:
failures.append(f"{prefix} exceeds its {limit}-day maximum")
return failures
def evaluate_audit(
policy: dict[str, Any],
evidence: object,
*,
scanner_status: int,
today: date | None = None,
) -> dict[str, Any]:
"""Evaluate uv audit JSON without discarding its meaningful exit status."""
failures = validate_policy(policy, today=today)
if scanner_status not in (0, 1):
failures.append(f"uv audit failed operationally with status {scanner_status}")
if not isinstance(evidence, dict):
failures.append("uv audit evidence must be a JSON object")
vulnerabilities: list[object] = []
adverse_statuses: list[object] = []
else:
vulnerabilities = evidence.get("vulnerabilities", [])
adverse_statuses = evidence.get("adverse_statuses", [])
if not isinstance(vulnerabilities, list):
failures.append("uv audit vulnerabilities must be a list")
vulnerabilities = []
if not isinstance(adverse_statuses, list):
failures.append("uv audit adverse_statuses must be a list")
adverse_statuses = []
suppressions = policy.get("vulnerability", {}).get("suppressions", [])
matched: set[str] = set()
suppressed: list[str] = []
for item in vulnerabilities:
if not isinstance(item, dict) or not isinstance(item.get("id"), str):
failures.append("uv audit contains a malformed vulnerability")
continue
identifiers = {item["id"]}
aliases = item.get("aliases", [])
if isinstance(aliases, list):
identifiers.update(alias for alias in aliases if isinstance(alias, str))
suppression = next(
(
candidate
for candidate in suppressions
if isinstance(candidate, dict) and candidate.get("id") in identifiers
),
None,
)
if suppression is None:
failures.append(f"unsuppressed vulnerability: {item['id']}")
continue
has_fix = bool(item.get("fix_versions"))
if suppression.get("fix_available") != has_fix:
failures.append(
f"suppression fixability is stale for {suppression.get('id')}: "
f"scanner reports fix_available={has_fix}",
)
continue
matched.add(suppression["id"])
suppressed.append(item["id"])
for suppression in suppressions:
if isinstance(suppression, dict) and suppression.get("id") not in matched:
failures.append(f"unused vulnerability suppression: {suppression.get('id')}")
if adverse_statuses:
failures.append(f"uv audit reported {len(adverse_statuses)} adverse package statuses")
if scanner_status == 0 and vulnerabilities:
failures.append("uv audit returned success while reporting vulnerabilities")
if scanner_status == 1 and not vulnerabilities and not adverse_statuses:
failures.append("uv audit returned failure without reviewable findings")
return {
"passed": not failures,
"scanner_status": scanner_status,
"vulnerability_count": len(vulnerabilities),
"suppressed": sorted(suppressed),
"failures": failures,
}
def evaluate_licenses(policy: dict[str, Any], inventory: object) -> dict[str, Any]:
"""Enforce the exact license-expression allow list."""
failures = validate_policy(policy)
if not isinstance(inventory, list):
failures.append("license inventory must be a JSON list")
inventory = []
license_policy = policy.get("license", {})
allowed = set(license_policy.get("allowed_expressions", []))
excluded = set(license_policy.get("excluded_packages", []))
checked = 0
for item in inventory:
if not isinstance(item, dict):
failures.append("license inventory contains a malformed row")
continue
name = item.get("Name")
expression = item.get("License")
if name in excluded:
continue
checked += 1
if expression not in allowed:
failures.append(f"unapproved license expression for {name}: {expression}")
if checked == 0:
failures.append("license inventory contains no packaged dependencies")
return {"passed": not failures, "checked_packages": checked, "failures": failures}
def evaluate_secrets(
policy: dict[str, Any],
report: object,
*,
scanner_status: int,
today: date | None = None,
) -> dict[str, Any]:
"""Require redacted Gitleaks evidence and match only bounded fingerprints."""
failures = validate_policy(policy, today=today)
if scanner_status not in (0, 1):
failures.append(f"gitleaks failed operationally with status {scanner_status}")
if not isinstance(report, list):
failures.append("gitleaks report must be a JSON list")
report = []
suppressions = policy.get("secret", {}).get("suppressions", [])
matched: set[str] = set()
suppressed: list[str] = []
for finding in report:
if not isinstance(finding, dict):
failures.append("gitleaks report contains a malformed finding")
continue
fingerprint = finding.get("Fingerprint")
if finding.get("Secret") != "REDACTED" or "REDACTED" not in str(
finding.get("Match", ""),
):
failures.append(f"gitleaks finding is not fully redacted: {fingerprint}")
suppression = next(
(
candidate
for candidate in suppressions
if isinstance(candidate, dict) and candidate.get("fingerprint") == fingerprint
),
None,
)
if suppression is None:
failures.append(f"unsuppressed secret finding: {fingerprint}")
else:
matched.add(fingerprint)
suppressed.append(fingerprint)
for suppression in suppressions:
if isinstance(suppression, dict) and suppression.get("fingerprint") not in matched:
failures.append(
f"unused secret suppression: {suppression.get('fingerprint')}",
)
if scanner_status == 0 and report:
failures.append("gitleaks returned success while reporting findings")
if scanner_status == 1 and not report:
failures.append("gitleaks returned failure without reviewable findings")
return {
"passed": not failures,
"scanner_status": scanner_status,
"finding_count": len(report),
"suppressed": sorted(suppressed),
"failures": failures,
}
def bind_sbom(sbom: object, wheel: Path) -> dict[str, Any]:
"""Bind a CycloneDX SBOM to the exact built wheel by SHA-256."""
if not isinstance(sbom, dict) or sbom.get("bomFormat") != "CycloneDX":
raise ValueError("SBOM must be a CycloneDX JSON object")
if sbom.get("specVersion") != "1.5":
raise ValueError("SBOM must use CycloneDX 1.5")
metadata = sbom.get("metadata")
if not isinstance(metadata, dict):
raise ValueError("SBOM is missing metadata")
component = metadata.get("component")
if not isinstance(component, dict) or component.get("name") != "obliteratus":
raise ValueError("SBOM metadata must describe obliteratus")
digest = hashlib.sha256(wheel.read_bytes()).hexdigest()
component["hashes"] = [{"alg": "SHA-256", "content": digest}]
properties = component.setdefault("properties", [])
if not isinstance(properties, list):
raise ValueError("SBOM component properties must be a list")
properties.append({"name": "obliteratus:distribution-file", "value": wheel.name})
return sbom
def _status(path: Path) -> int:
return int(path.read_text(encoding="utf-8").strip())
def _write_decision(path: Path, decision: dict[str, Any]) -> int:
path.write_text(json.dumps(decision, indent=2, sort_keys=True) + "\n", encoding="utf-8")
for failure in decision.get("failures", []):
print(f"supply-chain gate failed: {failure}")
if decision.get("passed"):
print("supply-chain gate passed")
return 0
return 1
def _parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=__doc__)
commands = parser.add_subparsers(dest="command", required=True)
policy = commands.add_parser("policy")
policy.add_argument("--policy", type=Path, required=True)
for name in ("audit", "secrets"):
command = commands.add_parser(name)
command.add_argument("--policy", type=Path, required=True)
command.add_argument("--evidence", type=Path, required=True)
command.add_argument("--scanner-status", type=Path, required=True)
command.add_argument("--decision", type=Path, required=True)
licenses = commands.add_parser("licenses")
licenses.add_argument("--policy", type=Path, required=True)
licenses.add_argument("--evidence", type=Path, required=True)
licenses.add_argument("--decision", type=Path, required=True)
sbom = commands.add_parser("sbom")
sbom.add_argument("--input", type=Path, required=True)
sbom.add_argument("--wheel", type=Path, required=True)
sbom.add_argument("--output", type=Path, required=True)
return parser
def main() -> int:
args = _parser().parse_args()
if args.command == "policy":
failures = validate_policy(_read_json(args.policy))
return _write_decision(Path("/dev/null"), {"passed": not failures, "failures": failures})
if args.command == "sbom":
bound = bind_sbom(_read_json(args.input), args.wheel)
args.output.write_text(json.dumps(bound, indent=2) + "\n", encoding="utf-8")
print(f"bound SBOM to {args.wheel.name}")
return 0
policy = _read_json(args.policy)
evidence = _read_json(args.evidence)
if args.command == "audit":
decision = evaluate_audit(
policy,
evidence,
scanner_status=_status(args.scanner_status),
)
elif args.command == "secrets":
decision = evaluate_secrets(
policy,
evidence,
scanner_status=_status(args.scanner_status),
)
else:
decision = evaluate_licenses(policy, evidence)
return _write_decision(args.decision, decision)
if __name__ == "__main__":
raise SystemExit(main())
+24
View File
@@ -45,3 +45,27 @@ def test_actionlint_version_and_checksum_match_manifest():
assert f'ACTIONLINT_VERSION: "{version.removeprefix("v")}"' in workflow
assert f'ACTIONLINT_SHA256: "{pin.removeprefix("sha256:")}"' in workflow
def test_uv_and_gitleaks_pins_match_manifest():
workflow = WORKFLOW.read_text(encoding="utf-8")
entries = _manifest_entries()
uv_pin, uv_version = entries[("tool", "astral-sh/uv")]
gitleaks_pin, gitleaks_version = entries[("tool", "gitleaks/gitleaks")]
assert uv_pin == f"pypi:{uv_version.removeprefix('v')}"
assert f'UV_VERSION: "{uv_version.removeprefix("v")}"' in workflow
assert f'GITLEAKS_VERSION: "{gitleaks_version.removeprefix("v")}"' in workflow
assert f'GITLEAKS_SHA256: "{gitleaks_pin.removeprefix("sha256:")}"' in workflow
def test_ci_requires_the_committed_lock_and_strict_policy_gate():
workflow = WORKFLOW.read_text(encoding="utf-8")
assert "uv lock --check" in workflow
assert "uv sync --locked" in workflow
assert "scripts/check_supply_chain_policy.py audit" in workflow
assert "scripts/check_supply_chain_policy.py secrets" in workflow
assert "scripts/check_supply_chain_policy.py licenses" in workflow
supply_chain_job = workflow.split(" supply-chain:\n", maxsplit=1)[1]
assert "|| true" not in supply_chain_job
+200
View File
@@ -0,0 +1,200 @@
"""Tests for reproducible supply-chain policy decisions."""
from __future__ import annotations
import importlib.util
from datetime import date
from pathlib import Path
ROOT = Path(__file__).parents[1]
SCRIPT = ROOT / "scripts" / "check_supply_chain_policy.py"
SPEC = importlib.util.spec_from_file_location("check_supply_chain_policy", SCRIPT)
assert SPEC is not None and SPEC.loader is not None
MODULE = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(MODULE)
def _policy() -> dict[str, object]:
return {
"schema_version": 1,
"vulnerability": {
"fail_severities": ["unknown", "low", "medium", "high", "critical"],
"max_fixed_suppression_days": 7,
"max_unfixed_suppression_days": 90,
"suppressions": [],
},
"secret": {
"report_redaction_percent": 100,
"max_suppression_days": 30,
"suppressions": [],
},
"license": {
"allowed_expressions": ["MIT"],
"excluded_packages": ["obliteratus"],
},
}
def test_committed_policy_is_valid():
policy = MODULE._read_json(ROOT / "ci" / "supply-chain-policy.json")
assert MODULE.validate_policy(policy) == []
def test_policy_rejects_expired_and_overlong_suppressions():
policy = _policy()
policy["vulnerability"]["suppressions"] = [
{
"id": "GHSA-example",
"reason": "temporary",
"approved_on": "2026-01-01",
"expires": "2026-01-10",
"fix_available": True,
},
]
failures = MODULE.validate_policy(policy, today=date(2026, 1, 11))
assert "suppression[0] expired on 2026-01-10" in failures
assert "suppression[0] exceeds its 7-day maximum" in failures
def test_audit_blocks_every_unsuppressed_vulnerability():
evidence = {
"vulnerabilities": [
{"id": "GHSA-example", "aliases": ["CVE-example"], "fix_versions": ["2.0"]},
],
"adverse_statuses": [],
}
decision = MODULE.evaluate_audit(_policy(), evidence, scanner_status=1)
assert decision["passed"] is False
assert decision["failures"] == ["unsuppressed vulnerability: GHSA-example"]
def test_audit_rejects_unexplained_scanner_failure():
evidence = {"vulnerabilities": [], "adverse_statuses": []}
decision = MODULE.evaluate_audit(_policy(), evidence, scanner_status=1)
assert decision["passed"] is False
assert decision["failures"] == [
"uv audit returned failure without reviewable findings",
]
def test_audit_accepts_bounded_alias_suppression():
policy = _policy()
policy["vulnerability"]["suppressions"] = [
{
"id": "CVE-example",
"reason": "upgrade lands this week",
"approved_on": "2026-01-01",
"expires": "2026-01-08",
"fix_available": True,
},
]
evidence = {
"vulnerabilities": [
{"id": "GHSA-example", "aliases": ["CVE-example"], "fix_versions": ["2.0"]},
],
"adverse_statuses": [],
}
decision = MODULE.evaluate_audit(
policy,
evidence,
scanner_status=1,
today=date(2026, 1, 2),
)
assert decision["passed"] is True
assert decision["suppressed"] == ["GHSA-example"]
def test_audit_rejects_stale_fixability_claim():
policy = _policy()
policy["vulnerability"]["suppressions"] = [
{
"id": "GHSA-example",
"reason": "waiting for a fix",
"approved_on": "2026-01-01",
"expires": "2026-03-01",
"fix_available": False,
},
]
evidence = {
"vulnerabilities": [{"id": "GHSA-example", "fix_versions": ["2.0"]}],
"adverse_statuses": [],
}
decision = MODULE.evaluate_audit(
policy,
evidence,
scanner_status=1,
today=date(2026, 1, 2),
)
assert decision["passed"] is False
assert any("fixability is stale" in failure for failure in decision["failures"])
def test_license_gate_is_exact_and_excludes_first_party_package():
inventory = [
{"Name": "obliteratus", "License": "AGPL-3.0-or-later"},
{"Name": "safe", "License": "MIT"},
{"Name": "unknown", "License": "UNKNOWN"},
]
decision = MODULE.evaluate_licenses(_policy(), inventory)
assert decision["passed"] is False
assert decision["checked_packages"] == 2
assert decision["failures"] == ["unapproved license expression for unknown: UNKNOWN"]
def test_secret_gate_requires_redaction_even_for_suppressed_finding():
policy = _policy()
policy["secret"]["suppressions"] = [
{
"fingerprint": "file:rule:1",
"reason": "documented fixture",
"approved_on": "2026-01-01",
"expires": "2026-01-15",
},
]
report = [
{
"Fingerprint": "file:rule:1",
"Secret": "not-redacted",
"Match": "token=not-redacted",
},
]
decision = MODULE.evaluate_secrets(
policy,
report,
scanner_status=1,
today=date(2026, 1, 2),
)
assert decision["passed"] is False
assert any("not fully redacted" in failure for failure in decision["failures"])
def test_bind_sbom_adds_exact_wheel_hash(tmp_path):
wheel = tmp_path / "obliteratus-0.1.2-py3-none-any.whl"
wheel.write_bytes(b"wheel")
sbom = {
"bomFormat": "CycloneDX",
"specVersion": "1.5",
"metadata": {"component": {"name": "obliteratus"}},
}
bound = MODULE.bind_sbom(sbom, wheel)
component = bound["metadata"]["component"]
assert component["hashes"] == [
{
"alg": "SHA-256",
"content": "ba59926159d2aa256eb8739b8da7e2b574b960e1202c6d624cbe981cef996c91",
},
]
assert component["properties"] == [
{"name": "obliteratus:distribution-file", "value": wheel.name},
]
Generated
+4457
View File
File diff suppressed because it is too large Load Diff