diff --git a/.aiwg/architecture/sketch-testing-quality-program.md b/.aiwg/architecture/sketch-testing-quality-program.md new file mode 100644 index 0000000..0f4c1fe --- /dev/null +++ b/.aiwg/architecture/sketch-testing-quality-program.md @@ -0,0 +1,26 @@ +# Architecture sketch: layered verification + +The verification system has four explicit layers: + +1. **Mandatory PR gate** — CPU-only, offline unit and boundary-contract tests, + package build, lint, branch/line coverage, warnings, JUnit, and artifacts. +2. **Offline integration gate** — tiny local model/config fixtures exercise + pipeline composition, checkpoint/save/reload, evaluation, and reporting. +3. **Quality-depth gate** — property tests, repeat tests, and selective mutation + testing for high-consequence pure modules. +4. **Conditional environment gates** — accelerator, optional backend, download, + network, and remote-service jobs with explicit prerequisites. + +pytest markers form the selection contract between layers. GitHub Actions jobs +produce retained evidence and never silently convert a required failure into a +success. Tool versions remain project-controlled. The default gate must not +access external services, inherit user model caches, or require credentials. + +Coverage policy evolves by wave: + +- measurement wave: preserve the 49% line floor and establish branch baseline; +- boundary wave: repository line coverage at least 55%, changed lines at least + 90%, and touched critical modules at least 70%; +- integration wave: repository line coverage at least 60%; +- mature CPU-testable target: at least 80% line and 75% branch coverage, with + exclusions limited to documented conditional environment code. diff --git a/.aiwg/requirements/UC-testing-quality-program.md b/.aiwg/requirements/UC-testing-quality-program.md new file mode 100644 index 0000000..0b76130 --- /dev/null +++ b/.aiwg/requirements/UC-testing-quality-program.md @@ -0,0 +1,30 @@ +# Use cases: testing quality program + +## UC-TQ-01 — Contributor verifies a pull request offline + +Given a clean checkout on Python 3.10–3.12 without a GPU, network access, model +cache, or service credentials, a contributor can run the canonical commands and +receive deterministic pass/fail results plus retained JUnit and coverage data. + +## UC-TQ-02 — Maintainer evaluates changed-code risk + +A maintainer can see line and branch coverage, changed-line evidence, warnings, +packaging validity, and the exact test layer exercised before merging a change. + +## UC-TQ-03 — Release operator verifies installed distributions + +A release operator can build both distribution formats, validate their metadata, +install each in isolation, import the package, read its version, and execute the +public CLI without relying on the source tree. + +## UC-TQ-04 — Developer exercises model boundaries without downloads + +A developer can verify loader, device, quantization, architecture, pipeline, +checkpoint, evaluation, and reporting contracts with local deterministic fakes +and a tiny offline model fixture. + +## UC-TQ-05 — Operator validates conditional environments + +An operator can intentionally run clearly documented CUDA, bitsandbytes, MPS, +MLX, model-download, network, and remote checks without weakening or surprising +the default pull-request gate. diff --git a/.aiwg/risks/risks-testing-quality-program.md b/.aiwg/risks/risks-testing-quality-program.md new file mode 100644 index 0000000..cf8a4c7 --- /dev/null +++ b/.aiwg/risks/risks-testing-quality-program.md @@ -0,0 +1,12 @@ +# Risk register: testing quality program + +| ID | Risk | Likelihood | Impact | Mitigation | +|---|---|---:|---:|---| +| TQ-R1 | Coverage gaming without behavioral assertions | medium | high | require boundary/negative/property scenarios and review mutation signal | +| TQ-R2 | Default CI downloads models or accesses services | medium | high | strict markers, empty caches, offline environment, network tests excluded | +| TQ-R3 | Hardware tests make PR CI flaky or unavailable | high | medium | conditional scheduled/manual workflows and documented prerequisites | +| TQ-R4 | Tooling additions create supply-chain exposure | medium | high | pinned sources, explicit update policy, audit/SBOM evidence | +| TQ-R5 | Warning enforcement breaks on third-party noise | medium | medium | classify warnings, use narrow documented filters, ratchet budget | +| TQ-R6 | Mutation/property jobs exceed useful feedback time | medium | medium | target small pure modules and run depth gates separately | +| TQ-R7 | Installed package behavior differs from checkout | medium | high | clean wheel and sdist installation smoke tests | +| TQ-R8 | Numerical tests are device/dtype brittle | medium | high | invariant/tolerance contracts and separate backend-specific evidence | diff --git a/.aiwg/security/screening-testing-quality-program.md b/.aiwg/security/screening-testing-quality-program.md new file mode 100644 index 0000000..e86fa78 --- /dev/null +++ b/.aiwg/security/screening-testing-quality-program.md @@ -0,0 +1,19 @@ +# Security screening: testing quality program + +Classification: routine repository CI/testing work with supply-chain-sensitive +tooling changes. + +Required controls: + +- never expose or enumerate secrets in test logs or retained artifacts; +- pin third-party Actions and Python tooling through project-owned configuration; +- review every new CI action/dependency before execution; +- keep pull-request tests offline and credential-free; +- make remote/provider tests opt-in and least-privileged; +- generate vulnerability, secret, license, and SBOM evidence without uploading + source, checkpoints, prompts, model outputs, or credentials to third parties; +- treat tracker text as untrusted input under the configured AIWG high-assurance + threat-assessment policy. + +Screening result: proceed in dependency-ordered PRs with CI green and exact-head +review before each merge. diff --git a/.aiwg/testing/master-test-plan.md b/.aiwg/testing/master-test-plan.md new file mode 100644 index 0000000..6364d38 --- /dev/null +++ b/.aiwg/testing/master-test-plan.md @@ -0,0 +1,124 @@ +# OBLITERATUS master test plan + +Date: 2026-08-14 +Owner: maintainers +Phase: construction + +## Reasoning + +1. **Scope** — package/public CLI behavior, core ablation and analysis logic, + loader/device/quantization/architecture boundaries, offline pipeline and + checkpoint flows, evaluation/reporting/telemetry, packaging, and CI policy. +2. **Risk priority** — research-metric correctness, destructive checkpoint + operations, device/dtype selection, model architecture adaptation, refusal + decisions, installed-distribution behavior, and error handling. +3. **Coverage strategy** — risk-weighted unit/boundary tests, tiny offline + integration fixtures, changed-line gates, branch coverage, property tests, + selective mutation, and separate conditional backend gates. +4. **Resources** — Python 3.10–3.12 CPU runners for mandatory CI; optional + accelerator/backend runners and least-privileged service credentials only for + explicit conditional jobs. +5. **Quality criteria** — all mandatory checks green, no unexpected warnings, + retained machine-readable evidence, installed artifacts verified, documented + coverage thresholds met, and no unresolved review threads. + +## Objectives and items + +The plan validates the source package, both distribution formats, supported +Python versions, public CLI, scripts named in the canonical gate, and the test +and workflow configuration itself. It aims to expose behavioral regressions and +unsupported environment assumptions before merge. + +## In scope + +- deterministic pure/unit and boundary-contract tests; +- negative and error-path behavior; +- offline tiny-model integration and save/reload contracts; +- evaluation, reporting, telemetry, and research-metric invariants; +- packaging metadata and installed CLI/import behavior; +- CI workflow policy, evidence, supply-chain checks, and explicit markers; +- conditional accelerator, optional backend, network, and remote workflows. + +## Out of scope for the mandatory PR job + +- large model downloads or benchmark-quality model runs; +- credentials, remote execution, or production services; +- CUDA, MPS, MLX, and bitsandbytes availability; +- performance claims requiring dedicated hardware. + +These remain in scope for conditional gates and release evidence. + +## Approach and deliverables + +| Layer | Required timing | Deliverables | +|---|---|---| +| CPU unit/boundary | every PR | pytest/JUnit, line+branch coverage, warning result | +| package | every PR | sdist/wheel, metadata check, clean install smoke | +| offline integration | every PR after stabilization | tiny-model pipeline, checkpoint/save/reload, eval/report | +| quality depth | PR or scheduled by cost | property/repeat tests, selected mutation report | +| conditional environment | manual/scheduled/release | backend-specific JUnit/log evidence | + +## Environment and data + +The mandatory environment uses a clean checkout, an isolated Python environment, +CPU-only execution, no service credentials, no network/model downloads, and +synthetic deterministic fixtures. Integration fixtures must be small enough for +the repository and may not embed third-party model weights without documented +license/provenance. + +## Entry and exit criteria + +Entry: + +- clean base commit and dependency set; +- test scope and marker declared; +- acceptance criteria mapped to tests; +- external/hardware prerequisites separated from default CI. + +Exit for each issue: + +- implementation and relevant negative/boundary tests merged; +- complete required suite and hosted CI green; +- branch/line/changed-line thresholds for that wave met; +- generated evidence retained and reviewed; +- no unresolved issue/PR feedback; +- documentation and tracker state reconciled. + +Program exit: + +- Phases 0–2 are merged before ordinary feature work resumes; +- all planned waves are delivered or explicitly accepted as conditional release + gates with a runnable workflow and owner; +- CPU-testable code reaches the mature 80% line / 75% branch target, or every + remaining exclusion has a documented environment-bound rationale. + +## Milestones + +1. Evidence, strict pytest policy, workflow validation. +2. Installed distributions and supply-chain evidence. +3. Boundary contracts and 55% repository line coverage. +4. Offline integration and 60% repository line coverage. +5. Research-integrity, property, repeat, and selected mutation gates. +6. Conditional backend/service workflows and mature CPU-testable targets. + +## Metrics and reporting + +- tests passed/failed/skipped by marker and Python version; +- line and branch coverage overall and by critical module; +- changed-line coverage; +- warning count/category; +- distribution validation/install results; +- repeat/flaky failures and mutation score for selected modules; +- conditional workflow freshness and result. + +## References + +- @.aiwg/requirements/UC-testing-quality-program.md +- @.aiwg/architecture/sketch-testing-quality-program.md +- @.aiwg/risks/risks-testing-quality-program.md +- @.aiwg/security/screening-testing-quality-program.md +- @.aiwg/working/issue-planner/research-synthesis.md +- @WORKSPACE.md +- @pyproject.toml +- @.github/workflows/ci.yml +- @tests/ diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 4904e1b..c329a6d 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -4,7 +4,7 @@ ## Tests -- [ ] `python -m ruff check --select F obliteratus tests scripts/gemma4_12b_recursive_loop.py` +- [ ] `python -m ruff check --select F obliteratus tests scripts/check_coverage_thresholds.py scripts/gemma4_12b_recursive_loop.py` - [ ] `python -m ruff check --select E501 --statistics obliteratus tests scripts/gemma4_12b_recursive_loop.py` reviewed as known non-blocking line-length debt - [ ] `python -c 'import obliteratus; print(getattr(obliteratus, "__version__", "import ok"))'` - [ ] `python -m obliteratus --help` diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3bec41c..f27f9aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,11 +95,17 @@ jobs: run: python -m pip install "ruff==0.8.6" - name: Enforce Ruff F gate - run: python -m ruff check --select F obliteratus tests scripts/gemma4_12b_recursive_loop.py + run: >- + python -m ruff check --select F obliteratus tests + scripts/check_coverage_thresholds.py + scripts/gemma4_12b_recursive_loop.py - name: Report E501 legacy baseline if: always() - run: python -m ruff check --select E501 --statistics obliteratus tests scripts/gemma4_12b_recursive_loop.py || true + run: >- + python -m ruff check --select E501 --statistics obliteratus tests + scripts/check_coverage_thresholds.py + scripts/gemma4_12b_recursive_loop.py || true test: name: Tests py${{ matrix.python-version }} @@ -112,6 +118,13 @@ jobs: - "3.10" - "3.11" - "3.12" + env: + CUDA_VISIBLE_DEVICES: "" + HF_DATASETS_OFFLINE: "1" + HF_HUB_DISABLE_TELEMETRY: "1" + HF_HUB_OFFLINE: "1" + TOKENIZERS_PARALLELISM: "false" + TRANSFORMERS_OFFLINE: "1" steps: - name: Check out repository @@ -151,4 +164,28 @@ jobs: python -m obliteratus --help - name: Run tests with coverage - run: python -m pytest + run: | + mkdir -p test-results + python -m pytest \ + -m "not slow and not gpu and not mps and not mlx and not network and not download and not remote" \ + --cov-branch \ + --cov-fail-under=0 \ + --junitxml="test-results/junit-py${{ matrix.python-version }}.xml" \ + --cov-report="xml:test-results/coverage-py${{ matrix.python-version }}.xml" \ + --cov-report="json:test-results/coverage-py${{ matrix.python-version }}.json" + + - name: Enforce line and branch coverage floors + run: >- + python scripts/check_coverage_thresholds.py + "test-results/coverage-py${{ matrix.python-version }}.json" + --min-line 49 + --min-branch 36 + + - name: Upload test and coverage evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: test-evidence-py${{ matrix.python-version }} + path: test-results/ + if-no-files-found: error + retention-days: 14 diff --git a/.gitignore b/.gitignore index 5bd04d3..cfc1e77 100644 --- a/.gitignore +++ b/.gitignore @@ -84,3 +84,4 @@ docs/EFFICIENCY_AUDIT.md .coverage.* coverage.xml htmlcov/ +test-results/ diff --git a/WORKSPACE.md b/WORKSPACE.md index 4494f1f..63f4d69 100644 --- a/WORKSPACE.md +++ b/WORKSPACE.md @@ -42,8 +42,10 @@ accelerator, or remote-execution credentials. Canonical required checks: -- `python -m ruff check --select F obliteratus tests scripts/gemma4_12b_recursive_loop.py` +- `python -m ruff check --select F obliteratus tests scripts/check_coverage_thresholds.py scripts/gemma4_12b_recursive_loop.py` - `python -m pytest` (includes the measured 49% coverage floor) +- CI additionally enforces the measured 36% branch-coverage floor from its + retained coverage JSON report. - `python -m build --sdist --wheel` - `python -c 'import obliteratus; print(obliteratus.__version__)'` - `python -m obliteratus --help` diff --git a/pyproject.toml b/pyproject.toml index d6cd070..329941f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,7 +75,27 @@ select = ["F"] [tool.pytest.ini_options] testpaths = ["tests"] -addopts = "--cov=obliteratus --cov-report=term-missing --cov-fail-under=49" +addopts = [ + "--strict-config", + "--strict-markers", + "--cov=obliteratus", + "--cov-report=term-missing", + "--cov-fail-under=49", +] +filterwarnings = [ + "error", +] +markers = [ + "cpu: deterministic tests supported by the mandatory CPU-only gate", + "integration: offline integration tests spanning multiple components", + "slow: tests excluded from the fast pull-request gate because of runtime", + "gpu: tests that require a CUDA-capable runner", + "mps: tests that require Apple Metal Performance Shaders", + "mlx: tests that require Apple MLX", + "network: tests that access external network resources", + "download: tests that download model or dataset artifacts", + "remote: tests that require a remote execution provider or credentials", +] [tool.uv] exclude-newer = "3 days" diff --git a/scripts/check_coverage_thresholds.py b/scripts/check_coverage_thresholds.py new file mode 100644 index 0000000..cdb8f6e --- /dev/null +++ b/scripts/check_coverage_thresholds.py @@ -0,0 +1,66 @@ +"""Enforce separate line and branch floors from a coverage.py JSON report.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + + +def validate_coverage( + report: dict[str, Any], *, min_line: float, min_branch: float, +) -> list[str]: + """Return human-readable failures for coverage totals below their floors.""" + totals = report.get("totals") + if not isinstance(totals, dict): + return ["coverage report is missing the totals object"] + + failures: list[str] = [] + metrics = ( + ("line", "percent_statements_covered", min_line), + ("branch", "percent_branches_covered", min_branch), + ) + for label, key, minimum in metrics: + value = totals.get(key) + if not isinstance(value, (int, float)): + failures.append(f"coverage report is missing numeric {key}") + elif value < minimum: + failures.append( + f"{label} coverage {value:.2f}% is below the {minimum:.2f}% floor", + ) + return failures + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("report", type=Path, help="coverage.py JSON report") + parser.add_argument("--min-line", type=float, required=True) + parser.add_argument("--min-branch", type=float, required=True) + return parser + + +def main() -> int: + args = _parser().parse_args() + report = json.loads(args.report.read_text(encoding="utf-8")) + failures = validate_coverage( + report, + min_line=args.min_line, + min_branch=args.min_branch, + ) + if failures: + for failure in failures: + print(f"coverage gate failed: {failure}") + return 1 + + totals = report["totals"] + print( + "coverage gate passed: " + f"line={totals['percent_statements_covered']:.2f}% " + f"branch={totals['percent_branches_covered']:.2f}%", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/conftest.py b/tests/conftest.py index 8fbf11a..c2fde3d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,12 +2,37 @@ from __future__ import annotations +import socket from unittest.mock import MagicMock import pytest import torch +_EXTERNAL_MARKERS = ("network", "download", "remote") + + +@pytest.fixture(autouse=True) +def offline_test_environment(request, monkeypatch): + """Fail accidental network access and force offline Hugging Face behavior.""" + if any(request.node.get_closest_marker(name) for name in _EXTERNAL_MARKERS): + return + + monkeypatch.setenv("HF_DATASETS_OFFLINE", "1") + monkeypatch.setenv("HF_HUB_DISABLE_TELEMETRY", "1") + monkeypatch.setenv("HF_HUB_OFFLINE", "1") + monkeypatch.setenv("TRANSFORMERS_OFFLINE", "1") + + def reject_network(*_args, **_kwargs): + raise RuntimeError( + "unmarked tests may not access the network; add an explicit " + "network, download, or remote marker", + ) + + monkeypatch.setattr(socket, "create_connection", reject_network) + monkeypatch.setattr(socket.socket, "connect", reject_network) + + # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- diff --git a/tests/test_abliterate.py b/tests/test_abliterate.py index 6080c90..db85022 100644 --- a/tests/test_abliterate.py +++ b/tests/test_abliterate.py @@ -1067,7 +1067,12 @@ class TestAttentionHeadSurgery: # SOTA technique #6: SAE feature-level abliteration # --------------------------------------------------------------------------- +@pytest.mark.filterwarnings( + "ignore:SAE held-out reconstruction MSE.*:UserWarning", +) class TestSAEAbliteration: + """Exercise deliberately undertrained SAEs without weakening global warnings.""" + def test_sae_train_and_reconstruct(self): """SAE should train and reconstruct activations.""" from obliteratus.analysis.sae_abliteration import train_sae diff --git a/tests/test_coverage_thresholds.py b/tests/test_coverage_thresholds.py new file mode 100644 index 0000000..5e44751 --- /dev/null +++ b/tests/test_coverage_thresholds.py @@ -0,0 +1,45 @@ +"""Tests for the separate line/branch coverage policy gate.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + + +SCRIPT = Path(__file__).parents[1] / "scripts" / "check_coverage_thresholds.py" +SPEC = importlib.util.spec_from_file_location("check_coverage_thresholds", SCRIPT) +assert SPEC is not None and SPEC.loader is not None +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +def _report(line: float = 49.0, branch: float = 36.0) -> dict[str, object]: + return { + "totals": { + "percent_statements_covered": line, + "percent_branches_covered": branch, + }, + } + + +def test_validate_coverage_accepts_exact_floors(): + assert MODULE.validate_coverage( + _report(), min_line=49.0, min_branch=36.0, + ) == [] + + +def test_validate_coverage_reports_each_regression(): + failures = MODULE.validate_coverage( + _report(line=48.9, branch=35.9), min_line=49.0, min_branch=36.0, + ) + + assert failures == [ + "line coverage 48.90% is below the 49.00% floor", + "branch coverage 35.90% is below the 36.00% floor", + ] + + +def test_validate_coverage_rejects_malformed_totals(): + assert MODULE.validate_coverage( + {}, min_line=49.0, min_branch=36.0, + ) == ["coverage report is missing the totals object"] diff --git a/tests/test_new_analysis_modules.py b/tests/test_new_analysis_modules.py index e2060d7..f6d7193 100644 --- a/tests/test_new_analysis_modules.py +++ b/tests/test_new_analysis_modules.py @@ -369,7 +369,12 @@ class TestActivationPatcher: # Tests: Enhanced SAE Decomposition Pipeline # =========================================================================== +@pytest.mark.filterwarnings( + "ignore:SAE held-out reconstruction MSE.*:UserWarning", +) class TestSAEDecompositionPipeline: + """Use small training budgets while keeping unrelated warnings fatal.""" + def test_basic_pipeline(self): harmful, harmless, _ = _make_activations(hidden_dim=16, n_per_class=30, separation=2.0) diff --git a/tests/test_test_policy.py b/tests/test_test_policy.py new file mode 100644 index 0000000..d347b9b --- /dev/null +++ b/tests/test_test_policy.py @@ -0,0 +1,20 @@ +"""Executable contracts for the mandatory CPU/offline pytest policy.""" + +from __future__ import annotations + +import os +import socket + +import pytest + + +def test_unmarked_tests_run_with_offline_hugging_face_policy(): + assert os.environ["HF_DATASETS_OFFLINE"] == "1" + assert os.environ["HF_HUB_DISABLE_TELEMETRY"] == "1" + assert os.environ["HF_HUB_OFFLINE"] == "1" + assert os.environ["TRANSFORMERS_OFFLINE"] == "1" + + +def test_unmarked_tests_cannot_open_network_connections(): + with pytest.raises(RuntimeError, match="unmarked tests may not access the network"): + socket.create_connection(("example.invalid", 443))