From 37d008d462a6c3b2f04bd9046593e728870227a2 Mon Sep 17 00:00:00 2001 From: Joseph Magly <1159087+jmagly@users.noreply.github.com> Date: Sat, 15 Aug 2026 01:07:47 -0400 Subject: [PATCH] test: establish Gate 1 quality baseline (#90) Establishes the mandatory testing, coverage, repeatability, mutation, packaging, supply-chain, and AIWG workspace baseline before feature integration. --- .aiwg/aiwg.config | 53 +++- .aiwg/gates/test-quality-gate.md | 13 + .aiwg/plugins/bt6-maintainer/README.md | 40 ++- .aiwg/plugins/bt6-maintainer/manifest.json | 2 +- .../bt6-maintainer/payload/manifest.json | 2 +- .aiwg/quickref.config.json | 8 + .aiwg/testing/coverage-report.md | 21 ++ .aiwg/testing/master-test-plan.md | 46 +-- .aiwg/testing/regression-report.md | 14 + .aiwg/testing/test-execution-report.md | 41 +++ .github/workflows/ci.yml | 4 +- CONTRIBUTING.md | 14 +- README.md | 6 +- WORKSPACE.md | 6 +- ci/test-quality-policy.json | 29 +- ci/test-risk-map.json | 44 ++- obliteratus/evaluation/__init__.py | 177 ++++++----- obliteratus/evaluation/lm_eval_integration.py | 4 +- obliteratus/reporting/report.py | 19 +- obliteratus/telemetry.py | 6 +- pyproject.toml | 4 +- scripts/check_quality_policy.py | 8 +- scripts/run_repeat_gate.py | 3 + tests/test_evaluation_reporting_contracts.py | 131 ++++++++ tests/test_lm_eval_reporting_contracts.py | 133 ++++++++ tests/test_quality_policy.py | 27 +- tests/test_telemetry_failure_contracts.py | 295 ++++++++++++++++++ 27 files changed, 976 insertions(+), 174 deletions(-) create mode 100644 .aiwg/gates/test-quality-gate.md create mode 100644 .aiwg/quickref.config.json create mode 100644 .aiwg/testing/coverage-report.md create mode 100644 .aiwg/testing/regression-report.md create mode 100644 .aiwg/testing/test-execution-report.md create mode 100644 tests/test_evaluation_reporting_contracts.py create mode 100644 tests/test_lm_eval_reporting_contracts.py create mode 100644 tests/test_telemetry_failure_contracts.py diff --git a/.aiwg/aiwg.config b/.aiwg/aiwg.config index 423ef1e..cd2d748 100644 --- a/.aiwg/aiwg.config +++ b/.aiwg/aiwg.config @@ -4,11 +4,38 @@ "providers": [ "codex" ], + "workspace": { + "name": "bt6-obliteratus-maintenance" + }, + "repos": [ + { + "name": "obliteratus", + "path": ".", + "provider": "github", + "allowed": [ + "read", + "write", + "commit", + "push", + "issue-comment", + "service-action" + ] + }, + { + "name": "bt6-aiwg-plugins", + "path": "../bt6-aiwg-plugins", + "provider": "github", + "allowed": [ + "read" + ], + "notes": "Authoritative BT6 code-audit and maintainer workflow source; mutations require separate authorization." + } + ], "installed": { "all": { - "version": "2026.8.8", + "version": "2026.8.9", "source": "bundled", - "installedAt": "2026-08-15T02:40:58.412Z", + "installedAt": "2026-08-15T04:22:13.831Z", "deployedTo": { "codex": { "agents": 0, @@ -20,9 +47,9 @@ "manifestHash": "sha256:21afaf03f52084ab895a6862fd45c284ba94366ee420eaa1f67e1b4acdc1be7c" }, "bt6-maintainer": { - "version": "0.2.0", + "version": "0.3.0", "source": "project-local", - "installedAt": "2026-08-15T02:40:59.927Z", + "installedAt": "2026-08-15T04:23:14.957Z", "deployedTo": { "codex": { "agents": 5, @@ -31,7 +58,7 @@ "rules": 1 } }, - "manifestHash": "sha256:66a87462e0567646017a8e90f5ed7f31ba9959fcd2fd2b7f4486c33c32204c45", + "manifestHash": "sha256:509e05c707975f4b2cd7a5c086c9545882022e2f68e921f45e2d31616334532c", "localPath": ".aiwg/plugins/bt6-maintainer/", "localType": "plugin", "manifestVersion": "1", @@ -65,7 +92,21 @@ } } }, - "scripts": {}, + "scripts": { + "deploy": "aiwg use all --provider codex && aiwg use bt6-maintainer --provider codex", + "doctor": "aiwg doctor --verbose", + "sync": "aiwg sync" + }, + "remotes": { + "primary": "origin", + "issue_tracker": "origin", + "ci": "origin", + "tracker_actor": { + "login": "jmagly", + "via": "gh", + "forbid_actors": [] + } + }, "security": { "threatAssessment": { "schemaVersion": "1", diff --git a/.aiwg/gates/test-quality-gate.md b/.aiwg/gates/test-quality-gate.md new file mode 100644 index 0000000..e174355 --- /dev/null +++ b/.aiwg/gates/test-quality-gate.md @@ -0,0 +1,13 @@ +# Gate 1 quality decision + +Date: 2026-08-15 +Decision: CONDITIONAL PASS + +All local Gate 1 criteria pass: mandatory CPU tests, repository and mature-scope +coverage, warning budget, repeatability, bounded mutation, packaging, locked +resolution, and supply-chain policy. No P0/P1 defect or flaky quarantine was +identified. + +The sole remaining condition is exact-head hosted CI after the branch is +committed and pushed. Gate 1 becomes PASS only when all required GitHub Actions +jobs are green at that immutable head and review threads are resolved. diff --git a/.aiwg/plugins/bt6-maintainer/README.md b/.aiwg/plugins/bt6-maintainer/README.md index b48aa14..21c2a43 100644 --- a/.aiwg/plugins/bt6-maintainer/README.md +++ b/.aiwg/plugins/bt6-maintainer/README.md @@ -33,9 +33,15 @@ been verified. └── provenance/ ``` -## Usage +## Installation -On AIWG 2026.7.24 or newer, place the wrapper under the consuming repository's +The repository root README is the canonical installation guide. It covers the +full AIWG package, lightweight `@aiwg/cli`, and the dependency-free manual +installer for Claude Code and Codex. + +### AIWG-managed deployment + +On AIWG 2026.8.0 or newer, place the wrapper under the consuming repository's `.aiwg/plugins/` directory and deploy it directly: ```bash @@ -63,6 +69,23 @@ Inspect health: aiwg doctor --project-local ``` +### Native deployment without AIWG + +From the repository root: + +```bash +node scripts/install-bt6-maintainer.mjs \ + --platform all \ + --target /absolute/path/to/consumer \ + --with-profile \ + --dry-run +``` + +The helper installs native provider artifacts plus a `.bt6-maintainer/` +support tree and refuses collisions unless `--force` is explicit. See the +[repository README](../../../README.md#route-c-no-aiwg-installation) for the +review/apply and verification sequence. + AIWG #1998 currently prevents reliable automated removal of freshly deployed namespaced skill files. Inspect provider paths and preserve the registry record needed for recovery; do not use `--force` without verifying exact ownership. @@ -70,7 +93,7 @@ needed for recovery; do not use `--force` without verifying exact ownership. ## Packaging status The wrapper follows AIWG's project-local plugin schema and contains an addon -payload under `payload/`. AIWG 2026.7.24 validates, packages, and directly +payload under `payload/`. AIWG 2026.8.0 validates, packages, and directly deploys the wrapper. Remaining lifecycle gaps are: - [#1996](https://git.integrolabs.net/roctinam/aiwg/issues/1996) — legacy @@ -87,10 +110,11 @@ npm run check npm run test:smoke ``` -The smoke test deploys the payload as a project-local addon in an isolated -temporary repository for provider parity. Direct wrapper deployment is also -covered during adoption; automated removal remains outside the passing smoke -gate until #1998 is resolved. +The smoke test deploys the wrapper in an isolated temporary repository for +provider parity. Manual installer tests cover dry-run, native Claude and Codex +layouts, support files, idempotency, collision refusal, and explicit +replacement. Automated removal remains outside the passing smoke gate until +#1998 is resolved. ## Supported repository families @@ -104,7 +128,7 @@ gate until #1998 is resolved. - Claude: full addon deployment (agents, skills, and guardrail rule). - Codex: full deployment through AIWG's shared Agent Skills surface plus agent - TOMLs and guardrail rule on AIWG 2026.7.24 or newer. + TOMLs and guardrail rule on AIWG 2026.8.0 or newer. ## Source and license diff --git a/.aiwg/plugins/bt6-maintainer/manifest.json b/.aiwg/plugins/bt6-maintainer/manifest.json index 537c166..f1cdb37 100644 --- a/.aiwg/plugins/bt6-maintainer/manifest.json +++ b/.aiwg/plugins/bt6-maintainer/manifest.json @@ -2,7 +2,7 @@ "id": "bt6-maintainer", "type": "plugin", "name": "bt6-maintainer", - "version": "0.2.0", + "version": "0.3.0", "description": "Cross-repository maintenance and external-provider review for BT6 research and support tooling.", "manifestVersion": "1", "platforms": { diff --git a/.aiwg/plugins/bt6-maintainer/payload/manifest.json b/.aiwg/plugins/bt6-maintainer/payload/manifest.json index 386fb29..6a804ba 100644 --- a/.aiwg/plugins/bt6-maintainer/payload/manifest.json +++ b/.aiwg/plugins/bt6-maintainer/payload/manifest.json @@ -2,7 +2,7 @@ "id": "bt6-maintainer", "type": "addon", "name": "bt6-maintainer", - "version": "0.2.0", + "version": "0.3.0", "description": "Cross-repository queue, review, issue, provider-trust, and merge stewardship for BT6 research and support tooling.", "manifestVersion": "1", "platforms": { diff --git a/.aiwg/quickref.config.json b/.aiwg/quickref.config.json new file mode 100644 index 0000000..c5d11d2 --- /dev/null +++ b/.aiwg/quickref.config.json @@ -0,0 +1,8 @@ +{ + "version": "1", + "project": { + "id": "obliteratus", + "name": "OBLITERATUS", + "description": "Managed project-specific orientation for OBLITERATUS." + } +} diff --git a/.aiwg/testing/coverage-report.md b/.aiwg/testing/coverage-report.md new file mode 100644 index 0000000..0624145 --- /dev/null +++ b/.aiwg/testing/coverage-report.md @@ -0,0 +1,21 @@ +# Gate 1 coverage report + +Date: 2026-08-15 +Coverage format: coverage.py branch JSON v3 + +| Scope | Measured | Enforced floor | Status | +|---|---:|---:|---| +| Repository statements | 70.20% | 70% | PASS | +| Repository branches | 57.41% | 55% | PASS | +| Mature CPU statements | 92.23% | 90% | PASS | +| Mature CPU branches | 80.36% | 78% | PASS | +| Changed executable lines | 100.00% | 90% | PASS | + +Critical changed surfaces are fully or strongly covered: evaluation baselines +100%/90.91%, evaluator 100%/97.37%, lm-eval adapter 100%/100%, report +100%/98.21%, and telemetry 91.62%/81.88% (statements/branches). + +The immutable floors are encoded in `ci/test-quality-policy.json`, pytest, +the CI workflow, and `scripts/check_quality_policy.py`. Exact-base +touched-module no-regression is rerun after the candidate has a commit SHA and +again by hosted CI. diff --git a/.aiwg/testing/master-test-plan.md b/.aiwg/testing/master-test-plan.md index e70a654..f9264da 100644 --- a/.aiwg/testing/master-test-plan.md +++ b/.aiwg/testing/master-test-plan.md @@ -1,21 +1,25 @@ # OBLITERATUS master test plan -Date: 2026-08-14 +Date: 2026-08-15 Owner: maintainers Phase: construction ## Current verified baseline -Baseline commit: `340173e203757757451893347445ac8bd5fdc816` +Gate 1 candidate base: `62a480fd6a8f9ef091e60ba8dbf6b0ca94a8b785` -The mandatory offline CPU selection currently completes locally in 72.55 -seconds with 1,195 passing tests, 9 conditionally deselected tests, no skips, -and no warnings. Branch-enabled coverage is 66.44% line and 52.08% branch. The -documented mature CPU-testable scope is 89.36% line and 75.41% branch. The -repository also enforces 90% changed-line coverage, per-touched-module line and -branch no-regression, a 70% selective mutation score, and zero unexpected -warnings. Hosted CI is green for Python 3.10, 3.11, and 3.12, packaging, -quality depth, supply chain, and the current exact `main` head. +The mandatory offline CPU selection currently completes from a clean, +non-editable Python 3.12 environment in 68.22 seconds with 1,468 passing tests, +9 conditionally deselected tests, no skips, and no warnings. Branch-enabled +coverage is 70.20% statements and 57.41% branches. The documented mature +CPU-testable scope is 92.23% statements and 80.36% branches. The repository +enforces 90% changed-line coverage, per-touched-module line and branch +no-regression, a 75% selective mutation floor, and zero unexpected warnings. +The current bounded campaign kills 809 of 886 mutants (91.31%), and 316 +high-consequence tests pass in each of three file orders and hash seeds with no +flake candidates. Package, installed wheel/sdist, vulnerability, secret, +license, and SBOM checks are green locally. Exact-head hosted CI remains the +final Gate 1 merge condition. The latest conditional run is green for its selected pinned-model, external-evaluation, loopback-network, and operator-UI gates. CUDA, bitsandbytes, MPS, MLX, and remote execution were not selected and therefore @@ -114,7 +118,7 @@ Program exit: - Phases 0–2 are merged before ordinary feature work resumes; - all planned waves are delivered or explicitly accepted as conditional release gates with a runnable workflow and owner; -- CPU-testable code reaches the mature 80% line / 75% branch target, or every +- CPU-testable code reaches the mature 90% line / 78% branch target, or every remaining exclusion has a documented environment-bound rationale. ## Per-PR test contract @@ -191,16 +195,16 @@ Gate 1 requires all of the following on exact-head hosted CI: | Surface | Verified baseline | Wave B target | |---|---:|---:| -| Repository | 66.44% line / 52.08% branch | ≥70% / ≥55% | -| Mature CPU scope | 89.36% / 75.41% | ≥90% / ≥78% | -| `cli.py` | 97.91% / 81.43% | preserve ≥97% / raise to ≥85% | -| `config.py` | 98.73% / 81.25% | preserve ≥98% / raise to ≥90% | -| `device.py` | 96.47% / 92.42% | preserve ≥96% / ≥92% | -| `architecture_profiles.py` | 88.62% / 82.95% | ≥92% / ≥85% | -| `models/loader.py` | 85.79% / 84.09% | ≥90% / ≥87% | -| `evaluation/evaluator.py` | 72.00% / 71.05% | ≥85% / ≥80% | -| `reporting/report.py` | 82.94% / 64.81% | ≥90% / ≥80% | -| `telemetry.py` | 71.07% / 62.40% | ≥80% / ≥75% | +| Repository | 70.20% line / 57.41% branch | ≥70% / ≥55% | +| Mature CPU scope | 92.23% / 80.36% | ≥90% / ≥78% | +| `cli.py` | 97.93% / 89.15% | preserve ≥97% / raise to ≥85% | +| `config.py` | 100.00% / 90.00% | preserve ≥98% / raise to ≥90% | +| `device.py` | 100.00% / 95.00% | preserve ≥96% / ≥92% | +| `architecture_profiles.py` | 97.52% / 92.31% | ≥92% / ≥85% | +| `models/loader.py` | 91.01% / 88.14% | ≥90% / ≥87% | +| `evaluation/evaluator.py` | 100.00% / 97.37% | ≥85% / ≥80% | +| `reporting/report.py` | 100.00% / 98.21% | ≥90% / ≥80% | +| `telemetry.py` | 91.62% / 81.88% | ≥80% / ≥75% | Large runtime-bound modules must be split conceptually into CPU-testable decision logic and conditional execution. CPU-testable helpers extracted from diff --git a/.aiwg/testing/regression-report.md b/.aiwg/testing/regression-report.md new file mode 100644 index 0000000..b4fd37e --- /dev/null +++ b/.aiwg/testing/regression-report.md @@ -0,0 +1,14 @@ +# Gate 1 regression report + +Date: 2026-08-15 +Base: `62a480fd6a8f9ef091e60ba8dbf6b0ca94a8b785` + +- Mandatory CPU regressions: none; 1,468 tests pass with no warnings. +- Determinism regressions: none; three orders and hash seeds are green. +- Packaging regressions: none in wheel, sdist, metadata, import, or CLI contracts. +- Supply-chain regressions: none in vulnerability, secret, license, lock, or SBOM checks. +- Coverage direction: repository and mature-scope measures exceed every Gate 1 target. + +The exact committed-head touched-module comparison is intentionally deferred +until the candidate commit exists; hosted CI independently regenerates coverage +for the exact base SHA before allowing merge. diff --git a/.aiwg/testing/test-execution-report.md b/.aiwg/testing/test-execution-report.md new file mode 100644 index 0000000..1301916 --- /dev/null +++ b/.aiwg/testing/test-execution-report.md @@ -0,0 +1,41 @@ +# Gate 1 test execution report + +Date: 2026-08-15 +Scope: mandatory offline CPU baseline, quality depth, packaging, and supply chain +Status: PASS locally; exact-head hosted CI pending + +## Results + +| Gate | Result | +|---|---| +| Mandatory CPU | 1,468 passed, 9 conditional deselections, 0 failures/skips/warnings | +| Repository coverage | 70.20% statements, 57.41% branches | +| Mature CPU scope | 92.23% statements, 80.36% branches | +| Changed executable lines | 100% on the candidate worktree | +| Repeat | 316 tests × 3 orders/seeds; no flakes or consistent failures | +| Mutation | 809/886 killed (91.31%); 72 survived; 5 timed out | +| Package | wheel and sdist build, metadata, isolated imports, and both CLI paths pass | +| Supply chain | 0 vulnerabilities, 0 secret findings, 100 licenses accepted, SBOM bound | + +The mandatory lane ran in a clean non-editable Python 3.12 environment with +offline Hugging Face and accelerator settings. Wall time was 68.22 seconds and +maximum RSS was 791,476 KB. The mutation lane ran under the CI-pinned Python +3.12/tool configuration in 69.79 seconds with maximum RSS of 1,399,824 KB. + +## Defects found and corrected + +1. The lm-eval adapter treated a measured accuracy of `0.0` as missing. +2. Public sanitizers did not reliably remove Windows paths on non-Windows hosts. +3. Report heatmap generation surfaced a dependency deprecation as an unexpected warning. +4. The evaluation package eagerly loaded heavyweight model dependencies for pure adapters; + it now preserves the public API through lazy imports. + +Each correction has a focused regression contract. Evaluation/reporting and +telemetry failure contracts are also part of the deterministic repeat gate; +the pure lm-eval adapter is part of bounded mutation testing. + +## Remaining conditional evidence + +CUDA, bitsandbytes, MPS, MLX, remote execution, real model downloads, and live +external evaluation remain owned by their conditional gates. They are not +required for this offline pull-request baseline and were not rerun here. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 425b15e..55fcb49 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -328,8 +328,8 @@ jobs: fi "$TEST_ENV/bin/python" scripts/check_coverage_thresholds.py \ "test-results/coverage-py${{ matrix.python-version }}.json" \ - --min-line 60 \ - --min-branch 42 \ + --min-line 70 \ + --min-branch 55 \ --min-file obliteratus/device.py=70 \ --min-file obliteratus/models/loader.py=70 \ --min-file obliteratus/architecture_profiles.py=70 \ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 34cd83c..e072247 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,13 +32,13 @@ python -m obliteratus --help ``` All tests must pass before submitting a PR. Tests are designed to run on CPU without downloading models. -The mandatory gate currently requires at least 60% repository statement coverage, -42% branch coverage, 90% coverage of changed executable lines, and 70% statement +The mandatory gate currently requires at least 70% repository statement coverage, +55% branch coverage, 90% coverage of changed executable lines, and 70% statement coverage in the device, loader, architecture-profile, CLI, simulated MLX, evaluation-metric, reporting, community-contribution, and telemetry boundary modules. New changes should raise these floors rather than consume the existing -margin. A separately measured mature CPU-only scope must remain at or above 80% -statement and 75% branch coverage; its environment-bound exclusions and their +margin. A separately measured mature CPU-only scope must remain at or above 90% +statement and 78% branch coverage; its environment-bound exclusions and their conditional-test ownership are versioned in `ci/test-quality-policy.json`. Every changed production module is also compared with coverage generated from the exact base commit. Line and branch coverage may not regress independently, @@ -47,15 +47,15 @@ changed-line requirement. The quality-depth job repeats the highest-consequence pure tests three times in different file orders and with different deterministic hash seeds. It also runs -selective mutation testing over configuration parsing and the coverage-policy -gate. The initial mutation floor is 70%. Run these checks locally with: +selective mutation testing over configuration, quality-policy, and pure +evaluation contracts. The mutation floor is 75%. Run these checks locally with: ```bash uv sync --locked --extra dev --group quality python scripts/run_repeat_gate.py --output test-results/repeat-gate.json mutmut run --max-children 4 mutmut export-cicd-stats -python scripts/check_mutation_score.py mutants/mutmut-cicd-stats.json --minimum 70 +python scripts/check_mutation_score.py mutants/mutmut-cicd-stats.json --minimum 75 ``` The repeat and mutation selections are intentionally bounded; on the Wave A diff --git a/README.md b/README.md index ac7d99a..59a7b07 100644 --- a/README.md +++ b/README.md @@ -761,14 +761,14 @@ pip install -e ".[dev]" pytest ``` -The mandatory CPU suite contains more than 1,100 tests, including a +The mandatory CPU suite contains more than 1,400 tests, including a repository-owned synthetic model that exercises the offline pipeline, installed-wheel CLI, study runner, transactional checkpoint recovery, and resumable auto-obliteration state. The suite also covers model/device/quantization/MLX boundaries, all analysis modules, architecture detection, visualization sanitization, community contributions, edge cases, and evaluation metrics. CI enforces at least -60% repository statement coverage, 42% repository branch coverage, 90% -changed-line coverage, and 80% statement / 75% branch coverage for the documented +70% repository statement coverage, 55% repository branch coverage, 90% +changed-line coverage, and 90% statement / 78% branch coverage for the documented mature CPU-only scope. Deterministic property, order-repeat, and selective mutation gates provide additional depth for numerical and policy-critical behavior. CI also rejects line or branch regressions in each touched production module by diff --git a/WORKSPACE.md b/WORKSPACE.md index 8cdde33..939f009 100644 --- a/WORKSPACE.md +++ b/WORKSPACE.md @@ -43,12 +43,12 @@ accelerator, or remote-execution credentials. Canonical required checks: - the exact Ruff F and actionlint command set in [.github/workflows/ci.yml](.github/workflows/ci.yml); -- `python -m pytest` with at least 60% repository line coverage and 42% branch +- `python -m pytest` with at least 70% repository line coverage and 55% branch coverage; - at least 90% changed-line coverage plus no line or branch regression in any touched production module, compared with coverage from the exact base commit; -- at least 80% line and 75% branch coverage for the documented mature - CPU-testable scope, plus a 70% selective mutation score and zero unexpected +- at least 90% line and 78% branch coverage for the documented mature + CPU-testable scope, plus a 75% selective mutation score and zero unexpected warnings; - `python -m build --sdist --wheel` - `python -c 'import obliteratus; print(obliteratus.__version__)'` diff --git a/ci/test-quality-policy.json b/ci/test-quality-policy.json index c5a1f4d..842063c 100644 --- a/ci/test-quality-policy.json +++ b/ci/test-quality-policy.json @@ -1,11 +1,11 @@ { "schema_version": 1, "minimums": { - "repository_statement": 60.0, - "repository_branch": 42.0, + "repository_statement": 70.0, + "repository_branch": 55.0, "changed_line": 90.0, - "mature_cpu_statement": 80.0, - "mature_cpu_branch": 75.0, + "mature_cpu_statement": 90.0, + "mature_cpu_branch": 78.0, "mutation_score": 75.0, "warning_budget": 0 }, @@ -55,20 +55,6 @@ "conditional_issue": "https://github.com/elder-plinius/OBLITERATUS/issues/71", "conditional_gate": "network-services" }, - { - "path": "obliteratus/evaluation/baselines.py", - "boundary": "model-runtime", - "rationale": "Control baselines mutate and repeatedly evaluate a live transformer pipeline.", - "conditional_issue": "https://github.com/elder-plinius/OBLITERATUS/issues/71", - "conditional_gate": "model-download-runtime" - }, - { - "path": "obliteratus/evaluation/evaluator.py", - "boundary": "model-runtime", - "rationale": "The remaining branches tokenize datasets and execute causal-LM or classifier forward passes.", - "conditional_issue": "https://github.com/elder-plinius/OBLITERATUS/issues/71", - "conditional_gate": "model-download-runtime" - }, { "path": "obliteratus/evaluation/heretic_eval.py", "boundary": "external-evaluator", @@ -76,13 +62,6 @@ "conditional_issue": "https://github.com/elder-plinius/OBLITERATUS/issues/71", "conditional_gate": "external-evaluation" }, - { - "path": "obliteratus/evaluation/lm_eval_integration.py", - "boundary": "external-evaluator", - "rationale": "This optional adapter requires lm-evaluation-harness, benchmark downloads, and a model runtime.", - "conditional_issue": "https://github.com/elder-plinius/OBLITERATUS/issues/71", - "conditional_gate": "external-evaluation" - }, { "path": "obliteratus/informed_pipeline.py", "boundary": "model-runtime", diff --git a/ci/test-risk-map.json b/ci/test-risk-map.json index 027df86..cb7591c 100644 --- a/ci/test-risk-map.json +++ b/ci/test-risk-map.json @@ -129,7 +129,8 @@ "tests/test_hard_negative.py", "tests/test_prompt_sources.py", "tests/test_module_imports.py", - "tests/test_telemetry.py" + "tests/test_telemetry.py", + "tests/test_telemetry_failure_contracts.py" ] }, { @@ -201,7 +202,9 @@ "tests/test_benchmark_plots.py", "tests/test_benchmarks.py", "tests/test_evaluator.py", + "tests/test_evaluation_reporting_contracts.py", "tests/test_heretic_eval.py", + "tests/test_lm_eval_reporting_contracts.py", "tests/test_metrics.py", "tests/test_property_contracts.py" ] @@ -212,7 +215,11 @@ "description": "Report schemas, deterministic serialization, plotting, and persisted result artifacts", "contract_types": ["research-output", "persistence", "numerical-invariant"], "paths": ["obliteratus/reporting/report.py"], - "required_tests": ["tests/test_report.py"] + "required_tests": [ + "tests/test_report.py", + "tests/test_evaluation_reporting_contracts.py", + "tests/test_lm_eval_reporting_contracts.py" + ] }, { "id": "ablation-strategies", @@ -268,14 +275,21 @@ "path": "obliteratus/telemetry.py", "risk_class": "cpu-contract", "risk": "research telemetry schema, aggregation, and filesystem behavior", - "required_tests": ["tests/test_telemetry.py"], + "required_tests": [ + "tests/test_telemetry.py", + "tests/test_telemetry_failure_contracts.py" + ], "conditional_gates": [] }, { "path": "obliteratus/reporting/report.py", "risk_class": "cpu-contract", "risk": "report schema, output paths, serialization, and plotting contracts", - "required_tests": ["tests/test_report.py"], + "required_tests": [ + "tests/test_report.py", + "tests/test_evaluation_reporting_contracts.py", + "tests/test_lm_eval_reporting_contracts.py" + ], "conditional_gates": [] }, { @@ -363,14 +377,22 @@ "path": "obliteratus/evaluation/baselines.py", "risk_class": "mixed-runtime", "risk": "control baseline mutation and comparative evaluation", - "required_tests": ["tests/test_evaluator.py", "tests/conditional/test_model_download_runtime.py"], + "required_tests": [ + "tests/test_evaluator.py", + "tests/test_evaluation_reporting_contracts.py", + "tests/conditional/test_model_download_runtime.py" + ], "conditional_gates": ["model-download-runtime"] }, { "path": "obliteratus/evaluation/evaluator.py", "risk_class": "mixed-runtime", "risk": "dataset bounds, causal/classifier evaluation, and result semantics", - "required_tests": ["tests/test_evaluator.py", "tests/test_offline_integration.py"], + "required_tests": [ + "tests/test_evaluator.py", + "tests/test_evaluation_reporting_contracts.py", + "tests/test_offline_integration.py" + ], "conditional_gates": ["model-download-runtime"] }, { @@ -403,9 +425,13 @@ }, { "path": "obliteratus/evaluation/lm_eval_integration.py", - "risk_class": "conditional-runtime", - "risk": "lm-evaluation-harness integration and benchmark result translation", - "required_tests": ["tests/test_module_imports.py", "tests/conditional/test_external_evaluation_runtime.py"], + "risk_class": "mixed-runtime", + "risk": "external benchmark adapter arguments, metric extraction, measured-zero semantics, and comparison output", + "required_tests": [ + "tests/test_lm_eval_reporting_contracts.py", + "tests/test_module_imports.py", + "tests/conditional/test_external_evaluation_runtime.py" + ], "conditional_gates": ["external-evaluation"] }, { diff --git a/obliteratus/evaluation/__init__.py b/obliteratus/evaluation/__init__.py index bd97fed..0d2b53f 100644 --- a/obliteratus/evaluation/__init__.py +++ b/obliteratus/evaluation/__init__.py @@ -1,71 +1,108 @@ -from obliteratus.evaluation.evaluator import Evaluator -from obliteratus.evaluation.metrics import perplexity, accuracy, f1_score_metric -from obliteratus.evaluation.advanced_metrics import ( - refusal_rate, - refusal_rate_with_ci, - token_kl_divergence, - first_token_kl_divergence, - effective_rank, - effective_rank_change, - activation_cosine_similarity, - linear_cka, - refusal_projection_magnitude, - AbliterationEvalResult, - format_eval_report, -) -from obliteratus.evaluation.baselines import ( - random_direction_ablation, - direction_specificity_test, -) -from obliteratus.evaluation.heretic_eval import ( - arditi_refusal_rate, - harmbench_asr, - unload_harmbench_classifier, - first_token_kl_on_prompts, - run_lm_eval, - load_jailbreakbench_prompts, - run_full_heretic_eval, - format_comparison_table, - HereticComparisonResult, - LM_EVAL_BENCHMARKS, -) -from obliteratus.evaluation.lm_eval_integration import ( - run_benchmarks, - compare_models, -) +"""Evaluation APIs with lazy imports for optional and heavyweight boundaries.""" -__all__ = [ - "Evaluator", - "perplexity", - "accuracy", - "f1_score_metric", - "refusal_rate", - "token_kl_divergence", - "first_token_kl_divergence", - "effective_rank", - "effective_rank_change", - "activation_cosine_similarity", - "linear_cka", - "refusal_projection_magnitude", - "AbliterationEvalResult", - "format_eval_report", - # Community-standard evaluation (Heretics/Arditi protocol) - "arditi_refusal_rate", - "harmbench_asr", - "unload_harmbench_classifier", - "first_token_kl_on_prompts", - "run_lm_eval", - "load_jailbreakbench_prompts", - "run_full_heretic_eval", - "format_comparison_table", - "HereticComparisonResult", - "LM_EVAL_BENCHMARKS", - # lm-evaluation-harness integration - "run_benchmarks", - "compare_models", - # Statistical evaluation - "refusal_rate_with_ci", - # Baselines - "random_direction_ablation", - "direction_specificity_test", -] +from __future__ import annotations + +from importlib import import_module +from typing import Any + + +_EXPORTS = { + "Evaluator": ("obliteratus.evaluation.evaluator", "Evaluator"), + "perplexity": ("obliteratus.evaluation.metrics", "perplexity"), + "accuracy": ("obliteratus.evaluation.metrics", "accuracy"), + "f1_score_metric": ("obliteratus.evaluation.metrics", "f1_score_metric"), + "refusal_rate": ("obliteratus.evaluation.advanced_metrics", "refusal_rate"), + "refusal_rate_with_ci": ( + "obliteratus.evaluation.advanced_metrics", + "refusal_rate_with_ci", + ), + "token_kl_divergence": ( + "obliteratus.evaluation.advanced_metrics", + "token_kl_divergence", + ), + "first_token_kl_divergence": ( + "obliteratus.evaluation.advanced_metrics", + "first_token_kl_divergence", + ), + "effective_rank": ("obliteratus.evaluation.advanced_metrics", "effective_rank"), + "effective_rank_change": ( + "obliteratus.evaluation.advanced_metrics", + "effective_rank_change", + ), + "activation_cosine_similarity": ( + "obliteratus.evaluation.advanced_metrics", + "activation_cosine_similarity", + ), + "linear_cka": ("obliteratus.evaluation.advanced_metrics", "linear_cka"), + "refusal_projection_magnitude": ( + "obliteratus.evaluation.advanced_metrics", + "refusal_projection_magnitude", + ), + "AbliterationEvalResult": ( + "obliteratus.evaluation.advanced_metrics", + "AbliterationEvalResult", + ), + "format_eval_report": ( + "obliteratus.evaluation.advanced_metrics", + "format_eval_report", + ), + "random_direction_ablation": ( + "obliteratus.evaluation.baselines", + "random_direction_ablation", + ), + "direction_specificity_test": ( + "obliteratus.evaluation.baselines", + "direction_specificity_test", + ), + "arditi_refusal_rate": ("obliteratus.evaluation.heretic_eval", "arditi_refusal_rate"), + "harmbench_asr": ("obliteratus.evaluation.heretic_eval", "harmbench_asr"), + "unload_harmbench_classifier": ( + "obliteratus.evaluation.heretic_eval", + "unload_harmbench_classifier", + ), + "first_token_kl_on_prompts": ( + "obliteratus.evaluation.heretic_eval", + "first_token_kl_on_prompts", + ), + "run_lm_eval": ("obliteratus.evaluation.heretic_eval", "run_lm_eval"), + "load_jailbreakbench_prompts": ( + "obliteratus.evaluation.heretic_eval", + "load_jailbreakbench_prompts", + ), + "run_full_heretic_eval": ( + "obliteratus.evaluation.heretic_eval", + "run_full_heretic_eval", + ), + "format_comparison_table": ( + "obliteratus.evaluation.heretic_eval", + "format_comparison_table", + ), + "HereticComparisonResult": ( + "obliteratus.evaluation.heretic_eval", + "HereticComparisonResult", + ), + "LM_EVAL_BENCHMARKS": ( + "obliteratus.evaluation.heretic_eval", + "LM_EVAL_BENCHMARKS", + ), + "run_benchmarks": ( + "obliteratus.evaluation.lm_eval_integration", + "run_benchmarks", + ), + "compare_models": ( + "obliteratus.evaluation.lm_eval_integration", + "compare_models", + ), +} + +__all__ = list(_EXPORTS) + + +def __getattr__(name: str) -> Any: + try: + module_name, attribute = _EXPORTS[name] + except KeyError: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None + value = getattr(import_module(module_name), attribute) + globals()[name] = value + return value diff --git a/obliteratus/evaluation/lm_eval_integration.py b/obliteratus/evaluation/lm_eval_integration.py index 34b50ca..8a485ff 100644 --- a/obliteratus/evaluation/lm_eval_integration.py +++ b/obliteratus/evaluation/lm_eval_integration.py @@ -103,7 +103,9 @@ def run_benchmarks( scores: dict[str, float] = {} for task_name, task_result in results.get("results", {}).items(): # lm-eval uses "acc" or "acc_norm" depending on the task - acc = task_result.get("acc,none") or task_result.get("acc_norm,none") + acc = task_result.get("acc,none") + if acc is None: + acc = task_result.get("acc_norm,none") if acc is not None: scores[task_name] = acc else: diff --git a/obliteratus/reporting/report.py b/obliteratus/reporting/report.py index 697b4f7..3b445c7 100644 --- a/obliteratus/reporting/report.py +++ b/obliteratus/reporting/report.py @@ -5,8 +5,9 @@ from __future__ import annotations import json import math import re +import warnings from dataclasses import dataclass, field -from pathlib import Path +from pathlib import Path, PureWindowsPath from typing import Any import pandas as pd @@ -19,8 +20,10 @@ _SENSITIVE_KEY_RE = re.compile( def _sanitize_label(text: str, max_len: int = 80) -> str: """Strip filesystem paths, tokens, and overly-long strings from labels.""" - if text.startswith("/") or re.match(r"^[A-Za-z]:[\\/]", text): + if text.startswith("/"): text = Path(text).name + elif re.match(r"^[A-Za-z]:[\\/]", text): + text = PureWindowsPath(text).name text = re.sub( r"(?:/[A-Za-z0-9_.-]+){2,}", lambda match: Path(match.group()).name, text, ) @@ -257,7 +260,17 @@ class AblationReport: pivot.columns = [c.replace("_pct_change", "") for c in pivot.columns] fig, ax = plt.subplots(figsize=(max(6, len(pivot.columns) * 2), max(4, len(pivot) * 0.4))) - sns.heatmap(pivot, annot=True, fmt=".1f", cmap="RdYlGn_r", center=0, ax=ax) + # Seaborn currently calls Matplotlib's pending-deprecated ``set_bad`` + # internally. Keep the repository's zero-warning contract focused on + # our code while containing this specific third-party compatibility + # warning to the call that emits it. + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message="The set_bad function will be deprecated.*", + category=PendingDeprecationWarning, + ) + sns.heatmap(pivot, annot=True, fmt=".1f", cmap="RdYlGn_r", center=0, ax=ax) ax.set_title(f"Ablation % Change — {_sanitize_label(self.model_name)}") plt.tight_layout() diff --git a/obliteratus/telemetry.py b/obliteratus/telemetry.py index d33c88a..190b776 100644 --- a/obliteratus/telemetry.py +++ b/obliteratus/telemetry.py @@ -39,7 +39,7 @@ import threading import uuid from dataclasses import dataclass, field, asdict from datetime import datetime, timezone -from pathlib import Path +from pathlib import Path, PureWindowsPath from typing import Any logger = logging.getLogger(__name__) @@ -62,8 +62,10 @@ _PUBLIC_METRIC_RANGES: dict[str, tuple[float | None, float | None]] = { def _sanitize_public_text(value: Any, max_len: int = 240) -> str: text = str(value) - if text.startswith("/") or re.match(r"^[A-Za-z]:[\\/]", text): + if text.startswith("/"): text = Path(text).name + elif re.match(r"^[A-Za-z]:[\\/]", text): + text = PureWindowsPath(text).name text = re.sub(r"\bhf_[A-Za-z0-9]{6,}\b", "", text) text = re.sub(r"\bgh[pousr]_[A-Za-z0-9]{12,}\b", "", text) text = re.sub(r"\bgithub_pat_[A-Za-z0-9_]{12,}\b", "", text) diff --git a/pyproject.toml b/pyproject.toml index 5a158d2..e432961 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -95,7 +95,7 @@ addopts = [ "--strict-markers", "--cov=obliteratus", "--cov-report=term-missing", - "--cov-fail-under=60", + "--cov-fail-under=70", ] filterwarnings = [ "error", @@ -121,6 +121,7 @@ only_mutate = [ "obliteratus/runtime_contracts.py", "obliteratus/persistence_contracts.py", "obliteratus/remote_contracts.py", + "obliteratus/evaluation/lm_eval_integration.py", "scripts/check_coverage_thresholds.py", ] pytest_add_cli_args = ["--no-cov", "-q"] @@ -128,6 +129,7 @@ pytest_add_cli_args_test_selection = [ "tests/test_config.py", "tests/test_config_properties.py", "tests/test_coverage_thresholds.py", + "tests/test_lm_eval_reporting_contracts.py", "tests/test_numerical_contracts.py", "tests/test_persistence_contracts.py", "tests/test_remote_contracts.py", diff --git a/scripts/check_quality_policy.py b/scripts/check_quality_policy.py index 60edd32..516b28f 100644 --- a/scripts/check_quality_policy.py +++ b/scripts/check_quality_policy.py @@ -13,11 +13,11 @@ from typing import Any BASELINE_FLOORS = { - "repository_statement": 60.0, - "repository_branch": 42.0, + "repository_statement": 70.0, + "repository_branch": 55.0, "changed_line": 90.0, - "mature_cpu_statement": 80.0, - "mature_cpu_branch": 75.0, + "mature_cpu_statement": 90.0, + "mature_cpu_branch": 78.0, "mutation_score": 75.0, "warning_budget": 0.0, } diff --git a/scripts/run_repeat_gate.py b/scripts/run_repeat_gate.py index c2d53c4..d9d9eb5 100644 --- a/scripts/run_repeat_gate.py +++ b/scripts/run_repeat_gate.py @@ -18,6 +18,8 @@ DEFAULT_TESTS = ( "tests/test_config.py", "tests/test_config_properties.py", "tests/test_coverage_thresholds.py", + "tests/test_evaluation_reporting_contracts.py", + "tests/test_lm_eval_reporting_contracts.py", "tests/test_numerical_contracts.py", "tests/test_persistence_contracts.py", "tests/test_property_contracts.py", @@ -25,6 +27,7 @@ DEFAULT_TESTS = ( "tests/test_metrics.py", "tests/test_remote_contracts.py", "tests/test_runtime_contracts.py", + "tests/test_telemetry_failure_contracts.py", ) HASH_SEEDS = ("0", "1", "8675309") diff --git a/tests/test_evaluation_reporting_contracts.py b/tests/test_evaluation_reporting_contracts.py new file mode 100644 index 0000000..b8ef29e --- /dev/null +++ b/tests/test_evaluation_reporting_contracts.py @@ -0,0 +1,131 @@ +"""Boundary contracts for evaluation baselines, adapters, and public reports.""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +import torch +from torch import nn + +from obliteratus.evaluation.baselines import ( + direction_specificity_test, + random_direction_ablation, +) +from obliteratus.evaluation.evaluator import Evaluator + + +class _Encoding(dict): + def to(self, _device): + return self + + +class _ClassificationDataset: + def __init__(self, texts, labels): + self.texts = list(texts) + self.labels = list(labels) + self.selected = None + + def __len__(self): + return len(self.texts) + + def __getitem__(self, key): + if isinstance(key, slice): + return {"text": self.texts[key], "label": self.labels[key]} + return {"text": self.texts[key], "label": self.labels[key]} + + def select(self, indices): + indices = list(indices) + self.selected = indices + return _ClassificationDataset( + [self.texts[index] for index in indices], + [self.labels[index] for index in indices], + ) + + +class _ClassificationModel(nn.Module): + def __init__(self, batches): + super().__init__() + self.anchor = nn.Parameter(torch.zeros(())) + self.batches = list(batches) + + def forward(self, **_encodings): + return SimpleNamespace(logits=torch.tensor(self.batches.pop(0))) + + +def test_evaluator_dispatches_classification_and_rejects_unknown_tasks(): + dataset = _ClassificationDataset(["a", "b", "ignored"], [1, 0, 1]) + tokenizer = Mock(return_value=_Encoding(input_ids=torch.ones((2, 1), dtype=torch.long))) + model = _ClassificationModel([[[0.0, 2.0], [3.0, 0.0]]]) + handle = SimpleNamespace(model=model, tokenizer=tokenizer, task="classification") + + result = Evaluator( + handle, + dataset, + metrics=["accuracy", "f1"], + batch_size=2, + max_samples=2, + ).evaluate() + + assert dataset.selected == [0, 1] + assert result == {"accuracy": 1.0, "f1": 1.0} + tokenizer.assert_called_once() + + handle.task = "unsupported" + with pytest.raises(ValueError, match="Unsupported task: unsupported"): + Evaluator(handle, dataset).evaluate() + + +def test_classification_returns_only_requested_metrics(): + dataset = _ClassificationDataset(["a"], [0]) + tokenizer = Mock(return_value=_Encoding(input_ids=torch.ones((1, 1), dtype=torch.long))) + model = _ClassificationModel([[[2.0, 0.0]]]) + handle = SimpleNamespace(model=model, tokenizer=tokenizer, task="classification") + + assert Evaluator(handle, dataset, metrics=["accuracy"]).evaluate() == {"accuracy": 1.0} + + +def _pipeline(**overrides): + values = { + "_strong_layers": [0, 1], + "refusal_directions": {0: torch.tensor([1.0, 0.0]), 1: torch.tensor([0.0, 1.0])}, + "_harmful_means": {0: torch.tensor([2.0, 0.0]), 1: torch.tensor([0.0, 4.0])}, + "_harmless_means": {0: torch.tensor([0.5, 0.0]), 1: torch.tensor([0.0, 1.0])}, + } + values.update(overrides) + return SimpleNamespace(**values) + + +def test_random_direction_baseline_handles_missing_and_cleared_activations(): + missing = _pipeline(_strong_layers=[], refusal_directions={}) + assert "no directions" in random_direction_ablation(missing).details["error"].lower() + + cleared = _pipeline(_harmful_means={}) + assert "activations cleared" in random_direction_ablation(cleared).details["error"] + + +def test_random_direction_baseline_is_seeded_and_reports_trial_statistics(): + first = random_direction_ablation(_pipeline(), n_trials=4, seed=17) + second = random_direction_ablation(_pipeline(), n_trials=4, seed=17) + + assert first == second + assert first.baseline_name == "random_direction" + assert first.n_trials == 4 + assert len(first.refusal_rates) == 4 + assert first.refusal_rate == first.mean_refusal_rate + assert first.std_refusal_rate >= 0 + assert first.details == {"hidden_dim": 2, "n_strong_layers": 2} + + +def test_direction_specificity_covers_missing_partial_and_complete_inputs(): + assert direction_specificity_test(_pipeline(_strong_layers=[], refusal_directions={})) == { + "error": "No directions available" + } + partial = _pipeline(_harmless_means={}) + assert "activations cleared" in direction_specificity_test(partial)["error"] + + result = direction_specificity_test(_pipeline()) + assert result["harmful_projection"] == 3.0 + assert result["harmless_projection"] == 0.75 + assert result["specificity_ratio"] == 4.0 diff --git a/tests/test_lm_eval_reporting_contracts.py b/tests/test_lm_eval_reporting_contracts.py new file mode 100644 index 0000000..f4a9bd6 --- /dev/null +++ b/tests/test_lm_eval_reporting_contracts.py @@ -0,0 +1,133 @@ +"""Pure contracts for the lm-eval adapter and public report boundary.""" + +from __future__ import annotations + +import sys +from types import SimpleNamespace +from unittest.mock import Mock + +import matplotlib.pyplot as plt +import pytest + +from obliteratus.reporting.report import ( + AblationReport, + AblationResult, + _sanitize_public_value, +) +from obliteratus.evaluation import lm_eval_integration as LM_EVAL + + +def test_lm_eval_missing_dependency_has_actionable_error(monkeypatch): + monkeypatch.setitem(sys.modules, "lm_eval", None) + with pytest.raises(ImportError, match="pip install lm-eval>=0.4.0"): + LM_EVAL.run_benchmarks("model") + + +def test_lm_eval_preserves_measured_zero_and_falls_back_to_numeric_metric(monkeypatch): + simple_evaluate = Mock(return_value={ + "results": { + "zero": {"acc,none": 0.0, "acc_norm,none": 0.75}, + "normalized": {"acc_norm,none": 0.6}, + "fallback": {"alias": "name", "stderr": 0.02, "score": 0.4}, + "empty": {"alias": "empty"}, + } + }) + monkeypatch.setitem(sys.modules, "lm_eval", SimpleNamespace(simple_evaluate=simple_evaluate)) + + scores = LM_EVAL.run_benchmarks( + "org/model", + tasks=["zero", "normalized", "fallback", "empty"], + device="cpu", + batch_size=3, + num_fewshot=2, + limit=5, + ) + + assert scores == {"zero": 0.0, "normalized": 0.6, "fallback": 0.02} + simple_evaluate.assert_called_once_with( + model="hf", + model_args="pretrained=org/model,device=cpu", + tasks=["zero", "normalized", "fallback", "empty"], + batch_size=3, + num_fewshot=2, + limit=5, + ) + + +def test_lm_eval_defaults_and_model_comparison(monkeypatch): + simple_evaluate = Mock(return_value={"results": {"task": {"acc,none": 0.5}}}) + monkeypatch.setitem(sys.modules, "lm_eval", SimpleNamespace(simple_evaluate=simple_evaluate)) + assert LM_EVAL.run_benchmarks("model", tasks=["task"]) == {"task": 0.5} + assert simple_evaluate.call_args.kwargs["model_args"] == "pretrained=model" + + responses = iter([{"a": 0.8, "shared": 0.5}, {"b": 0.4, "shared": 0.7}]) + monkeypatch.setattr( + LM_EVAL, + "run_benchmarks", + lambda *_args, **_kwargs: next(responses), + ) + assert LM_EVAL.compare_models("original", "abliterated") == { + "a": {"original": 0.8, "abliterated": 0.0, "delta": -0.8}, + "b": {"original": 0.0, "abliterated": 0.4, "delta": 0.4}, + "shared": {"original": 0.5, "abliterated": 0.7, "delta": pytest.approx(0.2)}, + } + + +def test_report_sanitizes_sequences_objects_windows_paths_and_long_labels(): + custom = SimpleNamespace(value="/private/path/item") + sanitized = _sanitize_public_value({ + "items": ("C:\\private\\model.bin", custom), + "api-key": "must disappear", + "finite": 1.5, + "infinite": float("inf"), + }) + assert "api-key" not in sanitized + assert sanitized["finite"] == 1.5 + assert sanitized["infinite"] is None + assert sanitized["items"][0] == "model.bin" + assert "private/path" not in sanitized["items"][1] + + report = AblationReport(model_name="x" * 100) + assert report.to_dict()["model_name"].endswith("...") + assert len(report.to_dict()["model_name"]) == 80 + + +def test_report_summary_empty_and_populated(capsys): + AblationReport("empty").print_summary() + assert "No ablation results" in capsys.readouterr().out + + report = AblationReport("model") + report.add_baseline({"score": 0.0, "missing": None}) + report.add_result(AblationResult("s", "c", "d", {"score": 1.0, "missing": None})) + report.print_summary() + output = capsys.readouterr().out + assert "Ablation Results: model" in output + assert "unavailable" in output + + +def test_report_plot_boundaries(monkeypatch, tmp_path): + report = AblationReport("model") + report.add_baseline({"score": 2.0}) + report.add_result(AblationResult("s", "positive", "d", {"score": 3.0})) + report.add_result(AblationResult("s", "negative", "d", {"score": 1.0})) + + impact = tmp_path / "nested" / "impact.png" + impact.parent.mkdir() + report.plot_impact(output_path=impact) + assert impact.stat().st_size > 0 + + heatmap = tmp_path / "heatmap.png" + report.plot_heatmap(heatmap) + assert heatmap.stat().st_size > 0 + + show = Mock() + monkeypatch.setattr(plt, "show", show) + report.plot_impact(metric="score") + report.plot_heatmap() + assert show.call_count == 2 + + no_delta = AblationReport("model", baseline_metrics={"score": None}) + no_delta.add_result(AblationResult("s", "c", "d", {"score": 1.0})) + with pytest.raises(ValueError, match="No delta column"): + no_delta.plot_impact("score") + no_delta.plot_heatmap() diff --git a/tests/test_quality_policy.py b/tests/test_quality_policy.py index 588ee46..277f468 100644 --- a/tests/test_quality_policy.py +++ b/tests/test_quality_policy.py @@ -19,8 +19,21 @@ def test_mutation_campaign_preloads_native_modules_before_covered_line_discovery assert "mutate_only_covered_lines = true" in mutmut_config assert '"obliteratus/runtime_contracts.py"' in mutmut_config assert '"obliteratus/persistence_contracts.py"' in mutmut_config + assert '"obliteratus/evaluation/lm_eval_integration.py"' in mutmut_config + assert '"obliteratus/reporting/report.py"' not in mutmut_config assert '"tests/test_runtime_contracts.py"' in mutmut_config assert '"tests/test_persistence_contracts.py"' in mutmut_config + assert '"tests/test_lm_eval_reporting_contracts.py"' in mutmut_config + assert '"tests/test_telemetry_failure_contracts.py"' not in mutmut_config + assert '"tests/test_evaluation_reporting_contracts.py"' in Path( + "scripts/run_repeat_gate.py", + ).read_text() + assert '"tests/test_lm_eval_reporting_contracts.py"' in Path( + "scripts/run_repeat_gate.py", + ).read_text() + assert '"tests/test_telemetry_failure_contracts.py"' in Path( + "scripts/run_repeat_gate.py", + ).read_text() assert "import torch, yaml; from mutmut.__main__ import cli; cli()" in workflow @@ -54,9 +67,9 @@ def _coverage(): "obliteratus/pure.py": { "summary": { "num_statements": 100, - "covered_lines": 80, - "num_branches": 20, - "covered_branches": 15, + "covered_lines": 90, + "num_branches": 100, + "covered_branches": 78, }, }, "obliteratus/external.py": { @@ -76,8 +89,8 @@ def test_policy_and_exact_mature_floors_pass(): assert quality.validate_policy(policy) == [] measurement, failures = quality.validate_mature_cpu_scope(_coverage(), policy) assert failures == [] - assert measurement["line_percent"] == 80 - assert measurement["branch_percent"] == 75 + assert measurement["line_percent"] == 90 + assert measurement["branch_percent"] == 78 def test_floor_regression_requires_structured_reviewed_exception(): @@ -109,10 +122,10 @@ def test_exclusions_require_unique_traceable_environment_boundaries(): def test_mature_scope_rejects_regression_and_stale_exclusion(): policy = _policy() report = _coverage() - report["files"]["obliteratus/pure.py"]["summary"]["covered_lines"] = 79 + report["files"]["obliteratus/pure.py"]["summary"]["covered_lines"] = 89 _, failures = quality.validate_mature_cpu_scope(report, policy) assert failures == [ - "mature CPU line coverage 79.00% is below the 80.00% floor", + "mature CPU line coverage 89.00% is below the 90.00% floor", ] del report["files"]["obliteratus/external.py"] _, failures = quality.measure_mature_cpu_scope(report, policy) diff --git a/tests/test_telemetry_failure_contracts.py b/tests/test_telemetry_failure_contracts.py new file mode 100644 index 0000000..4dbd31b --- /dev/null +++ b/tests/test_telemetry_failure_contracts.py @@ -0,0 +1,295 @@ +"""Failure, storage, and Hub boundary contracts for opt-in telemetry.""" + +from __future__ import annotations + +import sys +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +import torch + +import obliteratus.telemetry as telemetry + + +@pytest.fixture(autouse=True) +def _restore_telemetry_globals(monkeypatch, tmp_path): + monkeypatch.setattr(telemetry, "TELEMETRY_FILE", tmp_path / "telemetry.jsonl") + monkeypatch.setattr(telemetry, "_TELEMETRY_DIR", tmp_path) + monkeypatch.setattr(telemetry, "_TELEMETRY_REPO", "") + monkeypatch.setattr(telemetry, "_hub_repo_created", False) + monkeypatch.setattr(telemetry, "_hub_sync_last", 0.0) + monkeypatch.setattr(telemetry, "_restore_done", False) + telemetry._sync_in_progress.clear() + + +def test_public_text_sanitizes_windows_paths_and_truncates(): + assert telemetry._sanitize_public_text(r"C:\private\model.bin") == "model.bin" + assert telemetry._sanitize_public_text("x" * 20, max_len=8) == "xxxxx..." + assert telemetry._sanitize_public_value(object()) + + +def test_telemetry_directory_prefers_explicit_and_home(monkeypatch, tmp_path): + explicit = tmp_path / "explicit" + monkeypatch.setenv("OBLITERATUS_DATA_DIR", str(explicit)) + monkeypatch.setattr(telemetry, "_ON_HF_SPACES", False) + assert telemetry._telemetry_dir() == explicit + + monkeypatch.setattr(telemetry, "_test_writable", lambda path: path.name == ".obliteratus") + monkeypatch.setattr(Path, "home", staticmethod(lambda: tmp_path / "home")) + assert telemetry._telemetry_dir() == tmp_path / "home" / ".obliteratus" + + +def test_telemetry_directory_retries_hf_mount_then_uses_it(monkeypatch): + monkeypatch.delenv("OBLITERATUS_DATA_DIR", raising=False) + monkeypatch.setattr(telemetry, "_ON_HF_SPACES", True) + monkeypatch.setattr(Path, "exists", lambda self: str(self) == "/data") + attempts = iter([False, True]) + monkeypatch.setattr(telemetry, "_test_writable", lambda _path: next(attempts)) + sleep = Mock() + monkeypatch.setattr(telemetry.time, "sleep", sleep) + + assert telemetry._telemetry_dir() == Path("/data/obliteratus") + sleep.assert_called_once_with(1) + + +def test_telemetry_directory_has_ephemeral_fallback(monkeypatch, tmp_path): + monkeypatch.delenv("OBLITERATUS_DATA_DIR", raising=False) + monkeypatch.setattr(telemetry, "_ON_HF_SPACES", False) + monkeypatch.setattr(telemetry, "_test_writable", lambda _path: False) + monkeypatch.setattr(Path, "home", staticmethod(lambda: tmp_path / "home")) + assert telemetry._telemetry_dir() == Path("/tmp/obliteratus_telemetry") + + +class _HubApi: + instances: list["_HubApi"] = [] + + def __init__(self, **kwargs): + self.kwargs = kwargs + self.create_repo = Mock() + self.repo_info = Mock() + self.upload_file = Mock() + self.list_repo_files = Mock(return_value=[]) + self.instances.append(self) + + +def _hub_module(api_class=_HubApi, **members): + return SimpleNamespace(HfApi=api_class, **members) + + +def test_ensure_hub_repo_create_and_existing_fallback(monkeypatch): + _HubApi.instances.clear() + monkeypatch.setitem(sys.modules, "huggingface_hub", _hub_module()) + assert telemetry._ensure_hub_repo("org/data") is True + _HubApi.instances[-1].create_repo.assert_called_once() + assert telemetry._ensure_hub_repo("org/data") is True + assert len(_HubApi.instances) == 1 + + telemetry._hub_repo_created = False + + class ExistingApi(_HubApi): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.create_repo.side_effect = RuntimeError("cannot create") + + monkeypatch.setitem(sys.modules, "huggingface_hub", _hub_module(ExistingApi)) + assert telemetry._ensure_hub_repo("org/data") is True + ExistingApi.instances[-1].repo_info.assert_called_once() + + +def test_ensure_hub_repo_fails_closed(monkeypatch): + class FailingApi(_HubApi): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.create_repo.side_effect = RuntimeError("create") + self.repo_info.side_effect = RuntimeError("lookup") + + monkeypatch.setitem(sys.modules, "huggingface_hub", _hub_module(FailingApi)) + assert telemetry._ensure_hub_repo("org/data") is False + + monkeypatch.setitem(sys.modules, "huggingface_hub", None) + assert telemetry._ensure_hub_repo("org/data") is False + + +def test_background_sync_short_circuits_and_uploads(monkeypatch, tmp_path): + telemetry._sync_in_progress.set() + telemetry._sync_to_hub_bg() + telemetry._sync_in_progress.clear() + + telemetry._sync_to_hub_bg() + assert not telemetry._sync_in_progress.is_set() + + telemetry._TELEMETRY_REPO = "org/data" + telemetry._sync_to_hub_bg() + assert not telemetry._sync_in_progress.is_set() + + telemetry.TELEMETRY_FILE.write_text("{}\n") + _HubApi.instances.clear() + monkeypatch.setitem(sys.modules, "huggingface_hub", _hub_module()) + monkeypatch.setattr(telemetry, "_ensure_hub_repo", lambda _repo: True) + monkeypatch.setattr(telemetry, "_instance_slug", lambda: "slug") + telemetry._sync_to_hub_bg() + _HubApi.instances[-1].upload_file.assert_called_once_with( + path_or_fileobj=str(telemetry.TELEMETRY_FILE), + path_in_repo="data/slug.jsonl", + repo_id="org/data", + repo_type="dataset", + commit_message="Auto-sync telemetry from slug", + ) + assert not telemetry._sync_in_progress.is_set() + + +def test_sync_scheduler_enforces_configuration_enablement_and_debounce(monkeypatch): + thread = Mock() + monkeypatch.setattr(telemetry.threading, "Thread", Mock(return_value=thread)) + monkeypatch.setattr(telemetry, "is_enabled", lambda: True) + + telemetry._schedule_hub_sync() + thread.start.assert_not_called() + + telemetry._TELEMETRY_REPO = "org/data" + monkeypatch.setattr(telemetry, "is_enabled", lambda: False) + telemetry._schedule_hub_sync() + thread.start.assert_not_called() + + monkeypatch.setattr(telemetry, "is_enabled", lambda: True) + monkeypatch.setattr(telemetry.time, "time", lambda: 100.0) + telemetry._schedule_hub_sync() + thread.start.assert_called_once() + telemetry._schedule_hub_sync() + thread.start.assert_called_once() + + +def test_hf_api_fetch_handles_listing_errors_and_file_errors(monkeypatch, tmp_path): + class ListingApi(_HubApi): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.list_repo_files.return_value = ["README.md"] + + monkeypatch.setitem(sys.modules, "huggingface_hub", _hub_module(ListingApi, hf_hub_download=Mock())) + assert telemetry._fetch_via_hf_api("org/data", 2) == [] + + class BrokenListingApi(_HubApi): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.list_repo_files.side_effect = RuntimeError("offline") + + monkeypatch.setitem(sys.modules, "huggingface_hub", _hub_module(BrokenListingApi, hf_hub_download=Mock())) + assert telemetry._fetch_via_hf_api("org/data", 2) == [] + + +def test_git_clone_fetch_parses_bounded_records_and_cleans_up(monkeypatch): + def fake_run(command, **_kwargs): + clone_dir = Path(command[-1]) + data = clone_dir / "data" + data.mkdir() + (data / "a.jsonl").write_text('\n{"id": 1}\ninvalid\n{"id": 2}\n') + return SimpleNamespace(returncode=0, stderr="") + + monkeypatch.setattr("subprocess.run", fake_run) + assert telemetry._fetch_via_git_clone("org/data", 1) == [{"id": 1}] + + +def test_git_clone_fetch_handles_failure_and_missing_data(monkeypatch): + monkeypatch.setattr( + "subprocess.run", + lambda *_args, **_kwargs: SimpleNamespace(returncode=1, stderr="denied"), + ) + assert telemetry._fetch_via_git_clone("org/data", 2) == [] + + monkeypatch.setattr( + "subprocess.run", + lambda *_args, **_kwargs: SimpleNamespace(returncode=0, stderr=""), + ) + assert telemetry._fetch_via_git_clone("org/data", 2) == [] + + +def test_gpu_detection_and_peak_vram(monkeypatch): + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(torch.cuda, "get_device_name", lambda _index: "GPU") + monkeypatch.setattr( + torch.cuda, + "get_device_properties", + lambda _index: SimpleNamespace(total_memory=8 * 1024**3), + ) + monkeypatch.setattr(torch.cuda, "max_memory_allocated", lambda: 3 * 1024**3) + monkeypatch.setattr(torch.cuda, "max_memory_reserved", lambda: 4 * 1024**3) + assert telemetry._detect_gpu() == ("GPU", 8.0) + assert telemetry._get_peak_vram() == { + "peak_allocated_gb": 3.0, + "peak_reserved_gb": 4.0, + } + assert telemetry._detect_model_family("org/Qwen-model") == "qwen" + assert telemetry._detect_model_family("org/other") == "unknown" + + +def test_direction_stats_and_excise_details_cover_optional_techniques(monkeypatch): + pipeline = SimpleNamespace( + refusal_directions={0: torch.tensor([1.0, 0.0]), 1: torch.tensor([0.0, 1.0])}, + refusal_subspaces={0: torch.eye(2)}, + _excise_modified_count=2, + _refusal_heads={0: [1, 2]}, + _sae_directions={0: torch.ones(2)}, + _expert_safety_scores={0: 1.0}, + _layer_excise_weights={0: 0.2, 1: 0.8}, + _expert_directions={0: torch.ones(2)}, + _steering_hooks=[object()], + invert_refusal=True, + project_embeddings=True, + activation_steering=True, + expert_transplant=True, + ) + stats = telemetry._direction_stats(pipeline) + assert stats["direction_norms"] == {"0": 1.0, "1": 1.0} + assert stats["mean_direction_persistence"] == 0.0 + assert stats["effective_ranks"] == {"0": 2.0} + + details = telemetry._extract_excise_details(pipeline) + assert details["modified_count"] == 2 + assert details["total_heads_projected"] == 2 + assert details["adaptive_weight_min"] == 0.2 + assert details["adaptive_weight_max"] == 0.8 + assert set(details["used_techniques"]) == { + "head_surgery", "sae_features", "expert_gating", "layer_adaptive", + "per_expert", "activation_steering", "inversion", "embedding_projection", + "expert_transplant", + } + + monkeypatch.setattr(torch.linalg, "svdvals", Mock(side_effect=RuntimeError("svd"))) + assert "effective_ranks" not in telemetry._direction_stats(pipeline) + + +def test_send_and_pipeline_failures_are_best_effort(monkeypatch, tmp_path): + monkeypatch.setattr(telemetry, "is_enabled", lambda: True) + monkeypatch.setattr(telemetry, "TELEMETRY_FILE", tmp_path / "missing" / "file.jsonl") + telemetry._send_sync({"schema_version": 2}) + + logger = Mock() + monkeypatch.setattr(telemetry, "logger", logger) + pipeline = SimpleNamespace(handle=SimpleNamespace(summary=Mock(side_effect=RuntimeError("summary")))) + telemetry.maybe_send_pipeline_report(pipeline) + telemetry.maybe_send_informed_report(pipeline, SimpleNamespace()) + assert logger.debug.call_count == 2 + + +def test_push_to_hub_failure_paths(monkeypatch, tmp_path): + assert telemetry.push_to_hub() is False + + telemetry.TELEMETRY_FILE.write_text('{}\n') + monkeypatch.setattr(telemetry, "read_telemetry", lambda: [{}]) + monkeypatch.setattr(telemetry, "_ensure_hub_repo", lambda _repo: False) + assert telemetry.push_to_hub("org/data") is False + + monkeypatch.setitem(sys.modules, "huggingface_hub", None) + monkeypatch.setattr(telemetry, "_ensure_hub_repo", lambda _repo: True) + assert telemetry.push_to_hub("org/data") is False + + +def test_restore_and_background_restore_absorb_boundary_failures(monkeypatch): + telemetry._TELEMETRY_REPO = "org/data" + monkeypatch.setattr(telemetry, "fetch_hub_records", Mock(side_effect=RuntimeError("offline"))) + assert telemetry.restore_from_hub() == 0 + + monkeypatch.setattr(telemetry, "restore_from_hub", Mock(side_effect=RuntimeError("offline"))) + telemetry._restore_from_hub_bg()