mirror of
https://github.com/elder-plinius/OBLITERATUS.git
synced 2026-08-17 16:37:30 +02:00
test: establish Gate 1 quality baseline (#90)
Establishes the mandatory testing, coverage, repeatability, mutation, packaging, supply-chain, and AIWG workspace baseline before feature integration.
This commit is contained in:
Joseph Magly
+47
-6
@@ -4,11 +4,38 @@
|
||||
"providers": [
|
||||
"codex"
|
||||
],
|
||||
"workspace": {
|
||||
"name": "bt6-obliteratus-maintenance"
|
||||
},
|
||||
"repos": [
|
||||
{
|
||||
"name": "obliteratus",
|
||||
"path": ".",
|
||||
"provider": "github",
|
||||
"allowed": [
|
||||
"read",
|
||||
"write",
|
||||
"commit",
|
||||
"push",
|
||||
"issue-comment",
|
||||
"service-action"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "bt6-aiwg-plugins",
|
||||
"path": "../bt6-aiwg-plugins",
|
||||
"provider": "github",
|
||||
"allowed": [
|
||||
"read"
|
||||
],
|
||||
"notes": "Authoritative BT6 code-audit and maintainer workflow source; mutations require separate authorization."
|
||||
}
|
||||
],
|
||||
"installed": {
|
||||
"all": {
|
||||
"version": "2026.8.8",
|
||||
"version": "2026.8.9",
|
||||
"source": "bundled",
|
||||
"installedAt": "2026-08-15T02:40:58.412Z",
|
||||
"installedAt": "2026-08-15T04:22:13.831Z",
|
||||
"deployedTo": {
|
||||
"codex": {
|
||||
"agents": 0,
|
||||
@@ -20,9 +47,9 @@
|
||||
"manifestHash": "sha256:21afaf03f52084ab895a6862fd45c284ba94366ee420eaa1f67e1b4acdc1be7c"
|
||||
},
|
||||
"bt6-maintainer": {
|
||||
"version": "0.2.0",
|
||||
"version": "0.3.0",
|
||||
"source": "project-local",
|
||||
"installedAt": "2026-08-15T02:40:59.927Z",
|
||||
"installedAt": "2026-08-15T04:23:14.957Z",
|
||||
"deployedTo": {
|
||||
"codex": {
|
||||
"agents": 5,
|
||||
@@ -31,7 +58,7 @@
|
||||
"rules": 1
|
||||
}
|
||||
},
|
||||
"manifestHash": "sha256:66a87462e0567646017a8e90f5ed7f31ba9959fcd2fd2b7f4486c33c32204c45",
|
||||
"manifestHash": "sha256:509e05c707975f4b2cd7a5c086c9545882022e2f68e921f45e2d31616334532c",
|
||||
"localPath": ".aiwg/plugins/bt6-maintainer/",
|
||||
"localType": "plugin",
|
||||
"manifestVersion": "1",
|
||||
@@ -65,7 +92,21 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"scripts": {},
|
||||
"scripts": {
|
||||
"deploy": "aiwg use all --provider codex && aiwg use bt6-maintainer --provider codex",
|
||||
"doctor": "aiwg doctor --verbose",
|
||||
"sync": "aiwg sync"
|
||||
},
|
||||
"remotes": {
|
||||
"primary": "origin",
|
||||
"issue_tracker": "origin",
|
||||
"ci": "origin",
|
||||
"tracker_actor": {
|
||||
"login": "jmagly",
|
||||
"via": "gh",
|
||||
"forbid_actors": []
|
||||
}
|
||||
},
|
||||
"security": {
|
||||
"threatAssessment": {
|
||||
"schemaVersion": "1",
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
# Gate 1 quality decision
|
||||
|
||||
Date: 2026-08-15
|
||||
Decision: CONDITIONAL PASS
|
||||
|
||||
All local Gate 1 criteria pass: mandatory CPU tests, repository and mature-scope
|
||||
coverage, warning budget, repeatability, bounded mutation, packaging, locked
|
||||
resolution, and supply-chain policy. No P0/P1 defect or flaky quarantine was
|
||||
identified.
|
||||
|
||||
The sole remaining condition is exact-head hosted CI after the branch is
|
||||
committed and pushed. Gate 1 becomes PASS only when all required GitHub Actions
|
||||
jobs are green at that immutable head and review threads are resolved.
|
||||
@@ -33,9 +33,15 @@ been verified.
|
||||
└── provenance/
|
||||
```
|
||||
|
||||
## Usage
|
||||
## Installation
|
||||
|
||||
On AIWG 2026.7.24 or newer, place the wrapper under the consuming repository's
|
||||
The repository root README is the canonical installation guide. It covers the
|
||||
full AIWG package, lightweight `@aiwg/cli`, and the dependency-free manual
|
||||
installer for Claude Code and Codex.
|
||||
|
||||
### AIWG-managed deployment
|
||||
|
||||
On AIWG 2026.8.0 or newer, place the wrapper under the consuming repository's
|
||||
`.aiwg/plugins/` directory and deploy it directly:
|
||||
|
||||
```bash
|
||||
@@ -63,6 +69,23 @@ Inspect health:
|
||||
aiwg doctor --project-local
|
||||
```
|
||||
|
||||
### Native deployment without AIWG
|
||||
|
||||
From the repository root:
|
||||
|
||||
```bash
|
||||
node scripts/install-bt6-maintainer.mjs \
|
||||
--platform all \
|
||||
--target /absolute/path/to/consumer \
|
||||
--with-profile \
|
||||
--dry-run
|
||||
```
|
||||
|
||||
The helper installs native provider artifacts plus a `.bt6-maintainer/`
|
||||
support tree and refuses collisions unless `--force` is explicit. See the
|
||||
[repository README](../../../README.md#route-c-no-aiwg-installation) for the
|
||||
review/apply and verification sequence.
|
||||
|
||||
AIWG #1998 currently prevents reliable automated removal of freshly deployed
|
||||
namespaced skill files. Inspect provider paths and preserve the registry record
|
||||
needed for recovery; do not use `--force` without verifying exact ownership.
|
||||
@@ -70,7 +93,7 @@ needed for recovery; do not use `--force` without verifying exact ownership.
|
||||
## Packaging status
|
||||
|
||||
The wrapper follows AIWG's project-local plugin schema and contains an addon
|
||||
payload under `payload/`. AIWG 2026.7.24 validates, packages, and directly
|
||||
payload under `payload/`. AIWG 2026.8.0 validates, packages, and directly
|
||||
deploys the wrapper. Remaining lifecycle gaps are:
|
||||
|
||||
- [#1996](https://git.integrolabs.net/roctinam/aiwg/issues/1996) — legacy
|
||||
@@ -87,10 +110,11 @@ npm run check
|
||||
npm run test:smoke
|
||||
```
|
||||
|
||||
The smoke test deploys the payload as a project-local addon in an isolated
|
||||
temporary repository for provider parity. Direct wrapper deployment is also
|
||||
covered during adoption; automated removal remains outside the passing smoke
|
||||
gate until #1998 is resolved.
|
||||
The smoke test deploys the wrapper in an isolated temporary repository for
|
||||
provider parity. Manual installer tests cover dry-run, native Claude and Codex
|
||||
layouts, support files, idempotency, collision refusal, and explicit
|
||||
replacement. Automated removal remains outside the passing smoke gate until
|
||||
#1998 is resolved.
|
||||
|
||||
## Supported repository families
|
||||
|
||||
@@ -104,7 +128,7 @@ gate until #1998 is resolved.
|
||||
|
||||
- Claude: full addon deployment (agents, skills, and guardrail rule).
|
||||
- Codex: full deployment through AIWG's shared Agent Skills surface plus agent
|
||||
TOMLs and guardrail rule on AIWG 2026.7.24 or newer.
|
||||
TOMLs and guardrail rule on AIWG 2026.8.0 or newer.
|
||||
|
||||
## Source and license
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"id": "bt6-maintainer",
|
||||
"type": "plugin",
|
||||
"name": "bt6-maintainer",
|
||||
"version": "0.2.0",
|
||||
"version": "0.3.0",
|
||||
"description": "Cross-repository maintenance and external-provider review for BT6 research and support tooling.",
|
||||
"manifestVersion": "1",
|
||||
"platforms": {
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"id": "bt6-maintainer",
|
||||
"type": "addon",
|
||||
"name": "bt6-maintainer",
|
||||
"version": "0.2.0",
|
||||
"version": "0.3.0",
|
||||
"description": "Cross-repository queue, review, issue, provider-trust, and merge stewardship for BT6 research and support tooling.",
|
||||
"manifestVersion": "1",
|
||||
"platforms": {
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"version": "1",
|
||||
"project": {
|
||||
"id": "obliteratus",
|
||||
"name": "OBLITERATUS",
|
||||
"description": "Managed project-specific orientation for OBLITERATUS."
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
# Gate 1 coverage report
|
||||
|
||||
Date: 2026-08-15
|
||||
Coverage format: coverage.py branch JSON v3
|
||||
|
||||
| Scope | Measured | Enforced floor | Status |
|
||||
|---|---:|---:|---|
|
||||
| Repository statements | 70.20% | 70% | PASS |
|
||||
| Repository branches | 57.41% | 55% | PASS |
|
||||
| Mature CPU statements | 92.23% | 90% | PASS |
|
||||
| Mature CPU branches | 80.36% | 78% | PASS |
|
||||
| Changed executable lines | 100.00% | 90% | PASS |
|
||||
|
||||
Critical changed surfaces are fully or strongly covered: evaluation baselines
|
||||
100%/90.91%, evaluator 100%/97.37%, lm-eval adapter 100%/100%, report
|
||||
100%/98.21%, and telemetry 91.62%/81.88% (statements/branches).
|
||||
|
||||
The immutable floors are encoded in `ci/test-quality-policy.json`, pytest,
|
||||
the CI workflow, and `scripts/check_quality_policy.py`. Exact-base
|
||||
touched-module no-regression is rerun after the candidate has a commit SHA and
|
||||
again by hosted CI.
|
||||
@@ -1,21 +1,25 @@
|
||||
# OBLITERATUS master test plan
|
||||
|
||||
Date: 2026-08-14
|
||||
Date: 2026-08-15
|
||||
Owner: maintainers
|
||||
Phase: construction
|
||||
|
||||
## Current verified baseline
|
||||
|
||||
Baseline commit: `340173e203757757451893347445ac8bd5fdc816`
|
||||
Gate 1 candidate base: `62a480fd6a8f9ef091e60ba8dbf6b0ca94a8b785`
|
||||
|
||||
The mandatory offline CPU selection currently completes locally in 72.55
|
||||
seconds with 1,195 passing tests, 9 conditionally deselected tests, no skips,
|
||||
and no warnings. Branch-enabled coverage is 66.44% line and 52.08% branch. The
|
||||
documented mature CPU-testable scope is 89.36% line and 75.41% branch. The
|
||||
repository also enforces 90% changed-line coverage, per-touched-module line and
|
||||
branch no-regression, a 70% selective mutation score, and zero unexpected
|
||||
warnings. Hosted CI is green for Python 3.10, 3.11, and 3.12, packaging,
|
||||
quality depth, supply chain, and the current exact `main` head.
|
||||
The mandatory offline CPU selection currently completes from a clean,
|
||||
non-editable Python 3.12 environment in 68.22 seconds with 1,468 passing tests,
|
||||
9 conditionally deselected tests, no skips, and no warnings. Branch-enabled
|
||||
coverage is 70.20% statements and 57.41% branches. The documented mature
|
||||
CPU-testable scope is 92.23% statements and 80.36% branches. The repository
|
||||
enforces 90% changed-line coverage, per-touched-module line and branch
|
||||
no-regression, a 75% selective mutation floor, and zero unexpected warnings.
|
||||
The current bounded campaign kills 809 of 886 mutants (91.31%), and 316
|
||||
high-consequence tests pass in each of three file orders and hash seeds with no
|
||||
flake candidates. Package, installed wheel/sdist, vulnerability, secret,
|
||||
license, and SBOM checks are green locally. Exact-head hosted CI remains the
|
||||
final Gate 1 merge condition.
|
||||
The latest conditional run is green for its selected pinned-model,
|
||||
external-evaluation, loopback-network, and operator-UI gates. CUDA,
|
||||
bitsandbytes, MPS, MLX, and remote execution were not selected and therefore
|
||||
@@ -114,7 +118,7 @@ Program exit:
|
||||
- Phases 0–2 are merged before ordinary feature work resumes;
|
||||
- all planned waves are delivered or explicitly accepted as conditional release
|
||||
gates with a runnable workflow and owner;
|
||||
- CPU-testable code reaches the mature 80% line / 75% branch target, or every
|
||||
- CPU-testable code reaches the mature 90% line / 78% branch target, or every
|
||||
remaining exclusion has a documented environment-bound rationale.
|
||||
|
||||
## Per-PR test contract
|
||||
@@ -191,16 +195,16 @@ Gate 1 requires all of the following on exact-head hosted CI:
|
||||
|
||||
| Surface | Verified baseline | Wave B target |
|
||||
|---|---:|---:|
|
||||
| Repository | 66.44% line / 52.08% branch | ≥70% / ≥55% |
|
||||
| Mature CPU scope | 89.36% / 75.41% | ≥90% / ≥78% |
|
||||
| `cli.py` | 97.91% / 81.43% | preserve ≥97% / raise to ≥85% |
|
||||
| `config.py` | 98.73% / 81.25% | preserve ≥98% / raise to ≥90% |
|
||||
| `device.py` | 96.47% / 92.42% | preserve ≥96% / ≥92% |
|
||||
| `architecture_profiles.py` | 88.62% / 82.95% | ≥92% / ≥85% |
|
||||
| `models/loader.py` | 85.79% / 84.09% | ≥90% / ≥87% |
|
||||
| `evaluation/evaluator.py` | 72.00% / 71.05% | ≥85% / ≥80% |
|
||||
| `reporting/report.py` | 82.94% / 64.81% | ≥90% / ≥80% |
|
||||
| `telemetry.py` | 71.07% / 62.40% | ≥80% / ≥75% |
|
||||
| Repository | 70.20% line / 57.41% branch | ≥70% / ≥55% |
|
||||
| Mature CPU scope | 92.23% / 80.36% | ≥90% / ≥78% |
|
||||
| `cli.py` | 97.93% / 89.15% | preserve ≥97% / raise to ≥85% |
|
||||
| `config.py` | 100.00% / 90.00% | preserve ≥98% / raise to ≥90% |
|
||||
| `device.py` | 100.00% / 95.00% | preserve ≥96% / ≥92% |
|
||||
| `architecture_profiles.py` | 97.52% / 92.31% | ≥92% / ≥85% |
|
||||
| `models/loader.py` | 91.01% / 88.14% | ≥90% / ≥87% |
|
||||
| `evaluation/evaluator.py` | 100.00% / 97.37% | ≥85% / ≥80% |
|
||||
| `reporting/report.py` | 100.00% / 98.21% | ≥90% / ≥80% |
|
||||
| `telemetry.py` | 91.62% / 81.88% | ≥80% / ≥75% |
|
||||
|
||||
Large runtime-bound modules must be split conceptually into CPU-testable
|
||||
decision logic and conditional execution. CPU-testable helpers extracted from
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
# Gate 1 regression report
|
||||
|
||||
Date: 2026-08-15
|
||||
Base: `62a480fd6a8f9ef091e60ba8dbf6b0ca94a8b785`
|
||||
|
||||
- Mandatory CPU regressions: none; 1,468 tests pass with no warnings.
|
||||
- Determinism regressions: none; three orders and hash seeds are green.
|
||||
- Packaging regressions: none in wheel, sdist, metadata, import, or CLI contracts.
|
||||
- Supply-chain regressions: none in vulnerability, secret, license, lock, or SBOM checks.
|
||||
- Coverage direction: repository and mature-scope measures exceed every Gate 1 target.
|
||||
|
||||
The exact committed-head touched-module comparison is intentionally deferred
|
||||
until the candidate commit exists; hosted CI independently regenerates coverage
|
||||
for the exact base SHA before allowing merge.
|
||||
@@ -0,0 +1,41 @@
|
||||
# Gate 1 test execution report
|
||||
|
||||
Date: 2026-08-15
|
||||
Scope: mandatory offline CPU baseline, quality depth, packaging, and supply chain
|
||||
Status: PASS locally; exact-head hosted CI pending
|
||||
|
||||
## Results
|
||||
|
||||
| Gate | Result |
|
||||
|---|---|
|
||||
| Mandatory CPU | 1,468 passed, 9 conditional deselections, 0 failures/skips/warnings |
|
||||
| Repository coverage | 70.20% statements, 57.41% branches |
|
||||
| Mature CPU scope | 92.23% statements, 80.36% branches |
|
||||
| Changed executable lines | 100% on the candidate worktree |
|
||||
| Repeat | 316 tests × 3 orders/seeds; no flakes or consistent failures |
|
||||
| Mutation | 809/886 killed (91.31%); 72 survived; 5 timed out |
|
||||
| Package | wheel and sdist build, metadata, isolated imports, and both CLI paths pass |
|
||||
| Supply chain | 0 vulnerabilities, 0 secret findings, 100 licenses accepted, SBOM bound |
|
||||
|
||||
The mandatory lane ran in a clean non-editable Python 3.12 environment with
|
||||
offline Hugging Face and accelerator settings. Wall time was 68.22 seconds and
|
||||
maximum RSS was 791,476 KB. The mutation lane ran under the CI-pinned Python
|
||||
3.12/tool configuration in 69.79 seconds with maximum RSS of 1,399,824 KB.
|
||||
|
||||
## Defects found and corrected
|
||||
|
||||
1. The lm-eval adapter treated a measured accuracy of `0.0` as missing.
|
||||
2. Public sanitizers did not reliably remove Windows paths on non-Windows hosts.
|
||||
3. Report heatmap generation surfaced a dependency deprecation as an unexpected warning.
|
||||
4. The evaluation package eagerly loaded heavyweight model dependencies for pure adapters;
|
||||
it now preserves the public API through lazy imports.
|
||||
|
||||
Each correction has a focused regression contract. Evaluation/reporting and
|
||||
telemetry failure contracts are also part of the deterministic repeat gate;
|
||||
the pure lm-eval adapter is part of bounded mutation testing.
|
||||
|
||||
## Remaining conditional evidence
|
||||
|
||||
CUDA, bitsandbytes, MPS, MLX, remote execution, real model downloads, and live
|
||||
external evaluation remain owned by their conditional gates. They are not
|
||||
required for this offline pull-request baseline and were not rerun here.
|
||||
@@ -328,8 +328,8 @@ jobs:
|
||||
fi
|
||||
"$TEST_ENV/bin/python" scripts/check_coverage_thresholds.py \
|
||||
"test-results/coverage-py${{ matrix.python-version }}.json" \
|
||||
--min-line 60 \
|
||||
--min-branch 42 \
|
||||
--min-line 70 \
|
||||
--min-branch 55 \
|
||||
--min-file obliteratus/device.py=70 \
|
||||
--min-file obliteratus/models/loader.py=70 \
|
||||
--min-file obliteratus/architecture_profiles.py=70 \
|
||||
|
||||
+7
-7
@@ -32,13 +32,13 @@ python -m obliteratus --help
|
||||
```
|
||||
|
||||
All tests must pass before submitting a PR. Tests are designed to run on CPU without downloading models.
|
||||
The mandatory gate currently requires at least 60% repository statement coverage,
|
||||
42% branch coverage, 90% coverage of changed executable lines, and 70% statement
|
||||
The mandatory gate currently requires at least 70% repository statement coverage,
|
||||
55% branch coverage, 90% coverage of changed executable lines, and 70% statement
|
||||
coverage in the device, loader, architecture-profile, CLI, simulated MLX,
|
||||
evaluation-metric, reporting, community-contribution, and telemetry boundary
|
||||
modules. New changes should raise these floors rather than consume the existing
|
||||
margin. A separately measured mature CPU-only scope must remain at or above 80%
|
||||
statement and 75% branch coverage; its environment-bound exclusions and their
|
||||
margin. A separately measured mature CPU-only scope must remain at or above 90%
|
||||
statement and 78% branch coverage; its environment-bound exclusions and their
|
||||
conditional-test ownership are versioned in `ci/test-quality-policy.json`.
|
||||
Every changed production module is also compared with coverage generated from
|
||||
the exact base commit. Line and branch coverage may not regress independently,
|
||||
@@ -47,15 +47,15 @@ changed-line requirement.
|
||||
|
||||
The quality-depth job repeats the highest-consequence pure tests three times in
|
||||
different file orders and with different deterministic hash seeds. It also runs
|
||||
selective mutation testing over configuration parsing and the coverage-policy
|
||||
gate. The initial mutation floor is 70%. Run these checks locally with:
|
||||
selective mutation testing over configuration, quality-policy, and pure
|
||||
evaluation contracts. The mutation floor is 75%. Run these checks locally with:
|
||||
|
||||
```bash
|
||||
uv sync --locked --extra dev --group quality
|
||||
python scripts/run_repeat_gate.py --output test-results/repeat-gate.json
|
||||
mutmut run --max-children 4
|
||||
mutmut export-cicd-stats
|
||||
python scripts/check_mutation_score.py mutants/mutmut-cicd-stats.json --minimum 70
|
||||
python scripts/check_mutation_score.py mutants/mutmut-cicd-stats.json --minimum 75
|
||||
```
|
||||
|
||||
The repeat and mutation selections are intentionally bounded; on the Wave A
|
||||
|
||||
@@ -761,14 +761,14 @@ pip install -e ".[dev]"
|
||||
pytest
|
||||
```
|
||||
|
||||
The mandatory CPU suite contains more than 1,100 tests, including a
|
||||
The mandatory CPU suite contains more than 1,400 tests, including a
|
||||
repository-owned synthetic model that exercises the offline pipeline,
|
||||
installed-wheel CLI, study runner, transactional checkpoint recovery, and resumable
|
||||
auto-obliteration state. The suite also covers model/device/quantization/MLX
|
||||
boundaries, all analysis modules, architecture detection, visualization sanitization,
|
||||
community contributions, edge cases, and evaluation metrics. CI enforces at least
|
||||
60% repository statement coverage, 42% repository branch coverage, 90%
|
||||
changed-line coverage, and 80% statement / 75% branch coverage for the documented
|
||||
70% repository statement coverage, 55% repository branch coverage, 90%
|
||||
changed-line coverage, and 90% statement / 78% branch coverage for the documented
|
||||
mature CPU-only scope. Deterministic property, order-repeat, and selective mutation
|
||||
gates provide additional depth for numerical and policy-critical behavior. CI
|
||||
also rejects line or branch regressions in each touched production module by
|
||||
|
||||
+3
-3
@@ -43,12 +43,12 @@ accelerator, or remote-execution credentials.
|
||||
Canonical required checks:
|
||||
|
||||
- the exact Ruff F and actionlint command set in [.github/workflows/ci.yml](.github/workflows/ci.yml);
|
||||
- `python -m pytest` with at least 60% repository line coverage and 42% branch
|
||||
- `python -m pytest` with at least 70% repository line coverage and 55% branch
|
||||
coverage;
|
||||
- at least 90% changed-line coverage plus no line or branch regression in any
|
||||
touched production module, compared with coverage from the exact base commit;
|
||||
- at least 80% line and 75% branch coverage for the documented mature
|
||||
CPU-testable scope, plus a 70% selective mutation score and zero unexpected
|
||||
- at least 90% line and 78% branch coverage for the documented mature
|
||||
CPU-testable scope, plus a 75% selective mutation score and zero unexpected
|
||||
warnings;
|
||||
- `python -m build --sdist --wheel`
|
||||
- `python -c 'import obliteratus; print(obliteratus.__version__)'`
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"minimums": {
|
||||
"repository_statement": 60.0,
|
||||
"repository_branch": 42.0,
|
||||
"repository_statement": 70.0,
|
||||
"repository_branch": 55.0,
|
||||
"changed_line": 90.0,
|
||||
"mature_cpu_statement": 80.0,
|
||||
"mature_cpu_branch": 75.0,
|
||||
"mature_cpu_statement": 90.0,
|
||||
"mature_cpu_branch": 78.0,
|
||||
"mutation_score": 75.0,
|
||||
"warning_budget": 0
|
||||
},
|
||||
@@ -55,20 +55,6 @@
|
||||
"conditional_issue": "https://github.com/elder-plinius/OBLITERATUS/issues/71",
|
||||
"conditional_gate": "network-services"
|
||||
},
|
||||
{
|
||||
"path": "obliteratus/evaluation/baselines.py",
|
||||
"boundary": "model-runtime",
|
||||
"rationale": "Control baselines mutate and repeatedly evaluate a live transformer pipeline.",
|
||||
"conditional_issue": "https://github.com/elder-plinius/OBLITERATUS/issues/71",
|
||||
"conditional_gate": "model-download-runtime"
|
||||
},
|
||||
{
|
||||
"path": "obliteratus/evaluation/evaluator.py",
|
||||
"boundary": "model-runtime",
|
||||
"rationale": "The remaining branches tokenize datasets and execute causal-LM or classifier forward passes.",
|
||||
"conditional_issue": "https://github.com/elder-plinius/OBLITERATUS/issues/71",
|
||||
"conditional_gate": "model-download-runtime"
|
||||
},
|
||||
{
|
||||
"path": "obliteratus/evaluation/heretic_eval.py",
|
||||
"boundary": "external-evaluator",
|
||||
@@ -76,13 +62,6 @@
|
||||
"conditional_issue": "https://github.com/elder-plinius/OBLITERATUS/issues/71",
|
||||
"conditional_gate": "external-evaluation"
|
||||
},
|
||||
{
|
||||
"path": "obliteratus/evaluation/lm_eval_integration.py",
|
||||
"boundary": "external-evaluator",
|
||||
"rationale": "This optional adapter requires lm-evaluation-harness, benchmark downloads, and a model runtime.",
|
||||
"conditional_issue": "https://github.com/elder-plinius/OBLITERATUS/issues/71",
|
||||
"conditional_gate": "external-evaluation"
|
||||
},
|
||||
{
|
||||
"path": "obliteratus/informed_pipeline.py",
|
||||
"boundary": "model-runtime",
|
||||
|
||||
+35
-9
@@ -129,7 +129,8 @@
|
||||
"tests/test_hard_negative.py",
|
||||
"tests/test_prompt_sources.py",
|
||||
"tests/test_module_imports.py",
|
||||
"tests/test_telemetry.py"
|
||||
"tests/test_telemetry.py",
|
||||
"tests/test_telemetry_failure_contracts.py"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -201,7 +202,9 @@
|
||||
"tests/test_benchmark_plots.py",
|
||||
"tests/test_benchmarks.py",
|
||||
"tests/test_evaluator.py",
|
||||
"tests/test_evaluation_reporting_contracts.py",
|
||||
"tests/test_heretic_eval.py",
|
||||
"tests/test_lm_eval_reporting_contracts.py",
|
||||
"tests/test_metrics.py",
|
||||
"tests/test_property_contracts.py"
|
||||
]
|
||||
@@ -212,7 +215,11 @@
|
||||
"description": "Report schemas, deterministic serialization, plotting, and persisted result artifacts",
|
||||
"contract_types": ["research-output", "persistence", "numerical-invariant"],
|
||||
"paths": ["obliteratus/reporting/report.py"],
|
||||
"required_tests": ["tests/test_report.py"]
|
||||
"required_tests": [
|
||||
"tests/test_report.py",
|
||||
"tests/test_evaluation_reporting_contracts.py",
|
||||
"tests/test_lm_eval_reporting_contracts.py"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "ablation-strategies",
|
||||
@@ -268,14 +275,21 @@
|
||||
"path": "obliteratus/telemetry.py",
|
||||
"risk_class": "cpu-contract",
|
||||
"risk": "research telemetry schema, aggregation, and filesystem behavior",
|
||||
"required_tests": ["tests/test_telemetry.py"],
|
||||
"required_tests": [
|
||||
"tests/test_telemetry.py",
|
||||
"tests/test_telemetry_failure_contracts.py"
|
||||
],
|
||||
"conditional_gates": []
|
||||
},
|
||||
{
|
||||
"path": "obliteratus/reporting/report.py",
|
||||
"risk_class": "cpu-contract",
|
||||
"risk": "report schema, output paths, serialization, and plotting contracts",
|
||||
"required_tests": ["tests/test_report.py"],
|
||||
"required_tests": [
|
||||
"tests/test_report.py",
|
||||
"tests/test_evaluation_reporting_contracts.py",
|
||||
"tests/test_lm_eval_reporting_contracts.py"
|
||||
],
|
||||
"conditional_gates": []
|
||||
},
|
||||
{
|
||||
@@ -363,14 +377,22 @@
|
||||
"path": "obliteratus/evaluation/baselines.py",
|
||||
"risk_class": "mixed-runtime",
|
||||
"risk": "control baseline mutation and comparative evaluation",
|
||||
"required_tests": ["tests/test_evaluator.py", "tests/conditional/test_model_download_runtime.py"],
|
||||
"required_tests": [
|
||||
"tests/test_evaluator.py",
|
||||
"tests/test_evaluation_reporting_contracts.py",
|
||||
"tests/conditional/test_model_download_runtime.py"
|
||||
],
|
||||
"conditional_gates": ["model-download-runtime"]
|
||||
},
|
||||
{
|
||||
"path": "obliteratus/evaluation/evaluator.py",
|
||||
"risk_class": "mixed-runtime",
|
||||
"risk": "dataset bounds, causal/classifier evaluation, and result semantics",
|
||||
"required_tests": ["tests/test_evaluator.py", "tests/test_offline_integration.py"],
|
||||
"required_tests": [
|
||||
"tests/test_evaluator.py",
|
||||
"tests/test_evaluation_reporting_contracts.py",
|
||||
"tests/test_offline_integration.py"
|
||||
],
|
||||
"conditional_gates": ["model-download-runtime"]
|
||||
},
|
||||
{
|
||||
@@ -403,9 +425,13 @@
|
||||
},
|
||||
{
|
||||
"path": "obliteratus/evaluation/lm_eval_integration.py",
|
||||
"risk_class": "conditional-runtime",
|
||||
"risk": "lm-evaluation-harness integration and benchmark result translation",
|
||||
"required_tests": ["tests/test_module_imports.py", "tests/conditional/test_external_evaluation_runtime.py"],
|
||||
"risk_class": "mixed-runtime",
|
||||
"risk": "external benchmark adapter arguments, metric extraction, measured-zero semantics, and comparison output",
|
||||
"required_tests": [
|
||||
"tests/test_lm_eval_reporting_contracts.py",
|
||||
"tests/test_module_imports.py",
|
||||
"tests/conditional/test_external_evaluation_runtime.py"
|
||||
],
|
||||
"conditional_gates": ["external-evaluation"]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -1,71 +1,108 @@
|
||||
from obliteratus.evaluation.evaluator import Evaluator
|
||||
from obliteratus.evaluation.metrics import perplexity, accuracy, f1_score_metric
|
||||
from obliteratus.evaluation.advanced_metrics import (
|
||||
refusal_rate,
|
||||
refusal_rate_with_ci,
|
||||
token_kl_divergence,
|
||||
first_token_kl_divergence,
|
||||
effective_rank,
|
||||
effective_rank_change,
|
||||
activation_cosine_similarity,
|
||||
linear_cka,
|
||||
refusal_projection_magnitude,
|
||||
AbliterationEvalResult,
|
||||
format_eval_report,
|
||||
)
|
||||
from obliteratus.evaluation.baselines import (
|
||||
random_direction_ablation,
|
||||
direction_specificity_test,
|
||||
)
|
||||
from obliteratus.evaluation.heretic_eval import (
|
||||
arditi_refusal_rate,
|
||||
harmbench_asr,
|
||||
unload_harmbench_classifier,
|
||||
first_token_kl_on_prompts,
|
||||
run_lm_eval,
|
||||
load_jailbreakbench_prompts,
|
||||
run_full_heretic_eval,
|
||||
format_comparison_table,
|
||||
HereticComparisonResult,
|
||||
LM_EVAL_BENCHMARKS,
|
||||
)
|
||||
from obliteratus.evaluation.lm_eval_integration import (
|
||||
run_benchmarks,
|
||||
compare_models,
|
||||
)
|
||||
"""Evaluation APIs with lazy imports for optional and heavyweight boundaries."""
|
||||
|
||||
__all__ = [
|
||||
"Evaluator",
|
||||
"perplexity",
|
||||
"accuracy",
|
||||
"f1_score_metric",
|
||||
"refusal_rate",
|
||||
"token_kl_divergence",
|
||||
"first_token_kl_divergence",
|
||||
"effective_rank",
|
||||
"effective_rank_change",
|
||||
"activation_cosine_similarity",
|
||||
"linear_cka",
|
||||
"refusal_projection_magnitude",
|
||||
"AbliterationEvalResult",
|
||||
"format_eval_report",
|
||||
# Community-standard evaluation (Heretics/Arditi protocol)
|
||||
"arditi_refusal_rate",
|
||||
"harmbench_asr",
|
||||
"unload_harmbench_classifier",
|
||||
"first_token_kl_on_prompts",
|
||||
"run_lm_eval",
|
||||
"load_jailbreakbench_prompts",
|
||||
"run_full_heretic_eval",
|
||||
"format_comparison_table",
|
||||
"HereticComparisonResult",
|
||||
"LM_EVAL_BENCHMARKS",
|
||||
# lm-evaluation-harness integration
|
||||
"run_benchmarks",
|
||||
"compare_models",
|
||||
# Statistical evaluation
|
||||
"refusal_rate_with_ci",
|
||||
# Baselines
|
||||
"random_direction_ablation",
|
||||
"direction_specificity_test",
|
||||
]
|
||||
from __future__ import annotations
|
||||
|
||||
from importlib import import_module
|
||||
from typing import Any
|
||||
|
||||
|
||||
_EXPORTS = {
|
||||
"Evaluator": ("obliteratus.evaluation.evaluator", "Evaluator"),
|
||||
"perplexity": ("obliteratus.evaluation.metrics", "perplexity"),
|
||||
"accuracy": ("obliteratus.evaluation.metrics", "accuracy"),
|
||||
"f1_score_metric": ("obliteratus.evaluation.metrics", "f1_score_metric"),
|
||||
"refusal_rate": ("obliteratus.evaluation.advanced_metrics", "refusal_rate"),
|
||||
"refusal_rate_with_ci": (
|
||||
"obliteratus.evaluation.advanced_metrics",
|
||||
"refusal_rate_with_ci",
|
||||
),
|
||||
"token_kl_divergence": (
|
||||
"obliteratus.evaluation.advanced_metrics",
|
||||
"token_kl_divergence",
|
||||
),
|
||||
"first_token_kl_divergence": (
|
||||
"obliteratus.evaluation.advanced_metrics",
|
||||
"first_token_kl_divergence",
|
||||
),
|
||||
"effective_rank": ("obliteratus.evaluation.advanced_metrics", "effective_rank"),
|
||||
"effective_rank_change": (
|
||||
"obliteratus.evaluation.advanced_metrics",
|
||||
"effective_rank_change",
|
||||
),
|
||||
"activation_cosine_similarity": (
|
||||
"obliteratus.evaluation.advanced_metrics",
|
||||
"activation_cosine_similarity",
|
||||
),
|
||||
"linear_cka": ("obliteratus.evaluation.advanced_metrics", "linear_cka"),
|
||||
"refusal_projection_magnitude": (
|
||||
"obliteratus.evaluation.advanced_metrics",
|
||||
"refusal_projection_magnitude",
|
||||
),
|
||||
"AbliterationEvalResult": (
|
||||
"obliteratus.evaluation.advanced_metrics",
|
||||
"AbliterationEvalResult",
|
||||
),
|
||||
"format_eval_report": (
|
||||
"obliteratus.evaluation.advanced_metrics",
|
||||
"format_eval_report",
|
||||
),
|
||||
"random_direction_ablation": (
|
||||
"obliteratus.evaluation.baselines",
|
||||
"random_direction_ablation",
|
||||
),
|
||||
"direction_specificity_test": (
|
||||
"obliteratus.evaluation.baselines",
|
||||
"direction_specificity_test",
|
||||
),
|
||||
"arditi_refusal_rate": ("obliteratus.evaluation.heretic_eval", "arditi_refusal_rate"),
|
||||
"harmbench_asr": ("obliteratus.evaluation.heretic_eval", "harmbench_asr"),
|
||||
"unload_harmbench_classifier": (
|
||||
"obliteratus.evaluation.heretic_eval",
|
||||
"unload_harmbench_classifier",
|
||||
),
|
||||
"first_token_kl_on_prompts": (
|
||||
"obliteratus.evaluation.heretic_eval",
|
||||
"first_token_kl_on_prompts",
|
||||
),
|
||||
"run_lm_eval": ("obliteratus.evaluation.heretic_eval", "run_lm_eval"),
|
||||
"load_jailbreakbench_prompts": (
|
||||
"obliteratus.evaluation.heretic_eval",
|
||||
"load_jailbreakbench_prompts",
|
||||
),
|
||||
"run_full_heretic_eval": (
|
||||
"obliteratus.evaluation.heretic_eval",
|
||||
"run_full_heretic_eval",
|
||||
),
|
||||
"format_comparison_table": (
|
||||
"obliteratus.evaluation.heretic_eval",
|
||||
"format_comparison_table",
|
||||
),
|
||||
"HereticComparisonResult": (
|
||||
"obliteratus.evaluation.heretic_eval",
|
||||
"HereticComparisonResult",
|
||||
),
|
||||
"LM_EVAL_BENCHMARKS": (
|
||||
"obliteratus.evaluation.heretic_eval",
|
||||
"LM_EVAL_BENCHMARKS",
|
||||
),
|
||||
"run_benchmarks": (
|
||||
"obliteratus.evaluation.lm_eval_integration",
|
||||
"run_benchmarks",
|
||||
),
|
||||
"compare_models": (
|
||||
"obliteratus.evaluation.lm_eval_integration",
|
||||
"compare_models",
|
||||
),
|
||||
}
|
||||
|
||||
__all__ = list(_EXPORTS)
|
||||
|
||||
|
||||
def __getattr__(name: str) -> Any:
|
||||
try:
|
||||
module_name, attribute = _EXPORTS[name]
|
||||
except KeyError:
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None
|
||||
value = getattr(import_module(module_name), attribute)
|
||||
globals()[name] = value
|
||||
return value
|
||||
|
||||
@@ -103,7 +103,9 @@ def run_benchmarks(
|
||||
scores: dict[str, float] = {}
|
||||
for task_name, task_result in results.get("results", {}).items():
|
||||
# lm-eval uses "acc" or "acc_norm" depending on the task
|
||||
acc = task_result.get("acc,none") or task_result.get("acc_norm,none")
|
||||
acc = task_result.get("acc,none")
|
||||
if acc is None:
|
||||
acc = task_result.get("acc_norm,none")
|
||||
if acc is not None:
|
||||
scores[task_name] = acc
|
||||
else:
|
||||
|
||||
@@ -5,8 +5,9 @@ from __future__ import annotations
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import warnings
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from pathlib import Path, PureWindowsPath
|
||||
from typing import Any
|
||||
|
||||
import pandas as pd
|
||||
@@ -19,8 +20,10 @@ _SENSITIVE_KEY_RE = re.compile(
|
||||
|
||||
def _sanitize_label(text: str, max_len: int = 80) -> str:
|
||||
"""Strip filesystem paths, tokens, and overly-long strings from labels."""
|
||||
if text.startswith("/") or re.match(r"^[A-Za-z]:[\\/]", text):
|
||||
if text.startswith("/"):
|
||||
text = Path(text).name
|
||||
elif re.match(r"^[A-Za-z]:[\\/]", text):
|
||||
text = PureWindowsPath(text).name
|
||||
text = re.sub(
|
||||
r"(?:/[A-Za-z0-9_.-]+){2,}", lambda match: Path(match.group()).name, text,
|
||||
)
|
||||
@@ -257,7 +260,17 @@ class AblationReport:
|
||||
pivot.columns = [c.replace("_pct_change", "") for c in pivot.columns]
|
||||
|
||||
fig, ax = plt.subplots(figsize=(max(6, len(pivot.columns) * 2), max(4, len(pivot) * 0.4)))
|
||||
sns.heatmap(pivot, annot=True, fmt=".1f", cmap="RdYlGn_r", center=0, ax=ax)
|
||||
# Seaborn currently calls Matplotlib's pending-deprecated ``set_bad``
|
||||
# internally. Keep the repository's zero-warning contract focused on
|
||||
# our code while containing this specific third-party compatibility
|
||||
# warning to the call that emits it.
|
||||
with warnings.catch_warnings():
|
||||
warnings.filterwarnings(
|
||||
"ignore",
|
||||
message="The set_bad function will be deprecated.*",
|
||||
category=PendingDeprecationWarning,
|
||||
)
|
||||
sns.heatmap(pivot, annot=True, fmt=".1f", cmap="RdYlGn_r", center=0, ax=ax)
|
||||
ax.set_title(f"Ablation % Change — {_sanitize_label(self.model_name)}")
|
||||
|
||||
plt.tight_layout()
|
||||
|
||||
@@ -39,7 +39,7 @@ import threading
|
||||
import uuid
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from pathlib import Path, PureWindowsPath
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -62,8 +62,10 @@ _PUBLIC_METRIC_RANGES: dict[str, tuple[float | None, float | None]] = {
|
||||
|
||||
def _sanitize_public_text(value: Any, max_len: int = 240) -> str:
|
||||
text = str(value)
|
||||
if text.startswith("/") or re.match(r"^[A-Za-z]:[\\/]", text):
|
||||
if text.startswith("/"):
|
||||
text = Path(text).name
|
||||
elif re.match(r"^[A-Za-z]:[\\/]", text):
|
||||
text = PureWindowsPath(text).name
|
||||
text = re.sub(r"\bhf_[A-Za-z0-9]{6,}\b", "<TOKEN>", text)
|
||||
text = re.sub(r"\bgh[pousr]_[A-Za-z0-9]{12,}\b", "<TOKEN>", text)
|
||||
text = re.sub(r"\bgithub_pat_[A-Za-z0-9_]{12,}\b", "<TOKEN>", text)
|
||||
|
||||
+3
-1
@@ -95,7 +95,7 @@ addopts = [
|
||||
"--strict-markers",
|
||||
"--cov=obliteratus",
|
||||
"--cov-report=term-missing",
|
||||
"--cov-fail-under=60",
|
||||
"--cov-fail-under=70",
|
||||
]
|
||||
filterwarnings = [
|
||||
"error",
|
||||
@@ -121,6 +121,7 @@ only_mutate = [
|
||||
"obliteratus/runtime_contracts.py",
|
||||
"obliteratus/persistence_contracts.py",
|
||||
"obliteratus/remote_contracts.py",
|
||||
"obliteratus/evaluation/lm_eval_integration.py",
|
||||
"scripts/check_coverage_thresholds.py",
|
||||
]
|
||||
pytest_add_cli_args = ["--no-cov", "-q"]
|
||||
@@ -128,6 +129,7 @@ pytest_add_cli_args_test_selection = [
|
||||
"tests/test_config.py",
|
||||
"tests/test_config_properties.py",
|
||||
"tests/test_coverage_thresholds.py",
|
||||
"tests/test_lm_eval_reporting_contracts.py",
|
||||
"tests/test_numerical_contracts.py",
|
||||
"tests/test_persistence_contracts.py",
|
||||
"tests/test_remote_contracts.py",
|
||||
|
||||
@@ -13,11 +13,11 @@ from typing import Any
|
||||
|
||||
|
||||
BASELINE_FLOORS = {
|
||||
"repository_statement": 60.0,
|
||||
"repository_branch": 42.0,
|
||||
"repository_statement": 70.0,
|
||||
"repository_branch": 55.0,
|
||||
"changed_line": 90.0,
|
||||
"mature_cpu_statement": 80.0,
|
||||
"mature_cpu_branch": 75.0,
|
||||
"mature_cpu_statement": 90.0,
|
||||
"mature_cpu_branch": 78.0,
|
||||
"mutation_score": 75.0,
|
||||
"warning_budget": 0.0,
|
||||
}
|
||||
|
||||
@@ -18,6 +18,8 @@ DEFAULT_TESTS = (
|
||||
"tests/test_config.py",
|
||||
"tests/test_config_properties.py",
|
||||
"tests/test_coverage_thresholds.py",
|
||||
"tests/test_evaluation_reporting_contracts.py",
|
||||
"tests/test_lm_eval_reporting_contracts.py",
|
||||
"tests/test_numerical_contracts.py",
|
||||
"tests/test_persistence_contracts.py",
|
||||
"tests/test_property_contracts.py",
|
||||
@@ -25,6 +27,7 @@ DEFAULT_TESTS = (
|
||||
"tests/test_metrics.py",
|
||||
"tests/test_remote_contracts.py",
|
||||
"tests/test_runtime_contracts.py",
|
||||
"tests/test_telemetry_failure_contracts.py",
|
||||
)
|
||||
HASH_SEEDS = ("0", "1", "8675309")
|
||||
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
"""Boundary contracts for evaluation baselines, adapters, and public reports."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from obliteratus.evaluation.baselines import (
|
||||
direction_specificity_test,
|
||||
random_direction_ablation,
|
||||
)
|
||||
from obliteratus.evaluation.evaluator import Evaluator
|
||||
|
||||
|
||||
class _Encoding(dict):
|
||||
def to(self, _device):
|
||||
return self
|
||||
|
||||
|
||||
class _ClassificationDataset:
|
||||
def __init__(self, texts, labels):
|
||||
self.texts = list(texts)
|
||||
self.labels = list(labels)
|
||||
self.selected = None
|
||||
|
||||
def __len__(self):
|
||||
return len(self.texts)
|
||||
|
||||
def __getitem__(self, key):
|
||||
if isinstance(key, slice):
|
||||
return {"text": self.texts[key], "label": self.labels[key]}
|
||||
return {"text": self.texts[key], "label": self.labels[key]}
|
||||
|
||||
def select(self, indices):
|
||||
indices = list(indices)
|
||||
self.selected = indices
|
||||
return _ClassificationDataset(
|
||||
[self.texts[index] for index in indices],
|
||||
[self.labels[index] for index in indices],
|
||||
)
|
||||
|
||||
|
||||
class _ClassificationModel(nn.Module):
|
||||
def __init__(self, batches):
|
||||
super().__init__()
|
||||
self.anchor = nn.Parameter(torch.zeros(()))
|
||||
self.batches = list(batches)
|
||||
|
||||
def forward(self, **_encodings):
|
||||
return SimpleNamespace(logits=torch.tensor(self.batches.pop(0)))
|
||||
|
||||
|
||||
def test_evaluator_dispatches_classification_and_rejects_unknown_tasks():
|
||||
dataset = _ClassificationDataset(["a", "b", "ignored"], [1, 0, 1])
|
||||
tokenizer = Mock(return_value=_Encoding(input_ids=torch.ones((2, 1), dtype=torch.long)))
|
||||
model = _ClassificationModel([[[0.0, 2.0], [3.0, 0.0]]])
|
||||
handle = SimpleNamespace(model=model, tokenizer=tokenizer, task="classification")
|
||||
|
||||
result = Evaluator(
|
||||
handle,
|
||||
dataset,
|
||||
metrics=["accuracy", "f1"],
|
||||
batch_size=2,
|
||||
max_samples=2,
|
||||
).evaluate()
|
||||
|
||||
assert dataset.selected == [0, 1]
|
||||
assert result == {"accuracy": 1.0, "f1": 1.0}
|
||||
tokenizer.assert_called_once()
|
||||
|
||||
handle.task = "unsupported"
|
||||
with pytest.raises(ValueError, match="Unsupported task: unsupported"):
|
||||
Evaluator(handle, dataset).evaluate()
|
||||
|
||||
|
||||
def test_classification_returns_only_requested_metrics():
|
||||
dataset = _ClassificationDataset(["a"], [0])
|
||||
tokenizer = Mock(return_value=_Encoding(input_ids=torch.ones((1, 1), dtype=torch.long)))
|
||||
model = _ClassificationModel([[[2.0, 0.0]]])
|
||||
handle = SimpleNamespace(model=model, tokenizer=tokenizer, task="classification")
|
||||
|
||||
assert Evaluator(handle, dataset, metrics=["accuracy"]).evaluate() == {"accuracy": 1.0}
|
||||
|
||||
|
||||
def _pipeline(**overrides):
|
||||
values = {
|
||||
"_strong_layers": [0, 1],
|
||||
"refusal_directions": {0: torch.tensor([1.0, 0.0]), 1: torch.tensor([0.0, 1.0])},
|
||||
"_harmful_means": {0: torch.tensor([2.0, 0.0]), 1: torch.tensor([0.0, 4.0])},
|
||||
"_harmless_means": {0: torch.tensor([0.5, 0.0]), 1: torch.tensor([0.0, 1.0])},
|
||||
}
|
||||
values.update(overrides)
|
||||
return SimpleNamespace(**values)
|
||||
|
||||
|
||||
def test_random_direction_baseline_handles_missing_and_cleared_activations():
|
||||
missing = _pipeline(_strong_layers=[], refusal_directions={})
|
||||
assert "no directions" in random_direction_ablation(missing).details["error"].lower()
|
||||
|
||||
cleared = _pipeline(_harmful_means={})
|
||||
assert "activations cleared" in random_direction_ablation(cleared).details["error"]
|
||||
|
||||
|
||||
def test_random_direction_baseline_is_seeded_and_reports_trial_statistics():
|
||||
first = random_direction_ablation(_pipeline(), n_trials=4, seed=17)
|
||||
second = random_direction_ablation(_pipeline(), n_trials=4, seed=17)
|
||||
|
||||
assert first == second
|
||||
assert first.baseline_name == "random_direction"
|
||||
assert first.n_trials == 4
|
||||
assert len(first.refusal_rates) == 4
|
||||
assert first.refusal_rate == first.mean_refusal_rate
|
||||
assert first.std_refusal_rate >= 0
|
||||
assert first.details == {"hidden_dim": 2, "n_strong_layers": 2}
|
||||
|
||||
|
||||
def test_direction_specificity_covers_missing_partial_and_complete_inputs():
|
||||
assert direction_specificity_test(_pipeline(_strong_layers=[], refusal_directions={})) == {
|
||||
"error": "No directions available"
|
||||
}
|
||||
partial = _pipeline(_harmless_means={})
|
||||
assert "activations cleared" in direction_specificity_test(partial)["error"]
|
||||
|
||||
result = direction_specificity_test(_pipeline())
|
||||
assert result["harmful_projection"] == 3.0
|
||||
assert result["harmless_projection"] == 0.75
|
||||
assert result["specificity_ratio"] == 4.0
|
||||
@@ -0,0 +1,133 @@
|
||||
"""Pure contracts for the lm-eval adapter and public report boundary."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
import pytest
|
||||
|
||||
from obliteratus.reporting.report import (
|
||||
AblationReport,
|
||||
AblationResult,
|
||||
_sanitize_public_value,
|
||||
)
|
||||
from obliteratus.evaluation import lm_eval_integration as LM_EVAL
|
||||
|
||||
|
||||
def test_lm_eval_missing_dependency_has_actionable_error(monkeypatch):
|
||||
monkeypatch.setitem(sys.modules, "lm_eval", None)
|
||||
with pytest.raises(ImportError, match="pip install lm-eval>=0.4.0"):
|
||||
LM_EVAL.run_benchmarks("model")
|
||||
|
||||
|
||||
def test_lm_eval_preserves_measured_zero_and_falls_back_to_numeric_metric(monkeypatch):
|
||||
simple_evaluate = Mock(return_value={
|
||||
"results": {
|
||||
"zero": {"acc,none": 0.0, "acc_norm,none": 0.75},
|
||||
"normalized": {"acc_norm,none": 0.6},
|
||||
"fallback": {"alias": "name", "stderr": 0.02, "score": 0.4},
|
||||
"empty": {"alias": "empty"},
|
||||
}
|
||||
})
|
||||
monkeypatch.setitem(sys.modules, "lm_eval", SimpleNamespace(simple_evaluate=simple_evaluate))
|
||||
|
||||
scores = LM_EVAL.run_benchmarks(
|
||||
"org/model",
|
||||
tasks=["zero", "normalized", "fallback", "empty"],
|
||||
device="cpu",
|
||||
batch_size=3,
|
||||
num_fewshot=2,
|
||||
limit=5,
|
||||
)
|
||||
|
||||
assert scores == {"zero": 0.0, "normalized": 0.6, "fallback": 0.02}
|
||||
simple_evaluate.assert_called_once_with(
|
||||
model="hf",
|
||||
model_args="pretrained=org/model,device=cpu",
|
||||
tasks=["zero", "normalized", "fallback", "empty"],
|
||||
batch_size=3,
|
||||
num_fewshot=2,
|
||||
limit=5,
|
||||
)
|
||||
|
||||
|
||||
def test_lm_eval_defaults_and_model_comparison(monkeypatch):
|
||||
simple_evaluate = Mock(return_value={"results": {"task": {"acc,none": 0.5}}})
|
||||
monkeypatch.setitem(sys.modules, "lm_eval", SimpleNamespace(simple_evaluate=simple_evaluate))
|
||||
assert LM_EVAL.run_benchmarks("model", tasks=["task"]) == {"task": 0.5}
|
||||
assert simple_evaluate.call_args.kwargs["model_args"] == "pretrained=model"
|
||||
|
||||
responses = iter([{"a": 0.8, "shared": 0.5}, {"b": 0.4, "shared": 0.7}])
|
||||
monkeypatch.setattr(
|
||||
LM_EVAL,
|
||||
"run_benchmarks",
|
||||
lambda *_args, **_kwargs: next(responses),
|
||||
)
|
||||
assert LM_EVAL.compare_models("original", "abliterated") == {
|
||||
"a": {"original": 0.8, "abliterated": 0.0, "delta": -0.8},
|
||||
"b": {"original": 0.0, "abliterated": 0.4, "delta": 0.4},
|
||||
"shared": {"original": 0.5, "abliterated": 0.7, "delta": pytest.approx(0.2)},
|
||||
}
|
||||
|
||||
|
||||
def test_report_sanitizes_sequences_objects_windows_paths_and_long_labels():
|
||||
custom = SimpleNamespace(value="/private/path/item")
|
||||
sanitized = _sanitize_public_value({
|
||||
"items": ("C:\\private\\model.bin", custom),
|
||||
"api-key": "must disappear",
|
||||
"finite": 1.5,
|
||||
"infinite": float("inf"),
|
||||
})
|
||||
assert "api-key" not in sanitized
|
||||
assert sanitized["finite"] == 1.5
|
||||
assert sanitized["infinite"] is None
|
||||
assert sanitized["items"][0] == "model.bin"
|
||||
assert "private/path" not in sanitized["items"][1]
|
||||
|
||||
report = AblationReport(model_name="x" * 100)
|
||||
assert report.to_dict()["model_name"].endswith("...")
|
||||
assert len(report.to_dict()["model_name"]) == 80
|
||||
|
||||
|
||||
def test_report_summary_empty_and_populated(capsys):
|
||||
AblationReport("empty").print_summary()
|
||||
assert "No ablation results" in capsys.readouterr().out
|
||||
|
||||
report = AblationReport("model")
|
||||
report.add_baseline({"score": 0.0, "missing": None})
|
||||
report.add_result(AblationResult("s", "c", "d", {"score": 1.0, "missing": None}))
|
||||
report.print_summary()
|
||||
output = capsys.readouterr().out
|
||||
assert "Ablation Results: model" in output
|
||||
assert "unavailable" in output
|
||||
|
||||
|
||||
def test_report_plot_boundaries(monkeypatch, tmp_path):
|
||||
report = AblationReport("model")
|
||||
report.add_baseline({"score": 2.0})
|
||||
report.add_result(AblationResult("s", "positive", "d", {"score": 3.0}))
|
||||
report.add_result(AblationResult("s", "negative", "d", {"score": 1.0}))
|
||||
|
||||
impact = tmp_path / "nested" / "impact.png"
|
||||
impact.parent.mkdir()
|
||||
report.plot_impact(output_path=impact)
|
||||
assert impact.stat().st_size > 0
|
||||
|
||||
heatmap = tmp_path / "heatmap.png"
|
||||
report.plot_heatmap(heatmap)
|
||||
assert heatmap.stat().st_size > 0
|
||||
|
||||
show = Mock()
|
||||
monkeypatch.setattr(plt, "show", show)
|
||||
report.plot_impact(metric="score")
|
||||
report.plot_heatmap()
|
||||
assert show.call_count == 2
|
||||
|
||||
no_delta = AblationReport("model", baseline_metrics={"score": None})
|
||||
no_delta.add_result(AblationResult("s", "c", "d", {"score": 1.0}))
|
||||
with pytest.raises(ValueError, match="No delta column"):
|
||||
no_delta.plot_impact("score")
|
||||
no_delta.plot_heatmap()
|
||||
@@ -19,8 +19,21 @@ def test_mutation_campaign_preloads_native_modules_before_covered_line_discovery
|
||||
assert "mutate_only_covered_lines = true" in mutmut_config
|
||||
assert '"obliteratus/runtime_contracts.py"' in mutmut_config
|
||||
assert '"obliteratus/persistence_contracts.py"' in mutmut_config
|
||||
assert '"obliteratus/evaluation/lm_eval_integration.py"' in mutmut_config
|
||||
assert '"obliteratus/reporting/report.py"' not in mutmut_config
|
||||
assert '"tests/test_runtime_contracts.py"' in mutmut_config
|
||||
assert '"tests/test_persistence_contracts.py"' in mutmut_config
|
||||
assert '"tests/test_lm_eval_reporting_contracts.py"' in mutmut_config
|
||||
assert '"tests/test_telemetry_failure_contracts.py"' not in mutmut_config
|
||||
assert '"tests/test_evaluation_reporting_contracts.py"' in Path(
|
||||
"scripts/run_repeat_gate.py",
|
||||
).read_text()
|
||||
assert '"tests/test_lm_eval_reporting_contracts.py"' in Path(
|
||||
"scripts/run_repeat_gate.py",
|
||||
).read_text()
|
||||
assert '"tests/test_telemetry_failure_contracts.py"' in Path(
|
||||
"scripts/run_repeat_gate.py",
|
||||
).read_text()
|
||||
assert "import torch, yaml; from mutmut.__main__ import cli; cli()" in workflow
|
||||
|
||||
|
||||
@@ -54,9 +67,9 @@ def _coverage():
|
||||
"obliteratus/pure.py": {
|
||||
"summary": {
|
||||
"num_statements": 100,
|
||||
"covered_lines": 80,
|
||||
"num_branches": 20,
|
||||
"covered_branches": 15,
|
||||
"covered_lines": 90,
|
||||
"num_branches": 100,
|
||||
"covered_branches": 78,
|
||||
},
|
||||
},
|
||||
"obliteratus/external.py": {
|
||||
@@ -76,8 +89,8 @@ def test_policy_and_exact_mature_floors_pass():
|
||||
assert quality.validate_policy(policy) == []
|
||||
measurement, failures = quality.validate_mature_cpu_scope(_coverage(), policy)
|
||||
assert failures == []
|
||||
assert measurement["line_percent"] == 80
|
||||
assert measurement["branch_percent"] == 75
|
||||
assert measurement["line_percent"] == 90
|
||||
assert measurement["branch_percent"] == 78
|
||||
|
||||
|
||||
def test_floor_regression_requires_structured_reviewed_exception():
|
||||
@@ -109,10 +122,10 @@ def test_exclusions_require_unique_traceable_environment_boundaries():
|
||||
def test_mature_scope_rejects_regression_and_stale_exclusion():
|
||||
policy = _policy()
|
||||
report = _coverage()
|
||||
report["files"]["obliteratus/pure.py"]["summary"]["covered_lines"] = 79
|
||||
report["files"]["obliteratus/pure.py"]["summary"]["covered_lines"] = 89
|
||||
_, failures = quality.validate_mature_cpu_scope(report, policy)
|
||||
assert failures == [
|
||||
"mature CPU line coverage 79.00% is below the 80.00% floor",
|
||||
"mature CPU line coverage 89.00% is below the 90.00% floor",
|
||||
]
|
||||
del report["files"]["obliteratus/external.py"]
|
||||
_, failures = quality.measure_mature_cpu_scope(report, policy)
|
||||
|
||||
@@ -0,0 +1,295 @@
|
||||
"""Failure, storage, and Hub boundary contracts for opt-in telemetry."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import obliteratus.telemetry as telemetry
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _restore_telemetry_globals(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(telemetry, "TELEMETRY_FILE", tmp_path / "telemetry.jsonl")
|
||||
monkeypatch.setattr(telemetry, "_TELEMETRY_DIR", tmp_path)
|
||||
monkeypatch.setattr(telemetry, "_TELEMETRY_REPO", "")
|
||||
monkeypatch.setattr(telemetry, "_hub_repo_created", False)
|
||||
monkeypatch.setattr(telemetry, "_hub_sync_last", 0.0)
|
||||
monkeypatch.setattr(telemetry, "_restore_done", False)
|
||||
telemetry._sync_in_progress.clear()
|
||||
|
||||
|
||||
def test_public_text_sanitizes_windows_paths_and_truncates():
|
||||
assert telemetry._sanitize_public_text(r"C:\private\model.bin") == "model.bin"
|
||||
assert telemetry._sanitize_public_text("x" * 20, max_len=8) == "xxxxx..."
|
||||
assert telemetry._sanitize_public_value(object())
|
||||
|
||||
|
||||
def test_telemetry_directory_prefers_explicit_and_home(monkeypatch, tmp_path):
|
||||
explicit = tmp_path / "explicit"
|
||||
monkeypatch.setenv("OBLITERATUS_DATA_DIR", str(explicit))
|
||||
monkeypatch.setattr(telemetry, "_ON_HF_SPACES", False)
|
||||
assert telemetry._telemetry_dir() == explicit
|
||||
|
||||
monkeypatch.setattr(telemetry, "_test_writable", lambda path: path.name == ".obliteratus")
|
||||
monkeypatch.setattr(Path, "home", staticmethod(lambda: tmp_path / "home"))
|
||||
assert telemetry._telemetry_dir() == tmp_path / "home" / ".obliteratus"
|
||||
|
||||
|
||||
def test_telemetry_directory_retries_hf_mount_then_uses_it(monkeypatch):
|
||||
monkeypatch.delenv("OBLITERATUS_DATA_DIR", raising=False)
|
||||
monkeypatch.setattr(telemetry, "_ON_HF_SPACES", True)
|
||||
monkeypatch.setattr(Path, "exists", lambda self: str(self) == "/data")
|
||||
attempts = iter([False, True])
|
||||
monkeypatch.setattr(telemetry, "_test_writable", lambda _path: next(attempts))
|
||||
sleep = Mock()
|
||||
monkeypatch.setattr(telemetry.time, "sleep", sleep)
|
||||
|
||||
assert telemetry._telemetry_dir() == Path("/data/obliteratus")
|
||||
sleep.assert_called_once_with(1)
|
||||
|
||||
|
||||
def test_telemetry_directory_has_ephemeral_fallback(monkeypatch, tmp_path):
|
||||
monkeypatch.delenv("OBLITERATUS_DATA_DIR", raising=False)
|
||||
monkeypatch.setattr(telemetry, "_ON_HF_SPACES", False)
|
||||
monkeypatch.setattr(telemetry, "_test_writable", lambda _path: False)
|
||||
monkeypatch.setattr(Path, "home", staticmethod(lambda: tmp_path / "home"))
|
||||
assert telemetry._telemetry_dir() == Path("/tmp/obliteratus_telemetry")
|
||||
|
||||
|
||||
class _HubApi:
|
||||
instances: list["_HubApi"] = []
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.kwargs = kwargs
|
||||
self.create_repo = Mock()
|
||||
self.repo_info = Mock()
|
||||
self.upload_file = Mock()
|
||||
self.list_repo_files = Mock(return_value=[])
|
||||
self.instances.append(self)
|
||||
|
||||
|
||||
def _hub_module(api_class=_HubApi, **members):
|
||||
return SimpleNamespace(HfApi=api_class, **members)
|
||||
|
||||
|
||||
def test_ensure_hub_repo_create_and_existing_fallback(monkeypatch):
|
||||
_HubApi.instances.clear()
|
||||
monkeypatch.setitem(sys.modules, "huggingface_hub", _hub_module())
|
||||
assert telemetry._ensure_hub_repo("org/data") is True
|
||||
_HubApi.instances[-1].create_repo.assert_called_once()
|
||||
assert telemetry._ensure_hub_repo("org/data") is True
|
||||
assert len(_HubApi.instances) == 1
|
||||
|
||||
telemetry._hub_repo_created = False
|
||||
|
||||
class ExistingApi(_HubApi):
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.create_repo.side_effect = RuntimeError("cannot create")
|
||||
|
||||
monkeypatch.setitem(sys.modules, "huggingface_hub", _hub_module(ExistingApi))
|
||||
assert telemetry._ensure_hub_repo("org/data") is True
|
||||
ExistingApi.instances[-1].repo_info.assert_called_once()
|
||||
|
||||
|
||||
def test_ensure_hub_repo_fails_closed(monkeypatch):
|
||||
class FailingApi(_HubApi):
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.create_repo.side_effect = RuntimeError("create")
|
||||
self.repo_info.side_effect = RuntimeError("lookup")
|
||||
|
||||
monkeypatch.setitem(sys.modules, "huggingface_hub", _hub_module(FailingApi))
|
||||
assert telemetry._ensure_hub_repo("org/data") is False
|
||||
|
||||
monkeypatch.setitem(sys.modules, "huggingface_hub", None)
|
||||
assert telemetry._ensure_hub_repo("org/data") is False
|
||||
|
||||
|
||||
def test_background_sync_short_circuits_and_uploads(monkeypatch, tmp_path):
|
||||
telemetry._sync_in_progress.set()
|
||||
telemetry._sync_to_hub_bg()
|
||||
telemetry._sync_in_progress.clear()
|
||||
|
||||
telemetry._sync_to_hub_bg()
|
||||
assert not telemetry._sync_in_progress.is_set()
|
||||
|
||||
telemetry._TELEMETRY_REPO = "org/data"
|
||||
telemetry._sync_to_hub_bg()
|
||||
assert not telemetry._sync_in_progress.is_set()
|
||||
|
||||
telemetry.TELEMETRY_FILE.write_text("{}\n")
|
||||
_HubApi.instances.clear()
|
||||
monkeypatch.setitem(sys.modules, "huggingface_hub", _hub_module())
|
||||
monkeypatch.setattr(telemetry, "_ensure_hub_repo", lambda _repo: True)
|
||||
monkeypatch.setattr(telemetry, "_instance_slug", lambda: "slug")
|
||||
telemetry._sync_to_hub_bg()
|
||||
_HubApi.instances[-1].upload_file.assert_called_once_with(
|
||||
path_or_fileobj=str(telemetry.TELEMETRY_FILE),
|
||||
path_in_repo="data/slug.jsonl",
|
||||
repo_id="org/data",
|
||||
repo_type="dataset",
|
||||
commit_message="Auto-sync telemetry from slug",
|
||||
)
|
||||
assert not telemetry._sync_in_progress.is_set()
|
||||
|
||||
|
||||
def test_sync_scheduler_enforces_configuration_enablement_and_debounce(monkeypatch):
|
||||
thread = Mock()
|
||||
monkeypatch.setattr(telemetry.threading, "Thread", Mock(return_value=thread))
|
||||
monkeypatch.setattr(telemetry, "is_enabled", lambda: True)
|
||||
|
||||
telemetry._schedule_hub_sync()
|
||||
thread.start.assert_not_called()
|
||||
|
||||
telemetry._TELEMETRY_REPO = "org/data"
|
||||
monkeypatch.setattr(telemetry, "is_enabled", lambda: False)
|
||||
telemetry._schedule_hub_sync()
|
||||
thread.start.assert_not_called()
|
||||
|
||||
monkeypatch.setattr(telemetry, "is_enabled", lambda: True)
|
||||
monkeypatch.setattr(telemetry.time, "time", lambda: 100.0)
|
||||
telemetry._schedule_hub_sync()
|
||||
thread.start.assert_called_once()
|
||||
telemetry._schedule_hub_sync()
|
||||
thread.start.assert_called_once()
|
||||
|
||||
|
||||
def test_hf_api_fetch_handles_listing_errors_and_file_errors(monkeypatch, tmp_path):
|
||||
class ListingApi(_HubApi):
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.list_repo_files.return_value = ["README.md"]
|
||||
|
||||
monkeypatch.setitem(sys.modules, "huggingface_hub", _hub_module(ListingApi, hf_hub_download=Mock()))
|
||||
assert telemetry._fetch_via_hf_api("org/data", 2) == []
|
||||
|
||||
class BrokenListingApi(_HubApi):
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.list_repo_files.side_effect = RuntimeError("offline")
|
||||
|
||||
monkeypatch.setitem(sys.modules, "huggingface_hub", _hub_module(BrokenListingApi, hf_hub_download=Mock()))
|
||||
assert telemetry._fetch_via_hf_api("org/data", 2) == []
|
||||
|
||||
|
||||
def test_git_clone_fetch_parses_bounded_records_and_cleans_up(monkeypatch):
|
||||
def fake_run(command, **_kwargs):
|
||||
clone_dir = Path(command[-1])
|
||||
data = clone_dir / "data"
|
||||
data.mkdir()
|
||||
(data / "a.jsonl").write_text('\n{"id": 1}\ninvalid\n{"id": 2}\n')
|
||||
return SimpleNamespace(returncode=0, stderr="")
|
||||
|
||||
monkeypatch.setattr("subprocess.run", fake_run)
|
||||
assert telemetry._fetch_via_git_clone("org/data", 1) == [{"id": 1}]
|
||||
|
||||
|
||||
def test_git_clone_fetch_handles_failure_and_missing_data(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"subprocess.run",
|
||||
lambda *_args, **_kwargs: SimpleNamespace(returncode=1, stderr="denied"),
|
||||
)
|
||||
assert telemetry._fetch_via_git_clone("org/data", 2) == []
|
||||
|
||||
monkeypatch.setattr(
|
||||
"subprocess.run",
|
||||
lambda *_args, **_kwargs: SimpleNamespace(returncode=0, stderr=""),
|
||||
)
|
||||
assert telemetry._fetch_via_git_clone("org/data", 2) == []
|
||||
|
||||
|
||||
def test_gpu_detection_and_peak_vram(monkeypatch):
|
||||
monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
|
||||
monkeypatch.setattr(torch.cuda, "get_device_name", lambda _index: "GPU")
|
||||
monkeypatch.setattr(
|
||||
torch.cuda,
|
||||
"get_device_properties",
|
||||
lambda _index: SimpleNamespace(total_memory=8 * 1024**3),
|
||||
)
|
||||
monkeypatch.setattr(torch.cuda, "max_memory_allocated", lambda: 3 * 1024**3)
|
||||
monkeypatch.setattr(torch.cuda, "max_memory_reserved", lambda: 4 * 1024**3)
|
||||
assert telemetry._detect_gpu() == ("GPU", 8.0)
|
||||
assert telemetry._get_peak_vram() == {
|
||||
"peak_allocated_gb": 3.0,
|
||||
"peak_reserved_gb": 4.0,
|
||||
}
|
||||
assert telemetry._detect_model_family("org/Qwen-model") == "qwen"
|
||||
assert telemetry._detect_model_family("org/other") == "unknown"
|
||||
|
||||
|
||||
def test_direction_stats_and_excise_details_cover_optional_techniques(monkeypatch):
|
||||
pipeline = SimpleNamespace(
|
||||
refusal_directions={0: torch.tensor([1.0, 0.0]), 1: torch.tensor([0.0, 1.0])},
|
||||
refusal_subspaces={0: torch.eye(2)},
|
||||
_excise_modified_count=2,
|
||||
_refusal_heads={0: [1, 2]},
|
||||
_sae_directions={0: torch.ones(2)},
|
||||
_expert_safety_scores={0: 1.0},
|
||||
_layer_excise_weights={0: 0.2, 1: 0.8},
|
||||
_expert_directions={0: torch.ones(2)},
|
||||
_steering_hooks=[object()],
|
||||
invert_refusal=True,
|
||||
project_embeddings=True,
|
||||
activation_steering=True,
|
||||
expert_transplant=True,
|
||||
)
|
||||
stats = telemetry._direction_stats(pipeline)
|
||||
assert stats["direction_norms"] == {"0": 1.0, "1": 1.0}
|
||||
assert stats["mean_direction_persistence"] == 0.0
|
||||
assert stats["effective_ranks"] == {"0": 2.0}
|
||||
|
||||
details = telemetry._extract_excise_details(pipeline)
|
||||
assert details["modified_count"] == 2
|
||||
assert details["total_heads_projected"] == 2
|
||||
assert details["adaptive_weight_min"] == 0.2
|
||||
assert details["adaptive_weight_max"] == 0.8
|
||||
assert set(details["used_techniques"]) == {
|
||||
"head_surgery", "sae_features", "expert_gating", "layer_adaptive",
|
||||
"per_expert", "activation_steering", "inversion", "embedding_projection",
|
||||
"expert_transplant",
|
||||
}
|
||||
|
||||
monkeypatch.setattr(torch.linalg, "svdvals", Mock(side_effect=RuntimeError("svd")))
|
||||
assert "effective_ranks" not in telemetry._direction_stats(pipeline)
|
||||
|
||||
|
||||
def test_send_and_pipeline_failures_are_best_effort(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(telemetry, "is_enabled", lambda: True)
|
||||
monkeypatch.setattr(telemetry, "TELEMETRY_FILE", tmp_path / "missing" / "file.jsonl")
|
||||
telemetry._send_sync({"schema_version": 2})
|
||||
|
||||
logger = Mock()
|
||||
monkeypatch.setattr(telemetry, "logger", logger)
|
||||
pipeline = SimpleNamespace(handle=SimpleNamespace(summary=Mock(side_effect=RuntimeError("summary"))))
|
||||
telemetry.maybe_send_pipeline_report(pipeline)
|
||||
telemetry.maybe_send_informed_report(pipeline, SimpleNamespace())
|
||||
assert logger.debug.call_count == 2
|
||||
|
||||
|
||||
def test_push_to_hub_failure_paths(monkeypatch, tmp_path):
|
||||
assert telemetry.push_to_hub() is False
|
||||
|
||||
telemetry.TELEMETRY_FILE.write_text('{}\n')
|
||||
monkeypatch.setattr(telemetry, "read_telemetry", lambda: [{}])
|
||||
monkeypatch.setattr(telemetry, "_ensure_hub_repo", lambda _repo: False)
|
||||
assert telemetry.push_to_hub("org/data") is False
|
||||
|
||||
monkeypatch.setitem(sys.modules, "huggingface_hub", None)
|
||||
monkeypatch.setattr(telemetry, "_ensure_hub_repo", lambda _repo: True)
|
||||
assert telemetry.push_to_hub("org/data") is False
|
||||
|
||||
|
||||
def test_restore_and_background_restore_absorb_boundary_failures(monkeypatch):
|
||||
telemetry._TELEMETRY_REPO = "org/data"
|
||||
monkeypatch.setattr(telemetry, "fetch_hub_records", Mock(side_effect=RuntimeError("offline")))
|
||||
assert telemetry.restore_from_hub() == 0
|
||||
|
||||
monkeypatch.setattr(telemetry, "restore_from_hub", Mock(side_effect=RuntimeError("offline")))
|
||||
telemetry._restore_from_hub_bg()
|
||||
Reference in New Issue
Block a user