test: enforce whitened direction contracts

ci/test-risk-map.json

@@ -145,6 +145,7 @@
         "obliteratus/analysis/leace.py",
         "obliteratus/analysis/logit_lens.py",
         "obliteratus/analysis/multi_token_position.py",
+        "obliteratus/analysis/numerical_contracts.py",
         "obliteratus/analysis/probing_classifiers.py",
         "obliteratus/analysis/residual_stream.py",
         "obliteratus/analysis/riemannian_manifold.py",
@@ -168,6 +169,7 @@
         "tests/test_leace.py",
         "tests/test_logit_lens.py",
         "tests/test_new_analysis_modules.py",
+        "tests/test_numerical_contracts.py",
         "tests/test_novel_analysis.py",
         "tests/test_visualization.py"
       ]

obliteratus/analysis/numerical_contracts.py

@@ -0,0 +1,47 @@
+"""Pure validation contracts shared by numerical analysis routines."""
+
+from __future__ import annotations
+
+import math
+
+
+def validate_whitened_parameters(
+    regularization_eps: object,
+    min_variance_ratio: object,
+) -> tuple[float, float]:
+    """Validate and normalize whitened-SVD tuning parameters."""
+    if (
+        isinstance(regularization_eps, bool)
+        or not isinstance(regularization_eps, (int, float))
+        or not math.isfinite(regularization_eps)
+        or regularization_eps <= 0
+    ):
+        raise ValueError("regularization_eps must be a finite positive number")
+    if (
+        isinstance(min_variance_ratio, bool)
+        or not isinstance(min_variance_ratio, (int, float))
+        or not math.isfinite(min_variance_ratio)
+        or not 0 <= min_variance_ratio < 1
+    ):
+        raise ValueError("min_variance_ratio must be in the interval [0, 1)")
+    return float(regularization_eps), float(min_variance_ratio)
+
+
+def validate_whitened_request(
+    harmful_count: int,
+    harmless_count: int,
+    n_directions: object,
+) -> int:
+    """Validate paired sample counts and requested direction count."""
+    if harmful_count <= 0 or harmless_count <= 0:
+        raise ValueError("harmful and harmless activations must both be non-empty")
+    if harmful_count != harmless_count:
+        raise ValueError(
+            "harmful and harmless activations must have equal sample counts, got "
+            f"{harmful_count} and {harmless_count}",
+        )
+    if isinstance(n_directions, bool) or not isinstance(n_directions, int):
+        raise ValueError("n_directions must be a positive integer")
+    if n_directions <= 0:
+        raise ValueError("n_directions must be a positive integer")
+    return n_directions

obliteratus/analysis/whitened_svd.py

@@ -31,6 +31,11 @@ from dataclasses import dataclass
 
 import torch
 
+from obliteratus.analysis.numerical_contracts import (
+    validate_whitened_parameters,
+    validate_whitened_request,
+)
+
 
 @dataclass
 class WhitenedSVDResult:
@@ -67,8 +72,53 @@ class WhitenedSVDExtractor:
                 below which dimensions are truncated. Prevents amplifying
                 noise in near-degenerate dimensions.
         """
-        self.regularization_eps = regularization_eps
-        self.min_variance_ratio = min_variance_ratio
+        self.regularization_eps, self.min_variance_ratio = validate_whitened_parameters(
+            regularization_eps,
+            min_variance_ratio,
+        )
+
+    @staticmethod
+    def _stack_activation_pair(
+        harmful_activations: list[torch.Tensor],
+        harmless_activations: list[torch.Tensor],
+    ) -> tuple[torch.Tensor, torch.Tensor]:
+        """Validate and stack paired activation samples into two 2D tensors."""
+        def normalize(samples: list[torch.Tensor], name: str) -> torch.Tensor:
+            normalized = []
+            width = None
+            device = None
+            for index, sample in enumerate(samples):
+                if not isinstance(sample, torch.Tensor):
+                    raise ValueError(f"{name} activation {index} must be a tensor")
+                if sample.dim() == 2 and sample.shape[0] == 1:
+                    sample = sample.squeeze(0)
+                if sample.dim() != 1 or sample.numel() == 0:
+                    raise ValueError(
+                        f"{name} activation {index} must be a non-empty vector "
+                        "or a single-row matrix",
+                    )
+                if not torch.isfinite(sample).all():
+                    raise ValueError(f"{name} activations must contain only finite values")
+                if width is None:
+                    width = sample.shape[0]
+                    device = sample.device
+                elif sample.shape[0] != width:
+                    raise ValueError(f"{name} activations must have a consistent width")
+                elif sample.device != device:
+                    raise ValueError(f"{name} activations must be on a single device")
+                normalized.append(sample)
+            return torch.stack(normalized).float()
+
+        harmful = normalize(harmful_activations, "harmful")
+        harmless = normalize(harmless_activations, "harmless")
+        if harmful.shape[1] != harmless.shape[1]:
+            raise ValueError(
+                "harmful and harmless activations must have the same width, got "
+                f"{harmful.shape[1]} and {harmless.shape[1]}",
+            )
+        if harmful.device != harmless.device:
+            raise ValueError("harmful and harmless activations must be on the same device")
+        return harmful, harmless
 
     def extract(
         self,
@@ -88,13 +138,12 @@ class WhitenedSVDExtractor:
         Returns:
             WhitenedSVDResult with directions in original activation space.
         """
-        H = torch.stack(harmful_activations).float()   # (n, d)
-        B = torch.stack(harmless_activations).float()   # (n, d)
-
-        if H.dim() == 3:
-            H = H.squeeze(1)
-        if B.dim() == 3:
-            B = B.squeeze(1)
+        n_directions = validate_whitened_request(
+            len(harmful_activations),
+            len(harmless_activations),
+            n_directions,
+        )
+        H, B = self._stack_activation_pair(harmful_activations, harmless_activations)
 
         n_samples, d = B.shape
 
@@ -123,7 +172,7 @@ class WhitenedSVDExtractor:
 
         # Step 3: Truncate near-degenerate dimensions
         threshold = max_eig * self.min_variance_ratio
-        valid_mask = eigenvalues > threshold
+        valid_mask = eigenvalues >= threshold
         eigenvalues_valid = eigenvalues[valid_mask]
         eigenvectors_valid = eigenvectors[:, valid_mask]
 
@@ -140,6 +189,8 @@ class WhitenedSVDExtractor:
 
         # Step 6: Compute whitened difference and SVD
         D_whitened = H_whitened - B_whitened  # (n, k_valid)
+        if torch.linalg.vector_norm(D_whitened) <= 1e-12:
+            raise ValueError("cannot extract a refusal direction without activation difference")
 
         k = min(n_directions, D_whitened.shape[0], D_whitened.shape[1])
         U, S, Vh = torch.linalg.svd(D_whitened, full_matrices=False)

tests/test_analysis.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 
+import pytest
 import torch
 
 from obliteratus.analysis.whitened_svd import WhitenedSVDExtractor, WhitenedSVDResult
@@ -15,6 +16,53 @@ from obliteratus.analysis.activation_probing import ActivationProbe, ProbeResult
 # ---------------------------------------------------------------------------
 
 class TestWhitenedSVD:
+    @pytest.mark.parametrize("regularization_eps", [0, -1e-4, float("nan"), True])
+    def test_rejects_invalid_regularization(self, regularization_eps):
+        with pytest.raises(ValueError, match="finite positive"):
+            WhitenedSVDExtractor(regularization_eps=regularization_eps)
+
+    @pytest.mark.parametrize("min_variance_ratio", [-0.1, 1.0, float("inf"), True])
+    def test_rejects_invalid_variance_ratio(self, min_variance_ratio):
+        with pytest.raises(ValueError, match=r"interval \[0, 1\)"):
+            WhitenedSVDExtractor(min_variance_ratio=min_variance_ratio)
+
+    @pytest.mark.parametrize("n_directions", [-1, 0, 1.5, True])
+    def test_rejects_invalid_direction_count(self, n_directions):
+        with pytest.raises(ValueError, match="positive integer"):
+            WhitenedSVDExtractor().extract(
+                [torch.ones(4)],
+                [torch.zeros(4)],
+                n_directions=n_directions,
+            )
+
+    def test_rejects_empty_unpaired_and_mismatched_activations(self):
+        extractor = WhitenedSVDExtractor()
+        with pytest.raises(ValueError, match="both be non-empty"):
+            extractor.extract([], [])
+        with pytest.raises(ValueError, match="equal sample counts"):
+            extractor.extract([torch.ones(4)] * 2, [torch.zeros(4)])
+        with pytest.raises(ValueError, match="same width"):
+            extractor.extract([torch.ones(4)], [torch.zeros(3)])
+        with pytest.raises(ValueError, match="consistent width"):
+            extractor.extract(
+                [torch.ones(4), torch.ones(3)],
+                [torch.zeros(4), torch.zeros(4)],
+            )
+        with pytest.raises(ValueError, match="must be a tensor"):
+            extractor.extract([object()], [torch.zeros(4)])
+        with pytest.raises(ValueError, match="non-empty vector"):
+            extractor.extract([torch.ones(2, 4)], [torch.zeros(4)])
+
+    def test_single_sample_signal_uses_regularized_zero_covariance(self):
+        result = WhitenedSVDExtractor().extract(
+            [torch.tensor([0.0, 2.0, 0.0])],
+            [torch.zeros(3)],
+            n_directions=1,
+        )
+        assert result.directions.shape == (1, 3)
+        assert result.directions.norm() == pytest.approx(1.0)
+        assert result.variance_explained == pytest.approx(1.0)
+
     def test_basic_extraction(self):
         """Whitened SVD should extract directions from activation differences."""
         torch.manual_seed(42)

tests/test_edge_cases.py

@@ -45,33 +45,20 @@ class TestNaNInfHandling:
     """Test that modules handle degenerate inputs gracefully."""
 
     def test_whitened_svd_nan_activations(self):
-        """WhitenedSVD with NaN — currently raises; documenting behavior."""
+        """WhitenedSVD rejects non-finite research inputs explicitly."""
         harmful = [torch.tensor([float("nan"), 1.0, 2.0]) for _ in range(5)]
         harmless = [torch.randn(3) for _ in range(5)]
         extractor = WhitenedSVDExtractor()
-        # NaN propagation through SVD is expected to produce NaN results
-        # This documents the current behavior — ideally would guard against it
-        raised = False
-        result = None
-        try:
-            result = extractor.extract(harmful, harmless)
-        except (RuntimeError, ValueError):
-            raised = True
-        # Either it raised an exception (acceptable) or returned a result with NaNs
-        assert raised or result is not None, (
-            "Should either raise on NaN input or return a result"
-        )
+        with pytest.raises(ValueError, match="finite"):
+            extractor.extract(harmful, harmless)
 
     def test_whitened_svd_zero_activations(self):
-        """WhitenedSVD with all-zero activations."""
+        """WhitenedSVD reports that identical activation sets have no signal."""
         harmful = [torch.zeros(8) for _ in range(5)]
         harmless = [torch.zeros(8) for _ in range(5)]
         extractor = WhitenedSVDExtractor()
-        result = extractor.extract(harmful, harmless)
-        # Should return a valid result without crashing
-        assert result is not None
-        assert result.directions is not None
-        assert result.singular_values is not None
+        with pytest.raises(ValueError, match="without activation difference"):
+            extractor.extract(harmful, harmless)
 
     def test_concept_cone_nan_direction(self):
         """ConceptConeAnalyzer with NaN in activations — documenting behavior."""
@@ -478,11 +465,11 @@ class TestExceptionPaths:
     """Tests for error handling and boundary conditions."""
 
     def test_whitened_svd_mismatched_dims(self):
-        """Harmful and harmless with different hidden dims should fail or handle gracefully."""
+        """Harmful and harmless vectors must use the same hidden width."""
         harmful = [torch.randn(64) for _ in range(10)]
         harmless = [torch.randn(32) for _ in range(10)]
         extractor = WhitenedSVDExtractor()
-        with pytest.raises(Exception):
+        with pytest.raises(ValueError, match="same width"):
             extractor.extract(harmful, harmless, n_directions=1)
 
     def test_whitened_svd_single_sample(self):

tests/test_numerical_contracts.py

@@ -0,0 +1,63 @@
+"""Mutation-focused tests for pure numerical input contracts."""
+
+from __future__ import annotations
+
+import pytest
+
+from obliteratus.analysis.numerical_contracts import (
+    validate_whitened_parameters,
+    validate_whitened_request,
+)
+
+
+@pytest.mark.parametrize("regularization_eps", [0, -1e-4, float("nan"), float("inf"), True, "1e-4"])
+def test_regularization_must_be_a_finite_positive_number(regularization_eps):
+    with pytest.raises(ValueError, match="finite positive"):
+        validate_whitened_parameters(regularization_eps, 0.01)
+
+
+@pytest.mark.parametrize(
+    "min_variance_ratio",
+    [-0.1, 1.0, float("nan"), float("inf"), True, "0.01"],
+)
+def test_variance_ratio_must_be_in_the_half_open_unit_interval(min_variance_ratio):
+    with pytest.raises(ValueError, match=r"interval \[0, 1\)"):
+        validate_whitened_parameters(1e-4, min_variance_ratio)
+
+
+@pytest.mark.parametrize(
+    ("regularization_eps", "min_variance_ratio", "expected"),
+    [(1, 0, (1.0, 0.0)), (1e-4, 0.999, (1e-4, 0.999))],
+)
+def test_valid_parameters_are_normalized_to_floats(
+    regularization_eps,
+    min_variance_ratio,
+    expected,
+):
+    assert validate_whitened_parameters(regularization_eps, min_variance_ratio) == expected
+
+
+@pytest.mark.parametrize(
+    ("harmful_count", "harmless_count", "message"),
+    [(0, 0, "both be non-empty"), (1, 0, "both be non-empty"), (0, 1, "both be non-empty")],
+)
+def test_activation_sets_must_both_be_nonempty(harmful_count, harmless_count, message):
+    with pytest.raises(ValueError, match=message):
+        validate_whitened_request(harmful_count, harmless_count, 1)
+
+
+@pytest.mark.parametrize(("harmful_count", "harmless_count"), [(1, 2), (3, 1)])
+def test_activation_sets_must_have_equal_sample_counts(harmful_count, harmless_count):
+    with pytest.raises(ValueError, match=f"got {harmful_count} and {harmless_count}"):
+        validate_whitened_request(harmful_count, harmless_count, 1)
+
+
+@pytest.mark.parametrize("n_directions", [-1, 0, 1.5, True, "1"])
+def test_direction_count_must_be_a_positive_integer(n_directions):
+    with pytest.raises(ValueError, match="positive integer"):
+        validate_whitened_request(2, 2, n_directions)
+
+
+@pytest.mark.parametrize("n_directions", [1, 2, 100])
+def test_valid_direction_count_is_returned(n_directions):
+    assert validate_whitened_request(2, 2, n_directions) == n_directions

tests/test_property_contracts.py

@@ -2,10 +2,13 @@
 
 from __future__ import annotations
 
+import math
+
 import pytest
 import torch
 from hypothesis import given, seed, settings, strategies as st
 
+from obliteratus.analysis.whitened_svd import WhitenedSVDExtractor
 from obliteratus.evaluation.advanced_metrics import (
     _is_refusal,
     linear_cka,
@@ -122,3 +125,102 @@ def test_refusal_classification_is_invariant_to_case_and_leading_space(
 ):
     transformed = " " * leading_space + (refusal.upper() if upper else refusal.lower())
     assert _is_refusal(transformed, mode="combined")
+
+
+@seed(7007)
+@PROPERTY_SETTINGS
+@given(
+    n_samples=st.integers(3, 12),
+    hidden_dim=st.integers(3, 16),
+    offset=st.floats(-50, 50, allow_nan=False, allow_infinity=False),
+)
+def test_whitened_direction_is_invariant_to_common_activation_translation(
+    n_samples, hidden_dim, offset,
+):
+    generator = torch.Generator().manual_seed(n_samples * 100 + hidden_dim)
+    harmless = torch.randn(n_samples, hidden_dim, generator=generator)
+    signal = torch.linspace(-1.0, 1.0, hidden_dim)
+    harmful = harmless + signal
+    extractor = WhitenedSVDExtractor()
+
+    original = extractor.extract(list(harmful), list(harmless), n_directions=1)
+    translated = extractor.extract(
+        list(harmful + offset),
+        list(harmless + offset),
+        n_directions=1,
+    )
+
+    alignment = torch.dot(original.directions[0], translated.directions[0]).abs()
+    assert alignment == pytest.approx(1.0, abs=2e-4)
+    assert translated.variance_explained == pytest.approx(
+        original.variance_explained,
+        abs=2e-5,
+    )
+
+
+@seed(7008)
+@PROPERTY_SETTINGS
+@given(permutation=st.permutations(tuple(range(8))))
+def test_whitened_direction_is_invariant_to_joint_sample_permutation(permutation):
+    generator = torch.Generator().manual_seed(7008)
+    harmless = torch.randn(8, 10, generator=generator)
+    harmful = harmless + torch.linspace(-2.0, 2.0, 10)
+    extractor = WhitenedSVDExtractor()
+
+    original = extractor.extract(list(harmful), list(harmless), n_directions=1)
+    permuted = extractor.extract(
+        [harmful[index] for index in permutation],
+        [harmless[index] for index in permutation],
+        n_directions=1,
+    )
+
+    alignment = torch.dot(original.directions[0], permuted.directions[0]).abs()
+    assert alignment == pytest.approx(1.0, abs=2e-5)
+    assert permuted.singular_values == pytest.approx(original.singular_values, rel=2e-5)
+
+
+@seed(7009)
+@PROPERTY_SETTINGS
+@given(
+    n_samples=st.integers(2, 12),
+    hidden_dim=st.integers(2, 16),
+    requested=st.integers(1, 8),
+)
+def test_whitened_outputs_obey_normalization_ordering_and_bounds(
+    n_samples, hidden_dim, requested,
+):
+    generator = torch.Generator().manual_seed(n_samples * 1000 + hidden_dim)
+    harmless = torch.randn(n_samples, hidden_dim, generator=generator)
+    harmful = harmless + torch.randn(n_samples, hidden_dim, generator=generator)
+    result = WhitenedSVDExtractor(min_variance_ratio=0).extract(
+        list(harmful),
+        list(harmless),
+        n_directions=requested,
+    )
+
+    expected_count = min(requested, n_samples, hidden_dim)
+    assert result.directions.shape == (expected_count, hidden_dim)
+    assert result.directions.norm(dim=1) == pytest.approx(torch.ones(expected_count))
+    assert torch.all(result.singular_values >= 0)
+    assert torch.all(result.singular_values[:-1] >= result.singular_values[1:])
+    assert 0.0 <= result.variance_explained <= 1.0
+    assert math.isfinite(result.condition_number)
+    assert math.isfinite(result.effective_rank)
+
+
+@pytest.mark.parametrize("dtype", [torch.float16, torch.float32, torch.float64])
+def test_whitened_direction_normalizes_supported_cpu_input_dtypes(dtype):
+    harmless = torch.tensor(
+        [[0, 0, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1]],
+        dtype=dtype,
+    )
+    harmful = harmless + torch.tensor([0, 0, 2], dtype=dtype)
+
+    result = WhitenedSVDExtractor().extract(
+        list(harmful),
+        list(harmless),
+        n_directions=1,
+    )
+
+    assert result.directions.dtype == torch.float32
+    assert result.directions.norm() == pytest.approx(1.0)