diff --git a/ci/test-risk-map.json b/ci/test-risk-map.json index 2bbf2dc..19a9f4a 100644 --- a/ci/test-risk-map.json +++ b/ci/test-risk-map.json @@ -145,6 +145,7 @@ "obliteratus/analysis/leace.py", "obliteratus/analysis/logit_lens.py", "obliteratus/analysis/multi_token_position.py", + "obliteratus/analysis/numerical_contracts.py", "obliteratus/analysis/probing_classifiers.py", "obliteratus/analysis/residual_stream.py", "obliteratus/analysis/riemannian_manifold.py", @@ -168,6 +169,7 @@ "tests/test_leace.py", "tests/test_logit_lens.py", "tests/test_new_analysis_modules.py", + "tests/test_numerical_contracts.py", "tests/test_novel_analysis.py", "tests/test_visualization.py" ] diff --git a/obliteratus/analysis/numerical_contracts.py b/obliteratus/analysis/numerical_contracts.py new file mode 100644 index 0000000..cdc6f4a --- /dev/null +++ b/obliteratus/analysis/numerical_contracts.py @@ -0,0 +1,47 @@ +"""Pure validation contracts shared by numerical analysis routines.""" + +from __future__ import annotations + +import math + + +def validate_whitened_parameters( + regularization_eps: object, + min_variance_ratio: object, +) -> tuple[float, float]: + """Validate and normalize whitened-SVD tuning parameters.""" + if ( + isinstance(regularization_eps, bool) + or not isinstance(regularization_eps, (int, float)) + or not math.isfinite(regularization_eps) + or regularization_eps <= 0 + ): + raise ValueError("regularization_eps must be a finite positive number") + if ( + isinstance(min_variance_ratio, bool) + or not isinstance(min_variance_ratio, (int, float)) + or not math.isfinite(min_variance_ratio) + or not 0 <= min_variance_ratio < 1 + ): + raise ValueError("min_variance_ratio must be in the interval [0, 1)") + return float(regularization_eps), float(min_variance_ratio) + + +def validate_whitened_request( + harmful_count: int, + harmless_count: int, + n_directions: object, +) -> int: + """Validate paired sample counts and requested direction count.""" + if harmful_count <= 0 or harmless_count <= 0: + raise ValueError("harmful and harmless activations must both be non-empty") + if harmful_count != harmless_count: + raise ValueError( + "harmful and harmless activations must have equal sample counts, got " + f"{harmful_count} and {harmless_count}", + ) + if isinstance(n_directions, bool) or not isinstance(n_directions, int): + raise ValueError("n_directions must be a positive integer") + if n_directions <= 0: + raise ValueError("n_directions must be a positive integer") + return n_directions diff --git a/obliteratus/analysis/whitened_svd.py b/obliteratus/analysis/whitened_svd.py index bcb7746..55d16bc 100644 --- a/obliteratus/analysis/whitened_svd.py +++ b/obliteratus/analysis/whitened_svd.py @@ -31,6 +31,11 @@ from dataclasses import dataclass import torch +from obliteratus.analysis.numerical_contracts import ( + validate_whitened_parameters, + validate_whitened_request, +) + @dataclass class WhitenedSVDResult: @@ -67,8 +72,53 @@ class WhitenedSVDExtractor: below which dimensions are truncated. Prevents amplifying noise in near-degenerate dimensions. """ - self.regularization_eps = regularization_eps - self.min_variance_ratio = min_variance_ratio + self.regularization_eps, self.min_variance_ratio = validate_whitened_parameters( + regularization_eps, + min_variance_ratio, + ) + + @staticmethod + def _stack_activation_pair( + harmful_activations: list[torch.Tensor], + harmless_activations: list[torch.Tensor], + ) -> tuple[torch.Tensor, torch.Tensor]: + """Validate and stack paired activation samples into two 2D tensors.""" + def normalize(samples: list[torch.Tensor], name: str) -> torch.Tensor: + normalized = [] + width = None + device = None + for index, sample in enumerate(samples): + if not isinstance(sample, torch.Tensor): + raise ValueError(f"{name} activation {index} must be a tensor") + if sample.dim() == 2 and sample.shape[0] == 1: + sample = sample.squeeze(0) + if sample.dim() != 1 or sample.numel() == 0: + raise ValueError( + f"{name} activation {index} must be a non-empty vector " + "or a single-row matrix", + ) + if not torch.isfinite(sample).all(): + raise ValueError(f"{name} activations must contain only finite values") + if width is None: + width = sample.shape[0] + device = sample.device + elif sample.shape[0] != width: + raise ValueError(f"{name} activations must have a consistent width") + elif sample.device != device: + raise ValueError(f"{name} activations must be on a single device") + normalized.append(sample) + return torch.stack(normalized).float() + + harmful = normalize(harmful_activations, "harmful") + harmless = normalize(harmless_activations, "harmless") + if harmful.shape[1] != harmless.shape[1]: + raise ValueError( + "harmful and harmless activations must have the same width, got " + f"{harmful.shape[1]} and {harmless.shape[1]}", + ) + if harmful.device != harmless.device: + raise ValueError("harmful and harmless activations must be on the same device") + return harmful, harmless def extract( self, @@ -88,13 +138,12 @@ class WhitenedSVDExtractor: Returns: WhitenedSVDResult with directions in original activation space. """ - H = torch.stack(harmful_activations).float() # (n, d) - B = torch.stack(harmless_activations).float() # (n, d) - - if H.dim() == 3: - H = H.squeeze(1) - if B.dim() == 3: - B = B.squeeze(1) + n_directions = validate_whitened_request( + len(harmful_activations), + len(harmless_activations), + n_directions, + ) + H, B = self._stack_activation_pair(harmful_activations, harmless_activations) n_samples, d = B.shape @@ -123,7 +172,7 @@ class WhitenedSVDExtractor: # Step 3: Truncate near-degenerate dimensions threshold = max_eig * self.min_variance_ratio - valid_mask = eigenvalues > threshold + valid_mask = eigenvalues >= threshold eigenvalues_valid = eigenvalues[valid_mask] eigenvectors_valid = eigenvectors[:, valid_mask] @@ -140,6 +189,8 @@ class WhitenedSVDExtractor: # Step 6: Compute whitened difference and SVD D_whitened = H_whitened - B_whitened # (n, k_valid) + if torch.linalg.vector_norm(D_whitened) <= 1e-12: + raise ValueError("cannot extract a refusal direction without activation difference") k = min(n_directions, D_whitened.shape[0], D_whitened.shape[1]) U, S, Vh = torch.linalg.svd(D_whitened, full_matrices=False) diff --git a/tests/test_analysis.py b/tests/test_analysis.py index 1fb03c7..58b61c6 100644 --- a/tests/test_analysis.py +++ b/tests/test_analysis.py @@ -3,6 +3,7 @@ from __future__ import annotations +import pytest import torch from obliteratus.analysis.whitened_svd import WhitenedSVDExtractor, WhitenedSVDResult @@ -15,6 +16,53 @@ from obliteratus.analysis.activation_probing import ActivationProbe, ProbeResult # --------------------------------------------------------------------------- class TestWhitenedSVD: + @pytest.mark.parametrize("regularization_eps", [0, -1e-4, float("nan"), True]) + def test_rejects_invalid_regularization(self, regularization_eps): + with pytest.raises(ValueError, match="finite positive"): + WhitenedSVDExtractor(regularization_eps=regularization_eps) + + @pytest.mark.parametrize("min_variance_ratio", [-0.1, 1.0, float("inf"), True]) + def test_rejects_invalid_variance_ratio(self, min_variance_ratio): + with pytest.raises(ValueError, match=r"interval \[0, 1\)"): + WhitenedSVDExtractor(min_variance_ratio=min_variance_ratio) + + @pytest.mark.parametrize("n_directions", [-1, 0, 1.5, True]) + def test_rejects_invalid_direction_count(self, n_directions): + with pytest.raises(ValueError, match="positive integer"): + WhitenedSVDExtractor().extract( + [torch.ones(4)], + [torch.zeros(4)], + n_directions=n_directions, + ) + + def test_rejects_empty_unpaired_and_mismatched_activations(self): + extractor = WhitenedSVDExtractor() + with pytest.raises(ValueError, match="both be non-empty"): + extractor.extract([], []) + with pytest.raises(ValueError, match="equal sample counts"): + extractor.extract([torch.ones(4)] * 2, [torch.zeros(4)]) + with pytest.raises(ValueError, match="same width"): + extractor.extract([torch.ones(4)], [torch.zeros(3)]) + with pytest.raises(ValueError, match="consistent width"): + extractor.extract( + [torch.ones(4), torch.ones(3)], + [torch.zeros(4), torch.zeros(4)], + ) + with pytest.raises(ValueError, match="must be a tensor"): + extractor.extract([object()], [torch.zeros(4)]) + with pytest.raises(ValueError, match="non-empty vector"): + extractor.extract([torch.ones(2, 4)], [torch.zeros(4)]) + + def test_single_sample_signal_uses_regularized_zero_covariance(self): + result = WhitenedSVDExtractor().extract( + [torch.tensor([0.0, 2.0, 0.0])], + [torch.zeros(3)], + n_directions=1, + ) + assert result.directions.shape == (1, 3) + assert result.directions.norm() == pytest.approx(1.0) + assert result.variance_explained == pytest.approx(1.0) + def test_basic_extraction(self): """Whitened SVD should extract directions from activation differences.""" torch.manual_seed(42) diff --git a/tests/test_edge_cases.py b/tests/test_edge_cases.py index 4184c0f..7a0b702 100644 --- a/tests/test_edge_cases.py +++ b/tests/test_edge_cases.py @@ -45,33 +45,20 @@ class TestNaNInfHandling: """Test that modules handle degenerate inputs gracefully.""" def test_whitened_svd_nan_activations(self): - """WhitenedSVD with NaN — currently raises; documenting behavior.""" + """WhitenedSVD rejects non-finite research inputs explicitly.""" harmful = [torch.tensor([float("nan"), 1.0, 2.0]) for _ in range(5)] harmless = [torch.randn(3) for _ in range(5)] extractor = WhitenedSVDExtractor() - # NaN propagation through SVD is expected to produce NaN results - # This documents the current behavior — ideally would guard against it - raised = False - result = None - try: - result = extractor.extract(harmful, harmless) - except (RuntimeError, ValueError): - raised = True - # Either it raised an exception (acceptable) or returned a result with NaNs - assert raised or result is not None, ( - "Should either raise on NaN input or return a result" - ) + with pytest.raises(ValueError, match="finite"): + extractor.extract(harmful, harmless) def test_whitened_svd_zero_activations(self): - """WhitenedSVD with all-zero activations.""" + """WhitenedSVD reports that identical activation sets have no signal.""" harmful = [torch.zeros(8) for _ in range(5)] harmless = [torch.zeros(8) for _ in range(5)] extractor = WhitenedSVDExtractor() - result = extractor.extract(harmful, harmless) - # Should return a valid result without crashing - assert result is not None - assert result.directions is not None - assert result.singular_values is not None + with pytest.raises(ValueError, match="without activation difference"): + extractor.extract(harmful, harmless) def test_concept_cone_nan_direction(self): """ConceptConeAnalyzer with NaN in activations — documenting behavior.""" @@ -478,11 +465,11 @@ class TestExceptionPaths: """Tests for error handling and boundary conditions.""" def test_whitened_svd_mismatched_dims(self): - """Harmful and harmless with different hidden dims should fail or handle gracefully.""" + """Harmful and harmless vectors must use the same hidden width.""" harmful = [torch.randn(64) for _ in range(10)] harmless = [torch.randn(32) for _ in range(10)] extractor = WhitenedSVDExtractor() - with pytest.raises(Exception): + with pytest.raises(ValueError, match="same width"): extractor.extract(harmful, harmless, n_directions=1) def test_whitened_svd_single_sample(self): diff --git a/tests/test_numerical_contracts.py b/tests/test_numerical_contracts.py new file mode 100644 index 0000000..6216b06 --- /dev/null +++ b/tests/test_numerical_contracts.py @@ -0,0 +1,63 @@ +"""Mutation-focused tests for pure numerical input contracts.""" + +from __future__ import annotations + +import pytest + +from obliteratus.analysis.numerical_contracts import ( + validate_whitened_parameters, + validate_whitened_request, +) + + +@pytest.mark.parametrize("regularization_eps", [0, -1e-4, float("nan"), float("inf"), True, "1e-4"]) +def test_regularization_must_be_a_finite_positive_number(regularization_eps): + with pytest.raises(ValueError, match="finite positive"): + validate_whitened_parameters(regularization_eps, 0.01) + + +@pytest.mark.parametrize( + "min_variance_ratio", + [-0.1, 1.0, float("nan"), float("inf"), True, "0.01"], +) +def test_variance_ratio_must_be_in_the_half_open_unit_interval(min_variance_ratio): + with pytest.raises(ValueError, match=r"interval \[0, 1\)"): + validate_whitened_parameters(1e-4, min_variance_ratio) + + +@pytest.mark.parametrize( + ("regularization_eps", "min_variance_ratio", "expected"), + [(1, 0, (1.0, 0.0)), (1e-4, 0.999, (1e-4, 0.999))], +) +def test_valid_parameters_are_normalized_to_floats( + regularization_eps, + min_variance_ratio, + expected, +): + assert validate_whitened_parameters(regularization_eps, min_variance_ratio) == expected + + +@pytest.mark.parametrize( + ("harmful_count", "harmless_count", "message"), + [(0, 0, "both be non-empty"), (1, 0, "both be non-empty"), (0, 1, "both be non-empty")], +) +def test_activation_sets_must_both_be_nonempty(harmful_count, harmless_count, message): + with pytest.raises(ValueError, match=message): + validate_whitened_request(harmful_count, harmless_count, 1) + + +@pytest.mark.parametrize(("harmful_count", "harmless_count"), [(1, 2), (3, 1)]) +def test_activation_sets_must_have_equal_sample_counts(harmful_count, harmless_count): + with pytest.raises(ValueError, match=f"got {harmful_count} and {harmless_count}"): + validate_whitened_request(harmful_count, harmless_count, 1) + + +@pytest.mark.parametrize("n_directions", [-1, 0, 1.5, True, "1"]) +def test_direction_count_must_be_a_positive_integer(n_directions): + with pytest.raises(ValueError, match="positive integer"): + validate_whitened_request(2, 2, n_directions) + + +@pytest.mark.parametrize("n_directions", [1, 2, 100]) +def test_valid_direction_count_is_returned(n_directions): + assert validate_whitened_request(2, 2, n_directions) == n_directions diff --git a/tests/test_property_contracts.py b/tests/test_property_contracts.py index fa2df02..fed02ca 100644 --- a/tests/test_property_contracts.py +++ b/tests/test_property_contracts.py @@ -2,10 +2,13 @@ from __future__ import annotations +import math + import pytest import torch from hypothesis import given, seed, settings, strategies as st +from obliteratus.analysis.whitened_svd import WhitenedSVDExtractor from obliteratus.evaluation.advanced_metrics import ( _is_refusal, linear_cka, @@ -122,3 +125,102 @@ def test_refusal_classification_is_invariant_to_case_and_leading_space( ): transformed = " " * leading_space + (refusal.upper() if upper else refusal.lower()) assert _is_refusal(transformed, mode="combined") + + +@seed(7007) +@PROPERTY_SETTINGS +@given( + n_samples=st.integers(3, 12), + hidden_dim=st.integers(3, 16), + offset=st.floats(-50, 50, allow_nan=False, allow_infinity=False), +) +def test_whitened_direction_is_invariant_to_common_activation_translation( + n_samples, hidden_dim, offset, +): + generator = torch.Generator().manual_seed(n_samples * 100 + hidden_dim) + harmless = torch.randn(n_samples, hidden_dim, generator=generator) + signal = torch.linspace(-1.0, 1.0, hidden_dim) + harmful = harmless + signal + extractor = WhitenedSVDExtractor() + + original = extractor.extract(list(harmful), list(harmless), n_directions=1) + translated = extractor.extract( + list(harmful + offset), + list(harmless + offset), + n_directions=1, + ) + + alignment = torch.dot(original.directions[0], translated.directions[0]).abs() + assert alignment == pytest.approx(1.0, abs=2e-4) + assert translated.variance_explained == pytest.approx( + original.variance_explained, + abs=2e-5, + ) + + +@seed(7008) +@PROPERTY_SETTINGS +@given(permutation=st.permutations(tuple(range(8)))) +def test_whitened_direction_is_invariant_to_joint_sample_permutation(permutation): + generator = torch.Generator().manual_seed(7008) + harmless = torch.randn(8, 10, generator=generator) + harmful = harmless + torch.linspace(-2.0, 2.0, 10) + extractor = WhitenedSVDExtractor() + + original = extractor.extract(list(harmful), list(harmless), n_directions=1) + permuted = extractor.extract( + [harmful[index] for index in permutation], + [harmless[index] for index in permutation], + n_directions=1, + ) + + alignment = torch.dot(original.directions[0], permuted.directions[0]).abs() + assert alignment == pytest.approx(1.0, abs=2e-5) + assert permuted.singular_values == pytest.approx(original.singular_values, rel=2e-5) + + +@seed(7009) +@PROPERTY_SETTINGS +@given( + n_samples=st.integers(2, 12), + hidden_dim=st.integers(2, 16), + requested=st.integers(1, 8), +) +def test_whitened_outputs_obey_normalization_ordering_and_bounds( + n_samples, hidden_dim, requested, +): + generator = torch.Generator().manual_seed(n_samples * 1000 + hidden_dim) + harmless = torch.randn(n_samples, hidden_dim, generator=generator) + harmful = harmless + torch.randn(n_samples, hidden_dim, generator=generator) + result = WhitenedSVDExtractor(min_variance_ratio=0).extract( + list(harmful), + list(harmless), + n_directions=requested, + ) + + expected_count = min(requested, n_samples, hidden_dim) + assert result.directions.shape == (expected_count, hidden_dim) + assert result.directions.norm(dim=1) == pytest.approx(torch.ones(expected_count)) + assert torch.all(result.singular_values >= 0) + assert torch.all(result.singular_values[:-1] >= result.singular_values[1:]) + assert 0.0 <= result.variance_explained <= 1.0 + assert math.isfinite(result.condition_number) + assert math.isfinite(result.effective_rank) + + +@pytest.mark.parametrize("dtype", [torch.float16, torch.float32, torch.float64]) +def test_whitened_direction_normalizes_supported_cpu_input_dtypes(dtype): + harmless = torch.tensor( + [[0, 0, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1]], + dtype=dtype, + ) + harmful = harmless + torch.tensor([0, 0, 2], dtype=dtype) + + result = WhitenedSVDExtractor().extract( + list(harmful), + list(harmless), + n_directions=1, + ) + + assert result.directions.dtype == torch.float32 + assert result.directions.norm() == pytest.approx(1.0)