diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c61de0c..b3c1c9f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -78,10 +78,19 @@ python scripts/check_test_risk_map.py ``` The machine-readable source-to-test ownership graph lives in -`ci/test-risk-map.json`. Each behavior-changing PR must add a focused regression -or contract test, relevant negative/boundary coverage, and propagation/runtime -evidence for public options. Hardware or service changes require both a -deterministic boundary test and their mapped conditional gate. +`ci/test-risk-map.json`. Its source inventory covers `app.py` and every Python +module under `obliteratus/`; every production file must belong to exactly one +named contract surface with an owner, contract types, and existing test files. +Adding a module without updating that graph fails `check_test_risk_map.py`. +Targeted risk-module entries add stricter conditional-gate and critical-path +requirements on top of the exhaustive contract ownership layer. + +Each behavior-changing PR must add a focused regression or contract test, +relevant negative/boundary coverage, and propagation/runtime evidence for +public options. Hardware or service changes require both a deterministic +boundary test and their mapped conditional gate. When adding or moving source, +update its contract surface and, when applicable, its targeted risk-module and +conditional-gate entries in the same PR. ## Code Style diff --git a/ci/conditional-test-policy.json b/ci/conditional-test-policy.json index 8616da5..a50443f 100644 --- a/ci/conditional-test-policy.json +++ b/ci/conditional-test-policy.json @@ -67,6 +67,7 @@ "prerequisites": "locked spaces extra; no public listener", "expected_cost": "under 10 runner-minutes", "coverage_paths": [ + "app.py", "obliteratus/interactive.py", "obliteratus/local_ui.py", "obliteratus/ui_watchtower.py" diff --git a/ci/test-risk-map.json b/ci/test-risk-map.json index f4ad80e..f95e66f 100644 --- a/ci/test-risk-map.json +++ b/ci/test-risk-map.json @@ -1,7 +1,238 @@ { - "schema_version": 1, + "schema_version": 2, "owner": "OBLITERATUS maintainers", + "source_inventory": { + "roots": ["app.py", "obliteratus"] + }, + "contract_surfaces": [ + { + "id": "package-entrypoints", + "owner": "packaging maintainers", + "description": "Installed package exports, module entry points, console dispatch, and optional application construction", + "contract_types": ["package-entrypoint", "public-interface", "operator-ui"], + "paths": [ + "app.py", + "obliteratus/__init__.py", + "obliteratus/__main__.py", + "obliteratus/analysis/__init__.py", + "obliteratus/evaluation/__init__.py", + "obliteratus/models/__init__.py", + "obliteratus/reporting/__init__.py", + "obliteratus/strategies/__init__.py" + ], + "required_tests": [ + "tests/test_cli.py", + "tests/test_cli_boundaries.py", + "tests/test_module_imports.py", + "tests/conditional/test_operator_ui.py" + ] + }, + { + "id": "core-pipeline", + "owner": "abliteration pipeline maintainers", + "description": "Model mutation, orchestration, automated search, study execution, sweeps, and tournament lifecycle", + "contract_types": ["model-mutation", "model-runtime", "orchestration", "persistence"], + "paths": [ + "obliteratus/abliterate.py", + "obliteratus/auto_obliterate.py", + "obliteratus/bayesian_optimizer.py", + "obliteratus/informed_pipeline.py", + "obliteratus/lora_ablation.py", + "obliteratus/runner.py", + "obliteratus/sweep.py", + "obliteratus/tourney.py" + ], + "required_tests": [ + "tests/test_abliterate.py", + "tests/test_abliterate_extended.py", + "tests/test_auto_obliterate.py", + "tests/test_informed_pipeline.py", + "tests/test_offline_integration.py", + "tests/test_runner_boundaries.py", + "tests/test_checkpoint_atomicity.py" + ] + }, + { + "id": "configuration-and-selection", + "owner": "runtime compatibility maintainers", + "description": "Configuration, defaults, model profiles, architecture detection, device selection, loading, and preset contracts", + "contract_types": ["configuration", "architecture-selection", "device-boundary", "model-runtime"], + "paths": [ + "obliteratus/adaptive_defaults.py", + "obliteratus/architecture_profiles.py", + "obliteratus/config.py", + "obliteratus/device.py", + "obliteratus/mlx_backend.py", + "obliteratus/model_profile.py", + "obliteratus/models/loader.py", + "obliteratus/presets.py", + "obliteratus/study_presets.py" + ], + "required_tests": [ + "tests/test_adaptive_defaults.py", + "tests/test_architecture_profiles.py", + "tests/test_config.py", + "tests/test_config_properties.py", + "tests/test_device_boundaries.py", + "tests/test_loader_boundaries.py", + "tests/test_mlx_backend_boundaries.py", + "tests/test_model_profile.py", + "tests/test_study_presets.py" + ] + }, + { + "id": "public-interface-and-services", + "owner": "operator interface maintainers", + "description": "CLI and interactive interfaces, remote execution, service adapters, scheduling, and UI boundaries", + "contract_types": ["public-interface", "external-service", "remote-execution", "operator-ui"], + "paths": [ + "obliteratus/bestiary_sync.py", + "obliteratus/cli.py", + "obliteratus/interactive.py", + "obliteratus/local_ui.py", + "obliteratus/models_client.py", + "obliteratus/remote.py", + "obliteratus/ui_watchtower.py", + "obliteratus/watchtower.py" + ], + "required_tests": [ + "tests/test_cli.py", + "tests/test_cli_boundaries.py", + "tests/test_remote_boundaries.py", + "tests/conditional/test_network_services.py", + "tests/conditional/test_operator_ui.py", + "tests/conditional/test_remote_runtime.py" + ] + }, + { + "id": "research-inputs-and-provenance", + "owner": "research integrity maintainers", + "description": "Prompt sources, hard negatives, community records, reproducibility controls, and telemetry schemas", + "contract_types": ["research-input", "research-output", "reproducibility", "persistence"], + "paths": [ + "obliteratus/community.py", + "obliteratus/hard_negative.py", + "obliteratus/prompts.py", + "obliteratus/reproducibility.py", + "obliteratus/telemetry.py" + ], + "required_tests": [ + "tests/test_community.py", + "tests/test_hard_negative.py", + "tests/test_prompt_sources.py", + "tests/test_module_imports.py", + "tests/test_telemetry.py" + ] + }, + { + "id": "analysis-methods", + "owner": "mechanistic analysis maintainers", + "description": "Mechanistic analysis algorithms, numerical invariants, transfer methods, and visualization contracts", + "contract_types": ["research-metric", "numerical-invariant", "research-output", "model-runtime"], + "paths": [ + "obliteratus/analysis/activation_patching.py", + "obliteratus/analysis/activation_probing.py", + "obliteratus/analysis/alignment_imprint.py", + "obliteratus/analysis/anti_ouroboros.py", + "obliteratus/analysis/bayesian_kernel_projection.py", + "obliteratus/analysis/causal_tracing.py", + "obliteratus/analysis/concept_geometry.py", + "obliteratus/analysis/conditional_abliteration.py", + "obliteratus/analysis/cross_layer.py", + "obliteratus/analysis/cross_model_transfer.py", + "obliteratus/analysis/defense_robustness.py", + "obliteratus/analysis/leace.py", + "obliteratus/analysis/logit_lens.py", + "obliteratus/analysis/multi_token_position.py", + "obliteratus/analysis/probing_classifiers.py", + "obliteratus/analysis/residual_stream.py", + "obliteratus/analysis/riemannian_manifold.py", + "obliteratus/analysis/sae_abliteration.py", + "obliteratus/analysis/sparse_surgery.py", + "obliteratus/analysis/spectral_certification.py", + "obliteratus/analysis/steering_vectors.py", + "obliteratus/analysis/tuned_lens.py", + "obliteratus/analysis/utils.py", + "obliteratus/analysis/visualization.py", + "obliteratus/analysis/wasserstein_optimal.py", + "obliteratus/analysis/wasserstein_transfer.py", + "obliteratus/analysis/whitened_svd.py" + ], + "required_tests": [ + "tests/test_analysis.py", + "tests/test_analysis_utils.py", + "tests/test_breakthrough_modules.py", + "tests/test_causal_and_transfer.py", + "tests/test_defense_robustness.py", + "tests/test_leace.py", + "tests/test_logit_lens.py", + "tests/test_new_analysis_modules.py", + "tests/test_novel_analysis.py", + "tests/test_visualization.py" + ] + }, + { + "id": "evaluation-and-benchmarks", + "owner": "evaluation maintainers", + "description": "Evaluation routing, benchmark execution, plots, statistical metrics, and external adapter results", + "contract_types": ["research-metric", "numerical-invariant", "research-output", "external-service"], + "paths": [ + "obliteratus/evaluation/advanced_metrics.py", + "obliteratus/evaluation/baselines.py", + "obliteratus/evaluation/benchmark_plots.py", + "obliteratus/evaluation/benchmarks.py", + "obliteratus/evaluation/evaluator.py", + "obliteratus/evaluation/heretic_eval.py", + "obliteratus/evaluation/lm_eval_integration.py", + "obliteratus/evaluation/metrics.py" + ], + "required_tests": [ + "tests/test_advanced_metrics.py", + "tests/test_benchmark_plots.py", + "tests/test_benchmarks.py", + "tests/test_evaluator.py", + "tests/test_heretic_eval.py", + "tests/test_metrics.py", + "tests/test_property_contracts.py" + ] + }, + { + "id": "reporting", + "owner": "research reporting maintainers", + "description": "Report schemas, deterministic serialization, plotting, and persisted result artifacts", + "contract_types": ["research-output", "persistence", "numerical-invariant"], + "paths": ["obliteratus/reporting/report.py"], + "required_tests": ["tests/test_report.py"] + }, + { + "id": "ablation-strategies", + "owner": "ablation strategy maintainers", + "description": "Strategy registration, enumeration, architecture navigation, mutation, and restoration behavior", + "contract_types": ["ablation-strategy", "architecture-selection", "model-mutation"], + "paths": [ + "obliteratus/strategies/base.py", + "obliteratus/strategies/embedding_ablation.py", + "obliteratus/strategies/ffn_ablation.py", + "obliteratus/strategies/head_pruning.py", + "obliteratus/strategies/layer_removal.py", + "obliteratus/strategies/registry.py", + "obliteratus/strategies/utils.py" + ], + "required_tests": [ + "tests/test_strategies.py", + "tests/test_gemma4_support.py", + "tests/test_study_presets.py" + ] + } + ], "modules": [ + { + "path": "app.py", + "risk_class": "conditional-runtime", + "risk": "optional application construction, UI wiring, authentication, and listener safety", + "required_tests": ["tests/conditional/test_operator_ui.py"], + "conditional_gates": ["operator-ui"] + }, { "path": "obliteratus/cli.py", "risk_class": "cpu-contract", diff --git a/scripts/check_test_risk_map.py b/scripts/check_test_risk_map.py index 87ed03f..dc14a86 100644 --- a/scripts/check_test_risk_map.py +++ b/scripts/check_test_risk_map.py @@ -5,11 +5,33 @@ from __future__ import annotations import argparse import json +import re from pathlib import Path from typing import Any RISK_CLASSES = {"cpu-contract", "mixed-runtime", "conditional-runtime"} +CONTRACT_TYPES = { + "ablation-strategy", + "architecture-selection", + "configuration", + "device-boundary", + "external-service", + "model-mutation", + "model-runtime", + "numerical-invariant", + "operator-ui", + "orchestration", + "package-entrypoint", + "persistence", + "public-interface", + "remote-execution", + "reproducibility", + "research-input", + "research-metric", + "research-output", +} +SURFACE_ID = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") def _load_object(path: Path, label: str, errors: list[str]) -> dict[str, Any]: @@ -24,6 +46,152 @@ def _load_object(path: Path, label: str, errors: list[str]) -> dict[str, Any]: return value +def _project_root(risk_path: Path, quality_path: Path) -> Path: + """Resolve the checkout root even when a test supplies temporary policy files.""" + + candidates = [ + risk_path.resolve().parent.parent, + quality_path.resolve().parent.parent, + Path.cwd().resolve(), + ] + for candidate in candidates: + if (candidate / "obliteratus").is_dir() and (candidate / "tests").is_dir(): + return candidate + return Path.cwd().resolve() + + +def _validate_test_paths( + tests: object, + *, + label: str, + project_root: Path, + errors: list[str], +) -> None: + if not isinstance(tests, list) or not tests: + errors.append(f"{label} requires at least one test") + return + if len(tests) != len(set(map(str, tests))): + errors.append(f"{label} has duplicate test paths") + for test_path in tests: + if not isinstance(test_path, str) or not test_path.startswith("tests/"): + errors.append(f"{label} has invalid test path: {test_path!r}") + elif not (project_root / test_path).is_file(): + errors.append(f"{label} maps missing test path: {test_path}") + + +def _inventory_sources( + inventory: object, + *, + project_root: Path, + errors: list[str], +) -> set[str]: + if not isinstance(inventory, dict): + errors.append("test risk map source_inventory must be an object") + return set() + roots = inventory.get("roots") + if not isinstance(roots, list) or not roots: + errors.append("test risk map source_inventory requires non-empty roots") + return set() + if len(roots) != len(set(map(str, roots))): + errors.append("test risk map source_inventory has duplicate roots") + + sources: set[str] = set() + for value in roots: + if not isinstance(value, str) or not value or Path(value).is_absolute(): + errors.append(f"source_inventory has invalid root: {value!r}") + continue + root = project_root / value + if root.is_file(): + if root.suffix != ".py": + errors.append(f"source_inventory root is not Python source: {value}") + else: + sources.add(root.relative_to(project_root).as_posix()) + elif root.is_dir(): + sources.update( + path.relative_to(project_root).as_posix() + for path in root.rglob("*.py") + if path.is_file() + ) + else: + errors.append(f"source_inventory root does not exist: {value}") + return sources + + +def _validate_contract_surfaces( + surfaces: object, + *, + inventory_sources: set[str], + project_root: Path, + errors: list[str], +) -> dict[str, str]: + if not isinstance(surfaces, list) or not surfaces: + errors.append("test risk map contract_surfaces must be a non-empty list") + return {} + + surface_ids: set[str] = set() + source_owner: dict[str, str] = {} + for index, surface in enumerate(surfaces): + label = f"contract surface {index}" + if not isinstance(surface, dict): + errors.append(f"{label} must be an object") + continue + surface_id = surface.get("id") + if not isinstance(surface_id, str) or not SURFACE_ID.fullmatch(surface_id): + errors.append(f"{label} has invalid id: {surface_id!r}") + surface_id = f"index-{index}" + elif surface_id in surface_ids: + errors.append(f"duplicate contract surface id: {surface_id}") + surface_ids.add(surface_id) + label = f"contract surface {surface_id}" + + if not isinstance(surface.get("owner"), str) or not surface["owner"].strip(): + errors.append(f"{label} requires a non-empty owner") + if not isinstance(surface.get("description"), str) or not surface["description"].strip(): + errors.append(f"{label} requires a non-empty description") + + contract_types = surface.get("contract_types") + if not isinstance(contract_types, list) or not contract_types: + errors.append(f"{label} requires at least one contract type") + else: + if len(contract_types) != len(set(map(str, contract_types))): + errors.append(f"{label} has duplicate contract types") + for contract_type in contract_types: + if not isinstance(contract_type, str) or contract_type not in CONTRACT_TYPES: + errors.append(f"{label} has invalid contract type: {contract_type!r}") + + _validate_test_paths( + surface.get("required_tests"), + label=label, + project_root=project_root, + errors=errors, + ) + + paths = surface.get("paths") + if not isinstance(paths, list) or not paths: + errors.append(f"{label} requires at least one source path") + continue + if len(paths) != len(set(map(str, paths))): + errors.append(f"{label} has duplicate source paths") + for source_path in paths: + if not isinstance(source_path, str) or not source_path: + errors.append(f"{label} has invalid source path: {source_path!r}") + continue + if source_path not in inventory_sources: + errors.append(f"{label} maps source outside inventory: {source_path}") + previous = source_owner.get(source_path) + if previous is not None: + errors.append( + f"production source path has multiple contract owners: " + f"{source_path} ({previous}, {surface_id})" + ) + else: + source_owner[source_path] = surface_id + + for source_path in sorted(inventory_sources - source_owner.keys()): + errors.append(f"unmapped production source path: {source_path}") + return source_owner + + def validate(risk_path: Path, quality_path: Path, conditional_path: Path) -> list[str]: """Return structural and cross-policy failures for the risk map.""" @@ -34,11 +202,24 @@ def validate(risk_path: Path, quality_path: Path, conditional_path: Path) -> lis if errors: return errors - if risk.get("schema_version") != 1: - errors.append("test risk map schema_version must be 1") + project_root = _project_root(risk_path, quality_path) + if risk.get("schema_version") != 2: + errors.append("test risk map schema_version must be 2") if not isinstance(risk.get("owner"), str) or not risk["owner"].strip(): errors.append("test risk map requires a non-empty owner") + inventory_sources = _inventory_sources( + risk.get("source_inventory"), + project_root=project_root, + errors=errors, + ) + source_contracts = _validate_contract_surfaces( + risk.get("contract_surfaces"), + inventory_sources=inventory_sources, + project_root=project_root, + errors=errors, + ) + gates = conditional.get("gates") gate_by_id = { gate.get("id"): gate @@ -62,30 +243,33 @@ def validate(risk_path: Path, quality_path: Path, conditional_path: Path) -> lis if path in module_by_path: errors.append(f"duplicate risk module path: {path}") module_by_path[path] = module - if not Path(path).is_file(): + if not (project_root / path).is_file(): errors.append(f"risk module maps missing source path: {path}") - if module.get("risk_class") not in RISK_CLASSES: + if path not in source_contracts: + errors.append(f"risk module is missing a contract owner: {path}") + risk_class = module.get("risk_class") + if not isinstance(risk_class, str) or risk_class not in RISK_CLASSES: errors.append(f"risk module {path} has invalid risk_class") if not isinstance(module.get("risk"), str) or not module["risk"].strip(): errors.append(f"risk module {path} requires a non-empty risk") - tests = module.get("required_tests") - if not isinstance(tests, list) or not tests: - errors.append(f"risk module {path} requires at least one test") - else: - for test_path in tests: - if not isinstance(test_path, str) or not test_path.startswith("tests/"): - errors.append(f"risk module {path} has invalid test path: {test_path!r}") - elif not Path(test_path).is_file(): - errors.append(f"risk module {path} maps missing test path: {test_path}") + _validate_test_paths( + module.get("required_tests"), + label=f"risk module {path}", + project_root=project_root, + errors=errors, + ) module_gates = module.get("conditional_gates") if not isinstance(module_gates, list): errors.append(f"risk module {path} conditional_gates must be a list") continue - if len(module_gates) != len(set(module_gates)): + if len(module_gates) != len(set(map(str, module_gates))): errors.append(f"risk module {path} has duplicate conditional gates") for gate_id in module_gates: + if not isinstance(gate_id, str): + errors.append(f"risk module {path} has invalid conditional gate: {gate_id!r}") + continue gate = gate_by_id.get(gate_id) if gate is None: errors.append(f"risk module {path} references unknown gate {gate_id}")