Enhance project configuration and structure: Added build-system requirements in pyproject.toml, updated README for installation instructions, and refactored attack classes to utilize a new create_test_case method. Introduced a plugins.yaml for component registration and removed the legacy registry initialization file.

evaluators/__init__.py

@@ -1,4 +1,4 @@
-from .base_evaluator import BaseEvaluator
+from core.base_classes import BaseEvaluator
 from .default_judge import DefaultJudge
 
 __all__ = [

evaluators/base_evaluator.py

@@ -1,34 +0,0 @@
-"""
-Evaluator base class - new architecture implementation
-"""
-
-import logging
-from abc import ABC, abstractmethod
-from typing import Dict, Any
-
-from core.data_formats import ModelResponse, EvaluationResult
-
-
-class BaseEvaluator(ABC):
-    """Evaluator abstract base class"""
-
-    def __init__(self, config: Dict[str, Any] = None):
-        self.config = config or {}
-        self.logger = logging.getLogger(self.__class__.__name__)
-        self.name = self.__class__.__name__.lower()
-
-    @abstractmethod
-    def evaluate_response(
-        self, model_response: ModelResponse, **kwargs
-    ) -> EvaluationResult:
-        """
-        Evaluate model response
-
-        Args:
-            model_response: Model response
-            **kwargs: Additional parameters
-
-        Returns:
-            Evaluation result
-        """
-        pass

evaluators/default_judge.py

@@ -5,7 +5,7 @@ Adapted to new architecture, using BaseEvaluator base class
 
 from typing import Dict, Any, List
 
-from .base_evaluator import BaseEvaluator
+from core.base_classes import BaseEvaluator
 from core.data_formats import ModelResponse, EvaluationResult
 
 # Import original judge_utils
@@ -22,12 +22,18 @@ class DefaultJudge(BaseEvaluator):
     """
 
     def __init__(self, config: Dict[str, Any] = None):
+        config = config or {}
         super().__init__(config)
 
         # Get parameters from configuration
         # Configuration should include model, max_tokens, temperature, max_workers, etc.
-        self.judge_model_name = config.get("model")
-        self.success_threshold = config.get("success_threshold", 3)
+        self.judge_model_name = self.config.get("model")
+        if not self.judge_model_name:
+            raise ValueError(
+                "DefaultJudge requires `evaluation.evaluator_params.default_judge.model` "
+                "to be set to a valid model name."
+            )
+        self.success_threshold = self.config.get("success_threshold", 3)
 
         # Use unified LLM initialization
         from core.unified_registry import UNIFIED_REGISTRY