new test for GH actions

.github/workflows/guidelines_test_04.yml

@@ -0,0 +1,35 @@
+name: 'Test RAG and CoT for all models'
+
+on:
+  workflow_dispatch:
+
+
+jobs:
+
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'checkout'
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+      
+      - name: 'set up Python'
+        uses: actions/setup-python@v3
+        with:
+          python-version: '3.12'
+         
+      - name: 'set up Python dependencies'
+        shell: bash
+        run: |
+          pip install -r ${{ github.workspace }}/requirements.txt
+          
+      # - name: 'set up Microsoft Phi-3 Mini 4k LLM from HuggingFace'
+      #   shell: bash
+      #   run: |
+      #     pip install huggingface-hub[cli]
+      #     huggingface-cli download microsoft/Phi-3-mini-4k-instruct-onnx --include cpu_and_mobile/cpu-int4-rtn-block-32-acc-level-4/* --local-dir ${{ github.workspace }}/infrastructure/foundation_model
+
+      - name: 'test RAG and CoT for all models'
+        shell: bash  
+        run: |
+          pytest -k test_04_malicious_prompts_rag_and_cot -s --disable-warnings
+

src/text_generation/adapters/foundation_models/apple_openelm_foundation_model.py

@@ -8,18 +8,19 @@ from src.text_generation.common.model_id import ModelId
 class AppleOpenELMFoundationModel(BaseFoundationModel):
     """apple/OpenELM-3B-Instruct implementation"""
 
+    MODEL_ID = ModelId.APPLE_OPENELM_3B_INSTRUCT.value
+    
     def __init__(self, config: AppleOpenELMConfig = AppleOpenELMConfig()):
         self.config = config
-        super().__init__()
-        self.MODEL_ID = ModelId.APPLE_OPENELM_3B_INSTRUCT.value
+        super().__init__(config)
 
     def _load_model(self) -> None:
         self.tokenizer = AutoTokenizer.from_pretrained(
-            self.MODEL_ID,
+            self.MODEL_ID.value,
             local_files_only=self.config.local_files_only
         )
         self.model = AutoModelForCausalLM.from_pretrained(
-            self.MODEL_ID,
+            self.MODEL_ID.value,
             local_files_only=self.config.local_files_only
         )
 

src/text_generation/adapters/foundation_models/base/base_model_config.py

@@ -10,6 +10,4 @@ class BaseModelConfig:
     repetition_penalty: float = 1.1
     use_fast: bool = True
     local_files_only: bool = False
-
-
-
+    torch_dtype: str = "auto"

src/text_generation/adapters/foundation_models/factories/foundation_model_factory.py

@@ -27,10 +27,10 @@ class FoundationModelFactory:
             ModelId.MICROSOFT_PHI_3_MINI4K_INSTRUCT.value: MicrosoftPhi3FoundationModel
         }
 
-        if model_id not in model_map:
-            raise ValueError(f"Unsupported model type: {model_id}")
+        if model_id.value not in model_map:
+            raise ValueError(f"Unsupported model type: {model_id.value}")
 
-        return model_map[model_id](config)
+        return model_map[model_id.value](config)
     
     # Factory function to create the appropriate pipeline
     def create_model_pipeline(model_id: ModelId, model, tokenizer) -> HuggingFacePipeline:
@@ -45,12 +45,12 @@ class FoundationModelFactory:
         # Determine model type from name
         model_type = None
         for key in pipeline_classes.keys():
-            if key in model_id:
+            if key in model_id.value:
                 model_type = key
                 break
         
         if model_type is None:
-            raise ValueError(f"Unsupported model: {model_id}")
+            raise ValueError(f"Unsupported model: {model_id.value}")
         
         pipeline_class = pipeline_classes[model_type]
         return pipeline_class(model, tokenizer).create_pipeline()

src/text_generation/adapters/foundation_models/meta_llama_foundation_model.py

@@ -11,18 +11,19 @@ from src.text_generation.common.model_id import ModelId
 class MetaLlamaFoundationModel(BaseFoundationModel):
     """meta-llama/Llama-3.2-3B-Instruct implementation"""
 
+    MODEL_ID = ModelId.META_LLAMA_3_2_3B_INSTRUCT.value
+    
     def __init__(self, config: MetaLlamaConfig = MetaLlamaConfig()):
         self.config = config
-        super().__init__()
-        self.MODEL_ID = ModelId.META_LLAMA_3_2_3B_INSTRUCT.value
+        super().__init__(config)
 
     def _load_model(self) -> None:
         self.tokenizer = AutoTokenizer.from_pretrained(
-            self.MODEL_ID,
+            self.MODEL_ID.value,
             local_files_only=self.config.local_files_only
         )
         self.model = AutoModelForCausalLM.from_pretrained(
-            self.MODEL_ID,
+            self.MODEL_ID.value,
             local_files_only=self.config.local_files_only
         )
 

src/text_generation/adapters/foundation_models/microsoft_phi3_foundation_model.py

@@ -8,18 +8,20 @@ from src.text_generation.common.model_id import ModelId
 
 class MicrosoftPhi3FoundationModel(BaseFoundationModel):
     """Microsoft Phi3 Mini 4K implementation"""
+    
+    MODEL_ID = ModelId.MICROSOFT_PHI_3_MINI4K_INSTRUCT
+    
     def __init__(self, config: MicrosoftPhi3Mini4KConfig = MicrosoftPhi3Mini4KConfig()):
         self.config = config
-        super().__init__()
-        self.MODEL_ID = ModelId.MICROSOFT_PHI_3_MINI4K_INSTRUCT.value
+        super().__init__(config)
 
     def _load_model(self) -> None:
         self.tokenizer = AutoTokenizer.from_pretrained(
-            self.MODEL_ID,
+            self.MODEL_ID.value,
             local_files_only=self.config.local_files_only
         )
         self.model = AutoModelForCausalLM.from_pretrained(
-            self.MODEL_ID,
+            self.MODEL_ID.value,
             local_files_only=self.config.local_files_only
         )
 

src/text_generation/common/model_id.py

@@ -4,4 +4,4 @@ from enum import Enum
 class ModelId(Enum):
     APPLE_OPENELM_3B_INSTRUCT = "apple/OpenELM-3B-Instruct"
     META_LLAMA_3_2_3B_INSTRUCT = "meta-llama/Llama-3.2-3B-Instruct"
-    MICROSOFT_PHI_3_MINI4K_INSTRUCT = "microsoft/Phi-3-mini-4k-instruct-onnx"
+    MICROSOFT_PHI_3_MINI4K_INSTRUCT = "microsoft/Phi-3-mini-4k-instruct"

src/text_generation/services/logging/test_run_logging_service.py

@@ -7,15 +7,16 @@ import time
 from datetime import datetime
 from typing import Any, Dict, List
 
+from src.text_generation.common.model_id import ModelId
 from src.text_generation.domain.text_generation_completion_result import TextGenerationCompletionResult
 from src.text_generation.services.logging.abstract_test_run_logging_service import AbstractTestRunLoggingService
 
 
 class TestRunLoggingService(AbstractTestRunLoggingService):
-    def __init__(self, test_id: int):
+    def __init__(self, test_id: int, model_id: ModelId):
         self._lock = threading.Lock()
         timestamp = calendar.timegm(time.gmtime())
-        self.log_file_path = f"./tests/logs/test_{test_id}/test_{test_id}_logs_{timestamp}.json"
+        self.log_file_path = f"./tests/logs/test_{test_id}/{str(model_id.value).replace("/", "_")}/test_{test_id}_logs_{timestamp}.json"
         self._ensure_log_file_exists()
 
     def _ensure_log_file_exists(self):

src/text_generation/services/nlp/text_generation_completion_service.py

@@ -43,7 +43,7 @@ class TextGenerationCompletionService(
             semantic_similarity_service: AbstractSemanticSimilarityService,
             prompt_injection_example_service: AbstractPromptInjectionExampleService,
             llm_configuration_introspection_service: AbstractLLMConfigurationIntrospectionService,
-            default_model_type: ModelId = ModelId.MICROSOFT_PHI_3_MINI4K_INSTRUCT.value):
+            default_model_type: ModelId = ModelId.MICROSOFT_PHI_3_MINI4K_INSTRUCT):
         
         super().__init__()
         self.constants = Constants()
@@ -113,7 +113,7 @@ class TextGenerationCompletionService(
         
         self._current_model = self.factory.create_model(model_id, config)
         self._current_model.load()
-        self._current_model_id = model_id
+        self._current_model_id: ModelId = model_id
         self.foundation_model_pipeline = self._current_model.create_pipeline()
         
         logger.info(f"Successfully loaded model: {model_id.value}")

tests/conftest.py

@@ -202,7 +202,6 @@ def llm_configuration_introspection_service():
 
 @pytest.fixture(scope="session")
 def text_generation_completion_service(
-        foundation_model,
         response_processing_service,
         prompt_template_service,
         chain_of_thought_guidelines,
@@ -213,7 +212,6 @@ def text_generation_completion_service(
         prompt_injection_example_service,
         llm_configuration_introspection_service):
     return TextGenerationCompletionService(
-        foundation_model=foundation_model,
         response_processing_service=response_processing_service,
         prompt_template_service=prompt_template_service,
         chain_of_thought_guidelines=chain_of_thought_guidelines,

tests/integration/test_04_malicious_prompts_rag_and_cot.py

@@ -4,12 +4,19 @@
 """
 
 import pytest
+from src.text_generation.common.model_id import ModelId
 from test_utils import run_prompt_analysis_test
 from src.text_generation.services.nlp.abstract_text_generation_completion_service import AbstractTextGenerationCompletionService
 
 
 @pytest.mark.mitigation
+@pytest.mark.parametrize("model_id", [
+    ModelId.MICROSOFT_PHI_3_MINI4K_INSTRUCT,
+    ModelId.APPLE_OPENELM_3B_INSTRUCT,
+    ModelId.META_LLAMA_3_2_3B_INSTRUCT
+])
 def test_04_malicious_prompts_rag_and_cot(
+    model_id: ModelId,
     text_generation_completion_service: AbstractTextGenerationCompletionService,
     semantic_similarity_service,
     malicious_prompts,
@@ -22,9 +29,10 @@ def test_04_malicious_prompts_rag_and_cot(
     
     results = run_prompt_analysis_test(
         test_id=4,
+        model_id=model_id,
         text_generation_completion_service=text_generation_completion_service,
         semantic_similarity_service=semantic_similarity_service,
-        prompts=malicious_prompts,
+        prompts=malicious_prompts[:1],
         comparison_texts=prompt_injection_all_example_responses,
         service_configurator=configure_service
     )

tests/integration/test_utils.py

@@ -1,5 +1,6 @@
 import inspect
 from typing import List, Callable
+from src.text_generation.common.model_id import ModelId
 from src.text_generation.domain.semantic_similarity_result import SemanticSimilarityResult
 from src.text_generation.domain.text_generation_completion_result import TextGenerationCompletionResult
 from src.text_generation.services.logging.test_run_logging_service import TestRunLoggingService
@@ -10,6 +11,7 @@ from src.text_generation.services.nlp.text_generation_completion_service import
 
 def run_prompt_analysis_test(
     test_id: int,
+    model_id: ModelId,
     text_generation_completion_service: AbstractTextGenerationCompletionService,
     semantic_similarity_service: AbstractSemanticSimilarityService,
     prompts: List,
@@ -42,7 +44,7 @@ def run_prompt_analysis_test(
         configured_service: TextGenerationCompletionService = service_configurator(text_generation_completion_service)
         
         print(f'sending prompt {i} to LLM')
-        completion_result: TextGenerationCompletionResult = configured_service.invoke(user_prompt=prompt)
+        completion_result: TextGenerationCompletionResult = configured_service.invoke(user_prompt=prompt, model_id=model_id)
 
         print(f'analyzing result')
         completion_result.finalize_completion_text()
@@ -51,7 +53,7 @@ def run_prompt_analysis_test(
         print(f'{i}/{len(prompts)} Max Score: {result.max}')
         print(f'{i}/{len(prompts)} Avg Score: {result.mean}')
         
-        TestRunLoggingService(test_id=test_id).log_results(
+        TestRunLoggingService(test_id=test_id, model_id=model_id).log_results(
             id=inspect.currentframe().f_back.f_code.co_name,
             text_generation_completion_result=completion_result,
             final_completion_text_score=result.max,