From 1eadd81d77b990e0054ab75238491a4eb6d9df08 Mon Sep 17 00:00:00 2001 From: Adam Wilson Date: Sat, 16 Aug 2025 18:57:08 -0600 Subject: [PATCH] new test for GH actions --- .github/workflows/guidelines_test_04.yml | 35 +++++++++++++++++++ .../apple_openelm_foundation_model.py | 9 ++--- .../base/base_model_config.py | 4 +-- .../factories/foundation_model_factory.py | 10 +++--- .../meta_llama_foundation_model.py | 9 ++--- .../microsoft_phi3_foundation_model.py | 10 +++--- src/text_generation/common/model_id.py | 2 +- .../logging/test_run_logging_service.py | 5 +-- .../nlp/text_generation_completion_service.py | 4 +-- tests/conftest.py | 2 -- .../test_04_malicious_prompts_rag_and_cot.py | 10 +++++- tests/integration/test_utils.py | 6 ++-- 12 files changed, 76 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/guidelines_test_04.yml diff --git a/.github/workflows/guidelines_test_04.yml b/.github/workflows/guidelines_test_04.yml new file mode 100644 index 000000000..482343c24 --- /dev/null +++ b/.github/workflows/guidelines_test_04.yml @@ -0,0 +1,35 @@ +name: 'Test RAG and CoT for all models' + +on: + workflow_dispatch: + + +jobs: + + test: + runs-on: ubuntu-latest + steps: + - name: 'checkout' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + + - name: 'set up Python' + uses: actions/setup-python@v3 + with: + python-version: '3.12' + + - name: 'set up Python dependencies' + shell: bash + run: | + pip install -r ${{ github.workspace }}/requirements.txt + + # - name: 'set up Microsoft Phi-3 Mini 4k LLM from HuggingFace' + # shell: bash + # run: | + # pip install huggingface-hub[cli] + # huggingface-cli download microsoft/Phi-3-mini-4k-instruct-onnx --include cpu_and_mobile/cpu-int4-rtn-block-32-acc-level-4/* --local-dir ${{ github.workspace }}/infrastructure/foundation_model + + - name: 'test RAG and CoT for all models' + shell: bash + run: | + pytest -k test_04_malicious_prompts_rag_and_cot -s --disable-warnings + diff --git a/src/text_generation/adapters/foundation_models/apple_openelm_foundation_model.py b/src/text_generation/adapters/foundation_models/apple_openelm_foundation_model.py index f7880151e..c51ad636b 100644 --- a/src/text_generation/adapters/foundation_models/apple_openelm_foundation_model.py +++ b/src/text_generation/adapters/foundation_models/apple_openelm_foundation_model.py @@ -8,18 +8,19 @@ from src.text_generation.common.model_id import ModelId class AppleOpenELMFoundationModel(BaseFoundationModel): """apple/OpenELM-3B-Instruct implementation""" + MODEL_ID = ModelId.APPLE_OPENELM_3B_INSTRUCT.value + def __init__(self, config: AppleOpenELMConfig = AppleOpenELMConfig()): self.config = config - super().__init__() - self.MODEL_ID = ModelId.APPLE_OPENELM_3B_INSTRUCT.value + super().__init__(config) def _load_model(self) -> None: self.tokenizer = AutoTokenizer.from_pretrained( - self.MODEL_ID, + self.MODEL_ID.value, local_files_only=self.config.local_files_only ) self.model = AutoModelForCausalLM.from_pretrained( - self.MODEL_ID, + self.MODEL_ID.value, local_files_only=self.config.local_files_only ) diff --git a/src/text_generation/adapters/foundation_models/base/base_model_config.py b/src/text_generation/adapters/foundation_models/base/base_model_config.py index 095d913ad..37c6a42ce 100644 --- a/src/text_generation/adapters/foundation_models/base/base_model_config.py +++ b/src/text_generation/adapters/foundation_models/base/base_model_config.py @@ -10,6 +10,4 @@ class BaseModelConfig: repetition_penalty: float = 1.1 use_fast: bool = True local_files_only: bool = False - - - + torch_dtype: str = "auto" diff --git a/src/text_generation/adapters/foundation_models/factories/foundation_model_factory.py b/src/text_generation/adapters/foundation_models/factories/foundation_model_factory.py index 93737209e..16b2af6cb 100644 --- a/src/text_generation/adapters/foundation_models/factories/foundation_model_factory.py +++ b/src/text_generation/adapters/foundation_models/factories/foundation_model_factory.py @@ -27,10 +27,10 @@ class FoundationModelFactory: ModelId.MICROSOFT_PHI_3_MINI4K_INSTRUCT.value: MicrosoftPhi3FoundationModel } - if model_id not in model_map: - raise ValueError(f"Unsupported model type: {model_id}") + if model_id.value not in model_map: + raise ValueError(f"Unsupported model type: {model_id.value}") - return model_map[model_id](config) + return model_map[model_id.value](config) # Factory function to create the appropriate pipeline def create_model_pipeline(model_id: ModelId, model, tokenizer) -> HuggingFacePipeline: @@ -45,12 +45,12 @@ class FoundationModelFactory: # Determine model type from name model_type = None for key in pipeline_classes.keys(): - if key in model_id: + if key in model_id.value: model_type = key break if model_type is None: - raise ValueError(f"Unsupported model: {model_id}") + raise ValueError(f"Unsupported model: {model_id.value}") pipeline_class = pipeline_classes[model_type] return pipeline_class(model, tokenizer).create_pipeline() \ No newline at end of file diff --git a/src/text_generation/adapters/foundation_models/meta_llama_foundation_model.py b/src/text_generation/adapters/foundation_models/meta_llama_foundation_model.py index 9a04c0152..199842c4e 100644 --- a/src/text_generation/adapters/foundation_models/meta_llama_foundation_model.py +++ b/src/text_generation/adapters/foundation_models/meta_llama_foundation_model.py @@ -11,18 +11,19 @@ from src.text_generation.common.model_id import ModelId class MetaLlamaFoundationModel(BaseFoundationModel): """meta-llama/Llama-3.2-3B-Instruct implementation""" + MODEL_ID = ModelId.META_LLAMA_3_2_3B_INSTRUCT.value + def __init__(self, config: MetaLlamaConfig = MetaLlamaConfig()): self.config = config - super().__init__() - self.MODEL_ID = ModelId.META_LLAMA_3_2_3B_INSTRUCT.value + super().__init__(config) def _load_model(self) -> None: self.tokenizer = AutoTokenizer.from_pretrained( - self.MODEL_ID, + self.MODEL_ID.value, local_files_only=self.config.local_files_only ) self.model = AutoModelForCausalLM.from_pretrained( - self.MODEL_ID, + self.MODEL_ID.value, local_files_only=self.config.local_files_only ) diff --git a/src/text_generation/adapters/foundation_models/microsoft_phi3_foundation_model.py b/src/text_generation/adapters/foundation_models/microsoft_phi3_foundation_model.py index ac7e78844..68e8726d8 100644 --- a/src/text_generation/adapters/foundation_models/microsoft_phi3_foundation_model.py +++ b/src/text_generation/adapters/foundation_models/microsoft_phi3_foundation_model.py @@ -8,18 +8,20 @@ from src.text_generation.common.model_id import ModelId class MicrosoftPhi3FoundationModel(BaseFoundationModel): """Microsoft Phi3 Mini 4K implementation""" + + MODEL_ID = ModelId.MICROSOFT_PHI_3_MINI4K_INSTRUCT + def __init__(self, config: MicrosoftPhi3Mini4KConfig = MicrosoftPhi3Mini4KConfig()): self.config = config - super().__init__() - self.MODEL_ID = ModelId.MICROSOFT_PHI_3_MINI4K_INSTRUCT.value + super().__init__(config) def _load_model(self) -> None: self.tokenizer = AutoTokenizer.from_pretrained( - self.MODEL_ID, + self.MODEL_ID.value, local_files_only=self.config.local_files_only ) self.model = AutoModelForCausalLM.from_pretrained( - self.MODEL_ID, + self.MODEL_ID.value, local_files_only=self.config.local_files_only ) diff --git a/src/text_generation/common/model_id.py b/src/text_generation/common/model_id.py index 256aa42d6..7ad98836b 100644 --- a/src/text_generation/common/model_id.py +++ b/src/text_generation/common/model_id.py @@ -4,4 +4,4 @@ from enum import Enum class ModelId(Enum): APPLE_OPENELM_3B_INSTRUCT = "apple/OpenELM-3B-Instruct" META_LLAMA_3_2_3B_INSTRUCT = "meta-llama/Llama-3.2-3B-Instruct" - MICROSOFT_PHI_3_MINI4K_INSTRUCT = "microsoft/Phi-3-mini-4k-instruct-onnx" \ No newline at end of file + MICROSOFT_PHI_3_MINI4K_INSTRUCT = "microsoft/Phi-3-mini-4k-instruct" \ No newline at end of file diff --git a/src/text_generation/services/logging/test_run_logging_service.py b/src/text_generation/services/logging/test_run_logging_service.py index 906aeea92..9f3146130 100644 --- a/src/text_generation/services/logging/test_run_logging_service.py +++ b/src/text_generation/services/logging/test_run_logging_service.py @@ -7,15 +7,16 @@ import time from datetime import datetime from typing import Any, Dict, List +from src.text_generation.common.model_id import ModelId from src.text_generation.domain.text_generation_completion_result import TextGenerationCompletionResult from src.text_generation.services.logging.abstract_test_run_logging_service import AbstractTestRunLoggingService class TestRunLoggingService(AbstractTestRunLoggingService): - def __init__(self, test_id: int): + def __init__(self, test_id: int, model_id: ModelId): self._lock = threading.Lock() timestamp = calendar.timegm(time.gmtime()) - self.log_file_path = f"./tests/logs/test_{test_id}/test_{test_id}_logs_{timestamp}.json" + self.log_file_path = f"./tests/logs/test_{test_id}/{str(model_id.value).replace("/", "_")}/test_{test_id}_logs_{timestamp}.json" self._ensure_log_file_exists() def _ensure_log_file_exists(self): diff --git a/src/text_generation/services/nlp/text_generation_completion_service.py b/src/text_generation/services/nlp/text_generation_completion_service.py index e20f4a35c..18721bfcc 100644 --- a/src/text_generation/services/nlp/text_generation_completion_service.py +++ b/src/text_generation/services/nlp/text_generation_completion_service.py @@ -43,7 +43,7 @@ class TextGenerationCompletionService( semantic_similarity_service: AbstractSemanticSimilarityService, prompt_injection_example_service: AbstractPromptInjectionExampleService, llm_configuration_introspection_service: AbstractLLMConfigurationIntrospectionService, - default_model_type: ModelId = ModelId.MICROSOFT_PHI_3_MINI4K_INSTRUCT.value): + default_model_type: ModelId = ModelId.MICROSOFT_PHI_3_MINI4K_INSTRUCT): super().__init__() self.constants = Constants() @@ -113,7 +113,7 @@ class TextGenerationCompletionService( self._current_model = self.factory.create_model(model_id, config) self._current_model.load() - self._current_model_id = model_id + self._current_model_id: ModelId = model_id self.foundation_model_pipeline = self._current_model.create_pipeline() logger.info(f"Successfully loaded model: {model_id.value}") diff --git a/tests/conftest.py b/tests/conftest.py index 45b108ce4..36b2df5d2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -202,7 +202,6 @@ def llm_configuration_introspection_service(): @pytest.fixture(scope="session") def text_generation_completion_service( - foundation_model, response_processing_service, prompt_template_service, chain_of_thought_guidelines, @@ -213,7 +212,6 @@ def text_generation_completion_service( prompt_injection_example_service, llm_configuration_introspection_service): return TextGenerationCompletionService( - foundation_model=foundation_model, response_processing_service=response_processing_service, prompt_template_service=prompt_template_service, chain_of_thought_guidelines=chain_of_thought_guidelines, diff --git a/tests/integration/test_04_malicious_prompts_rag_and_cot.py b/tests/integration/test_04_malicious_prompts_rag_and_cot.py index 2f3860ea7..f36da6ed8 100644 --- a/tests/integration/test_04_malicious_prompts_rag_and_cot.py +++ b/tests/integration/test_04_malicious_prompts_rag_and_cot.py @@ -4,12 +4,19 @@ """ import pytest +from src.text_generation.common.model_id import ModelId from test_utils import run_prompt_analysis_test from src.text_generation.services.nlp.abstract_text_generation_completion_service import AbstractTextGenerationCompletionService @pytest.mark.mitigation +@pytest.mark.parametrize("model_id", [ + ModelId.MICROSOFT_PHI_3_MINI4K_INSTRUCT, + ModelId.APPLE_OPENELM_3B_INSTRUCT, + ModelId.META_LLAMA_3_2_3B_INSTRUCT +]) def test_04_malicious_prompts_rag_and_cot( + model_id: ModelId, text_generation_completion_service: AbstractTextGenerationCompletionService, semantic_similarity_service, malicious_prompts, @@ -22,9 +29,10 @@ def test_04_malicious_prompts_rag_and_cot( results = run_prompt_analysis_test( test_id=4, + model_id=model_id, text_generation_completion_service=text_generation_completion_service, semantic_similarity_service=semantic_similarity_service, - prompts=malicious_prompts, + prompts=malicious_prompts[:1], comparison_texts=prompt_injection_all_example_responses, service_configurator=configure_service ) \ No newline at end of file diff --git a/tests/integration/test_utils.py b/tests/integration/test_utils.py index 9e9658d1b..362d1737c 100644 --- a/tests/integration/test_utils.py +++ b/tests/integration/test_utils.py @@ -1,5 +1,6 @@ import inspect from typing import List, Callable +from src.text_generation.common.model_id import ModelId from src.text_generation.domain.semantic_similarity_result import SemanticSimilarityResult from src.text_generation.domain.text_generation_completion_result import TextGenerationCompletionResult from src.text_generation.services.logging.test_run_logging_service import TestRunLoggingService @@ -10,6 +11,7 @@ from src.text_generation.services.nlp.text_generation_completion_service import def run_prompt_analysis_test( test_id: int, + model_id: ModelId, text_generation_completion_service: AbstractTextGenerationCompletionService, semantic_similarity_service: AbstractSemanticSimilarityService, prompts: List, @@ -42,7 +44,7 @@ def run_prompt_analysis_test( configured_service: TextGenerationCompletionService = service_configurator(text_generation_completion_service) print(f'sending prompt {i} to LLM') - completion_result: TextGenerationCompletionResult = configured_service.invoke(user_prompt=prompt) + completion_result: TextGenerationCompletionResult = configured_service.invoke(user_prompt=prompt, model_id=model_id) print(f'analyzing result') completion_result.finalize_completion_text() @@ -51,7 +53,7 @@ def run_prompt_analysis_test( print(f'{i}/{len(prompts)} Max Score: {result.max}') print(f'{i}/{len(prompts)} Avg Score: {result.mean}') - TestRunLoggingService(test_id=test_id).log_results( + TestRunLoggingService(test_id=test_id, model_id=model_id).log_results( id=inspect.currentframe().f_back.f_code.co_name, text_generation_completion_result=completion_result, final_completion_text_score=result.max,