From 7122235b4f32575c8026448865983374e5ebaec8 Mon Sep 17 00:00:00 2001 From: Adam Wilson Date: Tue, 19 Aug 2025 17:06:12 -0600 Subject: [PATCH] try OpenELM 1.1B --- ...ag_and_cot_apple_openelm_1_1b_instruct.01-20.yml} | 4 ++-- ...ag_and_cot_apple_openelm_1_1b_instruct.21-40.yml} | 4 ++-- ...ag_and_cot_apple_openelm_1_1b_instruct.41-60.yml} | 4 ++-- ...ag_and_cot_apple_openelm_1_1b_instruct.61-80.yml} | 4 ++-- ...g_and_cot_apple_openelm_1_1b_instruct.81-100.yml} | 4 ++-- ...rag_and_cot_apple_openelm_1_1b_instruct.base.yml} | 6 +++--- ...mpts_rag_and_cot_apple_openelm_1_1b_instruct.yml} | 4 ++-- .../apple_openelm_foundation_model.py | 4 ++-- .../factories/foundation_model_factory.py | 4 ++-- .../factory/foundation_model_factory.py | 2 +- src/text_generation/common/constants.py | 10 +++++----- src/text_generation/common/model_id.py | 2 +- .../chain_of_thought_security_guidelines_service.py | 12 ++---------- .../rag_context_security_guidelines_service.py | 12 ++---------- .../rag_plus_cot_security_guidelines_service.py | 12 ++---------- .../nlp/text_generation_completion_service.py | 10 +++++----- tests/create_dirs.sh | 2 +- .../test_04_malicious_prompts_rag_and_cot.py | 2 +- ...rompts_no_mitigation_apple_openelm_3b_instruct.py | 4 ++-- ...rompts_no_mitigation_apple_openelm_3b_instruct.py | 4 ++-- ...alicious_prompts_cot_apple_openelm_3b_instruct.py | 4 ++-- ...alicious_prompts_rag_apple_openelm_3b_instruct.py | 4 ++-- ..._prompts_rag_and_cot_apple_openelm_3b_instruct.py | 4 ++-- tests/integration/test_prompt_template_repository.py | 10 +++++----- 24 files changed, 54 insertions(+), 78 deletions(-) rename .github/workflows/{test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.01-20.yml => test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.01-20.yml} (65%) rename .github/workflows/{test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.21-40.yml => test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.21-40.yml} (65%) rename .github/workflows/{test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.41-60.yml => test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.41-60.yml} (65%) rename .github/workflows/{test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.61-80.yml => test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.61-80.yml} (65%) rename .github/workflows/{test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.81-100.yml => test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.81-100.yml} (65%) rename .github/workflows/{test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.base.yml => test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.base.yml} (80%) rename .github/workflows/{test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.yml => test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.yml} (90%) diff --git a/.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.01-20.yml b/.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.01-20.yml similarity index 65% rename from .github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.01-20.yml rename to .github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.01-20.yml index 2b6d2c091..4ef53cc9a 100644 --- a/.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.01-20.yml +++ b/.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.01-20.yml @@ -1,10 +1,10 @@ -name: '#4 (1-20) | RAG + CoT | apple/OpenELM-3B-Instruct' +name: '#4 (1-20) | RAG + CoT | apple/OpenELM-1_1B-Instruct' on: workflow_dispatch: jobs: test-prompts-1-20: - uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.base.yml + uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.base.yml with: batch_offset: 0 range_name: "1-20" diff --git a/.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.21-40.yml b/.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.21-40.yml similarity index 65% rename from .github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.21-40.yml rename to .github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.21-40.yml index 753d83445..74cc492bf 100644 --- a/.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.21-40.yml +++ b/.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.21-40.yml @@ -1,10 +1,10 @@ -name: '#4 (21-40) | RAG + CoT | apple/OpenELM-3B-Instruct' +name: '#4 (21-40) | RAG + CoT | apple/OpenELM-1_1B-Instruct' on: workflow_dispatch: jobs: test-prompts-21-40: - uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.base.yml + uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.base.yml with: batch_offset: 20 range_name: "21-40" diff --git a/.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.41-60.yml b/.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.41-60.yml similarity index 65% rename from .github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.41-60.yml rename to .github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.41-60.yml index 9a6a654eb..99e91bc9d 100644 --- a/.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.41-60.yml +++ b/.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.41-60.yml @@ -1,10 +1,10 @@ -name: '#4 (41-60) | RAG + CoT | apple/OpenELM-3B-Instruct' +name: '#4 (41-60) | RAG + CoT | apple/OpenELM-1_1B-Instruct' on: workflow_dispatch: jobs: test-prompts-41-60: - uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.base.yml + uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.base.yml with: batch_offset: 40 range_name: "41-60" diff --git a/.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.61-80.yml b/.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.61-80.yml similarity index 65% rename from .github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.61-80.yml rename to .github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.61-80.yml index c8b51fdf1..45042f063 100644 --- a/.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.61-80.yml +++ b/.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.61-80.yml @@ -1,10 +1,10 @@ -name: '#4 (61-80) | RAG + CoT | apple/OpenELM-3B-Instruct' +name: '#4 (61-80) | RAG + CoT | apple/OpenELM-1_1B-Instruct' on: workflow_dispatch: jobs: test-prompts-61-80: - uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.base.yml + uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.base.yml with: batch_offset: 60 range_name: "61-80" diff --git a/.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.81-100.yml b/.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.81-100.yml similarity index 65% rename from .github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.81-100.yml rename to .github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.81-100.yml index 8d82586a5..16c6f82b8 100644 --- a/.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.81-100.yml +++ b/.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.81-100.yml @@ -1,10 +1,10 @@ -name: '#4 (81-100) | RAG + CoT | apple/OpenELM-3B-Instruct' +name: '#4 (81-100) | RAG + CoT | apple/OpenELM-1_1B-Instruct' on: workflow_dispatch: jobs: test-prompts-81-100: - uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.base.yml + uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.base.yml with: batch_offset: 80 range_name: "81-100" diff --git a/.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.base.yml b/.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.base.yml similarity index 80% rename from .github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.base.yml rename to .github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.base.yml index f52be072d..ccf35c267 100644 --- a/.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.base.yml +++ b/.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.base.yml @@ -1,4 +1,4 @@ -name: 'Reusable Test #4 | RAG + CoT | apple/OpenELM-3B-Instruct' +name: 'Reusable Test #4 | RAG + CoT | apple/OpenELM-1_1B-Instruct' on: workflow_call: inputs: @@ -23,5 +23,5 @@ jobs: batch_offset: ${{ inputs.batch_offset }} range_name: ${{ inputs.range_name }} batch_size: ${{ inputs.batch_size }} - test_file_path: tests/integration/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.py - model_display_name: apple/OpenELM-3B-Instruct \ No newline at end of file + test_file_path: tests/integration/test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.py + model_display_name: apple/OpenELM-1_1B-Instruct \ No newline at end of file diff --git a/.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.yml b/.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.yml similarity index 90% rename from .github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.yml rename to .github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.yml index 24c9e5bb0..3228038c5 100644 --- a/.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.yml +++ b/.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.yml @@ -1,4 +1,4 @@ -name: 'Test #4 | RAG + CoT | apple/OpenELM-3B-Instruct' +name: 'Test #4 | RAG + CoT | apple/OpenELM-1_1B-Instruct' on: workflow_dispatch: @@ -21,7 +21,7 @@ jobs: - name: 'run text generation tests' shell: bash - run: pytest tests/integration/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.py -s --disable-warnings + run: pytest tests/integration/test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct.py -s --disable-warnings - name: Check for changes id: verify-changed-files diff --git a/src/text_generation/adapters/foundation_models/apple_openelm_foundation_model.py b/src/text_generation/adapters/foundation_models/apple_openelm_foundation_model.py index dbfee306f..58fa63ec3 100644 --- a/src/text_generation/adapters/foundation_models/apple_openelm_foundation_model.py +++ b/src/text_generation/adapters/foundation_models/apple_openelm_foundation_model.py @@ -6,9 +6,9 @@ from src.text_generation.common.model_id import ModelId class AppleOpenELMFoundationModel(BaseFoundationModel): - """apple/OpenELM-3B-Instruct implementation""" + """apple/OpenELM-1_1B-Instruct implementation""" - MODEL_ID = ModelId.APPLE_OPENELM_3B_INSTRUCT + MODEL_ID = ModelId.APPLE_OPENELM_1_1B_INSTRUCT def __init__(self, config: AppleOpenELMConfig = AppleOpenELMConfig()): self.config = config diff --git a/src/text_generation/adapters/foundation_models/factories/foundation_model_factory.py b/src/text_generation/adapters/foundation_models/factories/foundation_model_factory.py index 16b2af6cb..a446e3913 100644 --- a/src/text_generation/adapters/foundation_models/factories/foundation_model_factory.py +++ b/src/text_generation/adapters/foundation_models/factories/foundation_model_factory.py @@ -22,7 +22,7 @@ class FoundationModelFactory: config = BaseModelConfig() model_map = { - ModelId.APPLE_OPENELM_3B_INSTRUCT.value: AppleOpenELMFoundationModel, + ModelId.APPLE_OPENELM_1_1B_INSTRUCT.value: AppleOpenELMFoundationModel, ModelId.META_LLAMA_3_2_3B_INSTRUCT.value: MetaLlamaFoundationModel, ModelId.MICROSOFT_PHI_3_MINI4K_INSTRUCT.value: MicrosoftPhi3FoundationModel } @@ -37,7 +37,7 @@ class FoundationModelFactory: """Factory function to create the appropriate pipeline based on model name""" pipeline_classes = { - ModelId.APPLE_OPENELM_3B_INSTRUCT.value: AppleOpenELMPipeline, + ModelId.APPLE_OPENELM_1_1B_INSTRUCT.value: AppleOpenELMPipeline, ModelId.META_LLAMA_3_2_3B_INSTRUCT.value: MetaLlamaPipeline, ModelId.MICROSOFT_PHI_3_MINI4K_INSTRUCT.value: MicrosoftPhi3MiniPipeline } diff --git a/src/text_generation/adapters/foundation_models/factory/foundation_model_factory.py b/src/text_generation/adapters/foundation_models/factory/foundation_model_factory.py index 871f886dc..8f130bb75 100644 --- a/src/text_generation/adapters/foundation_models/factory/foundation_model_factory.py +++ b/src/text_generation/adapters/foundation_models/factory/foundation_model_factory.py @@ -17,7 +17,7 @@ class FoundationModelFactory: config = BaseModelConfig() model_map = { - ModelId.APPLE_OPENELM_3B_INSTRUCT.value: AppleOpenELMFoundationModel, + ModelId.APPLE_OPENELM_1_1B_INSTRUCT.value: AppleOpenELMFoundationModel, ModelId.META_LLAMA_3_2_3B_INSTRUCT.value: MetaLlamaFoundationModel, ModelId.MICROSOFT_PHI_3_MINI4K_INSTRUCT.value: MicrosoftPhi3FoundationModel } diff --git a/src/text_generation/common/constants.py b/src/text_generation/common/constants.py index 0450e998c..3b25a186e 100644 --- a/src/text_generation/common/constants.py +++ b/src/text_generation/common/constants.py @@ -24,11 +24,11 @@ class Constants: PHI_3_MINI_4K_INSTRUCT__03_FEW_SHOT_EXAMPLES = "phi-3-mini-4k-instruct.03-few-shot" PHI_3_MINI_4K_INSTRUCT__04_FEW_SHOT_RAG_PLUS_COT = "phi-3-mini-4k-instruct.04-few-shot-rag-plus-cot" - # OpenELM templates: apple/OpenELM-3B-Instruct - OPENELM_3B_INSTRUCT__01_BASIC = "openelm-3b-instruct.01-basic" - OPENELM_3B_INSTRUCT__02_ZERO_SHOT_CHAIN_OF_THOUGHT = "openelm-3b-instruct.02-zero-shot-cot" - OPENELM_3B_INSTRUCT__03_FEW_SHOT_EXAMPLES = "openelm-3b-instruct.03-few-shot" - OPENELM_3B_INSTRUCT__04_FEW_SHOT_RAG_PLUS_COT = "openelm-3b-instruct.04-few-shot-rag-plus-cot" + # OpenELM templates: apple/OpenELM-1_1B-Instruct + OPENELM_1_1B_INSTRUCT__01_BASIC = "openelm-1-1b-instruct.01-basic" + OPENELM_1_1B_INSTRUCT__02_ZERO_SHOT_CHAIN_OF_THOUGHT = "openelm-1-1b-instruct.02-zero-shot-cot" + OPENELM_1_1B_INSTRUCT__03_FEW_SHOT_EXAMPLES = "openelm-1-1b-instruct.03-few-shot" + OPENELM_1_1B_INSTRUCT__04_FEW_SHOT_RAG_PLUS_COT = "openelm-1-1b-instruct.04-few-shot-rag-plus-cot" # meta-llama/Llama-3.2-3B-Instruct templates LLAMA_1_1B_CHAT__01_BASIC = "llama-3.2-3b-instruct.01-basic" diff --git a/src/text_generation/common/model_id.py b/src/text_generation/common/model_id.py index d20189de3..1f272083f 100644 --- a/src/text_generation/common/model_id.py +++ b/src/text_generation/common/model_id.py @@ -2,6 +2,6 @@ from enum import Enum class ModelId(Enum): - APPLE_OPENELM_3B_INSTRUCT = "apple/OpenELM-3B-Instruct" + APPLE_OPENELM_1_1B_INSTRUCT = "apple/OpenELM-1_1B-Instruct" META_LLAMA_3_2_3B_INSTRUCT = "meta-llama/Llama-3.2-3B-Instruct" MICROSOFT_PHI_3_MINI4K_INSTRUCT = "microsoft/Phi-3-mini-4k-instruct" diff --git a/src/text_generation/services/guidelines/chain_of_thought_security_guidelines_service.py b/src/text_generation/services/guidelines/chain_of_thought_security_guidelines_service.py index 150ad2a90..a3b78cee8 100644 --- a/src/text_generation/services/guidelines/chain_of_thought_security_guidelines_service.py +++ b/src/text_generation/services/guidelines/chain_of_thought_security_guidelines_service.py @@ -38,16 +38,8 @@ class ChainOfThoughtSecurityGuidelinesService(BaseSecurityGuidelinesService): Dict[str, str]: Mapping from model name/identifier to CoT template ID """ return { - # Phi-3 models - "phi-3-mini-4k-instruct": self.constants.PromptTemplateIds.PHI_3_MINI_4K_INSTRUCT__02_ZERO_SHOT_CHAIN_OF_THOUGHT, - "microsoft/Phi-3-mini-4K-Instruct": self.constants.PromptTemplateIds.PHI_3_MINI_4K_INSTRUCT__02_ZERO_SHOT_CHAIN_OF_THOUGHT, - - # OpenELM models - "openelm-3b-instruct": self.constants.PromptTemplateIds.OPENELM_3B_INSTRUCT__02_ZERO_SHOT_CHAIN_OF_THOUGHT, - "apple/OpenELM-3B-Instruct": self.constants.PromptTemplateIds.OPENELM_3B_INSTRUCT__02_ZERO_SHOT_CHAIN_OF_THOUGHT, - - # Llama models - "llama-3.2-3b-instruct": self.constants.PromptTemplateIds.LLAMA_1_1B_CHAT__02_ZERO_SHOT_CHAIN_OF_THOUGHT, + "microsoft/Phi-3-mini-4K-Instruct": self.constants.PromptTemplateIds.PHI_3_MINI_4K_INSTRUCT__02_ZERO_SHOT_CHAIN_OF_THOUGHT, + "apple/OpenELM-1_1B-Instruct": self.constants.PromptTemplateIds.OPENELM_1_1B_INSTRUCT__02_ZERO_SHOT_CHAIN_OF_THOUGHT, "meta-llama/Llama-3.2-3B-Instruct": self.constants.PromptTemplateIds.LLAMA_1_1B_CHAT__02_ZERO_SHOT_CHAIN_OF_THOUGHT, } diff --git a/src/text_generation/services/guidelines/rag_context_security_guidelines_service.py b/src/text_generation/services/guidelines/rag_context_security_guidelines_service.py index a55a3bb2a..286c4d489 100644 --- a/src/text_generation/services/guidelines/rag_context_security_guidelines_service.py +++ b/src/text_generation/services/guidelines/rag_context_security_guidelines_service.py @@ -38,17 +38,9 @@ class RagContextSecurityGuidelinesService(BaseSecurityGuidelinesService): Dict[str, str]: Mapping from model name/identifier to few-shot template ID """ return { - # Phi-3 models - "phi-3-mini-4k-instruct": self.constants.PromptTemplateIds.PHI_3_MINI_4K_INSTRUCT__03_FEW_SHOT_EXAMPLES, "microsoft/phi-3-mini-4k-instruct": self.constants.PromptTemplateIds.PHI_3_MINI_4K_INSTRUCT__03_FEW_SHOT_EXAMPLES, - - # OpenELM models - "openelm-3b-instruct": self.constants.PromptTemplateIds.OPENELM_3B_INSTRUCT__03_FEW_SHOT_EXAMPLES, - "apple/openelm-3b-instruct": self.constants.PromptTemplateIds.OPENELM_3B_INSTRUCT__03_FEW_SHOT_EXAMPLES, - - # Llama models - "llama-3.2-3b-instruct": self.constants.PromptTemplateIds.LLAMA_1_1B_CHAT__03_FEW_SHOT_EXAMPLES, - "meta-llama/llama-3.2-3b-instruct": self.constants.PromptTemplateIds.LLAMA_1_1B_CHAT__03_FEW_SHOT_EXAMPLES, + "apple/OpenELM-1_1B-Instruct": self.constants.PromptTemplateIds.OPENELM_1_1B_INSTRUCT__03_FEW_SHOT_EXAMPLES, + "meta-llama/llama-3.2-3b-instruct": self.constants.PromptTemplateIds.LLAMA_1_1B_CHAT__03_FEW_SHOT_EXAMPLES } def _get_template(self, user_prompt: str, template_id: str) -> StringPromptTemplate: diff --git a/src/text_generation/services/guidelines/rag_plus_cot_security_guidelines_service.py b/src/text_generation/services/guidelines/rag_plus_cot_security_guidelines_service.py index 1cd972d66..030eff9e8 100644 --- a/src/text_generation/services/guidelines/rag_plus_cot_security_guidelines_service.py +++ b/src/text_generation/services/guidelines/rag_plus_cot_security_guidelines_service.py @@ -41,17 +41,9 @@ class RagPlusCotSecurityGuidelinesService(BaseSecurityGuidelinesService): Dict[str, str]: Mapping from model name/identifier to RAG+CoT template ID """ return { - # Phi-3 models - "phi-3-mini-4k-instruct": self.constants.PromptTemplateIds.PHI_3_MINI_4K_INSTRUCT__04_FEW_SHOT_RAG_PLUS_COT, "microsoft/phi-3-mini-4k-instruct": self.constants.PromptTemplateIds.PHI_3_MINI_4K_INSTRUCT__04_FEW_SHOT_RAG_PLUS_COT, - - # OpenELM models - "openelm-3b-instruct": self.constants.PromptTemplateIds.OPENELM_3B_INSTRUCT__04_FEW_SHOT_RAG_PLUS_COT, - "apple/openelm-3b-instruct": self.constants.PromptTemplateIds.OPENELM_3B_INSTRUCT__04_FEW_SHOT_RAG_PLUS_COT, - - # Llama models - "llama-3.2-3b-instruct": self.constants.PromptTemplateIds.LLAMA_1_1B_CHAT__04_FEW_SHOT_RAG_PLUS_COT, - "meta-llama/llama-3.2-3b-instruct": self.constants.PromptTemplateIds.LLAMA_1_1B_CHAT__04_FEW_SHOT_RAG_PLUS_COT, + "apple/OpenELM-1_1B-Instruct": self.constants.PromptTemplateIds.OPENELM_1_1B_INSTRUCT__04_FEW_SHOT_RAG_PLUS_COT, + "meta-llama/llama-3.2-3b-instruct": self.constants.PromptTemplateIds.LLAMA_1_1B_CHAT__04_FEW_SHOT_RAG_PLUS_COT } def _get_template(self, user_prompt: str, template_id: str) -> StringPromptTemplate: diff --git a/src/text_generation/services/nlp/text_generation_completion_service.py b/src/text_generation/services/nlp/text_generation_completion_service.py index 4d4fbaa0d..779710fd0 100644 --- a/src/text_generation/services/nlp/text_generation_completion_service.py +++ b/src/text_generation/services/nlp/text_generation_completion_service.py @@ -106,11 +106,11 @@ class TextGenerationCompletionService(AbstractTextGenerationCompletionService): }, # OpenELM models - "apple/openelm-3b-instruct": { - PromptTemplateType.BASIC.value: self.constants.PromptTemplateIds.OPENELM_3B_INSTRUCT__01_BASIC, - PromptTemplateType.ZERO_SHOT_COT.value: self.constants.PromptTemplateIds.OPENELM_3B_INSTRUCT__02_ZERO_SHOT_CHAIN_OF_THOUGHT, - PromptTemplateType.FEW_SHOT.value: self.constants.PromptTemplateIds.OPENELM_3B_INSTRUCT__03_FEW_SHOT_EXAMPLES, - PromptTemplateType.RAG_PLUS_COT.value: self.constants.PromptTemplateIds.OPENELM_3B_INSTRUCT__04_FEW_SHOT_RAG_PLUS_COT, + "apple/OpenELM-1_1B-Instruct": { + PromptTemplateType.BASIC.value: self.constants.PromptTemplateIds.OPENELM_1_1B_INSTRUCT__01_BASIC, + PromptTemplateType.ZERO_SHOT_COT.value: self.constants.PromptTemplateIds.OPENELM_1_1B_INSTRUCT__02_ZERO_SHOT_CHAIN_OF_THOUGHT, + PromptTemplateType.FEW_SHOT.value: self.constants.PromptTemplateIds.OPENELM_1_1B_INSTRUCT__03_FEW_SHOT_EXAMPLES, + PromptTemplateType.RAG_PLUS_COT.value: self.constants.PromptTemplateIds.OPENELM_1_1B_INSTRUCT__04_FEW_SHOT_RAG_PLUS_COT, }, # Llama models diff --git a/tests/create_dirs.sh b/tests/create_dirs.sh index 4c2cb648e..689bb8390 100755 --- a/tests/create_dirs.sh +++ b/tests/create_dirs.sh @@ -2,7 +2,7 @@ # Array of model names models=( - "apple/OpenELM-3B-Instruct" + "apple/OpenELM-1_1B-Instruct" "meta-llama/Llama-3.2-3B-Instruct" "microsoft/Phi-3-mini-4k-instruct" ) diff --git a/tests/integration/_archive/test_04_malicious_prompts_rag_and_cot.py b/tests/integration/_archive/test_04_malicious_prompts_rag_and_cot.py index f36da6ed8..eadff3a65 100644 --- a/tests/integration/_archive/test_04_malicious_prompts_rag_and_cot.py +++ b/tests/integration/_archive/test_04_malicious_prompts_rag_and_cot.py @@ -12,7 +12,7 @@ from src.text_generation.services.nlp.abstract_text_generation_completion_servic @pytest.mark.mitigation @pytest.mark.parametrize("model_id", [ ModelId.MICROSOFT_PHI_3_MINI4K_INSTRUCT, - ModelId.APPLE_OPENELM_3B_INSTRUCT, + ModelId.APPLE_OPENELM_1_1B_INSTRUCT, ModelId.META_LLAMA_3_2_3B_INSTRUCT ]) def test_04_malicious_prompts_rag_and_cot( diff --git a/tests/integration/test_00_benign_prompts_no_mitigation_apple_openelm_3b_instruct.py b/tests/integration/test_00_benign_prompts_no_mitigation_apple_openelm_3b_instruct.py index 5b9f3abc4..3632b5704 100644 --- a/tests/integration/test_00_benign_prompts_no_mitigation_apple_openelm_3b_instruct.py +++ b/tests/integration/test_00_benign_prompts_no_mitigation_apple_openelm_3b_instruct.py @@ -3,7 +3,7 @@ from src.text_generation.services.nlp.abstract_text_generation_completion_servic from tests.integration.test_utils import get_prompt_batch, run_prompt_analysis_test -def test_00_benign_prompts_no_mitigation_apple_openelm_3b_instruct( +def test_00_benign_prompts_no_mitigation_apple_openelm_1_1b_instruct( text_generation_completion_service: AbstractTextGenerationCompletionService, semantic_similarity_service, math_prompts, @@ -19,7 +19,7 @@ def test_00_benign_prompts_no_mitigation_apple_openelm_3b_instruct( results = run_prompt_analysis_test( test_id=0, - model_id=ModelId.APPLE_OPENELM_3B_INSTRUCT, + model_id=ModelId.APPLE_OPENELM_1_1B_INSTRUCT, text_generation_completion_service=text_generation_completion_service, semantic_similarity_service=semantic_similarity_service, prompts=prompt_batch, diff --git a/tests/integration/test_01_malicious_prompts_no_mitigation_apple_openelm_3b_instruct.py b/tests/integration/test_01_malicious_prompts_no_mitigation_apple_openelm_3b_instruct.py index 52aa48c91..7e7bea020 100644 --- a/tests/integration/test_01_malicious_prompts_no_mitigation_apple_openelm_3b_instruct.py +++ b/tests/integration/test_01_malicious_prompts_no_mitigation_apple_openelm_3b_instruct.py @@ -7,7 +7,7 @@ import pytest @pytest.mark.mitigation -def test_01_malicious_prompts_no_mitigation_apple_openelm_3b_instruct( +def test_01_malicious_prompts_no_mitigation_apple_openelm_1_1b_instruct( text_generation_completion_service: AbstractTextGenerationCompletionService, semantic_similarity_service, malicious_prompts, @@ -20,7 +20,7 @@ def test_01_malicious_prompts_no_mitigation_apple_openelm_3b_instruct( results = run_prompt_analysis_test( test_id=1, - model_id=ModelId.APPLE_OPENELM_3B_INSTRUCT, + model_id=ModelId.APPLE_OPENELM_1_1B_INSTRUCT, text_generation_completion_service=text_generation_completion_service, semantic_similarity_service=semantic_similarity_service, prompts=prompt_batch, diff --git a/tests/integration/test_02_malicious_prompts_cot_apple_openelm_3b_instruct.py b/tests/integration/test_02_malicious_prompts_cot_apple_openelm_3b_instruct.py index e10912b7c..6e2d49c29 100644 --- a/tests/integration/test_02_malicious_prompts_cot_apple_openelm_3b_instruct.py +++ b/tests/integration/test_02_malicious_prompts_cot_apple_openelm_3b_instruct.py @@ -7,7 +7,7 @@ import pytest @pytest.mark.mitigation -def test_02_malicious_prompts_cot_apple_openelm_3b_instruct( +def test_02_malicious_prompts_cot_apple_openelm_1_1b_instruct( text_generation_completion_service: AbstractTextGenerationCompletionService, semantic_similarity_service, malicious_prompts, @@ -21,7 +21,7 @@ def test_02_malicious_prompts_cot_apple_openelm_3b_instruct( results = run_prompt_analysis_test( test_id=2, - model_id=ModelId.APPLE_OPENELM_3B_INSTRUCT, + model_id=ModelId.APPLE_OPENELM_1_1B_INSTRUCT, text_generation_completion_service=text_generation_completion_service, semantic_similarity_service=semantic_similarity_service, prompts=prompt_batch, diff --git a/tests/integration/test_03_malicious_prompts_rag_apple_openelm_3b_instruct.py b/tests/integration/test_03_malicious_prompts_rag_apple_openelm_3b_instruct.py index 82cf3b423..153af3e74 100644 --- a/tests/integration/test_03_malicious_prompts_rag_apple_openelm_3b_instruct.py +++ b/tests/integration/test_03_malicious_prompts_rag_apple_openelm_3b_instruct.py @@ -7,7 +7,7 @@ import pytest @pytest.mark.mitigation -def test_03_malicious_prompts_rag_apple_openelm_3b_instruct( +def test_03_malicious_prompts_rag_apple_openelm_1_1b_instruct( text_generation_completion_service: AbstractTextGenerationCompletionService, semantic_similarity_service, malicious_prompts, @@ -21,7 +21,7 @@ def test_03_malicious_prompts_rag_apple_openelm_3b_instruct( results = run_prompt_analysis_test( test_id=3, - model_id=ModelId.APPLE_OPENELM_3B_INSTRUCT, + model_id=ModelId.APPLE_OPENELM_1_1B_INSTRUCT, text_generation_completion_service=text_generation_completion_service, semantic_similarity_service=semantic_similarity_service, prompts=prompt_batch, diff --git a/tests/integration/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.py b/tests/integration/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.py index 180310c00..55e21a9ac 100644 --- a/tests/integration/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.py +++ b/tests/integration/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.py @@ -7,7 +7,7 @@ import pytest @pytest.mark.mitigation -def test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct( +def test_04_malicious_prompts_rag_and_cot_apple_openelm_1_1b_instruct( text_generation_completion_service: AbstractTextGenerationCompletionService, semantic_similarity_service, malicious_prompts, @@ -22,7 +22,7 @@ def test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct( results = run_prompt_analysis_test( test_id=4, - model_id=ModelId.APPLE_OPENELM_3B_INSTRUCT, + model_id=ModelId.APPLE_OPENELM_1_1B_INSTRUCT, text_generation_completion_service=text_generation_completion_service, semantic_similarity_service=semantic_similarity_service, prompts=prompt_batch, diff --git a/tests/integration/test_prompt_template_repository.py b/tests/integration/test_prompt_template_repository.py index 3a1b60cff..3e9edeeb7 100644 --- a/tests/integration/test_prompt_template_repository.py +++ b/tests/integration/test_prompt_template_repository.py @@ -161,7 +161,7 @@ def test_4_save_rag_plus_cot_template_llama(constants, prompt_template_service): """ - Model: apple/OpenELM-3B-Instruct + Model: apple/OpenELM-1_1B-Instruct """ @pytest.mark.repository @@ -175,7 +175,7 @@ def test_1_save_basic_template_apple_openelm(constants, prompt_template_service) ### Response: """, input_variables=[constants.INPUT_VARIABLE_TOKEN] ) - id = constants.PromptTemplateIds.OPENELM_3B_INSTRUCT__01_BASIC + id = constants.PromptTemplateIds.OPENELM_1_1B_INSTRUCT__01_BASIC prompt_template_service.add(id, prompt_template) @pytest.mark.repository @@ -191,7 +191,7 @@ def test_2_save_zero_shot_cot_template_apple_openelm(constants, prompt_template_ ### Response: """, input_variables=[constants.INPUT_VARIABLE_TOKEN] ) - id = constants.PromptTemplateIds.OPENELM_3B_INSTRUCT__02_ZERO_SHOT_CHAIN_OF_THOUGHT + id = constants.PromptTemplateIds.OPENELM_1_1B_INSTRUCT__02_ZERO_SHOT_CHAIN_OF_THOUGHT prompt_template_service.add(id, prompt_template) @pytest.mark.repository @@ -209,7 +209,7 @@ def test_3_save_rag_few_shot_template_apple_openelm(constants, prompt_template_s template=template_string ) - template_id = constants.PromptTemplateIds.OPENELM_3B_INSTRUCT__03_FEW_SHOT_EXAMPLES + template_id = constants.PromptTemplateIds.OPENELM_1_1B_INSTRUCT__03_FEW_SHOT_EXAMPLES prompt_template_service.add(template_id, prompt_template) @pytest.mark.repository @@ -229,5 +229,5 @@ def test_4_save_rag_plus_cot_template_apple_openelm(constants, prompt_template_s template=template_string ) - template_id = constants.PromptTemplateIds.OPENELM_3B_INSTRUCT__04_FEW_SHOT_RAG_PLUS_COT + template_id = constants.PromptTemplateIds.OPENELM_1_1B_INSTRUCT__04_FEW_SHOT_RAG_PLUS_COT prompt_template_service.add(template_id, prompt_template)