Merge pull request #26 from lightbroker/model-support-expansion

Model support expansion

.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.01-20.yml

@@ -0,0 +1,11 @@
+name: '#4 (1-20) | RAG + CoT | apple/OpenELM-3B-Instruct'
+on:
+  workflow_dispatch:
+
+jobs:
+  test-prompts-1-20:
+    uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.base.yml
+    with:
+      batch_offset: 0
+      range_name: "1-20"
+      batch_size: 2

.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.21-40.yml

@@ -0,0 +1,11 @@
+name: '#4 (21-40) | RAG + CoT | apple/OpenELM-3B-Instruct'
+on:
+  workflow_dispatch:
+
+jobs:
+  test-prompts-21-40:
+    uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.base.yml
+    with:
+      batch_offset: 20
+      range_name: "21-40"
+      batch_size: 2

.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.41-60.yml

@@ -0,0 +1,11 @@
+name: '#4 (41-60) | RAG + CoT | apple/OpenELM-3B-Instruct'
+on:
+  workflow_dispatch:
+
+jobs:
+  test-prompts-41-60:
+    uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.base.yml
+    with:
+      batch_offset: 40
+      range_name: "41-60"
+      batch_size: 2

.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.61-80.yml

@@ -0,0 +1,11 @@
+name: '#4 (61-80) | RAG + CoT | apple/OpenELM-3B-Instruct'
+on:
+  workflow_dispatch:
+
+jobs:
+  test-prompts-61-80:
+    uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.base.yml
+    with:
+      batch_offset: 60
+      range_name: "61-80"
+      batch_size: 2

.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.81-100.yml

@@ -0,0 +1,11 @@
+name: '#4 (81-100) | RAG + CoT | apple/OpenELM-3B-Instruct'
+on:
+  workflow_dispatch:
+
+jobs:
+  test-prompts-81-100:
+    uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.base.yml
+    with:
+      batch_offset: 80
+      range_name: "81-100"
+      batch_size: 2

.github/workflows/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.base.yml

@@ -0,0 +1,27 @@
+name: 'Reusable Test #4 | RAG + CoT | apple/OpenELM-3B-Instruct'
+on:
+  workflow_call:
+    inputs:
+      batch_offset:
+        description: 'Starting prompt index offset'
+        required: true
+        type: number
+      range_name:
+        description: 'Human readable range name (e.g., "1-20")'
+        required: true
+        type: string
+      batch_size:
+        description: 'Number of prompts per batch'
+        required: false
+        type: number
+        default: 2
+
+jobs:
+  test:
+    uses: ./.github/workflows/test_04.abstract_base.yml
+    with:
+      batch_offset: ${{ inputs.batch_offset }}
+      range_name: ${{ inputs.range_name }}
+      batch_size: ${{ inputs.batch_size }}
+      test_file_path: tests/integration/test_04_malicious_prompts_rag_and_cot_apple_openelm_3b_instruct.py
+      model_display_name: apple/OpenELM-3B-Instruct

.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.01-20.yml

@@ -1,10 +1,10 @@
-name: '#4 (1-20) | RAG + CoT | microsoft/Phi-3-mini-4k-instruct'
+name: '#4 (1-20) | RAG + CoT | meta-llama/Llama-3.2-3B-Instruct'
 on:
   workflow_dispatch:
 
 jobs:
   test-prompts-1-20:
-    uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.base.yml
+    uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.base.yml
     with:
       batch_offset: 0
       range_name: "1-20"

.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.21-40.yml

@@ -1,10 +1,10 @@
-name: '#4 (21-40) | RAG + CoT | microsoft/Phi-3-mini-4k-instruct'
+name: '#4 (21-40) | RAG + CoT | meta-llama/Llama-3.2-3B-Instruct'
 on:
   workflow_dispatch:
 
 jobs:
   test-prompts-21-40:
-    uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.base.yml
+    uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.base.yml
     with:
       batch_offset: 20
       range_name: "21-40"

.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.41-60.yml

@@ -1,10 +1,10 @@
-name: '#4 (41-60) | RAG + CoT | microsoft/Phi-3-mini-4k-instruct'
+name: '#4 (41-60) | RAG + CoT | meta-llama/Llama-3.2-3B-Instruct'
 on:
   workflow_dispatch:
 
 jobs:
   test-prompts-41-60:
-    uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.base.yml
+    uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.base.yml
     with:
       batch_offset: 40
       range_name: "41-60"

.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.61-80.yml

@@ -1,10 +1,10 @@
-name: '#4 (61-80) | RAG + CoT | microsoft/Phi-3-mini-4k-instruct'
+name: '#4 (61-80) | RAG + CoT | meta-llama/Llama-3.2-3B-Instruct'
 on:
   workflow_dispatch:
 
 jobs:
   test-prompts-61-80:
-    uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.base.yml
+    uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.base.yml
     with:
       batch_offset: 60
       range_name: "61-80"

.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.81-100.yml

@@ -1,10 +1,10 @@
-name: '#4 (81-100) | RAG + CoT | microsoft/Phi-3-mini-4k-instruct'
+name: '#4 (81-100) | RAG + CoT | meta-llama/Llama-3.2-3B-Instruct'
 on:
   workflow_dispatch:
 
 jobs:
   test-prompts-81-100:
-    uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.base.yml
+    uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.base.yml
     with:
       batch_offset: 80
       range_name: "81-100"

src/text_generation/services/logging/test_run_logging_service.py

@@ -22,9 +22,15 @@ class TestRunLoggingService(AbstractTestRunLoggingService):
     ):
         self._lock = threading.Lock()
         timestamp = calendar.timegm(time.gmtime())
-        self.log_file_path = f"./tests/logs/test_{test_id}/{str(model_id.value).replace("/", "_")}/{start}_{end}/test_{str(test_id).lower()}_logs_{timestamp}.json"
+        base_path = os.environ.get('TEST_RUNS', '.')
+        self.log_file_path = os.path.join(base_path, str(f"test_{test_id}/{str(model_id.value).replace("/", "_")}/{start}_{end}/test_{str(test_id).lower()}_logs_{timestamp}.json").lower())
+
+        # Ensure directory structure exists
+        os.makedirs(os.path.dirname(self.log_file_path), exist_ok=True)
+
         self._ensure_log_file_exists()
 
+
     def _ensure_log_file_exists(self):
         if not os.path.exists(self.log_file_path):
             with open(self.log_file_path, 'w') as f:

tests/conftest.py

@@ -75,6 +75,7 @@ def setup_test_environment():
     os.environ["LOG_LEVEL"] = "DEBUG"
     os.environ["PROMPT_TEMPLATES_DIR"] = "./infrastructure/prompt_templates"
     os.environ["INJECTION_DATA_DIR"] = "./tests/security/tests/results/01_garak_no_guidelines"
+    os.environ["TEST_RUNS"] = "./tests/logs"
     os.environ["MODEL_BASE_DIR"] = "./infrastructure/foundation_model"
     os.environ["MODEL_CPU_DIR"] = "cpu_and_mobile/cpu-int4-rtn-block-32-acc-level-4"
     os.environ["MODEL_DATA_FILENAME"] = "phi3-mini-4k-instruct-cpu-int4-rtn-block-32-acc-level-4.onnx.data"

tests/create_dirs.sh

@@ -28,7 +28,7 @@ for i in {1..5}; do
         for model in "${models[@]}"; do
             # Replace / with _ and convert to lowercase for directory name
             dir_name="${model//\//_}"
-            dir_name="${dir_name,,}"  # Convert to lowercase
+            dir_name="${dir_name}"  # Convert to lowercase
             
             # Create the full directory path
             full_path="logs/test_${i}/${dir_name}/${range}"