From 42e4962063a20994710cd65a447b43b4037bbbdf Mon Sep 17 00:00:00 2001 From: Adam Wilson Date: Mon, 18 Aug 2025 20:24:36 -0600 Subject: [PATCH] base classes --- .github/workflows/test_04.abstract_base.yml | 72 +++++++++++++++++++ ...d_cot_meta_llama_3_2_3b_instruct.01-20.yml | 11 +++ ...d_cot_meta_llama_3_2_3b_instruct.21-40.yml | 11 +++ ...d_cot_meta_llama_3_2_3b_instruct.41-60.yml | 11 +++ ...d_cot_meta_llama_3_2_3b_instruct.61-80.yml | 11 +++ ..._cot_meta_llama_3_2_3b_instruct.81-100.yml | 11 +++ ...nd_cot_meta_llama_3_2_3b_instruct.base.yml | 27 +++++++ ...rag_and_cot_meta_llama_3_2_3b_instruct.yml | 42 ----------- ..._microsoft_phi_3_mini4k_instruct.01-20.yml | 2 +- ..._microsoft_phi_3_mini4k_instruct.21-40.yml | 2 +- ..._microsoft_phi_3_mini4k_instruct.41-60.yml | 2 +- ..._microsoft_phi_3_mini4k_instruct.61-80.yml | 2 +- ...microsoft_phi_3_mini4k_instruct.81-100.yml | 2 +- ...t_microsoft_phi_3_mini4k_instruct.base.yml | 46 ++---------- ...nd_cot_microsoft_phi_3_mini4k_instruct.yml | 44 ------------ .../logging/test_run_logging_service.py | 2 +- 16 files changed, 167 insertions(+), 131 deletions(-) create mode 100644 .github/workflows/test_04.abstract_base.yml create mode 100644 .github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.01-20.yml create mode 100644 .github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.21-40.yml create mode 100644 .github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.41-60.yml create mode 100644 .github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.61-80.yml create mode 100644 .github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.81-100.yml create mode 100644 .github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.base.yml delete mode 100644 .github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.yml delete mode 100644 .github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.yml diff --git a/.github/workflows/test_04.abstract_base.yml b/.github/workflows/test_04.abstract_base.yml new file mode 100644 index 000000000..2c1cde516 --- /dev/null +++ b/.github/workflows/test_04.abstract_base.yml @@ -0,0 +1,72 @@ +name: 'Reusable Test Runner | RAG + CoT | Generic' +on: + workflow_call: + inputs: + batch_offset: + description: 'Starting prompt index offset' + required: true + type: number + range_name: + description: 'Human readable range name (e.g., "1-20")' + required: true + type: string + test_file_path: + description: 'Path to the test file to run' + required: true + type: string + model_display_name: + description: 'Human readable model name for workflow title' + required: true + type: string + batch_size: + description: 'Number of prompts per batch' + required: false + type: number + default: 2 + +jobs: + test: + runs-on: ubuntu-latest + strategy: + # Always 10 batches per workflow + matrix: + batch: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + fail-fast: false + steps: + - name: 'checkout' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + + - name: 'set up Python' + uses: actions/setup-python@v3 + with: + python-version: '3.12' + + - name: 'set up Python dependencies' + shell: bash + run: pip install -r ${{ github.workspace }}/requirements.txt + + - name: 'run text generation tests - ${{ inputs.model_display_name }} - range ${{ inputs.range_name }} batch ${{ matrix.batch }}' + shell: bash + env: + PROMPT_BATCH: ${{ matrix.batch }} + BATCH_SIZE: ${{ inputs.batch_size }} + BATCH_OFFSET: ${{ inputs.batch_offset }} + run: pytest ${{ inputs.test_file_path }} -s --disable-warnings + + - name: Check for changes + id: verify-changed-files + run: | + if [ -n "$(git status --porcelain)" ]; then + echo "changed=true" >> $GITHUB_OUTPUT + else + echo "changed=false" >> $GITHUB_OUTPUT + fi + + - name: Commit and push changes + if: steps.verify-changed-files.outputs.changed == 'true' + run: | + git config --local user.email "42450907+lightbroker@users.noreply.github.com" + git config --local user.name "Adam Wilson" + git add . + git commit -m "Auto-generated files from ${{ inputs.model_display_name }} range ${{ inputs.range_name }} batch ${{ matrix.batch }} [skip ci]" + git push \ No newline at end of file diff --git a/.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.01-20.yml b/.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.01-20.yml new file mode 100644 index 000000000..a8e3d4470 --- /dev/null +++ b/.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.01-20.yml @@ -0,0 +1,11 @@ +name: '#4 (1-20) | RAG + CoT | microsoft/Phi-3-mini-4k-instruct' +on: + workflow_dispatch: + +jobs: + test-prompts-1-20: + uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.base.yml + with: + batch_offset: 0 + range_name: "1-20" + batch_size: 2 \ No newline at end of file diff --git a/.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.21-40.yml b/.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.21-40.yml new file mode 100644 index 000000000..56efda8e4 --- /dev/null +++ b/.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.21-40.yml @@ -0,0 +1,11 @@ +name: '#4 (21-40) | RAG + CoT | microsoft/Phi-3-mini-4k-instruct' +on: + workflow_dispatch: + +jobs: + test-prompts-21-40: + uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.base.yml + with: + batch_offset: 20 + range_name: "21-40" + batch_size: 2 \ No newline at end of file diff --git a/.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.41-60.yml b/.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.41-60.yml new file mode 100644 index 000000000..474d66bca --- /dev/null +++ b/.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.41-60.yml @@ -0,0 +1,11 @@ +name: '#4 (41-60) | RAG + CoT | microsoft/Phi-3-mini-4k-instruct' +on: + workflow_dispatch: + +jobs: + test-prompts-41-60: + uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.base.yml + with: + batch_offset: 40 + range_name: "41-60" + batch_size: 2 \ No newline at end of file diff --git a/.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.61-80.yml b/.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.61-80.yml new file mode 100644 index 000000000..7f078d0a7 --- /dev/null +++ b/.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.61-80.yml @@ -0,0 +1,11 @@ +name: '#4 (61-80) | RAG + CoT | microsoft/Phi-3-mini-4k-instruct' +on: + workflow_dispatch: + +jobs: + test-prompts-61-80: + uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.base.yml + with: + batch_offset: 60 + range_name: "61-80" + batch_size: 2 \ No newline at end of file diff --git a/.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.81-100.yml b/.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.81-100.yml new file mode 100644 index 000000000..f44cb09ba --- /dev/null +++ b/.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.81-100.yml @@ -0,0 +1,11 @@ +name: '#4 (81-100) | RAG + CoT | microsoft/Phi-3-mini-4k-instruct' +on: + workflow_dispatch: + +jobs: + test-prompts-81-100: + uses: ./.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.base.yml + with: + batch_offset: 80 + range_name: "81-100" + batch_size: 2 \ No newline at end of file diff --git a/.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.base.yml b/.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.base.yml new file mode 100644 index 000000000..8420c7437 --- /dev/null +++ b/.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.base.yml @@ -0,0 +1,27 @@ +name: 'Reusable Test #4 | RAG + CoT | meta-llama/Llama-3.2-3B-Instruct' +on: + workflow_call: + inputs: + batch_offset: + description: 'Starting prompt index offset' + required: true + type: number + range_name: + description: 'Human readable range name (e.g., "1-20")' + required: true + type: string + batch_size: + description: 'Number of prompts per batch' + required: false + type: number + default: 2 + +jobs: + test: + uses: ./.github/workflows/test_04.abstract_base.yml + with: + batch_offset: ${{ inputs.batch_offset }} + range_name: ${{ inputs.range_name }} + batch_size: ${{ inputs.batch_size }} + test_file_path: tests/integration/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.py + model_display_name: meta-llama/Llama-3.2-3B-Instruct \ No newline at end of file diff --git a/.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.yml b/.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.yml deleted file mode 100644 index f8775f4f9..000000000 --- a/.github/workflows/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: 'Test #4 | RAG + CoT | meta-llama/Llama-3.2-3B-Instruct' - -on: - workflow_dispatch: - -jobs: - test: - runs-on: ubuntu-latest - steps: - - name: 'checkout' - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - - - name: 'set up Python' - uses: actions/setup-python@v3 - with: - python-version: '3.12' - - - name: 'set up Python dependencies' - shell: bash - run: pip install -r ${{ github.workspace }}/requirements.txt - - - name: 'run text generation tests' - shell: bash - run: pytest tests/integration/test_04_malicious_prompts_rag_and_cot_meta_llama_3_2_3b_instruct.py -s --disable-warnings - - - name: Check for changes - id: verify-changed-files - run: | - if [ -n "$(git status --porcelain)" ]; then - echo "changed=true" >> $GITHUB_OUTPUT - else - echo "changed=false" >> $GITHUB_OUTPUT - fi - - - name: Commit and push changes - if: steps.verify-changed-files.outputs.changed == 'true' - run: | - git config --local user.email "42450907+lightbroker@users.noreply.github.com" - git config --local user.name "Adam Wilson" - git add . - git commit -m "Auto-generated files from workflow [skip ci]" - git push \ No newline at end of file diff --git a/.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.01-20.yml b/.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.01-20.yml index 941db8259..a8e3d4470 100644 --- a/.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.01-20.yml +++ b/.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.01-20.yml @@ -1,4 +1,4 @@ -name: '#4 | RAG + CoT | microsoft/Phi-3-mini-4k-instruct (1-20)' +name: '#4 (1-20) | RAG + CoT | microsoft/Phi-3-mini-4k-instruct' on: workflow_dispatch: diff --git a/.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.21-40.yml b/.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.21-40.yml index ed6d1d152..56efda8e4 100644 --- a/.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.21-40.yml +++ b/.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.21-40.yml @@ -1,4 +1,4 @@ -name: '#4 | RAG + CoT | microsoft/Phi-3-mini-4k-instruct (21-40)' +name: '#4 (21-40) | RAG + CoT | microsoft/Phi-3-mini-4k-instruct' on: workflow_dispatch: diff --git a/.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.41-60.yml b/.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.41-60.yml index 7f53f4b23..474d66bca 100644 --- a/.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.41-60.yml +++ b/.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.41-60.yml @@ -1,4 +1,4 @@ -name: '#4 | RAG + CoT | microsoft/Phi-3-mini-4k-instruct (41-60)' +name: '#4 (41-60) | RAG + CoT | microsoft/Phi-3-mini-4k-instruct' on: workflow_dispatch: diff --git a/.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.61-80.yml b/.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.61-80.yml index 889d121de..7f078d0a7 100644 --- a/.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.61-80.yml +++ b/.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.61-80.yml @@ -1,4 +1,4 @@ -name: '#4 | RAG + CoT | microsoft/Phi-3-mini-4k-instruct (61-80)' +name: '#4 (61-80) | RAG + CoT | microsoft/Phi-3-mini-4k-instruct' on: workflow_dispatch: diff --git a/.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.81-100.yml b/.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.81-100.yml index 545b87463..f44cb09ba 100644 --- a/.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.81-100.yml +++ b/.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.81-100.yml @@ -1,4 +1,4 @@ -name: '#4 | RAG + CoT | microsoft/Phi-3-mini-4k-instruct (81-100)' +name: '#4 (81-100) | RAG + CoT | microsoft/Phi-3-mini-4k-instruct' on: workflow_dispatch: diff --git a/.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.base.yml b/.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.base.yml index b513e57fc..336f1f97b 100644 --- a/.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.base.yml +++ b/.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.base.yml @@ -18,42 +18,10 @@ on: jobs: test: - runs-on: ubuntu-latest - strategy: - # Always 10 batches per workflow - matrix: - batch: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - fail-fast: false - steps: - - name: 'checkout' - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - - name: 'set up Python' - uses: actions/setup-python@v3 - with: - python-version: '3.12' - - name: 'set up Python dependencies' - shell: bash - run: pip install -r ${{ github.workspace }}/requirements.txt - - name: 'run text generation tests - range ${{ inputs.range_name }} batch ${{ matrix.batch }}' - shell: bash - env: - PROMPT_BATCH: ${{ matrix.batch }} - BATCH_SIZE: ${{ inputs.batch_size }} - BATCH_OFFSET: ${{ inputs.batch_offset }} - run: pytest tests/integration/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.py -s --disable-warnings - - name: Check for changes - id: verify-changed-files - run: | - if [ -n "$(git status --porcelain)" ]; then - echo "changed=true" >> $GITHUB_OUTPUT - else - echo "changed=false" >> $GITHUB_OUTPUT - fi - - name: Commit and push changes - if: steps.verify-changed-files.outputs.changed == 'true' - run: | - git config --local user.email "42450907+lightbroker@users.noreply.github.com" - git config --local user.name "Adam Wilson" - git add . - git commit -m "Auto-generated files from range ${{ inputs.range_name }} batch ${{ matrix.batch }} [skip ci]" - git push \ No newline at end of file + uses: ./.github/workflows/test_04.abstract_base.yml + with: + batch_offset: ${{ inputs.batch_offset }} + range_name: ${{ inputs.range_name }} + batch_size: ${{ inputs.batch_size }} + test_file_path: tests/integration/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.py + model_display_name: microsoft/Phi-3-mini-4k-instruct \ No newline at end of file diff --git a/.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.yml b/.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.yml deleted file mode 100644 index e8c07a758..000000000 --- a/.github/workflows/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: 'Test #4 | RAG + CoT | microsoft/Phi-3-mini-4k-instruct | Prompts 1-20' -on: - workflow_dispatch: -jobs: - test: - runs-on: ubuntu-latest - strategy: - # Matrix for batches 1-10, each testing 2 prompts (prompts 1-20 total) - matrix: - batch: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - fail-fast: false - steps: - - name: 'checkout' - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - - name: 'set up Python' - uses: actions/setup-python@v3 - with: - python-version: '3.12' - - name: 'set up Python dependencies' - shell: bash - run: pip install -r ${{ github.workspace }}/requirements.txt - - name: 'run text generation tests - batch ${{ matrix.batch }}' - shell: bash - env: - PROMPT_BATCH: ${{ matrix.batch }} - BATCH_SIZE: 2 - BATCH_OFFSET: 0 # Prompts 1-20 (offset 0) - run: pytest tests/integration/test_04_malicious_prompts_rag_and_cot_microsoft_phi_3_mini4k_instruct.py -s --disable-warnings - - name: Check for changes - id: verify-changed-files - run: | - if [ -n "$(git status --porcelain)" ]; then - echo "changed=true" >> $GITHUB_OUTPUT - else - echo "changed=false" >> $GITHUB_OUTPUT - fi - - name: Commit and push changes (batch ${{ matrix.batch }}) - if: steps.verify-changed-files.outputs.changed == 'true' - run: | - git config --local user.email "42450907+lightbroker@users.noreply.github.com" - git config --local user.name "Adam Wilson" - git add . - git commit -m "Auto-generated files from workflow batch ${{ matrix.batch }} (prompts 1-20) [skip ci]" - git push \ No newline at end of file diff --git a/src/text_generation/services/logging/test_run_logging_service.py b/src/text_generation/services/logging/test_run_logging_service.py index 2e42a260c..23d1cc075 100644 --- a/src/text_generation/services/logging/test_run_logging_service.py +++ b/src/text_generation/services/logging/test_run_logging_service.py @@ -22,7 +22,7 @@ class TestRunLoggingService(AbstractTestRunLoggingService): ): self._lock = threading.Lock() timestamp = calendar.timegm(time.gmtime()) - self.log_file_path = f"./tests/logs/test_{test_id}/{str(model_id.value).replace("/", "_")}/{start}_{end}/test_{test_id}_logs_{timestamp}.json" + self.log_file_path = f"./tests/logs/test_{test_id}/{str(model_id.value).replace("/", "_")}/{start}_{end}/test_{str(test_id).lower()}_logs_{timestamp}.json" self._ensure_log_file_exists() def _ensure_log_file_exists(self):